From f461a1dd0246880fe9b01d83dc3fcd4e3b30044e Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Fri, 12 Aug 2022 15:05:47 +0000 Subject: [PATCH 001/106] Add preamble script from global workflow. --- ush/preamble.sh | 94 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 ush/preamble.sh diff --git a/ush/preamble.sh b/ush/preamble.sh new file mode 100644 index 0000000000..632011d1e9 --- /dev/null +++ b/ush/preamble.sh @@ -0,0 +1,94 @@ +#!/bin/bash + +# +#----------------------------------------------------------------------- +# +# Preamble script to be SOURCED at the beginning of every script. Sets +# useful PS4 and optionally turns on set -x and set -eu. Also sets up +# crude script timing and provides a postamble that runs on exit. +# +# Syntax: +# preamble.sh [id] +# +# Aruguments: +# id: Optional identifier string. Use when running the same script +# multiple times in the same job (e.g. MPMD) +# +# Input environment variables: +# TRACE (TRUE/FALSE): Whether to echo every command (set -x) [default: "TRUE"] +# STRICT (TRUE/FALSE): Whether to exit immediately on error or undefined variable +# (set -eu) [default: "TRUE"] +# +# +#----------------------------------------------------------------------- +# +set +x +if [[ -v '1' ]]; then + id="(${1})" +else + id="" +fi + +# Record the start time so we can calculate the elapsed time later +start_time=$(date +%s) + +# Get the base name of the calling script +_calling_script=$(basename ${BASH_SOURCE[1]}) + +# Announce the script has begun +echo "Begin ${_calling_script} at $(date -u)" + +# Stage our variables +export STRICT=${STRICT:-"TRUE"} +export TRACE=${DEBUG:-"FALSE"} +export ERR_EXIT_CMD="" +export TRACE_CMD="" + +if [[ $STRICT == "TRUE" ]]; then + # Exit on error and undefined variable + export ERR_EXIT_CMD="set -eu" +fi +if [[ $TRACE == "TRUE" ]]; then + export TRACE_CMD="set -x" + # Print the script name and line number of each command as it is executed + export PS4='+ $(basename $BASH_SOURCE)[$LINEFALSE]'"$id: " +fi + +function postamble() { + # + #----------------------------------------------------------------------- + # + # Commands to execute when a script ends. + # + # Syntax: + # postamble script start_time rc + # + # Arguments: + # script: name of the script ending + # start_time: start time of script (in seconds) + # rc: the exit code of the script + # + #----------------------------------------------------------------------- + # + + set +x + script=${1} + start_time=${2} + rc=${3} + + # Calculate the elapsed time + end_time=$(date +%s) + elapsed_sec=$((end_time - start_time)) + elapsed=$(date -d@${elapsed_sec} -u +%H:%M:%S) + + # Announce the script has ended, then pass the error code up + echo "End ${script} at $(date -u) with error code ${rc:-0} (time elapsed: ${elapsed})" + exit ${rc} +} + +# Place the postamble in a trap so it is always called no matter how the script exits +trap "postamble ${_calling_script} ${start_time} \$?" EXIT + +# Turn on our settings +$ERR_EXIT_CMD +$TRACE_CMD From 9fd4f3de881fb6788809f4fc06445b915576262c Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Fri, 12 Aug 2022 16:27:11 +0000 Subject: [PATCH 002/106] Call preamble script in j-jobs and ex-scripts --- jobs/JREGIONAL_GET_EXTRN_MDL_FILES | 2 +- jobs/JREGIONAL_GET_OBS_CCPA | 2 +- jobs/JREGIONAL_GET_OBS_MRMS | 2 +- jobs/JREGIONAL_GET_OBS_NDAS | 2 +- jobs/JREGIONAL_MAKE_GRID | 2 +- jobs/JREGIONAL_MAKE_ICS | 2 +- jobs/JREGIONAL_MAKE_LBCS | 2 +- jobs/JREGIONAL_MAKE_OROG | 2 +- jobs/JREGIONAL_MAKE_SFC_CLIMO | 2 +- jobs/JREGIONAL_RUN_FCST | 2 +- jobs/JREGIONAL_RUN_POST | 2 +- jobs/JREGIONAL_RUN_VX_ENSGRID | 2 +- jobs/JREGIONAL_RUN_VX_ENSGRID_MEAN | 2 +- jobs/JREGIONAL_RUN_VX_ENSGRID_PROB | 2 +- jobs/JREGIONAL_RUN_VX_ENSPOINT | 2 +- jobs/JREGIONAL_RUN_VX_ENSPOINT_MEAN | 2 +- jobs/JREGIONAL_RUN_VX_ENSPOINT_PROB | 2 +- jobs/JREGIONAL_RUN_VX_GRIDSTAT | 2 +- jobs/JREGIONAL_RUN_VX_POINTSTAT | 2 +- scripts/exregional_get_ccpa_files.sh | 33 +++++++++++++++++++++ scripts/exregional_get_extrn_mdl_files.sh | 2 +- scripts/exregional_get_mrms_files.sh | 36 +++++++++++++++++++++-- scripts/exregional_get_ndas_files.sh | 34 +++++++++++++++++++++ scripts/exregional_make_grid.sh | 2 +- scripts/exregional_make_ics.sh | 2 +- scripts/exregional_make_lbcs.sh | 2 +- scripts/exregional_make_orog.sh | 2 +- scripts/exregional_make_sfc_climo.sh | 2 +- scripts/exregional_run_ensgridvx.sh | 3 +- scripts/exregional_run_ensgridvx_mean.sh | 3 +- scripts/exregional_run_ensgridvx_prob.sh | 3 +- scripts/exregional_run_enspointvx.sh | 3 +- scripts/exregional_run_enspointvx_mean.sh | 3 +- scripts/exregional_run_enspointvx_prob.sh | 3 +- scripts/exregional_run_fcst.sh | 2 +- scripts/exregional_run_gridstatvx.sh | 3 +- scripts/exregional_run_pointstatvx.sh | 3 +- scripts/exregional_run_post.sh | 2 +- ush/load_modules_run_task.sh | 2 +- 39 files changed, 137 insertions(+), 46 deletions(-) diff --git a/jobs/JREGIONAL_GET_EXTRN_MDL_FILES b/jobs/JREGIONAL_GET_EXTRN_MDL_FILES index 31c2963866..b528c38563 100755 --- a/jobs/JREGIONAL_GET_EXTRN_MDL_FILES +++ b/jobs/JREGIONAL_GET_EXTRN_MDL_FILES @@ -38,7 +38,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u +x; } > /dev/null 2>&1 +{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_GET_OBS_CCPA b/jobs/JREGIONAL_GET_OBS_CCPA index bd1760ee0d..2fb9067d96 100755 --- a/jobs/JREGIONAL_GET_OBS_CCPA +++ b/jobs/JREGIONAL_GET_OBS_CCPA @@ -29,7 +29,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u +x; } > /dev/null 2>&1 +{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_GET_OBS_MRMS b/jobs/JREGIONAL_GET_OBS_MRMS index f5d6349745..9c66d3bbd0 100755 --- a/jobs/JREGIONAL_GET_OBS_MRMS +++ b/jobs/JREGIONAL_GET_OBS_MRMS @@ -25,7 +25,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u +x; } > /dev/null 2>&1 +{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_GET_OBS_NDAS b/jobs/JREGIONAL_GET_OBS_NDAS index ab44e21b8f..132f1e4936 100755 --- a/jobs/JREGIONAL_GET_OBS_NDAS +++ b/jobs/JREGIONAL_GET_OBS_NDAS @@ -25,7 +25,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u +x; } > /dev/null 2>&1 +{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_MAKE_GRID b/jobs/JREGIONAL_MAKE_GRID index bbf22e0241..451cbe635e 100755 --- a/jobs/JREGIONAL_MAKE_GRID +++ b/jobs/JREGIONAL_MAKE_GRID @@ -115,7 +115,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u +x; } > /dev/null 2>&1 +{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_MAKE_ICS b/jobs/JREGIONAL_MAKE_ICS index a5e5894f10..99ba807577 100755 --- a/jobs/JREGIONAL_MAKE_ICS +++ b/jobs/JREGIONAL_MAKE_ICS @@ -17,7 +17,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u +x; } > /dev/null 2>&1 +{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_MAKE_LBCS b/jobs/JREGIONAL_MAKE_LBCS index bc7afbaf71..30d8bb56c4 100755 --- a/jobs/JREGIONAL_MAKE_LBCS +++ b/jobs/JREGIONAL_MAKE_LBCS @@ -17,7 +17,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u +x; } > /dev/null 2>&1 +{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_MAKE_OROG b/jobs/JREGIONAL_MAKE_OROG index 50b6c03697..2fdc5c983f 100755 --- a/jobs/JREGIONAL_MAKE_OROG +++ b/jobs/JREGIONAL_MAKE_OROG @@ -17,7 +17,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u +x; } > /dev/null 2>&1 +{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_MAKE_SFC_CLIMO b/jobs/JREGIONAL_MAKE_SFC_CLIMO index 24af27bfa3..bf0100af06 100755 --- a/jobs/JREGIONAL_MAKE_SFC_CLIMO +++ b/jobs/JREGIONAL_MAKE_SFC_CLIMO @@ -17,7 +17,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u +x; } > /dev/null 2>&1 +{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_RUN_FCST b/jobs/JREGIONAL_RUN_FCST index dbd4c80c4b..962dfc45ee 100755 --- a/jobs/JREGIONAL_RUN_FCST +++ b/jobs/JREGIONAL_RUN_FCST @@ -27,7 +27,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u +x; } > /dev/null 2>&1 +{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_RUN_POST b/jobs/JREGIONAL_RUN_POST index 86a30470bb..cbddabc9f9 100755 --- a/jobs/JREGIONAL_RUN_POST +++ b/jobs/JREGIONAL_RUN_POST @@ -26,7 +26,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u +x; } > /dev/null 2>&1 +{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_RUN_VX_ENSGRID b/jobs/JREGIONAL_RUN_VX_ENSGRID index 867c75fc84..2c2f74a618 100755 --- a/jobs/JREGIONAL_RUN_VX_ENSGRID +++ b/jobs/JREGIONAL_RUN_VX_ENSGRID @@ -26,7 +26,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u +x; } > /dev/null 2>&1 +{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_RUN_VX_ENSGRID_MEAN b/jobs/JREGIONAL_RUN_VX_ENSGRID_MEAN index a75558d928..208cd80e67 100755 --- a/jobs/JREGIONAL_RUN_VX_ENSGRID_MEAN +++ b/jobs/JREGIONAL_RUN_VX_ENSGRID_MEAN @@ -26,7 +26,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u +x; } > /dev/null 2>&1 +{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_RUN_VX_ENSGRID_PROB b/jobs/JREGIONAL_RUN_VX_ENSGRID_PROB index a8a2c43be5..c649d9177d 100755 --- a/jobs/JREGIONAL_RUN_VX_ENSGRID_PROB +++ b/jobs/JREGIONAL_RUN_VX_ENSGRID_PROB @@ -26,7 +26,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u +x; } > /dev/null 2>&1 +{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_RUN_VX_ENSPOINT b/jobs/JREGIONAL_RUN_VX_ENSPOINT index fc30f076fb..c844b72020 100755 --- a/jobs/JREGIONAL_RUN_VX_ENSPOINT +++ b/jobs/JREGIONAL_RUN_VX_ENSPOINT @@ -24,7 +24,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u +x; } > /dev/null 2>&1 +{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_RUN_VX_ENSPOINT_MEAN b/jobs/JREGIONAL_RUN_VX_ENSPOINT_MEAN index a7d937c12e..9a00a05176 100755 --- a/jobs/JREGIONAL_RUN_VX_ENSPOINT_MEAN +++ b/jobs/JREGIONAL_RUN_VX_ENSPOINT_MEAN @@ -24,7 +24,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u +x; } > /dev/null 2>&1 +{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_RUN_VX_ENSPOINT_PROB b/jobs/JREGIONAL_RUN_VX_ENSPOINT_PROB index 818e588e51..0320c9162a 100755 --- a/jobs/JREGIONAL_RUN_VX_ENSPOINT_PROB +++ b/jobs/JREGIONAL_RUN_VX_ENSPOINT_PROB @@ -24,7 +24,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u +x; } > /dev/null 2>&1 +{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_RUN_VX_GRIDSTAT b/jobs/JREGIONAL_RUN_VX_GRIDSTAT index 8537cb3ec5..10ebb74fa2 100755 --- a/jobs/JREGIONAL_RUN_VX_GRIDSTAT +++ b/jobs/JREGIONAL_RUN_VX_GRIDSTAT @@ -26,7 +26,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u +x; } > /dev/null 2>&1 +{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_RUN_VX_POINTSTAT b/jobs/JREGIONAL_RUN_VX_POINTSTAT index 084543695f..bd83283ce9 100755 --- a/jobs/JREGIONAL_RUN_VX_POINTSTAT +++ b/jobs/JREGIONAL_RUN_VX_POINTSTAT @@ -24,7 +24,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u +x; } > /dev/null 2>&1 +{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_get_ccpa_files.sh b/scripts/exregional_get_ccpa_files.sh index b0ab11f5bb..9a404e4557 100755 --- a/scripts/exregional_get_ccpa_files.sh +++ b/scripts/exregional_get_ccpa_files.sh @@ -1,9 +1,33 @@ #!/bin/bash +# +#----------------------------------------------------------------------- +# +# Source the variable definitions file and the bash utility functions. +# +#----------------------------------------------------------------------- +# +. ${GLOBAL_VAR_DEFNS_FP} +. $USHDIR/source_util_funcs.sh +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# +{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# # This script reorganizes the CCPA data into a more intuitive structure: # A valid YYYYMMDD directory is created, and all files for the valid day are placed within the directory. # Supported accumulations: 01h, 03h, and 06h. NOTE: Accumulation is currently hardcoded to 01h. # The verification uses MET/pcp-combine to sum 01h files into desired accumulations. +# +#----------------------------------------------------------------------- +# # Top-level CCPA directory ccpa_dir=${OBS_DIR}/.. @@ -224,4 +248,13 @@ while [[ ${current_fcst} -le ${fcst_length} ]]; do echo "Current fcst hr=${current_fcst}" done +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/func- +# tion. +# +#----------------------------------------------------------------------- +# +{ restore_shell_opts; } > /dev/null 2>&1 diff --git a/scripts/exregional_get_extrn_mdl_files.sh b/scripts/exregional_get_extrn_mdl_files.sh index 5a3fe04cf3..1529d3d8d8 100755 --- a/scripts/exregional_get_extrn_mdl_files.sh +++ b/scripts/exregional_get_extrn_mdl_files.sh @@ -17,7 +17,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u +x; } > /dev/null 2>&1 +{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_get_mrms_files.sh b/scripts/exregional_get_mrms_files.sh index d0abf8f9bb..b6ef59699b 100755 --- a/scripts/exregional_get_mrms_files.sh +++ b/scripts/exregional_get_mrms_files.sh @@ -1,10 +1,32 @@ #!/bin/bash +# +#----------------------------------------------------------------------- +# +# Source the variable definitions file and the bash utility functions. +# +#----------------------------------------------------------------------- +# . ${GLOBAL_VAR_DEFNS_FP} - +. $USHDIR/source_util_funcs.sh +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# +{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# # This script pulls MRMS data from the NOAA HPSS # Top-level MRMS directory -set -x +# +#----------------------------------------------------------------------- +# + mrms_dir=${OBS_DIR}/.. if [[ ! -d "$mrms_dir" ]]; then mkdir -p $mrms_dir @@ -139,3 +161,13 @@ while [[ ${cur_ut} -le ${end_valid_ut} ]]; do cur_ut=`expr ${start_valid_ut} + ${fcst_sec}` done +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/func- +# tion. +# +#----------------------------------------------------------------------- +# +{ restore_shell_opts; } > /dev/null 2>&1 + diff --git a/scripts/exregional_get_ndas_files.sh b/scripts/exregional_get_ndas_files.sh index 0105f2edb0..5592addb1a 100755 --- a/scripts/exregional_get_ndas_files.sh +++ b/scripts/exregional_get_ndas_files.sh @@ -1,7 +1,31 @@ #!/bin/bash +# +#----------------------------------------------------------------------- +# +# Source the variable definitions file and the bash utility functions. +# +#----------------------------------------------------------------------- +# +. ${GLOBAL_VAR_DEFNS_FP} +. $USHDIR/source_util_funcs.sh +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# +{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# # This script reorganizes the NDAS data into a more intuitive structure: # A valid YYYYMMDD directory is created, and all files for the valid day are placed within the directory. +# +#----------------------------------------------------------------------- +# # Top-level NDAS directory ndas_dir=${OBS_DIR}/.. @@ -143,3 +167,13 @@ echo "vhh_noZero=$vhh_noZero" echo "new fcst=${current_fcst}" done +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/func- +# tion. +# +#----------------------------------------------------------------------- +# +{ restore_shell_opts; } > /dev/null 2>&1 + diff --git a/scripts/exregional_make_grid.sh b/scripts/exregional_make_grid.sh index 73a478b421..b350a6ab8a 100755 --- a/scripts/exregional_make_grid.sh +++ b/scripts/exregional_make_grid.sh @@ -25,7 +25,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u +x; } > /dev/null 2>&1 +{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index b7a8dbe05a..ffd5e6c6bd 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -17,7 +17,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u +x; } > /dev/null 2>&1 +{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_make_lbcs.sh b/scripts/exregional_make_lbcs.sh index 6dc3ba369a..f4bf3a6056 100755 --- a/scripts/exregional_make_lbcs.sh +++ b/scripts/exregional_make_lbcs.sh @@ -17,7 +17,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u +x; } > /dev/null 2>&1 +{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_make_orog.sh b/scripts/exregional_make_orog.sh index 6589f6aa3b..d83cefd873 100755 --- a/scripts/exregional_make_orog.sh +++ b/scripts/exregional_make_orog.sh @@ -17,7 +17,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u +x; } > /dev/null 2>&1 +{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_make_sfc_climo.sh b/scripts/exregional_make_sfc_climo.sh index 061aed0dbb..5036656488 100755 --- a/scripts/exregional_make_sfc_climo.sh +++ b/scripts/exregional_make_sfc_climo.sh @@ -17,7 +17,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u +x; } > /dev/null 2>&1 +{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_run_ensgridvx.sh b/scripts/exregional_run_ensgridvx.sh index 8b8736ec42..99fc494838 100755 --- a/scripts/exregional_run_ensgridvx.sh +++ b/scripts/exregional_run_ensgridvx.sh @@ -1,5 +1,4 @@ #!/bin/bash -set -x # #----------------------------------------------------------------------- @@ -18,7 +17,7 @@ set -x # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u +x; } > /dev/null 2>&1 +{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_run_ensgridvx_mean.sh b/scripts/exregional_run_ensgridvx_mean.sh index faa56d5caa..2164ad8222 100755 --- a/scripts/exregional_run_ensgridvx_mean.sh +++ b/scripts/exregional_run_ensgridvx_mean.sh @@ -1,5 +1,4 @@ #!/bin/bash -set -x # #----------------------------------------------------------------------- @@ -18,7 +17,7 @@ set -x # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u +x; } > /dev/null 2>&1 +{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_run_ensgridvx_prob.sh b/scripts/exregional_run_ensgridvx_prob.sh index dc32f599a4..adfaffb4a6 100755 --- a/scripts/exregional_run_ensgridvx_prob.sh +++ b/scripts/exregional_run_ensgridvx_prob.sh @@ -1,5 +1,4 @@ #!/bin/bash -set -x # #----------------------------------------------------------------------- @@ -18,7 +17,7 @@ set -x # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u +x; } > /dev/null 2>&1 +{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_run_enspointvx.sh b/scripts/exregional_run_enspointvx.sh index 3ac4401c27..2cac598f7f 100755 --- a/scripts/exregional_run_enspointvx.sh +++ b/scripts/exregional_run_enspointvx.sh @@ -1,5 +1,4 @@ #!/bin/bash -set -x # #----------------------------------------------------------------------- @@ -18,7 +17,7 @@ set -x # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u +x; } > /dev/null 2>&1 +{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_run_enspointvx_mean.sh b/scripts/exregional_run_enspointvx_mean.sh index cd9ee7004c..d9341bb540 100755 --- a/scripts/exregional_run_enspointvx_mean.sh +++ b/scripts/exregional_run_enspointvx_mean.sh @@ -1,5 +1,4 @@ #!/bin/bash -set -x # #----------------------------------------------------------------------- @@ -18,7 +17,7 @@ set -x # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u +x; } > /dev/null 2>&1 +{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_run_enspointvx_prob.sh b/scripts/exregional_run_enspointvx_prob.sh index 4e025125cb..94cf10e92d 100755 --- a/scripts/exregional_run_enspointvx_prob.sh +++ b/scripts/exregional_run_enspointvx_prob.sh @@ -1,5 +1,4 @@ #!/bin/bash -set -x # #----------------------------------------------------------------------- @@ -18,7 +17,7 @@ set -x # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u +x; } > /dev/null 2>&1 +{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index eb9dc0bc49..f185d38b25 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -17,7 +17,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u +x; } > /dev/null 2>&1 +{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_run_gridstatvx.sh b/scripts/exregional_run_gridstatvx.sh index b6b5671814..8bce19afa9 100755 --- a/scripts/exregional_run_gridstatvx.sh +++ b/scripts/exregional_run_gridstatvx.sh @@ -1,5 +1,4 @@ #!/bin/bash -set -x # #----------------------------------------------------------------------- @@ -18,7 +17,7 @@ set -x # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u +x; } > /dev/null 2>&1 +{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_run_pointstatvx.sh b/scripts/exregional_run_pointstatvx.sh index 8afed1dd4a..580a61b706 100755 --- a/scripts/exregional_run_pointstatvx.sh +++ b/scripts/exregional_run_pointstatvx.sh @@ -1,5 +1,4 @@ #!/bin/bash -set -x # #----------------------------------------------------------------------- @@ -18,7 +17,7 @@ set -x # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u +x; } > /dev/null 2>&1 +{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_run_post.sh b/scripts/exregional_run_post.sh index cb8c3d5d5e..7dd476f175 100755 --- a/scripts/exregional_run_post.sh +++ b/scripts/exregional_run_post.sh @@ -17,7 +17,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u +x; } > /dev/null 2>&1 +{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/ush/load_modules_run_task.sh b/ush/load_modules_run_task.sh index f1a62ddc3a..310876a1ec 100755 --- a/ush/load_modules_run_task.sh +++ b/ush/load_modules_run_task.sh @@ -19,7 +19,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u +x; } > /dev/null 2>&1 +{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # From 950b4618a446db68799eb431f847f615db346f2f Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Fri, 12 Aug 2022 19:18:33 +0000 Subject: [PATCH 003/106] Call preamble in other scripts. --- tests/WE2E/get_WE2Etest_names_subdirs_descs.sh | 2 +- tests/WE2E/run_WE2E_tests.sh | 2 +- ush/cmp_expt_to_baseline.sh | 2 +- ush/init_env.sh | 2 +- ush/make_grid_mosaic_file.sh | 2 +- ush/preamble.sh | 2 +- ush/source_machine_file.sh | 10 +++++----- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh b/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh index 1dfe48483d..e277591320 100755 --- a/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh +++ b/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh @@ -252,7 +252,7 @@ function get_WE2Etest_names_subdirs_descs() { # #----------------------------------------------------------------------- # - { save_shell_opts; set -u +x; } > /dev/null 2>&1 + { save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/tests/WE2E/run_WE2E_tests.sh b/tests/WE2E/run_WE2E_tests.sh index af9b63b488..bc8a473248 100755 --- a/tests/WE2E/run_WE2E_tests.sh +++ b/tests/WE2E/run_WE2E_tests.sh @@ -70,7 +70,7 @@ WE2Edir="$testsdir/WE2E" # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u +x; } > /dev/null 2>&1 +{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/ush/cmp_expt_to_baseline.sh b/ush/cmp_expt_to_baseline.sh index b7459593ab..c7eab46524 100755 --- a/ush/cmp_expt_to_baseline.sh +++ b/ush/cmp_expt_to_baseline.sh @@ -47,7 +47,7 @@ scrfunc_dir=$( dirname "${scrfunc_fp}" ) # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u +x; } > /dev/null 2>&1 +{ save_shell_opts; . ${scrfunc_dir}/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/ush/init_env.sh b/ush/init_env.sh index d883766325..957688ad9f 100644 --- a/ush/init_env.sh +++ b/ush/init_env.sh @@ -13,7 +13,7 @@ # function init_env() { - { save_shell_opts; set -u +x; } > /dev/null 2>&1 + { save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 local valid_args=( \ "env_init_scripts_fps" \ diff --git a/ush/make_grid_mosaic_file.sh b/ush/make_grid_mosaic_file.sh index 8abba8c334..5d8d9f06c2 100644 --- a/ush/make_grid_mosaic_file.sh +++ b/ush/make_grid_mosaic_file.sh @@ -17,7 +17,7 @@ function make_grid_mosaic_file() { # #----------------------------------------------------------------------- # - { save_shell_opts; set -u +x; } > /dev/null 2>&1 + { save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/ush/preamble.sh b/ush/preamble.sh index 632011d1e9..d552a41196 100644 --- a/ush/preamble.sh +++ b/ush/preamble.sh @@ -51,7 +51,7 @@ fi if [[ $TRACE == "TRUE" ]]; then export TRACE_CMD="set -x" # Print the script name and line number of each command as it is executed - export PS4='+ $(basename $BASH_SOURCE)[$LINEFALSE]'"$id: " + export PS4='+ $(basename $BASH_SOURCE)[$LINENO]'"$id: " fi function postamble() { diff --git a/ush/source_machine_file.sh b/ush/source_machine_file.sh index 532335962b..6d7e0de0e1 100644 --- a/ush/source_machine_file.sh +++ b/ush/source_machine_file.sh @@ -1,16 +1,16 @@ #!/bin/bash +# +#----------------------------------------------------------------------- # # This script is a wrapper that sources the machine file but also allows # for certain pre- and post-source commands to be executed (e.g. setting # of "set -x" shell option depending on the DEBUG variable). # +#----------------------------------------------------------------------- +# -if [ -z ${DEBUG+x} ] || [ "$DEBUG" != "TRUE" ]; then - { save_shell_opts; set -u +x; } > /dev/null 2>&1 -else - { save_shell_opts; set -u -x; } > /dev/null 2>&1 -fi +{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 source ${MACHINE_FILE} From 223b50b726c6a3cb8032f5ba68e604e0d5e99636 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Fri, 12 Aug 2022 19:59:57 +0000 Subject: [PATCH 004/106] Make names of j-jobs and ex-scripts consistent. --- jobs/JREGIONAL_GET_OBS_CCPA | 2 +- jobs/JREGIONAL_GET_OBS_MRMS | 2 +- jobs/JREGIONAL_GET_OBS_NDAS | 2 +- jobs/JREGIONAL_RUN_VX_ENSGRID | 2 +- jobs/JREGIONAL_RUN_VX_ENSGRID_MEAN | 2 +- jobs/JREGIONAL_RUN_VX_ENSGRID_PROB | 2 +- jobs/JREGIONAL_RUN_VX_ENSPOINT | 2 +- jobs/JREGIONAL_RUN_VX_ENSPOINT_MEAN | 2 +- jobs/JREGIONAL_RUN_VX_ENSPOINT_PROB | 2 +- jobs/JREGIONAL_RUN_VX_GRIDSTAT | 2 +- jobs/JREGIONAL_RUN_VX_POINTSTAT | 2 +- ...{exregional_get_ccpa_files.sh => exregional_get_obs_ccpa.sh} | 0 ...{exregional_get_mrms_files.sh => exregional_get_obs_mrms.sh} | 0 ...{exregional_get_ndas_files.sh => exregional_get_obs_ndas.sh} | 0 ...exregional_run_ensgridvx.sh => exregional_run_vx_ensgrid.sh} | 0 ..._run_ensgridvx_mean.sh => exregional_run_vx_ensgrid_mean.sh} | 0 ..._run_ensgridvx_prob.sh => exregional_run_vx_ensgrid_prob.sh} | 0 ...regional_run_enspointvx.sh => exregional_run_vx_enspoint.sh} | 0 ...un_enspointvx_mean.sh => exregional_run_vx_enspoint_mean.sh} | 0 ...un_enspointvx_prob.sh => exregional_run_vx_enspoint_prob.sh} | 0 ...regional_run_gridstatvx.sh => exregional_run_vx_gridstat.sh} | 0 ...gional_run_pointstatvx.sh => exregional_run_vx_pointstat.sh} | 0 22 files changed, 11 insertions(+), 11 deletions(-) rename scripts/{exregional_get_ccpa_files.sh => exregional_get_obs_ccpa.sh} (100%) rename scripts/{exregional_get_mrms_files.sh => exregional_get_obs_mrms.sh} (100%) rename scripts/{exregional_get_ndas_files.sh => exregional_get_obs_ndas.sh} (100%) rename scripts/{exregional_run_ensgridvx.sh => exregional_run_vx_ensgrid.sh} (100%) rename scripts/{exregional_run_ensgridvx_mean.sh => exregional_run_vx_ensgrid_mean.sh} (100%) rename scripts/{exregional_run_ensgridvx_prob.sh => exregional_run_vx_ensgrid_prob.sh} (100%) rename scripts/{exregional_run_enspointvx.sh => exregional_run_vx_enspoint.sh} (100%) rename scripts/{exregional_run_enspointvx_mean.sh => exregional_run_vx_enspoint_mean.sh} (100%) rename scripts/{exregional_run_enspointvx_prob.sh => exregional_run_vx_enspoint_prob.sh} (100%) rename scripts/{exregional_run_gridstatvx.sh => exregional_run_vx_gridstat.sh} (100%) rename scripts/{exregional_run_pointstatvx.sh => exregional_run_vx_pointstat.sh} (100%) diff --git a/jobs/JREGIONAL_GET_OBS_CCPA b/jobs/JREGIONAL_GET_OBS_CCPA index 2fb9067d96..0a3d101569 100755 --- a/jobs/JREGIONAL_GET_OBS_CCPA +++ b/jobs/JREGIONAL_GET_OBS_CCPA @@ -67,7 +67,7 @@ for verification purposes. # #----------------------------------------------------------------------- # -$SCRIPTSDIR/exregional_get_ccpa_files.sh || \ +$SCRIPTSDIR/exregional_get_obs_ccpa.sh || \ print_err_msg_exit "\ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." # diff --git a/jobs/JREGIONAL_GET_OBS_MRMS b/jobs/JREGIONAL_GET_OBS_MRMS index 9c66d3bbd0..951cd8e272 100755 --- a/jobs/JREGIONAL_GET_OBS_MRMS +++ b/jobs/JREGIONAL_GET_OBS_MRMS @@ -67,7 +67,7 @@ echo "VAR=${VAR}" for field in ${VAR[@]}; do export field echo "Field=${field}" - $SCRIPTSDIR/exregional_get_mrms_files.sh || \ + $SCRIPTSDIR/exregional_get_obs_mrms.sh || \ print_err_msg_exit "\ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." done diff --git a/jobs/JREGIONAL_GET_OBS_NDAS b/jobs/JREGIONAL_GET_OBS_NDAS index 132f1e4936..2d6bee77a0 100755 --- a/jobs/JREGIONAL_GET_OBS_NDAS +++ b/jobs/JREGIONAL_GET_OBS_NDAS @@ -63,7 +63,7 @@ for verification purposes. # #----------------------------------------------------------------------- # -$SCRIPTSDIR/exregional_get_ndas_files.sh || \ +$SCRIPTSDIR/exregional_get_obs_ndas.sh || \ print_err_msg_exit "\ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." # diff --git a/jobs/JREGIONAL_RUN_VX_ENSGRID b/jobs/JREGIONAL_RUN_VX_ENSGRID index 2c2f74a618..e66d696620 100755 --- a/jobs/JREGIONAL_RUN_VX_ENSGRID +++ b/jobs/JREGIONAL_RUN_VX_ENSGRID @@ -62,7 +62,7 @@ on gridded fields by initialization time for all forecast hours. # #----------------------------------------------------------------------- # -$SCRIPTSDIR/exregional_run_ensgridvx.sh \ +$SCRIPTSDIR/exregional_run_vx_ensgrid.sh \ cycle_dir="${CYCLE_DIR}" || \ print_err_msg_exit "\ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." diff --git a/jobs/JREGIONAL_RUN_VX_ENSGRID_MEAN b/jobs/JREGIONAL_RUN_VX_ENSGRID_MEAN index 208cd80e67..7bcc601c09 100755 --- a/jobs/JREGIONAL_RUN_VX_ENSGRID_MEAN +++ b/jobs/JREGIONAL_RUN_VX_ENSGRID_MEAN @@ -62,7 +62,7 @@ on gridded fields by initialization time for all forecast hours. # #----------------------------------------------------------------------- # -$SCRIPTSDIR/exregional_run_ensgridvx_mean.sh \ +$SCRIPTSDIR/exregional_run_vx_ensgrid_mean.sh \ cycle_dir="${CYCLE_DIR}" || \ print_err_msg_exit "\ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." diff --git a/jobs/JREGIONAL_RUN_VX_ENSGRID_PROB b/jobs/JREGIONAL_RUN_VX_ENSGRID_PROB index c649d9177d..7a7a4614bf 100755 --- a/jobs/JREGIONAL_RUN_VX_ENSGRID_PROB +++ b/jobs/JREGIONAL_RUN_VX_ENSGRID_PROB @@ -62,7 +62,7 @@ on gridded fields by initialization time for all forecast hours. # #----------------------------------------------------------------------- # -$SCRIPTSDIR/exregional_run_ensgridvx_prob.sh \ +$SCRIPTSDIR/exregional_run_vx_ensgrid_prob.sh \ cycle_dir="${CYCLE_DIR}" || \ print_err_msg_exit "\ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." diff --git a/jobs/JREGIONAL_RUN_VX_ENSPOINT b/jobs/JREGIONAL_RUN_VX_ENSPOINT index c844b72020..56e1409707 100755 --- a/jobs/JREGIONAL_RUN_VX_ENSPOINT +++ b/jobs/JREGIONAL_RUN_VX_ENSPOINT @@ -60,7 +60,7 @@ by initialitation time for all forecast hours. # #----------------------------------------------------------------------- # -$SCRIPTSDIR/exregional_run_enspointvx.sh \ +$SCRIPTSDIR/exregional_run_vx_enspoint.sh \ cycle_dir="${CYCLE_DIR}" || \ print_err_msg_exit "\ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." diff --git a/jobs/JREGIONAL_RUN_VX_ENSPOINT_MEAN b/jobs/JREGIONAL_RUN_VX_ENSPOINT_MEAN index 9a00a05176..c58c6efa07 100755 --- a/jobs/JREGIONAL_RUN_VX_ENSPOINT_MEAN +++ b/jobs/JREGIONAL_RUN_VX_ENSPOINT_MEAN @@ -60,7 +60,7 @@ by initialitation time for all forecast hours. # #----------------------------------------------------------------------- # -$SCRIPTSDIR/exregional_run_enspointvx_mean.sh \ +$SCRIPTSDIR/exregional_run_vx_enspoint_mean.sh \ cycle_dir="${CYCLE_DIR}" || \ print_err_msg_exit "\ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." diff --git a/jobs/JREGIONAL_RUN_VX_ENSPOINT_PROB b/jobs/JREGIONAL_RUN_VX_ENSPOINT_PROB index 0320c9162a..cafbb58dea 100755 --- a/jobs/JREGIONAL_RUN_VX_ENSPOINT_PROB +++ b/jobs/JREGIONAL_RUN_VX_ENSPOINT_PROB @@ -60,7 +60,7 @@ by initialitation time for all forecast hours. # #----------------------------------------------------------------------- # -$SCRIPTSDIR/exregional_run_enspointvx_prob.sh \ +$SCRIPTSDIR/exregional_run_vx_enspoint_prob.sh \ cycle_dir="${CYCLE_DIR}" || \ print_err_msg_exit "\ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." diff --git a/jobs/JREGIONAL_RUN_VX_GRIDSTAT b/jobs/JREGIONAL_RUN_VX_GRIDSTAT index 10ebb74fa2..28ea5d94a3 100755 --- a/jobs/JREGIONAL_RUN_VX_GRIDSTAT +++ b/jobs/JREGIONAL_RUN_VX_GRIDSTAT @@ -62,7 +62,7 @@ by initialization time for all forecast hours. # #----------------------------------------------------------------------- # -$SCRIPTSDIR/exregional_run_gridstatvx.sh \ +$SCRIPTSDIR/exregional_run_vx_gridstat.sh \ cycle_dir="${CYCLE_DIR}" || \ print_err_msg_exit "\ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." diff --git a/jobs/JREGIONAL_RUN_VX_POINTSTAT b/jobs/JREGIONAL_RUN_VX_POINTSTAT index bd83283ce9..b5a5be324c 100755 --- a/jobs/JREGIONAL_RUN_VX_POINTSTAT +++ b/jobs/JREGIONAL_RUN_VX_POINTSTAT @@ -60,7 +60,7 @@ by initialitation time for all forecast hours. # #----------------------------------------------------------------------- # -$SCRIPTSDIR/exregional_run_pointstatvx.sh \ +$SCRIPTSDIR/exregional_run_vx_pointstat.sh \ cycle_dir="${CYCLE_DIR}" || \ print_err_msg_exit "\ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." diff --git a/scripts/exregional_get_ccpa_files.sh b/scripts/exregional_get_obs_ccpa.sh similarity index 100% rename from scripts/exregional_get_ccpa_files.sh rename to scripts/exregional_get_obs_ccpa.sh diff --git a/scripts/exregional_get_mrms_files.sh b/scripts/exregional_get_obs_mrms.sh similarity index 100% rename from scripts/exregional_get_mrms_files.sh rename to scripts/exregional_get_obs_mrms.sh diff --git a/scripts/exregional_get_ndas_files.sh b/scripts/exregional_get_obs_ndas.sh similarity index 100% rename from scripts/exregional_get_ndas_files.sh rename to scripts/exregional_get_obs_ndas.sh diff --git a/scripts/exregional_run_ensgridvx.sh b/scripts/exregional_run_vx_ensgrid.sh similarity index 100% rename from scripts/exregional_run_ensgridvx.sh rename to scripts/exregional_run_vx_ensgrid.sh diff --git a/scripts/exregional_run_ensgridvx_mean.sh b/scripts/exregional_run_vx_ensgrid_mean.sh similarity index 100% rename from scripts/exregional_run_ensgridvx_mean.sh rename to scripts/exregional_run_vx_ensgrid_mean.sh diff --git a/scripts/exregional_run_ensgridvx_prob.sh b/scripts/exregional_run_vx_ensgrid_prob.sh similarity index 100% rename from scripts/exregional_run_ensgridvx_prob.sh rename to scripts/exregional_run_vx_ensgrid_prob.sh diff --git a/scripts/exregional_run_enspointvx.sh b/scripts/exregional_run_vx_enspoint.sh similarity index 100% rename from scripts/exregional_run_enspointvx.sh rename to scripts/exregional_run_vx_enspoint.sh diff --git a/scripts/exregional_run_enspointvx_mean.sh b/scripts/exregional_run_vx_enspoint_mean.sh similarity index 100% rename from scripts/exregional_run_enspointvx_mean.sh rename to scripts/exregional_run_vx_enspoint_mean.sh diff --git a/scripts/exregional_run_enspointvx_prob.sh b/scripts/exregional_run_vx_enspoint_prob.sh similarity index 100% rename from scripts/exregional_run_enspointvx_prob.sh rename to scripts/exregional_run_vx_enspoint_prob.sh diff --git a/scripts/exregional_run_gridstatvx.sh b/scripts/exregional_run_vx_gridstat.sh similarity index 100% rename from scripts/exregional_run_gridstatvx.sh rename to scripts/exregional_run_vx_gridstat.sh diff --git a/scripts/exregional_run_pointstatvx.sh b/scripts/exregional_run_vx_pointstat.sh similarity index 100% rename from scripts/exregional_run_pointstatvx.sh rename to scripts/exregional_run_vx_pointstat.sh From 8e91ad9c56af64af6c2231944b9e3094075b5e6e Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Sat, 13 Aug 2022 16:00:28 +0000 Subject: [PATCH 005/106] Working towards nco vars in table 1. --- jobs/JREGIONAL_GET_EXTRN_MDL_FILES | 4 +- jobs/JREGIONAL_GET_OBS_CCPA | 4 +- jobs/JREGIONAL_GET_OBS_MRMS | 4 +- jobs/JREGIONAL_GET_OBS_NDAS | 4 +- jobs/JREGIONAL_MAKE_GRID | 4 +- jobs/JREGIONAL_MAKE_ICS | 4 +- jobs/JREGIONAL_MAKE_LBCS | 4 +- jobs/JREGIONAL_MAKE_OROG | 4 +- jobs/JREGIONAL_MAKE_SFC_CLIMO | 4 +- jobs/JREGIONAL_RUN_FCST | 4 +- jobs/JREGIONAL_RUN_POST | 4 +- jobs/JREGIONAL_RUN_VX_ENSGRID | 4 +- jobs/JREGIONAL_RUN_VX_ENSGRID_MEAN | 4 +- jobs/JREGIONAL_RUN_VX_ENSGRID_PROB | 4 +- jobs/JREGIONAL_RUN_VX_ENSPOINT | 4 +- jobs/JREGIONAL_RUN_VX_ENSPOINT_MEAN | 4 +- jobs/JREGIONAL_RUN_VX_ENSPOINT_PROB | 4 +- jobs/JREGIONAL_RUN_VX_GRIDSTAT | 4 +- jobs/JREGIONAL_RUN_VX_POINTSTAT | 4 +- scripts/exregional_get_extrn_mdl_files.sh | 8 ++-- scripts/exregional_get_obs_ccpa.sh | 4 +- scripts/exregional_get_obs_mrms.sh | 6 +-- scripts/exregional_get_obs_ndas.sh | 4 +- scripts/exregional_make_grid.sh | 20 +++++----- scripts/exregional_make_ics.sh | 12 +++--- scripts/exregional_make_lbcs.sh | 10 ++--- scripts/exregional_make_orog.sh | 16 ++++---- scripts/exregional_make_sfc_climo.sh | 10 ++--- scripts/exregional_run_fcst.sh | 12 +++--- scripts/exregional_run_post.sh | 8 ++-- scripts/exregional_run_vx_ensgrid.sh | 4 +- scripts/exregional_run_vx_ensgrid_mean.sh | 4 +- scripts/exregional_run_vx_ensgrid_prob.sh | 4 +- scripts/exregional_run_vx_enspoint.sh | 4 +- scripts/exregional_run_vx_enspoint_mean.sh | 4 +- scripts/exregional_run_vx_enspoint_prob.sh | 4 +- scripts/exregional_run_vx_gridstat.sh | 4 +- scripts/exregional_run_vx_pointstat.sh | 4 +- ush/calculate_cost.py | 2 +- ush/config_defaults.yaml | 8 +++- ush/create_diag_table_file.py | 8 ++-- ush/create_model_configure_file.py | 8 ++-- ush/generate_FV3LAM_wflow.py | 4 +- ush/init_env.sh | 2 +- ush/launch_FV3LAM_wflow.sh | 12 +++--- ush/load_modules_run_task.sh | 8 ++-- ush/make_grid_mosaic_file.sh | 4 +- ush/set_FV3nml_ens_stoch_seeds.py | 8 ++-- ush/set_FV3nml_sfc_climo_filenames.py | 8 ++-- ush/set_predef_grid_params.py | 4 +- ush/setup.py | 45 ++++++++++++---------- ush/source_machine_file.sh | 2 +- 52 files changed, 174 insertions(+), 167 deletions(-) diff --git a/jobs/JREGIONAL_GET_EXTRN_MDL_FILES b/jobs/JREGIONAL_GET_EXTRN_MDL_FILES index b528c38563..3ad85fd582 100755 --- a/jobs/JREGIONAL_GET_EXTRN_MDL_FILES +++ b/jobs/JREGIONAL_GET_EXTRN_MDL_FILES @@ -29,7 +29,7 @@ #----------------------------------------------------------------------- # . ${GLOBAL_VAR_DEFNS_FP} -. $USHDIR/source_util_funcs.sh +. $USHrrfs/source_util_funcs.sh # #----------------------------------------------------------------------- # @@ -38,7 +38,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 +{ save_shell_opts; . $USHrrfs/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_GET_OBS_CCPA b/jobs/JREGIONAL_GET_OBS_CCPA index 0a3d101569..bc5d5f25e7 100755 --- a/jobs/JREGIONAL_GET_OBS_CCPA +++ b/jobs/JREGIONAL_GET_OBS_CCPA @@ -20,7 +20,7 @@ #----------------------------------------------------------------------- # . ${GLOBAL_VAR_DEFNS_FP} -. $USHDIR/source_util_funcs.sh +. $USHrrfs/source_util_funcs.sh # #----------------------------------------------------------------------- # @@ -29,7 +29,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 +{ save_shell_opts; . $USHrrfs/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_GET_OBS_MRMS b/jobs/JREGIONAL_GET_OBS_MRMS index 951cd8e272..62be9861e5 100755 --- a/jobs/JREGIONAL_GET_OBS_MRMS +++ b/jobs/JREGIONAL_GET_OBS_MRMS @@ -16,7 +16,7 @@ #----------------------------------------------------------------------- # . ${GLOBAL_VAR_DEFNS_FP} -. $USHDIR/source_util_funcs.sh +. $USHrrfs/source_util_funcs.sh # #----------------------------------------------------------------------- # @@ -25,7 +25,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 +{ save_shell_opts; . $USHrrfs/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_GET_OBS_NDAS b/jobs/JREGIONAL_GET_OBS_NDAS index 2d6bee77a0..218335a435 100755 --- a/jobs/JREGIONAL_GET_OBS_NDAS +++ b/jobs/JREGIONAL_GET_OBS_NDAS @@ -16,7 +16,7 @@ #----------------------------------------------------------------------- # . ${GLOBAL_VAR_DEFNS_FP} -. $USHDIR/source_util_funcs.sh +. $USHrrfs/source_util_funcs.sh # #----------------------------------------------------------------------- # @@ -25,7 +25,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 +{ save_shell_opts; . $USHrrfs/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_MAKE_GRID b/jobs/JREGIONAL_MAKE_GRID index 451cbe635e..44bfb48db6 100755 --- a/jobs/JREGIONAL_MAKE_GRID +++ b/jobs/JREGIONAL_MAKE_GRID @@ -106,7 +106,7 @@ #----------------------------------------------------------------------- # . ${GLOBAL_VAR_DEFNS_FP} -. $USHDIR/source_util_funcs.sh +. $USHrrfs/source_util_funcs.sh # #----------------------------------------------------------------------- # @@ -115,7 +115,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 +{ save_shell_opts; . $USHrrfs/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_MAKE_ICS b/jobs/JREGIONAL_MAKE_ICS index 99ba807577..d07b11db4e 100755 --- a/jobs/JREGIONAL_MAKE_ICS +++ b/jobs/JREGIONAL_MAKE_ICS @@ -8,7 +8,7 @@ #----------------------------------------------------------------------- # . ${GLOBAL_VAR_DEFNS_FP} -. $USHDIR/source_util_funcs.sh +. $USHrrfs/source_util_funcs.sh # #----------------------------------------------------------------------- # @@ -17,7 +17,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 +{ save_shell_opts; . $USHrrfs/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_MAKE_LBCS b/jobs/JREGIONAL_MAKE_LBCS index 30d8bb56c4..8f669b2da8 100755 --- a/jobs/JREGIONAL_MAKE_LBCS +++ b/jobs/JREGIONAL_MAKE_LBCS @@ -8,7 +8,7 @@ #----------------------------------------------------------------------- # . ${GLOBAL_VAR_DEFNS_FP} -. $USHDIR/source_util_funcs.sh +. $USHrrfs/source_util_funcs.sh # #----------------------------------------------------------------------- # @@ -17,7 +17,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 +{ save_shell_opts; . $USHrrfs/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_MAKE_OROG b/jobs/JREGIONAL_MAKE_OROG index 2fdc5c983f..6a697cd88f 100755 --- a/jobs/JREGIONAL_MAKE_OROG +++ b/jobs/JREGIONAL_MAKE_OROG @@ -8,7 +8,7 @@ #----------------------------------------------------------------------- # . ${GLOBAL_VAR_DEFNS_FP} -. $USHDIR/source_util_funcs.sh +. $USHrrfs/source_util_funcs.sh # #----------------------------------------------------------------------- # @@ -17,7 +17,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 +{ save_shell_opts; . $USHrrfs/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_MAKE_SFC_CLIMO b/jobs/JREGIONAL_MAKE_SFC_CLIMO index bf0100af06..2219c40075 100755 --- a/jobs/JREGIONAL_MAKE_SFC_CLIMO +++ b/jobs/JREGIONAL_MAKE_SFC_CLIMO @@ -8,7 +8,7 @@ #----------------------------------------------------------------------- # . ${GLOBAL_VAR_DEFNS_FP} -. $USHDIR/source_util_funcs.sh +. $USHrrfs/source_util_funcs.sh # #----------------------------------------------------------------------- # @@ -17,7 +17,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 +{ save_shell_opts; . $USHrrfs/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_RUN_FCST b/jobs/JREGIONAL_RUN_FCST index 962dfc45ee..1a6a7de6f4 100755 --- a/jobs/JREGIONAL_RUN_FCST +++ b/jobs/JREGIONAL_RUN_FCST @@ -18,7 +18,7 @@ #----------------------------------------------------------------------- # . ${GLOBAL_VAR_DEFNS_FP} -. $USHDIR/source_util_funcs.sh +. $USHrrfs/source_util_funcs.sh # #----------------------------------------------------------------------- # @@ -27,7 +27,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 +{ save_shell_opts; . $USHrrfs/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_RUN_POST b/jobs/JREGIONAL_RUN_POST index cbddabc9f9..ad8c85c177 100755 --- a/jobs/JREGIONAL_RUN_POST +++ b/jobs/JREGIONAL_RUN_POST @@ -17,7 +17,7 @@ #----------------------------------------------------------------------- # . ${GLOBAL_VAR_DEFNS_FP} -. $USHDIR/source_util_funcs.sh +. $USHrrfs/source_util_funcs.sh # #----------------------------------------------------------------------- # @@ -26,7 +26,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 +{ save_shell_opts; . $USHrrfs/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_RUN_VX_ENSGRID b/jobs/JREGIONAL_RUN_VX_ENSGRID index e66d696620..eb58ec8d01 100755 --- a/jobs/JREGIONAL_RUN_VX_ENSGRID +++ b/jobs/JREGIONAL_RUN_VX_ENSGRID @@ -17,7 +17,7 @@ #----------------------------------------------------------------------- # . ${GLOBAL_VAR_DEFNS_FP} -. $USHDIR/source_util_funcs.sh +. $USHrrfs/source_util_funcs.sh # #----------------------------------------------------------------------- # @@ -26,7 +26,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 +{ save_shell_opts; . $USHrrfs/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_RUN_VX_ENSGRID_MEAN b/jobs/JREGIONAL_RUN_VX_ENSGRID_MEAN index 7bcc601c09..fc84ac8260 100755 --- a/jobs/JREGIONAL_RUN_VX_ENSGRID_MEAN +++ b/jobs/JREGIONAL_RUN_VX_ENSGRID_MEAN @@ -17,7 +17,7 @@ #----------------------------------------------------------------------- # . ${GLOBAL_VAR_DEFNS_FP} -. $USHDIR/source_util_funcs.sh +. $USHrrfs/source_util_funcs.sh # #----------------------------------------------------------------------- # @@ -26,7 +26,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 +{ save_shell_opts; . $USHrrfs/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_RUN_VX_ENSGRID_PROB b/jobs/JREGIONAL_RUN_VX_ENSGRID_PROB index 7a7a4614bf..b83e9de35f 100755 --- a/jobs/JREGIONAL_RUN_VX_ENSGRID_PROB +++ b/jobs/JREGIONAL_RUN_VX_ENSGRID_PROB @@ -17,7 +17,7 @@ #----------------------------------------------------------------------- # . ${GLOBAL_VAR_DEFNS_FP} -. $USHDIR/source_util_funcs.sh +. $USHrrfs/source_util_funcs.sh # #----------------------------------------------------------------------- # @@ -26,7 +26,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 +{ save_shell_opts; . $USHrrfs/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_RUN_VX_ENSPOINT b/jobs/JREGIONAL_RUN_VX_ENSPOINT index 56e1409707..44ea4187d5 100755 --- a/jobs/JREGIONAL_RUN_VX_ENSPOINT +++ b/jobs/JREGIONAL_RUN_VX_ENSPOINT @@ -15,7 +15,7 @@ #----------------------------------------------------------------------- # . ${GLOBAL_VAR_DEFNS_FP} -. $USHDIR/source_util_funcs.sh +. $USHrrfs/source_util_funcs.sh # #----------------------------------------------------------------------- # @@ -24,7 +24,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 +{ save_shell_opts; . $USHrrfs/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_RUN_VX_ENSPOINT_MEAN b/jobs/JREGIONAL_RUN_VX_ENSPOINT_MEAN index c58c6efa07..08fe8c7fd8 100755 --- a/jobs/JREGIONAL_RUN_VX_ENSPOINT_MEAN +++ b/jobs/JREGIONAL_RUN_VX_ENSPOINT_MEAN @@ -15,7 +15,7 @@ #----------------------------------------------------------------------- # . ${GLOBAL_VAR_DEFNS_FP} -. $USHDIR/source_util_funcs.sh +. $USHrrfs/source_util_funcs.sh # #----------------------------------------------------------------------- # @@ -24,7 +24,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 +{ save_shell_opts; . $USHrrfs/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_RUN_VX_ENSPOINT_PROB b/jobs/JREGIONAL_RUN_VX_ENSPOINT_PROB index cafbb58dea..8a54570430 100755 --- a/jobs/JREGIONAL_RUN_VX_ENSPOINT_PROB +++ b/jobs/JREGIONAL_RUN_VX_ENSPOINT_PROB @@ -15,7 +15,7 @@ #----------------------------------------------------------------------- # . ${GLOBAL_VAR_DEFNS_FP} -. $USHDIR/source_util_funcs.sh +. $USHrrfs/source_util_funcs.sh # #----------------------------------------------------------------------- # @@ -24,7 +24,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 +{ save_shell_opts; . $USHrrfs/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_RUN_VX_GRIDSTAT b/jobs/JREGIONAL_RUN_VX_GRIDSTAT index 28ea5d94a3..cce81b004f 100755 --- a/jobs/JREGIONAL_RUN_VX_GRIDSTAT +++ b/jobs/JREGIONAL_RUN_VX_GRIDSTAT @@ -17,7 +17,7 @@ #----------------------------------------------------------------------- # . ${GLOBAL_VAR_DEFNS_FP} -. $USHDIR/source_util_funcs.sh +. $USHrrfs/source_util_funcs.sh # #----------------------------------------------------------------------- # @@ -26,7 +26,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 +{ save_shell_opts; . $USHrrfs/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_RUN_VX_POINTSTAT b/jobs/JREGIONAL_RUN_VX_POINTSTAT index b5a5be324c..b075bb9caf 100755 --- a/jobs/JREGIONAL_RUN_VX_POINTSTAT +++ b/jobs/JREGIONAL_RUN_VX_POINTSTAT @@ -15,7 +15,7 @@ #----------------------------------------------------------------------- # . ${GLOBAL_VAR_DEFNS_FP} -. $USHDIR/source_util_funcs.sh +. $USHrrfs/source_util_funcs.sh # #----------------------------------------------------------------------- # @@ -24,7 +24,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 +{ save_shell_opts; . $USHrrfs/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_get_extrn_mdl_files.sh b/scripts/exregional_get_extrn_mdl_files.sh index 1529d3d8d8..f1267b26ff 100755 --- a/scripts/exregional_get_extrn_mdl_files.sh +++ b/scripts/exregional_get_extrn_mdl_files.sh @@ -8,7 +8,7 @@ #----------------------------------------------------------------------- # . ${GLOBAL_VAR_DEFNS_FP} -. $USHDIR/source_util_funcs.sh +. $USHrrfs/source_util_funcs.sh # #----------------------------------------------------------------------- # @@ -17,7 +17,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 +{ save_shell_opts; . $USHrrfs/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # @@ -154,10 +154,10 @@ fi #----------------------------------------------------------------------- # cmd=" -python3 -u ${USHDIR}/retrieve_data.py \ +python3 -u ${USHrrfs}/retrieve_data.py \ --debug \ --anl_or_fcst ${anl_or_fcst} \ - --config ${USHDIR}/templates/data_locations.yml \ + --config ${USHrrfs}/templates/data_locations.yml \ --cycle_date ${extrn_mdl_cdate} \ --data_stores ${data_stores} \ --external_model ${extrn_mdl_name} \ diff --git a/scripts/exregional_get_obs_ccpa.sh b/scripts/exregional_get_obs_ccpa.sh index 9a404e4557..729e1feab8 100755 --- a/scripts/exregional_get_obs_ccpa.sh +++ b/scripts/exregional_get_obs_ccpa.sh @@ -8,7 +8,7 @@ #----------------------------------------------------------------------- # . ${GLOBAL_VAR_DEFNS_FP} -. $USHDIR/source_util_funcs.sh +. $USHrrfs/source_util_funcs.sh # #----------------------------------------------------------------------- # @@ -17,7 +17,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 +{ save_shell_opts; . $USHrrfs/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_get_obs_mrms.sh b/scripts/exregional_get_obs_mrms.sh index b6ef59699b..b0c7d805cd 100755 --- a/scripts/exregional_get_obs_mrms.sh +++ b/scripts/exregional_get_obs_mrms.sh @@ -8,7 +8,7 @@ #----------------------------------------------------------------------- # . ${GLOBAL_VAR_DEFNS_FP} -. $USHDIR/source_util_funcs.sh +. $USHrrfs/source_util_funcs.sh # #----------------------------------------------------------------------- # @@ -17,7 +17,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 +{ save_shell_opts; . $USHrrfs/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # @@ -146,7 +146,7 @@ while [[ ${cur_ut} -le ${end_valid_ut} ]]; do hour=0 while [[ ${hour} -le 23 ]]; do echo "hour=${hour}" - python ${USHDIR}/mrms_pull_topofhour.py ${vyyyy}${vmm}${vdd}${hour} ${mrms_proc} ${mrms_raw} ${field_base_name} ${level} + python ${USHrrfs}/mrms_pull_topofhour.py ${vyyyy}${vmm}${vdd}${hour} ${mrms_proc} ${mrms_raw} ${field_base_name} ${level} hour=$((${hour} + 1)) # hourly increment done fi diff --git a/scripts/exregional_get_obs_ndas.sh b/scripts/exregional_get_obs_ndas.sh index 5592addb1a..4c5520de17 100755 --- a/scripts/exregional_get_obs_ndas.sh +++ b/scripts/exregional_get_obs_ndas.sh @@ -8,7 +8,7 @@ #----------------------------------------------------------------------- # . ${GLOBAL_VAR_DEFNS_FP} -. $USHDIR/source_util_funcs.sh +. $USHrrfs/source_util_funcs.sh # #----------------------------------------------------------------------- # @@ -17,7 +17,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 +{ save_shell_opts; . $USHrrfs/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_make_grid.sh b/scripts/exregional_make_grid.sh index b350a6ab8a..6e73f4fc5e 100755 --- a/scripts/exregional_make_grid.sh +++ b/scripts/exregional_make_grid.sh @@ -8,7 +8,7 @@ #----------------------------------------------------------------------- # . ${GLOBAL_VAR_DEFNS_FP} -. $USHDIR/source_util_funcs.sh +. $USHrrfs/source_util_funcs.sh # #----------------------------------------------------------------------- # @@ -16,7 +16,7 @@ # #----------------------------------------------------------------------- # -. $USHDIR/make_grid_mosaic_file.sh +. $USHrrfs/make_grid_mosaic_file.sh # #----------------------------------------------------------------------- # @@ -25,7 +25,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 +{ save_shell_opts; . $USHrrfs/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # @@ -81,7 +81,7 @@ print_input_args valid_args # #----------------------------------------------------------------------- # -source $USHDIR/source_machine_file.sh +source $USHrrfs/source_machine_file.sh eval ${PRE_TASK_CMDS} if [ -z "${RUN_CMD_SERIAL:-}" ] ; then @@ -222,7 +222,7 @@ elif [ "${GRID_GEN_METHOD}" = "ESGgrid" ]; then exec_fn="regional_esg_grid" fi -exec_fp="$EXECDIR/${exec_fn}" +exec_fp="$EXECrrfs/${exec_fn}" if [ ! -f "${exec_fp}" ]; then print_err_msg_exit "\ The executable (exec_fp) for generating the grid file does not exist: @@ -313,7 +313,7 @@ generation executable (exec_fp): # # Call the python script to create the namelist file. # - ${USHDIR}/set_namelist.py -q -u "$settings" -o ${rgnl_grid_nml_fp} || \ + ${USHrrfs}/set_namelist.py -q -u "$settings" -o ${rgnl_grid_nml_fp} || \ print_err_msg_exit "\ Call to python script set_namelist.py to set the variables in the regional_esg_grid namelist file failed. Parameters passed to this script @@ -356,7 +356,7 @@ Grid file generation completed successfully." #----------------------------------------------------------------------- # exec_fn="global_equiv_resol" -exec_fp="$EXECDIR/${exec_fn}" +exec_fp="$EXECrrfs/${exec_fn}" if [ ! -f "${exec_fp}" ]; then print_err_msg_exit "\ The executable (exec_fp) for calculating the regional grid's global uniform @@ -458,7 +458,7 @@ fi # Set the name and path to the executable and make sure that it exists. # exec_fn="shave" -exec_fp="$EXECDIR/${exec_fn}" +exec_fp="$EXECrrfs/${exec_fn}" if [ ! -f "${exec_fp}" ]; then print_err_msg_exit " \ The executable (exec_fp) for \"shaving\" down the halo in the grid file @@ -587,7 +587,7 @@ halo failed." # #----------------------------------------------------------------------- # -python3 $USHDIR/link_fix.py \ +python3 $USHrrfs/link_fix.py \ --path-to-defns ${GLOBAL_VAR_DEFNS_FP} \ --file-group "grid" || \ print_err_msg_exit "\ @@ -605,7 +605,7 @@ failed." # #----------------------------------------------------------------------- # -python3 $USHDIR/set_FV3nml_sfc_climo_filenames.py \ +python3 $USHrrfs/set_FV3nml_sfc_climo_filenames.py \ --path-to-defns ${GLOBAL_VAR_DEFNS_FP} \ || print_err_msg_exit "\ Call to function to set surface climatology file names in the FV3 namelist diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index ffd5e6c6bd..f1bf4c83a6 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -8,7 +8,7 @@ #----------------------------------------------------------------------- # . ${GLOBAL_VAR_DEFNS_FP} -. $USHDIR/source_util_funcs.sh +. $USHrrfs/source_util_funcs.sh # #----------------------------------------------------------------------- # @@ -17,7 +17,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 +{ save_shell_opts; . $USHrrfs/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # @@ -86,7 +86,7 @@ export OMP_STACKSIZE=${OMP_STACKSIZE_MAKE_ICS} # #----------------------------------------------------------------------- # -source $USHDIR/source_machine_file.sh +source $USHrrfs/source_machine_file.sh eval ${PRE_TASK_CMDS} nprocs=$(( NNODES_MAKE_ICS*PPN_MAKE_ICS )) @@ -483,7 +483,7 @@ hh="${EXTRN_MDL_CDATE:8:2}" #----------------------------------------------------------------------- # exec_fn="chgres_cube" -exec_fp="$EXECDIR/${exec_fn}" +exec_fp="$EXECrrfs/${exec_fn}" if [ ! -f "${exec_fp}" ]; then print_err_msg_exit "\ The executable (exec_fp) for generating initial conditions on the FV3-LAM @@ -555,7 +555,7 @@ settings=" # Call the python script to create the namelist file. # nml_fn="fort.41" -${USHDIR}/set_namelist.py -q -u "$settings" -o ${nml_fn} || \ +${USHrrfs}/set_namelist.py -q -u "$settings" -o ${nml_fn} || \ print_err_msg_exit "\ Call to python script set_namelist.py to set the variables in the namelist file read in by the ${exec_fn} executable failed. Parameters passed to @@ -618,7 +618,7 @@ if [ "${USE_FVCOM}" = "TRUE" ]; then #Format for fvcom_time: YYYY-MM-DDTHH:00:00.000000 fvcom_exec_fn="fvcom_to_FV3" - fvcom_exec_fp="$EXECDIR/${fvcom_exec_fn}" + fvcom_exec_fp="$EXECrrfs/${fvcom_exec_fn}" fvcom_time="${DATE_FIRST_CYCL:0:4}-${DATE_FIRST_CYCL:4:2}-${DATE_FIRST_CYCL:6:2}T${CYCL_HRS[0]}:00:00.000000" if [ ! -f "${fvcom_exec_fp}" ]; then print_err_msg_exit "\ diff --git a/scripts/exregional_make_lbcs.sh b/scripts/exregional_make_lbcs.sh index f4bf3a6056..98fba2be96 100755 --- a/scripts/exregional_make_lbcs.sh +++ b/scripts/exregional_make_lbcs.sh @@ -8,7 +8,7 @@ #----------------------------------------------------------------------- # . ${GLOBAL_VAR_DEFNS_FP} -. $USHDIR/source_util_funcs.sh +. $USHrrfs/source_util_funcs.sh # #----------------------------------------------------------------------- # @@ -17,7 +17,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 +{ save_shell_opts; . $USHrrfs/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # @@ -86,7 +86,7 @@ export OMP_STACKSIZE=${OMP_STACKSIZE_MAKE_LBCS} # #----------------------------------------------------------------------- # -source $USHDIR/source_machine_file.sh +source $USHrrfs/source_machine_file.sh eval ${PRE_TASK_CMDS} nprocs=$(( NNODES_MAKE_LBCS*PPN_MAKE_LBCS )) @@ -341,7 +341,7 @@ esac #----------------------------------------------------------------------- # exec_fn="chgres_cube" -exec_fp="$EXECDIR/${exec_fn}" +exec_fp="$EXECrrfs/${exec_fn}" if [ ! -f "${exec_fp}" ]; then print_err_msg_exit "\ The executable (exec_fp) for generating initial conditions on the FV3-LAM @@ -468,7 +468,7 @@ settings=" # Call the python script to create the namelist file. # nml_fn="fort.41" - ${USHDIR}/set_namelist.py -q -u "$settings" -o ${nml_fn} || \ + ${USHrrfs}/set_namelist.py -q -u "$settings" -o ${nml_fn} || \ print_err_msg_exit "\ Call to python script set_namelist.py to set the variables in the namelist file read in by the ${exec_fn} executable failed. Parameters passed to diff --git a/scripts/exregional_make_orog.sh b/scripts/exregional_make_orog.sh index d83cefd873..244d72751b 100755 --- a/scripts/exregional_make_orog.sh +++ b/scripts/exregional_make_orog.sh @@ -8,7 +8,7 @@ #----------------------------------------------------------------------- # . ${GLOBAL_VAR_DEFNS_FP} -. $USHDIR/source_util_funcs.sh +. $USHrrfs/source_util_funcs.sh # #----------------------------------------------------------------------- # @@ -17,7 +17,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 +{ save_shell_opts; . $USHrrfs/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # @@ -86,7 +86,7 @@ export OMP_STACKSIZE=${OMP_STACKSIZE_MAKE_OROG} # #----------------------------------------------------------------------- # -source $USHDIR/source_machine_file.sh +source $USHrrfs/source_machine_file.sh eval ${PRE_TASK_CMDS} if [ -z "${RUN_CMD_SERIAL:-}" ] ; then @@ -130,7 +130,7 @@ mkdir_vrfy -p "${shave_dir}" # file and make sure that it exists. # exec_fn="orog" -exec_fp="$EXECDIR/${exec_fn}" +exec_fp="$EXECrrfs/${exec_fn}" if [ ! -f "${exec_fp}" ]; then print_err_msg_exit "\ The executable (exec_fp) for generating the orography file does not exist: @@ -291,7 +291,7 @@ ${NH4} EOF exec_fn="orog_gsl" - exec_fp="$EXECDIR/${exec_fn}" + exec_fp="$EXECrrfs/${exec_fn}" if [ ! -f "${exec_fp}" ]; then print_err_msg_exit "\ The executable (exec_fp) for generating the GSL orography GWD data files @@ -379,7 +379,7 @@ fi # Set the name and path to the executable and make sure that it exists. # exec_fn="filter_topo" -exec_fp="$EXECDIR/${exec_fn}" +exec_fp="$EXECrrfs/${exec_fn}" if [ ! -f "${exec_fp}" ]; then print_err_msg_exit "\ The executable (exec_fp) for filtering the raw orography does not exist: @@ -485,7 +485,7 @@ Filtering of orography complete." # Set the name and path to the executable and make sure that it exists. # exec_fn="shave" -exec_fp="$EXECDIR/${exec_fn}" +exec_fp="$EXECrrfs/${exec_fn}" if [ ! -f "${exec_fp}" ]; then print_err_msg_exit "\ The executable (exec_fp) for \"shaving\" down the halo in the orography @@ -572,7 +572,7 @@ cd_vrfy - # #----------------------------------------------------------------------- # -python3 $USHDIR/link_fix.py \ +python3 $USHrrfs/link_fix.py \ --path-to-defns ${GLOBAL_VAR_DEFNS_FP} \ --file-group "orog" || \ print_err_msg_exit "\ diff --git a/scripts/exregional_make_sfc_climo.sh b/scripts/exregional_make_sfc_climo.sh index 5036656488..637ed9f78c 100755 --- a/scripts/exregional_make_sfc_climo.sh +++ b/scripts/exregional_make_sfc_climo.sh @@ -8,7 +8,7 @@ #----------------------------------------------------------------------- # . ${GLOBAL_VAR_DEFNS_FP} -. $USHDIR/source_util_funcs.sh +. $USHrrfs/source_util_funcs.sh # #----------------------------------------------------------------------- # @@ -17,7 +17,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 +{ save_shell_opts; . $USHrrfs/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # @@ -128,7 +128,7 @@ EOF # #----------------------------------------------------------------------- # -source $USHDIR/source_machine_file.sh +source $USHrrfs/source_machine_file.sh eval ${PRE_TASK_CMDS} nprocs=$(( NNODES_MAKE_SFC_CLIMO*PPN_MAKE_SFC_CLIMO )) @@ -152,7 +152,7 @@ fi # Set the name and path to the executable and make sure that it exists. # exec_fn="sfc_climo_gen" -exec_fp="$EXECDIR/${exec_fn}" +exec_fp="$EXECrrfs/${exec_fn}" if [ ! -f "${exec_fp}" ]; then print_err_msg_exit "\ The executable (exec_fp) for generating the surface climatology files @@ -233,7 +233,7 @@ esac # #----------------------------------------------------------------------- # -python3 $USHDIR/link_fix.py \ +python3 $USHrrfs/link_fix.py \ --path-to-defns ${GLOBAL_VAR_DEFNS_FP} \ --file-group "sfc_climo" || \ print_err_msg_exit "\ diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index f185d38b25..bcdbf4e361 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -8,7 +8,7 @@ #----------------------------------------------------------------------- # . ${GLOBAL_VAR_DEFNS_FP} -. $USHDIR/source_util_funcs.sh +. $USHrrfs/source_util_funcs.sh # #----------------------------------------------------------------------- # @@ -17,7 +17,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 +{ save_shell_opts; . $USHrrfs/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # @@ -89,7 +89,7 @@ export OMP_STACKSIZE=${OMP_STACKSIZE_RUN_FCST} # #----------------------------------------------------------------------- # -source $USHDIR/source_machine_file.sh +source $USHrrfs/source_machine_file.sh eval ${PRE_TASK_CMDS} nprocs=$(( NNODES_RUN_FCST*PPN_RUN_FCST )) @@ -445,7 +445,7 @@ fi if [ "${DO_ENSEMBLE}" = TRUE ] && ([ "${DO_SPP}" = TRUE ] || [ "${DO_SPPT}" = TRUE ] || [ "${DO_SHUM}" = TRUE ] || \ [ "${DO_SKEB}" = TRUE ] || [ "${DO_LSM_SPP}" = TRUE ]); then - python3 $USHDIR/set_FV3nml_ens_stoch_seeds.py \ + python3 $USHrrfs/set_FV3nml_ens_stoch_seeds.py \ --path-to-defns ${GLOBAL_VAR_DEFNS_FP} \ --cdate "$cdate" || print_err_msg_exit "\ Call to function to create the ensemble-based namelist for the current @@ -465,7 +465,7 @@ fi # #----------------------------------------------------------------------- # -python3 $USHDIR/create_model_configure_file.py \ +python3 $USHrrfs/create_model_configure_file.py \ --path-to-defns ${GLOBAL_VAR_DEFNS_FP} \ --cdate "$cdate" \ --run-dir "${run_dir}" \ @@ -484,7 +484,7 @@ cycle's (cdate) run directory (run_dir) failed: # #----------------------------------------------------------------------- # -python3 $USHDIR/create_diag_table_file.py \ +python3 $USHrrfs/create_diag_table_file.py \ --path-to-defns ${GLOBAL_VAR_DEFNS_FP} \ --run-dir "${run_dir}" || print_err_msg_exit "\ Call to function to create a diag table file for the current cycle's diff --git a/scripts/exregional_run_post.sh b/scripts/exregional_run_post.sh index 7dd476f175..3dad30bfb0 100755 --- a/scripts/exregional_run_post.sh +++ b/scripts/exregional_run_post.sh @@ -8,7 +8,7 @@ #----------------------------------------------------------------------- # . ${GLOBAL_VAR_DEFNS_FP} -. $USHDIR/source_util_funcs.sh +. $USHrrfs/source_util_funcs.sh # #----------------------------------------------------------------------- # @@ -17,7 +17,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 +{ save_shell_opts; . $USHrrfs/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # @@ -92,7 +92,7 @@ export OMP_STACKSIZE=${OMP_STACKSIZE_RUN_POST} # #----------------------------------------------------------------------- # -source $USHDIR/source_machine_file.sh +source $USHrrfs/source_machine_file.sh eval ${PRE_TASK_CMDS} nprocs=$(( NNODES_RUN_POST*PPN_RUN_POST )) @@ -244,7 +244,7 @@ EOF print_info_msg "$VERBOSE" " Starting post-processing for fhr = $fhr hr..." -${RUN_CMD_POST} ${EXECDIR}/upp.x < itag || print_err_msg_exit "\ +${RUN_CMD_POST} ${EXECrrfs}/upp.x < itag || print_err_msg_exit "\ Call to executable to run post for forecast hour $fhr returned with non- zero exit code." # diff --git a/scripts/exregional_run_vx_ensgrid.sh b/scripts/exregional_run_vx_ensgrid.sh index 99fc494838..2058163ac1 100755 --- a/scripts/exregional_run_vx_ensgrid.sh +++ b/scripts/exregional_run_vx_ensgrid.sh @@ -8,7 +8,7 @@ #----------------------------------------------------------------------- # . ${GLOBAL_VAR_DEFNS_FP} -. $USHDIR/source_util_funcs.sh +. $USHrrfs/source_util_funcs.sh # #----------------------------------------------------------------------- # @@ -17,7 +17,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 +{ save_shell_opts; . $USHrrfs/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_run_vx_ensgrid_mean.sh b/scripts/exregional_run_vx_ensgrid_mean.sh index 2164ad8222..5bfe44f165 100755 --- a/scripts/exregional_run_vx_ensgrid_mean.sh +++ b/scripts/exregional_run_vx_ensgrid_mean.sh @@ -8,7 +8,7 @@ #----------------------------------------------------------------------- # . ${GLOBAL_VAR_DEFNS_FP} -. $USHDIR/source_util_funcs.sh +. $USHrrfs/source_util_funcs.sh # #----------------------------------------------------------------------- # @@ -17,7 +17,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 +{ save_shell_opts; . $USHrrfs/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_run_vx_ensgrid_prob.sh b/scripts/exregional_run_vx_ensgrid_prob.sh index adfaffb4a6..4f8b11ffaa 100755 --- a/scripts/exregional_run_vx_ensgrid_prob.sh +++ b/scripts/exregional_run_vx_ensgrid_prob.sh @@ -8,7 +8,7 @@ #----------------------------------------------------------------------- # . ${GLOBAL_VAR_DEFNS_FP} -. $USHDIR/source_util_funcs.sh +. $USHrrfs/source_util_funcs.sh # #----------------------------------------------------------------------- # @@ -17,7 +17,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 +{ save_shell_opts; . $USHrrfs/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_run_vx_enspoint.sh b/scripts/exregional_run_vx_enspoint.sh index 2cac598f7f..b90929ca83 100755 --- a/scripts/exregional_run_vx_enspoint.sh +++ b/scripts/exregional_run_vx_enspoint.sh @@ -8,7 +8,7 @@ #----------------------------------------------------------------------- # . ${GLOBAL_VAR_DEFNS_FP} -. $USHDIR/source_util_funcs.sh +. $USHrrfs/source_util_funcs.sh # #----------------------------------------------------------------------- # @@ -17,7 +17,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 +{ save_shell_opts; . $USHrrfs/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_run_vx_enspoint_mean.sh b/scripts/exregional_run_vx_enspoint_mean.sh index d9341bb540..a705fce5bd 100755 --- a/scripts/exregional_run_vx_enspoint_mean.sh +++ b/scripts/exregional_run_vx_enspoint_mean.sh @@ -8,7 +8,7 @@ #----------------------------------------------------------------------- # . ${GLOBAL_VAR_DEFNS_FP} -. $USHDIR/source_util_funcs.sh +. $USHrrfs/source_util_funcs.sh # #----------------------------------------------------------------------- # @@ -17,7 +17,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 +{ save_shell_opts; . $USHrrfs/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_run_vx_enspoint_prob.sh b/scripts/exregional_run_vx_enspoint_prob.sh index 94cf10e92d..0df2a0ca48 100755 --- a/scripts/exregional_run_vx_enspoint_prob.sh +++ b/scripts/exregional_run_vx_enspoint_prob.sh @@ -8,7 +8,7 @@ #----------------------------------------------------------------------- # . ${GLOBAL_VAR_DEFNS_FP} -. $USHDIR/source_util_funcs.sh +. $USHrrfs/source_util_funcs.sh # #----------------------------------------------------------------------- # @@ -17,7 +17,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 +{ save_shell_opts; . $USHrrfs/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_run_vx_gridstat.sh b/scripts/exregional_run_vx_gridstat.sh index 8bce19afa9..29fd34a84b 100755 --- a/scripts/exregional_run_vx_gridstat.sh +++ b/scripts/exregional_run_vx_gridstat.sh @@ -8,7 +8,7 @@ #----------------------------------------------------------------------- # . ${GLOBAL_VAR_DEFNS_FP} -. $USHDIR/source_util_funcs.sh +. $USHrrfs/source_util_funcs.sh # #----------------------------------------------------------------------- # @@ -17,7 +17,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 +{ save_shell_opts; . $USHrrfs/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_run_vx_pointstat.sh b/scripts/exregional_run_vx_pointstat.sh index 580a61b706..2fc66f7de1 100755 --- a/scripts/exregional_run_vx_pointstat.sh +++ b/scripts/exregional_run_vx_pointstat.sh @@ -8,7 +8,7 @@ #----------------------------------------------------------------------- # . ${GLOBAL_VAR_DEFNS_FP} -. $USHDIR/source_util_funcs.sh +. $USHrrfs/source_util_funcs.sh # #----------------------------------------------------------------------- # @@ -17,7 +17,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 +{ save_shell_opts; . $USHrrfs/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/ush/calculate_cost.py b/ush/calculate_cost.py index 3fc1455c3a..f1b3f3496d 100755 --- a/ush/calculate_cost.py +++ b/ush/calculate_cost.py @@ -94,7 +94,7 @@ def calculate_cost(config_fn): class Testing(unittest.TestCase): def test_calculate_cost(self): - USHDIR = os.path.dirname(os.path.abspath(__file__)) + USHrrfs = os.path.dirname(os.path.abspath(__file__)) params = calculate_cost(None) self.assertCountEqual(params, [36, 1987440, 36, 28689]) diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index a57af40b18..247a0512c2 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -420,13 +420,17 @@ workflow: # # $PTMP/com/$NET/$model_ver/$RUN.$yyyymmdd/$hh # + # For more information on NCO standards + # + # https://www.nco.ncep.noaa.gov/idsb/implementation_standards/ImplementationStandards.v11.0.0.pdf + # #----------------------------------------------------------------------- # - COMIN: "/path/of/directory/containing/data/files/for/IC/LBCS" envir: "para" NET: "rrfs" - model_ver: "v1.0.0" RUN: "rrfs" + model_ver: "v1.0.0" + COMIN: "/path/of/directory/containing/data/files/for/IC/LBCS" STMP: "/base/path/of/directory/containing/model/input/and/raw/output/files" PTMP: "/base/path/of/directory/containing/postprocessed/output/files" # diff --git a/ush/create_diag_table_file.py b/ush/create_diag_table_file.py index 9c5b0f2e01..9b0a341c7a 100644 --- a/ush/create_diag_table_file.py +++ b/ush/create_diag_table_file.py @@ -126,18 +126,18 @@ def parse_args(argv): class Testing(unittest.TestCase): def test_create_diag_table_file(self): - path = os.path.join(os.getenv("USHDIR"), "test_data") + path = os.path.join(os.getenv("USHrrfs"), "test_data") self.assertTrue(create_diag_table_file(run_dir=path)) def setUp(self): - USHDIR = os.path.dirname(os.path.abspath(__file__)) + USHrrfs = os.path.dirname(os.path.abspath(__file__)) DIAG_TABLE_FN = "diag_table" DIAG_TABLE_TMPL_FP = os.path.join( - USHDIR, "templates", f"{DIAG_TABLE_FN}.FV3_GFS_v15p2" + USHrrfs, "templates", f"{DIAG_TABLE_FN}.FV3_GFS_v15p2" ) set_env_var("DEBUG", True) set_env_var("VERBOSE", True) - set_env_var("USHDIR", USHDIR) + set_env_var("USHrrfs", USHrrfs) set_env_var("DIAG_TABLE_FN", DIAG_TABLE_FN) set_env_var("DIAG_TABLE_TMPL_FP", DIAG_TABLE_TMPL_FP) set_env_var("CRES", "C48") diff --git a/ush/create_model_configure_file.py b/ush/create_model_configure_file.py index dd522981a1..5cac49b613 100644 --- a/ush/create_model_configure_file.py +++ b/ush/create_model_configure_file.py @@ -306,7 +306,7 @@ def parse_args(argv): class Testing(unittest.TestCase): def test_create_model_configure_file(self): - path = os.path.join(os.getenv("USHDIR"), "test_data") + path = os.path.join(os.getenv("USHrrfs"), "test_data") self.assertTrue( create_model_configure_file( run_dir=path, @@ -318,9 +318,9 @@ def test_create_model_configure_file(self): ) def setUp(self): - USHDIR = os.path.dirname(os.path.abspath(__file__)) + USHrrfs = os.path.dirname(os.path.abspath(__file__)) MODEL_CONFIG_FN = "model_configure" - MODEL_CONFIG_TMPL_FP = os.path.join(USHDIR, "templates", MODEL_CONFIG_FN) + MODEL_CONFIG_TMPL_FP = os.path.join(USHrrfs, "templates", MODEL_CONFIG_FN) set_env_var("DEBUG", True) set_env_var("VERBOSE", True) @@ -328,7 +328,7 @@ def setUp(self): set_env_var("PRINT_ESMF", True) set_env_var("CPL", True) set_env_var("WRITE_DOPOST", True) - set_env_var("USHDIR", USHDIR) + set_env_var("USHrrfs", USHrrfs) set_env_var("MODEL_CONFIG_FN", MODEL_CONFIG_FN) set_env_var("MODEL_CONFIG_TMPL_FP", MODEL_CONFIG_TMPL_FP) set_env_var("PE_MEMBER01", 24) diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index 5140c8dbc8..e9326d4f2a 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -955,7 +955,7 @@ def generate_FV3LAM_wflow(): # # ----------------------------------------------------------------------- # - cp_vrfy(os.path.join(USHDIR, EXPT_CONFIG_FN), EXPTDIR) + cp_vrfy(os.path.join(USHrrfs, EXPT_CONFIG_FN), EXPTDIR) # # ----------------------------------------------------------------------- # @@ -1046,7 +1046,7 @@ def generate_FV3LAM_wflow(): print("Getting NOMADS online data") print(f"NOMADS_file_type= {NOMADS_file_type}") cd_vrfy(EXPTDIR) - NOMADS_script = os.path.join(USHDIR, "NOMADS_get_extrn_mdl_files.h") + NOMADS_script = os.path.join(USHrrfs, "NOMADS_get_extrn_mdl_files.h") run_command( f"""{NOMADS_script} {date_to_str(DATE_FIRST_CYCL,format="%Y%m%d")} \ {CYCL_HRS} {NOMADS_file_type} {FCST_LEN_HRS} {LBC_SPEC_INTVL_HRS}""" diff --git a/ush/init_env.sh b/ush/init_env.sh index 957688ad9f..2463528f5e 100644 --- a/ush/init_env.sh +++ b/ush/init_env.sh @@ -13,7 +13,7 @@ # function init_env() { - { save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 + { save_shell_opts; . $USHrrfs/preamble.sh; } > /dev/null 2>&1 local valid_args=( \ "env_init_scripts_fps" \ diff --git a/ush/launch_FV3LAM_wflow.sh b/ush/launch_FV3LAM_wflow.sh index 1f7f692026..1a417a3eb5 100755 --- a/ush/launch_FV3LAM_wflow.sh +++ b/ush/launch_FV3LAM_wflow.sh @@ -95,10 +95,10 @@ fi #----------------------------------------------------------------------- # . $exptdir/var_defns.sh -. $USHDIR/source_util_funcs.sh -. $USHDIR/source_machine_file.sh -. $USHDIR/constants.sh -. $USHDIR/init_env.sh +. $USHrrfs/source_util_funcs.sh +. $USHrrfs/source_machine_file.sh +. $USHrrfs/constants.sh +. $USHrrfs/init_env.sh # #----------------------------------------------------------------------- # @@ -396,10 +396,10 @@ script for this experiment: # if [ "${called_from_cron}" = "TRUE" ]; then MACHINE=$MACHINE CRONTAB_LINE=$CRONTAB_LINE \ - python3 $USHDIR/get_crontab_contents.py --delete --called-from-cron + python3 $USHrrfs/get_crontab_contents.py --delete --called-from-cron else MACHINE=$MACHINE CRONTAB_LINE=$CRONTAB_LINE \ - python3 $USHDIR/get_crontab_contents.py --delete + python3 $USHrrfs/get_crontab_contents.py --delete fi fi # diff --git a/ush/load_modules_run_task.sh b/ush/load_modules_run_task.sh index 310876a1ec..4e08ed62f0 100755 --- a/ush/load_modules_run_task.sh +++ b/ush/load_modules_run_task.sh @@ -8,9 +8,9 @@ #----------------------------------------------------------------------- # . ${GLOBAL_VAR_DEFNS_FP} -. $USHDIR/source_util_funcs.sh -. $USHDIR/source_machine_file.sh -. $USHDIR/init_env.sh +. $USHrrfs/source_util_funcs.sh +. $USHrrfs/source_machine_file.sh +. $USHrrfs/init_env.sh # #----------------------------------------------------------------------- # @@ -19,7 +19,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 +{ save_shell_opts; . $USHrrfs/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/ush/make_grid_mosaic_file.sh b/ush/make_grid_mosaic_file.sh index 5d8d9f06c2..684b4b27be 100644 --- a/ush/make_grid_mosaic_file.sh +++ b/ush/make_grid_mosaic_file.sh @@ -17,7 +17,7 @@ function make_grid_mosaic_file() { # #----------------------------------------------------------------------- # - { save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 + { save_shell_opts; . $USHrrfs/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # @@ -85,7 +85,7 @@ function make_grid_mosaic_file() { #----------------------------------------------------------------------- # exec_fn="make_solo_mosaic" - exec_fp="$EXECDIR/${exec_fn}" + exec_fp="$EXECrrfs/${exec_fn}" if [ ! -f "${exec_fp}" ]; then print_err_msg_exit "\ The executable (exec_fp) for generating the grid mosaic file does not diff --git a/ush/set_FV3nml_ens_stoch_seeds.py b/ush/set_FV3nml_ens_stoch_seeds.py index ab7da1880b..9ebde47c91 100644 --- a/ush/set_FV3nml_ens_stoch_seeds.py +++ b/ush/set_FV3nml_ens_stoch_seeds.py @@ -172,10 +172,10 @@ def setUp(self): set_env_var("DEBUG", True) set_env_var("VERBOSE", True) self.cdate = datetime(2021, 1, 1) - USHDIR = os.path.dirname(os.path.abspath(__file__)) - EXPTDIR = os.path.join(USHDIR, "test_data", "expt") + USHrrfs = os.path.dirname(os.path.abspath(__file__)) + EXPTDIR = os.path.join(USHrrfs, "test_data", "expt") cp_vrfy( - os.path.join(USHDIR, f"templates{os.sep}input.nml.FV3"), + os.path.join(USHrrfs, f"templates{os.sep}input.nml.FV3"), os.path.join(EXPTDIR, "input.nml"), ) for i in range(2): @@ -187,7 +187,7 @@ def setUp(self): ), ) - set_env_var("USHDIR", USHDIR) + set_env_var("USHrrfs", USHrrfs) set_env_var("CYCLE_BASEDIR", EXPTDIR) set_env_var("ENSMEM_INDX", 2) set_env_var("FV3_NML_FN", "input.nml") diff --git a/ush/set_FV3nml_sfc_climo_filenames.py b/ush/set_FV3nml_sfc_climo_filenames.py index a55f0575f5..33086f8c35 100644 --- a/ush/set_FV3nml_sfc_climo_filenames.py +++ b/ush/set_FV3nml_sfc_climo_filenames.py @@ -149,15 +149,15 @@ def setUp(self): define_macos_utilities() set_env_var("DEBUG", True) set_env_var("VERBOSE", True) - USHDIR = os.path.dirname(os.path.abspath(__file__)) - EXPTDIR = os.path.join(USHDIR, "test_data", "expt") + USHrrfs = os.path.dirname(os.path.abspath(__file__)) + EXPTDIR = os.path.join(USHrrfs, "test_data", "expt") FIXLAM = os.path.join(EXPTDIR, "fix_lam") mkdir_vrfy("-p", FIXLAM) cp_vrfy( - os.path.join(USHDIR, f"templates{os.sep}input.nml.FV3"), + os.path.join(USHrrfs, f"templates{os.sep}input.nml.FV3"), os.path.join(EXPTDIR, "input.nml"), ) - set_env_var("USHDIR", USHDIR) + set_env_var("USHrrfs", USHrrfs) set_env_var("EXPTDIR", EXPTDIR) set_env_var("FIXLAM", FIXLAM) set_env_var("DO_ENSEMBLE", False) diff --git a/ush/set_predef_grid_params.py b/ush/set_predef_grid_params.py index f7a56b542d..b9c205e177 100644 --- a/ush/set_predef_grid_params.py +++ b/ush/set_predef_grid_params.py @@ -36,8 +36,8 @@ def set_predef_grid_params(): ] import_vars(env_vars=IMPORTS) - USHDIR = os.path.dirname(os.path.abspath(__file__)) - params_dict = load_config_file(os.path.join(USHDIR, "predef_grid_params.yaml")) + USHrrfs = os.path.dirname(os.path.abspath(__file__)) + params_dict = load_config_file(os.path.join(USHrrfs, "predef_grid_params.yaml")) params_dict = params_dict[PREDEF_GRID_NAME] # if QUILTING = False, remove key diff --git a/ush/setup.py b/ush/setup.py index 2f1577b0f3..9bc33a702d 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -348,18 +348,18 @@ def setup(): # # Define some other useful paths # - global USHDIR, SCRIPTSDIR, JOBSDIR, SORCDIR, SRC_DIR, PARMDIR, MODULES_DIR - global EXECDIR, TEMPLATE_DIR, VX_CONFIG_DIR, METPLUS_CONF, MET_CONFIG + global USHrrfs, SCRIPTSDIR, JOBSDIR, SORCDIR, SRC_DIR, PARMrrfs, MODULES_DIR + global EXECrrfs, TEMPLATE_DIR, VX_CONFIG_DIR, METPLUS_CONF, MET_CONFIG - USHDIR = os.path.join(SR_WX_APP_TOP_DIR, "ush") - SCRIPTSDIR = os.path.join(SR_WX_APP_TOP_DIR, "scripts") - JOBSDIR = os.path.join(SR_WX_APP_TOP_DIR, "jobs") - SORCDIR = os.path.join(SR_WX_APP_TOP_DIR, "sorc") + USHrrfs = os.path.join(HOMErrfs, "ush") + SCRIPTSDIR = os.path.join(HOMErrfs, "scripts") + JOBSDIR = os.path.join(HOMErrfs, "jobs") + SORCDIR = os.path.join(HOMErrfs, "sorc") SRC_DIR = os.path.join(SR_WX_APP_TOP_DIR, "src") - PARMDIR = os.path.join(SR_WX_APP_TOP_DIR, "parm") - MODULES_DIR = os.path.join(SR_WX_APP_TOP_DIR, "modulefiles") - EXECDIR = os.path.join(SR_WX_APP_TOP_DIR, EXEC_SUBDIR) - TEMPLATE_DIR = os.path.join(USHDIR, "templates") + PARMrrfs = os.path.join(HOMErrfs, "parm") + MODULES_DIR = os.path.join(HOMErrfs, "modulefiles") + EXECrrfs = os.path.join(SR_WX_APP_TOP_DIR, EXEC_SUBDIR) + TEMPLATE_DIR = os.path.join(USHrrfs, "templates") VX_CONFIG_DIR = os.path.join(TEMPLATE_DIR, "parm") METPLUS_CONF = os.path.join(TEMPLATE_DIR, "parm", "metplus") MET_CONFIG = os.path.join(TEMPLATE_DIR, "parm", "met") @@ -380,7 +380,7 @@ def setup(): MACHINE = uppercase(MACHINE) RELATIVE_LINK_FLAG = "--relative" MACHINE_FILE = MACHINE_FILE or os.path.join( - USHDIR, "machine", f"{lowercase(MACHINE)}.sh" + USHrrfs, "machine", f"{lowercase(MACHINE)}.sh" ) machine_cfg = load_shell_config(MACHINE_FILE) import_vars(dictionary=machine_cfg) @@ -896,7 +896,7 @@ def setup(): # # $COMROOT/$NET/$envir/$RUN.$yyyymmdd/$hh # - # Below, we set COMROOT in terms of PTMP as COMROOT="$PTMP/com". COMOROOT + # Below, we set COMROOT in terms of PTMP as COMROOT="$PTMP/com". COMROOT # is not used by the workflow in community mode. # # COMOUT_BASEDIR: @@ -913,7 +913,7 @@ def setup(): # # ----------------------------------------------------------------------- # - global LOGDIR, FIXam, FIXclim, FIXLAM, CYCLE_BASEDIR, COMROOT + global LOGDIR, FIXam, FIXclim, FIXLAM, CYCLE_BASEDIR, COMROOT, DCOMROOT global COMOUT_BASEDIR, POST_OUTPUT_DOMAIN_NAME LOGDIR = os.path.join(EXPTDIR, "log") @@ -927,6 +927,7 @@ def setup(): CYCLE_BASEDIR = os.path.join(STMP, "tmpnwprd", RUN) check_for_preexist_dir_file(CYCLE_BASEDIR, PREEXISTING_DIR_METHOD) COMROOT = os.path.join(PTMP, "com") + DCOMROOT = os.path.join(PTMP, "dcom") COMOUT_BASEDIR = os.path.join(COMROOT, NET, model_ver) check_for_preexist_dir_file(COMOUT_BASEDIR, PREEXISTING_DIR_METHOD) @@ -934,6 +935,7 @@ def setup(): CYCLE_BASEDIR = EXPTDIR COMROOT = "" + DCOMROOT = "" COMOUT_BASEDIR = "" # @@ -1214,7 +1216,7 @@ def setup(): # ----------------------------------------------------------------------- # global FV3_EXEC_FP - FV3_EXEC_FP = os.path.join(EXECDIR, FV3_EXEC_FN) + FV3_EXEC_FP = os.path.join(EXECrrfs, FV3_EXEC_FN) # # ----------------------------------------------------------------------- # @@ -1227,7 +1229,7 @@ def setup(): # ----------------------------------------------------------------------- # global WFLOW_LAUNCH_SCRIPT_FP, WFLOW_LAUNCH_LOG_FP, CRONTAB_LINE - WFLOW_LAUNCH_SCRIPT_FP = os.path.join(USHDIR, WFLOW_LAUNCH_SCRIPT_FN) + WFLOW_LAUNCH_SCRIPT_FP = os.path.join(USHrrfs, WFLOW_LAUNCH_SCRIPT_FN) WFLOW_LAUNCH_LOG_FP = os.path.join(EXPTDIR, WFLOW_LAUNCH_LOG_FN) if USE_CRON_TO_RELAUNCH: CRONTAB_LINE = ( @@ -1245,7 +1247,7 @@ def setup(): # ----------------------------------------------------------------------- # global LOAD_MODULES_RUN_TASK_FP - LOAD_MODULES_RUN_TASK_FP = os.path.join(USHDIR, "load_modules_run_task.sh") + LOAD_MODULES_RUN_TASK_FP = os.path.join(USHrrfs, "load_modules_run_task.sh") # # ----------------------------------------------------------------------- # @@ -1825,13 +1827,13 @@ def setup(): # # 1) Copying the default workflow/experiment configuration file (speci- # fied by EXPT_DEFAULT_CONFIG_FN and located in the shell script di- - # rectory specified by USHDIR) to the experiment directory and rena- + # rectory specified by USHrrfs) to the experiment directory and rena- # ming it to the name specified by GLOBAL_VAR_DEFNS_FN. # # 2) Resetting the default variable values in this file to their current # values. This is necessary because these variables may have been # reset by the user-specified configuration file (if one exists in - # USHDIR) and/or by this setup script, e.g. because predef_domain is + # USHrrfs) and/or by this setup script, e.g. because predef_domain is # set to a valid non-empty value. # # 3) Appending to the variable definitions file any new variables intro- @@ -1889,14 +1891,15 @@ def setup(): # ----------------------------------------------------------------------- # "SR_WX_APP_TOP_DIR": SR_WX_APP_TOP_DIR, - "USHDIR": USHDIR, + "HOMErrfs": HOMErrfs, + "USHrrfs": USHrrfs, "SCRIPTSDIR": SCRIPTSDIR, "JOBSDIR": JOBSDIR, "SORCDIR": SORCDIR, "SRC_DIR": SRC_DIR, - "PARMDIR": PARMDIR, + "PARMrrfs": PARMrrfs, "MODULES_DIR": MODULES_DIR, - "EXECDIR": EXECDIR, + "EXECrrfs": EXECrrfs, "FIXam": FIXam, "FIXclim": FIXclim, "FIXLAM": FIXLAM, diff --git a/ush/source_machine_file.sh b/ush/source_machine_file.sh index 6d7e0de0e1..ac40cefade 100644 --- a/ush/source_machine_file.sh +++ b/ush/source_machine_file.sh @@ -10,7 +10,7 @@ #----------------------------------------------------------------------- # -{ save_shell_opts; . $USHDIR/preamble.sh; } > /dev/null 2>&1 +{ save_shell_opts; . $USHrrfs/preamble.sh; } > /dev/null 2>&1 source ${MACHINE_FILE} From d59b628cd7543b92b12ef94bf0727d4dd5a0019a Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Sun, 14 Aug 2022 00:51:39 +0000 Subject: [PATCH 006/106] Change default bin directory to exec --- ush/config_defaults.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 247a0512c2..84d6ce0748 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -369,7 +369,7 @@ workflow: # EXPT_BASEDIR: "" EXPT_SUBDIR: "" - EXEC_SUBDIR: "bin" + EXEC_SUBDIR: "exec" # #----------------------------------------------------------------------- # From 75ca156cf80b984f9aaf1076c9b3f0907aceb333 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Mon, 15 Aug 2022 00:11:03 +0000 Subject: [PATCH 007/106] Appen FATAL ERROR to print_err_msg_exit. --- ush/bash_utils/print_msg.sh | 2 +- ush/python_utils/print_msg.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/bash_utils/print_msg.sh b/ush/bash_utils/print_msg.sh index d837329421..f48e0c2f8b 100644 --- a/ush/bash_utils/print_msg.sh +++ b/ush/bash_utils/print_msg.sh @@ -299,7 +299,7 @@ the header and footer. err_msg="\n$1" fi - printf "${msg_header}${err_msg}${msg_footer}\n" 1>&2 + printf "FATAL ERROR: ${msg_header}${err_msg}${msg_footer}\n" 1>&2 exit 1 fi diff --git a/ush/python_utils/print_msg.py b/ush/python_utils/print_msg.py index e943ab46da..606284e140 100644 --- a/ush/python_utils/print_msg.py +++ b/ush/python_utils/print_msg.py @@ -19,7 +19,7 @@ def print_err_msg_exit(error_msg="", stack_trace=True): traceback.print_stack(file=sys.stderr) msg_footer = "\nExiting with nonzero status." - print(dedent(error_msg) + msg_footer, file=sys.stderr) + print("FATAL ERROR: " + dedent(error_msg) + msg_footer, file=sys.stderr) sys.exit(1) From b15165a04d780e1f96e6cb9b5ae1a26c3fb88f74 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Tue, 16 Aug 2022 00:12:04 +0000 Subject: [PATCH 008/106] Replace some cp, cd, mkdir calls with their corresponding _vrfy versions --- scripts/exregional_get_obs_ccpa.sh | 52 +++++++++++++++--------------- scripts/exregional_get_obs_mrms.sh | 12 +++---- scripts/exregional_get_obs_ndas.sh | 24 +++++++------- 3 files changed, 44 insertions(+), 44 deletions(-) diff --git a/scripts/exregional_get_obs_ccpa.sh b/scripts/exregional_get_obs_ccpa.sh index 729e1feab8..a365c5cf82 100755 --- a/scripts/exregional_get_obs_ccpa.sh +++ b/scripts/exregional_get_obs_ccpa.sh @@ -32,19 +32,19 @@ # Top-level CCPA directory ccpa_dir=${OBS_DIR}/.. if [[ ! -d "$ccpa_dir" ]]; then - mkdir -p $ccpa_dir + mkdir_vrfy -p $ccpa_dir fi # CCPA data from HPSS ccpa_raw=$ccpa_dir/raw if [[ ! -d "$ccpa_raw" ]]; then - mkdir -p $ccpa_raw + mkdir_vrfy -p $ccpa_raw fi # Reorganized CCPA location ccpa_proc=$ccpa_dir/proc if [[ ! -d "$ccpa_proc" ]]; then - mkdir -p $ccpa_proc + mkdir_vrfy -p $ccpa_proc fi # Accumulation is for accumulation of CCPA data to pull (hardcoded to 01h, see note above.) @@ -102,19 +102,19 @@ while [[ ${current_fcst} -le ${fcst_length} ]]; do # Create necessary raw and prop directories if [[ ! -d "$ccpa_raw/${vyyyymmdd}" ]]; then - mkdir -p $ccpa_raw/${vyyyymmdd} + mkdir_vrfy -p $ccpa_raw/${vyyyymmdd} fi if [[ ! -d "$ccpa_raw/${vyyyymmdd_m1}" ]]; then - mkdir -p $ccpa_raw/${vyyyymmdd_m1} + mkdir_vrfy -p $ccpa_raw/${vyyyymmdd_m1} fi if [[ ! -d "$ccpa_raw/${vyyyymmdd_p1}" ]]; then - mkdir -p $ccpa_raw/${vyyyymmdd_p1} + mkdir_vrfy -p $ccpa_raw/${vyyyymmdd_p1} fi if [[ ! -d "$ccpa_proc/${vyyyymmdd}" ]]; then - mkdir -p $ccpa_proc/${vyyyymmdd} + mkdir_vrfy -p $ccpa_proc/${vyyyymmdd} fi # Name of CCPA tar file on HPSS is dependent on date. Logic accounts for files from 2019 until Sept. 2020. @@ -161,13 +161,13 @@ while [[ ${current_fcst} -le ${fcst_length} ]]; do if [[ ${accum} == "01" ]]; then # Check if valid hour is 00 if [[ ${vhh_noZero} -ge 19 && ${vhh_noZero} -le 23 ]]; then - cd $ccpa_raw/${vyyyymmdd_p1} + cd_vrfy $ccpa_raw/${vyyyymmdd_p1} # Pull CCPA data from HPSS TarCommand="htar -xvf ${TarFile_p1} \`htar -tf ${TarFile_p1} | egrep \"ccpa.t${vhh}z.${accum}h.hrap.conus.gb2\" | awk '{print $7}'\`" echo "CALLING: ${TarCommand}" htar -xvf ${TarFile_p1} `htar -tf ${TarFile_p1} | egrep "ccpa.t${vhh}z.${accum}h.hrap.conus.gb2" | awk '{print $7}'` else - cd $ccpa_raw/${vyyyymmdd} + cd_vrfy $ccpa_raw/${vyyyymmdd} # Pull CCPA data from HPSS TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"ccpa.t${vhh}z.${accum}h.hrap.conus.gb2\" | awk '{print $7}'\`" echo "CALLING: ${TarCommand}" @@ -177,35 +177,35 @@ while [[ ${current_fcst} -le ${fcst_length} ]]; do # One hour CCPA files have incorrect metadeta in the files under the "00" directory from 20180718 to 20210504. # After data is pulled, reorganize into correct valid yyyymmdd structure. if [[ ${vhh_noZero} -ge 1 && ${vhh_noZero} -le 6 ]]; then - cp $ccpa_raw/${vyyyymmdd}/06/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} + cp_vrfy $ccpa_raw/${vyyyymmdd}/06/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} elif [[ ${vhh_noZero} -ge 7 && ${vhh_noZero} -le 12 ]]; then - cp $ccpa_raw/${vyyyymmdd}/12/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} + cp_vrfy $ccpa_raw/${vyyyymmdd}/12/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} elif [[ ${vhh_noZero} -ge 13 && ${vhh_noZero} -le 18 ]]; then - cp $ccpa_raw/${vyyyymmdd}/18/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} + cp_vrfy $ccpa_raw/${vyyyymmdd}/18/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} elif [[ ${vhh_noZero} -ge 19 && ${vhh_noZero} -le 23 ]]; then if [[ ${vyyyymmdd} -ge 20180718 && ${vyyyymmdd} -le 20210504 ]]; then wgrib2 $ccpa_raw/${vyyyymmdd_p1}/00/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 -set_date -24hr -grib $ccpa_proc/${vyyyymmdd}/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 -s else - cp $ccpa_raw/${vyyyymmdd_p1}/00/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} + cp_vrfy $ccpa_raw/${vyyyymmdd_p1}/00/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} fi elif [[ ${vhh_noZero} -eq 0 ]]; then if [[ ${vyyyymmdd} -ge 20180718 && ${vyyyymmdd} -le 20210504 ]]; then wgrib2 $ccpa_raw/${vyyyymmdd}/00/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 -set_date -24hr -grib $ccpa_proc/${vyyyymmdd}/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 -s else - cp $ccpa_raw/${vyyyymmdd}/00/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} + cp_vrfy $ccpa_raw/${vyyyymmdd}/00/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} fi fi elif [[ ${accum} == "03" ]]; then # Check if valid hour is 21 if [[ ${vhh_noZero} -ne 21 ]]; then - cd $ccpa_raw/${vyyyymmdd} + cd_vrfy $ccpa_raw/${vyyyymmdd} # Pull CCPA data from HPSS TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"ccpa.t${vhh}z.${accum}h.hrap.conus.gb2\" | awk '{print $7}'\`" echo "CALLING: ${TarCommand}" htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "ccpa.t${vhh}z.${accum}h.hrap.conus.gb2" | awk '{print $7}'` elif [[ ${vhh_noZero} -eq 21 ]]; then - cd $ccpa_raw/${vyyyymmdd_p1} + cd_vrfy $ccpa_raw/${vyyyymmdd_p1} # Pull CCPA data from HPSS TarCommand="htar -xvf ${TarFile_p1} \`htar -tf ${TarFile_p1} | egrep \"ccpa.t${vhh}z.${accum}h.hrap.conus.gb2\" | awk '{print $7}'\`" echo "CALLING: ${TarCommand}" @@ -213,32 +213,32 @@ while [[ ${current_fcst} -le ${fcst_length} ]]; do fi if [[ ${vhh_noZero} -eq 0 ]]; then - cp $ccpa_raw/${vyyyymmdd}/00/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} + cp_vrfy $ccpa_raw/${vyyyymmdd}/00/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} elif [[ ${vhh_noZero} -eq 3 || ${vhh_noZero} -eq 6 ]]; then - cp $ccpa_raw/${vyyyymmdd}/06/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} + cp_vrfy $ccpa_raw/${vyyyymmdd}/06/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} elif [[ ${vhh_noZero} -eq 9 || ${vhh_noZero} -eq 12 ]]; then - cp $ccpa_raw/${vyyyymmdd}/12/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} + cp_vrfy $ccpa_raw/${vyyyymmdd}/12/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} elif [[ ${vhh_noZero} -eq 15 || ${vhh_noZero} -eq 18 ]]; then - cp $ccpa_raw/${vyyyymmdd}/18/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} + cp_vrfy $ccpa_raw/${vyyyymmdd}/18/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} elif [[ ${vhh_noZero} -eq 21 ]]; then - cp $ccpa_raw/${vyyyymmdd_p1}/00/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} + cp_vrfy $ccpa_raw/${vyyyymmdd_p1}/00/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} fi elif [[ ${accum} == "06" ]]; then - cd $ccpa_raw/${vyyyymmdd} + cd_vrfy $ccpa_raw/${vyyyymmdd} # Pull CCPA data from HPSS TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"ccpa.t${vhh}z.${accum}h.hrap.conus.gb2\" | awk '{print $7}'\`" echo "CALLING: ${TarCommand}" htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "ccpa.t${vhh}z.${accum}h.hrap.conus.gb2" | awk '{print $7}'` if [[ ${vhh_noZero} -eq 0 ]]; then - cp $ccpa_raw/${vyyyymmdd}/00/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} + cp_vrfy $ccpa_raw/${vyyyymmdd}/00/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} elif [[ ${vhh_noZero} -eq 6 ]]; then - cp $ccpa_raw/${vyyyymmdd}/06/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} + cp_vrfy $ccpa_raw/${vyyyymmdd}/06/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} elif [[ ${vhh_noZero} -eq 12 ]]; then - cp $ccpa_raw/${vyyyymmdd}/12/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} + cp_vrfy $ccpa_raw/${vyyyymmdd}/12/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} elif [[ ${vhh_noZero} -eq 18 ]]; then - cp $ccpa_raw/${vyyyymmdd}/18/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} + cp_vrfy $ccpa_raw/${vyyyymmdd}/18/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} fi fi fi diff --git a/scripts/exregional_get_obs_mrms.sh b/scripts/exregional_get_obs_mrms.sh index b0c7d805cd..aec156841d 100755 --- a/scripts/exregional_get_obs_mrms.sh +++ b/scripts/exregional_get_obs_mrms.sh @@ -29,19 +29,19 @@ mrms_dir=${OBS_DIR}/.. if [[ ! -d "$mrms_dir" ]]; then - mkdir -p $mrms_dir + mkdir_vrfy -p $mrms_dir fi # MRMS data from HPSS mrms_raw=$mrms_dir/raw if [[ ! -d "$mrms_raw" ]]; then - mkdir -p $mrms_raw + mkdir_vrfy -p $mrms_raw fi # Reorganized MRMS location mrms_proc=$mrms_dir/proc if [[ ! -d "$mrms_proc" ]]; then - mkdir -p $mrms_proc + mkdir_vrfy -p $mrms_proc fi # Initialization @@ -83,7 +83,7 @@ while [[ ${cur_ut} -le ${end_valid_ut} ]]; do # Create necessary raw and proc directories if [[ ! -d "$mrms_raw/${vyyyymmdd}" ]]; then - mkdir -p $mrms_raw/${vyyyymmdd} + mkdir_vrfy -p $mrms_raw/${vyyyymmdd} fi # Set field of interest from the MRMS products, including name and level information. @@ -104,7 +104,7 @@ while [[ ${cur_ut} -le ${end_valid_ut} ]]; do echo "MRMS FILE:${mrms_file}" if [[ ! -f "${mrms_file}" ]]; then - cd $mrms_raw/${vyyyymmdd} + cd_vrfy $mrms_raw/${vyyyymmdd} # Name of MRMS tar file on HPSS is dependent on date. Logic accounts for files from 2019 until Sept. 2020. if [[ ${vyyyymmdd} -ge 20190101 && ${vyyyymmdd} -lt 20200303 ]]; then @@ -140,7 +140,7 @@ while [[ ${cur_ut} -le ${end_valid_ut} ]]; do exit ${Status} else if [[ ! -d "$mrms_proc/${vyyyymmdd}" ]]; then - mkdir -p $mrms_proc/${vyyyymmdd} + mkdir_vrfy -p $mrms_proc/${vyyyymmdd} fi hour=0 diff --git a/scripts/exregional_get_obs_ndas.sh b/scripts/exregional_get_obs_ndas.sh index 4c5520de17..85261cc14c 100755 --- a/scripts/exregional_get_obs_ndas.sh +++ b/scripts/exregional_get_obs_ndas.sh @@ -30,19 +30,19 @@ # Top-level NDAS directory ndas_dir=${OBS_DIR}/.. if [[ ! -d "$ndas_dir" ]]; then - mkdir -p $ndas_dir + mkdir_vrfy -p $ndas_dir fi # NDAS data from HPSS ndas_raw=$ndas_dir/raw if [[ ! -d "$ndas_raw" ]]; then - mkdir -p $ndas_raw + mkdir_vrfy -p $ndas_raw fi # Reorganized NDAS location ndas_proc=$ndas_dir/proc if [[ ! -d "$ndas_proc" ]]; then - mkdir -p $ndas_proc + mkdir_vrfy -p $ndas_proc fi # Initialization @@ -127,9 +127,9 @@ echo "vhh_noZero=$vhh_noZero" if [[ ! -f "${ndas_file}" ]]; then if [[ ! -d "$ndas_raw/${vyyyymmdd}${vhh}" ]]; then - mkdir -p $ndas_raw/${vyyyymmdd}${vhh} + mkdir_vrfy -p $ndas_raw/${vyyyymmdd}${vhh} fi - cd $ndas_raw/${vyyyymmdd}${vhh} + cd_vrfy $ndas_raw/${vyyyymmdd}${vhh} # Name of NDAS tar file on HPSS is dependent on date. Logic accounts for files from 2019 until July 2020. if [[ ${vyyyymmdd} -ge 20190101 && ${vyyyymmdd} -le 20190820 ]]; then @@ -150,17 +150,17 @@ echo "vhh_noZero=$vhh_noZero" fi if [[ ! -d "$ndas_proc" ]]; then - mkdir -p $ndas_proc + mkdir_vrfy -p $ndas_proc fi if [[ ${vhh_noZero} -eq 0 || ${vhh} -eq 6 || ${vhh} -eq 12 || ${vhh} -eq 18 ]]; then #echo "$ndas_raw/${vyyyymmdd}${vhh}/nam.t${vhh}z.prepbufr.tm00.nr $ndas_proc/prepbufr.ndas.${vyyyymmdd}${vhh}" - cp $ndas_raw/${vyyyymmdd}${vhh}/nam.t${vhh}z.prepbufr.tm00.nr $ndas_proc/prepbufr.ndas.${vyyyymmdd}${vhh} - cp $ndas_raw/${vyyyymmdd}${vhh}/nam.t${vhh}z.prepbufr.tm01.nr $ndas_proc/prepbufr.ndas.${vyyyymmdd_m1h}${vhh_m1h} - cp $ndas_raw/${vyyyymmdd}${vhh}/nam.t${vhh}z.prepbufr.tm02.nr $ndas_proc/prepbufr.ndas.${vyyyymmdd_m2h}${vhh_m2h} - cp $ndas_raw/${vyyyymmdd}${vhh}/nam.t${vhh}z.prepbufr.tm03.nr $ndas_proc/prepbufr.ndas.${vyyyymmdd_m3h}${vhh_m3h} - cp $ndas_raw/${vyyyymmdd}${vhh}/nam.t${vhh}z.prepbufr.tm04.nr $ndas_proc/prepbufr.ndas.${vyyyymmdd_m4h}${vhh_m4h} - cp $ndas_raw/${vyyyymmdd}${vhh}/nam.t${vhh}z.prepbufr.tm05.nr $ndas_proc/prepbufr.ndas.${vyyyymmdd_m5h}${vhh_m5h} + cp_vrfy $ndas_raw/${vyyyymmdd}${vhh}/nam.t${vhh}z.prepbufr.tm00.nr $ndas_proc/prepbufr.ndas.${vyyyymmdd}${vhh} + cp_vrfy $ndas_raw/${vyyyymmdd}${vhh}/nam.t${vhh}z.prepbufr.tm01.nr $ndas_proc/prepbufr.ndas.${vyyyymmdd_m1h}${vhh_m1h} + cp_vrfy $ndas_raw/${vyyyymmdd}${vhh}/nam.t${vhh}z.prepbufr.tm02.nr $ndas_proc/prepbufr.ndas.${vyyyymmdd_m2h}${vhh_m2h} + cp_vrfy $ndas_raw/${vyyyymmdd}${vhh}/nam.t${vhh}z.prepbufr.tm03.nr $ndas_proc/prepbufr.ndas.${vyyyymmdd_m3h}${vhh_m3h} + cp_vrfy $ndas_raw/${vyyyymmdd}${vhh}/nam.t${vhh}z.prepbufr.tm04.nr $ndas_proc/prepbufr.ndas.${vyyyymmdd_m4h}${vhh_m4h} + cp_vrfy $ndas_raw/${vyyyymmdd}${vhh}/nam.t${vhh}z.prepbufr.tm05.nr $ndas_proc/prepbufr.ndas.${vyyyymmdd_m5h}${vhh_m5h} fi fi current_fcst=$((${current_fcst} + 6)) From b4004d7a77e4af4ff7d223db0530bf24e399c18f Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Tue, 16 Aug 2022 02:25:18 +0000 Subject: [PATCH 009/106] Add job and jobid to the job-card. --- ush/load_modules_run_task.sh | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/ush/load_modules_run_task.sh b/ush/load_modules_run_task.sh index 4e08ed62f0..8cb55e42b1 100755 --- a/ush/load_modules_run_task.sh +++ b/ush/load_modules_run_task.sh @@ -86,6 +86,23 @@ jjob_fp="$2" # #----------------------------------------------------------------------- # +# For NCO mode we need to define job and jobid +# +#----------------------------------------------------------------------- +# +if [ ! -z ${SLURM_JOB_ID} ]; then + export job=${SLURM_JOB_NAME} + export jobid=${job}.${SLURM_JOB_ID} +elif [ ! -z ${PBS_JOBID} ]; then + export job=${PBS_JOBNAME} + export jobid=${job}.${PBS_JOB_ID} +else + export job=${task_name} + export jobid=${job}.$$ +fi +# +#----------------------------------------------------------------------- +# # Loading ufs-srweather-app build module files # #----------------------------------------------------------------------- From a6c1173c6b004a4e0600a183523c943ecb8ba032 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Tue, 16 Aug 2022 02:56:56 +0000 Subject: [PATCH 010/106] Add cyc and subcyc to rocoto xml --- ush/templates/FV3LAM_wflow.xml | 69 +++++++++++++++++++++++++++------- 1 file changed, 56 insertions(+), 13 deletions(-) diff --git a/ush/templates/FV3LAM_wflow.xml b/ush/templates/FV3LAM_wflow.xml index 219798afb1..58846325e2 100644 --- a/ush/templates/FV3LAM_wflow.xml +++ b/ush/templates/FV3LAM_wflow.xml @@ -278,6 +278,8 @@ MODULES_RUN_TASK_FP script. GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; PDY@Y@m@d + cyc@H + subcyc@M CDATE@Y@m@d@H CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H ICS_OR_LBCSICS @@ -304,6 +306,8 @@ MODULES_RUN_TASK_FP script. GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; PDY@Y@m@d + cyc@H + subcyc@M CDATE@Y@m@d@H CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H ICS_OR_LBCSLBCS @@ -340,6 +344,8 @@ MODULES_RUN_TASK_FP script. GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; PDY@Y@m@d + cyc@H + subcyc@M CDATE@Y@m@d@H CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} @@ -387,6 +393,8 @@ MODULES_RUN_TASK_FP script. GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; PDY@Y@m@d + cyc@H + subcyc@M CDATE@Y@m@d@H CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} @@ -439,6 +447,8 @@ MODULES_RUN_TASK_FP script. GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; PDY@Y@m@d + cyc@H + subcyc@M CDATE@Y@m@d@H CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} @@ -492,12 +502,13 @@ later below for other output times. GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; PDY@Y@m@d - CDATE@Y@m@d@H - CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H - SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} cyc@H + subcyc@M fhr#fhr# fmn#fmn# + CDATE@Y@m@d@H + CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} - + @@ -93,6 +93,13 @@ Directories and files. +{%- if run_envir == "nco" %} + + +{%- else %} + + +{%- endif %} @@ -518,8 +525,8 @@ names in the dependencies. - &COMIN_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/dynf{{ first_fv3_file_tstr }}.nc - &COMIN_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/phyf{{ first_fv3_file_tstr }}.nc + &DYN_DIR;f{{ first_fv3_file_tstr }}.nc + &PHY_DIR;f{{ first_fv3_file_tstr }}.nc @@ -570,8 +577,8 @@ for other output times. - &COMIN_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/dynf#fhr#:#fmn#:00.nc - &COMIN_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/phyf#fhr#:#fmn#:00.nc + &DYN_DIR;f#fhr#:#fmn#:00.nc + &PHY_DIR;f#fhr#:#fmn#:00.nc @@ -643,11 +650,11 @@ always zero). {%- if sub_hourly_post %} - &COMIN_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/dynf#fhr#:#fmn#:00.nc - &COMIN_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/phyf#fhr#:#fmn#:00.nc + &DYN_DIR;f#fhr#:#fmn#:00.nc + &PHY_DIR;f#fhr#:#fmn#:00.nc {%- else %} - &COMIN_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/dynf#fhr#.nc - &COMIN_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/phyf#fhr#.nc + &DYN_DIR;f#fhr#.nc + &PHY_DIR;f#fhr#.nc {%- endif %} @@ -705,8 +712,8 @@ the tag to be identical to the ones above for other output times. - &COMIN_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/dynf#fhr#:#fmn#:00.nc - &COMIN_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/phyf#fhr#:#fmn#:00.nc + &DYN_DIR;f#fhr#:#fmn#:00.nc + &PHY_DIR;f#fhr#:#fmn#:00.nc From 4b2bc0e1f4b976d9719c5218860a68fa6c5f6d39 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Sat, 20 Aug 2022 15:03:38 +0000 Subject: [PATCH 032/106] Use OPSROOT instead of PTMP and STMP. --- scripts/exregional_run_fcst.sh | 2 +- scripts/exregional_run_post.sh | 2 +- tests/WE2E/run_WE2E_tests.sh | 103 ++++----------------------------- ush/config.nco.yaml | 3 +- ush/config_defaults.yaml | 30 +--------- ush/job_preamble.sh | 14 +++-- ush/setup.py | 54 ++++------------- 7 files changed, 35 insertions(+), 173 deletions(-) diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index b8f7aa867f..5ba88a4cda 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -479,7 +479,7 @@ Call to function to create a diag table file for the current cycle's # #----------------------------------------------------------------------- # -if [ ! -z ${DATAROOT} ] && [ "${DATA}" != "${COMOUT}" ]; then +if [ "${RUN_ENVIR}" = "nco" ]; then # create comout directory mkdir_vrfy -p ${COMOUT} diff --git a/scripts/exregional_run_post.sh b/scripts/exregional_run_post.sh index 631cb7c3f1..8feee72c7a 100755 --- a/scripts/exregional_run_post.sh +++ b/scripts/exregional_run_post.sh @@ -169,7 +169,7 @@ fi # # Set the names of the forecast model's write-component output files. # -if [ -z ${DATAROOT} ]; then +if [ "${RUN_ENVIR}" != "nco" ]; then dyn_file="${DATA}/dynf${fhr}${mnts_secs_str}.nc" phy_file="${DATA}/phyf${fhr}${mnts_secs_str}.nc" else diff --git a/tests/WE2E/run_WE2E_tests.sh b/tests/WE2E/run_WE2E_tests.sh index 841c10d0ec..f53e6a5bb2 100755 --- a/tests/WE2E/run_WE2E_tests.sh +++ b/tests/WE2E/run_WE2E_tests.sh @@ -92,8 +92,7 @@ Usage: [verbose=\"...\"] \\ [generate_csv_file=\"...\"] \\ [machine_file=\"...\"] \\ - [stmp=\"...\"] \\ - [ptmp=\"...\"] \\ + [opsroot=\"...\"] \\ [compiler=\"...\"] \\ [build_mod_fn=\"...\"] @@ -192,29 +191,8 @@ machine_file: Optional argument specifying the full path to a machine configuration file. If not set, a supported platform machine file may be used. -stmp: -Optional argument used to explicitly set the experiment variable STMP in -the experiment configuration files of all the WE2E tests the user wants -to run that are in NCO mode, i.e. they have test configuration files that -set the experiment variable RUN_ENVIR to \"nco\". (A description of -STMP can be found in the default experiment configuration file.) If -stmp is specified in the call to this script, its value is used to set -STMP in the configuration files of all tests that will run in NCO mode. -If it is not specified, STMP is (effectively) set as follows in the -configuration files (of all NCO mode tests to be run): - - STMP=\$( readlink -f \"\$SR_WX_APP_TOP_DIR/../nco_dirs/stmp\" \) - -Here, SR_WX_APP_TOP_DIR is the base directory in which the UFS SRW App -repository is cloned. Note that it is not possible to specify a different -value for STMP for each test via this argument; all tests will use the -same value for STMP (either the value specified in the call to this -script or the default value above). Note also that the value of this -argument is not used for any tests that are not in NCO mode. - -ptmp: -Same as the argument \"stmp\" described above but for setting the -experiment variable PTMP for all tests that will run in NCO mode. +opsroot: +Operations root directory in NCO mode compiler: Optional argument used to explicitly set the experiment variable COMPILER @@ -340,8 +318,7 @@ valid_args=( \ "verbose" \ "generate_csv_file" \ "machine_file" \ - "stmp" \ - "ptmp" \ + "opsroot" \ "compiler" \ "build_mod_fn" \ ) @@ -928,82 +905,22 @@ SFC_CLIMO_DIR=\"${SFC_CLIMO_DIR}\"" # expt_config_str=${expt_config_str}" # -# In order to prevent simultaneous WE2E (Workflow End-to-End) tests that -# are running in NCO mode and which run the same cycles from interfering -# with each other, for each cycle, each such test must have a distinct -# path to the following two directories: -# -# 1) The directory in which the cycle-dependent model input files, symlinks -# to cycle-independent input files, and raw (i.e. before post-processing) -# forecast output files for a given cycle are stored. The path to this -# directory is -# -# \$STMP/tmpnwprd/\$RUN/\$cdate -# -# where cdate is the starting year (yyyy), month (mm), day (dd) and -# hour of the cycle in the form yyyymmddhh. -# -# 2) The directory in which the output files from the post-processor (UPP) -# for a given cycle are stored. The path to this directory is -# -# \$PTMP/com/\$NET/\$model_ver/\$RUN.\$yyyymmdd/\$hh -# -# Here, we make the first directory listed above unique to a WE2E test -# by setting RUN to the name of the current test. This will also make -# the second directory unique because it also conains the variable RUN -# in its full path, but if this directory -- or set of directories since -# it involves a set of cycles and forecast hours -- already exists from -# a previous run of the same test, then it is much less confusing to the -# user to first move or delete this set of directories during the workflow -# generation step and then start the experiment (whether we move or delete -# depends on the setting of PREEXISTING_DIR_METHOD). For this purpose, -# it is most convenient to put this set of directories under an umbrella -# directory that has the same name as the experiment. This can be done -# by setting the variable envir to the name of the current test. Since -# as mentiond above we will store this name in RUN, below we simply set -# envir to the same value as RUN (which is just EXPT_SUBDIR). Then, for -# this test, the UPP output will be located in the directory -# -# \$PTMP/com/\$NET/\we2e/\$RUN.\$yyyymmdd/\$hh +# Set NCO mode RUN and model_ver # RUN=\"\${EXPT_SUBDIR}\" model_ver="we2e"" # -# Set COMIN. - - COMIN=${TEST_COMIN:-} - - if [ ! -d "${COMIN:-}" ] ; then - print_err_msg_exit "\ -The directory (COMIN) that needs to be specified when running the -workflow in NCO mode (RUN_ENVIR set to \"nco\") AND using the FV3GFS or -the GSMGFS as the external model for ICs and/or LBCs has not been specified -for this machine (MACHINE): - MACHINE= \"${MACHINE}\"" - fi - - expt_config_str=${expt_config_str}" -# -# Directory that needs to be specified when running the workflow in NCO -# mode (RUN_ENVIR set to \"nco\"). -# -COMIN=\"${COMIN}\"" - -# -# Set STMP and PTMP. +# Set OPSROOT. # - nco_basedir=$( readlink -f "$SR_WX_APP_TOP_DIR/../nco_dirs" ) - STMP=${stmp:-"${nco_basedir}/stmp"} - PTMP=${ptmp:-"${nco_basedir}/ptmp"} + nco_basedir=$( readlink -f "$homerrfs/../../nco_dirs" ) + OPSROOT=${opsroot:-"${nco_basedir}/OPSROOT"} expt_config_str=${expt_config_str}" # -# Directories STMP and PTMP that need to be specified when running the -# workflow in NCO-mode (i.e. RUN_ENVIR set to "nco"). +# Set NCO mode OPSROOT # -STMP=\"${STMP}\" -PTMP=\"${PTMP}\"" +OPSROOT=\"${OPSROOT}\"" fi # diff --git a/ush/config.nco.yaml b/ush/config.nco.yaml index df6bcf5a98..c1de5421dd 100644 --- a/ush/config.nco.yaml +++ b/ush/config.nco.yaml @@ -15,8 +15,7 @@ workflow: NET: rrfs model_ver: v1.0 RUN: rrfs_test - STMP: /path/to/stmp/directory - PTMP: /path/to/ptmp/directory + OPSROOT: /path/to/opsroot/directory CCPP_PHYS_SUITE: FV3_GFS_v16 DATE_FIRST_CYCL: '20220407' DATE_LAST_CYCL: '20220407' diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 0c298401eb..28602fa7ee 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -376,12 +376,6 @@ workflow: # Set variables that are only used in NCO mode (i.e. when RUN_ENVIR is # set to "nco"). Definitions: # - # COMIN: - # Directory containing files generated by the external model (FV3GFS, NAM, - # HRRR, etc) that the initial and lateral boundary condition generation tasks - # need in order to create initial and boundary condition files for a given - # cycle on the native FV3-LAM grid. - # # envir, NET, model_ver, RUN: # Standard environment variables defined in the NCEP Central Operations WCOSS # Implementation Standards document as follows: @@ -400,26 +394,9 @@ workflow: # Name of model run (third level of com directory structure). # In general, same as $NET # - # STMP: - # The beginning portion of the directory that will contain cycle-dependent - # model input files, symlinks to cycle-independent input files, and raw - # (i.e. before post-processing) forecast output files for a given cycle. - # For a cycle that starts on the date specified by yyyymmdd and hour - # specified by hh (where yyyymmdd and hh are as described above) [so that - # the cycle date (cdate) is given by cdate: "${yyyymmdd}${hh}"], the - # directory in which the aforementioned files will be located is: - # - # $STMP/tmpnwprd/$RUN/$cdate - # - # PTMP: - # The beginning portion of the directory that will contain the output - # files from the post-processor (UPP) for a given cycle. For a cycle - # that starts on the date specified by yyyymmdd and hour specified by hh - # (where yyyymmdd and hh are as described above), the directory in which - # the UPP output files will be placed will be: + # OPSROOT: + # The operations root directory in NCO mode. # - # $PTMP/com/$NET/$model_ver/$RUN.$yyyymmdd/$hh - # # For more information on NCO standards # # https://www.nco.ncep.noaa.gov/idsb/implementation_standards/ImplementationStandards.v11.0.0.pdf @@ -430,8 +407,7 @@ workflow: NET: "rrfs" RUN: "rrfs" model_ver: "v1.0.0" - STMP: "" - PTMP: "" + OPSROOT: "" # #----------------------------------------------------------------------- # diff --git a/ush/job_preamble.sh b/ush/job_preamble.sh index 32c957aba7..66e13335cf 100644 --- a/ush/job_preamble.sh +++ b/ush/job_preamble.sh @@ -8,7 +8,7 @@ #----------------------------------------------------------------------- # export DATA= -if [ ! -z ${DATAROOT} ]; then +if [ "${RUN_ENVIR}" = "nco" ]; then export DATA=${DATAROOT}/${jobid} mkdir_vrfy -p $DATA cd $DATA @@ -21,9 +21,11 @@ fi #----------------------------------------------------------------------- # export cycle="t${cyc}${subcyc}z" -if [ ! -z $(command -v setpdy.sh) ]; then - setpdy.sh - . ./PDY +if [ "${RUN_ENVIR}" = "nco" ]; then + if [ ! -z $(command -v setpdy.sh) ]; then + setpdy.sh + . ./PDY + fi fi # #----------------------------------------------------------------------- @@ -41,7 +43,7 @@ export pgmerr="errfile" # #----------------------------------------------------------------------- # -if [ ! -z ${DATAROOT} ]; then +if [ "${RUN_ENVIR}" = "nco" ]; then export COMIN="${COMIN_BASEDIR}/${RUN}.${PDY}/${cyc}" export COMOUT="${COMOUT_BASEDIR}/${RUN}.${PDY}" else @@ -63,7 +65,7 @@ function job_postamble() { fi # Remove temp directory - if [ ! -z ${DATA} ]; then + if [ "${RUN_ENVIR}" = "nco" ]; then [[ $KEEPDATA = "FALSE" ]] && rm -rf $DATA fi diff --git a/ush/setup.py b/ush/setup.py index a62cb437c4..636a21c632 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -883,30 +883,6 @@ def setup(): # the fixed files containing the grid, orography, and surface climatology # on the native FV3-LAM grid. # - # COMIN_BASEDIR: - # The base directory in which the directories for the various cycles will - # be placed. - # - # COMROOT: - # In NCO mode, this is the full path to the "com" directory under which - # output from the RUN_POST_TN task will be placed. Note that this output - # is not placed directly under COMROOT but several directories further - # down. More specifically, for a cycle starting at yyyymmddhh, it is at - # - # $COMROOT/$NET/$envir/$RUN.$yyyymmdd/$hh - # - # Below, we set COMROOT in terms of PTMP as COMROOT="$PTMP/com". COMROOT - # is not used by the workflow in community mode. - # - # COMOUT_BASEDIR: - # In NCO mode, this is the base directory directly under which the output - # from the RUN_POST_TN task will be placed, i.e. it is the cycle-independent - # portion of the RUN_POST_TN task's output directory. It is given by - # - # $COMROOT/$NET/$model_ver - # - # COMOUT_BASEDIR is not used by the workflow in community mode. - # # POST_OUTPUT_DOMAIN_NAME: # The PREDEF_GRID_NAME is set by default. # @@ -930,26 +906,18 @@ def setup(): if RUN_ENVIR == "nco": try: - OPSROOT + COMROOT except NameError: - if PTMP is not None: - OPSROOT = PTMP - COMROOT = os.path.join(OPSROOT, "com") - PACKAGEROOT = os.path.join(OPSROOT, "packages") - DATAROOT = os.path.join(OPSROOT, "tmp") - DCOMROOT = os.path.join(OPSROOT, "dcom") - else: - OPSROOT = None - COMROOT = None - PACKAGEROOT = None - DATAROOT = None - DCOMROOT = None - - if OPSROOT is not None: - COMIN_BASEDIR = os.path.join(COMROOT, NET, model_ver) - check_for_preexist_dir_file(COMIN_BASEDIR, PREEXISTING_DIR_METHOD) - COMOUT_BASEDIR = os.path.join(COMROOT, NET, model_ver) - check_for_preexist_dir_file(COMOUT_BASEDIR, PREEXISTING_DIR_METHOD) + OPSROOT = OPSROOT or EXPTDIR + COMROOT = os.path.join(OPSROOT, "com") + PACKAGEROOT = os.path.join(OPSROOT, "packages") + DATAROOT = os.path.join(OPSROOT, "tmp") + DCOMROOT = os.path.join(OPSROOT, "dcom") + + COMIN_BASEDIR = os.path.join(COMROOT, NET, model_ver) + check_for_preexist_dir_file(COMIN_BASEDIR, PREEXISTING_DIR_METHOD) + COMOUT_BASEDIR = os.path.join(COMROOT, NET, model_ver) + check_for_preexist_dir_file(COMOUT_BASEDIR, PREEXISTING_DIR_METHOD) else: From fb0a8485943717249500309376952ce7daa903aa Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Sat, 20 Aug 2022 15:37:02 +0000 Subject: [PATCH 033/106] Move nco vars in config_defaults. --- ush/config.nco.yaml | 10 ++--- ush/config_defaults.yaml | 81 +++++++++++++++++++++------------------- ush/job_preamble.sh | 2 +- ush/setup.py | 2 +- 4 files changed, 50 insertions(+), 45 deletions(-) diff --git a/ush/config.nco.yaml b/ush/config.nco.yaml index c1de5421dd..1380087859 100644 --- a/ush/config.nco.yaml +++ b/ush/config.nco.yaml @@ -11,11 +11,6 @@ workflow: USE_CRON_TO_RELAUNCH: true CRON_RELAUNCH_INTVL_MNTS: 3 EXPT_SUBDIR: test_nco - COMIN: /scratch1/NCEPDEV/rstprod/com/gfs/prod - NET: rrfs - model_ver: v1.0 - RUN: rrfs_test - OPSROOT: /path/to/opsroot/directory CCPP_PHYS_SUITE: FV3_GFS_v16 DATE_FIRST_CYCL: '20220407' DATE_LAST_CYCL: '20220407' @@ -25,6 +20,11 @@ workflow: PREEXISTING_DIR_METHOD: rename VERBOSE: true COMPILER: intel +nco: + NET: rrfs + model_ver: v1.0 + RUN: rrfs_test + OPSROOT: /path/to/opsroot/directory task_get_extrn_ics: EXTRN_MDL_NAME_ICS: FV3GFS FV3GFS_FILE_FMT_ICS: grib2 diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 28602fa7ee..7bb9e41048 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -373,44 +373,6 @@ workflow: # #----------------------------------------------------------------------- # - # Set variables that are only used in NCO mode (i.e. when RUN_ENVIR is - # set to "nco"). Definitions: - # - # envir, NET, model_ver, RUN: - # Standard environment variables defined in the NCEP Central Operations WCOSS - # Implementation Standards document as follows: - # - # envir: - # Set to "test" during the initial testing phase, "para" when running - # in parallel (on a schedule), and "prod" in production. - # - # NET: - # Model name (first level of com directory structure) - # - # model_ver: - # Version number of package in three digits (second level of com directory) - # - # RUN: - # Name of model run (third level of com directory structure). - # In general, same as $NET - # - # OPSROOT: - # The operations root directory in NCO mode. - # - # For more information on NCO standards - # - # https://www.nco.ncep.noaa.gov/idsb/implementation_standards/ImplementationStandards.v11.0.0.pdf - # - #----------------------------------------------------------------------- - # - envir: "para" - NET: "rrfs" - RUN: "rrfs" - model_ver: "v1.0.0" - OPSROOT: "" - # - #----------------------------------------------------------------------- - # # Set the separator character(s) to use in the names of the grid, mosaic, # and orography fixed files. # @@ -687,6 +649,49 @@ workflow: VX_ENSGRID_PROB_REFC_TN: "run_ensgridvx_prob_refc" MAXTRIES_VX_ENSGRID_PROB_REFC: 1 +#---------------------------- +# NCO specific variables +#----------------------------- +nco: + # + #----------------------------------------------------------------------- + # + # Set variables that are only used in NCO mode (i.e. when RUN_ENVIR is + # set to "nco"). Definitions: + # + # envir, NET, model_ver, RUN: + # Standard environment variables defined in the NCEP Central Operations WCOSS + # Implementation Standards document as follows: + # + # envir: + # Set to "test" during the initial testing phase, "para" when running + # in parallel (on a schedule), and "prod" in production. + # + # NET: + # Model name (first level of com directory structure) + # + # model_ver: + # Version number of package in three digits (second level of com directory) + # + # RUN: + # Name of model run (third level of com directory structure). + # In general, same as $NET + # + # OPSROOT: + # The operations root directory in NCO mode. + # + # For more information on NCO standards + # + # https://www.nco.ncep.noaa.gov/idsb/implementation_standards/ImplementationStandards.v11.0.0.pdf + # + #----------------------------------------------------------------------- + # + envir: "para" + NET: "rrfs" + RUN: "rrfs" + model_ver: "v1.0.0" + OPSROOT: "" + #---------------------------- # WORKFLOW SWITCHES config parameters #----------------------------- diff --git a/ush/job_preamble.sh b/ush/job_preamble.sh index 66e13335cf..3663a5bfa7 100644 --- a/ush/job_preamble.sh +++ b/ush/job_preamble.sh @@ -20,7 +20,7 @@ fi # #----------------------------------------------------------------------- # -export cycle="t${cyc}${subcyc}z" +export cycle="t${cyc}z" if [ "${RUN_ENVIR}" = "nco" ]; then if [ ! -z $(command -v setpdy.sh) ]; then setpdy.sh diff --git a/ush/setup.py b/ush/setup.py index 636a21c632..07551bbeda 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -2171,7 +2171,7 @@ def setup(): "MAILCC": MAILCC, } - cfg_d["nco"] = settings + cfg_d["nco"].update(settings) # # ----------------------------------------------------------------------- # From 5f84dd85e01496f9947149e9d6008fbf64def67e Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Sat, 20 Aug 2022 16:35:13 +0000 Subject: [PATCH 034/106] Move logdir location to COMROOT. --- ush/setup.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ush/setup.py b/ush/setup.py index 07551bbeda..b192469108 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -897,7 +897,6 @@ def setup(): global module_ver, extmodel_ver global KEEPDATA, MAILTO, MAILCC - LOGDIR = os.path.join(EXPTDIR, "log") FIXam = os.path.join(EXPTDIR, "fix_am") FIXclim = os.path.join(EXPTDIR, "fix_clim") @@ -919,6 +918,8 @@ def setup(): COMOUT_BASEDIR = os.path.join(COMROOT, NET, model_ver) check_for_preexist_dir_file(COMOUT_BASEDIR, PREEXISTING_DIR_METHOD) + LOGDIR = os.path.join(COMROOT,"output", "logs") + else: COMIN_BASEDIR = EXPTDIR @@ -929,6 +930,8 @@ def setup(): DATAROOT = None DCOMROOT = None + LOGDIR = os.path.join(EXPTDIR, "log") + try: DBNROOT except NameError: From ac36ceac56361c581bdb8c6c5dad69b1c149b486 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Sat, 20 Aug 2022 17:19:17 +0000 Subject: [PATCH 035/106] Set all root directories to EXPTDIR in community mode. --- ush/setup.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ush/setup.py b/ush/setup.py index b192469108..eb0be1924e 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -924,11 +924,11 @@ def setup(): COMIN_BASEDIR = EXPTDIR COMOUT_BASEDIR = EXPTDIR - OPSROOT = None - COMROOT = None - PACKAGEROOT = None - DATAROOT = None - DCOMROOT = None + OPSROOT = EXPTDIR + COMROOT = EXPTDIR + PACKAGEROOT = EXPTDIR + DATAROOT = EXPTDIR + DCOMROOT = EXPTDIR LOGDIR = os.path.join(EXPTDIR, "log") From add637bbfbb6753ce0a3c617a9e794f445330875 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Sat, 20 Aug 2022 22:55:19 +0000 Subject: [PATCH 036/106] Use pgmout and pgmerr. --- scripts/exregional_get_extrn_mdl_files.sh | 2 +- scripts/exregional_make_grid.sh | 10 +++++----- scripts/exregional_make_ics.sh | 6 +++--- scripts/exregional_make_lbcs.sh | 2 +- scripts/exregional_make_orog.sh | 10 +++++----- scripts/exregional_run_fcst.sh | 2 +- scripts/exregional_run_post.sh | 2 +- ush/job_preamble.sh | 9 +++++++-- 8 files changed, 24 insertions(+), 19 deletions(-) diff --git a/scripts/exregional_get_extrn_mdl_files.sh b/scripts/exregional_get_extrn_mdl_files.sh index de74e2a139..6489e98682 100755 --- a/scripts/exregional_get_extrn_mdl_files.sh +++ b/scripts/exregional_get_extrn_mdl_files.sh @@ -138,7 +138,7 @@ python3 -u ${USHrrfs}/retrieve_data.py \ --summary_file ${EXTRN_MDL_VAR_DEFNS_FN} \ $additional_flags" -$cmd || print_err_msg_exit "\ +$cmd >>$pgmout 2>$pgmerr || print_err_msg_exit "\ Call to retrieve_data.py failed with a non-zero exit status. The command was: diff --git a/scripts/exregional_make_grid.sh b/scripts/exregional_make_grid.sh index 55edc2bc72..068e0a2912 100755 --- a/scripts/exregional_make_grid.sh +++ b/scripts/exregional_make_grid.sh @@ -233,7 +233,7 @@ if [ "${GRID_GEN_METHOD}" = "GFDLgrid" ]; then --iend_nest ${IEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG} \ --jend_nest ${JEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG} \ --halo 1 \ - --great_circle_algorithm || \ + --great_circle_algorithm >>$pgmout 2>$pgmerr || \ print_err_msg_exit "\ Call to executable (exec_fp) that generates grid files returned with nonzero exit code. @@ -291,7 +291,7 @@ $settings" # # Call the executable that generates the grid file. # - $RUN_CMD_SERIAL ${exec_fp} ${rgnl_grid_nml_fp} || \ + $RUN_CMD_SERIAL ${exec_fp} ${rgnl_grid_nml_fp} >>$pgmout 2>$pgmerr || \ print_err_msg_exit "\ Call to executable (exec_fp) that generates a ESGgrid-type regional grid returned with nonzero exit code: @@ -330,7 +330,7 @@ cubed-sphere grid equivalent resolution does not exist: Please ensure that you've built this executable." fi -$RUN_CMD_SERIAL ${exec_fp} "${grid_fp}" || \ +$RUN_CMD_SERIAL ${exec_fp} "${grid_fp}" >>$pgmout 2>$pgmerr || \ print_err_msg_exit "\ Call to executable (exec_fp) that calculates the regional grid's global uniform cubed-sphere grid equivalent resolution returned with nonzero exit @@ -459,7 +459,7 @@ printf "%s %s %s %s %s\n" \ $NX $NY ${NH3} \"${unshaved_fp}\" \"${shaved_fp}\" \ > ${nml_fn} -$RUN_CMD_SERIAL ${exec_fp} < ${nml_fn} || \ +$RUN_CMD_SERIAL ${exec_fp} < ${nml_fn} >>$pgmout 2>$pgmerr || \ print_err_msg_exit "\ Call to executable (exec_fp) to generate a grid file with a ${NH3}-cell-wide halo from the grid file with a ${NHW}-cell-wide halo returned with nonzero @@ -485,7 +485,7 @@ printf "%s %s %s %s %s\n" \ $NX $NY ${NH4} \"${unshaved_fp}\" \"${shaved_fp}\" \ > ${nml_fn} -$RUN_CMD_SERIAL ${exec_fp} < ${nml_fn} || \ +$RUN_CMD_SERIAL ${exec_fp} < ${nml_fn} >>$pgmout 2>$pgmerr || \ print_err_msg_exit "\ Call to executable (exec_fp) to generate a grid file with a ${NH4}-cell-wide halo from the grid file with a ${NHW}-cell-wide halo returned with nonzero diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index a40af82cf1..c043e6f9dd 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -557,7 +557,7 @@ $settings" # exit code of chgres_cube is nonzero. A similar thing happens in the # forecast task. # -${RUN_CMD_UTILS} ${exec_fp} || \ +${RUN_CMD_UTILS} ${exec_fp} >>$pgmout 2>$pgmerr || \ print_err_msg_exit "\ Call to executable (exec_fp) to generate surface and initial conditions (ICs) files for the FV3-LAM failed: @@ -617,8 +617,8 @@ Please check the following user defined variables: cp_vrfy ${fvcom_data_fp} ${INPUT_DATA}/fvcom.nc cd_vrfy ${INPUT_DATA} - ${RUN_CMD_UTILS} ${fvcom_exec_fn} sfc_data.tile${TILE_RGNL}.halo${NH0}.nc fvcom.nc ${FVCOM_WCSTART} ${fvcom_time}|| \ - print_err_msg_exit "\ + ${RUN_CMD_UTILS} ${fvcom_exec_fn} sfc_data.tile${TILE_RGNL}.halo${NH0}.nc fvcom.nc ${FVCOM_WCSTART} ${fvcom_time} \ + >>$pgmout 2>$pgmerr || print_err_msg_exit "\ Call to executable (fvcom_exe) to modify sfc fields for FV3-LAM failed: fvcom_exe = \"${fvcom_exe}\" The following variables were being used: diff --git a/scripts/exregional_make_lbcs.sh b/scripts/exregional_make_lbcs.sh index 7ce25a1b49..e648b9dabb 100755 --- a/scripts/exregional_make_lbcs.sh +++ b/scripts/exregional_make_lbcs.sh @@ -470,7 +470,7 @@ $settings" # exit code of chgres_cube is nonzero. A similar thing happens in the # forecast task. # - ${RUN_CMD_UTILS} ${exec_fp} || \ + ${RUN_CMD_UTILS} ${exec_fp} >>$pgmout 2>$pgmerr || \ print_err_msg_exit "\ Call to executable (exec_fp) to generate lateral boundary conditions (LBCs) file for the FV3-LAM for forecast hour fhr failed: diff --git a/scripts/exregional_make_orog.sh b/scripts/exregional_make_orog.sh index 2f2ce1c31e..8bd24f9297 100755 --- a/scripts/exregional_make_orog.sh +++ b/scripts/exregional_make_orog.sh @@ -210,7 +210,7 @@ cat "${input_redirect_fn}" print_info_msg "$VERBOSE" "\ Starting orography file generation..." -${RUN_CMD_SERIAL} "${exec_fp}" < "${input_redirect_fn}" || \ +${RUN_CMD_SERIAL} "${exec_fp}" < "${input_redirect_fn}" >>$pgmout 2>$pgmerr || \ print_err_msg_exit "\ Call to executable (exec_fp) that generates the raw orography file returned with nonzero exit code: @@ -282,7 +282,7 @@ Please ensure that you've built this executable." print_info_msg "$VERBOSE" " Starting orography file generation..." - ${RUN_CMD_SERIAL} "${exec_fp}" < "${input_redirect_fn}" || \ + ${RUN_CMD_SERIAL} "${exec_fp}" < "${input_redirect_fn}" >>$pgmout 2>$pgmerr || \ print_err_msg_exit "\ Call to executable (exec_fp) that generates the GSL orography GWD data files returned with nonzero exit code: @@ -430,7 +430,7 @@ cd_vrfy "${filter_dir}" print_info_msg "$VERBOSE" " Starting filtering of orography..." -${RUN_CMD_SERIAL} "${exec_fp}" || \ +${RUN_CMD_SERIAL} "${exec_fp}" >>$pgmout 2>$pgmerr || \ print_err_msg_exit "\ Call to executable that generates filtered orography file returned with non-zero exit code." @@ -499,7 +499,7 @@ printf "%s %s %s %s %s\n" \ $NX $NY ${NH0} \"${unshaved_fp}\" \"${shaved_fp}\" \ > ${nml_fn} -${RUN_CMD_SERIAL} ${exec_fp} < ${nml_fn} || \ +${RUN_CMD_SERIAL} ${exec_fp} < ${nml_fn} >>$pgmout 2>$pgmerr || \ print_err_msg_exit "\ Call to executable (exec_fp) to generate a (filtered) orography file with a ${NH0}-cell-wide halo from the orography file with a {NHW}-cell-wide halo @@ -525,7 +525,7 @@ printf "%s %s %s %s %s\n" \ $NX $NY ${NH4} \"${unshaved_fp}\" \"${shaved_fp}\" \ > ${nml_fn} -${RUN_CMD_SERIAL} ${exec_fp} < ${nml_fn} || \ +${RUN_CMD_SERIAL} ${exec_fp} < ${nml_fn} >>$pgmout 2>$pgmerr || \ print_err_msg_exit "\ Call to executable (exec_fp) to generate a (filtered) orography file with a ${NH4}-cell-wide halo from the orography file with a {NHW}-cell-wide halo diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 5ba88a4cda..d425dcfdc1 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -511,7 +511,7 @@ fi # #----------------------------------------------------------------------- # -${RUN_CMD_FCST} ${FV3_EXEC_FP} || print_err_msg_exit "\ +${RUN_CMD_FCST} ${FV3_EXEC_FP} >>$pgmout 2>$pgmerr || print_err_msg_exit "\ Call to executable to run FV3-LAM forecast returned with nonzero exit code." # diff --git a/scripts/exregional_run_post.sh b/scripts/exregional_run_post.sh index 8feee72c7a..b76e899f2c 100755 --- a/scripts/exregional_run_post.sh +++ b/scripts/exregional_run_post.sh @@ -219,7 +219,7 @@ EOF print_info_msg "$VERBOSE" " Starting post-processing for fhr = $fhr hr..." -${RUN_CMD_POST} ${EXECrrfs}/upp.x < itag || print_err_msg_exit "\ +${RUN_CMD_POST} ${EXECrrfs}/upp.x < itag >>$pgmout 2>$pgmerr || print_err_msg_exit "\ Call to executable to run post for forecast hour $fhr returned with non- zero exit code." # diff --git a/ush/job_preamble.sh b/ush/job_preamble.sh index 3663a5bfa7..afa5567087 100644 --- a/ush/job_preamble.sh +++ b/ush/job_preamble.sh @@ -34,8 +34,13 @@ fi # #----------------------------------------------------------------------- # -export pgmout="OUTPUT.$$" -export pgmerr="errfile" +if [ "${RUN_ENVIR}" = "nco" ]; then + export pgmout="${DATA}/OUTPUT.$$" + export pgmerr="${DATA}/errfile" +else + export pgmout="${LOGDIR}/OUTPUT.$$" + export pgmerr="${LOGDIR}/errfile" +fi # #----------------------------------------------------------------------- # From e246b3f2907318a390000c6bfc218cf705017785 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Sun, 21 Aug 2022 00:05:19 +0000 Subject: [PATCH 037/106] Fix inline post. --- scripts/exregional_run_fcst.sh | 12 +++++------- ush/job_preamble.sh | 1 + 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index d425dcfdc1..548f1763df 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -529,14 +529,12 @@ if [ ${WRITE_DOPOST} = "TRUE" ]; then cyc=$hh fmn="00" - if [ "${RUN_ENVIR}" = "nco" ]; then - postprd_dir="${COMOUT}/$cyc${SLASH_ENSMEM_SUBDIR}" - else - postprd_dir="${DATA}/postprd" + if [ "${RUN_ENVIR}" != "nco" ]; then + export COMOUT="${DATA}/postprd" fi - mkdir_vrfy -p "${postprd_dir}" + mkdir_vrfy -p "${COMOUT}" - cd_vrfy ${postprd_dir} + cd_vrfy ${COMOUT} for fhr in $(seq -f "%03g" 0 ${FCST_LEN_HRS}); do @@ -558,7 +556,7 @@ if [ ${WRITE_DOPOST} = "TRUE" ]; then for fid in "${fids[@]}"; do FID=$(echo_uppercase $fid) post_orig_fn="${FID}.${post_fn_suffix}" - post_renamed_fn="${NET}.t${cyc}z.${fid}.${post_renamed_fn_suffix}" + post_renamed_fn="${NET}.t${cyc}z.mem${ENSMEM_INDX}.${fid}.${post_renamed_fn_suffix}" mv_vrfy ${DATA}/${post_orig_fn} ${post_renamed_fn} ln_vrfy -fs ${post_renamed_fn} ${FID}${symlink_suffix} done diff --git a/ush/job_preamble.sh b/ush/job_preamble.sh index afa5567087..2428e07074 100644 --- a/ush/job_preamble.sh +++ b/ush/job_preamble.sh @@ -71,6 +71,7 @@ function job_postamble() { # Remove temp directory if [ "${RUN_ENVIR}" = "nco" ]; then + cd ${DATAROOT} [[ $KEEPDATA = "FALSE" ]] && rm -rf $DATA fi From 892daff5359425592244064c5009db52171e6182 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Sun, 21 Aug 2022 01:34:54 +0000 Subject: [PATCH 038/106] Make pgmout/err redirection work with community mode. --- scripts/exregional_get_extrn_mdl_files.sh | 2 +- scripts/exregional_make_grid.sh | 13 ++++++------- scripts/exregional_make_ics.sh | 8 ++++---- scripts/exregional_make_lbcs.sh | 3 +-- scripts/exregional_make_orog.sh | 11 +++++------ scripts/exregional_make_sfc_climo.sh | 3 +-- scripts/exregional_run_fcst.sh | 3 +-- scripts/exregional_run_post.sh | 3 +-- ush/job_preamble.sh | 18 ++++++++++-------- 9 files changed, 30 insertions(+), 34 deletions(-) diff --git a/scripts/exregional_get_extrn_mdl_files.sh b/scripts/exregional_get_extrn_mdl_files.sh index 6489e98682..de74e2a139 100755 --- a/scripts/exregional_get_extrn_mdl_files.sh +++ b/scripts/exregional_get_extrn_mdl_files.sh @@ -138,7 +138,7 @@ python3 -u ${USHrrfs}/retrieve_data.py \ --summary_file ${EXTRN_MDL_VAR_DEFNS_FN} \ $additional_flags" -$cmd >>$pgmout 2>$pgmerr || print_err_msg_exit "\ +$cmd || print_err_msg_exit "\ Call to retrieve_data.py failed with a non-zero exit status. The command was: diff --git a/scripts/exregional_make_grid.sh b/scripts/exregional_make_grid.sh index 068e0a2912..41f94e5591 100755 --- a/scripts/exregional_make_grid.sh +++ b/scripts/exregional_make_grid.sh @@ -68,7 +68,6 @@ if [ -z "${RUN_CMD_SERIAL:-}" ] ; then Run command was not set in machine file. \ Please set RUN_CMD_SERIAL for your platform" else - RUN_CMD_SERIAL=$(eval echo ${RUN_CMD_SERIAL}) print_info_msg "$VERBOSE" " All executables will be submitted with command \'${RUN_CMD_SERIAL}\'." fi @@ -217,7 +216,7 @@ if [ "${GRID_GEN_METHOD}" = "GFDLgrid" ]; then # for the 6 global tiles. However, after this call we will only need the # regional grid file. # - $RUN_CMD_SERIAL ${exec_fp} \ + eval $RUN_CMD_SERIAL ${exec_fp} \ --grid_type gnomonic_ed \ --nlon ${nx_t6sg} \ --grid_name ${grid_name} \ @@ -233,7 +232,7 @@ if [ "${GRID_GEN_METHOD}" = "GFDLgrid" ]; then --iend_nest ${IEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG} \ --jend_nest ${JEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG} \ --halo 1 \ - --great_circle_algorithm >>$pgmout 2>$pgmerr || \ + --great_circle_algorithm ${REDIRECT_OUT_ERR} || \ print_err_msg_exit "\ Call to executable (exec_fp) that generates grid files returned with nonzero exit code. @@ -291,7 +290,7 @@ $settings" # # Call the executable that generates the grid file. # - $RUN_CMD_SERIAL ${exec_fp} ${rgnl_grid_nml_fp} >>$pgmout 2>$pgmerr || \ + eval $RUN_CMD_SERIAL ${exec_fp} ${rgnl_grid_nml_fp} ${REDIRECT_OUT_ERR} || \ print_err_msg_exit "\ Call to executable (exec_fp) that generates a ESGgrid-type regional grid returned with nonzero exit code: @@ -330,7 +329,7 @@ cubed-sphere grid equivalent resolution does not exist: Please ensure that you've built this executable." fi -$RUN_CMD_SERIAL ${exec_fp} "${grid_fp}" >>$pgmout 2>$pgmerr || \ +eval $RUN_CMD_SERIAL ${exec_fp} "${grid_fp}" ${REDIRECT_OUT_ERR} || \ print_err_msg_exit "\ Call to executable (exec_fp) that calculates the regional grid's global uniform cubed-sphere grid equivalent resolution returned with nonzero exit @@ -459,7 +458,7 @@ printf "%s %s %s %s %s\n" \ $NX $NY ${NH3} \"${unshaved_fp}\" \"${shaved_fp}\" \ > ${nml_fn} -$RUN_CMD_SERIAL ${exec_fp} < ${nml_fn} >>$pgmout 2>$pgmerr || \ +eval $RUN_CMD_SERIAL ${exec_fp} < ${nml_fn} ${REDIRECT_OUT_ERR} || \ print_err_msg_exit "\ Call to executable (exec_fp) to generate a grid file with a ${NH3}-cell-wide halo from the grid file with a ${NHW}-cell-wide halo returned with nonzero @@ -485,7 +484,7 @@ printf "%s %s %s %s %s\n" \ $NX $NY ${NH4} \"${unshaved_fp}\" \"${shaved_fp}\" \ > ${nml_fn} -$RUN_CMD_SERIAL ${exec_fp} < ${nml_fn} >>$pgmout 2>$pgmerr || \ +eval $RUN_CMD_SERIAL ${exec_fp} < ${nml_fn} ${REDIRECT_OUT_ERR} || \ print_err_msg_exit "\ Call to executable (exec_fp) to generate a grid file with a ${NH4}-cell-wide halo from the grid file with a ${NHW}-cell-wide halo returned with nonzero diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index c043e6f9dd..d63aaa6657 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -73,7 +73,6 @@ if [ -z "${RUN_CMD_UTILS:-}" ] ; then Run command was not set in machine file. \ Please set RUN_CMD_UTILS for your platform" else - RUN_CMD_UTILS=$(eval echo ${RUN_CMD_UTILS}) print_info_msg "$VERBOSE" " All executables will be submitted with command \'${RUN_CMD_UTILS}\'." fi @@ -557,7 +556,7 @@ $settings" # exit code of chgres_cube is nonzero. A similar thing happens in the # forecast task. # -${RUN_CMD_UTILS} ${exec_fp} >>$pgmout 2>$pgmerr || \ +eval ${RUN_CMD_UTILS} ${exec_fp} ${REDIRECT_OUT_ERR} || \ print_err_msg_exit "\ Call to executable (exec_fp) to generate surface and initial conditions (ICs) files for the FV3-LAM failed: @@ -617,8 +616,9 @@ Please check the following user defined variables: cp_vrfy ${fvcom_data_fp} ${INPUT_DATA}/fvcom.nc cd_vrfy ${INPUT_DATA} - ${RUN_CMD_UTILS} ${fvcom_exec_fn} sfc_data.tile${TILE_RGNL}.halo${NH0}.nc fvcom.nc ${FVCOM_WCSTART} ${fvcom_time} \ - >>$pgmout 2>$pgmerr || print_err_msg_exit "\ + eval ${RUN_CMD_UTILS} ${fvcom_exec_fn} \ + sfc_data.tile${TILE_RGNL}.halo${NH0}.nc fvcom.nc ${FVCOM_WCSTART} ${fvcom_time} \ + ${REDIRECT_OUT_ERR} || print_err_msg_exit "\ Call to executable (fvcom_exe) to modify sfc fields for FV3-LAM failed: fvcom_exe = \"${fvcom_exe}\" The following variables were being used: diff --git a/scripts/exregional_make_lbcs.sh b/scripts/exregional_make_lbcs.sh index e648b9dabb..3af44289a6 100755 --- a/scripts/exregional_make_lbcs.sh +++ b/scripts/exregional_make_lbcs.sh @@ -73,7 +73,6 @@ if [ -z "${RUN_CMD_UTILS:-}" ] ; then Run command was not set in machine file. \ Please set RUN_CMD_UTILS for your platform" else - RUN_CMD_UTILS=$(eval echo ${RUN_CMD_UTILS}) print_info_msg "$VERBOSE" " All executables will be submitted with command \'${RUN_CMD_UTILS}\'." fi @@ -470,7 +469,7 @@ $settings" # exit code of chgres_cube is nonzero. A similar thing happens in the # forecast task. # - ${RUN_CMD_UTILS} ${exec_fp} >>$pgmout 2>$pgmerr || \ + eval ${RUN_CMD_UTILS} ${exec_fp} ${REDIRECT_OUT_ERR} || \ print_err_msg_exit "\ Call to executable (exec_fp) to generate lateral boundary conditions (LBCs) file for the FV3-LAM for forecast hour fhr failed: diff --git a/scripts/exregional_make_orog.sh b/scripts/exregional_make_orog.sh index 8bd24f9297..0382f4b6a2 100755 --- a/scripts/exregional_make_orog.sh +++ b/scripts/exregional_make_orog.sh @@ -73,7 +73,6 @@ if [ -z "${RUN_CMD_SERIAL:-}" ] ; then Run command was not set in machine file. \ Please set RUN_CMD_SERIAL for your platform" else - RUN_CMD_SERIAL=$(eval echo ${RUN_CMD_SERIAL}) print_info_msg "$VERBOSE" " All executables will be submitted with command \'${RUN_CMD_SERIAL}\'." fi @@ -210,7 +209,7 @@ cat "${input_redirect_fn}" print_info_msg "$VERBOSE" "\ Starting orography file generation..." -${RUN_CMD_SERIAL} "${exec_fp}" < "${input_redirect_fn}" >>$pgmout 2>$pgmerr || \ +eval ${RUN_CMD_SERIAL} "${exec_fp}" < "${input_redirect_fn}" ${REDIRECT_OUT_ERR} || \ print_err_msg_exit "\ Call to executable (exec_fp) that generates the raw orography file returned with nonzero exit code: @@ -282,7 +281,7 @@ Please ensure that you've built this executable." print_info_msg "$VERBOSE" " Starting orography file generation..." - ${RUN_CMD_SERIAL} "${exec_fp}" < "${input_redirect_fn}" >>$pgmout 2>$pgmerr || \ + eval ${RUN_CMD_SERIAL} "${exec_fp}" < "${input_redirect_fn}" ${REDIRECT_OUT_ERR} || \ print_err_msg_exit "\ Call to executable (exec_fp) that generates the GSL orography GWD data files returned with nonzero exit code: @@ -430,7 +429,7 @@ cd_vrfy "${filter_dir}" print_info_msg "$VERBOSE" " Starting filtering of orography..." -${RUN_CMD_SERIAL} "${exec_fp}" >>$pgmout 2>$pgmerr || \ +eval ${RUN_CMD_SERIAL} "${exec_fp}" ${REDIRECT_OUT_ERR} || \ print_err_msg_exit "\ Call to executable that generates filtered orography file returned with non-zero exit code." @@ -499,7 +498,7 @@ printf "%s %s %s %s %s\n" \ $NX $NY ${NH0} \"${unshaved_fp}\" \"${shaved_fp}\" \ > ${nml_fn} -${RUN_CMD_SERIAL} ${exec_fp} < ${nml_fn} >>$pgmout 2>$pgmerr || \ +eval ${RUN_CMD_SERIAL} ${exec_fp} < ${nml_fn} ${REDIRECT_OUT_ERR} || \ print_err_msg_exit "\ Call to executable (exec_fp) to generate a (filtered) orography file with a ${NH0}-cell-wide halo from the orography file with a {NHW}-cell-wide halo @@ -525,7 +524,7 @@ printf "%s %s %s %s %s\n" \ $NX $NY ${NH4} \"${unshaved_fp}\" \"${shaved_fp}\" \ > ${nml_fn} -${RUN_CMD_SERIAL} ${exec_fp} < ${nml_fn} >>$pgmout 2>$pgmerr || \ +eval ${RUN_CMD_SERIAL} ${exec_fp} < ${nml_fn} ${REDIRECT_OUT_ERR} || \ print_err_msg_exit "\ Call to executable (exec_fp) to generate a (filtered) orography file with a ${NH4}-cell-wide halo from the orography file with a {NHW}-cell-wide halo diff --git a/scripts/exregional_make_sfc_climo.sh b/scripts/exregional_make_sfc_climo.sh index 6fa769808b..9c07cd91b6 100755 --- a/scripts/exregional_make_sfc_climo.sh +++ b/scripts/exregional_make_sfc_climo.sh @@ -116,7 +116,6 @@ if [ -z "${RUN_CMD_UTILS:-}" ] ; then Run command was not set in machine file. \ Please set RUN_CMD_UTILS for your platform" else - RUN_CMD_UTILS=$(eval echo ${RUN_CMD_UTILS}) print_info_msg "$VERBOSE" " All executables will be submitted with command \'${RUN_CMD_UTILS}\'." fi @@ -139,7 +138,7 @@ does not exist: Please ensure that you've built this executable." fi -${RUN_CMD_UTILS} ${exec_fp} || \ +eval ${RUN_CMD_UTILS} ${exec_fp} ${REDIRECT_OUT_ERR} || \ print_err_msg_exit "\ Call to executable (exec_fp) to generate surface climatology files returned with nonzero exit code: diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 548f1763df..f809077fa1 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -72,7 +72,6 @@ if [ -z "${RUN_CMD_FCST:-}" ] ; then Run command was not set in machine file. \ Please set RUN_CMD_FCST for your platform" else - RUN_CMD_FCST=$(eval echo ${RUN_CMD_FCST}) print_info_msg "$VERBOSE" " All executables will be submitted with command \'${RUN_CMD_FCST}\'." fi @@ -511,7 +510,7 @@ fi # #----------------------------------------------------------------------- # -${RUN_CMD_FCST} ${FV3_EXEC_FP} >>$pgmout 2>$pgmerr || print_err_msg_exit "\ +eval ${RUN_CMD_FCST} ${FV3_EXEC_FP} ${REDIRECT_OUT_ERR} || print_err_msg_exit "\ Call to executable to run FV3-LAM forecast returned with nonzero exit code." # diff --git a/scripts/exregional_run_post.sh b/scripts/exregional_run_post.sh index b76e899f2c..056d845957 100755 --- a/scripts/exregional_run_post.sh +++ b/scripts/exregional_run_post.sh @@ -71,7 +71,6 @@ if [ -z "${RUN_CMD_POST:-}" ] ; then Run command was not set in machine file. \ Please set RUN_CMD_POST for your platform" else - RUN_CMD_POST=$(eval echo ${RUN_CMD_POST}) print_info_msg "$VERBOSE" " All executables will be submitted with command \'${RUN_CMD_POST}\'." fi @@ -219,7 +218,7 @@ EOF print_info_msg "$VERBOSE" " Starting post-processing for fhr = $fhr hr..." -${RUN_CMD_POST} ${EXECrrfs}/upp.x < itag >>$pgmout 2>$pgmerr || print_err_msg_exit "\ +eval ${RUN_CMD_POST} ${EXECrrfs}/upp.x < itag ${REDIRECT_OUT_ERR} || print_err_msg_exit "\ Call to executable to run post for forecast hour $fhr returned with non- zero exit code." # diff --git a/ush/job_preamble.sh b/ush/job_preamble.sh index 2428e07074..ad75a1d1e3 100644 --- a/ush/job_preamble.sh +++ b/ush/job_preamble.sh @@ -37,9 +37,11 @@ fi if [ "${RUN_ENVIR}" = "nco" ]; then export pgmout="${DATA}/OUTPUT.$$" export pgmerr="${DATA}/errfile" + export REDIRECT_OUT_ERR=">>${pgmout} 2>${pgmerr}" else - export pgmout="${LOGDIR}/OUTPUT.$$" - export pgmerr="${LOGDIR}/errfile" + export pgmout= + export pgmerr= + export REDIRECT_OUT_ERR= fi # #----------------------------------------------------------------------- @@ -64,13 +66,13 @@ fi # function job_postamble() { - # Print output file to stdout - if [ -e "$pgmout" ]; then - cat $pgmout - fi - - # Remove temp directory if [ "${RUN_ENVIR}" = "nco" ]; then + # Print output file to stdout + if [ -e "$pgmout" ]; then + cat $pgmout + fi + + # Remove temp directory cd ${DATAROOT} [[ $KEEPDATA = "FALSE" ]] && rm -rf $DATA fi From c63d81c00e0cd762c93f90d1fe0efef48614ad95 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Sun, 21 Aug 2022 02:03:03 +0000 Subject: [PATCH 039/106] Use print_err in get_obs_mrms. --- scripts/exregional_get_obs_mrms.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/scripts/exregional_get_obs_mrms.sh b/scripts/exregional_get_obs_mrms.sh index aec156841d..74f297b1de 100755 --- a/scripts/exregional_get_obs_mrms.sh +++ b/scripts/exregional_get_obs_mrms.sh @@ -118,8 +118,7 @@ while [[ ${cur_ut} -le ${end_valid_ut} ]]; do if [[ ${Status} == 0 ]]; then TarFile="/NCEPPROD/hpssprod/runhistory/rh${vyyyy}/${vyyyy}${vmm}/${vyyyy}${vmm}${vdd}/ldmdata.tide.${vyyyy}${vmm}${vdd}.tar" else - echo "ERROR: MRMS data not available for ${vyyyy}${vmm}${vdd}!" - exit + print_err_msg_exit "MRMS data not available for ${vyyyy}${vmm}${vdd}!" fi fi fi @@ -135,9 +134,9 @@ while [[ ${cur_ut} -le ${end_valid_ut} ]]; do Status=$? if [[ ${Status} != 0 ]]; then - echo "WARNING: Bad return status (${Status}) for date \"${CurDate}\". Did you forget to run \"module load hpss\"?" - echo "WARNING: ${TarCommand}" - exit ${Status} + print_err_msg_exit "Bad return status (${Status}) for date \"${CurDate}\".\ +Did you forget to run \"module load hpss\"?\ + COMMAND: ${TarCommand}" else if [[ ! -d "$mrms_proc/${vyyyymmdd}" ]]; then mkdir_vrfy -p $mrms_proc/${vyyyymmdd} From 51f6b5c630b73346ff7ef11222a000f8bebf8aba Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Sun, 21 Aug 2022 17:11:41 +0000 Subject: [PATCH 040/106] Add prep_step. --- scripts/exregional_make_grid.sh | 5 +++++ scripts/exregional_make_ics.sh | 2 ++ scripts/exregional_make_lbcs.sh | 1 + scripts/exregional_make_orog.sh | 5 +++++ scripts/exregional_make_sfc_climo.sh | 1 + scripts/exregional_run_fcst.sh | 1 + scripts/exregional_run_post.sh | 1 + ush/job_preamble.sh | 19 +++++++++++++++++++ 8 files changed, 35 insertions(+) diff --git a/scripts/exregional_make_grid.sh b/scripts/exregional_make_grid.sh index 41f94e5591..fe37735ac9 100755 --- a/scripts/exregional_make_grid.sh +++ b/scripts/exregional_make_grid.sh @@ -216,6 +216,7 @@ if [ "${GRID_GEN_METHOD}" = "GFDLgrid" ]; then # for the 6 global tiles. However, after this call we will only need the # regional grid file. # + PREP_STEP eval $RUN_CMD_SERIAL ${exec_fp} \ --grid_type gnomonic_ed \ --nlon ${nx_t6sg} \ @@ -290,6 +291,7 @@ $settings" # # Call the executable that generates the grid file. # + PREP_STEP eval $RUN_CMD_SERIAL ${exec_fp} ${rgnl_grid_nml_fp} ${REDIRECT_OUT_ERR} || \ print_err_msg_exit "\ Call to executable (exec_fp) that generates a ESGgrid-type regional grid @@ -329,6 +331,7 @@ cubed-sphere grid equivalent resolution does not exist: Please ensure that you've built this executable." fi +PREP_STEP eval $RUN_CMD_SERIAL ${exec_fp} "${grid_fp}" ${REDIRECT_OUT_ERR} || \ print_err_msg_exit "\ Call to executable (exec_fp) that calculates the regional grid's global @@ -458,6 +461,7 @@ printf "%s %s %s %s %s\n" \ $NX $NY ${NH3} \"${unshaved_fp}\" \"${shaved_fp}\" \ > ${nml_fn} +PREP_STEP eval $RUN_CMD_SERIAL ${exec_fp} < ${nml_fn} ${REDIRECT_OUT_ERR} || \ print_err_msg_exit "\ Call to executable (exec_fp) to generate a grid file with a ${NH3}-cell-wide @@ -484,6 +488,7 @@ printf "%s %s %s %s %s\n" \ $NX $NY ${NH4} \"${unshaved_fp}\" \"${shaved_fp}\" \ > ${nml_fn} +PREP_STEP eval $RUN_CMD_SERIAL ${exec_fp} < ${nml_fn} ${REDIRECT_OUT_ERR} || \ print_err_msg_exit "\ Call to executable (exec_fp) to generate a grid file with a ${NH4}-cell-wide diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index d63aaa6657..41642b817a 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -556,6 +556,7 @@ $settings" # exit code of chgres_cube is nonzero. A similar thing happens in the # forecast task. # +PREP_STEP eval ${RUN_CMD_UTILS} ${exec_fp} ${REDIRECT_OUT_ERR} || \ print_err_msg_exit "\ Call to executable (exec_fp) to generate surface and initial conditions @@ -616,6 +617,7 @@ Please check the following user defined variables: cp_vrfy ${fvcom_data_fp} ${INPUT_DATA}/fvcom.nc cd_vrfy ${INPUT_DATA} + PREP_STEP eval ${RUN_CMD_UTILS} ${fvcom_exec_fn} \ sfc_data.tile${TILE_RGNL}.halo${NH0}.nc fvcom.nc ${FVCOM_WCSTART} ${fvcom_time} \ ${REDIRECT_OUT_ERR} || print_err_msg_exit "\ diff --git a/scripts/exregional_make_lbcs.sh b/scripts/exregional_make_lbcs.sh index 3af44289a6..ba57b4b72b 100755 --- a/scripts/exregional_make_lbcs.sh +++ b/scripts/exregional_make_lbcs.sh @@ -469,6 +469,7 @@ $settings" # exit code of chgres_cube is nonzero. A similar thing happens in the # forecast task. # + PREP_STEP eval ${RUN_CMD_UTILS} ${exec_fp} ${REDIRECT_OUT_ERR} || \ print_err_msg_exit "\ Call to executable (exec_fp) to generate lateral boundary conditions (LBCs) diff --git a/scripts/exregional_make_orog.sh b/scripts/exregional_make_orog.sh index 0382f4b6a2..8fea0a4684 100755 --- a/scripts/exregional_make_orog.sh +++ b/scripts/exregional_make_orog.sh @@ -209,6 +209,7 @@ cat "${input_redirect_fn}" print_info_msg "$VERBOSE" "\ Starting orography file generation..." +PREP_STEP eval ${RUN_CMD_SERIAL} "${exec_fp}" < "${input_redirect_fn}" ${REDIRECT_OUT_ERR} || \ print_err_msg_exit "\ Call to executable (exec_fp) that generates the raw orography file returned @@ -281,6 +282,7 @@ Please ensure that you've built this executable." print_info_msg "$VERBOSE" " Starting orography file generation..." + PREP_STEP eval ${RUN_CMD_SERIAL} "${exec_fp}" < "${input_redirect_fn}" ${REDIRECT_OUT_ERR} || \ print_err_msg_exit "\ Call to executable (exec_fp) that generates the GSL orography GWD data files @@ -429,6 +431,7 @@ cd_vrfy "${filter_dir}" print_info_msg "$VERBOSE" " Starting filtering of orography..." +PREP_STEP eval ${RUN_CMD_SERIAL} "${exec_fp}" ${REDIRECT_OUT_ERR} || \ print_err_msg_exit "\ Call to executable that generates filtered orography file returned with @@ -498,6 +501,7 @@ printf "%s %s %s %s %s\n" \ $NX $NY ${NH0} \"${unshaved_fp}\" \"${shaved_fp}\" \ > ${nml_fn} +PREP_STEP eval ${RUN_CMD_SERIAL} ${exec_fp} < ${nml_fn} ${REDIRECT_OUT_ERR} || \ print_err_msg_exit "\ Call to executable (exec_fp) to generate a (filtered) orography file with @@ -524,6 +528,7 @@ printf "%s %s %s %s %s\n" \ $NX $NY ${NH4} \"${unshaved_fp}\" \"${shaved_fp}\" \ > ${nml_fn} +PREP_STEP eval ${RUN_CMD_SERIAL} ${exec_fp} < ${nml_fn} ${REDIRECT_OUT_ERR} || \ print_err_msg_exit "\ Call to executable (exec_fp) to generate a (filtered) orography file with diff --git a/scripts/exregional_make_sfc_climo.sh b/scripts/exregional_make_sfc_climo.sh index 9c07cd91b6..570fb57656 100755 --- a/scripts/exregional_make_sfc_climo.sh +++ b/scripts/exregional_make_sfc_climo.sh @@ -138,6 +138,7 @@ does not exist: Please ensure that you've built this executable." fi +PREP_STEP eval ${RUN_CMD_UTILS} ${exec_fp} ${REDIRECT_OUT_ERR} || \ print_err_msg_exit "\ Call to executable (exec_fp) to generate surface climatology files returned diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index f809077fa1..b6e9990b08 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -510,6 +510,7 @@ fi # #----------------------------------------------------------------------- # +PREP_STEP eval ${RUN_CMD_FCST} ${FV3_EXEC_FP} ${REDIRECT_OUT_ERR} || print_err_msg_exit "\ Call to executable to run FV3-LAM forecast returned with nonzero exit code." diff --git a/scripts/exregional_run_post.sh b/scripts/exregional_run_post.sh index 056d845957..752cbac412 100755 --- a/scripts/exregional_run_post.sh +++ b/scripts/exregional_run_post.sh @@ -218,6 +218,7 @@ EOF print_info_msg "$VERBOSE" " Starting post-processing for fhr = $fhr hr..." +PREP_STEP eval ${RUN_CMD_POST} ${EXECrrfs}/upp.x < itag ${REDIRECT_OUT_ERR} || print_err_msg_exit "\ Call to executable to run post for forecast hour $fhr returned with non- zero exit code." diff --git a/ush/job_preamble.sh b/ush/job_preamble.sh index ad75a1d1e3..cc5c38fc18 100644 --- a/ush/job_preamble.sh +++ b/ush/job_preamble.sh @@ -35,13 +35,32 @@ fi #----------------------------------------------------------------------- # if [ "${RUN_ENVIR}" = "nco" ]; then + export pgm="$(basename $0):$LINENO" export pgmout="${DATA}/OUTPUT.$$" export pgmerr="${DATA}/errfile" export REDIRECT_OUT_ERR=">>${pgmout} 2>${pgmerr}" + + function PREP_STEP() { + if [ ! -z $(command -v prep_step) ]; then + . prep_step + else + # Append header + if [ -n "$pgm" ] && [ -n "$pgmout" ]; then + echo "$pgm" >> $pgmout + fi + # Remove error file + if [ -f $pgmerr ]; then + rm $pgmerr + fi + fi + } + export -f PREP_STEP else + export pgm= export pgmout= export pgmerr= export REDIRECT_OUT_ERR= + export PREP_STEP= fi # #----------------------------------------------------------------------- From cee6d36f02fefd784fa17bbe4598e6c6a8bbe4f9 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Tue, 23 Aug 2022 16:08:15 +0000 Subject: [PATCH 041/106] Add post_step. --- scripts/exregional_make_grid.sh | 5 +++++ scripts/exregional_make_ics.sh | 2 ++ scripts/exregional_make_lbcs.sh | 1 + scripts/exregional_make_orog.sh | 5 +++++ scripts/exregional_make_sfc_climo.sh | 1 + scripts/exregional_run_fcst.sh | 1 + scripts/exregional_run_post.sh | 1 + ush/job_preamble.sh | 20 ++++++++++++-------- 8 files changed, 28 insertions(+), 8 deletions(-) diff --git a/scripts/exregional_make_grid.sh b/scripts/exregional_make_grid.sh index fe37735ac9..a6bd142d1c 100755 --- a/scripts/exregional_make_grid.sh +++ b/scripts/exregional_make_grid.sh @@ -238,6 +238,7 @@ if [ "${GRID_GEN_METHOD}" = "GFDLgrid" ]; then Call to executable (exec_fp) that generates grid files returned with nonzero exit code. exec_fp = \"${exec_fp}\"" + POST_STEP # # Set the name of the regional grid file generated by the above call. # @@ -297,6 +298,7 @@ $settings" Call to executable (exec_fp) that generates a ESGgrid-type regional grid returned with nonzero exit code: exec_fp = \"${exec_fp}\"" + POST_STEP # # Set the name of the regional grid file generated by the above call. # This must be the same name as in the regional_esg_grid code. @@ -338,6 +340,7 @@ Call to executable (exec_fp) that calculates the regional grid's global uniform cubed-sphere grid equivalent resolution returned with nonzero exit code: exec_fp = \"${exec_fp}\"" +POST_STEP # Make sure 'ncdump' is available before we try to use it if ! command -v ncdump &> /dev/null @@ -471,6 +474,7 @@ exit code: The namelist file (nml_fn) used in this call is in directory DATA: nml_fn = \"${nml_fn}\" DATA = \"${DATA}\"" +POST_STEP mv_vrfy ${shaved_fp} ${GRID_DIR} # # Create an input namelist file for the shave executable to generate a @@ -498,6 +502,7 @@ exit code: The namelist file (nml_fn) used in this call is in directory DATA: nml_fn = \"${nml_fn}\" DATA = \"${DATA}\"" +POST_STEP mv_vrfy ${shaved_fp} ${GRID_DIR} # # Change location to the original directory. diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index 41642b817a..56eb52c328 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -567,6 +567,7 @@ The external model from which the ICs files are to be generated is: The external model files that are inputs to the executable (exec_fp) are located in the following directory: extrn_mdl_staging_dir = \"${extrn_mdl_staging_dir}\"" +POST_STEP # #----------------------------------------------------------------------- # @@ -631,6 +632,7 @@ The following variables were being used: INPUT_DATA = \"${INPUT_DATA}\" fvcom_exe_dir = \"${fvcom_exe_dir}\" fvcom_exe = \"${fvcom_exe}\"" + POST_STEP fi # #----------------------------------------------------------------------- diff --git a/scripts/exregional_make_lbcs.sh b/scripts/exregional_make_lbcs.sh index ba57b4b72b..9a2fb246a1 100755 --- a/scripts/exregional_make_lbcs.sh +++ b/scripts/exregional_make_lbcs.sh @@ -481,6 +481,7 @@ The external model from which the LBCs files are to be generated is: The external model files that are inputs to the executable (exec_fp) are located in the following directory: extrn_mdl_staging_dir = \"${extrn_mdl_staging_dir}\"" + POST_STEP # # Move LBCs file for the current lateral boundary update time to the LBCs # work directory. Note that we rename the file by including in its name diff --git a/scripts/exregional_make_orog.sh b/scripts/exregional_make_orog.sh index 8fea0a4684..887b64d9b1 100755 --- a/scripts/exregional_make_orog.sh +++ b/scripts/exregional_make_orog.sh @@ -215,6 +215,7 @@ eval ${RUN_CMD_SERIAL} "${exec_fp}" < "${input_redirect_fn}" ${REDIRECT_OUT_ERR Call to executable (exec_fp) that generates the raw orography file returned with nonzero exit code: exec_fp = \"${exec_fp}\"" +POST_STEP # # Change location to the original directory. @@ -288,6 +289,7 @@ Starting orography file generation..." Call to executable (exec_fp) that generates the GSL orography GWD data files returned with nonzero exit code: exec_fp = \"${exec_fp}\"" + POST_STEP mv_vrfy "${CRES}${DOT_OR_USCORE}oro_data_ss.tile${TILE_RGNL}.halo${NH0}.nc" \ "${CRES}${DOT_OR_USCORE}oro_data_ls.tile${TILE_RGNL}.halo${NH0}.nc" \ @@ -436,6 +438,7 @@ eval ${RUN_CMD_SERIAL} "${exec_fp}" ${REDIRECT_OUT_ERR} || \ print_err_msg_exit "\ Call to executable that generates filtered orography file returned with non-zero exit code." +POST_STEP # # For clarity, rename the filtered orography file in filter_dir # such that its new name contains the halo size. @@ -511,6 +514,7 @@ returned with nonzero exit code: The namelist file (nml_fn) used in this call is in directory shave_dir: nml_fn = \"${nml_fn}\" shave_dir = \"${shave_dir}\"" +POST_STEP mv_vrfy ${shaved_fp} ${OROG_DIR} # # Create an input namelist file for the shave executable to generate an @@ -538,6 +542,7 @@ returned with nonzero exit code: The namelist file (nml_fn) used in this call is in directory shave_dir: nml_fn = \"${nml_fn}\" shave_dir = \"${shave_dir}\"" +POST_STEP mv_vrfy "${shaved_fp}" "${OROG_DIR}" # # Change location to the original directory. diff --git a/scripts/exregional_make_sfc_climo.sh b/scripts/exregional_make_sfc_climo.sh index 570fb57656..4ebd21dd3f 100755 --- a/scripts/exregional_make_sfc_climo.sh +++ b/scripts/exregional_make_sfc_climo.sh @@ -144,6 +144,7 @@ print_err_msg_exit "\ Call to executable (exec_fp) to generate surface climatology files returned with nonzero exit code: exec_fp = \"${exec_fp}\"" +POST_STEP # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index b6e9990b08..7e36a46655 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -514,6 +514,7 @@ PREP_STEP eval ${RUN_CMD_FCST} ${FV3_EXEC_FP} ${REDIRECT_OUT_ERR} || print_err_msg_exit "\ Call to executable to run FV3-LAM forecast returned with nonzero exit code." +POST_STEP # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_run_post.sh b/scripts/exregional_run_post.sh index 752cbac412..31f1e91c30 100755 --- a/scripts/exregional_run_post.sh +++ b/scripts/exregional_run_post.sh @@ -222,6 +222,7 @@ PREP_STEP eval ${RUN_CMD_POST} ${EXECrrfs}/upp.x < itag ${REDIRECT_OUT_ERR} || print_err_msg_exit "\ Call to executable to run post for forecast hour $fhr returned with non- zero exit code." +POST_STEP # #----------------------------------------------------------------------- # diff --git a/ush/job_preamble.sh b/ush/job_preamble.sh index cc5c38fc18..f7257b9181 100644 --- a/ush/job_preamble.sh +++ b/ush/job_preamble.sh @@ -3,7 +3,7 @@ # #----------------------------------------------------------------------- # -# Set cycle and run setpdy to initialize PDYm and PDYp variables +# Create a temp working directory (DATA) and cd into it. # #----------------------------------------------------------------------- # @@ -35,12 +35,13 @@ fi #----------------------------------------------------------------------- # if [ "${RUN_ENVIR}" = "nco" ]; then - export pgm="$(basename $0):$LINENO" export pgmout="${DATA}/OUTPUT.$$" export pgmerr="${DATA}/errfile" export REDIRECT_OUT_ERR=">>${pgmout} 2>${pgmerr}" + export pgmout_lines=1 function PREP_STEP() { + export pgm="$(basename ${0})" if [ ! -z $(command -v prep_step) ]; then . prep_step else @@ -54,13 +55,21 @@ if [ "${RUN_ENVIR}" = "nco" ]; then fi fi } + function POST_STEP() { + if [ -f $pgmout ]; then + tail -n +${pgmout_lines} $pgmout + pgmout_line=$( wc -l $pgmout ) + pgmout_lines=$((pgmout_lines + 1)) + fi + } export -f PREP_STEP + export -f POST_STEP else - export pgm= export pgmout= export pgmerr= export REDIRECT_OUT_ERR= export PREP_STEP= + export POST_STEP= fi # #----------------------------------------------------------------------- @@ -86,11 +95,6 @@ fi function job_postamble() { if [ "${RUN_ENVIR}" = "nco" ]; then - # Print output file to stdout - if [ -e "$pgmout" ]; then - cat $pgmout - fi - # Remove temp directory cd ${DATAROOT} [[ $KEEPDATA = "FALSE" ]] && rm -rf $DATA From ccf77cc83ae22a166e9d63058461dbdb33720466 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Tue, 23 Aug 2022 19:15:17 +0000 Subject: [PATCH 042/106] Add dbn_alert to post-processed grib2 output. --- scripts/exregional_run_fcst.sh | 4 ++++ scripts/exregional_run_post.sh | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 7e36a46655..1a2241390d 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -560,6 +560,10 @@ if [ ${WRITE_DOPOST} = "TRUE" ]; then post_renamed_fn="${NET}.t${cyc}z.mem${ENSMEM_INDX}.${fid}.${post_renamed_fn_suffix}" mv_vrfy ${DATA}/${post_orig_fn} ${post_renamed_fn} ln_vrfy -fs ${post_renamed_fn} ${FID}${symlink_suffix} + # DBN alert + if [ $SENDDBN = "TRUE" ]; then + $DBNROOT/bin/dbn_alert MODEL rrfs_post ${job} ${COMOUT}/${post_renamed_fn} + fi done done diff --git a/scripts/exregional_run_post.sh b/scripts/exregional_run_post.sh index 31f1e91c30..f921348a90 100755 --- a/scripts/exregional_run_post.sh +++ b/scripts/exregional_run_post.sh @@ -284,6 +284,10 @@ for fid in "${fids[@]}"; do create_symlink_to_file target="${post_renamed_fn}" \ symlink="${FID}${symlink_suffix}" \ relative="TRUE" + # DBN alert + if [ $SENDDBN = "TRUE" ]; then + $DBNROOT/bin/dbn_alert MODEL rrfs_post ${job} ${COMOUT}/${post_renamed_fn} + fi done rm_vrfy -rf ${DATA_FHR} From cc4c8de86c0a23b694e970787c6d5d7f3339d063 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Fri, 26 Aug 2022 02:47:04 +0000 Subject: [PATCH 043/106] Download extrn files directly to COMIN. --- jobs/JREGIONAL_GET_EXTRN_MDL_FILES | 6 +++++- jobs/JREGIONAL_MAKE_ICS | 4 ++-- jobs/JREGIONAL_MAKE_LBCS | 4 ++-- jobs/JREGIONAL_RUN_FCST | 4 ++-- jobs/JREGIONAL_RUN_POST | 2 +- scripts/exregional_get_extrn_mdl_files.sh | 7 ++++++- scripts/exregional_make_ics.sh | 8 ++++++-- scripts/exregional_make_lbcs.sh | 8 ++++++-- ush/config_defaults.yaml | 2 +- ush/job_preamble.sh | 2 +- 10 files changed, 32 insertions(+), 15 deletions(-) diff --git a/jobs/JREGIONAL_GET_EXTRN_MDL_FILES b/jobs/JREGIONAL_GET_EXTRN_MDL_FILES index 1922788252..c5ab7f3403 100755 --- a/jobs/JREGIONAL_GET_EXTRN_MDL_FILES +++ b/jobs/JREGIONAL_GET_EXTRN_MDL_FILES @@ -208,7 +208,11 @@ esac # #----------------------------------------------------------------------- # -export EXTRN_MDL_STAGING_DIR="${COMIN}/${EXTRN_MDL_NAME}/for_${ICS_OR_LBCS}" +if [ $RUN_ENVIR = "nco" ]; then + export EXTRN_MDL_STAGING_DIR="${COMIN}" +else + export EXTRN_MDL_STAGING_DIR="${COMIN}/${cyc}/${EXTRN_MDL_NAME}/for_${ICS_OR_LBCS}" +fi mkdir_vrfy -p "${EXTRN_MDL_STAGING_DIR}" cd_vrfy "${EXTRN_MDL_STAGING_DIR}" # diff --git a/jobs/JREGIONAL_MAKE_ICS b/jobs/JREGIONAL_MAKE_ICS index 36e56d1411..59ba2944b1 100755 --- a/jobs/JREGIONAL_MAKE_ICS +++ b/jobs/JREGIONAL_MAKE_ICS @@ -55,7 +55,7 @@ for the FV3 (in NetCDF format). # #----------------------------------------------------------------------- # -export INPUT_DATA="${COMIN}${SLASH_ENSMEM_SUBDIR}/INPUT" +export INPUT_DATA="${COMIN}/${cyc}${SLASH_ENSMEM_SUBDIR}/INPUT" mkdir_vrfy -p "${INPUT_DATA}" # #----------------------------------------------------------------------- @@ -64,7 +64,7 @@ mkdir_vrfy -p "${INPUT_DATA}" # #----------------------------------------------------------------------- # -DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}}" +DATA="${DATA:-${COMIN}/${cyc}${SLASH_ENSMEM_SUBDIR}}" mkdir_vrfy -p "${DATA}" # #----------------------------------------------------------------------- diff --git a/jobs/JREGIONAL_MAKE_LBCS b/jobs/JREGIONAL_MAKE_LBCS index 5b4c8f2a1b..7d7c88a6bd 100755 --- a/jobs/JREGIONAL_MAKE_LBCS +++ b/jobs/JREGIONAL_MAKE_LBCS @@ -55,7 +55,7 @@ hour zero). # #----------------------------------------------------------------------- # -export INPUT_DATA="${COMIN}${SLASH_ENSMEM_SUBDIR}/INPUT" +export INPUT_DATA="${COMIN}/${cyc}${SLASH_ENSMEM_SUBDIR}/INPUT" mkdir_vrfy -p "${INPUT_DATA}" # #----------------------------------------------------------------------- @@ -64,7 +64,7 @@ mkdir_vrfy -p "${INPUT_DATA}" # #----------------------------------------------------------------------- # -DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}}" +DATA="${DATA:-${COMIN}/${cyc}${SLASH_ENSMEM_SUBDIR}}" mkdir_vrfy -p "${DATA}" # #----------------------------------------------------------------------- diff --git a/jobs/JREGIONAL_RUN_FCST b/jobs/JREGIONAL_RUN_FCST index 3a22ff9be9..fea76522cb 100755 --- a/jobs/JREGIONAL_RUN_FCST +++ b/jobs/JREGIONAL_RUN_FCST @@ -63,7 +63,7 @@ the specified cycle. # #----------------------------------------------------------------------- # -export INPUT_DATA="${COMIN}${SLASH_ENSMEM_SUBDIR}" +export INPUT_DATA="${COMIN}/${cyc}${SLASH_ENSMEM_SUBDIR}" # #----------------------------------------------------------------------- # @@ -71,7 +71,7 @@ export INPUT_DATA="${COMIN}${SLASH_ENSMEM_SUBDIR}" # #----------------------------------------------------------------------- # -DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}}" +DATA="${DATA:-${COMIN}/${cyc}${SLASH_ENSMEM_SUBDIR}}" mkdir_vrfy -p ${DATA}/INPUT mkdir_vrfy -p ${DATA}/RESTART # diff --git a/jobs/JREGIONAL_RUN_POST b/jobs/JREGIONAL_RUN_POST index 725a9b8410..baaa75c64d 100755 --- a/jobs/JREGIONAL_RUN_POST +++ b/jobs/JREGIONAL_RUN_POST @@ -62,7 +62,7 @@ on the output files corresponding to a specified forecast hour. # #----------------------------------------------------------------------- # -DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}}" +DATA="${DATA:-${COMIN}/${cyc}${SLASH_ENSMEM_SUBDIR}}" # # If SUB_HOURLY_POST is not set to "TRUE", ensure that the forecast # minutes (fmn) are set to "00". This is necessary in order to pass diff --git a/scripts/exregional_get_extrn_mdl_files.sh b/scripts/exregional_get_extrn_mdl_files.sh index de74e2a139..65963cb6ca 100755 --- a/scripts/exregional_get_extrn_mdl_files.sh +++ b/scripts/exregional_get_extrn_mdl_files.sh @@ -125,6 +125,11 @@ fi # #----------------------------------------------------------------------- # +if [ $RUN_ENVIR = "nco" ]; then + EXTRN_DEFNS="${EXTRN_MDL_VAR_DEFNS_FN}_${cyc}_${EXTRN_MDL_NAME}_${ICS_OR_LBCS}" +else + EXTRN_DEFNS="${EXTRN_MDL_VAR_DEFNS_FN}" +fi cmd=" python3 -u ${USHrrfs}/retrieve_data.py \ --debug \ @@ -135,7 +140,7 @@ python3 -u ${USHrrfs}/retrieve_data.py \ --external_model ${EXTRN_MDL_NAME} \ --fcst_hrs ${fcst_hrs[@]} \ --output_path ${EXTRN_MDL_STAGING_DIR} \ - --summary_file ${EXTRN_MDL_VAR_DEFNS_FN} \ + --summary_file ${EXTRN_DEFNS} \ $additional_flags" $cmd || print_err_msg_exit "\ diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index 56eb52c328..d1b5dbc1b1 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -86,8 +86,12 @@ fi # #----------------------------------------------------------------------- # -extrn_mdl_staging_dir="${COMIN}/${EXTRN_MDL_NAME_ICS}/for_ICS" -extrn_mdl_var_defns_fp="${extrn_mdl_staging_dir}/${EXTRN_MDL_VAR_DEFNS_FN}" +if [ $RUN_ENVIR = "nco" ]; then + extrn_mdl_staging_dir="${COMIN}" +else + extrn_mdl_staging_dir="${COMIN}/${cyc}/${EXTRN_MDL_NAME_ICS}/for_ICS" +fi +extrn_mdl_var_defns_fp="${extrn_mdl_staging_dir}/${EXTRN_MDL_VAR_DEFNS_FN}_${cyc}_${EXTRN_MDL_NAME_ICS}_ICS" . ${extrn_mdl_var_defns_fp} # #----------------------------------------------------------------------- diff --git a/scripts/exregional_make_lbcs.sh b/scripts/exregional_make_lbcs.sh index 9a2fb246a1..b7d3a589c4 100755 --- a/scripts/exregional_make_lbcs.sh +++ b/scripts/exregional_make_lbcs.sh @@ -84,8 +84,12 @@ fi # #----------------------------------------------------------------------- # -extrn_mdl_staging_dir="${COMIN}/${EXTRN_MDL_NAME_LBCS}/for_LBCS" -extrn_mdl_var_defns_fp="${extrn_mdl_staging_dir}/${EXTRN_MDL_VAR_DEFNS_FN}" +if [ $RUN_ENVIR = "nco" ]; then + extrn_mdl_staging_dir="${COMIN}" +else + extrn_mdl_staging_dir="${COMIN}/${cyc}/${EXTRN_MDL_NAME_ICS}/for_LBCS" +fi +extrn_mdl_var_defns_fp="${extrn_mdl_staging_dir}/${EXTRN_MDL_VAR_DEFNS_FN}_${cyc}_${EXTRN_MDL_NAME_LBCS}_LBCS" . ${extrn_mdl_var_defns_fp} # #----------------------------------------------------------------------- diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 7bb9e41048..901d4fd4f3 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -494,7 +494,7 @@ workflow: FCST_MODEL: "ufs-weather-model" WFLOW_XML_FN: "FV3LAM_wflow.xml" GLOBAL_VAR_DEFNS_FN: "var_defns.sh" - EXTRN_MDL_VAR_DEFNS_FN: "extrn_mdl_var_defns.sh" + EXTRN_MDL_VAR_DEFNS_FN: "extrn_mdl_var_defns" WFLOW_LAUNCH_SCRIPT_FN: "launch_FV3LAM_wflow.sh" WFLOW_LAUNCH_LOG_FN: "log.launch_FV3LAM_wflow" # diff --git a/ush/job_preamble.sh b/ush/job_preamble.sh index f7257b9181..6fa206413a 100644 --- a/ush/job_preamble.sh +++ b/ush/job_preamble.sh @@ -79,7 +79,7 @@ fi #----------------------------------------------------------------------- # if [ "${RUN_ENVIR}" = "nco" ]; then - export COMIN="${COMIN_BASEDIR}/${RUN}.${PDY}/${cyc}" + export COMIN="${COMIN_BASEDIR}/${RUN}.${PDY}" export COMOUT="${COMOUT_BASEDIR}/${RUN}.${PDY}" else export COMIN="${COMIN_BASEDIR}/${PDY}${cyc}" From 6dfdd0c9e9347f5b47cdb7757acc89e21d56fb1c Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Fri, 26 Aug 2022 05:20:48 +0000 Subject: [PATCH 044/106] Make make_ics/lbcs directly output to COMIN. --- jobs/JREGIONAL_MAKE_ICS | 6 +++++- jobs/JREGIONAL_MAKE_LBCS | 6 +++++- jobs/JREGIONAL_RUN_FCST | 6 +++++- scripts/exregional_make_ics.sh | 10 ++++----- scripts/exregional_make_lbcs.sh | 2 +- scripts/exregional_run_fcst.sh | 38 ++++++++++++++++----------------- scripts/exregional_run_post.sh | 6 +++--- 7 files changed, 42 insertions(+), 32 deletions(-) diff --git a/jobs/JREGIONAL_MAKE_ICS b/jobs/JREGIONAL_MAKE_ICS index 59ba2944b1..68570d032a 100755 --- a/jobs/JREGIONAL_MAKE_ICS +++ b/jobs/JREGIONAL_MAKE_ICS @@ -55,7 +55,11 @@ for the FV3 (in NetCDF format). # #----------------------------------------------------------------------- # -export INPUT_DATA="${COMIN}/${cyc}${SLASH_ENSMEM_SUBDIR}/INPUT" +if [ $RUN_ENVIR = "nco" ]; then + export INPUT_DATA="${COMIN}" +else + export INPUT_DATA="${COMIN}/${cyc}${SLASH_ENSMEM_SUBDIR}/INPUT" +fi mkdir_vrfy -p "${INPUT_DATA}" # #----------------------------------------------------------------------- diff --git a/jobs/JREGIONAL_MAKE_LBCS b/jobs/JREGIONAL_MAKE_LBCS index 7d7c88a6bd..28dad2a6d8 100755 --- a/jobs/JREGIONAL_MAKE_LBCS +++ b/jobs/JREGIONAL_MAKE_LBCS @@ -55,7 +55,11 @@ hour zero). # #----------------------------------------------------------------------- # -export INPUT_DATA="${COMIN}/${cyc}${SLASH_ENSMEM_SUBDIR}/INPUT" +if [ $RUN_ENVIR = "nco" ]; then + export INPUT_DATA="${COMIN}" +else + export INPUT_DATA="${COMIN}/${cyc}${SLASH_ENSMEM_SUBDIR}/INPUT" +fi mkdir_vrfy -p "${INPUT_DATA}" # #----------------------------------------------------------------------- diff --git a/jobs/JREGIONAL_RUN_FCST b/jobs/JREGIONAL_RUN_FCST index fea76522cb..5925b575db 100755 --- a/jobs/JREGIONAL_RUN_FCST +++ b/jobs/JREGIONAL_RUN_FCST @@ -63,7 +63,11 @@ the specified cycle. # #----------------------------------------------------------------------- # -export INPUT_DATA="${COMIN}/${cyc}${SLASH_ENSMEM_SUBDIR}" +if [ $RUN_ENVIR = "nco" ]; then + export INPUT_DATA="${COMIN}" +else + export INPUT_DATA="${COMIN}/${cyc}${SLASH_ENSMEM_SUBDIR}/INPUT" +fi # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index d1b5dbc1b1..a6bb15d5d2 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -581,14 +581,14 @@ POST_STEP #----------------------------------------------------------------------- # mv_vrfy out.atm.tile${TILE_RGNL}.nc \ - ${INPUT_DATA}/gfs_data.tile${TILE_RGNL}.halo${NH0}.nc + ${INPUT_DATA}/gfs_data.${cycle}.mem${ENSMEM_INDX}.tile${TILE_RGNL}.halo${NH0}.nc mv_vrfy out.sfc.tile${TILE_RGNL}.nc \ - ${INPUT_DATA}/sfc_data.tile${TILE_RGNL}.halo${NH0}.nc + ${INPUT_DATA}/sfc_data.${cycle}.mem${ENSMEM_INDX}.tile${TILE_RGNL}.halo${NH0}.nc -mv_vrfy gfs_ctrl.nc ${INPUT_DATA} +mv_vrfy gfs_ctrl.nc ${INPUT_DATA}/gfs_ctrl.${cycle}.mem${ENSMEM_INDX}.nc -mv_vrfy gfs.bndy.nc ${INPUT_DATA}/gfs_bndy.tile${TILE_RGNL}.000.nc +mv_vrfy gfs.bndy.nc ${INPUT_DATA}/gfs_bndy.${cycle}.mem${ENSMEM_INDX}.tile${TILE_RGNL}.f000.nc # #----------------------------------------------------------------------- # @@ -624,7 +624,7 @@ Please check the following user defined variables: cd_vrfy ${INPUT_DATA} PREP_STEP eval ${RUN_CMD_UTILS} ${fvcom_exec_fn} \ - sfc_data.tile${TILE_RGNL}.halo${NH0}.nc fvcom.nc ${FVCOM_WCSTART} ${fvcom_time} \ + sfc_data.${cycle}.mem${ENSMEM_INDX}.tile${TILE_RGNL}.halo${NH0}.nc fvcom.nc ${FVCOM_WCSTART} ${fvcom_time} \ ${REDIRECT_OUT_ERR} || print_err_msg_exit "\ Call to executable (fvcom_exe) to modify sfc fields for FV3-LAM failed: fvcom_exe = \"${fvcom_exe}\" diff --git a/scripts/exregional_make_lbcs.sh b/scripts/exregional_make_lbcs.sh index b7d3a589c4..11ff625812 100755 --- a/scripts/exregional_make_lbcs.sh +++ b/scripts/exregional_make_lbcs.sh @@ -493,7 +493,7 @@ located in the following directory: # that of the external model since their start times may be offset). # fcst_hhh_FV3LAM=$( printf "%03d" "${LBC_SPEC_FCST_HRS[$i]}" ) - mv_vrfy gfs.bndy.nc ${INPUT_DATA}/gfs_bndy.tile7.${fcst_hhh_FV3LAM}.nc + mv_vrfy gfs.bndy.nc ${INPUT_DATA}/gfs_bndy.${cycle}.mem${ENSMEM_INDX}.tile7.f${fcst_hhh_FV3LAM}.nc done # diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 1a2241390d..4c0f4ae0de 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -232,37 +232,35 @@ of the current run directory (DATA), where cd_vrfy ${DATA}/INPUT -# -# DATA and INPUT_DATA are different -# -if [ "${DATA}" != "${INPUT_DATA}" ]; then - - relative_link_flag="FALSE" - for f_nm_path in ${INPUT_DATA}/INPUT/*; do - symlink=$( basename "${f_nm_path}" ) - target="${f_nm_path}" - create_symlink_to_file target="$target" symlink="$symlink" \ - relative="${relative_link_flag}" - done - -fi - # # The symlinks to be created point to files in the same directory (INPUT), # so it's most straightforward to use relative paths. # relative_link_flag="FALSE" -target="${INPUT_DATA}/INPUT/gfs_data.tile${TILE_RGNL}.halo${NH0}.nc" +target="${INPUT_DATA}/gfs_data.${cycle}.mem${ENSMEM_INDX}.tile${TILE_RGNL}.halo${NH0}.nc" symlink="gfs_data.nc" create_symlink_to_file target="$target" symlink="$symlink" \ relative="${relative_link_flag}" -target="${INPUT_DATA}/INPUT/sfc_data.tile${TILE_RGNL}.halo${NH0}.nc" +target="${INPUT_DATA}/sfc_data.${cycle}.mem${ENSMEM_INDX}.tile${TILE_RGNL}.halo${NH0}.nc" symlink="sfc_data.nc" create_symlink_to_file target="$target" symlink="$symlink" \ relative="${relative_link_flag}" +target="${INPUT_DATA}/gfs_ctrl.${cycle}.mem${ENSMEM_INDX}.nc" +symlink="gfs_ctrl.nc" +create_symlink_to_file target="$target" symlink="$symlink" \ + relative="${relative_link_flag}" + + +for fhr in $(seq -f "%03g" 0 ${LBC_SPEC_INTVL_HRS} ${FCST_LEN_HRS}); do + target="${INPUT_DATA}/gfs_bndy.${cycle}.mem${ENSMEM_INDX}.tile${TILE_RGNL}.f${fhr}.nc" + symlink="gfs_bndy.tile${TILE_RGNL}.${fhr}.nc" + create_symlink_to_file target="$target" symlink="$symlink" \ + relative="${relative_link_flag}" +done + # #----------------------------------------------------------------------- # @@ -495,8 +493,8 @@ if [ "${RUN_ENVIR}" = "nco" ]; then # create the symlinks for fhr in $(seq -f "%03g" 0 ${FCST_LEN_HRS}); do - ln_vrfy -sf "${COMOUT}/${NET}.t${cyc}z.mem${ENSMEM_INDX}.dynf${fhr}${mnts_secs_str}.nc" "dynf${fhr}${mnts_secs_str}.nc" - ln_vrfy -sf "${COMOUT}/${NET}.t${cyc}z.mem${ENSMEM_INDX}.phyf${fhr}${mnts_secs_str}.nc" "phyf${fhr}${mnts_secs_str}.nc" + ln_vrfy -sf "${COMOUT}/${NET}.${cycle}.mem${ENSMEM_INDX}.dynf${fhr}${mnts_secs_str}.nc" "dynf${fhr}${mnts_secs_str}.nc" + ln_vrfy -sf "${COMOUT}/${NET}.${cycle}.mem${ENSMEM_INDX}.phyf${fhr}${mnts_secs_str}.nc" "phyf${fhr}${mnts_secs_str}.nc" done fi # @@ -557,7 +555,7 @@ if [ ${WRITE_DOPOST} = "TRUE" ]; then for fid in "${fids[@]}"; do FID=$(echo_uppercase $fid) post_orig_fn="${FID}.${post_fn_suffix}" - post_renamed_fn="${NET}.t${cyc}z.mem${ENSMEM_INDX}.${fid}.${post_renamed_fn_suffix}" + post_renamed_fn="${NET}.${cycle}.mem${ENSMEM_INDX}.${fid}.${post_renamed_fn_suffix}" mv_vrfy ${DATA}/${post_orig_fn} ${post_renamed_fn} ln_vrfy -fs ${post_renamed_fn} ${FID}${symlink_suffix} # DBN alert diff --git a/scripts/exregional_run_post.sh b/scripts/exregional_run_post.sh index f921348a90..0e863d3878 100755 --- a/scripts/exregional_run_post.sh +++ b/scripts/exregional_run_post.sh @@ -172,8 +172,8 @@ if [ "${RUN_ENVIR}" != "nco" ]; then dyn_file="${DATA}/dynf${fhr}${mnts_secs_str}.nc" phy_file="${DATA}/phyf${fhr}${mnts_secs_str}.nc" else - dyn_file="${COMOUT}/${NET}.t${cyc}z.mem${ENSMEM_INDX}.dynf${fhr}${mnts_secs_str}.nc" - phy_file="${COMOUT}/${NET}.t${cyc}z.mem${ENSMEM_INDX}.phyf${fhr}${mnts_secs_str}.nc" + dyn_file="${COMOUT}/${NET}.${cycle}.mem${ENSMEM_INDX}.dynf${fhr}${mnts_secs_str}.nc" + phy_file="${COMOUT}/${NET}.${cycle}.mem${ENSMEM_INDX}.phyf${fhr}${mnts_secs_str}.nc" fi # # Set parameters that specify the actual time (not forecast time) of the @@ -279,7 +279,7 @@ fids=( "prslev" "natlev" ) for fid in "${fids[@]}"; do FID=$(echo_uppercase $fid) post_orig_fn="${FID}.${post_fn_suffix}" - post_renamed_fn="${NET}.t${cyc}z.mem${ENSMEM_INDX}.${fid}.${post_renamed_fn_suffix}" + post_renamed_fn="${NET}.${cycle}.mem${ENSMEM_INDX}.${fid}.${post_renamed_fn_suffix}" mv_vrfy ${DATA_FHR}/${post_orig_fn} ${post_renamed_fn} create_symlink_to_file target="${post_renamed_fn}" \ symlink="${FID}${symlink_suffix}" \ From e0fee4a749b571daa6ff818b3d00332688be2c23 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Fri, 26 Aug 2022 10:29:42 +0000 Subject: [PATCH 045/106] Change names of extrn_mdl_var_defns files. --- scripts/exregional_get_extrn_mdl_files.sh | 4 ++-- scripts/exregional_make_ics.sh | 2 +- scripts/exregional_make_lbcs.sh | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/exregional_get_extrn_mdl_files.sh b/scripts/exregional_get_extrn_mdl_files.sh index 65963cb6ca..55b848caa0 100755 --- a/scripts/exregional_get_extrn_mdl_files.sh +++ b/scripts/exregional_get_extrn_mdl_files.sh @@ -126,9 +126,9 @@ fi #----------------------------------------------------------------------- # if [ $RUN_ENVIR = "nco" ]; then - EXTRN_DEFNS="${EXTRN_MDL_VAR_DEFNS_FN}_${cyc}_${EXTRN_MDL_NAME}_${ICS_OR_LBCS}" + EXTRN_DEFNS="${EXTRN_MDL_VAR_DEFNS_FN}.${cycle}.${EXTRN_MDL_NAME}.${ICS_OR_LBCS}.sh" else - EXTRN_DEFNS="${EXTRN_MDL_VAR_DEFNS_FN}" + EXTRN_DEFNS="${EXTRN_MDL_VAR_DEFNS_FN}.sh" fi cmd=" python3 -u ${USHrrfs}/retrieve_data.py \ diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index a6bb15d5d2..a53f1e9081 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -91,7 +91,7 @@ if [ $RUN_ENVIR = "nco" ]; then else extrn_mdl_staging_dir="${COMIN}/${cyc}/${EXTRN_MDL_NAME_ICS}/for_ICS" fi -extrn_mdl_var_defns_fp="${extrn_mdl_staging_dir}/${EXTRN_MDL_VAR_DEFNS_FN}_${cyc}_${EXTRN_MDL_NAME_ICS}_ICS" +extrn_mdl_var_defns_fp="${extrn_mdl_staging_dir}/${EXTRN_MDL_VAR_DEFNS_FN}.${cycle}.${EXTRN_MDL_NAME_ICS}.ICS.sh" . ${extrn_mdl_var_defns_fp} # #----------------------------------------------------------------------- diff --git a/scripts/exregional_make_lbcs.sh b/scripts/exregional_make_lbcs.sh index 11ff625812..66ea81045e 100755 --- a/scripts/exregional_make_lbcs.sh +++ b/scripts/exregional_make_lbcs.sh @@ -89,7 +89,7 @@ if [ $RUN_ENVIR = "nco" ]; then else extrn_mdl_staging_dir="${COMIN}/${cyc}/${EXTRN_MDL_NAME_ICS}/for_LBCS" fi -extrn_mdl_var_defns_fp="${extrn_mdl_staging_dir}/${EXTRN_MDL_VAR_DEFNS_FN}_${cyc}_${EXTRN_MDL_NAME_LBCS}_LBCS" +extrn_mdl_var_defns_fp="${extrn_mdl_staging_dir}/${EXTRN_MDL_VAR_DEFNS_FN}.${cycle}.${EXTRN_MDL_NAME_LBCS}.LBCS.sh" . ${extrn_mdl_var_defns_fp} # #----------------------------------------------------------------------- From d647d94cbf943346476677cdfab78fc933791873 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Fri, 26 Aug 2022 11:07:59 +0000 Subject: [PATCH 046/106] Name fixes for DO_ENSEMBLE=false, dyn/phy --- scripts/exregional_make_ics.sh | 10 +++++----- scripts/exregional_make_lbcs.sh | 2 +- scripts/exregional_run_fcst.sh | 14 +++++++------- scripts/exregional_run_post.sh | 8 ++++---- ush/job_preamble.sh | 12 ++++++++++++ 5 files changed, 29 insertions(+), 17 deletions(-) diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index a53f1e9081..3bdcee06d5 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -581,14 +581,14 @@ POST_STEP #----------------------------------------------------------------------- # mv_vrfy out.atm.tile${TILE_RGNL}.nc \ - ${INPUT_DATA}/gfs_data.${cycle}.mem${ENSMEM_INDX}.tile${TILE_RGNL}.halo${NH0}.nc + ${INPUT_DATA}/gfs_data.${cycle}${dot_ensmem}.tile${TILE_RGNL}.halo${NH0}.nc mv_vrfy out.sfc.tile${TILE_RGNL}.nc \ - ${INPUT_DATA}/sfc_data.${cycle}.mem${ENSMEM_INDX}.tile${TILE_RGNL}.halo${NH0}.nc + ${INPUT_DATA}/sfc_data.${cycle}${dot_ensmem}.tile${TILE_RGNL}.halo${NH0}.nc -mv_vrfy gfs_ctrl.nc ${INPUT_DATA}/gfs_ctrl.${cycle}.mem${ENSMEM_INDX}.nc +mv_vrfy gfs_ctrl.nc ${INPUT_DATA}/gfs_ctrl.${cycle}${dot_ensmem}.nc -mv_vrfy gfs.bndy.nc ${INPUT_DATA}/gfs_bndy.${cycle}.mem${ENSMEM_INDX}.tile${TILE_RGNL}.f000.nc +mv_vrfy gfs.bndy.nc ${INPUT_DATA}/gfs_bndy.${cycle}${dot_ensmem}.tile${TILE_RGNL}.f000.nc # #----------------------------------------------------------------------- # @@ -624,7 +624,7 @@ Please check the following user defined variables: cd_vrfy ${INPUT_DATA} PREP_STEP eval ${RUN_CMD_UTILS} ${fvcom_exec_fn} \ - sfc_data.${cycle}.mem${ENSMEM_INDX}.tile${TILE_RGNL}.halo${NH0}.nc fvcom.nc ${FVCOM_WCSTART} ${fvcom_time} \ + sfc_data.${cycle}${dot_ensmem}.tile${TILE_RGNL}.halo${NH0}.nc fvcom.nc ${FVCOM_WCSTART} ${fvcom_time} \ ${REDIRECT_OUT_ERR} || print_err_msg_exit "\ Call to executable (fvcom_exe) to modify sfc fields for FV3-LAM failed: fvcom_exe = \"${fvcom_exe}\" diff --git a/scripts/exregional_make_lbcs.sh b/scripts/exregional_make_lbcs.sh index 66ea81045e..7ca68a0d2a 100755 --- a/scripts/exregional_make_lbcs.sh +++ b/scripts/exregional_make_lbcs.sh @@ -493,7 +493,7 @@ located in the following directory: # that of the external model since their start times may be offset). # fcst_hhh_FV3LAM=$( printf "%03d" "${LBC_SPEC_FCST_HRS[$i]}" ) - mv_vrfy gfs.bndy.nc ${INPUT_DATA}/gfs_bndy.${cycle}.mem${ENSMEM_INDX}.tile7.f${fcst_hhh_FV3LAM}.nc + mv_vrfy gfs.bndy.nc ${INPUT_DATA}/gfs_bndy.${cycle}${dot_ensmem}.tile7.f${fcst_hhh_FV3LAM}.nc done # diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 4c0f4ae0de..ba19be67cd 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -238,24 +238,24 @@ cd_vrfy ${DATA}/INPUT # relative_link_flag="FALSE" -target="${INPUT_DATA}/gfs_data.${cycle}.mem${ENSMEM_INDX}.tile${TILE_RGNL}.halo${NH0}.nc" +target="${INPUT_DATA}/gfs_data.${cycle}${dot_ensmem}.tile${TILE_RGNL}.halo${NH0}.nc" symlink="gfs_data.nc" create_symlink_to_file target="$target" symlink="$symlink" \ relative="${relative_link_flag}" -target="${INPUT_DATA}/sfc_data.${cycle}.mem${ENSMEM_INDX}.tile${TILE_RGNL}.halo${NH0}.nc" +target="${INPUT_DATA}/sfc_data.${cycle}${dot_ensmem}.tile${TILE_RGNL}.halo${NH0}.nc" symlink="sfc_data.nc" create_symlink_to_file target="$target" symlink="$symlink" \ relative="${relative_link_flag}" -target="${INPUT_DATA}/gfs_ctrl.${cycle}.mem${ENSMEM_INDX}.nc" +target="${INPUT_DATA}/gfs_ctrl.${cycle}${dot_ensmem}.nc" symlink="gfs_ctrl.nc" create_symlink_to_file target="$target" symlink="$symlink" \ relative="${relative_link_flag}" for fhr in $(seq -f "%03g" 0 ${LBC_SPEC_INTVL_HRS} ${FCST_LEN_HRS}); do - target="${INPUT_DATA}/gfs_bndy.${cycle}.mem${ENSMEM_INDX}.tile${TILE_RGNL}.f${fhr}.nc" + target="${INPUT_DATA}/gfs_bndy.${cycle}${dot_ensmem}.tile${TILE_RGNL}.f${fhr}.nc" symlink="gfs_bndy.tile${TILE_RGNL}.${fhr}.nc" create_symlink_to_file target="$target" symlink="$symlink" \ relative="${relative_link_flag}" @@ -493,8 +493,8 @@ if [ "${RUN_ENVIR}" = "nco" ]; then # create the symlinks for fhr in $(seq -f "%03g" 0 ${FCST_LEN_HRS}); do - ln_vrfy -sf "${COMOUT}/${NET}.${cycle}.mem${ENSMEM_INDX}.dynf${fhr}${mnts_secs_str}.nc" "dynf${fhr}${mnts_secs_str}.nc" - ln_vrfy -sf "${COMOUT}/${NET}.${cycle}.mem${ENSMEM_INDX}.phyf${fhr}${mnts_secs_str}.nc" "phyf${fhr}${mnts_secs_str}.nc" + ln_vrfy -sf "${COMOUT}/${NET}.${cycle}${dot_ensmem}.dyn.f${fhr}${mnts_secs_str}.nc" "dynf${fhr}${mnts_secs_str}.nc" + ln_vrfy -sf "${COMOUT}/${NET}.${cycle}${dot_ensmem}.phy.f${fhr}${mnts_secs_str}.nc" "phyf${fhr}${mnts_secs_str}.nc" done fi # @@ -555,7 +555,7 @@ if [ ${WRITE_DOPOST} = "TRUE" ]; then for fid in "${fids[@]}"; do FID=$(echo_uppercase $fid) post_orig_fn="${FID}.${post_fn_suffix}" - post_renamed_fn="${NET}.${cycle}.mem${ENSMEM_INDX}.${fid}.${post_renamed_fn_suffix}" + post_renamed_fn="${NET}.${cycle}${dot_ensmem}.${fid}.${post_renamed_fn_suffix}" mv_vrfy ${DATA}/${post_orig_fn} ${post_renamed_fn} ln_vrfy -fs ${post_renamed_fn} ${FID}${symlink_suffix} # DBN alert diff --git a/scripts/exregional_run_post.sh b/scripts/exregional_run_post.sh index 0e863d3878..81a244d3ef 100755 --- a/scripts/exregional_run_post.sh +++ b/scripts/exregional_run_post.sh @@ -172,8 +172,8 @@ if [ "${RUN_ENVIR}" != "nco" ]; then dyn_file="${DATA}/dynf${fhr}${mnts_secs_str}.nc" phy_file="${DATA}/phyf${fhr}${mnts_secs_str}.nc" else - dyn_file="${COMOUT}/${NET}.${cycle}.mem${ENSMEM_INDX}.dynf${fhr}${mnts_secs_str}.nc" - phy_file="${COMOUT}/${NET}.${cycle}.mem${ENSMEM_INDX}.phyf${fhr}${mnts_secs_str}.nc" + dyn_file="${COMOUT}/${NET}.${cycle}${dot_ensmem}.dyn.f${fhr}${mnts_secs_str}.nc" + phy_file="${COMOUT}/${NET}.${cycle}${dot_ensmem}.phy.f${fhr}${mnts_secs_str}.nc" fi # # Set parameters that specify the actual time (not forecast time) of the @@ -274,12 +274,12 @@ post_renamed_fn_suffix="f${fhr}${post_mn_or_null}.${POST_OUTPUT_DOMAIN_NAME}.gri # cd_vrfy "${COMOUT}" basetime=$( $DATE_UTIL --date "$yyyymmdd $hh" +%y%j%H%M ) -symlink_suffix="_mem${ENSMEM_INDX}_${basetime}f${fhr}${post_mn}" +symlink_suffix="${dot_ensmem}.${basetime}f${fhr}${post_mn}" fids=( "prslev" "natlev" ) for fid in "${fids[@]}"; do FID=$(echo_uppercase $fid) post_orig_fn="${FID}.${post_fn_suffix}" - post_renamed_fn="${NET}.${cycle}.mem${ENSMEM_INDX}.${fid}.${post_renamed_fn_suffix}" + post_renamed_fn="${NET}.${cycle}${dot_ensmem}.${fid}.${post_renamed_fn_suffix}" mv_vrfy ${DATA_FHR}/${post_orig_fn} ${post_renamed_fn} create_symlink_to_file target="${post_renamed_fn}" \ symlink="${FID}${symlink_suffix}" \ diff --git a/ush/job_preamble.sh b/ush/job_preamble.sh index 6fa206413a..7d478da3bc 100644 --- a/ush/job_preamble.sh +++ b/ush/job_preamble.sh @@ -30,6 +30,18 @@ fi # #----------------------------------------------------------------------- # +# Set ensemble member name in file names +# +#----------------------------------------------------------------------- +# +if [ "${RUN_ENVIR}" = "nco" ] && [ "${DO_ENSEMBLE}" = "TRUE" ]; then + export dot_ensmem=".mem${ENSMEM_INDX}" +else + export dot_ensmem= +fi +# +#----------------------------------------------------------------------- +# # Set pgmout and pgmerr files # #----------------------------------------------------------------------- From 3df5d2c75d0fb3dd40604fc4ce1b3006a9796dbb Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Fri, 26 Aug 2022 12:13:38 +0000 Subject: [PATCH 047/106] Don't create symlinks to grib2 files in NCO mode. --- scripts/exregional_run_post.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/exregional_run_post.sh b/scripts/exregional_run_post.sh index 81a244d3ef..b473ee5888 100755 --- a/scripts/exregional_run_post.sh +++ b/scripts/exregional_run_post.sh @@ -281,9 +281,11 @@ for fid in "${fids[@]}"; do post_orig_fn="${FID}.${post_fn_suffix}" post_renamed_fn="${NET}.${cycle}${dot_ensmem}.${fid}.${post_renamed_fn_suffix}" mv_vrfy ${DATA_FHR}/${post_orig_fn} ${post_renamed_fn} - create_symlink_to_file target="${post_renamed_fn}" \ + if [ $RUN_ENVIR != "nco" ]; then + create_symlink_to_file target="${post_renamed_fn}" \ symlink="${FID}${symlink_suffix}" \ relative="TRUE" + fi # DBN alert if [ $SENDDBN = "TRUE" ]; then $DBNROOT/bin/dbn_alert MODEL rrfs_post ${job} ${COMOUT}/${post_renamed_fn} From ae343bcad38c6740d82b3360cc4d450d36c139a5 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Fri, 26 Aug 2022 13:43:58 +0000 Subject: [PATCH 048/106] Append rrfs to make_ics/lbcs output. --- scripts/exregional_make_ics.sh | 10 +++++----- scripts/exregional_make_lbcs.sh | 2 +- scripts/exregional_run_fcst.sh | 8 ++++---- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index 3bdcee06d5..3066b203be 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -581,14 +581,14 @@ POST_STEP #----------------------------------------------------------------------- # mv_vrfy out.atm.tile${TILE_RGNL}.nc \ - ${INPUT_DATA}/gfs_data.${cycle}${dot_ensmem}.tile${TILE_RGNL}.halo${NH0}.nc + ${INPUT_DATA}/${NET}.${cycle}${dot_ensmem}.gfs_data.tile${TILE_RGNL}.halo${NH0}.nc mv_vrfy out.sfc.tile${TILE_RGNL}.nc \ - ${INPUT_DATA}/sfc_data.${cycle}${dot_ensmem}.tile${TILE_RGNL}.halo${NH0}.nc + ${INPUT_DATA}/${NET}.${cycle}${dot_ensmem}.sfc_data.tile${TILE_RGNL}.halo${NH0}.nc -mv_vrfy gfs_ctrl.nc ${INPUT_DATA}/gfs_ctrl.${cycle}${dot_ensmem}.nc +mv_vrfy gfs_ctrl.nc ${INPUT_DATA}/${NET}.${cycle}${dot_ensmem}.gfs_ctrl.nc -mv_vrfy gfs.bndy.nc ${INPUT_DATA}/gfs_bndy.${cycle}${dot_ensmem}.tile${TILE_RGNL}.f000.nc +mv_vrfy gfs.bndy.nc ${INPUT_DATA}/${NET}.${cycle}${dot_ensmem}.gfs_bndy.tile${TILE_RGNL}.f000.nc # #----------------------------------------------------------------------- # @@ -624,7 +624,7 @@ Please check the following user defined variables: cd_vrfy ${INPUT_DATA} PREP_STEP eval ${RUN_CMD_UTILS} ${fvcom_exec_fn} \ - sfc_data.${cycle}${dot_ensmem}.tile${TILE_RGNL}.halo${NH0}.nc fvcom.nc ${FVCOM_WCSTART} ${fvcom_time} \ + ${NET}.${cycle}${dot_ensmem}.sfc_data.tile${TILE_RGNL}.halo${NH0}.nc fvcom.nc ${FVCOM_WCSTART} ${fvcom_time} \ ${REDIRECT_OUT_ERR} || print_err_msg_exit "\ Call to executable (fvcom_exe) to modify sfc fields for FV3-LAM failed: fvcom_exe = \"${fvcom_exe}\" diff --git a/scripts/exregional_make_lbcs.sh b/scripts/exregional_make_lbcs.sh index 7ca68a0d2a..d778cded15 100755 --- a/scripts/exregional_make_lbcs.sh +++ b/scripts/exregional_make_lbcs.sh @@ -493,7 +493,7 @@ located in the following directory: # that of the external model since their start times may be offset). # fcst_hhh_FV3LAM=$( printf "%03d" "${LBC_SPEC_FCST_HRS[$i]}" ) - mv_vrfy gfs.bndy.nc ${INPUT_DATA}/gfs_bndy.${cycle}${dot_ensmem}.tile7.f${fcst_hhh_FV3LAM}.nc + mv_vrfy gfs.bndy.nc ${INPUT_DATA}/${NET}.${cycle}${dot_ensmem}.gfs_bndy.tile7.f${fcst_hhh_FV3LAM}.nc done # diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index ba19be67cd..c01640c963 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -238,24 +238,24 @@ cd_vrfy ${DATA}/INPUT # relative_link_flag="FALSE" -target="${INPUT_DATA}/gfs_data.${cycle}${dot_ensmem}.tile${TILE_RGNL}.halo${NH0}.nc" +target="${INPUT_DATA}/${NET}.${cycle}${dot_ensmem}.gfs_data.tile${TILE_RGNL}.halo${NH0}.nc" symlink="gfs_data.nc" create_symlink_to_file target="$target" symlink="$symlink" \ relative="${relative_link_flag}" -target="${INPUT_DATA}/sfc_data.${cycle}${dot_ensmem}.tile${TILE_RGNL}.halo${NH0}.nc" +target="${INPUT_DATA}/${NET}.${cycle}${dot_ensmem}.sfc_data.tile${TILE_RGNL}.halo${NH0}.nc" symlink="sfc_data.nc" create_symlink_to_file target="$target" symlink="$symlink" \ relative="${relative_link_flag}" -target="${INPUT_DATA}/gfs_ctrl.${cycle}${dot_ensmem}.nc" +target="${INPUT_DATA}/${NET}.${cycle}${dot_ensmem}.gfs_ctrl.nc" symlink="gfs_ctrl.nc" create_symlink_to_file target="$target" symlink="$symlink" \ relative="${relative_link_flag}" for fhr in $(seq -f "%03g" 0 ${LBC_SPEC_INTVL_HRS} ${FCST_LEN_HRS}); do - target="${INPUT_DATA}/gfs_bndy.${cycle}${dot_ensmem}.tile${TILE_RGNL}.f${fhr}.nc" + target="${INPUT_DATA}/${NET}.${cycle}${dot_ensmem}.gfs_bndy.tile${TILE_RGNL}.f${fhr}.nc" symlink="gfs_bndy.tile${TILE_RGNL}.${fhr}.nc" create_symlink_to_file target="$target" symlink="$symlink" \ relative="${relative_link_flag}" From 178e4d201f4fa1c7aa136f0687513c20cb5cbf18 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Fri, 26 Aug 2022 17:22:07 +0000 Subject: [PATCH 049/106] Modify extrn_mdl_var_defns names. --- scripts/exregional_get_extrn_mdl_files.sh | 2 +- scripts/exregional_make_ics.sh | 2 +- scripts/exregional_make_lbcs.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/exregional_get_extrn_mdl_files.sh b/scripts/exregional_get_extrn_mdl_files.sh index 55b848caa0..5f4a93f601 100755 --- a/scripts/exregional_get_extrn_mdl_files.sh +++ b/scripts/exregional_get_extrn_mdl_files.sh @@ -126,7 +126,7 @@ fi #----------------------------------------------------------------------- # if [ $RUN_ENVIR = "nco" ]; then - EXTRN_DEFNS="${EXTRN_MDL_VAR_DEFNS_FN}.${cycle}.${EXTRN_MDL_NAME}.${ICS_OR_LBCS}.sh" + EXTRN_DEFNS="${NET}.${cycle}.${EXTRN_MDL_NAME}.${ICS_OR_LBCS}.${EXTRN_MDL_VAR_DEFNS_FN}.sh" else EXTRN_DEFNS="${EXTRN_MDL_VAR_DEFNS_FN}.sh" fi diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index 3066b203be..5d317a3393 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -91,7 +91,7 @@ if [ $RUN_ENVIR = "nco" ]; then else extrn_mdl_staging_dir="${COMIN}/${cyc}/${EXTRN_MDL_NAME_ICS}/for_ICS" fi -extrn_mdl_var_defns_fp="${extrn_mdl_staging_dir}/${EXTRN_MDL_VAR_DEFNS_FN}.${cycle}.${EXTRN_MDL_NAME_ICS}.ICS.sh" +extrn_mdl_var_defns_fp="${extrn_mdl_staging_dir}/${NET}.${cycle}.${EXTRN_MDL_NAME_ICS}.ICS.${EXTRN_MDL_VAR_DEFNS_FN}.sh" . ${extrn_mdl_var_defns_fp} # #----------------------------------------------------------------------- diff --git a/scripts/exregional_make_lbcs.sh b/scripts/exregional_make_lbcs.sh index d778cded15..d9d7e02f8c 100755 --- a/scripts/exregional_make_lbcs.sh +++ b/scripts/exregional_make_lbcs.sh @@ -89,7 +89,7 @@ if [ $RUN_ENVIR = "nco" ]; then else extrn_mdl_staging_dir="${COMIN}/${cyc}/${EXTRN_MDL_NAME_ICS}/for_LBCS" fi -extrn_mdl_var_defns_fp="${extrn_mdl_staging_dir}/${EXTRN_MDL_VAR_DEFNS_FN}.${cycle}.${EXTRN_MDL_NAME_LBCS}.LBCS.sh" +extrn_mdl_var_defns_fp="${extrn_mdl_staging_dir}/${NET}.${cycle}.${EXTRN_MDL_NAME_LBCS}.LBCS.${EXTRN_MDL_VAR_DEFNS_FN}.sh" . ${extrn_mdl_var_defns_fp} # #----------------------------------------------------------------------- From 7a7e3240c0b9b6f2a68a95c1500548c54426aed9 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Sat, 27 Aug 2022 01:27:19 +0000 Subject: [PATCH 050/106] Move forecast output to DATA/RUN.PDY. This location can be used to store output of other tasks as well. --- scripts/exregional_run_fcst.sh | 9 +++------ scripts/exregional_run_post.sh | 4 ++-- ush/job_preamble.sh | 34 ++++++++++++++++++---------------- 3 files changed, 23 insertions(+), 24 deletions(-) diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index c01640c963..a6af733cf7 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -472,15 +472,12 @@ Call to function to create a diag table file for the current cycle's # #----------------------------------------------------------------------- # -# Pre-generate symlinks to forecast output in DATA pointing to COMOUT +# Pre-generate symlinks to forecast output in DATA pointing to DATA_SHARED # #----------------------------------------------------------------------- # if [ "${RUN_ENVIR}" = "nco" ]; then - # create comout directory - mkdir_vrfy -p ${COMOUT} - # first set suffix for minutes and seconds of forecast time mnts_secs_str="" if [ "${SUB_HOURLY_POST}" = "TRUE" ]; then @@ -493,8 +490,8 @@ if [ "${RUN_ENVIR}" = "nco" ]; then # create the symlinks for fhr in $(seq -f "%03g" 0 ${FCST_LEN_HRS}); do - ln_vrfy -sf "${COMOUT}/${NET}.${cycle}${dot_ensmem}.dyn.f${fhr}${mnts_secs_str}.nc" "dynf${fhr}${mnts_secs_str}.nc" - ln_vrfy -sf "${COMOUT}/${NET}.${cycle}${dot_ensmem}.phy.f${fhr}${mnts_secs_str}.nc" "phyf${fhr}${mnts_secs_str}.nc" + ln_vrfy -sf "${DATA_SHARED}/${NET}.${cycle}${dot_ensmem}.dyn.f${fhr}${mnts_secs_str}.nc" "dynf${fhr}${mnts_secs_str}.nc" + ln_vrfy -sf "${DATA_SHARED}/${NET}.${cycle}${dot_ensmem}.phy.f${fhr}${mnts_secs_str}.nc" "phyf${fhr}${mnts_secs_str}.nc" done fi # diff --git a/scripts/exregional_run_post.sh b/scripts/exregional_run_post.sh index b473ee5888..4d626452f8 100755 --- a/scripts/exregional_run_post.sh +++ b/scripts/exregional_run_post.sh @@ -172,8 +172,8 @@ if [ "${RUN_ENVIR}" != "nco" ]; then dyn_file="${DATA}/dynf${fhr}${mnts_secs_str}.nc" phy_file="${DATA}/phyf${fhr}${mnts_secs_str}.nc" else - dyn_file="${COMOUT}/${NET}.${cycle}${dot_ensmem}.dyn.f${fhr}${mnts_secs_str}.nc" - phy_file="${COMOUT}/${NET}.${cycle}${dot_ensmem}.phy.f${fhr}${mnts_secs_str}.nc" + dyn_file="${DATA_SHARED}/${NET}.${cycle}${dot_ensmem}.dyn.f${fhr}${mnts_secs_str}.nc" + phy_file="${DATA_SHARED}/${NET}.${cycle}${dot_ensmem}.phy.f${fhr}${mnts_secs_str}.nc" fi # # Set parameters that specify the actual time (not forecast time) of the diff --git a/ush/job_preamble.sh b/ush/job_preamble.sh index 7d478da3bc..c737848d85 100644 --- a/ush/job_preamble.sh +++ b/ush/job_preamble.sh @@ -1,5 +1,18 @@ #!/bin/bash +# +#----------------------------------------------------------------------- +# +# Set cycle and ensemble member names in file/diectory names +# +#----------------------------------------------------------------------- +# +export cycle="t${cyc}z" +if [ "${RUN_ENVIR}" = "nco" ] && [ "${DO_ENSEMBLE}" = "TRUE" ]; then + export dot_ensmem=".mem${ENSMEM_INDX}" +else + export dot_ensmem= +fi # #----------------------------------------------------------------------- # @@ -8,19 +21,20 @@ #----------------------------------------------------------------------- # export DATA= +export DATA_SHARED= if [ "${RUN_ENVIR}" = "nco" ]; then export DATA=${DATAROOT}/${jobid} - mkdir_vrfy -p $DATA + export DATA_SHARED=${DATAROOT}/${RUN}.${PDY} + mkdir_vrfy -p $DATA $DATA_SHARED cd $DATA fi # #----------------------------------------------------------------------- # -# Set cycle and run setpdy to initialize PDYm and PDYp variables +# Run setpdy to initialize PDYm and PDYp variables # #----------------------------------------------------------------------- # -export cycle="t${cyc}z" if [ "${RUN_ENVIR}" = "nco" ]; then if [ ! -z $(command -v setpdy.sh) ]; then setpdy.sh @@ -30,18 +44,6 @@ fi # #----------------------------------------------------------------------- # -# Set ensemble member name in file names -# -#----------------------------------------------------------------------- -# -if [ "${RUN_ENVIR}" = "nco" ] && [ "${DO_ENSEMBLE}" = "TRUE" ]; then - export dot_ensmem=".mem${ENSMEM_INDX}" -else - export dot_ensmem= -fi -# -#----------------------------------------------------------------------- -# # Set pgmout and pgmerr files # #----------------------------------------------------------------------- @@ -109,7 +111,7 @@ function job_postamble() { if [ "${RUN_ENVIR}" = "nco" ]; then # Remove temp directory cd ${DATAROOT} - [[ $KEEPDATA = "FALSE" ]] && rm -rf $DATA + [[ $KEEPDATA = "FALSE" ]] && rm -rf $DATA $DATA_SHARED fi # Print exit message From 548b824b1ac401b958c3dde86f73a7a8db9c78b4 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Mon, 29 Aug 2022 12:50:48 +0000 Subject: [PATCH 051/106] Move templates to parm. --- {ush/templates => parm}/FV3.input.yml | 0 {ush/templates => parm}/FV3LAM_wflow.xml | 0 {ush/templates => parm}/README.xml_templating.md | 0 {ush/templates => parm}/data_locations.yml | 0 {ush/templates => parm}/data_table | 0 {ush/templates => parm}/diag_table.FV3_GFS_2017_gfdlmp | 0 {ush/templates => parm}/diag_table.FV3_GFS_2017_gfdlmp_regional | 0 .../diag_table.FV3_GFS_v15_thompson_mynn_lam3km | 0 {ush/templates => parm}/diag_table.FV3_GFS_v15p2 | 0 {ush/templates => parm}/diag_table.FV3_GFS_v16 | 0 {ush/templates => parm}/diag_table.FV3_HRRR | 0 {ush/templates => parm}/diag_table.FV3_RRFS_v0 | 0 {ush/templates => parm}/diag_table.FV3_RRFS_v1beta | 0 {ush/templates => parm}/field_table.FV3_GFS_2017_gfdlmp | 0 {ush/templates => parm}/field_table.FV3_GFS_2017_gfdlmp_regional | 0 .../field_table.FV3_GFS_v15_thompson_mynn_lam3km | 0 {ush/templates => parm}/field_table.FV3_GFS_v15p2 | 0 {ush/templates => parm}/field_table.FV3_GFS_v16 | 0 {ush/templates => parm}/field_table.FV3_HRRR | 0 {ush/templates => parm}/field_table.FV3_RRFS_v0 | 0 {ush/templates => parm}/field_table.FV3_RRFS_v1beta | 0 {ush/templates => parm}/input.nml.FV3 | 0 {ush/templates => parm}/model_configure | 0 {ush/templates => parm}/nems.configure | 0 {ush/templates => parm}/parm/metplus/EnsembleStat_APCP01h.conf | 0 {ush/templates => parm}/parm/metplus/EnsembleStat_APCP03h.conf | 0 {ush/templates => parm}/parm/metplus/EnsembleStat_APCP06h.conf | 0 {ush/templates => parm}/parm/metplus/EnsembleStat_APCP24h.conf | 0 {ush/templates => parm}/parm/metplus/EnsembleStat_REFC.conf | 0 {ush/templates => parm}/parm/metplus/EnsembleStat_RETOP.conf | 0 {ush/templates => parm}/parm/metplus/EnsembleStat_conus_sfc.conf | 0 {ush/templates => parm}/parm/metplus/EnsembleStat_upper_air.conf | 0 {ush/templates => parm}/parm/metplus/GridStat_APCP01h.conf | 0 {ush/templates => parm}/parm/metplus/GridStat_APCP01h_mean.conf | 0 {ush/templates => parm}/parm/metplus/GridStat_APCP01h_prob.conf | 0 {ush/templates => parm}/parm/metplus/GridStat_APCP03h.conf | 0 {ush/templates => parm}/parm/metplus/GridStat_APCP03h_mean.conf | 0 {ush/templates => parm}/parm/metplus/GridStat_APCP03h_prob.conf | 0 {ush/templates => parm}/parm/metplus/GridStat_APCP06h.conf | 0 {ush/templates => parm}/parm/metplus/GridStat_APCP06h_mean.conf | 0 {ush/templates => parm}/parm/metplus/GridStat_APCP06h_prob.conf | 0 {ush/templates => parm}/parm/metplus/GridStat_APCP24h.conf | 0 {ush/templates => parm}/parm/metplus/GridStat_APCP24h_mean.conf | 0 {ush/templates => parm}/parm/metplus/GridStat_APCP24h_prob.conf | 0 {ush/templates => parm}/parm/metplus/GridStat_REFC.conf | 0 {ush/templates => parm}/parm/metplus/GridStat_REFC_mean.conf | 0 {ush/templates => parm}/parm/metplus/GridStat_REFC_prob.conf | 0 {ush/templates => parm}/parm/metplus/GridStat_RETOP.conf | 0 {ush/templates => parm}/parm/metplus/GridStat_RETOP_mean.conf | 0 {ush/templates => parm}/parm/metplus/GridStat_RETOP_prob.conf | 0 {ush/templates => parm}/parm/metplus/PointStat_conus_sfc.conf | 0 .../templates => parm}/parm/metplus/PointStat_conus_sfc_mean.conf | 0 .../templates => parm}/parm/metplus/PointStat_conus_sfc_prob.conf | 0 {ush/templates => parm}/parm/metplus/PointStat_upper_air.conf | 0 .../templates => parm}/parm/metplus/PointStat_upper_air_mean.conf | 0 .../templates => parm}/parm/metplus/PointStat_upper_air_prob.conf | 0 {ush/templates => parm}/parm/metplus/common.conf | 0 {ush/templates => parm}/regional_grid.nml | 0 58 files changed, 0 insertions(+), 0 deletions(-) rename {ush/templates => parm}/FV3.input.yml (100%) rename {ush/templates => parm}/FV3LAM_wflow.xml (100%) rename {ush/templates => parm}/README.xml_templating.md (100%) rename {ush/templates => parm}/data_locations.yml (100%) rename {ush/templates => parm}/data_table (100%) rename {ush/templates => parm}/diag_table.FV3_GFS_2017_gfdlmp (100%) rename {ush/templates => parm}/diag_table.FV3_GFS_2017_gfdlmp_regional (100%) rename {ush/templates => parm}/diag_table.FV3_GFS_v15_thompson_mynn_lam3km (100%) rename {ush/templates => parm}/diag_table.FV3_GFS_v15p2 (100%) rename {ush/templates => parm}/diag_table.FV3_GFS_v16 (100%) rename {ush/templates => parm}/diag_table.FV3_HRRR (100%) rename {ush/templates => parm}/diag_table.FV3_RRFS_v0 (100%) rename {ush/templates => parm}/diag_table.FV3_RRFS_v1beta (100%) rename {ush/templates => parm}/field_table.FV3_GFS_2017_gfdlmp (100%) rename {ush/templates => parm}/field_table.FV3_GFS_2017_gfdlmp_regional (100%) rename {ush/templates => parm}/field_table.FV3_GFS_v15_thompson_mynn_lam3km (100%) rename {ush/templates => parm}/field_table.FV3_GFS_v15p2 (100%) rename {ush/templates => parm}/field_table.FV3_GFS_v16 (100%) rename {ush/templates => parm}/field_table.FV3_HRRR (100%) rename {ush/templates => parm}/field_table.FV3_RRFS_v0 (100%) rename {ush/templates => parm}/field_table.FV3_RRFS_v1beta (100%) rename {ush/templates => parm}/input.nml.FV3 (100%) rename {ush/templates => parm}/model_configure (100%) rename {ush/templates => parm}/nems.configure (100%) rename {ush/templates => parm}/parm/metplus/EnsembleStat_APCP01h.conf (100%) rename {ush/templates => parm}/parm/metplus/EnsembleStat_APCP03h.conf (100%) rename {ush/templates => parm}/parm/metplus/EnsembleStat_APCP06h.conf (100%) rename {ush/templates => parm}/parm/metplus/EnsembleStat_APCP24h.conf (100%) rename {ush/templates => parm}/parm/metplus/EnsembleStat_REFC.conf (100%) rename {ush/templates => parm}/parm/metplus/EnsembleStat_RETOP.conf (100%) rename {ush/templates => parm}/parm/metplus/EnsembleStat_conus_sfc.conf (100%) rename {ush/templates => parm}/parm/metplus/EnsembleStat_upper_air.conf (100%) rename {ush/templates => parm}/parm/metplus/GridStat_APCP01h.conf (100%) rename {ush/templates => parm}/parm/metplus/GridStat_APCP01h_mean.conf (100%) rename {ush/templates => parm}/parm/metplus/GridStat_APCP01h_prob.conf (100%) rename {ush/templates => parm}/parm/metplus/GridStat_APCP03h.conf (100%) rename {ush/templates => parm}/parm/metplus/GridStat_APCP03h_mean.conf (100%) rename {ush/templates => parm}/parm/metplus/GridStat_APCP03h_prob.conf (100%) rename {ush/templates => parm}/parm/metplus/GridStat_APCP06h.conf (100%) rename {ush/templates => parm}/parm/metplus/GridStat_APCP06h_mean.conf (100%) rename {ush/templates => parm}/parm/metplus/GridStat_APCP06h_prob.conf (100%) rename {ush/templates => parm}/parm/metplus/GridStat_APCP24h.conf (100%) rename {ush/templates => parm}/parm/metplus/GridStat_APCP24h_mean.conf (100%) rename {ush/templates => parm}/parm/metplus/GridStat_APCP24h_prob.conf (100%) rename {ush/templates => parm}/parm/metplus/GridStat_REFC.conf (100%) rename {ush/templates => parm}/parm/metplus/GridStat_REFC_mean.conf (100%) rename {ush/templates => parm}/parm/metplus/GridStat_REFC_prob.conf (100%) rename {ush/templates => parm}/parm/metplus/GridStat_RETOP.conf (100%) rename {ush/templates => parm}/parm/metplus/GridStat_RETOP_mean.conf (100%) rename {ush/templates => parm}/parm/metplus/GridStat_RETOP_prob.conf (100%) rename {ush/templates => parm}/parm/metplus/PointStat_conus_sfc.conf (100%) rename {ush/templates => parm}/parm/metplus/PointStat_conus_sfc_mean.conf (100%) rename {ush/templates => parm}/parm/metplus/PointStat_conus_sfc_prob.conf (100%) rename {ush/templates => parm}/parm/metplus/PointStat_upper_air.conf (100%) rename {ush/templates => parm}/parm/metplus/PointStat_upper_air_mean.conf (100%) rename {ush/templates => parm}/parm/metplus/PointStat_upper_air_prob.conf (100%) rename {ush/templates => parm}/parm/metplus/common.conf (100%) rename {ush/templates => parm}/regional_grid.nml (100%) diff --git a/ush/templates/FV3.input.yml b/parm/FV3.input.yml similarity index 100% rename from ush/templates/FV3.input.yml rename to parm/FV3.input.yml diff --git a/ush/templates/FV3LAM_wflow.xml b/parm/FV3LAM_wflow.xml similarity index 100% rename from ush/templates/FV3LAM_wflow.xml rename to parm/FV3LAM_wflow.xml diff --git a/ush/templates/README.xml_templating.md b/parm/README.xml_templating.md similarity index 100% rename from ush/templates/README.xml_templating.md rename to parm/README.xml_templating.md diff --git a/ush/templates/data_locations.yml b/parm/data_locations.yml similarity index 100% rename from ush/templates/data_locations.yml rename to parm/data_locations.yml diff --git a/ush/templates/data_table b/parm/data_table similarity index 100% rename from ush/templates/data_table rename to parm/data_table diff --git a/ush/templates/diag_table.FV3_GFS_2017_gfdlmp b/parm/diag_table.FV3_GFS_2017_gfdlmp similarity index 100% rename from ush/templates/diag_table.FV3_GFS_2017_gfdlmp rename to parm/diag_table.FV3_GFS_2017_gfdlmp diff --git a/ush/templates/diag_table.FV3_GFS_2017_gfdlmp_regional b/parm/diag_table.FV3_GFS_2017_gfdlmp_regional similarity index 100% rename from ush/templates/diag_table.FV3_GFS_2017_gfdlmp_regional rename to parm/diag_table.FV3_GFS_2017_gfdlmp_regional diff --git a/ush/templates/diag_table.FV3_GFS_v15_thompson_mynn_lam3km b/parm/diag_table.FV3_GFS_v15_thompson_mynn_lam3km similarity index 100% rename from ush/templates/diag_table.FV3_GFS_v15_thompson_mynn_lam3km rename to parm/diag_table.FV3_GFS_v15_thompson_mynn_lam3km diff --git a/ush/templates/diag_table.FV3_GFS_v15p2 b/parm/diag_table.FV3_GFS_v15p2 similarity index 100% rename from ush/templates/diag_table.FV3_GFS_v15p2 rename to parm/diag_table.FV3_GFS_v15p2 diff --git a/ush/templates/diag_table.FV3_GFS_v16 b/parm/diag_table.FV3_GFS_v16 similarity index 100% rename from ush/templates/diag_table.FV3_GFS_v16 rename to parm/diag_table.FV3_GFS_v16 diff --git a/ush/templates/diag_table.FV3_HRRR b/parm/diag_table.FV3_HRRR similarity index 100% rename from ush/templates/diag_table.FV3_HRRR rename to parm/diag_table.FV3_HRRR diff --git a/ush/templates/diag_table.FV3_RRFS_v0 b/parm/diag_table.FV3_RRFS_v0 similarity index 100% rename from ush/templates/diag_table.FV3_RRFS_v0 rename to parm/diag_table.FV3_RRFS_v0 diff --git a/ush/templates/diag_table.FV3_RRFS_v1beta b/parm/diag_table.FV3_RRFS_v1beta similarity index 100% rename from ush/templates/diag_table.FV3_RRFS_v1beta rename to parm/diag_table.FV3_RRFS_v1beta diff --git a/ush/templates/field_table.FV3_GFS_2017_gfdlmp b/parm/field_table.FV3_GFS_2017_gfdlmp similarity index 100% rename from ush/templates/field_table.FV3_GFS_2017_gfdlmp rename to parm/field_table.FV3_GFS_2017_gfdlmp diff --git a/ush/templates/field_table.FV3_GFS_2017_gfdlmp_regional b/parm/field_table.FV3_GFS_2017_gfdlmp_regional similarity index 100% rename from ush/templates/field_table.FV3_GFS_2017_gfdlmp_regional rename to parm/field_table.FV3_GFS_2017_gfdlmp_regional diff --git a/ush/templates/field_table.FV3_GFS_v15_thompson_mynn_lam3km b/parm/field_table.FV3_GFS_v15_thompson_mynn_lam3km similarity index 100% rename from ush/templates/field_table.FV3_GFS_v15_thompson_mynn_lam3km rename to parm/field_table.FV3_GFS_v15_thompson_mynn_lam3km diff --git a/ush/templates/field_table.FV3_GFS_v15p2 b/parm/field_table.FV3_GFS_v15p2 similarity index 100% rename from ush/templates/field_table.FV3_GFS_v15p2 rename to parm/field_table.FV3_GFS_v15p2 diff --git a/ush/templates/field_table.FV3_GFS_v16 b/parm/field_table.FV3_GFS_v16 similarity index 100% rename from ush/templates/field_table.FV3_GFS_v16 rename to parm/field_table.FV3_GFS_v16 diff --git a/ush/templates/field_table.FV3_HRRR b/parm/field_table.FV3_HRRR similarity index 100% rename from ush/templates/field_table.FV3_HRRR rename to parm/field_table.FV3_HRRR diff --git a/ush/templates/field_table.FV3_RRFS_v0 b/parm/field_table.FV3_RRFS_v0 similarity index 100% rename from ush/templates/field_table.FV3_RRFS_v0 rename to parm/field_table.FV3_RRFS_v0 diff --git a/ush/templates/field_table.FV3_RRFS_v1beta b/parm/field_table.FV3_RRFS_v1beta similarity index 100% rename from ush/templates/field_table.FV3_RRFS_v1beta rename to parm/field_table.FV3_RRFS_v1beta diff --git a/ush/templates/input.nml.FV3 b/parm/input.nml.FV3 similarity index 100% rename from ush/templates/input.nml.FV3 rename to parm/input.nml.FV3 diff --git a/ush/templates/model_configure b/parm/model_configure similarity index 100% rename from ush/templates/model_configure rename to parm/model_configure diff --git a/ush/templates/nems.configure b/parm/nems.configure similarity index 100% rename from ush/templates/nems.configure rename to parm/nems.configure diff --git a/ush/templates/parm/metplus/EnsembleStat_APCP01h.conf b/parm/parm/metplus/EnsembleStat_APCP01h.conf similarity index 100% rename from ush/templates/parm/metplus/EnsembleStat_APCP01h.conf rename to parm/parm/metplus/EnsembleStat_APCP01h.conf diff --git a/ush/templates/parm/metplus/EnsembleStat_APCP03h.conf b/parm/parm/metplus/EnsembleStat_APCP03h.conf similarity index 100% rename from ush/templates/parm/metplus/EnsembleStat_APCP03h.conf rename to parm/parm/metplus/EnsembleStat_APCP03h.conf diff --git a/ush/templates/parm/metplus/EnsembleStat_APCP06h.conf b/parm/parm/metplus/EnsembleStat_APCP06h.conf similarity index 100% rename from ush/templates/parm/metplus/EnsembleStat_APCP06h.conf rename to parm/parm/metplus/EnsembleStat_APCP06h.conf diff --git a/ush/templates/parm/metplus/EnsembleStat_APCP24h.conf b/parm/parm/metplus/EnsembleStat_APCP24h.conf similarity index 100% rename from ush/templates/parm/metplus/EnsembleStat_APCP24h.conf rename to parm/parm/metplus/EnsembleStat_APCP24h.conf diff --git a/ush/templates/parm/metplus/EnsembleStat_REFC.conf b/parm/parm/metplus/EnsembleStat_REFC.conf similarity index 100% rename from ush/templates/parm/metplus/EnsembleStat_REFC.conf rename to parm/parm/metplus/EnsembleStat_REFC.conf diff --git a/ush/templates/parm/metplus/EnsembleStat_RETOP.conf b/parm/parm/metplus/EnsembleStat_RETOP.conf similarity index 100% rename from ush/templates/parm/metplus/EnsembleStat_RETOP.conf rename to parm/parm/metplus/EnsembleStat_RETOP.conf diff --git a/ush/templates/parm/metplus/EnsembleStat_conus_sfc.conf b/parm/parm/metplus/EnsembleStat_conus_sfc.conf similarity index 100% rename from ush/templates/parm/metplus/EnsembleStat_conus_sfc.conf rename to parm/parm/metplus/EnsembleStat_conus_sfc.conf diff --git a/ush/templates/parm/metplus/EnsembleStat_upper_air.conf b/parm/parm/metplus/EnsembleStat_upper_air.conf similarity index 100% rename from ush/templates/parm/metplus/EnsembleStat_upper_air.conf rename to parm/parm/metplus/EnsembleStat_upper_air.conf diff --git a/ush/templates/parm/metplus/GridStat_APCP01h.conf b/parm/parm/metplus/GridStat_APCP01h.conf similarity index 100% rename from ush/templates/parm/metplus/GridStat_APCP01h.conf rename to parm/parm/metplus/GridStat_APCP01h.conf diff --git a/ush/templates/parm/metplus/GridStat_APCP01h_mean.conf b/parm/parm/metplus/GridStat_APCP01h_mean.conf similarity index 100% rename from ush/templates/parm/metplus/GridStat_APCP01h_mean.conf rename to parm/parm/metplus/GridStat_APCP01h_mean.conf diff --git a/ush/templates/parm/metplus/GridStat_APCP01h_prob.conf b/parm/parm/metplus/GridStat_APCP01h_prob.conf similarity index 100% rename from ush/templates/parm/metplus/GridStat_APCP01h_prob.conf rename to parm/parm/metplus/GridStat_APCP01h_prob.conf diff --git a/ush/templates/parm/metplus/GridStat_APCP03h.conf b/parm/parm/metplus/GridStat_APCP03h.conf similarity index 100% rename from ush/templates/parm/metplus/GridStat_APCP03h.conf rename to parm/parm/metplus/GridStat_APCP03h.conf diff --git a/ush/templates/parm/metplus/GridStat_APCP03h_mean.conf b/parm/parm/metplus/GridStat_APCP03h_mean.conf similarity index 100% rename from ush/templates/parm/metplus/GridStat_APCP03h_mean.conf rename to parm/parm/metplus/GridStat_APCP03h_mean.conf diff --git a/ush/templates/parm/metplus/GridStat_APCP03h_prob.conf b/parm/parm/metplus/GridStat_APCP03h_prob.conf similarity index 100% rename from ush/templates/parm/metplus/GridStat_APCP03h_prob.conf rename to parm/parm/metplus/GridStat_APCP03h_prob.conf diff --git a/ush/templates/parm/metplus/GridStat_APCP06h.conf b/parm/parm/metplus/GridStat_APCP06h.conf similarity index 100% rename from ush/templates/parm/metplus/GridStat_APCP06h.conf rename to parm/parm/metplus/GridStat_APCP06h.conf diff --git a/ush/templates/parm/metplus/GridStat_APCP06h_mean.conf b/parm/parm/metplus/GridStat_APCP06h_mean.conf similarity index 100% rename from ush/templates/parm/metplus/GridStat_APCP06h_mean.conf rename to parm/parm/metplus/GridStat_APCP06h_mean.conf diff --git a/ush/templates/parm/metplus/GridStat_APCP06h_prob.conf b/parm/parm/metplus/GridStat_APCP06h_prob.conf similarity index 100% rename from ush/templates/parm/metplus/GridStat_APCP06h_prob.conf rename to parm/parm/metplus/GridStat_APCP06h_prob.conf diff --git a/ush/templates/parm/metplus/GridStat_APCP24h.conf b/parm/parm/metplus/GridStat_APCP24h.conf similarity index 100% rename from ush/templates/parm/metplus/GridStat_APCP24h.conf rename to parm/parm/metplus/GridStat_APCP24h.conf diff --git a/ush/templates/parm/metplus/GridStat_APCP24h_mean.conf b/parm/parm/metplus/GridStat_APCP24h_mean.conf similarity index 100% rename from ush/templates/parm/metplus/GridStat_APCP24h_mean.conf rename to parm/parm/metplus/GridStat_APCP24h_mean.conf diff --git a/ush/templates/parm/metplus/GridStat_APCP24h_prob.conf b/parm/parm/metplus/GridStat_APCP24h_prob.conf similarity index 100% rename from ush/templates/parm/metplus/GridStat_APCP24h_prob.conf rename to parm/parm/metplus/GridStat_APCP24h_prob.conf diff --git a/ush/templates/parm/metplus/GridStat_REFC.conf b/parm/parm/metplus/GridStat_REFC.conf similarity index 100% rename from ush/templates/parm/metplus/GridStat_REFC.conf rename to parm/parm/metplus/GridStat_REFC.conf diff --git a/ush/templates/parm/metplus/GridStat_REFC_mean.conf b/parm/parm/metplus/GridStat_REFC_mean.conf similarity index 100% rename from ush/templates/parm/metplus/GridStat_REFC_mean.conf rename to parm/parm/metplus/GridStat_REFC_mean.conf diff --git a/ush/templates/parm/metplus/GridStat_REFC_prob.conf b/parm/parm/metplus/GridStat_REFC_prob.conf similarity index 100% rename from ush/templates/parm/metplus/GridStat_REFC_prob.conf rename to parm/parm/metplus/GridStat_REFC_prob.conf diff --git a/ush/templates/parm/metplus/GridStat_RETOP.conf b/parm/parm/metplus/GridStat_RETOP.conf similarity index 100% rename from ush/templates/parm/metplus/GridStat_RETOP.conf rename to parm/parm/metplus/GridStat_RETOP.conf diff --git a/ush/templates/parm/metplus/GridStat_RETOP_mean.conf b/parm/parm/metplus/GridStat_RETOP_mean.conf similarity index 100% rename from ush/templates/parm/metplus/GridStat_RETOP_mean.conf rename to parm/parm/metplus/GridStat_RETOP_mean.conf diff --git a/ush/templates/parm/metplus/GridStat_RETOP_prob.conf b/parm/parm/metplus/GridStat_RETOP_prob.conf similarity index 100% rename from ush/templates/parm/metplus/GridStat_RETOP_prob.conf rename to parm/parm/metplus/GridStat_RETOP_prob.conf diff --git a/ush/templates/parm/metplus/PointStat_conus_sfc.conf b/parm/parm/metplus/PointStat_conus_sfc.conf similarity index 100% rename from ush/templates/parm/metplus/PointStat_conus_sfc.conf rename to parm/parm/metplus/PointStat_conus_sfc.conf diff --git a/ush/templates/parm/metplus/PointStat_conus_sfc_mean.conf b/parm/parm/metplus/PointStat_conus_sfc_mean.conf similarity index 100% rename from ush/templates/parm/metplus/PointStat_conus_sfc_mean.conf rename to parm/parm/metplus/PointStat_conus_sfc_mean.conf diff --git a/ush/templates/parm/metplus/PointStat_conus_sfc_prob.conf b/parm/parm/metplus/PointStat_conus_sfc_prob.conf similarity index 100% rename from ush/templates/parm/metplus/PointStat_conus_sfc_prob.conf rename to parm/parm/metplus/PointStat_conus_sfc_prob.conf diff --git a/ush/templates/parm/metplus/PointStat_upper_air.conf b/parm/parm/metplus/PointStat_upper_air.conf similarity index 100% rename from ush/templates/parm/metplus/PointStat_upper_air.conf rename to parm/parm/metplus/PointStat_upper_air.conf diff --git a/ush/templates/parm/metplus/PointStat_upper_air_mean.conf b/parm/parm/metplus/PointStat_upper_air_mean.conf similarity index 100% rename from ush/templates/parm/metplus/PointStat_upper_air_mean.conf rename to parm/parm/metplus/PointStat_upper_air_mean.conf diff --git a/ush/templates/parm/metplus/PointStat_upper_air_prob.conf b/parm/parm/metplus/PointStat_upper_air_prob.conf similarity index 100% rename from ush/templates/parm/metplus/PointStat_upper_air_prob.conf rename to parm/parm/metplus/PointStat_upper_air_prob.conf diff --git a/ush/templates/parm/metplus/common.conf b/parm/parm/metplus/common.conf similarity index 100% rename from ush/templates/parm/metplus/common.conf rename to parm/parm/metplus/common.conf diff --git a/ush/templates/regional_grid.nml b/parm/regional_grid.nml similarity index 100% rename from ush/templates/regional_grid.nml rename to parm/regional_grid.nml From 5ac662ddda018a78500400a501c796026119561f Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Mon, 29 Aug 2022 12:55:12 +0000 Subject: [PATCH 052/106] Fix for new parm location. --- .github/workflows/python_func_tests.yaml | 2 +- parm/FV3.input.yml | 2 +- parm/README.xml_templating.md | 2 +- scripts/exregional_get_extrn_mdl_files.sh | 2 +- ush/create_diag_table_file.py | 2 +- ush/generate_FV3LAM_wflow.py | 2 +- ush/retrieve_data.py | 6 +++--- ush/set_FV3nml_ens_stoch_seeds.py | 2 +- ush/set_FV3nml_sfc_climo_filenames.py | 2 +- ush/set_namelist.py | 6 +++--- ush/setup.py | 26 +++++++++++------------ ush/test_retrieve_data.py | 2 +- 12 files changed, 28 insertions(+), 28 deletions(-) diff --git a/.github/workflows/python_func_tests.yaml b/.github/workflows/python_func_tests.yaml index 80a3f191fc..c35291fe11 100644 --- a/.github/workflows/python_func_tests.yaml +++ b/.github/workflows/python_func_tests.yaml @@ -3,7 +3,7 @@ on: push: paths: - ush/*retrieve_data.py - - ush/templates/data_locations.yml + - parm/data_locations.yml pull_request: env: diff --git a/parm/FV3.input.yml b/parm/FV3.input.yml index 0e9276de25..1066d43971 100644 --- a/parm/FV3.input.yml +++ b/parm/FV3.input.yml @@ -1,7 +1,7 @@ # This configuration file maintains the modifications that need to be # made to the base FV3 namelist specified in # -# ush/templates/input.nml.FV3 +# parm/input.nml.FV3 # # to obtain the namelist for each physics suite that the SRW App can # run with. To build a namelist for one of these configurations, use diff --git a/parm/README.xml_templating.md b/parm/README.xml_templating.md index 0d2c58cf1b..f446e09d65 100644 --- a/parm/README.xml_templating.md +++ b/parm/README.xml_templating.md @@ -121,7 +121,7 @@ These all work together like this: In `ush/generate_workflow.sh`, the `run_task_make_grid` flag is set to "False" to turn off running verification. -In `ush/templates/FV3LAM_wflow.xml`: +In `parm/FV3LAM_wflow.xml`: 1. Set an entity for use in dependencies. 2. Add a Jinja if statement around the optional task. diff --git a/scripts/exregional_get_extrn_mdl_files.sh b/scripts/exregional_get_extrn_mdl_files.sh index 5f4a93f601..c2d5ffc866 100755 --- a/scripts/exregional_get_extrn_mdl_files.sh +++ b/scripts/exregional_get_extrn_mdl_files.sh @@ -134,7 +134,7 @@ cmd=" python3 -u ${USHrrfs}/retrieve_data.py \ --debug \ --anl_or_fcst ${anl_or_fcst} \ - --config ${USHrrfs}/templates/data_locations.yml \ + --config ${PARMrrfs}/data_locations.yml \ --cycle_date ${EXTRN_MDL_CDATE} \ --data_stores ${data_stores} \ --external_model ${EXTRN_MDL_NAME} \ diff --git a/ush/create_diag_table_file.py b/ush/create_diag_table_file.py index 9b0a341c7a..616bd4f1b7 100644 --- a/ush/create_diag_table_file.py +++ b/ush/create_diag_table_file.py @@ -133,7 +133,7 @@ def setUp(self): USHrrfs = os.path.dirname(os.path.abspath(__file__)) DIAG_TABLE_FN = "diag_table" DIAG_TABLE_TMPL_FP = os.path.join( - USHrrfs, "templates", f"{DIAG_TABLE_FN}.FV3_GFS_v15p2" + PARMrrfs, f"{DIAG_TABLE_FN}.FV3_GFS_v15p2" ) set_env_var("DEBUG", True) set_env_var("VERBOSE", True) diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index 8fd55afd11..25359eb290 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -138,7 +138,7 @@ def generate_FV3LAM_wflow(): # if WORKFLOW_MANAGER == "rocoto": - template_xml_fp = os.path.join(TEMPLATE_DIR, WFLOW_XML_FN) + template_xml_fp = os.path.join(PARMrrfs, WFLOW_XML_FN) print_info_msg( f''' diff --git a/ush/retrieve_data.py b/ush/retrieve_data.py index 2792e7ba28..fbcadf2686 100755 --- a/ush/retrieve_data.py +++ b/ush/retrieve_data.py @@ -6,7 +6,7 @@ data locations on disk. Several supported data streams are included in -ush/templates/data_locations.yml, which provides locations and naming +parm/data_locations.yml, which provides locations and naming conventions for files commonly used with the SRW App. Provide the file to this tool via the --config flag. Users are welcome to provide their own file with alternative locations and naming conventions. @@ -16,7 +16,7 @@ templates. The file names follow those included in the --config file by default, or can be user-supplied via the --file_name flag. That flag takes a YAML-formatted string that follows the same conventions outlined -in the ush/templates/data_locations.yml file for naming files. +in the parm/data_locations.yml file for naming files. To see usage for this script: @@ -899,7 +899,7 @@ def parse_args(argv): "--config", help="Full path to a configuration file containing paths and \ naming conventions for known data streams. The default included \ - in this repository is in ush/templates/data_locations.yml", + in this repository is in parm/data_locations.yml", type=config_exists, ) parser.add_argument( diff --git a/ush/set_FV3nml_ens_stoch_seeds.py b/ush/set_FV3nml_ens_stoch_seeds.py index 80818fd4b0..51be55add2 100644 --- a/ush/set_FV3nml_ens_stoch_seeds.py +++ b/ush/set_FV3nml_ens_stoch_seeds.py @@ -175,7 +175,7 @@ def setUp(self): USHrrfs = os.path.dirname(os.path.abspath(__file__)) EXPTDIR = os.path.join(USHrrfs, "test_data", "expt") cp_vrfy( - os.path.join(USHrrfs, f"templates{os.sep}input.nml.FV3"), + os.path.join(PARMrrfs, "input.nml.FV3"), os.path.join(EXPTDIR, "input.nml"), ) for i in range(2): diff --git a/ush/set_FV3nml_sfc_climo_filenames.py b/ush/set_FV3nml_sfc_climo_filenames.py index 33086f8c35..b9db5baf81 100644 --- a/ush/set_FV3nml_sfc_climo_filenames.py +++ b/ush/set_FV3nml_sfc_climo_filenames.py @@ -154,7 +154,7 @@ def setUp(self): FIXLAM = os.path.join(EXPTDIR, "fix_lam") mkdir_vrfy("-p", FIXLAM) cp_vrfy( - os.path.join(USHrrfs, f"templates{os.sep}input.nml.FV3"), + os.path.join(USHrrfs, "input.nml.FV3"), os.path.join(EXPTDIR, "input.nml"), ) set_env_var("USHrrfs", USHrrfs) diff --git a/ush/set_namelist.py b/ush/set_namelist.py index 90594dff98..e578d3201f 100755 --- a/ush/set_namelist.py +++ b/ush/set_namelist.py @@ -27,7 +27,7 @@ The optional base configuration file (provided via the -c command line argument) contains the known set of configurations used and supported by the community, if -using the one provided in ush/templates/FV3.input.yml. If maintaining this file +using the one provided in parm/FV3.input.yml. If maintaining this file for a different set of configurations, ensure that the heirarchy is such that it names the configuration at the top level (section), and the subsequent sections match those in the F90 namelist that will be updated. @@ -40,7 +40,7 @@ To produce a namelist (fv3_expt.nml) by specifying a physics package: - set_namelist.py -n templates/input.nml.FV3 -c templates/FV3.input.yml FV3_HRRR + set_namelist.py -n ../parm/input.nml.FV3 -c ../parm/FV3.input.yml FV3_HRRR -o fv3_expt.nml To produce a YAML file (fv3_namelist.yml) from a user namelist: @@ -49,7 +49,7 @@ To produce a YAML file (fv3_my_namelist.yml) with differences from base nml: - set_namelist.py -n templates/input.nml.FV3 -i my_namelist.nml -t yaml + set_namelist.py -n ../parm/input.nml.FV3 -i my_namelist.nml -t yaml -o fv3_my_namelist.nml Expected behavior: diff --git a/ush/setup.py b/ush/setup.py index eb0be1924e..26567e94c1 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -349,7 +349,7 @@ def setup(): # Define some other useful paths # global USHrrfs, SCRIPTSDIR, JOBSDIR, SORCrrfs, PARMrrfs, MODULESrrfs - global EXECrrfs, TEMPLATE_DIR, VX_CONFIG_DIR, METPLUS_CONF, MET_CONFIG + global EXECrrfs, PARMrrfs, VX_CONFIG_DIR, METPLUS_CONF, MET_CONFIG USHrrfs = os.path.join(HOMErrfs, "ush") SCRIPTSDIR = os.path.join(HOMErrfs, "scripts") @@ -358,10 +358,10 @@ def setup(): PARMrrfs = os.path.join(HOMErrfs, "parm") MODULESrrfs = os.path.join(HOMErrfs, "modulefiles") EXECrrfs = os.path.join(SR_WX_APP_TOP_DIR, EXEC_SUBDIR) - TEMPLATE_DIR = os.path.join(USHrrfs, "templates") - VX_CONFIG_DIR = os.path.join(TEMPLATE_DIR, "parm") - METPLUS_CONF = os.path.join(TEMPLATE_DIR, "parm", "metplus") - MET_CONFIG = os.path.join(TEMPLATE_DIR, "parm", "met") + PARMrrfs = os.path.join(HOMErrfs, "parm") + VX_CONFIG_DIR = os.path.join(PARMrrfs, "parm") + METPLUS_CONF = os.path.join(PARMrrfs, "parm", "metplus") + MET_CONFIG = os.path.join(PARMrrfs, "parm", "met") # # ----------------------------------------------------------------------- @@ -1050,14 +1050,14 @@ def setup(): MODEL_CONFIG_TMPL_FN = MODEL_CONFIG_TMPL_FN or MODEL_CONFIG_FN NEMS_CONFIG_TMPL_FN = NEMS_CONFIG_TMPL_FN or NEMS_CONFIG_FN - DATA_TABLE_TMPL_FP = os.path.join(TEMPLATE_DIR, DATA_TABLE_TMPL_FN) - DIAG_TABLE_TMPL_FP = os.path.join(TEMPLATE_DIR, DIAG_TABLE_TMPL_FN) - FIELD_TABLE_TMPL_FP = os.path.join(TEMPLATE_DIR, FIELD_TABLE_TMPL_FN) - FV3_NML_BASE_SUITE_FP = os.path.join(TEMPLATE_DIR, FV3_NML_BASE_SUITE_FN) - FV3_NML_YAML_CONFIG_FP = os.path.join(TEMPLATE_DIR, FV3_NML_YAML_CONFIG_FN) + DATA_TABLE_TMPL_FP = os.path.join(PARMrrfs, DATA_TABLE_TMPL_FN) + DIAG_TABLE_TMPL_FP = os.path.join(PARMrrfs, DIAG_TABLE_TMPL_FN) + FIELD_TABLE_TMPL_FP = os.path.join(PARMrrfs, FIELD_TABLE_TMPL_FN) + FV3_NML_BASE_SUITE_FP = os.path.join(PARMrrfs, FV3_NML_BASE_SUITE_FN) + FV3_NML_YAML_CONFIG_FP = os.path.join(PARMrrfs, FV3_NML_YAML_CONFIG_FN) FV3_NML_BASE_ENS_FP = os.path.join(EXPTDIR, FV3_NML_BASE_ENS_FN) - MODEL_CONFIG_TMPL_FP = os.path.join(TEMPLATE_DIR, MODEL_CONFIG_TMPL_FN) - NEMS_CONFIG_TMPL_FP = os.path.join(TEMPLATE_DIR, NEMS_CONFIG_TMPL_FN) + MODEL_CONFIG_TMPL_FP = os.path.join(PARMrrfs, MODEL_CONFIG_TMPL_FN) + NEMS_CONFIG_TMPL_FP = os.path.join(PARMrrfs, NEMS_CONFIG_TMPL_FN) # # ----------------------------------------------------------------------- # @@ -1924,7 +1924,7 @@ def setup(): "FIXgsm": FIXgsm, "FIXaer": FIXaer, "FIXlut": FIXlut, - "TEMPLATE_DIR": TEMPLATE_DIR, + "PARMrrfs": PARMrrfs, "VX_CONFIG_DIR": VX_CONFIG_DIR, "METPLUS_CONF": METPLUS_CONF, "MET_CONFIG": MET_CONFIG, diff --git a/ush/test_retrieve_data.py b/ush/test_retrieve_data.py index 78cc0e8cca..b2ef60503b 100644 --- a/ush/test_retrieve_data.py +++ b/ush/test_retrieve_data.py @@ -28,7 +28,7 @@ class FunctionalTesting(unittest.TestCase): def setUp(self): self.path = os.path.dirname(__file__) - self.config = f'{self.path}/templates/data_locations.yml' + self.config = f'{self.path}/../parm/data_locations.yml' @unittest.skipIf(os.environ.get('CI') == "true", "Skipping HPSS tests") def test_fv3gfs_grib2_lbcs_from_hpss(self): From 0e7afbf837fcf3b00361af33d5fc6f75ccbd84d7 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Mon, 29 Aug 2022 13:38:35 +0000 Subject: [PATCH 053/106] Move metplus one level up. --- parm/{parm => }/metplus/EnsembleStat_APCP01h.conf | 0 parm/{parm => }/metplus/EnsembleStat_APCP03h.conf | 0 parm/{parm => }/metplus/EnsembleStat_APCP06h.conf | 0 parm/{parm => }/metplus/EnsembleStat_APCP24h.conf | 0 parm/{parm => }/metplus/EnsembleStat_REFC.conf | 0 parm/{parm => }/metplus/EnsembleStat_RETOP.conf | 0 parm/{parm => }/metplus/EnsembleStat_conus_sfc.conf | 0 parm/{parm => }/metplus/EnsembleStat_upper_air.conf | 0 parm/{parm => }/metplus/GridStat_APCP01h.conf | 0 parm/{parm => }/metplus/GridStat_APCP01h_mean.conf | 0 parm/{parm => }/metplus/GridStat_APCP01h_prob.conf | 0 parm/{parm => }/metplus/GridStat_APCP03h.conf | 0 parm/{parm => }/metplus/GridStat_APCP03h_mean.conf | 0 parm/{parm => }/metplus/GridStat_APCP03h_prob.conf | 0 parm/{parm => }/metplus/GridStat_APCP06h.conf | 0 parm/{parm => }/metplus/GridStat_APCP06h_mean.conf | 0 parm/{parm => }/metplus/GridStat_APCP06h_prob.conf | 0 parm/{parm => }/metplus/GridStat_APCP24h.conf | 0 parm/{parm => }/metplus/GridStat_APCP24h_mean.conf | 0 parm/{parm => }/metplus/GridStat_APCP24h_prob.conf | 0 parm/{parm => }/metplus/GridStat_REFC.conf | 0 parm/{parm => }/metplus/GridStat_REFC_mean.conf | 0 parm/{parm => }/metplus/GridStat_REFC_prob.conf | 0 parm/{parm => }/metplus/GridStat_RETOP.conf | 0 parm/{parm => }/metplus/GridStat_RETOP_mean.conf | 0 parm/{parm => }/metplus/GridStat_RETOP_prob.conf | 0 parm/{parm => }/metplus/PointStat_conus_sfc.conf | 0 parm/{parm => }/metplus/PointStat_conus_sfc_mean.conf | 0 parm/{parm => }/metplus/PointStat_conus_sfc_prob.conf | 0 parm/{parm => }/metplus/PointStat_upper_air.conf | 0 parm/{parm => }/metplus/PointStat_upper_air_mean.conf | 0 parm/{parm => }/metplus/PointStat_upper_air_prob.conf | 0 parm/{parm => }/metplus/common.conf | 0 ush/create_model_configure_file.py | 2 +- ush/set_FV3nml_sfc_climo_filenames.py | 2 +- ush/setup.py | 6 +++--- 36 files changed, 5 insertions(+), 5 deletions(-) rename parm/{parm => }/metplus/EnsembleStat_APCP01h.conf (100%) rename parm/{parm => }/metplus/EnsembleStat_APCP03h.conf (100%) rename parm/{parm => }/metplus/EnsembleStat_APCP06h.conf (100%) rename parm/{parm => }/metplus/EnsembleStat_APCP24h.conf (100%) rename parm/{parm => }/metplus/EnsembleStat_REFC.conf (100%) rename parm/{parm => }/metplus/EnsembleStat_RETOP.conf (100%) rename parm/{parm => }/metplus/EnsembleStat_conus_sfc.conf (100%) rename parm/{parm => }/metplus/EnsembleStat_upper_air.conf (100%) rename parm/{parm => }/metplus/GridStat_APCP01h.conf (100%) rename parm/{parm => }/metplus/GridStat_APCP01h_mean.conf (100%) rename parm/{parm => }/metplus/GridStat_APCP01h_prob.conf (100%) rename parm/{parm => }/metplus/GridStat_APCP03h.conf (100%) rename parm/{parm => }/metplus/GridStat_APCP03h_mean.conf (100%) rename parm/{parm => }/metplus/GridStat_APCP03h_prob.conf (100%) rename parm/{parm => }/metplus/GridStat_APCP06h.conf (100%) rename parm/{parm => }/metplus/GridStat_APCP06h_mean.conf (100%) rename parm/{parm => }/metplus/GridStat_APCP06h_prob.conf (100%) rename parm/{parm => }/metplus/GridStat_APCP24h.conf (100%) rename parm/{parm => }/metplus/GridStat_APCP24h_mean.conf (100%) rename parm/{parm => }/metplus/GridStat_APCP24h_prob.conf (100%) rename parm/{parm => }/metplus/GridStat_REFC.conf (100%) rename parm/{parm => }/metplus/GridStat_REFC_mean.conf (100%) rename parm/{parm => }/metplus/GridStat_REFC_prob.conf (100%) rename parm/{parm => }/metplus/GridStat_RETOP.conf (100%) rename parm/{parm => }/metplus/GridStat_RETOP_mean.conf (100%) rename parm/{parm => }/metplus/GridStat_RETOP_prob.conf (100%) rename parm/{parm => }/metplus/PointStat_conus_sfc.conf (100%) rename parm/{parm => }/metplus/PointStat_conus_sfc_mean.conf (100%) rename parm/{parm => }/metplus/PointStat_conus_sfc_prob.conf (100%) rename parm/{parm => }/metplus/PointStat_upper_air.conf (100%) rename parm/{parm => }/metplus/PointStat_upper_air_mean.conf (100%) rename parm/{parm => }/metplus/PointStat_upper_air_prob.conf (100%) rename parm/{parm => }/metplus/common.conf (100%) diff --git a/parm/parm/metplus/EnsembleStat_APCP01h.conf b/parm/metplus/EnsembleStat_APCP01h.conf similarity index 100% rename from parm/parm/metplus/EnsembleStat_APCP01h.conf rename to parm/metplus/EnsembleStat_APCP01h.conf diff --git a/parm/parm/metplus/EnsembleStat_APCP03h.conf b/parm/metplus/EnsembleStat_APCP03h.conf similarity index 100% rename from parm/parm/metplus/EnsembleStat_APCP03h.conf rename to parm/metplus/EnsembleStat_APCP03h.conf diff --git a/parm/parm/metplus/EnsembleStat_APCP06h.conf b/parm/metplus/EnsembleStat_APCP06h.conf similarity index 100% rename from parm/parm/metplus/EnsembleStat_APCP06h.conf rename to parm/metplus/EnsembleStat_APCP06h.conf diff --git a/parm/parm/metplus/EnsembleStat_APCP24h.conf b/parm/metplus/EnsembleStat_APCP24h.conf similarity index 100% rename from parm/parm/metplus/EnsembleStat_APCP24h.conf rename to parm/metplus/EnsembleStat_APCP24h.conf diff --git a/parm/parm/metplus/EnsembleStat_REFC.conf b/parm/metplus/EnsembleStat_REFC.conf similarity index 100% rename from parm/parm/metplus/EnsembleStat_REFC.conf rename to parm/metplus/EnsembleStat_REFC.conf diff --git a/parm/parm/metplus/EnsembleStat_RETOP.conf b/parm/metplus/EnsembleStat_RETOP.conf similarity index 100% rename from parm/parm/metplus/EnsembleStat_RETOP.conf rename to parm/metplus/EnsembleStat_RETOP.conf diff --git a/parm/parm/metplus/EnsembleStat_conus_sfc.conf b/parm/metplus/EnsembleStat_conus_sfc.conf similarity index 100% rename from parm/parm/metplus/EnsembleStat_conus_sfc.conf rename to parm/metplus/EnsembleStat_conus_sfc.conf diff --git a/parm/parm/metplus/EnsembleStat_upper_air.conf b/parm/metplus/EnsembleStat_upper_air.conf similarity index 100% rename from parm/parm/metplus/EnsembleStat_upper_air.conf rename to parm/metplus/EnsembleStat_upper_air.conf diff --git a/parm/parm/metplus/GridStat_APCP01h.conf b/parm/metplus/GridStat_APCP01h.conf similarity index 100% rename from parm/parm/metplus/GridStat_APCP01h.conf rename to parm/metplus/GridStat_APCP01h.conf diff --git a/parm/parm/metplus/GridStat_APCP01h_mean.conf b/parm/metplus/GridStat_APCP01h_mean.conf similarity index 100% rename from parm/parm/metplus/GridStat_APCP01h_mean.conf rename to parm/metplus/GridStat_APCP01h_mean.conf diff --git a/parm/parm/metplus/GridStat_APCP01h_prob.conf b/parm/metplus/GridStat_APCP01h_prob.conf similarity index 100% rename from parm/parm/metplus/GridStat_APCP01h_prob.conf rename to parm/metplus/GridStat_APCP01h_prob.conf diff --git a/parm/parm/metplus/GridStat_APCP03h.conf b/parm/metplus/GridStat_APCP03h.conf similarity index 100% rename from parm/parm/metplus/GridStat_APCP03h.conf rename to parm/metplus/GridStat_APCP03h.conf diff --git a/parm/parm/metplus/GridStat_APCP03h_mean.conf b/parm/metplus/GridStat_APCP03h_mean.conf similarity index 100% rename from parm/parm/metplus/GridStat_APCP03h_mean.conf rename to parm/metplus/GridStat_APCP03h_mean.conf diff --git a/parm/parm/metplus/GridStat_APCP03h_prob.conf b/parm/metplus/GridStat_APCP03h_prob.conf similarity index 100% rename from parm/parm/metplus/GridStat_APCP03h_prob.conf rename to parm/metplus/GridStat_APCP03h_prob.conf diff --git a/parm/parm/metplus/GridStat_APCP06h.conf b/parm/metplus/GridStat_APCP06h.conf similarity index 100% rename from parm/parm/metplus/GridStat_APCP06h.conf rename to parm/metplus/GridStat_APCP06h.conf diff --git a/parm/parm/metplus/GridStat_APCP06h_mean.conf b/parm/metplus/GridStat_APCP06h_mean.conf similarity index 100% rename from parm/parm/metplus/GridStat_APCP06h_mean.conf rename to parm/metplus/GridStat_APCP06h_mean.conf diff --git a/parm/parm/metplus/GridStat_APCP06h_prob.conf b/parm/metplus/GridStat_APCP06h_prob.conf similarity index 100% rename from parm/parm/metplus/GridStat_APCP06h_prob.conf rename to parm/metplus/GridStat_APCP06h_prob.conf diff --git a/parm/parm/metplus/GridStat_APCP24h.conf b/parm/metplus/GridStat_APCP24h.conf similarity index 100% rename from parm/parm/metplus/GridStat_APCP24h.conf rename to parm/metplus/GridStat_APCP24h.conf diff --git a/parm/parm/metplus/GridStat_APCP24h_mean.conf b/parm/metplus/GridStat_APCP24h_mean.conf similarity index 100% rename from parm/parm/metplus/GridStat_APCP24h_mean.conf rename to parm/metplus/GridStat_APCP24h_mean.conf diff --git a/parm/parm/metplus/GridStat_APCP24h_prob.conf b/parm/metplus/GridStat_APCP24h_prob.conf similarity index 100% rename from parm/parm/metplus/GridStat_APCP24h_prob.conf rename to parm/metplus/GridStat_APCP24h_prob.conf diff --git a/parm/parm/metplus/GridStat_REFC.conf b/parm/metplus/GridStat_REFC.conf similarity index 100% rename from parm/parm/metplus/GridStat_REFC.conf rename to parm/metplus/GridStat_REFC.conf diff --git a/parm/parm/metplus/GridStat_REFC_mean.conf b/parm/metplus/GridStat_REFC_mean.conf similarity index 100% rename from parm/parm/metplus/GridStat_REFC_mean.conf rename to parm/metplus/GridStat_REFC_mean.conf diff --git a/parm/parm/metplus/GridStat_REFC_prob.conf b/parm/metplus/GridStat_REFC_prob.conf similarity index 100% rename from parm/parm/metplus/GridStat_REFC_prob.conf rename to parm/metplus/GridStat_REFC_prob.conf diff --git a/parm/parm/metplus/GridStat_RETOP.conf b/parm/metplus/GridStat_RETOP.conf similarity index 100% rename from parm/parm/metplus/GridStat_RETOP.conf rename to parm/metplus/GridStat_RETOP.conf diff --git a/parm/parm/metplus/GridStat_RETOP_mean.conf b/parm/metplus/GridStat_RETOP_mean.conf similarity index 100% rename from parm/parm/metplus/GridStat_RETOP_mean.conf rename to parm/metplus/GridStat_RETOP_mean.conf diff --git a/parm/parm/metplus/GridStat_RETOP_prob.conf b/parm/metplus/GridStat_RETOP_prob.conf similarity index 100% rename from parm/parm/metplus/GridStat_RETOP_prob.conf rename to parm/metplus/GridStat_RETOP_prob.conf diff --git a/parm/parm/metplus/PointStat_conus_sfc.conf b/parm/metplus/PointStat_conus_sfc.conf similarity index 100% rename from parm/parm/metplus/PointStat_conus_sfc.conf rename to parm/metplus/PointStat_conus_sfc.conf diff --git a/parm/parm/metplus/PointStat_conus_sfc_mean.conf b/parm/metplus/PointStat_conus_sfc_mean.conf similarity index 100% rename from parm/parm/metplus/PointStat_conus_sfc_mean.conf rename to parm/metplus/PointStat_conus_sfc_mean.conf diff --git a/parm/parm/metplus/PointStat_conus_sfc_prob.conf b/parm/metplus/PointStat_conus_sfc_prob.conf similarity index 100% rename from parm/parm/metplus/PointStat_conus_sfc_prob.conf rename to parm/metplus/PointStat_conus_sfc_prob.conf diff --git a/parm/parm/metplus/PointStat_upper_air.conf b/parm/metplus/PointStat_upper_air.conf similarity index 100% rename from parm/parm/metplus/PointStat_upper_air.conf rename to parm/metplus/PointStat_upper_air.conf diff --git a/parm/parm/metplus/PointStat_upper_air_mean.conf b/parm/metplus/PointStat_upper_air_mean.conf similarity index 100% rename from parm/parm/metplus/PointStat_upper_air_mean.conf rename to parm/metplus/PointStat_upper_air_mean.conf diff --git a/parm/parm/metplus/PointStat_upper_air_prob.conf b/parm/metplus/PointStat_upper_air_prob.conf similarity index 100% rename from parm/parm/metplus/PointStat_upper_air_prob.conf rename to parm/metplus/PointStat_upper_air_prob.conf diff --git a/parm/parm/metplus/common.conf b/parm/metplus/common.conf similarity index 100% rename from parm/parm/metplus/common.conf rename to parm/metplus/common.conf diff --git a/ush/create_model_configure_file.py b/ush/create_model_configure_file.py index 5cac49b613..4b0adef63e 100644 --- a/ush/create_model_configure_file.py +++ b/ush/create_model_configure_file.py @@ -320,7 +320,7 @@ def test_create_model_configure_file(self): def setUp(self): USHrrfs = os.path.dirname(os.path.abspath(__file__)) MODEL_CONFIG_FN = "model_configure" - MODEL_CONFIG_TMPL_FP = os.path.join(USHrrfs, "templates", MODEL_CONFIG_FN) + MODEL_CONFIG_TMPL_FP = os.path.join(PARMrrfs, MODEL_CONFIG_FN) set_env_var("DEBUG", True) set_env_var("VERBOSE", True) diff --git a/ush/set_FV3nml_sfc_climo_filenames.py b/ush/set_FV3nml_sfc_climo_filenames.py index b9db5baf81..bb2308472e 100644 --- a/ush/set_FV3nml_sfc_climo_filenames.py +++ b/ush/set_FV3nml_sfc_climo_filenames.py @@ -154,7 +154,7 @@ def setUp(self): FIXLAM = os.path.join(EXPTDIR, "fix_lam") mkdir_vrfy("-p", FIXLAM) cp_vrfy( - os.path.join(USHrrfs, "input.nml.FV3"), + os.path.join(PARMrrfs, "input.nml.FV3"), os.path.join(EXPTDIR, "input.nml"), ) set_env_var("USHrrfs", USHrrfs) diff --git a/ush/setup.py b/ush/setup.py index 26567e94c1..286838e2a5 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -359,9 +359,9 @@ def setup(): MODULESrrfs = os.path.join(HOMErrfs, "modulefiles") EXECrrfs = os.path.join(SR_WX_APP_TOP_DIR, EXEC_SUBDIR) PARMrrfs = os.path.join(HOMErrfs, "parm") - VX_CONFIG_DIR = os.path.join(PARMrrfs, "parm") - METPLUS_CONF = os.path.join(PARMrrfs, "parm", "metplus") - MET_CONFIG = os.path.join(PARMrrfs, "parm", "met") + VX_CONFIG_DIR = PARMrrfs + METPLUS_CONF = os.path.join(PARMrrfs, "metplus") + MET_CONFIG = os.path.join(PARMrrfs, "met") # # ----------------------------------------------------------------------- From 490f1deccb19157330cbb3f9cfd9d3acf934e840 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Mon, 29 Aug 2022 16:39:31 +0000 Subject: [PATCH 054/106] Fixes for community mode. --- jobs/JREGIONAL_GET_EXTRN_MDL_FILES | 2 +- jobs/JREGIONAL_MAKE_ICS | 4 ++-- jobs/JREGIONAL_MAKE_LBCS | 4 ++-- jobs/JREGIONAL_RUN_FCST | 4 ++-- jobs/JREGIONAL_RUN_POST | 2 +- scripts/exregional_make_ics.sh | 5 +++-- scripts/exregional_make_lbcs.sh | 5 +++-- ush/job_preamble.sh | 12 ++++++++---- 8 files changed, 22 insertions(+), 16 deletions(-) diff --git a/jobs/JREGIONAL_GET_EXTRN_MDL_FILES b/jobs/JREGIONAL_GET_EXTRN_MDL_FILES index c5ab7f3403..f8adfc5ec5 100755 --- a/jobs/JREGIONAL_GET_EXTRN_MDL_FILES +++ b/jobs/JREGIONAL_GET_EXTRN_MDL_FILES @@ -211,7 +211,7 @@ esac if [ $RUN_ENVIR = "nco" ]; then export EXTRN_MDL_STAGING_DIR="${COMIN}" else - export EXTRN_MDL_STAGING_DIR="${COMIN}/${cyc}/${EXTRN_MDL_NAME}/for_${ICS_OR_LBCS}" + export EXTRN_MDL_STAGING_DIR="${COMIN}/${EXTRN_MDL_NAME}/for_${ICS_OR_LBCS}" fi mkdir_vrfy -p "${EXTRN_MDL_STAGING_DIR}" cd_vrfy "${EXTRN_MDL_STAGING_DIR}" diff --git a/jobs/JREGIONAL_MAKE_ICS b/jobs/JREGIONAL_MAKE_ICS index 68570d032a..327c302182 100755 --- a/jobs/JREGIONAL_MAKE_ICS +++ b/jobs/JREGIONAL_MAKE_ICS @@ -58,7 +58,7 @@ for the FV3 (in NetCDF format). if [ $RUN_ENVIR = "nco" ]; then export INPUT_DATA="${COMIN}" else - export INPUT_DATA="${COMIN}/${cyc}${SLASH_ENSMEM_SUBDIR}/INPUT" + export INPUT_DATA="${COMIN}${SLASH_ENSMEM_SUBDIR}/INPUT" fi mkdir_vrfy -p "${INPUT_DATA}" # @@ -68,7 +68,7 @@ mkdir_vrfy -p "${INPUT_DATA}" # #----------------------------------------------------------------------- # -DATA="${DATA:-${COMIN}/${cyc}${SLASH_ENSMEM_SUBDIR}}" +DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}}" mkdir_vrfy -p "${DATA}" # #----------------------------------------------------------------------- diff --git a/jobs/JREGIONAL_MAKE_LBCS b/jobs/JREGIONAL_MAKE_LBCS index 28dad2a6d8..8d5656ae5f 100755 --- a/jobs/JREGIONAL_MAKE_LBCS +++ b/jobs/JREGIONAL_MAKE_LBCS @@ -58,7 +58,7 @@ hour zero). if [ $RUN_ENVIR = "nco" ]; then export INPUT_DATA="${COMIN}" else - export INPUT_DATA="${COMIN}/${cyc}${SLASH_ENSMEM_SUBDIR}/INPUT" + export INPUT_DATA="${COMIN}${SLASH_ENSMEM_SUBDIR}/INPUT" fi mkdir_vrfy -p "${INPUT_DATA}" # @@ -68,7 +68,7 @@ mkdir_vrfy -p "${INPUT_DATA}" # #----------------------------------------------------------------------- # -DATA="${DATA:-${COMIN}/${cyc}${SLASH_ENSMEM_SUBDIR}}" +DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}}" mkdir_vrfy -p "${DATA}" # #----------------------------------------------------------------------- diff --git a/jobs/JREGIONAL_RUN_FCST b/jobs/JREGIONAL_RUN_FCST index 5925b575db..40811f3928 100755 --- a/jobs/JREGIONAL_RUN_FCST +++ b/jobs/JREGIONAL_RUN_FCST @@ -66,7 +66,7 @@ the specified cycle. if [ $RUN_ENVIR = "nco" ]; then export INPUT_DATA="${COMIN}" else - export INPUT_DATA="${COMIN}/${cyc}${SLASH_ENSMEM_SUBDIR}/INPUT" + export INPUT_DATA="${COMIN}${SLASH_ENSMEM_SUBDIR}/INPUT" fi # #----------------------------------------------------------------------- @@ -75,7 +75,7 @@ fi # #----------------------------------------------------------------------- # -DATA="${DATA:-${COMIN}/${cyc}${SLASH_ENSMEM_SUBDIR}}" +DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}}" mkdir_vrfy -p ${DATA}/INPUT mkdir_vrfy -p ${DATA}/RESTART # diff --git a/jobs/JREGIONAL_RUN_POST b/jobs/JREGIONAL_RUN_POST index baaa75c64d..725a9b8410 100755 --- a/jobs/JREGIONAL_RUN_POST +++ b/jobs/JREGIONAL_RUN_POST @@ -62,7 +62,7 @@ on the output files corresponding to a specified forecast hour. # #----------------------------------------------------------------------- # -DATA="${DATA:-${COMIN}/${cyc}${SLASH_ENSMEM_SUBDIR}}" +DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}}" # # If SUB_HOURLY_POST is not set to "TRUE", ensure that the forecast # minutes (fmn) are set to "00". This is necessary in order to pass diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index 5d317a3393..e079fdb7c1 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -88,10 +88,11 @@ fi # if [ $RUN_ENVIR = "nco" ]; then extrn_mdl_staging_dir="${COMIN}" + extrn_mdl_var_defns_fp="${extrn_mdl_staging_dir}/${NET}.${cycle}.${EXTRN_MDL_NAME_ICS}.ICS.${EXTRN_MDL_VAR_DEFNS_FN}.sh" else - extrn_mdl_staging_dir="${COMIN}/${cyc}/${EXTRN_MDL_NAME_ICS}/for_ICS" + extrn_mdl_staging_dir="${COMIN}/${EXTRN_MDL_NAME_ICS}/for_ICS" + extrn_mdl_var_defns_fp="${extrn_mdl_staging_dir}/${EXTRN_MDL_VAR_DEFNS_FN}.sh" fi -extrn_mdl_var_defns_fp="${extrn_mdl_staging_dir}/${NET}.${cycle}.${EXTRN_MDL_NAME_ICS}.ICS.${EXTRN_MDL_VAR_DEFNS_FN}.sh" . ${extrn_mdl_var_defns_fp} # #----------------------------------------------------------------------- diff --git a/scripts/exregional_make_lbcs.sh b/scripts/exregional_make_lbcs.sh index d9d7e02f8c..49f161d101 100755 --- a/scripts/exregional_make_lbcs.sh +++ b/scripts/exregional_make_lbcs.sh @@ -86,10 +86,11 @@ fi # if [ $RUN_ENVIR = "nco" ]; then extrn_mdl_staging_dir="${COMIN}" + extrn_mdl_var_defns_fp="${extrn_mdl_staging_dir}/${NET}.${cycle}.${EXTRN_MDL_NAME_LBCS}.LBCS.${EXTRN_MDL_VAR_DEFNS_FN}.sh" else - extrn_mdl_staging_dir="${COMIN}/${cyc}/${EXTRN_MDL_NAME_ICS}/for_LBCS" + extrn_mdl_staging_dir="${COMIN}/${EXTRN_MDL_NAME_ICS}/for_LBCS" + extrn_mdl_var_defns_fp="${extrn_mdl_staging_dir}/${EXTRN_MDL_VAR_DEFNS_FN}.sh" fi -extrn_mdl_var_defns_fp="${extrn_mdl_staging_dir}/${NET}.${cycle}.${EXTRN_MDL_NAME_LBCS}.LBCS.${EXTRN_MDL_VAR_DEFNS_FN}.sh" . ${extrn_mdl_var_defns_fp} # #----------------------------------------------------------------------- diff --git a/ush/job_preamble.sh b/ush/job_preamble.sh index c737848d85..ec671b7bc7 100644 --- a/ush/job_preamble.sh +++ b/ush/job_preamble.sh @@ -76,15 +76,19 @@ if [ "${RUN_ENVIR}" = "nco" ]; then pgmout_lines=$((pgmout_lines + 1)) fi } - export -f PREP_STEP - export -f POST_STEP else export pgmout= export pgmerr= export REDIRECT_OUT_ERR= - export PREP_STEP= - export POST_STEP= + function PREP_STEP() { + : + } + function POST_STEP() { + : + } fi +export -f PREP_STEP +export -f POST_STEP # #----------------------------------------------------------------------- # From e864a012066b9659990c5f7a41213b2111ac514b Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Mon, 29 Aug 2022 17:23:17 +0000 Subject: [PATCH 055/106] Rename SCRIPTSDIR and JOBSDIR. --- jobs/JREGIONAL_GET_EXTRN_MDL_FILES | 2 +- jobs/JREGIONAL_GET_OBS_CCPA | 2 +- jobs/JREGIONAL_GET_OBS_MRMS | 2 +- jobs/JREGIONAL_GET_OBS_NDAS | 2 +- jobs/JREGIONAL_MAKE_GRID | 2 +- jobs/JREGIONAL_MAKE_ICS | 2 +- jobs/JREGIONAL_MAKE_LBCS | 2 +- jobs/JREGIONAL_MAKE_OROG | 2 +- jobs/JREGIONAL_MAKE_SFC_CLIMO | 2 +- jobs/JREGIONAL_RUN_FCST | 2 +- jobs/JREGIONAL_RUN_POST | 2 +- jobs/JREGIONAL_RUN_VX_ENSGRID | 2 +- jobs/JREGIONAL_RUN_VX_ENSGRID_MEAN | 2 +- jobs/JREGIONAL_RUN_VX_ENSGRID_PROB | 2 +- jobs/JREGIONAL_RUN_VX_ENSPOINT | 2 +- jobs/JREGIONAL_RUN_VX_ENSPOINT_MEAN | 2 +- jobs/JREGIONAL_RUN_VX_ENSPOINT_PROB | 2 +- jobs/JREGIONAL_RUN_VX_GRIDSTAT | 2 +- jobs/JREGIONAL_RUN_VX_POINTSTAT | 2 +- parm/FV3LAM_wflow.xml | 90 +++++++++++------------ scripts/exregional_run_vx_ensgrid.sh | 2 +- scripts/exregional_run_vx_ensgrid_mean.sh | 2 +- scripts/exregional_run_vx_ensgrid_prob.sh | 2 +- scripts/exregional_run_vx_gridstat.sh | 2 +- ush/generate_FV3LAM_wflow.py | 4 +- ush/setup.py | 10 +-- ush/wrappers/run_fcst.sh | 2 +- ush/wrappers/run_get_ics.sh | 2 +- ush/wrappers/run_get_lbcs.sh | 2 +- ush/wrappers/run_gridensvx.sh | 6 +- ush/wrappers/run_gridvx.sh | 2 +- ush/wrappers/run_make_grid.sh | 2 +- ush/wrappers/run_make_ics.sh | 2 +- ush/wrappers/run_make_lbcs.sh | 2 +- ush/wrappers/run_make_orog.sh | 2 +- ush/wrappers/run_make_sfc_climo.sh | 2 +- ush/wrappers/run_pointensvx.sh | 6 +- ush/wrappers/run_pointvx.sh | 2 +- ush/wrappers/run_post.sh | 2 +- 39 files changed, 92 insertions(+), 92 deletions(-) diff --git a/jobs/JREGIONAL_GET_EXTRN_MDL_FILES b/jobs/JREGIONAL_GET_EXTRN_MDL_FILES index f8adfc5ec5..782f54eccb 100755 --- a/jobs/JREGIONAL_GET_EXTRN_MDL_FILES +++ b/jobs/JREGIONAL_GET_EXTRN_MDL_FILES @@ -222,7 +222,7 @@ cd_vrfy "${EXTRN_MDL_STAGING_DIR}" # #----------------------------------------------------------------------- # -$SCRIPTSDIR/exregional_get_extrn_mdl_files.sh || \ +$SCRIPTSrrfs/exregional_get_extrn_mdl_files.sh || \ print_err_msg_exit "\ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." # diff --git a/jobs/JREGIONAL_GET_OBS_CCPA b/jobs/JREGIONAL_GET_OBS_CCPA index 6114a56c88..f40523ec8a 100755 --- a/jobs/JREGIONAL_GET_OBS_CCPA +++ b/jobs/JREGIONAL_GET_OBS_CCPA @@ -68,7 +68,7 @@ for verification purposes. # #----------------------------------------------------------------------- # -$SCRIPTSDIR/exregional_get_obs_ccpa.sh || \ +$SCRIPTSrrfs/exregional_get_obs_ccpa.sh || \ print_err_msg_exit "\ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." # diff --git a/jobs/JREGIONAL_GET_OBS_MRMS b/jobs/JREGIONAL_GET_OBS_MRMS index 798570f098..8792da2fcd 100755 --- a/jobs/JREGIONAL_GET_OBS_MRMS +++ b/jobs/JREGIONAL_GET_OBS_MRMS @@ -68,7 +68,7 @@ echo "VAR=${VAR}" for field in ${VAR[@]}; do export field echo "Field=${field}" - $SCRIPTSDIR/exregional_get_obs_mrms.sh || \ + $SCRIPTSrrfs/exregional_get_obs_mrms.sh || \ print_err_msg_exit "\ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." done diff --git a/jobs/JREGIONAL_GET_OBS_NDAS b/jobs/JREGIONAL_GET_OBS_NDAS index 62c008dceb..dab4c58624 100755 --- a/jobs/JREGIONAL_GET_OBS_NDAS +++ b/jobs/JREGIONAL_GET_OBS_NDAS @@ -64,7 +64,7 @@ for verification purposes. # #----------------------------------------------------------------------- # -$SCRIPTSDIR/exregional_get_obs_ndas.sh || \ +$SCRIPTSrrfs/exregional_get_obs_ndas.sh || \ print_err_msg_exit "\ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." # diff --git a/jobs/JREGIONAL_MAKE_GRID b/jobs/JREGIONAL_MAKE_GRID index 136574e892..23bdc0cbc0 100755 --- a/jobs/JREGIONAL_MAKE_GRID +++ b/jobs/JREGIONAL_MAKE_GRID @@ -171,7 +171,7 @@ mkdir_vrfy -p "$DATA" # #----------------------------------------------------------------------- # -${SCRIPTSDIR}/exregional_make_grid.sh || \ +${SCRIPTSrrfs}/exregional_make_grid.sh || \ print_err_msg_exit "\ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." # diff --git a/jobs/JREGIONAL_MAKE_ICS b/jobs/JREGIONAL_MAKE_ICS index 327c302182..fffc075755 100755 --- a/jobs/JREGIONAL_MAKE_ICS +++ b/jobs/JREGIONAL_MAKE_ICS @@ -77,7 +77,7 @@ mkdir_vrfy -p "${DATA}" # #----------------------------------------------------------------------- # -$SCRIPTSDIR/exregional_make_ics.sh || \ +$SCRIPTSrrfs/exregional_make_ics.sh || \ print_err_msg_exit "\ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." # diff --git a/jobs/JREGIONAL_MAKE_LBCS b/jobs/JREGIONAL_MAKE_LBCS index 8d5656ae5f..95f36e5fb2 100755 --- a/jobs/JREGIONAL_MAKE_LBCS +++ b/jobs/JREGIONAL_MAKE_LBCS @@ -77,7 +77,7 @@ mkdir_vrfy -p "${DATA}" # #----------------------------------------------------------------------- # -$SCRIPTSDIR/exregional_make_lbcs.sh || \ +$SCRIPTSrrfs/exregional_make_lbcs.sh || \ print_err_msg_exit "\ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." # diff --git a/jobs/JREGIONAL_MAKE_OROG b/jobs/JREGIONAL_MAKE_OROG index 3c559e36c4..36620f21a8 100755 --- a/jobs/JREGIONAL_MAKE_OROG +++ b/jobs/JREGIONAL_MAKE_OROG @@ -53,7 +53,7 @@ This is the J-job script for the task that generates orography files. # #----------------------------------------------------------------------- # -${SCRIPTSDIR}/exregional_make_orog.sh || \ +${SCRIPTSrrfs}/exregional_make_orog.sh || \ print_err_msg_exit "\ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." # diff --git a/jobs/JREGIONAL_MAKE_SFC_CLIMO b/jobs/JREGIONAL_MAKE_SFC_CLIMO index b674db32ee..36002d81ac 100755 --- a/jobs/JREGIONAL_MAKE_SFC_CLIMO +++ b/jobs/JREGIONAL_MAKE_SFC_CLIMO @@ -74,7 +74,7 @@ mkdir_vrfy $DATA # #----------------------------------------------------------------------- # -${SCRIPTSDIR}/exregional_make_sfc_climo.sh || \ +${SCRIPTSrrfs}/exregional_make_sfc_climo.sh || \ print_err_msg_exit "\ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." # diff --git a/jobs/JREGIONAL_RUN_FCST b/jobs/JREGIONAL_RUN_FCST index 40811f3928..231a7364f5 100755 --- a/jobs/JREGIONAL_RUN_FCST +++ b/jobs/JREGIONAL_RUN_FCST @@ -86,7 +86,7 @@ mkdir_vrfy -p ${DATA}/RESTART # #----------------------------------------------------------------------- # -$SCRIPTSDIR/exregional_run_fcst.sh || \ +$SCRIPTSrrfs/exregional_run_fcst.sh || \ print_err_msg_exit "\ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." # diff --git a/jobs/JREGIONAL_RUN_POST b/jobs/JREGIONAL_RUN_POST index 725a9b8410..99c4439318 100755 --- a/jobs/JREGIONAL_RUN_POST +++ b/jobs/JREGIONAL_RUN_POST @@ -126,7 +126,7 @@ fi # #----------------------------------------------------------------------- # -$SCRIPTSDIR/exregional_run_post.sh || \ +$SCRIPTSrrfs/exregional_run_post.sh || \ print_err_msg_exit "\ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." # diff --git a/jobs/JREGIONAL_RUN_VX_ENSGRID b/jobs/JREGIONAL_RUN_VX_ENSGRID index c37d9ad08f..5c78ead423 100755 --- a/jobs/JREGIONAL_RUN_VX_ENSGRID +++ b/jobs/JREGIONAL_RUN_VX_ENSGRID @@ -63,7 +63,7 @@ on gridded fields by initialization time for all forecast hours. # #----------------------------------------------------------------------- # -$SCRIPTSDIR/exregional_run_vx_ensgrid.sh || \ +$SCRIPTSrrfs/exregional_run_vx_ensgrid.sh || \ print_err_msg_exit "\ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." # diff --git a/jobs/JREGIONAL_RUN_VX_ENSGRID_MEAN b/jobs/JREGIONAL_RUN_VX_ENSGRID_MEAN index 595794d107..56b0b6ec5e 100755 --- a/jobs/JREGIONAL_RUN_VX_ENSGRID_MEAN +++ b/jobs/JREGIONAL_RUN_VX_ENSGRID_MEAN @@ -63,7 +63,7 @@ on gridded fields by initialization time for all forecast hours. # #----------------------------------------------------------------------- # -$SCRIPTSDIR/exregional_run_vx_ensgrid_mean.sh || \ +$SCRIPTSrrfs/exregional_run_vx_ensgrid_mean.sh || \ print_err_msg_exit "\ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." # diff --git a/jobs/JREGIONAL_RUN_VX_ENSGRID_PROB b/jobs/JREGIONAL_RUN_VX_ENSGRID_PROB index 465e51d798..b845c4e2d7 100755 --- a/jobs/JREGIONAL_RUN_VX_ENSGRID_PROB +++ b/jobs/JREGIONAL_RUN_VX_ENSGRID_PROB @@ -63,7 +63,7 @@ on gridded fields by initialization time for all forecast hours. # #----------------------------------------------------------------------- # -$SCRIPTSDIR/exregional_run_vx_ensgrid_prob.sh || \ +$SCRIPTSrrfs/exregional_run_vx_ensgrid_prob.sh || \ print_err_msg_exit "\ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." # diff --git a/jobs/JREGIONAL_RUN_VX_ENSPOINT b/jobs/JREGIONAL_RUN_VX_ENSPOINT index c7772c5082..86e8da8e0c 100755 --- a/jobs/JREGIONAL_RUN_VX_ENSPOINT +++ b/jobs/JREGIONAL_RUN_VX_ENSPOINT @@ -54,7 +54,7 @@ by initialitation time for all forecast hours. # #----------------------------------------------------------------------- # -$SCRIPTSDIR/exregional_run_vx_enspoint.sh || \ +$SCRIPTSrrfs/exregional_run_vx_enspoint.sh || \ print_err_msg_exit "\ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." # diff --git a/jobs/JREGIONAL_RUN_VX_ENSPOINT_MEAN b/jobs/JREGIONAL_RUN_VX_ENSPOINT_MEAN index b5ca84a104..3a4855d09a 100755 --- a/jobs/JREGIONAL_RUN_VX_ENSPOINT_MEAN +++ b/jobs/JREGIONAL_RUN_VX_ENSPOINT_MEAN @@ -54,7 +54,7 @@ by initialitation time for all forecast hours. # #----------------------------------------------------------------------- # -$SCRIPTSDIR/exregional_run_vx_enspoint_mean.sh || \ +$SCRIPTSrrfs/exregional_run_vx_enspoint_mean.sh || \ print_err_msg_exit "\ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." # diff --git a/jobs/JREGIONAL_RUN_VX_ENSPOINT_PROB b/jobs/JREGIONAL_RUN_VX_ENSPOINT_PROB index 1dd52eb97d..02634fbc9d 100755 --- a/jobs/JREGIONAL_RUN_VX_ENSPOINT_PROB +++ b/jobs/JREGIONAL_RUN_VX_ENSPOINT_PROB @@ -54,7 +54,7 @@ by initialitation time for all forecast hours. # #----------------------------------------------------------------------- # -$SCRIPTSDIR/exregional_run_vx_enspoint_prob.sh || \ +$SCRIPTSrrfs/exregional_run_vx_enspoint_prob.sh || \ print_err_msg_exit "\ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." # diff --git a/jobs/JREGIONAL_RUN_VX_GRIDSTAT b/jobs/JREGIONAL_RUN_VX_GRIDSTAT index 4518eae4d4..a44d2700ff 100755 --- a/jobs/JREGIONAL_RUN_VX_GRIDSTAT +++ b/jobs/JREGIONAL_RUN_VX_GRIDSTAT @@ -63,7 +63,7 @@ by initialization time for all forecast hours. # #----------------------------------------------------------------------- # -$SCRIPTSDIR/exregional_run_vx_gridstat.sh || \ +$SCRIPTSrrfs/exregional_run_vx_gridstat.sh || \ print_err_msg_exit "\ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." # diff --git a/jobs/JREGIONAL_RUN_VX_POINTSTAT b/jobs/JREGIONAL_RUN_VX_POINTSTAT index 6f67227d7c..0b2a95d5d8 100755 --- a/jobs/JREGIONAL_RUN_VX_POINTSTAT +++ b/jobs/JREGIONAL_RUN_VX_POINTSTAT @@ -54,7 +54,7 @@ by initialitation time for all forecast hours. # #----------------------------------------------------------------------- # -$SCRIPTSDIR/exregional_run_vx_pointstat.sh || \ +$SCRIPTSrrfs/exregional_run_vx_pointstat.sh || \ print_err_msg_exit "\ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." # diff --git a/parm/FV3LAM_wflow.xml b/parm/FV3LAM_wflow.xml index 41c5c5cd1f..744d628571 100644 --- a/parm/FV3LAM_wflow.xml +++ b/parm/FV3LAM_wflow.xml @@ -86,9 +86,9 @@ specifies the number of processes per node being used (the PPN_... entities). - + - + @@ -166,7 +166,7 @@ we are already sourcing other files in the J-job anyway. --> leave empty for REFC and RETOP export FHR=`echo $(seq 0 ${ACCUM} ${FCST_LEN_HRS}) | cut -d" " -f2-` -${JOBSDIR}/JREGIONAL_RUN_VX_ENSGRID +${JOBSrrfs}/JREGIONAL_RUN_VX_ENSGRID -${JOBSDIR}/JREGIONAL_RUN_VX_ENSGRID_MEAN +${JOBSrrfs}/JREGIONAL_RUN_VX_ENSGRID_MEAN -${JOBSDIR}/JREGIONAL_RUN_VX_ENSGRID_PROB +${JOBSrrfs}/JREGIONAL_RUN_VX_ENSGRID_PROB diff --git a/ush/wrappers/run_gridvx.sh b/ush/wrappers/run_gridvx.sh index a96383f6ed..00326d975b 100755 --- a/ush/wrappers/run_gridvx.sh +++ b/ush/wrappers/run_gridvx.sh @@ -15,5 +15,5 @@ export ACCUM="06" # 01 03 06 24 --> leave empty for REFC and RETOP export FHR=`echo $(seq 0 ${ACCUM} ${FCST_LEN_HRS}) | cut -d" " -f2-` -${JOBSDIR}/JREGIONAL_RUN_VX_GRIDSTAT +${JOBSrrfs}/JREGIONAL_RUN_VX_GRIDSTAT diff --git a/ush/wrappers/run_make_grid.sh b/ush/wrappers/run_make_grid.sh index 96fd8a4ec4..2d0a3c98a3 100755 --- a/ush/wrappers/run_make_grid.sh +++ b/ush/wrappers/run_make_grid.sh @@ -5,4 +5,4 @@ source ${GLOBAL_VAR_DEFNS_FP} export CDATE=${DATE_FIRST_CYCL}${CYCL_HRS} export CYCLE_DIR=${EXPTDIR}/${CDATE} -${JOBSDIR}/JREGIONAL_MAKE_GRID +${JOBSrrfs}/JREGIONAL_MAKE_GRID diff --git a/ush/wrappers/run_make_ics.sh b/ush/wrappers/run_make_ics.sh index de3d3de6d2..bf63e9680a 100755 --- a/ush/wrappers/run_make_ics.sh +++ b/ush/wrappers/run_make_ics.sh @@ -6,5 +6,5 @@ export CDATE=${DATE_FIRST_CYCL}${CYCL_HRS} export CYCLE_DIR=${EXPTDIR}/${CDATE} export SLASH_ENSMEM_SUBDIR="" -${JOBSDIR}/JREGIONAL_MAKE_ICS +${JOBSrrfs}/JREGIONAL_MAKE_ICS diff --git a/ush/wrappers/run_make_lbcs.sh b/ush/wrappers/run_make_lbcs.sh index adae78ea4a..697e1add5c 100755 --- a/ush/wrappers/run_make_lbcs.sh +++ b/ush/wrappers/run_make_lbcs.sh @@ -6,5 +6,5 @@ export CDATE=${DATE_FIRST_CYCL}${CYCL_HRS} export CYCLE_DIR=${EXPTDIR}/${CDATE} export SLASH_ENSMEM_SUBDIR="" -${JOBSDIR}/JREGIONAL_MAKE_LBCS +${JOBSrrfs}/JREGIONAL_MAKE_LBCS diff --git a/ush/wrappers/run_make_orog.sh b/ush/wrappers/run_make_orog.sh index cf1adab4c4..3fd255d1f5 100755 --- a/ush/wrappers/run_make_orog.sh +++ b/ush/wrappers/run_make_orog.sh @@ -5,4 +5,4 @@ source ${GLOBAL_VAR_DEFNS_FP} export CDATE=${DATE_FIRST_CYCL}${CYCL_HRS} export CYCLE_DIR=${EXPTDIR}/${CDATE} -${JOBSDIR}/JREGIONAL_MAKE_OROG +${JOBSrrfs}/JREGIONAL_MAKE_OROG diff --git a/ush/wrappers/run_make_sfc_climo.sh b/ush/wrappers/run_make_sfc_climo.sh index 06c9260793..1972db228f 100755 --- a/ush/wrappers/run_make_sfc_climo.sh +++ b/ush/wrappers/run_make_sfc_climo.sh @@ -5,4 +5,4 @@ source ${GLOBAL_VAR_DEFNS_FP} export CDATE=${DATE_FIRST_CYCL}${CYCL_HRS} export CYCLE_DIR=${EXPTDIR}/${CDATE} -${JOBSDIR}/JREGIONAL_MAKE_SFC_CLIMO +${JOBSrrfs}/JREGIONAL_MAKE_SFC_CLIMO diff --git a/ush/wrappers/run_pointensvx.sh b/ush/wrappers/run_pointensvx.sh index 142d9ec833..8bd85b35f2 100755 --- a/ush/wrappers/run_pointensvx.sh +++ b/ush/wrappers/run_pointensvx.sh @@ -12,9 +12,9 @@ export OBS_DIR=${NDAS_OBS_DIR} export FHR=`echo $(seq 0 1 ${FCST_LEN_HRS})` -${JOBSDIR}/JREGIONAL_RUN_VX_ENSPOINT +${JOBSrrfs}/JREGIONAL_RUN_VX_ENSPOINT -${JOBSDIR}/JREGIONAL_RUN_VX_ENSPOINT_MEAN +${JOBSrrfs}/JREGIONAL_RUN_VX_ENSPOINT_MEAN -${JOBSDIR}/JREGIONAL_RUN_VX_ENSPOINT_PROB +${JOBSrrfs}/JREGIONAL_RUN_VX_ENSPOINT_PROB diff --git a/ush/wrappers/run_pointvx.sh b/ush/wrappers/run_pointvx.sh index 527c16f37d..74c76bbfdf 100755 --- a/ush/wrappers/run_pointvx.sh +++ b/ush/wrappers/run_pointvx.sh @@ -13,5 +13,5 @@ export OBS_DIR=${NDAS_OBS_DIR} export FHR=`echo $(seq 0 1 ${FCST_LEN_HRS})` -${JOBSDIR}/JREGIONAL_RUN_VX_POINTSTAT +${JOBSrrfs}/JREGIONAL_RUN_VX_POINTSTAT diff --git a/ush/wrappers/run_post.sh b/ush/wrappers/run_post.sh index 1c8f3d4dfb..4f7c8d293b 100755 --- a/ush/wrappers/run_post.sh +++ b/ush/wrappers/run_post.sh @@ -11,5 +11,5 @@ export ENSMEM_INDX="" num_fcst_hrs=${FCST_LEN_HRS} for (( i=0; i<=$((num_fcst_hrs)); i++ )); do export fhr=`printf "%03i" ${i}` - ${JOBSDIR}/JREGIONAL_RUN_POST + ${JOBSrrfs}/JREGIONAL_RUN_POST done From 77adc34ca5960c7b421ef8916dd3751bca994c36 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Mon, 29 Aug 2022 18:04:03 +0000 Subject: [PATCH 056/106] Move all FIX** directories in to a fix/ directory. --- scripts/exregional_make_grid.sh | 4 ++-- scripts/exregional_make_ics.sh | 6 +++--- scripts/exregional_make_lbcs.sh | 6 +++--- scripts/exregional_make_orog.sh | 8 +++---- scripts/exregional_make_sfc_climo.sh | 4 ++-- scripts/exregional_run_fcst.sh | 18 ++++++++-------- ush/config_defaults.yaml | 2 +- ush/create_diag_table_file.py | 1 + ush/create_model_configure_file.py | 1 + ush/generate_FV3LAM_wflow.py | 2 +- ush/link_fix.py | 20 +++++++++--------- ush/set_FV3nml_ens_stoch_seeds.py | 1 + ush/set_FV3nml_sfc_climo_filenames.py | 11 +++++----- ush/setup.py | 30 +++++++++++++-------------- 14 files changed, 59 insertions(+), 55 deletions(-) diff --git a/scripts/exregional_make_grid.sh b/scripts/exregional_make_grid.sh index a6bd142d1c..5637d1d6f8 100755 --- a/scripts/exregional_make_grid.sh +++ b/scripts/exregional_make_grid.sh @@ -556,7 +556,7 @@ halo failed." # #----------------------------------------------------------------------- # -# Create symlinks in the FIXLAM directory to the grid and mosaic files +# Create symlinks in the FIXlam directory to the grid and mosaic files # generated above in the GRID_DIR directory. # #----------------------------------------------------------------------- @@ -575,7 +575,7 @@ failed." # paths to the surface climatology files. These files will either already # be avaialable in a user-specified directory (SFC_CLIMO_DIR) or will be # generated by the MAKE_SFC_CLIMO_TN task. They (or symlinks to them) -# will be placed (or wll already exist) in the FIXLAM directory. +# will be placed (or wll already exist) in the FIXlam directory. # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index e079fdb7c1..3d93331531 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -498,9 +498,9 @@ fi # settings=" 'config': { - 'fix_dir_target_grid': ${FIXLAM}, - 'mosaic_file_target_grid': ${FIXLAM}/${CRES}${DOT_OR_USCORE}mosaic.halo$((10#${NH4})).nc, - 'orog_dir_target_grid': ${FIXLAM}, + 'fix_dir_target_grid': ${FIXlam}, + 'mosaic_file_target_grid': ${FIXlam}/${CRES}${DOT_OR_USCORE}mosaic.halo$((10#${NH4})).nc, + 'orog_dir_target_grid': ${FIXlam}, 'orog_files_target_grid': ${CRES}${DOT_OR_USCORE}oro_data.tile${TILE_RGNL}.halo$((10#${NH4})).nc, 'vcoord_file_target_grid': ${FIXam}/global_hyblev.l65.txt, 'varmap_file': ${UFS_UTILS_DIR}/parm/varmap_tables/${varmap_file}, diff --git a/scripts/exregional_make_lbcs.sh b/scripts/exregional_make_lbcs.sh index 49f161d101..61dc1ebaa9 100755 --- a/scripts/exregional_make_lbcs.sh +++ b/scripts/exregional_make_lbcs.sh @@ -422,9 +422,9 @@ list file has not specified for this external LBC model (EXTRN_MDL_NAME_LBCS): # settings=" 'config': { - 'fix_dir_target_grid': ${FIXLAM}, - 'mosaic_file_target_grid': ${FIXLAM}/${CRES}${DOT_OR_USCORE}mosaic.halo$((10#${NH4})).nc, - 'orog_dir_target_grid': ${FIXLAM}, + 'fix_dir_target_grid': ${FIXlam}, + 'mosaic_file_target_grid': ${FIXlam}/${CRES}${DOT_OR_USCORE}mosaic.halo$((10#${NH4})).nc, + 'orog_dir_target_grid': ${FIXlam}, 'orog_files_target_grid': ${CRES}${DOT_OR_USCORE}oro_data.tile${TILE_RGNL}.halo$((10#${NH4})).nc, 'vcoord_file_target_grid': ${FIXam}/global_hyblev.l65.txt, 'varmap_file': ${UFS_UTILS_DIR}/parm/varmap_tables/${varmap_file}, diff --git a/scripts/exregional_make_orog.sh b/scripts/exregional_make_orog.sh index 887b64d9b1..3fee8d6b96 100755 --- a/scripts/exregional_make_orog.sh +++ b/scripts/exregional_make_orog.sh @@ -145,11 +145,11 @@ cp_vrfy ${TOPO_DIR}/gmted2010.30sec.int fort.235 #----------------------------------------------------------------------- # mosaic_fn="${CRES}${DOT_OR_USCORE}mosaic.halo${NHW}.nc" -mosaic_fp="$FIXLAM/${mosaic_fn}" +mosaic_fp="${FIXlam}/${mosaic_fn}" grid_fn=$( get_charvar_from_netcdf "${mosaic_fp}" "gridfiles" ) || print_err_msg_exit "\ get_charvar_from_netcdf function failed." -grid_fp="${FIXLAM}/${grid_fn}" +grid_fp="${FIXlam}/${grid_fn}" # #----------------------------------------------------------------------- # @@ -250,10 +250,10 @@ if [ "${CCPP_PHYS_SUITE}" = "FV3_HRRR" ]; then mkdir_vrfy -p ${DATA} cd_vrfy ${DATA} mosaic_fn_gwd="${CRES}${DOT_OR_USCORE}mosaic.halo${NH4}.nc" - mosaic_fp_gwd="$FIXLAM/${mosaic_fn_gwd}" + mosaic_fp_gwd="${FIXlam}/${mosaic_fn_gwd}" grid_fn_gwd=$( get_charvar_from_netcdf "${mosaic_fp_gwd}" "gridfiles" ) || \ print_err_msg_exit "get_charvar_from_netcdf function failed." - grid_fp_gwd="${FIXLAM}/${grid_fn_gwd}" + grid_fp_gwd="${FIXlam}/${grid_fn_gwd}" ls_fn="geo_em.d01.lat-lon.2.5m.HGT_M.nc" ss_fn="HGT.Beljaars_filtered.lat-lon.30s_res.nc" create_symlink_to_file target="${grid_fp_gwd}" symlink="${DATA}/${grid_fn_gwd}" \ diff --git a/scripts/exregional_make_sfc_climo.sh b/scripts/exregional_make_sfc_climo.sh index 4ebd21dd3f..1d74efc072 100755 --- a/scripts/exregional_make_sfc_climo.sh +++ b/scripts/exregional_make_sfc_climo.sh @@ -90,8 +90,8 @@ input_slope_type_file="${SFC_CLIMO_INPUT_DIR}/slope_type.1.0.nc" input_soil_type_file="${SFC_CLIMO_INPUT_DIR}/soil_type.statsgo.0.05.nc" input_vegetation_type_file="${SFC_CLIMO_INPUT_DIR}/vegetation_type.igbp.0.05.nc" input_vegetation_greenness_file="${SFC_CLIMO_INPUT_DIR}/vegetation_greenness.0.144.nc" -mosaic_file_mdl="${FIXLAM}/${CRES}${DOT_OR_USCORE}mosaic.halo${NH4}.nc" -orog_dir_mdl="${FIXLAM}" +mosaic_file_mdl="${FIXlam}/${CRES}${DOT_OR_USCORE}mosaic.halo${NH4}.nc" +orog_dir_mdl="${FIXlam}" orog_files_mdl="${CRES}${DOT_OR_USCORE}oro_data.tile${TILE_RGNL}.halo${NH4}.nc" halo=${NH4} maximum_snow_albedo_method="bilinear" diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index a6af733cf7..8913813ec2 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -88,7 +88,7 @@ Creating links in the INPUT subdirectory of the current run directory to the grid and (filtered) orography files ..." -# Create links to fix files in the FIXLAM directory. +# Create links to fix files in the FIXlam directory. cd_vrfy ${DATA}/INPUT @@ -111,14 +111,14 @@ else fi # Symlink to mosaic file with a completely different name. -#target="${FIXLAM}/${CRES}${DOT_OR_USCORE}mosaic.halo${NH4}.nc" # Should this point to this halo4 file or a halo3 file??? -target="${FIXLAM}/${CRES}${DOT_OR_USCORE}mosaic.halo${NH3}.nc" # Should this point to this halo4 file or a halo3 file??? +#target="${FIXlam}/${CRES}${DOT_OR_USCORE}mosaic.halo${NH4}.nc" # Should this point to this halo4 file or a halo3 file??? +target="${FIXlam}/${CRES}${DOT_OR_USCORE}mosaic.halo${NH3}.nc" # Should this point to this halo4 file or a halo3 file??? symlink="grid_spec.nc" create_symlink_to_file target="$target" symlink="$symlink" \ relative="${relative_link_flag}" ## Symlink to halo-3 grid file with "halo3" stripped from name. -#target="${FIXLAM}/${CRES}${DOT_OR_USCORE}grid.tile${TILE_RGNL}.halo${NH3}.nc" +#target="${FIXlam}/${CRES}${DOT_OR_USCORE}grid.tile${TILE_RGNL}.halo${NH3}.nc" #if [ "${RUN_TASK_MAKE_SFC_CLIMO}" = "TRUE" ] && \ # [ "${GRID_GEN_METHOD}" = "GFDLgrid" ] && \ # [ "${GFDLgrid_USE_NUM_CELLS_IN_FILENAMES}" = "FALSE" ]; then @@ -131,7 +131,7 @@ create_symlink_to_file target="$target" symlink="$symlink" \ mosaic_fn="grid_spec.nc" grid_fn=$( get_charvar_from_netcdf "${mosaic_fn}" "gridfiles" ) -target="${FIXLAM}/${grid_fn}" +target="${FIXlam}/${grid_fn}" symlink="${grid_fn}" create_symlink_to_file target="$target" symlink="$symlink" \ relative="${relative_link_flag}" @@ -148,7 +148,7 @@ create_symlink_to_file target="$target" symlink="$symlink" \ # Note that even though the message says "Stopped", the task still con- # sumes core-hours. # -target="${FIXLAM}/${CRES}${DOT_OR_USCORE}grid.tile${TILE_RGNL}.halo${NH4}.nc" +target="${FIXlam}/${CRES}${DOT_OR_USCORE}grid.tile${TILE_RGNL}.halo${NH4}.nc" symlink="grid.tile${TILE_RGNL}.halo${NH4}.nc" create_symlink_to_file target="$target" symlink="$symlink" \ relative="${relative_link_flag}" @@ -166,7 +166,7 @@ else fi # Symlink to halo-0 orography file with "${CRES}_" and "halo0" stripped from name. -target="${FIXLAM}/${CRES}${DOT_OR_USCORE}oro_data.tile${TILE_RGNL}.halo${NH0}.nc" +target="${FIXlam}/${CRES}${DOT_OR_USCORE}oro_data.tile${TILE_RGNL}.halo${NH0}.nc" symlink="oro_data.nc" create_symlink_to_file target="$target" symlink="$symlink" \ relative="${relative_link_flag}" @@ -183,7 +183,7 @@ create_symlink_to_file target="$target" symlink="$symlink" \ # Note that even though the message says "Stopped", the task still con- # sumes core-hours. # -target="${FIXLAM}/${CRES}${DOT_OR_USCORE}oro_data.tile${TILE_RGNL}.halo${NH4}.nc" +target="${FIXlam}/${CRES}${DOT_OR_USCORE}oro_data.tile${TILE_RGNL}.halo${NH4}.nc" symlink="oro_data.tile${TILE_RGNL}.halo${NH4}.nc" create_symlink_to_file target="$target" symlink="$symlink" \ relative="${relative_link_flag}" @@ -199,7 +199,7 @@ if [ "${CCPP_PHYS_SUITE}" = "FV3_HRRR" ]; then fileids=( "ss" "ls" ) for fileid in "${fileids[@]}"; do - target="${FIXLAM}/${CRES}${DOT_OR_USCORE}oro_data_${fileid}.tile${TILE_RGNL}.halo${NH0}.nc" + target="${FIXlam}/${CRES}${DOT_OR_USCORE}oro_data_${fileid}.tile${TILE_RGNL}.halo${NH0}.nc" symlink="oro_data_${fileid}.nc" create_symlink_to_file target="$target" symlink="$symlink" \ relative="${relative_link_flag}" diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 901d4fd4f3..102b6da635 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -1643,7 +1643,7 @@ task_run_fcst: # model's namelist file that represent the relative or absolute paths of # various fixed files (the first column of the array, where columns are # delineated by the pipe symbol "|") to the full paths to surface climatology - # files (on the native FV3-LAM grid) in the FIXLAM directory derived from + # files (on the native FV3-LAM grid) in the FIXlam directory derived from # the corresponding surface climatology fields (the second column of the # array). # diff --git a/ush/create_diag_table_file.py b/ush/create_diag_table_file.py index 616bd4f1b7..129ca1ab94 100644 --- a/ush/create_diag_table_file.py +++ b/ush/create_diag_table_file.py @@ -131,6 +131,7 @@ def test_create_diag_table_file(self): def setUp(self): USHrrfs = os.path.dirname(os.path.abspath(__file__)) + PARMrrfs = os.path.join(USHrrfs, "..", "parm") DIAG_TABLE_FN = "diag_table" DIAG_TABLE_TMPL_FP = os.path.join( PARMrrfs, f"{DIAG_TABLE_FN}.FV3_GFS_v15p2" diff --git a/ush/create_model_configure_file.py b/ush/create_model_configure_file.py index 4b0adef63e..63e29ebe62 100644 --- a/ush/create_model_configure_file.py +++ b/ush/create_model_configure_file.py @@ -319,6 +319,7 @@ def test_create_model_configure_file(self): def setUp(self): USHrrfs = os.path.dirname(os.path.abspath(__file__)) + PARMrrfs = os.path.join(USHrrfs, "..", "parm") MODEL_CONFIG_FN = "model_configure" MODEL_CONFIG_TMPL_FP = os.path.join(PARMrrfs, MODEL_CONFIG_FN) diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index a3c763083d..790792a7c2 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -942,7 +942,7 @@ def generate_FV3LAM_wflow(): # If not running the MAKE_GRID_TN task (which implies the workflow will # use pregenerated grid files), set the namelist variables specifying # the paths to surface climatology files. These files are located in - # (or have symlinks that point to them) in the FIXLAM directory. + # (or have symlinks that point to them) in the FIXlam directory. # # Note that if running the MAKE_GRID_TN task, this action usually cannot # be performed here but must be performed in that task because the names diff --git a/ush/link_fix.py b/ush/link_fix.py index ea6503da0d..752fb7743f 100755 --- a/ush/link_fix.py +++ b/ush/link_fix.py @@ -43,7 +43,7 @@ def link_fix(verbose, file_group): # # ----------------------------------------------------------------------- # - # Create symlinks in the FIXLAM directory pointing to the grid files. + # Create symlinks in the FIXlam directory pointing to the grid files. # These symlinks are needed by the make_orog, make_sfc_climo, make_ic, # make_lbc, and/or run_fcst tasks. # @@ -55,7 +55,7 @@ def link_fix(verbose, file_group): # ----------------------------------------------------------------------- # print_info_msg( - f"Creating links in the FIXLAM directory to the grid files...", verbose=verbose + f"Creating links in the FIXlam directory to the grid files...", verbose=verbose ) # # ----------------------------------------------------------------------- @@ -65,7 +65,7 @@ def link_fix(verbose, file_group): # # # For grid files (i.e. file_group set to "grid"), symlinks are created - # in the FIXLAM directory to files (of the same names) in the GRID_DIR. + # in the FIXlam directory to files (of the same names) in the GRID_DIR. # These symlinks/files and the reason each is needed is listed below: # # 1) "C*.mosaic.halo${NHW}.nc" @@ -265,18 +265,18 @@ def link_fix(verbose, file_group): # ----------------------------------------------------------------------- # # In creating the various symlinks below, it is convenient to work in - # the FIXLAM directory. We will change directory back to the original + # the FIXlam directory. We will change directory back to the original # later below. # # ----------------------------------------------------------------------- # SAVE_DIR = os.getcwd() - cd_vrfy(FIXLAM) + cd_vrfy(FIXlam) # # ----------------------------------------------------------------------- # # Use the set of full file paths generated above as the link targets to - # create symlinks to these files in the FIXLAM directory. + # create symlinks to these files in the FIXlam directory. # # ----------------------------------------------------------------------- # @@ -345,7 +345,7 @@ def link_fix(verbose, file_group): create_symlink_to_file(target, symlink, True) # # In order to be able to specify the surface climatology file names in - # the forecast model's namelist file, in the FIXLAM directory a symlink + # the forecast model's namelist file, in the FIXlam directory a symlink # must be created for each surface climatology field that has "tile1" in # its name (and no "halo") and which points to the corresponding "tile7.halo0" # file. @@ -413,9 +413,9 @@ def test_link_fix(self): def setUp(self): define_macos_utilities() TEST_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "test_data") - FIXLAM = os.path.join(TEST_DIR, "expt", "fix_lam") - mkdir_vrfy("-p", FIXLAM) - set_env_var("FIXLAM", FIXLAM) + FIXlam = os.path.join(TEST_DIR, "expt", "fix_lam") + mkdir_vrfy("-p", FIXlam) + set_env_var("FIXlam", FIXlam) set_env_var("DOT_OR_USCORE", "_") set_env_var("TILE_RGNL", 7) set_env_var("NH0", 0) diff --git a/ush/set_FV3nml_ens_stoch_seeds.py b/ush/set_FV3nml_ens_stoch_seeds.py index 51be55add2..86722ac439 100644 --- a/ush/set_FV3nml_ens_stoch_seeds.py +++ b/ush/set_FV3nml_ens_stoch_seeds.py @@ -173,6 +173,7 @@ def setUp(self): set_env_var("VERBOSE", True) self.cdate = datetime(2021, 1, 1) USHrrfs = os.path.dirname(os.path.abspath(__file__)) + PARMrrfs = os.path.join(USHrrfs, "..", "parm") EXPTDIR = os.path.join(USHrrfs, "test_data", "expt") cp_vrfy( os.path.join(PARMrrfs, "input.nml.FV3"), diff --git a/ush/set_FV3nml_sfc_climo_filenames.py b/ush/set_FV3nml_sfc_climo_filenames.py index bb2308472e..419b45721a 100644 --- a/ush/set_FV3nml_sfc_climo_filenames.py +++ b/ush/set_FV3nml_sfc_climo_filenames.py @@ -34,7 +34,7 @@ def set_FV3nml_sfc_climo_filenames(): climatology files on the FV3LAM native grid (which are either pregenerated or created by the MAKE_SFC_CLIMO_TN task). Note that the workflow generation scripts create symlinks to these surface climatology files - in the FIXLAM directory, and the values in the namelist file that get + in the FIXlam directory, and the values in the namelist file that get set by this function are relative or full paths to these links. Args: @@ -70,7 +70,7 @@ def set_FV3nml_sfc_climo_filenames(): check_var_valid_value(sfc_climo_field_name, SFC_CLIMO_FIELDS) - fp = os.path.join(FIXLAM, f"{CRES}.{sfc_climo_field_name}.{suffix}") + fp = os.path.join(FIXlam, f"{CRES}.{sfc_climo_field_name}.{suffix}") if RUN_ENVIR != "nco": fp = os.path.relpath(os.path.realpath(fp), start=dummy_run_dir) @@ -150,16 +150,17 @@ def setUp(self): set_env_var("DEBUG", True) set_env_var("VERBOSE", True) USHrrfs = os.path.dirname(os.path.abspath(__file__)) + PARMrrfs = os.path.join(USHrrfs, "..", "parm") EXPTDIR = os.path.join(USHrrfs, "test_data", "expt") - FIXLAM = os.path.join(EXPTDIR, "fix_lam") - mkdir_vrfy("-p", FIXLAM) + FIXlam = os.path.join(EXPTDIR, "fix_lam") + mkdir_vrfy("-p", FIXlam) cp_vrfy( os.path.join(PARMrrfs, "input.nml.FV3"), os.path.join(EXPTDIR, "input.nml"), ) set_env_var("USHrrfs", USHrrfs) set_env_var("EXPTDIR", EXPTDIR) - set_env_var("FIXLAM", FIXLAM) + set_env_var("FIXlam", FIXlam) set_env_var("DO_ENSEMBLE", False) set_env_var("CRES", "C3357") set_env_var("RUN_ENVIR", "nco") diff --git a/ush/setup.py b/ush/setup.py index 02776b2e0a..782bfc07a2 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -349,7 +349,7 @@ def setup(): # Define some other useful paths # global USHrrfs, SCRIPTSrrfs, JOBSrrfs, SORCrrfs, PARMrrfs, MODULESrrfs - global EXECrrfs, PARMrrfs, VX_CONFIG_DIR, METPLUS_CONF, MET_CONFIG + global EXECrrfs, PARMrrfs, FIXrrfs, VX_CONFIG_DIR, METPLUS_CONF, MET_CONFIG USHrrfs = os.path.join(HOMErrfs, "ush") SCRIPTSrrfs = os.path.join(HOMErrfs, "scripts") @@ -358,7 +358,7 @@ def setup(): PARMrrfs = os.path.join(HOMErrfs, "parm") MODULESrrfs = os.path.join(HOMErrfs, "modulefiles") EXECrrfs = os.path.join(SR_WX_APP_TOP_DIR, EXEC_SUBDIR) - PARMrrfs = os.path.join(HOMErrfs, "parm") + FIXrrfs = os.path.join(HOMErrfs, "fix") VX_CONFIG_DIR = PARMrrfs METPLUS_CONF = os.path.join(PARMrrfs, "metplus") MET_CONFIG = os.path.join(PARMrrfs, "met") @@ -878,7 +878,7 @@ def setup(): # This is the directory that will contain the MERRA2 aerosol climatology # data file and lookup tables for optics properties # - # FIXLAM: + # FIXlam: # This is the directory that will contain the fixed files or symlinks to # the fixed files containing the grid, orography, and surface climatology # on the native FV3-LAM grid. @@ -888,7 +888,7 @@ def setup(): # # ----------------------------------------------------------------------- # - global LOGDIR, FIXam, FIXclim, FIXLAM + global LOGDIR, FIXam, FIXclim, FIXlam global POST_OUTPUT_DOMAIN_NAME global COMIN_BASEDIR, COMOUT_BASEDIR @@ -898,9 +898,9 @@ def setup(): global KEEPDATA, MAILTO, MAILCC - FIXam = os.path.join(EXPTDIR, "fix_am") - FIXclim = os.path.join(EXPTDIR, "fix_clim") - FIXLAM = os.path.join(EXPTDIR, "fix_lam") + FIXam = os.path.join(FIXrrfs, "fix_am") + FIXclim = os.path.join(FIXrrfs, "fix_clim") + FIXlam = os.path.join(FIXrrfs, "fix_lam") if RUN_ENVIR == "nco": @@ -1631,21 +1631,21 @@ def setup(): # ----------------------------------------------------------------------- # # If not running the MAKE_GRID_TN, MAKE_OROG_TN, and/or MAKE_SFC_CLIMO - # tasks, create symlinks under the FIXLAM directory to pregenerated grid, + # tasks, create symlinks under the FIXlam directory to pregenerated grid, # orography, and surface climatology files. In the process, also set # RES_IN_FIXLAM_FILENAMES, which is the resolution of the grid (in units # of number of grid points on an equivalent global uniform cubed-sphere - # grid) used in the names of the fixed files in the FIXLAM directory. + # grid) used in the names of the fixed files in the FIXlam directory. # # ----------------------------------------------------------------------- # - mkdir_vrfy(f' -p "{FIXLAM}"') + mkdir_vrfy(f' -p "{FIXlam}"') RES_IN_FIXLAM_FILENAMES = "" # # ----------------------------------------------------------------------- # # If the grid file generation task in the workflow is going to be skipped - # (because pregenerated files are available), create links in the FIXLAM + # (because pregenerated files are available), create links in the FIXlam # directory to the pregenerated grid files. # # ----------------------------------------------------------------------- @@ -1666,7 +1666,7 @@ def setup(): # # If the orography file generation task in the workflow is going to be # skipped (because pregenerated files are available), create links in - # the FIXLAM directory to the pregenerated orography files. + # the FIXlam directory to the pregenerated orography files. # # ----------------------------------------------------------------------- # @@ -1691,7 +1691,7 @@ def setup(): # # If the surface climatology file generation task in the workflow is # going to be skipped (because pregenerated files are available), create - # links in the FIXLAM directory to the pregenerated surface climatology + # links in the FIXlam directory to the pregenerated surface climatology # files. # # ----------------------------------------------------------------------- @@ -1918,13 +1918,13 @@ def setup(): "PARMrrfs": PARMrrfs, "MODULESrrfs": MODULESrrfs, "EXECrrfs": EXECrrfs, + "FIXrrfs": FIXrrfs, "FIXam": FIXam, "FIXclim": FIXclim, - "FIXLAM": FIXLAM, + "FIXlam": FIXlam, "FIXgsm": FIXgsm, "FIXaer": FIXaer, "FIXlut": FIXlut, - "PARMrrfs": PARMrrfs, "VX_CONFIG_DIR": VX_CONFIG_DIR, "METPLUS_CONF": METPLUS_CONF, "MET_CONFIG": MET_CONFIG, From 6bb881f893475d4a6d9d60751af08dec244f2831 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Mon, 29 Aug 2022 21:27:18 +0000 Subject: [PATCH 057/106] Make FIXrrfs be EXPTDIR for community mode. --- ush/setup.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ush/setup.py b/ush/setup.py index 782bfc07a2..e4a8b08cb7 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -358,7 +358,6 @@ def setup(): PARMrrfs = os.path.join(HOMErrfs, "parm") MODULESrrfs = os.path.join(HOMErrfs, "modulefiles") EXECrrfs = os.path.join(SR_WX_APP_TOP_DIR, EXEC_SUBDIR) - FIXrrfs = os.path.join(HOMErrfs, "fix") VX_CONFIG_DIR = PARMrrfs METPLUS_CONF = os.path.join(PARMrrfs, "metplus") MET_CONFIG = os.path.join(PARMrrfs, "met") @@ -897,6 +896,10 @@ def setup(): global module_ver, extmodel_ver global KEEPDATA, MAILTO, MAILCC + if RUN_ENVIR == "nco": + FIXrrfs = os.path.join(HOMErrfs, "fix") + else: + FIXrrfs = EXPTDIR FIXam = os.path.join(FIXrrfs, "fix_am") FIXclim = os.path.join(FIXrrfs, "fix_clim") From 69f37d83c8c7552df2d806ef91b300f3e3085ce5 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Tue, 30 Aug 2022 10:59:03 +0000 Subject: [PATCH 058/106] Symlink upp and ufs_utils parm files to top level parm directory. --- parm/ufs_utils | 1 + parm/upp | 1 + scripts/exregional_make_ics.sh | 2 +- scripts/exregional_make_lbcs.sh | 2 +- scripts/exregional_run_fcst.sh | 6 +++--- scripts/exregional_run_post.sh | 8 ++++---- 6 files changed, 11 insertions(+), 9 deletions(-) create mode 120000 parm/ufs_utils create mode 120000 parm/upp diff --git a/parm/ufs_utils b/parm/ufs_utils new file mode 120000 index 0000000000..87c46354a5 --- /dev/null +++ b/parm/ufs_utils @@ -0,0 +1 @@ +../../sorc/UFS_UTILS/parm/ \ No newline at end of file diff --git a/parm/upp b/parm/upp new file mode 120000 index 0000000000..6a5fa737a4 --- /dev/null +++ b/parm/upp @@ -0,0 +1 @@ +../../sorc/UPP/parm/ \ No newline at end of file diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index 3d93331531..61e8d3876e 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -503,7 +503,7 @@ settings=" 'orog_dir_target_grid': ${FIXlam}, 'orog_files_target_grid': ${CRES}${DOT_OR_USCORE}oro_data.tile${TILE_RGNL}.halo$((10#${NH4})).nc, 'vcoord_file_target_grid': ${FIXam}/global_hyblev.l65.txt, - 'varmap_file': ${UFS_UTILS_DIR}/parm/varmap_tables/${varmap_file}, + 'varmap_file': ${PARMrrfs}/ufs_utils/varmap_tables/${varmap_file}, 'data_dir_input_grid': ${extrn_mdl_staging_dir}, 'atm_files_input_grid': ${fn_atm}, 'sfc_files_input_grid': ${fn_sfc}, diff --git a/scripts/exregional_make_lbcs.sh b/scripts/exregional_make_lbcs.sh index 61dc1ebaa9..1cf8d7b5d0 100755 --- a/scripts/exregional_make_lbcs.sh +++ b/scripts/exregional_make_lbcs.sh @@ -427,7 +427,7 @@ settings=" 'orog_dir_target_grid': ${FIXlam}, 'orog_files_target_grid': ${CRES}${DOT_OR_USCORE}oro_data.tile${TILE_RGNL}.halo$((10#${NH4})).nc, 'vcoord_file_target_grid': ${FIXam}/global_hyblev.l65.txt, - 'varmap_file': ${UFS_UTILS_DIR}/parm/varmap_tables/${varmap_file}, + 'varmap_file': ${PARMrrfs}/ufs_utils/varmap_tables/${varmap_file}, 'data_dir_input_grid': ${extrn_mdl_staging_dir}, 'atm_files_input_grid': ${fn_atm}, 'grib2_file_input_grid': \"${fn_grib2}\", diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 8913813ec2..edfcc365c8 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -394,7 +394,7 @@ create_symlink_to_file target="${FIELD_DICT_FP}" \ relative="${relative_link_flag}" if [ ${WRITE_DOPOST} = "TRUE" ]; then - cp_vrfy ${UPP_DIR}/parm/nam_micro_lookup.dat ./eta_micro_lookup.dat + cp_vrfy ${PARMrrfs}/upp/nam_micro_lookup.dat ./eta_micro_lookup.dat if [ ${USE_CUSTOM_POST_CONFIG_FILE} = "TRUE" ]; then post_config_fp="${CUSTOM_POST_CONFIG_FP}" print_info_msg " @@ -402,7 +402,7 @@ if [ ${WRITE_DOPOST} = "TRUE" ]; then CUSTOM_POST_CONFIG_FP = \"${CUSTOM_POST_CONFIG_FP}\" ====================================================================" else - post_config_fp="${UPP_DIR}/parm/postxconfig-NT-fv3lam.txt" + post_config_fp="${PARMrrfs}/upp/postxconfig-NT-fv3lam.txt" print_info_msg " ==================================================================== post_config_fp = \"${post_config_fp}\" @@ -410,7 +410,7 @@ if [ ${WRITE_DOPOST} = "TRUE" ]; then fi cp_vrfy ${post_config_fp} ./postxconfig-NT_FH00.txt cp_vrfy ${post_config_fp} ./postxconfig-NT.txt - cp_vrfy ${UPP_DIR}/parm/params_grib2_tbl_new . + cp_vrfy ${PARMrrfs}/upp/params_grib2_tbl_new . # Set itag for inline-post: cat > itag < Date: Tue, 30 Aug 2022 11:20:13 +0000 Subject: [PATCH 059/106] Remove UPP_DIR and UFS_UTILS_DIR. --- ush/setup.py | 46 ---------------------------------------------- 1 file changed, 46 deletions(-) diff --git a/ush/setup.py b/ush/setup.py index e4a8b08cb7..ce89647a76 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -302,50 +302,6 @@ def setup(): build the executable, and then rerun the workflow.""" ) # - # Get the base directory of the UFS_UTILS codes. - # - external_name = "ufs_utils" - UFS_UTILS_DIR = get_ini_value(cfg, external_name, property_name) - - if not UFS_UTILS_DIR: - print_err_msg_exit( - f""" - Externals.cfg does not contain "{external_name}".""" - ) - - UFS_UTILS_DIR = os.path.join(SR_WX_APP_TOP_DIR, UFS_UTILS_DIR) - if not os.path.exists(UFS_UTILS_DIR): - print_err_msg_exit( - f""" - The base directory in which the UFS utilities source codes should be lo- - cated (UFS_UTILS_DIR) does not exist: - UFS_UTILS_DIR = \"{UFS_UTILS_DIR}\" - Please clone the external repository containing the code in this direct- - ory, build the executables, and then rerun the workflow.""" - ) - # - # Get the base directory of the UPP code. - # - external_name = "UPP" - UPP_DIR = get_ini_value(cfg, external_name, property_name) - if not UPP_DIR: - print_err_msg_exit( - f""" - Externals.cfg does not contain "{external_name}".""" - ) - - UPP_DIR = os.path.join(SR_WX_APP_TOP_DIR, UPP_DIR) - if not os.path.exists(UPP_DIR): - print_err_msg_exit( - f""" - The base directory in which the UPP source code should be located - (UPP_DIR) does not exist: - UPP_DIR = \"{UPP_DIR}\" - Please clone the external repository containing the code in this directory, - build the executable, and then rerun the workflow.""" - ) - - # # Define some other useful paths # global USHrrfs, SCRIPTSrrfs, JOBSrrfs, SORCrrfs, PARMrrfs, MODULESrrfs @@ -1932,10 +1888,8 @@ def setup(): "METPLUS_CONF": METPLUS_CONF, "MET_CONFIG": MET_CONFIG, "UFS_WTHR_MDL_DIR": UFS_WTHR_MDL_DIR, - "UFS_UTILS_DIR": UFS_UTILS_DIR, "SFC_CLIMO_INPUT_DIR": SFC_CLIMO_INPUT_DIR, "TOPO_DIR": TOPO_DIR, - "UPP_DIR": UPP_DIR, "EXPTDIR": EXPTDIR, "LOGDIR": LOGDIR, "GRID_DIR": GRID_DIR, From 6f9ff94765b4de49658a0e0434d47c096a69a013 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Wed, 31 Aug 2022 15:15:38 +0000 Subject: [PATCH 060/106] Define cycle with subcyc when it is non-zero. --- ush/job_preamble.sh | 6 +++++- ush/setup.py | 5 ----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/ush/job_preamble.sh b/ush/job_preamble.sh index ec671b7bc7..5bac0f9c1e 100644 --- a/ush/job_preamble.sh +++ b/ush/job_preamble.sh @@ -7,7 +7,11 @@ # #----------------------------------------------------------------------- # -export cycle="t${cyc}z" +if [ $subcyc -eq 0 ]; then + export cycle="t${cyc}z" +else + export cycle="t${cyc}${subcyc}z" +fi if [ "${RUN_ENVIR}" = "nco" ] && [ "${DO_ENSEMBLE}" = "TRUE" ]; then export dot_ensmem=".mem${ENSMEM_INDX}" else diff --git a/ush/setup.py b/ush/setup.py index ce89647a76..80fa9db35d 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -849,7 +849,6 @@ def setup(): global OPSROOT, COMROOT, PACKAGEROOT, DATAROOT, DCOMROOT, DBNROOT global SENDECF, SENDDBN, SENDDBN_NTC, SENDCOM, SENDWEB - global module_ver, extmodel_ver global KEEPDATA, MAILTO, MAILCC if RUN_ENVIR == "nco": @@ -900,8 +899,6 @@ def setup(): SENDDBN_NTC = False SENDCOM = False SENDWEB = False - module_ver = "8.5.2" - extmodel_ver = "00" KEEPDATA = True MAILTO = None MAILCC = None @@ -2124,8 +2121,6 @@ def setup(): "SENDDBN_NTC": SENDDBN_NTC, "SENDCOM": SENDCOM, "SENDWEB": SENDWEB, - "module_ver": module_ver, - "extmodel_ver": extmodel_ver, "KEEPDATA": KEEPDATA, "MAILTO": MAILTO, "MAILCC": MAILCC, From 737b1989e8d14b067701455a3e9c4e8196ef3772 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Wed, 31 Aug 2022 16:57:33 +0000 Subject: [PATCH 061/106] Don't delete COMIN_BASEDIR if it already exists. --- ush/setup.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/ush/setup.py b/ush/setup.py index 80fa9db35d..1bd6701139 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -851,15 +851,6 @@ def setup(): global SENDECF, SENDDBN, SENDDBN_NTC, SENDCOM, SENDWEB global KEEPDATA, MAILTO, MAILCC - if RUN_ENVIR == "nco": - FIXrrfs = os.path.join(HOMErrfs, "fix") - else: - FIXrrfs = EXPTDIR - - FIXam = os.path.join(FIXrrfs, "fix_am") - FIXclim = os.path.join(FIXrrfs, "fix_clim") - FIXlam = os.path.join(FIXrrfs, "fix_lam") - if RUN_ENVIR == "nco": try: @@ -872,11 +863,10 @@ def setup(): DCOMROOT = os.path.join(OPSROOT, "dcom") COMIN_BASEDIR = os.path.join(COMROOT, NET, model_ver) - check_for_preexist_dir_file(COMIN_BASEDIR, PREEXISTING_DIR_METHOD) COMOUT_BASEDIR = os.path.join(COMROOT, NET, model_ver) - check_for_preexist_dir_file(COMOUT_BASEDIR, PREEXISTING_DIR_METHOD) LOGDIR = os.path.join(COMROOT,"output", "logs") + FIXrrfs = os.path.join(HOMErrfs, "fix") else: @@ -889,6 +879,11 @@ def setup(): DCOMROOT = EXPTDIR LOGDIR = os.path.join(EXPTDIR, "log") + FIXrrfs = EXPTDIR + + FIXam = os.path.join(FIXrrfs, "fix_am") + FIXclim = os.path.join(FIXrrfs, "fix_clim") + FIXlam = os.path.join(FIXrrfs, "fix_lam") try: DBNROOT From f40da27a48ce4b476249bf503de3005b40809986 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Wed, 31 Aug 2022 18:27:42 +0000 Subject: [PATCH 062/106] Disassociate NCO mode from pre-generated grid. --- parm/FV3LAM_wflow.xml | 9 ++ ush/config_defaults.yaml | 6 +- ush/setup.py | 301 +++++++++++---------------------------- 3 files changed, 98 insertions(+), 218 deletions(-) diff --git a/parm/FV3LAM_wflow.xml b/parm/FV3LAM_wflow.xml index 744d628571..aef46787a7 100644 --- a/parm/FV3LAM_wflow.xml +++ b/parm/FV3LAM_wflow.xml @@ -196,6 +196,9 @@ MODULES_RUN_TASK_FP script. &LOGDIR;/&MAKE_GRID_TN;.log GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + PDY@Y@m@d + cyc@H + subcyc@M {%- endif %} @@ -218,6 +221,9 @@ MODULES_RUN_TASK_FP script. &LOGDIR;/&MAKE_OROG_TN;.log GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + PDY@Y@m@d + cyc@H + subcyc@M @@ -248,6 +254,9 @@ MODULES_RUN_TASK_FP script. &LOGDIR;/&MAKE_SFC_CLIMO_TN;.log GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + PDY@Y@m@d + cyc@H + subcyc@M diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 102b6da635..3323e822ac 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -792,7 +792,7 @@ task_make_grid: # #----------------------------------------------------------------------- # - GRID_DIR: "/path/to/pregenerated/grid/files" + GRID_DIR: "" # #----------------------------------------------------------------------- # @@ -1073,7 +1073,7 @@ task_make_orog: KMP_AFFINITY_MAKE_OROG: "disabled" OMP_NUM_THREADS_MAKE_OROG: 6 OMP_STACKSIZE_MAKE_OROG: "2048m" - OROG_DIR: "/path/to/pregenerated/orog/files" + OROG_DIR: "" #---------------------------- # MAKE SFC CLIMO config parameters @@ -1087,7 +1087,7 @@ task_make_sfc_climo: KMP_AFFINITY_MAKE_SFC_CLIMO: "scatter" OMP_NUM_THREADS_MAKE_SFC_CLIMO: 1 OMP_STACKSIZE_MAKE_SFC_CLIMO: "1024m" - SFC_CLIMO_DIR: "/path/to/pregenerated/surface/climo/files" + SFC_CLIMO_DIR: "" #---------------------------- # EXTRN ICS config parameters diff --git a/ush/setup.py b/ush/setup.py index 1bd6701139..6c441678f8 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -1218,6 +1218,47 @@ def setup(): # global LOAD_MODULES_RUN_TASK_FP LOAD_MODULES_RUN_TASK_FP = os.path.join(USHrrfs, "load_modules_run_task.sh") + + # + # ----------------------------------------------------------------------- + # + # Turn off some tasks that can not be run in NCO mode + # + # ----------------------------------------------------------------------- + # + global RUN_TASK_MAKE_GRID, RUN_TASK_MAKE_OROG, RUN_TASK_MAKE_SFC_CLIMO + global RUN_TASK_VX_GRIDSTAT, RUN_TASK_VX_POINTSTAT, RUN_TASK_VX_ENSGRID, RUN_TASK_VX_ENSPOINT + + if RUN_ENVIR == "nco": + if RUN_TASK_VX_GRIDSTAT: + RUN_TASK_VX_GRIDSTAT = False + print_info_msg("""Turning off task RUN_TASK_VX_GRIDSTAT""") + if RUN_TASK_VX_POINTSTAT: + RUN_TASK_VX_POINTSTAT = False + print_info_msg("""Turning off task RUN_TASK_VX_POINTSTAT""") + if RUN_TASK_VX_ENSPOINT: + RUN_TASK_VX_ENSPOINT = False + print_info_msg("""Turning off task RUN_TASK_VX_ENSPOINT""") + if RUN_TASK_VX_ENSGRID: + RUN_TASK_VX_ENSGRID = False + print_info_msg("""Turning off task RUN_TASK_VX_ENSGRID""") + + # + # ----------------------------------------------------------------------- + # + # Make sure that DO_ENSEMBLE is set to TRUE when running ensemble vx. + # + # ----------------------------------------------------------------------- + # + if (not DO_ENSEMBLE) and (RUN_TASK_VX_ENSGRID or RUN_TASK_VX_ENSPOINT): + print_err_msg_exit( + f''' + Ensemble verification can not be run unless running in ensemble mode: + DO_ENSEMBLE = \"{DO_ENSEMBLE}\" + RUN_TASK_VX_ENSGRID = \"{RUN_TASK_VX_ENSGRID}\" + RUN_TASK_VX_ENSPOINT = \"{RUN_TASK_VX_ENSPOINT}\"''' + ) + # # ----------------------------------------------------------------------- # @@ -1242,225 +1283,47 @@ def setup(): # # ---------------------------------------------------------------------- # - global RUN_TASK_MAKE_GRID, RUN_TASK_MAKE_OROG, RUN_TASK_MAKE_SFC_CLIMO global GRID_DIR, OROG_DIR, SFC_CLIMO_DIR - global RUN_TASK_VX_GRIDSTAT, RUN_TASK_VX_POINTSTAT, RUN_TASK_VX_ENSGRID # - # ----------------------------------------------------------------------- - # - # Make sure that DO_ENSEMBLE is set to TRUE when running ensemble vx. + # If RUN_TASK_MAKE_GRID is set to False, the workflow will look for + # the pregenerated grid files in GRID_DIR. In this case, make sure that + # GRID_DIR exists. Otherwise, set it to a predefined location under the + # experiment directory (EXPTDIR). # - # ----------------------------------------------------------------------- - # - if (not DO_ENSEMBLE) and (RUN_TASK_VX_ENSGRID or RUN_TASK_VX_ENSPOINT): - print_err_msg_exit( - f''' - Ensemble verification can not be run unless running in ensemble mode: - DO_ENSEMBLE = \"{DO_ENSEMBLE}\" - RUN_TASK_VX_ENSGRID = \"{RUN_TASK_VX_ENSGRID}\" - RUN_TASK_VX_ENSPOINT = \"{RUN_TASK_VX_ENSPOINT}\"''' - ) - - if RUN_ENVIR == "nco": - - nco_fix_dir = os.path.join(DOMAIN_PREGEN_BASEDIR, PREDEF_GRID_NAME) - if not os.path.exists(nco_fix_dir): - print_err_msg_exit( - f''' - The directory (nco_fix_dir) that should contain the pregenerated grid, - orography, and surface climatology files does not exist: - nco_fix_dir = \"{nco_fix_dir}\"''' - ) - - if RUN_TASK_MAKE_GRID or (not RUN_TASK_MAKE_GRID and GRID_DIR != nco_fix_dir): - - msg = f""" - When RUN_ENVIR is set to \"nco\", the workflow assumes that pregenerated - grid files already exist in the directory - - {DOMAIN_PREGEN_BASEDIR}/{PREDEF_GRID_NAME} - - where - - DOMAIN_PREGEN_BASEDIR = \"{DOMAIN_PREGEN_BASEDIR}\" - PREDEF_GRID_NAME = \"{PREDEF_GRID_NAME}\" - - Thus, the MAKE_GRID_TN task must not be run (i.e. RUN_TASK_MAKE_GRID must - be set to \"FALSE\"), and the directory in which to look for the grid - files (i.e. GRID_DIR) must be set to the one above. Current values for - these quantities are: - - RUN_TASK_MAKE_GRID = \"{RUN_TASK_MAKE_GRID}\" - GRID_DIR = \"{GRID_DIR}\" - - Resetting RUN_TASK_MAKE_GRID to \"FALSE\" and GRID_DIR to the one above. - Reset values are: - """ - - RUN_TASK_MAKE_GRID = False - GRID_DIR = nco_fix_dir + if not RUN_TASK_MAKE_GRID: + if (GRID_DIR is None) or (not os.path.exists(GRID_DIR)): + GRID_DIR = os.path.join(DOMAIN_PREGEN_BASEDIR, PREDEF_GRID_NAME) - msg += f""" - RUN_TASK_MAKE_GRID = \"{RUN_TASK_MAKE_GRID}\" + msg = f"""Setting GRID_DIR to: GRID_DIR = \"{GRID_DIR}\" """ - - print_info_msg(msg) - - if RUN_TASK_MAKE_OROG or (not RUN_TASK_MAKE_OROG and OROG_DIR != nco_fix_dir): - - msg = f""" - When RUN_ENVIR is set to \"nco\", the workflow assumes that pregenerated - orography files already exist in the directory - {DOMAIN_PREGEN_BASEDIR}/{PREDEF_GRID_NAME} - - where - - DOMAIN_PREGEN_BASEDIR = \"{DOMAIN_PREGEN_BASEDIR}\" - PREDEF_GRID_NAME = \"{PREDEF_GRID_NAME}\" - - Thus, the MAKE_OROG_TN task must not be run (i.e. RUN_TASK_MAKE_OROG must - be set to \"FALSE\"), and the directory in which to look for the orography - files (i.e. OROG_DIR) must be set to the one above. Current values for - these quantities are: - - RUN_TASK_MAKE_OROG = \"{RUN_TASK_MAKE_OROG}\" - OROG_DIR = \"{OROG_DIR}\" - - Resetting RUN_TASK_MAKE_OROG to \"FALSE\" and OROG_DIR to the one above. - Reset values are: - """ - - RUN_TASK_MAKE_OROG = False - OROG_DIR = nco_fix_dir - - msg += f""" - RUN_TASK_MAKE_OROG = \"{RUN_TASK_MAKE_OROG}\" - OROG_DIR = \"{OROG_DIR}\" - """ - - print_info_msg(msg) - - if RUN_TASK_MAKE_SFC_CLIMO or ( - not RUN_TASK_MAKE_SFC_CLIMO and SFC_CLIMO_DIR != nco_fix_dir - ): - - msg = f""" - When RUN_ENVIR is set to \"nco\", the workflow assumes that pregenerated - surface climatology files already exist in the directory - - {DOMAIN_PREGEN_BASEDIR}/{PREDEF_GRID_NAME} - - where - - DOMAIN_PREGEN_BASEDIR = \"{DOMAIN_PREGEN_BASEDIR}\" - PREDEF_GRID_NAME = \"{PREDEF_GRID_NAME}\" - - Thus, the MAKE_SFC_CLIMO_TN task must not be run (i.e. RUN_TASK_MAKE_SFC_CLIMO - must be set to \"FALSE\"), and the directory in which to look for the - surface climatology files (i.e. SFC_CLIMO_DIR) must be set to the one - above. Current values for these quantities are: - - RUN_TASK_MAKE_SFC_CLIMO = \"{RUN_TASK_MAKE_SFC_CLIMO}\" - SFC_CLIMO_DIR = \"{SFC_CLIMO_DIR}\" - - Resetting RUN_TASK_MAKE_SFC_CLIMO to \"FALSE\" and SFC_CLIMO_DIR to the - one above. Reset values are: - """ - - RUN_TASK_MAKE_SFC_CLIMO = False - SFC_CLIMO_DIR = nco_fix_dir - - msg += f""" - RUN_TASK_MAKE_SFC_CLIMO = \"{RUN_TASK_MAKE_SFC_CLIMO}\" - SFC_CLIMO_DIR = \"{SFC_CLIMO_DIR}\" - """ - - print_info_msg(msg) - - if RUN_TASK_VX_GRIDSTAT: - - msg = f""" - When RUN_ENVIR is set to \"nco\", it is assumed that the verification - will not be run. - RUN_TASK_VX_GRIDSTAT = \"{RUN_TASK_VX_GRIDSTAT}\" - Resetting RUN_TASK_VX_GRIDSTAT to \"FALSE\" - Reset value is:""" - - RUN_TASK_VX_GRIDSTAT = False - - msg += f""" - RUN_TASK_VX_GRIDSTAT = \"{RUN_TASK_VX_GRIDSTAT}\" - """ - - print_info_msg(msg) - - if RUN_TASK_VX_POINTSTAT: - - msg = f""" - When RUN_ENVIR is set to \"nco\", it is assumed that the verification - will not be run. - RUN_TASK_VX_POINTSTAT = \"{RUN_TASK_VX_POINTSTAT}\" - Resetting RUN_TASK_VX_POINTSTAT to \"FALSE\" - Reset value is:""" - - RUN_TASK_VX_POINTSTAT = False - - msg = f""" - RUN_TASK_VX_POINTSTAT = \"{RUN_TASK_VX_POINTSTAT}\" - """ - print_info_msg(msg) - if RUN_TASK_VX_ENSGRID: - - msg = f""" - When RUN_ENVIR is set to \"nco\", it is assumed that the verification - will not be run. - RUN_TASK_VX_ENSGRID = \"{RUN_TASK_VX_ENSGRID}\" - Resetting RUN_TASK_VX_ENSGRID to \"FALSE\" - Reset value is:""" - - RUN_TASK_VX_ENSGRID = False - - msg += f""" - RUN_TASK_VX_ENSGRID = \"{RUN_TASK_VX_ENSGRID}\" - """ - - print_info_msg(msg) - - # - # ----------------------------------------------------------------------- - # - # Now consider community mode. - # - # ----------------------------------------------------------------------- - # - else: - # - # If RUN_TASK_MAKE_GRID is set to False, the workflow will look for - # the pregenerated grid files in GRID_DIR. In this case, make sure that - # GRID_DIR exists. Otherwise, set it to a predefined location under the - # experiment directory (EXPTDIR). - # - if not RUN_TASK_MAKE_GRID: - if not os.path.exists(GRID_DIR): + if not os.path.exists(GRID_DIR): print_err_msg_exit( f''' The directory (GRID_DIR) that should contain the pregenerated grid files does not exist: GRID_DIR = \"{GRID_DIR}\"''' ) - else: - GRID_DIR = os.path.join(EXPTDIR, "grid") - # - # If RUN_TASK_MAKE_OROG is set to False, the workflow will look for - # the pregenerated orography files in OROG_DIR. In this case, make sure - # that OROG_DIR exists. Otherwise, set it to a predefined location under - # the experiment directory (EXPTDIR). - # - if not RUN_TASK_MAKE_OROG: + else: + GRID_DIR = os.path.join(EXPTDIR, "grid") + # + # If RUN_TASK_MAKE_OROG is set to False, the workflow will look for + # the pregenerated orography files in OROG_DIR. In this case, make sure + # that OROG_DIR exists. Otherwise, set it to a predefined location under + # the experiment directory (EXPTDIR). + # + if not RUN_TASK_MAKE_OROG: + if (OROG_DIR is None) or (not os.path.exists(OROG_DIR)): + OROG_DIR = os.path.join(DOMAIN_PREGEN_BASEDIR, PREDEF_GRID_NAME) + + msg = f"""Setting OROG_DIR to: + OROG_DIR = \"{OROG_DIR}\" + """ + print_info_msg(msg) + if not os.path.exists(OROG_DIR): print_err_msg_exit( f''' @@ -1468,15 +1331,23 @@ def setup(): files does not exist: OROG_DIR = \"{OROG_DIR}\"''' ) - else: - OROG_DIR = os.path.join(EXPTDIR, "orog") - # - # If RUN_TASK_MAKE_SFC_CLIMO is set to False, the workflow will look - # for the pregenerated surface climatology files in SFC_CLIMO_DIR. In - # this case, make sure that SFC_CLIMO_DIR exists. Otherwise, set it to - # a predefined location under the experiment directory (EXPTDIR). - # - if not RUN_TASK_MAKE_SFC_CLIMO: + else: + OROG_DIR = os.path.join(EXPTDIR, "orog") + # + # If RUN_TASK_MAKE_SFC_CLIMO is set to False, the workflow will look + # for the pregenerated surface climatology files in SFC_CLIMO_DIR. In + # this case, make sure that SFC_CLIMO_DIR exists. Otherwise, set it to + # a predefined location under the experiment directory (EXPTDIR). + # + if not RUN_TASK_MAKE_SFC_CLIMO: + if (SFC_CLIMO_DIR is None) or (not os.path.exists(SFC_CLIMO_DIR)): + SFC_CLIMO_DIR = os.path.join(DOMAIN_PREGEN_BASEDIR, PREDEF_GRID_NAME) + + msg = f"""Setting SFC_CLIMO_DIR to: + SFC_CLIMO_DIR = \"{SFC_CLIMO_DIR}\" + """ + print_info_msg(msg) + if not os.path.exists(SFC_CLIMO_DIR): print_err_msg_exit( f''' @@ -1484,8 +1355,8 @@ def setup(): climatology files does not exist: SFC_CLIMO_DIR = \"{SFC_CLIMO_DIR}\"''' ) - else: - SFC_CLIMO_DIR = os.path.join(EXPTDIR, "sfc_climo") + else: + SFC_CLIMO_DIR = os.path.join(EXPTDIR, "sfc_climo") # ----------------------------------------------------------------------- # From 3606a5194af2bd8af4c7b967d149427f741c8fae Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Wed, 31 Aug 2022 22:16:09 +0000 Subject: [PATCH 063/106] Don't choose fix location based on RUN_ENVIR. --- ush/setup.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/ush/setup.py b/ush/setup.py index 6c441678f8..636e88b9e9 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -851,6 +851,7 @@ def setup(): global SENDECF, SENDDBN, SENDDBN_NTC, SENDCOM, SENDWEB global KEEPDATA, MAILTO, MAILCC + # Main directory locations if RUN_ENVIR == "nco": try: @@ -866,7 +867,6 @@ def setup(): COMOUT_BASEDIR = os.path.join(COMROOT, NET, model_ver) LOGDIR = os.path.join(COMROOT,"output", "logs") - FIXrrfs = os.path.join(HOMErrfs, "fix") else: @@ -879,11 +879,6 @@ def setup(): DCOMROOT = EXPTDIR LOGDIR = os.path.join(EXPTDIR, "log") - FIXrrfs = EXPTDIR - - FIXam = os.path.join(FIXrrfs, "fix_am") - FIXclim = os.path.join(FIXrrfs, "fix_clim") - FIXlam = os.path.join(FIXrrfs, "fix_lam") try: DBNROOT @@ -1243,6 +1238,16 @@ def setup(): RUN_TASK_VX_ENSGRID = False print_info_msg("""Turning off task RUN_TASK_VX_ENSGRID""") + # Fix file location + if RUN_TASK_MAKE_GRID: + FIXrrfs = EXPTDIR + else: + FIXrrfs = os.path.join(HOMErrfs, "fix") + + FIXam = os.path.join(FIXrrfs, "fix_am") + FIXclim = os.path.join(FIXrrfs, "fix_clim") + FIXlam = os.path.join(FIXrrfs, "fix_lam") + # # ----------------------------------------------------------------------- # From 7e155b2390f394a5228e7812e6a856e916746d62 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Thu, 1 Sep 2022 03:05:10 +0000 Subject: [PATCH 064/106] Bug fix in make_lbcs. --- scripts/exregional_make_lbcs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/exregional_make_lbcs.sh b/scripts/exregional_make_lbcs.sh index 1cf8d7b5d0..90e6e53149 100755 --- a/scripts/exregional_make_lbcs.sh +++ b/scripts/exregional_make_lbcs.sh @@ -88,7 +88,7 @@ if [ $RUN_ENVIR = "nco" ]; then extrn_mdl_staging_dir="${COMIN}" extrn_mdl_var_defns_fp="${extrn_mdl_staging_dir}/${NET}.${cycle}.${EXTRN_MDL_NAME_LBCS}.LBCS.${EXTRN_MDL_VAR_DEFNS_FN}.sh" else - extrn_mdl_staging_dir="${COMIN}/${EXTRN_MDL_NAME_ICS}/for_LBCS" + extrn_mdl_staging_dir="${COMIN}/${EXTRN_MDL_NAME_LBCS}/for_LBCS" extrn_mdl_var_defns_fp="${extrn_mdl_staging_dir}/${EXTRN_MDL_VAR_DEFNS_FN}.sh" fi . ${extrn_mdl_var_defns_fp} From ca152f9976d931c672d67e850ecbf44d2b96bbbe Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Thu, 1 Sep 2022 03:46:32 +0000 Subject: [PATCH 065/106] Add flag to symlink or copy fix files. --- ush/config_defaults.yaml | 2 ++ ush/generate_FV3LAM_wflow.py | 16 +++++++++++----- ush/setup.py | 12 ++++++++---- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 3323e822ac..7ce1c8a4da 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -1660,6 +1660,8 @@ task_run_fcst: # Because the default values are dependent on the platform, we set these # to a null string which will then be overwritten in setup.sh unless the # user has specified a different value in config.sh + # + SYMLINK_FIX_FILES: true FIXgsm: "" FIXaer: "" FIXlut: "" diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index 790792a7c2..5f8341d796 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -510,9 +510,11 @@ def generate_FV3LAM_wflow(): # # ----------------------------------------------------------------------- # - # First, consider NCO mode. + + # + # Symlink fix files # - if RUN_ENVIR == "nco": + if SYMLINK_FIX_FILES: ln_vrfy(f'''-fsn "{FIXgsm}" "{FIXam}"''') # @@ -537,7 +539,7 @@ def generate_FV3LAM_wflow(): the experiment generation script.""" ) # - # Now consider community mode. + # Copy relevant fix files. # else: @@ -579,8 +581,12 @@ def generate_FV3LAM_wflow(): check_for_preexist_dir_file(FIXclim, "delete") mkdir_vrfy("-p", FIXclim) - cp_vrfy(os.path.join(FIXaer, "merra2.aerclim*.nc"), FIXclim) - cp_vrfy(os.path.join(FIXlut, "optics*.dat"), FIXclim) + if SYMLINK_FIX_FILES: + ln_vrfy("-fsn", os.path.join(FIXaer, "merra2.aerclim*.nc"), FIXclim) + ln_vrfy("-fsn", os.path.join(FIXlut, "optics*.dat"), FIXclim) + else: + cp_vrfy(os.path.join(FIXaer, "merra2.aerclim*.nc"), FIXclim) + cp_vrfy(os.path.join(FIXlut, "optics*.dat"), FIXclim) # # ----------------------------------------------------------------------- # diff --git a/ush/setup.py b/ush/setup.py index 636e88b9e9..5fb0b4643b 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -1223,20 +1223,24 @@ def setup(): # global RUN_TASK_MAKE_GRID, RUN_TASK_MAKE_OROG, RUN_TASK_MAKE_SFC_CLIMO global RUN_TASK_VX_GRIDSTAT, RUN_TASK_VX_POINTSTAT, RUN_TASK_VX_ENSGRID, RUN_TASK_VX_ENSPOINT + global SYMLINK_FIX_FILES if RUN_ENVIR == "nco": if RUN_TASK_VX_GRIDSTAT: RUN_TASK_VX_GRIDSTAT = False - print_info_msg("""Turning off task RUN_TASK_VX_GRIDSTAT""") + print_info_msg("""Setting RUN_TASK_VX_GRIDSTAT={RUN_TASK_VX_GRIDSTAT}""") if RUN_TASK_VX_POINTSTAT: RUN_TASK_VX_POINTSTAT = False - print_info_msg("""Turning off task RUN_TASK_VX_POINTSTAT""") + print_info_msg("""Setting RUN_TASK_VX_POINTSTAT={RUN_TASK_VX_POINTSTAT}""") if RUN_TASK_VX_ENSPOINT: RUN_TASK_VX_ENSPOINT = False - print_info_msg("""Turning off task RUN_TASK_VX_ENSPOINT""") + print_info_msg("""Setting RUN_TASK_VX_ENSPOINT={RUN_TASK_VX_ENSPOINT}""") if RUN_TASK_VX_ENSGRID: RUN_TASK_VX_ENSGRID = False - print_info_msg("""Turning off task RUN_TASK_VX_ENSGRID""") + print_info_msg("""Setting RUN_TASK_VX_ENSGRID={RUN_TASK_VX_ENSGRID}""") + if not SYMLINK_FIX_FILES: + SYMLINK_FIX_FILES = True + print_info_msg("""Setting SYMLINK_FIX_FILES={SYMLINK_FIX_FILES}""") # Fix file location if RUN_TASK_MAKE_GRID: From 369ce06413a2e4dc144082a42ad0817392b2bf89 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Thu, 1 Sep 2022 09:34:39 +0000 Subject: [PATCH 066/106] Change slurm log file locations --- parm/FV3LAM_wflow.xml | 71 +++++++++++++++++++++++++++++++++++++------ ush/setup.py | 3 +- 2 files changed, 62 insertions(+), 12 deletions(-) diff --git a/parm/FV3LAM_wflow.xml b/parm/FV3LAM_wflow.xml index aef46787a7..21ad3122eb 100644 --- a/parm/FV3LAM_wflow.xml +++ b/parm/FV3LAM_wflow.xml @@ -87,12 +87,12 @@ specifies the number of processes per node being used (the PPN_... entities). Directories and files. --> - + {%- if run_envir == "nco" %} @@ -101,6 +101,16 @@ Directories and files. {%- endif %} +{%- if run_envir == "nco" %} +{%- if machine == "WCOSS2" %} + +{%- else %} + +{%- endif %} +{%- else %} + +{%- endif %} + @@ -199,6 +209,7 @@ MODULES_RUN_TASK_FP script. PDY@Y@m@d cyc@H subcyc@M + LOGDIR&LOGDIR; {%- endif %} @@ -224,11 +235,12 @@ MODULES_RUN_TASK_FP script. PDY@Y@m@d cyc@H subcyc@M + LOGDIR&LOGDIR; - &LOGDIR;/&MAKE_GRID_TN;_task_complete.txt + &LOGDIR;/&MAKE_GRID_TN;_task_complete.txt &RUN_TASK_MAKE_GRID;FALSE @@ -257,17 +269,18 @@ MODULES_RUN_TASK_FP script. PDY@Y@m@d cyc@H subcyc@M + LOGDIR&LOGDIR; - &LOGDIR;/&MAKE_GRID_TN;_task_complete.txt + &LOGDIR;/&MAKE_GRID_TN;_task_complete.txt &RUN_TASK_MAKE_GRID;FALSE - &LOGDIR;/&MAKE_OROG_TN;_task_complete.txt + &LOGDIR;/&MAKE_OROG_TN;_task_complete.txt &RUN_TASK_MAKE_OROG;FALSE @@ -297,6 +310,7 @@ MODULES_RUN_TASK_FP script. PDY@Y@m@d cyc@H subcyc@M + LOGDIR&LOGDIR; CDATE@Y@m@d@H ICS_OR_LBCSICS @@ -324,6 +338,7 @@ MODULES_RUN_TASK_FP script. PDY@Y@m@d cyc@H subcyc@M + LOGDIR&LOGDIR; CDATE@Y@m@d@H ICS_OR_LBCSLBCS @@ -361,6 +376,7 @@ MODULES_RUN_TASK_FP script. PDY@Y@m@d cyc@H subcyc@M + LOGDIR&LOGDIR; CDATE@Y@m@d@H SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} ENSMEM_INDX#{{ ensmem_indx_name }}# @@ -370,17 +386,17 @@ MODULES_RUN_TASK_FP script. - &LOGDIR;/&MAKE_GRID_TN;_task_complete.txt + &LOGDIR;/&MAKE_GRID_TN;_task_complete.txt &RUN_TASK_MAKE_GRID;FALSE - &LOGDIR;/&MAKE_OROG_TN;_task_complete.txt + &LOGDIR;/&MAKE_OROG_TN;_task_complete.txt &RUN_TASK_MAKE_OROG;FALSE - &LOGDIR;/&MAKE_SFC_CLIMO_TN;_task_complete.txt + &LOGDIR;/&MAKE_SFC_CLIMO_TN;_task_complete.txt &RUN_TASK_MAKE_SFC_CLIMO;FALSE @@ -410,6 +426,7 @@ MODULES_RUN_TASK_FP script. PDY@Y@m@d cyc@H subcyc@M + LOGDIR&LOGDIR; CDATE@Y@m@d@H SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} ENSMEM_INDX#{{ ensmem_indx_name }}# @@ -419,17 +436,17 @@ MODULES_RUN_TASK_FP script. - &LOGDIR;/&MAKE_GRID_TN;_task_complete.txt + &LOGDIR;/&MAKE_GRID_TN;_task_complete.txt &RUN_TASK_MAKE_GRID;FALSE - &LOGDIR;/&MAKE_OROG_TN;_task_complete.txt + &LOGDIR;/&MAKE_OROG_TN;_task_complete.txt &RUN_TASK_MAKE_OROG;FALSE - &LOGDIR;/&MAKE_SFC_CLIMO_TN;_task_complete.txt + &LOGDIR;/&MAKE_SFC_CLIMO_TN;_task_complete.txt &RUN_TASK_MAKE_SFC_CLIMO;FALSE @@ -464,6 +481,7 @@ MODULES_RUN_TASK_FP script. PDY@Y@m@d cyc@H subcyc@M + LOGDIR&LOGDIR; CDATE@Y@m@d@H SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} ENSMEM_INDX#{{ ensmem_indx_name }}# @@ -518,6 +536,7 @@ later below for other output times. PDY@Y@m@d cyc@H subcyc@M + LOGDIR&LOGDIR; fhr#fhr# fmn#fmn# CDATE@Y@m@d@H @@ -576,6 +595,7 @@ for other output times. PDY@Y@m@d cyc@H subcyc@M + LOGDIR&LOGDIR; fhr#fhr# fmn#fmn# CDATE@Y@m@d@H @@ -644,6 +664,7 @@ always zero). PDY@Y@m@d cyc@H subcyc@M + LOGDIR&LOGDIR; {%- if sub_hourly_post %} fhr#fhr# fmn#fmn# @@ -711,6 +732,7 @@ the tag to be identical to the ones above for other output times. PDY@Y@m@d cyc@H subcyc@M + LOGDIR&LOGDIR; fhr#fhr# fmn#fmn# CDATE@Y@m@d@H @@ -757,6 +779,7 @@ the tag to be identical to the ones above for other output times. PDY@Y@m@d cyc@H subcyc@M + LOGDIR&LOGDIR; FHR {% for h in range(0, fcst_len_hrs+1) %}{{ " %02d" % h }}{% endfor %} ACCUM01 @@ -787,6 +810,7 @@ the tag to be identical to the ones above for other output times. PDY@Y@m@d cyc@H subcyc@M + LOGDIR&LOGDIR; FHR {% for h in range(0, fcst_len_hrs+1) %}{{ " %02d" % h }}{% endfor %} SCRIPTSrrfs&SCRIPTSrrfs; VARREFC RETOP @@ -818,6 +842,7 @@ the tag to be identical to the ones above for other output times. PDY@Y@m@d cyc@H subcyc@M + LOGDIR&LOGDIR; FHR {% for h in range(0, fcst_len_hrs+1) %}{{ " %02d" % h }}{% endfor %} @@ -847,6 +872,7 @@ the tag to be identical to the ones above for other output times. PDY@Y@m@d cyc@H subcyc@M + LOGDIR&LOGDIR; FHR {% for h in range(1, fcst_len_hrs+1) %}{{ " %02d" % h }}{% endfor %} VARAPCP ACCUM01 @@ -901,6 +927,7 @@ the tag to be identical to the ones above for other output times. PDY@Y@m@d cyc@H subcyc@M + LOGDIR&LOGDIR; FHR {% for h in range(1, fcst_len_hrs+1) %}{{ " %02d" % h }}{% endfor %} VARREFC {%- if do_ensemble %} @@ -954,6 +981,7 @@ the tag to be identical to the ones above for other output times. PDY@Y@m@d cyc@H subcyc@M + LOGDIR&LOGDIR; FHR {% for h in range(1, fcst_len_hrs+1) %}{{ " %02d" % h }}{% endfor %} VARRETOP {%- if do_ensemble %} @@ -1007,6 +1035,7 @@ the tag to be identical to the ones above for other output times. PDY@Y@m@d cyc@H subcyc@M + LOGDIR&LOGDIR; FHR {% for h in range(3, fcst_len_hrs+1, 3) %}{{ " %02d" % h }}{% endfor %} VARAPCP ACCUM03 @@ -1046,6 +1075,7 @@ the tag to be identical to the ones above for other output times. PDY@Y@m@d cyc@H subcyc@M + LOGDIR&LOGDIR; FHR {% for h in range(6, fcst_len_hrs+1, 6) %}{{ " %02d" % h }}{% endfor %} VARAPCP ACCUM06 @@ -1085,6 +1115,7 @@ the tag to be identical to the ones above for other output times. PDY@Y@m@d cyc@H subcyc@M + LOGDIR&LOGDIR; FHR {% for h in range(24, fcst_len_hrs+1, 24) %}{{ " %02d" % h }}{% endfor %} VARAPCP ACCUM24 @@ -1124,6 +1155,7 @@ the tag to be identical to the ones above for other output times. PDY@Y@m@d cyc@H subcyc@M + LOGDIR&LOGDIR; FHR {% for h in range(0, fcst_len_hrs+1) %}{{ " %02d" % h }}{% endfor %} {%- if do_ensemble %} SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} @@ -1180,6 +1212,7 @@ the tag to be identical to the ones above for other output times. PDY@Y@m@d cyc@H subcyc@M + LOGDIR&LOGDIR; FHR {% for h in range(1, fcst_len_hrs+1) %}{{ " %02d" % h }}{% endfor %} VARAPCP ACCUM01 @@ -1215,6 +1248,7 @@ the tag to be identical to the ones above for other output times. PDY@Y@m@d cyc@H subcyc@M + LOGDIR&LOGDIR; FHR {% for h in range(3, fcst_len_hrs+1, 3) %}{{ " %02d" % h }}{% endfor %} VARAPCP ACCUM03 @@ -1250,6 +1284,7 @@ the tag to be identical to the ones above for other output times. PDY@Y@m@d cyc@H subcyc@M + LOGDIR&LOGDIR; FHR {% for h in range(6, fcst_len_hrs+1, 6) %}{{ " %02d" % h }}{% endfor %} VARAPCP ACCUM06 @@ -1285,6 +1320,7 @@ the tag to be identical to the ones above for other output times. PDY@Y@m@d cyc@H subcyc@M + LOGDIR&LOGDIR; FHR {% for h in range(24, fcst_len_hrs+1, 24) %}{{ " %02d" % h }}{% endfor %} VARAPCP ACCUM24 @@ -1319,6 +1355,7 @@ the tag to be identical to the ones above for other output times. PDY@Y@m@d cyc@H subcyc@M + LOGDIR&LOGDIR; FHR {% for h in range(1, fcst_len_hrs+1) %}{{ " %02d" % h }}{% endfor %} VARREFC @@ -1352,6 +1389,7 @@ the tag to be identical to the ones above for other output times. PDY@Y@m@d cyc@H subcyc@M + LOGDIR&LOGDIR; FHR {% for h in range(1, fcst_len_hrs+1) %}{{ " %02d" % h }}{% endfor %} VARRETOP @@ -1384,6 +1422,7 @@ the tag to be identical to the ones above for other output times. PDY@Y@m@d cyc@H subcyc@M + LOGDIR&LOGDIR; FHR {% for h in range(1, fcst_len_hrs+1) %}{{ " %02d" % h }}{% endfor %} VARAPCP ACCUM01 @@ -1418,6 +1457,7 @@ the tag to be identical to the ones above for other output times. PDY@Y@m@d cyc@H subcyc@M + LOGDIR&LOGDIR; FHR {% for h in range(1, fcst_len_hrs+1) %}{{ " %02d" % h }}{% endfor %} VARAPCP ACCUM01 @@ -1452,6 +1492,7 @@ the tag to be identical to the ones above for other output times. PDY@Y@m@d cyc@H subcyc@M + LOGDIR&LOGDIR; FHR {% for h in range(3, fcst_len_hrs+1, 3) %}{{ " %02d" % h }}{% endfor %} VARAPCP ACCUM03 @@ -1486,6 +1527,7 @@ the tag to be identical to the ones above for other output times. PDY@Y@m@d cyc@H subcyc@M + LOGDIR&LOGDIR; FHR {% for h in range(3, fcst_len_hrs+1, 3) %}{{ " %02d" % h }}{% endfor %} VARAPCP ACCUM03 @@ -1521,6 +1563,7 @@ the tag to be identical to the ones above for other output times. PDY@Y@m@d cyc@H subcyc@M + LOGDIR&LOGDIR; FHR {% for h in range(6, fcst_len_hrs+1, 6) %}{{ " %02d" % h }}{% endfor %} VARAPCP ACCUM06 @@ -1555,6 +1598,7 @@ the tag to be identical to the ones above for other output times. PDY@Y@m@d cyc@H subcyc@M + LOGDIR&LOGDIR; FHR {% for h in range(6, fcst_len_hrs+1, 6) %}{{ " %02d" % h }}{% endfor %} VARAPCP ACCUM06 @@ -1590,6 +1634,7 @@ the tag to be identical to the ones above for other output times. PDY@Y@m@d cyc@H subcyc@M + LOGDIR&LOGDIR; FHR {% for h in range(24, fcst_len_hrs+1, 24) %}{{ " %02d" % h }}{% endfor %} VARAPCP ACCUM24 @@ -1624,6 +1669,7 @@ the tag to be identical to the ones above for other output times. PDY@Y@m@d cyc@H subcyc@M + LOGDIR&LOGDIR; FHR {% for h in range(24, fcst_len_hrs+1, 24) %}{{ " %02d" % h }}{% endfor %} VARAPCP ACCUM24 @@ -1658,6 +1704,7 @@ the tag to be identical to the ones above for other output times. PDY@Y@m@d cyc@H subcyc@M + LOGDIR&LOGDIR; FHR {% for h in range(1, fcst_len_hrs+1) %}{{ " %02d" % h }}{% endfor %} VARREFC @@ -1691,6 +1738,7 @@ the tag to be identical to the ones above for other output times. PDY@Y@m@d cyc@H subcyc@M + LOGDIR&LOGDIR; FHR {% for h in range(1, fcst_len_hrs+1) %}{{ " %02d" % h }}{% endfor %} VARRETOP @@ -1725,6 +1773,7 @@ the tag to be identical to the ones above for other output times. PDY@Y@m@d cyc@H subcyc@M + LOGDIR&LOGDIR; FHR {% for h in range(0, fcst_len_hrs+1) %}{{ " %02d" % h }}{% endfor %} @@ -1756,6 +1805,7 @@ the tag to be identical to the ones above for other output times. PDY@Y@m@d cyc@H subcyc@M + LOGDIR&LOGDIR; FHR {% for h in range(0, fcst_len_hrs+1) %}{{ " %02d" % h }}{% endfor %} @@ -1787,6 +1837,7 @@ the tag to be identical to the ones above for other output times. PDY@Y@m@d cyc@H subcyc@M + LOGDIR&LOGDIR; FHR {% for h in range(0, fcst_len_hrs+1) %}{{ " %02d" % h }}{% endfor %} diff --git a/ush/setup.py b/ush/setup.py index 5fb0b4643b..89440c322f 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -866,7 +866,7 @@ def setup(): COMIN_BASEDIR = os.path.join(COMROOT, NET, model_ver) COMOUT_BASEDIR = os.path.join(COMROOT, NET, model_ver) - LOGDIR = os.path.join(COMROOT,"output", "logs") + LOGDIR = os.path.join(COMROOT,"output") else: @@ -1763,7 +1763,6 @@ def setup(): "SFC_CLIMO_INPUT_DIR": SFC_CLIMO_INPUT_DIR, "TOPO_DIR": TOPO_DIR, "EXPTDIR": EXPTDIR, - "LOGDIR": LOGDIR, "GRID_DIR": GRID_DIR, "OROG_DIR": OROG_DIR, "SFC_CLIMO_DIR": SFC_CLIMO_DIR, From fe22589b9af2d5f056cf9086761806195a4229b8 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Thu, 1 Sep 2022 14:01:23 +0000 Subject: [PATCH 067/106] Minor fix for inline post in nco mode. --- scripts/exregional_run_fcst.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index edfcc365c8..666c5b9eb3 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -554,7 +554,9 @@ if [ ${WRITE_DOPOST} = "TRUE" ]; then post_orig_fn="${FID}.${post_fn_suffix}" post_renamed_fn="${NET}.${cycle}${dot_ensmem}.${fid}.${post_renamed_fn_suffix}" mv_vrfy ${DATA}/${post_orig_fn} ${post_renamed_fn} - ln_vrfy -fs ${post_renamed_fn} ${FID}${symlink_suffix} + if [ $RUN_ENVIR != "nco" ]; then + ln_vrfy -fs ${post_renamed_fn} ${FID}${symlink_suffix} + fi # DBN alert if [ $SENDDBN = "TRUE" ]; then $DBNROOT/bin/dbn_alert MODEL rrfs_post ${job} ${COMOUT}/${post_renamed_fn} From e19888719f2a31c8d4c5f79ac173158fb547d1bb Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Fri, 2 Sep 2022 00:43:25 +0000 Subject: [PATCH 068/106] Add unique workflow ID to avoid clashes between different runs, while keeping the relation between different tasks, which PID can not do. --- jobs/JREGIONAL_MAKE_GRID | 2 +- jobs/JREGIONAL_MAKE_OROG | 2 +- jobs/JREGIONAL_MAKE_SFC_CLIMO | 2 +- parm/FV3LAM_wflow.xml | 110 +++++++++++++++++----------------- ush/config_defaults.yaml | 8 +++ ush/generate_FV3LAM_wflow.py | 1 + ush/job_preamble.sh | 2 +- ush/setup.py | 15 ++++- 8 files changed, 81 insertions(+), 61 deletions(-) diff --git a/jobs/JREGIONAL_MAKE_GRID b/jobs/JREGIONAL_MAKE_GRID index 23bdc0cbc0..fc6a36d76b 100755 --- a/jobs/JREGIONAL_MAKE_GRID +++ b/jobs/JREGIONAL_MAKE_GRID @@ -197,7 +197,7 @@ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." # #----------------------------------------------------------------------- # -touch "$LOGDIR/make_grid_task_complete.txt" +touch "$LOGDIR/make_grid_task_complete_${WORKFLOW_ID}.txt" # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_MAKE_OROG b/jobs/JREGIONAL_MAKE_OROG index 36620f21a8..de0705a2f7 100755 --- a/jobs/JREGIONAL_MAKE_OROG +++ b/jobs/JREGIONAL_MAKE_OROG @@ -79,7 +79,7 @@ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." # #----------------------------------------------------------------------- # -touch "$LOGDIR/make_orog_task_complete.txt" +touch "$LOGDIR/make_orog_task_complete_${WORKFLOW_ID}.txt" # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_MAKE_SFC_CLIMO b/jobs/JREGIONAL_MAKE_SFC_CLIMO index 36002d81ac..6d6f106b45 100755 --- a/jobs/JREGIONAL_MAKE_SFC_CLIMO +++ b/jobs/JREGIONAL_MAKE_SFC_CLIMO @@ -100,7 +100,7 @@ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." # #----------------------------------------------------------------------- # -touch "$LOGDIR/make_sfc_climo_task_complete.txt" +touch "$LOGDIR/make_sfc_climo_task_complete_${WORKFLOW_ID}.txt" # #----------------------------------------------------------------------- # diff --git a/parm/FV3LAM_wflow.xml b/parm/FV3LAM_wflow.xml index 21ad3122eb..759d9378fe 100644 --- a/parm/FV3LAM_wflow.xml +++ b/parm/FV3LAM_wflow.xml @@ -102,14 +102,12 @@ Directories and files. {%- endif %} {%- if run_envir == "nco" %} -{%- if machine == "WCOSS2" %} {%- else %} - -{%- endif %} -{%- else %} {%- endif %} + + @@ -157,7 +155,7 @@ tasks; and the "FCST" type is used for the RUN_FCST_TN task. {%- endfor %} - &LOGDIR;/FV3LAM_wflow.log + &LOGDIR;/FV3LAM_wflow_{{ workflow_id }}.log - &LOGDIR;/&MAKE_GRID_TN;_task_complete.txt + &LOGDIR;/&MAKE_GRID_TN;&CMPEXT; &RUN_TASK_MAKE_GRID;FALSE @@ -263,7 +261,7 @@ MODULES_RUN_TASK_FP script. &SLURM_NATIVE_CMD; {%- endif %} &MAKE_SFC_CLIMO_TN; - &LOGDIR;/&MAKE_SFC_CLIMO_TN;.log + &LOGDIR;/&MAKE_SFC_CLIMO_TN;&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; PDY@Y@m@d @@ -275,12 +273,12 @@ MODULES_RUN_TASK_FP script. - &LOGDIR;/&MAKE_GRID_TN;_task_complete.txt + &LOGDIR;/&MAKE_GRID_TN;&CMPEXT; &RUN_TASK_MAKE_GRID;FALSE - &LOGDIR;/&MAKE_OROG_TN;_task_complete.txt + &LOGDIR;/&MAKE_OROG_TN;&CMPEXT; &RUN_TASK_MAKE_OROG;FALSE @@ -304,7 +302,7 @@ MODULES_RUN_TASK_FP script. &SLURM_NATIVE_CMD; {%- endif %} &GET_EXTRN_ICS_TN; - &LOGDIR;/&GET_EXTRN_ICS_TN;_@Y@m@d@H.log + &LOGDIR;/&GET_EXTRN_ICS_TN;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; PDY@Y@m@d @@ -332,7 +330,7 @@ MODULES_RUN_TASK_FP script. &SLURM_NATIVE_CMD; {%- endif %} &GET_EXTRN_LBCS_TN; - &LOGDIR;/&GET_EXTRN_LBCS_TN;_@Y@m@d@H.log + &LOGDIR;/&GET_EXTRN_LBCS_TN;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; PDY@Y@m@d @@ -370,7 +368,7 @@ MODULES_RUN_TASK_FP script. &SLURM_NATIVE_CMD; {%- endif %} &MAKE_ICS_TN;{{ uscore_ensmem_name }} - &LOGDIR;/&MAKE_ICS_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log + &LOGDIR;/&MAKE_ICS_TN;{{ uscore_ensmem_name }}_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; PDY@Y@m@d @@ -386,17 +384,17 @@ MODULES_RUN_TASK_FP script. - &LOGDIR;/&MAKE_GRID_TN;_task_complete.txt + &LOGDIR;/&MAKE_GRID_TN;&CMPEXT; &RUN_TASK_MAKE_GRID;FALSE - &LOGDIR;/&MAKE_OROG_TN;_task_complete.txt + &LOGDIR;/&MAKE_OROG_TN;&CMPEXT; &RUN_TASK_MAKE_OROG;FALSE - &LOGDIR;/&MAKE_SFC_CLIMO_TN;_task_complete.txt + &LOGDIR;/&MAKE_SFC_CLIMO_TN;&CMPEXT; &RUN_TASK_MAKE_SFC_CLIMO;FALSE @@ -420,7 +418,7 @@ MODULES_RUN_TASK_FP script. &SLURM_NATIVE_CMD; {%- endif %} &MAKE_LBCS_TN;{{ uscore_ensmem_name }} - &LOGDIR;/&MAKE_LBCS_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log + &LOGDIR;/&MAKE_LBCS_TN;{{ uscore_ensmem_name }}_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; PDY@Y@m@d @@ -436,17 +434,17 @@ MODULES_RUN_TASK_FP script. - &LOGDIR;/&MAKE_GRID_TN;_task_complete.txt + &LOGDIR;/&MAKE_GRID_TN;&CMPEXT; &RUN_TASK_MAKE_GRID;FALSE - &LOGDIR;/&MAKE_OROG_TN;_task_complete.txt + &LOGDIR;/&MAKE_OROG_TN;&CMPEXT; &RUN_TASK_MAKE_OROG;FALSE - &LOGDIR;/&MAKE_SFC_CLIMO_TN;_task_complete.txt + &LOGDIR;/&MAKE_SFC_CLIMO_TN;&CMPEXT; &RUN_TASK_MAKE_SFC_CLIMO;FALSE @@ -475,7 +473,7 @@ MODULES_RUN_TASK_FP script. {%- endif %} {{ wtime_run_fcst }} &RUN_FCST_TN;{{ uscore_ensmem_name }} - &LOGDIR;/&RUN_FCST_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log + &LOGDIR;/&RUN_FCST_TN;{{ uscore_ensmem_name }}_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; PDY@Y@m@d @@ -530,7 +528,7 @@ later below for other output times. &SLURM_NATIVE_CMD; {%- endif %} &RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmn# - &LOGDIR;/&RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmn#_@Y@m@d@H.log + &LOGDIR;/&RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmn#_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; PDY@Y@m@d @@ -589,7 +587,7 @@ for other output times. &SLURM_NATIVE_CMD; {%- endif %} &RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmn# - &LOGDIR;/&RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmn#_@Y@m@d@H.log + &LOGDIR;/&RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmn#_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; PDY@Y@m@d @@ -654,10 +652,10 @@ always zero). {%- endif %} {%- if sub_hourly_post %} &RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmn# - &LOGDIR;/&RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmn#_@Y@m@d@H.log + &LOGDIR;/&RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmn#_@Y@m@d@H&LOGEXT; {%- else %} &RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr# - &LOGDIR;/&RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr#_@Y@m@d@H.log + &LOGDIR;/&RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr#_@Y@m@d@H&LOGEXT; {%- endif %} GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; @@ -726,7 +724,7 @@ the tag to be identical to the ones above for other output times. &SLURM_NATIVE_CMD; {%- endif %} &RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmn# - &LOGDIR;/&RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmn#_@Y@m@d@H.log + &LOGDIR;/&RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr##fmn#_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; PDY@Y@m@d @@ -771,7 +769,7 @@ the tag to be identical to the ones above for other output times. &SLURM_NATIVE_CMD; {%- endif %} &GET_OBS_CCPA_TN; - &LOGDIR;/&GET_OBS_CCPA_TN;_@Y@m@d@H.log + &LOGDIR;/&GET_OBS_CCPA_TN;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&CCPA_OBS_DIR; @@ -802,7 +800,7 @@ the tag to be identical to the ones above for other output times. &SLURM_NATIVE_CMD; {%- endif %} &GET_OBS_MRMS_TN; - &LOGDIR;/&GET_OBS_MRMS_TN;_@Y@m@d@H.log + &LOGDIR;/&GET_OBS_MRMS_TN;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&MRMS_OBS_DIR; @@ -834,7 +832,7 @@ the tag to be identical to the ones above for other output times. &SLURM_NATIVE_CMD; {%- endif %} &GET_OBS_NDAS_TN; - &LOGDIR;/&GET_OBS_NDAS_TN;_@Y@m@d@H.log + &LOGDIR;/&GET_OBS_NDAS_TN;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&NDAS_OBS_DIR; @@ -864,7 +862,7 @@ the tag to be identical to the ones above for other output times. &SLURM_NATIVE_CMD; {%- endif %} &VX_GRIDSTAT_TN; - &LOGDIR;/&VX_GRIDSTAT_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log + &LOGDIR;/&VX_GRIDSTAT_TN;{{ uscore_ensmem_name }}_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&CCPA_OBS_DIR; @@ -919,7 +917,7 @@ the tag to be identical to the ones above for other output times. &SLURM_NATIVE_CMD; {%- endif %} &VX_GRIDSTAT_REFC_TN; - &LOGDIR;/&VX_GRIDSTAT_REFC_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log + &LOGDIR;/&VX_GRIDSTAT_REFC_TN;{{ uscore_ensmem_name }}_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&MRMS_OBS_DIR; @@ -973,7 +971,7 @@ the tag to be identical to the ones above for other output times. &SLURM_NATIVE_CMD; {%- endif %} &VX_GRIDSTAT_RETOP_TN; - &LOGDIR;/&VX_GRIDSTAT_RETOP_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log + &LOGDIR;/&VX_GRIDSTAT_RETOP_TN;{{ uscore_ensmem_name }}_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&MRMS_OBS_DIR; @@ -1027,7 +1025,7 @@ the tag to be identical to the ones above for other output times. &SLURM_NATIVE_CMD; {%- endif %} &VX_GRIDSTAT_03h_TN; - &LOGDIR;/&VX_GRIDSTAT_03h_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log + &LOGDIR;/&VX_GRIDSTAT_03h_TN;{{ uscore_ensmem_name }}_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&CCPA_OBS_DIR; @@ -1067,7 +1065,7 @@ the tag to be identical to the ones above for other output times. &SLURM_NATIVE_CMD; {%- endif %} &VX_GRIDSTAT_06h_TN; - &LOGDIR;/&VX_GRIDSTAT_06h_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log + &LOGDIR;/&VX_GRIDSTAT_06h_TN;{{ uscore_ensmem_name }}_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&CCPA_OBS_DIR; @@ -1107,7 +1105,7 @@ the tag to be identical to the ones above for other output times. &SLURM_NATIVE_CMD; {%- endif %} &VX_GRIDSTAT_24h_TN; - &LOGDIR;/&VX_GRIDSTAT_24h_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log + &LOGDIR;/&VX_GRIDSTAT_24h_TN;{{ uscore_ensmem_name }}_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&CCPA_OBS_DIR; @@ -1147,7 +1145,7 @@ the tag to be identical to the ones above for other output times. &SLURM_NATIVE_CMD; {%- endif %} &VX_POINTSTAT_TN; - &LOGDIR;/&VX_POINTSTAT_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log + &LOGDIR;/&VX_POINTSTAT_TN;{{ uscore_ensmem_name }}_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&NDAS_OBS_DIR; @@ -1204,7 +1202,7 @@ the tag to be identical to the ones above for other output times. &SLURM_NATIVE_CMD; {%- endif %} &VX_ENSGRID_TN; - &LOGDIR;/&VX_ENSGRID_TN;_@Y@m@d@H.log + &LOGDIR;/&VX_ENSGRID_TN;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&CCPA_OBS_DIR; @@ -1240,7 +1238,7 @@ the tag to be identical to the ones above for other output times. &SLURM_NATIVE_CMD; {%- endif %} &VX_ENSGRID_03h_TN; - &LOGDIR;/&VX_ENSGRID_03h_TN;_@Y@m@d@H.log + &LOGDIR;/&VX_ENSGRID_03h_TN;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&CCPA_OBS_DIR; @@ -1276,7 +1274,7 @@ the tag to be identical to the ones above for other output times. &SLURM_NATIVE_CMD; {%- endif %} &VX_ENSGRID_06h_TN; - &LOGDIR;/&VX_ENSGRID_06h_TN;_@Y@m@d@H.log + &LOGDIR;/&VX_ENSGRID_06h_TN;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&CCPA_OBS_DIR; @@ -1312,7 +1310,7 @@ the tag to be identical to the ones above for other output times. &SLURM_NATIVE_CMD; {%- endif %} &VX_ENSGRID_24h_TN; - &LOGDIR;/&VX_ENSGRID_24h_TN;_@Y@m@d@H.log + &LOGDIR;/&VX_ENSGRID_24h_TN;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&CCPA_OBS_DIR; @@ -1347,7 +1345,7 @@ the tag to be identical to the ones above for other output times. &SLURM_NATIVE_CMD; {%- endif %} &VX_ENSGRID_REFC_TN; - &LOGDIR;/&VX_ENSGRID_REFC_TN;_@Y@m@d@H.log + &LOGDIR;/&VX_ENSGRID_REFC_TN;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&MRMS_OBS_DIR; @@ -1381,7 +1379,7 @@ the tag to be identical to the ones above for other output times. &SLURM_NATIVE_CMD; {%- endif %} &VX_ENSGRID_RETOP_TN; - &LOGDIR;/&VX_ENSGRID_RETOP_TN;_@Y@m@d@H.log + &LOGDIR;/&VX_ENSGRID_RETOP_TN;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&MRMS_OBS_DIR; @@ -1414,7 +1412,7 @@ the tag to be identical to the ones above for other output times. &SLURM_NATIVE_CMD; {%- endif %} &VX_ENSGRID_MEAN_TN; - &LOGDIR;/&VX_ENSGRID_MEAN_TN;_@Y@m@d@H.log + &LOGDIR;/&VX_ENSGRID_MEAN_TN;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&CCPA_OBS_DIR; @@ -1449,7 +1447,7 @@ the tag to be identical to the ones above for other output times. &SLURM_NATIVE_CMD; {%- endif %} &VX_ENSGRID_PROB_TN; - &LOGDIR;/&VX_ENSGRID_PROB_TN;_@Y@m@d@H.log + &LOGDIR;/&VX_ENSGRID_PROB_TN;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&CCPA_OBS_DIR; @@ -1484,7 +1482,7 @@ the tag to be identical to the ones above for other output times. &SLURM_NATIVE_CMD; {%- endif %} &VX_ENSGRID_MEAN_03h_TN; - &LOGDIR;/&VX_ENSGRID_MEAN_03h_TN;_@Y@m@d@H.log + &LOGDIR;/&VX_ENSGRID_MEAN_03h_TN;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&CCPA_OBS_DIR; @@ -1519,7 +1517,7 @@ the tag to be identical to the ones above for other output times. &SLURM_NATIVE_CMD; {%- endif %} &VX_ENSGRID_PROB_03h_TN; - &LOGDIR;/&VX_ENSGRID_PROB_03h_TN;_@Y@m@d@H.log + &LOGDIR;/&VX_ENSGRID_PROB_03h_TN;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&CCPA_OBS_DIR; @@ -1555,7 +1553,7 @@ the tag to be identical to the ones above for other output times. &SLURM_NATIVE_CMD; {%- endif %} &VX_ENSGRID_MEAN_06h_TN; - &LOGDIR;/&VX_ENSGRID_MEAN_06h_TN;_@Y@m@d@H.log + &LOGDIR;/&VX_ENSGRID_MEAN_06h_TN;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&CCPA_OBS_DIR; @@ -1590,7 +1588,7 @@ the tag to be identical to the ones above for other output times. &SLURM_NATIVE_CMD; {%- endif %} &VX_ENSGRID_PROB_06h_TN; - &LOGDIR;/&VX_ENSGRID_PROB_06h_TN;_@Y@m@d@H.log + &LOGDIR;/&VX_ENSGRID_PROB_06h_TN;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&CCPA_OBS_DIR; @@ -1626,7 +1624,7 @@ the tag to be identical to the ones above for other output times. &SLURM_NATIVE_CMD; {%- endif %} &VX_ENSGRID_MEAN_24h_TN; - &LOGDIR;/&VX_ENSGRID_MEAN_24h_TN;_@Y@m@d@H.log + &LOGDIR;/&VX_ENSGRID_MEAN_24h_TN;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&CCPA_OBS_DIR; @@ -1661,7 +1659,7 @@ the tag to be identical to the ones above for other output times. &SLURM_NATIVE_CMD; {%- endif %} &VX_ENSGRID_PROB_24h_TN; - &LOGDIR;/&VX_ENSGRID_PROB_24h_TN;_@Y@m@d@H.log + &LOGDIR;/&VX_ENSGRID_PROB_24h_TN;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&CCPA_OBS_DIR; @@ -1696,7 +1694,7 @@ the tag to be identical to the ones above for other output times. &SLURM_NATIVE_CMD; {%- endif %} &VX_ENSGRID_PROB_REFC_TN; - &LOGDIR;/&VX_ENSGRID_PROB_REFC_TN;_@Y@m@d@H.log + &LOGDIR;/&VX_ENSGRID_PROB_REFC_TN;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&MRMS_OBS_DIR; @@ -1730,7 +1728,7 @@ the tag to be identical to the ones above for other output times. &SLURM_NATIVE_CMD; {%- endif %} &VX_ENSGRID_PROB_RETOP_TN; - &LOGDIR;/&VX_ENSGRID_PROB_RETOP_TN;_@Y@m@d@H.log + &LOGDIR;/&VX_ENSGRID_PROB_RETOP_TN;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&MRMS_OBS_DIR; @@ -1765,7 +1763,7 @@ the tag to be identical to the ones above for other output times. &SLURM_NATIVE_CMD; {%- endif %} &VX_ENSPOINT_TN; - &LOGDIR;/&VX_ENSPOINT_TN;_@Y@m@d@H.log + &LOGDIR;/&VX_ENSPOINT_TN;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&NDAS_OBS_DIR; @@ -1797,7 +1795,7 @@ the tag to be identical to the ones above for other output times. &SLURM_NATIVE_CMD; {%- endif %} &VX_ENSPOINT_MEAN_TN; - &LOGDIR;/&VX_ENSPOINT_MEAN_TN;_@Y@m@d@H.log + &LOGDIR;/&VX_ENSPOINT_MEAN_TN;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&NDAS_OBS_DIR; @@ -1829,7 +1827,7 @@ the tag to be identical to the ones above for other output times. &SLURM_NATIVE_CMD; {%- endif %} &VX_ENSPOINT_PROB_TN; - &LOGDIR;/&VX_ENSPOINT_PROB_TN;_@Y@m@d@H.log + &LOGDIR;/&VX_ENSPOINT_PROB_TN;_@Y@m@d@H&LOGEXT; GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&NDAS_OBS_DIR; diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 7ce1c8a4da..d73750c42c 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -320,6 +320,14 @@ platform: # WORKFLOW config parameters #----------------------------- workflow: + # + #----------------------------------------------------------------------- + # + # Unique ID for workflow run that will be set in setup.py + # + #----------------------------------------------------------------------- + # + WORKFLOW_ID: "" # #----------------------------------------------------------------------- # diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index 5f8341d796..ca42f3d947 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -178,6 +178,7 @@ def generate_FV3LAM_wflow(): "queue_fcst": QUEUE_FCST, "machine": MACHINE, "slurm_native_cmd": SLURM_NATIVE_CMD, + "workflow_id": WORKFLOW_ID, # # Run environment # diff --git a/ush/job_preamble.sh b/ush/job_preamble.sh index 5bac0f9c1e..6eb15e28b8 100644 --- a/ush/job_preamble.sh +++ b/ush/job_preamble.sh @@ -28,7 +28,7 @@ export DATA= export DATA_SHARED= if [ "${RUN_ENVIR}" = "nco" ]; then export DATA=${DATAROOT}/${jobid} - export DATA_SHARED=${DATAROOT}/${RUN}.${PDY} + export DATA_SHARED=${DATAROOT}/${RUN}.${PDY}.${WORKFLOW_ID} mkdir_vrfy -p $DATA $DATA_SHARED cd $DATA fi diff --git a/ush/setup.py b/ush/setup.py index 89440c322f..a4d68389d5 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -115,6 +115,19 @@ def setup(): # cfg_c = load_config_file(os.path.join(ushdir, CONSTANTS_FN)) import_vars(dictionary=cfg_c) + + # + # ----------------------------------------------------------------------- + # + # Generate a unique number for this workflow run. This maybe used to + # get unique log file names for example + # + # ----------------------------------------------------------------------- + # + global WORKFLOW_ID + WORKFLOW_ID = "id_" + str(int(datetime.datetime.now().timestamp())) + print_info_msg(f"""WORKFLOW ID = {WORKFLOW_ID}""") + # # ----------------------------------------------------------------------- # @@ -866,7 +879,7 @@ def setup(): COMIN_BASEDIR = os.path.join(COMROOT, NET, model_ver) COMOUT_BASEDIR = os.path.join(COMROOT, NET, model_ver) - LOGDIR = os.path.join(COMROOT,"output") + LOGDIR = os.path.join(OPSROOT,"output") else: From ecc6d5975e772a857675c7d94a3a4e40970696c8 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Fri, 2 Sep 2022 03:08:25 +0000 Subject: [PATCH 069/106] Make verification tasks NCO complaint. --- parm/metplus/EnsembleStat_APCP01h.conf | 6 +-- parm/metplus/EnsembleStat_APCP03h.conf | 15 +++--- parm/metplus/EnsembleStat_APCP06h.conf | 15 +++--- parm/metplus/EnsembleStat_APCP24h.conf | 15 +++--- parm/metplus/EnsembleStat_REFC.conf | 6 +-- parm/metplus/EnsembleStat_RETOP.conf | 6 +-- parm/metplus/EnsembleStat_conus_sfc.conf | 10 ++-- parm/metplus/EnsembleStat_upper_air.conf | 10 ++-- parm/metplus/GridStat_APCP01h.conf | 6 +-- parm/metplus/GridStat_APCP01h_mean.conf | 2 +- parm/metplus/GridStat_APCP01h_prob.conf | 2 +- parm/metplus/GridStat_APCP03h.conf | 12 ++--- parm/metplus/GridStat_APCP03h_mean.conf | 4 +- parm/metplus/GridStat_APCP03h_prob.conf | 4 +- parm/metplus/GridStat_APCP06h.conf | 12 ++--- parm/metplus/GridStat_APCP06h_mean.conf | 4 +- parm/metplus/GridStat_APCP06h_prob.conf | 4 +- parm/metplus/GridStat_APCP24h.conf | 12 ++--- parm/metplus/GridStat_APCP24h_mean.conf | 4 +- parm/metplus/GridStat_APCP24h_prob.conf | 4 +- parm/metplus/GridStat_REFC.conf | 6 +-- parm/metplus/GridStat_REFC_mean.conf | 2 +- parm/metplus/GridStat_REFC_prob.conf | 2 +- parm/metplus/GridStat_RETOP.conf | 6 +-- parm/metplus/GridStat_RETOP_mean.conf | 2 +- parm/metplus/GridStat_RETOP_prob.conf | 2 +- parm/metplus/PointStat_conus_sfc.conf | 12 ++--- parm/metplus/PointStat_conus_sfc_mean.conf | 10 ++-- parm/metplus/PointStat_conus_sfc_prob.conf | 10 ++-- parm/metplus/PointStat_upper_air.conf | 12 ++--- parm/metplus/PointStat_upper_air_mean.conf | 10 ++-- parm/metplus/PointStat_upper_air_prob.conf | 10 ++-- scripts/exregional_run_vx_ensgrid.sh | 42 +++++++++------ scripts/exregional_run_vx_ensgrid_mean.sh | 37 +++++++++---- scripts/exregional_run_vx_ensgrid_prob.sh | 37 +++++++++---- scripts/exregional_run_vx_enspoint.sh | 31 ++++++++--- scripts/exregional_run_vx_enspoint_mean.sh | 33 +++++++++--- scripts/exregional_run_vx_enspoint_prob.sh | 33 +++++++++--- scripts/exregional_run_vx_gridstat.sh | 62 ++++++++++++---------- scripts/exregional_run_vx_pointstat.sh | 52 ++++++++++++------ ush/job_preamble.sh | 2 +- ush/setup.py | 12 ----- 42 files changed, 345 insertions(+), 233 deletions(-) diff --git a/parm/metplus/EnsembleStat_APCP01h.conf b/parm/metplus/EnsembleStat_APCP01h.conf index 33599d9f79..bdefb0d203 100644 --- a/parm/metplus/EnsembleStat_APCP01h.conf +++ b/parm/metplus/EnsembleStat_APCP01h.conf @@ -186,8 +186,8 @@ OBS_VAR1_OPTIONS = {FCST_VAR1_OPTIONS} [dir] # Input and output data directories -INPUT_BASE = {ENV[EXPTDIR]}/{ENV[CDATE]} -OUTPUT_BASE = {ENV[EXPTDIR]} +INPUT_BASE = {ENV[INPUT_BASE]} +OUTPUT_BASE = {ENV[OUTPUT_BASE]} # Forecast model input directory for ensemble_stat FCST_ENSEMBLE_STAT_INPUT_DIR = {INPUT_BASE} @@ -221,7 +221,7 @@ STAGING_DIR = {OUTPUT_BASE}/stage/APCP_01 # FCST_ENSEMBLE_STAT_INPUT_TEMPLATE - comma separated list of ensemble members # or a single line, - filename wildcard characters may be used, ? or *. -FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = mem*/postprd/{ENV[NET]}.t{init?fmt=%H}z.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}.grib2 +FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = {ENV[MEM_STAR]}{ENV[POSTPRD]}{ENV[NET]}.t{init?fmt=%H}z*.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}.grib2 # Template to look for point observations. # Example precip24_2010010112.nc diff --git a/parm/metplus/EnsembleStat_APCP03h.conf b/parm/metplus/EnsembleStat_APCP03h.conf index 2691dc2dca..17971d7b4f 100644 --- a/parm/metplus/EnsembleStat_APCP03h.conf +++ b/parm/metplus/EnsembleStat_APCP03h.conf @@ -11,6 +11,7 @@ LOOP_ORDER = times # Loop through ensmeble members for making precip buckets PCP_COMBINE_CUSTOM_LOOP_LIST = membegin_end_incr(1,{ENV[NUM_ENS_MEMBERS]},1,{ENV[NUM_PAD]}) +ENSEMBLE_STAT_CUSTOM_LOOP_LIST = membegin_end_incr(1,{ENV[NUM_ENS_MEMBERS]},1,{ENV[NUM_PAD]}) # LOOP_BY: Set to INIT to loop over initialization times LOOP_BY = INIT @@ -216,16 +217,16 @@ OBS_VAR1_OPTIONS = {FCST_VAR1_OPTIONS} [dir] # Input and output data directories -INPUT_BASE = {ENV[EXPTDIR]}/{ENV[CDATE]} -OUTPUT_BASE = {ENV[EXPTDIR]} +INPUT_BASE = {ENV[INPUT_BASE]} +OUTPUT_BASE = {ENV[OUTPUT_BASE]} # Input and output fcst data directories for pcp-combine FCST_PCP_COMBINE_INPUT_DIR = {INPUT_BASE} -FCST_PCP_COMBINE_OUTPUT_DIR = {INPUT_BASE} +FCST_PCP_COMBINE_OUTPUT_DIR = {ENV[MEM_BASE]} # Input and output obs data directories for pcp-combine OBS_PCP_COMBINE_INPUT_DIR = {ENV[OBS_DIR]} -OBS_PCP_COMBINE_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pcp_combine +OBS_PCP_COMBINE_OUTPUT_DIR = {ENV[OUTPUT_BASE]}/metprd/pcp_combine # Forecast model input directory for ensemble_stat FCST_ENSEMBLE_STAT_INPUT_DIR = {FCST_PCP_COMBINE_OUTPUT_DIR} @@ -256,8 +257,8 @@ STAGING_DIR = {OUTPUT_BASE}/stage/APCP_03 [filename_templates] # Need to have PCPCombine output data to individual member directories. -FCST_PCP_COMBINE_INPUT_TEMPLATE = {custom?fmt=%s}/postprd/{ENV[NET]}.t{init?fmt=%H}z.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}.grib2 -FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {custom?fmt=%s}/metprd/pcp_combine/{ENV[NET]}.t{init?fmt=%H}z.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}_a03h +FCST_PCP_COMBINE_INPUT_TEMPLATE = {ENV[MEM_CUSTOM]}{ENV[POSTPRD]}{ENV[NET]}.t{init?fmt=%H}z{ENV[DOT_MEM_CUSTOM]}.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}.grib2 +FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {ENV[MEM_CUSTOM]}metprd/pcp_combine/{ENV[NET]}.t{init?fmt=%H}z{ENV[DOT_MEM_CUSTOM]}.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}_a03h # Input and output template for obs pcp-combine files OBS_PCP_COMBINE_INPUT_TEMPLATE = {OBS_PCP_COMBINE_INPUT_DIR}/{valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.01h.hrap.conus.gb2 @@ -266,7 +267,7 @@ OBS_PCP_COMBINE_OUTPUT_TEMPLATE = {OBS_PCP_COMBINE_OUTPUT_DIR}/{valid?fmt=%Y%m%d # FCST_ENSEMBLE_STAT_INPUT_TEMPLATE - comma separated list of ensemble members # or a single line, - filename wildcard characters may be used, ? or *. -FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = membegin_end_incr(1,{ENV[NUM_ENS_MEMBERS]},1,{ENV[NUM_PAD]})/metprd/pcp_combine/{ENV[NET]}.t{init?fmt=%H}z.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}_a03h +FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = {ENV[MEM_CUSTOM]}metprd/pcp_combine/{ENV[NET]}.t{init?fmt=%H}z{ENV[DOT_MEM_CUSTOM]}.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}_a03h # Template to look for point observations. # Example precip24_2010010112.nc diff --git a/parm/metplus/EnsembleStat_APCP06h.conf b/parm/metplus/EnsembleStat_APCP06h.conf index 8527e23635..b497b31000 100644 --- a/parm/metplus/EnsembleStat_APCP06h.conf +++ b/parm/metplus/EnsembleStat_APCP06h.conf @@ -11,6 +11,7 @@ LOOP_ORDER = times # Loop through ensmeble members for making precip buckets PCP_COMBINE_CUSTOM_LOOP_LIST = membegin_end_incr(1,{ENV[NUM_ENS_MEMBERS]},1,{ENV[NUM_PAD]}) +ENSEMBLE_STAT_CUSTOM_LOOP_LIST = membegin_end_incr(1,{ENV[NUM_ENS_MEMBERS]},1,{ENV[NUM_PAD]}) # LOOP_BY: Set to INIT to loop over initialization times LOOP_BY = INIT @@ -217,16 +218,16 @@ OBS_VAR1_OPTIONS = {FCST_VAR1_OPTIONS} [dir] # Input and output data directories -INPUT_BASE = {ENV[EXPTDIR]}/{ENV[CDATE]} -OUTPUT_BASE = {ENV[EXPTDIR]} +INPUT_BASE = {ENV[INPUT_BASE]} +OUTPUT_BASE = {ENV[OUTPUT_BASE]} # Input and output fcst data directories for pcp-combine FCST_PCP_COMBINE_INPUT_DIR = {INPUT_BASE} -FCST_PCP_COMBINE_OUTPUT_DIR = {INPUT_BASE} +FCST_PCP_COMBINE_OUTPUT_DIR = {ENV[MEM_BASE]} # Input and output obs data directories for pcp-combine OBS_PCP_COMBINE_INPUT_DIR = {ENV[OBS_DIR]} -OBS_PCP_COMBINE_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pcp_combine +OBS_PCP_COMBINE_OUTPUT_DIR = {ENV[OUTPUT_BASE]}/metprd/pcp_combine # Forecast model input directory for ensemble_stat FCST_ENSEMBLE_STAT_INPUT_DIR = {FCST_PCP_COMBINE_OUTPUT_DIR} @@ -257,8 +258,8 @@ STAGING_DIR = {OUTPUT_BASE}/stage/APCP_01 [filename_templates] # Need to have PCPCombine output data to individual member directories. -FCST_PCP_COMBINE_INPUT_TEMPLATE = {custom?fmt=%s}/postprd/{ENV[NET]}.t{init?fmt=%H}z.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}.grib2 -FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {custom?fmt=%s}/metprd/pcp_combine/{ENV[NET]}.t{init?fmt=%H}z.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}_a06h +FCST_PCP_COMBINE_INPUT_TEMPLATE = {ENV[MEM_CUSTOM]}{ENV[POSTPRD]}{ENV[NET]}.t{init?fmt=%H}z{ENV[DOT_MEM_CUSTOM]}.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}.grib2 +FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {ENV[MEM_CUSTOM]}metprd/pcp_combine/{ENV[NET]}.t{init?fmt=%H}z{ENV[DOT_MEM_CUSTOM]}.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}_a06h # Input and output template for obs pcp-combine files OBS_PCP_COMBINE_INPUT_TEMPLATE = {OBS_PCP_COMBINE_INPUT_DIR}/{valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.01h.hrap.conus.gb2 @@ -267,7 +268,7 @@ OBS_PCP_COMBINE_OUTPUT_TEMPLATE = {OBS_PCP_COMBINE_OUTPUT_DIR}/{valid?fmt=%Y%m%d # FCST_ENSEMBLE_STAT_INPUT_TEMPLATE - comma separated list of ensemble members # or a single line, - filename wildcard characters may be used, ? or *. -FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = membegin_end_incr(1,{ENV[NUM_ENS_MEMBERS]},1,{ENV[NUM_PAD]})/metprd/pcp_combine/{ENV[NET]}.t{init?fmt=%H}z.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}_a06h +FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = {ENV[MEM_CUSTOM]}metprd/pcp_combine/{ENV[NET]}.t{init?fmt=%H}z{ENV[DOT_MEM_CUSTOM]}.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}_a06h # Template to look for point observations. # Example precip24_2010010112.nc diff --git a/parm/metplus/EnsembleStat_APCP24h.conf b/parm/metplus/EnsembleStat_APCP24h.conf index c3ed6a3a99..f0b7b5638d 100644 --- a/parm/metplus/EnsembleStat_APCP24h.conf +++ b/parm/metplus/EnsembleStat_APCP24h.conf @@ -11,6 +11,7 @@ LOOP_ORDER = times # Loop through ensmeble members for making precip buckets PCP_COMBINE_CUSTOM_LOOP_LIST = membegin_end_incr(1,{ENV[NUM_ENS_MEMBERS]},1,{ENV[NUM_PAD]}) +ENSEMBLE_STAT_CUSTOM_LOOP_LIST = membegin_end_incr(1,{ENV[NUM_ENS_MEMBERS]},1,{ENV[NUM_PAD]}) # LOOP_BY: Set to INIT to loop over initialization times LOOP_BY = INIT @@ -217,16 +218,16 @@ OBS_VAR1_OPTIONS = {FCST_VAR1_OPTIONS} [dir] # Input and output data directories -INPUT_BASE = {ENV[EXPTDIR]}/{ENV[CDATE]} -OUTPUT_BASE = {ENV[EXPTDIR]} +INPUT_BASE = {ENV[INPUT_BASE]} +OUTPUT_BASE = {ENV[OUTPUT_BASE]} # Input and output fcst data directories for pcp-combine FCST_PCP_COMBINE_INPUT_DIR = {INPUT_BASE} -FCST_PCP_COMBINE_OUTPUT_DIR = {INPUT_BASE} +FCST_PCP_COMBINE_OUTPUT_DIR = {ENV[MEM_BASE]} # Input and output obs data directories for pcp-combine OBS_PCP_COMBINE_INPUT_DIR = {ENV[OBS_DIR]} -OBS_PCP_COMBINE_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pcp_combine +OBS_PCP_COMBINE_OUTPUT_DIR = {ENV[OUTPUT_BASE]}/metprd/pcp_combine # Forecast model input directory for ensemble_stat FCST_ENSEMBLE_STAT_INPUT_DIR = {FCST_PCP_COMBINE_OUTPUT_DIR} @@ -257,8 +258,8 @@ STAGING_DIR = {OUTPUT_BASE}/stage/APCP_01 [filename_templates] # Need to have PCPCombine output data to individual member directories. -FCST_PCP_COMBINE_INPUT_TEMPLATE = {custom?fmt=%s}/postprd/{ENV[NET]}.t{init?fmt=%H}z.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}.grib2 -FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {custom?fmt=%s}/metprd/pcp_combine/{ENV[NET]}.t{init?fmt=%H}z.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}_a24h +FCST_PCP_COMBINE_INPUT_TEMPLATE = {ENV[MEM_CUSTOM]}{ENV[POSTPRD]}{ENV[NET]}.t{init?fmt=%H}z{ENV[DOT_MEM_CUSTOM]}.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}.grib2 +FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {ENV[MEM_CUSTOM]}metprd/pcp_combine/{ENV[NET]}.t{init?fmt=%H}z{ENV[DOT_MEM_CUSTOM]}.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}_a24h # Input and output template for obs pcp-combine files OBS_PCP_COMBINE_INPUT_TEMPLATE = {OBS_PCP_COMBINE_INPUT_DIR}/{valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.01h.hrap.conus.gb2 @@ -267,7 +268,7 @@ OBS_PCP_COMBINE_OUTPUT_TEMPLATE = {OBS_PCP_COMBINE_OUTPUT_DIR}/{valid?fmt=%Y%m%d # FCST_ENSEMBLE_STAT_INPUT_TEMPLATE - comma separated list of ensemble members # or a single line, - filename wildcard characters may be used, ? or *. -FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = membegin_end_incr(1,{ENV[NUM_ENS_MEMBERS]},1,{ENV[NUM_PAD]})/metprd/pcp_combine/{ENV[NET]}.t{init?fmt=%H}z.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}_a24h +FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = {ENV[MEM_CUSTOM]}metprd/pcp_combine/{ENV[NET]}.t{init?fmt=%H}z{ENV[DOT_MEM_CUSTOM]}.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}_a24h # Template to look for point observations. # Example precip24_2010010112.nc diff --git a/parm/metplus/EnsembleStat_REFC.conf b/parm/metplus/EnsembleStat_REFC.conf index 123af0f0a2..180988656b 100644 --- a/parm/metplus/EnsembleStat_REFC.conf +++ b/parm/metplus/EnsembleStat_REFC.conf @@ -185,8 +185,8 @@ OBS_VAR1_OPTIONS = censor_thresh = lt-20; censor_val = -20.0; cnt_thresh = [ >15 [dir] # Input and output data directories -INPUT_BASE = {ENV[EXPTDIR]}/{ENV[CDATE]} -OUTPUT_BASE = {ENV[EXPTDIR]} +INPUT_BASE = {ENV[INPUT_BASE]} +OUTPUT_BASE = {ENV[OUTPUT_BASE]} # Forecast model input directory for ensemble_stat FCST_ENSEMBLE_STAT_INPUT_DIR = {INPUT_BASE} @@ -220,7 +220,7 @@ STAGING_DIR = {OUTPUT_BASE}/stage/REFC # FCST_ENSEMBLE_STAT_INPUT_TEMPLATE - comma separated list of ensemble members # or a single line, - filename wildcard characters may be used, ? or *. -FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = mem*/postprd/{ENV[NET]}.t{init?fmt=%H}z.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}.grib2 +FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = {ENV[MEM_STAR]}{ENV[POSTPRD]}{ENV[NET]}.t{init?fmt=%H}z*.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}.grib2 # Template to look for point observations. # Example precip24_2010010112.nc diff --git a/parm/metplus/EnsembleStat_RETOP.conf b/parm/metplus/EnsembleStat_RETOP.conf index 4fcbc9bcf2..7f58de0de7 100644 --- a/parm/metplus/EnsembleStat_RETOP.conf +++ b/parm/metplus/EnsembleStat_RETOP.conf @@ -188,8 +188,8 @@ OBS_VAR1_OPTIONS = censor_thresh = lt-20; censor_val = -20.0; cnt_thresh = [ >15 [dir] # Input and output data directories -INPUT_BASE = {ENV[EXPTDIR]}/{ENV[CDATE]} -OUTPUT_BASE = {ENV[EXPTDIR]} +INPUT_BASE = {ENV[INPUT_BASE]} +OUTPUT_BASE = {ENV[OUTPUT_BASE]} # Forecast model input directory for ensemble_stat FCST_ENSEMBLE_STAT_INPUT_DIR = {INPUT_BASE} @@ -223,7 +223,7 @@ STAGING_DIR = {OUTPUT_BASE}/stage/RETOP # FCST_ENSEMBLE_STAT_INPUT_TEMPLATE - comma separated list of ensemble members # or a single line, - filename wildcard characters may be used, ? or *. -FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = mem*/postprd/{ENV[NET]}.t{init?fmt=%H}z.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}.grib2 +FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = {ENV[MEM_STAR]}{ENV[POSTPRD]}{ENV[NET]}.t{init?fmt=%H}z*.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}.grib2 # Template to look for point observations. # Example precip24_2010010112.nc diff --git a/parm/metplus/EnsembleStat_conus_sfc.conf b/parm/metplus/EnsembleStat_conus_sfc.conf index 9f93af38f6..7c429eda98 100644 --- a/parm/metplus/EnsembleStat_conus_sfc.conf +++ b/parm/metplus/EnsembleStat_conus_sfc.conf @@ -283,10 +283,10 @@ OBS_VAR6_THRESH = <152, <305, >=914 [dir] # Input and output directories for pb2nc PB2NC_INPUT_DIR = {ENV[OBS_DIR]} -PB2NC_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pb2nc +PB2NC_OUTPUT_DIR = {ENV[OUTPUT_BASE]}/metprd/pb2nc # Forecast model input directory for ensemble_stat -FCST_ENSEMBLE_STAT_INPUT_DIR = {ENV[EXPTDIR]}/{ENV[CDATE]} +FCST_ENSEMBLE_STAT_INPUT_DIR = {ENV[INPUT_BASE]} # Point observation input dir for ensemble_stat OBS_ENSEMBLE_STAT_POINT_INPUT_DIR = {PB2NC_OUTPUT_DIR} @@ -303,11 +303,11 @@ ENSEMBLE_STAT_CLIMO_MEAN_INPUT_DIR = ENSEMBLE_STAT_CLIMO_STDEV_INPUT_DIR = # output directory for ensemble_stat -OUTPUT_BASE = {ENV[EXPTDIR]} +OUTPUT_BASE = {ENV[OUTPUT_BASE]} ENSEMBLE_STAT_OUTPUT_DIR = {OUTPUT_BASE} # directory containing log files -LOG_DIR = {ENV[EXPTDIR]}/log +LOG_DIR = {ENV[LOG_DIR]} # directory for staging data STAGING_DIR = {OUTPUT_BASE}/stage/conus_sfc @@ -323,7 +323,7 @@ PB2NC_OUTPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc # or a single line, - filename wildcard characters may be used, ? or *. FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = - mem*/postprd/{ENV[NET]}.t{init?fmt=%H}z.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}.grib2 + {ENV[MEM_STAR]}{ENV[POSTPRD]}{ENV[NET]}.t{init?fmt=%H}z*.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}.grib2 # Template to look for point observations. # Example precip24_2010010112.nc diff --git a/parm/metplus/EnsembleStat_upper_air.conf b/parm/metplus/EnsembleStat_upper_air.conf index c3e24b259d..57aa33273e 100644 --- a/parm/metplus/EnsembleStat_upper_air.conf +++ b/parm/metplus/EnsembleStat_upper_air.conf @@ -353,10 +353,10 @@ OBS_VAR12_THRESH = <500, <1500, >1500 [dir] # Input and output directories for pb2nc PB2NC_INPUT_DIR = {ENV[OBS_DIR]} -PB2NC_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pb2nc +PB2NC_OUTPUT_DIR = {ENV[OUTPUT_BASE]}/metprd/pb2nc # Forecast model input directory for ensemble_stat -FCST_ENSEMBLE_STAT_INPUT_DIR = {ENV[EXPTDIR]}/{ENV[CDATE]} +FCST_ENSEMBLE_STAT_INPUT_DIR = {ENV[INPUT_BASE]} # Point observation input dir for ensemble_stat OBS_ENSEMBLE_STAT_POINT_INPUT_DIR = {PB2NC_OUTPUT_DIR} @@ -373,11 +373,11 @@ ENSEMBLE_STAT_CLIMO_MEAN_INPUT_DIR = ENSEMBLE_STAT_CLIMO_STDEV_INPUT_DIR = # output directory for ensemble_stat -OUTPUT_BASE = {ENV[EXPTDIR]} +OUTPUT_BASE = {ENV[OUTPUT_BASE]} ENSEMBLE_STAT_OUTPUT_DIR = {OUTPUT_BASE} # directory containing log files -LOG_DIR = {ENV[EXPTDIR]}/log +LOG_DIR = {ENV[LOG_DIR]} # directory for staging data STAGING_DIR = {OUTPUT_BASE}/stage/upper_air @@ -393,7 +393,7 @@ PB2NC_OUTPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc # or a single line, - filename wildcard characters may be used, ? or *. FCST_ENSEMBLE_STAT_INPUT_TEMPLATE = - mem*/postprd/{ENV[NET]}.t{init?fmt=%H}z.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}.grib2 + {ENV[MEM_STAR]}{ENV[POSTPRD]}{ENV[NET]}.t{init?fmt=%H}z*.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}.grib2 # Template to look for point observations. # Example precip24_2010010112.nc diff --git a/parm/metplus/GridStat_APCP01h.conf b/parm/metplus/GridStat_APCP01h.conf index 2c6aac65aa..aec4f10375 100644 --- a/parm/metplus/GridStat_APCP01h.conf +++ b/parm/metplus/GridStat_APCP01h.conf @@ -52,13 +52,13 @@ OBS_PCP_COMBINE_RUN = False # Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only #LOG_GRID_STAT_VERBOSITY = 2 -LOG_DIR = {ENV[EXPTDIR]}/log +LOG_DIR = {ENV[LOG_DIR]} # Specify the name of the metplus.log file LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} # Specify where the location and name of the final metplus_final.conf -METPLUS_CONF={OUTPUT_BASE}/metprd/grid_stat/metplus_final.APCP_01h.conf +METPLUS_CONF={OUTPUT_BASE}/metprd/grid_stat/metplus_final.APCP_01h{ENV[DOT_ENSMEM]}.conf # Location of MET config file to pass to GridStat GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped @@ -253,7 +253,7 @@ STAGING_DIR = {OUTPUT_BASE}/stage/APCP_01h [filename_templates] # Template to look for forecast input to GridStat relative to FCST_GRID_STAT_INPUT_DIR -FCST_GRID_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}.grib2 +FCST_GRID_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z{ENV[DOT_ENSMEM]}.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}.grib2 # Template to look for observation input to GridStat relative to OBS_GRID_STAT_INPUT_DIR OBS_GRID_STAT_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.01h.hrap.conus.gb2 diff --git a/parm/metplus/GridStat_APCP01h_mean.conf b/parm/metplus/GridStat_APCP01h_mean.conf index 26740647e6..9df901f396 100644 --- a/parm/metplus/GridStat_APCP01h_mean.conf +++ b/parm/metplus/GridStat_APCP01h_mean.conf @@ -244,7 +244,7 @@ GRID_STAT_CLIMO_MEAN_INPUT_DIR = GRID_STAT_CLIMO_STDEV_INPUT_DIR = # directory to write output from GridStat -OUTPUT_BASE = {ENV[EXPTDIR]} +OUTPUT_BASE = {ENV[OUTPUT_BASE]} GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} STAGING_DIR = {OUTPUT_BASE}/stage/APCP_01h_mean diff --git a/parm/metplus/GridStat_APCP01h_prob.conf b/parm/metplus/GridStat_APCP01h_prob.conf index d507983f2c..ec08a36644 100644 --- a/parm/metplus/GridStat_APCP01h_prob.conf +++ b/parm/metplus/GridStat_APCP01h_prob.conf @@ -255,7 +255,7 @@ GRID_STAT_CLIMO_MEAN_INPUT_DIR = GRID_STAT_CLIMO_STDEV_INPUT_DIR = # directory to write output from GridStat -OUTPUT_BASE = {ENV[EXPTDIR]} +OUTPUT_BASE = {ENV[OUTPUT_BASE]} GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} STAGING_DIR = {OUTPUT_BASE}/stage/APCP_01h_prob diff --git a/parm/metplus/GridStat_APCP03h.conf b/parm/metplus/GridStat_APCP03h.conf index c017806171..e1be9d0f46 100644 --- a/parm/metplus/GridStat_APCP03h.conf +++ b/parm/metplus/GridStat_APCP03h.conf @@ -77,13 +77,13 @@ FCST_PCP_COMBINE_CONSTANT_INIT = True # Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only #LOG_GRID_STAT_VERBOSITY = 2 -LOG_DIR = {ENV[EXPTDIR]}/log +LOG_DIR = {ENV[LOG_DIR]} # Specify the name of the metplus.log file LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} # Specify where the location and name of the final metplus_final.conf -METPLUS_CONF={OUTPUT_BASE}/metprd/grid_stat/metplus_final.APCP_03h.conf +METPLUS_CONF={OUTPUT_BASE}/metprd/grid_stat/metplus_final.APCP_03h{ENV[DOT_ENSMEM]}.conf # Location of MET config file to pass to GridStat GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped @@ -274,7 +274,7 @@ GRID_STAT_CLIMO_STDEV_INPUT_DIR = # directory to write output from PCPCombine and GridStat OUTPUT_BASE = {ENV[OUTPUT_BASE]} FCST_PCP_COMBINE_OUTPUT_DIR = {OUTPUT_BASE}/metprd/pcp_combine -OBS_PCP_COMBINE_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pcp_combine +OBS_PCP_COMBINE_OUTPUT_DIR = {ENV[OUTPUT_BASE]}/metprd/pcp_combine GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} STAGING_DIR = {OUTPUT_BASE}/stage/APCP_03h @@ -283,7 +283,7 @@ STAGING_DIR = {OUTPUT_BASE}/stage/APCP_03h [filename_templates] # Template to look for forecast input to PCPCombine and GridStat relative to FCST_GRID_STAT_INPUT_DIR -FCST_PCP_COMBINE_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}.grib2 +FCST_PCP_COMBINE_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z{ENV[DOT_ENSMEM]}.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}.grib2 FCST_GRID_STAT_INPUT_TEMPLATE = {FCST_PCP_COMBINE_OUTPUT_TEMPLATE} # Template to look for observation input to PCPCombine and GridStat relative to OBS_GRID_STAT_INPUT_DIR @@ -291,8 +291,8 @@ OBS_PCP_COMBINE_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.01h.hr OBS_GRID_STAT_INPUT_TEMPLATE = {OBS_PCP_COMBINE_OUTPUT_TEMPLATE} # Optional subdirectories relative to GRID_STAT_OUTPUT_DIR to write output from PCPCombine and GridStat -FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}_a{level?fmt=%HH}h -OBS_PCP_COMBINE_OUTPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.hrap.conus.gb2_a{level?fmt=%HH}h +FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z{ENV[DOT_ENSMEM]}.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}_a{level?fmt=%HH}h +OBS_PCP_COMBINE_OUTPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z{ENV[DOT_ENSMEM]}.hrap.conus.gb2_a{level?fmt=%HH}h GRID_STAT_OUTPUT_TEMPLATE = metprd/grid_stat # Template to look for climatology input to GridStat relative to GRID_STAT_CLIMO_MEAN_INPUT_DIR diff --git a/parm/metplus/GridStat_APCP03h_mean.conf b/parm/metplus/GridStat_APCP03h_mean.conf index d912c7f8d9..d8759e2734 100644 --- a/parm/metplus/GridStat_APCP03h_mean.conf +++ b/parm/metplus/GridStat_APCP03h_mean.conf @@ -233,7 +233,7 @@ INPUT_BASE = {ENV[INPUT_BASE]} FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE} # directory containing observation input to GridStat -OBS_GRID_STAT_INPUT_DIR = {ENV[EXPTDIR]}/metprd/pcp_combine +OBS_GRID_STAT_INPUT_DIR = {ENV[OUTPUT_BASE]}/metprd/pcp_combine # directory containing climatology mean input to GridStat # Not used in this example @@ -244,7 +244,7 @@ GRID_STAT_CLIMO_MEAN_INPUT_DIR = GRID_STAT_CLIMO_STDEV_INPUT_DIR = # directory to write output from GridStat -OUTPUT_BASE = {ENV[EXPTDIR]} +OUTPUT_BASE = {ENV[OUTPUT_BASE]} GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} STAGING_DIR = {OUTPUT_BASE}/stage/APCP_03h_mean diff --git a/parm/metplus/GridStat_APCP03h_prob.conf b/parm/metplus/GridStat_APCP03h_prob.conf index 30cd5ad102..357a6495d3 100644 --- a/parm/metplus/GridStat_APCP03h_prob.conf +++ b/parm/metplus/GridStat_APCP03h_prob.conf @@ -243,7 +243,7 @@ INPUT_BASE = {ENV[INPUT_BASE]} FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE} # directory containing observation input to GridStat -OBS_GRID_STAT_INPUT_DIR = {ENV[EXPTDIR]}/metprd/pcp_combine +OBS_GRID_STAT_INPUT_DIR = {ENV[OUTPUT_BASE]}/metprd/pcp_combine # directory containing climatology mean input to GridStat # Not used in this example @@ -254,7 +254,7 @@ GRID_STAT_CLIMO_MEAN_INPUT_DIR = GRID_STAT_CLIMO_STDEV_INPUT_DIR = # directory to write output from GridStat -OUTPUT_BASE = {ENV[EXPTDIR]} +OUTPUT_BASE = {ENV[OUTPUT_BASE]} GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} STAGING_DIR = {OUTPUT_BASE}/stage/APCP_03h_prob diff --git a/parm/metplus/GridStat_APCP06h.conf b/parm/metplus/GridStat_APCP06h.conf index 0e4d66531e..efcf52da92 100644 --- a/parm/metplus/GridStat_APCP06h.conf +++ b/parm/metplus/GridStat_APCP06h.conf @@ -77,13 +77,13 @@ FCST_PCP_COMBINE_CONSTANT_INIT = True # Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only #LOG_GRID_STAT_VERBOSITY = 2 -LOG_DIR = {ENV[EXPTDIR]}/log +LOG_DIR = {ENV[LOG_DIR]} # Specify the name of the metplus.log file LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} # Specify where the location and name of the final metplus_final.conf -METPLUS_CONF={OUTPUT_BASE}/metprd/grid_stat/metplus_final.APCP_06h.conf +METPLUS_CONF={OUTPUT_BASE}/metprd/grid_stat/metplus_final.APCP_06h{ENV[DOT_ENSMEM]}.conf # Location of MET config file to pass to GridStat GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped @@ -274,7 +274,7 @@ GRID_STAT_CLIMO_STDEV_INPUT_DIR = # directory to write output from PCPCombine and GridStat OUTPUT_BASE = {ENV[OUTPUT_BASE]} FCST_PCP_COMBINE_OUTPUT_DIR = {OUTPUT_BASE}/metprd/pcp_combine -OBS_PCP_COMBINE_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pcp_combine +OBS_PCP_COMBINE_OUTPUT_DIR = {ENV[OUTPUT_BASE]}/metprd/pcp_combine GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} STAGING_DIR = {OUTPUT_BASE}/stage/APCP_06h @@ -283,7 +283,7 @@ STAGING_DIR = {OUTPUT_BASE}/stage/APCP_06h [filename_templates] # Template to look for forecast input to PCPCombine and GridStat relative to FCST_GRID_STAT_INPUT_DIR -FCST_PCP_COMBINE_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}.grib2 +FCST_PCP_COMBINE_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z{ENV[DOT_ENSMEM]}.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}.grib2 FCST_GRID_STAT_INPUT_TEMPLATE = {FCST_PCP_COMBINE_OUTPUT_TEMPLATE} # Template to look for observation input to PCPCombine and GridStat relative to OBS_GRID_STAT_INPUT_DIR @@ -291,8 +291,8 @@ OBS_PCP_COMBINE_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.01h.hr OBS_GRID_STAT_INPUT_TEMPLATE = {OBS_PCP_COMBINE_OUTPUT_TEMPLATE} # Optional subdirectories relative to GRID_STAT_OUTPUT_DIR to write output from PCPCombine and GridStat -FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}_a{level?fmt=%HH}h -OBS_PCP_COMBINE_OUTPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.hrap.conus.gb2_a{level?fmt=%HH}h +FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z{ENV[DOT_ENSMEM]}.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}_a{level?fmt=%HH}h +OBS_PCP_COMBINE_OUTPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z{ENV[DOT_ENSMEM]}.hrap.conus.gb2_a{level?fmt=%HH}h GRID_STAT_OUTPUT_TEMPLATE = metprd/grid_stat # Template to look for climatology input to GridStat relative to GRID_STAT_CLIMO_MEAN_INPUT_DIR diff --git a/parm/metplus/GridStat_APCP06h_mean.conf b/parm/metplus/GridStat_APCP06h_mean.conf index 264321eef3..c2049d41c2 100644 --- a/parm/metplus/GridStat_APCP06h_mean.conf +++ b/parm/metplus/GridStat_APCP06h_mean.conf @@ -233,7 +233,7 @@ INPUT_BASE = {ENV[INPUT_BASE]} FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE} # directory containing observation input to GridStat -OBS_GRID_STAT_INPUT_DIR = {ENV[EXPTDIR]}/metprd/pcp_combine +OBS_GRID_STAT_INPUT_DIR = {ENV[OUTPUT_BASE]}/metprd/pcp_combine # directory containing climatology mean input to GridStat # Not used in this example @@ -244,7 +244,7 @@ GRID_STAT_CLIMO_MEAN_INPUT_DIR = GRID_STAT_CLIMO_STDEV_INPUT_DIR = # directory to write output from GridStat -OUTPUT_BASE = {ENV[EXPTDIR]} +OUTPUT_BASE = {ENV[OUTPUT_BASE]} GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} STAGING_DIR = {OUTPUT_BASE}/stage/APCP_06h_mean diff --git a/parm/metplus/GridStat_APCP06h_prob.conf b/parm/metplus/GridStat_APCP06h_prob.conf index 0bfdf4eaa9..6f32e711d6 100644 --- a/parm/metplus/GridStat_APCP06h_prob.conf +++ b/parm/metplus/GridStat_APCP06h_prob.conf @@ -243,7 +243,7 @@ INPUT_BASE = {ENV[INPUT_BASE]} FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE} # directory containing observation input to GridStat -OBS_GRID_STAT_INPUT_DIR = {ENV[EXPTDIR]}/metprd/pcp_combine +OBS_GRID_STAT_INPUT_DIR = {ENV[OUTPUT_BASE]}/metprd/pcp_combine # directory containing climatology mean input to GridStat # Not used in this example @@ -254,7 +254,7 @@ GRID_STAT_CLIMO_MEAN_INPUT_DIR = GRID_STAT_CLIMO_STDEV_INPUT_DIR = # directory to write output from GridStat -OUTPUT_BASE = {ENV[EXPTDIR]} +OUTPUT_BASE = {ENV[OUTPUT_BASE]} GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} STAGING_DIR = {OUTPUT_BASE}/stage/APCP_06h_prob diff --git a/parm/metplus/GridStat_APCP24h.conf b/parm/metplus/GridStat_APCP24h.conf index 7ae700cf31..ebba69d1dc 100644 --- a/parm/metplus/GridStat_APCP24h.conf +++ b/parm/metplus/GridStat_APCP24h.conf @@ -77,13 +77,13 @@ FCST_PCP_COMBINE_CONSTANT_INIT = True # Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only #LOG_GRID_STAT_VERBOSITY = 2 -LOG_DIR = {ENV[EXPTDIR]}/log +LOG_DIR = {ENV[LOG_DIR]} # Specify the name of the metplus.log file LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} # Specify where the location and name of the final metplus_final.conf -METPLUS_CONF={OUTPUT_BASE}/metprd/grid_stat/metplus_final.APCP_24h.conf +METPLUS_CONF={OUTPUT_BASE}/metprd/grid_stat/metplus_final.APCP_24h{ENV[DOT_ENSMEM]}.conf # Location of MET config file to pass to GridStat GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped @@ -274,7 +274,7 @@ GRID_STAT_CLIMO_STDEV_INPUT_DIR = # directory to write output from PCPCombine and GridStat OUTPUT_BASE = {ENV[OUTPUT_BASE]} FCST_PCP_COMBINE_OUTPUT_DIR = {OUTPUT_BASE}/metprd/pcp_combine -OBS_PCP_COMBINE_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pcp_combine +OBS_PCP_COMBINE_OUTPUT_DIR = {ENV[OUTPUT_BASE]}/metprd/pcp_combine GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} STAGING_DIR = {OUTPUT_BASE}/stage/APCP_24h @@ -283,7 +283,7 @@ STAGING_DIR = {OUTPUT_BASE}/stage/APCP_24h [filename_templates] # Template to look for forecast input to PCPCombine and GridStat relative to FCST_GRID_STAT_INPUT_DIR -FCST_PCP_COMBINE_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}.grib2 +FCST_PCP_COMBINE_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z{ENV[DOT_ENSMEM]}.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}.grib2 FCST_GRID_STAT_INPUT_TEMPLATE = {FCST_PCP_COMBINE_OUTPUT_TEMPLATE} # Template to look for observation input to PCPCombine and GridStat relative to OBS_GRID_STAT_INPUT_DIR @@ -291,8 +291,8 @@ OBS_PCP_COMBINE_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.01h.hr OBS_GRID_STAT_INPUT_TEMPLATE = {OBS_PCP_COMBINE_OUTPUT_TEMPLATE} # Optional subdirectories relative to GRID_STAT_OUTPUT_DIR to write output from PCPCombine and GridStat -FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}_a{level?fmt=%HH}h -OBS_PCP_COMBINE_OUTPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z.hrap.conus.gb2_a{level?fmt=%HH}h +FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z{ENV[DOT_ENSMEM]}.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}_a{level?fmt=%HH}h +OBS_PCP_COMBINE_OUTPUT_TEMPLATE = {valid?fmt=%Y%m%d}/ccpa.t{valid?fmt=%H}z{ENV[DOT_ENSMEM]}.hrap.conus.gb2_a{level?fmt=%HH}h GRID_STAT_OUTPUT_TEMPLATE = metprd/grid_stat # Template to look for climatology input to GridStat relative to GRID_STAT_CLIMO_MEAN_INPUT_DIR diff --git a/parm/metplus/GridStat_APCP24h_mean.conf b/parm/metplus/GridStat_APCP24h_mean.conf index eaf9d5fb30..d7b9083266 100644 --- a/parm/metplus/GridStat_APCP24h_mean.conf +++ b/parm/metplus/GridStat_APCP24h_mean.conf @@ -233,7 +233,7 @@ INPUT_BASE = {ENV[INPUT_BASE]} FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE} # directory containing observation input to GridStat -OBS_GRID_STAT_INPUT_DIR = {ENV[EXPTDIR]}/metprd/pcp_combine +OBS_GRID_STAT_INPUT_DIR = {ENV[OUTPUT_BASE]}/metprd/pcp_combine # directory containing climatology mean input to GridStat # Not used in this example @@ -244,7 +244,7 @@ GRID_STAT_CLIMO_MEAN_INPUT_DIR = GRID_STAT_CLIMO_STDEV_INPUT_DIR = # directory to write output from GridStat -OUTPUT_BASE = {ENV[EXPTDIR]} +OUTPUT_BASE = {ENV[OUTPUT_BASE]} GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} STAGING_DIR = {OUTPUT_BASE}/stage/APCP_24h_mean diff --git a/parm/metplus/GridStat_APCP24h_prob.conf b/parm/metplus/GridStat_APCP24h_prob.conf index 397abdc42e..61c08ac530 100644 --- a/parm/metplus/GridStat_APCP24h_prob.conf +++ b/parm/metplus/GridStat_APCP24h_prob.conf @@ -243,7 +243,7 @@ INPUT_BASE = {ENV[INPUT_BASE]} FCST_GRID_STAT_INPUT_DIR = {INPUT_BASE} # directory containing observation input to GridStat -OBS_GRID_STAT_INPUT_DIR = {ENV[EXPTDIR]}/metprd/pcp_combine +OBS_GRID_STAT_INPUT_DIR = {ENV[OUTPUT_BASE]}/metprd/pcp_combine # directory containing climatology mean input to GridStat # Not used in this example @@ -254,7 +254,7 @@ GRID_STAT_CLIMO_MEAN_INPUT_DIR = GRID_STAT_CLIMO_STDEV_INPUT_DIR = # directory to write output from GridStat -OUTPUT_BASE = {ENV[EXPTDIR]} +OUTPUT_BASE = {ENV[OUTPUT_BASE]} GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} STAGING_DIR = {OUTPUT_BASE}/stage/APCP_24h_prob diff --git a/parm/metplus/GridStat_REFC.conf b/parm/metplus/GridStat_REFC.conf index 027723470e..8c0b939740 100644 --- a/parm/metplus/GridStat_REFC.conf +++ b/parm/metplus/GridStat_REFC.conf @@ -52,13 +52,13 @@ OBS_PCP_COMBINE_RUN = False # Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only #LOG_GRID_STAT_VERBOSITY = 2 -LOG_DIR = {ENV[EXPTDIR]}/log +LOG_DIR = {ENV[LOG_DIR]} # Specify the name of the metplus.log file LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} # Specify where the location and name of the final metplus_final.conf -METPLUS_CONF={OUTPUT_BASE}/metprd/grid_stat/metplus_final.REFC.conf +METPLUS_CONF={OUTPUT_BASE}/metprd/grid_stat/metplus_final.REFC{ENV[DOT_ENSMEM]}.conf # Location of MET config file to pass to GridStat GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped @@ -263,7 +263,7 @@ STAGING_DIR = {OUTPUT_BASE}/stage/REFC [filename_templates] # Template to look for forecast input to GridStat relative to FCST_GRID_STAT_INPUT_DIR -FCST_GRID_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}.grib2 +FCST_GRID_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z{ENV[DOT_ENSMEM]}.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}.grib2 # Template to look for observation input to GridStat relative to OBS_GRID_STAT_INPUT_DIR OBS_GRID_STAT_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/MergedReflectivityQCComposite_00.50_{valid?fmt=%Y%m%d}-{valid?fmt=%H}0000.grib2 diff --git a/parm/metplus/GridStat_REFC_mean.conf b/parm/metplus/GridStat_REFC_mean.conf index 6fad42b133..11299bb848 100644 --- a/parm/metplus/GridStat_REFC_mean.conf +++ b/parm/metplus/GridStat_REFC_mean.conf @@ -254,7 +254,7 @@ GRID_STAT_CLIMO_MEAN_INPUT_DIR = GRID_STAT_CLIMO_STDEV_INPUT_DIR = # directory to write output from GridStat -OUTPUT_BASE = {ENV[EXPTDIR]} +OUTPUT_BASE = {ENV[OUTPUT_BASE]} GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} STAGING_DIR = {OUTPUT_BASE}/stage/REFC_mean diff --git a/parm/metplus/GridStat_REFC_prob.conf b/parm/metplus/GridStat_REFC_prob.conf index 997fe1274a..1abcc8803d 100644 --- a/parm/metplus/GridStat_REFC_prob.conf +++ b/parm/metplus/GridStat_REFC_prob.conf @@ -279,7 +279,7 @@ GRID_STAT_CLIMO_MEAN_INPUT_DIR = GRID_STAT_CLIMO_STDEV_INPUT_DIR = # directory to write output from GridStat -OUTPUT_BASE = {ENV[EXPTDIR]} +OUTPUT_BASE = {ENV[OUTPUT_BASE]} GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} STAGING_DIR = {OUTPUT_BASE}/stage/REFC_prob diff --git a/parm/metplus/GridStat_RETOP.conf b/parm/metplus/GridStat_RETOP.conf index 0b00292112..c1d203106f 100644 --- a/parm/metplus/GridStat_RETOP.conf +++ b/parm/metplus/GridStat_RETOP.conf @@ -52,13 +52,13 @@ OBS_PCP_COMBINE_RUN = False # Verbosity of MET output - overrides LOG_VERBOSITY for GridStat only #LOG_GRID_STAT_VERBOSITY = 2 -LOG_DIR = {ENV[EXPTDIR]}/log +LOG_DIR = {ENV[LOG_DIR]} # Specify the name of the metplus.log file LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]} # Specify where the location and name of the final metplus_final.conf -METPLUS_CONF={OUTPUT_BASE}/metprd/grid_stat/metplus_final.RETOP.conf +METPLUS_CONF={OUTPUT_BASE}/metprd/grid_stat/metplus_final.RETOP{ENV[DOT_ENSMEM]}.conf # Location of MET config file to pass to GridStat GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped @@ -263,7 +263,7 @@ STAGING_DIR = {OUTPUT_BASE}/stage/RETOP [filename_templates] # Template to look for forecast input to GridStat relative to FCST_GRID_STAT_INPUT_DIR -FCST_GRID_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}.grib2 +FCST_GRID_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z{ENV[DOT_ENSMEM]}.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}.grib2 # Template to look for observation input to GridStat relative to OBS_GRID_STAT_INPUT_DIR OBS_GRID_STAT_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/EchoTop_18_00.50_{valid?fmt=%Y%m%d}-{valid?fmt=%H%M%S}.grib2 diff --git a/parm/metplus/GridStat_RETOP_mean.conf b/parm/metplus/GridStat_RETOP_mean.conf index b0b17dbbd0..3970c4b674 100644 --- a/parm/metplus/GridStat_RETOP_mean.conf +++ b/parm/metplus/GridStat_RETOP_mean.conf @@ -254,7 +254,7 @@ GRID_STAT_CLIMO_MEAN_INPUT_DIR = GRID_STAT_CLIMO_STDEV_INPUT_DIR = # directory to write output from GridStat -OUTPUT_BASE = {ENV[EXPTDIR]} +OUTPUT_BASE = {ENV[OUTPUT_BASE]} GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} STAGING_DIR = {OUTPUT_BASE}/stage/RETOP_mean diff --git a/parm/metplus/GridStat_RETOP_prob.conf b/parm/metplus/GridStat_RETOP_prob.conf index 27142979c0..49a5f67138 100644 --- a/parm/metplus/GridStat_RETOP_prob.conf +++ b/parm/metplus/GridStat_RETOP_prob.conf @@ -262,7 +262,7 @@ GRID_STAT_CLIMO_MEAN_INPUT_DIR = GRID_STAT_CLIMO_STDEV_INPUT_DIR = # directory to write output from GridStat -OUTPUT_BASE = {ENV[EXPTDIR]} +OUTPUT_BASE = {ENV[OUTPUT_BASE]} GRID_STAT_OUTPUT_DIR = {OUTPUT_BASE} STAGING_DIR = {OUTPUT_BASE}/stage/RETOP_prob diff --git a/parm/metplus/PointStat_conus_sfc.conf b/parm/metplus/PointStat_conus_sfc.conf index a0994f222e..1fc815e201 100644 --- a/parm/metplus/PointStat_conus_sfc.conf +++ b/parm/metplus/PointStat_conus_sfc.conf @@ -240,7 +240,7 @@ OBS_VAR11_THRESH = <152, <305, <914, <1520, <3040, >=914 PB2NC_INPUT_DIR = {ENV[OBS_DIR]} # directory to write output from -PB2NC_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pb2nc +PB2NC_OUTPUT_DIR = {ENV[OUTPUT_BASE]}/metprd/pb2nc INPUT_BASE = {ENV[INPUT_BASE]} @@ -258,7 +258,7 @@ POINT_STAT_CLIMO_STDEV_INPUT_DIR = OUTPUT_BASE = {ENV[OUTPUT_BASE]} POINT_STAT_OUTPUT_DIR = {OUTPUT_BASE} -LOG_DIR = {ENV[EXPTDIR]}/log +LOG_DIR = {ENV[LOG_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage/conus_sfc @@ -268,13 +268,13 @@ STAGING_DIR = {OUTPUT_BASE}/stage/conus_sfc # Template to look for prepbvur input to PB2NC relative to PB2NC_INPUT_DIR PB2NC_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H} # Template to use to write output from PB2NC -PB2NC_OUTPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc +PB2NC_OUTPUT_TEMPLATE = prepbufr.ndas{ENV[DOT_ENSMEM]}.{valid?fmt=%Y%m%d%H}.nc # Template to look for forecast input to PointStat relative to FCST_POINT_STAT_INPUT_DIR -FCST_POINT_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}.grib2 +FCST_POINT_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z{ENV[DOT_ENSMEM]}.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}.grib2 # Template to look for observation input to PointStat relative to OBS_POINT_STAT_INPUT_DIR -OBS_POINT_STAT_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc +OBS_POINT_STAT_INPUT_TEMPLATE = prepbufr.ndas{ENV[DOT_ENSMEM]}.{valid?fmt=%Y%m%d%H}.nc # Template to look for climatology input to PointStat relative to POINT_STAT_CLIMO_MEAN_INPUT_DIR # Not used in this example @@ -291,4 +291,4 @@ POINT_STAT_OUTPUT_TEMPLATE = metprd/point_stat LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]}_sfc # Specify where the location and name of the final metplus_final.conf -METPLUS_CONF={OUTPUT_BASE}/metprd/point_stat/metplus_final.conus_surface.conf +METPLUS_CONF={OUTPUT_BASE}/metprd/point_stat/metplus_final.conus_surface{ENV[DOT_ENSMEM]}.conf diff --git a/parm/metplus/PointStat_conus_sfc_mean.conf b/parm/metplus/PointStat_conus_sfc_mean.conf index 98c23a30d2..83af83c302 100644 --- a/parm/metplus/PointStat_conus_sfc_mean.conf +++ b/parm/metplus/PointStat_conus_sfc_mean.conf @@ -215,7 +215,7 @@ OBS_VAR3_THRESH = >=5, >=10, >=15 PB2NC_INPUT_DIR = {ENV[OBS_DIR]} # directory to write output from -PB2NC_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pb2nc +PB2NC_OUTPUT_DIR = {ENV[OUTPUT_BASE]}/metprd/pb2nc INPUT_BASE = {ENV[INPUT_BASE]} @@ -230,10 +230,10 @@ POINT_STAT_CLIMO_MEAN_INPUT_DIR = # Not used in this example POINT_STAT_CLIMO_STDEV_INPUT_DIR = -OUTPUT_BASE = {ENV[EXPTDIR]} +OUTPUT_BASE = {ENV[OUTPUT_BASE]} POINT_STAT_OUTPUT_DIR = {OUTPUT_BASE} -LOG_DIR = {ENV[EXPTDIR]}/log +LOG_DIR = {ENV[LOG_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage/conus_sfc_mean @@ -243,13 +243,13 @@ STAGING_DIR = {OUTPUT_BASE}/stage/conus_sfc_mean # Template to look for prepbvur input to PB2NC relative to PB2NC_INPUT_DIR PB2NC_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H} # Template to use to write output from PB2NC -PB2NC_OUTPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc +PB2NC_OUTPUT_TEMPLATE = prepbufr.ndas{ENV[DOT_ENSMEM]}.{valid?fmt=%Y%m%d%H}.nc # Template to look for forecast input to PointStat relative to FCST_POINT_STAT_INPUT_DIR FCST_POINT_STAT_INPUT_TEMPLATE = ensemble_stat_{ENV[MODEL]}_ADPSFC_{OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc # Template to look for observation input to PointStat relative to OBS_POINT_STAT_INPUT_DIR -OBS_POINT_STAT_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc +OBS_POINT_STAT_INPUT_TEMPLATE = prepbufr.ndas{ENV[DOT_ENSMEM]}.{valid?fmt=%Y%m%d%H}.nc # Template to look for climatology input to PointStat relative to POINT_STAT_CLIMO_MEAN_INPUT_DIR # Not used in this example diff --git a/parm/metplus/PointStat_conus_sfc_prob.conf b/parm/metplus/PointStat_conus_sfc_prob.conf index 66bbc06020..0c2e290782 100644 --- a/parm/metplus/PointStat_conus_sfc_prob.conf +++ b/parm/metplus/PointStat_conus_sfc_prob.conf @@ -391,7 +391,7 @@ FCST_PROB_IN_GRIB_PDS = False PB2NC_INPUT_DIR = {ENV[OBS_DIR]} # directory to write output from -PB2NC_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pb2nc +PB2NC_OUTPUT_DIR = {ENV[OUTPUT_BASE]}/metprd/pb2nc INPUT_BASE = {ENV[INPUT_BASE]} @@ -406,10 +406,10 @@ POINT_STAT_CLIMO_MEAN_INPUT_DIR = # Not used in this example POINT_STAT_CLIMO_STDEV_INPUT_DIR = -OUTPUT_BASE = {ENV[EXPTDIR]} +OUTPUT_BASE = {ENV[OUTPUT_BASE]} POINT_STAT_OUTPUT_DIR = {OUTPUT_BASE} -LOG_DIR = {ENV[EXPTDIR]}/log +LOG_DIR = {ENV[LOG_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage/conus_sfc_prob @@ -419,13 +419,13 @@ STAGING_DIR = {OUTPUT_BASE}/stage/conus_sfc_prob # Template to look for prepbvur input to PB2NC relative to PB2NC_INPUT_DIR PB2NC_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H} # Template to use to write output from PB2NC -PB2NC_OUTPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc +PB2NC_OUTPUT_TEMPLATE = prepbufr.ndas{ENV[DOT_ENSMEM]}.{valid?fmt=%Y%m%d%H}.nc # Template to look for forecast input to PointStat relative to FCST_POINT_STAT_INPUT_DIR FCST_POINT_STAT_INPUT_TEMPLATE = ensemble_stat_{ENV[MODEL]}_ADPSFC_{OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc # Template to look for observation input to PointStat relative to OBS_POINT_STAT_INPUT_DIR -OBS_POINT_STAT_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc +OBS_POINT_STAT_INPUT_TEMPLATE = prepbufr.ndas{ENV[DOT_ENSMEM]}.{valid?fmt=%Y%m%d%H}.nc # Template to look for climatology input to PointStat relative to POINT_STAT_CLIMO_MEAN_INPUT_DIR # Not used in this example diff --git a/parm/metplus/PointStat_upper_air.conf b/parm/metplus/PointStat_upper_air.conf index 23591bec11..822f9bae1e 100644 --- a/parm/metplus/PointStat_upper_air.conf +++ b/parm/metplus/PointStat_upper_air.conf @@ -240,7 +240,7 @@ OBS_VAR11_OPTIONS = desc = "RI"; PB2NC_INPUT_DIR = {ENV[OBS_DIR]} # directory to write output from -PB2NC_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pb2nc +PB2NC_OUTPUT_DIR = {ENV[OUTPUT_BASE]}/metprd/pb2nc INPUT_BASE = {ENV[INPUT_BASE]} @@ -258,7 +258,7 @@ POINT_STAT_CLIMO_STDEV_INPUT_DIR = OUTPUT_BASE = {ENV[OUTPUT_BASE]} POINT_STAT_OUTPUT_DIR = {OUTPUT_BASE} -LOG_DIR = {ENV[EXPTDIR]}/log +LOG_DIR = {ENV[LOG_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage/upper_air @@ -269,13 +269,13 @@ STAGING_DIR = {OUTPUT_BASE}/stage/upper_air PB2NC_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H} # Template to use to write output from PB2NC -PB2NC_OUTPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc +PB2NC_OUTPUT_TEMPLATE = prepbufr.ndas{ENV[DOT_ENSMEM]}.{valid?fmt=%Y%m%d%H}.nc # Template to look for forecast input to PointStat relative to FCST_POINT_STAT_INPUT_DIR -FCST_POINT_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}.grib2 +FCST_POINT_STAT_INPUT_TEMPLATE = {ENV[NET]}.t{init?fmt=%H}z{ENV[DOT_ENSMEM]}.prslev.f{lead?fmt=%HHH}.{ENV[POST_OUTPUT_DOMAIN_NAME]}.grib2 # Template to look for observation input to PointStat relative to OBS_POINT_STAT_INPUT_DIR -OBS_POINT_STAT_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc +OBS_POINT_STAT_INPUT_TEMPLATE = prepbufr.ndas{ENV[DOT_ENSMEM]}.{valid?fmt=%Y%m%d%H}.nc # Template to look for climatology input to PointStat relative to POINT_STAT_CLIMO_MEAN_INPUT_DIR # Not used in this example @@ -292,4 +292,4 @@ POINT_STAT_OUTPUT_TEMPLATE = metprd/point_stat LOG_METPLUS = {LOG_DIR}/metplus.log.{ENV[LOG_SUFFIX]}_upa # Specify where the location and name of the final metplus_final.conf -METPLUS_CONF={OUTPUT_BASE}/metprd/point_stat/metplus_final.upper_air.conf +METPLUS_CONF={OUTPUT_BASE}/metprd/point_stat/metplus_final.upper_air{ENV[DOT_ENSMEM]}.conf diff --git a/parm/metplus/PointStat_upper_air_mean.conf b/parm/metplus/PointStat_upper_air_mean.conf index 4b5e917100..b2a9dd593e 100644 --- a/parm/metplus/PointStat_upper_air_mean.conf +++ b/parm/metplus/PointStat_upper_air_mean.conf @@ -290,7 +290,7 @@ OBS_VAR12_THRESH = <500, <1500, >1500 PB2NC_INPUT_DIR = {ENV[OBS_DIR]} # directory to write output from -PB2NC_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pb2nc +PB2NC_OUTPUT_DIR = {ENV[OUTPUT_BASE]}/metprd/pb2nc INPUT_BASE = {ENV[INPUT_BASE]} @@ -305,10 +305,10 @@ POINT_STAT_CLIMO_MEAN_INPUT_DIR = # Not used in this example POINT_STAT_CLIMO_STDEV_INPUT_DIR = -OUTPUT_BASE = {ENV[EXPTDIR]} +OUTPUT_BASE = {ENV[OUTPUT_BASE]} POINT_STAT_OUTPUT_DIR = {OUTPUT_BASE} -LOG_DIR = {ENV[EXPTDIR]}/log +LOG_DIR = {ENV[LOG_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage/upper_air_mean @@ -319,13 +319,13 @@ STAGING_DIR = {OUTPUT_BASE}/stage/upper_air_mean PB2NC_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H} # Template to use to write output from PB2NC -PB2NC_OUTPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc +PB2NC_OUTPUT_TEMPLATE = prepbufr.ndas{ENV[DOT_ENSMEM]}.{valid?fmt=%Y%m%d%H}.nc # Template to look for forecast input to PointStat relative to FCST_POINT_STAT_INPUT_DIR FCST_POINT_STAT_INPUT_TEMPLATE = ensemble_stat_{ENV[MODEL]}_ADPUPA_{OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc # Template to look for observation input to PointStat relative to OBS_POINT_STAT_INPUT_DIR -OBS_POINT_STAT_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc +OBS_POINT_STAT_INPUT_TEMPLATE = prepbufr.ndas{ENV[DOT_ENSMEM]}.{valid?fmt=%Y%m%d%H}.nc # Template to look for climatology input to PointStat relative to POINT_STAT_CLIMO_MEAN_INPUT_DIR # Not used in this example diff --git a/parm/metplus/PointStat_upper_air_prob.conf b/parm/metplus/PointStat_upper_air_prob.conf index 213e1f04f5..092c13970e 100644 --- a/parm/metplus/PointStat_upper_air_prob.conf +++ b/parm/metplus/PointStat_upper_air_prob.conf @@ -514,7 +514,7 @@ FCST_PROB_IN_GRIB_PDS = False PB2NC_INPUT_DIR = {ENV[OBS_DIR]} # directory to write output from -PB2NC_OUTPUT_DIR = {ENV[EXPTDIR]}/metprd/pb2nc +PB2NC_OUTPUT_DIR = {ENV[OUTPUT_BASE]}/metprd/pb2nc INPUT_BASE = {ENV[INPUT_BASE]} @@ -529,10 +529,10 @@ POINT_STAT_CLIMO_MEAN_INPUT_DIR = # Not used in this example POINT_STAT_CLIMO_STDEV_INPUT_DIR = -OUTPUT_BASE = {ENV[EXPTDIR]} +OUTPUT_BASE = {ENV[OUTPUT_BASE]} POINT_STAT_OUTPUT_DIR = {OUTPUT_BASE} -LOG_DIR = {ENV[EXPTDIR]}/log +LOG_DIR = {ENV[LOG_DIR]} STAGING_DIR = {OUTPUT_BASE}/stage/upper_air_prob @@ -543,13 +543,13 @@ STAGING_DIR = {OUTPUT_BASE}/stage/upper_air_prob PB2NC_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H} # Template to use to write output from PB2NC -PB2NC_OUTPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc +PB2NC_OUTPUT_TEMPLATE = prepbufr.ndas{ENV[DOT_ENSMEM]}.{valid?fmt=%Y%m%d%H}.nc # Template to look for forecast input to PointStat relative to FCST_POINT_STAT_INPUT_DIR FCST_POINT_STAT_INPUT_TEMPLATE = ensemble_stat_{ENV[MODEL]}_ADPUPA_{OBTYPE}_{valid?fmt=%Y%m%d}_{valid?fmt=%H%M%S}V_ens.nc # Template to look for observation input to PointStat relative to OBS_POINT_STAT_INPUT_DIR -OBS_POINT_STAT_INPUT_TEMPLATE = prepbufr.ndas.{valid?fmt=%Y%m%d%H}.nc +OBS_POINT_STAT_INPUT_TEMPLATE = prepbufr.ndas{ENV[DOT_ENSMEM]}.{valid?fmt=%Y%m%d%H}.nc # Template to look for climatology input to PointStat relative to POINT_STAT_CLIMO_MEAN_INPUT_DIR # Not used in this example diff --git a/scripts/exregional_run_vx_ensgrid.sh b/scripts/exregional_run_vx_ensgrid.sh index f8209e2e74..73bb39796d 100755 --- a/scripts/exregional_run_vx_ensgrid.sh +++ b/scripts/exregional_run_vx_ensgrid.sh @@ -82,35 +82,45 @@ NUM_PAD=${NDIGITS_ENSMEM_NAMES} # #----------------------------------------------------------------------- # -# Create LOG_SUFFIX to read into METplus conf files. +# Pick a directory structure for METplus output files # #----------------------------------------------------------------------- # +if [ $RUN_ENVIR = "nco" ]; then + export INPUT_BASE=$COMIN + export OUTPUT_BASE=$COMOUT/metout + export MEM_BASE=$OUTPUT_BASE + export LOG_DIR=$LOGDIR -if [ ${VAR} == "APCP" ]; then - LOG_SUFFIX=ensgrid_${CDATE}_${VAR}_${ACCUM}h + export POSTPRD= + export MEM_STAR= + export MEM_CUSTOM= + export DOT_MEM_CUSTOM=".{custom?fmt=%s}" else - LOG_SUFFIX=ensgrid_${CDATE}_${VAR} + export INPUT_BASE=$EXPTDIR/$CDATE + export OUTPUT_BASE=$EXPTDIR + export MEM_BASE=$EXPTDIR/$CDATE + export LOG_DIR=${EXPTDIR}/log + + export POSTPRD="postprd/" + export MEM_STAR="mem*/" + export MEM_CUSTOM="{custom?fmt=%s}/" + export DOT_MEM_CUSTOM= fi +export DOT_ENSMEM=${dot_ensmem} # #----------------------------------------------------------------------- # -# Make sure directories in which output files will be placed exist. +# Create LOG_SUFFIX to read into METplus conf files. # #----------------------------------------------------------------------- # -mkdir_vrfy -p "${EXPTDIR}/${CDATE}/metprd/ensemble_stat" # Output directory for ensemble_stat tool. -# -# If the variable is accumulated precipitation for a time interval -# (bucket) other than 1 hour, the MET/METplus tools called below will -# include pcp_combine. In that case, create (if necessary) directories -# needed by pcp_combine. -# -if [ "${VAR}" = "APCP" ] && [ "${ACCUM: -1}" != "1" ]; then - mkdir_vrfy -p "${EXPTDIR}/metprd/pcp_combine" # For observations - mkdir_vrfy -p "${EXPTDIR}/${CDATE}/metprd/pcp_combine" # For forecast +if [ ${VAR} == "APCP" ]; then + LOG_SUFFIX=ensgrid_${CDATE}_${VAR}_${ACCUM}h +else + LOG_SUFFIX=ensgrid_${CDATE}_${VAR} fi # @@ -120,8 +130,6 @@ fi # #----------------------------------------------------------------------- # -export SCRIPTSrrfs -export EXPTDIR export MET_INSTALL_DIR export MET_BIN_EXEC export METPLUS_PATH diff --git a/scripts/exregional_run_vx_ensgrid_mean.sh b/scripts/exregional_run_vx_ensgrid_mean.sh index 2d3220c84a..dcd3913afb 100755 --- a/scripts/exregional_run_vx_ensgrid_mean.sh +++ b/scripts/exregional_run_vx_ensgrid_mean.sh @@ -79,26 +79,46 @@ export fhr_list # #----------------------------------------------------------------------- # -# Create INPUT_BASE and LOG_SUFFIX to read into METplus conf files. +# Pick a directory structure for METplus output files # #----------------------------------------------------------------------- # -INPUT_BASE=${EXPTDIR}/${CDATE}/metprd/ensemble_stat +if [ $RUN_ENVIR = "nco" ]; then + export INPUT_BASE=$COMOUT/metout/${CDATE}/metprd/ensemble_stat + export OUTPUT_BASE=$COMOUT/metout + export MEM_BASE=$OUTPUT_BASE + export LOG_DIR=$LOGDIR -if [ ${VAR} == "APCP" ]; then - LOG_SUFFIX=ensgrid_mean_${CDATE}_${VAR}_${ACCUM}h + export POSTPRD= + export MEM_STAR= + export MEM_CUSTOM= + export DOT_MEM_CUSTOM=".{custom?fmt=%s}" else - LOG_SUFFIX=ensgrid_mean_${CDATE}_${VAR} + export INPUT_BASE=${EXPTDIR}/${CDATE}/metprd/ensemble_stat + export OUTPUT_BASE=$EXPTDIR + export MEM_BASE=$EXPTDIR/$CDATE + export LOG_DIR=${EXPTDIR}/log + + export POSTPRD="postprd/" + export MEM_STAR="mem*/" + export MEM_CUSTOM="{custom?fmt=%s}/" + export DOT_MEM_CUSTOM= fi +export DOT_ENSMEM=${dot_ensmem} # #----------------------------------------------------------------------- # -# Make sure directory in which output files will be placed exist. +# Create INPUT_BASE and LOG_SUFFIX to read into METplus conf files. # #----------------------------------------------------------------------- # -mkdir_vrfy -p "${EXPTDIR}/${CDATE}/metprd/ensemble_stat_mean" # Output directory for grid_stat tool. + +if [ ${VAR} == "APCP" ]; then + LOG_SUFFIX=ensgrid_mean_${CDATE}_${VAR}_${ACCUM}h +else + LOG_SUFFIX=ensgrid_mean_${CDATE}_${VAR} +fi # #----------------------------------------------------------------------- @@ -119,8 +139,6 @@ fi # #----------------------------------------------------------------------- # -export SCRIPTSrrfs -export EXPTDIR export MET_INSTALL_DIR export MET_BIN_EXEC export METPLUS_PATH @@ -129,7 +147,6 @@ export MET_CONFIG export MODEL export NET export POST_OUTPUT_DOMAIN_NAME -export INPUT_BASE export LOG_SUFFIX # diff --git a/scripts/exregional_run_vx_ensgrid_prob.sh b/scripts/exregional_run_vx_ensgrid_prob.sh index 70a7aa82f8..52a972a9ff 100755 --- a/scripts/exregional_run_vx_ensgrid_prob.sh +++ b/scripts/exregional_run_vx_ensgrid_prob.sh @@ -79,26 +79,46 @@ export fhr_list # #----------------------------------------------------------------------- # -# Create INPUT_BASE and LOG_SUFFIX to read into METplus conf files. +# Pick a directory structure for METplus output files # #----------------------------------------------------------------------- # -INPUT_BASE=${EXPTDIR}/${CDATE}/metprd/ensemble_stat +if [ $RUN_ENVIR = "nco" ]; then + export INPUT_BASE=$COMOUT/metout/${CDATE}/metprd/ensemble_stat + export OUTPUT_BASE=$COMOUT/metout + export MEM_BASE=$OUTPUT_BASE + export LOG_DIR=$LOGDIR -if [ ${VAR} == "APCP" ]; then - LOG_SUFFIX=ensgrid_prob_${CDATE}_${VAR}_${ACCUM}h + export POSTPRD= + export MEM_STAR= + export MEM_CUSTOM= + export DOT_MEM_CUSTOM=".{custom?fmt=%s}" else - LOG_SUFFIX=ensgrid_prob_${CDATE}_${VAR} + export INPUT_BASE=${EXPTDIR}/${CDATE}/metprd/ensemble_stat + export OUTPUT_BASE=$EXPTDIR + export MEM_BASE=$EXPTDIR/$CDATE + export LOG_DIR=${EXPTDIR}/log + + export POSTPRD="postprd/" + export MEM_STAR="mem*/" + export MEM_CUSTOM="{custom?fmt=%s}/" + export DOT_MEM_CUSTOM= fi +export DOT_ENSMEM=${dot_ensmem} # #----------------------------------------------------------------------- # -# Make sure directory in which output files will be placed exist. +# Create INPUT_BASE and LOG_SUFFIX to read into METplus conf files. # #----------------------------------------------------------------------- # -mkdir_vrfy -p "${EXPTDIR}/${CDATE}/metprd/ensemble_stat_prob" # Output directory for grid_stat tool. + +if [ ${VAR} == "APCP" ]; then + LOG_SUFFIX=ensgrid_prob_${CDATE}_${VAR}_${ACCUM}h +else + LOG_SUFFIX=ensgrid_prob_${CDATE}_${VAR} +fi # #----------------------------------------------------------------------- @@ -119,9 +139,6 @@ fi # #----------------------------------------------------------------------- # -export SCRIPTSrrfs -export INPUT_BASE -export EXPTDIR export MET_INSTALL_DIR export MET_BIN_EXEC export METPLUS_PATH diff --git a/scripts/exregional_run_vx_enspoint.sh b/scripts/exregional_run_vx_enspoint.sh index 1a3ffd0d5e..e4be4b2cd5 100755 --- a/scripts/exregional_run_vx_enspoint.sh +++ b/scripts/exregional_run_vx_enspoint.sh @@ -78,21 +78,41 @@ export fhr_list # #----------------------------------------------------------------------- # -# Create LOG_SUFFIX to read into METplus conf files. +# Pick a directory structure for METplus output files # #----------------------------------------------------------------------- # -LOG_SUFFIX=enspoint_${CDATE} +if [ $RUN_ENVIR = "nco" ]; then + export INPUT_BASE=$COMIN + export OUTPUT_BASE=$COMOUT/metout + export MEM_BASE=$OUTPUT_BASE + export LOG_DIR=$LOGDIR + + export POSTPRD= + export MEM_STAR= + export MEM_CUSTOM= + export DOT_MEM_CUSTOM=".{custom?fmt=%s}" +else + export INPUT_BASE=$EXPTDIR/$CDATE + export OUTPUT_BASE=$EXPTDIR + export MEM_BASE=$EXPTDIR/$CDATE + export LOG_DIR=${EXPTDIR}/log + + export POSTPRD="postprd/" + export MEM_STAR="mem*/" + export MEM_CUSTOM="{custom?fmt=%s}/" + export DOT_MEM_CUSTOM= +fi +export DOT_ENSMEM=${dot_ensmem} # #----------------------------------------------------------------------- # -# Make sure directories in which output files will be placed exist. +# Create LOG_SUFFIX to read into METplus conf files. # #----------------------------------------------------------------------- # -mkdir_vrfy -p "${EXPTDIR}/metprd/pb2nc" # Output directory for pb2nc tool. -mkdir_vrfy -p "${EXPTDIR}/${CDATE}/metprd/ensemble_stat" # Output directory for ensemble_stat tool. +LOG_SUFFIX=enspoint_${CDATE} # #----------------------------------------------------------------------- @@ -113,7 +133,6 @@ fi # #----------------------------------------------------------------------- # -export EXPTDIR export LOG_SUFFIX export MET_INSTALL_DIR export MET_BIN_EXEC diff --git a/scripts/exregional_run_vx_enspoint_mean.sh b/scripts/exregional_run_vx_enspoint_mean.sh index 6fd6c797df..fac966a232 100755 --- a/scripts/exregional_run_vx_enspoint_mean.sh +++ b/scripts/exregional_run_vx_enspoint_mean.sh @@ -79,22 +79,41 @@ export fhr_list # #----------------------------------------------------------------------- # -# Create INPUT_BASE and LOG_SUFFIX to read into METplus conf files. +# Pick a directory structure for METplus output files # #----------------------------------------------------------------------- # -INPUT_BASE=${EXPTDIR}/${CDATE}/metprd/ensemble_stat -LOG_SUFFIX=enspoint_mean_${CDATE} +if [ $RUN_ENVIR = "nco" ]; then + export INPUT_BASE=$COMOUT/metout/${CDATE}/metprd/ensemble_stat + export OUTPUT_BASE=$COMOUT/metout + export MEM_BASE=$OUTPUT_BASE + export LOG_DIR=$LOGDIR + + export POSTPRD= + export MEM_STAR= + export MEM_CUSTOM= + export DOT_MEM_CUSTOM=".{custom?fmt=%s}" +else + export INPUT_BASE=${EXPTDIR}/${CDATE}/metprd/ensemble_stat + export OUTPUT_BASE=$EXPTDIR + export MEM_BASE=$EXPTDIR/$CDATE + export LOG_DIR=${EXPTDIR}/log + + export POSTPRD="postprd/" + export MEM_STAR="mem*/" + export MEM_CUSTOM="{custom?fmt=%s}/" + export DOT_MEM_CUSTOM= +fi +export DOT_ENSMEM=${dot_ensmem} # #----------------------------------------------------------------------- # -# Make sure directories in which output files will be placed exist. +# Create INPUT_BASE and LOG_SUFFIX to read into METplus conf files. # #----------------------------------------------------------------------- # -mkdir_vrfy -p "${EXPTDIR}/metprd/pb2nc" # Output directory for pb2nc tool. -mkdir_vrfy -p "${EXPTDIR}/${CDATE}/metprd/ensemble_stat_mean" # Output directory for point_stat tool. +LOG_SUFFIX=enspoint_mean_${CDATE} # #----------------------------------------------------------------------- @@ -115,8 +134,6 @@ fi # #----------------------------------------------------------------------- # -export EXPTDIR -export INPUT_BASE export LOG_SUFFIX export MET_INSTALL_DIR export MET_BIN_EXEC diff --git a/scripts/exregional_run_vx_enspoint_prob.sh b/scripts/exregional_run_vx_enspoint_prob.sh index 90fef31710..fdb397aa4e 100755 --- a/scripts/exregional_run_vx_enspoint_prob.sh +++ b/scripts/exregional_run_vx_enspoint_prob.sh @@ -79,22 +79,41 @@ export fhr_list # #----------------------------------------------------------------------- # -# Create INPUT_BASE and LOG_SUFFIX to read into METplus conf files. +# Pick a directory structure for METplus output files # #----------------------------------------------------------------------- # -INPUT_BASE=${EXPTDIR}/${CDATE}/metprd/ensemble_stat -LOG_SUFFIX=enspoint_prob_${CDATE} +if [ $RUN_ENVIR = "nco" ]; then + export INPUT_BASE=$COMOUT/metout/${CDATE}/metprd/ensemble_stat + export OUTPUT_BASE=$COMOUT/metout + export MEM_BASE=$OUTPUT_BASE + export LOG_DIR=$LOGDIR + + export POSTPRD= + export MEM_STAR= + export MEM_CUSTOM= + export DOT_MEM_CUSTOM=".{custom?fmt=%s}" +else + export INPUT_BASE=${EXPTDIR}/${CDATE}/metprd/ensemble_stat + export OUTPUT_BASE=$EXPTDIR + export MEM_BASE=$EXPTDIR/$CDATE + export LOG_DIR=${EXPTDIR}/log + + export POSTPRD="postprd/" + export MEM_STAR="mem*/" + export MEM_CUSTOM="{custom?fmt=%s}/" + export DOT_MEM_CUSTOM= +fi +export DOT_ENSMEM=${dot_ensmem} # #----------------------------------------------------------------------- # -# Make sure directories in which output files will be placed exist. +# Create INPUT_BASE and LOG_SUFFIX to read into METplus conf files. # #----------------------------------------------------------------------- # -mkdir_vrfy -p "${EXPTDIR}/metprd/pb2nc" # Output directory for pb2nc tool. -mkdir_vrfy -p "${EXPTDIR}/${CDATE}/metprd/ensemble_stat_prob" # Output directory for point_stat tool. +LOG_SUFFIX=enspoint_prob_${CDATE} # #----------------------------------------------------------------------- @@ -115,8 +134,6 @@ fi # #----------------------------------------------------------------------- # -export EXPTDIR -export INPUT_BASE export LOG_SUFFIX export MET_INSTALL_DIR export MET_BIN_EXEC diff --git a/scripts/exregional_run_vx_gridstat.sh b/scripts/exregional_run_vx_gridstat.sh index 207ad31ea9..2d78593a5b 100755 --- a/scripts/exregional_run_vx_gridstat.sh +++ b/scripts/exregional_run_vx_gridstat.sh @@ -67,6 +67,41 @@ export fhr_last fhr_list=`echo ${FHR} | $SED "s/ /,/g"` export fhr_list +# +#----------------------------------------------------------------------- +# +# Pick a directory structure for METplus output files +# +#----------------------------------------------------------------------- +# +if [ $RUN_ENVIR = "nco" ]; then + export INPUT_BASE=$COMIN + export OUTPUT_BASE=$COMOUT/metout + export MEM_BASE=$OUTPUT_BASE + export LOG_DIR=$LOGDIR + + export POSTPRD= + export MEM_STAR= + export MEM_CUSTOM= + export DOT_MEM_CUSTOM=".{custom?fmt=%s}" +else + if [[ ${DO_ENSEMBLE} == "FALSE" ]]; then + export INPUT_BASE=${EXPTDIR}/${CDATE}/postprd + export OUTPUT_BASE=${EXPTDIR}/${CDATE} + else + export INPUT_BASE=${EXPTDIR}/${CDATE}/${SLASH_ENSMEM_SUBDIR}/postprd + export OUTPUT_BASE=${EXPTDIR}/${CDATE}/${SLASH_ENSMEM_SUBDIR} + fi + export MEM_BASE=$EXPTDIR/$CDATE + export LOG_DIR=${EXPTDIR}/log + + export POSTPRD="postprd/" + export MEM_STAR="mem*/" + export MEM_CUSTOM="{custom?fmt=%s}/" + export DOT_MEM_CUSTOM= +fi +export DOT_ENSMEM=${dot_ensmem} + # #----------------------------------------------------------------------- # @@ -75,16 +110,12 @@ export fhr_list #----------------------------------------------------------------------- # if [[ ${DO_ENSEMBLE} == "FALSE" ]]; then - INPUT_BASE=${EXPTDIR}/${CDATE}/postprd - OUTPUT_BASE=${EXPTDIR}/${CDATE} if [ ${VAR} == "APCP" ]; then LOG_SUFFIX=gridstat_${CDATE}_${VAR}_${ACCUM}h else LOG_SUFFIX=gridstat_${CDATE}_${VAR} fi elif [[ ${DO_ENSEMBLE} == "TRUE" ]]; then - INPUT_BASE=${EXPTDIR}/${CDATE}/${SLASH_ENSMEM_SUBDIR}/postprd - OUTPUT_BASE=${EXPTDIR}/${CDATE}/${SLASH_ENSMEM_SUBDIR} ENSMEM=`echo ${SLASH_ENSMEM_SUBDIR} | cut -d"/" -f2` MODEL=${MODEL}_${ENSMEM} if [ ${VAR} == "APCP" ]; then @@ -94,25 +125,6 @@ elif [[ ${DO_ENSEMBLE} == "TRUE" ]]; then fi fi -# -#----------------------------------------------------------------------- -# -# Make sure directories in which output files will be placed exist. -# -#----------------------------------------------------------------------- -# -mkdir_vrfy -p "${OUTPUT_BASE}/metprd/grid_stat" -# -# If the variable is accumulated precipitation for a time interval -# (bucket) other than 1 hour, the MET/METplus tools called below will -# include pcp_combine. In that case, create (if necessary) directories -# needed by pcp_combine. -# -if [ "${VAR}" = "APCP" ] && [ "${ACCUM: -1}" != "1" ]; then - mkdir_vrfy -p "${EXPTDIR}/metprd/pcp_combine" # For observations - mkdir_vrfy -p "${OUTPUT_BASE}/metprd/pcp_combine" # For forecast -fi - # #----------------------------------------------------------------------- # @@ -133,10 +145,6 @@ fi # #----------------------------------------------------------------------- # -export SCRIPTSrrfs -export EXPTDIR -export INPUT_BASE -export OUTPUT_BASE export LOG_SUFFIX export MET_INSTALL_DIR export MET_BIN_EXEC diff --git a/scripts/exregional_run_vx_pointstat.sh b/scripts/exregional_run_vx_pointstat.sh index 6ee8bf5640..b985e93c4d 100755 --- a/scripts/exregional_run_vx_pointstat.sh +++ b/scripts/exregional_run_vx_pointstat.sh @@ -70,31 +70,52 @@ export fhr_list # #----------------------------------------------------------------------- # -# Create INPUT_BASE to read into METplus conf files. +# Pick a directory structure for METplus output files # #----------------------------------------------------------------------- # -if [[ ${DO_ENSEMBLE} == "FALSE" ]]; then - INPUT_BASE=${EXPTDIR}/${CDATE}/postprd - OUTPUT_BASE=${EXPTDIR}/${CDATE} - LOG_SUFFIX=pointstat_${CDATE} -elif [[ ${DO_ENSEMBLE} == "TRUE" ]]; then - INPUT_BASE=${EXPTDIR}/${CDATE}/${SLASH_ENSMEM_SUBDIR}/postprd - OUTPUT_BASE=${EXPTDIR}/${CDATE}/${SLASH_ENSMEM_SUBDIR} - ENSMEM=`echo ${SLASH_ENSMEM_SUBDIR} | cut -d"/" -f2` - MODEL=${MODEL}_${ENSMEM} - LOG_SUFFIX=pointstat_${CDATE}_${ENSMEM} +if [ $RUN_ENVIR = "nco" ]; then + export INPUT_BASE=$COMIN + export OUTPUT_BASE=$COMOUT/metout + export MEM_BASE=$OUTPUT_BASE + export LOG_DIR=$LOGDIR + + export POSTPRD= + export MEM_STAR= + export MEM_CUSTOM= + export DOT_MEM_CUSTOM=".{custom?fmt=%s}" +else + if [[ ${DO_ENSEMBLE} == "FALSE" ]]; then + export INPUT_BASE=${EXPTDIR}/${CDATE}/postprd + export OUTPUT_BASE=${EXPTDIR}/${CDATE} + else + export INPUT_BASE=${EXPTDIR}/${CDATE}/${SLASH_ENSMEM_SUBDIR}/postprd + export OUTPUT_BASE=${EXPTDIR}/${CDATE}/${SLASH_ENSMEM_SUBDIR} + fi + export MEM_BASE=$EXPTDIR/$CDATE + export LOG_DIR=${EXPTDIR}/log + + export POSTPRD="postprd/" + export MEM_STAR="mem*/" + export MEM_CUSTOM="{custom?fmt=%s}/" + export DOT_MEM_CUSTOM= fi +export DOT_ENSMEM=${dot_ensmem} # #----------------------------------------------------------------------- # -# Make sure directories in which output files will be placed exist. +# Create INPUT_BASE to read into METplus conf files. # #----------------------------------------------------------------------- # -mkdir_vrfy -p "${EXPTDIR}/metprd/pb2nc" # Output directory for pb2nc tool. -mkdir_vrfy -p "${OUTPUT_BASE}/metprd/point_stat" # Output directory for point_stat tool. +if [[ ${DO_ENSEMBLE} == "FALSE" ]]; then + LOG_SUFFIX=pointstat_${CDATE} +elif [[ ${DO_ENSEMBLE} == "TRUE" ]]; then + ENSMEM=`echo ${SLASH_ENSMEM_SUBDIR} | cut -d"/" -f2` + MODEL=${MODEL}_${ENSMEM} + LOG_SUFFIX=pointstat_${CDATE}_${ENSMEM} +fi # #----------------------------------------------------------------------- @@ -115,9 +136,6 @@ fi # #----------------------------------------------------------------------- # -export EXPTDIR -export INPUT_BASE -export OUTPUT_BASE export LOG_SUFFIX export MET_INSTALL_DIR export MET_BIN_EXEC diff --git a/ush/job_preamble.sh b/ush/job_preamble.sh index 6eb15e28b8..97d5893c55 100644 --- a/ush/job_preamble.sh +++ b/ush/job_preamble.sh @@ -12,7 +12,7 @@ if [ $subcyc -eq 0 ]; then else export cycle="t${cyc}${subcyc}z" fi -if [ "${RUN_ENVIR}" = "nco" ] && [ "${DO_ENSEMBLE}" = "TRUE" ]; then +if [ "${RUN_ENVIR}" = "nco" ] && [ "${DO_ENSEMBLE}" = "TRUE" ] && [ ! -z $ENSMEM_INDX ]; then export dot_ensmem=".mem${ENSMEM_INDX}" else export dot_ensmem= diff --git a/ush/setup.py b/ush/setup.py index a4d68389d5..72db75388e 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -1239,18 +1239,6 @@ def setup(): global SYMLINK_FIX_FILES if RUN_ENVIR == "nco": - if RUN_TASK_VX_GRIDSTAT: - RUN_TASK_VX_GRIDSTAT = False - print_info_msg("""Setting RUN_TASK_VX_GRIDSTAT={RUN_TASK_VX_GRIDSTAT}""") - if RUN_TASK_VX_POINTSTAT: - RUN_TASK_VX_POINTSTAT = False - print_info_msg("""Setting RUN_TASK_VX_POINTSTAT={RUN_TASK_VX_POINTSTAT}""") - if RUN_TASK_VX_ENSPOINT: - RUN_TASK_VX_ENSPOINT = False - print_info_msg("""Setting RUN_TASK_VX_ENSPOINT={RUN_TASK_VX_ENSPOINT}""") - if RUN_TASK_VX_ENSGRID: - RUN_TASK_VX_ENSGRID = False - print_info_msg("""Setting RUN_TASK_VX_ENSGRID={RUN_TASK_VX_ENSGRID}""") if not SYMLINK_FIX_FILES: SYMLINK_FIX_FILES = True print_info_msg("""Setting SYMLINK_FIX_FILES={SYMLINK_FIX_FILES}""") From dcecfe3cd9864a8a3f6dc4b89e0076fe6a03c55d Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Mon, 5 Sep 2022 19:34:13 +0000 Subject: [PATCH 070/106] Pass RUN_ENVIR to we2e script. --- tests/WE2E/run_WE2E_tests.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/WE2E/run_WE2E_tests.sh b/tests/WE2E/run_WE2E_tests.sh index f53e6a5bb2..801cbdfd47 100755 --- a/tests/WE2E/run_WE2E_tests.sh +++ b/tests/WE2E/run_WE2E_tests.sh @@ -93,6 +93,7 @@ Usage: [generate_csv_file=\"...\"] \\ [machine_file=\"...\"] \\ [opsroot=\"...\"] \\ + [run_envir=\"...\"] \\ [compiler=\"...\"] \\ [build_mod_fn=\"...\"] @@ -194,6 +195,9 @@ file. If not set, a supported platform machine file may be used. opsroot: Operations root directory in NCO mode +run_envir: +Overrides RUN_ENVIR variable to a new value ( nco or community ) + compiler: Optional argument used to explicitly set the experiment variable COMPILER in the experiment configuration files of all the WE2E tests the user @@ -319,6 +323,7 @@ valid_args=( \ "generate_csv_file" \ "machine_file" \ "opsroot" \ + "run_envir" \ "compiler" \ "build_mod_fn" \ ) @@ -829,6 +834,20 @@ VERBOSE=\"${VERBOSE}\"" # End of section from this test's configuration file. #----------------------------------------------------------------------- #-----------------------------------------------------------------------" + +# +# Set RUN_ENVIR from the $run_envir argument passed to this script +# +if [ ! -z ${run_envir} ]; then + expt_config_str=${expt_config_str}" +# +# Set RUN_ENVIR +# +RUN_ENVIR=${run_envir}" + + RUN_ENVIR=${run_envir} +fi + # #----------------------------------------------------------------------- # From c25a46c62352caee98a2f7d64c7cf95d83167bca Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Thu, 8 Sep 2022 23:47:34 +0000 Subject: [PATCH 071/106] Fixes for merge conflicts. --- devbuild.sh | 2 +- tests/WE2E/run_WE2E_tests.sh | 2 +- tests/WE2E/setup_WE2E_tests.sh | 2 +- ush/setup.py | 6 +++++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/devbuild.sh b/devbuild.sh index d9326388c6..79c9199322 100755 --- a/devbuild.sh +++ b/devbuild.sh @@ -102,7 +102,7 @@ SRW_DIR=$(cd "$(dirname "$(readlink -f -n "${BASH_SOURCE[0]}" )" )" && pwd -P) MACHINE_SETUP=${SRW_DIR}/src/UFS_UTILS/sorc/machine-setup.sh BUILD_DIR="${SRW_DIR}/build" INSTALL_DIR=${SRW_DIR} -BIN_DIR="bin" #change this to "exec" for NCO mode later +BIN_DIR="exec" COMPILER="" APPLICATION="" CCPP_SUITES="" diff --git a/tests/WE2E/run_WE2E_tests.sh b/tests/WE2E/run_WE2E_tests.sh index 801cbdfd47..9ed4a56c2d 100755 --- a/tests/WE2E/run_WE2E_tests.sh +++ b/tests/WE2E/run_WE2E_tests.sh @@ -28,7 +28,7 @@ scrfunc_dir=$( dirname "${scrfunc_fp}" ) #----------------------------------------------------------------------- # # Set the full path to the top-level directory of the UFS SRW App -# repository. We denote this path by SR_WX_APP_TOP_DIR. The current script +# repository. We denote this path by homerrfs. The current script # should be located in the "tests/WE2E" subdirectory under this directory. # Thus, SR_WX_APP_TOP_DIR is the directory two levels above the directory # in which the current script is located. diff --git a/tests/WE2E/setup_WE2E_tests.sh b/tests/WE2E/setup_WE2E_tests.sh index fbe03cff33..407a27d0d7 100755 --- a/tests/WE2E/setup_WE2E_tests.sh +++ b/tests/WE2E/setup_WE2E_tests.sh @@ -68,7 +68,7 @@ auto_file=${scrfunc_dir}/machine_suites/${machine}.txt # Use exec_subdir consistent with the automated build. #---------------------------------------------------------------------- -exec_subdir='bin_intel/bin' +exec_subdir='bin_intel/exec' #----------------------------------------------------------------------- # Run E2E Tests diff --git a/ush/setup.py b/ush/setup.py index 72db75388e..9593d8383a 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -293,6 +293,10 @@ def setup(): property_name = "local_path" cfg = load_ini_config(mng_extrns_cfg_fn) # + # Get the path to the workflow scripts + # + HOMErrfs = SR_WX_APP_TOP_DIR + # # Get the base directory of the FV3 forecast model code. # external_name = FCST_MODEL @@ -326,7 +330,7 @@ def setup(): SORCrrfs = os.path.join(HOMErrfs, "sorc") PARMrrfs = os.path.join(HOMErrfs, "parm") MODULESrrfs = os.path.join(HOMErrfs, "modulefiles") - EXECrrfs = os.path.join(SR_WX_APP_TOP_DIR, EXEC_SUBDIR) + EXECrrfs = os.path.join(HOMErrfs, EXEC_SUBDIR) VX_CONFIG_DIR = PARMrrfs METPLUS_CONF = os.path.join(PARMrrfs, "metplus") MET_CONFIG = os.path.join(PARMrrfs, "met") From d0ffd80de10d460bdf86595a719d47c247e9a1db Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Tue, 30 Aug 2022 19:16:38 +0000 Subject: [PATCH 072/106] Add versions for wcoss2. --- modulefiles/build_wcoss2_intel | 66 +++++++++++++++++----------------- versions/build.ver | 1 + versions/build.ver.wcoss2 | 35 ++++++++++++++++++ versions/run.ver | 1 + versions/run.ver.wcoss2 | 1 + 5 files changed, 71 insertions(+), 33 deletions(-) create mode 120000 versions/build.ver create mode 100644 versions/build.ver.wcoss2 create mode 120000 versions/run.ver create mode 100644 versions/run.ver.wcoss2 diff --git a/modulefiles/build_wcoss2_intel b/modulefiles/build_wcoss2_intel index f7adefb4f7..c60f9ee68c 100644 --- a/modulefiles/build_wcoss2_intel +++ b/modulefiles/build_wcoss2_intel @@ -7,43 +7,43 @@ proc ModulesHelp { } { module-whatis "Loads libraries needed for building SRW on WCOSS2 (Cactus/Dogwood)" -module load envvar/1.0 +module load envvar/$::env(envvar_ver) -module load PrgEnv-intel/8.1.0 -module load intel/19.1.3.304 -module load craype/2.7.13 -module load cray-mpich/8.1.7 +module load PrgEnv-intel/$::env(PrgEnv_intel_ver) +module load intel/$::env(intel_ver) +module load craype/$::env(craype_ver) +module load cray-mpich/$::env(cray_mpich_ver) -module load cmake/3.20.2 +module load cmake/$::env(cmake_ver) setenv HPC_OPT /apps/ops/para/libs -module use /apps/ops/para/libs/modulefiles/compiler/intel/19.1.3.304 -module use /apps/ops/para/libs/modulefiles/mpi/intel/19.1.3.304/cray-mpich/8.1.7 - -module load jasper/2.0.25 -module load zlib/1.2.11 -module load libpng/1.6.37 -module load hdf5/1.10.6 -module load netcdf/4.7.4 -module load pio/2.5.2 -module load esmf/8.3.0b09 -module load fms/2022.01 -module load bacio/2.4.1 -module load crtm/2.3.0 -module load g2/3.4.5 -module load g2tmpl/1.10.0 -module load ip/3.3.3 -module load sp/2.3.3 -module load w3nco/2.4.1 - -module load libjpeg/9c -module load cray-pals/1.1.3 - -module load w3emc/2.9.2 -module load nemsio/2.5.2 -module load sigio/2.3.2 -module load sfcio/1.4.1 -module load wrf_io/1.2.0 +module use /apps/ops/para/libs/modulefiles/compiler/intel/$::env(intel_ver) +module use /apps/ops/para/libs/modulefiles/mpi/intel/$::env(intel_ver)/cray-mpich/$::env(cray_mpich_ver) + +module load jasper/$::env(jasper_ver) +module load zlib/$::env(zlib_ver) +module load libpng/$::env(libpng_ver) +module load hdf5/$::env(hdf5_ver) +module load netcdf/$::env(netcdf_ver) +module load pio/$::env(pio_ver) +module load esmf/$::env(esmf_ver) +module load fms/$::env(fms_ver) +module load bacio/$::env(bacio_ver) +module load crtm/$::env(crtm_ver) +module load g2/$::env(g2_ver) +module load g2tmpl/$::env(g2tmpl_ver) +module load ip/$::env(ip_ver) +module load sp/$::env(sp_ver) +module load w3nco/$::env(w3nco_ver) + +module load libjpeg/$::env(libjpeg_ver) +module load cray-pals/$::env(cray_pals_ver) + +module load w3emc/$::env(w3emc_ver) +module load nemsio/$::env(nemsio_ver) +module load sigio/$::env(sigio_ver) +module load sfcio/$::env(sfcio_ver) +module load wrf_io/$::env(wrfio_ver) setenv CMAKE_C_COMPILER cc setenv CMAKE_CXX_COMPILER CC diff --git a/versions/build.ver b/versions/build.ver new file mode 120000 index 0000000000..fb0c92cf93 --- /dev/null +++ b/versions/build.ver @@ -0,0 +1 @@ +build.ver.wcoss2 \ No newline at end of file diff --git a/versions/build.ver.wcoss2 b/versions/build.ver.wcoss2 new file mode 100644 index 0000000000..9a89f97716 --- /dev/null +++ b/versions/build.ver.wcoss2 @@ -0,0 +1,35 @@ +#!/bin/bash + +export module_ver=8.5.2 +export gfs_ver=v16.2 + +export envvar_ver=1.0 +export PrgEnv_intel_ver=8.1.0 +export intel_ver=19.1.3.304 +export craype_ver=2.7.13 +export cray_mpich_ver=8.1.7 +export cmake_ver=3.20.2 +export intel_ver=19.1.3.304 +export cray_mpich_ver=8.1.7 +export jasper_ver=2.0.25 +export zlib_ver=1.2.11 +export libpng_ver=1.6.37 +export hdf5_ver=1.10.6 +export netcdf_ver=4.7.4 +export pio_ver=2.5.2 +export esmf_ver=8.3.0b09 +export fms_ver=2022.01 +export bacio_ver=2.4.1 +export crtm_ver=2.3.0 +export g2_ver=3.4.5 +export g2tmpl_ver=1.10.0 +export ip_ver=3.3.3 +export sp_ver=2.3.3 +export w3nco_ver=2.4.1 +export libjpeg_ver=9c +export cray_pals_ver=1.1.3 +export w3emc_ver=2.9.2 +export nemsio_ver=2.5.2 +export sigio_ver=2.3.2 +export sfcio_ver=1.4.1 +export wrf_io_ver=1.2.0 diff --git a/versions/run.ver b/versions/run.ver new file mode 120000 index 0000000000..feb792992b --- /dev/null +++ b/versions/run.ver @@ -0,0 +1 @@ +run.ver.wcoss2 \ No newline at end of file diff --git a/versions/run.ver.wcoss2 b/versions/run.ver.wcoss2 new file mode 100644 index 0000000000..a9bf588e2f --- /dev/null +++ b/versions/run.ver.wcoss2 @@ -0,0 +1 @@ +#!/bin/bash From b695afb5945edca3a7180c457f67bb4d5f51f5f3 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Mon, 12 Sep 2022 01:05:38 +0000 Subject: [PATCH 073/106] Fix symlinks. --- parm/ufs_utils | 2 +- parm/upp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/ufs_utils b/parm/ufs_utils index 87c46354a5..d9985c4046 120000 --- a/parm/ufs_utils +++ b/parm/ufs_utils @@ -1 +1 @@ -../../sorc/UFS_UTILS/parm/ \ No newline at end of file +../sorc/UFS_UTILS/parm/ \ No newline at end of file diff --git a/parm/upp b/parm/upp index 6a5fa737a4..f124b19863 120000 --- a/parm/upp +++ b/parm/upp @@ -1 +1 @@ -../../sorc/UPP/parm/ \ No newline at end of file +../sorc/UPP/parm/ \ No newline at end of file From 5bca8f7fd7b81b99061cdb2ab9decf370ca2060b Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Mon, 12 Sep 2022 19:36:24 +0000 Subject: [PATCH 074/106] Minor changes. --- tests/WE2E/get_expts_status.sh | 10 +++++----- tests/WE2E/run_WE2E_tests.sh | 16 ++++++++-------- ush/config_defaults.yaml | 4 ++-- ush/load_modules_run_task.sh | 8 ++++---- 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/tests/WE2E/get_expts_status.sh b/tests/WE2E/get_expts_status.sh index f14879861b..8eb65d5e19 100755 --- a/tests/WE2E/get_expts_status.sh +++ b/tests/WE2E/get_expts_status.sh @@ -44,13 +44,13 @@ scrfunc_dir=$( dirname "${scrfunc_fp}" ) #----------------------------------------------------------------------- # # The current script should be located in the "tests" subdirectory of the -# workflow's top-level directory, which we denote by SR_WX_APP_TOP_DIR. -# SR_WX_APP_TOP_DIR is the directory two levels above the directory in -# which the current script is located; Set SR_WX_APP_TOP_DIR accordingly. +# workflow's top-level directory, which we denote by HOMErrfs. Thus, +# HOMErrfs is the directory one level above the directory in which the +# current script is located. Set HOMErrfs accordingly. # #----------------------------------------------------------------------- # -SR_WX_APP_TOP_DIR=${scrfunc_dir%/*/*} +HOMErrfs=${scrfunc_dir%/*/*} # #----------------------------------------------------------------------- # @@ -58,7 +58,7 @@ SR_WX_APP_TOP_DIR=${scrfunc_dir%/*/*} # #----------------------------------------------------------------------- # -ushdir="$SR_WX_APP_TOP_DIR/ush" +ushdir="$HOMErrfs/ush" # #----------------------------------------------------------------------- # diff --git a/tests/WE2E/run_WE2E_tests.sh b/tests/WE2E/run_WE2E_tests.sh index 9ed4a56c2d..238a8d48d1 100755 --- a/tests/WE2E/run_WE2E_tests.sh +++ b/tests/WE2E/run_WE2E_tests.sh @@ -28,23 +28,23 @@ scrfunc_dir=$( dirname "${scrfunc_fp}" ) #----------------------------------------------------------------------- # # Set the full path to the top-level directory of the UFS SRW App -# repository. We denote this path by homerrfs. The current script +# repository. We denote this path by HOMErrfs. The current script # should be located in the "tests/WE2E" subdirectory under this directory. -# Thus, SR_WX_APP_TOP_DIR is the directory two levels above the directory -# in which the current script is located. +# Thus, HOMErrfs is the directory two levels above the directory in which +# the current script is located. # #----------------------------------------------------------------------- # -SR_WX_APP_TOP_DIR=${scrfunc_dir%/*/*} +HOMErrfs=${scrfunc_dir%/*/*} # #----------------------------------------------------------------------- # -# Set other directories that depend on SR_WX_APP_TOP_DIR. +# Set other directories that depend on HOMErrfs. # #----------------------------------------------------------------------- # -ushdir="$SR_WX_APP_TOP_DIR/ush" -testsdir="$SR_WX_APP_TOP_DIR/tests" +ushdir="$HOMErrfs/ush" +testsdir="$HOMErrfs/tests" WE2Edir="$testsdir/WE2E" # #----------------------------------------------------------------------- @@ -932,7 +932,7 @@ model_ver="we2e"" # # Set OPSROOT. # - nco_basedir=$( readlink -f "$homerrfs/../../nco_dirs" ) + nco_basedir=$( readlink -f "$HOMErrfs/../../nco_dirs" ) OPSROOT=${opsroot:-"${nco_basedir}/OPSROOT"} expt_config_str=${expt_config_str}" diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index d73750c42c..950d604ced 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -356,7 +356,7 @@ workflow: # EXPT_BASEDIR: # The base directory in which the experiment directory will be created. # If this is not specified or if it is set to an empty string, it will - # default to ${SR_WX_APP_TOP_DIR}/../expt_dirs. + # default to ${HOMErrfs}/../expt_dirs. # # EXPT_SUBDIR: # The name that the experiment directory (without the full path) will @@ -1563,7 +1563,7 @@ task_run_fcst: # commonly used set of grid-dependent parameters. The predefined grid # parameters are specified in the script # - # $SR_WX_APP_TOP_DIR/ush/set_predef_grid_params.sh + # $HOMErrfs/ush/set_predef_grid_params.sh # #----------------------------------------------------------------------- # diff --git a/ush/load_modules_run_task.sh b/ush/load_modules_run_task.sh index caa5bdffc2..b440d21546 100755 --- a/ush/load_modules_run_task.sh +++ b/ush/load_modules_run_task.sh @@ -134,17 +134,17 @@ for the workflow task specified by task_name failed: # # The full path to a module file for a given task is # -# $SR_WX_APP_TOP_DIR/modulefiles/$machine/${task_name}.local +# $HOMErrfs/modulefiles/$machine/${task_name}.local # -# where SR_WX_APP_TOP_DIR is the base directory of the workflow, machine is the +# where HOMErrfs is the base directory of the workflow, machine is the # name of the machine that we're running on (in lowercase), and task_- # name is the name of the current task (an input to this script). # #----------------------------------------------------------------------- # -modules_dir="$SR_WX_APP_TOP_DIR/modulefiles/tasks/$machine" +modules_dir="$HOMErrfs/modulefiles/tasks/$machine" modulefile_name="${task_name}" -default_modules_dir="$SR_WX_APP_TOP_DIR/modulefiles" +default_modules_dir="$HOMErrfs/modulefiles" # #----------------------------------------------------------------------- # From 08e63a0c9aacc12385ec06bf1883f905509c876b Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Tue, 13 Sep 2022 03:02:36 +0000 Subject: [PATCH 075/106] Move grid/orog/sfcc completion files to EXPTDIR/grid/orog etc. --- jobs/JREGIONAL_MAKE_GRID | 2 +- jobs/JREGIONAL_MAKE_OROG | 2 +- jobs/JREGIONAL_MAKE_SFC_CLIMO | 2 +- parm/FV3LAM_wflow.xml | 21 +++++++++++---------- ush/generate_FV3LAM_wflow.py | 1 + 5 files changed, 15 insertions(+), 13 deletions(-) diff --git a/jobs/JREGIONAL_MAKE_GRID b/jobs/JREGIONAL_MAKE_GRID index fc6a36d76b..d754e2ba03 100755 --- a/jobs/JREGIONAL_MAKE_GRID +++ b/jobs/JREGIONAL_MAKE_GRID @@ -197,7 +197,7 @@ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." # #----------------------------------------------------------------------- # -touch "$LOGDIR/make_grid_task_complete_${WORKFLOW_ID}.txt" +touch "$GRID_DIR/make_grid_task_complete.txt" # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_MAKE_OROG b/jobs/JREGIONAL_MAKE_OROG index de0705a2f7..d07224af61 100755 --- a/jobs/JREGIONAL_MAKE_OROG +++ b/jobs/JREGIONAL_MAKE_OROG @@ -79,7 +79,7 @@ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." # #----------------------------------------------------------------------- # -touch "$LOGDIR/make_orog_task_complete_${WORKFLOW_ID}.txt" +touch "$OROG_DIR/make_orog_task_complete.txt" # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_MAKE_SFC_CLIMO b/jobs/JREGIONAL_MAKE_SFC_CLIMO index 6d6f106b45..51fece5200 100755 --- a/jobs/JREGIONAL_MAKE_SFC_CLIMO +++ b/jobs/JREGIONAL_MAKE_SFC_CLIMO @@ -100,7 +100,7 @@ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." # #----------------------------------------------------------------------- # -touch "$LOGDIR/make_sfc_climo_task_complete_${WORKFLOW_ID}.txt" +touch "$SFC_CLIMO_DIR/make_sfc_climo_task_complete.txt" # #----------------------------------------------------------------------- # diff --git a/parm/FV3LAM_wflow.xml b/parm/FV3LAM_wflow.xml index 759d9378fe..ae77983933 100644 --- a/parm/FV3LAM_wflow.xml +++ b/parm/FV3LAM_wflow.xml @@ -92,6 +92,7 @@ Directories and files. + {%- if run_envir == "nco" %} @@ -107,7 +108,7 @@ Directories and files. {%- endif %} - + @@ -238,7 +239,7 @@ MODULES_RUN_TASK_FP script. - &LOGDIR;/&MAKE_GRID_TN;&CMPEXT; + &EXPTDIR;/grid/&MAKE_GRID_TN;&CMPEXT; &RUN_TASK_MAKE_GRID;FALSE @@ -273,12 +274,12 @@ MODULES_RUN_TASK_FP script. - &LOGDIR;/&MAKE_GRID_TN;&CMPEXT; + &EXPTDIR;/grid/&MAKE_GRID_TN;&CMPEXT; &RUN_TASK_MAKE_GRID;FALSE - &LOGDIR;/&MAKE_OROG_TN;&CMPEXT; + &EXPTDIR;/orog/&MAKE_OROG_TN;&CMPEXT; &RUN_TASK_MAKE_OROG;FALSE @@ -384,17 +385,17 @@ MODULES_RUN_TASK_FP script. - &LOGDIR;/&MAKE_GRID_TN;&CMPEXT; + &EXPTDIR;/grid/&MAKE_GRID_TN;&CMPEXT; &RUN_TASK_MAKE_GRID;FALSE - &LOGDIR;/&MAKE_OROG_TN;&CMPEXT; + &EXPTDIR;/orog/&MAKE_OROG_TN;&CMPEXT; &RUN_TASK_MAKE_OROG;FALSE - &LOGDIR;/&MAKE_SFC_CLIMO_TN;&CMPEXT; + &EXPTDIR;/sfc_climo/&MAKE_SFC_CLIMO_TN;&CMPEXT; &RUN_TASK_MAKE_SFC_CLIMO;FALSE @@ -434,17 +435,17 @@ MODULES_RUN_TASK_FP script. - &LOGDIR;/&MAKE_GRID_TN;&CMPEXT; + &EXPTDIR;/grid/&MAKE_GRID_TN;&CMPEXT; &RUN_TASK_MAKE_GRID;FALSE - &LOGDIR;/&MAKE_OROG_TN;&CMPEXT; + &EXPTDIR;/orog/&MAKE_OROG_TN;&CMPEXT; &RUN_TASK_MAKE_OROG;FALSE - &LOGDIR;/&MAKE_SFC_CLIMO_TN;&CMPEXT; + &EXPTDIR;/sfc_climo/&MAKE_SFC_CLIMO_TN;&CMPEXT; &RUN_TASK_MAKE_SFC_CLIMO;FALSE diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index ca42f3d947..ad6a38ce54 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -374,6 +374,7 @@ def generate_FV3LAM_wflow(): # # Directories and files. # + "exptdir": EXPTDIR, "jobsrrfs": JOBSrrfs, "logdir": LOGDIR, "scriptsrrfs": SCRIPTSrrfs, From fb403fb1914b6b0c301794e32ecdfb7e9940f3e9 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Tue, 13 Sep 2022 04:17:10 +0000 Subject: [PATCH 076/106] Output modified namelist file with seeds in current directory. --- ush/set_FV3nml_ens_stoch_seeds.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/ush/set_FV3nml_ens_stoch_seeds.py b/ush/set_FV3nml_ens_stoch_seeds.py index 86722ac439..6121cc516a 100644 --- a/ush/set_FV3nml_ens_stoch_seeds.py +++ b/ush/set_FV3nml_ens_stoch_seeds.py @@ -14,6 +14,7 @@ date_to_str, mkdir_vrfy, cp_vrfy, + cd_vrfy, str_to_type, import_vars, set_env_var, @@ -57,12 +58,7 @@ def set_FV3nml_ens_stoch_seeds(cdate): # # ----------------------------------------------------------------------- # - ensmem_name = f"mem{ENSMEM_INDX}" - - fv3_nml_ensmem_fp = os.path.join( - COMIN_BASEDIR, - f'{date_to_str(cdate,format="%Y%m%d%H")}{os.sep}{ensmem_name}{os.sep}{FV3_NML_FN}', - ) + fv3_nml_ensmem_fp = f"{os.getcwd()}{os.sep}{FV3_NML_FN}" ensmem_num = ENSMEM_INDX @@ -175,6 +171,10 @@ def setUp(self): USHrrfs = os.path.dirname(os.path.abspath(__file__)) PARMrrfs = os.path.join(USHrrfs, "..", "parm") EXPTDIR = os.path.join(USHrrfs, "test_data", "expt") + print( + os.path.join(PARMrrfs, "input.nml.FV3"), + os.path.join(EXPTDIR, "input.nml"), + ) cp_vrfy( os.path.join(PARMrrfs, "input.nml.FV3"), os.path.join(EXPTDIR, "input.nml"), @@ -188,8 +188,9 @@ def setUp(self): ), ) + cd_vrfy(f'{EXPTDIR}{os.sep}{date_to_str(self.cdate,format="%Y%m%d%H")}{os.sep}mem2') + set_env_var("USHrrfs", USHrrfs) - set_env_var("COMIN_BASEDIR", EXPTDIR) set_env_var("ENSMEM_INDX", 2) set_env_var("FV3_NML_FN", "input.nml") set_env_var("FV3_NML_FP", os.path.join(EXPTDIR, "input.nml")) From fcdffac21c2418f38c5a19603222ff6f82686711 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Tue, 13 Sep 2022 13:10:37 +0000 Subject: [PATCH 077/106] Fixes for unittests. --- .github/workflows/build.yml | 7 +------ .github/workflows/python_unittests.yaml | 4 +++- ush/check_ruc_lsm.py | 3 ++- ush/set_FV3nml_ens_stoch_seeds.py | 5 +---- ush/set_FV3nml_sfc_climo_filenames.py | 1 + ush/set_ozone_param.py | 3 ++- ush/set_thompson_mp_fix_files.py | 3 ++- 7 files changed, 12 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4703c58749..130ffd2e49 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,10 +12,5 @@ jobs: - run: | cd ufs-srweather-app ./manage_externals/checkout_externals - module use modulefiles - module load build_cheyenne_intel - mdkir build - cd build - cmake -DCMAKE_INSTALL_PREFIX=.. .. - make -j 4 + ./devbuild.sh -p=cheynne -c=intel - run: echo "🍏 This job's status is ${{ job.status }}." diff --git a/.github/workflows/python_unittests.yaml b/.github/workflows/python_unittests.yaml index b67039c923..6e8a7f2464 100644 --- a/.github/workflows/python_unittests.yaml +++ b/.github/workflows/python_unittests.yaml @@ -23,5 +23,7 @@ jobs: run: | cd ush python3 -m unittest -b python_utils/test_python_utils.py - python3 -m unittest -b *.py + files=$(find . -maxdepth 1 -name '*.py' -a ! -name 'test_retrieve_data.py' -exec basename {} \;) + files=$(echo $(echo "${files[@]}")) + python3 -m unittest -b ${files[@]} diff --git a/ush/check_ruc_lsm.py b/ush/check_ruc_lsm.py index 44dc72266d..b7bda0f1c9 100644 --- a/ush/check_ruc_lsm.py +++ b/ush/check_ruc_lsm.py @@ -30,8 +30,9 @@ def check_ruc_lsm(ccpp_phys_suite_fp): class Testing(unittest.TestCase): def test_check_ruc_lsm(self): + USHrrfs = os.path.dirname(os.path.abspath(__file__)) self.assertTrue( - check_ruc_lsm(ccpp_phys_suite_fp=f"test_data{os.sep}suite_FV3_GSD_SAR.xml") + check_ruc_lsm(ccpp_phys_suite_fp=f"{USHrrfs}{os.sep}test_data{os.sep}suite_FV3_GSD_SAR.xml") ) def setUp(self): diff --git a/ush/set_FV3nml_ens_stoch_seeds.py b/ush/set_FV3nml_ens_stoch_seeds.py index 6121cc516a..0f2bce4969 100644 --- a/ush/set_FV3nml_ens_stoch_seeds.py +++ b/ush/set_FV3nml_ens_stoch_seeds.py @@ -171,10 +171,7 @@ def setUp(self): USHrrfs = os.path.dirname(os.path.abspath(__file__)) PARMrrfs = os.path.join(USHrrfs, "..", "parm") EXPTDIR = os.path.join(USHrrfs, "test_data", "expt") - print( - os.path.join(PARMrrfs, "input.nml.FV3"), - os.path.join(EXPTDIR, "input.nml"), - ) + mkdir_vrfy("-p", EXPTDIR) cp_vrfy( os.path.join(PARMrrfs, "input.nml.FV3"), os.path.join(EXPTDIR, "input.nml"), diff --git a/ush/set_FV3nml_sfc_climo_filenames.py b/ush/set_FV3nml_sfc_climo_filenames.py index 419b45721a..483948ead5 100644 --- a/ush/set_FV3nml_sfc_climo_filenames.py +++ b/ush/set_FV3nml_sfc_climo_filenames.py @@ -154,6 +154,7 @@ def setUp(self): EXPTDIR = os.path.join(USHrrfs, "test_data", "expt") FIXlam = os.path.join(EXPTDIR, "fix_lam") mkdir_vrfy("-p", FIXlam) + mkdir_vrfy("-p", EXPTDIR) cp_vrfy( os.path.join(PARMrrfs, "input.nml.FV3"), os.path.join(EXPTDIR, "input.nml"), diff --git a/ush/set_ozone_param.py b/ush/set_ozone_param.py index 65bde9675d..c4462b8ddb 100644 --- a/ush/set_ozone_param.py +++ b/ush/set_ozone_param.py @@ -187,10 +187,11 @@ def set_ozone_param(ccpp_phys_suite_fp): class Testing(unittest.TestCase): def test_set_ozone_param(self): + USHrrfs = os.path.dirname(os.path.abspath(__file__)) self.assertEqual( "ozphys_2015", set_ozone_param( - ccpp_phys_suite_fp=f"test_data{os.sep}suite_FV3_GSD_SAR.xml" + ccpp_phys_suite_fp=f"{USHrrfs}{os.sep}test_data{os.sep}suite_FV3_GSD_SAR.xml" ), ) diff --git a/ush/set_thompson_mp_fix_files.py b/ush/set_thompson_mp_fix_files.py index 7c9ffec727..c0dad9bf35 100644 --- a/ush/set_thompson_mp_fix_files.py +++ b/ush/set_thompson_mp_fix_files.py @@ -127,10 +127,11 @@ def set_thompson_mp_fix_files(ccpp_phys_suite_fp, thompson_mp_climo_fn): class Testing(unittest.TestCase): def test_set_thompson_mp_fix_files(self): + USHrrfs = os.path.dirname(os.path.abspath(__file__)) self.assertEqual( True, set_thompson_mp_fix_files( - ccpp_phys_suite_fp=f"test_data{os.sep}suite_FV3_GSD_SAR.xml", + ccpp_phys_suite_fp=f"{USHrrfs}{os.sep}test_data{os.sep}suite_FV3_GSD_SAR.xml", thompson_mp_climo_fn="Thompson_MP_MONTHLY_CLIMO.nc", ), ) From 2a92159586bdaa0daee8089ca9bb29cf198e2c9c Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Tue, 13 Sep 2022 19:04:00 +0000 Subject: [PATCH 078/106] Bugfix wrf_io version --- modulefiles/build_wcoss2_intel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modulefiles/build_wcoss2_intel b/modulefiles/build_wcoss2_intel index c60f9ee68c..8e771b6a80 100644 --- a/modulefiles/build_wcoss2_intel +++ b/modulefiles/build_wcoss2_intel @@ -43,7 +43,7 @@ module load w3emc/$::env(w3emc_ver) module load nemsio/$::env(nemsio_ver) module load sigio/$::env(sigio_ver) module load sfcio/$::env(sfcio_ver) -module load wrf_io/$::env(wrfio_ver) +module load wrf_io/$::env(wrf_io_ver) setenv CMAKE_C_COMPILER cc setenv CMAKE_CXX_COMPILER CC From 9d5551e3fdb88405214b1ffb543fcf52bdc004ec Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Wed, 14 Sep 2022 19:18:09 +0000 Subject: [PATCH 079/106] Fix CI issue with bin locations. --- test/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/build.sh b/test/build.sh index b48c9ad5fc..e269d2bdad 100755 --- a/test/build.sh +++ b/test/build.sh @@ -136,7 +136,7 @@ declare -a executables_rrfs_utl_created=( adjust_soiltq.exe \ for compiler in "${compilers[@]}"; do BUILD_DIR=${TOP_DIR}/build_${compiler} BIN_DIR=${TOP_DIR}/bin_${compiler} - EXEC_DIR=${BIN_DIR}/bin + EXEC_DIR=${BIN_DIR}/exec if [ $build_it -eq 0 ] ; then ./devbuild.sh --platform=${machine} --compiler=${compiler} --build-dir=${BUILD_DIR} --install-dir=${BIN_DIR} \ --remove all || fail "Build ${machine} ${compiler} FAILED" From 6a52e6259620eb794e49f392f4e02b3929df91bd Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Wed, 14 Sep 2022 21:22:04 +0000 Subject: [PATCH 080/106] Allow NCO root directories to be set individually. --- ush/setup.py | 50 +++++++++++++++++++++++++++++--------------------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/ush/setup.py b/ush/setup.py index 9593d8383a..47bf4f9c16 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -871,14 +871,16 @@ def setup(): # Main directory locations if RUN_ENVIR == "nco": - try: - COMROOT - except NameError: - OPSROOT = OPSROOT or EXPTDIR - COMROOT = os.path.join(OPSROOT, "com") - PACKAGEROOT = os.path.join(OPSROOT, "packages") - DATAROOT = os.path.join(OPSROOT, "tmp") - DCOMROOT = os.path.join(OPSROOT, "dcom") + try: OPSROOT + except NameError: OPSROOT = OPSROOT or EXPTDIR + try: COMROOT + except NameError: COMROOT = os.path.join(OPSROOT, "com") + try: PACKAGEROOT + except NameError: PACKAGEROOT = os.path.join(OPSROOT, "packages") + try: DATAROOT + except NameError: DATAROOT = os.path.join(OPSROOT, "tmp") + try: DCOMROOT + except NameError: DCOMROOT = os.path.join(OPSROOT, "dcom") COMIN_BASEDIR = os.path.join(COMROOT, NET, model_ver) COMOUT_BASEDIR = os.path.join(COMROOT, NET, model_ver) @@ -897,21 +899,27 @@ def setup(): LOGDIR = os.path.join(EXPTDIR, "log") - try: - DBNROOT - except NameError: - DBNROOT = None - SENDECF = False - SENDDBN = False - SENDDBN_NTC = False - SENDCOM = False - SENDWEB = False - KEEPDATA = True - MAILTO = None - MAILCC = None + try: DBNROOT + except NameError: DBNROOT = None + try: SENDECF + except NameError: SENDECF = False + try: SENDDBN + except NameError: SENDDBN = False + try: SENDDBN_NTC + except NameError: SENDDBN_NTC = False + try: SENDCOM + except NameError: SENDCOM = False + try: SENDWEB + except NameError: SENDWEB = False + try: KEEPDATA + except NameError: KEEPDATA = True + try: MAILTO + except NameError: MAILTO = None + try: MAILCC + except NameError: MAILCC = None # create NCO directories - if OPSROOT is not None: + if RUN_ENVIR == "nco": mkdir_vrfy(f' -p "{OPSROOT}"') mkdir_vrfy(f' -p "{COMROOT}"') mkdir_vrfy(f' -p "{PACKAGEROOT}"') From 6b14cd1b9a776128d708d646b9d378f68a312888 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Thu, 15 Sep 2022 03:24:19 +0000 Subject: [PATCH 081/106] Don't append workflow id in community mode. --- parm/FV3LAM_wflow.xml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/parm/FV3LAM_wflow.xml b/parm/FV3LAM_wflow.xml index ae77983933..847a71a914 100644 --- a/parm/FV3LAM_wflow.xml +++ b/parm/FV3LAM_wflow.xml @@ -104,10 +104,11 @@ Directories and files. {%- if run_envir == "nco" %} + {%- else %} + {%- endif %} - @@ -156,7 +157,11 @@ tasks; and the "FCST" type is used for the RUN_FCST_TN task. {%- endfor %} + {%- if run_envir == "nco" %} &LOGDIR;/FV3LAM_wflow_{{ workflow_id }}.log + {%- else %} + &LOGDIR;/FV3LAM_wflow.log + {%- endif %} - - + + @@ -180,7 +180,7 @@ we are already sourcing other files in the J-job anyway. --> leave empty for REFC and RETOP export FHR=`echo $(seq 0 ${ACCUM} ${FCST_LEN_HRS}) | cut -d" " -f2-` -${JOBSrrfs}/JREGIONAL_RUN_VX_ENSGRID +${JOBSdir}/JREGIONAL_RUN_VX_ENSGRID -${JOBSrrfs}/JREGIONAL_RUN_VX_ENSGRID_MEAN +${JOBSdir}/JREGIONAL_RUN_VX_ENSGRID_MEAN -${JOBSrrfs}/JREGIONAL_RUN_VX_ENSGRID_PROB +${JOBSdir}/JREGIONAL_RUN_VX_ENSGRID_PROB diff --git a/ush/wrappers/run_gridvx.sh b/ush/wrappers/run_gridvx.sh index 00326d975b..71a577ef51 100755 --- a/ush/wrappers/run_gridvx.sh +++ b/ush/wrappers/run_gridvx.sh @@ -15,5 +15,5 @@ export ACCUM="06" # 01 03 06 24 --> leave empty for REFC and RETOP export FHR=`echo $(seq 0 ${ACCUM} ${FCST_LEN_HRS}) | cut -d" " -f2-` -${JOBSrrfs}/JREGIONAL_RUN_VX_GRIDSTAT +${JOBSdir}/JREGIONAL_RUN_VX_GRIDSTAT diff --git a/ush/wrappers/run_make_grid.sh b/ush/wrappers/run_make_grid.sh index 2d0a3c98a3..9b2242f0f4 100755 --- a/ush/wrappers/run_make_grid.sh +++ b/ush/wrappers/run_make_grid.sh @@ -5,4 +5,4 @@ source ${GLOBAL_VAR_DEFNS_FP} export CDATE=${DATE_FIRST_CYCL}${CYCL_HRS} export CYCLE_DIR=${EXPTDIR}/${CDATE} -${JOBSrrfs}/JREGIONAL_MAKE_GRID +${JOBSdir}/JREGIONAL_MAKE_GRID diff --git a/ush/wrappers/run_make_ics.sh b/ush/wrappers/run_make_ics.sh index bf63e9680a..8ecfce2799 100755 --- a/ush/wrappers/run_make_ics.sh +++ b/ush/wrappers/run_make_ics.sh @@ -6,5 +6,5 @@ export CDATE=${DATE_FIRST_CYCL}${CYCL_HRS} export CYCLE_DIR=${EXPTDIR}/${CDATE} export SLASH_ENSMEM_SUBDIR="" -${JOBSrrfs}/JREGIONAL_MAKE_ICS +${JOBSdir}/JREGIONAL_MAKE_ICS diff --git a/ush/wrappers/run_make_lbcs.sh b/ush/wrappers/run_make_lbcs.sh index 697e1add5c..7377444738 100755 --- a/ush/wrappers/run_make_lbcs.sh +++ b/ush/wrappers/run_make_lbcs.sh @@ -6,5 +6,5 @@ export CDATE=${DATE_FIRST_CYCL}${CYCL_HRS} export CYCLE_DIR=${EXPTDIR}/${CDATE} export SLASH_ENSMEM_SUBDIR="" -${JOBSrrfs}/JREGIONAL_MAKE_LBCS +${JOBSdir}/JREGIONAL_MAKE_LBCS diff --git a/ush/wrappers/run_make_orog.sh b/ush/wrappers/run_make_orog.sh index 3fd255d1f5..cf1972e6de 100755 --- a/ush/wrappers/run_make_orog.sh +++ b/ush/wrappers/run_make_orog.sh @@ -5,4 +5,4 @@ source ${GLOBAL_VAR_DEFNS_FP} export CDATE=${DATE_FIRST_CYCL}${CYCL_HRS} export CYCLE_DIR=${EXPTDIR}/${CDATE} -${JOBSrrfs}/JREGIONAL_MAKE_OROG +${JOBSdir}/JREGIONAL_MAKE_OROG diff --git a/ush/wrappers/run_make_sfc_climo.sh b/ush/wrappers/run_make_sfc_climo.sh index 1972db228f..538d6f8924 100755 --- a/ush/wrappers/run_make_sfc_climo.sh +++ b/ush/wrappers/run_make_sfc_climo.sh @@ -5,4 +5,4 @@ source ${GLOBAL_VAR_DEFNS_FP} export CDATE=${DATE_FIRST_CYCL}${CYCL_HRS} export CYCLE_DIR=${EXPTDIR}/${CDATE} -${JOBSrrfs}/JREGIONAL_MAKE_SFC_CLIMO +${JOBSdir}/JREGIONAL_MAKE_SFC_CLIMO diff --git a/ush/wrappers/run_pointensvx.sh b/ush/wrappers/run_pointensvx.sh index 8bd85b35f2..f066ea8676 100755 --- a/ush/wrappers/run_pointensvx.sh +++ b/ush/wrappers/run_pointensvx.sh @@ -12,9 +12,9 @@ export OBS_DIR=${NDAS_OBS_DIR} export FHR=`echo $(seq 0 1 ${FCST_LEN_HRS})` -${JOBSrrfs}/JREGIONAL_RUN_VX_ENSPOINT +${JOBSdir}/JREGIONAL_RUN_VX_ENSPOINT -${JOBSrrfs}/JREGIONAL_RUN_VX_ENSPOINT_MEAN +${JOBSdir}/JREGIONAL_RUN_VX_ENSPOINT_MEAN -${JOBSrrfs}/JREGIONAL_RUN_VX_ENSPOINT_PROB +${JOBSdir}/JREGIONAL_RUN_VX_ENSPOINT_PROB diff --git a/ush/wrappers/run_pointvx.sh b/ush/wrappers/run_pointvx.sh index 74c76bbfdf..5f87d29fbc 100755 --- a/ush/wrappers/run_pointvx.sh +++ b/ush/wrappers/run_pointvx.sh @@ -13,5 +13,5 @@ export OBS_DIR=${NDAS_OBS_DIR} export FHR=`echo $(seq 0 1 ${FCST_LEN_HRS})` -${JOBSrrfs}/JREGIONAL_RUN_VX_POINTSTAT +${JOBSdir}/JREGIONAL_RUN_VX_POINTSTAT diff --git a/ush/wrappers/run_post.sh b/ush/wrappers/run_post.sh index 4f7c8d293b..6e63226d21 100755 --- a/ush/wrappers/run_post.sh +++ b/ush/wrappers/run_post.sh @@ -11,5 +11,5 @@ export ENSMEM_INDX="" num_fcst_hrs=${FCST_LEN_HRS} for (( i=0; i<=$((num_fcst_hrs)); i++ )); do export fhr=`printf "%03i" ${i}` - ${JOBSrrfs}/JREGIONAL_RUN_POST + ${JOBSdir}/JREGIONAL_RUN_POST done From 42ea00d2b399e6d55ed12c1a9174997f3c423b01 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Fri, 16 Sep 2022 21:08:09 +0000 Subject: [PATCH 085/106] Add unittest for whole workflow now that the merge made it possible. --- .github/workflows/python_unittests.yaml | 4 ++- ush/generate_FV3LAM_wflow.py | 44 ++++++++++++------------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/.github/workflows/python_unittests.yaml b/.github/workflows/python_unittests.yaml index 6e8a7f2464..0b780e2340 100644 --- a/.github/workflows/python_unittests.yaml +++ b/.github/workflows/python_unittests.yaml @@ -15,12 +15,14 @@ jobs: run: | sudo apt-get update sudo apt-get install python3 python3-pip netcdf-bin - sudo pip3 install pyyaml jinja2 f90nml + sudo pip3 install pyyaml jinja2==2.11 f90nml sudo pip3 install numpy matplotlib basemap + sudo ln -sf /bin/bash /bin/sh # Run python unittests - name: Run python unittests run: | + ./manage_externals/checkout_externals ufs-weather-model cd ush python3 -m unittest -b python_utils/test_python_utils.py files=$(find . -maxdepth 1 -name '*.py' -a ! -name 'test_retrieve_data.py' -exec basename {} \;) diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index c0c0027e9c..63144fe5f6 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -4,6 +4,7 @@ import sys import platform import subprocess +import unittest from multiprocessing import Process from textwrap import dedent from datetime import datetime, timedelta @@ -25,6 +26,7 @@ check_for_preexist_dir_file, cfg_to_yaml_str, find_pattern_in_str, + set_env_var, ) from setup import setup @@ -518,28 +520,15 @@ def generate_FV3LAM_wflow(): # if SYMLINK_FIX_FILES: + print_info_msg( + f''' + Symlinking fixed files from system directory (FIXgsm) to a subdirectory (FIXam): + FIXgsm = \"{FIXgsm}\" + FIXam = \"{FIXam}\"''', + verbose=VERBOSE, + ) + ln_vrfy(f'''-fsn "{FIXgsm}" "{FIXam}"''') - # - # Resolve the target directory that the FIXam symlink points to and check - # that it exists. - # - try: - path_resolved = os.path.realpath(FIXam) - except: - path_resolved = FIXam - if not os.path.exists(path_resolved): - print_err_msg_exit( - f""" - In order to be able to generate a forecast experiment in NCO mode (i.e. - when RUN_ENVIR set to \"nco\"), the path specified by FIXam after resolving - all symlinks (path_resolved) must be an existing directory (but in this - case isn't): - RUN_ENVIR = \"{RUN_ENVIR}\" - FIXam = \"{FIXam}\" - path_resolved = \"{path_resolved}\" - Please ensure that path_resolved is an existing directory and then rerun - the experiment generation script.""" - ) # # Copy relevant fix files. # @@ -547,8 +536,7 @@ def generate_FV3LAM_wflow(): print_info_msg( f''' - Copying fixed files from system directory (FIXgsm) to a subdirectory - (FIXam) in the experiment directory: + Copying fixed files from system directory (FIXgsm) to a subdirectory (FIXam): FIXgsm = \"{FIXgsm}\" FIXam = \"{FIXam}\"''', verbose=VERBOSE, @@ -1165,3 +1153,13 @@ def workflow_func(): log_fp = \"{log_fp}\" Stopping.""" ) + +class Testing(unittest.TestCase): + def test_generate_FV3LAM_wflow(self): + USHdir = os.path.dirname(os.path.abspath(__file__)) + ln_vrfy("-fs", f"{USHdir}/config.community.yaml", f"{USHdir}/config.yaml") + generate_FV3LAM_wflow() + + def setUp(self): + set_env_var("DEBUG", False) + set_env_var("VERBOSE", False) From 2a6b1074804befe62fc61cf17020c1bd38de54c0 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Sat, 17 Sep 2022 00:29:03 +0000 Subject: [PATCH 086/106] Remove unused process_args utility. --- ush/python_utils/__init__.py | 1 - ush/python_utils/process_args.py | 74 --------------------------- ush/python_utils/test_python_utils.py | 15 ------ ush/set_predef_grid_params.py | 1 - 4 files changed, 91 deletions(-) delete mode 100644 ush/python_utils/process_args.py diff --git a/ush/python_utils/__init__.py b/ush/python_utils/__init__.py index 7b297432e6..8a74a595d0 100644 --- a/ush/python_utils/__init__.py +++ b/ush/python_utils/__init__.py @@ -29,7 +29,6 @@ from .interpol_to_arbit_CRES import interpol_to_arbit_CRES from .print_input_args import print_input_args from .print_msg import print_info_msg, print_err_msg_exit -from .process_args import process_args from .run_command import run_command from .get_charvar_from_netcdf import get_charvar_from_netcdf from .xml_parser import load_xml_file, has_tag_with_value diff --git a/ush/python_utils/process_args.py b/ush/python_utils/process_args.py deleted file mode 100644 index 3eea16f034..0000000000 --- a/ush/python_utils/process_args.py +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env python3 - -from textwrap import dedent - -from .print_msg import print_err_msg_exit -from .check_var_valid_value import check_var_valid_value - - -def process_args(valid_args, **kwargs): - """Function to process a list of variable name-value pairs. - It checks whether each argument is a valid argument or not. - - Args: - valid_args: List of valid arguments - **kwargs: keyword arguments - Returns: - A dictionary of all valid (arg,value) pairs - """ - - if valid_args[0] == "__unset__": - valid_arg_names = [] - else: - valid_arg_names = valid_args - num_valid_args = len(valid_arg_names) - num_arg_val_pairs = len(kwargs) - - if num_arg_val_pairs > num_valid_args: - print_err_msg_exit( - f""" - The number of argument-value pairs specified on the command line (num_- - arg_val_pairs) must be less than or equal to the number of valid argu- - ments (num_valid_args) specified in the array valid_arg_names: - num_arg_val_pairs = {num_arg_val_pairs} - num_valid_args = {num_valid_args} - valid_arg_names = ( {valid_arg_names})""" - ) - - if num_valid_args == 0: - return None - - values_args = [None] * num_valid_args - - for i, a in enumerate(valid_args): - if a is None: - print_err_msg_exit( - f''' - The list of valid arguments (valid_arg_names) cannot contain empty elements, - but the element with index i={i} is empty: - valid_arg_names = ( {valid_arg_names}) - valid_arg_names[{i}] = \"{valid_arg_names[i]}\"''' - ) - - for arg_name, arg_value in kwargs.items(): - err_msg = dedent( - f""" - The specified argument name (arg_name) in the current argument-value - pair (arg_val_pair) is not valid: - arg_name = \"{arg_name}\" - arg_val = \"{arg_value}\"\n""" - ) - check_var_valid_value(arg_name, valid_arg_names, err_msg) - - idx = valid_arg_names.index(arg_name) - if values_args[idx] is not None: - print_err_msg_exit( - f""" - The current argument has already been assigned a value: - arg_name = \"{arg_name}\" - key_value_pair = {kwargs} - Please assign values to arguments only once on the command line.""" - ) - values_args[idx] = arg_value - - return dict(zip(valid_args, values_args)) diff --git a/ush/python_utils/test_python_utils.py b/ush/python_utils/test_python_utils.py index eb997c0bb3..7e25fdaf5b 100644 --- a/ush/python_utils/test_python_utils.py +++ b/ush/python_utils/test_python_utils.py @@ -95,21 +95,6 @@ def test_define_macos_utilities(self): self.assertEqual(val, "MYVAL") self.assertEqual(os.getenv("SED"), "gsed" if os.uname() == "Darwin" else "sed") - def test_process_args(self): - valid_args = ["arg1", "arg2", "arg3", "arg4"] - values = process_args( - valid_args, arg2="bye", arg3="hello", arg4=["this", "is", "an", "array"] - ) - self.assertEqual( - values, - { - "arg1": None, - "arg2": "bye", - "arg3": "hello", - "arg4": ["this", "is", "an", "array"], - }, - ) - def test_print_input_args(self): valid_args = {"arg1": 1, "arg2": 2, "arg3": 3, "arg4": 4} self.assertEqual(print_input_args(valid_args), 4) diff --git a/ush/set_predef_grid_params.py b/ush/set_predef_grid_params.py index ba88d5a5d7..2e48e5b4a8 100644 --- a/ush/set_predef_grid_params.py +++ b/ush/set_predef_grid_params.py @@ -4,7 +4,6 @@ import os from python_utils import ( - process_args, import_vars, export_vars, set_env_var, From f6c63ac24707618869d64599dfe4ccee7778367b Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Mon, 19 Sep 2022 17:41:58 +0000 Subject: [PATCH 087/106] Remove hard coded paths from configs. --- ush/config.community.yaml | 10 +++++----- ush/config.nco.yaml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ush/config.community.yaml b/ush/config.community.yaml index 53c97a218b..65f3a998cc 100644 --- a/ush/config.community.yaml +++ b/ush/config.community.yaml @@ -7,11 +7,11 @@ user: ACCOUNT: an_account platform: MODEL: FV3_GFS_v16_CONUS_25km - MET_INSTALL_DIR: path/to/MET - METPLUS_PATH: path/to/METPlus - CCPA_OBS_DIR: /path/to/processed/CCPA/data - MRMS_OBS_DIR: /path/to/processed/MRMS/data - NDAS_OBS_DIR: /path/to/processed/NDAS/data + MET_INSTALL_DIR: "" + METPLUS_PATH: "" + CCPA_OBS_DIR: "" + MRMS_OBS_DIR: "" + NDAS_OBS_DIR: "" workflow: EXPT_SUBDIR: test_community CCPP_PHYS_SUITE: FV3_GFS_v16 diff --git a/ush/config.nco.yaml b/ush/config.nco.yaml index 1380087859..7763e0bc1d 100644 --- a/ush/config.nco.yaml +++ b/ush/config.nco.yaml @@ -24,7 +24,7 @@ nco: NET: rrfs model_ver: v1.0 RUN: rrfs_test - OPSROOT: /path/to/opsroot/directory + OPSROOT: "" task_get_extrn_ics: EXTRN_MDL_NAME_ICS: FV3GFS FV3GFS_FILE_FMT_ICS: grib2 From 1de5a679e556712f40b1a58cbad738d5a8bc81d8 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Mon, 19 Sep 2022 23:27:44 +0000 Subject: [PATCH 088/106] Don't replace existing var value with None. --- ush/python_utils/environment.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ush/python_utils/environment.py b/ush/python_utils/environment.py index 5263b92346..1655150649 100644 --- a/ush/python_utils/environment.py +++ b/ush/python_utils/environment.py @@ -223,7 +223,9 @@ def import_vars(dictionary=None, target_dict=None, env_vars=None): env_vars = {k: dictionary[k] if k in dictionary else None for k in env_vars} for k, v in env_vars.items(): - target_dict[k] = str_to_list(v) + # Don't replace variable with empty value + if not ((k in target_dict) and (v == "")): + target_dict[k] = str_to_list(v) def export_vars(dictionary=None, source_dict=None, env_vars=None): From 8c3199f5f032bdff764ed7db15767e0ed89759b3 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Mon, 19 Sep 2022 23:28:19 +0000 Subject: [PATCH 089/106] Add config.nco to unittest. --- ush/generate_FV3LAM_wflow.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/ush/generate_FV3LAM_wflow.py b/ush/generate_FV3LAM_wflow.py index 63144fe5f6..76493c53f1 100755 --- a/ush/generate_FV3LAM_wflow.py +++ b/ush/generate_FV3LAM_wflow.py @@ -1156,9 +1156,26 @@ def workflow_func(): class Testing(unittest.TestCase): def test_generate_FV3LAM_wflow(self): + + # run workflows in separate process to avoid conflict + def workflow_func(): + generate_FV3LAM_wflow() + + def run_workflow(): + p = Process(target=workflow_func) + p.start() + p.join() + USHdir = os.path.dirname(os.path.abspath(__file__)) + + # community test case ln_vrfy("-fs", f"{USHdir}/config.community.yaml", f"{USHdir}/config.yaml") - generate_FV3LAM_wflow() + run_workflow() + + # nco test case + set_env_var("OPSROOT", f"{USHdir}/../../nco_dirs") + ln_vrfy("-fs", f"{USHdir}/config.nco.yaml", f"{USHdir}/config.yaml") + run_workflow() def setUp(self): set_env_var("DEBUG", False) From f5b03fa8e8a2c542fe2499c88e622189b3c600ab Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Tue, 20 Sep 2022 16:09:42 -0500 Subject: [PATCH 090/106] Fix for Orion issue. --- tests/WE2E/get_WE2Etest_names_subdirs_descs.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh b/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh index 6670edb8a4..cab45a092f 100755 --- a/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh +++ b/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh @@ -996,7 +996,9 @@ configuration files of the primary WE2E tests... for (( k=0; k<=$((num_vars_to_extract-1)); k++ )); do var_name="${vars_to_extract[$k]}" + set +e cmd=$( grep "^[ ]*${var_name}=" <<< "${config_content}" ) + set -e eval $cmd if [ -z "${!var_name+x}" ]; then From e87291e45dc58c735e58eac817294d1896dd95f2 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Tue, 20 Sep 2022 22:07:47 +0000 Subject: [PATCH 091/106] Fix default OPSROOT location in run_we2e. --- tests/WE2E/run_WE2E_tests.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/WE2E/run_WE2E_tests.sh b/tests/WE2E/run_WE2E_tests.sh index 4f875ca2ca..f7b049f0ef 100755 --- a/tests/WE2E/run_WE2E_tests.sh +++ b/tests/WE2E/run_WE2E_tests.sh @@ -932,8 +932,7 @@ model_ver="we2e"" # # Set OPSROOT. # - nco_basedir=$( readlink -f "$HOMEdir/../../nco_dirs" ) - OPSROOT=${opsroot:-"${nco_basedir}/OPSROOT"} + OPSROOT=${opsroot:-$( readlink -f "$HOMEdir/../nco_dirs" )} expt_config_str=${expt_config_str}" # From 41fd5baa2f472f9d8470f462402ccbecd1f43986 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Tue, 20 Sep 2022 22:10:21 +0000 Subject: [PATCH 092/106] Modeify setup_we2e script to run fundamental tests on all machines. --- tests/WE2E/machine_suites/{hera.txt => fundamental.txt} | 0 tests/WE2E/machine_suites/jet.txt | 9 --------- tests/WE2E/setup_WE2E_tests.sh | 4 ++-- 3 files changed, 2 insertions(+), 11 deletions(-) rename tests/WE2E/machine_suites/{hera.txt => fundamental.txt} (100%) delete mode 100644 tests/WE2E/machine_suites/jet.txt diff --git a/tests/WE2E/machine_suites/hera.txt b/tests/WE2E/machine_suites/fundamental.txt similarity index 100% rename from tests/WE2E/machine_suites/hera.txt rename to tests/WE2E/machine_suites/fundamental.txt diff --git a/tests/WE2E/machine_suites/jet.txt b/tests/WE2E/machine_suites/jet.txt deleted file mode 100644 index 0887e6c58e..0000000000 --- a/tests/WE2E/machine_suites/jet.txt +++ /dev/null @@ -1,9 +0,0 @@ -grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2 -grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 -grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR -grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_v15p2 -grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_HRRR -grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta -grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR -grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta -nco_grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR diff --git a/tests/WE2E/setup_WE2E_tests.sh b/tests/WE2E/setup_WE2E_tests.sh index 407a27d0d7..e6f7576e4f 100755 --- a/tests/WE2E/setup_WE2E_tests.sh +++ b/tests/WE2E/setup_WE2E_tests.sh @@ -35,7 +35,7 @@ function usage { } -machines=( hera jet ) +machines=( hera jet cheyenne orion wcoss2 gaea odin singularity macos noaacloud ) if [ "$1" = "-h" ] ; then usage ; fi [[ $# -le 1 ]] && usage @@ -62,7 +62,7 @@ EXPTS_DIR=${TOP_DIR}/expt_dirs # Set the path to the machine-specific test suite file. #----------------------------------------------------------------------- -auto_file=${scrfunc_dir}/machine_suites/${machine}.txt +auto_file=${scrfunc_dir}/machine_suites/fundamental.txt #---------------------------------------------------------------------- # Use exec_subdir consistent with the automated build. From f3b1b3f75a051a07d06f59f6caccfd8f4e277705 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Wed, 21 Sep 2022 03:47:28 +0000 Subject: [PATCH 093/106] Fix conflicting ics/lbcs temp location by moving to DATA. --- jobs/JREGIONAL_GET_EXTRN_MDL_FILES | 5 +++-- ush/retrieve_data.py | 10 ++-------- ush/test_retrieve_data.py | 24 ++++++++++++++---------- 3 files changed, 19 insertions(+), 20 deletions(-) diff --git a/jobs/JREGIONAL_GET_EXTRN_MDL_FILES b/jobs/JREGIONAL_GET_EXTRN_MDL_FILES index b3e8c484d3..2eb7797256 100755 --- a/jobs/JREGIONAL_GET_EXTRN_MDL_FILES +++ b/jobs/JREGIONAL_GET_EXTRN_MDL_FILES @@ -210,11 +210,12 @@ esac # if [ $RUN_ENVIR = "nco" ]; then export EXTRN_MDL_STAGING_DIR="${COMIN}" + mkdir_vrfy -p "${EXTRN_MDL_STAGING_DIR}" else export EXTRN_MDL_STAGING_DIR="${COMIN}/${EXTRN_MDL_NAME}/for_${ICS_OR_LBCS}" + mkdir_vrfy -p "${EXTRN_MDL_STAGING_DIR}" + cd_vrfy "${EXTRN_MDL_STAGING_DIR}" fi -mkdir_vrfy -p "${EXTRN_MDL_STAGING_DIR}" -cd_vrfy "${EXTRN_MDL_STAGING_DIR}" # #----------------------------------------------------------------------- # diff --git a/ush/retrieve_data.py b/ush/retrieve_data.py index fbcadf2686..52f217bb85 100755 --- a/ush/retrieve_data.py +++ b/ush/retrieve_data.py @@ -47,7 +47,7 @@ def clean_up_output_dir(expected_subdir, local_archive, output_path, source_path unavailable = {} # Check to make sure the files exist on disk for file_path in source_paths: - local_file_path = os.path.join(output_path, file_path.lstrip("/")) + local_file_path = os.path.join( os.getcwd(), file_path.lstrip("/") ) if not os.path.exists(local_file_path): logging.info(f"File does not exist: {local_file_path}") unavailable["hpss"] = source_paths @@ -347,7 +347,7 @@ def get_requested_files(cla, file_templates, input_locs, method="disk", **kwargs input_locs = input_locs if isinstance(input_locs, list) else [input_locs] - orig_path = os.path.dirname(__file__) + orig_path = os.getcwd() unavailable = [] locs_files = pair_locs_with_files(input_locs, file_templates, check_all) @@ -509,9 +509,7 @@ def hpss_requested_files(cla, file_names, store_specs, members=-1, ens_group=-1) output_path = fill_template(cla.output_path, cla.cycle_date, mem=mem) logging.info(f"Will place files in {os.path.abspath(output_path)}") - orig_path = os.path.dirname(__file__) logging.debug(f"CWD: {os.getcwd()}") - os.chdir(orig_path) if mem != -1: archive_internal_dir = fill_template( @@ -522,8 +520,6 @@ def hpss_requested_files(cla, file_names, store_specs, members=-1, ens_group=-1) output_path = create_target_path(output_path) logging.info(f"Will place files in {os.path.abspath(output_path)}") - os.chdir(output_path) - source_paths = [] for fcst_hr in cla.fcst_hrs: for file_name in file_names: @@ -582,8 +578,6 @@ def hpss_requested_files(cla, file_names, store_specs, members=-1, ens_group=-1) if not expected == unavailable: return unavailable - expected - os.chdir(orig_path) - return {} diff --git a/ush/test_retrieve_data.py b/ush/test_retrieve_data.py index b2ef60503b..3ffdf4744c 100644 --- a/ush/test_retrieve_data.py +++ b/ush/test_retrieve_data.py @@ -38,6 +38,7 @@ def test_fv3gfs_grib2_lbcs_from_hpss(self): ''' with tempfile.TemporaryDirectory(dir='.') as tmp_dir: + os.chdir(tmp_dir) args = [ '--anl_or_fcst', 'fcst', @@ -55,7 +56,6 @@ def test_fv3gfs_grib2_lbcs_from_hpss(self): # Verify files exist in temp dir - os.chdir(os.path.dirname(__file__)) path = os.path.join(tmp_dir, '*') files_on_disk = glob.glob(path) self.assertEqual(len(files_on_disk), 3) @@ -68,6 +68,7 @@ def test_fv3gfs_netcdf_lbcs_from_hpss(self): ''' with tempfile.TemporaryDirectory(dir='.') as tmp_dir: + os.chdir(tmp_dir) args = [ '--anl_or_fcst', 'fcst', @@ -85,7 +86,6 @@ def test_fv3gfs_netcdf_lbcs_from_hpss(self): # Verify files exist in temp dir - os.chdir(os.path.dirname(__file__)) path = os.path.join(tmp_dir, '*') files_on_disk = glob.glob(path) self.assertEqual(len(files_on_disk), 2) @@ -97,7 +97,9 @@ def test_gdas_ics_from_aws(self): ''' with tempfile.TemporaryDirectory(dir='.') as tmp_dir: - out_path_tmpl = f'{tmp_dir}/mem{{mem:03d}}' + os.chdir(tmp_dir) + + out_path_tmpl = f'mem{{mem:03d}}' args = [ '--anl_or_fcst', 'anl', @@ -131,7 +133,9 @@ def test_gefs_grib2_ics_from_aws(self): ''' with tempfile.TemporaryDirectory(dir='.') as tmp_dir: - out_path_tmpl = f'{tmp_dir}/mem{{mem:03d}}' + os.chdir(tmp_dir) + + out_path_tmpl = f'mem{{mem:03d}}' args = [ '--anl_or_fcst', 'anl', @@ -164,6 +168,7 @@ def test_hrrr_ics_from_hpss(self): ''' Get HRRR ICS from hpss ''' with tempfile.TemporaryDirectory(dir='.') as tmp_dir: + os.chdir(tmp_dir) args = [ '--anl_or_fcst', 'anl', @@ -180,7 +185,6 @@ def test_hrrr_ics_from_hpss(self): # Verify files exist in temp dir - os.chdir(os.path.dirname(__file__)) path = os.path.join(tmp_dir, '*') files_on_disk = glob.glob(path) self.assertEqual(len(files_on_disk), 1) @@ -191,6 +195,7 @@ def test_hrrr_lbcs_from_hpss(self): ''' Get HRRR LBCS from hpss for 3 hour boundary conditions ''' with tempfile.TemporaryDirectory(dir='.') as tmp_dir: + os.chdir(tmp_dir) args = [ '--anl_or_fcst', 'fcst', @@ -207,7 +212,6 @@ def test_hrrr_lbcs_from_hpss(self): # Verify files exist in temp dir - os.chdir(os.path.dirname(__file__)) path = os.path.join(tmp_dir, '*') files_on_disk = glob.glob(path) self.assertEqual(len(files_on_disk), 8) @@ -217,6 +221,7 @@ def test_hrrr_ics_from_aws(self): ''' Get HRRR ICS from aws ''' with tempfile.TemporaryDirectory(dir='.') as tmp_dir: + os.chdir(tmp_dir) args = [ '--anl_or_fcst', 'anl', @@ -233,7 +238,6 @@ def test_hrrr_ics_from_aws(self): # Verify files exist in temp dir - os.chdir(os.path.dirname(__file__)) path = os.path.join(tmp_dir, '*') files_on_disk = glob.glob(path) self.assertEqual(len(files_on_disk), 1) @@ -243,6 +247,7 @@ def test_hrrr_lbcs_from_aws(self): ''' Get HRRR LBCS from aws for 3 hour boundary conditions ''' with tempfile.TemporaryDirectory(dir='.') as tmp_dir: + os.chdir(tmp_dir) args = [ '--anl_or_fcst', 'fcst', @@ -259,7 +264,6 @@ def test_hrrr_lbcs_from_aws(self): # Verify files exist in temp dir - os.chdir(os.path.dirname(__file__)) path = os.path.join(tmp_dir, '*') files_on_disk = glob.glob(path) self.assertEqual(len(files_on_disk), 8) @@ -270,6 +274,7 @@ def test_rap_ics_from_aws(self): ''' Get RAP ICS from aws offset by 3 hours ''' with tempfile.TemporaryDirectory(dir='.') as tmp_dir: + os.chdir(tmp_dir) args = [ '--anl_or_fcst', 'anl', @@ -286,7 +291,6 @@ def test_rap_ics_from_aws(self): # Verify files exist in temp dir - os.chdir(os.path.dirname(__file__)) path = os.path.join(tmp_dir, '*') files_on_disk = glob.glob(path) self.assertEqual(len(files_on_disk), 1) @@ -297,6 +301,7 @@ def test_rap_lbcs_from_aws(self): by 3 hours. Use 09Z start time for longer LBCS.''' with tempfile.TemporaryDirectory(dir='.') as tmp_dir: + os.chdir(tmp_dir) args = [ '--anl_or_fcst', 'fcst', @@ -313,7 +318,6 @@ def test_rap_lbcs_from_aws(self): # Verify files exist in temp dir - os.chdir(os.path.dirname(__file__)) path = os.path.join(tmp_dir, '*') files_on_disk = glob.glob(path) self.assertEqual(len(files_on_disk), 5) From 1ecee87cb9d54f40a2b4ac4d10f44e52bf864012 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Wed, 21 Sep 2022 19:53:13 +0000 Subject: [PATCH 094/106] Bug fix in load_modules taken from PR #353. --- devbuild.sh | 2 +- ush/load_modules_run_task.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/devbuild.sh b/devbuild.sh index 79c9199322..e6e1999767 100755 --- a/devbuild.sh +++ b/devbuild.sh @@ -38,7 +38,7 @@ OPTIONS --install-dir=INSTALL_DIR installation prefix --bin-dir=BIN_DIR - installation binary directory name ("bin" by default; any name is available) + installation binary directory name ("exec" by default; any name is available) --build-type=BUILD_TYPE build type; defaults to RELEASE (e.g. DEBUG | RELEASE | RELWITHDEBINFO) diff --git a/ush/load_modules_run_task.sh b/ush/load_modules_run_task.sh index 8db0274275..c17610d124 100755 --- a/ush/load_modules_run_task.sh +++ b/ush/load_modules_run_task.sh @@ -109,7 +109,7 @@ fi # machine=$(echo_lowercase $MACHINE) -source "${SR_WX_APP_TOP_DIR}/etc/lmod-setup.sh" +source "${SR_WX_APP_TOP_DIR}/etc/lmod-setup.sh" ${machine} module use "${SR_WX_APP_TOP_DIR}/modulefiles" module load "${BUILD_MOD_FN}" || print_err_msg_exit "\ Loading of platform- and compiler-specific module file (BUILD_MOD_FN) From e54b868e6b9df647986673d0957cad90c16f84a3 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Thu, 22 Sep 2022 20:54:43 +0000 Subject: [PATCH 095/106] Specify default shell instead of symlinking. --- .github/workflows/python_unittests.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python_unittests.yaml b/.github/workflows/python_unittests.yaml index 0b780e2340..7c76288a8d 100644 --- a/.github/workflows/python_unittests.yaml +++ b/.github/workflows/python_unittests.yaml @@ -1,5 +1,8 @@ name: Python unittests on: [push, pull_request, workflow_dispatch] +defaults: + run: + shell: bash jobs: python_unittests: @@ -17,7 +20,6 @@ jobs: sudo apt-get install python3 python3-pip netcdf-bin sudo pip3 install pyyaml jinja2==2.11 f90nml sudo pip3 install numpy matplotlib basemap - sudo ln -sf /bin/bash /bin/sh # Run python unittests - name: Run python unittests @@ -25,6 +27,7 @@ jobs: ./manage_externals/checkout_externals ufs-weather-model cd ush python3 -m unittest -b python_utils/test_python_utils.py + # exclude test_retrieve_data that is tested in functional test files=$(find . -maxdepth 1 -name '*.py' -a ! -name 'test_retrieve_data.py' -exec basename {} \;) files=$(echo $(echo "${files[@]}")) python3 -m unittest -b ${files[@]} From 404ad02d89e0953d2fc107d28fde6472b6fa8dbe Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Thu, 22 Sep 2022 22:10:36 +0000 Subject: [PATCH 096/106] Turn off grid/orog/sfc_climo tasks for NCO test cases. --- ..._RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.yaml | 4 ++++ ...FV3GFS_lbcs_FV3GFS_suite_GFS_v15_thompson_mynn_lam3km.yaml | 4 ++++ ...d_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR.yaml | 4 ++++ .../WE2E/test_configs/wflow_features/config.nco_ensemble.yaml | 4 ++++ ush/config.nco.yaml | 4 ++++ 5 files changed, 20 insertions(+) diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.yaml b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.yaml index 8d3841bfe1..33d5345776 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.yaml +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.yaml @@ -13,6 +13,10 @@ workflow: - 0 FCST_LEN_HRS: 6 PREEXISTING_DIR_METHOD: rename +workflow_switches: + RUN_TASK_MAKE_GRID: false + RUN_TASK_MAKE_OROG: false + RUN_TASK_MAKE_SFC_CLIMO: false task_get_extrn_ics: EXTRN_MDL_NAME_ICS: FV3GFS FV3GFS_FILE_FMT_ICS: grib2 diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15_thompson_mynn_lam3km.yaml b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15_thompson_mynn_lam3km.yaml index 90dbe92d3f..b99e27b26c 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15_thompson_mynn_lam3km.yaml +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15_thompson_mynn_lam3km.yaml @@ -13,6 +13,10 @@ workflow: - 0 FCST_LEN_HRS: 6 PREEXISTING_DIR_METHOD: rename +workflow_switches: + RUN_TASK_MAKE_GRID: false + RUN_TASK_MAKE_OROG: false + RUN_TASK_MAKE_SFC_CLIMO: false task_get_extrn_ics: EXTRN_MDL_NAME_ICS: FV3GFS FV3GFS_FILE_FMT_ICS: grib2 diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR.yaml b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR.yaml index 445209e424..62aa0df848 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR.yaml +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR.yaml @@ -13,6 +13,10 @@ workflow: - 0 FCST_LEN_HRS: 6 PREEXISTING_DIR_METHOD: rename +workflow_switches: + RUN_TASK_MAKE_GRID: false + RUN_TASK_MAKE_OROG: false + RUN_TASK_MAKE_SFC_CLIMO: false task_get_extrn_ics: EXTRN_MDL_NAME_ICS: HRRR USE_USER_STAGED_EXTRN_FILES: true diff --git a/tests/WE2E/test_configs/wflow_features/config.nco_ensemble.yaml b/tests/WE2E/test_configs/wflow_features/config.nco_ensemble.yaml index 87b2654faa..de4c7c414a 100644 --- a/tests/WE2E/test_configs/wflow_features/config.nco_ensemble.yaml +++ b/tests/WE2E/test_configs/wflow_features/config.nco_ensemble.yaml @@ -20,6 +20,10 @@ workflow: - 12 FCST_LEN_HRS: 6 PREEXISTING_DIR_METHOD: rename +workflow_switches: + RUN_TASK_MAKE_GRID: false + RUN_TASK_MAKE_OROG: false + RUN_TASK_MAKE_SFC_CLIMO: false task_get_extrn_ics: EXTRN_MDL_NAME_ICS: FV3GFS USE_USER_STAGED_EXTRN_FILES: true diff --git a/ush/config.nco.yaml b/ush/config.nco.yaml index 7763e0bc1d..b91195eabf 100644 --- a/ush/config.nco.yaml +++ b/ush/config.nco.yaml @@ -20,6 +20,10 @@ workflow: PREEXISTING_DIR_METHOD: rename VERBOSE: true COMPILER: intel +workflow_switches: + RUN_TASK_MAKE_GRID: false + RUN_TASK_MAKE_OROG: false + RUN_TASK_MAKE_SFC_CLIMO: false nco: NET: rrfs model_ver: v1.0 From d4d39d600148628aabaffc0d2e4c3e670eabf8ac Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Fri, 23 Sep 2022 02:16:33 +0000 Subject: [PATCH 097/106] Use PDY and cyc in ex-scripts. --- jobs/JREGIONAL_GET_EXTRN_MDL_FILES | 4 ++-- scripts/exregional_get_obs_ccpa.sh | 5 ++--- scripts/exregional_get_obs_mrms.sh | 5 ++--- scripts/exregional_get_obs_ndas.sh | 5 ++--- scripts/exregional_run_fcst.sh | 4 ++-- scripts/exregional_run_post.sh | 5 ++--- scripts/exregional_run_vx_ensgrid.sh | 5 ++--- scripts/exregional_run_vx_ensgrid_mean.sh | 5 ++--- scripts/exregional_run_vx_ensgrid_prob.sh | 5 ++--- scripts/exregional_run_vx_enspoint.sh | 5 ++--- scripts/exregional_run_vx_enspoint_mean.sh | 5 ++--- scripts/exregional_run_vx_enspoint_prob.sh | 5 ++--- scripts/exregional_run_vx_gridstat.sh | 5 ++--- scripts/exregional_run_vx_pointstat.sh | 5 ++--- 14 files changed, 28 insertions(+), 40 deletions(-) diff --git a/jobs/JREGIONAL_GET_EXTRN_MDL_FILES b/jobs/JREGIONAL_GET_EXTRN_MDL_FILES index 2eb7797256..a6b1902a14 100755 --- a/jobs/JREGIONAL_GET_EXTRN_MDL_FILES +++ b/jobs/JREGIONAL_GET_EXTRN_MDL_FILES @@ -107,8 +107,8 @@ fi #----------------------------------------------------------------------- # -hh=${CDATE:8:2} -yyyymmdd=${CDATE:0:8} +yyyymmdd=${PDY} +hh=${cyc} export EXTRN_MDL_CDATE=$( $DATE_UTIL --utc --date "${yyyymmdd} ${hh} UTC - ${TIME_OFFSET_HRS} hours" "+%Y%m%d%H" ) # diff --git a/scripts/exregional_get_obs_ccpa.sh b/scripts/exregional_get_obs_ccpa.sh index 8879abb962..8607861c63 100755 --- a/scripts/exregional_get_obs_ccpa.sh +++ b/scripts/exregional_get_obs_ccpa.sh @@ -52,9 +52,8 @@ fi accum=01 # Initialization -yyyymmdd=${CDATE:0:8} -hh=${CDATE:8:2} -cyc=$hh +yyyymmdd=${PDY} +hh=${cyc} init=${CDATE}${hh} diff --git a/scripts/exregional_get_obs_mrms.sh b/scripts/exregional_get_obs_mrms.sh index 8c06902c1b..f6da0cac86 100755 --- a/scripts/exregional_get_obs_mrms.sh +++ b/scripts/exregional_get_obs_mrms.sh @@ -45,9 +45,8 @@ if [[ ! -d "$mrms_proc" ]]; then fi # Initialization -yyyymmdd=${CDATE:0:8} -hh=${CDATE:8:2} -cyc=$hh +yyyymmdd=${PDY} +hh=${cyc} start_valid=${CDATE}${hh} diff --git a/scripts/exregional_get_obs_ndas.sh b/scripts/exregional_get_obs_ndas.sh index a49fbaf147..b7587b18e6 100755 --- a/scripts/exregional_get_obs_ndas.sh +++ b/scripts/exregional_get_obs_ndas.sh @@ -46,9 +46,8 @@ if [[ ! -d "$ndas_proc" ]]; then fi # Initialization -yyyymmdd=${CDATE:0:8} -hh=${CDATE:8:2} -cyc=$hh +yyyymmdd=${PDY} +hh=${cyc} init=${CDATE}${hh} diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 22066433a3..2a72f34043 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -520,8 +520,8 @@ POST_STEP # if [ ${WRITE_DOPOST} = "TRUE" ]; then - yyyymmdd=${CDATE:0:8} - hh=${CDATE:8:2} + yyyymmdd=${PDY} + hh=${cyc} cyc=$hh fmn="00" diff --git a/scripts/exregional_run_post.sh b/scripts/exregional_run_post.sh index 9e71390365..da1a117545 100755 --- a/scripts/exregional_run_post.sh +++ b/scripts/exregional_run_post.sh @@ -135,9 +135,8 @@ fi # #----------------------------------------------------------------------- # -yyyymmdd=${CDATE:0:8} -hh=${CDATE:8:2} -cyc=$hh +yyyymmdd=${PDY} +hh=${cyc} # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_run_vx_ensgrid.sh b/scripts/exregional_run_vx_ensgrid.sh index 82d1c29713..c6a7b4c6a2 100755 --- a/scripts/exregional_run_vx_ensgrid.sh +++ b/scripts/exregional_run_vx_ensgrid.sh @@ -65,9 +65,8 @@ print_info_msg "$VERBOSE" "Starting ensemble-stat verification" # #----------------------------------------------------------------------- # -yyyymmdd=${CDATE:0:8} -hh=${CDATE:8:2} -cyc=$hh +yyyymmdd=${PDY} +hh=${cyc} export CDATE export hh diff --git a/scripts/exregional_run_vx_ensgrid_mean.sh b/scripts/exregional_run_vx_ensgrid_mean.sh index 1cc0510535..9679ae60ea 100755 --- a/scripts/exregional_run_vx_ensgrid_mean.sh +++ b/scripts/exregional_run_vx_ensgrid_mean.sh @@ -64,9 +64,8 @@ print_info_msg "$VERBOSE" "Starting grid-stat verification" # #----------------------------------------------------------------------- # -yyyymmdd=${CDATE:0:8} -hh=${CDATE:8:2} -cyc=$hh +yyyymmdd=${PDY} +hh=${cyc} export CDATE export hh diff --git a/scripts/exregional_run_vx_ensgrid_prob.sh b/scripts/exregional_run_vx_ensgrid_prob.sh index 11e932cdc9..f21a31bb70 100755 --- a/scripts/exregional_run_vx_ensgrid_prob.sh +++ b/scripts/exregional_run_vx_ensgrid_prob.sh @@ -64,9 +64,8 @@ print_info_msg "$VERBOSE" "Starting grid-stat verification" # #----------------------------------------------------------------------- # -yyyymmdd=${CDATE:0:8} -hh=${CDATE:8:2} -cyc=$hh +yyyymmdd=${PDY} +hh=${cyc} export CDATE export hh diff --git a/scripts/exregional_run_vx_enspoint.sh b/scripts/exregional_run_vx_enspoint.sh index d26a942826..b2e459ad4a 100755 --- a/scripts/exregional_run_vx_enspoint.sh +++ b/scripts/exregional_run_vx_enspoint.sh @@ -63,9 +63,8 @@ print_info_msg "$VERBOSE" "Starting point-based ensemble-stat verification" # #----------------------------------------------------------------------- # -yyyymmdd=${CDATE:0:8} -hh=${CDATE:8:2} -cyc=$hh +yyyymmdd=${PDY} +hh=${cyc} export CDATE export hh diff --git a/scripts/exregional_run_vx_enspoint_mean.sh b/scripts/exregional_run_vx_enspoint_mean.sh index ce196a6189..556342dee3 100755 --- a/scripts/exregional_run_vx_enspoint_mean.sh +++ b/scripts/exregional_run_vx_enspoint_mean.sh @@ -64,9 +64,8 @@ print_info_msg "$VERBOSE" "Starting point-stat verification" # #----------------------------------------------------------------------- # -yyyymmdd=${CDATE:0:8} -hh=${CDATE:8:2} -cyc=$hh +yyyymmdd=${PDY} +hh=${cyc} export CDATE export hh diff --git a/scripts/exregional_run_vx_enspoint_prob.sh b/scripts/exregional_run_vx_enspoint_prob.sh index 2d7645d788..fc44c24e5b 100755 --- a/scripts/exregional_run_vx_enspoint_prob.sh +++ b/scripts/exregional_run_vx_enspoint_prob.sh @@ -64,9 +64,8 @@ print_info_msg "$VERBOSE" "Starting point-stat verification" # #----------------------------------------------------------------------- # -yyyymmdd=${CDATE:0:8} -hh=${CDATE:8:2} -cyc=$hh +yyyymmdd=${PDY} +hh=${cyc} export CDATE export hh diff --git a/scripts/exregional_run_vx_gridstat.sh b/scripts/exregional_run_vx_gridstat.sh index c10144b5de..2d5f34184e 100755 --- a/scripts/exregional_run_vx_gridstat.sh +++ b/scripts/exregional_run_vx_gridstat.sh @@ -55,9 +55,8 @@ the UPP output files by initialization time for all forecast hours. # #----------------------------------------------------------------------- # -yyyymmdd=${CDATE:0:8} -hh=${CDATE:8:2} -cyc=$hh +yyyymmdd=${PDY} +hh=${cyc} export CDATE export hh diff --git a/scripts/exregional_run_vx_pointstat.sh b/scripts/exregional_run_vx_pointstat.sh index 2d6eba74f2..ce2518563b 100755 --- a/scripts/exregional_run_vx_pointstat.sh +++ b/scripts/exregional_run_vx_pointstat.sh @@ -55,9 +55,8 @@ the UPP output files by initialization time for all forecast hours. # #----------------------------------------------------------------------- # -yyyymmdd=${CDATE:0:8} -hh=${CDATE:8:2} -cyc=$hh +yyyymmdd=${PDY} +hh=${cyc} export CDATE export hh From 277381dea4ddf4c12973ea2eaf273fcd9b6dfd8a Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Fri, 23 Sep 2022 02:19:28 +0000 Subject: [PATCH 098/106] Remove CDATE from xml and define int job_preamble. --- parm/FV3LAM_wflow.xml | 38 -------------------------------------- ush/job_preamble.sh | 1 + 2 files changed, 1 insertion(+), 38 deletions(-) diff --git a/parm/FV3LAM_wflow.xml b/parm/FV3LAM_wflow.xml index d123f71191..14ca173a03 100644 --- a/parm/FV3LAM_wflow.xml +++ b/parm/FV3LAM_wflow.xml @@ -315,7 +315,6 @@ MODULES_RUN_TASK_FP script. cyc@H subcyc@M LOGDIR&LOGDIR; - CDATE@Y@m@d@H ICS_OR_LBCSICS @@ -343,7 +342,6 @@ MODULES_RUN_TASK_FP script. cyc@H subcyc@M LOGDIR&LOGDIR; - CDATE@Y@m@d@H ICS_OR_LBCSLBCS @@ -381,7 +379,6 @@ MODULES_RUN_TASK_FP script. cyc@H subcyc@M LOGDIR&LOGDIR; - CDATE@Y@m@d@H SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} ENSMEM_INDX#{{ ensmem_indx_name }}# @@ -431,7 +428,6 @@ MODULES_RUN_TASK_FP script. cyc@H subcyc@M LOGDIR&LOGDIR; - CDATE@Y@m@d@H SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} ENSMEM_INDX#{{ ensmem_indx_name }}# @@ -486,7 +482,6 @@ MODULES_RUN_TASK_FP script. cyc@H subcyc@M LOGDIR&LOGDIR; - CDATE@Y@m@d@H SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} ENSMEM_INDX#{{ ensmem_indx_name }}# @@ -543,7 +538,6 @@ later below for other output times. LOGDIR&LOGDIR; fhr#fhr# fmn#fmn# - CDATE@Y@m@d@H SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} ENSMEM_INDX#{{ ensmem_indx_name }}# @@ -602,7 +596,6 @@ for other output times. LOGDIR&LOGDIR; fhr#fhr# fmn#fmn# - CDATE@Y@m@d@H SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} ENSMEM_INDX#{{ ensmem_indx_name }}# @@ -675,7 +668,6 @@ always zero). {%- else %} fhr#fhr# {%- endif %} - CDATE@Y@m@d@H SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} ENSMEM_INDX#{{ ensmem_indx_name }}# @@ -739,7 +731,6 @@ the tag to be identical to the ones above for other output times. LOGDIR&LOGDIR; fhr#fhr# fmn#fmn# - CDATE@Y@m@d@H SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} ENSMEM_INDX#{{ ensmem_indx_name }}# @@ -779,7 +770,6 @@ the tag to be identical to the ones above for other output times. GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&CCPA_OBS_DIR; - CDATE@Y@m@d@H PDY@Y@m@d cyc@H subcyc@M @@ -810,7 +800,6 @@ the tag to be identical to the ones above for other output times. GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&MRMS_OBS_DIR; - CDATE@Y@m@d@H PDY@Y@m@d cyc@H subcyc@M @@ -842,7 +831,6 @@ the tag to be identical to the ones above for other output times. GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&NDAS_OBS_DIR; - CDATE@Y@m@d@H PDY@Y@m@d cyc@H subcyc@M @@ -872,7 +860,6 @@ the tag to be identical to the ones above for other output times. GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&CCPA_OBS_DIR; - CDATE@Y@m@d@H PDY@Y@m@d cyc@H subcyc@M @@ -927,7 +914,6 @@ the tag to be identical to the ones above for other output times. GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&MRMS_OBS_DIR; - CDATE@Y@m@d@H PDY@Y@m@d cyc@H subcyc@M @@ -981,7 +967,6 @@ the tag to be identical to the ones above for other output times. GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&MRMS_OBS_DIR; - CDATE@Y@m@d@H PDY@Y@m@d cyc@H subcyc@M @@ -1035,7 +1020,6 @@ the tag to be identical to the ones above for other output times. GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&CCPA_OBS_DIR; - CDATE@Y@m@d@H PDY@Y@m@d cyc@H subcyc@M @@ -1075,7 +1059,6 @@ the tag to be identical to the ones above for other output times. GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&CCPA_OBS_DIR; - CDATE@Y@m@d@H PDY@Y@m@d cyc@H subcyc@M @@ -1115,7 +1098,6 @@ the tag to be identical to the ones above for other output times. GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&CCPA_OBS_DIR; - CDATE@Y@m@d@H PDY@Y@m@d cyc@H subcyc@M @@ -1155,7 +1137,6 @@ the tag to be identical to the ones above for other output times. GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&NDAS_OBS_DIR; - CDATE@Y@m@d@H PDY@Y@m@d cyc@H subcyc@M @@ -1212,7 +1193,6 @@ the tag to be identical to the ones above for other output times. GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&CCPA_OBS_DIR; - CDATE@Y@m@d@H PDY@Y@m@d cyc@H subcyc@M @@ -1248,7 +1228,6 @@ the tag to be identical to the ones above for other output times. GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&CCPA_OBS_DIR; - CDATE@Y@m@d@H PDY@Y@m@d cyc@H subcyc@M @@ -1284,7 +1263,6 @@ the tag to be identical to the ones above for other output times. GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&CCPA_OBS_DIR; - CDATE@Y@m@d@H PDY@Y@m@d cyc@H subcyc@M @@ -1320,7 +1298,6 @@ the tag to be identical to the ones above for other output times. GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&CCPA_OBS_DIR; - CDATE@Y@m@d@H PDY@Y@m@d cyc@H subcyc@M @@ -1355,7 +1332,6 @@ the tag to be identical to the ones above for other output times. GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&MRMS_OBS_DIR; - CDATE@Y@m@d@H PDY@Y@m@d cyc@H subcyc@M @@ -1389,7 +1365,6 @@ the tag to be identical to the ones above for other output times. GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&MRMS_OBS_DIR; - CDATE@Y@m@d@H PDY@Y@m@d cyc@H subcyc@M @@ -1422,7 +1397,6 @@ the tag to be identical to the ones above for other output times. GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&CCPA_OBS_DIR; - CDATE@Y@m@d@H PDY@Y@m@d cyc@H subcyc@M @@ -1457,7 +1431,6 @@ the tag to be identical to the ones above for other output times. GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&CCPA_OBS_DIR; - CDATE@Y@m@d@H PDY@Y@m@d cyc@H subcyc@M @@ -1492,7 +1465,6 @@ the tag to be identical to the ones above for other output times. GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&CCPA_OBS_DIR; - CDATE@Y@m@d@H PDY@Y@m@d cyc@H subcyc@M @@ -1527,7 +1499,6 @@ the tag to be identical to the ones above for other output times. GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&CCPA_OBS_DIR; - CDATE@Y@m@d@H PDY@Y@m@d cyc@H subcyc@M @@ -1563,7 +1534,6 @@ the tag to be identical to the ones above for other output times. GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&CCPA_OBS_DIR; - CDATE@Y@m@d@H PDY@Y@m@d cyc@H subcyc@M @@ -1598,7 +1568,6 @@ the tag to be identical to the ones above for other output times. GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&CCPA_OBS_DIR; - CDATE@Y@m@d@H PDY@Y@m@d cyc@H subcyc@M @@ -1634,7 +1603,6 @@ the tag to be identical to the ones above for other output times. GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&CCPA_OBS_DIR; - CDATE@Y@m@d@H PDY@Y@m@d cyc@H subcyc@M @@ -1669,7 +1637,6 @@ the tag to be identical to the ones above for other output times. GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&CCPA_OBS_DIR; - CDATE@Y@m@d@H PDY@Y@m@d cyc@H subcyc@M @@ -1704,7 +1671,6 @@ the tag to be identical to the ones above for other output times. GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&MRMS_OBS_DIR; - CDATE@Y@m@d@H PDY@Y@m@d cyc@H subcyc@M @@ -1738,7 +1704,6 @@ the tag to be identical to the ones above for other output times. GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&MRMS_OBS_DIR; - CDATE@Y@m@d@H PDY@Y@m@d cyc@H subcyc@M @@ -1773,7 +1738,6 @@ the tag to be identical to the ones above for other output times. GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&NDAS_OBS_DIR; - CDATE@Y@m@d@H PDY@Y@m@d cyc@H subcyc@M @@ -1805,7 +1769,6 @@ the tag to be identical to the ones above for other output times. GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&NDAS_OBS_DIR; - CDATE@Y@m@d@H PDY@Y@m@d cyc@H subcyc@M @@ -1837,7 +1800,6 @@ the tag to be identical to the ones above for other output times. GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; OBS_DIR&NDAS_OBS_DIR; - CDATE@Y@m@d@H PDY@Y@m@d cyc@H subcyc@M diff --git a/ush/job_preamble.sh b/ush/job_preamble.sh index 97d5893c55..4e90c58dfd 100644 --- a/ush/job_preamble.sh +++ b/ush/job_preamble.sh @@ -45,6 +45,7 @@ if [ "${RUN_ENVIR}" = "nco" ]; then . ./PDY fi fi +export CDATE=${PDY}${cyc} # #----------------------------------------------------------------------- # From 0580a1879f8c943a9c66acdb6a489984da092298 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Fri, 23 Sep 2022 04:38:47 +0000 Subject: [PATCH 099/106] Use machine specific list of tests if available. --- test/build.sh | 6 +++--- tests/WE2E/setup_WE2E_tests.sh | 7 +++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/test/build.sh b/test/build.sh index e269d2bdad..cdf8d4e29b 100755 --- a/test/build.sh +++ b/test/build.sh @@ -135,10 +135,10 @@ declare -a executables_rrfs_utl_created=( adjust_soiltq.exe \ #----------------------------------------------------------------------- for compiler in "${compilers[@]}"; do BUILD_DIR=${TOP_DIR}/build_${compiler} - BIN_DIR=${TOP_DIR}/bin_${compiler} - EXEC_DIR=${BIN_DIR}/exec + INSTALL_DIR=${TOP_DIR}/install_${compiler} + EXEC_DIR=${INSTALL_DIR}/exec if [ $build_it -eq 0 ] ; then - ./devbuild.sh --platform=${machine} --compiler=${compiler} --build-dir=${BUILD_DIR} --install-dir=${BIN_DIR} \ + ./devbuild.sh --platform=${machine} --compiler=${compiler} --build-dir=${BUILD_DIR} --install-dir=${INSTALL_DIR} \ --remove all || fail "Build ${machine} ${compiler} FAILED" fi # End of skip build for testing diff --git a/tests/WE2E/setup_WE2E_tests.sh b/tests/WE2E/setup_WE2E_tests.sh index e6f7576e4f..b2412fef0a 100755 --- a/tests/WE2E/setup_WE2E_tests.sh +++ b/tests/WE2E/setup_WE2E_tests.sh @@ -62,13 +62,16 @@ EXPTS_DIR=${TOP_DIR}/expt_dirs # Set the path to the machine-specific test suite file. #----------------------------------------------------------------------- -auto_file=${scrfunc_dir}/machine_suites/fundamental.txt +auto_file=${scrfunc_dir}/machine_suites/${machine}.txt +if [ ! -f ${auto_file} ]; then + auto_file=${scrfunc_dir}/machine_suites/fundamental.txt +fi #---------------------------------------------------------------------- # Use exec_subdir consistent with the automated build. #---------------------------------------------------------------------- -exec_subdir='bin_intel/exec' +exec_subdir='install_intel/exec' #----------------------------------------------------------------------- # Run E2E Tests From 49fd6d75531eaabb9319b764b7034af6c32bca83 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Fri, 23 Sep 2022 19:26:27 +0000 Subject: [PATCH 100/106] Run all tests in community mode so that the last NCO test case gets reported as finished. --- tests/WE2E/setup_WE2E_tests.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/WE2E/setup_WE2E_tests.sh b/tests/WE2E/setup_WE2E_tests.sh index b2412fef0a..6769c42939 100755 --- a/tests/WE2E/setup_WE2E_tests.sh +++ b/tests/WE2E/setup_WE2E_tests.sh @@ -93,5 +93,6 @@ module list tests_file=${auto_file} \ machine=${machine} \ account=${account} \ - exec_subdir=${exec_subdir} + exec_subdir=${exec_subdir} \ + run_envir="community" From 95b7ef0356cd234c27f7c64c17b341dd25527465 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Sat, 24 Sep 2022 02:50:11 +0000 Subject: [PATCH 101/106] Minor changes --- tests/WE2E/setup_WE2E_tests.sh | 6 +++--- ush/setup.py | 6 ------ 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/tests/WE2E/setup_WE2E_tests.sh b/tests/WE2E/setup_WE2E_tests.sh index 6769c42939..8116cd9f14 100755 --- a/tests/WE2E/setup_WE2E_tests.sh +++ b/tests/WE2E/setup_WE2E_tests.sh @@ -11,12 +11,12 @@ # The script is dependent on a successful build of this repo using the # test/build.sh script in the ufs-srweather-app repository. The UFS # build must be completed in a particular manner for this script to -# function properly, notably the location of the build and bin +# function properly, notably the location of the build and install # directories: # BUILD_DIR=${APP_DIR}/build_${compiler} -# BIN_DIR=${APP_DIR}/bin_${compiler} +# INSTALL_DIR=${APP_DIR}/install_${compiler} # -# Example: ./end_to_end_tests.sh hera zrtrr +# Example: ./setup_WE2E_tests.sh hera zrtrr #---------------------------------------------------------------------- #----------------------------------------------------------------------- diff --git a/ush/setup.py b/ush/setup.py index 7710898cc5..3e6138d372 100644 --- a/ush/setup.py +++ b/ush/setup.py @@ -1249,12 +1249,6 @@ def setup(): # global RUN_TASK_MAKE_GRID, RUN_TASK_MAKE_OROG, RUN_TASK_MAKE_SFC_CLIMO global RUN_TASK_VX_GRIDSTAT, RUN_TASK_VX_POINTSTAT, RUN_TASK_VX_ENSGRID, RUN_TASK_VX_ENSPOINT - global SYMLINK_FIX_FILES - - if RUN_ENVIR == "nco": - if not SYMLINK_FIX_FILES: - SYMLINK_FIX_FILES = True - print_info_msg("""Setting SYMLINK_FIX_FILES={SYMLINK_FIX_FILES}""") # Fix file location if RUN_TASK_MAKE_GRID: From f4bc96e9a887c718e04c5761b96ad3023bcade55 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Sat, 24 Sep 2022 18:25:10 +0000 Subject: [PATCH 102/106] Avoid using preamble in functions. --- ush/init_env.sh | 2 +- ush/make_grid_mosaic_file.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/init_env.sh b/ush/init_env.sh index ee6aee4be0..d883766325 100644 --- a/ush/init_env.sh +++ b/ush/init_env.sh @@ -13,7 +13,7 @@ # function init_env() { - { save_shell_opts; . $USHdir/preamble.sh; } > /dev/null 2>&1 + { save_shell_opts; set -u +x; } > /dev/null 2>&1 local valid_args=( \ "env_init_scripts_fps" \ diff --git a/ush/make_grid_mosaic_file.sh b/ush/make_grid_mosaic_file.sh index a68ae19b35..b0066a399a 100644 --- a/ush/make_grid_mosaic_file.sh +++ b/ush/make_grid_mosaic_file.sh @@ -17,7 +17,7 @@ function make_grid_mosaic_file() { # #----------------------------------------------------------------------- # - { save_shell_opts; . $USHdir/preamble.sh; } > /dev/null 2>&1 + { save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # From a268b9f9954a413bf8c873c1002223f5771c2197 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Sat, 24 Sep 2022 18:29:11 +0000 Subject: [PATCH 103/106] Use preamble in function too. --- ush/preamble.sh | 92 ++++++++++++++++++++++++++----------------------- 1 file changed, 49 insertions(+), 43 deletions(-) diff --git a/ush/preamble.sh b/ush/preamble.sh index d552a41196..458ab7ac6c 100644 --- a/ush/preamble.sh +++ b/ush/preamble.sh @@ -23,21 +23,14 @@ #----------------------------------------------------------------------- # set +x + +# Id for MPMD jobs if [[ -v '1' ]]; then - id="(${1})" + local id="(${1})" else - id="" + local id="" fi -# Record the start time so we can calculate the elapsed time later -start_time=$(date +%s) - -# Get the base name of the calling script -_calling_script=$(basename ${BASH_SOURCE[1]}) - -# Announce the script has begun -echo "Begin ${_calling_script} at $(date -u)" - # Stage our variables export STRICT=${STRICT:-"TRUE"} export TRACE=${DEBUG:-"FALSE"} @@ -54,40 +47,53 @@ if [[ $TRACE == "TRUE" ]]; then export PS4='+ $(basename $BASH_SOURCE)[$LINENO]'"$id: " fi -function postamble() { - # - #----------------------------------------------------------------------- - # - # Commands to execute when a script ends. - # - # Syntax: - # postamble script start_time rc - # - # Arguments: - # script: name of the script ending - # start_time: start time of script (in seconds) - # rc: the exit code of the script - # - #----------------------------------------------------------------------- - # +# Only when not called from a function +if [[ -z ${FUNCNAME[0]} ]]; then - set +x - script=${1} - start_time=${2} - rc=${3} + function postamble() { + # + #----------------------------------------------------------------------- + # + # Commands to execute when a script ends. + # + # Syntax: + # postamble script start_time rc + # + # Arguments: + # script: name of the script ending + # start_time: start time of script (in seconds) + # rc: the exit code of the script + # + #----------------------------------------------------------------------- + # + + set +x + script=${1} + start_time=${2} + rc=${3} + + # Calculate the elapsed time + end_time=$(date +%s) + elapsed_sec=$((end_time - start_time)) + elapsed=$(date -d@${elapsed_sec} -u +%H:%M:%S) + + # Announce the script has ended, then pass the error code up + echo "End ${script} at $(date -u) with error code ${rc:-0} (time elapsed: ${elapsed})" + exit ${rc} + } + + # Record the start time so we can calculate the elapsed time later + local start_time=$(date +%s) + + # Get the base name of the calling script + local _calling_script=$(basename ${BASH_SOURCE[1]}) + + # Announce the script has begun + echo "Begin ${_calling_script} at $(date -u)" - # Calculate the elapsed time - end_time=$(date +%s) - elapsed_sec=$((end_time - start_time)) - elapsed=$(date -d@${elapsed_sec} -u +%H:%M:%S) - - # Announce the script has ended, then pass the error code up - echo "End ${script} at $(date -u) with error code ${rc:-0} (time elapsed: ${elapsed})" - exit ${rc} -} - -# Place the postamble in a trap so it is always called no matter how the script exits -trap "postamble ${_calling_script} ${start_time} \$?" EXIT + # Place the postamble in a trap so it is always called no matter how the script exits + trap "postamble ${_calling_script} ${start_time} \$?" EXIT +fi # Turn on our settings $ERR_EXIT_CMD From ce7928086c09d7bb01259a6f353725acf0bc0368 Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Sat, 24 Sep 2022 18:48:52 +0000 Subject: [PATCH 104/106] Turn on debugging for utility functions. --- tests/WE2E/run_WE2E_tests.sh | 7 ++ ush/bash_utils/boolify.sh | 2 +- ush/bash_utils/change_case.sh | 4 +- ush/bash_utils/check_for_preexist_dir_file.sh | 2 +- ush/bash_utils/check_var_valid_value.sh | 2 +- ush/bash_utils/count_files.sh | 2 +- ush/bash_utils/create_symlink_to_file.sh | 2 +- ush/bash_utils/filesys_cmds_vrfy.sh | 14 ++-- ush/bash_utils/get_bash_file_contents.sh | 2 +- ush/bash_utils/get_charvar_from_netcdf.sh | 2 +- ush/bash_utils/get_elem_inds.sh | 2 +- .../get_manage_externals_config_property.sh | 2 +- ush/bash_utils/interpol_to_arbit_CRES.sh | 2 +- ush/bash_utils/is_array.sh | 2 +- ush/bash_utils/is_element_of.sh | 2 +- ush/bash_utils/print_input_args.sh | 2 +- ush/bash_utils/print_msg.sh | 4 +- ush/bash_utils/process_args.sh | 2 +- ush/bash_utils/set_bash_param.sh | 2 +- ush/bash_utils/set_file_param.sh | 2 +- ush/bash_utils/source_config.sh | 2 +- ush/init_env.sh | 2 +- ush/make_grid_mosaic_file.sh | 2 +- ush/preamble.sh | 67 +++++++++++-------- 24 files changed, 75 insertions(+), 59 deletions(-) diff --git a/tests/WE2E/run_WE2E_tests.sh b/tests/WE2E/run_WE2E_tests.sh index f7b049f0ef..31df790145 100755 --- a/tests/WE2E/run_WE2E_tests.sh +++ b/tests/WE2E/run_WE2E_tests.sh @@ -89,6 +89,7 @@ Usage: [exec_subdir=\"...\"] \\ [use_cron_to_relaunch=\"...\"] \\ [cron_relaunch_intvl_mnts=\"...\"] \\ + [debug=\"...\"] \\ [verbose=\"...\"] \\ [generate_csv_file=\"...\"] \\ [machine_file=\"...\"] \\ @@ -172,6 +173,9 @@ that the value of this argument matters only if the argument use_cron_to_relaunch is not explicitly set to \"FALSE\" in the call to this script. +debug: +If true, run test case in debugging mode. + verbose: Optional argument used to explicitly set the experiment variable VERBOSE in the experiment configuration files of all the WE2E tests the user @@ -319,6 +323,7 @@ valid_args=( \ "exec_subdir" \ "use_cron_to_relaunch" \ "cron_relaunch_intvl_mnts" \ + "debug" \ "verbose" \ "generate_csv_file" \ "machine_file" \ @@ -764,6 +769,7 @@ Please correct and rerun." EXEC_SUBDIR="${exec_subdir}" USE_CRON_TO_RELAUNCH=${use_cron_to_relaunch:-"TRUE"} CRON_RELAUNCH_INTVL_MNTS=${cron_relaunch_intvl_mnts:-"02"} + DEBUG=${debug:-"FALSE"} VERBOSE=${verbose:-"TRUE"} MACHINE_FILE=${machine_file:-"${USHdir}/machine/${machine,,}.sh"} @@ -812,6 +818,7 @@ MACHINE_FILE=\"${MACHINE_FILE}\" # # Flag specifying whether to run in verbose mode. # +DEBUG=\"${DEBUG}\" VERBOSE=\"${VERBOSE}\"" # #----------------------------------------------------------------------- diff --git a/ush/bash_utils/boolify.sh b/ush/bash_utils/boolify.sh index fe32090fda..577b7c16eb 100644 --- a/ush/bash_utils/boolify.sh +++ b/ush/bash_utils/boolify.sh @@ -16,7 +16,7 @@ function boolify() { # #----------------------------------------------------------------------- # - { save_shell_opts; set -u +x; } > /dev/null 2>&1 + { save_shell_opts; . ${USHdir}/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/ush/bash_utils/change_case.sh b/ush/bash_utils/change_case.sh index 2bd82d1306..752ead9018 100644 --- a/ush/bash_utils/change_case.sh +++ b/ush/bash_utils/change_case.sh @@ -24,7 +24,7 @@ function echo_uppercase() { # #----------------------------------------------------------------------- # - { save_shell_opts; set -u +x; } > /dev/null 2>&1 + { save_shell_opts; . ${USHdir}/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # @@ -139,7 +139,7 @@ function echo_lowercase() { # #----------------------------------------------------------------------- # - { save_shell_opts; set -u +x; } > /dev/null 2>&1 + { save_shell_opts; . ${USHdir}/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/ush/bash_utils/check_for_preexist_dir_file.sh b/ush/bash_utils/check_for_preexist_dir_file.sh index dac2688a31..53dab67588 100644 --- a/ush/bash_utils/check_for_preexist_dir_file.sh +++ b/ush/bash_utils/check_for_preexist_dir_file.sh @@ -16,7 +16,7 @@ function check_for_preexist_dir_file() { # #----------------------------------------------------------------------- # - { save_shell_opts; set -u +x; } > /dev/null 2>&1 + { save_shell_opts; . ${USHdir}/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/ush/bash_utils/check_var_valid_value.sh b/ush/bash_utils/check_var_valid_value.sh index 576ad6b1b8..21288184db 100644 --- a/ush/bash_utils/check_var_valid_value.sh +++ b/ush/bash_utils/check_var_valid_value.sh @@ -15,7 +15,7 @@ function check_var_valid_value() { # #----------------------------------------------------------------------- # - { save_shell_opts; set -u +x; } > /dev/null 2>&1 + { save_shell_opts; . ${USHdir}/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/ush/bash_utils/count_files.sh b/ush/bash_utils/count_files.sh index c80f342cb1..9ca22f8c09 100644 --- a/ush/bash_utils/count_files.sh +++ b/ush/bash_utils/count_files.sh @@ -15,7 +15,7 @@ function count_files() { # #----------------------------------------------------------------------- # - { save_shell_opts; set -u +x; } > /dev/null 2>&1 + { save_shell_opts; . ${USHdir}/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/ush/bash_utils/create_symlink_to_file.sh b/ush/bash_utils/create_symlink_to_file.sh index c14a35bdc6..38099ffb8e 100644 --- a/ush/bash_utils/create_symlink_to_file.sh +++ b/ush/bash_utils/create_symlink_to_file.sh @@ -21,7 +21,7 @@ function create_symlink_to_file() { # #----------------------------------------------------------------------- # - { save_shell_opts; set -u +x; } > /dev/null 2>&1 + { save_shell_opts; . ${USHdir}/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/ush/bash_utils/filesys_cmds_vrfy.sh b/ush/bash_utils/filesys_cmds_vrfy.sh index d147fc1079..0f7d842467 100644 --- a/ush/bash_utils/filesys_cmds_vrfy.sh +++ b/ush/bash_utils/filesys_cmds_vrfy.sh @@ -18,7 +18,7 @@ function filesys_cmd_vrfy() { # #----------------------------------------------------------------------- # - { save_shell_opts; set -u +x; } > /dev/null 2>&1 + { save_shell_opts; . ${USHdir}/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # @@ -239,37 +239,37 @@ $output" # function cp_vrfy() { - { save_shell_opts; set -u +x; } > /dev/null 2>&1 + { save_shell_opts; . ${USHdir}/preamble.sh; } > /dev/null 2>&1 filesys_cmd_vrfy "cp" "$@" { restore_shell_opts; } > /dev/null 2>&1 } function mv_vrfy() { - { save_shell_opts; set -u +x; } > /dev/null 2>&1 + { save_shell_opts; . ${USHdir}/preamble.sh; } > /dev/null 2>&1 filesys_cmd_vrfy "mv" "$@" { restore_shell_opts; } > /dev/null 2>&1 } function rm_vrfy() { - { save_shell_opts; set -u +x; } > /dev/null 2>&1 + { save_shell_opts; . ${USHdir}/preamble.sh; } > /dev/null 2>&1 filesys_cmd_vrfy "rm" "$@" { restore_shell_opts; } > /dev/null 2>&1 } function ln_vrfy() { - { save_shell_opts; set -u +x; } > /dev/null 2>&1 + { save_shell_opts; . ${USHdir}/preamble.sh; } > /dev/null 2>&1 filesys_cmd_vrfy "$LN_UTIL" "$@" { restore_shell_opts; } > /dev/null 2>&1 } function mkdir_vrfy() { - { save_shell_opts; set -u +x; } > /dev/null 2>&1 + { save_shell_opts; . ${USHdir}/preamble.sh; } > /dev/null 2>&1 filesys_cmd_vrfy "mkdir" "$@" { restore_shell_opts; } > /dev/null 2>&1 } function cd_vrfy() { - { save_shell_opts; set -u +x; } > /dev/null 2>&1 + { save_shell_opts; . ${USHdir}/preamble.sh; } > /dev/null 2>&1 filesys_cmd_vrfy "cd" "$@" { restore_shell_opts; } > /dev/null 2>&1 } diff --git a/ush/bash_utils/get_bash_file_contents.sh b/ush/bash_utils/get_bash_file_contents.sh index fdfb626922..ff958abd3a 100644 --- a/ush/bash_utils/get_bash_file_contents.sh +++ b/ush/bash_utils/get_bash_file_contents.sh @@ -17,7 +17,7 @@ # function get_bash_file_contents() { - { save_shell_opts; set -u +x; } > /dev/null 2>&1 + { save_shell_opts; . ${USHdir}/preamble.sh; } > /dev/null 2>&1 local valid_args=( \ "fp" \ diff --git a/ush/bash_utils/get_charvar_from_netcdf.sh b/ush/bash_utils/get_charvar_from_netcdf.sh index ac70cf68da..e7a0904db7 100644 --- a/ush/bash_utils/get_charvar_from_netcdf.sh +++ b/ush/bash_utils/get_charvar_from_netcdf.sh @@ -25,7 +25,7 @@ function get_charvar_from_netcdf() { # #----------------------------------------------------------------------- # - { save_shell_opts; set -u +x; } > /dev/null 2>&1 + { save_shell_opts; . ${USHdir}/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/ush/bash_utils/get_elem_inds.sh b/ush/bash_utils/get_elem_inds.sh index 89a9c41227..4545fa4678 100644 --- a/ush/bash_utils/get_elem_inds.sh +++ b/ush/bash_utils/get_elem_inds.sh @@ -14,7 +14,7 @@ function get_elem_inds() { # #----------------------------------------------------------------------- # - { save_shell_opts; set -u +x; } > /dev/null 2>&1 + { save_shell_opts; . ${USHdir}/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/ush/bash_utils/get_manage_externals_config_property.sh b/ush/bash_utils/get_manage_externals_config_property.sh index 370d448f2a..318804927b 100644 --- a/ush/bash_utils/get_manage_externals_config_property.sh +++ b/ush/bash_utils/get_manage_externals_config_property.sh @@ -14,7 +14,7 @@ function get_manage_externals_config_property() { # #----------------------------------------------------------------------- # - { save_shell_opts; set -u +x; } > /dev/null 2>&1 + { save_shell_opts; . ${USHdir}/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/ush/bash_utils/interpol_to_arbit_CRES.sh b/ush/bash_utils/interpol_to_arbit_CRES.sh index bc4c6e8336..2f4681d93a 100644 --- a/ush/bash_utils/interpol_to_arbit_CRES.sh +++ b/ush/bash_utils/interpol_to_arbit_CRES.sh @@ -17,7 +17,7 @@ function interpol_to_arbit_CRES() { # #----------------------------------------------------------------------- # - { save_shell_opts; set -u +x; } > /dev/null 2>&1 + { save_shell_opts; . ${USHdir}/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/ush/bash_utils/is_array.sh b/ush/bash_utils/is_array.sh index c831f313e2..234c3a13ac 100644 --- a/ush/bash_utils/is_array.sh +++ b/ush/bash_utils/is_array.sh @@ -21,7 +21,7 @@ function is_array() { # #----------------------------------------------------------------------- # - { save_shell_opts; set -u +x; } > /dev/null 2>&1 + { save_shell_opts; . ${USHdir}/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/ush/bash_utils/is_element_of.sh b/ush/bash_utils/is_element_of.sh index 3ae9ce1cc5..639c076d93 100644 --- a/ush/bash_utils/is_element_of.sh +++ b/ush/bash_utils/is_element_of.sh @@ -14,7 +14,7 @@ function is_element_of() { # #----------------------------------------------------------------------- # - { save_shell_opts; set -u +x; } > /dev/null 2>&1 + { save_shell_opts; . ${USHdir}/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/ush/bash_utils/print_input_args.sh b/ush/bash_utils/print_input_args.sh index d5ba5bd53c..ab097de4fd 100644 --- a/ush/bash_utils/print_input_args.sh +++ b/ush/bash_utils/print_input_args.sh @@ -21,7 +21,7 @@ function print_input_args() { # #----------------------------------------------------------------------- # - { save_shell_opts; set -u +x; } > /dev/null 2>&1 + { save_shell_opts; . ${USHdir}/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/ush/bash_utils/print_msg.sh b/ush/bash_utils/print_msg.sh index f48e0c2f8b..aa66a9bd90 100644 --- a/ush/bash_utils/print_msg.sh +++ b/ush/bash_utils/print_msg.sh @@ -23,7 +23,7 @@ function print_info_msg() { # #----------------------------------------------------------------------- # - { save_shell_opts; set -u +x; } > /dev/null 2>&1 + { save_shell_opts; . ${USHdir}/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # @@ -172,7 +172,7 @@ function print_err_msg_exit() { # #----------------------------------------------------------------------- # - { save_shell_opts; set -u +x; } > /dev/null 2>&1 + { save_shell_opts; . ${USHdir}/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/ush/bash_utils/process_args.sh b/ush/bash_utils/process_args.sh index 203bdedcf7..40cc68f4f0 100644 --- a/ush/bash_utils/process_args.sh +++ b/ush/bash_utils/process_args.sh @@ -89,7 +89,7 @@ function process_args() { # #----------------------------------------------------------------------- # - { save_shell_opts; set -u +x; } > /dev/null 2>&1 + { save_shell_opts; . ${USHdir}/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/ush/bash_utils/set_bash_param.sh b/ush/bash_utils/set_bash_param.sh index dd22206077..de4ebbf0ef 100644 --- a/ush/bash_utils/set_bash_param.sh +++ b/ush/bash_utils/set_bash_param.sh @@ -15,7 +15,7 @@ function set_bash_param() { # #----------------------------------------------------------------------- # - { save_shell_opts; set -u +x; } > /dev/null 2>&1 + { save_shell_opts; . ${USHdir}/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/ush/bash_utils/set_file_param.sh b/ush/bash_utils/set_file_param.sh index 3e0d133494..a67b65dc70 100644 --- a/ush/bash_utils/set_file_param.sh +++ b/ush/bash_utils/set_file_param.sh @@ -15,7 +15,7 @@ function set_file_param() { # #----------------------------------------------------------------------- # - { save_shell_opts; set -u +x; } > /dev/null 2>&1 + { save_shell_opts; . ${USHdir}/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/ush/bash_utils/source_config.sh b/ush/bash_utils/source_config.sh index 271c462814..5d0d047d00 100644 --- a/ush/bash_utils/source_config.sh +++ b/ush/bash_utils/source_config.sh @@ -14,7 +14,7 @@ function config_to_str() { # #----------------------------------------------------------------------- # - { save_shell_opts; set -u +x; } > /dev/null 2>&1 + { save_shell_opts; . ${USHdir}/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- diff --git a/ush/init_env.sh b/ush/init_env.sh index d883766325..9a653fd6a3 100644 --- a/ush/init_env.sh +++ b/ush/init_env.sh @@ -13,7 +13,7 @@ # function init_env() { - { save_shell_opts; set -u +x; } > /dev/null 2>&1 + { save_shell_opts; . ${USHdir}/preamble.sh; } > /dev/null 2>&1 local valid_args=( \ "env_init_scripts_fps" \ diff --git a/ush/make_grid_mosaic_file.sh b/ush/make_grid_mosaic_file.sh index b0066a399a..3890f93a22 100644 --- a/ush/make_grid_mosaic_file.sh +++ b/ush/make_grid_mosaic_file.sh @@ -17,7 +17,7 @@ function make_grid_mosaic_file() { # #----------------------------------------------------------------------- # - { save_shell_opts; set -u +x; } > /dev/null 2>&1 + { save_shell_opts; . ${USHdir}/preamble.sh; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/ush/preamble.sh b/ush/preamble.sh index 458ab7ac6c..0357ace76f 100644 --- a/ush/preamble.sh +++ b/ush/preamble.sh @@ -24,30 +24,13 @@ # set +x -# Id for MPMD jobs -if [[ -v '1' ]]; then - local id="(${1})" -else - local id="" -fi - -# Stage our variables -export STRICT=${STRICT:-"TRUE"} -export TRACE=${DEBUG:-"FALSE"} -export ERR_EXIT_CMD="" -export TRACE_CMD="" - -if [[ $STRICT == "TRUE" ]]; then - # Exit on error and undefined variable - export ERR_EXIT_CMD="set -eu" -fi -if [[ $TRACE == "TRUE" ]]; then - export TRACE_CMD="set -x" - # Print the script name and line number of each command as it is executed - export PS4='+ $(basename $BASH_SOURCE)[$LINENO]'"$id: " -fi - -# Only when not called from a function +# +#----------------------------------------------------------------------- +# +# Print time elaspsed in a script (not done when called from a function) +# +#----------------------------------------------------------------------- +# if [[ -z ${FUNCNAME[0]} ]]; then function postamble() { @@ -83,10 +66,10 @@ if [[ -z ${FUNCNAME[0]} ]]; then } # Record the start time so we can calculate the elapsed time later - local start_time=$(date +%s) + start_time=$(date +%s) # Get the base name of the calling script - local _calling_script=$(basename ${BASH_SOURCE[1]}) + _calling_script=$(basename ${BASH_SOURCE[1]}) # Announce the script has begun echo "Begin ${_calling_script} at $(date -u)" @@ -95,6 +78,32 @@ if [[ -z ${FUNCNAME[0]} ]]; then trap "postamble ${_calling_script} ${start_time} \$?" EXIT fi -# Turn on our settings -$ERR_EXIT_CMD -$TRACE_CMD +# +#----------------------------------------------------------------------- +# +# Turn on flags for degugging and strict error checking. +# Also set PS4 +# +#----------------------------------------------------------------------- +# +STRICT=${STRICT:-"TRUE"} +TRACE=${DEBUG:-"FALSE"} + +if [[ $STRICT == "TRUE" ]]; then + # Exit on error and undefined variable + set -eu +fi +if [[ $TRACE == "TRUE" ]]; then + # Turn on debugging + set -x + + # Id for MPMD jobs + if [[ -v '1' ]]; then + id="(${1})" + else + id="" + fi + # Print the script name and line number of each command as it is executed + export PS4='+ $(basename $BASH_SOURCE)[$LINENO]'"$id: " +fi + From 3c44bc7ab4dd4c9a4acaab0fb697404cc2912e1f Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Sat, 24 Sep 2022 19:14:16 +0000 Subject: [PATCH 105/106] Turn on debug & verbose in CI. --- tests/WE2E/setup_WE2E_tests.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/WE2E/setup_WE2E_tests.sh b/tests/WE2E/setup_WE2E_tests.sh index 8116cd9f14..1605582a86 100755 --- a/tests/WE2E/setup_WE2E_tests.sh +++ b/tests/WE2E/setup_WE2E_tests.sh @@ -94,5 +94,7 @@ module list machine=${machine} \ account=${account} \ exec_subdir=${exec_subdir} \ + debug="TRUE" \ + verbose="TRUE" \ run_envir="community" From 21f9c965d10a0a33fdab8cbd4aec6083010eac6e Mon Sep 17 00:00:00 2001 From: Daniel Abdi Date: Sat, 24 Sep 2022 19:30:26 +0000 Subject: [PATCH 106/106] Turn off set -e for init_env --- ush/init_env.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/init_env.sh b/ush/init_env.sh index 9a653fd6a3..368cf9818c 100644 --- a/ush/init_env.sh +++ b/ush/init_env.sh @@ -35,9 +35,9 @@ Attempting to source script: # The scripts being sourced here may have undefined variables, but since # they are system scripts outside of the SRW App, they cannot be changed. # Thus, we allow for undefined variables by temporarily using "set +u". - set +u + set +eu source "$fp" && print_info_msg "$DEBUG" "Succeeded." - set -u + set -eu else print_err_msg_exit "\ The script to source does not exist or is not a regular file: