From 5392d360585889456ec042acb5d2c24303184908 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Fri, 19 Nov 2021 04:32:43 -0700 Subject: [PATCH 01/35] Change "set -x" to "set -x" to reduce clutter in output. --- ush/check_ruc_lsm.sh | 2 +- ush/create_diag_table_file.sh | 2 +- ush/create_model_configure_file.sh | 2 +- ush/link_fix.sh | 2 +- ush/make_grid_mosaic_file.sh | 2 +- ush/set_FV3nml_sfc_climo_filenames.sh | 2 +- ush/set_FV3nml_stoch_params.sh | 2 +- ush/set_ozone_param.sh | 2 +- ush/set_thompson_mp_fix_files.sh | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ush/check_ruc_lsm.sh b/ush/check_ruc_lsm.sh index 8d51cc01b..35e4db195 100644 --- a/ush/check_ruc_lsm.sh +++ b/ush/check_ruc_lsm.sh @@ -20,7 +20,7 @@ function check_ruc_lsm() { # #----------------------------------------------------------------------- # - { save_shell_opts; set -u -x; } > /dev/null 2>&1 + { save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/ush/create_diag_table_file.sh b/ush/create_diag_table_file.sh index 3c409031a..41ca941bf 100644 --- a/ush/create_diag_table_file.sh +++ b/ush/create_diag_table_file.sh @@ -15,7 +15,7 @@ function create_diag_table_file() { # #----------------------------------------------------------------------- # - { save_shell_opts; set -u -x; } > /dev/null 2>&1 + { save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/ush/create_model_configure_file.sh b/ush/create_model_configure_file.sh index ccd55d00f..8f814ac59 100644 --- a/ush/create_model_configure_file.sh +++ b/ush/create_model_configure_file.sh @@ -15,7 +15,7 @@ function create_model_configure_file() { # #----------------------------------------------------------------------- # - { save_shell_opts; set -u -x; } > /dev/null 2>&1 + { save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/ush/link_fix.sh b/ush/link_fix.sh index 34bcd5a53..0bfd5c03d 100755 --- a/ush/link_fix.sh +++ b/ush/link_fix.sh @@ -16,7 +16,7 @@ function link_fix() { # #----------------------------------------------------------------------- # - { save_shell_opts; set -u -x; } > /dev/null 2>&1 + { save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/ush/make_grid_mosaic_file.sh b/ush/make_grid_mosaic_file.sh index 8a81feb14..5adf2ac30 100644 --- a/ush/make_grid_mosaic_file.sh +++ b/ush/make_grid_mosaic_file.sh @@ -17,7 +17,7 @@ function make_grid_mosaic_file() { # #----------------------------------------------------------------------- # - { save_shell_opts; set -u -x; } > /dev/null 2>&1 + { save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/ush/set_FV3nml_sfc_climo_filenames.sh b/ush/set_FV3nml_sfc_climo_filenames.sh index 13f2ecdf5..d6f9dc5d4 100644 --- a/ush/set_FV3nml_sfc_climo_filenames.sh +++ b/ush/set_FV3nml_sfc_climo_filenames.sh @@ -20,7 +20,7 @@ function set_FV3nml_sfc_climo_filenames() { # #----------------------------------------------------------------------- # - { save_shell_opts; set -u -x; } > /dev/null 2>&1 + { save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/ush/set_FV3nml_stoch_params.sh b/ush/set_FV3nml_stoch_params.sh index cfe115d52..8f08bb566 100644 --- a/ush/set_FV3nml_stoch_params.sh +++ b/ush/set_FV3nml_stoch_params.sh @@ -22,7 +22,7 @@ function set_FV3nml_stoch_params() { # #----------------------------------------------------------------------- # - { save_shell_opts; set -u -x; } > /dev/null 2>&1 + { save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/ush/set_ozone_param.sh b/ush/set_ozone_param.sh index c85749279..22abe3371 100644 --- a/ush/set_ozone_param.sh +++ b/ush/set_ozone_param.sh @@ -34,7 +34,7 @@ function set_ozone_param() { # #----------------------------------------------------------------------- # - { save_shell_opts; set -u -x; } > /dev/null 2>&1 + { save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/ush/set_thompson_mp_fix_files.sh b/ush/set_thompson_mp_fix_files.sh index 667ad522f..7bad26c2b 100644 --- a/ush/set_thompson_mp_fix_files.sh +++ b/ush/set_thompson_mp_fix_files.sh @@ -22,7 +22,7 @@ function set_thompson_mp_fix_files() { # #----------------------------------------------------------------------- # - { save_shell_opts; set -u -x; } > /dev/null 2>&1 + { save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # From 10b6e3a387a29c53492b8d436efcf8e0f3668757 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Fri, 19 Nov 2021 04:34:30 -0700 Subject: [PATCH 02/35] Change printing of input arguments to a function or script only if the new experiment variable DEBUG is set to TRUE (as opposed to VERBOSE being set to TRUE). --- ush/bash_utils/print_input_args.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/ush/bash_utils/print_input_args.sh b/ush/bash_utils/print_input_args.sh index 72b7744b3..957cec524 100644 --- a/ush/bash_utils/print_input_args.sh +++ b/ush/bash_utils/print_input_args.sh @@ -168,16 +168,15 @@ have been set as follows: # #----------------------------------------------------------------------- # -# If a global variable named VERBOSE is not defined, print out the mes- -# sage. If it is defined, print out the message only if VERBOSE is set -# to TRUE. +# If a global variable named DEBUG is not defined, print out the message. +# If it is defined, print out the message only if DEBUG is set to TRUE. # #----------------------------------------------------------------------- # - if [ -z ${VERBOSE+x} ]; then + if [ -z ${DEBUG+x} ]; then print_info_msg "$msg" else - print_info_msg "$VERBOSE" "$msg" + print_info_msg "$DEBUG" "$msg" fi # #----------------------------------------------------------------------- From 666cdea7f96d60569b51a264127cf4e4781b46bb Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Fri, 19 Nov 2021 04:36:40 -0700 Subject: [PATCH 03/35] Change printing out of informational message only if the new experiment variable DEBUG is set to TRUE (as opposed to VERBOSE being set to TRUE). Also, for brevity, rename variable file_full_path to file_fp. --- ush/bash_utils/set_file_param.sh | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/ush/bash_utils/set_file_param.sh b/ush/bash_utils/set_file_param.sh index d45451483..3e0d13349 100644 --- a/ush/bash_utils/set_file_param.sh +++ b/ush/bash_utils/set_file_param.sh @@ -53,11 +53,11 @@ Incorrect number of arguments specified: Usage: - ${func_name} file_full_path param value + ${func_name} file_fp param value where the arguments are defined as follows: - file_full_path: + file_fp: Full path to the file in which the specified parameter's value will be set. @@ -76,7 +76,7 @@ where the arguments are defined as follows: # #----------------------------------------------------------------------- # - local file_full_path="$1" + local file_fp="$1" local param="$2" local value="$3" # @@ -86,7 +86,7 @@ where the arguments are defined as follows: # #----------------------------------------------------------------------- # - local file="${file_full_path##*/}" + local file="${file_fp##*/}" # #----------------------------------------------------------------------- # @@ -94,7 +94,7 @@ where the arguments are defined as follows: # #----------------------------------------------------------------------- # - print_info_msg "$VERBOSE" "\ + print_info_msg "$DEBUG" "\ Setting parameter \"$param\" in file \"$file\" to \"$value\" ..." # #----------------------------------------------------------------------- @@ -141,7 +141,7 @@ Setting parameter \"$param\" in file \"$file\" to \"$value\" ..." "${GLOBAL_VAR_DEFNS_FN}") regex_search="(^\s*$param=)(\".*\")?([^ \"]*)?(\(.*\))?(\s*[#].*)?" regex_replace="\1$value\5" -# set_bash_param "${file_full_path}" "$param" "$value" +# set_bash_param "${file_fp}" "$param" "$value" ;; # #----------------------------------------------------------------------- @@ -168,15 +168,15 @@ specified for this file: # #----------------------------------------------------------------------- # - grep -q -E "${regex_search}" "${file_full_path}" + grep -q -E "${regex_search}" "${file_fp}" if [ $? -eq 0 ]; then - $SED -i -r -e "s%${regex_search}%${regex_replace}%" "${file_full_path}" + $SED -i -r -e "s%${regex_search}%${regex_replace}%" "${file_fp}" else print_err_msg_exit "\ -Specified file (file_full_path) does not contain the searched-for regu- -lar expression (regex_search): - file_full_path = \"${file_full_path}\" +The specified file (file_fp) does not contain the searched-for regular +expression (regex_search): + file_fp = \"${file_fp}\" param = \"$param\" value = \"$value\" regex_search = ${regex_search}" From 24b49bffb371c4dfbce692333fafab95a6b31ef8 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Fri, 19 Nov 2021 04:45:05 -0700 Subject: [PATCH 04/35] Add new experiment variable DEBUG to config_defaults.sh and use it instead of VERBOSE to reduce clutter in output. --- ush/config_defaults.sh | 14 ++++++++++--- ush/setup.sh | 45 ++++++++++++++++++++++++++++++++++------- ush/valid_param_vals.sh | 1 + 3 files changed, 50 insertions(+), 10 deletions(-) diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index ba3711f90..4bdfdfc6e 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -1171,13 +1171,21 @@ PREEXISTING_DIR_METHOD="delete" # #----------------------------------------------------------------------- # -# Set VERBOSE. This is a flag that determines whether or not the experiment -# generation and workflow task scripts tend to print out more informational -# messages. +# Set flags for more detailed messages. Defintitions: +# +# VERBOSE: +# This is a flag that determines whether or not the experiment generation +# and workflow task scripts tend to print out more informational messages. +# +# DEBUG: +# This is a flag that determines whether or not very detailed debugging +# messages are printed to out. Note that if DEBUG is set to TRUE, then +# VERBOSE will also get reset to TRUE if it isn't already. # #----------------------------------------------------------------------- # VERBOSE="TRUE" +DEBUG="FALSE" # #----------------------------------------------------------------------- # diff --git a/ush/setup.sh b/ush/setup.sh index 1dcc748ab..a633f98b4 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -152,6 +152,36 @@ fi # #----------------------------------------------------------------------- # +# Make sure that DEBUG is set to a valid value. +# +#----------------------------------------------------------------------- +# +check_var_valid_value "DEBUG" "valid_vals_DEBUG" +# +# Set DEBUG to either "TRUE" or "FALSE" so we don't have to consider +# other valid values later on. +# +DEBUG=$(echo_uppercase $DEBUG) +if [ "$DEBUG" = "TRUE" ] || \ + [ "$DEBUG" = "YES" ]; then + DEBUG="TRUE" +elif [ "$DEBUG" = "FALSE" ] || \ + [ "$DEBUG" = "NO" ]; then + DEBUG="FALSE" +fi +# +#----------------------------------------------------------------------- +# +# If DEBUG is set to "TRUE", make sure that VERBOSE is also set to "TRUE". +# +#----------------------------------------------------------------------- +# +if [ "$DEBUG" = "TRUE" ]; then + VERBOSE="TRUE" +fi +# +#----------------------------------------------------------------------- +# # Make sure that USE_CRON_TO_RELAUNCH is set to a valid value. # #----------------------------------------------------------------------- @@ -792,9 +822,9 @@ fi # #----------------------------------------------------------------------- # -DATE_OR_NULL=$( printf "%s" "${DATE_FIRST_CYCL}" | \ +date_or_null=$( printf "%s" "${DATE_FIRST_CYCL}" | \ $SED -n -r -e "s/^([0-9]{8})$/\1/p" ) -if [ -z "${DATE_OR_NULL}" ]; then +if [ -z "${date_or_null}" ]; then print_err_msg_exit "\ DATE_FIRST_CYCL must be a string consisting of exactly 8 digits of the form \"YYYYMMDD\", where YYYY is the 4-digit year, MM is the 2-digit @@ -802,9 +832,9 @@ month, and DD is the 2-digit day-of-month. DATE_FIRST_CYCL = \"${DATE_FIRST_CYCL}\"" fi -DATE_OR_NULL=$( printf "%s" "${DATE_LAST_CYCL}" | \ +date_or_null=$( printf "%s" "${DATE_LAST_CYCL}" | \ $SED -n -r -e "s/^([0-9]{8})$/\1/p" ) -if [ -z "${DATE_OR_NULL}" ]; then +if [ -z "${date_or_null}" ]; then print_err_msg_exit "\ DATE_LAST_CYCL must be a string consisting of exactly 8 digits of the form \"YYYYMMDD\", where YYYY is the 4-digit year, MM is the 2-digit @@ -2549,8 +2579,9 @@ line_list=$( $SED -r \ -e "/^$/d" \ ${GLOBAL_VAR_DEFNS_FP} ) -print_info_msg "$VERBOSE" " -The variable \"line_list\" contains: +print_info_msg "$DEBUG" " +Before updating default values of experiment variables, the variable +\"line_list\" contains: ${line_list} " @@ -2613,7 +2644,7 @@ while read crnt_line; do # if [ ! -z $var_name ]; then - print_info_msg "$VERBOSE" " + print_info_msg "$DEBUG" " var_name = \"${var_name}\"" # # If the variable specified in var_name is set in the current environ- diff --git a/ush/valid_param_vals.sh b/ush/valid_param_vals.sh index 1b3acf00a..8104038da 100644 --- a/ush/valid_param_vals.sh +++ b/ush/valid_param_vals.sh @@ -3,6 +3,7 @@ # valid_vals_RUN_ENVIR=("nco" "community") valid_vals_VERBOSE=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") +valid_vals_DEBUG=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_MACHINE=("WCOSS_CRAY" "WCOSS_DELL_P3" "HERA" "ORION" "JET" "ODIN" "CHEYENNE" "STAMPEDE" "LINUX" "MACOS") valid_vals_SCHED=("slurm" "pbspro" "lsf" "lsfcray" "none") valid_vals_FCST_MODEL=("ufs-weather-model" "fv3gfs_aqm") From b2759887037aad78837dc9bacb8518adabd9c167 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Fri, 19 Nov 2021 05:01:30 -0700 Subject: [PATCH 05/35] Changes to allow use of "template" variables in config_defaults.sh and/or config.sh, i.e. variables whose definitions contain other variables, e.g. MY_CMD="ls \${SOME_DIR}". --- ush/setup.sh | 211 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 133 insertions(+), 78 deletions(-) diff --git a/ush/setup.sh b/ush/setup.sh index a633f98b4..4fda4bb42 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -2599,10 +2599,10 @@ read -r -d '' str_to_insert << EOM #----------------------------------------------------------------------- #----------------------------------------------------------------------- # Section 1: -# This section is a copy of the default workflow/experiment configura- -# tion file config_defaults.sh in the shell scripts directory USHDIR ex- -# cept that variable values have been updated to those set by the setup -# script (setup.sh). +# This section is a copy of the default experiment configuration file +# (${EXPT_DEFAULT_CONFIG_FN}) in the shell scripts directory specified by USHDIR +# except that variable values have been updated to those for the experiment +# (as opposed to the default values). #----------------------------------------------------------------------- #----------------------------------------------------------------------- # @@ -2611,55 +2611,74 @@ EOM # Replace all occurrences of actual newlines in the variable str_to_insert # with escaped backslash-n. This is needed for the sed command below to # work properly (i.e. to avoid it failing with an "unterminated `s' command" -# message). +# error message). # str_to_insert=${str_to_insert//$'\n'/\\n} # -# Insert str_to_insert into GLOBAL_VAR_DEFNS_FP right after the line -# containing the name of the interpreter (i.e. the line that starts with -# the string "#!", e.g. "#!/bin/bash"). +# Insert str_to_insert at the top of the file GLOBAL_VAR_DEFNS_FP. # -regexp="(^#!.*)" -$SED -i -r -e "s|$regexp|\1\n\n${str_to_insert}\n|g" ${GLOBAL_VAR_DEFNS_FP} +$SED -i -r -e "1i${str_to_insert}\n" ${GLOBAL_VAR_DEFNS_FP} +# +# Set the flag that specifies whether or not array variables will be +# recorded in the variable definitions file on one line or one element +# per line. Then, if writing arrays one element per line (i.e. multiline), +# set an escaped-newline character that needs to be included after every +# element of each array as the newline character in order for sed to +# write the line properly. +# +multiline_arrays="TRUE" +#multiline_arrays="FALSE" +esc_nl_or_null="" +if [ "${multiline_arrays}" = "TRUE" ]; then + esc_nl_or_null='\\\n' +fi # # Loop through the lines in line_list. # +print_info_msg "$VERBOSE" " +Generating the global experiment variable definitions file specified by +GLOBAL_VAR_DEFNS_FN: + GLOBAL_VAR_DEFNS_FN = \"${GLOBAL_VAR_DEFNS_FN}\" +Full path to this file is: + GLOBAL_VAR_DEFNS_FP = \"${GLOBAL_VAR_DEFNS_FP}\" +For more detailed information, set DEBUG to \"TRUE\" in the experiment +configuration file (${EXPT_CONFIG_FN}\")." + +template_var_names=() +template_var_values=() while read crnt_line; do # # Try to obtain the name of the variable being set on the current line. # This will be successful only if the line consists of one or more char- # acters representing the name of a variable (recall that in generating -# the variable line_list, all leading spaces in the lines in the file -# have been stripped out), followed by an equal sign, followed by zero -# or more characters representing the value that the variable is being -# set to. +# the variable line_list, leading spaces on each line were stripped out), +# followed by an equal sign, followed by zero or more characters +# representing the value that the variable is being set to. # var_name=$( printf "%s" "${crnt_line}" | $SED -n -r -e "s/^([^ ]*)=.*/\1/p" ) -#echo -#echo "============================" -#printf "%s\n" "var_name = \"${var_name}\"" # -# If var_name is not empty, then a variable name was found in the cur- -# rent line in line_list. +# If var_name is not empty, then a variable name was found on the current +# line in line_list. # - if [ ! -z $var_name ]; then + if [ ! -z ${var_name} ]; then print_info_msg "$DEBUG" " var_name = \"${var_name}\"" # -# If the variable specified in var_name is set in the current environ- -# ment (to either an empty or non-empty string), get its value and in- -# sert it in the variable definitions file on the line where that varia- -# ble is defined. Note that +# If the variable specified in var_name is set in the current environment +# (to either an empty or non-empty string), get its value and insert it +# in the variable definitions file on the line where that variable is +# defined. Note that # # ${!var_name+x} # # will retrun the string "x" if the variable specified in var_name is # set (to either an empty or non-empty string), and it will return an -# empty string if the variable specified in var_name is unset (i.e. un- -# defined). +# empty string if the variable specified in var_name is unset (i.e. i +# undefined). # - if [ ! -z ${!var_name+x} ]; then + unset "var_value" + if [ ! -z "${!var_name+x}" ]; then # # The variable may be a scalar or an array. Thus, we first treat it as # an array and obtain the number of elements that it contains. @@ -2670,17 +2689,17 @@ var_name = \"${var_name}\"" # # We will now set the variable var_value to the string that needs to be # placed on the right-hand side of the assignment operator (=) on the -# appropriate line in variable definitions file. How this is done de- -# pends on whether the variable is a scalar or an array. +# appropriate line in the variable definitions file. How this is done +# depends on whether the variable is a scalar or an array. # # If the variable contains only one element, then it is a scalar. (It -# could be a 1-element array, but it is simpler to treat it as a sca- -# lar.) In this case, we enclose its value in double quotes and save -# the result in var_value. +# could be a 1-element array, but for simplicity, we treat that case as +# a scalar.) In this case, we enclose its value in double quotes and +# save the result in var_value. # - if [ "$num_elems" -eq 1 ]; then - var_value="${!var_name}" - var_value="\"${var_value}\"" + if [ "${num_elems}" -eq 1 ]; then + + var_value=\""${!var_name}"\" # # If the variable contains more than one element, then it is an array. # In this case, we build var_value in two steps as follows: @@ -2690,40 +2709,30 @@ var_name = \"${var_name}\"" # # 2) Place parentheses around the double-quoted list of array elements # generated in the first step. Note that there is no need to put a -# space before the closing parenthesis because in step 1, we have al- -# ready placed a space after the last element. +# space before the closing parenthesis because in step 1, we have +# already placed a space after the last element. # else - arrays_on_one_line="TRUE" - arrays_on_one_line="FALSE" - - if [ "${arrays_on_one_line}" = "TRUE" ]; then - var_value=$(printf "\"%s\" " "${!array_name_at}") -# var_value=$(printf "\"%s\" \\\\\\ \\\n" "${!array_name_at}") - else -# var_value=$(printf "%s" "\\\\\\n") - var_value="\\\\\n" - for (( i=0; i<${num_elems}; i++ )); do -# var_value=$(printf "%s\"%s\" %s" "${var_value}" "${array[$i]}" "\\\\\\n") - var_value="${var_value}\"${array[$i]}\" \\\\\n" -# var_value="${var_value}\"${array[$i]}\" " - done - fi - var_value="( $var_value)" +# var_value=$(printf "\"%s\" " "${!array_name_at}") + var_value="${esc_nl_or_null}" + for (( i=0; i<${num_elems}; i++ )); do + var_value="${var_value}\"${array[$i]}\" ${esc_nl_or_null}" + done + var_value="( ${var_value})" fi # -# If the variable specified in var_name is not set in the current envi- -# ronment (to either an empty or non-empty string), get its value and -# insert it in the variable definitions file on the line where that va- -# riable is defined. +# If for some reason the variable specified in var_name is not set in +# the current environment (to either an empty or non-empty string), below +# we will still include it in the variable definitions file and simply +# set it to a null string. Thus, here, we set its value (var_value) to +# an empty string). In this case, we also issue an informational message. # else print_info_msg " -The variable specified by \"var_name\" is not set in the current envi- -ronment: +The variable specified by \"var_name\" is not set in the current environment: var_name = \"${var_name}\" Setting its value in the variable definitions file to an empty string." @@ -2731,14 +2740,25 @@ Setting its value in the variable definitions file to an empty string." fi # +# If +# + dollar_or_null=$( printf "%s" "${var_value}" | \ + $SED -n -r -e "s/[^\$]*(\$).*/\1/p" ) + if [ -z "${dollar_or_null}" ]; then +# # Now place var_value on the right-hand side of the assignment statement -# on the appropriate line in variable definitions file. +# on the appropriate line in the variable definitions file. # - set_file_param "${GLOBAL_VAR_DEFNS_FP}" "${var_name}" "${var_value}" + set_file_param "${GLOBAL_VAR_DEFNS_FP}" "${var_name}" "${var_value}" + else + template_var_names+=( "${var_name}" ) + template_var_values+=( "${var_value}" ) + $SED -i "/^${var_name}=/d" ${GLOBAL_VAR_DEFNS_FP} + fi # -# If var_name is empty, then a variable name was not found in the cur- -# rent line in line_list. In this case, print out a warning and move on -# to the next line. +# If var_name is empty, then a variable name was not found on the current +# line in line_list. In this case, print out a warning and move on to +# the next line. # else @@ -2988,20 +3008,17 @@ fi # #----------------------------------------------------------------------- # -# Because RUN_CMD_FCST can include PE_MEMBER01 (and theoretically other -# variables calculated in this script), delete the first occurrence of it -# in the var_defns file, and write it again at the end. -# -#----------------------------------------------------------------------- -$SED -i '/^RUN_CMD_FCST=/d' $GLOBAL_VAR_DEFNS_FP -# -#----------------------------------------------------------------------- -# # Continue appending variable definitions to the variable definitions # file. # #----------------------------------------------------------------------- # +lbc_spec_fcst_hrs_str=$(printf "\"%s\" ${esc_nl_or_null}" "${LBC_SPEC_FCST_HRS[@]}") +lbc_spec_fcst_hrs_str=$(printf "( ${esc_nl_or_null}%s${esc_nl_or_null:+\n})" "${lbc_spec_fcst_hrs_str}") + +all_cdates_str=$(printf "\"%s\" ${esc_nl_or_null}" "${ALL_CDATES[@]}") +all_cdates_str=$(printf "( ${esc_nl_or_null}%s${esc_nl_or_null:+\n})" "${all_cdates_str}") + { cat << EOM >> ${GLOBAL_VAR_DEFNS_FP} # #----------------------------------------------------------------------- @@ -3061,7 +3078,7 @@ EXTRN_MDL_LBCS_OFFSET_HRS="${EXTRN_MDL_LBCS_OFFSET_HRS}" # #----------------------------------------------------------------------- # -LBC_SPEC_FCST_HRS=(${LBC_SPEC_FCST_HRS[@]}) +LBC_SPEC_FCST_HRS=${lbc_spec_fcst_hrs_str} # #----------------------------------------------------------------------- # @@ -3071,9 +3088,7 @@ LBC_SPEC_FCST_HRS=(${LBC_SPEC_FCST_HRS[@]}) #----------------------------------------------------------------------- # NUM_CYCLES="${NUM_CYCLES}" -ALL_CDATES=( \\ -$( printf "\"%s\" \\\\\n" "${ALL_CDATES[@]}" ) -) +ALL_CDATES=${all_cdates_str} # #----------------------------------------------------------------------- # @@ -3095,7 +3110,6 @@ FVCOM_FILE="${FVCOM_FILE}" # NCORES_PER_NODE="${NCORES_PER_NODE}" PE_MEMBER01="${PE_MEMBER01}" -RUN_CMD_FCST=$(eval echo ${RUN_CMD_FCST}) # #----------------------------------------------------------------------- # @@ -3109,6 +3123,47 @@ EOM } || print_err_msg_exit "\ Heredoc (cat) command to append new variable definitions to variable definitions file returned with a nonzero status." + + + +str_to_insert=" +# +#----------------------------------------------------------------------- +#----------------------------------------------------------------------- +# Section 3: +# This section defines templates variables, i.e. variables that contain +# other (unexpanded) variables in their definitions. See above for +# variable definitions. +#----------------------------------------------------------------------- +#----------------------------------------------------------------------- +#" + +num_template_vars="${#template_var_names[@]}" +for (( i=0; i<${num_template_vars}; i++ )); do + str_to_insert=${str_to_insert}" +${template_var_names[$i]}=" +done + +{ cat << EOM >> ${GLOBAL_VAR_DEFNS_FP} +${str_to_insert} +EOM +} || print_err_msg_exit "\ +Heredoc (cat) command to append new variable definitions to variable +definitions file returned with a nonzero status." + +for (( i=0; i<${num_template_vars}; i++ )); do + set_file_param "${GLOBAL_VAR_DEFNS_FP}" \ + "${template_var_names[$i]}" "${template_var_values[$i]}" +done +# +#----------------------------------------------------------------------- +# +# +# +#----------------------------------------------------------------------- +# +print_info_msg "$VERBOSE" " +Done generating the global experiment variable definitions file." # #----------------------------------------------------------------------- # From fc59ea0e3dd564a74ec450e0cf4d8782e36bea05 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Fri, 19 Nov 2021 11:17:20 -0700 Subject: [PATCH 06/35] Modifications to when messages are printed to screen (VERBOSE, DEBUG). --- ush/generate_FV3LAM_wflow.sh | 95 +++++++++++++++++++++--------------- ush/set_ozone_param.sh | 2 +- ush/setup.sh | 33 ++++++++++--- 3 files changed, 81 insertions(+), 49 deletions(-) diff --git a/ush/generate_FV3LAM_wflow.sh b/ush/generate_FV3LAM_wflow.sh index 0ee8eee0b..a9fc3a817 100755 --- a/ush/generate_FV3LAM_wflow.sh +++ b/ush/generate_FV3LAM_wflow.sh @@ -10,6 +10,15 @@ #----------------------------------------------------------------------- # function generate_FV3LAM_wflow() { +printf "\ +======================================================================== +======================================================================== + +Starting experiment generation... + +======================================================================== +======================================================================== +" # #----------------------------------------------------------------------- # @@ -20,10 +29,10 @@ function generate_FV3LAM_wflow() { #----------------------------------------------------------------------- # if [[ $(uname -s) == Darwin ]]; then - local scrfunc_fp=$( greadlink -f "${BASH_SOURCE[0]}" ) - else - local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) - fi + local scrfunc_fp=$( greadlink -f "${BASH_SOURCE[0]}" ) +else + local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +fi local scrfunc_fn=$( basename "${scrfunc_fp}" ) local scrfunc_dir=$( dirname "${scrfunc_fp}" ) # @@ -108,7 +117,6 @@ if [ $pyerrors -gt 0 ];then " fi - # #----------------------------------------------------------------------- # @@ -152,16 +160,26 @@ WFLOW_XML_FP="$EXPTDIR/${WFLOW_XML_FN}" # #----------------------------------------------------------------------- # -ensmem_indx_name="\"\"" -uscore_ensmem_name="\"\"" -slash_ensmem_subdir="\"\"" -if [ "${DO_ENSEMBLE}" = "TRUE" ]; then - ensmem_indx_name="mem" - uscore_ensmem_name="_mem#${ensmem_indx_name}#" - slash_ensmem_subdir="/mem#${ensmem_indx_name}#" -fi +if [ "${WORKFLOW_MANAGER}" = "rocoto" ]; then -settings="\ + template_xml_fp="${TEMPLATE_DIR}/${WFLOW_XML_FN}" + + print_info_msg " +Creating rocoto workflow XML file (WFLOW_XML_FP) from jinja template XML +file (template_xml_fp): + template_xml_fp = \"${template_xml_fp}\" + WFLOW_XML_FP = \"${WFLOW_XML_FP}\"" + + ensmem_indx_name="\"\"" + uscore_ensmem_name="\"\"" + slash_ensmem_subdir="\"\"" + if [ "${DO_ENSEMBLE}" = "TRUE" ]; then + ensmem_indx_name="mem" + uscore_ensmem_name="_mem#${ensmem_indx_name}#" + slash_ensmem_subdir="/mem#${ensmem_indx_name}#" + fi + + settings="\ # # Parameters needed by the job scheduler. # @@ -415,7 +433,7 @@ settings="\ 'first_fv3_file_tstr': "000:"`$DATE_UTIL -d "${DATE_FIRST_CYCL} +${DT_ATMOS} seconds" +%M:%S` " # End of "settings" variable. -print_info_msg $VERBOSE " + print_info_msg "$VERBOSE" " The variable \"settings\" specifying values of the rococo XML variables has been set as follows: #----------------------------------------------------------------------- @@ -423,12 +441,9 @@ settings = $settings" # -# Set the full path to the template rocoto XML file. Then call a python -# script to generate the experiment's actual XML file from this template -# file. +# Call the python script to generate the experiment's actual XML file +# from the jinja template file. # -if [ "${WORKFLOW_MANAGER}" = "rocoto" ]; then - template_xml_fp="${TEMPLATE_DIR}/${WFLOW_XML_FN}" $USHDIR/fill_jinja_template.py -q \ -u "${settings}" \ -t ${template_xml_fp} \ @@ -444,6 +459,7 @@ are: Namelist settings specified on command line: settings = $settings" + fi # #----------------------------------------------------------------------- @@ -453,7 +469,7 @@ fi # #----------------------------------------------------------------------- # -print_info_msg " +print_info_msg "$VERBOSE" " Creating symlink in the experiment directory (EXPTDIR) that points to the workflow launch script (WFLOW_LAUNCH_SCRIPT_FP): EXPTDIR = \"${EXPTDIR}\" @@ -476,7 +492,7 @@ if [ "${USE_CRON_TO_RELAUNCH}" = "TRUE" ]; then # time_stamp=$( $DATE_UTIL "+%F_%T" ) crontab_backup_fp="$EXPTDIR/crontab.bak.${time_stamp}" - print_info_msg " + print_info_msg "$VERBOSE" " Copying contents of user cron table to backup file: crontab_backup_fp = \"${crontab_backup_fp}\"" if [ "$MACHINE" = "WCOSS_DELL_P3" ]; then @@ -522,9 +538,9 @@ added: else - print_info_msg " -Adding the following line to the cron table in order to automatically -resubmit FV3-LAM workflow: + print_info_msg "$VERBOSE" " +Adding the following line to the user's cron table in order to automatically +resubmit SRW workflow: CRONTAB_LINE = \"${CRONTAB_LINE}\"" if [ "$MACHINE" = "WCOSS_DELL_P3" ];then @@ -637,8 +653,8 @@ cp_vrfy "${FIELD_DICT_IN_UWM_FP}" "${FIELD_DICT_FP}" # #----------------------------------------------------------------------- # -print_info_msg "$VERBOSE" " -Setting parameters in FV3 namelist file (FV3_NML_FP): +print_info_msg " +Setting parameters in weather model's namelist file (FV3_NML_FP): FV3_NML_FP = \"${FV3_NML_FP}\"" # # Set npx and npy, which are just NX plus 1 and NY plus 1, respectively. @@ -800,8 +816,8 @@ settings="$settings }" print_info_msg $VERBOSE " -The variable \"settings\" specifying values of the namelist variables -has been set as follows: +The variable \"settings\" specifying values of the weather model's +namelist variables has been set as follows: settings = $settings" @@ -886,15 +902,12 @@ print_info_msg " ======================================================================== ======================================================================== -Workflow generation completed. +Experiment generation completed. The experiment directory is: + + EXPTDIR=\"$EXPTDIR\" ======================================================================== ======================================================================== - -The experiment directory is: - - > EXPTDIR=\"$EXPTDIR\" - " # #----------------------------------------------------------------------- @@ -904,12 +917,16 @@ The experiment directory is: #----------------------------------------------------------------------- # if [ "${WORKFLOW_MANAGER}" = "rocoto" ]; then + print_info_msg "\ To launch the workflow, first ensure that you have a compatible version of rocoto available. For most pre-configured platforms, rocoto can be loaded via a module: + > module load rocoto + For more details on rocoto, see the User's Guide. + To launch the workflow, first ensure that you have a compatible version of rocoto loaded. For example, to load version 1.3.1 of rocoto, use @@ -938,17 +955,15 @@ Note that: 2) In order for the output of the rocotostat command to be up-to-date, the rocotorun command must be issued immediately before the rocoto- stat command. -" -fi -print_info_msg " + For automatic resubmission of the workflow (say every 3 minutes), the following line can be added to the user's crontab (use \"crontab -e\" to edit the cron table): */3 * * * * cd $EXPTDIR && ./launch_FV3LAM_wflow.sh - -Done. " + +fi # # If necessary, run the NOMADS script to source external model data. # diff --git a/ush/set_ozone_param.sh b/ush/set_ozone_param.sh index 22abe3371..bd6f5fd37 100644 --- a/ush/set_ozone_param.sh +++ b/ush/set_ozone_param.sh @@ -209,7 +209,7 @@ files in the FIXam directory is: " msg="$msg"$( printf "\"%s\" \\\\\n" "${CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING[@]}" ) msg="$msg"$( printf "\n)" ) - print_info_msg "$msg" + print_info_msg "$VERBOSE" "$msg" else diff --git a/ush/setup.sh b/ush/setup.sh index a633f98b4..49c7a3c2d 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -51,6 +51,11 @@ cd_vrfy ${scrfunc_dir} #----------------------------------------------------------------------- # . ./source_util_funcs.sh + +print_info_msg " +======================================================================== +Starting function ${func_name}() in \"${scrfunc_fn}\"... +========================================================================" # #----------------------------------------------------------------------- # @@ -172,11 +177,15 @@ fi # #----------------------------------------------------------------------- # -# If DEBUG is set to "TRUE", make sure that VERBOSE is also set to "TRUE". +# If DEBUG is set to "TRUE" but VERBOSE is set to "FALSE", reset VERBOSE +# to "TRUE" to print out all of the VERBOSE output (in addition to any +# DEBUG output). # #----------------------------------------------------------------------- # -if [ "$DEBUG" = "TRUE" ]; then +if [ "$DEBUG" = "TRUE" ] && [ "$VERBOSE" = "FALSE" ]; then + print_info_msg " +Resetting VERBOSE to \"TRUE\" because DEBUG has been set to \"TRUE\"..." VERBOSE="TRUE" fi # @@ -2580,8 +2589,8 @@ line_list=$( $SED -r \ ${GLOBAL_VAR_DEFNS_FP} ) print_info_msg "$DEBUG" " -Before updating default values of experiment variables, the variable -\"line_list\" contains: +Before updating default values of experiment variables to user-specified +values, the variable \"line_list\" contains: ${line_list} " @@ -2624,6 +2633,17 @@ $SED -i -r -e "s|$regexp|\1\n\n${str_to_insert}\n|g" ${GLOBAL_VAR_DEFNS_FP} # # Loop through the lines in line_list. # +print_info_msg " +Generating the global experiment variable definitions file specified by +GLOBAL_VAR_DEFNS_FN: + GLOBAL_VAR_DEFNS_FN = \"${GLOBAL_VAR_DEFNS_FN}\" +Full path to this file is: + GLOBAL_VAR_DEFNS_FP = \"${GLOBAL_VAR_DEFNS_FP}\" +For more detailed information, set DEBUG to \"TRUE\" in the experiment +configuration file (\"${EXPT_CONFIG_FN}\")." + +template_var_names=() +template_var_values=() while read crnt_line; do # # Try to obtain the name of the variable being set on the current line. @@ -2635,9 +2655,6 @@ while read crnt_line; do # set to. # var_name=$( printf "%s" "${crnt_line}" | $SED -n -r -e "s/^([^ ]*)=.*/\1/p" ) -#echo -#echo "============================" -#printf "%s\n" "var_name = \"${var_name}\"" # # If var_name is not empty, then a variable name was found in the cur- # rent line in line_list. @@ -3118,7 +3135,7 @@ definitions file returned with a nonzero status." # print_info_msg " ======================================================================== -Setup script completed successfully!!! +Function ${func_name}() in \"${scrfunc_fn}\" completed successfully!!! ========================================================================" # #----------------------------------------------------------------------- From 45c6ffb10b06eefb66b1944f6b438791ce914da9 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Fri, 19 Nov 2021 15:56:08 -0700 Subject: [PATCH 07/35] Change "set -x" to "set +x" in j-jobs to make output less compact. --- jobs/JREGIONAL_GET_OBS_CCPA | 2 +- jobs/JREGIONAL_GET_OBS_MRMS | 2 +- jobs/JREGIONAL_GET_OBS_NDAS | 2 +- jobs/JREGIONAL_MAKE_ICS | 2 +- jobs/JREGIONAL_RUN_POST | 2 +- jobs/JREGIONAL_RUN_VX_ENSGRID | 2 +- jobs/JREGIONAL_RUN_VX_ENSGRID_MEAN | 2 +- jobs/JREGIONAL_RUN_VX_ENSGRID_PROB | 2 +- jobs/JREGIONAL_RUN_VX_ENSPOINT | 2 +- jobs/JREGIONAL_RUN_VX_ENSPOINT_MEAN | 2 +- jobs/JREGIONAL_RUN_VX_ENSPOINT_PROB | 2 +- jobs/JREGIONAL_RUN_VX_GRIDSTAT | 2 +- jobs/JREGIONAL_RUN_VX_POINTSTAT | 2 +- 13 files changed, 13 insertions(+), 13 deletions(-) diff --git a/jobs/JREGIONAL_GET_OBS_CCPA b/jobs/JREGIONAL_GET_OBS_CCPA index e63010d0d..bd1760ee0 100755 --- a/jobs/JREGIONAL_GET_OBS_CCPA +++ b/jobs/JREGIONAL_GET_OBS_CCPA @@ -29,7 +29,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u -x; } > /dev/null 2>&1 +{ save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_GET_OBS_MRMS b/jobs/JREGIONAL_GET_OBS_MRMS index dfe1f68a6..f5d634974 100755 --- a/jobs/JREGIONAL_GET_OBS_MRMS +++ b/jobs/JREGIONAL_GET_OBS_MRMS @@ -25,7 +25,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u -x; } > /dev/null 2>&1 +{ save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_GET_OBS_NDAS b/jobs/JREGIONAL_GET_OBS_NDAS index 9514931cc..ab44e21b8 100755 --- a/jobs/JREGIONAL_GET_OBS_NDAS +++ b/jobs/JREGIONAL_GET_OBS_NDAS @@ -25,7 +25,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u -x; } > /dev/null 2>&1 +{ save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_MAKE_ICS b/jobs/JREGIONAL_MAKE_ICS index 9c5125f41..a5e5894f1 100755 --- a/jobs/JREGIONAL_MAKE_ICS +++ b/jobs/JREGIONAL_MAKE_ICS @@ -17,7 +17,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u -x; } > /dev/null 2>&1 +{ save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_RUN_POST b/jobs/JREGIONAL_RUN_POST index ed469dbe2..86a30470b 100755 --- a/jobs/JREGIONAL_RUN_POST +++ b/jobs/JREGIONAL_RUN_POST @@ -26,7 +26,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u -x; } > /dev/null 2>&1 +{ save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_RUN_VX_ENSGRID b/jobs/JREGIONAL_RUN_VX_ENSGRID index 641df6f66..867c75fc8 100755 --- a/jobs/JREGIONAL_RUN_VX_ENSGRID +++ b/jobs/JREGIONAL_RUN_VX_ENSGRID @@ -26,7 +26,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u -x; } > /dev/null 2>&1 +{ save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_RUN_VX_ENSGRID_MEAN b/jobs/JREGIONAL_RUN_VX_ENSGRID_MEAN index d8c4dcea0..a75558d92 100755 --- a/jobs/JREGIONAL_RUN_VX_ENSGRID_MEAN +++ b/jobs/JREGIONAL_RUN_VX_ENSGRID_MEAN @@ -26,7 +26,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u -x; } > /dev/null 2>&1 +{ save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_RUN_VX_ENSGRID_PROB b/jobs/JREGIONAL_RUN_VX_ENSGRID_PROB index 7ae543314..a8a2c43be 100755 --- a/jobs/JREGIONAL_RUN_VX_ENSGRID_PROB +++ b/jobs/JREGIONAL_RUN_VX_ENSGRID_PROB @@ -26,7 +26,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u -x; } > /dev/null 2>&1 +{ save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_RUN_VX_ENSPOINT b/jobs/JREGIONAL_RUN_VX_ENSPOINT index c240d7abe..fc30f076f 100755 --- a/jobs/JREGIONAL_RUN_VX_ENSPOINT +++ b/jobs/JREGIONAL_RUN_VX_ENSPOINT @@ -24,7 +24,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u -x; } > /dev/null 2>&1 +{ save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_RUN_VX_ENSPOINT_MEAN b/jobs/JREGIONAL_RUN_VX_ENSPOINT_MEAN index 376d07b99..a7d937c12 100755 --- a/jobs/JREGIONAL_RUN_VX_ENSPOINT_MEAN +++ b/jobs/JREGIONAL_RUN_VX_ENSPOINT_MEAN @@ -24,7 +24,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u -x; } > /dev/null 2>&1 +{ save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_RUN_VX_ENSPOINT_PROB b/jobs/JREGIONAL_RUN_VX_ENSPOINT_PROB index 1a47cfd12..818e588e5 100755 --- a/jobs/JREGIONAL_RUN_VX_ENSPOINT_PROB +++ b/jobs/JREGIONAL_RUN_VX_ENSPOINT_PROB @@ -24,7 +24,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u -x; } > /dev/null 2>&1 +{ save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_RUN_VX_GRIDSTAT b/jobs/JREGIONAL_RUN_VX_GRIDSTAT index b17fd2960..8537cb3ec 100755 --- a/jobs/JREGIONAL_RUN_VX_GRIDSTAT +++ b/jobs/JREGIONAL_RUN_VX_GRIDSTAT @@ -26,7 +26,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u -x; } > /dev/null 2>&1 +{ save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_RUN_VX_POINTSTAT b/jobs/JREGIONAL_RUN_VX_POINTSTAT index f995045ae..084543695 100755 --- a/jobs/JREGIONAL_RUN_VX_POINTSTAT +++ b/jobs/JREGIONAL_RUN_VX_POINTSTAT @@ -24,7 +24,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u -x; } > /dev/null 2>&1 +{ save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # From a966685665390d4ffd2e2e1b81bb37b9af7aea31 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Mon, 29 Nov 2021 12:31:44 -0700 Subject: [PATCH 08/35] Fix typo. --- ush/setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/setup.sh b/ush/setup.sh index 674628541..eb8752ecf 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -2683,7 +2683,7 @@ var_name = \"${var_name}\"" # # will retrun the string "x" if the variable specified in var_name is # set (to either an empty or non-empty string), and it will return an -# empty string if the variable specified in var_name is unset (i.e. i +# empty string if the variable specified in var_name is unset (i.e. is # undefined). # unset "var_value" From 52a1c9e5859c456a483527fa30344c5ddec8f2c9 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Thu, 9 Dec 2021 11:17:33 -0600 Subject: [PATCH 09/35] Set the variables BUILD_ENV_FN and WFLOW_ENV_FN in setup.sh (instead of launch_FV3LAM_wflow.sh and load_modules_run_task.sh) so that the correct values get recorded to the experiment's var_defns.sh file. --- ush/launch_FV3LAM_wflow.sh | 10 ++++------ ush/load_modules_run_task.sh | 3 +-- ush/setup.sh | 13 +++++++++++++ 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/ush/launch_FV3LAM_wflow.sh b/ush/launch_FV3LAM_wflow.sh index 76c964083..87a15dfbb 100755 --- a/ush/launch_FV3LAM_wflow.sh +++ b/ush/launch_FV3LAM_wflow.sh @@ -106,14 +106,12 @@ expt_name="${EXPT_SUBDIR}" # #----------------------------------------------------------------------- # -machine=$(echo_lowercase $MACHINE) -env_fn=${WFLOW_ENV_FN:-"wflow_${machine}.env"} -env_fp="${SR_WX_APP_TOP_DIR}/env/${env_fn}" +env_fp="${SR_WX_APP_TOP_DIR}/env/${WFLOW_ENV_FN}" module purge source "${env_fp}" || print_err_msg_exit "\ - Sourcing platform-specific environment file (env_fp) for -the workflow task failed : -env_fp = \"${env_fp}\"" +Sourcing platform-specific environment file (env_fp) for the workflow +task failed: + env_fp = \"${env_fp}\"" # #----------------------------------------------------------------------- # diff --git a/ush/load_modules_run_task.sh b/ush/load_modules_run_task.sh index 1f0d88126..17b6d5d3d 100755 --- a/ush/load_modules_run_task.sh +++ b/ush/load_modules_run_task.sh @@ -135,8 +135,7 @@ jjob_fp="$2" module purge machine=$(echo_lowercase $MACHINE) -env_fn=${BUILD_ENV_FN:-"build_${machine}_${COMPILER}.env"} -env_fp="${SR_WX_APP_TOP_DIR}/env/${env_fn}" +env_fp="${SR_WX_APP_TOP_DIR}/env/${BUILD_ENV_FN}" source "${env_fp}" || print_err_msg_exit "\ Sourcing platform- and compiler-specific environment file (env_fp) for the workflow task specified by task_name failed: diff --git a/ush/setup.sh b/ush/setup.sh index 674628541..e9a1de19b 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -698,6 +698,19 @@ fi # #----------------------------------------------------------------------- # +# Set the names of the build and workflow environment files (if not +# already specified by the user). These are the files that need to be +# sourced before building the component SRW App codes and running various +# workflow scripts, respectively. +# +#----------------------------------------------------------------------- +# +machine=$(echo_lowercase ${MACHINE}) +WFLOW_ENV_FN=${WFLOW_ENV_FN:-"wflow_${machine}.env"} +BUILD_ENV_FN=${BUILD_ENV_FN:-"build_${machine}_${COMPILER}.env"} +# +#----------------------------------------------------------------------- +# # Calculate PPN_RUN_FCST from NCORES_PER_NODE and OMP_NUM_THREADS_RUN_FCST # #----------------------------------------------------------------------- From 4beda25cfab2f54190a24bb35975b2672a1d8039 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Thu, 9 Dec 2021 11:19:14 -0600 Subject: [PATCH 10/35] Rewrite the way the variable definitions file (var_defns.sh) is created to allow for template variables, i.e. variables that are defined in terms of other variables in config_defaults.sh and config.sh, e.g. MY_VAR="${ANOTHER_VAR}". --- ush/setup.sh | 702 ++++++++++++++++++++++++--------------------------- 1 file changed, 335 insertions(+), 367 deletions(-) diff --git a/ush/setup.sh b/ush/setup.sh index e9a1de19b..c86935e61 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -871,32 +871,32 @@ fi # #----------------------------------------------------------------------- # -CYCL_HRS_str=$(printf "\"%s\" " "${CYCL_HRS[@]}") -CYCL_HRS_str="( $CYCL_HRS_str)" +cycl_hrs_str=$(printf "\"%s\" " "${CYCL_HRS[@]}") +cycl_hrs_str="( ${cycl_hrs_str})" i=0 -for CYCL in "${CYCL_HRS[@]}"; do +for cycl_hr in "${CYCL_HRS[@]}"; do - CYCL_OR_NULL=$( printf "%s" "$CYCL" | $SED -n -r -e "s/^([0-9]{2})$/\1/p" ) + cycl_hr_or_null=$( printf "%s" "${cycl_hr}" | $SED -n -r -e "s/^([0-9]{2})$/\1/p" ) - if [ -z "${CYCL_OR_NULL}" ]; then + if [ -z "${cycl_hr_or_null}" ]; then print_err_msg_exit "\ Each element of CYCL_HRS must be a string consisting of exactly 2 digits -(including a leading \"0\", if necessary) specifying an hour-of-day. Ele- -ment #$i of CYCL_HRS (where the index of the first element is 0) does not -have this form: - CYCL_HRS = $CYCL_HRS_str +(including a leading \"0\", if necessary) specifying an hour-of-day. +Element #$i of CYCL_HRS (where the index of the first element is 0) does +not have this form: + CYCL_HRS = ${cycl_hrs_str} CYCL_HRS[$i] = \"${CYCL_HRS[$i]}\"" fi - if [ "${CYCL_OR_NULL}" -lt "0" ] || \ - [ "${CYCL_OR_NULL}" -gt "23" ]; then + if [ "${cycl_hr_or_null}" -lt "0" ] || \ + [ "${cycl_hr_or_null}" -gt "23" ]; then print_err_msg_exit "\ -Each element of CYCL_HRS must be an integer between \"00\" and \"23\", in- -clusive (including a leading \"0\", if necessary), specifying an hour-of- -day. Element #$i of CYCL_HRS (where the index of the first element is 0) -does not have this form: - CYCL_HRS = $CYCL_HRS_str +Each element of CYCL_HRS must be an integer between \"00\" and \"23\", +inclusive (including a leading \"0\", if necessary), specifying an hour- +of-day. Element #$i of CYCL_HRS (where the index of the first element +is 0) does not have this form: + CYCL_HRS = ${cycl_hrs_str} CYCL_HRS[$i] = \"${CYCL_HRS[$i]}\"" fi @@ -917,7 +917,7 @@ done set_cycle_dates \ date_start="${DATE_FIRST_CYCL}" \ date_end="${DATE_LAST_CYCL}" \ - cycle_hrs="${CYCL_HRS_str}" \ + cycle_hrs="${cycl_hrs_str}" \ output_varname_all_cdates="ALL_CDATES" NUM_CYCLES="${#ALL_CDATES[@]}" @@ -1055,7 +1055,7 @@ One or more fix file directories have not been specified for this machine: TOPO_DIR = \"${TOPO_DIR:-\"\"} SFC_CLIMO_INPUT_DIR = \"${SFC_CLIMO_INPUT_DIR:-\"\"} FIXLAM_NCO_BASEDIR = \"${FIXLAM_NCO_BASEDIR:-\"\"} -You can specify the missing location(s) in config.sh" +You can specify the missing location(s) in ${EXPT_CONFIG_FN}." fi ;; @@ -1145,7 +1145,7 @@ check_var_valid_value \ # Set USE_CUSTOM_POST_CONFIG_FILE to either "TRUE" or "FALSE" so we don't # have to consider other valid values later on. # -USE_CUSTOM_POST_CONFIG_FILE=$(echo_uppercase $USE_CUSTOM_POST_CONFIG_FILE) +USE_CUSTOM_POST_CONFIG_FILE=$(echo_uppercase ${USE_CUSTOM_POST_CONFIG_FILE}) if [ "$USE_CUSTOM_POST_CONFIG_FILE" = "TRUE" ] || \ [ "$USE_CUSTOM_POST_CONFIG_FILE" = "YES" ]; then USE_CUSTOM_POST_CONFIG_FILE="TRUE" @@ -2113,7 +2113,6 @@ fi #----------------------------------------------------------------------- # . ./set_extrn_mdl_params.sh - # #----------------------------------------------------------------------- # @@ -2468,7 +2467,6 @@ fi #----------------------------------------------------------------------- # NNODES_RUN_FCST=$(( (PE_MEMBER01 + PPN_RUN_FCST - 1)/PPN_RUN_FCST )) - # #----------------------------------------------------------------------- # @@ -2516,130 +2514,89 @@ set_thompson_mp_fix_files \ # #----------------------------------------------------------------------- # -# Generate the shell script that will appear in the experiment directory -# (EXPTDIR) and will contain definitions of variables needed by the va- -# rious scripts in the workflow. We refer to this as the experiment/ -# workflow global variable definitions file. We will create this file -# by: -# -# 1) Copying the default workflow/experiment configuration file (speci- -# fied by EXPT_DEFAULT_CONFIG_FN and located in the shell script di- -# rectory specified by USHDIR) to the experiment directory and rena- -# ming it to the name specified by GLOBAL_VAR_DEFNS_FN. -# -# 2) Resetting the default variable values in this file to their current -# values. This is necessary because these variables may have been -# reset by the user-specified configuration file (if one exists in -# USHDIR) and/or by this setup script, e.g. because predef_domain is -# set to a valid non-empty value. -# -# 3) Appending to the variable definitions file any new variables intro- -# duced in this setup script that may be needed by the scripts that -# perform the various tasks in the workflow (and which source the va- -# riable defintions file). -# -# First, set the full path to the variable definitions file and copy the -# default configuration script into it. +# Set the full path to the experiment's variable definitions file. This +# file will contain definitions of variables (in bash syntax) needed by +# the various scripts in the workflow. # #----------------------------------------------------------------------- # -GLOBAL_VAR_DEFNS_FP="$EXPTDIR/$GLOBAL_VAR_DEFNS_FN" -cp_vrfy $USHDIR/${EXPT_DEFAULT_CONFIG_FN} ${GLOBAL_VAR_DEFNS_FP} +GLOBAL_VAR_DEFNS_FP="$EXPTDIR/${GLOBAL_VAR_DEFNS_FN}" # #----------------------------------------------------------------------- # -# +# Get the list of primary experiment variables and their default values +# from the default experiment configuration file (EXPT_DEFAULT_CONFIG_FN). +# By "primary", we mean those variables that are defined in the default +# configuration file and can be reset in the user-specified experiment +# configuration file (EXPT_CONFIG_FN). The default values will be +# updated below to user-specified ones and the result saved in the +# experiment's variable definitions file. # #----------------------------------------------------------------------- # - -# Read all lines of GLOBAL_VAR_DEFNS file into the variable line_list. -line_list=$( $SED -r -e "s/(.*)/\1/g" ${GLOBAL_VAR_DEFNS_FP} ) -# -# Loop through the lines in line_list and concatenate lines ending with -# the line bash continuation character "\". -# -rm_vrfy ${GLOBAL_VAR_DEFNS_FP} -while read crnt_line; do - printf "%s\n" "${crnt_line}" >> ${GLOBAL_VAR_DEFNS_FP} -done <<< "${line_list}" -# -#----------------------------------------------------------------------- -# -# The following comment block needs to be updated because now line_list -# may contain lines that are not assignment statements (e.g. it may con- -# tain if-statements). Such lines are ignored in the while-loop below. +print_info_msg " +Creating list of default experiment variable definitions..." # -# Reset each of the variables in the variable definitions file to its -# value in the current environment. To accomplish this, we: +# Read in all lines in the default experiment configuration file into the +# variable default_config_file_contents. In doing so: # -# 1) Create a list of variable settings by stripping out comments, blank -# lines, extraneous leading whitespace, etc from the variable defini- -# tions file (which is currently identical to the default workflow/ -# experiment configuration script) and saving the result in the vari- -# able line_list. Each line of line_list will have the form +# 1) Concatenate any line ending with the bash line continuation character +# (a backslash) with the following line. # -# VAR=... +# 2) Remove any leading and trailing whitespace. # -# where the VAR is a variable name and ... is the value from the de- -# fault configuration script (which does not necessarily correspond -# to the current value of the variable). +# Note that these two actions are automatically performed by the "read" +# utility in the while-loop below. # -# 2) Loop through each line of line_list. For each line, we extract the -# variable name (and save it in the variable var_name), get its value -# from the current environment (using bash indirection, i.e. -# ${!var_name}), and use the set_file_param() function to replace the -# value of the variable in the variable definitions script (denoted -# above by ...) with its current value. +default_config_file_contents=$( $SED -r -e "s/(.*)/\1/g" $USHDIR/${EXPT_DEFAULT_CONFIG_FN} ) +tmp="" +while read crnt_line; do + tmp="$tmp${crnt_line} +" +done <<< "${default_config_file_contents}" +default_config_file_contents="$tmp" # -#----------------------------------------------------------------------- +# Generate a variable (default_var_defns) containing a list of experiment +# variable definitions by stripping out any comment and empty lines from +# default_config_file_contents. # -# Also should remove trailing whitespace... -line_list=$( $SED -r \ - -e "s/^([ ]*)([^ ]+.*)/\2/g" \ - -e "/^#.*/d" \ - -e "/^$/d" \ - ${GLOBAL_VAR_DEFNS_FP} ) +default_var_defns=$( printf "${default_config_file_contents}" | \ + $SED -r -e "/^#.*/d" `# Remove comment lines.` \ + -e "/^$/d" `# Remove empty lines.` \ + ) print_info_msg "$DEBUG" " -Before updating default values of experiment variables to user-specified -values, the variable \"line_list\" contains: +The variable \"default_var_defns\" containing default values of primary +experiment variables is set as follows: -${line_list} +${default_var_defns} " # #----------------------------------------------------------------------- # -# Add a comment at the beginning of the variable definitions file that -# indicates that the first section of that file is (mostly) the same as -# the configuration file. -# -#----------------------------------------------------------------------- -# -read -r -d '' str_to_insert << EOM +# Create lists of primary experiment variable definitions containing +# updated values. By "updated", we mean non-default values. Values +# may have been updated due to the presence of user-specified values in +# the experiment configuration file (EXPT_CONFIG_FN) or due to other +# considerations (e.g. resetting depending on the platform the App is +# running on). # -#----------------------------------------------------------------------- -#----------------------------------------------------------------------- -# Section 1: -# This section is a copy of the default experiment configuration file -# (${EXPT_DEFAULT_CONFIG_FN}) in the shell scripts directory specified by USHDIR -# except that variable values have been updated to those for the experiment -# (as opposed to the default values). -#----------------------------------------------------------------------- -#----------------------------------------------------------------------- -# -EOM +# Note that we generate two lists: var_defns_notempl and var_defns_templ. +# var_defns_templ is for template variables, i.e. variables that contain +# references to other variables in their definitions, e.g. # -# Replace all occurrences of actual newlines in the variable str_to_insert -# with escaped backslash-n. This is needed for the sed command below to -# work properly (i.e. to avoid it failing with an "unterminated `s' command" -# error message). +# MY_VAR=\"\${ANOTHER_VAR}\" # -str_to_insert=${str_to_insert//$'\n'/\\n} +# while var_defns_notempl is for variables whose definitions contain +# only a literal string. These two types are treated separately +# because the template variables must be written at the end of the +# variable defintions file. This is in order to ensure that the +# (non-template) variables they reference have already been defined. # -# Insert str_to_insert at the top of the file GLOBAL_VAR_DEFNS_FP. +#----------------------------------------------------------------------- # -$SED -i -r -e "1i${str_to_insert}\n" ${GLOBAL_VAR_DEFNS_FP} +print_info_msg " +Creating lists of (updated) experiment variable definitions..." # # Set the flag that specifies whether or not array variables will be # recorded in the variable definitions file on one line or one element @@ -2650,37 +2607,38 @@ $SED -i -r -e "1i${str_to_insert}\n" ${GLOBAL_VAR_DEFNS_FP} # multiline_arrays="TRUE" #multiline_arrays="FALSE" -esc_nl_or_null="" +escbksl_nl_or_null="" if [ "${multiline_arrays}" = "TRUE" ]; then - esc_nl_or_null='\\\n' -fi -# -# Loop through the lines in line_list. -# -print_info_msg " -Generating the global experiment variable definitions file specified by -GLOBAL_VAR_DEFNS_FN: - GLOBAL_VAR_DEFNS_FN = \"${GLOBAL_VAR_DEFNS_FN}\" -Full path to this file is: - GLOBAL_VAR_DEFNS_FP = \"${GLOBAL_VAR_DEFNS_FP}\" -For more detailed information, set DEBUG to \"TRUE\" in the experiment -configuration file (\"${EXPT_CONFIG_FN}\")." - -template_var_names=() -template_var_values=() + escbksl_nl_or_null='\\\n' +fi +# +# Loop through the lines in default_var_defns. Reset the value of the +# variable on each line to the updated value (e.g. to a user-specified +# value, as opposed to the default value). Save the updated list of +# variables and values either in var_defns_notempl (if it is not a +# template variable, i.e. one that references other variables in its +# definition) or in var_defns_templ (if it is a template variable). +# Note that template variables are separated from non-template ones +# because they must be written at the end of the variable defintions +# file to ensure that the variables they reference have already been +# defined. +# +var_defns_notempl="" +var_defns_templ="" while read crnt_line; do # # Try to obtain the name of the variable being set on the current line. -# This will be successful only if the line consists of one or more char- -# acters representing the name of a variable (recall that in generating -# the variable line_list, leading spaces on each line were stripped out), -# followed by an equal sign, followed by zero or more characters -# representing the value that the variable is being set to. +# This will be successful only if the line consists of one or more non- +# whitespace characters representing the name of a variable followed by +# an equal sign, followed by zero or more characters representing the +# value that the variable is being set to. (Recall that in generating +# the variable default_var_defns, leading spaces on each line were +# stripped out). # var_name=$( printf "%s" "${crnt_line}" | $SED -n -r -e "s/^([^ ]*)=.*/\1/p" ) # # If var_name is not empty, then a variable name was found on the current -# line in line_list. +# line in default_var_defns. # if [ ! -z ${var_name} ]; then @@ -2688,16 +2646,15 @@ while read crnt_line; do var_name = \"${var_name}\"" # # If the variable specified in var_name is set in the current environment -# (to either an empty or non-empty string), get its value and insert it -# in the variable definitions file on the line where that variable is -# defined. Note that +# (to either an empty or non-empty string), get its value and save it in +# var_value. Note that # # ${!var_name+x} # # will retrun the string "x" if the variable specified in var_name is # set (to either an empty or non-empty string), and it will return an -# empty string if the variable specified in var_name is unset (i.e. i -# undefined). +# empty string if the variable specified in var_name is unset (i.e. if +# it is undefined). # unset "var_value" if [ ! -z "${!var_name+x}" ]; then @@ -2709,10 +2666,9 @@ var_name = \"${var_name}\"" array=("${!array_name_at}") num_elems="${#array[@]}" # -# We will now set the variable var_value to the string that needs to be -# placed on the right-hand side of the assignment operator (=) on the -# appropriate line in the variable definitions file. How this is done -# depends on whether the variable is a scalar or an array. +# Set var_value to the updated value of the current experiment variable. +# How this is done depends on whether the variable is a scalar or an +# array. # # If the variable contains only one element, then it is a scalar. (It # could be a 1-element array, but for simplicity, we treat that case as @@ -2727,19 +2683,20 @@ var_name = \"${var_name}\"" # In this case, we build var_value in two steps as follows: # # 1) Generate a string containing each element of the array in double -# quotes and followed by a space. +# quotes and followed by a space (and followed by an optional backslash +# and newline if multiline_arrays has been set to "TRUE"). # # 2) Place parentheses around the double-quoted list of array elements # generated in the first step. Note that there is no need to put a -# space before the closing parenthesis because in step 1, we have -# already placed a space after the last element. +# space before the closing parenthesis because during step 1 above, +# a space has already been placed after the last array element. # else -# var_value=$(printf "\"%s\" " "${!array_name_at}") - var_value="${esc_nl_or_null}" + var_value="" + printf -v "var_value" "${escbksl_nl_or_null}" for (( i=0; i<${num_elems}; i++ )); do - var_value="${var_value}\"${array[$i]}\" ${esc_nl_or_null}" + printf -v "var_value" "${var_value}\"${array[$i]}\" ${escbksl_nl_or_null}" done var_value="( ${var_value})" @@ -2762,59 +2719,96 @@ Setting its value in the variable definitions file to an empty string." fi # -# If +# Set the line containing the variable's definition. Then add the line +# to the appropriate list (var_defns_notempl or var_defns_templ) depending +# on whether the variable is a template or not. +# + var_defn="${var_name}=${var_value}" +# +# If the variable value contains a dollar sign, we assume it is a template +# variable, i.e. one whose definition contains a reference to the value +# of another variable. In this case, the line for this variable must +# be placed at the end of the variable definitions file to ensure that +# the variable that it refers to has already been defined (assuming the +# referenced variable is one of the experiment variables). # dollar_or_null=$( printf "%s" "${var_value}" | \ $SED -n -r -e "s/[^\$]*(\$).*/\1/p" ) if [ -z "${dollar_or_null}" ]; then -# -# Now place var_value on the right-hand side of the assignment statement -# on the appropriate line in the variable definitions file. -# - set_file_param "${GLOBAL_VAR_DEFNS_FP}" "${var_name}" "${var_value}" + printf -v "var_defns_notempl" "${var_defns_notempl}${var_defn}\n" else - template_var_names+=( "${var_name}" ) - template_var_values+=( "${var_value}" ) - $SED -i "/^${var_name}=/d" ${GLOBAL_VAR_DEFNS_FP} + printf -v "var_defns_templ" "${var_defns_templ}${var_defn}\n" fi # # If var_name is empty, then a variable name was not found on the current -# line in line_list. In this case, print out a warning and move on to -# the next line. +# line in default_var_defns. In this case, print out a warning and move +# on to the next line. # else print_info_msg " -Could not extract a variable name from the current line in \"line_list\" +Could not extract a variable name from the current line in \"default_var_defns\" (probably because it does not contain an equal sign with no spaces on either side): crnt_line = \"${crnt_line}\" var_name = \"${var_name}\" -Continuing to next line in \"line_list\"." +Continuing to next line in \"default_var_defns\"." fi -done <<< "${line_list}" +done <<< "${default_var_defns}" # #----------------------------------------------------------------------- # -# Append additional variable definitions (and comments) to the variable -# definitions file. These variables have been set above using the vari- -# ables in the default and local configuration scripts. These variables -# are needed by various tasks/scripts in the workflow. +# Construct the experiment's variable definitions file. Below, we first +# record the contents we want to place in this file in the variable +# var_defns_file_contents, and we then write the contents of this +# variable to the file. # #----------------------------------------------------------------------- # -{ cat << EOM >> ${GLOBAL_VAR_DEFNS_FP} +print_info_msg " +Generating the global experiment variable definitions file specified by +GLOBAL_VAR_DEFNS_FN: + GLOBAL_VAR_DEFNS_FN = \"${GLOBAL_VAR_DEFNS_FN}\" +Full path to this file is: + GLOBAL_VAR_DEFNS_FP = \"${GLOBAL_VAR_DEFNS_FP}\" +For more detailed information, set DEBUG to \"TRUE\" in the experiment +configuration file (\"${EXPT_CONFIG_FN}\")." +var_defns_file_contents="\ +# +#----------------------------------------------------------------------- +#----------------------------------------------------------------------- +# Section 1: +# This section contains (most of) the primary experiment variables, i.e. +# those variables that are defined in the default configuration file +# (${EXPT_DEFAULT_CONFIG_FN}) and that can be reset via the user-specified +# experiment configuration file (${EXPT_CONFIG_FN}). Note that primary variables +# that are template variables (i.e. contain in their definitions references +# to other variables) are placed towards the end of this file. +#----------------------------------------------------------------------- +#----------------------------------------------------------------------- +# +${var_defns_notempl}" +# +# Append derived/secondary variable definitions (as well as comments) to +# the contents of the variable definitions file. +# +ensmem_names_str=$(printf "${escbksl_nl_or_null}\"%s\" " "${ENSMEM_NAMES[@]}") +ensmem_names_str=$(printf "( %s${escbksl_nl_or_null})" "${ensmem_names_str}") + +fv3_nml_ensmem_fps_str=$(printf "${escbksl_nl_or_null}\"%s\" " "${FV3_NML_ENSMEM_FPS[@]}") +fv3_nml_ensmem_fps_str=$(printf "( %s${escbksl_nl_or_null})" "${fv3_nml_ensmem_fps_str}") + +var_defns_file_contents=${var_defns_file_contents}"\ # #----------------------------------------------------------------------- #----------------------------------------------------------------------- # Section 2: -# This section defines variables that have been derived from the ones -# above by the setup script (setup.sh) and which are needed by one or -# more of the scripts that perform the workflow tasks (those scripts -# source this variable definitions file). +# This section defines variables that have been derived from the primary +# set of experiment variables above (we refer to these as \"derived\" or +# \"secondary\" variables). #----------------------------------------------------------------------- #----------------------------------------------------------------------- # @@ -2822,15 +2816,15 @@ done <<< "${line_list}" # #----------------------------------------------------------------------- # -# Full path to workflow launcher script, its log file, and the line that -# gets added to the cron table to launch this script if USE_CRON_TO_RELAUNCH -# is set to TRUE. +# Full path to workflow (re)launch script, its log file, and the line +# that gets added to the cron table to launch this script if the flag +# USE_CRON_TO_RELAUNCH is set to \"TRUE\". # #----------------------------------------------------------------------- # -WFLOW_LAUNCH_SCRIPT_FP="${WFLOW_LAUNCH_SCRIPT_FP}" -WFLOW_LAUNCH_LOG_FP="${WFLOW_LAUNCH_LOG_FP}" -CRONTAB_LINE="${CRONTAB_LINE}" +WFLOW_LAUNCH_SCRIPT_FP=\"${WFLOW_LAUNCH_SCRIPT_FP}\" +WFLOW_LAUNCH_LOG_FP=\"${WFLOW_LAUNCH_LOG_FP}\" +CRONTAB_LINE=\"${CRONTAB_LINE}\" # #----------------------------------------------------------------------- # @@ -2838,41 +2832,41 @@ CRONTAB_LINE="${CRONTAB_LINE}" # #----------------------------------------------------------------------- # -SR_WX_APP_TOP_DIR="${SR_WX_APP_TOP_DIR}" -HOMErrfs="$HOMErrfs" -USHDIR="$USHDIR" -SCRIPTSDIR="$SCRIPTSDIR" -JOBSDIR="$JOBSDIR" -SORCDIR="$SORCDIR" -SRC_DIR="$SRC_DIR" -PARMDIR="$PARMDIR" -MODULES_DIR="${MODULES_DIR}" -EXECDIR="$EXECDIR" -FIXam="$FIXam" -FIXLAM="$FIXLAM" -FIXgsm="$FIXgsm" -COMROOT="$COMROOT" -COMOUT_BASEDIR="${COMOUT_BASEDIR}" -TEMPLATE_DIR="${TEMPLATE_DIR}" -VX_CONFIG_DIR="${VX_CONFIG_DIR}" -METPLUS_CONF="${METPLUS_CONF}" -MET_CONFIG="${MET_CONFIG}" -UFS_WTHR_MDL_DIR="${UFS_WTHR_MDL_DIR}" -UFS_UTILS_DIR="${UFS_UTILS_DIR}" -SFC_CLIMO_INPUT_DIR="${SFC_CLIMO_INPUT_DIR}" -TOPO_DIR="${TOPO_DIR}" -UPP_DIR="${UPP_DIR}" - -EXPTDIR="$EXPTDIR" -LOGDIR="$LOGDIR" -CYCLE_BASEDIR="${CYCLE_BASEDIR}" -GRID_DIR="${GRID_DIR}" -OROG_DIR="${OROG_DIR}" -SFC_CLIMO_DIR="${SFC_CLIMO_DIR}" - -NDIGITS_ENSMEM_NAMES="${NDIGITS_ENSMEM_NAMES}" -ENSMEM_NAMES=( $( printf "\"%s\" " "${ENSMEM_NAMES[@]}" )) -FV3_NML_ENSMEM_FPS=( $( printf "\"%s\" " "${FV3_NML_ENSMEM_FPS[@]}" )) +SR_WX_APP_TOP_DIR=\"${SR_WX_APP_TOP_DIR}\" +HOMErrfs=\"$HOMErrfs\" +USHDIR=\"$USHDIR\" +SCRIPTSDIR=\"$SCRIPTSDIR\" +JOBSDIR=\"$JOBSDIR\" +SORCDIR=\"$SORCDIR\" +SRC_DIR=\"$SRC_DIR\" +PARMDIR=\"$PARMDIR\" +MODULES_DIR=\"${MODULES_DIR}\" +EXECDIR=\"$EXECDIR\" +FIXam=\"$FIXam\" +FIXLAM=\"$FIXLAM\" +FIXgsm=\"$FIXgsm\" +COMROOT=\"$COMROOT\" +COMOUT_BASEDIR=\"${COMOUT_BASEDIR}\" +TEMPLATE_DIR=\"${TEMPLATE_DIR}\" +VX_CONFIG_DIR=\"${VX_CONFIG_DIR}\" +METPLUS_CONF=\"${METPLUS_CONF}\" +MET_CONFIG=\"${MET_CONFIG}\" +UFS_WTHR_MDL_DIR=\"${UFS_WTHR_MDL_DIR}\" +UFS_UTILS_DIR=\"${UFS_UTILS_DIR}\" +SFC_CLIMO_INPUT_DIR=\"${SFC_CLIMO_INPUT_DIR}\" +TOPO_DIR=\"${TOPO_DIR}\" +UPP_DIR=\"${UPP_DIR}\" + +EXPTDIR=\"$EXPTDIR\" +LOGDIR=\"$LOGDIR\" +CYCLE_BASEDIR=\"${CYCLE_BASEDIR}\" +GRID_DIR=\"${GRID_DIR}\" +OROG_DIR=\"${OROG_DIR}\" +SFC_CLIMO_DIR=\"${SFC_CLIMO_DIR}\" + +NDIGITS_ENSMEM_NAMES=\"${NDIGITS_ENSMEM_NAMES}\" +ENSMEM_NAMES=${ensmem_names_str} +FV3_NML_ENSMEM_FPS=${fv3_nml_ensmem_fps_str} # #----------------------------------------------------------------------- # @@ -2880,46 +2874,43 @@ FV3_NML_ENSMEM_FPS=( $( printf "\"%s\" " "${FV3_NML_ENSMEM_FPS[@]}" )) # #----------------------------------------------------------------------- # -GLOBAL_VAR_DEFNS_FP="${GLOBAL_VAR_DEFNS_FP}" -# Try this at some point instead of hard-coding it as above; it's a more -# flexible approach (if it works). -#GLOBAL_VAR_DEFNS_FP=$( $READLINK -f "${BASH_SOURCE[0]}" ) +GLOBAL_VAR_DEFNS_FP=\"${GLOBAL_VAR_DEFNS_FP}\" -DATA_TABLE_TMPL_FN="${DATA_TABLE_TMPL_FN}" -DIAG_TABLE_TMPL_FN="${DIAG_TABLE_TMPL_FN}" -FIELD_TABLE_TMPL_FN="${FIELD_TABLE_TMPL_FN}" -MODEL_CONFIG_TMPL_FN="${MODEL_CONFIG_TMPL_FN}" -NEMS_CONFIG_TMPL_FN="${NEMS_CONFIG_TMPL_FN}" +DATA_TABLE_TMPL_FN=\"${DATA_TABLE_TMPL_FN}\" +DIAG_TABLE_TMPL_FN=\"${DIAG_TABLE_TMPL_FN}\" +FIELD_TABLE_TMPL_FN=\"${FIELD_TABLE_TMPL_FN}\" +MODEL_CONFIG_TMPL_FN=\"${MODEL_CONFIG_TMPL_FN}\" +NEMS_CONFIG_TMPL_FN=\"${NEMS_CONFIG_TMPL_FN}\" -DATA_TABLE_TMPL_FP="${DATA_TABLE_TMPL_FP}" -DIAG_TABLE_TMPL_FP="${DIAG_TABLE_TMPL_FP}" -FIELD_TABLE_TMPL_FP="${FIELD_TABLE_TMPL_FP}" -FV3_NML_BASE_SUITE_FP="${FV3_NML_BASE_SUITE_FP}" -FV3_NML_YAML_CONFIG_FP="${FV3_NML_YAML_CONFIG_FP}" -FV3_NML_BASE_ENS_FP="${FV3_NML_BASE_ENS_FP}" -MODEL_CONFIG_TMPL_FP="${MODEL_CONFIG_TMPL_FP}" -NEMS_CONFIG_TMPL_FP="${NEMS_CONFIG_TMPL_FP}" +DATA_TABLE_TMPL_FP=\"${DATA_TABLE_TMPL_FP}\" +DIAG_TABLE_TMPL_FP=\"${DIAG_TABLE_TMPL_FP}\" +FIELD_TABLE_TMPL_FP=\"${FIELD_TABLE_TMPL_FP}\" +FV3_NML_BASE_SUITE_FP=\"${FV3_NML_BASE_SUITE_FP}\" +FV3_NML_YAML_CONFIG_FP=\"${FV3_NML_YAML_CONFIG_FP}\" +FV3_NML_BASE_ENS_FP=\"${FV3_NML_BASE_ENS_FP}\" +MODEL_CONFIG_TMPL_FP=\"${MODEL_CONFIG_TMPL_FP}\" +NEMS_CONFIG_TMPL_FP=\"${NEMS_CONFIG_TMPL_FP}\" -CCPP_PHYS_SUITE_FN="${CCPP_PHYS_SUITE_FN}" -CCPP_PHYS_SUITE_IN_CCPP_FP="${CCPP_PHYS_SUITE_IN_CCPP_FP}" -CCPP_PHYS_SUITE_FP="${CCPP_PHYS_SUITE_FP}" +CCPP_PHYS_SUITE_FN=\"${CCPP_PHYS_SUITE_FN}\" +CCPP_PHYS_SUITE_IN_CCPP_FP=\"${CCPP_PHYS_SUITE_IN_CCPP_FP}\" +CCPP_PHYS_SUITE_FP=\"${CCPP_PHYS_SUITE_FP}\" -FIELD_DICT_FN="${FIELD_DICT_FN}" -FIELD_DICT_IN_UWM_FP="${FIELD_DICT_IN_UWM_FP}" -FIELD_DICT_FP="${FIELD_DICT_FP}" +FIELD_DICT_FN=\"${FIELD_DICT_FN}\" +FIELD_DICT_IN_UWM_FP=\"${FIELD_DICT_IN_UWM_FP}\" +FIELD_DICT_FP=\"${FIELD_DICT_FP}\" -DATA_TABLE_FP="${DATA_TABLE_FP}" -FIELD_TABLE_FP="${FIELD_TABLE_FP}" -FV3_NML_FN="${FV3_NML_FN}" # This may not be necessary... -FV3_NML_FP="${FV3_NML_FP}" -NEMS_CONFIG_FP="${NEMS_CONFIG_FP}" +DATA_TABLE_FP=\"${DATA_TABLE_FP}\" +FIELD_TABLE_FP=\"${FIELD_TABLE_FP}\" +FV3_NML_FN=\"${FV3_NML_FN}\" # This may not be necessary... +FV3_NML_FP=\"${FV3_NML_FP}\" +NEMS_CONFIG_FP=\"${NEMS_CONFIG_FP}\" -FV3_EXEC_FP="${FV3_EXEC_FP}" +FV3_EXEC_FP=\"${FV3_EXEC_FP}\" -LOAD_MODULES_RUN_TASK_FP="${LOAD_MODULES_RUN_TASK_FP}" +LOAD_MODULES_RUN_TASK_FP=\"${LOAD_MODULES_RUN_TASK_FP}\" -THOMPSON_MP_CLIMO_FN="${THOMPSON_MP_CLIMO_FN}" -THOMPSON_MP_CLIMO_FP="${THOMPSON_MP_CLIMO_FP}" +THOMPSON_MP_CLIMO_FN=\"${THOMPSON_MP_CLIMO_FN}\" +THOMPSON_MP_CLIMO_FP=\"${THOMPSON_MP_CLIMO_FP}\" # #----------------------------------------------------------------------- # @@ -2927,106 +2918,99 @@ THOMPSON_MP_CLIMO_FP="${THOMPSON_MP_CLIMO_FP}" # #----------------------------------------------------------------------- # -RELATIVE_LINK_FLAG="${RELATIVE_LINK_FLAG}" +RELATIVE_LINK_FLAG=\"${RELATIVE_LINK_FLAG}\" # #----------------------------------------------------------------------- # # Parameters that indicate whether or not various parameterizations are -# included in and called by the phsics suite. +# included in and called by the physics suite. # #----------------------------------------------------------------------- # -SDF_USES_RUC_LSM="${SDF_USES_RUC_LSM}" -SDF_USES_THOMPSON_MP="${SDF_USES_THOMPSON_MP}" +SDF_USES_RUC_LSM=\"${SDF_USES_RUC_LSM}\" +SDF_USES_THOMPSON_MP=\"${SDF_USES_THOMPSON_MP}\" # #----------------------------------------------------------------------- # -# Grid configuration parameters needed regardless of grid generation me- -# thod used. +# Grid configuration parameters needed regardless of grid generation +# method used. # #----------------------------------------------------------------------- # -GTYPE="$GTYPE" -TILE_RGNL="${TILE_RGNL}" -NH0="${NH0}" -NH3="${NH3}" -NH4="${NH4}" +GTYPE=\"$GTYPE\" +TILE_RGNL=\"${TILE_RGNL}\" +NH0=\"${NH0}\" +NH3=\"${NH3}\" +NH4=\"${NH4}\" -LON_CTR="${LON_CTR}" -LAT_CTR="${LAT_CTR}" -NX="${NX}" -NY="${NY}" -NHW="${NHW}" -STRETCH_FAC="${STRETCH_FAC}" +LON_CTR=\"${LON_CTR}\" +LAT_CTR=\"${LAT_CTR}\" +NX=\"${NX}\" +NY=\"${NY}\" +NHW=\"${NHW}\" +STRETCH_FAC=\"${STRETCH_FAC}\" -RES_IN_FIXLAM_FILENAMES="${RES_IN_FIXLAM_FILENAMES}" +RES_IN_FIXLAM_FILENAMES=\"${RES_IN_FIXLAM_FILENAMES}\" # -# If running the make_grid task, CRES will be set to a null string du- -# the grid generation step. It will later be set to an actual value af- -# ter the make_grid task is complete. +# If running the make_grid task, CRES will be set to a null string during +# the grid generation step. It will later be set to an actual value after +# the make_grid task is complete. # -CRES="$CRES" -EOM -} || print_err_msg_exit "\ -Heredoc (cat) command to append new variable definitions to variable -definitions file returned with a nonzero status." +CRES=\"$CRES\" +" # #----------------------------------------------------------------------- # -# Append to the variable definitions file the defintions of grid parame- -# ters that are specific to the grid generation method used. +# Append to the variable definitions file the defintions of grid parameters +# that are specific to the grid generation method used. # #----------------------------------------------------------------------- # +grid_vars_str="" if [ "${GRID_GEN_METHOD}" = "GFDLgrid" ]; then - { cat << EOM >> ${GLOBAL_VAR_DEFNS_FP} + grid_vars_str="\ # #----------------------------------------------------------------------- # # Grid configuration parameters for a regional grid generated from a -# global parent cubed-sphere grid. This is the method originally sug- -# gested by GFDL since it allows GFDL's nested grid generator to be used -# to generate a regional grid. However, for large regional domains, it -# results in grids that have an unacceptably large range of cell sizes +# global parent cubed-sphere grid. This is the method originally +# suggested by GFDL since it allows GFDL's nested grid generator to be +# used to generate a regional grid. However, for large regional domains, +# it results in grids that have an unacceptably large range of cell sizes # (i.e. ratio of maximum to minimum cell size is not sufficiently close # to 1). # #----------------------------------------------------------------------- # -ISTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG="${ISTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG}" -IEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG="${IEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG}" -JSTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG="${JSTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG}" -JEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG="${JEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG}" -EOM -} || print_err_msg_exit "\ -Heredoc (cat) command to append grid parameters to variable definitions -file returned with a nonzero status." +ISTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG=\"${ISTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG}\" +IEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG=\"${IEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG}\" +JSTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG=\"${JSTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG}\" +JEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG=\"${JEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG}\" +" elif [ "${GRID_GEN_METHOD}" = "ESGgrid" ]; then - { cat << EOM >> ${GLOBAL_VAR_DEFNS_FP} + grid_vars_str="\ # #----------------------------------------------------------------------- # -# Grid configuration parameters for a regional grid generated indepen- -# dently of a global parent grid. This method was developed by Jim Pur- -# ser of EMC and results in very uniform grids (i.e. ratio of maximum to -# minimum cell size is very close to 1). +# Grid configuration parameters for a regional grid generated independently +# of a global parent grid. This method was developed by Jim Purser of +# EMC and results in very uniform grids (i.e. ratio of maximum to minimum +# cell size is very close to 1). # #----------------------------------------------------------------------- # -DEL_ANGLE_X_SG="${DEL_ANGLE_X_SG}" -DEL_ANGLE_Y_SG="${DEL_ANGLE_Y_SG}" -NEG_NX_OF_DOM_WITH_WIDE_HALO="${NEG_NX_OF_DOM_WITH_WIDE_HALO}" -NEG_NY_OF_DOM_WITH_WIDE_HALO="${NEG_NY_OF_DOM_WITH_WIDE_HALO}" -PAZI="${PAZI}" -EOM -} || print_err_msg_exit "\ -Heredoc (cat) command to append grid parameters to variable definitions -file returned with a nonzero status." +DEL_ANGLE_X_SG=\"${DEL_ANGLE_X_SG}\" +DEL_ANGLE_Y_SG=\"${DEL_ANGLE_Y_SG}\" +NEG_NX_OF_DOM_WITH_WIDE_HALO=\"${NEG_NX_OF_DOM_WITH_WIDE_HALO}\" +NEG_NY_OF_DOM_WITH_WIDE_HALO=\"${NEG_NY_OF_DOM_WITH_WIDE_HALO}\" +PAZI=\"${PAZI}\" +" fi +var_defns_file_contents="${var_defns_file_contents}${grid_vars_str}" # #----------------------------------------------------------------------- # @@ -3035,21 +3019,22 @@ fi # #----------------------------------------------------------------------- # -lbc_spec_fcst_hrs_str=$(printf "\"%s\" ${esc_nl_or_null}" "${LBC_SPEC_FCST_HRS[@]}") -lbc_spec_fcst_hrs_str=$(printf "( ${esc_nl_or_null}%s${esc_nl_or_null:+\n})" "${lbc_spec_fcst_hrs_str}") +lbc_spec_fcst_hrs_str=$(printf "${escbksl_nl_or_null}\"%s\" " "${LBC_SPEC_FCST_HRS[@]}") +lbc_spec_fcst_hrs_str=$(printf "( %s${escbksl_nl_or_null})" "${lbc_spec_fcst_hrs_str}") -all_cdates_str=$(printf "\"%s\" ${esc_nl_or_null}" "${ALL_CDATES[@]}") -all_cdates_str=$(printf "( ${esc_nl_or_null}%s${esc_nl_or_null:+\n})" "${all_cdates_str}") +all_cdates_str=$(printf "${escbksl_nl_or_null}\"%s\" " "${ALL_CDATES[@]}") +all_cdates_str=$(printf "( %s${escbksl_nl_or_null})" "${all_cdates_str}") -{ cat << EOM >> ${GLOBAL_VAR_DEFNS_FP} +var_defns_file_contents=${var_defns_file_contents}"\ # #----------------------------------------------------------------------- # -# CPL: parameter for coupling in model_configure +# Flag in the \"${MODEL_CONFIG_FN}\" file for coupling the ocean model to +# the weather model. # #----------------------------------------------------------------------- # -CPL="${CPL}" +CPL=\"${CPL}\" # #----------------------------------------------------------------------- # @@ -3058,11 +3043,11 @@ CPL="${CPL}" # #----------------------------------------------------------------------- # -OZONE_PARAM="${OZONE_PARAM}" +OZONE_PARAM=\"${OZONE_PARAM}\" # #----------------------------------------------------------------------- # -# If USE_USER_STAGED_EXTRN_FILES is set to "FALSE", this is the system +# If USE_USER_STAGED_EXTRN_FILES is set to \"FALSE\", this is the system # directory in which the workflow scripts will look for the files generated # by the external model specified in EXTRN_MDL_NAME_ICS. These files will # be used to generate the input initial condition and surface files for @@ -3070,11 +3055,11 @@ OZONE_PARAM="${OZONE_PARAM}" # #----------------------------------------------------------------------- # -EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS}" +EXTRN_MDL_SYSBASEDIR_ICS=\"${EXTRN_MDL_SYSBASEDIR_ICS}\" # #----------------------------------------------------------------------- # -# If USE_USER_STAGED_EXTRN_FILES is set to "FALSE", this is the system +# If USE_USER_STAGED_EXTRN_FILES is set to \"FALSE\", this is the system # directory in which the workflow scripts will look for the files generated # by the external model specified in EXTRN_MDL_NAME_LBCS. These files # will be used to generate the input lateral boundary condition files for @@ -3082,7 +3067,7 @@ EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS}" # #----------------------------------------------------------------------- # -EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS}" +EXTRN_MDL_SYSBASEDIR_LBCS=\"${EXTRN_MDL_SYSBASEDIR_LBCS}\" # #----------------------------------------------------------------------- # @@ -3091,7 +3076,7 @@ EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS}" # #----------------------------------------------------------------------- # -EXTRN_MDL_LBCS_OFFSET_HRS="${EXTRN_MDL_LBCS_OFFSET_HRS}" +EXTRN_MDL_LBCS_OFFSET_HRS=\"${EXTRN_MDL_LBCS_OFFSET_HRS}\" # #----------------------------------------------------------------------- # @@ -3104,25 +3089,29 @@ LBC_SPEC_FCST_HRS=${lbc_spec_fcst_hrs_str} # #----------------------------------------------------------------------- # -# The number of cycles for which to make forecasts and the list of starting -# dates/hours of these cycles. +# The number of cycles for which to make forecasts and the list of +# starting dates/hours of these cycles. # #----------------------------------------------------------------------- # -NUM_CYCLES="${NUM_CYCLES}" +NUM_CYCLES=\"${NUM_CYCLES}\" ALL_CDATES=${all_cdates_str} # #----------------------------------------------------------------------- # -# If USE_FVCOM is set to TRUE, then FVCOM data (located in FVCOM_DIR -# in FVCOM_FILE) will be used to update lower boundary conditions during -# make_ics. +# Parameters that determine whether FVCOM data will be used, and if so, +# their location. +# +# If USE_FVCOM is set to \"TRUE\", then FVCOM data (in the file FVCOM_FILE +# located in the directory FVCOM_DIR) will be used to update the surface +# boundary conditions during the initial conditions generation task +# (MAKE_ICS_TN). # #----------------------------------------------------------------------- # -USE_FVCOM="${USE_FVCOM}" -FVCOM_DIR="${FVCOM_DIR}" -FVCOM_FILE="${FVCOM_FILE}" +USE_FVCOM=\"${USE_FVCOM}\" +FVCOM_DIR=\"${FVCOM_DIR}\" +FVCOM_FILE=\"${FVCOM_FILE}\" # #----------------------------------------------------------------------- # @@ -3130,60 +3119,39 @@ FVCOM_FILE="${FVCOM_FILE}" # #----------------------------------------------------------------------- # -NCORES_PER_NODE="${NCORES_PER_NODE}" -PE_MEMBER01="${PE_MEMBER01}" +NCORES_PER_NODE=\"${NCORES_PER_NODE}\" +PE_MEMBER01=\"${PE_MEMBER01}\" # #----------------------------------------------------------------------- # -# IF DO_SPP="TRUE," N_VAR_SPP is the number of parameterizations that -# are perturbed with SPP, otherwise N_VAR_SPP=0. +# IF DO_SPP is set to \"TRUE\", N_VAR_SPP specifies the number of physics +# parameterizations that are perturbed with SPP. Otherwise, N_VAR_SPP +# is set 0. # #----------------------------------------------------------------------- # -N_VAR_SPP="${N_VAR_SPP}" -EOM -} || print_err_msg_exit "\ -Heredoc (cat) command to append new variable definitions to variable -definitions file returned with a nonzero status." - - - -str_to_insert=" +N_VAR_SPP=\"${N_VAR_SPP}\" # #----------------------------------------------------------------------- #----------------------------------------------------------------------- # Section 3: -# This section defines templates variables, i.e. variables that contain -# other (unexpanded) variables in their definitions. See above for -# variable definitions. -#----------------------------------------------------------------------- -#----------------------------------------------------------------------- -#" - -num_template_vars="${#template_var_names[@]}" -for (( i=0; i<${num_template_vars}; i++ )); do - str_to_insert=${str_to_insert}" -${template_var_names[$i]}=" -done - -{ cat << EOM >> ${GLOBAL_VAR_DEFNS_FP} -${str_to_insert} -EOM -} || print_err_msg_exit "\ -Heredoc (cat) command to append new variable definitions to variable -definitions file returned with a nonzero status." - -for (( i=0; i<${num_template_vars}; i++ )); do - set_file_param "${GLOBAL_VAR_DEFNS_FP}" \ - "${template_var_names[$i]}" "${template_var_values[$i]}" -done +# This section defines template variables, i.e. variables that contain +# references to other variables in their definitions, e.g. # +# MY_VAR=\"\${ANOTHER_VAR}\" +# +# They are placed towards the end of this file to ensure that the +# referenced variables have already been defined. +#----------------------------------------------------------------------- #----------------------------------------------------------------------- # +${var_defns_templ}" # +# Done with constructing the contents of the variable definitions file, +# so now write the contents to file. # -#----------------------------------------------------------------------- -# +printf "%s\n" "${var_defns_file_contents}" >> ${GLOBAL_VAR_DEFNS_FP} + print_info_msg "$VERBOSE" " Done generating the global experiment variable definitions file." # From 880bebe279fca2f0e22e8f214b9661bbbf9935c4 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Fri, 10 Dec 2021 13:26:40 -0700 Subject: [PATCH 11/35] Minor edits. --- ush/setup.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/setup.sh b/ush/setup.sh index c86935e61..9926ff90b 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -2585,7 +2585,7 @@ ${default_var_defns} # var_defns_templ is for template variables, i.e. variables that contain # references to other variables in their definitions, e.g. # -# MY_VAR=\"\${ANOTHER_VAR}\" +# MY_VAR="\${ANOTHER_VAR}" # # while var_defns_notempl is for variables whose definitions contain # only a literal string. These two types are treated separately @@ -3138,7 +3138,7 @@ N_VAR_SPP=\"${N_VAR_SPP}\" # This section defines template variables, i.e. variables that contain # references to other variables in their definitions, e.g. # -# MY_VAR=\"\${ANOTHER_VAR}\" +# MY_VAR="\${ANOTHER_VAR}" # # They are placed towards the end of this file to ensure that the # referenced variables have already been defined. From 2ff052d0e385a40fca3597e988894e964c9ecd2e Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Fri, 10 Dec 2021 13:27:56 -0700 Subject: [PATCH 12/35] Minor edits. --- ush/config_defaults.sh | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index 8c2ba74aa..07bee0f69 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -152,13 +152,14 @@ QUEUE_FCST="" # will be ignored unless WORKFLOW_MANAGER="none". Definitions: # # RUN_CMD_UTILS: -# The run command for pre-processing utilities (shave, orog, sfc_climo_gen, etc.) -# Can be left blank for smaller domains, in which case the executables will run -# without MPI. +# The run command for pre-processing utilities (shave, orog, sfc_climo_gen, +# etc.) Can be left blank for smaller domains, in which case the executables +# will run without MPI. # # RUN_CMD_FCST: -# The run command for the model forecast step. This will be appended to the end -# of the variable definitions file, so it can reference other variables. +# The run command for the model forecast step. This will be appended to +# the end of the variable definitions file, so it can reference other +# variables. # # RUN_CMD_POST: # The run command for post-processing (UPP). Can be left blank for smaller @@ -295,7 +296,7 @@ PTMP="/base/path/of/directory/containing/postprocessed/output/files" # #----------------------------------------------------------------------- # -# Set the sparator character(s) to use in the names of the grid, mosaic, +# Set the separator character(s) to use in the names of the grid, mosaic, # and orography fixed files. # # Ideally, the same separator should be used in the names of these fixed @@ -707,6 +708,7 @@ NOMADS_file_type="nemsio" # directory or the cycle directories under it. # #----------------------------------------------------------------------- +# CCPP_PHYS_SUITE="FV3_GFS_v15p2" # #----------------------------------------------------------------------- @@ -1712,9 +1714,9 @@ NUM_ENS_MEMBERS="1" # #----------------------------------------------------------------------- # -DO_SHUM="false" -DO_SPPT="false" -DO_SKEB="false" +DO_SHUM="FALSE" +DO_SPPT="FALSE" +DO_SKEB="FALSE" SHUM_MAG="0.006" #Variable "shum" in input.nml SHUM_LSCALE="150000" SHUM_TSCALE="21600" #Variable "shum_tau" in input.nml @@ -1728,7 +1730,7 @@ SKEB_LSCALE="150000" SKEB_TSCALE="21600" #Variable "skeb_tau" in input.nml SKEB_INT="3600" #Variable "skebint" in input.nml SKEB_VDOF="10" -USE_ZMTNBLCK="false" +USE_ZMTNBLCK="FALSE" # #----------------------------------------------------------------------- # @@ -1745,7 +1747,7 @@ USE_ZMTNBLCK="false" # #----------------------------------------------------------------------- # -DO_SPP="false" +DO_SPP="FALSE" SPP_VAR_LIST=( "pbl" ) SPP_MAG_LIST=( "0.2" ) #Variable "spp_prt_list" in input.nml SPP_LSCALE=( "150000.0" ) @@ -1860,6 +1862,3 @@ OMP_STACKSIZE_RUN_FCST="1024m" KMP_AFFINITY_RUN_POST="scatter" OMP_NUM_THREADS_RUN_POST="1" OMP_STACKSIZE_RUN_POST="1024m" -# -#----------------------------------------------------------------------- -# From a525cb5398969e26f20a99d6b0b262e969f39a95 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Fri, 10 Dec 2021 13:29:40 -0700 Subject: [PATCH 13/35] In constants.sh, add the variable VAR_REF_CHAR that contains the character that indicates a reference to a bash variable in a string (this character is the dollar sign). Then, in check_var_valid_value.sh, use this new variable to skip checking for valid values if a variable's value contains this character. --- ush/bash_utils/check_var_valid_value.sh | 25 ++++++++++++++++++------- ush/constants.sh | 16 ++++++++++++++++ 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/ush/bash_utils/check_var_valid_value.sh b/ush/bash_utils/check_var_valid_value.sh index 7a0e20707..4ea6068a1 100644 --- a/ush/bash_utils/check_var_valid_value.sh +++ b/ush/bash_utils/check_var_valid_value.sh @@ -108,19 +108,30 @@ The value specified in ${var_name} is not supported: # #----------------------------------------------------------------------- # -# Check whether var_value is equal to one of the elements of the array -# valid_var_values. If not, print out an error message and exit the -# calling script. +# If var_value contains the character specified by VAR_REF_CHAR, we +# assume the corresponding variable (var_name) is a template variable, +# i.e. one whose value contains a reference to another variable, e.g. +# +# MY_VAR="\${ANOTHER_VAR}" +# +# [In bash, VAR_REF_CHAR is a (escaped) dollar sign.] In this case, it +# does not make sense to check whether var_value is a valid value since +# its contents have not yet been expanded. If var_value doesn't contain +# VAR_REF_CHAR, it must contain a literal string. In this case, we check +# whether it is equal to one of the elements of the array valid_var_values. +# If not, we print out an error message and exit the calling script. # #----------------------------------------------------------------------- # - is_element_of "valid_var_values" "${var_value}" || { \ - valid_var_values_str=$(printf "\"%s\" " "${valid_var_values[@]}"); - print_err_msg_exit "\ + if [[ "${var_value}" != *"${VAR_REF_CHAR}"* ]]; then + is_element_of "valid_var_values" "${var_value}" || { \ + valid_var_values_str=$(printf "\"%s\" " "${valid_var_values[@]}"); + print_err_msg_exit "\ ${err_msg} ${var_name} must be set to one of the following: ${valid_var_values_str}"; \ - } + } + fi # #----------------------------------------------------------------------- # diff --git a/ush/constants.sh b/ush/constants.sh index fb19e61f8..5d32b3375 100644 --- a/ush/constants.sh +++ b/ush/constants.sh @@ -15,3 +15,19 @@ degs_per_radian=$( bc -l <<< "360.0/(2.0*$pi_geom)" ) # Radius of the Earth in meters. radius_Earth="6371200.0" +# +#----------------------------------------------------------------------- +# +# Character used to identify a reference (in bash) to a variable in a +# given string. In bash, this is the dollar sign. For example, if the +# string is +# +# "Hello, my name is $NAME." +# +# then the dollar sign indicates that $NAME is a reference to the contents +# of the variable NAME. +# +#----------------------------------------------------------------------- +# +VAR_REF_CHAR="\$" + From 0c2e5c412e2c328f2cc4d3a749ffedcd15264a65 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Fri, 10 Dec 2021 17:06:02 -0700 Subject: [PATCH 14/35] Minor changes. --- ush/bash_utils/print_input_args.sh | 2 +- ush/generate_FV3LAM_wflow.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ush/bash_utils/print_input_args.sh b/ush/bash_utils/print_input_args.sh index 957cec524..d5ba5bd53 100644 --- a/ush/bash_utils/print_input_args.sh +++ b/ush/bash_utils/print_input_args.sh @@ -169,7 +169,7 @@ have been set as follows: #----------------------------------------------------------------------- # # If a global variable named DEBUG is not defined, print out the message. -# If it is defined, print out the message only if DEBUG is set to TRUE. +# If it is defined, print out the message only if DEBUG is set to "TRUE". # #----------------------------------------------------------------------- # diff --git a/ush/generate_FV3LAM_wflow.sh b/ush/generate_FV3LAM_wflow.sh index 1370a7bc4..03c25f6aa 100755 --- a/ush/generate_FV3LAM_wflow.sh +++ b/ush/generate_FV3LAM_wflow.sh @@ -1087,8 +1087,8 @@ if [[ $retval == 0 ]]; then # else printf " -Experiment/workflow generation failed. Check the log file from the ex- -periment/workflow generation script in the file specified by log_fp: +Experiment generation failed. Check the log file from the experiment +generation script in the file specified by log_fp: log_fp = \"${log_fp}\" Stopping. " From 5365a1ef058fc78f5b07bbe99c8ffddc3c9fb1db Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Sun, 12 Dec 2021 02:26:30 -0700 Subject: [PATCH 15/35] Introduce new utility function get_bash_file_contents(). Details: 1) Introduce new utility function get_bash_file_contents() to get the contents of a bash script/function with comment lines, empty lines, and leading and trailing whitespace removed. This will be used to extract the contents of the default and user-specified experiment configuration files. 2) In compare_config_scripts.sh: a) Replace code with the new function get_bash_file_contents(). b) Clean up comments and output messages. c) Use input arguments to specify the paths/names to the default and user-specified experiment configuration files. 3) Rename file compare_config_scripts.sh (and its function within) to check_expt_config_vars(.sh). 4) Replace code in setup.sh with the new function get_bash_file_contents(). --- ush/check_expt_config_vars.sh | 161 ++++++++++++++++++++++++++++++++++ ush/compare_config_scripts.sh | 151 ------------------------------- ush/setup.sh | 52 ++++------- ush/source_util_funcs.sh | 9 ++ 4 files changed, 185 insertions(+), 188 deletions(-) create mode 100644 ush/check_expt_config_vars.sh delete mode 100644 ush/compare_config_scripts.sh diff --git a/ush/check_expt_config_vars.sh b/ush/check_expt_config_vars.sh new file mode 100644 index 000000000..8b996855d --- /dev/null +++ b/ush/check_expt_config_vars.sh @@ -0,0 +1,161 @@ +# +#----------------------------------------------------------------------- +# +# This file defines a function that checks that all experiment variables +# set in the user-specified experiment configuration file are defined (by +# being assigned default values) in the default experiment configuration +# file. If a variable is found in the former that is not defined in the +# latter, this function exits with an error message. +# +# This check is performed in order to prevent the user from defining +# arbitrary variables in the user-specified configuration file; the +# latter should be used to specify only varaibles that have already been +# defined in the default configuration file. +# +# Arguments are as follows: +# +# default_config_fp: +# The relative or full path to the default experiment configuration file. +# +# config_fp: +# The relative or full path to the user-specified experiment configuration +# file. +# +#----------------------------------------------------------------------- +# +function check_expt_config_vars() { +# +#----------------------------------------------------------------------- +# +# Source bash utility functions. +# +#----------------------------------------------------------------------- +# + . ${scrfunc_dir}/source_util_funcs.sh +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# + { save_shell_opts; set -u +x; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Specify the set of valid argument names for this script/function. Then +# process the arguments provided to this script/function (which should +# consist of a set of name-value pairs of the form arg1="value1", etc). +# +#----------------------------------------------------------------------- +# + local valid_args=( \ + "default_config_fp" \ + "config_fp" \ + ) + process_args valid_args "$@" +# +#----------------------------------------------------------------------- +# +# For debugging purposes, print out values of arguments passed to this +# script. Note that these will be printed out only if VERBOSE is set to +# TRUE. +# +#----------------------------------------------------------------------- +# + print_input_args "valid_args" +# +#----------------------------------------------------------------------- +# +# Declare local variables. +# +#----------------------------------------------------------------------- +# + local var_list_default \ + var_list_user \ + crnt_line \ + var_name \ + regex_search +# +#----------------------------------------------------------------------- +# +# Get the list of variable definitions, first from the default experiment +# configuration file and then from the user-specified experiment +# configuration file. +# +#----------------------------------------------------------------------- +# + get_bash_file_contents fp="${default_config_fp}" \ + output_varname_contents="var_list_default" + + get_bash_file_contents fp="${config_fp}" \ + output_varname_contents="var_list_user" +# +#----------------------------------------------------------------------- +# +# Loop through each line/variable in var_list_user. For each line, +# extract the the name of the variable that is being set (say VAR) and +# check that this variable is set somewhere in the default configuration +# file by verifying that a line that starts with "VAR=" exists in +# var_list_default. +# +#----------------------------------------------------------------------- +# + while read crnt_line; do +# +# Note that a variable name will be found only if the equal sign immediately +# follows the variable name. +# + var_name=$( printf "%s" "${crnt_line}" | $SED -n -r -e "s/^([^ =\"]*)=.*/\1/p") + + if [ -z "${var_name}" ]; then + + print_info_msg " +The current line (crnt_line) of the user-specified experiment configuration +file (config_fp) does not contain a variable name (i.e. var_name is empty): + config_fp = \"${config_fp}\" + crnt_line = \"${crnt_line}\" + var_name = \"${var_name}\" +Skipping to next line." + + else +# +# Use grep to search for the variable name (followed by an equal sign, +# all at the beginning of a line) in the list of variables in the default +# configuration file. +# +# Note that we use a herestring to input into grep the list of variables +# in the default configuration file. grep will return with a zero status +# if the specified string (regex_search) is not found in the default +# variables list and a nonzero status otherwise. Note also that we +# redirect the output of grep to null because we are only interested in +# its exit status. +# + regex_search="^${var_name}=" + grep "${regex_search}" <<< "${var_list_default}" > /dev/null 2>&1 || \ + print_err_msg_exit "\ +A variable (var_name) in the user-specified experiment configuration file +(config_fp) does not appear in the default experiment configuration file +(default_config_fp): + config_fp = \"${config_fp}\" + default_config_fp = \"${default_config_fp}\" + crnt_line = \"${crnt_line}\" + var_name = \"${var_name}\" +Please assign a default value to this variable in the default configuration +file and rerun." + + fi + + done <<< "${var_list_user}" +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/func- +# tion. +# +#----------------------------------------------------------------------- +# + { restore_shell_opts; } > /dev/null 2>&1 + +} diff --git a/ush/compare_config_scripts.sh b/ush/compare_config_scripts.sh deleted file mode 100644 index 791fa4e34..000000000 --- a/ush/compare_config_scripts.sh +++ /dev/null @@ -1,151 +0,0 @@ -# -#----------------------------------------------------------------------- -# -# This file defines and then calls a function that checks that all vari- -# ables defined in the user-specified experiment/workflow configuration -# file (whose file name is stored in the variable EXPT_CONFIG_FN) are -# also assigned default values in the default configuration file (whose -# file name is stored in the variable EXPT_DEFAULT_CONFIG_FN). -# -#----------------------------------------------------------------------- -# -function compare_config_scripts() { -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# -local scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) -local scrfunc_fn=$( basename "${scrfunc_fp}" ) -local scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Get the name of this function. -# -#----------------------------------------------------------------------- -# -local func_name="${FUNCNAME[0]}" -# -#----------------------------------------------------------------------- -# -# Source bash utility functions. -# -#----------------------------------------------------------------------- -# -. ${scrfunc_dir}/source_util_funcs.sh -# -#----------------------------------------------------------------------- -# -# Save current shell options (in a global array). Then set new options -# for this script/function. -# -#----------------------------------------------------------------------- -# -{ save_shell_opts; set -u +x; } > /dev/null 2>&1 -# -#----------------------------------------------------------------------- -# -# Create a list of variable settings in the default workflow/experiment -# file (script) by stripping out comments, blank lines, extraneous lead- -# ing whitespace, etc from that file and saving the result in the varia- -# ble var_list_default. Each line of var_list_default will have the -# form -# -# VAR=... -# -# where the VAR is a variable name and ... is the value (including any -# trailing comments). Then create an equivalent list for the local con- -# figuration file and save the result in var_list_local. -# -#----------------------------------------------------------------------- -# -var_list_default=$( \ -$SED -r \ - -e "s/^([ ]*)([^ ]+.*)/\2/g" \ - -e "/^#.*/d" \ - -e "/^$/d" \ - ${EXPT_DEFAULT_CONFIG_FN} \ -) - -var_list_local=$( \ -$SED -r \ - -e "s/^([ ]*)([^ ]+.*)/\2/g" \ - -e "/^#.*/d" \ - -e "/^$/d" \ - ${EXPT_CONFIG_FN} \ -) -# -#----------------------------------------------------------------------- -# -# Loop through each line of var_list_local. For each line, extract the -# the name of the variable that is being set (say VAR) and check that -# this variable is set somewhere in the default configuration file by -# verifying that a line that starts with "VAR=" exists in var_list_de- -# fault. -# -#----------------------------------------------------------------------- -# -while read crnt_line; do -# -# Note that a variable name will be found only if the equal sign immed- -# iately follows the variable name. -# - var_name=$( printf "%s" "${crnt_line}" | $SED -n -r -e "s/^([^ =\"]*)=.*/\1/p") - - if [ -z "${var_name}" ]; then - - print_info_msg " -Current line (crnt_line) of user-specified experiment/workflow configu- -ration file (EXPT_CONFIG_FN) does not contain a variable name (i.e. -var_name is empty): - EXPT_CONFIG_FN = \"${EXPT_CONFIG_FN}\" - crnt_line = \"${crnt_line}\" - var_name = \"${var_name}\" -Skipping to next line." - - else -# -# Use a herestring to input list of variables in the default configura- -# tion file to grep. Also, redirect the output to null because we are -# only interested in the exit status of grep (which will be nonzero if -# the specified regex was not found in the list).. -# - grep "^${var_name}=" <<< "${var_list_default}" > /dev/null 2>&1 || \ - print_err_msg_exit "\ -The variable specified by var_name in the user-specified experiment/ -workflow configuration file (EXPT_CONFIG_FN) does not appear in the de- -fault experiment/workflow configuration file (EXPT_DEFAULT_CONFIG_FN): - EXPT_CONFIG_FN = \"${EXPT_CONFIG_FN}\" - EXPT_DEFAULT_CONFIG_FN = \"${EXPT_DEFAULT_CONFIG_FN}\" - var_name = \"${var_name}\" -Please assign a default value to this variable in the default configura- -tion file and rerun." - - fi - -done <<< "${var_list_local}" -# -#----------------------------------------------------------------------- -# -# Restore the shell options saved at the beginning of this script/func- -# tion. -# -#----------------------------------------------------------------------- -# -{ restore_shell_opts; } > /dev/null 2>&1 - -} -# -#----------------------------------------------------------------------- -# -# Call the function defined above. -# -#----------------------------------------------------------------------- -# -compare_config_scripts - diff --git a/ush/setup.sh b/ush/setup.sh index 9926ff90b..4541ec652 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -63,6 +63,7 @@ Starting function ${func_name}() in \"${scrfunc_fn}\"... # #----------------------------------------------------------------------- # +. ./check_expt_config_vars.sh . ./set_cycle_dates.sh . ./set_gridparams_GFDLgrid.sh . ./set_gridparams_ESGgrid.sh @@ -111,7 +112,9 @@ if [ -f "${EXPT_CONFIG_FN}" ]; then # configuration file are also assigned default values in the default # configuration file. # - . ./compare_config_scripts.sh + check_expt_config_vars \ + default_config_fp="./${EXPT_DEFAULT_CONFIG_FN}" \ + config_fp="./${EXPT_CONFIG_FN}" # # Now source the user-specified configuration file. # @@ -2528,42 +2531,17 @@ GLOBAL_VAR_DEFNS_FP="$EXPTDIR/${GLOBAL_VAR_DEFNS_FN}" # from the default experiment configuration file (EXPT_DEFAULT_CONFIG_FN). # By "primary", we mean those variables that are defined in the default # configuration file and can be reset in the user-specified experiment -# configuration file (EXPT_CONFIG_FN). The default values will be -# updated below to user-specified ones and the result saved in the -# experiment's variable definitions file. +# configuration file (EXPT_CONFIG_FN). The default values will be updated +# below to user-specified ones and the result saved in the experiment's +# variable definitions file. # #----------------------------------------------------------------------- # print_info_msg " Creating list of default experiment variable definitions..." -# -# Read in all lines in the default experiment configuration file into the -# variable default_config_file_contents. In doing so: -# -# 1) Concatenate any line ending with the bash line continuation character -# (a backslash) with the following line. -# -# 2) Remove any leading and trailing whitespace. -# -# Note that these two actions are automatically performed by the "read" -# utility in the while-loop below. -# -default_config_file_contents=$( $SED -r -e "s/(.*)/\1/g" $USHDIR/${EXPT_DEFAULT_CONFIG_FN} ) -tmp="" -while read crnt_line; do - tmp="$tmp${crnt_line} -" -done <<< "${default_config_file_contents}" -default_config_file_contents="$tmp" -# -# Generate a variable (default_var_defns) containing a list of experiment -# variable definitions by stripping out any comment and empty lines from -# default_config_file_contents. -# -default_var_defns=$( printf "${default_config_file_contents}" | \ - $SED -r -e "/^#.*/d" `# Remove comment lines.` \ - -e "/^$/d" `# Remove empty lines.` \ - ) + +get_bash_file_contents fp="$USHDIR/${EXPT_DEFAULT_CONFIG_FN}" \ + output_varname_contents="default_var_defns" print_info_msg "$DEBUG" " The variable \"default_var_defns\" containing default values of primary @@ -2587,11 +2565,11 @@ ${default_var_defns} # # MY_VAR="\${ANOTHER_VAR}" # -# while var_defns_notempl is for variables whose definitions contain -# only a literal string. These two types are treated separately -# because the template variables must be written at the end of the -# variable defintions file. This is in order to ensure that the -# (non-template) variables they reference have already been defined. +# while var_defns_notempl is for variables whose definitions contain only +# a literal string. These two types are treated separately because the +# template variables must be written at the end of the variable defintions +# file. This is in order to ensure that the (non-template) variables +# they reference have already been defined. # #----------------------------------------------------------------------- # diff --git a/ush/source_util_funcs.sh b/ush/source_util_funcs.sh index 375543d35..60162aa40 100644 --- a/ush/source_util_funcs.sh +++ b/ush/source_util_funcs.sh @@ -211,6 +211,15 @@ function source_util_funcs() { #----------------------------------------------------------------------- # . ${bashutils_dir}/create_symlink_to_file.sh +# +#----------------------------------------------------------------------- +# +# Source the file containing the function that gets the stripped contents +# of a bash script or function. +# +#----------------------------------------------------------------------- +# + . ${bashutils_dir}/get_bash_file_contents.sh } source_util_funcs From 11fbbd41744ed91e2978cf024e77d764e3435a87 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Mon, 13 Dec 2021 10:54:35 -0700 Subject: [PATCH 16/35] Bug fix. --- tests/WE2E/get_WE2Etest_names_subdirs_descs.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh b/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh index 257cd8912..2183ead89 100755 --- a/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh +++ b/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh @@ -759,7 +759,7 @@ This is probably because it is a directory. Please correct and rerun." test_names=("${prim_test_names[@]}") test_subdirs=("${prim_test_subdirs[@]}") if [ "${num_alt_tests}" -gt "0" ]; then - test_names+=("${alt_test_subdirs[@]:-}") + test_names+=("${alt_test_names[@]:-}") test_subdirs+=("${alt_test_subdirs[@]:-}") fi # @@ -1025,7 +1025,7 @@ Please correct and rerun." # listed first. # # Finally, we extract from test_ids_and_inds_sorted the second number -# in each element (the one afte the first number, which is the test ID, +# in each element (the one after the first number, which is the test ID, # and the test type, which we no longer need), which is the original # array index before sorting, and save the results in the array sort_inds. # This array will contain the original indices in sorted order that we From 06a3d2a88e2005e853a3deb545b13f7f8ed2eae6 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Mon, 13 Dec 2021 10:55:03 -0700 Subject: [PATCH 17/35] Fix typo in message. --- tests/WE2E/run_WE2E_tests.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/WE2E/run_WE2E_tests.sh b/tests/WE2E/run_WE2E_tests.sh index b8eaeec74..dd1f50613 100755 --- a/tests/WE2E/run_WE2E_tests.sh +++ b/tests/WE2E/run_WE2E_tests.sh @@ -538,8 +538,8 @@ accordingly and rerun." if [ "${match_found}" = "FALSE" ]; then avail_WE2E_test_names_str=$( printf " \"%s\"\n" "${avail_WE2E_test_names[@]}" ) print_err_msg_exit "\ -The name current user-specified test to run (user_spec_test) does not -match any of the names (either primary or alternate) of the available +The name of the current user-specified test to run (user_spec_test) does +not match any of the names (either primary or alternate) of the available WE2E tests: user_spec_test = \"${user_spec_test}\" Valid values for user_spec_test consist of the names (primary or alternate) From dc69b5f0eb8a2d2a1a23e5a627ce187830cb6754 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Mon, 13 Dec 2021 11:00:18 -0700 Subject: [PATCH 18/35] Typo fixes to comments and output messages. --- ush/bash_utils/check_var_valid_value.sh | 2 +- ush/check_expt_config_vars.sh | 6 +++--- ush/setup.sh | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ush/bash_utils/check_var_valid_value.sh b/ush/bash_utils/check_var_valid_value.sh index 4ea6068a1..63c4857c1 100644 --- a/ush/bash_utils/check_var_valid_value.sh +++ b/ush/bash_utils/check_var_valid_value.sh @@ -112,7 +112,7 @@ The value specified in ${var_name} is not supported: # assume the corresponding variable (var_name) is a template variable, # i.e. one whose value contains a reference to another variable, e.g. # -# MY_VAR="\${ANOTHER_VAR}" +# MY_VAR='\${ANOTHER_VAR}' # # [In bash, VAR_REF_CHAR is a (escaped) dollar sign.] In this case, it # does not make sense to check whether var_value is a valid value since diff --git a/ush/check_expt_config_vars.sh b/ush/check_expt_config_vars.sh index 8b996855d..91dfde89d 100644 --- a/ush/check_expt_config_vars.sh +++ b/ush/check_expt_config_vars.sh @@ -135,9 +135,9 @@ Skipping to next line." regex_search="^${var_name}=" grep "${regex_search}" <<< "${var_list_default}" > /dev/null 2>&1 || \ print_err_msg_exit "\ -A variable (var_name) in the user-specified experiment configuration file -(config_fp) does not appear in the default experiment configuration file -(default_config_fp): +The variable (var_name) defined on the current line (crnt_line) of the +user-specified experiment configuration file (config_fp) does not appear +in the default experiment configuration file (default_config_fp): config_fp = \"${config_fp}\" default_config_fp = \"${default_config_fp}\" crnt_line = \"${crnt_line}\" diff --git a/ush/setup.sh b/ush/setup.sh index 4541ec652..1faebc73a 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -2563,7 +2563,7 @@ ${default_var_defns} # var_defns_templ is for template variables, i.e. variables that contain # references to other variables in their definitions, e.g. # -# MY_VAR="\${ANOTHER_VAR}" +# MY_VAR='\${ANOTHER_VAR}' # # while var_defns_notempl is for variables whose definitions contain only # a literal string. These two types are treated separately because the @@ -3116,7 +3116,7 @@ N_VAR_SPP=\"${N_VAR_SPP}\" # This section defines template variables, i.e. variables that contain # references to other variables in their definitions, e.g. # -# MY_VAR="\${ANOTHER_VAR}" +# MY_VAR='\${ANOTHER_VAR}' # # They are placed towards the end of this file to ensure that the # referenced variables have already been defined. From 0a6db5633ea47954b20649a6124b35256798c987 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Mon, 13 Dec 2021 11:01:43 -0700 Subject: [PATCH 19/35] Modify the WE2E test deactivate_tasks so that it can also be used to test use of template variables in the experiment configuration file. Include detailed comments on how template variables can be used. Add a alternate test (symlink) named template_vars that points to the deactivate_tasks test. --- .../wflow_features/config.deactivate_tasks.sh | 101 ++++++++++++++++-- .../wflow_features/config.template_vars.sh | 1 + 2 files changed, 96 insertions(+), 6 deletions(-) create mode 120000 tests/WE2E/test_configs/wflow_features/config.template_vars.sh diff --git a/tests/WE2E/test_configs/wflow_features/config.deactivate_tasks.sh b/tests/WE2E/test_configs/wflow_features/config.deactivate_tasks.sh index 52aad5cfa..d38d924f8 100644 --- a/tests/WE2E/test_configs/wflow_features/config.deactivate_tasks.sh +++ b/tests/WE2E/test_configs/wflow_features/config.deactivate_tasks.sh @@ -2,10 +2,26 @@ # TEST PURPOSE/DESCRIPTION: # ------------------------ # -# This test ensures that the various workflow tasks can be deactivated, -# i.e. removed from the Rocoto XML. Note that we leave the MAKE_GRID_TN, -# MAKE_OROG_TN, and MAKE_SFC_CLIMO_TN activated because there is a -# separate test for turning those off. +# This test has two purposes: +# +# 1) It checks that the various workflow tasks can be deactivated, i.e. +# removed from the Rocoto XML. +# 2) It checks the capability of the workflow to use "template" experiment +# variables, i.e. variables whose definitions include references to +# other variables, e.g. +# +# MY_VAR='\${ANOTHER_VAR}' +# +# Note that we do not deactivate all tasks in the workflow; we leave the +# MAKE_GRID_TN, MAKE_OROG_TN, and MAKE_SFC_CLIMO_TN activated because: +# +# 1) There is already a WE2E test that runs with these three tasks +# deactivated (that test is to ensure that pre-generated grid, +# orography, and surface climatology files can be used). +# 2) In checking the template variable capability, we want to make sure +# that the variable defintions file (GLOBAL_VAR_DEFNS_FN) generated +# does not have syntax or other errors in it by sourcing it in these +# three tasks. # RUN_ENVIR="community" @@ -14,13 +30,86 @@ PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_25km" CCPP_PHYS_SUITE="FV3_GFS_v15p2" -DATE_FIRST_CYCL="20190615" -DATE_LAST_CYCL="20190615" +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" +USE_USER_STAGED_EXTRN_FILES="TRUE" + +DATE_FIRST_CYCL="20190701" +DATE_LAST_CYCL="20190701" CYCL_HRS=( "00" ) +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" + RUN_TASK_GET_EXTRN_ICS="FALSE" RUN_TASK_GET_EXTRN_LBCS="FALSE" RUN_TASK_MAKE_ICS="FALSE" RUN_TASK_MAKE_LBCS="FALSE" RUN_TASK_RUN_FCST="FALSE" RUN_TASK_RUN_POST="FALSE" +# +# The following shows examples of how to define template variables. Here, +# we define RUN_CMD_UTILS, RUN_CMD_FCST, and RUN_CMD_POST as template +# variables. Note that these variables aren't actually needed to run +# any commands but are simply included for demonstration purposes. +# +# If we want the contents in RUN_CMD_UTILS, RUN_CMD_FCST, and RUN_CMD_POST +# to be expanded and/or executed when the variable definitions file +# (GLOBAL_VAR_DEFNS_FN) is sourced in a script or function, then we must +# escape bash's variable reference character "$" using backslashes, as +# follows: +# +# RUN_CMD_UTILS="echo \$yyyymmdd" +# RUN_CMD_FCST="mpirun -np \${PE_MEMBER01}" +# RUN_CMD_POST="\$( echo hello \$yyymmdd )" +# +# With this method, when the variable defintions file is sourced, the +# variable references on the right-hand sides (here $yyyymmdd and +# ${PE_MEMBER01}) as well as the command substitution [i.e. $( echo ...)] +# will first be evaluated/expanded and the results then combined with +# the remainder of each string on the right-hand side (if any) to obtain +# literal strings to assign to RUN_CMD_UTILS, RUN_CMD_FCST, and RUN_CMD_POST. +# Thus, the values that these three variables are assigned in the variable +# definitions file will not contain any variable references; they will +# contain only literal strings. +# +# In general, this is not what we want because the variables on the right- +# hand sides may not yet be defined at the time the variable definitions +# file is sourced. Thus, what we usually want is for variable references +# and command substitutions appearing on the right-hand sides in that +# file to be escaped. To do this, we must escape the "$" character twice, +# as follows: +# +# RUN_CMD_UTILS="echo \\\$yyyymmdd" +# RUN_CMD_FCST="mpirun -np \\\${PE_MEMBER01}" +# RUN_CMD_POST="\\\$( echo hello \\\$yyyymmdd )" +# +# A more compact way to do this is to use single quotes on the outside +# and eliminate the escaped-\ within, i.e. +# +# RUN_CMD_UTILS='echo \$yyyymmdd' +# RUN_CMD_FCST='mpirun -np \${PE_MEMBER01}' +# RUN_CMD_POST='\$( echo hello \$yyyymmdd )' +# +# This will cause $yyyymmdd, ${PE_MEMBER01}, and $(...) to be expanded +# only at the time RUN_CMD_UTILS, RUN_CMD_FCST, and RUN_CMD_POST are +# referenced in a file or script (using the syntax ${RUN_CMD_UTILS}, ...) +# and not earlier when the variable definitions file is sourced. That +# is what we do in this WE2E test. +# +# With the last set of settings above, in a given script or function we +# can source the variable defintions file and then use the three variables +# as follows: +# +# . $exptdir/var_defns.sh +# yyyymmdd="goodbye" +# eval ${RUN_CMD_UTILS} +# eval ${RUN_CMD_FCST} +# eval myvar=${RUN_CMD_POST} +# +# (The mpirun command in RUN_CMD_FCST will generate an error, but that is +# ok since using mpirun is not the primary concern here.) +# +RUN_CMD_UTILS='echo \$yyyymmdd' +RUN_CMD_FCST='mpirun -np \${PE_MEMBER01}' +RUN_CMD_POST='\$( echo hello \$yyyymmdd )' diff --git a/tests/WE2E/test_configs/wflow_features/config.template_vars.sh b/tests/WE2E/test_configs/wflow_features/config.template_vars.sh new file mode 120000 index 000000000..80ede5437 --- /dev/null +++ b/tests/WE2E/test_configs/wflow_features/config.template_vars.sh @@ -0,0 +1 @@ +config.deactivate_tasks.sh \ No newline at end of file From 40588ef15dbe7f2b159b55e97459fb83267e7a3b Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Mon, 13 Dec 2021 11:05:54 -0700 Subject: [PATCH 20/35] Add forgotten file. --- ush/bash_utils/get_bash_file_contents.sh | 119 +++++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 ush/bash_utils/get_bash_file_contents.sh diff --git a/ush/bash_utils/get_bash_file_contents.sh b/ush/bash_utils/get_bash_file_contents.sh new file mode 100644 index 000000000..afd55aa7c --- /dev/null +++ b/ush/bash_utils/get_bash_file_contents.sh @@ -0,0 +1,119 @@ +# +#----------------------------------------------------------------------- +# +# This file defines a function that returns the contents of a bash script/ +# function with all empty lines, comment lines, and leading and trailing +# whitespace removed. Arguments are as follows: +# +# fp: +# The relative or full path to the file containing the bash script or +# function. +# +# output_varname_contents: +# Name of the output variable that will contain the (processed) contents +# of the file. This is the output of the function. +# +#----------------------------------------------------------------------- +# +function get_bash_file_contents() { +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# + { save_shell_opts; set -u +x; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Specify the set of valid argument names for this script/function. Then +# process the arguments provided to this script/function (which should +# consist of a set of name-value pairs of the form arg1="value1", etc). +# +#----------------------------------------------------------------------- +# + local valid_args=( \ + "fp" \ + "output_varname_contents" \ + ) + process_args valid_args "$@" +# +#----------------------------------------------------------------------- +# +# For debugging purposes, print out values of arguments passed to this +# script. Note that these will be printed out only if VERBOSE is set to +# TRUE. +# +#----------------------------------------------------------------------- +# + print_input_args "valid_args" +# +#----------------------------------------------------------------------- +# +# Verify that the required arguments to this function have been specified. +# If not, print out an error message and exit. +# +#----------------------------------------------------------------------- +# + if [ -z "$fp" ]; then + print_err_msg_exit "\ +The argument \"fp\" specifying the relative or full path to the file to +read was not specified in the call to this function: + fp = \"$fp\"" + fi +# +#----------------------------------------------------------------------- +# +# Declare local variables. +# +#----------------------------------------------------------------------- +# + local contents \ + crnt_line +# +#----------------------------------------------------------------------- +# +# Read in all lines in the file. In doing so: +# +# 1) Concatenate any line ending with the bash line continuation character +# (a backslash) with the following line. +# 2) Remove any leading and trailing whitespace. +# +# Note that these two actions are automatically performed by the "read" +# utility in the while-loop below. +# +#----------------------------------------------------------------------- +# + contents="" + while read crnt_line; do + contents="${contents}${crnt_line} +" + done < "$fp" +# +# Strip out any comment and empty lines from contents. +# + contents=$( printf "${contents}" | \ + $SED -r -e "/^#.*/d" `# Remove comment lines.` \ + -e "/^$/d" `# Remove empty lines.` \ + ) +# +#----------------------------------------------------------------------- +# +# Set output variables. +# +#----------------------------------------------------------------------- +# + printf -v ${output_varname_contents} "${contents}" +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/function. +# +#----------------------------------------------------------------------- +# + { restore_shell_opts; } > /dev/null 2>&1 + +} + From f37de7e6b085c98aec1cdc8e5118c7b9f81cc447 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Mon, 13 Dec 2021 15:01:04 -0700 Subject: [PATCH 21/35] For clarity, change variable name from VAR_REF_CHAR to VARVALUE_REF_CHAR. --- ush/bash_utils/check_var_valid_value.sh | 18 +++++++++--------- ush/constants.sh | 7 +++---- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/ush/bash_utils/check_var_valid_value.sh b/ush/bash_utils/check_var_valid_value.sh index 63c4857c1..41e67d022 100644 --- a/ush/bash_utils/check_var_valid_value.sh +++ b/ush/bash_utils/check_var_valid_value.sh @@ -108,22 +108,23 @@ The value specified in ${var_name} is not supported: # #----------------------------------------------------------------------- # -# If var_value contains the character specified by VAR_REF_CHAR, we +# If var_value contains the character specified by VARVALUE_REF_CHAR, we # assume the corresponding variable (var_name) is a template variable, # i.e. one whose value contains a reference to another variable, e.g. # # MY_VAR='\${ANOTHER_VAR}' # -# [In bash, VAR_REF_CHAR is a (escaped) dollar sign.] In this case, it -# does not make sense to check whether var_value is a valid value since +# [In bash, VARVALUE_REF_CHAR is a (escaped) dollar sign.] In this case, +# it does not make sense to check whether var_value is a valid value since # its contents have not yet been expanded. If var_value doesn't contain -# VAR_REF_CHAR, it must contain a literal string. In this case, we check -# whether it is equal to one of the elements of the array valid_var_values. -# If not, we print out an error message and exit the calling script. +# VARVALUE_REF_CHAR, it must contain a literal string. In this case, we +# check whether it is equal to one of the elements of the array +# valid_var_values. If not, we print out an error message and exit the +# calling script. # #----------------------------------------------------------------------- # - if [[ "${var_value}" != *"${VAR_REF_CHAR}"* ]]; then + if [[ "${var_value}" != *"${VARVALUE_REF_CHAR}"* ]]; then is_element_of "valid_var_values" "${var_value}" || { \ valid_var_values_str=$(printf "\"%s\" " "${valid_var_values[@]}"); print_err_msg_exit "\ @@ -135,8 +136,7 @@ ${var_name} must be set to one of the following: # #----------------------------------------------------------------------- # -# Restore the shell options saved at the beginning of this script/func- -# tion. +# Restore the shell options saved at the beginning of this script/function. # #----------------------------------------------------------------------- # diff --git a/ush/constants.sh b/ush/constants.sh index 5d32b3375..f64f77168 100644 --- a/ush/constants.sh +++ b/ush/constants.sh @@ -18,9 +18,8 @@ radius_Earth="6371200.0" # #----------------------------------------------------------------------- # -# Character used to identify a reference (in bash) to a variable in a -# given string. In bash, this is the dollar sign. For example, if the -# string is +# Character (in bash) used in a string to reference the value stored in +# a variable. This is the dollar sign. For example, if the string is # # "Hello, my name is $NAME." # @@ -29,5 +28,5 @@ radius_Earth="6371200.0" # #----------------------------------------------------------------------- # -VAR_REF_CHAR="\$" +VARVALUE_REF_CHAR="\$" From 03d0767c39c2992f1e2dda5794cbc1459f047ee4 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Thu, 16 Dec 2021 09:57:55 -0700 Subject: [PATCH 22/35] Fix up comments. --- ush/setup.sh | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/ush/setup.sh b/ush/setup.sh index 686563787..05373d809 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -992,8 +992,7 @@ fi # Directory containing various executable files. # # TEMPLATE_DIR: -# Directory in which templates of various FV3-LAM input files are locat- -# ed. +# Directory in which templates of various FV3-LAM input files are located. # # UFS_WTHR_MDL_DIR: # Directory in which the (NEMS-enabled) FV3-LAM application is located. @@ -1559,20 +1558,20 @@ fi # # The workflow contains templates for the first six of these files. # Template files are versions of these files that contain placeholder -# (i.e. dummy) values for various parameters. The experiment/workflow -# generation scripts copy these templates to appropriate locations in -# the experiment directory (either the top of the experiment directory -# or one of the cycle subdirectories) and replace the placeholders in -# these copies by actual values specified in the experiment/workflow -# configuration file (or derived from such values). The scripts then -# use the resulting "actual" files as inputs to the forecast model. -# -# Note that the CCPP physics suite defintion file does not have a cor- -# responding template file because it does not contain any values that -# need to be replaced according to the experiment/workflow configura- -# tion. If using CCPP, this file simply needs to be copied over from -# its location in the forecast model's directory structure to the ex- -# periment directory. +# (i.e. dummy) values for various parameters. The experiment generation +# scripts copy these templates to appropriate locations in the experiment +# directory (either the top of the experiment directory or one of the +# cycle subdirectories) and replace the placeholders in these copies by +# actual values specified in the experiment configuration file (or derived +# from such values). The scripts then use the resulting "actual" files +# as inputs to the forecast model. +# +# Note that the CCPP physics suite defintion file does not have a +# corresponding template file because it does not contain any values +# that need to be replaced according to the experiment configuration. +# If using CCPP, this file simply needs to be copied over from its +# location in the forecast model's directory structure to the experiment +# directory. # # Below, we first set the names of the templates for the first six files # listed above. We then set the full paths to these template files. From bc2a78f541f0bc91250899e3d685e61b8d62a61b Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Thu, 16 Dec 2021 10:51:17 -0700 Subject: [PATCH 23/35] Enclose the template RUN_CMD_FCST in single quotes instead of double to ensure that it is not expanded when var_defns.sh is sourced but later when the eval command is called on RUN_CMD_FCST. --- ush/config_defaults.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index ea0f9baa6..db7837e55 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -168,7 +168,7 @@ QUEUE_FCST="" #----------------------------------------------------------------------- # RUN_CMD_UTILS="mpirun -np 1" -RUN_CMD_FCST="mpirun -np \${PE_MEMBER01}" +RUN_CMD_FCST='mpirun -np \${PE_MEMBER01}' RUN_CMD_POST="mpirun -np 1" # #----------------------------------------------------------------------- From 3d5c2c311afdde2e07d3de7e2aa1be8e23d2c3ba Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Thu, 16 Dec 2021 10:52:38 -0700 Subject: [PATCH 24/35] Use VARVALUE_REF_CHAR when searching for template variables; clean up comments. --- ush/setup.sh | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/ush/setup.sh b/ush/setup.sh index 05373d809..c6b499cdf 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -1414,7 +1414,7 @@ must set DT_SUBHOURLY_POST_MNTS to something other than zero." # # For now, the sub-hourly capability is restricted to having values of # DT_SUBHOURLY_POST_MNTS that evenly divide into 60 minutes. This is -# because the jinja rocoto XML template (FV3LAM_wflow.xml) assumes that +# because the jinja rocoto XML template (${WFLOW_XML_FN}) assumes that # model output is generated at the top of every hour (i.e. at 00 minutes). # This restricts DT_SUBHOURLY_POST_MNTS to the following values (inluding # both cases with and without a leading 0): @@ -2599,10 +2599,24 @@ ${default_var_defns} # MY_VAR='\${ANOTHER_VAR}' # # while var_defns_notempl is for variables whose definitions contain only -# a literal string. These two types are treated separately because the -# template variables must be written at the end of the variable defintions -# file. This is in order to ensure that the (non-template) variables -# they reference have already been defined. +# a literal string. These two types are treated separately. Non-template +# variables are written to the variable definitions file as they are +# encountered, whereas template variables (or simply "templates") are +# saved in an array and written at the end of the file. This is to +# ensure that any non-template variables that the templates reference +# are defined before the templates (so that undefined variable errors +# are not encountered). If a template is defined as above, i.e. enclosed +# in SINGLE quotes and with dollar signs escaped, this should not matter +# because variable references will not be expanded when the variable +# definitions file is sourced; they will be expanded only when the "eval" +# built-in command is used to evaluate the contents of the template. +# For this reason, use of single quotes is the recommended way of defining +# a template in the experiment configuration file. However, in case +# a template is enclosed in double quotes, any varaible references will +# be expanded/evaluated when the variable definitions file is sourced. +# In this case, unexpected behavior or failure may be encountered if the +# referenced variables are not already defined. To cover this scenario, +# we place the templates all at the end of the variable definitions file. # #----------------------------------------------------------------------- # @@ -2629,10 +2643,6 @@ fi # variables and values either in var_defns_notempl (if it is not a # template variable, i.e. one that references other variables in its # definition) or in var_defns_templ (if it is a template variable). -# Note that template variables are separated from non-template ones -# because they must be written at the end of the variable defintions -# file to ensure that the variables they reference have already been -# defined. # var_defns_notempl="" var_defns_templ="" @@ -2744,7 +2754,7 @@ Setting its value in the variable definitions file to an empty string." # referenced variable is one of the experiment variables). # dollar_or_null=$( printf "%s" "${var_value}" | \ - $SED -n -r -e "s/[^\$]*(\$).*/\1/p" ) + $SED -n -r -e "s/[^${VARVALUE_REF_CHAR}]*(${VARVALUE_REF_CHAR}).*/\1/p" ) if [ -z "${dollar_or_null}" ]; then printf -v "var_defns_notempl" "${var_defns_notempl}${var_defn}\n" else @@ -3151,8 +3161,6 @@ N_VAR_SPP=\"${N_VAR_SPP}\" # # MY_VAR='\${ANOTHER_VAR}' # -# They are placed towards the end of this file to ensure that the -# referenced variables have already been defined. #----------------------------------------------------------------------- #----------------------------------------------------------------------- # From dd5f7641d16e6321577a5ab3c57412d7a81eda2c Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Thu, 16 Dec 2021 11:42:06 -0700 Subject: [PATCH 25/35] Clean up comments regarding use of templates in the WE2E configuration file for templates. We trim most of the comments since there will be a companion PR into ufs-srweather-app that will include RST documentation for templates. --- .../wflow_features/config.deactivate_tasks.sh | 69 +++---------------- 1 file changed, 8 insertions(+), 61 deletions(-) diff --git a/tests/WE2E/test_configs/wflow_features/config.deactivate_tasks.sh b/tests/WE2E/test_configs/wflow_features/config.deactivate_tasks.sh index d38d924f8..9cd339341 100644 --- a/tests/WE2E/test_configs/wflow_features/config.deactivate_tasks.sh +++ b/tests/WE2E/test_configs/wflow_features/config.deactivate_tasks.sh @@ -50,66 +50,13 @@ RUN_TASK_RUN_POST="FALSE" # # The following shows examples of how to define template variables. Here, # we define RUN_CMD_UTILS, RUN_CMD_FCST, and RUN_CMD_POST as template -# variables. Note that these variables aren't actually needed to run -# any commands but are simply included for demonstration purposes. -# -# If we want the contents in RUN_CMD_UTILS, RUN_CMD_FCST, and RUN_CMD_POST -# to be expanded and/or executed when the variable definitions file -# (GLOBAL_VAR_DEFNS_FN) is sourced in a script or function, then we must -# escape bash's variable reference character "$" using backslashes, as -# follows: -# -# RUN_CMD_UTILS="echo \$yyyymmdd" -# RUN_CMD_FCST="mpirun -np \${PE_MEMBER01}" -# RUN_CMD_POST="\$( echo hello \$yyymmdd )" -# -# With this method, when the variable defintions file is sourced, the -# variable references on the right-hand sides (here $yyyymmdd and -# ${PE_MEMBER01}) as well as the command substitution [i.e. $( echo ...)] -# will first be evaluated/expanded and the results then combined with -# the remainder of each string on the right-hand side (if any) to obtain -# literal strings to assign to RUN_CMD_UTILS, RUN_CMD_FCST, and RUN_CMD_POST. -# Thus, the values that these three variables are assigned in the variable -# definitions file will not contain any variable references; they will -# contain only literal strings. -# -# In general, this is not what we want because the variables on the right- -# hand sides may not yet be defined at the time the variable definitions -# file is sourced. Thus, what we usually want is for variable references -# and command substitutions appearing on the right-hand sides in that -# file to be escaped. To do this, we must escape the "$" character twice, -# as follows: -# -# RUN_CMD_UTILS="echo \\\$yyyymmdd" -# RUN_CMD_FCST="mpirun -np \\\${PE_MEMBER01}" -# RUN_CMD_POST="\\\$( echo hello \\\$yyyymmdd )" -# -# A more compact way to do this is to use single quotes on the outside -# and eliminate the escaped-\ within, i.e. -# -# RUN_CMD_UTILS='echo \$yyyymmdd' -# RUN_CMD_FCST='mpirun -np \${PE_MEMBER01}' -# RUN_CMD_POST='\$( echo hello \$yyyymmdd )' -# -# This will cause $yyyymmdd, ${PE_MEMBER01}, and $(...) to be expanded -# only at the time RUN_CMD_UTILS, RUN_CMD_FCST, and RUN_CMD_POST are -# referenced in a file or script (using the syntax ${RUN_CMD_UTILS}, ...) -# and not earlier when the variable definitions file is sourced. That -# is what we do in this WE2E test. -# -# With the last set of settings above, in a given script or function we -# can source the variable defintions file and then use the three variables -# as follows: -# -# . $exptdir/var_defns.sh -# yyyymmdd="goodbye" -# eval ${RUN_CMD_UTILS} -# eval ${RUN_CMD_FCST} -# eval myvar=${RUN_CMD_POST} -# -# (The mpirun command in RUN_CMD_FCST will generate an error, but that is -# ok since using mpirun is not the primary concern here.) -# -RUN_CMD_UTILS='echo \$yyyymmdd' +# variables. Note that during this test, these templates aren't actually +# expanded/used (using bash's "eval" built-in command) anywhere in the +# scripts. They are included here simply included for demonstration +# purposes. If all goes well, the experiment generation system should +# place these templates at the end of the experiment's variable definitions +# file. +# +RUN_CMD_UTILS='cd \$yyyymmdd' RUN_CMD_FCST='mpirun -np \${PE_MEMBER01}' RUN_CMD_POST='\$( echo hello \$yyyymmdd )' From efdd5e1baeffb4b320ebca7d7998eea38ae59071 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Thu, 16 Dec 2021 14:02:52 -0700 Subject: [PATCH 26/35] Indent comments and remove obvious comments. --- ush/check_expt_config_vars.sh | 117 ++++++++++------------------------ 1 file changed, 33 insertions(+), 84 deletions(-) diff --git a/ush/check_expt_config_vars.sh b/ush/check_expt_config_vars.sh index 91dfde89d..53ce13a09 100644 --- a/ush/check_expt_config_vars.sh +++ b/ush/check_expt_config_vars.sh @@ -24,89 +24,45 @@ #----------------------------------------------------------------------- # function check_expt_config_vars() { -# -#----------------------------------------------------------------------- -# -# Source bash utility functions. -# -#----------------------------------------------------------------------- -# + . ${scrfunc_dir}/source_util_funcs.sh -# -#----------------------------------------------------------------------- -# -# Save current shell options (in a global array). Then set new options -# for this script/function. -# -#----------------------------------------------------------------------- -# + { save_shell_opts; set -u +x; } > /dev/null 2>&1 -# -#----------------------------------------------------------------------- -# -# Specify the set of valid argument names for this script/function. Then -# process the arguments provided to this script/function (which should -# consist of a set of name-value pairs of the form arg1="value1", etc). -# -#----------------------------------------------------------------------- -# + local valid_args=( \ "default_config_fp" \ "config_fp" \ ) process_args valid_args "$@" -# -#----------------------------------------------------------------------- -# -# For debugging purposes, print out values of arguments passed to this -# script. Note that these will be printed out only if VERBOSE is set to -# TRUE. -# -#----------------------------------------------------------------------- -# print_input_args "valid_args" -# -#----------------------------------------------------------------------- -# -# Declare local variables. -# -#----------------------------------------------------------------------- -# + local var_list_default \ var_list_user \ crnt_line \ var_name \ regex_search -# -#----------------------------------------------------------------------- -# -# Get the list of variable definitions, first from the default experiment -# configuration file and then from the user-specified experiment -# configuration file. -# -#----------------------------------------------------------------------- -# + # + # Get the list of variable definitions, first from the default experiment + # configuration file and then from the user-specified experiment + # configuration file. + # get_bash_file_contents fp="${default_config_fp}" \ output_varname_contents="var_list_default" get_bash_file_contents fp="${config_fp}" \ output_varname_contents="var_list_user" -# -#----------------------------------------------------------------------- -# -# Loop through each line/variable in var_list_user. For each line, -# extract the the name of the variable that is being set (say VAR) and -# check that this variable is set somewhere in the default configuration -# file by verifying that a line that starts with "VAR=" exists in -# var_list_default. -# -#----------------------------------------------------------------------- -# + # + # Loop through each line/variable in var_list_user. For each line, + # extract the the name of the variable that is being set (say VAR) and + # check that this variable is set somewhere in the default configuration + # file by verifying that a line that starts with "VAR=" exists in + # var_list_default. + # while read crnt_line; do -# -# Note that a variable name will be found only if the equal sign immediately -# follows the variable name. -# + # + # Note that a variable name will be found only if the equal sign immediately + # follows the variable name. + # var_name=$( printf "%s" "${crnt_line}" | $SED -n -r -e "s/^([^ =\"]*)=.*/\1/p") if [ -z "${var_name}" ]; then @@ -120,18 +76,18 @@ file (config_fp) does not contain a variable name (i.e. var_name is empty): Skipping to next line." else -# -# Use grep to search for the variable name (followed by an equal sign, -# all at the beginning of a line) in the list of variables in the default -# configuration file. -# -# Note that we use a herestring to input into grep the list of variables -# in the default configuration file. grep will return with a zero status -# if the specified string (regex_search) is not found in the default -# variables list and a nonzero status otherwise. Note also that we -# redirect the output of grep to null because we are only interested in -# its exit status. -# + # + # Use grep to search for the variable name (followed by an equal sign, + # all at the beginning of a line) in the list of variables in the default + # configuration file. + # + # Note that we use a herestring to input into grep the list of variables + # in the default configuration file. grep will return with a zero status + # if the specified string (regex_search) is not found in the default + # variables list and a nonzero status otherwise. Note also that we + # redirect the output of grep to null because we are only interested in + # its exit status. + # regex_search="^${var_name}=" grep "${regex_search}" <<< "${var_list_default}" > /dev/null 2>&1 || \ print_err_msg_exit "\ @@ -148,14 +104,7 @@ file and rerun." fi done <<< "${var_list_user}" -# -#----------------------------------------------------------------------- -# -# Restore the shell options saved at the beginning of this script/func- -# tion. -# -#----------------------------------------------------------------------- -# + { restore_shell_opts; } > /dev/null 2>&1 } From e8855898c71e682e6780334940cb2f3183ad831b Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Thu, 16 Dec 2021 14:10:06 -0700 Subject: [PATCH 27/35] Indent comments; remove obvious comments. --- ush/bash_utils/get_bash_file_contents.sh | 96 ++++++------------------ 1 file changed, 24 insertions(+), 72 deletions(-) diff --git a/ush/bash_utils/get_bash_file_contents.sh b/ush/bash_utils/get_bash_file_contents.sh index afd55aa7c..3b3ab7b30 100644 --- a/ush/bash_utils/get_bash_file_contents.sh +++ b/ush/bash_utils/get_bash_file_contents.sh @@ -16,103 +16,55 @@ #----------------------------------------------------------------------- # function get_bash_file_contents() { -# -#----------------------------------------------------------------------- -# -# Save current shell options (in a global array). Then set new options -# for this script/function. -# -#----------------------------------------------------------------------- -# + { save_shell_opts; set -u +x; } > /dev/null 2>&1 -# -#----------------------------------------------------------------------- -# -# Specify the set of valid argument names for this script/function. Then -# process the arguments provided to this script/function (which should -# consist of a set of name-value pairs of the form arg1="value1", etc). -# -#----------------------------------------------------------------------- -# + local valid_args=( \ "fp" \ "output_varname_contents" \ ) process_args valid_args "$@" -# -#----------------------------------------------------------------------- -# -# For debugging purposes, print out values of arguments passed to this -# script. Note that these will be printed out only if VERBOSE is set to -# TRUE. -# -#----------------------------------------------------------------------- -# print_input_args "valid_args" -# -#----------------------------------------------------------------------- -# -# Verify that the required arguments to this function have been specified. -# If not, print out an error message and exit. -# -#----------------------------------------------------------------------- -# + # + # Verify that the required arguments to this function have been specified. + # If not, print out an error message and exit. + # if [ -z "$fp" ]; then print_err_msg_exit "\ The argument \"fp\" specifying the relative or full path to the file to read was not specified in the call to this function: fp = \"$fp\"" fi -# -#----------------------------------------------------------------------- -# -# Declare local variables. -# -#----------------------------------------------------------------------- -# + local contents \ crnt_line -# -#----------------------------------------------------------------------- -# -# Read in all lines in the file. In doing so: -# -# 1) Concatenate any line ending with the bash line continuation character -# (a backslash) with the following line. -# 2) Remove any leading and trailing whitespace. -# -# Note that these two actions are automatically performed by the "read" -# utility in the while-loop below. -# -#----------------------------------------------------------------------- -# + # + # Read in all lines in the file. In doing so: + # + # 1) Concatenate any line ending with the bash line continuation character + # (a backslash) with the following line. + # 2) Remove any leading and trailing whitespace. + # + # Note that these two actions are automatically performed by the "read" + # utility in the while-loop below. + # contents="" while read crnt_line; do contents="${contents}${crnt_line} " done < "$fp" -# -# Strip out any comment and empty lines from contents. -# + # + # Strip out any comment and empty lines from contents. + # contents=$( printf "${contents}" | \ $SED -r -e "/^#.*/d" `# Remove comment lines.` \ -e "/^$/d" `# Remove empty lines.` \ ) -# -#----------------------------------------------------------------------- -# -# Set output variables. -# -#----------------------------------------------------------------------- -# + # + # Set output variables. + # printf -v ${output_varname_contents} "${contents}" -# -#----------------------------------------------------------------------- -# -# Restore the shell options saved at the beginning of this script/function. -# -#----------------------------------------------------------------------- -# + { restore_shell_opts; } > /dev/null 2>&1 } From fd288dd2987d0d0d3a738640c94c7e2f1f013205 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Fri, 17 Dec 2021 04:11:57 -0700 Subject: [PATCH 28/35] Bug fix -- include variables in setup.sh left out from previous manual merge of develop into this (feature/template_vars) branch. --- ush/setup.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ush/setup.sh b/ush/setup.sh index 7efcb73d3..ce963599d 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -2887,8 +2887,11 @@ PARMDIR=\"$PARMDIR\" MODULES_DIR=\"${MODULES_DIR}\" EXECDIR=\"$EXECDIR\" FIXam=\"$FIXam\" +FIXclim=\"$FIXclim\" FIXLAM=\"$FIXLAM\" FIXgsm=\"$FIXgsm\" +FIXaer=\"$FIXaer\" +FIXlut=\"$FIXlut\" COMROOT=\"$COMROOT\" COMOUT_BASEDIR=\"${COMOUT_BASEDIR}\" TEMPLATE_DIR=\"${TEMPLATE_DIR}\" From d1bd7b55faec18f8689b5ade65be5a9a9f8da7ba Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Fri, 17 Dec 2021 10:48:27 -0700 Subject: [PATCH 29/35] Fixes to comments and output messages (avoid hyphenation). --- ush/launch_FV3LAM_wflow.sh | 117 +++++++++++++++++++------------------ 1 file changed, 61 insertions(+), 56 deletions(-) diff --git a/ush/launch_FV3LAM_wflow.sh b/ush/launch_FV3LAM_wflow.sh index 87a15dfbb..4c881be2d 100755 --- a/ush/launch_FV3LAM_wflow.sh +++ b/ush/launch_FV3LAM_wflow.sh @@ -18,7 +18,13 @@ set -u #----------------------------------------------------------------------- # if [[ $(uname -s) == Darwin ]]; then - command -v greadlink >/dev/null 2>&1 || { echo >&2 "For Darwin-based operating systems (MacOS), the 'greadlink' utility is required to run the UFS SRW Application. Reference the User's Guide for more information about platform requirements. Aborting."; exit 1; } + command -v greadlink >/dev/null 2>&1 || { \ + echo >&2 "\ +For Darwin-based operating systems (MacOS), the 'greadlink' utility is +required to run the UFS SRW Application. Reference the User's Guide for +more information about platform requirements. Aborting."; \ + exit 1; \ + } scrfunc_fp=$( greadlink -f "${BASH_SOURCE[0]}" ) else scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) @@ -28,27 +34,26 @@ scrfunc_dir=$( dirname "${scrfunc_fp}" ) ushdir="${scrfunc_dir}" . $ushdir/source_util_funcs.sh - # #----------------------------------------------------------------------- # # Get the experiment directory. We assume that there is a symlink to # this script in the experiment directory, and this script is called via -# that symlink. Thus, finding the directory in which the symlink is lo- -# cated will give us the experiment directory. We find this by first +# that symlink. Thus, finding the directory in which the symlink is +# located will give us the experiment directory. We find this by first # obtaining the directory portion (i.e. the portion without the name of # this script) of the command that was used to called this script (i.e. -# "$0") and then use the "readlink -f" command to obtain the correspond- -# ing absolute path. This will work for all four of the following ways -# in which the symlink in the experiment directory pointing to this -# script may be called: +# "$0") and then use the "readlink -f" command to obtain the corresponding +# absolute path. This will work for all four of the following ways in +# which the symlink in the experiment directory pointing to this script +# may be called: # # 1) Call this script from the experiment directory: # > cd /path/to/experiment/directory # > launch_FV3LAM_wflow.sh # -# 2) Call this script from the experiment directory but using "./" be- -# fore the script name: +# 2) Call this script from the experiment directory but using "./" before +# the script name: # > cd /path/to/experiment/directory # > ./launch_FV3LAM_wflow.sh # @@ -56,9 +61,8 @@ ushdir="${scrfunc_dir}" # symlink in the experiment directory: # > /path/to/experiment/directory/launch_FV3LAM_wflow.sh # -# 4) Call this script from a directory that is several levels up from -# the experiment directory (but not necessarily at the root directo- -# ry): +# 4) Call this script from a directory that is several levels up from the +# experiment directory (but not necessarily at the root directory): # > cd /path/to # > experiment/directory/launch_FV3LAM_wflow.sh # @@ -75,7 +79,13 @@ ushdir="${scrfunc_dir}" # exptdir=$( dirname "$0" ) if [[ $(uname -s) == Darwin ]]; then - command -v greadlink >/dev/null 2>&1 || { echo >&2 "For Darwin-based operating systems (MacOS), the 'greadlink' utility is required to run the UFS SRW Application. Reference the User's Guide for more information about platform requirements. Aborting."; exit 1; } + command -v greadlink >/dev/null 2>&1 || { \ + echo >&2 "\ +For Darwin-based operating systems (MacOS), the 'greadlink' utility is +required to run the UFS SRW Application. Reference the User's Guide for +more information about platform requirements. Aborting."; + exit 1; + } exptdir=$( greadlink -f "$exptdir" ) else exptdir=$( readlink -f "$exptdir" ) @@ -139,20 +149,19 @@ wflow_status="IN PROGRESS" # #----------------------------------------------------------------------- # -cd "$exptdir" +cd_vrfy "$exptdir" # #----------------------------------------------------------------------- # -# Issue the rocotorun command to (re)launch the next task in the -# workflow. Then check for error messages in the output of rocotorun. -# If any are found, it means the end-to-end run of the workflow failed. -# In this case, we remove the crontab entry that launches the workflow, -# and we append an appropriate failure message at the end of the launch -# log file. +# Issue the rocotorun command to (re)launch the next task in the workflow. +# Then check for error messages in the output of rocotorun. If any are +# found, it means the end-to-end run of the workflow failed. In this +# case, we remove the crontab entry that launches the workflow, and we +# append an appropriate failure message at the end of the launch log +# file. # #----------------------------------------------------------------------- # - tmp_fn="rocotorun_output.txt" rocotorun_cmd="rocotorun -w \"${WFLOW_XML_FN}\" -d \"${rocoto_database_fn}\" -v 10" eval ${rocotorun_cmd} > ${tmp_fn} 2>&1 @@ -237,15 +246,15 @@ ${rocotostat_output} # ... # # Thus, the first row is a header line containing the column titles, and -# the remaining rows each correspond to one cycle in the workflow. Be- -# low, we are interested in the first and second columns of each row. -# The first column is a string containing the start time of the cycle -# (in the format YYYYMMDDHHmm, where YYYY is the 4-digit year, MM is the -# 2-digit month, DD is the 2-digit day of the month, HH is the 2-digit -# hour of the day, and mm is the 2-digit minute of the hour). The se- -# cond column is a string containing the state of the cycle. This can -# be "Active" or "Done". Below, we read in and store these two columns -# in (1-D) arrays. +# the remaining rows each correspond to one cycle in the workflow. Below, +# we are interested in the first and second columns of each row. The +# first column is a string containing the start time of the cycle (in the +# format YYYYMMDDHHmm, where YYYY is the 4-digit year, MM is the 2-digit +# month, DD is the 2-digit day of the month, HH is the 2-digit hour of +# the day, and mm is the 2-digit minute of the hour). The second column +# is a string containing the state of the cycle. This can be "Active" +# or "Done". Below, we read in and store these two columns in (1-D) +# arrays. # #----------------------------------------------------------------------- # @@ -257,9 +266,9 @@ cycle_status=() i=0 while read -r line; do # -# Note that the first line in rocotostat_output is a header line con- -# taining the column titles. Thus, we ignore it and consider only the -# remaining lines (of which there is one per cycle). +# Note that the first line in rocotostat_output is a header line containing +# the column titles. Thus, we ignore it and consider only the remaining +# lines (of which there is one per cycle). # if [ $i -gt 0 ]; then im1=$((i-1)) @@ -287,9 +296,9 @@ done # #----------------------------------------------------------------------- # -# If the number of completed cycles is equal to the total number of cy- -# cles, it means the end-to-end run of the workflow was successful. In -# this case, we reset the wflow_status to "SUCCESS". +# If the number of completed cycles is equal to the total number of cycles, +# it means the end-to-end run of the workflow was successful. In this +# case, we reset the wflow_status to "SUCCESS". # #----------------------------------------------------------------------- # @@ -299,8 +308,8 @@ fi # #----------------------------------------------------------------------- # -# Print informational messages about the workflow to the launch log -# file, including the workflow status. +# Print informational messages about the workflow to the launch log file, +# including the workflow status. # #----------------------------------------------------------------------- # @@ -331,25 +340,24 @@ if [ "${wflow_status}" = "SUCCESS" ] || \ msg=" The end-to-end run of the workflow for the forecast experiment specified -by expt_name has completed with the following workflow status (wflow_- -status): +by expt_name has completed with the following workflow status (wflow_status): expt_name = \"${expt_name}\" wflow_status = \"${wflow_status}\" " # # If a cron job was being used to periodically relaunch the workflow, we -# now remove the entry in the crontab corresponding to the workflow be- -# cause the end-to-end run of the workflow has now either succeeded or +# now remove the entry in the crontab corresponding to the workflow +# because the end-to-end run of the workflow has now either succeeded or # failed and will remain in that state without manual user intervention. -# Thus, there is no need to try to relaunch it. We also append a mes- -# sage to the completion message above to indicate this. +# Thus, there is no need to try to relaunch it. We also append a message +# to the completion message above to indicate this. # if [ "${USE_CRON_TO_RELAUNCH}" = "TRUE" ]; then msg="${msg}\ -Thus, there is no need to relaunch the workflow via a cron job. Remo- -ving from the crontab the line (CRONTAB_LINE) that calls the workflow -launch script for this experiment: +Thus, there is no need to relaunch the workflow via a cron job. Removing +from the crontab the line (CRONTAB_LINE) that calls the workflow launch +script for this experiment: CRONTAB_LINE = \"${CRONTAB_LINE}\" " # @@ -362,12 +370,13 @@ launch script for this experiment: $SED -r -e "s%[*]%\\\\*%g" ) # # In the string passed to the grep command below, we use the line start -# and line end anchors ("^" and "$", respectively) to ensure that we on- -# ly find lines in the crontab that contain exactly the string in cron- -# tab_line_esc_astr without any leading or trailing characters. +# and line end anchors ("^" and "$", respectively) to ensure that we +# only find lines in the crontab that contain exactly the string in +# crontab_line_esc_astr without any leading or trailing characters. # if [ "$MACHINE" = "WCOSS_DELL_P3" ];then - grep -v "^${crontab_line_esc_astr}$" "/u/$USER/cron/mycrontab" > tmpfile && mv tmpfile "/u/$USER/cron/mycrontab" + grep -v "^${crontab_line_esc_astr}$" "/u/$USER/cron/mycrontab" \ + > tmpfile && mv_vrfy tmpfile "/u/$USER/cron/mycrontab" else ( crontab -l | grep -v "^${crontab_line_esc_astr}$" ) | crontab - fi @@ -386,7 +395,3 @@ launch script for this experiment: fi fi - - - - From 7f11b806e02c51ea3a94bee9064f903c395dfdfb Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Wed, 29 Dec 2021 01:22:43 -0700 Subject: [PATCH 30/35] Change local variable name to lowercase and rename for clarity. --- ush/setup.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ush/setup.sh b/ush/setup.sh index ce963599d..ec57e8834 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -714,12 +714,14 @@ BUILD_ENV_FN=${BUILD_ENV_FN:-"build_${machine}_${COMPILER}.env"} # #----------------------------------------------------------------------- # -# Calculate PPN_RUN_FCST from NCORES_PER_NODE and OMP_NUM_THREADS_RUN_FCST +# Calculate a default value for the number of processes per node for the +# RUN_FCST_TN task. Then set PPN_RUN_FCST to this default value if +# PPN_RUN_FCST is not already specified by the user. # #----------------------------------------------------------------------- # -PPN_RUN_FCST_OPT="$(( ${NCORES_PER_NODE} / ${OMP_NUM_THREADS_RUN_FCST} ))" -PPN_RUN_FCST=${PPN_RUN_FCST:-${PPN_RUN_FCST_OPT}} +ppn_run_fcst_default="$(( ${NCORES_PER_NODE} / ${OMP_NUM_THREADS_RUN_FCST} ))" +PPN_RUN_FCST=${PPN_RUN_FCST:-${ppn_run_fcst_default}} # #----------------------------------------------------------------------- # From f37bfb6f1d4cf8d75067d84ecff60b6231e459b2 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Wed, 29 Dec 2021 03:39:34 -0700 Subject: [PATCH 31/35] Remove variable VARVALUE_REF_CHAR. --- ush/bash_utils/check_var_valid_value.sh | 21 ++++++++++----------- ush/constants.sh | 15 --------------- ush/setup.sh | 2 +- 3 files changed, 11 insertions(+), 27 deletions(-) diff --git a/ush/bash_utils/check_var_valid_value.sh b/ush/bash_utils/check_var_valid_value.sh index 41e67d022..f838dd1ff 100644 --- a/ush/bash_utils/check_var_valid_value.sh +++ b/ush/bash_utils/check_var_valid_value.sh @@ -108,23 +108,22 @@ The value specified in ${var_name} is not supported: # #----------------------------------------------------------------------- # -# If var_value contains the character specified by VARVALUE_REF_CHAR, we -# assume the corresponding variable (var_name) is a template variable, -# i.e. one whose value contains a reference to another variable, e.g. +# If var_value contains a dollar sign, we assume the corresponding variable +# (var_name) is a template variable, i.e. one whose value contains a +# reference to another variable, e.g. # # MY_VAR='\${ANOTHER_VAR}' # -# [In bash, VARVALUE_REF_CHAR is a (escaped) dollar sign.] In this case, -# it does not make sense to check whether var_value is a valid value since -# its contents have not yet been expanded. If var_value doesn't contain -# VARVALUE_REF_CHAR, it must contain a literal string. In this case, we -# check whether it is equal to one of the elements of the array -# valid_var_values. If not, we print out an error message and exit the -# calling script. +# In this case, we do nothing since it does not make sense to check +# whether var_value is a valid value (since its contents have not yet +# been expanded). If var_value doesn't contain a dollar sign, it must +# contain a literal string. In this case, we check whether it is equal +# to one of the elements of the array valid_var_values. If not, we +# print out an error message and exit the calling script. # #----------------------------------------------------------------------- # - if [[ "${var_value}" != *"${VARVALUE_REF_CHAR}"* ]]; then + if [[ "${var_value}" != *"\$"* ]]; then is_element_of "valid_var_values" "${var_value}" || { \ valid_var_values_str=$(printf "\"%s\" " "${valid_var_values[@]}"); print_err_msg_exit "\ diff --git a/ush/constants.sh b/ush/constants.sh index f64f77168..fb19e61f8 100644 --- a/ush/constants.sh +++ b/ush/constants.sh @@ -15,18 +15,3 @@ degs_per_radian=$( bc -l <<< "360.0/(2.0*$pi_geom)" ) # Radius of the Earth in meters. radius_Earth="6371200.0" -# -#----------------------------------------------------------------------- -# -# Character (in bash) used in a string to reference the value stored in -# a variable. This is the dollar sign. For example, if the string is -# -# "Hello, my name is $NAME." -# -# then the dollar sign indicates that $NAME is a reference to the contents -# of the variable NAME. -# -#----------------------------------------------------------------------- -# -VARVALUE_REF_CHAR="\$" - diff --git a/ush/setup.sh b/ush/setup.sh index ec57e8834..15f2bb25f 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -2779,7 +2779,7 @@ Setting its value in the variable definitions file to an empty string." # referenced variable is one of the experiment variables). # dollar_or_null=$( printf "%s" "${var_value}" | \ - $SED -n -r -e "s/[^${VARVALUE_REF_CHAR}]*(${VARVALUE_REF_CHAR}).*/\1/p" ) + $SED -n -r -e "s/[^\$]*(\$).*/\1/p" ) if [ -z "${dollar_or_null}" ]; then printf -v "var_defns_notempl" "${var_defns_notempl}${var_defn}\n" else From a41a5a3dbc39cc61c5b31903b260a1b7edb63ac6 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Thu, 6 Jan 2022 15:34:55 -0700 Subject: [PATCH 32/35] Fix comments; remove duplicate recording of NCORES_PER_NODE. --- ush/setup.sh | 30 ++++++++++++------------------ 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/ush/setup.sh b/ush/setup.sh index 15f2bb25f..2eaa487ba 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -1567,8 +1567,8 @@ fi # #----------------------------------------------------------------------- # -# The FV3 forecast model needs the following input files in the run di- -# rectory to start a forecast: +# The FV3 forecast model needs the following input files in the run +# directory to start a forecast: # # (1) The data table file # (2) The diagnostics table file @@ -1576,27 +1576,22 @@ fi # (4) The FV3 namelist file # (5) The model configuration file # (6) The NEMS configuration file -# -# If using CCPP, it also needs: -# # (7) The CCPP physics suite definition file # # The workflow contains templates for the first six of these files. # Template files are versions of these files that contain placeholder -# (i.e. dummy) values for various parameters. The experiment generation -# scripts copy these templates to appropriate locations in the experiment -# directory (either the top of the experiment directory or one of the -# cycle subdirectories) and replace the placeholders in these copies by -# actual values specified in the experiment configuration file (or derived -# from such values). The scripts then use the resulting "actual" files -# as inputs to the forecast model. -# -# Note that the CCPP physics suite defintion file does not have a +# (i.e. dummy) values for various parameters. The experiment generation +# and/or the forecast task (i.e. J-job) scripts copy these templates to +# appropriate locations in the experiment directory (e.g. to the top of +# the experiment directory, to one of the cycle subdirectories, etc) and +# replace the placeholders with actual values to obtain the files that +# are used as inputs to the forecast model. +# +# Note that the CCPP physics suite defintion file (SDF) does not have a # corresponding template file because it does not contain any values # that need to be replaced according to the experiment configuration. -# If using CCPP, this file simply needs to be copied over from its -# location in the forecast model's directory structure to the experiment -# directory. +# This file simply needs to be copied over from its location in the +# forecast model's directory structure to the experiment directory. # # Below, we first set the names of the templates for the first six files # listed above. We then set the full paths to these template files. @@ -3168,7 +3163,6 @@ FVCOM_FILE=\"${FVCOM_FILE}\" # #----------------------------------------------------------------------- # -NCORES_PER_NODE=\"${NCORES_PER_NODE}\" PE_MEMBER01=\"${PE_MEMBER01}\" # #----------------------------------------------------------------------- From ebe408e07bad75bae58ba173471eae357974ef45 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Mon, 10 Jan 2022 10:11:21 -0700 Subject: [PATCH 33/35] Modifications to simplify code under the assumption that all template variables will be expanded only at runtime (not within var_defns.sh). Details: * Removed code in setup.sh that places template variables at the end of the variable definitions file (var_defns.sh). This is no longer needed because templates will not be expanded within var_defns.sh. * Quoting of variable values in var_defns.sh: * Place all primary variable values in var_defns.sh in single quotes. If this is not done, bash will try to expand template variables, usually resulting in a "variable undefined" error. * For consistency with the use of single quotes for the primary variables in var_defns.sh, also place values of all secondary/derived variables in that file in single quotes. If this is not done, at least one variable (CRONTAB_LINE) that contains whitespace will cause an error. * Remove unnecessary presence of backslash escape characters in the definitions of the template variables in the WE2E test config.template_vars.sh. * Clean up code comments. --- .../wflow_features/config.deactivate_tasks.sh | 16 +- ush/setup.sh | 296 +++++++----------- 2 files changed, 126 insertions(+), 186 deletions(-) diff --git a/tests/WE2E/test_configs/wflow_features/config.deactivate_tasks.sh b/tests/WE2E/test_configs/wflow_features/config.deactivate_tasks.sh index 9cd339341..2375a648f 100644 --- a/tests/WE2E/test_configs/wflow_features/config.deactivate_tasks.sh +++ b/tests/WE2E/test_configs/wflow_features/config.deactivate_tasks.sh @@ -51,12 +51,10 @@ RUN_TASK_RUN_POST="FALSE" # The following shows examples of how to define template variables. Here, # we define RUN_CMD_UTILS, RUN_CMD_FCST, and RUN_CMD_POST as template # variables. Note that during this test, these templates aren't actually -# expanded/used (using bash's "eval" built-in command) anywhere in the -# scripts. They are included here simply included for demonstration -# purposes. If all goes well, the experiment generation system should -# place these templates at the end of the experiment's variable definitions -# file. -# -RUN_CMD_UTILS='cd \$yyyymmdd' -RUN_CMD_FCST='mpirun -np \${PE_MEMBER01}' -RUN_CMD_POST='\$( echo hello \$yyyymmdd )' +# expanded/used (something that would be done using bash's "eval" built-in +# command) anywhere in the scripts. They are included here only to verify +# that the test completes with some variables defined as templates. +# +RUN_CMD_UTILS='cd $yyyymmdd' +RUN_CMD_FCST='mpirun -np ${PE_MEMBER01}' +RUN_CMD_POST='echo hello $yyyymmdd' diff --git a/ush/setup.sh b/ush/setup.sh index 2eaa487ba..83e141f96 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -2045,7 +2045,6 @@ Reset value is:" print_info_msg "$msg" fi - # #----------------------------------------------------------------------- # @@ -2605,39 +2604,13 @@ ${default_var_defns} # #----------------------------------------------------------------------- # -# Create lists of primary experiment variable definitions containing +# Create a list of primary experiment variable definitions containing # updated values. By "updated", we mean non-default values. Values # may have been updated due to the presence of user-specified values in # the experiment configuration file (EXPT_CONFIG_FN) or due to other # considerations (e.g. resetting depending on the platform the App is # running on). # -# Note that we generate two lists: var_defns_notempl and var_defns_templ. -# var_defns_templ is for template variables, i.e. variables that contain -# references to other variables in their definitions, e.g. -# -# MY_VAR='\${ANOTHER_VAR}' -# -# while var_defns_notempl is for variables whose definitions contain only -# a literal string. These two types are treated separately. Non-template -# variables are written to the variable definitions file as they are -# encountered, whereas template variables (or simply "templates") are -# saved in an array and written at the end of the file. This is to -# ensure that any non-template variables that the templates reference -# are defined before the templates (so that undefined variable errors -# are not encountered). If a template is defined as above, i.e. enclosed -# in SINGLE quotes and with dollar signs escaped, this should not matter -# because variable references will not be expanded when the variable -# definitions file is sourced; they will be expanded only when the "eval" -# built-in command is used to evaluate the contents of the template. -# For this reason, use of single quotes is the recommended way of defining -# a template in the experiment configuration file. However, in case -# a template is enclosed in double quotes, any varaible references will -# be expanded/evaluated when the variable definitions file is sourced. -# In this case, unexpected behavior or failure may be encountered if the -# referenced variables are not already defined. To cover this scenario, -# we place the templates all at the end of the variable definitions file. -# #----------------------------------------------------------------------- # print_info_msg " @@ -2659,13 +2632,10 @@ fi # # Loop through the lines in default_var_defns. Reset the value of the # variable on each line to the updated value (e.g. to a user-specified -# value, as opposed to the default value). Save the updated list of -# variables and values either in var_defns_notempl (if it is not a -# template variable, i.e. one that references other variables in its -# definition) or in var_defns_templ (if it is a template variable). +# value, as opposed to the default value). The updated list of variables +# and values will be saved in var_defns. # -var_defns_notempl="" -var_defns_templ="" +var_defns="" while read crnt_line; do # # Try to obtain the name of the variable being set on the current line. @@ -2718,7 +2688,8 @@ var_name = \"${var_name}\"" # if [ "${num_elems}" -eq 1 ]; then - var_value=\""${!var_name}"\" + var_value="${!var_name}" + rhs="'${var_value}'" # # If the variable contains more than one element, then it is an array. # In this case, we build var_value in two steps as follows: @@ -2739,7 +2710,7 @@ var_name = \"${var_name}\"" for (( i=0; i<${num_elems}; i++ )); do printf -v "var_value" "${var_value}\"${array[$i]}\" ${escbksl_nl_or_null}" done - var_value="( ${var_value})" + rhs="( ${var_value})" fi # @@ -2756,30 +2727,15 @@ The variable specified by \"var_name\" is not set in the current environment: var_name = \"${var_name}\" Setting its value in the variable definitions file to an empty string." - var_value="\"\"" + rhs="''" fi # # Set the line containing the variable's definition. Then add the line -# to the appropriate list (var_defns_notempl or var_defns_templ) depending -# on whether the variable is a template or not. -# - var_defn="${var_name}=${var_value}" -# -# If the variable value contains a dollar sign, we assume it is a template -# variable, i.e. one whose definition contains a reference to the value -# of another variable. In this case, the line for this variable must -# be placed at the end of the variable definitions file to ensure that -# the variable that it refers to has already been defined (assuming the -# referenced variable is one of the experiment variables). -# - dollar_or_null=$( printf "%s" "${var_value}" | \ - $SED -n -r -e "s/[^\$]*(\$).*/\1/p" ) - if [ -z "${dollar_or_null}" ]; then - printf -v "var_defns_notempl" "${var_defns_notempl}${var_defn}\n" - else - printf -v "var_defns_templ" "${var_defns_templ}${var_defn}\n" - fi +# to the list of all variable definitions. +# + var_defn="${var_name}=$rhs" + printf -v "var_defns" "${var_defns}${var_defn}\n" # # If var_name is empty, then a variable name was not found on the current # line in default_var_defns. In this case, print out a warning and move @@ -2825,13 +2781,11 @@ var_defns_file_contents="\ # This section contains (most of) the primary experiment variables, i.e. # those variables that are defined in the default configuration file # (${EXPT_DEFAULT_CONFIG_FN}) and that can be reset via the user-specified -# experiment configuration file (${EXPT_CONFIG_FN}). Note that primary variables -# that are template variables (i.e. contain in their definitions references -# to other variables) are placed towards the end of this file. +# experiment configuration file (${EXPT_CONFIG_FN}). #----------------------------------------------------------------------- #----------------------------------------------------------------------- # -${var_defns_notempl}" +${var_defns}" # # Append derived/secondary variable definitions (as well as comments) to # the contents of the variable definitions file. @@ -2863,9 +2817,9 @@ var_defns_file_contents=${var_defns_file_contents}"\ # #----------------------------------------------------------------------- # -WFLOW_LAUNCH_SCRIPT_FP=\"${WFLOW_LAUNCH_SCRIPT_FP}\" -WFLOW_LAUNCH_LOG_FP=\"${WFLOW_LAUNCH_LOG_FP}\" -CRONTAB_LINE=\"${CRONTAB_LINE}\" +WFLOW_LAUNCH_SCRIPT_FP='${WFLOW_LAUNCH_SCRIPT_FP}' +WFLOW_LAUNCH_LOG_FP='${WFLOW_LAUNCH_LOG_FP}' +CRONTAB_LINE='${CRONTAB_LINE}' # #----------------------------------------------------------------------- # @@ -2873,42 +2827,42 @@ CRONTAB_LINE=\"${CRONTAB_LINE}\" # #----------------------------------------------------------------------- # -SR_WX_APP_TOP_DIR=\"${SR_WX_APP_TOP_DIR}\" -HOMErrfs=\"$HOMErrfs\" -USHDIR=\"$USHDIR\" -SCRIPTSDIR=\"$SCRIPTSDIR\" -JOBSDIR=\"$JOBSDIR\" -SORCDIR=\"$SORCDIR\" -SRC_DIR=\"$SRC_DIR\" -PARMDIR=\"$PARMDIR\" -MODULES_DIR=\"${MODULES_DIR}\" -EXECDIR=\"$EXECDIR\" -FIXam=\"$FIXam\" -FIXclim=\"$FIXclim\" -FIXLAM=\"$FIXLAM\" -FIXgsm=\"$FIXgsm\" -FIXaer=\"$FIXaer\" -FIXlut=\"$FIXlut\" -COMROOT=\"$COMROOT\" -COMOUT_BASEDIR=\"${COMOUT_BASEDIR}\" -TEMPLATE_DIR=\"${TEMPLATE_DIR}\" -VX_CONFIG_DIR=\"${VX_CONFIG_DIR}\" -METPLUS_CONF=\"${METPLUS_CONF}\" -MET_CONFIG=\"${MET_CONFIG}\" -UFS_WTHR_MDL_DIR=\"${UFS_WTHR_MDL_DIR}\" -UFS_UTILS_DIR=\"${UFS_UTILS_DIR}\" -SFC_CLIMO_INPUT_DIR=\"${SFC_CLIMO_INPUT_DIR}\" -TOPO_DIR=\"${TOPO_DIR}\" -UPP_DIR=\"${UPP_DIR}\" - -EXPTDIR=\"$EXPTDIR\" -LOGDIR=\"$LOGDIR\" -CYCLE_BASEDIR=\"${CYCLE_BASEDIR}\" -GRID_DIR=\"${GRID_DIR}\" -OROG_DIR=\"${OROG_DIR}\" -SFC_CLIMO_DIR=\"${SFC_CLIMO_DIR}\" - -NDIGITS_ENSMEM_NAMES=\"${NDIGITS_ENSMEM_NAMES}\" +SR_WX_APP_TOP_DIR='${SR_WX_APP_TOP_DIR}' +HOMErrfs='$HOMErrfs' +USHDIR='$USHDIR' +SCRIPTSDIR='$SCRIPTSDIR' +JOBSDIR='$JOBSDIR' +SORCDIR='$SORCDIR' +SRC_DIR='${SRC_DIR}' +PARMDIR='$PARMDIR' +MODULES_DIR='${MODULES_DIR}' +EXECDIR='$EXECDIR' +FIXam='$FIXam' +FIXclim='$FIXclim' +FIXLAM='$FIXLAM' +FIXgsm='$FIXgsm' +FIXaer='$FIXaer' +FIXlut='$FIXlut' +COMROOT='$COMROOT' +COMOUT_BASEDIR='${COMOUT_BASEDIR}' +TEMPLATE_DIR='${TEMPLATE_DIR}' +VX_CONFIG_DIR='${VX_CONFIG_DIR}' +METPLUS_CONF='${METPLUS_CONF}' +MET_CONFIG='${MET_CONFIG}' +UFS_WTHR_MDL_DIR='${UFS_WTHR_MDL_DIR}' +UFS_UTILS_DIR='${UFS_UTILS_DIR}' +SFC_CLIMO_INPUT_DIR='${SFC_CLIMO_INPUT_DIR}' +TOPO_DIR='${TOPO_DIR}' +UPP_DIR='${UPP_DIR}' + +EXPTDIR='$EXPTDIR' +LOGDIR='$LOGDIR' +CYCLE_BASEDIR='${CYCLE_BASEDIR}' +GRID_DIR='${GRID_DIR}' +OROG_DIR='${OROG_DIR}' +SFC_CLIMO_DIR='${SFC_CLIMO_DIR}' + +NDIGITS_ENSMEM_NAMES='${NDIGITS_ENSMEM_NAMES}' ENSMEM_NAMES=${ensmem_names_str} FV3_NML_ENSMEM_FPS=${fv3_nml_ensmem_fps_str} # @@ -2918,43 +2872,43 @@ FV3_NML_ENSMEM_FPS=${fv3_nml_ensmem_fps_str} # #----------------------------------------------------------------------- # -GLOBAL_VAR_DEFNS_FP=\"${GLOBAL_VAR_DEFNS_FP}\" +GLOBAL_VAR_DEFNS_FP='${GLOBAL_VAR_DEFNS_FP}' -DATA_TABLE_TMPL_FN=\"${DATA_TABLE_TMPL_FN}\" -DIAG_TABLE_TMPL_FN=\"${DIAG_TABLE_TMPL_FN}\" -FIELD_TABLE_TMPL_FN=\"${FIELD_TABLE_TMPL_FN}\" -MODEL_CONFIG_TMPL_FN=\"${MODEL_CONFIG_TMPL_FN}\" -NEMS_CONFIG_TMPL_FN=\"${NEMS_CONFIG_TMPL_FN}\" +DATA_TABLE_TMPL_FN='${DATA_TABLE_TMPL_FN}' +DIAG_TABLE_TMPL_FN='${DIAG_TABLE_TMPL_FN}' +FIELD_TABLE_TMPL_FN='${FIELD_TABLE_TMPL_FN}' +MODEL_CONFIG_TMPL_FN='${MODEL_CONFIG_TMPL_FN}' +NEMS_CONFIG_TMPL_FN='${NEMS_CONFIG_TMPL_FN}' -DATA_TABLE_TMPL_FP=\"${DATA_TABLE_TMPL_FP}\" -DIAG_TABLE_TMPL_FP=\"${DIAG_TABLE_TMPL_FP}\" -FIELD_TABLE_TMPL_FP=\"${FIELD_TABLE_TMPL_FP}\" -FV3_NML_BASE_SUITE_FP=\"${FV3_NML_BASE_SUITE_FP}\" -FV3_NML_YAML_CONFIG_FP=\"${FV3_NML_YAML_CONFIG_FP}\" -FV3_NML_BASE_ENS_FP=\"${FV3_NML_BASE_ENS_FP}\" -MODEL_CONFIG_TMPL_FP=\"${MODEL_CONFIG_TMPL_FP}\" -NEMS_CONFIG_TMPL_FP=\"${NEMS_CONFIG_TMPL_FP}\" +DATA_TABLE_TMPL_FP='${DATA_TABLE_TMPL_FP}' +DIAG_TABLE_TMPL_FP='${DIAG_TABLE_TMPL_FP}' +FIELD_TABLE_TMPL_FP='${FIELD_TABLE_TMPL_FP}' +FV3_NML_BASE_SUITE_FP='${FV3_NML_BASE_SUITE_FP}' +FV3_NML_YAML_CONFIG_FP='${FV3_NML_YAML_CONFIG_FP}' +FV3_NML_BASE_ENS_FP='${FV3_NML_BASE_ENS_FP}' +MODEL_CONFIG_TMPL_FP='${MODEL_CONFIG_TMPL_FP}' +NEMS_CONFIG_TMPL_FP='${NEMS_CONFIG_TMPL_FP}' -CCPP_PHYS_SUITE_FN=\"${CCPP_PHYS_SUITE_FN}\" -CCPP_PHYS_SUITE_IN_CCPP_FP=\"${CCPP_PHYS_SUITE_IN_CCPP_FP}\" -CCPP_PHYS_SUITE_FP=\"${CCPP_PHYS_SUITE_FP}\" +CCPP_PHYS_SUITE_FN='${CCPP_PHYS_SUITE_FN}' +CCPP_PHYS_SUITE_IN_CCPP_FP='${CCPP_PHYS_SUITE_IN_CCPP_FP}' +CCPP_PHYS_SUITE_FP='${CCPP_PHYS_SUITE_FP}' -FIELD_DICT_FN=\"${FIELD_DICT_FN}\" -FIELD_DICT_IN_UWM_FP=\"${FIELD_DICT_IN_UWM_FP}\" -FIELD_DICT_FP=\"${FIELD_DICT_FP}\" +FIELD_DICT_FN='${FIELD_DICT_FN}' +FIELD_DICT_IN_UWM_FP='${FIELD_DICT_IN_UWM_FP}' +FIELD_DICT_FP='${FIELD_DICT_FP}' -DATA_TABLE_FP=\"${DATA_TABLE_FP}\" -FIELD_TABLE_FP=\"${FIELD_TABLE_FP}\" -FV3_NML_FN=\"${FV3_NML_FN}\" # This may not be necessary... -FV3_NML_FP=\"${FV3_NML_FP}\" -NEMS_CONFIG_FP=\"${NEMS_CONFIG_FP}\" +DATA_TABLE_FP='${DATA_TABLE_FP}' +FIELD_TABLE_FP='${FIELD_TABLE_FP}' +FV3_NML_FN='${FV3_NML_FN}' +FV3_NML_FP='${FV3_NML_FP}' +NEMS_CONFIG_FP='${NEMS_CONFIG_FP}' -FV3_EXEC_FP=\"${FV3_EXEC_FP}\" +FV3_EXEC_FP='${FV3_EXEC_FP}' -LOAD_MODULES_RUN_TASK_FP=\"${LOAD_MODULES_RUN_TASK_FP}\" +LOAD_MODULES_RUN_TASK_FP='${LOAD_MODULES_RUN_TASK_FP}' -THOMPSON_MP_CLIMO_FN=\"${THOMPSON_MP_CLIMO_FN}\" -THOMPSON_MP_CLIMO_FP=\"${THOMPSON_MP_CLIMO_FP}\" +THOMPSON_MP_CLIMO_FN='${THOMPSON_MP_CLIMO_FN}' +THOMPSON_MP_CLIMO_FP='${THOMPSON_MP_CLIMO_FP}' # #----------------------------------------------------------------------- # @@ -2962,7 +2916,7 @@ THOMPSON_MP_CLIMO_FP=\"${THOMPSON_MP_CLIMO_FP}\" # #----------------------------------------------------------------------- # -RELATIVE_LINK_FLAG=\"${RELATIVE_LINK_FLAG}\" +RELATIVE_LINK_FLAG='${RELATIVE_LINK_FLAG}' # #----------------------------------------------------------------------- # @@ -2971,8 +2925,8 @@ RELATIVE_LINK_FLAG=\"${RELATIVE_LINK_FLAG}\" # #----------------------------------------------------------------------- # -SDF_USES_RUC_LSM=\"${SDF_USES_RUC_LSM}\" -SDF_USES_THOMPSON_MP=\"${SDF_USES_THOMPSON_MP}\" +SDF_USES_RUC_LSM='${SDF_USES_RUC_LSM}' +SDF_USES_THOMPSON_MP='${SDF_USES_THOMPSON_MP}' # #----------------------------------------------------------------------- # @@ -2981,26 +2935,26 @@ SDF_USES_THOMPSON_MP=\"${SDF_USES_THOMPSON_MP}\" # #----------------------------------------------------------------------- # -GTYPE=\"$GTYPE\" -TILE_RGNL=\"${TILE_RGNL}\" -NH0=\"${NH0}\" -NH3=\"${NH3}\" -NH4=\"${NH4}\" +GTYPE='$GTYPE' +TILE_RGNL='${TILE_RGNL}' +NH0='${NH0}' +NH3='${NH3}' +NH4='${NH4}' -LON_CTR=\"${LON_CTR}\" -LAT_CTR=\"${LAT_CTR}\" -NX=\"${NX}\" -NY=\"${NY}\" -NHW=\"${NHW}\" -STRETCH_FAC=\"${STRETCH_FAC}\" +LON_CTR='${LON_CTR}' +LAT_CTR='${LAT_CTR}' +NX='${NX}' +NY='${NY}' +NHW='${NHW}' +STRETCH_FAC='${STRETCH_FAC}' -RES_IN_FIXLAM_FILENAMES=\"${RES_IN_FIXLAM_FILENAMES}\" +RES_IN_FIXLAM_FILENAMES='${RES_IN_FIXLAM_FILENAMES}' # # If running the make_grid task, CRES will be set to a null string during # the grid generation step. It will later be set to an actual value after # the make_grid task is complete. # -CRES=\"$CRES\" +CRES='$CRES' " # #----------------------------------------------------------------------- @@ -3027,10 +2981,10 @@ if [ "${GRID_GEN_METHOD}" = "GFDLgrid" ]; then # #----------------------------------------------------------------------- # -ISTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG=\"${ISTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG}\" -IEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG=\"${IEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG}\" -JSTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG=\"${JSTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG}\" -JEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG=\"${JEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG}\" +ISTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG='${ISTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG}' +IEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG='${IEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG}' +JSTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG='${JSTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG}' +JEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG='${JEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG}' " elif [ "${GRID_GEN_METHOD}" = "ESGgrid" ]; then @@ -3046,11 +3000,11 @@ elif [ "${GRID_GEN_METHOD}" = "ESGgrid" ]; then # #----------------------------------------------------------------------- # -DEL_ANGLE_X_SG=\"${DEL_ANGLE_X_SG}\" -DEL_ANGLE_Y_SG=\"${DEL_ANGLE_Y_SG}\" -NEG_NX_OF_DOM_WITH_WIDE_HALO=\"${NEG_NX_OF_DOM_WITH_WIDE_HALO}\" -NEG_NY_OF_DOM_WITH_WIDE_HALO=\"${NEG_NY_OF_DOM_WITH_WIDE_HALO}\" -PAZI=\"${PAZI}\" +DEL_ANGLE_X_SG='${DEL_ANGLE_X_SG}' +DEL_ANGLE_Y_SG='${DEL_ANGLE_Y_SG}' +NEG_NX_OF_DOM_WITH_WIDE_HALO='${NEG_NX_OF_DOM_WITH_WIDE_HALO}' +NEG_NY_OF_DOM_WITH_WIDE_HALO='${NEG_NY_OF_DOM_WITH_WIDE_HALO}' +PAZI='${PAZI}' " fi @@ -3078,7 +3032,7 @@ var_defns_file_contents=${var_defns_file_contents}"\ # #----------------------------------------------------------------------- # -CPL=\"${CPL}\" +CPL='${CPL}' # #----------------------------------------------------------------------- # @@ -3087,7 +3041,7 @@ CPL=\"${CPL}\" # #----------------------------------------------------------------------- # -OZONE_PARAM=\"${OZONE_PARAM}\" +OZONE_PARAM='${OZONE_PARAM}' # #----------------------------------------------------------------------- # @@ -3099,7 +3053,7 @@ OZONE_PARAM=\"${OZONE_PARAM}\" # #----------------------------------------------------------------------- # -EXTRN_MDL_SYSBASEDIR_ICS=\"${EXTRN_MDL_SYSBASEDIR_ICS}\" +EXTRN_MDL_SYSBASEDIR_ICS='${EXTRN_MDL_SYSBASEDIR_ICS}' # #----------------------------------------------------------------------- # @@ -3111,7 +3065,7 @@ EXTRN_MDL_SYSBASEDIR_ICS=\"${EXTRN_MDL_SYSBASEDIR_ICS}\" # #----------------------------------------------------------------------- # -EXTRN_MDL_SYSBASEDIR_LBCS=\"${EXTRN_MDL_SYSBASEDIR_LBCS}\" +EXTRN_MDL_SYSBASEDIR_LBCS='${EXTRN_MDL_SYSBASEDIR_LBCS}' # #----------------------------------------------------------------------- # @@ -3120,7 +3074,7 @@ EXTRN_MDL_SYSBASEDIR_LBCS=\"${EXTRN_MDL_SYSBASEDIR_LBCS}\" # #----------------------------------------------------------------------- # -EXTRN_MDL_LBCS_OFFSET_HRS=\"${EXTRN_MDL_LBCS_OFFSET_HRS}\" +EXTRN_MDL_LBCS_OFFSET_HRS='${EXTRN_MDL_LBCS_OFFSET_HRS}' # #----------------------------------------------------------------------- # @@ -3138,7 +3092,7 @@ LBC_SPEC_FCST_HRS=${lbc_spec_fcst_hrs_str} # #----------------------------------------------------------------------- # -NUM_CYCLES=\"${NUM_CYCLES}\" +NUM_CYCLES='${NUM_CYCLES}' ALL_CDATES=${all_cdates_str} # #----------------------------------------------------------------------- @@ -3153,9 +3107,9 @@ ALL_CDATES=${all_cdates_str} # #----------------------------------------------------------------------- # -USE_FVCOM=\"${USE_FVCOM}\" -FVCOM_DIR=\"${FVCOM_DIR}\" -FVCOM_FILE=\"${FVCOM_FILE}\" +USE_FVCOM='${USE_FVCOM}' +FVCOM_DIR='${FVCOM_DIR}' +FVCOM_FILE='${FVCOM_FILE}' # #----------------------------------------------------------------------- # @@ -3163,7 +3117,7 @@ FVCOM_FILE=\"${FVCOM_FILE}\" # #----------------------------------------------------------------------- # -PE_MEMBER01=\"${PE_MEMBER01}\" +PE_MEMBER01='${PE_MEMBER01}' # #----------------------------------------------------------------------- # @@ -3173,20 +3127,8 @@ PE_MEMBER01=\"${PE_MEMBER01}\" # #----------------------------------------------------------------------- # -N_VAR_SPP=\"${N_VAR_SPP}\" -# -#----------------------------------------------------------------------- -#----------------------------------------------------------------------- -# Section 3: -# This section defines template variables, i.e. variables that contain -# references to other variables in their definitions, e.g. -# -# MY_VAR='\${ANOTHER_VAR}' -# -#----------------------------------------------------------------------- -#----------------------------------------------------------------------- -# -${var_defns_templ}" +N_VAR_SPP='${N_VAR_SPP}' +" # # Done with constructing the contents of the variable definitions file, # so now write the contents to file. From 6cd17867264d3445c8007a8950097da0c445b413 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Wed, 12 Jan 2022 09:19:36 -0700 Subject: [PATCH 34/35] Bug fix. --- tests/WE2E/get_expts_status.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/WE2E/get_expts_status.sh b/tests/WE2E/get_expts_status.sh index 5c05acafb..997bfb6b8 100755 --- a/tests/WE2E/get_expts_status.sh +++ b/tests/WE2E/get_expts_status.sh @@ -50,7 +50,7 @@ scrfunc_dir=$( dirname "${scrfunc_fp}" ) # #----------------------------------------------------------------------- # -homerrfs=${scrfunc_dir%/*} +homerrfs=${scrfunc_dir%/*/*} # #----------------------------------------------------------------------- # From d6847dc17ed40e0a1f1a2415a8046d0391f4f2bd Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Wed, 12 Jan 2022 09:34:23 -0700 Subject: [PATCH 35/35] Simplify syntax of if-statement test for detecting a dollar sign (to detect template variables). --- ush/bash_utils/check_var_valid_value.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/bash_utils/check_var_valid_value.sh b/ush/bash_utils/check_var_valid_value.sh index f838dd1ff..576ad6b1b 100644 --- a/ush/bash_utils/check_var_valid_value.sh +++ b/ush/bash_utils/check_var_valid_value.sh @@ -123,7 +123,7 @@ The value specified in ${var_name} is not supported: # #----------------------------------------------------------------------- # - if [[ "${var_value}" != *"\$"* ]]; then + if [[ "${var_value}" != *'$'* ]]; then is_element_of "valid_var_values" "${var_value}" || { \ valid_var_values_str=$(printf "\"%s\" " "${valid_var_values[@]}"); print_err_msg_exit "\