Enhance ability to use template variables

tests/WE2E/get_WE2Etest_names_subdirs_descs.sh

@@ -759,7 +759,7 @@ This is probably because it is a directory.  Please correct and rerun."
   test_names=("${prim_test_names[@]}")
   test_subdirs=("${prim_test_subdirs[@]}")
   if [ "${num_alt_tests}" -gt "0" ]; then
-    test_names+=("${alt_test_subdirs[@]:-}")
+    test_names+=("${alt_test_names[@]:-}")
     test_subdirs+=("${alt_test_subdirs[@]:-}")
   fi
 #
@@ -1025,7 +1025,7 @@ Please correct and rerun."
 # listed first.  
 #
 # Finally, we extract from test_ids_and_inds_sorted the second number 
-# in each element (the one afte the first number, which is the test ID,
+# in each element (the one after the first number, which is the test ID,
 # and the test type, which we no longer need), which is the original 
 # array index before sorting, and save the results in the array sort_inds.  
 # This array will contain the original indices in sorted order that we 

tests/WE2E/run_WE2E_tests.sh

@@ -538,8 +538,8 @@ accordingly and rerun."
   if [ "${match_found}" = "FALSE" ]; then
     avail_WE2E_test_names_str=$( printf "  \"%s\"\n" "${avail_WE2E_test_names[@]}" )
     print_err_msg_exit "\
-The name current user-specified test to run (user_spec_test) does not 
-match any of the names (either primary or alternate) of the available
+The name of the current user-specified test to run (user_spec_test) does 
+not match any of the names (either primary or alternate) of the available
 WE2E tests:
   user_spec_test = \"${user_spec_test}\"
 Valid values for user_spec_test consist of the names (primary or alternate)

tests/WE2E/test_configs/wflow_features/config.deactivate_tasks.sh

@@ -2,10 +2,26 @@
 # TEST PURPOSE/DESCRIPTION:
 # ------------------------
 #
-# This test ensures that the various workflow tasks can be deactivated,
-# i.e. removed from the Rocoto XML.  Note that we leave the MAKE_GRID_TN, 
-# MAKE_OROG_TN, and MAKE_SFC_CLIMO_TN activated because there is a 
-# separate test for turning those off.
+# This test has two purposes:
+#
+# 1) It checks that the various workflow tasks can be deactivated, i.e. 
+#    removed from the Rocoto XML.
+# 2) It checks the capability of the workflow to use "template" experiment 
+#    variables, i.e. variables whose definitions include references to 
+#    other variables, e.g.
+#
+#      MY_VAR='\${ANOTHER_VAR}'
+#
+# Note that we do not deactivate all tasks in the workflow; we leave the 
+# MAKE_GRID_TN, MAKE_OROG_TN, and MAKE_SFC_CLIMO_TN activated because:
+#
+# 1) There is already a WE2E test that runs with these three tasks
+#    deactivated (that test is to ensure that pre-generated grid, 
+#    orography, and surface climatology files can be used).
+# 2) In checking the template variable capability, we want to make sure
+#    that the variable defintions file (GLOBAL_VAR_DEFNS_FN) generated
+#    does not have syntax or other errors in it by sourcing it in these 
+#    three tasks.
 #
 
 RUN_ENVIR="community"
@@ -14,13 +30,86 @@ PREEXISTING_DIR_METHOD="rename"
 PREDEF_GRID_NAME="RRFS_CONUS_25km"
 CCPP_PHYS_SUITE="FV3_GFS_v15p2"
 
-DATE_FIRST_CYCL="20190615"
-DATE_LAST_CYCL="20190615"
+EXTRN_MDL_NAME_ICS="FV3GFS"
+EXTRN_MDL_NAME_LBCS="FV3GFS"
+USE_USER_STAGED_EXTRN_FILES="TRUE"
+
+DATE_FIRST_CYCL="20190701"
+DATE_LAST_CYCL="20190701"
 CYCL_HRS=( "00" )
 
+FCST_LEN_HRS="6"
+LBC_SPEC_INTVL_HRS="3"
+
 RUN_TASK_GET_EXTRN_ICS="FALSE"
 RUN_TASK_GET_EXTRN_LBCS="FALSE"
 RUN_TASK_MAKE_ICS="FALSE"
 RUN_TASK_MAKE_LBCS="FALSE"
 RUN_TASK_RUN_FCST="FALSE"
 RUN_TASK_RUN_POST="FALSE"
+#
+# The following shows examples of how to define template variables.  Here,
+# we define RUN_CMD_UTILS, RUN_CMD_FCST, and RUN_CMD_POST as template 
+# variables.  Note that these variables aren't actually needed to run
+# any commands but are simply included for demonstration purposes.
+#
+# If we want the contents in RUN_CMD_UTILS, RUN_CMD_FCST, and RUN_CMD_POST 
+# to be expanded and/or executed when the variable definitions file 
+# (GLOBAL_VAR_DEFNS_FN) is sourced in a script or function, then we must
+# escape bash's variable reference character "$" using backslashes, as 
+# follows:
+#
+#   RUN_CMD_UTILS="echo \$yyyymmdd"
+#   RUN_CMD_FCST="mpirun -np \${PE_MEMBER01}"
+#   RUN_CMD_POST="\$( echo hello \$yyymmdd )"
+#
+# With this method, when the variable defintions file is sourced, the 
+# variable references on the right-hand sides (here $yyyymmdd and 
+# ${PE_MEMBER01}) as well as the command substitution [i.e. $( echo ...)]
+# will first be evaluated/expanded and the results then combined with 
+# the remainder of each string on the right-hand side (if any) to obtain 
+# literal strings to assign to RUN_CMD_UTILS, RUN_CMD_FCST, and RUN_CMD_POST.  
+# Thus, the values that these three variables are assigned in the variable
+# definitions file will not contain any variable references; they will
+# contain only literal strings.
+#
+# In general, this is not what we want because the variables on the right-
+# hand sides may not yet be defined at the time the variable definitions
+# file is sourced.  Thus, what we usually want is for variable references
+# and command substitutions appearing on the right-hand sides in that 
+# file to be escaped.  To do this, we must escape the "$" character twice, 
+# as follows:
+#
+#   RUN_CMD_UTILS="echo \\\$yyyymmdd"
+#   RUN_CMD_FCST="mpirun -np \\\${PE_MEMBER01}"
+#   RUN_CMD_POST="\\\$( echo hello \\\$yyyymmdd )"
+#
+# A more compact way to do this is to use single quotes on the outside
+# and eliminate the escaped-\ within, i.e.
+#
+#   RUN_CMD_UTILS='echo \$yyyymmdd'
+#   RUN_CMD_FCST='mpirun -np \${PE_MEMBER01}'
+#   RUN_CMD_POST='\$( echo hello \$yyyymmdd )'
+#
+# This will cause $yyyymmdd, ${PE_MEMBER01}, and $(...) to be expanded
+# only at the time RUN_CMD_UTILS, RUN_CMD_FCST, and RUN_CMD_POST are 
+# referenced in a file or script (using the syntax ${RUN_CMD_UTILS}, ...)
+# and not earlier when the variable definitions file is sourced.  That
+# is what we do in this WE2E test.
+#
+# With the last set of settings above, in a given script or function we 
+# can source the variable defintions file and then use the three variables 
+# as follows:
+#
+#   . $exptdir/var_defns.sh
+#   yyyymmdd="goodbye"
+#   eval ${RUN_CMD_UTILS}
+#   eval ${RUN_CMD_FCST}
+#   eval myvar=${RUN_CMD_POST}
+#
+# (The mpirun command in RUN_CMD_FCST will generate an error, but that is
+# ok since using mpirun is not the primary concern here.)
+#
+RUN_CMD_UTILS='echo \$yyyymmdd'
+RUN_CMD_FCST='mpirun -np \${PE_MEMBER01}'
+RUN_CMD_POST='\$( echo hello \$yyyymmdd )'

tests/WE2E/test_configs/wflow_features/config.template_vars.sh

@@ -0,0 +1 @@
+config.deactivate_tasks.sh

ush/bash_utils/check_var_valid_value.sh

@@ -108,24 +108,35 @@ The value specified in ${var_name} is not supported:
 #
 #-----------------------------------------------------------------------
 #
-# Check whether var_value is equal to one of the elements of the array
-# valid_var_values.  If not, print out an error message and exit the 
+# If var_value contains the character specified by VARVALUE_REF_CHAR, we
+# assume the corresponding variable (var_name) is a template variable, 
+# i.e. one whose value contains a reference to another variable, e.g.
+#
+#   MY_VAR='\${ANOTHER_VAR}'
+#
+# [In bash, VARVALUE_REF_CHAR is a (escaped) dollar sign.]  In this case, 
+# it does not make sense to check whether var_value is a valid value since 
+# its contents have not yet been expanded.  If var_value doesn't contain 
+# VARVALUE_REF_CHAR, it must contain a literal string.  In this case, we 
+# check whether it is equal to one of the elements of the array 
+# valid_var_values.  If not, we print out an error message and exit the 
 # calling script.
 #
 #-----------------------------------------------------------------------
 #
-  is_element_of "valid_var_values" "${var_value}" || { \
-    valid_var_values_str=$(printf "\"%s\" " "${valid_var_values[@]}");
-    print_err_msg_exit "\
+  if [[ "${var_value}" != *"${VARVALUE_REF_CHAR}"* ]]; then
+    is_element_of "valid_var_values" "${var_value}" || { \
+      valid_var_values_str=$(printf "\"%s\" " "${valid_var_values[@]}");
+      print_err_msg_exit "\
 ${err_msg}
 ${var_name} must be set to one of the following:
   ${valid_var_values_str}"; \
-  }
+    }
+  fi
 #
 #-----------------------------------------------------------------------
 #
-# Restore the shell options saved at the beginning of this script/func-
-# tion.
+# Restore the shell options saved at the beginning of this script/function.
 #
 #-----------------------------------------------------------------------
 #

ush/bash_utils/get_bash_file_contents.sh

@@ -0,0 +1,119 @@
+#
+#-----------------------------------------------------------------------
+#
+# This file defines a function that returns the contents of a bash script/
+# function with all empty lines, comment lines, and leading and trailing 
+# whitespace removed.  Arguments are as follows:
+#
+# fp:
+# The relative or full path to the file containing the bash script or 
+# function.
+#
+# output_varname_contents:
+# Name of the output variable that will contain the (processed) contents 
+# of the file.  This is the output of the function.
+# 
+#-----------------------------------------------------------------------
+#
+function get_bash_file_contents() { 
+#
+#-----------------------------------------------------------------------
+#
+# Save current shell options (in a global array).  Then set new options
+# for this script/function.
+#
+#-----------------------------------------------------------------------
+#
+  { save_shell_opts; set -u +x; } > /dev/null 2>&1
+#
+#-----------------------------------------------------------------------
+#
+# Specify the set of valid argument names for this script/function.  Then
+# process the arguments provided to this script/function (which should
+# consist of a set of name-value pairs of the form arg1="value1", etc).
+#
+#-----------------------------------------------------------------------
+#
+  local valid_args=( \
+    "fp" \
+    "output_varname_contents" \
+    )
+  process_args valid_args "$@"
+#
+#-----------------------------------------------------------------------
+#
+# For debugging purposes, print out values of arguments passed to this
+# script.  Note that these will be printed out only if VERBOSE is set to
+# TRUE.
+#
+#-----------------------------------------------------------------------
+#
+  print_input_args "valid_args"
+#
+#-----------------------------------------------------------------------
+#
+# Verify that the required arguments to this function have been specified.
+# If not, print out an error message and exit.
+#
+#-----------------------------------------------------------------------
+#
+  if [ -z "$fp" ]; then
+    print_err_msg_exit "\
+The argument \"fp\" specifying the relative or full path to the file to
+read was not specified in the call to this function:
+  fp = \"$fp\""
+  fi
+#
+#-----------------------------------------------------------------------
+#
+# Declare local variables.
+#
+#-----------------------------------------------------------------------
+#
+  local contents \
+        crnt_line
+#
+#-----------------------------------------------------------------------
+#
+# Read in all lines in the file.  In doing so:
+#
+# 1) Concatenate any line ending with the bash line continuation character
+#    (a backslash) with the following line.
+# 2) Remove any leading and trailing whitespace.
+#
+# Note that these two actions are automatically performed by the "read"
+# utility in the while-loop below.
+#
+#-----------------------------------------------------------------------
+#
+  contents=""
+  while read crnt_line; do
+    contents="${contents}${crnt_line}
+"
+  done < "$fp"
+#
+# Strip out any comment and empty lines from contents.
+#
+  contents=$( printf "${contents}" | \
+              $SED -r -e "/^#.*/d"  `# Remove comment lines.` \
+                      -e "/^$/d"    `# Remove empty lines.` \
+            )
+#
+#-----------------------------------------------------------------------
+#
+# Set output variables.
+#
+#-----------------------------------------------------------------------
+#
+  printf -v ${output_varname_contents} "${contents}"
+#
+#-----------------------------------------------------------------------
+#
+# Restore the shell options saved at the beginning of this script/function.
+#
+#-----------------------------------------------------------------------
+#
+  { restore_shell_opts; } > /dev/null 2>&1
+
+}
+

ush/bash_utils/print_input_args.sh

@@ -169,7 +169,7 @@ have been set as follows:
 #-----------------------------------------------------------------------
 #
 # If a global variable named DEBUG is not defined, print out the message.  
-# If it is defined, print out the message only if DEBUG is set to TRUE.
+# If it is defined, print out the message only if DEBUG is set to "TRUE".
 #
 #-----------------------------------------------------------------------
 #