Enhance ability to use template variables

tests/WE2E/get_WE2Etest_names_subdirs_descs.sh

@@ -759,7 +759,7 @@ This is probably because it is a directory.  Please correct and rerun."
   test_names=("${prim_test_names[@]}")
   test_subdirs=("${prim_test_subdirs[@]}")
   if [ "${num_alt_tests}" -gt "0" ]; then
-    test_names+=("${alt_test_subdirs[@]:-}")
+    test_names+=("${alt_test_names[@]:-}")
     test_subdirs+=("${alt_test_subdirs[@]:-}")
   fi
 #
@@ -1025,7 +1025,7 @@ Please correct and rerun."
 # listed first.  
 #
 # Finally, we extract from test_ids_and_inds_sorted the second number 
-# in each element (the one afte the first number, which is the test ID,
+# in each element (the one after the first number, which is the test ID,
 # and the test type, which we no longer need), which is the original 
 # array index before sorting, and save the results in the array sort_inds.  
 # This array will contain the original indices in sorted order that we 

tests/WE2E/get_expts_status.sh

@@ -50,7 +50,7 @@ scrfunc_dir=$( dirname "${scrfunc_fp}" )
 #
 #-----------------------------------------------------------------------
 #
-homerrfs=${scrfunc_dir%/*}
+homerrfs=${scrfunc_dir%/*/*}
 #
 #-----------------------------------------------------------------------
 #

tests/WE2E/run_WE2E_tests.sh

@@ -538,8 +538,8 @@ accordingly and rerun."
   if [ "${match_found}" = "FALSE" ]; then
     avail_WE2E_test_names_str=$( printf "  \"%s\"\n" "${avail_WE2E_test_names[@]}" )
     print_err_msg_exit "\
-The name current user-specified test to run (user_spec_test) does not 
-match any of the names (either primary or alternate) of the available
+The name of the current user-specified test to run (user_spec_test) does 
+not match any of the names (either primary or alternate) of the available
 WE2E tests:
   user_spec_test = \"${user_spec_test}\"
 Valid values for user_spec_test consist of the names (primary or alternate)

tests/WE2E/test_configs/wflow_features/config.deactivate_tasks.sh

@@ -2,10 +2,26 @@
 # TEST PURPOSE/DESCRIPTION:
 # ------------------------
 #
-# This test ensures that the various workflow tasks can be deactivated,
-# i.e. removed from the Rocoto XML.  Note that we leave the MAKE_GRID_TN, 
-# MAKE_OROG_TN, and MAKE_SFC_CLIMO_TN activated because there is a 
-# separate test for turning those off.
+# This test has two purposes:
+#
+# 1) It checks that the various workflow tasks can be deactivated, i.e. 
+#    removed from the Rocoto XML.
+# 2) It checks the capability of the workflow to use "template" experiment 
+#    variables, i.e. variables whose definitions include references to 
+#    other variables, e.g.
+#
+#      MY_VAR='\${ANOTHER_VAR}'
+#
+# Note that we do not deactivate all tasks in the workflow; we leave the 
+# MAKE_GRID_TN, MAKE_OROG_TN, and MAKE_SFC_CLIMO_TN activated because:
+#
+# 1) There is already a WE2E test that runs with these three tasks
+#    deactivated (that test is to ensure that pre-generated grid, 
+#    orography, and surface climatology files can be used).
+# 2) In checking the template variable capability, we want to make sure
+#    that the variable defintions file (GLOBAL_VAR_DEFNS_FN) generated
+#    does not have syntax or other errors in it by sourcing it in these 
+#    three tasks.
 #
 
 RUN_ENVIR="community"
@@ -14,13 +30,31 @@ PREEXISTING_DIR_METHOD="rename"
 PREDEF_GRID_NAME="RRFS_CONUS_25km"
 CCPP_PHYS_SUITE="FV3_GFS_v15p2"
 
-DATE_FIRST_CYCL="20190615"
-DATE_LAST_CYCL="20190615"
+EXTRN_MDL_NAME_ICS="FV3GFS"
+EXTRN_MDL_NAME_LBCS="FV3GFS"
+USE_USER_STAGED_EXTRN_FILES="TRUE"
+
+DATE_FIRST_CYCL="20190701"
+DATE_LAST_CYCL="20190701"
 CYCL_HRS=( "00" )
 
+FCST_LEN_HRS="6"
+LBC_SPEC_INTVL_HRS="3"
+
 RUN_TASK_GET_EXTRN_ICS="FALSE"
 RUN_TASK_GET_EXTRN_LBCS="FALSE"
 RUN_TASK_MAKE_ICS="FALSE"
 RUN_TASK_MAKE_LBCS="FALSE"
 RUN_TASK_RUN_FCST="FALSE"
 RUN_TASK_RUN_POST="FALSE"
+#
+# The following shows examples of how to define template variables.  Here,
+# we define RUN_CMD_UTILS, RUN_CMD_FCST, and RUN_CMD_POST as template 
+# variables.  Note that during this test, these templates aren't actually 
+# expanded/used (something that would be done using bash's "eval" built-in 
+# command) anywhere in the scripts.  They are included here only to verify
+# that the test completes with some variables defined as templates.
+#
+RUN_CMD_UTILS='cd $yyyymmdd'
+RUN_CMD_FCST='mpirun -np ${PE_MEMBER01}'
+RUN_CMD_POST='echo hello $yyyymmdd'

tests/WE2E/test_configs/wflow_features/config.template_vars.sh

@@ -0,0 +1 @@
+config.deactivate_tasks.sh

ush/bash_utils/check_var_valid_value.sh

@@ -108,24 +108,34 @@ The value specified in ${var_name} is not supported:
 #
 #-----------------------------------------------------------------------
 #
-# Check whether var_value is equal to one of the elements of the array
-# valid_var_values.  If not, print out an error message and exit the 
-# calling script.
+# If var_value contains a dollar sign, we assume the corresponding variable 
+# (var_name) is a template variable, i.e. one whose value contains a 
+# reference to another variable, e.g.
+#
+#   MY_VAR='\${ANOTHER_VAR}'
+#
+# In this case, we do nothing since it does not make sense to check 
+# whether var_value is a valid value (since its contents have not yet 
+# been expanded).  If var_value doesn't contain a dollar sign, it must 
+# contain a literal string.  In this case, we check whether it is equal 
+# to one of the elements of the array valid_var_values.  If not, we 
+# print out an error message and exit the calling script.
 #
 #-----------------------------------------------------------------------
 #
-  is_element_of "valid_var_values" "${var_value}" || { \
-    valid_var_values_str=$(printf "\"%s\" " "${valid_var_values[@]}");
-    print_err_msg_exit "\
+  if [[ "${var_value}" != *'$'* ]]; then
+    is_element_of "valid_var_values" "${var_value}" || { \
+      valid_var_values_str=$(printf "\"%s\" " "${valid_var_values[@]}");
+      print_err_msg_exit "\
 ${err_msg}
 ${var_name} must be set to one of the following:
   ${valid_var_values_str}"; \
-  }
+    }
+  fi
 #
 #-----------------------------------------------------------------------
 #
-# Restore the shell options saved at the beginning of this script/func-
-# tion.
+# Restore the shell options saved at the beginning of this script/function.
 #
 #-----------------------------------------------------------------------
 #

ush/bash_utils/get_bash_file_contents.sh

@@ -0,0 +1,71 @@
+#
+#-----------------------------------------------------------------------
+#
+# This file defines a function that returns the contents of a bash script/
+# function with all empty lines, comment lines, and leading and trailing 
+# whitespace removed.  Arguments are as follows:
+#
+# fp:
+# The relative or full path to the file containing the bash script or 
+# function.
+#
+# output_varname_contents:
+# Name of the output variable that will contain the (processed) contents 
+# of the file.  This is the output of the function.
+# 
+#-----------------------------------------------------------------------
+#
+function get_bash_file_contents() { 
+
+  { save_shell_opts; set -u +x; } > /dev/null 2>&1
+
+  local valid_args=( \
+    "fp" \
+    "output_varname_contents" \
+    )
+  process_args valid_args "$@"
+  print_input_args "valid_args"
+  #
+  # Verify that the required arguments to this function have been specified.
+  # If not, print out an error message and exit.
+  #
+  if [ -z "$fp" ]; then
+    print_err_msg_exit "\
+The argument \"fp\" specifying the relative or full path to the file to
+read was not specified in the call to this function:
+  fp = \"$fp\""
+  fi
+
+  local contents \
+        crnt_line
+  #
+  # Read in all lines in the file.  In doing so:
+  #
+  # 1) Concatenate any line ending with the bash line continuation character
+  #    (a backslash) with the following line.
+  # 2) Remove any leading and trailing whitespace.
+  #
+  # Note that these two actions are automatically performed by the "read"
+  # utility in the while-loop below.
+  #
+  contents=""
+  while read crnt_line; do
+    contents="${contents}${crnt_line}
+"
+  done < "$fp"
+  #
+  # Strip out any comment and empty lines from contents.
+  #
+  contents=$( printf "${contents}" | \
+              $SED -r -e "/^#.*/d"  `# Remove comment lines.` \
+                      -e "/^$/d"    `# Remove empty lines.` \
+            )
+  #
+  # Set output variables.
+  #
+  printf -v ${output_varname_contents} "${contents}"
+
+  { restore_shell_opts; } > /dev/null 2>&1
+
+}
+

ush/bash_utils/print_input_args.sh

@@ -169,7 +169,7 @@ have been set as follows:
 #-----------------------------------------------------------------------
 #
 # If a global variable named DEBUG is not defined, print out the message.  
-# If it is defined, print out the message only if DEBUG is set to TRUE.
+# If it is defined, print out the message only if DEBUG is set to "TRUE".
 #
 #-----------------------------------------------------------------------
 #

ush/check_expt_config_vars.sh

@@ -0,0 +1,110 @@
+#
+#-----------------------------------------------------------------------
+#
+# This file defines a function that checks that all experiment variables 
+# set in the user-specified experiment configuration file are defined (by 
+# being assigned default values) in the default experiment configuration 
+# file.  If a variable is found in the former that is not defined in the 
+# latter, this function exits with an error message.  
+#
+# This check is performed in order to prevent the user from defining 
+# arbitrary variables in the user-specified configuration file; the 
+# latter should be used to specify only varaibles that have already been 
+# defined in the default configuration file.  
+#
+# Arguments are as follows:
+#
+# default_config_fp:
+# The relative or full path to the default experiment configuration file.
+#
+# config_fp:
+# The relative or full path to the user-specified experiment configuration 
+# file.
+#
+#-----------------------------------------------------------------------
+#
+function check_expt_config_vars() {
+
+  . ${scrfunc_dir}/source_util_funcs.sh
+
+  { save_shell_opts; set -u +x; } > /dev/null 2>&1
+
+  local valid_args=( \
+    "default_config_fp" \
+    "config_fp" \
+    )
+  process_args valid_args "$@"
+  print_input_args "valid_args"
+
+  local var_list_default \
+        var_list_user \
+        crnt_line \
+        var_name \
+        regex_search
+  #
+  # Get the list of variable definitions, first from the default experiment
+  # configuration file and then from the user-specified experiment
+  # configuration file.
+  #
+  get_bash_file_contents fp="${default_config_fp}" \
+                         output_varname_contents="var_list_default"
+
+  get_bash_file_contents fp="${config_fp}" \
+                         output_varname_contents="var_list_user"
+  #
+  # Loop through each line/variable in var_list_user.  For each line,
+  # extract the the name of the variable that is being set (say VAR) and
+  # check that this variable is set somewhere in the default configuration
+  # file by verifying that a line that starts with "VAR=" exists in
+  # var_list_default.
+  #
+  while read crnt_line; do
+    #
+    # Note that a variable name will be found only if the equal sign immediately
+    # follows the variable name.
+    #
+    var_name=$( printf "%s" "${crnt_line}" | $SED -n -r -e "s/^([^ =\"]*)=.*/\1/p")
+
+    if [ -z "${var_name}" ]; then
+
+      print_info_msg "
+The current line (crnt_line) of the user-specified experiment configuration
+file (config_fp) does not contain a variable name (i.e. var_name is empty):
+  config_fp = \"${config_fp}\"
+  crnt_line = \"${crnt_line}\"
+  var_name = \"${var_name}\"
+Skipping to next line."
+
+    else
+      #
+      # Use grep to search for the variable name (followed by an equal sign, 
+      # all at the beginning of a line) in the list of variables in the default
+      # configuration file.  
+      #
+      # Note that we use a herestring to input into grep the list of variables 
+      # in the default configuration file.  grep will return with a zero status 
+      # if the specified string (regex_search) is not found in the default 
+      # variables list and a nonzero status otherwise.  Note also that we 
+      # redirect the output of grep to null because we are only interested in 
+      # its exit status.
+      #
+      regex_search="^${var_name}="
+      grep "${regex_search}" <<< "${var_list_default}" > /dev/null 2>&1 || \
+      print_err_msg_exit "\
+The variable (var_name) defined on the current line (crnt_line) of the 
+user-specified experiment configuration file (config_fp) does not appear 
+in the default experiment configuration file (default_config_fp):
+  config_fp = \"${config_fp}\"
+  default_config_fp = \"${default_config_fp}\"
+  crnt_line = \"${crnt_line}\"
+  var_name = \"${var_name}\"
+Please assign a default value to this variable in the default configuration
+file and rerun."
+
+    fi
+
+  done <<< "${var_list_user}"
+
+  { restore_shell_opts; } > /dev/null 2>&1
+
+}