diff --git a/jobs/JREGIONAL_GET_EXTRN_MDL_FILES b/jobs/JREGIONAL_GET_EXTRN_MDL_FILES index 6b373895e..fed4ca8f2 100755 --- a/jobs/JREGIONAL_GET_EXTRN_MDL_FILES +++ b/jobs/JREGIONAL_GET_EXTRN_MDL_FILES @@ -33,15 +33,6 @@ # #----------------------------------------------------------------------- # -# Source the file defining the function that will be used to set various -# external-model-associated variables. -# -#----------------------------------------------------------------------- -# -. $USHDIR/get_extrn_mdl_file_dir_info.sh -# -#----------------------------------------------------------------------- -# # Save current shell options (in a global array). Then set new options # for this script/function. # @@ -167,102 +158,15 @@ check_var_valid_value "ICS_OR_LBCS" "valid_vals_ICS_OR_LBCS" # #----------------------------------------------------------------------- # -# Set the parameter anl_or_fcst that determines whether we want to get -# analysis or forecast files. This depends on whether we want these files -# to generate initial condition and surface field files or lateral boundary -# condition files. Also, set time_offset_hrs, which is the offset in -# hours between the current cycle's starting time and the starting time -# of the external model providing the LBCs. +# Set the name of the directory in which to stage the external model +# files (or symlinks to such files) for the current cycle. Then create +# the directory (if it doesn't already exist) and change location to it. # #----------------------------------------------------------------------- # -if [ "${ICS_OR_LBCS}" = "ICS" ]; then - anl_or_fcst="ANL" - time_offset_hrs="0" -elif [ "${ICS_OR_LBCS}" = "LBCS" ]; then - anl_or_fcst="FCST" - time_offset_hrs="${EXTRN_MDL_LBCS_OFFSET_HRS}" -fi -# -#----------------------------------------------------------------------- -# -# Set the name of and then create the directory in which to stage the -# external model files for the current cycle (if it doesn't already exist). -# Then change location to that directory. -# -#----------------------------------------------------------------------- -# -extrn_mdl_staging_dir="${CYCLE_DIR}/${EXTRN_MDL_NAME}/for_${ICS_OR_LBCS}" -mkdir_vrfy -p "${extrn_mdl_staging_dir}" -cd_vrfy "${extrn_mdl_staging_dir}" -# -#----------------------------------------------------------------------- -# -# Call the function that sets various external-model-associated variables. -# See the function defintion file for the definitions of these variables. -# -#----------------------------------------------------------------------- -# -get_extrn_mdl_file_dir_info \ - extrn_mdl_name="${EXTRN_MDL_NAME}" \ - anl_or_fcst="${anl_or_fcst}" \ - cdate_FV3LAM="${CDATE}" \ - time_offset_hrs="${time_offset_hrs}" \ - varname_extrn_mdl_cdate="extrn_mdl_cdate" \ - varname_extrn_mdl_lbc_spec_fhrs="extrn_mdl_lbc_spec_fhrs" \ - varname_extrn_mdl_fns_on_disk="extrn_mdl_fns_on_disk" \ - varname_extrn_mdl_fns_in_arcv="extrn_mdl_fns_in_arcv" \ - varname_extrn_mdl_sysdir="extrn_mdl_sysdir" \ - varname_extrn_mdl_arcv_fmt="extrn_mdl_arcv_fmt" \ - varname_extrn_mdl_arcv_fns="extrn_mdl_arcv_fns" \ - varname_extrn_mdl_arcv_fps="extrn_mdl_arcv_fps" \ - varname_extrn_mdl_arcvrel_dir="extrn_mdl_arcvrel_dir" || \ -print_err_msg_exit "\ -Call to function get_extrn_mdl_file_dir_info failed." -# -#----------------------------------------------------------------------- -# -# Set the directory in which to check for the external model files (which -# we refer to here as the "source" directory) to the default one set above -# for the current machine and external model. -# -#----------------------------------------------------------------------- -# -extrn_mdl_source_dir="${extrn_mdl_sysdir}" -# -#----------------------------------------------------------------------- -# -# If the user has specified that the external model files to be used for -# generating ICs or LBCs are staged, then reset extrn_mdl_source_dir to -# the user-specified directory in which these files are staged, and reset -# extrn_mdl_fns_on_disk to the user-specified array containing the names -# of the files. -# -#----------------------------------------------------------------------- -# -if [ "${USE_USER_STAGED_EXTRN_FILES}" = "TRUE" ]; then - - if [ "${ICS_OR_LBCS}" = "ICS" ]; then - extrn_mdl_source_dir="${EXTRN_MDL_SOURCE_BASEDIR_ICS}/$CDATE" - extrn_mdl_fns_on_disk=( $( printf "%s " "${EXTRN_MDL_FILES_ICS[@]}" )) - elif [ "${ICS_OR_LBCS}" = "LBCS" ]; then - extrn_mdl_source_dir="${EXTRN_MDL_SOURCE_BASEDIR_LBCS}/$CDATE" - extrn_mdl_fns_on_disk=( $( printf "%s " "${EXTRN_MDL_FILES_LBCS[@]}" )) - fi - - if [ ! -d "${extrn_mdl_source_dir}" ]; then - print_err_msg_exit "\ -The directory extrn_mdl_source_dir containing the user-staged external -model files does not exist: - extrn_mdl_source_dir = \"${extrn_mdl_source_dir}\" -Please ensure that the directory specified by extrn_mdl_source_dir exists -and that all the files specified in the array extrn_mdl_fns_on_disk exist -within it: - extrn_mdl_source_dir = \"${extrn_mdl_source_dir}\" - extrn_mdl_fns_on_disk = ( $( printf "\"%s\" " "${extrn_mdl_fns_on_disk[@]}" ))" - fi - -fi +staging_dir="${CYCLE_DIR}/${EXTRN_MDL_NAME}/for_${ICS_OR_LBCS}" +mkdir_vrfy -p "${staging_dir}" +cd_vrfy "${staging_dir}" # #----------------------------------------------------------------------- # @@ -270,25 +174,12 @@ fi # #----------------------------------------------------------------------- # -extrn_mdl_lbc_spec_fhrs_str="( "$( printf "\"%s\" " "${extrn_mdl_lbc_spec_fhrs[@]}" )")" -extrn_mdl_fns_on_disk_str="( "$( printf "\"%s\" " "${extrn_mdl_fns_on_disk[@]}" )")" -extrn_mdl_fns_in_arcv_str="( "$( printf "\"%s\" " "${extrn_mdl_fns_in_arcv[@]}" )")" -extrn_mdl_arcv_fns_str="( "$( printf "\"%s\" " "${extrn_mdl_arcv_fns[@]}" )")" -extrn_mdl_arcv_fps_str="( "$( printf "\"%s\" " "${extrn_mdl_arcv_fps[@]}" )")" +extrn_mdl_data_sources_str="( "$( printf "\"%s\" " "${EXTRN_MDL_DATA_SOURCES[@]}" )")" $SCRIPTSDIR/exregional_get_extrn_mdl_files.sh \ ics_or_lbcs="${ICS_OR_LBCS}" \ - use_user_staged_extrn_files="${USE_USER_STAGED_EXTRN_FILES}" \ - extrn_mdl_cdate="${extrn_mdl_cdate}" \ - extrn_mdl_lbc_spec_fhrs="${extrn_mdl_lbc_spec_fhrs_str}" \ - extrn_mdl_fns_on_disk="${extrn_mdl_fns_on_disk_str}" \ - extrn_mdl_fns_in_arcv="${extrn_mdl_fns_in_arcv_str}" \ - extrn_mdl_source_dir="${extrn_mdl_source_dir}" \ - extrn_mdl_staging_dir="${extrn_mdl_staging_dir}" \ - extrn_mdl_arcv_fmt="${extrn_mdl_arcv_fmt}" \ - extrn_mdl_arcv_fns="${extrn_mdl_arcv_fns_str}" \ - extrn_mdl_arcv_fps="${extrn_mdl_arcv_fps_str}" \ - extrn_mdl_arcvrel_dir="${extrn_mdl_arcvrel_dir}" || \ + data_sources="${extrn_mdl_data_sources_str}" \ + staging_dir="${staging_dir}" || \ print_err_msg_exit "\ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." # diff --git a/jobs/JREGIONAL_MAKE_ICS b/jobs/JREGIONAL_MAKE_ICS index 01027b4ac..8d9e75272 100755 --- a/jobs/JREGIONAL_MAKE_ICS +++ b/jobs/JREGIONAL_MAKE_ICS @@ -17,7 +17,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u -x; } > /dev/null 2>&1 +{ save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/jobs/JREGIONAL_RUN_POST b/jobs/JREGIONAL_RUN_POST index 68dcf95bd..6ecdcc561 100755 --- a/jobs/JREGIONAL_RUN_POST +++ b/jobs/JREGIONAL_RUN_POST @@ -26,7 +26,7 @@ # #----------------------------------------------------------------------- # -{ save_shell_opts; set -u -x; } > /dev/null 2>&1 +{ save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_get_extrn_mdl_files.sh b/scripts/exregional_get_extrn_mdl_files.sh index 35e8d373b..cde67ff29 100755 --- a/scripts/exregional_get_extrn_mdl_files.sh +++ b/scripts/exregional_get_extrn_mdl_files.sh @@ -12,6 +12,19 @@ # #----------------------------------------------------------------------- # +# Source required files. +# +#----------------------------------------------------------------------- +# +. $USHDIR/extrn_mdl/create_extrn_mdl_var_defns_file.sh +. $USHDIR/extrn_mdl/get_extrn_mdl_files_from_disk.sh +. $USHDIR/extrn_mdl/get_extrn_mdl_files_from_noaa_hpss.sh +. $USHDIR/extrn_mdl/get_extrn_mdl_files_from_nomads.sh +. $USHDIR/extrn_mdl/set_extrn_mdl_arcv_file_dir_names.sh +. $USHDIR/extrn_mdl/set_extrn_mdl_filenames.sh +# +#----------------------------------------------------------------------- +# # Save current shell options (in a global array). Then set new options # for this script/function. # @@ -56,19 +69,10 @@ boundary condition files for the FV3 will be generated. #----------------------------------------------------------------------- # valid_args=( \ -"ics_or_lbcs" \ -"use_user_staged_extrn_files" \ -"extrn_mdl_cdate" \ -"extrn_mdl_lbc_spec_fhrs" \ -"extrn_mdl_fns_on_disk" \ -"extrn_mdl_fns_in_arcv" \ -"extrn_mdl_source_dir" \ -"extrn_mdl_staging_dir" \ -"extrn_mdl_arcv_fmt" \ -"extrn_mdl_arcv_fns" \ -"extrn_mdl_arcv_fps" \ -"extrn_mdl_arcvrel_dir" \ -) + "ics_or_lbcs" \ + "data_sources" \ + "staging_dir" \ + ) process_args valid_args "$@" # #----------------------------------------------------------------------- @@ -79,722 +83,324 @@ process_args valid_args "$@" # #----------------------------------------------------------------------- # -print_input_args valid_args +print_input_args "valid_args" # #----------------------------------------------------------------------- # -# Set num_files_to_copy to the number of external model files that need -# to be copied or linked to from/at a location on disk. Then set -# extrn_mdl_fps_on_disk to the full paths of the external model files -# on disk. +# Set the name of the external model. # #----------------------------------------------------------------------- # -num_files_to_copy="${#extrn_mdl_fns_on_disk[@]}" -prefix="${extrn_mdl_source_dir}/" -extrn_mdl_fps_on_disk=( "${extrn_mdl_fns_on_disk[@]/#/$prefix}" ) +if [ "${ics_or_lbcs}" = "ICS" ]; then + extrn_mdl_name="${EXTRN_MDL_NAME_ICS}" +elif [ "${ics_or_lbcs}" = "LBCS" ]; then + extrn_mdl_name="${EXTRN_MDL_NAME_LBCS}" +fi # #----------------------------------------------------------------------- # -# Loop through the list of external model files and check whether they -# all exist on disk. The counter num_files_found_on_disk keeps track of -# the number of external model files that were actually found on disk in -# the directory specified by extrn_mdl_source_dir. -# -# If the location extrn_mdl_source_dir is a user-specified directory -# (i.e. if use_user_staged_extrn_files is set to "TRUE"), then if/when we -# encounter the first file that does not exist, we exit the script with -# an error message. If extrn_mdl_source_dir is a system directory (i.e. -# if use_user_staged_extrn_files is not set to "TRUE"), then if/when we -# encounter the first file that does not exist or exists but is younger -# than a certain age, we break out of the loop and try to fetch all the -# necessary external model files from HPSS. The age cutoff is to ensure -# that files are not still being written to. +# Set the number of hours to shift back the starting time of the external +# model for LBCs (relative to the starting time of the FV3LAM). # #----------------------------------------------------------------------- # -num_files_found_on_disk="0" -min_age="5" # Minimum file age, in minutes. - -for fp in "${extrn_mdl_fps_on_disk[@]}"; do - # - # If the external model file exists, then... - # - if [ -f "$fp" ]; then - # - # Increment the counter that keeps track of the number of external - # model files found on disk and print out an informational message. - # - num_files_found_on_disk=$(( num_files_found_on_disk+1 )) - print_info_msg " -File fp exists on disk: - fp = \"$fp\"" - # - # If we are NOT searching for user-staged external model files, then - # we also check that the current file is at least min_age minutes old. - # If not, we try searching for all the external model files on HPSS. - # - if [ "${use_user_staged_extrn_files}" != "TRUE" ]; then - - if [ $( find "$fp" -mmin +${min_age} ) ]; then - - print_info_msg " -File fp is older than the minimum required age of min_age minutes: - fp = \"$fp\" - min_age = ${min_age} minutes" - - else - - print_info_msg " -File fp is NOT older than the minumum required age of min_age minutes: - fp = \"$fp\" - min_age = ${min_age} minutes -Will try fetching all external model files from HPSS. Not checking -presence and age of remaining external model files on disk." - break - - fi - - fi - # - # If the external model file does not exist, then... - # - else - # - # If an external model file is not found and we are searching for it - # in a user-specified directory, print out an error message and exit. - # - if [ "${use_user_staged_extrn_files}" = "TRUE" ]; then - - print_err_msg_exit "\ -File fp does NOT exist on disk: - fp = \"$fp\" -Please ensure that the directory specified by extrn_mdl_source_dir exists -and that all the files specified in the array extrn_mdl_fns_on_disk exist -within it: - extrn_mdl_source_dir = \"${extrn_mdl_source_dir}\" - extrn_mdl_fns_on_disk = ( $( printf "\"%s\" " "${extrn_mdl_fns_on_disk[@]}" ))" - # - # If an external model file is not found and we are searching for it - # in a system directory, give up on the system directory and try instead - # to get all the external model files from HPSS. - # - else - - print_info_msg " -File fp does NOT exist on disk: - fp = \"$fp\" -Will try fetching all external model files from HPSS. Not checking -presence and age of remaining external model files on disk." - break - - fi +extrn_mdl_temporal_offset_hrs="0" +if [ "${ics_or_lbcs}" = "LBCS" ]; then - fi + case "${EXTRN_MDL_NAME_LBCS}" in + "GSMGFS") + extrn_mdl_temporal_offset_hrs="0" + ;; + "FV3GFS") + extrn_mdl_temporal_offset_hrs="0" + ;; + "RAP") + extrn_mdl_temporal_offset_hrs="3" + ;; + "HRRR") + extrn_mdl_temporal_offset_hrs="0" + ;; + "NAM") + extrn_mdl_temporal_offset_hrs="0" + ;; + esac -done +fi # #----------------------------------------------------------------------- # -# Set the variable (data_src) that determines the source of the external -# model files (either disk or HPSS). +# Extract from CDATE the starting date (without the hour) and hour-of-day +# of the current FV3LAM cycle. Then subtract the temporal offset given +# by extrn_mdl_temporal_offset_hrs (in units of hours) from CDATE to +# obtain the starting date and time of the external model, express the +# result in YYYYMMDDHH format, and save it in cdate. This is the starting +# time of the external model forecast. # #----------------------------------------------------------------------- # -if [ "${num_files_found_on_disk}" -eq "${num_files_to_copy}" ]; then - data_src="disk" -else - data_src="HPSS" -fi +parse_cdate \ + cdate="$CDATE" \ + outvarname_yyyymmdd="yyyymmdd" \ + outvarname_hh="hh" \ -if [ ${NOMADS} == "TRUE" ]; then - data_src="online" -fi +cdate=$( date --utc --date \ + "${yyyymmdd} ${hh} UTC - ${extrn_mdl_temporal_offset_hrs} hours" \ + "+%Y%m%d%H" ) # #----------------------------------------------------------------------- # -# If the source of the external model files is "disk", copy the files -# from the source directory on disk to a staging directory. +# If fetching external model files for the purpose of creating lateral +# boundary conditions (LBCs), set lbc_spec_fhrs to the array of forecast +# hours (of the external model, not of the FV3LAM) at which the LBCs are +# to be specified, starting with the 2nd such time (i.e. the one having +# array index 1). We do not include the first hour (hour 0) because at +# this initial time, the LBCs are obtained from the analysis fields +# provided by the external model (as opposed to a forecast field). # #----------------------------------------------------------------------- # -extrn_mdl_fns_on_disk_str="( "$( printf "\"%s\" " "${extrn_mdl_fns_on_disk[@]}" )")" +lbc_spec_fhrs=( "" ) -if [ "${data_src}" = "disk" ]; then - - if [ "${RUN_ENVIR}" = "nco" ]; then - - print_info_msg " -Creating links in staging directory (extrn_mdl_staging_dir) to external -model files on disk (extrn_mdl_fns_on_disk) in the source directory -(extrn_mdl_source_dir): - extrn_mdl_staging_dir = \"${extrn_mdl_staging_dir}\" - extrn_mdl_source_dir = \"${extrn_mdl_source_dir}\" - extrn_mdl_fns_on_disk = ${extrn_mdl_fns_on_disk_str}" - - ln_vrfy -sf -t ${extrn_mdl_staging_dir} ${extrn_mdl_fps_on_disk[@]} - - else +if [ "${ics_or_lbcs}" = "LBCS" ]; then - # - # If the external model files are user-staged, then simply link to - # them. Otherwise, if they are on the system disk, copy them to the - # staging directory. - # - if [ "${use_user_staged_extrn_files}" = "TRUE" ]; then - print_info_msg " -Creating symlinks in the staging directory (extrn_mdl_staging_dir) to the -external model files on disk (extrn_mdl_fns_on_disk) in the source directory -(extrn_mdl_source_dir): - extrn_mdl_source_dir = \"${extrn_mdl_source_dir}\" - extrn_mdl_fns_on_disk = ${extrn_mdl_fns_on_disk_str} - extrn_mdl_staging_dir = \"${extrn_mdl_staging_dir}\"" - ln_vrfy -sf -t ${extrn_mdl_staging_dir} ${extrn_mdl_fps_on_disk[@]} - else - print_info_msg " -Copying external model files on disk (extrn_mdl_fns_on_disk) from source -directory (extrn_mdl_source_dir) to staging directory (extrn_mdl_staging_dir): - extrn_mdl_source_dir = \"${extrn_mdl_source_dir}\" - extrn_mdl_fns_on_disk = ${extrn_mdl_fns_on_disk_str} - extrn_mdl_staging_dir = \"${extrn_mdl_staging_dir}\"" - cp_vrfy ${extrn_mdl_fps_on_disk[@]} ${extrn_mdl_staging_dir} - fi + lbc_spec_fhrs=( "${LBC_SPEC_FHRS[@]}" ) +# +# Add the temporal offset specified by extrn_mdl_temporal_offset_hrs +# (which is in units of hours) to the array of LBC update forecast hours +# to make up for the fact that the starting time of the external model +# is shifted back from that of the FV3LAM by this offset. After this +# addition, lbc_spec_fhrs will contain the LBC update forecast hours +# relative to the start time of the external model. +# + num_fhrs=${#lbc_spec_fhrs[@]} + for (( i=0; i<=$((num_fhrs-1)); i++ )); do + lbc_spec_fhrs[$i]=$(( ${lbc_spec_fhrs[$i]} + ${extrn_mdl_temporal_offset_hrs} )) + done - fi +fi # #----------------------------------------------------------------------- # -# Print message indicating successful completion of script. +# Loop through the specified external model data sources until we are +# able to get the external model files from one. # #----------------------------------------------------------------------- # - if [ "${ics_or_lbcs}" = "ICS" ]; then +num_data_sources="${#data_sources[@]}" +for (( i=0; i<${num_data_sources}; i++ )); do - print_info_msg " -======================================================================== -Successfully copied or linked to external model files on disk needed for -generating initial conditions and surface fields for the FV3 forecast!!! + data_src="${data_sources[i]}" -Exiting script: \"${scrfunc_fn}\" -In directory: \"${scrfunc_dir}\" -========================================================================" - - elif [ "${ics_or_lbcs}" = "LBCS" ]; then - - print_info_msg " -======================================================================== -Successfully copied or linked to external model files on disk needed for -generating lateral boundary conditions for the FV3 forecast!!! - -Exiting script: \"${scrfunc_fn}\" -In directory: \"${scrfunc_dir}\" -========================================================================" + print_info_msg " +Attempting to obtain external model data from current data source (data_src): + data_src = \"${data_src}\" +..." +# +# Get file names. +# + lbc_spec_fhrs_str="( "$( printf "\"%s\" " "${lbc_spec_fhrs[@]}" )")" + set_extrn_mdl_filenames \ + data_src="${data_src}" \ + extrn_mdl_name="${extrn_mdl_name}" \ + ics_or_lbcs="${ics_or_lbcs}" \ + cdate="$cdate" \ + lbc_spec_fhrs="${lbc_spec_fhrs_str}" \ + outvarname_fns="__fns" + + fns_str="( "$( printf "\"%s\" " "${__fns[@]}" )")" +# +# Data source is local disk. +# + if [ "${data_src}" = "disk" ]; then + + get_extrn_mdl_files_from_disk \ + extrn_mdl_name="${extrn_mdl_name}" \ + ics_or_lbcs="${ics_or_lbcs}" \ + cdate="$cdate" \ + staging_dir="${staging_dir}" \ + fns="${fns_str}" +# +# Data source is NOAA HPSS. +# + elif [ "${data_src}" = "noaa_hpss" ]; then + + set_extrn_mdl_arcv_file_dir_names \ + extrn_mdl_name="${extrn_mdl_name}" \ + ics_or_lbcs="${ics_or_lbcs}" \ + cdate="$cdate" \ + lbc_spec_fhrs="${lbc_spec_fhrs_str}" \ + outvarname_arcv_fmt="__arcv_fmt" \ + outvarname_arcv_fns="__arcv_fns" \ + outvarname_arcv_fps="__arcv_fps" \ + outvarname_arcvrel_dir="__arcvrel_dir" + + arcv_fns_str="( "$( printf "\"%s\" " "${__arcv_fns[@]}" )")" + arcv_fps_str="( "$( printf "\"%s\" " "${__arcv_fps[@]}" )")" + get_extrn_mdl_files_from_noaa_hpss \ + staging_dir="${staging_dir}" \ + arcv_fmt="${__arcv_fmt}" \ + arcv_fns="${arcv_fns_str}" \ + arcv_fps="${arcv_fps_str}" \ + arcvrel_dir="${__arcvrel_dir}" \ + fns="${fns_str}" +# +# Data source is NOMADS. +# + elif [ "${data_src}" = "nomads" ]; then + + set_extrn_mdl_arcv_file_dir_names \ + extrn_mdl_name="${extrn_mdl_name}" \ + ics_or_lbcs="${ics_or_lbcs}" \ + cdate="$cdate" \ + outvarname_arcvrel_dir="__arcvrel_dir" + + get_extrn_mdl_files_from_nomads \ + extrn_mdl_name="${extrn_mdl_name}" \ + ics_or_lbcs="${ics_or_lbcs}" \ + staging_dir="${staging_dir}" \ + arcvrel_dir="${__arcvrel_dir}" \ + fns="${fns_str}" fi # #----------------------------------------------------------------------- # -# If the source of the external model files is "HPSS", fetch them from -# HPSS. +# If the file retrieval from the current external model data source +# (data_src) failed, then print out a message and either try the next +# data source (if more are available) or exit. # #----------------------------------------------------------------------- # -elif [ "${data_src}" = "HPSS" ]; then + if [ $? -ne 0 ]; then + + if [ $i -eq "$((${num_data_sources}-1))" ]; then + data_sources_str="( "$( printf "\"%s\" " "${data_sources[@]}" )")" + print_err_msg_exit " +Failed to obtain the external model data files for generating "${ics_or_lbcs}" from +any of the data sources specified in data_sources, which are: + data_sources = ${data_sources_str}" + else + print_info_msg "\ +Failed to obtain the external model data files for generating "${ics_or_lbcs}" from +the current data source (data_src): + data_src = \"${data_src}\" +Will try the next data source specified in data_sources, which is: + \"${data_sources[$((i+1))]}\"" + fi # #----------------------------------------------------------------------- # -# Set extrn_mdl_fps_in_arcv to the full paths within the archive files of -# the external model files. +# If the file retrieval from the current external model data source +# (data_src) succeeded, there is no need to try to retrieve the files +# from the remaining data sources specified in data_sources. In this +# case, print out an appropriate message and break out of the for-loop +# over the locations. # #----------------------------------------------------------------------- # - prefix=${extrn_mdl_arcvrel_dir:+${extrn_mdl_arcvrel_dir}/} - extrn_mdl_fps_in_arcv=( "${extrn_mdl_fns_in_arcv[@]/#/$prefix}" ) + else - extrn_mdl_fps_in_arcv_str="( "$( printf "\"%s\" " "${extrn_mdl_fps_in_arcv[@]}" )")" - extrn_mdl_arcv_fps_str="( "$( printf "\"%s\" " "${extrn_mdl_arcv_fps[@]}" )")" + if [ "${data_src}" = "disk" ]; then - print_info_msg " -Fetching external model files from HPSS. The full paths to these files -in the archive file(s) (extrn_mdl_fps_in_arcv), the archive files on HPSS -in which these files are stored (extrn_mdl_arcv_fps), and the staging -directory to which they will be copied (extrn_mdl_staging_dir) are: - extrn_mdl_fps_in_arcv = ${extrn_mdl_fps_in_arcv_str} - extrn_mdl_arcv_fps = ${extrn_mdl_arcv_fps_str} - extrn_mdl_staging_dir = \"${extrn_mdl_staging_dir}\"" -# -#----------------------------------------------------------------------- -# -# Get the number of archive files to consider. -# -#----------------------------------------------------------------------- -# - num_arcv_files="${#extrn_mdl_arcv_fps[@]}" -# -#----------------------------------------------------------------------- -# -# Consider the case of the archive file to be fetched from HPSS being in -# tar format. -# -#----------------------------------------------------------------------- -# - if [ "${extrn_mdl_arcv_fmt}" = "tar" ]; then -# -#----------------------------------------------------------------------- -# -# Loop through the set of archive files specified in extrn_mdl_arcv_fps -# and extract a subset of the specified external model files from each. -# -#----------------------------------------------------------------------- -# - num_files_to_extract="${#extrn_mdl_fps_in_arcv[@]}" + if [ "${ics_or_lbcs}" = "ICS" ]; then - for (( narcv=0; narcv<${num_arcv_files}; narcv++ )); do + print_info_msg " +======================================================================== +Successfully copied or linked to external model files on disk needed for +generating initial conditions and surface fields for the FV3 forecast!!! - narcv_formatted=$( printf "%02d" $narcv ) - arcv_fp="${extrn_mdl_arcv_fps[$narcv]}" -# -# Before trying to extract (a subset of) the external model files from -# the current tar archive file (which is on HPSS), create a list of those -# external model files that are stored in the current tar archive file. -# For this purpose, we first use the "htar -tvf" command to list all the -# external model files that are in the current archive file and store the -# result in a log file. (This command also indirectly checks whether the -# archive file exists on HPSS.) We then grep this log file for each -# external model file and create a list containing only those external -# model files that exist in the current archive. -# -# Note that the "htar -tvf" command will fail if the tar archive file -# itself doesn't exist on HPSS, but it won't fail if any of the external -# model file names passed to it don't exist in the archive file. In the -# latter case, the missing files' names simply won't appear in the log -# file. -# - htar_log_fn="log.htar_tvf.${narcv_formatted}" - htar -tvf ${arcv_fp} ${extrn_mdl_fps_in_arcv[@]} >& ${htar_log_fn} || \ - print_err_msg_exit "\ -htar file list operation (\"htar -tvf ...\") failed. Check the log file -htar_log_fn in the staging directory (extrn_mdl_staging_di)r for details: - extrn_mdl_staging_dir = \"${extrn_mdl_staging_dir}\" - htar_log_fn = \"${htar_log_fn}\"" +Exiting script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" +========================================================================" - i=0 - files_in_crnt_arcv=() - for (( nfile=0; nfile<${num_files_to_extract}; nfile++ )); do - extrn_mdl_fp="${extrn_mdl_fps_in_arcv[$nfile]}" -# grep -n ${extrn_mdl_fp} ${htar_log_fn} 2>&1 && { \ - grep -n ${extrn_mdl_fp} ${htar_log_fn} > /dev/null 2>&1 && { \ - files_in_crnt_arcv[$i]="${extrn_mdl_fp}"; \ - i=$((i+1)); \ - } - done -# -# If none of the external model files were found in the current archive -# file, print out an error message and exit. -# - num_files_in_crnt_arcv=${#files_in_crnt_arcv[@]} - if [ ${num_files_in_crnt_arcv} -eq 0 ]; then - extrn_mdl_fps_in_arcv_str="( "$( printf "\"%s\" " "${extrn_mdl_fps_in_arcv[@]}" )")" - print_err_msg_exit "\ -The current archive file (arcv_fp) does not contain any of the external -model files listed in extrn_mdl_fps_in_arcv: - arcv_fp = \"${arcv_fp}\" - extrn_mdl_fps_in_arcv = ${extrn_mdl_fps_in_arcv_str} -The archive file should contain at least one external model file; otherwise, -it would not be needed." - fi -# -# Extract from the current tar archive file on HPSS all the external model -# files that exist in that archive file. Also, save the output of the -# "htar -xvf" command in a log file for debugging (if necessary). -# - htar_log_fn="log.htar_xvf.${narcv_formatted}" - htar -xvf ${arcv_fp} ${files_in_crnt_arcv[@]} >& ${htar_log_fn} || \ - print_err_msg_exit "\ -htar file extract operation (\"htar -xvf ...\") failed. Check the log -file htar_log_fn in the staging directory (extrn_mdl_staging_dir) for -details: - extrn_mdl_staging_dir = \"${extrn_mdl_staging_dir}\" - htar_log_fn = \"${htar_log_fn}\"" -# -# Note that the htar file extract operation above may return with a 0 -# exit code (success) even if one or more (or all) external model files -# that it is supposed to contain were not extracted. The names of those -# files that were not extracted will not be listed in the log file. Thus, -# we now check whether the log file contains the name of each external -# model file that should have been extracted. If any are missing, we -# print out a message and exit the script because initial condition and -# surface field files needed by FV3 cannot be generated without all the -# external model files. -# - for fp in "${files_in_crnt_arcv[@]}"; do -# -# If the file path is absolute (i.e. starts with a "/"), then drop the -# leading "/" because htar strips it before writing the file path to the -# log file. -# - fp=${fp#/} - - grep -n "${fp}" "${htar_log_fn}" > /dev/null 2>&1 || \ - print_err_msg_exit "\ -External model file fp not extracted from tar archive file arcv_fp: - arcv_fp = \"${arcv_fp}\" - fp = \"$fp\" -Check the log file htar_log_fn in the staging directory (extrn_mdl_staging_dir) -for details: - extrn_mdl_staging_dir = \"${extrn_mdl_staging_dir}\" - htar_log_fn = \"${htar_log_fn}\"" - - done - - done -# -#----------------------------------------------------------------------- -# -# For each external model file that was supposed to have been extracted -# from the set of specified archive files, loop through the extraction -# log files and check that it appears exactly once in one of the log files. -# If it doesn't appear at all, then it means that file was not extracted, -# and if it appears more than once, then something else is wrong. In -# either case, print out an error message and exit. -# -#----------------------------------------------------------------------- -# - for (( nfile=0; nfile<${num_files_to_extract}; nfile++ )); do - extrn_mdl_fp="${extrn_mdl_fps_in_arcv[$nfile]}" -# -# If the file path is absolute (i.e. starts with a "/"), then drop the -# leading "/" because htar strips it before writing the file path to the -# log file. -# - extrn_mdl_fp=${extrn_mdl_fp#/} - - num_occurs=0 - for (( narcv=0; narcv<${num_arcv_files}; narcv++ )); do - narcv_formatted=$( printf "%02d" $narcv ) - htar_log_fn="log.htar_xvf.${narcv_formatted}" - grep -n ${extrn_mdl_fp} ${htar_log_fn} > /dev/null 2>&1 && { \ - num_occurs=$((num_occurs+1)); \ - } - done - - if [ ${num_occurs} -eq 0 ]; then - print_err_msg_exit "\ -The current external model file (extrn_mdl_fp) does not appear in any of -the archive extraction log files: - extrn_mdl_fp = \"${extrn_mdl_fp}\" -Thus, it was not extracted, likely because it doesn't exist in any of the -archive files." - elif [ ${num_occurs} -gt 1 ]; then - print_err_msg_exit "\ -The current external model file (extrn_mdl_fp) appears more than once in -the archive extraction log files: - extrn_mdl_fp = \"${extrn_mdl_fp}\" -The number of times it occurs in the log files is: - num_occurs = ${num_occurs} -Thus, it was extracted from more than one archive file, with the last one -that was extracted overwriting all previous ones. This should normally -not happen." - fi + elif [ "${ics_or_lbcs}" = "LBCS" ]; then - done -# -#----------------------------------------------------------------------- -# -# If extrn_mdl_arcvrel_dir is not set to the current directory (i.e. it -# is not equal to "."), then the htar command will have created the -# subdirectory "./${extrn_mdl_arcvrel_dir}" under the current directory -# and placed the extracted files there. In that case, we move these -# extracted files back to the current directory and then remove the -# subdirectory created by htar. -# -#----------------------------------------------------------------------- -# - if [ "${extrn_mdl_arcvrel_dir}" != "." ]; then -# -# The code below works if extrn_mdl_arcvrel_dir starts with a "/" or a -# "./", which are the only case encountered thus far. The code may have -# to be modified to accomodate other cases. -# - if [ "${extrn_mdl_arcvrel_dir:0:1}" = "/" ] || \ - [ "${extrn_mdl_arcvrel_dir:0:2}" = "./" ]; then -# -# Strip the "/" or "./" from the beginning of extrn_mdl_arcvrel_dir to -# obtain the relative directory from which to move the extracted files -# to the current directory. Then move the files. -# - rel_dir=$( printf "%s" "${extrn_mdl_arcvrel_dir}" | \ - sed -r 's%^(\/|\.\/)([^/]*)(.*)%\2\3%' ) - mv_vrfy ${rel_dir}/* . -# -# Get the first subdirectory in rel_dir, i.e. the subdirectory before the -# first forward slash. This is the subdirectory that we want to remove -# since it no longer contains any files (only subdirectories). Then remove -# it. -# - subdir_to_remove=$( printf "%s" "${rel_dir}" | \ - sed -r 's%^([^/]*)(.*)%\1%' ) - rm_vrfy -rf ./${subdir_to_remove} -# -# If extrn_mdl_arcvrel_dir does not start with a "/" (and it is not -# equal to "."), then print out an error message and exit. -# - else + print_info_msg " +======================================================================== +Successfully copied or linked to external model files on disk needed for +generating lateral boundary conditions for the FV3 forecast!!! - print_err_msg_exit "\ -The archive-relative directory specified by extrn_mdl_arcvrel_dir [i.e. -the directory \"within\" the tar file(s) listed in extrn_mdl_arcv_fps] is -not the current directory (i.e. it is not \".\"), and it does not start -with a \"/\" or a \"./\": - extrn_mdl_arcvrel_dir = \"${extrn_mdl_arcvrel_dir}\" - extrn_mdl_arcv_fps = ${extrn_mdl_arcv_fps_str} -This script must be modified to account for this case." +Exiting script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" +========================================================================" fi - fi -# -#----------------------------------------------------------------------- -# -# Consider the case of the archive file to be fetched from HPSS being in -# zip format. -# -#----------------------------------------------------------------------- -# - elif [ "${extrn_mdl_arcv_fmt}" = "zip" ]; then -# -#----------------------------------------------------------------------- -# -# For archive files that are in "zip" format files, the array extrn_mdl_arcv_fps -# containing the list of archive files should contain only one element, -# i.e. there should be only one archive file to consider. Check for this. -# If this ever changes (e.g. due to the way an external model that uses -# the "zip" format archives its output files on HPSS), the code below must -# be modified to loop over all archive files. -# -#----------------------------------------------------------------------- -# - if [ "${num_arcv_files}" -gt 1 ]; then - print_err_msg_exit "\ -Currently, this script is coded to handle only one archive file if the -archive file format is specified to be \"zip\", but the number of archive -files (num_arcv_files) passed to this script is greater than 1: - extrn_mdl_arcv_fmt = \"${extrn_mdl_arcv_fmt}\" - num_arcv_files = ${num_arcv_files} -Please modify the script to handle more than one \"zip\" archive file. -Note that code already exists in this script that can handle multiple -archive files if the archive file format is specified to be \"tar\", so -that can be used as a guide for the \"zip\" case." - else - arcv_fn="${extrn_mdl_arcv_fns[0]}" - arcv_fp="${extrn_mdl_arcv_fps[0]}" - fi -# -#----------------------------------------------------------------------- -# -# Fetch the zip archive file from HPSS. -# -#----------------------------------------------------------------------- -# - hsi_log_fn="log.hsi_get" - hsi get "${arcv_fp}" >& ${hsi_log_fn} || \ - print_err_msg_exit "\ -hsi file get operation (\"hsi get ...\") failed. Check the log file -hsi_log_fn in the staging directory (extrn_mdl_staging_dir) for details: - extrn_mdl_staging_dir = \"${extrn_mdl_staging_dir}\" - hsi_log_fn = \"${hsi_log_fn}\"" -# -#----------------------------------------------------------------------- -# -# List the contents of the zip archive file and save the result in a log -# file. -# -#----------------------------------------------------------------------- -# - unzip_log_fn="log.unzip_lv" - unzip -l -v ${arcv_fn} >& ${unzip_log_fn} || \ - print_err_msg_exit "\ -unzip operation to list the contents of the zip archive file arcv_fn in -the staging directory (extrn_mdl_staging_dir) failed. Check the log -file unzip_log_fn in that directory for details: - arcv_fn = \"${arcv_fn}\" - extrn_mdl_staging_dir = \"${extrn_mdl_staging_dir}\" - unzip_log_fn = \"${unzip_log_fn}\"" -# -#----------------------------------------------------------------------- -# -# Check that the log file from the unzip command above contains the name -# of each external model file. If any are missing, then the corresponding -# files are not in the zip file and thus cannot be extracted. In that -# case, print out a message and exit the script because initial condition -# and surface field files for the FV3-LAM cannot be generated without all -# the external model files. -# -#----------------------------------------------------------------------- -# - for fp in "${extrn_mdl_fps_in_arcv[@]}"; do - grep -n "${fp}" "${unzip_log_fn}" > /dev/null 2>&1 || \ - print_err_msg_exit "\ -External model file fp does not exist in the zip archive file arcv_fn in -the staging directory (extrn_mdl_staging_dir). Check the log file -unzip_log_fn in that directory for the contents of the zip archive: - extrn_mdl_staging_dir = \"${extrn_mdl_staging_dir}\" - arcv_fn = \"${arcv_fn}\" - fp = \"$fp\" - unzip_log_fn = \"${unzip_log_fn}\"" - done -# -#----------------------------------------------------------------------- -# -# Extract the external model files from the zip file on HPSS. Note that -# the -o flag to unzip is needed to overwrite existing files. Otherwise, -# unzip will wait for user input as to whether the existing files should -# be overwritten. -# -#----------------------------------------------------------------------- -# - unzip_log_fn="log.unzip" - unzip -o "${arcv_fn}" ${extrn_mdl_fps_in_arcv[@]} >& ${unzip_log_fn} || \ - print_err_msg_exit "\ -unzip file extract operation (\"unzip -o ...\") failed. Check the log -file unzip_log_fn in the staging directory (extrn_mdl_staging_dir) for -details: - extrn_mdl_staging_dir = \"${extrn_mdl_staging_dir}\" - unzip_log_fn = \"${unzip_log_fn}\"" -# -# NOTE: -# If extrn_mdl_arcvrel_dir is not empty, the unzip command above will -# create a subdirectory under extrn_mdl_staging_dir and place the external -# model files there. We have not encountered this for the RAP and HRRR -# models, but it may happen for other models in the future. In that case, -# extra code must be included here to move the external model files from -# the subdirectory up to extrn_mdl_staging_dir and then the subdirectory -# (analogous to what is done above for the case of extrn_mdl_arcv_fmt set -# to "tar". -# - - fi -# -#----------------------------------------------------------------------- -# -# Print message indicating successful completion of script. -# -#----------------------------------------------------------------------- -# - if [ "${ics_or_lbcs}" = "ICS" ]; then + elif [ "${data_src}" = "noaa_hpss" ]; then + + if [ "${ics_or_lbcs}" = "ICS" ]; then - print_info_msg " + print_info_msg " ======================================================================== -External model files needed for generating initial condition and surface -fields for the FV3-LAM successfully fetched from HPSS!!! +External model files needed for generating initial condition and surface +fields for the FV3-LAM successfully fetched from NOAA HPSS!!! Exiting script: \"${scrfunc_fn}\" In directory: \"${scrfunc_dir}\" ========================================================================" - elif [ "${ics_or_lbcs}" = "LBCS" ]; then + elif [ "${ics_or_lbcs}" = "LBCS" ]; then - print_info_msg " + print_info_msg " ======================================================================== External model files needed for generating lateral boundary conditions -on the halo of the FV3-LAM's regional grid successfully fetched from -HPSS!!! +on the halo of the FV3-LAM's regional grid successfully fetched from +NOAA HPSS!!! Exiting script: \"${scrfunc_fn}\" In directory: \"${scrfunc_dir}\" ========================================================================" - fi + fi -elif [ "${data_src}" = "online" ]; then - print_info_msg " -======================================================================== -getting data from online nomads data sources -========================================================================" + elif [ "${data_src}" = "nomads" ]; then -# -#----------------------------------------------------------------------- -# -# Set extrn_mdl_fps to the full paths within the archive files of the -# external model output files. -# -#----------------------------------------------------------------------- -# - prefix=${extrn_mdl_arcvrel_dir:+${extrn_mdl_arcvrel_dir}/} - extrn_mdl_fps=( "${extrn_mdl_fns_on_disk[@]/#/$prefix}" ) + print_info_msg " +======================================================================== +External model files needed for generating initial and/or lateral boundary +conditions successfully fetched from NOMADS!!! - extrn_mdl_fps_str="( "$( printf "\"%s\" " "${extrn_mdl_fps[@]}" )")" +Exiting script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" +========================================================================" - print_info_msg " -Getting external model files from nomads: - extrn_mdl_fps= ${extrn_mdl_fps_str}" + fi - num_files_to_extract="${#extrn_mdl_fps[@]}" - wget_LOG_FN="log.wget.txt" - for (( nfile=0; nfile<${num_files_to_extract}; nfile++ )); do - cp ../../../${extrn_mdl_fps[$nfile]} . || \ - print_err_msg_exit "\ - onlie file ${extrn_mdl_fps[$nfile]} not found." - done + break + fi -fi +done # #----------------------------------------------------------------------- # -# Create a variable definitions file (a shell script) and save in it the -# values of several external-model-associated variables generated in this -# script that will be needed by downstream workflow tasks. +# Call a function to create a variable definitions file (in bash script +# syntax) that contains several external-model-associated variables +# generated in this script that will be needed by downstream workflow +# tasks. # #----------------------------------------------------------------------- # if [ "${ics_or_lbcs}" = "ICS" ]; then - extrn_mdl_var_defns_fn="${EXTRN_MDL_ICS_VAR_DEFNS_FN}" + var_defns_fn="${EXTRN_MDL_ICS_VAR_DEFNS_FN}" elif [ "${ics_or_lbcs}" = "LBCS" ]; then - extrn_mdl_var_defns_fn="${EXTRN_MDL_LBCS_VAR_DEFNS_FN}" -fi -extrn_mdl_var_defns_fp="${extrn_mdl_staging_dir}/${extrn_mdl_var_defns_fn}" -check_for_preexist_dir_file "${extrn_mdl_var_defns_fp}" "delete" - -if [ "${data_src}" = "disk" ]; then - extrn_mdl_fns_str="( "$( printf "\"%s\" " "${extrn_mdl_fns_on_disk[@]}" )")" -elif [ "${data_src}" = "HPSS" ]; then - extrn_mdl_fns_str="( "$( printf "\"%s\" " "${extrn_mdl_fns_in_arcv[@]}" )")" -elif [ "${data_src}" = "online" ]; then - extrn_mdl_fns_str="( "$( printf "\"%s\" " "${extrn_mdl_fns_on_disk[@]}" )")" + var_defns_fn="${EXTRN_MDL_LBCS_VAR_DEFNS_FN}" fi +var_defns_fp="${staging_dir}/${var_defns_fn}" -settings="\ -DATA_SRC=\"${data_src}\" -EXTRN_MDL_CDATE=\"${extrn_mdl_cdate}\" -EXTRN_MDL_STAGING_DIR=\"${extrn_mdl_staging_dir}\" -EXTRN_MDL_FNS=${extrn_mdl_fns_str}" -# -# If the external model files obtained above were for generating LBCS (as -# opposed to ICs), then add to the external model variable definitions -# file the array variable EXTRN_MDL_LBC_SPEC_FHRS containing the forecast -# hours at which the lateral boundary conditions are specified. -# -if [ "${ics_or_lbcs}" = "LBCS" ]; then - extrn_mdl_lbc_spec_fhrs_str="( "$( printf "\"%s\" " "${extrn_mdl_lbc_spec_fhrs[@]}" )")" - settings="$settings -EXTRN_MDL_LBC_SPEC_FHRS=${extrn_mdl_lbc_spec_fhrs_str}" -fi - -{ cat << EOM >> ${extrn_mdl_var_defns_fp} -$settings -EOM -} || print_err_msg_exit "\ -Heredoc (cat) command to create a variable definitions file associated -with the external model from which to generate ${ics_or_lbcs} returned with a -nonzero status. The full path to this variable definitions file is: - extrn_mdl_var_defns_fp = \"${extrn_mdl_var_defns_fp}\"" +create_extrn_mdl_var_defns_file \ + extrn_mdl_var_defns_fp="${var_defns_fp}" \ + ics_or_lbcs="${ics_or_lbcs}" \ + extrn_mdl_cdate="$cdate" \ + extrn_mdl_staging_dir="${staging_dir}" \ + extrn_mdl_fns="${fns_str}" \ + extrn_mdl_lbc_spec_fhrs="${lbc_spec_fhrs_str}" # #----------------------------------------------------------------------- # -# Restore the shell options saved at the beginning of this script/function. +# Restore the shell options saved at the beginning of this script or +# function. # #----------------------------------------------------------------------- # { restore_shell_opts; } > /dev/null 2>&1 - diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index 41b00ac63..fa243409b 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -56,8 +56,8 @@ This is the ex-script for the task that generates initial condition #----------------------------------------------------------------------- # valid_args=( \ -"ics_dir" \ -) + "ics_dir" \ + ) process_args valid_args "$@" # #----------------------------------------------------------------------- diff --git a/scripts/exregional_make_lbcs.sh b/scripts/exregional_make_lbcs.sh index 3865f092a..334df94dc 100755 --- a/scripts/exregional_make_lbcs.sh +++ b/scripts/exregional_make_lbcs.sh @@ -56,8 +56,8 @@ hour zero). #----------------------------------------------------------------------- # valid_args=( \ -"lbcs_dir" \ -) + "lbcs_dir" \ + ) process_args valid_args "$@" # #----------------------------------------------------------------------- @@ -540,7 +540,7 @@ located in the following directory: # the forecast hour of the FV3-LAM (which is not necessarily the same as # that of the external model since their start times may be offset). # - fcst_hhh_FV3LAM=$( printf "%03d" "${LBC_SPEC_FCST_HRS[$i]}" ) + fcst_hhh_FV3LAM=$( printf "%03d" "${LBC_SPEC_FHRS[$i]}" ) mv_vrfy gfs.bndy.nc ${lbcs_dir}/gfs_bndy.tile7.${fcst_hhh_FV3LAM}.nc done diff --git a/scripts/exregional_run_post.sh b/scripts/exregional_run_post.sh index 87d454813..c0eefcb48 100755 --- a/scripts/exregional_run_post.sh +++ b/scripts/exregional_run_post.sh @@ -245,11 +245,13 @@ phy_file="${run_dir}/phyf${fhr}${mnts_secs_str}.nc" # output. # post_time=$( date --utc --date "${yyyymmdd} ${hh} UTC + ${fhr} hours + ${fmn} minutes" "+%Y%m%d%H%M" ) -post_yyyy=${post_time:0:4} -post_mm=${post_time:4:2} -post_dd=${post_time:6:2} -post_hh=${post_time:8:2} -post_mn=${post_time:10:2} +parse_cdate \ + cdate="${post_time}" \ + outvarname_yyyy="post_yyyy" \ + outvarname_mm="post_mm" \ + outvarname_dd="post_dd" \ + outvarname_hh="post_hh" \ + outvarname_mn="post_mn" # # Create the input text file to the post-processor executable. # diff --git a/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh b/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh index 257cd8912..c881dc4c0 100755 --- a/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh +++ b/tests/WE2E/get_WE2Etest_names_subdirs_descs.sh @@ -430,6 +430,7 @@ function get_WE2Etest_names_subdirs_descs() { # category_subdirs=( \ "." \ + "get_extrn_mdl_files" \ "grids_extrn_mdls_suites_community" \ "grids_extrn_mdls_suites_nco" \ "release_SRW_v1" \ @@ -759,7 +760,7 @@ This is probably because it is a directory. Please correct and rerun." test_names=("${prim_test_names[@]}") test_subdirs=("${prim_test_subdirs[@]}") if [ "${num_alt_tests}" -gt "0" ]; then - test_names+=("${alt_test_subdirs[@]:-}") + test_names+=("${alt_test_names[@]:-}") test_subdirs+=("${alt_test_subdirs[@]:-}") fi # diff --git a/tests/WE2E/get_expts_status.sh b/tests/WE2E/get_expts_status.sh index 5c05acafb..5ef28ccd3 100755 --- a/tests/WE2E/get_expts_status.sh +++ b/tests/WE2E/get_expts_status.sh @@ -43,14 +43,14 @@ scrfunc_dir=$( dirname "${scrfunc_fp}" ) # #----------------------------------------------------------------------- # -# The current script should be located in the "tests" subdirectory of the -# workflow's top-level directory, which we denote by homerrfs. Thus, -# homerrfs is the directory one level above the directory in which the -# current script is located. Set homerrfs accordingly. +# The current script should be located in the "tests/WE2E" subdirectory +# of the workflow's top-level directory, which we denote by homerrfs. +# Thus, homerrfs is the directory two levels above the directory in which +# the current script is located. Set homerrfs accordingly. # #----------------------------------------------------------------------- # -homerrfs=${scrfunc_dir%/*} +homerrfs=${scrfunc_dir%/*/*} # #----------------------------------------------------------------------- # diff --git a/tests/WE2E/run_WE2E_tests.sh b/tests/WE2E/run_WE2E_tests.sh index 143be8084..d41b216f2 100755 --- a/tests/WE2E/run_WE2E_tests.sh +++ b/tests/WE2E/run_WE2E_tests.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash # #----------------------------------------------------------------------- @@ -15,8 +15,8 @@ # #----------------------------------------------------------------------- # -# Get the full path to the file in which this script or function is -# located (scrfunc_fp), the name of that file (scrfunc_fn), and the +# Get the full path to the file in which this script or function is +# located (scrfunc_fp), the name of that file (scrfunc_fn), and the # directory in which the file is located (scrfunc_dir). # #----------------------------------------------------------------------- @@ -27,10 +27,10 @@ scrfunc_dir=$( dirname "${scrfunc_fp}" ) # #----------------------------------------------------------------------- # -# Set the full path to the top-level directory of the regional_workflow -# repository. We denote this path by homerrfs. The current script +# Set the full path to the top-level directory of the regional_workflow +# repository. We denote this path by homerrfs. The current script # should be located in the "tests/WE2E" subdirectory under this directory. -# Thus, homerrfs is the directory two levels above the directory in which +# Thus, homerrfs is the directory two levels above the directory in which # the current script is located. # #----------------------------------------------------------------------- @@ -62,6 +62,7 @@ WE2Edir="$testsdir/WE2E" #----------------------------------------------------------------------- # . ${WE2Edir}/get_WE2Etest_names_subdirs_descs.sh +. ${WE2Edir}/set_user_specified_extrn_mdl_file_info.sh # #----------------------------------------------------------------------- # @@ -92,106 +93,106 @@ Usage: [stmp=\"...\"] \\ [ptmp=\"...\"] -The arguments in brackets are optional. The arguments are defined as +The arguments in brackets are optional. The arguments are defined as follows: tests_file: Name of file or relative or absolute path to file containing the list of -WE2E tests to run. This file must contain one test name per line, with +WE2E tests to run. This file must contain one test name per line, with no repeated names. This is a required argument. machine: Argument used to explicitly set the experiment variable MACHINE in the -experiment configuration files of all the WE2E tests the user wants to -run. (A description of MACHINE can be found in the default experiment +experiment configuration files of all the WE2E tests the user wants to +run. (A description of MACHINE can be found in the default experiment configuration file.) This is a required argument. account: Argument used to explicitly set the experiment variable ACCOUNT in the -experiment configuration files of all the WE2E tests the user wants to -run. (A description of ACCOUNT can be found in the default experiment +experiment configuration files of all the WE2E tests the user wants to +run. (A description of ACCOUNT can be found in the default experiment configuration file.) This is a required argument. expt_basedir: -Argument used to explicitly set the experiment variable EXPT_BASEDIR in +Argument used to explicitly set the experiment variable EXPT_BASEDIR in the experiment configuration files of all the WE2E tests the user wants -to run. (A description of EXPT_BASEDIR can be found in the default -experiment configuration file.) If expt_basedir is specified in the call -to this script, its value is used to set EXPT_BASEDIR in the configuration -files. If it is not specified, EXPT_BASEDIR is not set in the configuration -files, in which case the workflow generation script sets it to a default -value. Note that if expt_basedir is set to a relative path (e.g. -expt_basedir=\"testset1\" in the call to this script), then the workflow -generation script will set EXPT_BASEDIR for the experiment to a default -absolute path followed by \${expt_basedir}. This feature can be used to -group the WE2E tests into subdirectories for convenience, e.g. a set of -tests under subdirectory testset1, another set of tests under testset2, +to run. (A description of EXPT_BASEDIR can be found in the default +experiment configuration file.) If expt_basedir is specified in the call +to this script, its value is used to set EXPT_BASEDIR in the configuration +files. If it is not specified, EXPT_BASEDIR is not set in the configuration +files, in which case the workflow generation script sets it to a default +value. Note that if expt_basedir is set to a relative path (e.g. +expt_basedir=\"testset1\" in the call to this script), then the workflow +generation script will set EXPT_BASEDIR for the experiment to a default +absolute path followed by \${expt_basedir}. This feature can be used to +group the WE2E tests into subdirectories for convenience, e.g. a set of +tests under subdirectory testset1, another set of tests under testset2, etc. use_cron_to_relaunch: Argument used to explicitly set the experiment variable USE_CRON_TO_RELAUNCH -in the experiment configuration files of all the WE2E tests the user wants -to run. (A description of USE_CRON_TO_RELAUNCH can be found in the default -experiment configuration file.) If use_cron_to_relaunch is specified in -the call to this script, its value is used to set USE_CRON_TO_RELAUNCH -in the configuration files. If it is not specified, USE_CRON_TO_RELAUNCH -is set to \"TRUE\" in the configuration files, in which case cron jobs -are used to (re)launch the workflows for all tests (one cron job per test). -Thus, use_cron_to_relaunch needs to be specified only if the user wants +in the experiment configuration files of all the WE2E tests the user wants +to run. (A description of USE_CRON_TO_RELAUNCH can be found in the default +experiment configuration file.) If use_cron_to_relaunch is specified in +the call to this script, its value is used to set USE_CRON_TO_RELAUNCH +in the configuration files. If it is not specified, USE_CRON_TO_RELAUNCH +is set to \"TRUE\" in the configuration files, in which case cron jobs +are used to (re)launch the workflows for all tests (one cron job per test). +Thus, use_cron_to_relaunch needs to be specified only if the user wants to turn off use of cron jobs for all tests (by specifying use_cron_to_relaunch= -\"FALSE\" on the command line). Note that it is not possible to specify -a different value for USE_CRON_TO_RELAUNCH for each test via this argument; +\"FALSE\" on the command line). Note that it is not possible to specify +a different value for USE_CRON_TO_RELAUNCH for each test via this argument; either all tests use cron jobs or none do. cron_relaunch_intvl_mnts: Argument used to explicitly set the experiment variable CRON_RELAUNCH_INTVL_MNTS -in the experiment configuration files of all the WE2E tests the user wants -to run. (A description of CRON_RELAUNCH_INTVL_MNTS can be found in the -default experiment configuration file.) If cron_relaunch_intvl_mnts is -specified in the call to this script, its value is used to set -CRON_RELAUNCH_INTVL_MNTS in the configuration files. If it is not -specified, CRON_RELAUNCH_INTVL_MNTS is set to \"02\" (i.e. two minutes) -in the configuration files. Note that it is not possible to specify a -different value for CRON_RELAUNCH_INTVL_MNTS for each test via this -argument; all tests will use the same value for USE_CRON_TO_RELAUNCH -(either the value specified in the call to this script or the default -value of \"02\"). Note also that the value of this argument matters only -if the argument use_cron_to_relaunch is not explicitly set to \"FALSE\" +in the experiment configuration files of all the WE2E tests the user wants +to run. (A description of CRON_RELAUNCH_INTVL_MNTS can be found in the +default experiment configuration file.) If cron_relaunch_intvl_mnts is +specified in the call to this script, its value is used to set +CRON_RELAUNCH_INTVL_MNTS in the configuration files. If it is not +specified, CRON_RELAUNCH_INTVL_MNTS is set to \"02\" (i.e. two minutes) +in the configuration files. Note that it is not possible to specify a +different value for CRON_RELAUNCH_INTVL_MNTS for each test via this +argument; all tests will use the same value for USE_CRON_TO_RELAUNCH +(either the value specified in the call to this script or the default +value of \"02\"). Note also that the value of this argument matters only +if the argument use_cron_to_relaunch is not explicitly set to \"FALSE\" in the call to this script. verbose: Argument used to explicitly set the experiment variable VERBOSE in the experiment configuration files of all the WE2E tests the user wants to -run. (A description of VERBOSE can be found in the default experiment -configuration file.) If verbose is specified in the call to this script, -its value is used to set VERBOSE in the configuration files. If it is +run. (A description of VERBOSE can be found in the default experiment +configuration file.) If verbose is specified in the call to this script, +its value is used to set VERBOSE in the configuration files. If it is not specified, VERBOSE is set to \"TRUE\" in the configuration files. -Note that it is not possible to specify a different value for VERBOSE -for each test via this argument; either all tests will have VERBOSE set +Note that it is not possible to specify a different value for VERBOSE +for each test via this argument; either all tests will have VERBOSE set to \"TRUE\" or all will have it set to \"FALSE\". stmp: -Argument used to explicitly set the experiment variable STMP in the -experiment configuration files of all the WE2E tests the user wants to +Argument used to explicitly set the experiment variable STMP in the +experiment configuration files of all the WE2E tests the user wants to run that are in NCO mode, i.e. they have test configuration files that -set the experiment variable RUN_ENVIR to \"nco\". (A description of -STMP can be found in the default experiment configuration file.) If -stmp is specified in the call to this script, its value is used to set -STMP in the configuration files of all tests that will run in NCO mode. -If it is not specified, STMP is (effectively) set as follows in the +set the experiment variable RUN_ENVIR to \"nco\". (A description of +STMP can be found in the default experiment configuration file.) If +stmp is specified in the call to this script, its value is used to set +STMP in the configuration files of all tests that will run in NCO mode. +If it is not specified, STMP is (effectively) set as follows in the configuration files (of all NCO mode tests to be run): STMP=\$( readlink -f \"\$homerrfs/../../nco_dirs/stmp\" \) Here, homerrfs is the base directory in which the regional_workflow -repository is cloned. Note that it is not possible to specify a different +repository is cloned. Note that it is not possible to specify a different value for STMP for each test via this argument; all tests will use the -same value for STMP (either the value specified in the call to this -script or the default value above). Note also that the value of this +same value for STMP (either the value specified in the call to this +script or the default value above). Note also that the value of this argument is not used for any tests that are not in NCO mode. ptmp: -Same as the argument \"stmp\" described above but for setting the +Same as the argument \"stmp\" described above but for setting the experiment variable PTMP for all tests that will run in NCO mode. " # @@ -212,7 +213,7 @@ fi # # Specify the set of valid argument names for this script or function. # Then process the arguments provided to it on the command line (which -# should consist of a set of name-value pairs of the form arg1="value1", +# should consist of a set of name-value pairs of the form arg1="value1", # arg2="value2", etc). # #----------------------------------------------------------------------- @@ -254,7 +255,7 @@ to get help on how to use this script." if [ -z "${tests_file}" ]; then print_err_msg_exit "\ -The argument \"tests_file\" specifying the file containing a list of the +The argument \"tests_file\" specifying the file containing a list of the WE2E tests to run was not specified in the call to this script. \ ${help_msg}" fi @@ -268,102 +269,102 @@ fi if [ -z "${account}" ]; then print_err_msg_exit "\ -The argument \"account\" specifying the account under which to submit -jobs to the queue when running the WE2E tests was not specified in the +The argument \"account\" specifying the account under which to submit +jobs to the queue when running the WE2E tests was not specified in the call to this script. \ ${help_msg}" fi # #----------------------------------------------------------------------- # -# Get the full path to the file containing the list of user-specified +# Get the full path to the file containing the list of user-specified # WE2E tests to run. Then verify that the file exists. # #----------------------------------------------------------------------- # -user_spec_tests_fp=$( readlink -f "${tests_file}" ) +user_specified_tests_fp=$( readlink -f "${tests_file}" ) -if [ ! -f "${user_spec_tests_fp}" ]; then +if [ ! -f "${user_specified_tests_fp}" ]; then print_err_msg_exit "\ -The file containing the user-specified list of WE2E tests to run +The file containing the user-specified list of WE2E tests to run (tests_file) that is passed in as an argument to this script does not exit: tests_file = \"${tests_file}\" The full path to this script is: - user_spec_tests_fp = \"${user_spec_tests_fp}\" + user_specified_tests_fp = \"${user_specified_tests_fp}\" Please ensure that this file exists and rerun." fi # #----------------------------------------------------------------------- # -# Read in each line of the file specified by user_spec_tests_fp and add -# each non-empty line to the array user_spec_tests. Note that the read -# command will remove any leading and trailing whitespace from each line -# in user_spec_tests_fp [because it treats whatever character(s) the bash -# variable IFS (Internal Field Separator) is set to as word separators -# on each line, and IFS is by default set to a space, a tab, and a -# newline]. +# Read in each line of the file specified by user_specified_tests_fp and +# add each non-empty line to the array user_specified_tests. Note that +# the read command will remove any leading and trailing whitespace from +# each line in user_specified_tests_fp [because it treats whatever +# character(s) the bash variable IFS (Internal Field Separator) is set +# to as word separators on each line, and IFS is by default set to a +# space, a tab, and a newline]. # #----------------------------------------------------------------------- # -user_spec_tests=() +user_specified_tests=() while read -r line; do if [ ! -z "$line" ]; then - user_spec_tests+=("$line") + user_specified_tests+=("$line") fi -done < "${user_spec_tests_fp}" +done < "${user_specified_tests_fp}" # #----------------------------------------------------------------------- # -# Call a function to obtain the names of all available WE2E tests (i.e. -# not just the ones the user wants to run but all that are part of the -# WE2E testing system), the test IDs, and the category subdirectory in -# which each corresponding test configuration file is located. +# Call a function to obtain the names of all available WE2E tests (i.e. +# not just the ones the user wants to run but all that are part of the +# WE2E testing system), the test IDs, and the category subdirectory in +# which each corresponding test configuration file is located. # -# The array of test names (avail_WE2E_test_names) that the function -# called below returns contains both primary and alternate test names. -# A primary test name is a test name obtained from the name of a WE2E -# test configuration file that is an ordinary file, i.e. not a symlink, -# whereas an alternate name is one that is derived from the name of a +# The array of test names (avail_WE2E_test_names) that the function +# called below returns contains both primary and alternate test names. +# A primary test name is a test name obtained from the name of a WE2E +# test configuration file that is an ordinary file, i.e. not a symlink, +# whereas an alternate name is one that is derived from the name of a # symlink whose target is an ordinary test configuration file (but not -# another symlink). To be able to determine the set of test names that -# correspond to the same primary test, the function called also returns -# an array of test IDs (avail_WE2E_test_IDs) such that the IDs for a -# primary test name and all the alternate names that map to it (if any) -# are the same. These IDs will be used later below to ensure that the -# user does not list in the set of test names to run a given test more -# than once, e.g. by accidentally including in the list its primary name +# another symlink). To be able to determine the set of test names that +# correspond to the same primary test, the function called also returns +# an array of test IDs (avail_WE2E_test_IDs) such that the IDs for a +# primary test name and all the alternate names that map to it (if any) +# are the same. These IDs will be used later below to ensure that the +# user does not list in the set of test names to run a given test more +# than once, e.g. by accidentally including in the list its primary name # as well as one of its alternate names. # -# The category subdirectories in the array avail_WE2E_test_subdirs -# returned by the function called below are relative to the base +# The category subdirectories in the array avail_WE2E_test_subdirs +# returned by the function called below are relative to the base # directory under which the WE2E test configuration files are located. # This base directory is set by the function call below and is returned -# in the output variable avail_WE2E_test_configs_basedir. The i-th -# element of avail_WE2E_test_subdirs specifies the subdirectory under -# this base directory that contains the ordinary test configuration file -# (for a primary test name) or the symlink (for an alternate test name) -# corresponding to the i-th element (which may be a primary or alternate -# test name) in avail_WE2E_test_names. We refer to these subdirectories -# as "category" subdirectories because they are used for clarity to group +# in the output variable avail_WE2E_test_configs_basedir. The i-th +# element of avail_WE2E_test_subdirs specifies the subdirectory under +# this base directory that contains the ordinary test configuration file +# (for a primary test name) or the symlink (for an alternate test name) +# corresponding to the i-th element (which may be a primary or alternate +# test name) in avail_WE2E_test_names. We refer to these subdirectories +# as "category" subdirectories because they are used for clarity to group # the WE2E tests into types or categories. # -# Finally, note that the returned arrays +# Finally, note that the returned arrays # # avail_WE2E_test_names # avail_WE2E_test_ids # avail_WE2E_test_subdirs # -# are sorted in order of increasing test ID and such that for a given -# set of test names that share the same ID, the primary test name is +# are sorted in order of increasing test ID and such that for a given +# set of test names that share the same ID, the primary test name is # listed first followed by zero or more alternate names. As an example, # assume that there are three category subdirectories under the base -# directory specified by avail_WE2E_test_configs_basedir: dir1, dir2, -# and dir3. Also, assume that dir1 contains a test configuration file -# named config.primary_name.sh that is an ordinary file, and dir2 and dir3 +# directory specified by avail_WE2E_test_configs_basedir: dir1, dir2, +# and dir3. Also, assume that dir1 contains a test configuration file +# named config.primary_name.sh that is an ordinary file, and dir2 and dir3 # contain the following symlinks that point config.primary_name.sh: # -# ${avail_WE2E_test_configs_basedir}/dir2/config.alt_name_1.sh +# ${avail_WE2E_test_configs_basedir}/dir2/config.alt_name_1.sh # --> ${avail_WE2E_test_configs_basedir}/dir1/config.primary_name.sh # # ${avail_WE2E_test_configs_basedir}/dir3/config.alt_name_2.sh @@ -385,7 +386,7 @@ done < "${user_spec_tests_fp}" # avail_WE2E_test_subdirs[7]="dir1" # avail_WE2E_test_subdirs[8]="dir2" # avail_WE2E_test_subdirs[9]="dir3" -# +# #----------------------------------------------------------------------- # print_info_msg " @@ -405,19 +406,19 @@ num_avail_WE2E_tests="${#avail_WE2E_test_names[@]}" # #----------------------------------------------------------------------- # -# Loop through the elements of the array user_spec_tests and perform -# sanity checks. For each such element (i.e. for each WE2E test to run +# Loop through the elements of the array user_specified_tests and perform +# sanity checks. For each such element (i.e. for each WE2E test to run # specified by the user), make sure that: # # 1) The name of the test exists in the complete list of available WE2E # tests in avail_WE2E_test_names. -# 2) The test does not have an ID that is identical to a previously +# 2) The test does not have an ID that is identical to a previously # considered test in the user-specified list of tests to run (because # if so, it would be identical to that previously considered test, -# and it would be a waste of computational resources to run). +# and it would be a waste of computational resources to run). # -# If these requirements are met, add the test name to the list of tests -# to run in the array names_tests_to_run, and add the test's category +# If these requirements are met, add the test name to the list of tests +# to run in the array names_tests_to_run, and add the test's category # subdirectory to subdirs_tests_to_run. # #----------------------------------------------------------------------- @@ -431,34 +432,34 @@ subdirs_tests_to_run=() # # Initialize the array that will contain the remaining available WE2E # test names (including alternate names, if any) after finding a match -# for the i-th user-specified test name to run in user_spec_tests. +# for the i-th user-specified test name to run in user_specified_tests. # remaining_avail_WE2E_test_names=( "${avail_WE2E_test_names[@]}" ) -num_user_spec_tests="${#user_spec_tests[@]}" -for (( i=0; i<=$((num_user_spec_tests-1)); i++ )); do +num_user_specified_tests="${#user_specified_tests[@]}" +for (( i=0; i<${num_user_specified_tests}; i++ )); do - user_spec_test="${user_spec_tests[$i]}" + user_specified_test="${user_specified_tests[$i]}" print_info_msg "\ - Checking user-specified WE2E test: \"${user_spec_test}\"" + Checking user-specified WE2E test: \"${user_specified_test}\"" # -# For the current user-specified WE2E test (user_spec_test), loop through -# the list of all remaining available WE2E test names (i.e. the ones that -# haven't yet been matched to any of the user-specified test names to -# run) and make sure that: +# For the current user-specified WE2E test (user_specified_test), loop +# through the list of all remaining available WE2E test names (i.e. the +# ones that haven't yet been matched to any of the user-specified test +# names to run) and make sure that: # # 1) The name of the test exists (either as a primary test name or an # alternate test name) in the list of all available WE2E test names. -# 2) The test is not repeated in the user-specified list of tests to run, +# 2) The test is not repeated in the user-specified list of tests to run, # either under the same name or an alternate name (i.e. make sure that # it does not have the same test ID as a previously considered test). -# +# # Note that in the loop below, the index j gets set to only those elements -# of remaining_avail_WE2E_test_names that are defined [the syntax -# "${!some_array[@]}" expands to the indices of some_array that have -# defined elements]. We do this for efficiency; we unset elements of -# remaining_avail_WE2E_test_names that have already been matched with +# of remaining_avail_WE2E_test_names that are defined [the syntax +# "${!some_array[@]}" expands to the indices of some_array that have +# defined elements]. We do this for efficiency; we unset elements of +# remaining_avail_WE2E_test_names that have already been matched with # one of the user-specified test names to run because we know that any # remaining user-specified test names will not match those elements. # @@ -468,26 +469,27 @@ for (( i=0; i<=$((num_user_spec_tests-1)); i++ )); do test_name="${avail_WE2E_test_names[$j]}" test_id="${avail_WE2E_test_ids[$j]}" # -# Check whether the name of the current user-specified test (user_spec_test) -# matches any of the names in the full list of WE2E tests. If so: +# Check whether the name of the current user-specified test (in the +# variable user_specified_test) matches any of the names in the full +# list of WE2E tests. If so: # # 1) Set match_found to "TRUE". -# 2) Make sure that the test to run doesn't have a test ID that is -# identical to a previously considered test in the user-specified +# 2) Make sure that the test to run doesn't have a test ID that is +# identical to a previously considered test in the user-specified # list of tests to run (which would mean the two tests are identical). # If so, print out an error message and exit. -# - if [ "${test_name}" = "${user_spec_test}" ]; then +# + if [ "${test_name}" = "${user_specified_test}" ]; then match_found="TRUE" is_element_of "ids_tests_to_run" "${test_id}" && { - user_spec_tests_str=$(printf " \"%s\"\n" "${user_spec_tests[@]}") - user_spec_tests_str=$(printf "(\n%s\n )" "${user_spec_tests_str}") + user_specified_tests_str=$(printf " \"%s\"\n" "${user_specified_tests[@]}") + user_specified_tests_str=$(printf "(\n%s\n )" "${user_specified_tests_str}") all_names_for_test=() - for (( k=0; k<=$((num_avail_WE2E_tests-1)); k++ )); do + for (( k=0; k<${num_avail_WE2E_tests}; k++ )); do if [ "${avail_WE2E_test_ids[$k]}" = "${test_id}" ]; then all_names_for_test+=("${avail_WE2E_test_names[$k]}") fi @@ -495,29 +497,29 @@ for (( i=0; i<=$((num_user_spec_tests-1)); i++ )); do all_names_for_test_str=$(printf " \"%s\"\n" "${all_names_for_test[@]}") print_err_msg_exit "\ -The current user-specified test to run (user_spec_test) is already included -in the list of tests to run (user_spec_tests), either under the same name -or an alternate name: - user_spec_test = \"${user_spec_test}\" - user_spec_tests = ${user_spec_tests_str} +The current user-specified test to run (user_specified_test) is already +included in the list of tests to run (user_specified_tests), either under +the same name or an alternate name: + user_specified_test = \"${user_specified_test}\" + user_specified_tests = ${user_specified_tests_str} This test has the following primary and possible alternate names: ${all_names_for_test_str} -In order to avoid repeating the same WE2E test (and thus waste computational -resources), only one of these test names can be specified in the list of +In order to avoid repeating the same WE2E test (and thus waste computational +resources), only one of these test names can be specified in the list of tests to run. Please modify this list in the file - user_spec_tests_fp = \"${user_spec_tests_fp}\" + user_specified_tests_fp = \"${user_specified_tests_fp}\" accordingly and rerun." } # # Append the name of the current user-specified test, its ID, and its -# category subdirectory to the arrays that contain the sanity-checked -# versions of of these quantities. +# category subdirectory to the arrays that contain the sanity-checked +# versions of of these quantities. # - names_tests_to_run+=("${user_spec_test}") + names_tests_to_run+=("${user_specified_test}") ids_tests_to_run+=("${test_id}") subdirs_tests_to_run+=("${avail_WE2E_test_subdirs[$j]}") -# +# # Remove the j-th element of remaining_avail_WE2E_test_names so that for # the next user-specified test to run, we do not need to check whether # the j-th test is a match. Then break out of the loop over all remaining @@ -531,33 +533,33 @@ accordingly and rerun." done # # If match_found is still "FALSE" after exiting the loop above, then a -# match for the current user-specifed test to run was not found in the -# list of all WE2E tests -- neither as a primary test name nor as an +# match for the current user-specifed test to run was not found in the +# list of all WE2E tests -- neither as a primary test name nor as an # alternate name. In this case, print out an error message and exit. # if [ "${match_found}" = "FALSE" ]; then avail_WE2E_test_names_str=$( printf " \"%s\"\n" "${avail_WE2E_test_names[@]}" ) print_err_msg_exit "\ -The name current user-specified test to run (user_spec_test) does not -match any of the names (either primary or alternate) of the available +The name current user-specified test to run (user_specified_test) does +not match any of the names (either primary or alternate) of the available WE2E tests: - user_spec_test = \"${user_spec_test}\" -Valid values for user_spec_test consist of the names (primary or alternate) -of the available WE2E tests, which are: + user_specified_test = \"${user_specified_test}\" +Valid values for user_specified_test consist of the names (primary or +alternate) of the available WE2E tests, which are: ${avail_WE2E_test_names_str} Each name in the user-specified list of tests to run: - 1) Must match one of the (primary or alternate) test names of the + 1) Must match one of the (primary or alternate) test names of the availabe WE2E tests. 2) Must not be the primary or alternate name of a test that has its primary or one of its alternate names already included in the user- specified list of test to run, i.e. tests must not be repeated (in order not to waste computational resources). -Please modify the user-specified list of tests to run such that it adheres +Please modify the user-specified list of tests to run such that it adheres to the rules above and rerun. This list is in the file specified by the input variable tests_file: tests_file = \"${tests_file}\" The full path to this file is: - user_spec_tests_fp = \"${user_spec_tests_fp}\"" + user_specified_tests_fp = \"${user_specified_tests_fp}\"" fi done @@ -572,7 +574,7 @@ done num_tests_to_run="${#names_tests_to_run[@]}" tests_to_run_str=$( printf " \'%s\'\n" "${names_tests_to_run[@]}" ) print_info_msg " -After processing the user-specified list of WE2E tests to run, the number +After processing the user-specified list of WE2E tests to run, the number of tests to run (num_tests_to_run) is num_tests_to_run = ${num_tests_to_run} and the list of WE2E tests to run (one test per line) is @@ -586,7 +588,7 @@ ${tests_to_run_str}" # #----------------------------------------------------------------------- # -for (( i=0; i<=$((num_tests_to_run-1)); i++ )); do +for (( i=0; i<${num_tests_to_run}; i++ )); do test_name="${names_tests_to_run[$i]}" test_subdir="${subdirs_tests_to_run[$i]}" @@ -607,9 +609,9 @@ Please correct and rerun." # #----------------------------------------------------------------------- # -# Source the default experiment configuration file to set values of -# various experiment variables to their defaults. Then source the -# current WE2E test's configuration file to overwrite certain variables' +# Source the default experiment configuration file to set values of +# various experiment variables to their defaults. Then source the +# current WE2E test's configuration file to overwrite certain variables' # default values with test-specific ones. # #----------------------------------------------------------------------- @@ -619,11 +621,11 @@ Please correct and rerun." # #----------------------------------------------------------------------- # -# We will now construct a multiline variable consisting of the contents +# We will now construct a multiline variable consisting of the contents # that we want the experiment configuration file for this WE2E test to # have. Once this variable is constructed, we will write its contents # to the generic configuration file that the experiment generation script -# reads in (specified by the variable EXPT_CONFIG_FN in the default +# reads in (specified by the variable EXPT_CONFIG_FN in the default # configuration file config_defaults.sh sourced above) and then run that # script to generate an experiment for the current WE2E test. # @@ -633,24 +635,24 @@ Please correct and rerun." # and we append to it later below. # #----------------------------------------------------------------------- -# +# expt_config_str="" # #----------------------------------------------------------------------- # -# Set (and then write to expt_config_str) various experiment variables -# that depend on the input arguments to this script (as opposed to -# variable settings in the test configuration file specified by -# test_config_fp). Note that any values of these parameters specified -# in the default experiment configuration file (config_defaults.sh) -# or in the test configuraiton file (test_config_fp) that were sourced +# Set (and then write to expt_config_str) various experiment variables +# that depend on the input arguments to this script (as opposed to +# variable settings in the test configuration file specified by +# test_config_fp). Note that any values of these parameters specified +# in the default experiment configuration file (config_defaults.sh) +# or in the test configuraiton file (test_config_fp) that were sourced # above will be overwritten by the settings below. # -# Note also that if EXPT_BASEDIR ends up getting set to a null string, -# the experiment generation script that gets called further below will -# set it to a default path; if it gets set to a relative path, then the -# experiment generation script will set it to a path consisting of a -# default path with the relative path appended to it; and if it gets set +# Note also that if EXPT_BASEDIR ends up getting set to a null string, +# the experiment generation script that gets called further below will +# set it to a default path; if it gets set to a relative path, then the +# experiment generation script will set it to a path consisting of a +# default path with the relative path appended to it; and if it gets set # to an absolute path, then the workflow will leave it set to that path. # #----------------------------------------------------------------------- @@ -716,11 +718,11 @@ VERBOSE=\"${VERBOSE}\"" # #----------------------------------------------------------------------- # -# Modifications to the experiment configuration file if the WE2E test +# Modifications to the experiment configuration file if the WE2E test # uses pre-generated grid, orography, or surface climatology files. # -# If not running one or more of the grid, orography, and surface -# climatology file generation tasks, specify directories in which +# If not running one or more of the grid, orography, and surface +# climatology file generation tasks, specify directories in which # pregenerated versions of these files can be found. # #----------------------------------------------------------------------- @@ -786,7 +788,7 @@ SFC_CLIMO_DIR=\"${SFC_CLIMO_DIR}\"" # #----------------------------------------------------------------------- # -# Modifications to the experiment configuration file if running the WE2E +# Modifications to the experiment configuration file if running the WE2E # test in NCO mode. # #----------------------------------------------------------------------- @@ -848,28 +850,26 @@ envir=\"\${EXPT_SUBDIR}\"" FIXLAM_NCO_BASEDIR="/scratch2/BMC/det/FV3LAM_pregen" elif [ "$MACHINE" = "JET" ]; then FIXLAM_NCO_BASEDIR="/mnt/lfs4/BMC/wrfruc/FV3-LAM/pregen" - elif [ "$MACHINE" = "CHEYENNE" ]; then - FIXLAM_NCO_BASEDIR="/needs/to/be/specified" else print_err_msg_exit "\ -The base directory (FIXLAM_NCO_BASEDIR) in which the pregenerated grid, -orography, and surface climatology \"fixed\" files used in NCO mode are +The base directory (FIXLAM_NCO_BASEDIR) in which the pregenerated grid, +orography, and surface climatology \"fixed\" files used in NCO mode are located has not been specified for this machine (MACHINE): MACHINE= \"${MACHINE}\"" fi expt_config_str=${expt_config_str}" # -# The base directory in which the pregenerated grid, orography, and surface +# The base directory in which the pregenerated grid, orography, and surface # climatology \"fixed\" files used in NCO mode are located. In NCO mode, -# the workflow scripts will create symlinks (in the directory specified +# the workflow scripts will create symlinks (in the directory specified # by FIXLAM) to files in a subdirectory under FIXLAM_NCO_BASDEDIR, where -# the name of the subdirectory is the name of the predefined grid specified +# the name of the subdirectory is the name of the predefined grid specified # by PREDEF_GRID_NAME. # FIXLAM_NCO_BASEDIR=\"${FIXLAM_NCO_BASEDIR}\"" # -# Set COMINgfs if using the FV3GFS or the GSMGFS as the external model +# Set COMINgfs if using the FV3GFS or the GSMGFS as the external model # for ICs or LBCs. # if [ "${EXTRN_MDL_NAME_ICS}" = "FV3GFS" ] || \ @@ -924,99 +924,89 @@ PTMP=\"${PTMP}\"" # #----------------------------------------------------------------------- # -# Modifications to the experiment configuration file if the WE2E test -# uses user-staged external model files. +# Modifications to the experiment configuration file if the WE2E test +# may try to look for external model files on disk that are named using +# a user-specified naming convention (and are located in user-specified +# directories). # #----------------------------------------------------------------------- # - if [ "${USE_USER_STAGED_EXTRN_FILES}" = "TRUE" ]; then + if is_element_of "EXTRN_MDL_DATA_SOURCES" "disk"; then - if [ "$MACHINE" = "WCOSS_CRAY" ]; then - extrn_mdl_source_basedir="/gpfs/hps3/emc/meso/noscrub/UFS_SRW_App/extrn_mdl_files" - elif [ "$MACHINE" = "WCOSS_DELL_P3" ]; then - extrn_mdl_source_basedir="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/extrn_mdl_files" - elif [ "$MACHINE" = "HERA" ]; then - extrn_mdl_source_basedir="/scratch2/BMC/det/Gerard.Ketefian/UFS_CAM/staged_extrn_mdl_files" - elif [ "$MACHINE" = "JET" ]; then - extrn_mdl_source_basedir="/mnt/lfs1/BMC/fim/Gerard.Ketefian/UFS_CAM/staged_extrn_mdl_files" - elif [ "$MACHINE" = "CHEYENNE" ]; then - extrn_mdl_source_basedir="/glade/p/ral/jntp/UFS_SRW_app/staged_extrn_mdl_files" - elif [ "$MACHINE" = "ORION" ]; then - extrn_mdl_source_basedir="/work/noaa/gsd-fv3-dev/gsketefia/UFS/staged_extrn_mdl_files" - else - print_err_msg_exit "\ -The base directory (extrn_mdl_source_basedir) in which the user-staged -external model files should be located has not been specified for this -machine (MACHINE): - MACHINE= \"${MACHINE}\"" - fi - - EXTRN_MDL_SOURCE_BASEDIR_ICS="${extrn_mdl_source_basedir}/${EXTRN_MDL_NAME_ICS}" + if [ ! -z "${EXTRN_MDL_DIR_FILE_LAYOUT}" ]; then - if [ "${EXTRN_MDL_NAME_ICS}" = "FV3GFS" ] && [ "$MACHINE" = "HERA" ]; then - EXTRN_MDL_SOURCE_BASEDIR_ICS="${EXTRN_MDL_SOURCE_BASEDIR_ICS}/${FV3GFS_FILE_FMT_ICS}" - fi - - if [ "${EXTRN_MDL_NAME_ICS}" = "FV3GFS" ] || \ - [ "${EXTRN_MDL_NAME_ICS}" = "GSMGFS" ]; then - if [ "${FV3GFS_FILE_FMT_ICS}" = "nemsio" ]; then - EXTRN_MDL_FILES_ICS=( "gfs.atmanl.nemsio" "gfs.sfcanl.nemsio" ) - elif [ "${FV3GFS_FILE_FMT_ICS}" = "grib2" ]; then - EXTRN_MDL_FILES_ICS=( "gfs.pgrb2.0p25.f000" ) + extrn_mdl_basedir="" + if [ "$MACHINE" = "WCOSS_CRAY" ]; then + extrn_mdl_basedir="/gpfs/hps3/emc/meso/noscrub/UFS_SRW_App/extrn_mdl_files" + elif [ "$MACHINE" = "WCOSS_DELL_P3" ]; then + extrn_mdl_basedir="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/extrn_mdl_files" + elif [ "$MACHINE" = "HERA" ]; then + extrn_mdl_basedir="/scratch2/BMC/det/Gerard.Ketefian/UFS_CAM/staged_extrn_mdl_files" + elif [ "$MACHINE" = "JET" ]; then + extrn_mdl_basedir="/mnt/lfs1/BMC/gsd-fv3/Gerard.Ketefian/UFS_CAM/staged_extrn_mdl_files" + elif [ "$MACHINE" = "CHEYENNE" ]; then + extrn_mdl_basedir="/glade/p/ral/jntp/UFS_SRW_app/staged_extrn_mdl_files" + elif [ "$MACHINE" = "ORION" ]; then + extrn_mdl_basedir="/work/noaa/gsd-fv3-dev/gsketefia/UFS/staged_extrn_mdl_files" + else + print_err_msg_exit "\ +The base directory (extrn_mdl_basedir) in which the external model files +should be located has not been specified for this machine (MACHINE): + MACHINE= \"${MACHINE}\"" fi - elif [ "${EXTRN_MDL_NAME_ICS}" = "HRRR" ] || \ - [ "${EXTRN_MDL_NAME_ICS}" = "RAP" ]; then - EXTRN_MDL_FILES_ICS=( "${EXTRN_MDL_NAME_ICS,,}.out.for_f000" ) - elif [ "${EXTRN_MDL_NAME_ICS}" = "NAM" ]; then - EXTRN_MDL_FILES_ICS=( "${EXTRN_MDL_NAME_ICS,,}.out.for_f000" ) - fi - EXTRN_MDL_SOURCE_BASEDIR_LBCS="${extrn_mdl_source_basedir}/${EXTRN_MDL_NAME_LBCS}" + extrn_mdl_basedir_ics="${extrn_mdl_basedir}/${EXTRN_MDL_NAME_ICS}" + if [ "${EXTRN_MDL_NAME_ICS}" = "FV3GFS" ]; then + extrn_mdl_basedir_ics="${extrn_mdl_basedir_ics}/${FV3GFS_FILE_FMT_ICS}" + elif [ "${EXTRN_MDL_NAME_ICS}" = "HRRR" ] || \ + [ "${EXTRN_MDL_NAME_ICS}" = "RAP" ]; then + extrn_mdl_basedir_ics="${extrn_mdl_basedir_ics}/grib2" + fi - if [ "${EXTRN_MDL_NAME_LBCS}" = "FV3GFS" ] && [ "$MACHINE" = "HERA" ]; then - EXTRN_MDL_SOURCE_BASEDIR_LBCS="${EXTRN_MDL_SOURCE_BASEDIR_LBCS}/${FV3GFS_FILE_FMT_LBCS}" - fi + extrn_mdl_basedir_lbcs="${extrn_mdl_basedir}/${EXTRN_MDL_NAME_LBCS}" + if [ "${EXTRN_MDL_NAME_LBCS}" = "FV3GFS" ]; then + extrn_mdl_basedir_lbcs="${extrn_mdl_basedir_lbcs}/${FV3GFS_FILE_FMT_LBCS}" + elif [ "${EXTRN_MDL_NAME_LBCS}" = "HRRR" ] || \ + [ "${EXTRN_MDL_NAME_LBCS}" = "RAP" ]; then + extrn_mdl_basedir_lbcs="${extrn_mdl_basedir_lbcs}/grib2" + fi + expt_config_str=${expt_config_str}" # -# Make sure that the forecast length is evenly divisible by the interval -# between the times at which the lateral boundary conditions will be -# specified. +# Base directories in which to search for external mode files for +# generating ICs and LBCs. # - rem=$(( 10#${FCST_LEN_HRS} % 10#${LBC_SPEC_INTVL_HRS} )) - if [ "$rem" -ne "0" ]; then - print_err_msg_exit "\ -The forecast length (FCST_LEN_HRS) must be evenly divisible by the lateral -boundary conditions specification interval (LBC_SPEC_INTVL_HRS): - FCST_LEN_HRS = ${FCST_LEN_HRS} - LBC_SPEC_INTVL_HRS = ${LBC_SPEC_INTVL_HRS} - rem = FCST_LEN_HRS%%LBC_SPEC_INTVL_HRS = $rem" - fi - lbc_spec_times_hrs=( $( seq "${LBC_SPEC_INTVL_HRS}" "${LBC_SPEC_INTVL_HRS}" "${FCST_LEN_HRS}" ) ) - EXTRN_MDL_FILES_LBCS=( $( printf "%03d " "${lbc_spec_times_hrs[@]}" ) ) - if [ "${EXTRN_MDL_NAME_LBCS}" = "FV3GFS" ] || \ - [ "${EXTRN_MDL_NAME_LBCS}" = "GSMGFS" ]; then - if [ "${FV3GFS_FILE_FMT_LBCS}" = "nemsio" ]; then - EXTRN_MDL_FILES_LBCS=( "${EXTRN_MDL_FILES_LBCS[@]/#/gfs.atmf}" ) - EXTRN_MDL_FILES_LBCS=( "${EXTRN_MDL_FILES_LBCS[@]/%/.nemsio}" ) - elif [ "${FV3GFS_FILE_FMT_LBCS}" = "grib2" ]; then - EXTRN_MDL_FILES_LBCS=( "${EXTRN_MDL_FILES_LBCS[@]/#/gfs.pgrb2.0p25.f}" ) - fi - elif [ "${EXTRN_MDL_NAME_LBCS}" = "HRRR" ] || \ - [ "${EXTRN_MDL_NAME_LBCS}" = "RAP" ]; then - EXTRN_MDL_FILES_LBCS=( "${EXTRN_MDL_FILES_LBCS[@]/#/${EXTRN_MDL_NAME_LBCS,,}.out.for_f}" ) - elif [ "${EXTRN_MDL_NAME_LBCS}" = "NAM" ]; then - EXTRN_MDL_FILES_LBCS=( "${EXTRN_MDL_FILES_LBCS[@]/#/${EXTRN_MDL_NAME_LBCS,,}.out.for_f}" ) - fi +EXTRN_MDL_BASEDIRS_ICS=( $( printf "\"%s\" " "${extrn_mdl_basedir_ics}" )) +EXTRN_MDL_BASEDIRS_LBCS=( $( printf "\"%s\" " "${extrn_mdl_basedir_lbcs}" ))" +# +# If using a user-specified direcory structure and file naming convention, +# set the parameters needed for constructing the user-specified external +# model file names. +# + if [ "${EXTRN_MDL_DIR_FILE_LAYOUT}" = "user_spec" ]; then - expt_config_str=${expt_config_str}" + set_user_specified_extrn_mdl_file_info \ + extrn_mdl_name_ics="${EXTRN_MDL_NAME_ICS}" \ + extrn_mdl_name_lbcs="${EXTRN_MDL_NAME_LBCS}" \ + fv3gfs_file_fmt_ics="${FV3GFS_FILE_FMT_ICS}" \ + fv3gfs_file_fmt_lbcs="${FV3GFS_FILE_FMT_LBCS}" \ + outvarname_extrn_mdl_fns_ics="__extrn_mdl_fns_ics" \ + outvarname_extrn_mdl_fns_lbcs_prefix="__extrn_mdl_fns_lbcs_prefix" \ + outvarname_extrn_mdl_fns_lbcs_suffix="__extrn_mdl_fns_lbcs_suffix" + + expt_config_str=${expt_config_str}" # -# Locations and names of user-staged external model files for generating -# ICs and LBCs. +# Parameters needed to construct external model file names using a user- +# specified naming convention. These must be specified because +# EXTRN_MDL_DIR_FILE_LAYOUT has been set above to \"user_spec\". # -EXTRN_MDL_SOURCE_BASEDIR_ICS=\"${EXTRN_MDL_SOURCE_BASEDIR_ICS}\" -EXTRN_MDL_FILES_ICS=( $( printf "\"%s\" " "${EXTRN_MDL_FILES_ICS[@]}" )) -EXTRN_MDL_SOURCE_BASEDIR_LBCS=\"${EXTRN_MDL_SOURCE_BASEDIR_LBCS}\" -EXTRN_MDL_FILES_LBCS=( $( printf "\"%s\" " "${EXTRN_MDL_FILES_LBCS[@]}" ))" +EXTRN_MDL_FNS_ICS=( $( printf "\"%s\" " "${__extrn_mdl_fns_ics[@]}" )) +EXTRN_MDL_FNS_LBCS_PREFIX=\"${__extrn_mdl_fns_lbcs_prefix}\" +EXTRN_MDL_FNS_LBCS_SUFFIX=\"${__extrn_mdl_fns_lbcs_suffix}\"" + + fi + + fi fi # @@ -1031,50 +1021,52 @@ EXTRN_MDL_FILES_LBCS=( $( printf "\"%s\" " "${EXTRN_MDL_FILES_LBCS[@]}" ))" if [ "$MACHINE" = "WCOSS_CRAY" ]; then met_install_dir="/gpfs/hps/nco/ops/nwprod/met.v9.1.3" + met_bin_exec="exec" metplus_path="/gpfs/hps/nco/ops/nwprod/metplus.v3.1.1/METplus-3.1.1" ccpa_obs_dir="/gpfs/hps3/emc/meso/noscrub/UFS_SRW_App/obs_data/ccpa/proc" mrms_obs_dir="/gpfs/hps3/emc/meso/noscrub/UFS_SRW_App/obs_data/mrms/proc" ndas_obs_dir="/gpfs/hps3/emc/meso/noscrub/UFS_SRW_App/obs_data/ndas/proc" - met_bin_exec="exec" elif [ "$MACHINE" = "WCOSS_DELL_P3" ]; then met_install_dir="/gpfs/dell2/emc/verification/noscrub/emc.metplus/met/10.0.0" + met_bin_exec="exec" metplus_path="/gpfs/dell2/emc/verification/noscrub/emc.metplus/METplus/METplus-4.0.0" ccpa_obs_dir="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/obs_data/ccpa/proc" mrms_obs_dir="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/obs_data/mrms/proc" ndas_obs_dir="/gpfs/dell2/emc/modeling/noscrub/UFS_SRW_App/obs_data/ndas/proc" - met_bin_exec="exec" elif [ "$MACHINE" = "HERA" ]; then met_install_dir="/contrib/met/10.0.0" + met_bin_exec="bin" metplus_path="/contrib/METplus/METplus-4.0.0" ccpa_obs_dir="/scratch2/BMC/det/UFS_SRW_app/v1p0/obs_data/ccpa/proc" mrms_obs_dir="/scratch2/BMC/det/UFS_SRW_app/v1p0/obs_data/mrms/proc" ndas_obs_dir="/scratch2/BMC/det/UFS_SRW_app/v1p0/obs_data/ndas/proc" - met_bin_exec="bin" else print_err_msg_exit "\ -The MET and MET+ paths (MET_INSTALL_DIR and MET_INSTALL_DIR) or the observation directories -(CCPA_OBS_DIR, MRMS_OBS_DIR, NDAS_OBS_DIR) have not been specified for this machine (MACHINE): +The MET path and executable (MET_INSTALL_DIR and MET_BIN_EXEC), the MET+ +path (METPLUS_PATH) and/or the observations directories (CCPA_OBS_DIR, +MRMS_OBS_DIR, and NDAS_OBS_DIR) have not been specified for this machine +(MACHINE): MACHINE= \"${MACHINE}\"" fi expt_config_str=${expt_config_str}" # -# MET and MET+ paths. +# MET path and executable, MET+ path, and various observations paths. # -METPLUS_PATH=\"${metplus_path}\" MET_INSTALL_DIR=\"${met_install_dir}\" +MET_BIN_EXEC=\"${met_bin_exec}\" +METPLUS_PATH=\"${metplus_path}\" CCPA_OBS_DIR=\"${ccpa_obs_dir}\" MRMS_OBS_DIR=\"${mrms_obs_dir}\" -NDAS_OBS_DIR=\"${ndas_obs_dir}\" -MET_BIN_EXEC=\"${met_bin_exec}\"" +NDAS_OBS_DIR=\"${ndas_obs_dir}\"" fi # #----------------------------------------------------------------------- # # On some machines (e.g. cheyenne), some tasks often require multiple -# tries before they succeed. To make it more convenient to run the WE2E -# tests on these machines without manual intervention, change the number +# tries before they succeed. To make it more convenient to run the WE2E +# tests on these machines without manual intervention, change the number # of attempts for such tasks on those machines to be more than one. # #----------------------------------------------------------------------- @@ -1114,8 +1106,8 @@ MAXTRIES_RUN_POST=\"${MAXTRIES_RUN_POST}\"" # #----------------------------------------------------------------------- # -# Set the full path to the configuration file that the experiment -# generation script reads in. Then write the contents of expt_config_str +# Set the full path to the configuration file that the experiment +# generation script reads in. Then write the contents of expt_config_str # to that file. # #----------------------------------------------------------------------- @@ -1125,106 +1117,8 @@ MAXTRIES_RUN_POST=\"${MAXTRIES_RUN_POST}\"" # #----------------------------------------------------------------------- # -# The following are changes that need to be made directly to the -# experiment configuration file created above (as opposed to the -# experiment configuration string expt_config_str) because they involve -# resetting of values that have already been set in the experiment -# configuration file. -# -# If EXTRN_MDL_SYSBASEDIR_ICS has been specified in the current WE2E -# test's base configuration file, it must be set to one of the following: -# -# 1) The string "set_to_non_default_location_in_testing_script" in order -# to allow this script to set it to a valid location depending on the -# machine and external model (for ICs). -# -# 2) To an existing directory. If it is set to a directory, then this -# script ensures that the directory exists (via the check below). -# -#----------------------------------------------------------------------- -# - if [ ! -z "${EXTRN_MDL_SYSBASEDIR_ICS}" ]; then - - if [ "${EXTRN_MDL_SYSBASEDIR_ICS}" = "set_to_non_default_location_in_testing_script" ]; then - - EXTRN_MDL_SYSBASEDIR_ICS="" - if [ "$MACHINE" = "HERA" ]; then - if [ "${EXTRN_MDL_NAME_ICS}" = "FV3GFS" ]; then - EXTRN_MDL_SYSBASEDIR_ICS="/scratch2/BMC/det/UFS_SRW_app/dummy_FV3GFS_sys_dir" - fi - fi - - if [ -z "${EXTRN_MDL_SYSBASEDIR_ICS}" ]; then - print_err_msg_exit "\ -A non-default location for EXTRN_MDL_SYSBASEDIR_ICS for testing purposes -has not been specified for this machine (MACHINE) and external model for -initial conditions (EXTRN_MDL_NAME_ICS) combination: - MACHINE= \"${MACHINE}\" - EXTRN_MDL_NAME_ICS = \"${EXTRN_MDL_NAME_ICS}\"" - fi - - else - - if [ ! -d "${EXTRN_MDL_SYSBASEDIR_ICS}" ]; then - print_err_msg_exit "\ -The non-default location specified by EXTRN_MDL_SYSBASEDIR_ICS does not -exist or is not a directory: - EXTRN_MDL_NAME_ICS = \"${EXTRN_MDL_NAME_ICS}\"" - fi - - fi - - set_bash_param "${expt_config_fp}" \ - "EXTRN_MDL_SYSBASEDIR_ICS" "${EXTRN_MDL_SYSBASEDIR_ICS}" - - fi -# -#----------------------------------------------------------------------- -# -# Same as above but for EXTRN_MDL_SYSBASEDIR_LBCS. -# -#----------------------------------------------------------------------- -# - if [ ! -z "${EXTRN_MDL_SYSBASEDIR_LBCS}" ]; then - - if [ "${EXTRN_MDL_SYSBASEDIR_LBCS}" = "set_to_non_default_location_in_testing_script" ]; then - - EXTRN_MDL_SYSBASEDIR_LBCS="" - if [ "$MACHINE" = "HERA" ]; then - if [ "${EXTRN_MDL_NAME_LBCS}" = "FV3GFS" ]; then - EXTRN_MDL_SYSBASEDIR_LBCS="/scratch2/BMC/det/UFS_SRW_app/dummy_FV3GFS_sys_dir" - fi - fi - - if [ -z "${EXTRN_MDL_SYSBASEDIR_LBCS}" ]; then - print_err_msg_exit "\ -A non-default location for EXTRN_MDL_SYSBASEDIR_LBCS for testing purposes -has not been specified for this machine (MACHINE) and external model for -initial conditions (EXTRN_MDL_NAME_LBCS) combination: - MACHINE= \"${MACHINE}\" - EXTRN_MDL_NAME_LBCS = \"${EXTRN_MDL_NAME_LBCS}\"" - fi - - else - - if [ ! -d "${EXTRN_MDL_SYSBASEDIR_LBCS}" ]; then - print_err_msg_exit "\ -The non-default location specified by EXTRN_MDL_SYSBASEDIR_LBCS does not -exist or is not a directory: - EXTRN_MDL_NAME_LBCS = \"${EXTRN_MDL_NAME_LBCS}\"" - fi - - fi - - set_bash_param "${expt_config_fp}" \ - "EXTRN_MDL_SYSBASEDIR_LBCS" "${EXTRN_MDL_SYSBASEDIR_LBCS}" - - fi -# -#----------------------------------------------------------------------- -# -# Call the experiment generation script to generate an experiment -# directory and a rocoto workflow XML for the current WE2E test to run. +# Call the experiment generation script to generate an experiment +# directory and a rocoto workflow XML for the current WE2E test to run. # #----------------------------------------------------------------------- # @@ -1237,7 +1131,7 @@ done # #----------------------------------------------------------------------- # -# Restore the shell options saved at the beginning of this script or +# Restore the shell options saved at the beginning of this script or # function. # #----------------------------------------------------------------------- diff --git a/tests/WE2E/set_user_specified_extrn_mdl_file_info.sh b/tests/WE2E/set_user_specified_extrn_mdl_file_info.sh new file mode 100644 index 000000000..7945be97a --- /dev/null +++ b/tests/WE2E/set_user_specified_extrn_mdl_file_info.sh @@ -0,0 +1,161 @@ +# +#----------------------------------------------------------------------- +# +# This file defines a function that, for a given external model for ICs +# (initial conditions) and a given external model for LBCs (lateral +# boundary conditions), sets the parameters needed to construct the +# external model file names. The parameters that are set are: +# +# extrn_mdl_fns_ics: +# This is an array that specifies the set of files needed for generating +# ICs and surface fields on the native FV3LAM grid. This has to be an +# array because some external models store all the necessary information +# in a single file while other models store this information in multiple +# files, e.g. atmospheric fields in one file and surface fields in another. +# +# extrn_mdl_fns_lbcs_prefix, extrn_mdl_fns_lbcs_suffix: +# These two variables are the prefix and suffix that must be added to +# each 3-digit output forecast hour to construct the file names in which +# the information needed for generating LBCs are stored. For example, +# the file containing the information for forecast hour "003" would be +# given (in bash syntax) by +# +# ${extrn_mdl_fns_lbcs_prefix}003${extrn_mdl_fns_lbcs_suffix} +# +# The actual construction of the file names is not performed by this +# function. This function simply sets the prefix and suffix for the +# specified external model for LBCs. +# +#----------------------------------------------------------------------- +# +function set_user_specified_extrn_mdl_file_info() { +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# + { save_shell_opts; set -u +x; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Specify the set of valid argument names for this script/function. Then +# process the arguments provided to this script/function (which should +# consist of a set of name-value pairs of the form arg1="value1", etc). +# +#----------------------------------------------------------------------- +# + local valid_args=( \ + "extrn_mdl_name_ics" \ + "extrn_mdl_name_lbcs" \ + "fv3gfs_file_fmt_ics" \ + "fv3gfs_file_fmt_lbcs" \ + "outvarname_extrn_mdl_fns_ics" \ + "outvarname_extrn_mdl_fns_lbcs_prefix" \ + "outvarname_extrn_mdl_fns_lbcs_suffix" \ + ) + process_args valid_args "$@" +# +#----------------------------------------------------------------------- +# +# For debugging purposes, print out values of arguments passed to this +# script. Note that these will be printed out only if VERBOSE is set to +# TRUE. +# +#----------------------------------------------------------------------- +# + print_input_args valid_args +# +#----------------------------------------------------------------------- +# +# Declare local variables. +# +#----------------------------------------------------------------------- +# + local extrn_mdl_fns_lbcs_prefix \ + extrn_mdl_fns_lbcs_suffix \ + extrn_mdl_fns_ics \ + extrn_mdl_fns_ics_str +# +#----------------------------------------------------------------------- +# +# Set extrn_mdl_fns_ics according to the external model used for ICs. +# +#----------------------------------------------------------------------- +# + if [ "${extrn_mdl_name_ics}" = "FV3GFS" ] || \ + [ "${extrn_mdl_name_ics}" = "GSMGFS" ]; then + if [ "${fv3gfs_file_fmt_ics}" = "nemsio" ]; then + extrn_mdl_fns_ics=( "gfs.atmanl.nemsio" "gfs.sfcanl.nemsio" ) + elif [ "${fv3gfs_file_fmt_ics}" = "grib2" ]; then + extrn_mdl_fns_ics=( "gfs.pgrb2.0p25.f000" ) + elif [ "${fv3gfs_file_fmt_ics}" = "netcdf" ]; then + extrn_mdl_fns_ics=( "gfs.atmanl.nc" "gfs.sfcanl.nc" ) + fi + elif [ "${extrn_mdl_name_ics}" = "HRRR" ] || \ + [ "${extrn_mdl_name_ics}" = "RAP" ]; then + extrn_mdl_fns_ics=( "${extrn_mdl_name_ics,,}.out.f000" ) + elif [ "${extrn_mdl_name_ics}" = "NAM" ]; then + extrn_mdl_fns_ics=( "${extrn_mdl_name_ics,,}.out.f000" ) + fi +# +#----------------------------------------------------------------------- +# +# Set extrn_mdl_fns_lbcs_prefix and extrn_mdl_fns_lbcs_suffix according +# to the external model used for LBCs. +# +#----------------------------------------------------------------------- +# + if [ "${extrn_mdl_name_lbcs}" = "FV3GFS" ] || \ + [ "${extrn_mdl_name_lbcs}" = "GSMGFS" ]; then + if [ "${fv3gfs_file_fmt_lbcs}" = "nemsio" ]; then + extrn_mdl_fns_lbcs_prefix="gfs.atmf" + extrn_mdl_fns_lbcs_suffix=".nemsio" + elif [ "${fv3gfs_file_fmt_lbcs}" = "grib2" ]; then + extrn_mdl_fns_lbcs_prefix="gfs.pgrb2.0p25.f" + extrn_mdl_fns_lbcs_suffix="" + elif [ "${fv3gfs_file_fmt_lbcs}" = "netcdf" ]; then + extrn_mdl_fns_lbcs_prefix="gfs.atmf" + extrn_mdl_fns_lbcs_suffix=".nc" + fi + elif [ "${extrn_mdl_name_lbcs}" = "HRRR" ] || \ + [ "${extrn_mdl_name_lbcs}" = "RAP" ]; then + extrn_mdl_fns_lbcs_prefix="${extrn_mdl_name_lbcs,,}.out.f" + extrn_mdl_fns_lbcs_suffix="" + elif [ "${extrn_mdl_name_lbcs}" = "NAM" ]; then + extrn_mdl_fns_lbcs_prefix="" + extrn_mdl_fns_lbcs_suffix="" + fi +# +#----------------------------------------------------------------------- +# +# Use the eval function to set the output variables. Note that each of +# these is set only if the corresponding input variable specifying the +# name to use for the output variable is not empty. +# +#----------------------------------------------------------------------- +# + if [ ! -z "${outvarname_extrn_mdl_fns_ics}" ]; then + extrn_mdl_fns_ics_str="( "$( printf "\"%s\" " "${extrn_mdl_fns_ics[@]}" )")" + eval ${outvarname_extrn_mdl_fns_ics}="${extrn_mdl_fns_ics_str}" + fi + + if [ ! -z "${outvarname_extrn_mdl_fns_lbcs_prefix}" ]; then + eval ${outvarname_extrn_mdl_fns_lbcs_prefix}="${extrn_mdl_fns_lbcs_prefix}" + fi + + if [ ! -z "${outvarname_extrn_mdl_fns_lbcs_suffix}" ]; then + eval ${outvarname_extrn_mdl_fns_lbcs_suffix}="${extrn_mdl_fns_lbcs_suffix}" + fi +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/function. +# +#----------------------------------------------------------------------- +# + { restore_shell_opts; } > /dev/null 2>&1 + +} diff --git a/tests/WE2E/test_configs/get_extrn_mdl_files/config.from_disk__usedefaultdir_no__fdlayout_extrn_mdl__ics_FV3GFS_grib2__lbcs_FV3GFS_grib2__2021060306.sh b/tests/WE2E/test_configs/get_extrn_mdl_files/config.from_disk__usedefaultdir_no__fdlayout_extrn_mdl__ics_FV3GFS_grib2__lbcs_FV3GFS_grib2__2021060306.sh new file mode 100644 index 000000000..e98901af1 --- /dev/null +++ b/tests/WE2E/test_configs/get_extrn_mdl_files/config.from_disk__usedefaultdir_no__fdlayout_extrn_mdl__ics_FV3GFS_grib2__lbcs_FV3GFS_grib2__2021060306.sh @@ -0,0 +1,45 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test checks the capability of the workflow to retrieve from a +# user-specified location on disk (as opposed to a default location) +# grib2-formatted output files generated by the FV3GFS external model +# (from which ICs and LBCs will be derived) using the external model's +# default output directory structure and file naming convention. +# +# Note that in this test, we explicitly set EXTRN_MDL_DIR_FILE_LAYOUT +# to its default value of "native_to_extrn_mdl". This is in principle +# redundant, but the WE2E testing script checks whether or not this +# parameter is set in this file to determine whether or not it needs to +# set the base directories for ICs and LBCs to user-specified values. +# If so, it sets the base directories (EXTRN_MDL_BASEDIRS_ICS and +# EXTRN_MDL_BASEDIRS_LBCS) explicitly to user-specified values in the +# experiment configuration file that it generates for this test. If +# not, it leaves the base directories unset (so that they get set to the +# default locations during the experiment generation step). Thus, since +# we are specifying EXTRN_MDL_DIR_FILE_LAYOUT here, EXTRN_MDL_BASEDIRS_ICS +# and EXTRN_MDL_BASEDIRS_LBCS will also be explicitly set in the experiment +# configuration file. +# + +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +CCPP_PHYS_SUITE="FV3_GFS_v15p2" + +EXTRN_MDL_NAME_ICS="FV3GFS" +FV3GFS_FILE_FMT_ICS="grib2" +EXTRN_MDL_NAME_LBCS="FV3GFS" +FV3GFS_FILE_FMT_LBCS="grib2" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="native_to_extrn_mdl" + +DATE_FIRST_CYCL="20210603" +DATE_LAST_CYCL="20210603" +CYCL_HRS=( "06" ) + +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/WE2E/test_configs/get_extrn_mdl_files/config.from_disk__usedefaultdir_no__fdlayout_user_spec__ics_FV3GFS_grib2__lbcs_FV3GFS_grib2__2021060306.sh b/tests/WE2E/test_configs/get_extrn_mdl_files/config.from_disk__usedefaultdir_no__fdlayout_user_spec__ics_FV3GFS_grib2__lbcs_FV3GFS_grib2__2021060306.sh new file mode 100644 index 000000000..77a5103df --- /dev/null +++ b/tests/WE2E/test_configs/get_extrn_mdl_files/config.from_disk__usedefaultdir_no__fdlayout_user_spec__ics_FV3GFS_grib2__lbcs_FV3GFS_grib2__2021060306.sh @@ -0,0 +1,33 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test checks the capability of the workflow to retrieve from a +# user-specified location on disk (as opposed to a default location) +# grib2-formatted output files generated by the FV3GFS external model +# (from which ICs and LBCs will be derived) using a user-specified +# format for the external model's output directory structure and file +# naming convention (i.e. with EXTRN_MDL_DIR_FILE_LAYOUT set to +# "user_spec", as opposed to the default value of "extrn_mdl"). +# + +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +CCPP_PHYS_SUITE="FV3_GFS_v15p2" + +EXTRN_MDL_NAME_ICS="FV3GFS" +FV3GFS_FILE_FMT_ICS="grib2" +EXTRN_MDL_NAME_LBCS="FV3GFS" +FV3GFS_FILE_FMT_LBCS="grib2" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" + +DATE_FIRST_CYCL="20210603" +DATE_LAST_CYCL="20210603" +CYCL_HRS=( "06" ) + +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/WE2E/test_configs/get_extrn_mdl_files/config.from_disk__usedefaultdir_yes__fdlayout_extrn_mdl__ics_FV3GFS_grib2__lbcs_FV3GFS_grib2__1dayago_00Z.sh b/tests/WE2E/test_configs/get_extrn_mdl_files/config.from_disk__usedefaultdir_yes__fdlayout_extrn_mdl__ics_FV3GFS_grib2__lbcs_FV3GFS_grib2__1dayago_00Z.sh new file mode 100644 index 000000000..61f545b7e --- /dev/null +++ b/tests/WE2E/test_configs/get_extrn_mdl_files/config.from_disk__usedefaultdir_yes__fdlayout_extrn_mdl__ics_FV3GFS_grib2__lbcs_FV3GFS_grib2__1dayago_00Z.sh @@ -0,0 +1,36 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test checks the capability of the workflow to retrieve from the +# default location on disk grib2-formatted output files generated by the +# FV3GFS external model (from which ICs and LBCs will be derived) using +# the external model's default output directory structure and file naming +# convention (i.e. with EXTRN_MDL_DIR_FILE_LAYOUT left unspecified so +# that it takes on its default value of "native_to_extrn_mdl"). +# +# Note that the default locations on disk are system directories that +# usually contain only the most recent few days' files (e.g. the last +# two days). For this reason, the starting day of the forecast is set +# here to be "1 days ago" (i.e. yesterday). +# + +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +CCPP_PHYS_SUITE="FV3_GFS_v15p2" + +EXTRN_MDL_NAME_ICS="FV3GFS" +FV3GFS_FILE_FMT_ICS="grib2" +EXTRN_MDL_NAME_LBCS="FV3GFS" +FV3GFS_FILE_FMT_LBCS="grib2" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) + +DATE_FIRST_CYCL=$( date --utc --date="1 days ago" "+%Y%m%d" ) +DATE_LAST_CYCL="${DATE_FIRST_CYCL}" +CYCL_HRS=( "00" ) + +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/WE2E/test_configs/get_extrn_mdl_files/config.from_disk__usedefaultdir_yes__fdlayout_extrn_mdl__ics_FV3GFS_netcdf__lbcs_FV3GFS_netcdf__1dayago_00Z.sh b/tests/WE2E/test_configs/get_extrn_mdl_files/config.from_disk__usedefaultdir_yes__fdlayout_extrn_mdl__ics_FV3GFS_netcdf__lbcs_FV3GFS_netcdf__1dayago_00Z.sh new file mode 100644 index 000000000..87bd9a217 --- /dev/null +++ b/tests/WE2E/test_configs/get_extrn_mdl_files/config.from_disk__usedefaultdir_yes__fdlayout_extrn_mdl__ics_FV3GFS_netcdf__lbcs_FV3GFS_netcdf__1dayago_00Z.sh @@ -0,0 +1,36 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test checks the capability of the workflow to retrieve from the +# default location on disk netcdf-formatted output files generated by the +# FV3GFS external model (from which ICs and LBCs will be derived) using +# the external model's default output directory structure and file naming +# convention (i.e. with EXTRN_MDL_DIR_FILE_LAYOUT left unspecified so +# that it takes on its default value of "native_to_extrn_mdl"). +# +# Note that the default locations on disk are system directories that +# usually contain only the most recent few days' files (e.g. the last +# two days). For this reason, the starting day of the forecast is set +# here to be "1 days ago" (i.e. yesterday). +# + +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +CCPP_PHYS_SUITE="FV3_GFS_v15p2" + +EXTRN_MDL_NAME_ICS="FV3GFS" +FV3GFS_FILE_FMT_ICS="netcdf" +EXTRN_MDL_NAME_LBCS="FV3GFS" +FV3GFS_FILE_FMT_LBCS="netcdf" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) + +DATE_FIRST_CYCL=$( date --utc --date="1 days ago" "+%Y%m%d" ) +DATE_LAST_CYCL="${DATE_FIRST_CYCL}" +CYCL_HRS=( "00" ) + +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2019061200.sh b/tests/WE2E/test_configs/get_extrn_mdl_files/config.from_noaa_hpss__ics_FV3GFS_grib2__lbcs_FV3GFS_grib2__2019061200.sh similarity index 94% rename from tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2019061200.sh rename to tests/WE2E/test_configs/get_extrn_mdl_files/config.from_noaa_hpss__ics_FV3GFS_grib2__lbcs_FV3GFS_grib2__2019061200.sh index a835466b5..d98b9c2e9 100644 --- a/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2019061200.sh +++ b/tests/WE2E/test_configs/get_extrn_mdl_files/config.from_noaa_hpss__ics_FV3GFS_grib2__lbcs_FV3GFS_grib2__2019061200.sh @@ -15,9 +15,11 @@ PREDEF_GRID_NAME="RRFS_CONUS_25km" CCPP_PHYS_SUITE="FV3_GFS_v15p2" EXTRN_MDL_NAME_ICS="FV3GFS" -FV3GFS_FILE_FMT_LBCS="grib2" -EXTRN_MDL_NAME_LBCS="FV3GFS" FV3GFS_FILE_FMT_ICS="grib2" +EXTRN_MDL_NAME_LBCS="FV3GFS" +FV3GFS_FILE_FMT_LBCS="grib2" + +EXTRN_MDL_DATA_SOURCES=( "noaa_hpss" ) DATE_FIRST_CYCL="20190612" DATE_LAST_CYCL="20190612" diff --git a/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2019101818.sh b/tests/WE2E/test_configs/get_extrn_mdl_files/config.from_noaa_hpss__ics_FV3GFS_grib2__lbcs_FV3GFS_grib2__2019101818.sh similarity index 95% rename from tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2019101818.sh rename to tests/WE2E/test_configs/get_extrn_mdl_files/config.from_noaa_hpss__ics_FV3GFS_grib2__lbcs_FV3GFS_grib2__2019101818.sh index d7891e668..205b52027 100644 --- a/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2019101818.sh +++ b/tests/WE2E/test_configs/get_extrn_mdl_files/config.from_noaa_hpss__ics_FV3GFS_grib2__lbcs_FV3GFS_grib2__2019101818.sh @@ -17,9 +17,11 @@ PREDEF_GRID_NAME="RRFS_CONUS_25km" CCPP_PHYS_SUITE="FV3_GFS_v15p2" EXTRN_MDL_NAME_ICS="FV3GFS" -FV3GFS_FILE_FMT_LBCS="grib2" -EXTRN_MDL_NAME_LBCS="FV3GFS" FV3GFS_FILE_FMT_ICS="grib2" +EXTRN_MDL_NAME_LBCS="FV3GFS" +FV3GFS_FILE_FMT_LBCS="grib2" + +EXTRN_MDL_DATA_SOURCES=( "noaa_hpss" ) DATE_FIRST_CYCL="20191018" DATE_LAST_CYCL="20191018" diff --git a/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2020022518.sh b/tests/WE2E/test_configs/get_extrn_mdl_files/config.from_noaa_hpss__ics_FV3GFS_grib2__lbcs_FV3GFS_grib2__2020022518.sh similarity index 94% rename from tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2020022518.sh rename to tests/WE2E/test_configs/get_extrn_mdl_files/config.from_noaa_hpss__ics_FV3GFS_grib2__lbcs_FV3GFS_grib2__2020022518.sh index a4bea4ac1..38658b894 100644 --- a/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2020022518.sh +++ b/tests/WE2E/test_configs/get_extrn_mdl_files/config.from_noaa_hpss__ics_FV3GFS_grib2__lbcs_FV3GFS_grib2__2020022518.sh @@ -16,9 +16,11 @@ PREDEF_GRID_NAME="RRFS_CONUS_25km" CCPP_PHYS_SUITE="FV3_GFS_v15p2" EXTRN_MDL_NAME_ICS="FV3GFS" -FV3GFS_FILE_FMT_LBCS="grib2" -EXTRN_MDL_NAME_LBCS="FV3GFS" FV3GFS_FILE_FMT_ICS="grib2" +EXTRN_MDL_NAME_LBCS="FV3GFS" +FV3GFS_FILE_FMT_LBCS="grib2" + +EXTRN_MDL_DATA_SOURCES=( "noaa_hpss" ) DATE_FIRST_CYCL="20200225" DATE_LAST_CYCL="20200225" diff --git a/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2020022600.sh b/tests/WE2E/test_configs/get_extrn_mdl_files/config.from_noaa_hpss__ics_FV3GFS_grib2__lbcs_FV3GFS_grib2__2020022600.sh similarity index 94% rename from tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2020022600.sh rename to tests/WE2E/test_configs/get_extrn_mdl_files/config.from_noaa_hpss__ics_FV3GFS_grib2__lbcs_FV3GFS_grib2__2020022600.sh index 0f4863a0c..7521bca1e 100644 --- a/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2020022600.sh +++ b/tests/WE2E/test_configs/get_extrn_mdl_files/config.from_noaa_hpss__ics_FV3GFS_grib2__lbcs_FV3GFS_grib2__2020022600.sh @@ -16,9 +16,11 @@ PREDEF_GRID_NAME="RRFS_CONUS_25km" CCPP_PHYS_SUITE="FV3_GFS_v15p2" EXTRN_MDL_NAME_ICS="FV3GFS" -FV3GFS_FILE_FMT_LBCS="grib2" -EXTRN_MDL_NAME_LBCS="FV3GFS" FV3GFS_FILE_FMT_ICS="grib2" +EXTRN_MDL_NAME_LBCS="FV3GFS" +FV3GFS_FILE_FMT_LBCS="grib2" + +EXTRN_MDL_DATA_SOURCES=( "noaa_hpss" ) DATE_FIRST_CYCL="20200226" DATE_LAST_CYCL="20200226" diff --git a/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2021010100.sh b/tests/WE2E/test_configs/get_extrn_mdl_files/config.from_noaa_hpss__ics_FV3GFS_grib2__lbcs_FV3GFS_grib2__2021010100.sh similarity index 95% rename from tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2021010100.sh rename to tests/WE2E/test_configs/get_extrn_mdl_files/config.from_noaa_hpss__ics_FV3GFS_grib2__lbcs_FV3GFS_grib2__2021010100.sh index 70e0130eb..59aae73c8 100644 --- a/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_grib2_2021010100.sh +++ b/tests/WE2E/test_configs/get_extrn_mdl_files/config.from_noaa_hpss__ics_FV3GFS_grib2__lbcs_FV3GFS_grib2__2021010100.sh @@ -16,9 +16,11 @@ PREDEF_GRID_NAME="RRFS_CONUS_25km" CCPP_PHYS_SUITE="FV3_GFS_v15p2" EXTRN_MDL_NAME_ICS="FV3GFS" -FV3GFS_FILE_FMT_LBCS="grib2" -EXTRN_MDL_NAME_LBCS="FV3GFS" FV3GFS_FILE_FMT_ICS="grib2" +EXTRN_MDL_NAME_LBCS="FV3GFS" +FV3GFS_FILE_FMT_LBCS="grib2" + +EXTRN_MDL_DATA_SOURCES=( "noaa_hpss" ) DATE_FIRST_CYCL="20210101" DATE_LAST_CYCL="20210101" diff --git a/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2019061200.sh b/tests/WE2E/test_configs/get_extrn_mdl_files/config.from_noaa_hpss__ics_FV3GFS_nemsio__lbcs_FV3GFS_nemsio__2019061200.sh similarity index 94% rename from tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2019061200.sh rename to tests/WE2E/test_configs/get_extrn_mdl_files/config.from_noaa_hpss__ics_FV3GFS_nemsio__lbcs_FV3GFS_nemsio__2019061200.sh index 63cea8377..7e79ae3b4 100644 --- a/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2019061200.sh +++ b/tests/WE2E/test_configs/get_extrn_mdl_files/config.from_noaa_hpss__ics_FV3GFS_nemsio__lbcs_FV3GFS_nemsio__2019061200.sh @@ -15,9 +15,11 @@ PREDEF_GRID_NAME="RRFS_CONUS_25km" CCPP_PHYS_SUITE="FV3_GFS_v15p2" EXTRN_MDL_NAME_ICS="FV3GFS" -FV3GFS_FILE_FMT_LBCS="nemsio" -EXTRN_MDL_NAME_LBCS="FV3GFS" FV3GFS_FILE_FMT_ICS="nemsio" +EXTRN_MDL_NAME_LBCS="FV3GFS" +FV3GFS_FILE_FMT_LBCS="nemsio" + +EXTRN_MDL_DATA_SOURCES=( "noaa_hpss" ) DATE_FIRST_CYCL="20190612" DATE_LAST_CYCL="20190612" diff --git a/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2019101818.sh b/tests/WE2E/test_configs/get_extrn_mdl_files/config.from_noaa_hpss__ics_FV3GFS_nemsio__lbcs_FV3GFS_nemsio__2019101818.sh similarity index 95% rename from tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2019101818.sh rename to tests/WE2E/test_configs/get_extrn_mdl_files/config.from_noaa_hpss__ics_FV3GFS_nemsio__lbcs_FV3GFS_nemsio__2019101818.sh index cfac4ab62..55e7865af 100644 --- a/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2019101818.sh +++ b/tests/WE2E/test_configs/get_extrn_mdl_files/config.from_noaa_hpss__ics_FV3GFS_nemsio__lbcs_FV3GFS_nemsio__2019101818.sh @@ -17,9 +17,11 @@ PREDEF_GRID_NAME="RRFS_CONUS_25km" CCPP_PHYS_SUITE="FV3_GFS_v15p2" EXTRN_MDL_NAME_ICS="FV3GFS" -FV3GFS_FILE_FMT_LBCS="nemsio" -EXTRN_MDL_NAME_LBCS="FV3GFS" FV3GFS_FILE_FMT_ICS="nemsio" +EXTRN_MDL_NAME_LBCS="FV3GFS" +FV3GFS_FILE_FMT_LBCS="nemsio" + +EXTRN_MDL_DATA_SOURCES=( "noaa_hpss" ) DATE_FIRST_CYCL="20191018" DATE_LAST_CYCL="20191018" diff --git a/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2020022518.sh b/tests/WE2E/test_configs/get_extrn_mdl_files/config.from_noaa_hpss__ics_FV3GFS_nemsio__lbcs_FV3GFS_nemsio__2020022518.sh similarity index 94% rename from tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2020022518.sh rename to tests/WE2E/test_configs/get_extrn_mdl_files/config.from_noaa_hpss__ics_FV3GFS_nemsio__lbcs_FV3GFS_nemsio__2020022518.sh index e80507600..1b9344221 100644 --- a/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2020022518.sh +++ b/tests/WE2E/test_configs/get_extrn_mdl_files/config.from_noaa_hpss__ics_FV3GFS_nemsio__lbcs_FV3GFS_nemsio__2020022518.sh @@ -16,9 +16,11 @@ PREDEF_GRID_NAME="RRFS_CONUS_25km" CCPP_PHYS_SUITE="FV3_GFS_v15p2" EXTRN_MDL_NAME_ICS="FV3GFS" -FV3GFS_FILE_FMT_LBCS="nemsio" -EXTRN_MDL_NAME_LBCS="FV3GFS" FV3GFS_FILE_FMT_ICS="nemsio" +EXTRN_MDL_NAME_LBCS="FV3GFS" +FV3GFS_FILE_FMT_LBCS="nemsio" + +EXTRN_MDL_DATA_SOURCES=( "noaa_hpss" ) DATE_FIRST_CYCL="20200225" DATE_LAST_CYCL="20200225" diff --git a/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2020022600.sh b/tests/WE2E/test_configs/get_extrn_mdl_files/config.from_noaa_hpss__ics_FV3GFS_nemsio__lbcs_FV3GFS_nemsio__2020022600.sh similarity index 94% rename from tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2020022600.sh rename to tests/WE2E/test_configs/get_extrn_mdl_files/config.from_noaa_hpss__ics_FV3GFS_nemsio__lbcs_FV3GFS_nemsio__2020022600.sh index 873a13bef..6727283ed 100644 --- a/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2020022600.sh +++ b/tests/WE2E/test_configs/get_extrn_mdl_files/config.from_noaa_hpss__ics_FV3GFS_nemsio__lbcs_FV3GFS_nemsio__2020022600.sh @@ -16,9 +16,11 @@ PREDEF_GRID_NAME="RRFS_CONUS_25km" CCPP_PHYS_SUITE="FV3_GFS_v15p2" EXTRN_MDL_NAME_ICS="FV3GFS" -FV3GFS_FILE_FMT_LBCS="nemsio" -EXTRN_MDL_NAME_LBCS="FV3GFS" FV3GFS_FILE_FMT_ICS="nemsio" +EXTRN_MDL_NAME_LBCS="FV3GFS" +FV3GFS_FILE_FMT_LBCS="nemsio" + +EXTRN_MDL_DATA_SOURCES=( "noaa_hpss" ) DATE_FIRST_CYCL="20200226" DATE_LAST_CYCL="20200226" diff --git a/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2021010100.sh b/tests/WE2E/test_configs/get_extrn_mdl_files/config.from_noaa_hpss__ics_FV3GFS_nemsio__lbcs_FV3GFS_nemsio__2021010100.sh similarity index 95% rename from tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2021010100.sh rename to tests/WE2E/test_configs/get_extrn_mdl_files/config.from_noaa_hpss__ics_FV3GFS_nemsio__lbcs_FV3GFS_nemsio__2021010100.sh index 53421da0f..1a9de8159 100644 --- a/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2021010100.sh +++ b/tests/WE2E/test_configs/get_extrn_mdl_files/config.from_noaa_hpss__ics_FV3GFS_nemsio__lbcs_FV3GFS_nemsio__2021010100.sh @@ -16,9 +16,11 @@ PREDEF_GRID_NAME="RRFS_CONUS_25km" CCPP_PHYS_SUITE="FV3_GFS_v15p2" EXTRN_MDL_NAME_ICS="FV3GFS" -FV3GFS_FILE_FMT_LBCS="nemsio" -EXTRN_MDL_NAME_LBCS="FV3GFS" FV3GFS_FILE_FMT_ICS="nemsio" +EXTRN_MDL_NAME_LBCS="FV3GFS" +FV3GFS_FILE_FMT_LBCS="nemsio" + +EXTRN_MDL_DATA_SOURCES=( "noaa_hpss" ) DATE_FIRST_CYCL="20210101" DATE_LAST_CYCL="20210101" diff --git a/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_netcdf_2021062000.sh b/tests/WE2E/test_configs/get_extrn_mdl_files/config.from_noaa_hpss__ics_FV3GFS_netcdf__lbcs_FV3GFS_netcdf__2021062000.sh similarity index 93% rename from tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_netcdf_2021062000.sh rename to tests/WE2E/test_configs/get_extrn_mdl_files/config.from_noaa_hpss__ics_FV3GFS_netcdf__lbcs_FV3GFS_netcdf__2021062000.sh index 441690e81..be77fad43 100644 --- a/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_netcdf_2021062000.sh +++ b/tests/WE2E/test_configs/get_extrn_mdl_files/config.from_noaa_hpss__ics_FV3GFS_netcdf__lbcs_FV3GFS_netcdf__2021062000.sh @@ -14,9 +14,11 @@ PREDEF_GRID_NAME="RRFS_CONUS_25km" CCPP_PHYS_SUITE="FV3_GFS_v15p2" EXTRN_MDL_NAME_ICS="FV3GFS" -FV3GFS_FILE_FMT_LBCS="netcdf" -EXTRN_MDL_NAME_LBCS="FV3GFS" FV3GFS_FILE_FMT_ICS="netcdf" +EXTRN_MDL_NAME_LBCS="FV3GFS" +FV3GFS_FILE_FMT_LBCS="netcdf" + +EXTRN_MDL_DATA_SOURCES=( "noaa_hpss" ) DATE_FIRST_CYCL="20210620" DATE_LAST_CYCL="20210620" diff --git a/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_GSMGFS_lbcs_GSMGFS.sh b/tests/WE2E/test_configs/get_extrn_mdl_files/config.from_noaa_hpss__ics_GSMGFS__lbcs_GSMGFS__2019052000.sh similarity index 92% rename from tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_GSMGFS_lbcs_GSMGFS.sh rename to tests/WE2E/test_configs/get_extrn_mdl_files/config.from_noaa_hpss__ics_GSMGFS__lbcs_GSMGFS__2019052000.sh index 8c5c54193..50744492b 100644 --- a/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_GSMGFS_lbcs_GSMGFS.sh +++ b/tests/WE2E/test_configs/get_extrn_mdl_files/config.from_noaa_hpss__ics_GSMGFS__lbcs_GSMGFS__2019052000.sh @@ -16,6 +16,8 @@ CCPP_PHYS_SUITE="FV3_GFS_2017_gfdlmp" EXTRN_MDL_NAME_ICS="GSMGFS" EXTRN_MDL_NAME_LBCS="GSMGFS" +EXTRN_MDL_DATA_SOURCES=( "noaa_hpss" ) + DATE_FIRST_CYCL="20190520" DATE_LAST_CYCL="20190520" CYCL_HRS=( "00" ) diff --git a/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_HRRR_lbcs_RAP.sh b/tests/WE2E/test_configs/get_extrn_mdl_files/config.from_noaa_hpss__ics_HRRR__lbcs_RAP__2020080100.sh similarity index 94% rename from tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_HRRR_lbcs_RAP.sh rename to tests/WE2E/test_configs/get_extrn_mdl_files/config.from_noaa_hpss__ics_HRRR__lbcs_RAP__2020080100.sh index 84e93130f..34aa676c4 100644 --- a/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_HRRR_lbcs_RAP.sh +++ b/tests/WE2E/test_configs/get_extrn_mdl_files/config.from_noaa_hpss__ics_HRRR__lbcs_RAP__2020080100.sh @@ -18,6 +18,8 @@ CCPP_PHYS_SUITE="FV3_GSD_SAR" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" +EXTRN_MDL_DATA_SOURCES=( "noaa_hpss" ) + DATE_FIRST_CYCL="20200801" DATE_LAST_CYCL="20200801" CYCL_HRS=( "00" ) diff --git a/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_RAP_lbcs_RAP.sh b/tests/WE2E/test_configs/get_extrn_mdl_files/config.from_noaa_hpss__ics_RAP__lbcs_RAP__2019052000.sh similarity index 92% rename from tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_RAP_lbcs_RAP.sh rename to tests/WE2E/test_configs/get_extrn_mdl_files/config.from_noaa_hpss__ics_RAP__lbcs_RAP__2019052000.sh index 62fafdf86..38eaa7f02 100644 --- a/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_RAP_lbcs_RAP.sh +++ b/tests/WE2E/test_configs/get_extrn_mdl_files/config.from_noaa_hpss__ics_RAP__lbcs_RAP__2019052000.sh @@ -16,6 +16,8 @@ CCPP_PHYS_SUITE="FV3_GSD_SAR" EXTRN_MDL_NAME_ICS="RAP" EXTRN_MDL_NAME_LBCS="RAP" +EXTRN_MDL_DATA_SOURCES=( "noaa_hpss" ) + DATE_FIRST_CYCL="20190520" DATE_LAST_CYCL="20190520" CYCL_HRS=( "00" ) diff --git a/tests/WE2E/test_configs/get_extrn_mdl_files/config.from_nomads__ics_FV3GFS_grib2__lbcs_FV3GFS_grib2__last2days_00Z.sh b/tests/WE2E/test_configs/get_extrn_mdl_files/config.from_nomads__ics_FV3GFS_grib2__lbcs_FV3GFS_grib2__last2days_00Z.sh new file mode 100644 index 000000000..b7845536b --- /dev/null +++ b/tests/WE2E/test_configs/get_extrn_mdl_files/config.from_nomads__ics_FV3GFS_grib2__lbcs_FV3GFS_grib2__last2days_00Z.sh @@ -0,0 +1,35 @@ +# +# TEST PURPOSE/DESCRIPTION: +# ------------------------ +# +# This test checks the capability of the workflow to retrieve from NOMADS +# (NOAA Operational Model Archive and Distribution System) grib2-formatted +# output files generated by the FV3GFS external model (from which ICs and +# LBCs will be derived). +# +# Note that NOMADS hosts only the most recent few days' files. For this +# reason, the first cycle day of the experiment is set here to be "2 days +# ago" and the last cycle day is set to "1 days ago" (i.e. yesterday). +# We use two cycle days to test the capability of the scripts to fetch +# files from NOMADS for multiple cycles. +# + +RUN_ENVIR="community" +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +CCPP_PHYS_SUITE="FV3_GFS_v15p2" + +EXTRN_MDL_NAME_ICS="FV3GFS" +FV3GFS_FILE_FMT_ICS="grib2" +EXTRN_MDL_NAME_LBCS="FV3GFS" +FV3GFS_FILE_FMT_LBCS="grib2" + +EXTRN_MDL_DATA_SOURCES=( "nomads" ) + +DATE_FIRST_CYCL=$( date --utc --date="2 days ago" "+%Y%m%d" ) +DATE_LAST_CYCL=$( date --utc --date="1 days ago" "+%Y%m%d" ) +CYCL_HRS=( "00" ) + +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="3" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_CONUS_25km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_CONUS_25km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh index 34a61a6ad..e7794a5d3 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_CONUS_25km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_CONUS_25km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh @@ -16,7 +16,9 @@ CCPP_PHYS_SUITE="FV3_GFS_v16" EXTRN_MDL_NAME_ICS="FV3GFS" EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20190701" DATE_LAST_CYCL="20190701" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_CONUS_3km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_CONUS_3km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh index a081731e5..9c25b8f64 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_CONUS_3km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_CONUS_3km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh @@ -16,7 +16,9 @@ CCPP_PHYS_SUITE="FV3_GFS_v16" EXTRN_MDL_NAME_ICS="FV3GFS" EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20190701" DATE_LAST_CYCL="20190701" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_GSD_HRRR_AK_50km_ics_RAP_lbcs_RAP_suite_GSD_SAR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_GSD_HRRR_AK_50km_ics_RAP_lbcs_RAP_suite_GSD_SAR.sh index 18ad96810..8307a7ed8 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_GSD_HRRR_AK_50km_ics_RAP_lbcs_RAP_suite_GSD_SAR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_GSD_HRRR_AK_50km_ics_RAP_lbcs_RAP_suite_GSD_SAR.sh @@ -20,7 +20,9 @@ CCPP_PHYS_SUITE="FV3_GSD_SAR" EXTRN_MDL_NAME_ICS="RAP" EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20190520" DATE_LAST_CYCL="20190520" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_AK_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_AK_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh index 7a059e4ff..12bc29102 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_AK_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_AK_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh @@ -15,7 +15,9 @@ CCPP_PHYS_SUITE="FV3_GFS_v16" EXTRN_MDL_NAME_ICS="FV3GFS" EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20190701" DATE_LAST_CYCL="20190701" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_AK_13km_ics_RAP_lbcs_RAP_suite_GSD_SAR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_AK_13km_ics_RAP_lbcs_RAP_suite_GSD_SAR.sh index 44642ece9..1f9437eb8 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_AK_13km_ics_RAP_lbcs_RAP_suite_GSD_SAR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_AK_13km_ics_RAP_lbcs_RAP_suite_GSD_SAR.sh @@ -15,7 +15,9 @@ CCPP_PHYS_SUITE="FV3_GSD_SAR" EXTRN_MDL_NAME_ICS="RAP" EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20200801" DATE_LAST_CYCL="20200801" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_AK_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_AK_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh index d9d1151b0..c812163c1 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_AK_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_AK_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh @@ -15,7 +15,9 @@ CCPP_PHYS_SUITE="FV3_GFS_v16" EXTRN_MDL_NAME_ICS="FV3GFS" EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20190701" DATE_LAST_CYCL="20190701" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_AK_3km_ics_RAP_lbcs_RAP_suite_GSD_SAR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_AK_3km_ics_RAP_lbcs_RAP_suite_GSD_SAR.sh index e2da03a3b..c73dd0cdc 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_AK_3km_ics_RAP_lbcs_RAP_suite_GSD_SAR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_AK_3km_ics_RAP_lbcs_RAP_suite_GSD_SAR.sh @@ -15,7 +15,9 @@ CCPP_PHYS_SUITE="FV3_GSD_SAR" EXTRN_MDL_NAME_ICS="RAP" EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20200801" DATE_LAST_CYCL="20200801" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh index 7c6564a30..4ef996ed5 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh @@ -15,7 +15,9 @@ CCPP_PHYS_SUITE="FV3_GFS_v15p2" EXTRN_MDL_NAME_ICS="FV3GFS" EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20190701" DATE_LAST_CYCL="20190701" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh index de780b806..859cdc469 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh @@ -15,7 +15,9 @@ CCPP_PHYS_SUITE="FV3_GFS_v16" EXTRN_MDL_NAME_ICS="FV3GFS" EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20190701" DATE_LAST_CYCL="20190701" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh index 18a62f18d..4a5a10474 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh @@ -16,7 +16,9 @@ CCPP_PHYS_SUITE="FV3_GSD_SAR" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20200810" DATE_LAST_CYCL="20200810" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_HRRR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_HRRR.sh index bec80f24a..230704d3a 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_HRRR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_HRRR.sh @@ -16,7 +16,9 @@ CCPP_PHYS_SUITE="FV3_HRRR" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20200810" DATE_LAST_CYCL="20200810" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh index b3b420e00..ef396f2f3 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh @@ -16,7 +16,9 @@ CCPP_PHYS_SUITE="FV3_RRFS_v1alpha" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20200801" DATE_LAST_CYCL="20200801" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh index 3f39ab0f7..d21ac638e 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_13km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh @@ -16,7 +16,9 @@ CCPP_PHYS_SUITE="FV3_RRFS_v1beta" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20200801" DATE_LAST_CYCL="20200801" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp.sh index 2f7d6eccc..11c6fbc3c 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp.sh @@ -15,7 +15,9 @@ CCPP_PHYS_SUITE="FV3_GFS_2017_gfdlmp" EXTRN_MDL_NAME_ICS="FV3GFS" EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20190701" DATE_LAST_CYCL="20190701" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp_regional.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp_regional.sh index b21cfbe2c..a91f75827 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp_regional.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp_regional.sh @@ -15,7 +15,9 @@ CCPP_PHYS_SUITE="FV3_GFS_2017_gfdlmp_regional" EXTRN_MDL_NAME_ICS="FV3GFS" EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20190701" DATE_LAST_CYCL="20190701" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh index 5a36d5988..0353797e1 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh @@ -15,7 +15,9 @@ CCPP_PHYS_SUITE="FV3_GFS_v15p2" EXTRN_MDL_NAME_ICS="FV3GFS" EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20190701" DATE_LAST_CYCL="20190701" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh index 3c6841ddf..31bae0793 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh @@ -15,7 +15,9 @@ CCPP_PHYS_SUITE="FV3_GFS_v16" EXTRN_MDL_NAME_ICS="FV3GFS" EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20190701" DATE_LAST_CYCL="20190701" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GSD_SAR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GSD_SAR.sh index a313c8d1d..d592e615f 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GSD_SAR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GSD_SAR.sh @@ -15,7 +15,9 @@ CCPP_PHYS_SUITE="FV3_GSD_SAR" EXTRN_MDL_NAME_ICS="FV3GFS" EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20190701" DATE_LAST_CYCL="20190701" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.sh index 492a16336..9746f190e 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_HRRR.sh @@ -21,7 +21,9 @@ CCPP_PHYS_SUITE="FV3_HRRR" EXTRN_MDL_NAME_ICS="FV3GFS" FV3GFS_FILE_FMT_ICS="grib2" EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20200810" DATE_LAST_CYCL="20200810" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_CPT_v0.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_CPT_v0.sh index aae81d6eb..aea07c85f 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_CPT_v0.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_CPT_v0.sh @@ -15,7 +15,9 @@ CCPP_PHYS_SUITE="FV3_CPT_v0" EXTRN_MDL_NAME_ICS="GSMGFS" EXTRN_MDL_NAME_LBCS="GSMGFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20190520" DATE_LAST_CYCL="20190520" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_2017_gfdlmp.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_2017_gfdlmp.sh index 7e5b304ac..3fff1c7f7 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_2017_gfdlmp.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_2017_gfdlmp.sh @@ -15,7 +15,9 @@ CCPP_PHYS_SUITE="FV3_GFS_2017_gfdlmp" EXTRN_MDL_NAME_ICS="GSMGFS" EXTRN_MDL_NAME_LBCS="GSMGFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20190520" DATE_LAST_CYCL="20190520" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_v15p2.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_v15p2.sh index af3a13440..d6ab1668d 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_v15p2.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_v15p2.sh @@ -15,7 +15,9 @@ CCPP_PHYS_SUITE="FV3_GFS_v15p2" EXTRN_MDL_NAME_ICS="GSMGFS" EXTRN_MDL_NAME_LBCS="GSMGFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20190520" DATE_LAST_CYCL="20190520" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_v16.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_v16.sh index 616b0e744..8d72e5646 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_v16.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_GSMGFS_lbcs_GSMGFS_suite_GFS_v16.sh @@ -15,7 +15,9 @@ CCPP_PHYS_SUITE="FV3_GFS_v16" EXTRN_MDL_NAME_ICS="GSMGFS" EXTRN_MDL_NAME_LBCS="GSMGFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20190520" DATE_LAST_CYCL="20190520" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_GSD_SAR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_GSD_SAR.sh index cc239add0..35913ab5b 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_GSD_SAR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_GSD_SAR.sh @@ -15,7 +15,9 @@ CCPP_PHYS_SUITE="FV3_GSD_SAR" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="HRRR" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20200810" DATE_LAST_CYCL="20200810" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_HRRR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_HRRR.sh index 0e8ad1d04..9eeb6aa27 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_HRRR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_HRRR.sh @@ -15,7 +15,9 @@ CCPP_PHYS_SUITE="FV3_HRRR" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="HRRR" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20200810" DATE_LAST_CYCL="20200810" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta.sh index f236d2f35..dbe496b8e 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_HRRR_suite_RRFS_v1beta.sh @@ -15,7 +15,9 @@ CCPP_PHYS_SUITE="FV3_RRFS_v1beta" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="HRRR" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20200810" DATE_LAST_CYCL="20200810" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh index 1bd56f000..58664e9fa 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh @@ -16,7 +16,9 @@ CCPP_PHYS_SUITE="FV3_GSD_SAR" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20200810" DATE_LAST_CYCL="20200810" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_v0.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_v0.sh index 21ab6e7ab..7d0d7aec5 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_v0.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_v0.sh @@ -16,7 +16,9 @@ CCPP_PHYS_SUITE="FV3_GSD_v0" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20190520" DATE_LAST_CYCL="20190520" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh index 01471df07..6c5cabcb9 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh @@ -16,7 +16,9 @@ CCPP_PHYS_SUITE="FV3_HRRR" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20200810" DATE_LAST_CYCL="20200810" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh index 48f2f4c13..c76ad7dc7 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh @@ -16,7 +16,9 @@ CCPP_PHYS_SUITE="FV3_RRFS_v1alpha" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20200810" DATE_LAST_CYCL="20200810" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh index f5fabd981..5d764b222 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh @@ -16,7 +16,9 @@ CCPP_PHYS_SUITE="FV3_RRFS_v1beta" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20200810" DATE_LAST_CYCL="20200810" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GSD_SAR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GSD_SAR.sh index 8a7c1c102..27e50813b 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GSD_SAR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GSD_SAR.sh @@ -15,7 +15,9 @@ CCPP_PHYS_SUITE="FV3_GSD_SAR" EXTRN_MDL_NAME_ICS="NAM" EXTRN_MDL_NAME_LBCS="NAM" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20150602" DATE_LAST_CYCL="20150602" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_HRRR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_HRRR.sh index a91c10151..a0044f156 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_HRRR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_HRRR.sh @@ -15,7 +15,9 @@ CCPP_PHYS_SUITE="FV3_HRRR" EXTRN_MDL_NAME_ICS="NAM" EXTRN_MDL_NAME_LBCS="NAM" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20150602" DATE_LAST_CYCL="20150602" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_RRFS_v1beta.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_RRFS_v1beta.sh index a22466e5b..1b1bd4b31 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_RRFS_v1beta.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_RRFS_v1beta.sh @@ -15,7 +15,9 @@ CCPP_PHYS_SUITE="FV3_RRFS_v1beta" EXTRN_MDL_NAME_ICS="NAM" EXTRN_MDL_NAME_LBCS="NAM" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20150602" DATE_LAST_CYCL="20150602" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh index 528de40de..c4de29264 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh @@ -15,7 +15,9 @@ CCPP_PHYS_SUITE="FV3_GFS_v15p2" EXTRN_MDL_NAME_ICS="FV3GFS" EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20190701" DATE_LAST_CYCL="20190701" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh index cd2deecb8..1e744ba81 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh @@ -15,7 +15,9 @@ CCPP_PHYS_SUITE="FV3_GFS_v16" EXTRN_MDL_NAME_ICS="FV3GFS" EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20190701" DATE_LAST_CYCL="20190701" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh index 960f4b186..bded0330b 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh @@ -16,7 +16,9 @@ CCPP_PHYS_SUITE="FV3_GFS_v15p2" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20200801" DATE_LAST_CYCL="20200801" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh index 35301c466..14145bab0 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh @@ -16,7 +16,9 @@ CCPP_PHYS_SUITE="FV3_GSD_SAR" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20200810" DATE_LAST_CYCL="20200810" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh index 56b1ffb8b..3845b615f 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh @@ -16,7 +16,9 @@ CCPP_PHYS_SUITE="FV3_HRRR" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20200810" DATE_LAST_CYCL="20200810" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh index 5637d0b7a..964fd713c 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1alpha.sh @@ -16,7 +16,9 @@ CCPP_PHYS_SUITE="FV3_RRFS_v1alpha" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20200801" DATE_LAST_CYCL="20200801" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh index 07f66f9cc..1ea100507 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_RRFS_v1beta.sh @@ -16,7 +16,9 @@ CCPP_PHYS_SUITE="FV3_RRFS_v1beta" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20200801" DATE_LAST_CYCL="20200801" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GSD_v0.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GSD_v0.sh index 763945bf8..c37ee7fe5 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GSD_v0.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GSD_v0.sh @@ -15,7 +15,9 @@ CCPP_PHYS_SUITE="FV3_GSD_v0" EXTRN_MDL_NAME_ICS="FV3GFS" EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20190701" DATE_LAST_CYCL="20190701" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1alpha.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1alpha.sh index 9b4e0da93..8c7cf4699 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1alpha.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_NA_3km_ics_FV3GFS_lbcs_FV3GFS_suite_RRFS_v1alpha.sh @@ -19,7 +19,9 @@ CCPP_PHYS_SUITE="FV3_RRFS_v1alpha" EXTRN_MDL_NAME_ICS="FV3GFS" EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20190701" DATE_LAST_CYCL="20190701" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_SUBCONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_SUBCONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh index eac0a993a..d3c78c46a 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_SUBCONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_SUBCONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh @@ -15,7 +15,9 @@ CCPP_PHYS_SUITE="FV3_GFS_v16" EXTRN_MDL_NAME_ICS="FV3GFS" EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20190701" DATE_LAST_CYCL="20190701" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh index 3534d5df9..2d9f20339 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GFS_v15p2.sh @@ -16,7 +16,9 @@ CCPP_PHYS_SUITE="FV3_GFS_v15p2" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20200801" DATE_LAST_CYCL="20200801" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh index 4c2ff9f19..bd244914a 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh @@ -16,7 +16,9 @@ CCPP_PHYS_SUITE="FV3_GSD_SAR" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20200801" DATE_LAST_CYCL="20200801" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_CONUS_25km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_CONUS_25km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh index 3431f7cdc..129d83d45 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_CONUS_25km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_CONUS_25km_GFDLgrid_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16.sh @@ -16,7 +16,8 @@ CCPP_PHYS_SUITE="FV3_GFS_v16" EXTRN_MDL_NAME_ICS="FV3GFS" EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) DATE_FIRST_CYCL="20190901" DATE_LAST_CYCL="20190901" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh index 9bd42fa73..b8a59c045 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh @@ -15,7 +15,8 @@ CCPP_PHYS_SUITE="FV3_GSD_SAR" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) DATE_FIRST_CYCL="20200801" DATE_LAST_CYCL="20200801" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh index 300dcd666..3b5fc0563 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_25km_ics_HRRR_lbcs_RAP_suite_HRRR.sh @@ -15,7 +15,8 @@ CCPP_PHYS_SUITE="FV3_HRRR" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) DATE_FIRST_CYCL="20200208" DATE_LAST_CYCL="20200208" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp_regional.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp_regional.sh index d8eeef6c5..b4a5fd2ad 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp_regional.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_2017_gfdlmp_regional.sh @@ -15,7 +15,8 @@ CCPP_PHYS_SUITE="FV3_GFS_2017_gfdlmp_regional" EXTRN_MDL_NAME_ICS="FV3GFS" EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) DATE_FIRST_CYCL="20190901" DATE_LAST_CYCL="20190901" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh index b0d240892..df2117a72 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v15p2.sh @@ -17,7 +17,8 @@ EXTRN_MDL_NAME_ICS="FV3GFS" FV3GFS_FILE_FMT_ICS="grib2" EXTRN_MDL_NAME_LBCS="FV3GFS" FV3GFS_FILE_FMT_LBCS="grib2" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) DATE_FIRST_CYCL="20190901" DATE_LAST_CYCL="20190901" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh index eae6b7401..8625dad05 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh @@ -15,7 +15,8 @@ CCPP_PHYS_SUITE="FV3_GSD_SAR" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) DATE_FIRST_CYCL="20200801" DATE_LAST_CYCL="20200801" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh index 890395ba8..ca4ac07b7 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_CONUS_3km_ics_HRRR_lbcs_RAP_suite_HRRR.sh @@ -15,7 +15,8 @@ CCPP_PHYS_SUITE="FV3_HRRR" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) DATE_FIRST_CYCL="20200208" DATE_LAST_CYCL="20200208" diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh index 3770547ef..baeb48871 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_nco/config.nco_grid_RRFS_SUBCONUS_3km_ics_HRRR_lbcs_RAP_suite_GSD_SAR.sh @@ -15,7 +15,8 @@ CCPP_PHYS_SUITE="FV3_GSD_SAR" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) DATE_FIRST_CYCL="20200801" DATE_LAST_CYCL="20200801" diff --git a/tests/WE2E/test_configs/release_SRW_v1/config.GST_release_public_v1.sh b/tests/WE2E/test_configs/release_SRW_v1/config.GST_release_public_v1.sh index 0a5cec525..57a2bb1c9 100644 --- a/tests/WE2E/test_configs/release_SRW_v1/config.GST_release_public_v1.sh +++ b/tests/WE2E/test_configs/release_SRW_v1/config.GST_release_public_v1.sh @@ -16,7 +16,9 @@ EXTRN_MDL_NAME_ICS="FV3GFS" FV3GFS_FILE_FMT_ICS="grib2" EXTRN_MDL_NAME_LBCS="FV3GFS" FV3GFS_FILE_FMT_LBCS="grib2" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20190615" DATE_LAST_CYCL="20190615" diff --git a/tests/WE2E/test_configs/wflow_features/config.MET_verification.sh b/tests/WE2E/test_configs/wflow_features/config.MET_verification.sh index 8d7cc8d4c..9bb923ef6 100644 --- a/tests/WE2E/test_configs/wflow_features/config.MET_verification.sh +++ b/tests/WE2E/test_configs/wflow_features/config.MET_verification.sh @@ -2,7 +2,7 @@ # TEST PURPOSE/DESCRIPTION: # ------------------------ # -# This test is to ensure that the workflow running in community mode +# This test is to ensure that the workflow running in community mode # completes successfully with MET verification. # @@ -12,13 +12,14 @@ PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_25km" CCPP_PHYS_SUITE="FV3_GFS_v15p2" -USE_USER_STAGED_EXTRN_FILES="TRUE" - EXTRN_MDL_NAME_ICS="FV3GFS" FV3GFS_FILE_FMT_ICS="grib2" EXTRN_MDL_NAME_LBCS="FV3GFS" FV3GFS_FILE_FMT_LBCS="grib2" +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" + DATE_FIRST_CYCL="20190615" DATE_LAST_CYCL="20190615" CYCL_HRS=( "00" ) diff --git a/tests/WE2E/test_configs/wflow_features/config.community_ensemble_008mems.sh b/tests/WE2E/test_configs/wflow_features/config.community_ensemble_008mems.sh index 922367c72..d5536f0bb 100644 --- a/tests/WE2E/test_configs/wflow_features/config.community_ensemble_008mems.sh +++ b/tests/WE2E/test_configs/wflow_features/config.community_ensemble_008mems.sh @@ -1,12 +1,12 @@ -# +# # TEST PURPOSE/DESCRIPTION: # ------------------------ # # This test checks the capability of the workflow to run ensemble forecasts -# (i.e. DO_ENSEMBLE set to "TRUE") in community mode (i.e. RUN_ENVIR set +# (i.e. DO_ENSEMBLE set to "TRUE") in community mode (i.e. RUN_ENVIR set # to "community") with the number of ensemble members (NUM_ENS_MEMBERS) -# set to "008". The leading zeros in "008" should cause the ensemble -# members to be numbered "mem001", "mem002", ..., "mem008" (instead of, +# set to "008". The leading zeros in "008" should cause the ensemble +# members to be numbered "mem001", "mem002", ..., "mem008" (instead of, # for instance, "mem1", "mem2", ..., "mem8"). # @@ -18,7 +18,9 @@ CCPP_PHYS_SUITE="FV3_GFS_2017_gfdlmp" EXTRN_MDL_NAME_ICS="FV3GFS" EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20190701" DATE_LAST_CYCL="20190702" diff --git a/tests/WE2E/test_configs/wflow_features/config.community_ensemble_2mems.sh b/tests/WE2E/test_configs/wflow_features/config.community_ensemble_2mems.sh index c9b68006a..a6bb1d859 100644 --- a/tests/WE2E/test_configs/wflow_features/config.community_ensemble_2mems.sh +++ b/tests/WE2E/test_configs/wflow_features/config.community_ensemble_2mems.sh @@ -3,10 +3,10 @@ # ------------------------ # # This test checks the capability of the workflow to run ensemble forecasts -# (i.e. DO_ENSEMBLE set to "TRUE") in community mode (i.e. RUN_ENVIR set -# to "community") with the number of ensemble members (NUM_ENS_MEMBERS) -# set to "2". The lack of leading zeros in this "2" should cause the -# ensemble members to be named "mem1" and "mem2" (instead of, for instance, +# (i.e. DO_ENSEMBLE set to "TRUE") in community mode (i.e. RUN_ENVIR set +# to "community") with the number of ensemble members (NUM_ENS_MEMBERS) +# set to "2". The lack of leading zeros in this "2" should cause the +# ensemble members to be named "mem1" and "mem2" (instead of, for instance, # "mem01" and "mem02"). # # Note also that this test uses two cycle hours ("00" and "12") to test @@ -22,7 +22,9 @@ CCPP_PHYS_SUITE="FV3_GFS_2017_gfdlmp" EXTRN_MDL_NAME_ICS="FV3GFS" EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20190701" DATE_LAST_CYCL="20190702" diff --git a/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio.sh b/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio.sh deleted file mode 100644 index 17a9990c6..000000000 --- a/tests/WE2E/test_configs/wflow_features/config.get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio.sh +++ /dev/null @@ -1,24 +0,0 @@ -# -# TEST PURPOSE/DESCRIPTION: -# ------------------------ -# -# This test checks the capability of the workflow to retrieve from NOAA -# HPSS nemsio-formatted output files generated by the FV3GFS external -# model (from which ICs and LBCs will be derived). -# - -RUN_ENVIR="community" -PREEXISTING_DIR_METHOD="rename" - -PREDEF_GRID_NAME="RRFS_CONUS_25km" -CCPP_PHYS_SUITE="FV3_GFS_2017_gfdlmp" - -EXTRN_MDL_NAME_ICS="FV3GFS" -EXTRN_MDL_NAME_LBCS="FV3GFS" - -DATE_FIRST_CYCL="20190701" -DATE_LAST_CYCL="20190701" -CYCL_HRS=( "00" ) - -FCST_LEN_HRS="6" -LBC_SPEC_INTVL_HRS="3" diff --git a/tests/WE2E/test_configs/wflow_features/config.get_from_NOMADS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio.sh b/tests/WE2E/test_configs/wflow_features/config.get_from_NOMADS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio.sh deleted file mode 100644 index 04efe1fa2..000000000 --- a/tests/WE2E/test_configs/wflow_features/config.get_from_NOMADS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio.sh +++ /dev/null @@ -1,27 +0,0 @@ -# -# TEST PURPOSE/DESCRIPTION: -# ------------------------ -# -# This test checks the capability of the workflow to retrieve from NOMADS -# nemsio-formatted output files generated by the FV3GFS external model -# (from which ICs and LBCs will be derived). -# - -RUN_ENVIR="community" -PREEXISTING_DIR_METHOD="rename" - -PREDEF_GRID_NAME="RRFS_CONUS_25km" -CCPP_PHYS_SUITE="FV3_GFS_2017_gfdlmp" - -EXTRN_MDL_NAME_ICS="FV3GFS" -EXTRN_MDL_NAME_LBCS="FV3GFS" - -DATE_FIRST_CYCL="20200826" -DATE_LAST_CYCL="20200826" -CYCL_HRS=( "00" ) - -FCST_LEN_HRS="6" -LBC_SPEC_INTVL_HRS="3" - -NOMADS="TRUE" -NOMADS_file_type="NEMSIO" diff --git a/tests/WE2E/test_configs/wflow_features/config.inline_post.sh b/tests/WE2E/test_configs/wflow_features/config.inline_post.sh index e491f9b17..d20395e30 100644 --- a/tests/WE2E/test_configs/wflow_features/config.inline_post.sh +++ b/tests/WE2E/test_configs/wflow_features/config.inline_post.sh @@ -14,7 +14,9 @@ CCPP_PHYS_SUITE="FV3_GFS_v15p2" EXTRN_MDL_NAME_ICS="FV3GFS" EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20190701" DATE_LAST_CYCL="20190701" diff --git a/tests/WE2E/test_configs/wflow_features/config.nco_ensemble.sh b/tests/WE2E/test_configs/wflow_features/config.nco_ensemble.sh index 3b9c93f98..171b89c04 100644 --- a/tests/WE2E/test_configs/wflow_features/config.nco_ensemble.sh +++ b/tests/WE2E/test_configs/wflow_features/config.nco_ensemble.sh @@ -1,13 +1,13 @@ -# +# # TEST PURPOSE/DESCRIPTION: # ------------------------ # # This test checks the capability of the workflow to run ensemble forecasts -# (i.e. DO_ENSEMBLE set to "TRUE") in nco mode (i.e. RUN_ENVIR set to -# "nco") with the number of ensemble members (NUM_ENS_MEMBERS) set to -# "2". The lack of leading zeros in this "2" should cause the ensemble -# members to be named "mem1" and "mem2" (instead of, for instance, "mem01" -# and "mem02"). +# (i.e. DO_ENSEMBLE set to "TRUE") in nco mode (i.e. RUN_ENVIR set to +# "nco") with the number of ensemble members (NUM_ENS_MEMBERS) set to +# "2". The lack of leading zeros in this "2" should cause the ensemble +# members to be named "mem1" and "mem2" (instead of, for instance, "mem01" +# and "mem02"). # # Note also that this test uses two cycle hours ("12" and "18") to test # the capability of the workflow to run ensemble forecasts for more than @@ -22,7 +22,8 @@ CCPP_PHYS_SUITE="FV3_GFS_2017_gfdlmp_regional" EXTRN_MDL_NAME_ICS="FV3GFS" EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) DATE_FIRST_CYCL="20190901" DATE_LAST_CYCL="20190902" diff --git a/tests/WE2E/test_configs/wflow_features/config.nco_inline_post.sh b/tests/WE2E/test_configs/wflow_features/config.nco_inline_post.sh index 003b08234..8daf744e6 100644 --- a/tests/WE2E/test_configs/wflow_features/config.nco_inline_post.sh +++ b/tests/WE2E/test_configs/wflow_features/config.nco_inline_post.sh @@ -1,4 +1,4 @@ -# +# # TEST PURPOSE/DESCRIPTION: # ------------------------ # @@ -10,13 +10,12 @@ RUN_ENVIR="nco" PREEXISTING_DIR_METHOD="rename" PREDEF_GRID_NAME="RRFS_CONUS_25km" -WRITE_DOPOST="TRUE" - CCPP_PHYS_SUITE="FV3_GFS_v15p2" EXTRN_MDL_NAME_ICS="FV3GFS" EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) DATE_FIRST_CYCL="20190701" DATE_LAST_CYCL="20190701" @@ -25,4 +24,4 @@ CYCL_HRS=( "00" ) FCST_LEN_HRS="6" LBC_SPEC_INTVL_HRS="3" - +WRITE_DOPOST="TRUE" diff --git a/tests/WE2E/test_configs/wflow_features/config.new_ESGgrid.sh b/tests/WE2E/test_configs/wflow_features/config.new_ESGgrid.sh index b56681f54..acecafcee 100644 --- a/tests/WE2E/test_configs/wflow_features/config.new_ESGgrid.sh +++ b/tests/WE2E/test_configs/wflow_features/config.new_ESGgrid.sh @@ -2,7 +2,7 @@ # TEST PURPOSE/DESCRIPTION: # ------------------------ # -# This test checks the capability of the workflow to have the user +# This test checks the capability of the workflow to have the user # specify a new grid (as opposed to one of the predefined ones in the # workflow) of ESGgrid type. @@ -14,7 +14,9 @@ CCPP_PHYS_SUITE="FV3_GFS_2017_gfdlmp_regional" EXTRN_MDL_NAME_ICS="FV3GFS" EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20190701" DATE_LAST_CYCL="20190701" @@ -47,7 +49,7 @@ BLOCKSIZE="13" QUILTING="TRUE" if [ "$QUILTING" = "TRUE" ]; then WRTCMP_write_groups="1" - WRTCMP_write_tasks_per_group=$(( 1*LAYOUT_Y )) + WRTCMP_write_tasks_per_group=$(( 1*LAYOUT_Y )) WRTCMP_output_grid="lambert_conformal" WRTCMP_cen_lon="${ESGgrid_LON_CTR}" WRTCMP_cen_lat="${ESGgrid_LAT_CTR}" diff --git a/tests/WE2E/test_configs/wflow_features/config.new_GFDLgrid.sh b/tests/WE2E/test_configs/wflow_features/config.new_GFDLgrid.sh index d27148de2..30a54a05f 100644 --- a/tests/WE2E/test_configs/wflow_features/config.new_GFDLgrid.sh +++ b/tests/WE2E/test_configs/wflow_features/config.new_GFDLgrid.sh @@ -2,15 +2,15 @@ # TEST PURPOSE/DESCRIPTION: # ------------------------ # -# This test checks the capability of the workflow to have the user -# specify a new grid (as opposed to one of the predefined ones in the -# workflow) of GFDLgrid type. Note that this test sets the workflow -# variable +# This test checks the capability of the workflow to have the user +# specify a new grid (as opposed to one of the predefined ones in the +# workflow) of GFDLgrid type. Note that this test sets the workflow +# variable # -# GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES +# GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES # -# to "TRUE" (which is its default value); see the UFS SRW User's Guide -# for a description of this variable. +# to "TRUE" (which is its default value); see the UFS SRW User's Guide +# for a description of this variable. # # The difference between this test and the one named # @@ -20,7 +20,7 @@ # variable GFDLgrid_STRETCH_FAC very close to 1. Setting it exactly to # 1 used to cause the workflow to fail because it caused the GFDL grid # generator to assume a global grid. This bug should be rechecked, e.g. -# by setting GFDLgrid_STRETCH_FAC to exactly 1 below. If the grid +# by setting GFDLgrid_STRETCH_FAC to exactly 1 below. If the grid # generation succeeds, then this test can be removed. # @@ -31,7 +31,9 @@ CCPP_PHYS_SUITE="FV3_GFS_2017_gfdlmp" EXTRN_MDL_NAME_ICS="FV3GFS" EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20190701" DATE_LAST_CYCL="20190701" @@ -49,7 +51,7 @@ GFDLgrid_LAT_T6_CTR="38.5" GFDLgrid_STRETCH_FAC="1.0001" # Cannot be exactly 1.0 because then FV3 thinnks it's a global grid. This needs to be fixed in the ufs_weather_model repo. GFDLgrid_RES="96" GFDLgrid_REFINE_RATIO="2" - + #num_margin_cells_T6_left="9" #GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G=$(( num_margin_cells_T6_left + 1 )) GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G="10" diff --git a/tests/WE2E/test_configs/wflow_features/config.new_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_FALSE.sh b/tests/WE2E/test_configs/wflow_features/config.new_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_FALSE.sh index aef2636f2..ebc0e74a5 100644 --- a/tests/WE2E/test_configs/wflow_features/config.new_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_FALSE.sh +++ b/tests/WE2E/test_configs/wflow_features/config.new_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_FALSE.sh @@ -2,15 +2,15 @@ # TEST PURPOSE/DESCRIPTION: # ------------------------ # -# This test checks the capability of the workflow to have the user -# specify a new grid (as opposed to one of the predefined ones in the -# workflow) of GFDLgrid type. Note that this test sets the workflow -# variable +# This test checks the capability of the workflow to have the user +# specify a new grid (as opposed to one of the predefined ones in the +# workflow) of GFDLgrid type. Note that this test sets the workflow +# variable # -# GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES +# GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES # -# to "FALSE"; see the UFS SRW User's Guide for a description of this -# variable. +# to "FALSE"; see the UFS SRW User's Guide for a description of this +# variable. # RUN_ENVIR="community" @@ -20,7 +20,9 @@ CCPP_PHYS_SUITE="FV3_GFS_2017_gfdlmp" EXTRN_MDL_NAME_ICS="FV3GFS" EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20190701" DATE_LAST_CYCL="20190701" @@ -38,7 +40,7 @@ GFDLgrid_LAT_T6_CTR="38.5" GFDLgrid_STRETCH_FAC="1.5" GFDLgrid_RES="96" GFDLgrid_REFINE_RATIO="2" - + #num_margin_cells_T6_left="9" #GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G=$(( num_margin_cells_T6_left + 1 )) GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G="10" diff --git a/tests/WE2E/test_configs/wflow_features/config.new_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_TRUE.sh b/tests/WE2E/test_configs/wflow_features/config.new_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_TRUE.sh index 59ca0925e..dcc5bfa97 100644 --- a/tests/WE2E/test_configs/wflow_features/config.new_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_TRUE.sh +++ b/tests/WE2E/test_configs/wflow_features/config.new_GFDLgrid__GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES_eq_TRUE.sh @@ -2,15 +2,15 @@ # TEST PURPOSE/DESCRIPTION: # ------------------------ # -# This test checks the capability of the workflow to have the user -# specify a new grid (as opposed to one of the predefined ones in the -# workflow) of GFDLgrid type. Note that this test sets the workflow -# variable +# This test checks the capability of the workflow to have the user +# specify a new grid (as opposed to one of the predefined ones in the +# workflow) of GFDLgrid type. Note that this test sets the workflow +# variable # -# GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES +# GFDLgrid_USE_GFDLgrid_RES_IN_FILENAMES # -# to "TRUE" (which is its default value); see the UFS SRW User's Guide -# for a description of this variable. +# to "TRUE" (which is its default value); see the UFS SRW User's Guide +# for a description of this variable. # RUN_ENVIR="community" @@ -20,7 +20,9 @@ CCPP_PHYS_SUITE="FV3_GFS_2017_gfdlmp" EXTRN_MDL_NAME_ICS="FV3GFS" EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20190701" DATE_LAST_CYCL="20190701" @@ -38,7 +40,7 @@ GFDLgrid_LAT_T6_CTR="38.5" GFDLgrid_STRETCH_FAC="1.5" GFDLgrid_RES="96" GFDLgrid_REFINE_RATIO="2" - + #num_margin_cells_T6_left="9" #GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G=$(( num_margin_cells_T6_left + 1 )) GFDLgrid_ISTART_OF_RGNL_DOM_ON_T6G="10" diff --git a/tests/WE2E/test_configs/wflow_features/config.pregen_grid_orog_sfc_climo.sh b/tests/WE2E/test_configs/wflow_features/config.pregen_grid_orog_sfc_climo.sh index fa1d0f06a..96f2b311c 100644 --- a/tests/WE2E/test_configs/wflow_features/config.pregen_grid_orog_sfc_climo.sh +++ b/tests/WE2E/test_configs/wflow_features/config.pregen_grid_orog_sfc_climo.sh @@ -1,8 +1,8 @@ -# +# # TEST PURPOSE/DESCRIPTION: # ------------------------ # -# This test checks the capability of the workflow to use pregenerated +# This test checks the capability of the workflow to use pregenerated # grid, orography, and surface climatology files. # @@ -14,7 +14,9 @@ CCPP_PHYS_SUITE="FV3_GFS_v15p2" EXTRN_MDL_NAME_ICS="FV3GFS" EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20190701" DATE_LAST_CYCL="20190701" diff --git a/tests/WE2E/test_configs/wflow_features/config.specify_DOT_OR_USCORE.sh b/tests/WE2E/test_configs/wflow_features/config.specify_DOT_OR_USCORE.sh index 979d8e96e..150b150c4 100644 --- a/tests/WE2E/test_configs/wflow_features/config.specify_DOT_OR_USCORE.sh +++ b/tests/WE2E/test_configs/wflow_features/config.specify_DOT_OR_USCORE.sh @@ -1,21 +1,21 @@ -# +# # TEST PURPOSE/DESCRIPTION: # ------------------------ # -# This test checks the capability of the workflow to have the character -# (DOT_OR_USCORE) in the names of the input grid and orography files -# that comes after the C-resolution be set to a user-specified value. +# This test checks the capability of the workflow to have the character +# (DOT_OR_USCORE) in the names of the input grid and orography files +# that comes after the C-resolution be set to a user-specified value. # For example, a grid file may be named # # C403${DOT_OR_USCORE}grid.tile7.halo4.nc # -# where "C403" is the C-resolution for this specific grid and +# where "C403" is the C-resolution for this specific grid and # ${DOT_OR_USCORE} represents the contents of the workflow variable -# DOT_OR_USCORE (bash syntax). DOT_OR_USCORE is by default set to an -# underscore, but for consistency with the rest of the separators in the -# file name (as well as with the character after the C-resolution in the -# names of the surface climatology files), it should be a "." (a dot). -# The MAKE_GRID_TN and MAKE_OROG_TN tasks will name the grid and orography +# DOT_OR_USCORE (bash syntax). DOT_OR_USCORE is by default set to an +# underscore, but for consistency with the rest of the separators in the +# file name (as well as with the character after the C-resolution in the +# names of the surface climatology files), it should be a "." (a dot). +# The MAKE_GRID_TN and MAKE_OROG_TN tasks will name the grid and orography # files that they create using this character. # @@ -27,7 +27,9 @@ CCPP_PHYS_SUITE="FV3_GFS_2017_gfdlmp" EXTRN_MDL_NAME_ICS="GSMGFS" EXTRN_MDL_NAME_LBCS="GSMGFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20190520" DATE_LAST_CYCL="20190520" diff --git a/tests/WE2E/test_configs/wflow_features/config.specify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE.sh b/tests/WE2E/test_configs/wflow_features/config.specify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE.sh index 68fa31537..68f40c319 100644 --- a/tests/WE2E/test_configs/wflow_features/config.specify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE.sh +++ b/tests/WE2E/test_configs/wflow_features/config.specify_DT_ATMOS_LAYOUT_XY_BLOCKSIZE.sh @@ -1,9 +1,9 @@ -# +# # TEST PURPOSE/DESCRIPTION: # ------------------------ # -# This test checks the capability of the workflow to have the forecast -# model's time step (DT_ATMOS), its MPI layout (LAYOUT_X and LAYOUT_Y), +# This test checks the capability of the workflow to have the forecast +# model's time step (DT_ATMOS), its MPI layout (LAYOUT_X and LAYOUT_Y), # and its cache block size be set to user-specified values. # @@ -15,7 +15,9 @@ CCPP_PHYS_SUITE="FV3_GSD_SAR" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20200801" DATE_LAST_CYCL="20200801" diff --git a/tests/WE2E/test_configs/wflow_features/config.specify_EXTRN_MDL_BASEDIRS_ICS_LBCS.sh b/tests/WE2E/test_configs/wflow_features/config.specify_EXTRN_MDL_BASEDIRS_ICS_LBCS.sh new file mode 120000 index 000000000..72b708359 --- /dev/null +++ b/tests/WE2E/test_configs/wflow_features/config.specify_EXTRN_MDL_BASEDIRS_ICS_LBCS.sh @@ -0,0 +1 @@ +../get_extrn_mdl_files/config.from_disk__usedefaultdir_no__fdlayout_extrn_mdl__ics_FV3GFS_grib2__lbcs_FV3GFS_grib2__2021060306.sh \ No newline at end of file diff --git a/tests/WE2E/test_configs/wflow_features/config.specify_EXTRN_MDL_SYSBASEDIR_ICS_LBCS.sh b/tests/WE2E/test_configs/wflow_features/config.specify_EXTRN_MDL_SYSBASEDIR_ICS_LBCS.sh deleted file mode 100644 index 2ade79ff5..000000000 --- a/tests/WE2E/test_configs/wflow_features/config.specify_EXTRN_MDL_SYSBASEDIR_ICS_LBCS.sh +++ /dev/null @@ -1,29 +0,0 @@ -# -# TEST PURPOSE/DESCRIPTION: -# ------------------------ -# -# This test checks the capability of the workflow to have the base -# directories on the system disk in which the external model files are -# located be set to user-specified values. -# - -RUN_ENVIR="community" -PREEXISTING_DIR_METHOD="rename" - -PREDEF_GRID_NAME="RRFS_CONUS_25km" -CCPP_PHYS_SUITE="FV3_GFS_v15p2" - -EXTRN_MDL_NAME_ICS="FV3GFS" -FV3GFS_FILE_FMT_ICS="grib2" -EXTRN_MDL_NAME_LBCS="FV3GFS" -FV3GFS_FILE_FMT_LBCS="grib2" - -DATE_FIRST_CYCL="20210603" -DATE_LAST_CYCL="20210603" -CYCL_HRS=( "06" ) - -FCST_LEN_HRS="6" -LBC_SPEC_INTVL_HRS="3" - -EXTRN_MDL_SYSBASEDIR_ICS="set_to_non_default_location_in_testing_script" -EXTRN_MDL_SYSBASEDIR_LBCS="set_to_non_default_location_in_testing_script" diff --git a/tests/WE2E/test_configs/wflow_features/config.specify_RESTART_INTERVAL.sh b/tests/WE2E/test_configs/wflow_features/config.specify_RESTART_INTERVAL.sh index 3051cb5a8..88c5aa72a 100644 --- a/tests/WE2E/test_configs/wflow_features/config.specify_RESTART_INTERVAL.sh +++ b/tests/WE2E/test_configs/wflow_features/config.specify_RESTART_INTERVAL.sh @@ -1,10 +1,10 @@ -# +# # TEST PURPOSE/DESCRIPTION: # ------------------------ # -# This test checks the capability of the workflow to have the time +# This test checks the capability of the workflow to have the time # interval (RESTART_INTERVAL) at which restart files are written by the -# forecast model be set to a user-specified value. +# forecast model be set to a user-specified value. # RUN_ENVIR="community" @@ -15,7 +15,9 @@ CCPP_PHYS_SUITE="FV3_GFS_v15p2" EXTRN_MDL_NAME_ICS="FV3GFS" EXTRN_MDL_NAME_LBCS="FV3GFS" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20190701" DATE_LAST_CYCL="20190701" diff --git a/tests/WE2E/test_configs/wflow_features/config.subhourly_post.sh b/tests/WE2E/test_configs/wflow_features/config.subhourly_post.sh index 3edce24df..ff7d45b65 100644 --- a/tests/WE2E/test_configs/wflow_features/config.subhourly_post.sh +++ b/tests/WE2E/test_configs/wflow_features/config.subhourly_post.sh @@ -1,4 +1,4 @@ -# +# # TEST PURPOSE/DESCRIPTION: # ------------------------ # @@ -14,7 +14,9 @@ CCPP_PHYS_SUITE="FV3_RRFS_v1beta" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20200810" DATE_LAST_CYCL="20200810" diff --git a/tests/WE2E/test_configs/wflow_features/config.subhourly_post_ensemble_2mems.sh b/tests/WE2E/test_configs/wflow_features/config.subhourly_post_ensemble_2mems.sh index 337e99740..c9024749b 100644 --- a/tests/WE2E/test_configs/wflow_features/config.subhourly_post_ensemble_2mems.sh +++ b/tests/WE2E/test_configs/wflow_features/config.subhourly_post_ensemble_2mems.sh @@ -1,4 +1,4 @@ -# +# # TEST PURPOSE/DESCRIPTION: # ------------------------ # @@ -6,7 +6,7 @@ # that require the forecast model to write output files and perform post- # processing on a sub-hourly time interval. # -# This test is needed in addition to the one named "subhourly_post" +# This test is needed in addition to the one named "subhourly_post" # because in the jinja template file from which the rocoto workflow XML # is generated, the code changes that were made to add the subhourly # capability also involved changes to the ensemble capability. @@ -20,7 +20,9 @@ CCPP_PHYS_SUITE="FV3_RRFS_v1beta" EXTRN_MDL_NAME_ICS="HRRR" EXTRN_MDL_NAME_LBCS="RAP" -USE_USER_STAGED_EXTRN_FILES="TRUE" + +EXTRN_MDL_DATA_SOURCES=( "disk" ) +EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" DATE_FIRST_CYCL="20200810" DATE_LAST_CYCL="20200810" diff --git a/ush/NOMADS_get_extrn_mdl_files.sh b/ush/NOMADS_get_extrn_mdl_files.sh deleted file mode 100755 index 308aa8082..000000000 --- a/ush/NOMADS_get_extrn_mdl_files.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/bin/bash -# Command line arguments -if [ -z "$1" -o -z "$2" ]; then - echo "Usage: $0 yyyymmdd hh file_fmt nfcst nfcst_int" - echo "yyymmdd and hh are required and other variables are optional" - exit -fi -## date (year month day) and time (hour) -yyyymmdd=$1 #i.e. "20191224" -hh=$2 #i.e. "12" -## -## file format (grib2 or nemsio), the default format is grib2 -if [ "$#" -ge 3 ]; then - file_fmt=$3 -else - file_fmt="grib2" -fi -## forecast length, the default value are 6 hours -if [ "$#" -ge 4 ]; then - nfcst=$4 -else - nfcst=6 -fi -## forecast interval, the default interval are 3 hours -if [ "$#" -ge 5 ]; then - nfcst_int=$5 -else - nfcst_int=3 -fi - -# Get the data (do not need to edit anything after this point!) -yyyymm=$((yyyymmdd/100)) -#din_loc_ic=`./xmlquery DIN_LOC_IC --value` -mkdir -p gfs.$yyyymmdd/$hh -echo "Download files to $din_loc_ic/$yyyymm/$yyyymmdd ..." -cd gfs.$yyyymmdd/$hh - -#getting online analysis data -if [ $file_fmt == "grib2" ] || [ $file_fmt == "GRIB2" ]; then - wget -c https://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/gfs.$yyyymmdd/$hh/gfs.t${hh}z.pgrb2.0p25.f000 -else - wget -c https://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/gfs.$yyyymmdd/$hh/gfs.t${hh}z.atmanl.nemsio - wget -c https://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/gfs.$yyyymmdd/$hh/gfs.t${hh}z.sfcanl.nemsio -fi - -#getting online forecast data -ifcst=$nfcst_int -while [ $ifcst -le $nfcst ] -do -echo $ifcst - if [ $ifcst -le 99 ]; then - if [ $ifcst -le 9 ]; then - ifcst_str="00"$ifcst - else - ifcst_str="0"$ifcst - fi - else - ifcst_str="$ifcst" - fi - echo $ifcst_str -# -if [ $file_fmt == "grib2" ] || [ $file_fmt == "GRIB2" ]; then - wget -c https://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/gfs.$yyyymmdd/$hh/gfs.t${hh}z.pgrb2.0p25.f${ifcst_str} -else - wget -c https://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/gfs.$yyyymmdd/$hh/gfs.t${hh}z.atmf${ifcst_str}.nemsio - wget -c https://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/gfs.$yyyymmdd/$hh/gfs.t${hh}z.sfcf${ifcst_str}.nemsio -fi -# -ifcst=$[$ifcst+$nfcst_int] -done diff --git a/ush/bash_utils/check_for_preexist_dir_file.sh b/ush/bash_utils/check_for_preexist_dir_file.sh index 7b9b59772..01787e757 100644 --- a/ush/bash_utils/check_for_preexist_dir_file.sh +++ b/ush/bash_utils/check_for_preexist_dir_file.sh @@ -84,7 +84,7 @@ where the arguments are defined as follows: # #----------------------------------------------------------------------- # - local valid_vals_method=( "delete" "rename" "quit" ) + local valid_vals_method=( "delete" "rename" "quit" "none" ) check_var_valid_value "method" "valid_vals_method" # #----------------------------------------------------------------------- @@ -106,7 +106,6 @@ where the arguments are defined as follows: #----------------------------------------------------------------------- # "delete") - rm_vrfy -rf "${dir_or_file}" ;; # @@ -118,7 +117,6 @@ where the arguments are defined as follows: #----------------------------------------------------------------------- # "rename") - local i=1 local old_indx=$( printf "%03d" "$i" ) local old_dir_or_file="${dir_or_file}_old${old_indx}" @@ -127,8 +125,12 @@ where the arguments are defined as follows: old_indx=$( printf "%03d" "$i" ) old_dir_or_file="${dir_or_file}_old${old_indx}" done - - print_info_msg "$VERBOSE" " +# +# If a global variable named VERBOSE is defined, use it to determine +# whether or not to print out a message before renaming the file or +# directory. If VERBOSE is not defined, don't print out a message. +# + print_info_msg "${VERBOSE:-FALSE}" " Specified directory or file (dir_or_file) already exists: dir_or_file = \"${dir_or_file}\" Moving (renaming) preexisting directory or file to: @@ -146,11 +148,19 @@ Moving (renaming) preexisting directory or file to: #----------------------------------------------------------------------- # "quit") - print_err_msg_exit "\ Specified directory or file (dir_or_file) already exists: dir_or_file = \"${dir_or_file}\"" ;; +# +#----------------------------------------------------------------------- +# +# If method is set to "none", take no action. +# +#----------------------------------------------------------------------- +# + "none") + ;; esac @@ -166,5 +176,3 @@ Specified directory or file (dir_or_file) already exists: { restore_shell_opts; } > /dev/null 2>&1 } - - diff --git a/ush/bash_utils/create_symlink_to_file.sh b/ush/bash_utils/create_symlink_to_file.sh index e54bf0a36..e604c39dd 100644 --- a/ush/bash_utils/create_symlink_to_file.sh +++ b/ush/bash_utils/create_symlink_to_file.sh @@ -52,10 +52,10 @@ function create_symlink_to_file() { #----------------------------------------------------------------------- # local valid_args=( \ -"target" \ -"symlink" \ -"relative" \ - ) + "target" \ + "symlink" \ + "relative" \ + ) process_args valid_args "$@" # #----------------------------------------------------------------------- diff --git a/ush/bash_utils/is_array.sh b/ush/bash_utils/is_array.sh index 16be2fbb7..f76e0e567 100644 --- a/ush/bash_utils/is_array.sh +++ b/ush/bash_utils/is_array.sh @@ -4,7 +4,7 @@ # This file defines a function that is used to check whether a specified # variable is a bash array. It is called as follows: # -# is_array var_name +# is_array "var_name" # # Here, var_name is the name of the variable to check to determine whe- # ther or not it is an array. If the variable is an array, this func- diff --git a/ush/bash_utils/is_element_of.sh b/ush/bash_utils/is_element_of.sh index d3060db7d..a1a647f33 100644 --- a/ush/bash_utils/is_element_of.sh +++ b/ush/bash_utils/is_element_of.sh @@ -74,15 +74,15 @@ Use this function in a script as follows: array_name=("1" "2" "3 4" "5") str_to_match="2" - is_element_of "${str_to_match}" array_name + is_element_of array_name "${str_to_match}" echo $? # Should output 0. str_to_match="3 4" - is_element_of "${str_to_match}" array_name + is_element_of array_name "${str_to_match}" echo $? # Should output 0. str_to_match="6" - is_element_of "${str_to_match}" array_name + is_element_of array_name "${str_to_match}" echo $? # Should output 1. " diff --git a/ush/bash_utils/parse_cdate.sh b/ush/bash_utils/parse_cdate.sh new file mode 100644 index 000000000..cf72412c4 --- /dev/null +++ b/ush/bash_utils/parse_cdate.sh @@ -0,0 +1,209 @@ +# +#----------------------------------------------------------------------- +# +# This file defines a function that parses a given cycle date (cdate) and +# returns its various parts. It assumes that cdate is a string that +# consists of either 10 or 12 digits. If cdate is 10 digits long, this +# function assumes that it is a string of the form "yyyymmddhh" where +# yyyy is the four-digit year, mm is the two-digit month, dd is the two- +# digit day of the month, and hh is the two-digit hour of the day. In +# this case, the two-digit minutes (mn) are assumed to be "00". If cdate +# is 12 characters long, this function assumes it is a string of the form +# "yyyymmddhhmn" where yyyy, mm, dd, and hh are as defined above and mn +# is the two-digit minute. +# +#----------------------------------------------------------------------- +# +function parse_cdate() { +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# + { save_shell_opts; set -u +x; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Specify the set of valid argument names for this script/function. Then +# process the arguments provided to this script/function (which should +# consist of a set of name-value pairs of the form arg1="value1", etc). +# +#----------------------------------------------------------------------- +# + local valid_args=( \ + "cdate" \ + "outvarname_yyyymmdd" \ + "outvarname_yyyy" \ + "outvarname_mm" \ + "outvarname_dd" \ + "outvarname_hh" \ + "outvarname_mn" \ + ) + process_args valid_args "$@" +# +#----------------------------------------------------------------------- +# +# For debugging purposes, print out values of arguments passed to this +# script. Note that these will be printed out only if VERBOSE is set to +# TRUE. +# +#----------------------------------------------------------------------- +# + print_input_args valid_args +# +#----------------------------------------------------------------------- +# +# Declare local variables. +# +#----------------------------------------------------------------------- +# + local __dd \ + __hh \ + __mm \ + __mn \ + nchars_cdate \ + __yyyy \ + __yyyymmdd +# +#----------------------------------------------------------------------- +# +# Run the following checks on cdate: +# +# 1) Check that it is not empty. +# 2) Check that it only contains digits. +# 3) Check that it is either 10 or 12 characters long. +# +#----------------------------------------------------------------------- +# + if [ -z "${cdate}" ]; then + print_err_msg_exit "\ +The input argument \"cdate\" cannot be empty: + cdate = \"$cdate\"" + fi + + if ! [[ "${cdate}" =~ ^[0-9]+$ ]]; then + print_err_msg_exit "\ +The input argument \"cdate\" must contain only digits: + cdate = \"$cdate\"" + fi + + nchars_cdate=${#cdate} + if [ "${nchars_cdate}" -ne "10" ] && [ "${nchars_cdate}" -ne "12" ]; then + print_err_msg_exit "\ +The number of digits in the input argument \"cdate\" must be either 10 +(in which case it is assumed to have the form \"yyyymmddhh\") or 12 (in +which case it is assumed to have the form \"yyyymmddhhmn\" where mn is +the two-digit minute): + cdate = \"$cdate\" + nchars_cdate = ${nchars_cdate}" + fi +# +#----------------------------------------------------------------------- +# +# Extract the various parts of cdate. +# +#----------------------------------------------------------------------- +# + __yyyymmdd=${cdate:0:8} + __yyyy=${cdate:0:4} + __mm=${cdate:4:2} + __dd=${cdate:6:2} + __hh=${cdate:8:2} + if [ "${nchars_cdate}" -eq "10" ]; then + __mn="00" + elif [ "${nchars_cdate}" -eq "12" ]; then + __mn="${cdate:10:2}" + fi +# +#----------------------------------------------------------------------- +# +# Check the extracted values to make sure they're in range. +# +#----------------------------------------------------------------------- +# + mm_min="01" + mm_max="12" + if [ "$__mm" -lt "${mm_min}" ] || [ "$__mm" -gt "${mm_max}" ]; then + print_err_msg_exit "\ +The two-digit month (mm) extracted from cdate must be between \"${mm_min}\" and \"${mm_max}\", +inclusive: + cdate = \"$cdate\" + mm = \"$__mm\"" + fi + + dd_min="01" + dd_max="31" # Don't worry about some months having less than 31 days. + if [ "$__dd" -lt "${dd_min}" ] || [ "$__dd" -gt "${dd_max}" ]; then + print_err_msg_exit "\ +The two-digit day (dd) extracted from cdate must be between \"${dd_min}\" and \"${dd_max}\", +inclusive: + cdate = \"$cdate\" + dd = \"$__dd\"" + fi + + hh_min="00" + hh_max="23" + if [ "$__hh" -lt "${hh_min}" ] || [ "$__hh" -gt "${hh_max}" ]; then + print_err_msg_exit "\ +The two-digit hour (hh) extracted from cdate must be between \"${hh_min}\" and +\"${hh_max}\", inclusive: + cdate = \"$cdate\" + hh = \"$__hh\"" + fi + + mn_min="00" + mn_max="59" + if [ "$__mn" -lt "${mn_min}" ] || [ "$__mn" -gt "${mn_max}" ]; then + print_err_msg_exit "\ +The two-digit minute (mn) extracted from cdate must be between \"${mn_min}\" and +\"${mn_max}\", inclusive: + cdate = \"$cdate\" + mn = \"$__mn\"" + fi +# +#----------------------------------------------------------------------- +# +# Use the eval function to set the output variables. Note that each of +# these is set only if the corresponding input variable specifying the +# name to use for the output variable is not empty. +# +#----------------------------------------------------------------------- +# + if [ ! -z "${outvarname_yyyymmdd}" ]; then + eval ${outvarname_yyyymmdd}=$__yyyymmdd + fi + + if [ ! -z "${outvarname_yyyy}" ]; then + eval ${outvarname_yyyy}=$__yyyy + fi + + if [ ! -z "${outvarname_mm}" ]; then + eval ${outvarname_mm}=$__mm + fi + + if [ ! -z "${outvarname_dd}" ]; then + eval ${outvarname_dd}=$__dd + fi + + if [ ! -z "${outvarname_hh}" ]; then + eval ${outvarname_hh}=$__hh + fi + + if [ ! -z "${outvarname_mn}" ]; then + eval ${outvarname_mn}=$__mn + fi +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/func- +# tion. +# +#----------------------------------------------------------------------- +# + { restore_shell_opts; } > /dev/null 2>&1 + +} + diff --git a/ush/bash_utils/print_input_args.sh b/ush/bash_utils/print_input_args.sh index 10bbe4ca6..5c8257fc9 100644 --- a/ush/bash_utils/print_input_args.sh +++ b/ush/bash_utils/print_input_args.sh @@ -1,14 +1,14 @@ # #----------------------------------------------------------------------- # -# This file defines a function that prints to stdout the names and val- -# ues of a specified list of variables that are the valid arguments to -# the script or function that calls this function. It is mainly used -# for debugging to check that the argument values passed to the calling -# script/function have been set correctly. Note that if a global varia- -# ble named VERBOSE is not defined, the message will be printed out. If -# a global variable named VERBOSE is defined, then the message will be -# printed out only if VERBOSE is set to TRUE. +# This file defines a function that prints to stdout the names and values +# of a specified list of variables that are the valid arguments to the +# script or function that calls this function. It is mainly used for +# debugging to check that the argument values passed to the calling +# script or function have been set correctly. Note that if a global +# variable named VERBOSE is not defined, the message will be printed +# out. If a global variable named VERBOSE is defined, then the message +# will be printed out only if VERBOSE is set to TRUE. # #----------------------------------------------------------------------- # @@ -99,12 +99,12 @@ function. #----------------------------------------------------------------------- # local array_name_valid_caller_args \ - valid_caller_args \ - script_or_function \ + i \ + line \ msg \ num_valid_args \ - i \ - line + script_or_function \ + valid_caller_args # #----------------------------------------------------------------------- # @@ -168,17 +168,13 @@ have been set as follows: # #----------------------------------------------------------------------- # -# If a global variable named VERBOSE is not defined, print out the mes- -# sage. If it is defined, print out the message only if VERBOSE is set -# to TRUE. +# If a global variable named VERBOSE is defined, use it to determine +# whether or not to print out the message. If VERBOSE is not defined, +# don't print out the message. # #----------------------------------------------------------------------- # - if [ ! -v VERBOSE ]; then - print_info_msg "$msg" - else - print_info_msg "$VERBOSE" "$msg" - fi + print_info_msg "${VERBOSE:-FALSE}" "$msg" # #----------------------------------------------------------------------- # diff --git a/ush/bash_utils/process_args.sh b/ush/bash_utils/process_args.sh index 4363348ba..649c344bd 100644 --- a/ush/bash_utils/process_args.sh +++ b/ush/bash_utils/process_args.sh @@ -6,21 +6,20 @@ # refer to these pairs as argument-value pairs (or "arg-val" pairs for # short) because the variable names in these pairs represent the names # of arguments to the script or function that calls this function (which -# we refer to here as the "caller"). The first argument to this func- -# tion being the name of an array that contains a list of valid argument -# names that the caller is allowed to accept. Each arg-val pair must -# have the form +# we refer to here as the "caller"). The first argument to this function +# being the name of an array that contains a list of valid argument names +# that the caller is allowed to accept. Each arg-val pair must have the +# form # # ARG_NAME=VAR_VALUE # # where ARG_NAME is the name of an argument and VAR_VALUE is the value # to set that argument to. For each arg-val pair, this function creates -# a global variable named ARG_NAME and assigns to it the value VAR_VAL- -# UE. +# a global variable named ARG_NAME and assigns to it the value VAR_VALUE. # -# The purpose of this function is to provide a mechanism by which a pa- -# rent script, say parent.sh, can pass variable values to a child script -# or function, say child.sh, that makes it very clear which arguments of +# The purpose of this function is to provide a mechanism by which a parent +# script, say parent.sh, can pass variable values to a child script or +# function, say child.sh, that makes it very clear which arguments of # child.sh are being set and to what values. For example, parent.sh can # call child.sh as follows: # @@ -28,8 +27,8 @@ # child.sh arg3="Hello" arg2="bye" arg4=("this" "is" "an" "array") # ... # -# Then child.sh can use this function (process_args) as follows to pro- -# cess the arg-val pairs passed to it: +# Then child.sh can use this function (process_args) as follows to process +# the arg-val pairs passed to it: # # ... # valid_args=( "arg1" "arg2" "arg3" "arg4" ) @@ -41,8 +40,8 @@ # allowed to accept as arguments. Its name is passed to process_args as # the first argument. The "$@" appearing in the call to process_args # passes to process_args the list of arg-val pairs that parent.sh passes -# to child.sh as the second through N-th arguments. In the example -# above, "$@" represents: +# to child.sh as the second through N-th arguments. In the example above, +# "$@" represents: # # arg3="Hello" arg2="bye" arg4=("this" "is" "an" "array") # @@ -56,11 +55,11 @@ # # Note that: # -# 1) The set of arg-val pairs may list only a subset of the list of arg- -# uments declared in valid_args; the unlisted arguments will be set -# to the null string. In the example above, arg1 is set to the null -# string because it is not specified in any of the arg-val pairs in -# the call to child.sh in parent.sh. +# 1) The set of arg-val pairs may list only a subset of the list of +# arguments declared in valid_args; the unlisted arguments will be +# set to the null string. In the example above, arg1 is set to the +# null string because it is not specified in any of the arg-val pairs +# in the call to child.sh in parent.sh. # # 2) The arg-val pairs in the call to child.sh do not have to be in the # same order as the list of "declared" arguments in child.sh. For @@ -113,6 +112,16 @@ function process_args() { # #----------------------------------------------------------------------- # +# Get information about the script or function that calls this function. +# +#----------------------------------------------------------------------- +# + local caller_fp=$( readlink -f "${BASH_SOURCE[1]}" ) + local caller_fn=$( basename "${caller_fp}" ) + local caller_dir=$( dirname "${caller_fp}" ) +# +#----------------------------------------------------------------------- +# # Check arguments. # #----------------------------------------------------------------------- @@ -158,14 +167,20 @@ where the arguments are defined as follows: # #----------------------------------------------------------------------- # - local array_name_valid_arg_names \ - valid_arg_names_at \ - valid_arg_names \ - num_valid_args \ + local arg_already_specified \ + arg_name \ + arg_val_pair \ + arg_value \ + array_name_valid_arg_names \ + cmd_line \ + err_msg \ + i \ + is_array \ num_arg_val_pairs \ - i valid_arg_name arg_already_specified \ - arg_val_pair arg_name arg_value is_array \ - err_msg cmd_line + num_valid_args \ + valid_arg_name \ + valid_arg_names \ + valid_arg_names_at # #----------------------------------------------------------------------- # @@ -228,9 +243,12 @@ where the arguments are defined as follows: # if [ "${num_arg_val_pairs}" -gt "${num_valid_args}" ]; then print_err_msg_exit "\ -The number of argument-value pairs specified on the command line (num_- -arg_val_pairs) must be less than or equal to the number of valid argu- -ments (num_valid_args) specified in the array valid_arg_names: +In the call to the script or function specified by caller_fn (full path +given by caller_fp), the number of argument-value pairs (num_arg_val_pairs) +must be less than or equal to the number of valid arguments (num_valid_args) +specified in the array valid_arg_names: + caller_fn = \"${caller_fn}\" + caller_fp = \"${caller_fp}\" num_arg_val_pairs = ${num_arg_val_pairs} num_valid_args = ${num_valid_args} valid_arg_names = ( ${valid_arg_names_str})" @@ -267,17 +285,23 @@ ments (num_valid_args) specified in the array valid_arg_names: if [ "${valid_arg_name_no_spaces}" != "${valid_arg_name}" ]; then print_err_msg_exit "\ -The name of an argument in the list of valid arguments (valid_arg_names) -cannot contain any spaces, but the element with index i=${i} contains at -least one space: +In the call to the script or function specified by caller_fn (full path +given by caller_fp), the name of an argument in the list of valid arguments +(valid_arg_names) cannot contain any spaces, but the element with index +i=${i} contains at least one space: + caller_fn = \"${caller_fn}\" + caller_fp = \"${caller_fp}\" valid_arg_names = ( ${valid_arg_names_str}) valid_arg_names[$i] = \"${valid_arg_names[$i]}\"" fi if [ -z ${valid_arg_name} ]; then print_err_msg_exit "\ -The list of valid arguments (valid_arg_names) cannot contain empty elements, -but the element with index i=${i} is empty: +In the call to the script or function specified by caller_fn (full path +given by caller_fp), the list of valid arguments (valid_arg_names) cannot +contain empty elements, but the element with index i=${i} is empty: + caller_fn = \"${caller_fn}\" + caller_fp = \"${caller_fp}\" valid_arg_names = ( ${valid_arg_names_str}) valid_arg_names[$i] = \"${valid_arg_names[$i]}\"" fi @@ -330,10 +354,14 @@ but the element with index i=${i} is empty: #----------------------------------------------------------------------- # err_msg="\ -The specified argument name (arg_name) in the current argument-value -pair (arg_val_pair) is not valid: +In the call to the script or function specified by caller_fn (full path +given by caller_fp), the argument name (arg_name) in the current argument- +value pair (arg_val_pair) is not valid: + caller_fn = \"${caller_fn}\" + caller_fp = \"${caller_fp}\" arg_val_pair = \"${arg_val_pair}\" arg_name = \"${arg_name}\"" + check_var_valid_value "arg_name" "valid_arg_names" "${err_msg}" # #----------------------------------------------------------------------- diff --git a/ush/bash_utils/set_boolean_to_TRUE_or_FALSE.sh b/ush/bash_utils/set_boolean_to_TRUE_or_FALSE.sh new file mode 100644 index 000000000..9d4fbd991 --- /dev/null +++ b/ush/bash_utils/set_boolean_to_TRUE_or_FALSE.sh @@ -0,0 +1,99 @@ +# +#----------------------------------------------------------------------- +# +# This file defines a function that sets the value of a boolean variable +# to either "TRUE" (if it is set to other valid values that are equivalent +# to "TRUE", e.g. "true", "YES", "yes", etc) or "FALSE" (if it is set to +# other valid values that are equivalent to "FALSE", e.g. "false", "NO", +# "no", etc). +# +#----------------------------------------------------------------------- +# +function set_boolean_to_TRUE_or_FALSE() { +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# + { save_shell_opts; set -u +x; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Specify the set of valid argument names for this script/function. Then +# process the arguments provided to this script/function (which should +# consist of a set of name-value pairs of the form arg1="value1", etc). +# +#----------------------------------------------------------------------- +# + local valid_args=( \ + "boolean_value" \ + "outvarname_boolean" \ + ) + process_args valid_args "$@" +# +#----------------------------------------------------------------------- +# +# For debugging purposes, print out values of arguments passed to this +# script. Note that these will be printed out only if VERBOSE is set to +# TRUE. +# +#----------------------------------------------------------------------- +# + print_input_args "valid_args" +# +#----------------------------------------------------------------------- +# +# Declare local variables. +# +#----------------------------------------------------------------------- +# + local b +# +#----------------------------------------------------------------------- +# +# Make sure that boolean_value is set to a valid value. +# +#----------------------------------------------------------------------- +# + valid_vals_boolean=( "TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no" ) + check_var_valid_value "boolean_value" "valid_vals_BOOLEAN" +# +#----------------------------------------------------------------------- +# +# Set b to either "TRUE" or "FALSE" depending on the value of the input +# argument "boolean_value". +# +#----------------------------------------------------------------------- +# + b="${boolean_value}" + if [ "$b" = "true" ] || [ "$b" = "YES" ] || [ "$b" = "yes" ]; then + b="TRUE" + elif [ "$b" = "false" ] || [ "$b" = "NO" ] || [ "$b" = "no" ]; then + b="FALSE" + fi +# +#----------------------------------------------------------------------- +# +# Use the eval function to set the output variables. Note that each of +# these is set only if the corresponding input variable specifying the +# name to use for the output variable is not empty. +# +#----------------------------------------------------------------------- +# + if [ ! -z "${outvarname_boolean}" ]; then + eval ${outvarname_boolean}="$b" + fi +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/func- +# tion. +# +#----------------------------------------------------------------------- +# + { restore_shell_opts; } > /dev/null 2>&1 + +} diff --git a/ush/check_ruc_lsm.sh b/ush/check_ruc_lsm.sh index fe22d08ab..9dfeb2e5f 100644 --- a/ush/check_ruc_lsm.sh +++ b/ush/check_ruc_lsm.sh @@ -20,7 +20,7 @@ function check_ruc_lsm() { # #----------------------------------------------------------------------- # - { save_shell_opts; set -u -x; } > /dev/null 2>&1 + { save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/ush/config.community.sh b/ush/config.community.sh index b0c1f21f1..66a207437 100644 --- a/ush/config.community.sh +++ b/ush/config.community.sh @@ -20,9 +20,8 @@ DATE_LAST_CYCL="20190615" CYCL_HRS=( "00" ) EXTRN_MDL_NAME_ICS="FV3GFS" -EXTRN_MDL_NAME_LBCS="FV3GFS" - FV3GFS_FILE_FMT_ICS="grib2" +EXTRN_MDL_NAME_LBCS="FV3GFS" FV3GFS_FILE_FMT_LBCS="grib2" WTIME_RUN_FCST="01:00:00" @@ -42,20 +41,28 @@ RUN_TASK_GET_OBS_MRMS="FALSE" RUN_TASK_GET_OBS_NDAS="FALSE" RUN_TASK_VX_GRIDSTAT="FALSE" RUN_TASK_VX_POINTSTAT="FALSE" - -# -# Uncomment the following line in order to use user-staged external model -# files with locations and names as specified by EXTRN_MDL_SOURCE_BASEDIR_ICS/ -# LBCS and EXTRN_MDL_FILES_ICS/LBCS. # -#USE_USER_STAGED_EXTRN_FILES="TRUE" +# Uncomment the following lines to be able to use user-staged external +# model files. Note that the values of EXTRN_MDL_BASEDIRS_ICS and +# EXTRN_MDL_BASEDIRS_LBCS below are specifically for Hera. # -# The following is specifically for Hera. It will have to be modified -# if on another platform, using other dates, other external models, etc. -# Uncomment the following EXTRN_MDL_*_ICS/LBCS only when USE_USER_STAGED_EXTRN_FILES=TRUE +# If the lines below are left commented, the workflow scripts will first +# attempt to get the files from disk, then from NOAA HPSS, and finally +# (if needed) from NOMADS. The attempt from disk will fail because the +# default values that EXTRN_MDL_BASEDIRS_ICS and EXTRN_MDL_BASEDIRS_LBCS +# get assigned if they are not specified below will not contain the data +# for the date specified above (because that date is not recent, i.e. +# not within the last 2 weeks or 2 days, depending on the machine). +# Whether or not the attempt to get the files from NOAA HPSS will succeed +# or fail depends on whether the machine the experiment is running on +# has access to NOAA HPSS. Finally, the attempt to fetch from NOMADS +# will also fail because the date above is not recent (and only data +# that is relatively recent dates is available on NOMADS). # -#EXTRN_MDL_SOURCE_BASEDIR_ICS="/scratch2/BMC/det/UFS_SRW_app/v1p0/model_data/FV3GFS" -#EXTRN_MDL_FILES_ICS=( "gfs.pgrb2.0p25.f000" ) -#EXTRN_MDL_SOURCE_BASEDIR_LBCS="/path/to/model_data/FV3GFS" -#EXTRN_MDL_FILES_LBCS=( "gfs.pgrb2.0p25.f006" "gfs.pgrb2.0p25.f012" "gfs.pgrb2.0p25.f018" "gfs.pgrb2.0p25.f024" \ -# "gfs.pgrb2.0p25.f030" "gfs.pgrb2.0p25.f036" "gfs.pgrb2.0p25.f042" "gfs.pgrb2.0p25.f048" ) +#EXTRN_MDL_BASEDIRS_ICS="/scratch2/BMC/det/UFS_SRW_app/develop2/model_data/FV3GFS/grib2" +#EXTRN_MDL_BASEDIRS_LBCS="/scratch2/BMC/det/UFS_SRW_app/develop2/model_data/FV3GFS/grib2" +#EXTRN_MDL_DIR_FILE_LAYOUT="user_spec" +#EXTRN_MDL_FNS_ICS=( "gfs.pgrb2.0p25.f000" ) +#EXTRN_MDL_FNS_LBCS_PREFIX="gfs.pgrb2.0p25.f" +#EXTRN_MDL_FNS_LBCS_SUFFIX="" + diff --git a/ush/config.nco.sh b/ush/config.nco.sh index 3b2c84be6..39b914ed8 100644 --- a/ush/config.nco.sh +++ b/ush/config.nco.sh @@ -24,11 +24,30 @@ EXTRN_MDL_NAME_ICS="FV3GFS" EXTRN_MDL_NAME_LBCS="FV3GFS" # -# The following must be modified for different platforms and users. +# The following must be modified for different platforms and users. +# Definitions: +# +# RUN: +# Name of model run; used in forming path to output files. See details +# in config_defaults.sh. +# +# COMINgfs: +# Path to directory containing files from the external model (FV3GFS). +# +# FIXLAM_NCO_BASEDIR: +# Path to directory containing the pregenerated grid, orography, and +# surface climatology "fixed" files to use for the experiment. +# +# STMP: +# Path to directory STMP that mostly contains input files. +# +# PTMP: +# Path to directory PTMP in which the experiment's output files will be +# placed. # RUN="an_experiment" -COMINgfs="/scratch1/NCEPDEV/hwrf/noscrub/hafs-input/COMGFS" # Path to directory containing files from the external model (FV3GFS). -FIXLAM_NCO_BASEDIR="/scratch2/BMC/det/FV3LAM_pregen" # Path to directory containing the pregenerated grid, orography, and surface climatology "fixed" files to use for the experiment. -STMP="/scratch2/BMC/det/Gerard.Ketefian/UFS_CAM/NCO_dirs/stmp" # Path to directory STMP that mostly contains input files. -PTMP="/scratch2/BMC/det/Gerard.Ketefian/UFS_CAM/NCO_dirs/ptmp" # Path to directory PTMP in which the experiment's output files will be placed. +COMINgfs="/scratch2/BMC/det/Gerard.Ketefian/UFS_CAM/staged_extrn_mdl_files/FV3GFS/nemsio" +FIXLAM_NCO_BASEDIR="/scratch2/BMC/det/FV3LAM_pregen" +STMP="/scratch2/BMC/det/Gerard.Ketefian/UFS_CAM/NCO_dirs/stmp" +PTMP="/scratch2/BMC/det/Gerard.Ketefian/UFS_CAM/NCO_dirs/ptmp" diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index 64baa7a8a..736775618 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -226,7 +226,7 @@ EXPT_SUBDIR="" # #----------------------------------------------------------------------- # -COMINgfs="/base/path/of/directory/containing/gfs/input/files" +COMINgfs="" FIXLAM_NCO_BASEDIR="" STMP="/base/path/of/directory/containing/model/input/and/raw/output/files" NET="rrfs" @@ -522,28 +522,28 @@ NDAS_OBS_DIR="/path/to/observation-directory/ndas/proc" # Definitions: # # EXTRN_MDL_NAME_ICS: -#`The name of the external model that will provide fields from which -# initial condition (including and surface) files will be generated for +# The name of the external model that will provide fields from which +# initial condition (IC, including surface) files will be generated for # input into the forecast model. # # EXTRN_MDL_NAME_LBCS: -#`The name of the external model that will provide fields from which +# The name of the external model that will provide fields from which # lateral boundary condition (LBC) files will be generated for input into # the forecast model. # # LBC_SPEC_INTVL_HRS: # The interval (in integer hours) with which LBC files will be generated. -# We will refer to this as the boundary update interval. Note that the -# model specified in EXTRN_MDL_NAME_LBCS must have data available at a -# frequency greater than or equal to that implied by LBC_SPEC_INTVL_HRS. +# We will refer to this as the boundary specification interval. Note +# that the model specified in EXTRN_MDL_NAME_LBCS must have data available +# at a frequency greater than or equal to that implied by LBC_SPEC_INTVL_HRS. # For example, if LBC_SPEC_INTVL_HRS is set to 6, then the model must have # data availble at least every 6 hours. It is up to the user to ensure # that this is the case. # # FV3GFS_FILE_FMT_ICS: -# If using the FV3GFS model as the source of the ICs (i.e. if EXTRN_MDL_NAME_ICS -# is set to "FV3GFS"), this variable specifies the format of the model -# files to use when generating the ICs. +# If using the FV3GFS model as the source of the ICs (i.e. if +# EXTRN_MDL_NAME_ICS is set to "FV3GFS"), this variable specifies the +# format of the model files to use when generating the ICs. # # FV3GFS_FILE_FMT_LBCS: # If using the FV3GFS model as the source of the LBCs (i.e. if @@ -560,81 +560,146 @@ FV3GFS_FILE_FMT_LBCS="nemsio" # #----------------------------------------------------------------------- # -# Base directories in which to search for external model files. -# -# EXTRN_MDL_SYSBASEDIR_ICS: -# Base directory on the local machine containing external model files for -# generating ICs on the native grid. The way the full path containing -# these files is constructed depends on the user-specified external model -# for ICs, i.e. EXTRN_MDL_NAME_ICS. -# -# EXTRN_MDL_SYSBASEDIR_LBCS: -# Same as EXTRN_MDL_SYSBASEDIR_ICS but for LBCs. -# -# Note that these must be defined as null strings here so that if they -# are specified by the user in the experiment configuration file, they -# remain set to those values, and if not, they get set to machine-dependent -# values. -# -#----------------------------------------------------------------------- -# -EXTRN_MDL_SYSBASEDIR_ICS="" -EXTRN_MDL_SYSBASEDIR_LBCS="" -# -#----------------------------------------------------------------------- -# -# User-staged external model directories and files. Definitions: -# -# USE_USER_STAGED_EXTRN_FILES: -# Flag that determines whether or not the workflow will look for the -# external model files needed for generating ICs and LBCs in user-specified -# directories. -# -# EXTRN_MDL_SOURCE_BASEDIR_ICS: -# Directory in which to look for external model files for generating ICs. -# If USE_USER_STAGED_EXTRN_FILES is set to "TRUE", the workflow looks in -# this directory (specifically, in a subdirectory under this directory -# named "YYYYMMDDHH" consisting of the starting date and cycle hour of -# the forecast, where YYYY is the 4-digit year, MM the 2-digit month, DD -# the 2-digit day of the month, and HH the 2-digit hour of the day) for -# the external model files specified by the array EXTRN_MDL_FILES_ICS -# (these files will be used to generate the ICs on the native FV3-LAM -# grid). This variable is not used if USE_USER_STAGED_EXTRN_FILES is -# set to "FALSE". +# Parameters that specify how to obtain the external model files needed +# for generating ICs and LBCs on the native grid. Definitions: +# +# EXTRN_MDL_DATA_SOURCES: +# Array specifying the sources from which to try to obtain external model +# files for generating ICs or LBCs (which happens in the GET_EXTRN_ICS_TN +# and GET_EXTRN_LBCS_TN tasks). These sources are queried starting with +# the first and ending after the first one from which the files are +# successfully obtained or when all the data sources are exhausted (in +# which case there will be no external model files from which to generate +# ICs and/or LBCs, and the experiment will fail). Valid values that the +# elements of this array can take on and the respective behaviors they +# elicit are: +# +# * "disk" +# Look for external model files in local directories. The sets of +# base directories in which the files are assumed to be located are +# given by EXTRN_MDL_BASEDIRS_ICS and EXTRN_MDL_BASEDIRS_LBCS for ICs +# and LBCs, respectively. Here, by "base" directory, we mean the cycle- +# independent portion of the full directory in which the files are +# located. The way the cycle-dependent portion of the full directory +# as well as the file names are constructed depends on the parameter +# EXTRN_MDL_DIR_FILE_LAYOUT described below. # -# EXTRN_MDL_FILES_ICS: -# Array containing the names of the files to search for in the directory -# specified by EXTRN_MDL_SOURCE_BASEDIR_ICS. This variable is not used -# if USE_USER_STAGED_EXTRN_FILES is set to "FALSE". -# -# EXTRN_MDL_SOURCE_BASEDIR_LBCS: -# Analogous to EXTRN_MDL_SOURCE_BASEDIR_ICS but for LBCs instead of ICs. -# -# EXTRN_MDL_FILES_LBCS: -# Analogous to EXTRN_MDL_FILES_ICS but for LBCs instead of ICs. -# -#----------------------------------------------------------------------- -# -USE_USER_STAGED_EXTRN_FILES="FALSE" -EXTRN_MDL_SOURCE_BASEDIR_ICS="/base/dir/containing/user/staged/extrn/mdl/files/for/ICs" -EXTRN_MDL_FILES_ICS=( "ICS_file1" "ICS_file2" "..." ) -EXTRN_MDL_SOURCE_BASEDIR_LBCS="/base/dir/containing/user/staged/extrn/mdl/files/for/LBCs" -EXTRN_MDL_FILES_LBCS=( "LBCS_file1" "LBCS_file2" "..." ) +# * "noaa_hpss" +# Fetch external model files from NOAA's HPSS. All information needed +# to obtain the files [e.g. the path on NOAA HPSS of the archive files +# (e.g. tar or zip) that contain the external model files, the paths +# within the archive files where the latter are located, the names of +# the files, etc) are set in internal scripts according to the external +# model being considered (as specified by EXTRN_MDL_NAME_ICS or +# EXTRN_MDL_NAME_LBCS). +# +# * "nomads" +# Fetch external model files from NOMADS (NOAA Operational Model +# Archive and Distribution System). As with the "nomads" case, all +# information needed to fetch the files are set in internal scripts +# according to the external model. +# +# EXTRN_MDL_BASEDIRS_ICS: +# Array containing the set of base directories to search for external +# model file(s) for generating ICs on the native grid. Note that: +# +# * This is only used when searching for files on disk, i.e. when +# considering the element "disk" in EXTRN_MDL_DATA_SOURCES. +# +# * All elements of this array will be searched until the first one is +# encountered that contains the external model files. The remaining +# elements of this array are then ignored. +# +# * The experiment generation scripts will append to this array a default +# value for the base directory (that consists of a system directory) +# if such a default value is available for the combination of external +# model for ICs and machine on which the experiment is running. This +# is an attempt to have at least one location on disk available to +# search in case the user has not specified EXTRN_MDL_BASEDIRS_ICS in +# the experiment configuration file. It also tries to provide a backup +# location in case the ones specified by the user do not contain the +# required files. +# +# * By "base" directory we mean the cycle date-independent portion of a +# directory that may contain external model file(s). Thus, the files +# are not located directly under one of these base directories but in +# a sub(sub...)directory defined by some relative path. The full path +# to the file(s) is obtained by appending this relative path to one of +# the elements of EXTRN_MDL_BASEDIRS_ICS. This relative path depends +# on the current cycle's date and time (year, month, day, hour, and +# possibly minutes) and possibly other parameters, and the way in which +# it is constructed (i.e. how the date and time are parsed to form the +# relative path) itself depends on the setting of the experiment +# parameter EXTRN_MDL_DIR_FILE_LAYOUT (see below). +# +# EXTRN_MDL_BASEDIRS_LBCS: +# Same as EXTRN_MDL_BASEDIRS_ICS but for LBCs. +# +# EXTRN_MDL_DIR_FILE_LAYOUT: +# Directory structure and file naming convention to use when trying to +# obtain external model files from a base directory on disk. Valid +# values for this variable and the respective behaviors they elicit are: +# +# * "native_to_extrn_mdl" +# This is the default value and causes the workflow scripts to assume +# that the external model files have not been renamed and the directory +# structure in which they are stored has not been altered from the way +# they were either in an archive file or on a system disk, i.e. that +# the files use their original names and are located in their original +# cycle-dependent relative subdirectories (under the base directories +# specified by EXTRN_MDL_BASEDIRS_ICS and EXTRN_MDL_BASEDIRS_LBCS). +# The relative paths of these subdirectories are always constructed +# from the components of the current cycle's date and time, but the +# exact way in which this is done will depend on the external model +# and machine (because each such combination has its own output +# directory structure as well as file naming convention). +# +# * "user_spec" +# This value causes the workflow scripts to assume that, regardless of +# the external model and machine, the files are located in a subdirectory +# (under one of the base directories in EXTRN_MDL_BASEDIRS_ICS or +# EXTRN_MDL_BASEDIRS_LBCS) named "YYYYMMDDHH", where YYYY is the 4-digit +# cycle year, MM the 2-digit month, DD the 2-digit day of the month, +# and HH the 2-digit hour of the day. This is a way of imposing a +# standard directory structure to the external model files regardless +# of the external model and machine. +# +# EXTRN_MDL_FNS_ICS: +# If EXTRN_MDL_DIR_FILE_LAYOUT is set to "user_spec", this array specifies +# the names of the files from which to generate ICs and surface fields. +# This is an array because some external models (or external model and +# file format combinations) store all the information needed to generate +# ICs in a single file while others split this information between multiple +# files, e.g. one file for atmostpheric variables and another for surface +# variables. This variable is not used if EXTRN_MDL_DIR_FILE_LAYOUT is +# not set to "user_spec". +# +# EXTRN_MDL_FNS_LBCS_PREFIX, EXTRN_MDL_FNS_LBCS_SUFFIX: +# If EXTRN_MDL_DIR_FILE_LAYOUT is set to "user_spec", the prefix and +# suffix to add each 3-digit output forecast hour (of the external model) +# to obtain the names of the external model files from which to generate +# LBCs. For example, if we need output files at hours 3 and 6 from the +# external model, then (assuming EXTRN_MDL_DIR_FILE_LAYOUT is set to +# "user_spec") the scripts will look for files named # -#----------------------------------------------------------------------- +# ${EXTRN_MDL_FNS_LBCS_PREFIX}003${EXTRN_MDL_FNS_LBCS_SUFFIX} # -# Set NOMADS online data associated parameters. Definitions: +# and # -# NOMADS: -# Flag controlling whether or not using NOMADS online data. +# ${EXTRN_MDL_FNS_LBCS_PREFIX}006${EXTRN_MDL_FNS_LBCS_SUFFIX} # -# NOMADS_file_type: -# Flag controlling the format of data. +# This variable is not used if EXTRN_MDL_DIR_FILE_LAYOUT is not set to +# "user_spec". # #----------------------------------------------------------------------- # -NOMADS="FALSE" -NOMADS_file_type="nemsio" +EXTRN_MDL_DATA_SOURCES=( "disk" "noaa_hpss" "nomads" ) +EXTRN_MDL_BASEDIRS_ICS=( "" ) +EXTRN_MDL_BASEDIRS_LBCS=( "" ) +EXTRN_MDL_DIR_FILE_LAYOUT="native_to_extrn_mdl" +EXTRN_MDL_FNS_ICS=( "" ) +EXTRN_MDL_FNS_LBCS_PREFIX="" +EXTRN_MDL_FNS_LBCS_SUFFIX="" # #----------------------------------------------------------------------- # @@ -1152,7 +1217,7 @@ VX_POINTSTAT_TN="run_pointstatvx" #----------------------------------------------------------------------- # # Set flags (and related directories) that determine whether various -# workflow tasks should be run. Note that the MAKE_GRID_TN, MAKE_OROG_TN, +# workflow tasks should be run. Note that MAKE_GRID_TN, MAKE_OROG_TN, # and MAKE_SFC_CLIMO_TN are all cycle-independent tasks, i.e. if they # are to be run, they do so only once at the beginning of the workflow # before any cycles are run. Definitions: @@ -1180,6 +1245,12 @@ VX_POINTSTAT_TN="run_pointstatvx" # SFC_CLIMO_DIR: # Same as GRID_DIR but for the MAKE_SFC_CLIMO_TN task. # +# RUN_TASK_GET_EXTRN_ICS: +# Flag that determines whether the GET_EXTRN_ICS_TN task is to be run. +# +# RUN_TASK_GET_EXTRN_LBCS: +# Flag that determines whether the GET_EXTRN_LBCS_TN task is to be run. +# # RUN_TASK_RUN_POST: # Flag that determines whether the RUN_POST_TN task is to be run. # @@ -1202,6 +1273,10 @@ OROG_DIR="/path/to/pregenerated/orog/files" RUN_TASK_MAKE_SFC_CLIMO="TRUE" SFC_CLIMO_DIR="/path/to/pregenerated/surface/climo/files" +RUN_TASK_GET_EXTRN_ICS="TRUE" + +RUN_TASK_GET_EXTRN_LBCS="TRUE" + RUN_TASK_RUN_POST="TRUE" RUN_TASK_GET_OBS_CCPA="FALSE" @@ -1245,15 +1320,16 @@ SFC_CLIMO_FIELDS=( \ # task (orog.x and shave.x). Can be the same as FIXgsm. # # SFC_CLIMO_INPUT_DIR: -# The location on disk of the static surface climatology input fields, used by -# sfc_climo_gen. These files are only used if RUN_TASK_MAKE_SFC_CLIMO=TRUE +# The location on disk of the static surface climatology input fields +# used by the sfc_climo_gen code (called by the MAKE_SFC_CLIMO_TN task). +# These files are only used if RUN_TASK_MAKE_SFC_CLIMO is set to "TRUE". # # FNGLAC, ..., FNMSKH: # Names of (some of the) global data files that are assumed to exist in -# a system directory specified (this directory is machine-dependent; -# the experiment generation scripts will set it and store it in the -# variable FIXgsm). These file names also appear directly in the forecast -# model's input namelist file. +# a system directory specified (this directory is machine-dependent; the +# experiment generation scripts will set it and store it in the variable +# FIXgsm). These file names also appear directly in the forecast model's +# input namelist file. # # FIXgsm_FILES_TO_COPY_TO_FIXam: # If not running in NCO mode, this array contains the names of the files @@ -1297,7 +1373,8 @@ SFC_CLIMO_FIELDS=( \ # # Because the default values are dependent on the platform, we set these # to a null string which will then be overwritten in setup.sh unless the -# user has specified a different value in config.sh +# user has specified a different value in config.sh. +# FIXgsm="" TOPO_DIR="" SFC_CLIMO_INPUT_DIR="" @@ -1563,9 +1640,9 @@ NUM_ENS_MEMBERS="1" # #----------------------------------------------------------------------- # -DO_SHUM="false" -DO_SPPT="false" -DO_SKEB="false" +DO_SHUM="FALSE" +DO_SPPT="FALSE" +DO_SKEB="FALSE" SHUM_MAG="0.006" #Variable "shum" in input.nml SHUM_LSCALE="150000" SHUM_TSCALE="21600" #Variable "shum_tau" in input.nml @@ -1579,7 +1656,7 @@ SKEB_LSCALE="150000" SKEB_TSCALE="21600" #Variable "skeb_tau" in input.nml SKEB_INT="3600" #Variable "skebint" in input.nml SKEB_VDOF="10" -USE_ZMTNBLCK="false" +USE_ZMTNBLCK="FALSE" # This doesn't seem to be used anywhere in the workflow. Remove?? # #----------------------------------------------------------------------- # @@ -1596,7 +1673,7 @@ USE_ZMTNBLCK="false" # #----------------------------------------------------------------------- # -DO_SPP="false" +DO_SPP="FALSE" SPP_VAR_LIST=( "pbl" ) SPP_MAG_LIST=( "0.2" ) #Variable "spp_prt_list" in input.nml SPP_LSCALE=( "150000.0" ) diff --git a/ush/create_diag_table_file.sh b/ush/create_diag_table_file.sh index 10ad396ce..f6981b619 100644 --- a/ush/create_diag_table_file.sh +++ b/ush/create_diag_table_file.sh @@ -15,7 +15,7 @@ function create_diag_table_file() { # #----------------------------------------------------------------------- # - { save_shell_opts; set -u -x; } > /dev/null 2>&1 + { save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/ush/create_model_configure_file.sh b/ush/create_model_configure_file.sh index 23602932b..26b20ef96 100644 --- a/ush/create_model_configure_file.sh +++ b/ush/create_model_configure_file.sh @@ -15,7 +15,7 @@ function create_model_configure_file() { # #----------------------------------------------------------------------- # - { save_shell_opts; set -u -x; } > /dev/null 2>&1 + { save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/ush/extrn_mdl/check_nomads_access.sh b/ush/extrn_mdl/check_nomads_access.sh new file mode 100644 index 000000000..133a3691a --- /dev/null +++ b/ush/extrn_mdl/check_nomads_access.sh @@ -0,0 +1,109 @@ +# +#----------------------------------------------------------------------- +# +# This file defines a function that checks (using the "ping" utility) +# whether the machine on which it is running has access to NOAA's NOMADS +# (NOAA Operational Model Archive and Distribution System) host. The +# input arguments are as follows: +# +# host: +# The host to check access to. If this is not specified, it gets set +# to "nomads.ncep.noaa.gov". +# +# num_pings: +# The number of times to ping to successfully ping to verify access to +# the host. If this is not specified, it gets set to "4". +# +# wait_time_secs: +# The wait time (in units of seconds) before quitting the ping utility. +# If this is not specified, it gets set to "30". +# +# The "ping" utility called below will consider access successful if it +# can ping the host successfully num_ping times within wait_time_secs +# seconds. Otherwise, it will exit and consider access unsuccessful. +# In the latter case, this function will return with a non-zero return +# code. +# +#----------------------------------------------------------------------- +# +function check_nomads_access() { +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# + { save_shell_opts; set -u +x; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Specify the set of valid argument names for this script/function. Then +# process the arguments provided to this script/function (which should +# consist of a set of name-value pairs of the form arg1="value1", etc). +# +#----------------------------------------------------------------------- +# + local valid_args=( \ + "host" \ + "num_pings" \ + "wait_time_secs" \ + ) + process_args valid_args "$@" +# +#----------------------------------------------------------------------- +# +# For debugging purposes, print out values of arguments passed to this +# script or function. Note that these will be printed out only if an +# environment variable named VERBOSE exists and is set to TRUE. +# +#----------------------------------------------------------------------- +# + print_input_args "valid_args" +# +#----------------------------------------------------------------------- +# +# Set input arguments to default values if they are not specified in the +# call to this function. +# +#----------------------------------------------------------------------- +# + host="${host:-nomads.ncep.noaa.gov}" + num_pings="${num_pings:-4}" + wait_time_secs="${wait_time_secs:-30}" +# +#----------------------------------------------------------------------- +# +# Try pinging the host. +# +#----------------------------------------------------------------------- +# + print_info_msg " +Attempting to ping host ${num_pings} times for at most ${wait_time_secs} seconds: + host = \"$host\" +..." + + ping -c "${num_pings}" -w ${wait_time_secs} "$host" || { \ + print_info_msg " +Unable to ping the host after ${wait_time_secs} seconds: + host = \"$host\" +Returning with a nonzero return code. +"; + return 1; + } + + print_info_msg " +The host is accessible: + host = \"$host\"" +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script or +# function. +# +#----------------------------------------------------------------------- +# + { restore_shell_opts; } > /dev/null 2>&1 + +} diff --git a/ush/extrn_mdl/create_extrn_mdl_var_defns_file.sh b/ush/extrn_mdl/create_extrn_mdl_var_defns_file.sh new file mode 100644 index 000000000..a64f251d9 --- /dev/null +++ b/ush/extrn_mdl/create_extrn_mdl_var_defns_file.sh @@ -0,0 +1,117 @@ +# +#----------------------------------------------------------------------- +# +# This file defines a function that creates a variable definitions file +# (in bash script syntax) and save in it the values of several external- +# model-associated variables that may be needed by downstream workflow +# tasks. +# +#----------------------------------------------------------------------- +# +function create_extrn_mdl_var_defns_file() { +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# + { save_shell_opts; set -u +x; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Specify the set of valid argument names for this script/function. Then +# process the arguments provided to this script/function (which should +# consist of a set of name-value pairs of the form arg1="value1", etc). +# +#----------------------------------------------------------------------- +# + local valid_args=( \ + "extrn_mdl_var_defns_fp" \ + "ics_or_lbcs" \ + "extrn_mdl_cdate" \ + "extrn_mdl_staging_dir" \ + "extrn_mdl_fns" \ + "extrn_mdl_lbc_spec_fhrs" + ) + process_args valid_args "$@" +# +#----------------------------------------------------------------------- +# +# For debugging purposes, print out values of arguments passed to this +# script or function. Note that these will be printed out only if an +# environment variable named VERBOSE exists and is set to TRUE. +# +#----------------------------------------------------------------------- +# + print_input_args "valid_args" +# +#----------------------------------------------------------------------- +# +# Declare local variables. +# +#----------------------------------------------------------------------- +# + local extrn_mdl_fns_str \ + extrn_mdl_lbc_spec_fhrs_str \ + settings +# +#----------------------------------------------------------------------- +# +# If the external model variable definitions file already exists, delete +# it. +# +#----------------------------------------------------------------------- +# + check_for_preexist_dir_file "${extrn_mdl_var_defns_fp}" "delete" +# +#----------------------------------------------------------------------- +# +# Save the contents of the file in the variable "settings". +# +#----------------------------------------------------------------------- +# + extrn_mdl_fns_str="( "$( printf "\"%s\" " "${extrn_mdl_fns[@]}" )")" + + settings="\ +EXTRN_MDL_CDATE=\"${extrn_mdl_cdate}\" +EXTRN_MDL_STAGING_DIR=\"${extrn_mdl_staging_dir}\" +EXTRN_MDL_FNS=${extrn_mdl_fns_str}" +# +# If the external model files obtained above were for generating LBCS (as +# opposed to ICs), then add to the external model variable definitions +# file the array variable EXTRN_MDL_LBC_SPEC_FHRS containing the forecast +# hours at which the lateral boundary conditions are specified. +# + if [ "${ics_or_lbcs}" = "LBCS" ]; then + extrn_mdl_lbc_spec_fhrs_str="( "$( printf "\"%s\" " "${extrn_mdl_lbc_spec_fhrs[@]}" )")" + settings="$settings +EXTRN_MDL_LBC_SPEC_FHRS=${lbc_spec_fhrs_str}" + fi +# +#----------------------------------------------------------------------- +# +# Write the contents of "settings" to file. +# +#----------------------------------------------------------------------- +# + { cat << EOM >> "${extrn_mdl_var_defns_fp}" +$settings +EOM + } || print_err_msg_exit "\ +Heredoc (cat) command to create a variable definitions file associated +with the external model from which to generate ${ics_or_lbcs} returned with a +nonzero status. The full path to this variable definitions file is: + extrn_mdl_var_defns_fp = \"${extrn_mdl_var_defns_fp}\"" +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script or +# function. +# +#----------------------------------------------------------------------- +# + { restore_shell_opts; } > /dev/null 2>&1 + +} diff --git a/ush/extrn_mdl/get_extrn_mdl_files_from_disk.sh b/ush/extrn_mdl/get_extrn_mdl_files_from_disk.sh new file mode 100644 index 000000000..941c984ae --- /dev/null +++ b/ush/extrn_mdl/get_extrn_mdl_files_from_disk.sh @@ -0,0 +1,441 @@ +# +#----------------------------------------------------------------------- +# +# This file defines a function that does one of the following: +# +# 1) If running in NCO mode (RUN_ENVIR set to "nco") or if using a user- +# specified directory and file layout for the external model files +# (EXTRN_MDL_DIR_FILE_LAYOUT set to "user_spec"), creates symlinks in +# the current cycle's external model file staging directory (staging_dir) +# to the specified set of external model files (fns) in a specified +# directory (extrn_mdl_dir). +# +# 2) If running in community mode (RUN_ENVIR set to "community") and not +# using a user-specified and file layout for the external model files +# (EXTRN_MDL_DIR_FILE_LAYOUT not set to "user_spec"), copies the +# external model files to the staging directory. +# +#----------------------------------------------------------------------- +# +function get_extrn_mdl_files_from_disk() { +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# + { save_shell_opts; set -u +x; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Specify the set of valid argument names for this script/function. Then +# process the arguments provided to this script/function (which should +# consist of a set of name-value pairs of the form arg1="value1", etc). +# +#----------------------------------------------------------------------- +# + local valid_args=( \ + "extrn_mdl_name" \ + "ics_or_lbcs" \ + "cdate" \ + "staging_dir" \ + "fns" \ + ) + process_args valid_args "$@" +# +#----------------------------------------------------------------------- +# +# For debugging purposes, print out values of arguments passed to this +# script. Note that these will be printed out only if VERBOSE is set to +# TRUE. +# +#----------------------------------------------------------------------- +# + print_input_args "valid_args" +# +#----------------------------------------------------------------------- +# +# Declare local variables. +# +#----------------------------------------------------------------------- +# + local basedir \ + basedir_next \ + basedirs \ + basedirs_str \ + extrn_mdl_dir \ + fn \ + fns_str \ + fp \ + fps \ + hh \ + i \ + j \ + jp1 \ + min_file_age \ + msg \ + num_basedirs \ + num_files \ + num_files_obtained \ + prefix \ + rc \ + rel_path \ + slash_atmos_or_null \ + yyyymmdd +# +#----------------------------------------------------------------------- +# +# Set the array basedirs containing the set of base directories that may +# contain the external model files. Here, by base directory, we mean the +# beginning cycle-independent portion of the full path to the directory +# containing the external model files. Below, we will try each base +# directory specified in this array until we find one that contains the +# files. +# +#----------------------------------------------------------------------- +# + if [ "${ics_or_lbcs}" = "ICS" ]; then + basedirs=( "${EXTRN_MDL_BASEDIRS_ICS[@]:-}" ) + elif [ "${ics_or_lbcs}" = "LBCS" ]; then + basedirs=( "${EXTRN_MDL_BASEDIRS_LBCS[@]:-}" ) + fi + + basedirs_str="( "$( printf "\"%s\" " "${basedirs[@]}" )")" + num_basedirs="${#basedirs[@]}" + if [ ${num_basedirs} -eq 0 ]; then + print_info_msg " +The array containing base directories (basedirs) in which to look for +external model files is empty: + basedirs = ${basedirs_str} +Returning with a nonzero return code. +" + return 1 + fi +# +#----------------------------------------------------------------------- +# +# Set the relative path (rel_path) in which to search for external model +# files. The relative path is the cycle-dependent portion of the full +# path to the directory containing the files. This relative path will +# be appended to each base directory in basedirs and the resulting full +# directory searched for the files until the files are found (or until +# we run out of base directories to try). +# +# There are two ways in which the relative path may be set. Which is +# used depends on the value of EXTRN_MDL_DIR_FILE_LAYOUT as follows: +# +# 1) If EXTRN_MDL_DIR_FILE_LAYOUT is set to "native_to_extrn_mdl", the +# assumed directory structure is the one that is native to the external +# model (in which there may be multiple levels of subdirectories under +# the base directory). +# +# 2) If EXTRN_MDL_DIR_FILE_LAYOUT is set to "user_spec", the assumed +# directory structure is such that the external model files are located +# directly under a subdirectory named after the cycle date in the +# format "YYYYMMDDHH". +# +# In the first case, the way rel_path is set is machine-dependent because +# we attempt here to use the same directory structure as the one used in +# the system directory on the current machine in which the files from a +# given external model are stored (if such a system directory exists at +# all on the machine). Since different machines may use different +# directory structures for the same model, the relative path is in +# general machine dependent. In addition, since some machines won't +# contain system directories for certain models, rel_path may remain set +# below to its default value of a null string. +# +#----------------------------------------------------------------------- +# + if [ "${EXTRN_MDL_DIR_FILE_LAYOUT}" = "native_to_extrn_mdl" ]; then + + parse_cdate \ + cdate="$cdate" \ + outvarname_yyyymmdd="yyyymmdd" \ + outvarname_hh="hh" + + if [ "${extrn_mdl_name}" = "FV3GFS" ]; then + slash_atmos_or_null="" + if [ "${cdate}" -ge "2021032100" ]; then + slash_atmos_or_null="/atmos" + fi + fi + + rel_path="" + case "$MACHINE" in + + "WCOSS_CRAY") + case "${extrn_mdl_name}" in + "FV3GFS") + rel_path="gfs.${yyyymmdd}/${hh}${slash_atmos_or_null}" + ;; + esac + ;; + + "WCOSS_DELL_P3") + case "${extrn_mdl_name}" in + "FV3GFS") + rel_path="gfs.${yyyymmdd}/${hh}${slash_atmos_or_null}" + ;; + esac + ;; + + "HERA") + case "${extrn_mdl_name}" in + "FV3GFS") + rel_path="gfs.${yyyymmdd}/${hh}${slash_atmos_or_null}" + ;; + esac + ;; + + "JET") + case "${extrn_mdl_name}" in + "RAP") + rel_path="${yyyymmdd}${hh}/postprd" + ;; + "HRRR") + rel_path="${yyyymmdd}${hh}/postprd" + ;; + esac + ;; + + "ODIN") + case "${extrn_mdl_name}" in + "FV3GFS") + rel_path="${yyyymmdd}" + ;; + esac + ;; + + "CHEYENNE") + case "${extrn_mdl_name}" in + "FV3GFS") + rel_path="gfs.${yyyymmdd}/${hh}${slash_atmos_or_null}" + ;; + esac + ;; + + esac + + elif [ "${EXTRN_MDL_DIR_FILE_LAYOUT}" = "user_spec" ]; then + + rel_path="$cdate" + + fi +# +#----------------------------------------------------------------------- +# +# In NCO mode, to ensure that the external model files are complete (i.e. +# not still being written to), we require that they be at least min_file_age +# minutes old. Set this value. +# +#----------------------------------------------------------------------- +# + min_file_age="5" +# +#----------------------------------------------------------------------- +# +# Loop through the base directories in basedirs. For each one, append +# the relative path to obtain a full path and try to obtain the external +# model files from the directory represented by that path. +# +#----------------------------------------------------------------------- +# +# Initialize the return code from this function to a non-zero value. +# This will be reset to zero only after all the files have been obtained +# successfully. +# + rc=1 + + for (( j=0; j<${num_basedirs}; j++ )); do + + basedir="${basedirs[$j]}" +# +# Define quantities that may be needed below in error messages. +# + basedir_next="" + jp1=$((j+1)) + if [ $jp1 -ne ${num_basedirs} ]; then + basedir_next="${basedirs[$jp1]}" + fi +# +# Check if the current base directory is actually a directory, e.g. +# whether it exists at all. +# + if [ ! -d "$basedir" ]; then + msg=" +The base directory (basedir) in which to look for external model files +does not exist or is not a directory: + basedir = \"$basedir\"" + if [ ! -z "${basedir_next}" ]; then + msg=$msg" +Skipping to next base directory (basedir_next) in basedirs: + basedirs = ${basedirs_str} + basedir_next = \"${basedir_next}\" +" + fi + print_info_msg "$msg" + continue + fi +# +# Append the relative path to the base directory to obtain the full path +# to the directory containing the external model files (for the current +# cycle). Then check that the resulting full directory exists. +# + extrn_mdl_dir="$basedir/${rel_path}" + if [ ! -d "${extrn_mdl_dir}" ]; then + msg=" +The directory (extrn_mdl_dir) that should contain the external model +files for the current cycle does not exist or is not a directory: + extrn_mdl_dir = \"${extrn_mdl_dir}\"" + if [ ! -z "${basedir_next}" ]; then + msg=$msg" +Skipping to next base directory (basedir_next) in basedirs: + basedirs = ${basedirs_str} + basedir_next = \"${basedir_next}\" +" + fi + print_info_msg "$msg" + continue + fi +# +# Set the array fps containing the full paths of all the external model +# files to be obtained. +# + prefix="${extrn_mdl_dir}/" + fps=( "${fns[@]/#/$prefix}" ) +# +# Loop through the list of external model files and either create a +# symlink in the staging directory to each (if running in NCO mode or if +# using a user-specified directory and file layout) or copy each to the +# current cycle's staging directory (if running in community mode and +# not using a user-specified directory and file layout). In the latter +# case, the files are usually in a system directory and are available +# for only a few days. By copying the files, we ensure that they will +# be available later, e.g. for rerunning the experiment. +# + fns_str="( "$( printf "\"%s\" " "${fns[@]}" )")" + if [ "${RUN_ENVIR}" = "nco" ] || \ + [ "${EXTRN_MDL_DIR_FILE_LAYOUT}" = "user_spec" ]; then + print_info_msg " +Attempting to create symlinks in the current cycle's staging directory +(staging_dir) to the specified files (fns) in the external model directory +(extrn_mdl_dir): + extrn_mdl_dir = \"${extrn_mdl_dir}\" + fns = ${fns_str} + staging_dir = \"${staging_dir}\" +" + elif [ "${EXTRN_MDL_DIR_FILE_LAYOUT}" = "native_to_extrn_mdl" ]; then + print_info_msg " +Attempting to copy the specified files (fns) from the external model +directory (extrn_mdl_dir) to the current cycle's staging directory +(staging_dir): + extrn_mdl_dir = \"${extrn_mdl_dir}\" + fns = ${fns_str} + staging_dir = \"${staging_dir}\" +" + fi +# +# Loop through the set of files and try to obtain (copy or link to) each +# one. +# + num_files_obtained=0 + num_files="${#fps[@]}" + for (( i=0; i<${num_files}; i++ )); do + + fn="${fns[$i]}" + fp="${fps[$i]}" + + if [ ! -f "$fp" ]; then + msg=" +The external model file (fp) is not a regular file, probably because it +does not exist: + fp = \"$fp\"" + if [ ! -z "${basedir_next}" ]; then + msg=$msg" +Skipping to next base directory (basedir_next) in basedirs: + basedirs = ${basedirs_str} + basedir_next = \"${basedir_next}\" +" + fi + print_info_msg "$msg" + break + fi +# +# If in NCO mode, check that the file found is at least min_file_age +# minutes old. +# + if [ "${RUN_ENVIR}" = "nco" ] && \ + [ ! $( find "$fp" -mmin +${min_file_age} ) ]; then + print_info_msg " +In NCO mode, the external model file (fp) must be older than a minimum +value [min_file_age (in minutes), where file age is taken as the time +elapsed since the last modification time] to ensure that the file is not +still being written to, but the current file (fp) is younger: + fp = \"$fp\" + min_file_age = ${min_file_age} minutes" + if [ ! -z "${basedir_next}" ]; then + msg=$msg" +Skipping to next base directory (basedir_next) in basedirs: + basedirs = ${basedirs_str} + basedir_next = \"${basedir_next}\" +" + fi + print_info_msg "$msg" + break + fi +# +# Link to or copy the current file. +# + if [ "${RUN_ENVIR}" = "nco" ] || \ + [ "${EXTRN_MDL_DIR_FILE_LAYOUT}" = "user_spec" ]; then + print_info_msg " +Linking to file fn: + fn = \"$fn\"" + create_symlink_to_file target="$fp" \ + symlink="${staging_dir}/$fn" \ + relative="FALSE" + else + print_info_msg " +Copying file fn: + fn = \"$fn\"" + cp_vrfy "$fp" "${staging_dir}/$fn" + fi +# +# Increment the counter that keeps track of the number of external model +# files that have been obtained (i.e. copied or linked to). +# + num_files_obtained=$(( num_files_obtained+1 )) + + done +# +# If, after exiting the loop over the files, the number of files obtained +# is equal to the total number of files, then all files were successfully +# obtained. In this case, reset the return code to 0 (in case it is +# needed later below) and exit the loop over the base directories. +# + if [ "${num_files_obtained}" -eq "${num_files}" ]; then + rc=0 + break + fi + + done +# +# If, after exiting the loops above, the return code is non-zero, it +# means the files were not obtained succesfully from any of the base +# directories. In this case, return with the non-zero code. +# + if [ "$rc" -ne "0" ]; then + return "$rc" + fi +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/function. +# +#----------------------------------------------------------------------- +# + { restore_shell_opts; } > /dev/null 2>&1 + +} diff --git a/ush/extrn_mdl/get_extrn_mdl_files_from_noaa_hpss.sh b/ush/extrn_mdl/get_extrn_mdl_files_from_noaa_hpss.sh new file mode 100644 index 000000000..57e3c133e --- /dev/null +++ b/ush/extrn_mdl/get_extrn_mdl_files_from_noaa_hpss.sh @@ -0,0 +1,508 @@ +# +#----------------------------------------------------------------------- +# +# This file defines a function that fetches external model files from +# NOAA's HPSS (High Performance Storage System) and places them in the +# current cycle's external model file staging directory (staging_dir). +# +#----------------------------------------------------------------------- +# +function get_extrn_mdl_files_from_noaa_hpss() { +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# + { save_shell_opts; set -u +x; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Specify the set of valid argument names for this script/function. Then +# process the arguments provided to this script/function (which should +# consist of a set of name-value pairs of the form arg1="value1", etc). +# +#----------------------------------------------------------------------- +# + local valid_args=( \ + "staging_dir" \ + "arcv_fmt" \ + "arcv_fns" \ + "arcv_fps" \ + "arcvrel_dir" \ + "fns" \ + ) + process_args valid_args "$@" +# +#----------------------------------------------------------------------- +# +# For debugging purposes, print out values of arguments passed to this +# script/function. Note that these will be printed out only if VERBOSE +# is set to TRUE. +# +#----------------------------------------------------------------------- +# + print_input_args "valid_args" +# +#----------------------------------------------------------------------- +# +# Declare local variables. +# +#----------------------------------------------------------------------- +# + local arcv_dir \ + arcv_fn \ + arcv_fp \ + arcv_fps_str \ + files_in_crnt_arcv \ + first_lbc_fhr \ + fp \ + fps \ + fps_str \ + hh \ + hh_orig \ + hsi_log_fn \ + htar_log_fn \ + i \ + last_fhr_in_nemsioa \ + last_fhr_in_netcdfa \ + last_lbc_fhr \ + narcv \ + narcv_formatted \ + nfile \ + num_arcv_files \ + num_files_in_crnt_arcv \ + num_files_to_extract \ + num_occurs \ + prefix \ + rel_dir \ + slash_atmos_or_null \ + subdir_to_remove \ + suffix \ + unzip_log_fn +# +#----------------------------------------------------------------------- +# +# Set the array fps to the full paths within the archive files of the +# external model files. +# +#----------------------------------------------------------------------- +# + prefix=${arcvrel_dir:+${arcvrel_dir}/} + fps=( "${fns[@]/#/$prefix}" ) + + fps_str="( "$( printf "\"%s\" " "${fps[@]}" )")" + arcv_fps_str="( "$( printf "\"%s\" " "${arcv_fps[@]}" )")" + + print_info_msg " +Fetching external model files from NOAA HPSS. The full paths (fps) to +these files within the archive file(s), the archive files on NOAA HPSS +in which these files are stored (arcv_fps), and the staging directory +(staging_dir) to which they will be copied are: + fps = ${fps_str} + arcv_fps = ${arcv_fps_str} + staging_dir = \"${staging_dir}\"" +# +#----------------------------------------------------------------------- +# +# Get the number of archive files to consider. +# +#----------------------------------------------------------------------- +# + num_arcv_files="${#arcv_fps[@]}" +# +#----------------------------------------------------------------------- +# +# Consider the case of the archive file to be fetched from NOAA HPSS +# being in tar format. +# +#----------------------------------------------------------------------- +# + if [ "${arcv_fmt}" = "tar" ]; then +# +#----------------------------------------------------------------------- +# +# Loop through the set of archive files specified in arcv_fps and extract +# a subset of the specified external model files from each. +# +#----------------------------------------------------------------------- +# + num_files_to_extract="${#fps[@]}" + + for (( narcv=0; narcv<${num_arcv_files}; narcv++ )); do + + narcv_formatted=$( printf "%02d" $narcv ) + arcv_fp="${arcv_fps[$narcv]}" +# +# Before trying to extract (a subset of) the external model files from +# the current tar archive file (which is on NOAA HPSS), create a list of +# those external model files that are stored in the current tar archive +# file. For this purpose, we first use the "htar -tvf" command to list +# all the external model files that are in the current archive file and +# store the result in a log file. (This command also indirectly checks +# whether the archive file exists on NOAA HPSS.) We then grep this log +# file for each external model file and create a list containing only +# those external model files that exist in the current archive. +# +# Note that the "htar -tvf" command will fail if the tar archive file +# itself doesn't exist on NOAA HPSS, but it won't fail if any of the +# external model file names passed to it don't exist in the archive file. +# In the latter case, the missing files' names simply won't appear in +# the log file. +# + htar_log_fn="log.htar_tvf.${narcv_formatted}" + htar -tvf "${arcv_fp}" ${fps[@]} >& "${htar_log_fn}" || { \ + print_info_msg " +htar file list operation (\"htar -tvf ...\") failed. Check the log file +htar_log_fn in the staging directory (staging_di) for details: + staging_dir = \"${staging_dir}\" + htar_log_fn = \"${htar_log_fn}\" +Returning with a nonzero return code. +"; + return 1; + } + + + i=0 + files_in_crnt_arcv=() + for (( nfile=0; nfile<${num_files_to_extract}; nfile++ )); do + fp="${fps[$nfile]}" + grep -n "$fp" "${htar_log_fn}" > /dev/null 2>&1 && { \ + files_in_crnt_arcv[$i]="$fp"; \ + i=$((i+1)); \ + } + done +# +# If none of the external model files were found in the current archive +# file, print out an error message and exit. +# + num_files_in_crnt_arcv=${#files_in_crnt_arcv[@]} + if [ ${num_files_in_crnt_arcv} -eq 0 ]; then + fps_str="( "$( printf "\"%s\" " "${fps[@]}" )")" + print_info_msg " +The current archive file (arcv_fp) does not contain any of the external +model files listed in fps: + arcv_fp = \"${arcv_fp}\" + fps = ${fps_str} +The archive file should contain at least one external model file; otherwise, +it would not be needed. Returning with a nonzero return code. +" + return 1 + fi +# +# Extract from the current tar archive file on NOAA HPSS all the external +# model files that exist in that archive file. Also, save the output of +# the "htar -xvf" command in a log file for debugging (if necessary). +# + htar_log_fn="log.htar_xvf.${narcv_formatted}" + htar -xvf "${arcv_fp}" ${files_in_crnt_arcv[@]} >& "${htar_log_fn}" || { \ + print_info_msg " +htar file extract operation (\"htar -xvf ...\") failed. Check the log +file htar_log_fn in the staging directory (staging_dir) for details: + staging_dir = \"${staging_dir}\" + htar_log_fn = \"${htar_log_fn}\" +Returning with a nonzero return code. +"; + return 1; + } +# +# Note that the htar file extract operation above may return with a 0 +# exit code (success) even if one or more (or all) external model files +# that it is supposed to contain were not extracted. The names of those +# files that were not extracted will not be listed in the log file. Thus, +# we now check whether the log file contains the name of each external +# model file that should have been extracted. If any are missing, we +# print out a message and exit the script because initial condition and +# surface field files needed by FV3 cannot be generated without all the +# external model files. +# + for fp in "${files_in_crnt_arcv[@]}"; do +# +# If the file path is absolute (i.e. starts with a "/"), then drop the +# leading "/" because htar strips it before writing the file path to the +# log file. +# + fp=${fp#/} + + grep -n "$fp" "${htar_log_fn}" > /dev/null 2>&1 || { \ + print_info_msg " +External model file fp not extracted from tar archive file arcv_fp: + arcv_fp = \"${arcv_fp}\" + fp = \"$fp\" +Check the log file htar_log_fn in the staging directory (staging_dir) +for details: + staging_dir = \"${staging_dir}\" + htar_log_fn = \"${htar_log_fn}\" +Returning with a nonzero return code. +"; + return 1; + } + + done + + done +# +#----------------------------------------------------------------------- +# +# For each external model file that was supposed to have been extracted +# from the set of specified archive files, loop through the extraction +# log files and check that it appears exactly once in one of the log files. +# If it doesn't appear at all, then it means that file was not extracted, +# and if it appears more than once, then something else is wrong. In +# either case, print out an error message and exit. +# +#----------------------------------------------------------------------- +# + for (( nfile=0; nfile<${num_files_to_extract}; nfile++ )); do + + fp="${fps[$nfile]}" +# +# If the file path is absolute (i.e. starts with a "/"), then drop the +# leading "/" because htar strips it before writing the file path to the +# log file. +# + fp=${fp#/} + + num_occurs=0 + for (( narcv=0; narcv<${num_arcv_files}; narcv++ )); do + narcv_formatted=$( printf "%02d" $narcv ) + htar_log_fn="log.htar_xvf.${narcv_formatted}" + grep -n "$fp" "${htar_log_fn}" > /dev/null 2>&1 && { \ + num_occurs=$((num_occurs+1)); \ + } + done + + if [ ${num_occurs} -eq 0 ]; then + + print_info_msg " +The current external model file (fp) does not appear in any of the archive +extraction log files: + fp = \"$fp\" +Thus, it was not extracted, likely because it doesn't exist in any of the +archive files. Returning with a nonzero return code. +" + return 1 + + elif [ ${num_occurs} -gt 1 ]; then + + print_info_msg " +The current external model file (fp) appears more than once in the archive +extraction log files: + fp = \"$fp\" +The number of times it occurs in the log files is: + num_occurs = ${num_occurs} +Thus, it was extracted from more than one archive file, with the last one +that was extracted overwriting all previous ones. This should normally +not happen. Returning with a nonzero return code. +" + return 1 + + fi + + done +# +#----------------------------------------------------------------------- +# +# If arcvrel_dir is not set to the current directory (i.e. it is not +# equal to "."), then the htar command will have created the subdirectory +# "./${arcvrel_dir}" under the current directory and placed the extracted +# files there. In that case, we move these extracted files back to the +# current directory and then remove the subdirectory created by htar. +# +#----------------------------------------------------------------------- +# + if [ "${arcvrel_dir}" != "." ]; then +# +# The code below works if arcvrel_dir starts with a "/" or a "./", which +# are the only cases encountered thus far. The code will have to be +# modified to accomodate other cases. +# + if [ "${arcvrel_dir:0:1}" = "/" ] || \ + [ "${arcvrel_dir:0:2}" = "./" ]; then +# +# Strip the "/" or "./" from the beginning of arcvrel_dir to obtain the +# relative directory from which to move the extracted files to the current +# directory. Then move the files. +# + rel_dir=$( printf "%s" "${arcvrel_dir}" | \ + sed -r 's%^(\/|\.\/)([^/]*)(.*)%\2\3%' ) + mv_vrfy ${rel_dir}/* . +# +# Get the first subdirectory in rel_dir, i.e. the subdirectory before the +# first forward slash. This is the subdirectory that we want to remove +# since it no longer contains any files (only subdirectories). Then +# remove it. +# + subdir_to_remove=$( printf "%s" "${rel_dir}" | \ + sed -r 's%^([^/]*)(.*)%\1%' ) + rm_vrfy -rf ./${subdir_to_remove} +# +# If arcvrel_dir does not start with a "/" (and it is not equal to "."), +# then print out an error message and exit. +# + else + + print_info_msg " +The archive-relative directory specified by arcvrel_dir [i.e. the directory +\"within\" the tar file(s) listed in arcv_fps] is not the current directory +(i.e. it is not \".\"), and it does not start with a \"/\" or a \"./\": + arcvrel_dir = \"${arcvrel_dir}\" + arcv_fps = ${arcv_fps_str} +This script must be modified to account for this case. Returning with a +nonzero return code. +" + return 1 + + fi + + fi +# +#----------------------------------------------------------------------- +# +# Consider the case of the archive file to be fetched from NOAA HPSS +# being in zip format. +# +#----------------------------------------------------------------------- +# + elif [ "${arcv_fmt}" = "zip" ]; then +# +#----------------------------------------------------------------------- +# +# For archive files that are in "zip" format files, the array arcv_fps +# containing the list of archive files should contain only one element, +# i.e. there should be only one archive file to consider. Check for this. +# If this ever changes (e.g. due to the way an external model that uses +# the "zip" format archives its output files on NOAA HPSS), the code +# below must be modified to loop over all archive files. +# +#----------------------------------------------------------------------- +# + if [ "${num_arcv_files}" -gt 1 ]; then + print_info_msg " +Currently, this function is coded to handle only one archive file if the +archive file format is specified to be \"zip\", but the number of archive +files (num_arcv_files) passed to this function is greater than 1: + arcv_fmt = \"${arcv_fmt}\" + num_arcv_files = ${num_arcv_files} +Please modify the function to handle more than one \"zip\" archive file. +Note that code already exists in this function that can handle multiple +archive files if the archive file format is specified to be \"tar\", so +that can be used as a guide for the \"zip\" case. Returning with a +nonzero return code. +" + return 1 + else + arcv_fn="${arcv_fns[0]}" + arcv_fp="${arcv_fps[0]}" + fi +# +#----------------------------------------------------------------------- +# +# Fetch the zip archive file from NOAA HPSS. +# +#----------------------------------------------------------------------- +# + hsi_log_fn="log.hsi_get" + hsi get "${arcv_fp}" >& "${hsi_log_fn}" || { \ + print_info_msg " +hsi file get operation (\"hsi get ...\") failed. Check the log file +hsi_log_fn in the staging directory (staging_dir) for details: + staging_dir = \"${staging_dir}\" + hsi_log_fn = \"${hsi_log_fn}\" +Returning with a nonzero return code. +"; + return 1; + } +# +#----------------------------------------------------------------------- +# +# List the contents of the zip archive file and save the result in a log +# file. +# +#----------------------------------------------------------------------- +# + unzip_log_fn="log.unzip_lv" + unzip -l -v ${arcv_fn} >& "${unzip_log_fn}" || { \ + print_info_msg " +unzip operation to list the contents of the zip archive file arcv_fn in +the staging directory (staging_dir) failed. Check the log file +unzip_log_fn in that directory for details: + arcv_fn = \"${arcv_fn}\" + staging_dir = \"${staging_dir}\" + unzip_log_fn = \"${unzip_log_fn}\" +Returning with a nonzero return code. +"; + return 1; + } +# +#----------------------------------------------------------------------- +# +# Check that the log file from the unzip command above contains the name +# of each external model file. If any are missing, then the corresponding +# files are not in the zip file and thus cannot be extracted. In that +# case, print out a message and exit the function because initial condition +# and surface field files for the FV3-LAM cannot be generated without all +# the external model files. +# +#----------------------------------------------------------------------- +# + for fp in "${fps[@]}"; do + grep -n "$fp" "${unzip_log_fn}" > /dev/null 2>&1 || { \ + print_info_msg " +External model file fp does not exist in the zip archive file arcv_fn in +the staging directory (staging_dir). Check the log file unzip_log_fn in +that directory for the contents of the zip archive: + staging_dir = \"${staging_dir}\" + arcv_fn = \"${arcv_fn}\" + fp = \"$fp\" + unzip_log_fn = \"${unzip_log_fn}\" +Returning with a nonzero return code. +"; + return 1; + } + done +# +#----------------------------------------------------------------------- +# +# Extract the external model files from the zip file on NOAA HPSS. Note +# that the -o flag to unzip is needed to overwrite existing files. +# Otherwise, unzip will wait for user input as to whether the existing +# files should be overwritten. +# +#----------------------------------------------------------------------- +# + unzip_log_fn="log.unzip" + unzip -o "${arcv_fn}" ${fps[@]} >& "${unzip_log_fn}" || { \ + print_info_msg " +unzip file extract operation (\"unzip -o ...\") failed. Check the log +file unzip_log_fn in the staging directory (staging_dir) for details: + staging_dir = \"${staging_dir}\" + unzip_log_fn = \"${unzip_log_fn}\" +Returning with a nonzero return code. +"; + return 1; + } +# +# NOTE: +# If arcvrel_dir is not empty, the unzip command above will create a +# subdirectory under staging_dir and place the external model files there. +# We have not encountered this for the RAP and HRRR models, but it may +# happen for other models in the future. In that case, extra code must +# be included here to move the external model files from the subdirectory +# up to staging_dir and then the subdirectory (analogous to what is done +# above for the case of arcv_fmt set to "tar". +# + + fi +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/function. +# +#----------------------------------------------------------------------- +# + { restore_shell_opts; } > /dev/null 2>&1 + +} diff --git a/ush/extrn_mdl/get_extrn_mdl_files_from_nomads.sh b/ush/extrn_mdl/get_extrn_mdl_files_from_nomads.sh new file mode 100644 index 000000000..17f15e091 --- /dev/null +++ b/ush/extrn_mdl/get_extrn_mdl_files_from_nomads.sh @@ -0,0 +1,234 @@ +# +#----------------------------------------------------------------------- +# +# This file defines a function that fetches external model files from +# NOMADS (NOAA Operational Model Archive and Distribution System) and +# places them in the current cycle's external model file staging directory +# (staging_dir). +# +#----------------------------------------------------------------------- +# +function get_extrn_mdl_files_from_nomads() { +# +#----------------------------------------------------------------------- +# +# Get the full path to the file in which this script/function is located +# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in +# which the file is located (scrfunc_dir). +# +#----------------------------------------------------------------------- +# + local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) + local scrfunc_fn=$( basename "${scrfunc_fp}" ) + local scrfunc_dir=$( dirname "${scrfunc_fp}" ) +# +#----------------------------------------------------------------------- +# +# Get the name of this function. +# +#----------------------------------------------------------------------- +# + local func_name="${FUNCNAME[0]}" +# +#----------------------------------------------------------------------- +# +# Set directories. +# +#----------------------------------------------------------------------- +# + local ushdir="${scrfunc_dir%/*}" +# +#----------------------------------------------------------------------- +# +# Source necessary files. +# +#----------------------------------------------------------------------- +# + . $ushdir/extrn_mdl/check_nomads_access.sh +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# + { save_shell_opts; set -u +x; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Specify the set of valid argument names for this script/function. Then +# process the arguments provided to this script/function (which should +# consist of a set of name-value pairs of the form arg1="value1", etc). +# +#----------------------------------------------------------------------- +# + local valid_args=( \ + "check_access" \ + "extrn_mdl_name" \ + "ics_or_lbcs" \ + "staging_dir" \ + "arcvrel_dir" \ + "fns" \ + ) + process_args valid_args "$@" +# +#----------------------------------------------------------------------- +# +# For debugging purposes, print out values of arguments passed to this +# script or function. Note that these will be printed out only if an +# environment variable named VERBOSE exists and is set to TRUE. +# +#----------------------------------------------------------------------- +# + print_input_args "valid_args" +# +#----------------------------------------------------------------------- +# +# Declare local variables. +# +#----------------------------------------------------------------------- +# + local basedir \ + file_url \ + file_urls \ + fv3gfs_file_fmt \ + host_nomads \ + i \ + nomads_base_url \ + num_files \ + prefix \ + wait_time_secs \ + wget_log_fn +# +#----------------------------------------------------------------------- +# +# Currently, this script can fetch only FV3GFS files from NOMADS. Check +# that the specified external model is FV3GFS. +# +#----------------------------------------------------------------------- +# + if [ "${extrn_mdl_name}" != "FV3GFS" ]; then + print_info_msg " +Files generated by the specified external model (extrn_mdl_name) are not +available on NOMADS: + extrn_mdl_name = \"${extrn_mdl_name}\" +Returning with a nonzero return code. +" + return 1 + fi +# +#----------------------------------------------------------------------- +# +# Set the format of the FV3GFS files. Currently, the only supported +# format is "grib2". +# +#----------------------------------------------------------------------- +# + if [ "${ics_or_lbcs}" = "ICS" ]; then + fv3gfs_file_fmt="${FV3GFS_FILE_FMT_ICS}" + elif [ "${ics_or_lbcs}" = "LBCS" ]; then + fv3gfs_file_fmt="${FV3GFS_FILE_FMT_LBCS}" + fi + + if [ "${fv3gfs_file_fmt}" != "grib2" ]; then + print_info_msg " +Fetching of FV3GFS files of the specified format (fv3gfs_file_fmt) is +currently not available on NOMADS: + fv3gfs_file_fmt = \"${fv3gfs_file_fmt}\" +Returning with a nonzero return code. +" + return 1 + fi +# +#----------------------------------------------------------------------- +# +# If the check_access flag is set to TRUE or is unspecified, check whether +# the NOMADS host is accessible from the current machine. +# +#----------------------------------------------------------------------- +# + check_access="${check_access:-TRUE}" + host_nomads="nomads.ncep.noaa.gov" + if [ "${check_access}" = "TRUE" ]; then + print_info_msg " +Checking access to NOMADS..." + check_nomads_access host="${host_nomads}" || { \ + print_info_msg "\ +NOMADS is not accessible from this machine (MACHINE): + MACHINE = \"$MACHINE\" +Returning with a nonzero return code. +"; + return 1; + } + print_info_msg " +NOMADS is accessible from this machine (MACHINE): + MACHINE = \"$MACHINE\"" + fi +# +#----------------------------------------------------------------------- +# +# Set the URLs to the external model files. +# +#----------------------------------------------------------------------- +# +# First, set the base URL for NOMADS. +# + basedir="/pub/data/nccf/com/gfs/prod" + nomads_base_url="https://${host_nomads}$basedir" +# +# Now append to the NOMADS base URL the relative path to the files +# specified by arcvrel_dir (which depends on the cycle date and time) to +# get the full URL where the files are located. +# + prefix="${nomads_base_url}/${arcvrel_dir}/" +# +# For clarity, replace any occurrences of the substring "/./" in prefix +# with a single "/". +# + prefix=$( printf "%s" "$prefix" | sed -r -e 's|/\./|/|g' ) +# +# Finally, set the array containing the full URLs to each of the external +# model files. +# + file_urls=( "${fns[@]/#/$prefix}" ) +# +#----------------------------------------------------------------------- +# +# Loop through the file URLs and use the wget utility to get each file. +# The files will be placed in the directory specified by staging_dir. +# +#----------------------------------------------------------------------- +# + num_files="${#file_urls[@]}" + for (( i=0; i<${num_files}; i++ )); do + + file_url="${file_urls[$i]}" + + wget_log_fn="log.wget.${fns[$i]}" + wget --continue \ + --directory-prefix="${staging_dir}" \ + --output-file="${staging_dir}/${wget_log_fn}" \ + "${file_url}" || { \ + print_info_msg " +Fetching of file (file_url) from NOMADS failed: + file_url = \"${file_url}\" +Please check the log file (wget_log_fp) for details: + wget_log_fp = \"${wget_log_fp}\" +Returning with a nonzero return code. +"; + return 1; + } + + done +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script or +# function. +# +#----------------------------------------------------------------------- +# + { restore_shell_opts; } > /dev/null 2>&1 + +} diff --git a/ush/extrn_mdl/set_extrn_mdl_arcv_file_dir_names.sh b/ush/extrn_mdl/set_extrn_mdl_arcv_file_dir_names.sh new file mode 100644 index 000000000..f7694fd0e --- /dev/null +++ b/ush/extrn_mdl/set_extrn_mdl_arcv_file_dir_names.sh @@ -0,0 +1,321 @@ +# +#----------------------------------------------------------------------- +# +# This file defines a function that, for a given external model +# (extrn_mdl_name), a type of condition (initial or boundary, specified +# by ics_or_lbcs), and a starting date and hour for the external model +# (cdate), sets and returns the following NOAA HPSS-associated quantities: +# +# 1) The format of the archive file(s) containing the external model +# files, e.g. tar or zip. +# 2) The name(s) of the archive file(s). +# 3) The full paths in NOAA HPSS of the archive file(s). +# 4) The relative path in the archive files in which the external model +# files are located. This is the relative path with respect to the +# directory in which the archive is being extracted in which the +# external model files are located. +# +#----------------------------------------------------------------------- +# +function set_extrn_mdl_arcv_file_dir_names() { +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# + { save_shell_opts; set -u +x; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Specify the set of valid argument names for this script/function. Then +# process the arguments provided to this script/function (which should +# consist of a set of name-value pairs of the form arg1="value1", etc). +# +#----------------------------------------------------------------------- +# + local valid_args=( \ + "extrn_mdl_name" \ + "ics_or_lbcs" \ + "cdate" \ + "lbc_spec_fhrs" \ + "outvarname_arcv_fmt" \ + "outvarname_arcv_fns" \ + "outvarname_arcv_fps" \ + "outvarname_arcvrel_dir" + ) + process_args valid_args "$@" +# +#----------------------------------------------------------------------- +# +# For debugging purposes, print out values of arguments passed to this +# script. Note that these will be printed out only if VERBOSE is set to +# TRUE. +# +#----------------------------------------------------------------------- +# + print_input_args "valid_args" +# +#----------------------------------------------------------------------- +# +# Declare local variables. +# +#----------------------------------------------------------------------- +# + local anl_or_fcst \ + arcv_dir \ + arcv_fmt \ + arcv_fn \ + arcv_fns \ + arcv_fp \ + arcv_fps \ + arcv_fps_str \ + arcvrel_dir \ + fv3gfs_file_fmt +# +#----------------------------------------------------------------------- +# +# Set local variables that depend on whether we are considering ICs or +# LBCs. +# +#----------------------------------------------------------------------- +# + if [ "${ics_or_lbcs}" = "ICS" ]; then + anl_or_fcst="ANL" + fv3gfs_file_fmt="${FV3GFS_FILE_FMT_ICS}" + elif [ "${ics_or_lbcs}" = "LBCS" ]; then + anl_or_fcst="FCST" + fv3gfs_file_fmt="${FV3GFS_FILE_FMT_LBCS}" + fi +# +#----------------------------------------------------------------------- +# +# Extract from cdate the starting year, month, day, and hour of the +# external model forecast as well as the date without time (yyyymmdd). +# +#----------------------------------------------------------------------- +# + parse_cdate \ + cdate="$cdate" \ + outvarname_yyyymmdd="yyyymmdd" \ + outvarname_yyyy="yyyy" \ + outvarname_mm="mm" \ + outvarname_dd="dd" \ + outvarname_hh="hh" +# +#----------------------------------------------------------------------- +# +# Set parameters associated with the mass store (NOAA HPSS) for the +# specified cycle date of the external model (cdate). These parameters +# consist of: +# +# 1) The type of the archive file (e.g. tar, zip, etc). +# 2) The name of the archive file. +# 3) The full path in HPSS to the archive file. +# 4) The relative directory in the archive file in which the external +# model's output files are located. +# +#----------------------------------------------------------------------- +# + case "${extrn_mdl_name}" in + + "GSMGFS") + arcv_dir="/NCEPPROD/hpssprod/runhistory/rh${yyyy}/${yyyy}${mm}/${yyyymmdd}" + arcv_fmt="tar" + arcv_fns="gpfs_hps_nco_ops_com_gfs_prod_gfs.${cdate}." + if [ "${anl_or_fcst}" = "ANL" ]; then + arcv_fns="${arcv_fns}anl" + arcvrel_dir="." + elif [ "${anl_or_fcst}" = "FCST" ]; then + arcv_fns="${arcv_fns}sigma" + arcvrel_dir="/gpfs/hps/nco/ops/com/gfs/prod/gfs.${yyyymmdd}" + fi + arcv_fns="${arcv_fns}.${arcv_fmt}" + arcv_fps="${arcv_dir}/${arcv_fns}" + ;; + + "FV3GFS") + + if [ "${cdate}" -lt "2019061200" ]; then + arcv_dir="/NCEPDEV/emc-global/5year/emc.glopara/WCOSS_C/Q2FY19/prfv3rt3/${cdate}" + arcv_fns="" + elif [ "${cdate}" -ge "2019061200" ] && \ + [ "${cdate}" -lt "2020022600" ]; then + arcv_dir="/NCEPPROD/hpssprod/runhistory/rh${yyyy}/${yyyy}${mm}/${yyyymmdd}" + arcv_fns="gpfs_dell1_nco_ops_com_gfs_prod_gfs.${yyyymmdd}_${hh}." + elif [ "${cdate}" -ge "2020022600" ]; then + arcv_dir="/NCEPPROD/hpssprod/runhistory/rh${yyyy}/${yyyy}${mm}/${yyyymmdd}" + arcv_fns="com_gfs_prod_gfs.${yyyymmdd}_${hh}." + fi + + if [ "${fv3gfs_file_fmt}" = "nemsio" ]; then + + if [ "${anl_or_fcst}" = "ANL" ]; then + arcv_fns="${arcv_fns}gfs_nemsioa" + elif [ "${anl_or_fcst}" = "FCST" ]; then + last_fhr_in_nemsioa="39" + first_lbc_fhr="${lbc_spec_fhrs[0]}" + last_lbc_fhr="${lbc_spec_fhrs[-1]}" + if [ "${last_lbc_fhr}" -le "${last_fhr_in_nemsioa}" ]; then + arcv_fns="${arcv_fns}gfs_nemsioa" + elif [ "${first_lbc_fhr}" -gt "${last_fhr_in_nemsioa}" ]; then + arcv_fns="${arcv_fns}gfs_nemsiob" + else + arcv_fns=( "${arcv_fns}gfs_nemsioa" "${arcv_fns}gfs_nemsiob" ) + fi + fi + + elif [ "${fv3gfs_file_fmt}" = "grib2" ]; then + + arcv_fns="${arcv_fns}gfs_pgrb2" + + elif [ "${fv3gfs_file_fmt}" = "netcdf" ]; then + + if [ "${anl_or_fcst}" = "ANL" ]; then + arcv_fns="${arcv_fns}gfs_nca" + elif [ "${anl_or_fcst}" = "FCST" ]; then + last_fhr_in_netcdfa="39" + first_lbc_fhr="${lbc_spec_fhrs[0]}" + last_lbc_fhr="${lbc_spec_fhrs[-1]}" + if [ "${last_lbc_fhr}" -le "${last_fhr_in_netcdfa}" ]; then + arcv_fns="${arcv_fns}gfs_nca" + elif [ "${first_lbc_fhr}" -gt "${last_fhr_in_netcdfa}" ]; then + arcv_fns="${arcv_fns}gfs_ncb" + else + arcv_fns=( "${arcv_fns}gfs_nca" "${arcv_fns}gfs_ncb" ) + fi + fi + + fi + + arcv_fmt="tar" + + slash_atmos_or_null="" + if [ "${cdate}" -ge "2021032100" ]; then + slash_atmos_or_null="/atmos" + fi + arcvrel_dir="./gfs.${yyyymmdd}/${hh}${slash_atmos_or_null}" + + if is_array "arcv_fns"; then + suffix=".${arcv_fmt}" + arcv_fns=( "${arcv_fns[@]/%/$suffix}" ) + prefix="${arcv_dir}/" + arcv_fps=( "${arcv_fns[@]/#/$prefix}" ) + else + arcv_fns="${arcv_fns}.${arcv_fmt}" + arcv_fps="${arcv_dir}/${arcv_fns}" + fi + ;; + + + "RAP") +# +# Note that this is GSL RAPX data, not operational NCEP RAP data. An +# option for the latter may be added in the future. +# +# The zip archive files for RAPX are named such that the forecast files +# for odd-numbered starting hours (e.g. 01, 03, ..., 21, 23) are stored +# together with the forecast files for the corresponding preceding even- +# numbered starting hours (e.g. 00, 02, ..., 20, 22, respectively), in +# an archive file whose name contains only the even-numbered hour. Thus, +# in forming the name of the archive file, if the starting hour (hh) is +# odd, we reduce it by one to get the corresponding even-numbered hour +# and use that to form the archive file name. +# + hh_orig=$hh +# if it starts with a 0 (e.g. 00, 01, ..., 09), bash will treat it as an +# octal number, and 08 and 09 are illegal ocatal numbers for which the +# arithmetic operations below will fail. + hh=$((10#$hh)) + if [ $(($hh%2)) = 1 ]; then + hh=$((hh-1)) + fi +# Now that the arithmetic is done, recast hh as a two-digit string because +# that is needed in constructing the names below. + hh=$( printf "%02d\n" $hh ) + + arcv_dir="/BMC/fdr/Permanent/${yyyy}/${mm}/${dd}/data/fsl/rap/full/wrfnat" + arcv_fmt="zip" + arcv_fns="${yyyy}${mm}${dd}${hh}00.${arcv_fmt}" + arcv_fps="${arcv_dir}/${arcv_fns}" + arcvrel_dir="" +# +# Reset hh to its original value in case it is used again later below. +# + hh=${hh_orig} + ;; + + "HRRR") +# +# Note that this is GSL HRRRX data, not operational NCEP HRRR data. An +# option for the latter may be added in the future. +# + arcv_dir="/BMC/fdr/Permanent/${yyyy}/${mm}/${dd}/data/fsl/hrrr/conus/wrfnat" + arcv_fmt="zip" + arcv_fns="${yyyy}${mm}${dd}${hh}00.${arcv_fmt}" + arcv_fps="${arcv_dir}/${arcv_fns}" + arcvrel_dir="" + ;; + + "NAM") + arcv_dir="/NCEPPROD/hpssprod/runhistory/rh${yyyy}/${yyyy}${mm}/${yyyymmdd}" + arcv_fmt="tar" + arcv_fns="com_nam_prod_nam.${yyyy}${mm}${dd}${hh}.bgrid.${arcv_fmt}" + arcv_fps="${arcv_dir}/${arcv_fns}" + arcvrel_dir="" + ;; + + *) + print_err_msg_exit "\ +Archive file information has not been specified for this external model: + extrn_mdl_name = \"${extrn_mdl_name}\"" + ;; + + esac +# +# Depending on the experiment configuration, the above code may set +# arcv_fns and arcv_fps to either scalars or arrays. If they are not +# arrays, recast them as arrays because that is what is expected in the +# code below. +# + is_array "arcv_fns" || arcv_fns=( "${arcv_fns}" ) + is_array "arcv_fps" || arcv_fps=( "${arcv_fps}" ) +# +#----------------------------------------------------------------------- +# +# Use the eval function to set the output variables. Note that each of +# these is set only if the corresponding input variable specifying the +# name to use for the output variable is not empty. +# +#----------------------------------------------------------------------- +# + if [ ! -z "${outvarname_arcv_fmt}" ]; then + eval ${outvarname_arcv_fmt}=${arcv_fmt} + fi + + if [ ! -z "${outvarname_arcv_fns}" ]; then + arcv_fns_str="( "$( printf "\"%s\" " "${arcv_fns[@]}" )")" + eval ${outvarname_arcv_fns}=${arcv_fns_str} + fi + + if [ ! -z "${outvarname_arcv_fps}" ]; then + arcv_fps_str="( "$( printf "\"%s\" " "${arcv_fps[@]}" )")" + eval ${outvarname_arcv_fps}=${arcv_fps_str} + fi + + if [ ! -z "${outvarname_arcvrel_dir}" ]; then + eval ${outvarname_arcvrel_dir}=${arcvrel_dir} + fi +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/function. +# +#----------------------------------------------------------------------- +# + { restore_shell_opts; } > /dev/null 2>&1 + +} + diff --git a/ush/extrn_mdl/set_extrn_mdl_default_basedir.sh b/ush/extrn_mdl/set_extrn_mdl_default_basedir.sh new file mode 100644 index 000000000..c6dcfddc7 --- /dev/null +++ b/ush/extrn_mdl/set_extrn_mdl_default_basedir.sh @@ -0,0 +1,221 @@ +# +#----------------------------------------------------------------------- +# +# This file define a function that, for a specified external model +# (extrn_mdl_name), sets and returns the default value of the base +# directory (default_basedir) containing the files generated by that +# model, if such a directory exists on the machine on which the experiment +# is generated (and for the experiment mode -- nco or community). If +# such a directory does not exist, the function returns a null string +# for the default base directory. +# +#----------------------------------------------------------------------- +# +function set_extrn_mdl_default_basedir() { +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# + { save_shell_opts; set -u +x; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Specify the set of valid argument names for this script/function. Then +# process the arguments provided to this script/function (which should +# consist of a set of name-value pairs of the form arg1="value1", etc). +# +#----------------------------------------------------------------------- +# + local valid_args=( \ + "extrn_mdl_name" \ + "outvarname_extrn_mdl_default_basedir" \ + ) + process_args valid_args "$@" +# +#----------------------------------------------------------------------- +# +# For debugging purposes, print out values of arguments passed to this +# script. Note that these will be printed out only if VERBOSE is set to +# TRUE. +# +#----------------------------------------------------------------------- +# + print_input_args "valid_args" +# +#----------------------------------------------------------------------- +# +# Declare local variables. +# +#----------------------------------------------------------------------- +# + local default_basedir +# +#----------------------------------------------------------------------- +# +# Set the default value of the base directory (default_basedir) in which +# the files generated by the external model specified by extrn_mdl_name +# are located. The way this is set depends on the experiment mode (NCO +# or community), the machine on which the experiment is being generated, +# and the external model. +# +# Note that default_basedir may be unspecified for certain combinations +# of experiment mode, machine, and external model. In such casea, it +# will remain set to its initial value of a null string. +# +#----------------------------------------------------------------------- +# + default_basedir="" + + if [ "${RUN_ENVIR}" = "nco" ]; then + + default_basedir="$COMINgfs" + + else + + case "$MACHINE" in + + "WCOSS_CRAY") + case "${extrn_mdl_name}" in + "FV3GFS") + default_basedir="/gpfs/dell1/nco/ops/com/gfs/prod" + ;; + "RAP") + default_basedir="/gpfs/hps/nco/ops/com/rap/prod" + ;; + "HRRR") + default_basedir="/gpfs/hps/nco/ops/com/hrrr/prod" + ;; + "NAM") + default_basedir="/gpfs/dell1/nco/ops/com/nam/prod" + ;; + esac + ;; + + "WCOSS_DELL_P3") + case "${extrn_mdl_name}" in + "FV3GFS") + default_basedir="/gpfs/dell1/nco/ops/com/gfs/prod" + ;; + "RAP") + default_basedir="/gpfs/hps/nco/ops/com/rap/prod" + ;; + "HRRR") + default_basedir="/gpfs/hps/nco/ops/com/hrrr/prod" + ;; + "NAM") + default_basedir="/gpfs/dell1/nco/ops/com/nam/prod" + ;; + esac + ;; + + "HERA") + case "${extrn_mdl_name}" in + "FV3GFS") + default_basedir="/scratch1/NCEPDEV/rstprod/com/gfs/prod" + ;; + "RAP") + default_basedir="/scratch2/BMC/public/data/gsd/rap/full/wrfnat" + ;; + "HRRR") + default_basedir="/scratch2/BMC/public/data/gsd/hrrr/conus/wrfnat" + ;; + esac + ;; + + "JET") + case "${extrn_mdl_name}" in + "FV3GFS") + default_basedir="/public/data/grids/gfs/nemsio" + ;; + "RAP") + default_basedir="/misc/whome/rtrr/rap" + ;; + "HRRR") + default_basedir="/misc/whome/rtrr/hrrr" + ;; + esac + ;; + + "ODIN") + case "${extrn_mdl_name}" in + "GSMGFS") + default_basedir="/scratch/ywang/EPIC/GDAS/2019053000_mem001" + ;; + "FV3GFS") + default_basedir="/scratch/ywang/test_runs/FV3_regional/gfs" + ;; + esac + ;; + + "STAMPEDE") + case "${extrn_mdl_name}" in + "GSMGFS") + default_basedir="/scratch/00315/tg455890/GDAS/20190530/2019053000_mem001" + ;; + "FV3GFS") + default_basedir="/scratch/00315/tg455890/GDAS/20190530/2019053000_mem001" + ;; + esac + ;; + + "CHEYENNE") + case "${extrn_mdl_name}" in + "GSMGFS") + default_basedir="/glade/p/ral/jntp/UFS_CAM/COMGFS" + ;; + "FV3GFS") + default_basedir="/glade/p/ral/jntp/UFS_CAM/COMGFS" + ;; + esac + ;; + + esac + + fi +# +#----------------------------------------------------------------------- +# +# If default_basedir was not set above to a non-empty string, the returned +# value (in the variable specified by outvarname_extrn_mdl_default_basedir) +# will be a null string. In this case, print out an informational message. +# +#----------------------------------------------------------------------- +# + if [ -z "${default_basedir}" ]; then + print_info_msg " +A default value for the base directory (default_basedir) is +not specified for this combination of experiment mode (RUN_ENVIR), machine +(MACHINE), and external model (extrn_mdl_name): + RUN_ENVIR = \"${RUN_ENVIR}\" + MACHINE = \"${MACHINE}\" + extrn_mdl_name = \"${extrn_mdl_name}\" + default_basedir = \"${default_basedir}\" +Setting the output variable \"${outvarname_extrn_mdl_default_basedir}\" to a null string. +" + fi +# +#----------------------------------------------------------------------- +# +# Use the eval function to set the output variables. Note that each of +# these is set only if the corresponding input variable specifying the +# name to use for the output variable is not empty. +# +#----------------------------------------------------------------------- +# + if [ ! -z "${outvarname_extrn_mdl_default_basedir}" ]; then + eval ${outvarname_extrn_mdl_default_basedir}="${default_basedir}" + fi +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/function. +# +#----------------------------------------------------------------------- +# + { restore_shell_opts; } > /dev/null 2>&1 + +} diff --git a/ush/extrn_mdl/set_extrn_mdl_filenames.sh b/ush/extrn_mdl/set_extrn_mdl_filenames.sh new file mode 100644 index 000000000..748609cc5 --- /dev/null +++ b/ush/extrn_mdl/set_extrn_mdl_filenames.sh @@ -0,0 +1,463 @@ +# +#----------------------------------------------------------------------- +# +# This file defines a function that sets and returns the set of external +# model file names that are to be obtained (copied from disk, linked to, +# fetched from a URL, etc). The way these are set depends on: +# +# 1) The data source (data_src) from which the files are to be obtained. +# +# 2) The directory structure and file naming convention assumed for the +# external model (EXTRN_MDL_DIR_FILE_LAYOUT). +# +# 3) The external model (extrn_mdl_name) for which to obtain the files. +# +# 4) Whether the files obtained will be used to generate initial or +# lateral boundary conditions for the FV3LAM (ics_or_lbcs). +# +# 5) The starting date and hour of the external model forecast (cdate) +# for which the files will be obtained. Note that this will usually +# correspond to the starting date and time of the FV3GFS forecast for +# which ICs and LBCs will be generated, but this is not always the +# case. For example, when using RAP model output to generate LBCs +# for the FV3LAM, the staring time of the RAP forecast is set back 6 +# hours relative to the FV3LAM forecast. +# +# Note that for simplicity, this function first sets the file names for +# all three of the following possibilities: +# +# 1) File names in an archive file located on NOAA HPSS or file names on +# NOMADS (i.e. data_src set to "noaa_hpss" or "nomads"; they use the +# same names). For these possibilities, the file names are stored in +# the local variable fns_in_arcv. +# +# 2) File names on disk (data_src set to "disk") using the same file +# naming convention as in the external model (EXTRN_MDL_DIR_FILE_LAYOUT +# set to "native_to_extrn_mdl"). For this case, the file names are +# stored in the local variable fns_on_disk_in_sysdir. +# +# 3) File names on disk (data_src set to "disk") using a user-specified +# file naming convention (EXTRN_MDL_DIR_FILE_LAYOUT set to "user_spec"). +# For this case, the file names are stored in the local variable +# fns_on_disk_user_spec. +# +# The function then sets the set of file names to be returned to one of +# these three local variables (fns_in_arcv, fns_on_disk_in_sysdir, or +# fns_on_disk_user_spec) depending on the values of data_src and +# EXTRN_MDL_DIR_FILE_LAYOUT. +# +#----------------------------------------------------------------------- +# +function set_extrn_mdl_filenames() { +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# + { save_shell_opts; set -u +x; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Specify the set of valid argument names for this script/function. Then +# process the arguments provided to this script/function (which should +# consist of a set of name-value pairs of the form arg1="value1", etc). +# +#----------------------------------------------------------------------- +# + local valid_args=( \ + "data_src" \ + "extrn_mdl_name" \ + "ics_or_lbcs" \ + "cdate" \ + "lbc_spec_fhrs" \ + "outvarname_fns" \ + ) + process_args valid_args "$@" +# +#----------------------------------------------------------------------- +# +# For debugging purposes, print out values of arguments passed to this +# script. Note that these will be printed out only if VERBOSE is set to +# TRUE. +# +#----------------------------------------------------------------------- +# + print_input_args "valid_args" +# +#----------------------------------------------------------------------- +# +# Declare local variables. +# +#----------------------------------------------------------------------- +# + local dd \ + ddd \ + fcst_hh \ + fcst_hhh \ + fcst_mn \ + fns \ + fns_in_arcv \ + fns_on_disk_in_sysdir \ + fns_on_disk_user_spec \ + fv3gfs_file_fmt \ + hh \ + mm \ + mn \ + prefix \ + str \ + suffix \ + yy \ + yyyy +# +#----------------------------------------------------------------------- +# +# Extract from cdate the starting year, month, day, hour, and minute of +# the external model forecast. [The minute (mn) will get set to "00" +# since cdate does not contain minutes.] +# +#----------------------------------------------------------------------- +# + parse_cdate \ + cdate="$cdate" \ + outvarname_yyyy="yyyy" \ + outvarname_mm="mm" \ + outvarname_dd="dd" \ + outvarname_hh="hh" \ + outvarname_mn="mn" +# +#----------------------------------------------------------------------- +# +# Set additional parameters needed in forming the names of the external +# model files only under certain circumstances. +# +#----------------------------------------------------------------------- +# + if [ "${extrn_mdl_name}" = "RAP" ] || \ + [ "${extrn_mdl_name}" = "HRRR" ] || \ + [ "${extrn_mdl_name}" = "NAM" ] || \ + [ "${extrn_mdl_name}" = "FV3GFS" -a "${MACHINE}" = "JET" ]; then +# +# Get the Julian day-of-year of the starting date and time of the exter- +# nal model forecast. +# + ddd=$( date --utc --date "${yyyy}-${mm}-${dd} ${hh}:${mn} UTC" "+%j" ) +# +# Get the last two digits of the year of the starting date and time of +# the external model forecast. +# + yy=${yyyy:2:4} + + fi +# +#----------------------------------------------------------------------- +# +# First, consider initial condition files. +# +#----------------------------------------------------------------------- +# + if [ "${ics_or_lbcs}" = "ICS" ]; then +# +# First, set the names of the files (fns_in_arcv) as they would be found +# in an archive file (e.g. one obtained from NOAA HPSS) and as they would +# be found on disk (fns_on_disk_in_sysdir) in the system directory where +# the last few days of output are stored. +# + fcst_hh="00" + fcst_mn="00" + + case "${extrn_mdl_name}" in + + "GSMGFS") +# fns=( "atm" "sfc" "nst" ) + fns=( "atm" "sfc" ) + prefix="gfs.t${hh}z." + fns=( "${fns[@]/#/$prefix}" ) + suffix="anl.nemsio" + fns_in_arcv=( "${fns[@]/%/$suffix}" ) + fns_on_disk_in_sysdir=( "${fns[@]/%/$suffix}" ) + ;; + + "FV3GFS") + + fv3gfs_file_fmt="${FV3GFS_FILE_FMT_ICS}" + + if [ "${fv3gfs_file_fmt}" = "nemsio" ]; then + + fns=( "atm" "sfc" ) + suffix="anl.nemsio" + fns=( "${fns[@]/%/$suffix}" ) + + prefix="gfs.t${hh}z." + fns_in_arcv=( "${fns[@]/#/$prefix}" ) + + if [ "${MACHINE}" = "JET" ]; then + prefix="${yy}${ddd}${hh}00.gfs.t${hh}z." + else + prefix="gfs.t${hh}z." + fi + fns_on_disk_in_sysdir=( "${fns[@]/#/$prefix}" ) + + elif [ "${fv3gfs_file_fmt}" = "grib2" ]; then + +# GSK 12/16/2019: +# Turns out that the .f000 file contains certain necessary fields that +# are not in the .anl file, so switch to the former. +# fns=( "gfs.t${hh}z.pgrb2.0p25.anl" ) + fns_in_arcv=( "gfs.t${hh}z.pgrb2.0p25.f000" ) + fns_on_disk_in_sysdir=( "gfs.t${hh}z.pgrb2.0p25.f000" ) + + elif [ "${fv3gfs_file_fmt}" = "netcdf" ]; then + + fns=( "atm" "sfc" ) + suffix="anl.nc" + fns=( "${fns[@]/%/$suffix}" ) + + prefix="gfs.t${hh}z." + fns_in_arcv=( "${fns[@]/#/$prefix}" ) + + if [ "${MACHINE}" = "JET" ]; then + prefix="${yy}${ddd}${hh}00.gfs.t${hh}z." + else + prefix="gfs.t${hh}z." + fi + fns_on_disk_in_sysdir=( "${fns[@]/#/$prefix}" ) + + fi + ;; + + "RAP") +# +# Note that this is GSL RAPX data, not operational NCEP RAP data. An +# option for the latter may be added in the future. +# + fns_in_arcv=( "${yy}${ddd}${hh}${mn}${fcst_hh}${fcst_mn}" ) + if [ "${MACHINE}" = "JET" ]; then + fns_on_disk_in_sysdir=( "wrfnat_130_${fcst_hh}.grib2" ) + else + fns_on_disk_in_sysdir=( "${yy}${ddd}${hh}${mn}${fcst_hh}${fcst_mn}" ) + fi + ;; + + "HRRR") +# +# Note that this is GSL HRRRX data, not operational NCEP HRRR data. An +# option for the latter may be added in the future. +# + fns_in_arcv=( "${yy}${ddd}${hh}${mn}${fcst_hh}${fcst_mn}" ) + if [ "${MACHINE}" = "JET" ]; then + fns_on_disk_in_sysdir=( "wrfnat_hrconus_${fcst_hh}.grib2" ) + else + fns_on_disk_in_sysdir=( "${yy}${ddd}${hh}${mn}${fcst_hh}${fcst_mn}" ) + fi + ;; + + "NAM") + fns=( "" ) + prefix="nam.t${hh}z.bgrdsfi${hh}" + fns=( "${fns[@]/#/$prefix}" ) + suffix=".tm${hh}" + fns_in_arcv=( "${fns[@]/%/$suffix}" ) + fns_on_disk_in_sysdir=( "${fns[@]/%/$suffix}" ) + ;; + + *) + print_err_msg_exit "\ +The external model file names (either on disk or in archive files) have +not yet been specified for this combination of external model (extrn_mdl_name) +and ICs or LBCs (ics_or_lbcs): + extrn_mdl_name = \"${extrn_mdl_name}\" + ics_or_lbcs = \"${ics_or_lbcs}\"" + ;; + + esac +# +# Now set the names of the files on disk assuming a user-specified file +# naming convention. +# + fns_on_disk_user_spec=( "${EXTRN_MDL_FNS_ICS[@]}" ) +# +#----------------------------------------------------------------------- +# +# Now consider lateral boundary condition files. +# +#----------------------------------------------------------------------- +# + elif [ "${ics_or_lbcs}" = "LBCS" ]; then +# +# First, set the names of the files (fns_in_arcv) as they would be found +# in an archive file (e.g. one obtained from NOAA HPSS) and as they would +# be found on disk (fns_on_disk_in_sysdir) in the system directory where +# the last few days of output are stored. +# + fcst_hh=( $( printf "%02d " "${lbc_spec_fhrs[@]}" ) ) + fcst_hhh=( $( printf "%03d " "${lbc_spec_fhrs[@]}" ) ) + fcst_mn="00" + + case "${extrn_mdl_name}" in + + "GSMGFS") + prefix="gfs.t${hh}z.atmf" + fns=( "${fcst_hhh[@]/#/$prefix}" ) + suffix=".nemsio" + fns_in_arcv=( "${fns[@]/%/$suffix}" ) + fns_on_disk_in_sysdir=( "${fns[@]/%/$suffix}" ) + ;; + + "FV3GFS") + + fv3gfs_file_fmt="${FV3GFS_FILE_FMT_LBCS}" + + if [ "${fv3gfs_file_fmt}" = "nemsio" ]; then + + suffix=".nemsio" + fns=( "${fcst_hhh[@]/%/$suffix}" ) + + prefix="gfs.t${hh}z.atmf" + fns_in_arcv=( "${fns[@]/#/$prefix}" ) + + if [ "${MACHINE}" = "JET" ]; then + prefix="${yy}${ddd}${hh}00.gfs.t${hh}z.atmf" + else + prefix="gfs.t${hh}z.atmf" + fi + fns_on_disk_in_sysdir=( "${fns[@]/#/$prefix}" ) + + elif [ "${fv3gfs_file_fmt}" = "grib2" ]; then + + prefix="gfs.t${hh}z.pgrb2.0p25.f" + fns_in_arcv=( "${fcst_hhh[@]/#/$prefix}" ) + fns_on_disk_in_sysdir=( "${fcst_hhh[@]/#/$prefix}" ) + + elif [ "${fv3gfs_file_fmt}" = "netcdf" ]; then + + suffix=".nc" + fns=( "${fcst_hhh[@]/%/$suffix}" ) + + prefix="gfs.t${hh}z.atmf" + fns_in_arcv=( "${fns[@]/#/$prefix}" ) + + if [ "${MACHINE}" = "JET" ]; then + prefix="${yy}${ddd}${hh}00.gfs.t${hh}z.atmf" + else + prefix="gfs.t${hh}z.atmf" + fi + fns_on_disk_in_sysdir=( "${fns[@]/#/$prefix}" ) + + fi + ;; + + "RAP") +# +# Note that this is GSL RAPX data, not operational NCEP RAP data. An +# option for the latter may be added in the future. +# + prefix="${yy}${ddd}${hh}${mn}" + fns_in_arcv=( "${fcst_hh[@]/#/$prefix}" ) + suffix="${fcst_mn}" + fns_in_arcv=( "${fns_in_arcv[@]/%/$suffix}" ) + + if [ "${MACHINE}" = "JET" ]; then + prefix="wrfnat_130_" + suffix=".grib2" + else + prefix="${yy}${ddd}${hh}${mn}" + suffix="${fcst_mn}" + fi + fns_on_disk_in_sysdir=( "${fcst_hh[@]/#/$prefix}" ) + fns_on_disk_in_sysdir=( "${fns_on_disk_in_sysdir[@]/%/$suffix}" ) + ;; + + "HRRR") +# +# Note that this is GSL HRRRX data, not operational NCEP HRRR data. An +# option for the latter may be added in the future. +# + prefix="${yy}${ddd}${hh}${mn}" + fns_in_arcv=( "${fcst_hh[@]/#/$prefix}" ) + suffix="${fcst_mn}" + fns_in_arcv=( "${fns_in_arcv[@]/%/$suffix}" ) + + if [ "${MACHINE}" = "JET" ]; then + prefix="wrfnat_hrconus_" + suffix=".grib2" + else + prefix="${yy}${ddd}${hh}${mn}" + suffix="${fcst_mn}" + fi + fns_on_disk_in_sysdir=( "${fcst_hh[@]/#/$prefix}" ) + fns_on_disk_in_sysdir=( "${fns_on_disk_in_sysdir[@]/%/$suffix}" ) + ;; + + "NAM") + prefix="nam.t${hh}z.bgrdsf" + fns=( "${fcst_hhh[@]/#/$prefix}" ) + suffix="" + fns_in_arcv=( "${fns[@]/%/$suffix}" ) + fns_on_disk_in_sysdir=( "${fns[@]/%/$suffix}" ) + ;; + + *) + print_err_msg_exit "\ +The external model file names (either on disk or in archive files) have +not yet been specified for this combination of external model (extrn_mdl_name) +and ICs or LBCs (ics_or_lbcs): + extrn_mdl_name = \"${extrn_mdl_name}\" + ics_or_lbcs = \"${ics_or_lbcs}\"" + ;; + + esac +# +# Now set the names of the files on disk assuming a user-specified file +# naming convention. +# + fcst_hhh=( $( printf "%03d " "${lbc_spec_fhrs[@]}" ) ) + fns_on_disk_user_spec=( "${fcst_hhh[@]/#/${EXTRN_MDL_FNS_LBCS_PREFIX}}" ) + fns_on_disk_user_spec=( "${fns_on_disk_user_spec[@]/%/${EXTRN_MDL_FNS_LBCS_SUFFIX}}" ) + + fi +# +#----------------------------------------------------------------------- +# +# Set the variable containing the external file names that will be +# returned to the calling function or script. This depends on the data +# source as well as the directory structure and file naming convention +# assumed. +# +#----------------------------------------------------------------------- +# + if [ "${data_src}" = "disk" ]; then + if [ "${EXTRN_MDL_DIR_FILE_LAYOUT}" = "native_to_extrn_mdl" ]; then + fns=( "${fns_on_disk_in_sysdir[@]}" ) + elif [ "${EXTRN_MDL_DIR_FILE_LAYOUT}" = "user_spec" ]; then + fns=( "${fns_on_disk_user_spec[@]}" ) + fi + elif [ "${data_src}" = "noaa_hpss" ] || \ + [ "${data_src}" = "nomads" ]; then + fns=( "${fns_in_arcv[@]}" ) + fi +# +#----------------------------------------------------------------------- +# +# Use the eval function to set the output variables. Note that each of +# these is set only if the corresponding input variable specifying the +# name to use for the output variable is not empty. +# +#----------------------------------------------------------------------- +# + if [ ! -z "${outvarname_fns}" ]; then + str="( "$( printf "\"%s\" " "${fns[@]}" )")" + eval ${outvarname_fns}=$str + fi +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/function. +# +#----------------------------------------------------------------------- +# + { restore_shell_opts; } > /dev/null 2>&1 + +} diff --git a/ush/generate_FV3LAM_wflow.sh b/ush/generate_FV3LAM_wflow.sh index 07aee4036..a2cec7c54 100755 --- a/ush/generate_FV3LAM_wflow.sh +++ b/ush/generate_FV3LAM_wflow.sh @@ -277,12 +277,13 @@ settings="\ 'maxtries_vx_gridstat_24h': ${MAXTRIES_VX_GRIDSTAT_24h} 'maxtries_vx_pointstat': ${MAXTRIES_VX_POINTSTAT} # -# Flags that specify whether to run the preprocessing or -# verification-related tasks. +# Flags that specify whether to run various tasks. # 'run_task_make_grid': ${RUN_TASK_MAKE_GRID} 'run_task_make_orog': ${RUN_TASK_MAKE_OROG} 'run_task_make_sfc_climo': ${RUN_TASK_MAKE_SFC_CLIMO} + 'run_task_get_extrn_ics': ${RUN_TASK_GET_EXTRN_ICS} + 'run_task_get_extrn_lbcs': ${RUN_TASK_GET_EXTRN_LBCS} 'run_task_run_post': ${RUN_TASK_RUN_POST} 'run_task_get_obs_ccpa': ${RUN_TASK_GET_OBS_CCPA} 'run_task_get_obs_mrms': ${RUN_TASK_GET_OBS_MRMS} @@ -578,9 +579,9 @@ Setting parameters in FV3 namelist file (FV3_NML_FP): FV3_NML_FP = \"${FV3_NML_FP}\"" # # Set npx and npy, which are just NX plus 1 and NY plus 1, respectively. -# These need to be set in the FV3-LAM Fortran namelist file. They represent -# the number of cell vertices in the x and y directions on the regional -# grid. +# These need to be set in the FV3-LAM Fortran namelist file. They +# represent the number of cell vertices in the x and y directions on the +# regional grid. # npx=$((NX+1)) npy=$((NY+1)) @@ -786,11 +787,9 @@ $settings" # configurations is not known until the grid is created. # if [ "${RUN_TASK_MAKE_GRID}" = "FALSE" ]; then - set_FV3nml_sfc_climo_filenames || print_err_msg_exit "\ Call to function to set surface climatology file names in the FV3 namelist file failed." - fi # #----------------------------------------------------------------------- @@ -891,15 +890,6 @@ edit the cron table): Done. " # -# If necessary, run the NOMADS script to source external model data. -# -if [ "${NOMADS}" = "TRUE" ]; then - echo "Getting NOMADS online data" - echo "NOMADS_file_type=" $NOMADS_file_type - cd $EXPTDIR - $USHDIR/NOMADS_get_extrn_mdl_files.sh $DATE_FIRST_CYCL $CYCL_HRS $NOMADS_file_type $FCST_LEN_HRS $LBC_SPEC_INTVL_HRS -fi -# #----------------------------------------------------------------------- # # Restore the shell options saved at the beginning of this script/func- diff --git a/ush/get_FV3GFS_grib2_files_from_NOMADS.sh b/ush/get_FV3GFS_grib2_files_from_NOMADS.sh new file mode 100755 index 000000000..59817691e --- /dev/null +++ b/ush/get_FV3GFS_grib2_files_from_NOMADS.sh @@ -0,0 +1,641 @@ +#!/bin/bash + +# +#----------------------------------------------------------------------- +# +# This script gets external model files from NOMADS. Type +# +# get_FV3GFS_grib2_files_from_NOMADS.sh --help +# +# for a full description of how to use this script. +# +#----------------------------------------------------------------------- +# + +# +#----------------------------------------------------------------------- +# +# Get the full path to the file in which this script or function is +# located (scrfunc_fp), the name of that file (scrfunc_fn), and the +# directory in which the file is located (scrfunc_dir). +# +#----------------------------------------------------------------------- +# +scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fn=$( basename "${scrfunc_fp}" ) +scrfunc_dir=$( dirname "${scrfunc_fp}" ) +# +#----------------------------------------------------------------------- +# +# Set the full path to the top-level directory of the regional_workflow +# repository. We denote this path by homerrfs. +# +#----------------------------------------------------------------------- +# +homerrfs=${scrfunc_dir%/*} +# +#----------------------------------------------------------------------- +# +# Set other directories that depend on homerrfs. +# +#----------------------------------------------------------------------- +# +ushdir="$homerrfs/ush" +# +#----------------------------------------------------------------------- +# +# Source bash utility functions. +# +#----------------------------------------------------------------------- +# +. $ushdir/source_util_funcs.sh +# +#----------------------------------------------------------------------- +# +# Source other needed files. +# +#----------------------------------------------------------------------- +# +. $ushdir/config_defaults.sh +. $ushdir/extrn_mdl/check_nomads_access.sh +. $ushdir/extrn_mdl/create_extrn_mdl_var_defns_file.sh +. $ushdir/extrn_mdl/get_extrn_mdl_files_from_nomads.sh +. $ushdir/extrn_mdl/set_extrn_mdl_filenames.sh +. $ushdir/extrn_mdl/set_extrn_mdl_arcv_file_dir_names.sh +. $ushdir/valid_param_vals.sh +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script or function. +# +#----------------------------------------------------------------------- +# +{ save_shell_opts; set -u +x; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Set the usage message. +# +#----------------------------------------------------------------------- +# +usage_str="\ +Usage: + + ${scrfunc_fn} \\ + machine=... \\ + all_cdates=... \\ + [file_types=...] \\ + lbc_spec_fhrs=... \\ + [data_basedir=...] \\ + [data_relsubdir=...] \\ + [preexisting_dir_method=...] \\ + [verbose=...] + +The arguments in brackets are optional. Examples: + +1) On Cheyenne, to fetch from NOMADS the external model analysis file(s) + and the forecast files for hours 1, 2, and 5 of the external model + cycle starting at 2021081500, use + + ./${scrfunc_fn} \\ + machine=\"cheyenne\" \\ + all_cdates=\"2021081500\" \\ + lbc_spec_fhrs='( \"1\" \"2\" \"5\" )' + + This will place the files needed for generating both ICs and LBCs in + the subdirectory \"nomads_data/2021081500\" under the directory in + which this script is located. + +2) On Cheyenne, to fetch just the analysis files for the cycles starting + at 2021081500 and 2021081600 and place them in the subdirectories + \"./my_data/2021081500/analysis\" and \"./my_data/2021081600/analysis\", + respectively, under the directory in which this script is located, + use + + ./${scrfunc_fn} \\ + machine=\"cheyenne\" \\ + all_cdates='( \"2021081500\" \"2021081600\" )' \\ + file_types=\"ANL\" \\ + data_basedir=\"./my_data\" \\ + data_relsubir=\"analysis\" + +The arguments are defined as follows: + +machine: +Machine (platform) on which the script is running. + +all_cdates: +Array containing a set of cycle starting times (i.e. dates and hours) +for which to fetch files. Note that: +* The script will try to fetch from NOMADS the external model files + for each of the cycle times in all_cdates. If fetching fails for a + given cycle and file type combination (see description of file_types + below), the script continues to try the next such combination. +* Each element of all_cdates must have the form \"YYYYMMDDHH\", where + YYYYY is the four-digit year, MM is the two-digit month, DD is the + two-digit day, and HH is the two-digit hour of the cycle's starting + date and time. +* Normally, in the call to this script, all_cdates should be specified + as an array as follows: + all_cdates='( \"cdate1\" \"cdate2\" ... )' + However, if the file(s) for only one cycle time are to be fetched, + it may be specified as a scalar, i.e. + all_cdates=\"cdate1\" +* By default, for each cycle time, both the analysis files [needed to + generate initial conditions (ICs) for the FV3LAM] and the forecast + files [needed to generate lateral boundary conditions (LBCs)] are + fetched, but that can be changed via the argument file_types (see + below). + +file_types: +Array specifying which types of files (analysis and/or forecast) to +fetch for each cycle time. Note that: +* file_types may have at most two elements. Each element may be either + \"ANL\" or \"FCST\". If it contains only one element, then if that + element is set to \"ANL\", only the analysis files will be fetched + for each cycle, and if it is set to \"FCST\", only the forecast files + will be fetched. If it is set to the array + ( \"ANL\" \"FCST\" ) + or to the array + ( \"FCST\" \"ANL\" ) + then both analysis and forecast files will be fetched. +* This is an optional argument. If it is not specified in the call to + this script, it will get set to the array ( \"ANL\" \"FCST\" ), i.e. + both analysis and forecast files will be fetched. +* Normally, in the call to this script, file_types should be specified + as an array line, e.g. + file_types='( \"ANL\" )' + or + file_types='( \"FCST\" \"ANL\" )' + However, if only one type of file is to be fetched, it may be specified + as a scalar, e.g. + file_types=\"FCST\" + +lbc_spec_fhrs: +Array containing the forecat hours for which to fetch external model +forecast files. Note that: +* lbc_spec_fhrs must be specified if file_types contains the element + \"FCST\" or if file_types is not specified in the call to this script + (because in that case, file_types will get set to a default value that + includes \"FCST\"). +* lbc_spec_fhrs does not need to be specified if file_types is set in + the call to this script to a value that does not contain the element + \"FCST\", i.e. if file_types is set as + file_types='( \"ANL\" )' + or + file_types=\"ANL\" +* Normally, in the call to this script, lbc_spec_fhrs should be specified + as an array, e.g. + lbc_spec_fhrs='( \"1\" \"2\" ... )' + However, if the forecast file(s) are to be fetched for only a single + hour, it may be specified as a scalar, e.g. + lbc_spec_fhrs=\"2\" + +data_basedir: +The base directory under which the external model files will be stored. +Note that: +* This is an optional argument. If it is not specified in the call to + this script (or if it is set to a null string), it will get set to + \"nomads_data\". +* data_basedir may be set to an absolute or a relative directory. If + relative, it is with respect to the directory in which this script is + located. +* A subdirectory with a relative path of + \$cdate/\${data_relbasedir}\" + will be created under this base directory for each cycle time (cdate) + specified in all_cdates, and the external model files for the cycle + will be placed in this subdirectory. Here, cdate is the starting date + and time of the cycle (in the form \"YYYYMMDDHH\" described above), + and data_relsubdir is a cycle-independent relative path specified as + an argument to this script. Thus, the full path to the external model + files for the cycle will be + \${data_basedir}/\$cdate/\${data_relbasedir}\" + +data_relsubdir: +The relative path to append to each cycle date and time to obtain the +relative directory (with respect to data_basedir) in which to place the +external model files. Note that: +* The full path to the external model files for a given cycle date and + time of the form \"YYYYMMDDHH\" (cdate) is given by + \${data_basedir}/\$cdate/\${data_relbasedir}\" +* If data_relsubdir is not specified in the call to this script (or if + it is set to a null string), it will get set to an empty string. + +preexisting_dir_method: +Method to use to deal with preexisting data directories. For a given +cycle, the absolute path to the directory in which the external files +will be saved is + \${data_basedir}/\$cdate/\${data_relbasedir}\" +Note that: +* If preexisting_dir_method is not specified in the call to this script + (or if it set to a null string), it will get set to \"none\". +* Valid (and non-empty) values for preexisting_dir_method are: + $( printf "\"%s\" " "${valid_vals_PREEXISTING_DIR_METHOD[@]}" ) + The behavior each of these elicits is: + * \"delete\": + Delete the preexisting directory. + * \"rename\": + Rename the preexisting directory by appending to its name the string + \"_oldNNN\", where NNN is a 3-digit integer, e.g. \"_old003\". + * \"quit\": + Exit the script. + * \"none\": + Do nothing. + +verbose: +Flag that turns on/off extra informational messages. Currently, this is +only used to set the variable VERBOSE that other functions called from +this script look for in the environment. Valid values are \"TRUE\" and +\"FALSE\". If this is not set in the call to this script, it gets set +to \"FALSE\"." || \ +print_err_msg_exit "\ +Something went wrong while setting the variable \"usage_str\"." +# +#----------------------------------------------------------------------- +# +# If this script is being called without arguments, print out the usage +# message and exit with a nonzero code (failure). If it is being called +# with the appropriate help flag, print out the usage message and exit +# with a 0 code (success). +# +#----------------------------------------------------------------------- +# +help_flag="--help" +how_to_see_usage_str="\ +Type + ./${scrfunc_fn} ${help_flag} +to get help on usage." +if [ "$#" -eq "0" ]; then + print_info_msg "${how_to_see_usage_str}" + exit 1 +fi + +if [ "$#" -eq "1" -a "$1" = "${help_flag}" ]; then + print_info_msg "${usage_str}" + exit 0 +fi +# +#----------------------------------------------------------------------- +# +# Specify the set of valid argument names for this script or function. +# Then process the arguments provided to this script or function (which +# should consist of a set of name-value pairs of the form arg1="value1", +# etc). +# +#----------------------------------------------------------------------- +# +valid_args=( \ + "machine" \ + "all_cdates" \ + "file_types" \ + "lbc_spec_fhrs" \ + "data_basedir" \ + "data_relsubdir" \ + "preexisting_dir_method" \ + "verbose" \ + ) +process_args valid_args "$@" +# +#----------------------------------------------------------------------- +# +# For debugging purposes, print out values of arguments passed to this +# script or function. Note that these will be printed out only if VERBOSE +# is set to TRUE. +# +#----------------------------------------------------------------------- +# +print_input_args "valid_args" +# +#----------------------------------------------------------------------- +# +# Set string to print out in case of error to direct users to help. +# +#----------------------------------------------------------------------- +# +how_to_see_usage_str="\ +Type + ./${scrfunc_fn} ${help_flag} +to get help on usage." +# +#----------------------------------------------------------------------- +# +# Process input arguments. +# +#----------------------------------------------------------------------- +# +# If verbose is not set (or if is set to a null string), reset it to its +# default value ("FALSE"). Then ensure that it is set to a valid value. +# Finally, set the variable VERBOSE (that many of the functions called +# by this script look for in the environment) to verbose. +# +verbose="${verbose^^}" +verbose=${verbose:-"FALSE"} +check_var_valid_value "verbose" "valid_vals_BOOLEAN" +VERBOSE="$verbose" +# +# Certain arguments that are assumed to be arrays in the code below are +# (for convenience) allowed to be specified as scalars in the call to +# this script if they consist of a single element. If necessary, convert +# any such arguments to arrays. +# +is_array "all_cdates" || all_cdates=( "${all_cdates}" ) +is_array "file_types" || file_types=( "${file_types}" ) +is_array "lbc_spec_fhrs" || lbc_spec_fhrs=( "${lbc_spec_fhrs}" ) +# +# Ensure that the specified machine is valid. +# +MACHINE="${machine^^}" +check_var_valid_value "MACHINE" "valid_vals_MACHINE" +# +# Ensure that all the cycle times in all_cdates consist of only digits +# and are exactly 10 characters in length. +# +all_cdates_str="( "$( printf "\"%s\" " "${all_cdates[@]}" )")" +num_cdates="${#all_cdates[@]}" +for (( i=0; i<${num_cdates}; i++ )); do + + cdate="${all_cdates[$i]}" + + if ! [[ "${cdate}" =~ ^[0-9]+$ ]]; then + print_err_msg_exit "\ +Each cycle time specified in the array all_cdates must consist of only +digits (of the form \"YYYYMMDDHH\"), but the element with index $i does +not: + all_cdates = ${all_cdates_str} + all_cdates[$i] = \"$cdate\" +${how_to_see_usage_str}" + fi + + nchars_cdate=${#cdate} + if [ "${nchars_cdate}" -ne "10" ]; then + print_err_msg_exit "\ +Each cycle time specified in the array all_cdates must consist of exactly +10 digits (of the form \"YYYYMMDDHH\"), but the element with index $i does +not: + all_cdates = ${all_cdates_str} + all_cdates[$i] = \"$cdate\" +${how_to_see_usage_str}" + fi + +done +# +# If file_types is not set (or if its first element is set to an empty +# string), set it to its default value. This default value will cause +# both analysis and forecast files to be fetched for each cycle. +# +if [ -z "${file_types}" ]; then + file_types=( "ANL" "FCST" ) +fi +# +# Ensure that file_types has at most two elements and that all are set +# to distinct, valid values. +# +valid_vals_file_types=( "ANL" "FCST" ) +file_types_str="( "$( printf "\"%s\" " "${file_types[@]}" )")" +valid_vals_file_types_str=$( printf "\"%s\" " "${valid_vals_file_types[@]}" ) + +num_file_types="${#file_types[@]}" +for (( j=0; j<${num_file_types}; j++ )); do + check_var_valid_value "file_types[$j]" "valid_vals_file_types" +done + +if [ "${num_file_types}" -gt "2" ]; then + print_err_msg_exit "\ +At most two file types may be specified in the argument file_types but +there are ${num_file_types}: + file_types = ${file_types_str} +Valid values that the elements of file_types may take on are: + ${valid_vals_file_types_str} +${how_to_see_usage_str}" +fi + +if [ "${num_file_types}" -eq "2" ] && \ + [ "${file_types[0]}" = "${file_types[1]}" ]; then + print_err_msg_exit "\ +The two elements of file_types must be distinct but aren't: + file_types = ${file_types_str} +${how_to_see_usage_str}" +fi +# +# If data_basedir is not set (or if it is set to a null string), reset +# it to its default value. +# +data_basedir=${data_basedir:-"nomads_data"} +# +# If preexisting_dir_method is not set (or if is set to a null string), +# reset it to its default value. Then ensure that it is set to a valid +# value. +# +preexisting_dir_method=${preexisting_dir_method:-"none"} +check_var_valid_value "preexisting_dir_method" "valid_vals_PREEXISTING_DIR_METHOD" +# +# If forecast files are to be fetched, make sure that lbc_spec_fhrs is +# specified. +# +if is_element_of "file_types" "FCST"; then + if [ -z "${lbc_spec_fhrs}" ]; then + print_err_msg_exit "\ +The set of forecast hours (lbc_spec_fhrs) for which to fetch forecast +files cannot be empty when the array argument file_types contains the +element \"FCST\" or when file_types is not specified in the call to +this script (in which case both analysis and forecast files will be +fetched): + file_types = ${file_types_str} +${how_to_see_usage_str}" + fi +fi +# +# If lbc_spec_fhrs is specified in the call to this script (to a non- +# empty value), make sure that each of its elements consists of only +# digits. +# +lbc_spec_fhrs_str="( "$( printf "\"%s\" " "${lbc_spec_fhrs[@]}" )")" +num_lbc_spec_fhrs="${#lbc_spec_fhrs[@]}" +if [ ! -z "${lbc_spec_fhrs}" ]; then + for (( i=0; i<${num_lbc_spec_fhrs}; i++ )); do + lbc_spec_fhr="${lbc_spec_fhrs[$i]}" + if ! [[ "${lbc_spec_fhr}" =~ ^[0-9]+$ ]]; then + print_err_msg_exit "\ +Each forecast hour in lbc_spec_fhrs must consist of only digits, but the +element with index $i contains non-digit characters: + lbc_spec_fhrs = ${lbc_spec_fhrs_str} + lbc_spec_fhrs[$i] = \"${lbc_spec_fhr}\" +${how_to_see_usage_str}" + fi + done +fi +# +#----------------------------------------------------------------------- +# +# Set variables that fall into one of the following categories: +# +# 1) Are needed as input arguments in the functions called below. These +# are usually lower-case. +# +# 2) The functions called below assume exist in the environment (they +# would normally be available via the experiment's variable definitions +# file). These variables are usually upper-case. +# +#----------------------------------------------------------------------- +# +# This script considers only NOMADS as a data source. +# +data_src="nomads" +# +# As of 20210819, this script and the functions called below support +# fetching of only FV3GFS files of grib2 format (from NOMADS). Set the +# following variables accordingly. This may be changed in the future, +# in which case the variables below will have to be converted to arguments. +# +extrn_mdl_name="FV3GFS" +FV3GFS_FILE_FMT_ICS="grib2" +FV3GFS_FILE_FMT_LBCS="grib2" +# +#----------------------------------------------------------------------- +# +# Check whether the NOMADS host is accessible from the current machine. +# +#----------------------------------------------------------------------- +# +print_info_msg " +Checking access to NOMADS..." + +check_nomads_access || print_err_msg_exit "\ +NOMADS is not accessible from this machine (MACHINE): + MACHINE = \"$MACHINE\"" + +print_info_msg " +NOMADS is accessible from this machine (MACHINE): + MACHINE = \"$MACHINE\"" +# +#----------------------------------------------------------------------- +# +# Loop over all specified cycle times and file types (analysis and +# forecast) and try to get the external model files for each such +# combination. +# +#----------------------------------------------------------------------- +# +for (( i=0; i<${num_cdates}; i++ )); do + + for (( j=0; j<${num_file_types}; j++ )); do + + file_type="${file_types[$j]}" + if [ "${file_type}" = "ANL" ]; then + ics_or_lbcs="ICS" + elif [ "${file_type}" = "FCST" ]; then + ics_or_lbcs="LBCS" + fi + + cdate="${all_cdates[$i]}" + print_info_msg " +Attempting to fetch external model ${file_type} file(s) from NOMADS for: + cdate = \"$cdate\"" +# +# Create the full path to the directory in which the external model files +# for this cycle will be placed. Then, if preexisting_dir_method is not +# set to a null string, check if an identically named directory already +# exists and if so, deal with it as specified by preexisting_dir_method. +# Finally, create the directory. +# +# Note that the bash parameter expansion +# +# ${data_relsubdir:+/${data_relsubdir}} +# +# evaluates to a null string if data_relsubdir is null or unset, and it +# evaluates to the string "/${data_relsubdir}" if data_relsubdir is set. +# + staging_dir="${data_basedir}/$cdate${data_relsubdir:+/${data_relsubdir}}" + if [ ! -z "${preexisting_dir_method}" ]; then + check_for_preexist_dir_file "${data_basedir}" "${preexisting_dir_method}" + fi + mkdir_vrfy -p "${staging_dir}" +# +# Get the names of the external modle files on NOMADS (in __fns). +# + set_extrn_mdl_filenames \ + data_src="${data_src}" \ + extrn_mdl_name="${extrn_mdl_name}" \ + ics_or_lbcs="${ics_or_lbcs}" \ + cdate="$cdate" \ + lbc_spec_fhrs="${lbc_spec_fhrs_str}" \ + outvarname_fns="__fns" + + fns_str="( "$( printf "\"%s\" " "${__fns[@]}" )")" +# +# Get the relative (cycle-dependent) path (in __arcvrel_dir) that will +# be used to form the full URL to the external model files on NOMADS. +# + set_extrn_mdl_arcv_file_dir_names \ + extrn_mdl_name="${extrn_mdl_name}" \ + ics_or_lbcs="${ics_or_lbcs}" \ + cdate="$cdate" \ + outvarname_arcvrel_dir="__arcvrel_dir" +# +# Get the files from NOMADS. Note that we do not check access to NOMADS +# within the get_extrn_mdl_files_from_nomads function called below (i.e. +# we set check_access="FALSE") because we have already checked for access +# up above before entering this loop. +# + get_extrn_mdl_files_from_nomads \ + check_access="FALSE" \ + extrn_mdl_name="${extrn_mdl_name}" \ + ics_or_lbcs="${ics_or_lbcs}" \ + staging_dir="${staging_dir}" \ + arcvrel_dir="${__arcvrel_dir}" \ + fns="${fns_str}" +# +# Print out a message informing whether the fetching was successful. If +# it failed, we skip the rest of the loop and continue to try the next +# cycle time and file type combination. +# + if [ "$?" -eq "0" ]; then + print_info_msg " +Successfully fetched external model ${file_type} file(s) from NOMADS for +the current cycle (cdate): + cdate = \"$cdate\"" + else + print_info_msg "\ +Attempt to fetch external model ${file_type} files from NOMADS for the +current cycle time (cdate) failed: + cdate = \"$cdate\" +Skipping rest of current loop to try the next cycle time and file type +combination." + continue + fi +# +# Call a function to create a variable definitions file (in bash script +# syntax) that contains several external-model-associated variables +# generated in this script. If the external model files are to be used +# in the UFS SRW App workflow, these definition files will be needed by +# various workflow tasks. +# + if [ "${ics_or_lbcs}" = "ICS" ]; then + var_defns_fn="${EXTRN_MDL_ICS_VAR_DEFNS_FN}" + elif [ "${ics_or_lbcs}" = "LBCS" ]; then + var_defns_fn="${EXTRN_MDL_LBCS_VAR_DEFNS_FN}" + fi + var_defns_fp="${staging_dir}/${var_defns_fn}" + + create_extrn_mdl_var_defns_file \ + extrn_mdl_var_defns_fp="${var_defns_fp}" \ + ics_or_lbcs="${ics_or_lbcs}" \ + extrn_mdl_cdate="$cdate" \ + extrn_mdl_staging_dir="${staging_dir}" \ + extrn_mdl_fns="${fns_str}" \ + extrn_mdl_lbc_spec_fhrs="${lbc_spec_fhrs_str}" + + done + +done +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script or +# function. +# +#----------------------------------------------------------------------- +# +{ restore_shell_opts; } > /dev/null 2>&1 + diff --git a/ush/get_extrn_mdl_file_dir_info.sh b/ush/get_extrn_mdl_file_dir_info.sh deleted file mode 100755 index f2224559a..000000000 --- a/ush/get_extrn_mdl_file_dir_info.sh +++ /dev/null @@ -1,1025 +0,0 @@ -# -#----------------------------------------------------------------------- -# -# Source the variable definitions file and the bash utility functions. -# -#----------------------------------------------------------------------- -# -. ${GLOBAL_VAR_DEFNS_FP} -. $USHDIR/source_util_funcs.sh -# -#----------------------------------------------------------------------- -# -# This file defines a function that is used to obtain information (e.g. -# output file names, system and mass store file and/or directory names) -# for a specified external model, analysis or forecast, and cycle date. -# See the usage statement below for this function should be called and -# the definitions of the input parameters. -# -#----------------------------------------------------------------------- -# -function get_extrn_mdl_file_dir_info() { -# -#----------------------------------------------------------------------- -# -# Save current shell options (in a global array). Then set new options -# for this script/function. -# -#----------------------------------------------------------------------- -# - { save_shell_opts; set -u +x; } > /dev/null 2>&1 -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# - local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) - local scrfunc_fn=$( basename "${scrfunc_fp}" ) - local scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Get the name of this function. -# -#----------------------------------------------------------------------- -# - local func_name="${FUNCNAME[0]}" -# -#----------------------------------------------------------------------- -# -# Specify the set of valid argument names for this script/function. Then -# process the arguments provided to this script/function (which should -# consist of a set of name-value pairs of the form arg1="value1", etc). -# -#----------------------------------------------------------------------- -# - local valid_args=( \ - "extrn_mdl_name" \ - "anl_or_fcst" \ - "cdate_FV3LAM" \ - "time_offset_hrs" \ - "varname_extrn_mdl_cdate" \ - "varname_extrn_mdl_lbc_spec_fhrs" \ - "varname_extrn_mdl_fns_on_disk" \ - "varname_extrn_mdl_fns_in_arcv" \ - "varname_extrn_mdl_sysdir" \ - "varname_extrn_mdl_arcv_fmt" \ - "varname_extrn_mdl_arcv_fns" \ - "varname_extrn_mdl_arcv_fps" \ - "varname_extrn_mdl_arcvrel_dir" \ - ) - process_args valid_args "$@" -# -#----------------------------------------------------------------------- -# -# For debugging purposes, print out values of arguments passed to this -# script/function. Note that these will be printed out only if VERBOSE -# is set to TRUE. -# -#----------------------------------------------------------------------- -# - print_input_args valid_args -# -#----------------------------------------------------------------------- -# -# Check arguments. -# -#----------------------------------------------------------------------- -# -if [ 0 = 1 ]; then - - if [ "$#" -ne "13" ]; then - - print_err_msg_exit " -Incorrect number of arguments specified: - - Function name: \"${func_name}\" - Number of arguments specified: $# - -Usage: - - ${func_name} \ - extrn_mdl_name \ - anl_or_fcst \ - cdate_FV3LAM \ - time_offset_hrs \ - varname_extrn_mdl_cdate \ - varname_extrn_mdl_lbc_spec_fhrs \ - varname_extrn_mdl_fns \ - varname_extrn_mdl_sysdir \ - varname_extrn_mdl_arcv_fmt \ - varname_extrn_mdl_arcv_fns \ - varname_extrn_mdl_arcv_fps \ - varname_extrn_mdl_arcvrel_dir - -where the arguments are defined as follows: - - extrn_mdl_name: - Name of the external model, i.e. the name of the model providing the - fields from which files containing initial conditions, surface fields, - and/or lateral boundary conditions for the FV3-LAM will be generated. - - anl_or_fcst: - Flag that specifies whether the external model files we are interested - in obtaining are analysis or forecast files. - - cdate_FV3LAM: - The cycle date and time (hours only) for which we want to obtain file - and directory information. This has the form YYYYMMDDHH, where YYYY - is the four-digit starting year of the cycle, MM is the two-digit - month, DD is the two-digit day of the month, and HH is the two-digit - hour of day. - - time_offset_hrs: - The number of hours by which to shift back in time the start time of - the external model forecast from the specified cycle start time of the - FV3-LAM (cdate_FV3LAM). When getting directory and file information on - external model analysis files, this is normally set to 0. When get- - ting directory and file information on external model forecast files, - this may be set to a nonzero value to obtain information for an exter- - nal model run that started time_offset_hrs hours before cdate_FV3LAM - (instead of exactly at cdate_FV3LAM). Note that in this case, the - forecast hours (relative to the external model run's start time) at - which the lateral boundary conditions will be updated must be shifted - forward by time_offset_hrs hours relative to those for the FV3-LAM in - order to make up for the backward-in-time shift in the starting time - of the external model. - - varname_extrn_mdl_cdate: - Name of the global variable that will contain the starting date and - hour of the external model run. - - varname_extrn_mdl_lbc_spec_fhrs: - Name of the global variable that will contain the forecast hours (re- - lative to the starting time of the external model run, which is earli- - er than that of the FV3-LAM by time_offset_hrs hours) at which lateral - boundary condition (LBC) output files are obtained from the external - model (and will be used to update the LBCs of the FV3-LAM). - - varname_extrn_mdl_fns: - Name of the global variable that will contain the names of the exter- - nal model output files. - - varname_extrn_mdl_sysdir: - Name of the global variable that will contain the system directory in - which the externaml model output files may be stored. - - varname_extrn_mdl_arcv_fmt: - Name of the global variable that will contain the format of the ar- - chive file on HPSS in which the externaml model output files may be - stored. - - varname_extrn_mdl_arcv_fns: - Name of the global variable that will contain the name of the archive - file on HPSS in which the externaml model output files may be stored. - - varname_extrn_mdl_arcv_fps: - Name of the global variable that will contain the full path to the ar- - chive file on HPSS in which the externaml model output files may be - stored. - - varname_extrn_mdl_arcvrel_dir: - Name of the global variable that will contain the archive-relative di- - rectory, i.e. the directory \"inside\" the archive file in which the ex- - ternal model output files may be stored. -" - - fi - -fi - - -# -#----------------------------------------------------------------------- -# -# Declare additional local variables. -# -#----------------------------------------------------------------------- -# - local yyyy mm dd hh mn yyyymmdd \ - lbc_spec_fhrs i num_fhrs \ - yy ddd fcst_hhh fcst_hh fcst_mn \ - prefix suffix fns fns_on_disk fns_in_arcv \ - sysbasedir sysdir \ - arcv_dir arcv_fmt arcv_fns arcv_fps arcvrel_dir -# -#----------------------------------------------------------------------- -# -# Check input variables for valid values. -# -#----------------------------------------------------------------------- -# - anl_or_fcst="${anl_or_fcst^^}" - valid_vals_anl_or_fcst=( "ANL" "FCST" ) - check_var_valid_value "anl_or_fcst" "valid_vals_anl_or_fcst" -# -#----------------------------------------------------------------------- -# -# Extract from cdate_FV3LAM the starting year, month, day, and hour of -# the FV3-LAM cycle. Then subtract the temporal offset specified in -# time_offset_hrs (assumed to be given in units of hours) from cdate_FV3LAM -# to obtain the starting date and time of the external model, express the -# result in YYYYMMDDHH format, and save it in cdate. This is the starting -# time of the external model forecast. -# -#----------------------------------------------------------------------- -# - yyyy=${cdate_FV3LAM:0:4} - mm=${cdate_FV3LAM:4:2} - dd=${cdate_FV3LAM:6:2} - hh=${cdate_FV3LAM:8:2} - yyyymmdd=${cdate_FV3LAM:0:8} - - cdate=$( date --utc --date "${yyyymmdd} ${hh} UTC - ${time_offset_hrs} hours" "+%Y%m%d%H" ) -# -#----------------------------------------------------------------------- -# -# Extract from cdate the starting year, month, day, and hour of the external -# model forecast. Also, set the starting minute to "00" and get the date -# without the time-of-day. These are needed below in setting various -# directory and file names. -# -#----------------------------------------------------------------------- -# - yyyy=${cdate:0:4} - mm=${cdate:4:2} - dd=${cdate:6:2} - hh=${cdate:8:2} - mn="00" - yyyymmdd=${cdate:0:8} -# -#----------------------------------------------------------------------- -# -# Initialize lbc_spec_fhrs to an empty array. Then, if considering a -# forecast, reset lbc_spec_fhrs to the array of forecast hours at which -# the lateral boundary conditions (LBCs) are to be updated, starting with -# the 2nd such time (i.e. the one having array index 1). We do not include -# the first hour (hour 0) because at this initial time, the LBCs are -# obtained from the analysis fields provided by the external model (as -# opposed to a forecast field). -# -#----------------------------------------------------------------------- -# - lbc_spec_fhrs=( "" ) - - if [ "${anl_or_fcst}" = "FCST" ]; then - - lbc_spec_fhrs=( "${LBC_SPEC_FCST_HRS[@]}" ) -# -# Add the temporal offset specified in time_offset_hrs (assumed to be in -# units of hours) to the the array of LBC update forecast hours to make -# up for shifting the starting hour back in time. After this addition, -# lbc_spec_fhrs will contain the LBC update forecast hours relative to -# the start time of the external model run. -# - num_fhrs=${#lbc_spec_fhrs[@]} - for (( i=0; i<=$((num_fhrs-1)); i++ )); do - lbc_spec_fhrs[$i]=$(( ${lbc_spec_fhrs[$i]} + time_offset_hrs )) - done - - fi -# -#----------------------------------------------------------------------- -# -# Set additional parameters needed in forming the names of the external -# model files only under certain circumstances. -# -#----------------------------------------------------------------------- -# - if [ "${extrn_mdl_name}" = "RAP" ] || \ - [ "${extrn_mdl_name}" = "HRRR" ] || \ - [ "${extrn_mdl_name}" = "NAM" ] || \ - [ "${extrn_mdl_name}" = "FV3GFS" -a "${MACHINE}" = "JET" ]; then -# -# Get the Julian day-of-year of the starting date and time of the exter- -# nal model forecast. -# - ddd=$( date --utc --date "${yyyy}-${mm}-${dd} ${hh}:${mn} UTC" "+%j" ) -# -# Get the last two digits of the year of the starting date and time of -# the external model forecast. -# - yy=${yyyy:2:4} - - fi -# -#----------------------------------------------------------------------- -# -# Set the external model output file names that must be obtained (from -# disk if available, otherwise from HPSS). -# -#----------------------------------------------------------------------- -# - if [ "${anl_or_fcst}" = "ANL" ]; then - fv3gfs_file_fmt="${FV3GFS_FILE_FMT_ICS}" - elif [ "${anl_or_fcst}" = "FCST" ]; then - fv3gfs_file_fmt="${FV3GFS_FILE_FMT_LBCS}" - fi - - case "${anl_or_fcst}" in -# -#----------------------------------------------------------------------- -# -# Consider analysis files (possibly including surface files). -# -#----------------------------------------------------------------------- -# - "ANL") - - fcst_hh="00" - fcst_mn="00" - - case "${extrn_mdl_name}" in - - "GSMGFS") -# fns=( "atm" "sfc" "nst" ) - fns=( "atm" "sfc" ) - prefix="gfs.t${hh}z." - fns=( "${fns[@]/#/$prefix}" ) - suffix="anl.nemsio" - fns_on_disk=( "${fns[@]/%/$suffix}" ) - fns_in_arcv=( "${fns[@]/%/$suffix}" ) - ;; - - "FV3GFS") - - if [ "${fv3gfs_file_fmt}" = "nemsio" ]; then - - fns=( "atm" "sfc" ) - suffix="anl.nemsio" - fns=( "${fns[@]/%/$suffix}" ) - -# Set names of external files if searching on disk. - if [ "${MACHINE}" = "JET" ]; then - prefix="${yy}${ddd}${hh}00.gfs.t${hh}z." - else - prefix="gfs.t${hh}z." - fi - fns_on_disk=( "${fns[@]/#/$prefix}" ) - -# Set names of external files if searching in an archive file, e.g. from -# HPSS. - prefix="gfs.t${hh}z." - fns_in_arcv=( "${fns[@]/#/$prefix}" ) - - elif [ "${fv3gfs_file_fmt}" = "grib2" ]; then - -# GSK 12/16/2019: -# Turns out that the .f000 file contains certain necessary fields that -# are not in the .anl file, so switch to the former. -# fns=( "gfs.t${hh}z.pgrb2.0p25.anl" ) # Get only 0.25 degree files for now. -# fns=( "gfs.t${hh}z.pgrb2.0p25.f000" ) # Get only 0.25 degree files for now. - fns_on_disk=( "gfs.t${hh}z.pgrb2.0p25.f000" ) # Get only 0.25 degree files for now. - fns_in_arcv=( "gfs.t${hh}z.pgrb2.0p25.f000" ) # Get only 0.25 degree files for now. - - elif [ "${fv3gfs_file_fmt}" = "netcdf" ]; then - - fns=( "atm" "sfc" ) - suffix="anl.nc" - fns=( "${fns[@]/%/$suffix}" ) - -# Set names of external files if searching on disk. - if [ "${MACHINE}" = "JET" ]; then - prefix="${yy}${ddd}${hh}00.gfs.t${hh}z." - else - prefix="gfs.t${hh}z." - fi - fns_on_disk=( "${fns[@]/#/$prefix}" ) - -# Set names of external files if searching in an archive file, e.g. from -# HPSS. - prefix="gfs.t${hh}z." - fns_in_arcv=( "${fns[@]/#/$prefix}" ) - - fi - ;; - - "RAP") -# -# Note that this is GSL RAPX data, not operational NCEP RAP data. An option for the latter -# may be added in the future. -# - if [ "${MACHINE}" = "JET" ]; then - fns_on_disk=( "wrfnat_130_${fcst_hh}.grib2" ) - else - fns_on_disk=( "${yy}${ddd}${hh}${mn}${fcst_hh}${fcst_mn}" ) - fi - fns_in_arcv=( "${yy}${ddd}${hh}${mn}${fcst_hh}${fcst_mn}" ) - ;; - - "HRRR") -# -# Note that this is GSL HRRRX data, not operational NCEP HRRR data. An option for the latter -# may be added in the future. -# - if [ "${MACHINE}" = "JET" ]; then - fns_on_disk=( "wrfnat_hrconus_${fcst_hh}.grib2" ) - else - fns_on_disk=( "${yy}${ddd}${hh}${mn}${fcst_hh}${fcst_mn}" ) - fi - fns_in_arcv=( "${yy}${ddd}${hh}${mn}${fcst_hh}${fcst_mn}" ) - ;; - - "NAM") - fns=( "" ) - prefix="nam.t${hh}z.bgrdsfi${hh}" - fns=( "${fns[@]/#/$prefix}" ) - suffix=".tm${hh}" - fns_on_disk=( "${fns[@]/%/$suffix}" ) - fns_in_arcv=( "${fns[@]/%/$suffix}" ) - ;; - - *) - print_err_msg_exit "\ -The external model file names (either on disk or in archive files) have -not yet been specified for this combination of external model (extrn_mdl_name) -and analysis or forecast (anl_or_fcst): - extrn_mdl_name = \"${extrn_mdl_name}\" - anl_or_fcst = \"${anl_or_fcst}\"" - ;; - - esac - ;; -# -#----------------------------------------------------------------------- -# -# Consider forecast files. -# -#----------------------------------------------------------------------- -# - "FCST") - - fcst_mn="00" - - case "${extrn_mdl_name}" in - - "GSMGFS") - fcst_hhh=( $( printf "%03d " "${lbc_spec_fhrs[@]}" ) ) - prefix="gfs.t${hh}z.atmf" - fns=( "${fcst_hhh[@]/#/$prefix}" ) - suffix=".nemsio" - fns_on_disk=( "${fns[@]/%/$suffix}" ) - fns_in_arcv=( "${fns[@]/%/$suffix}" ) - ;; - - "FV3GFS") - - if [ "${fv3gfs_file_fmt}" = "nemsio" ]; then - - fcst_hhh=( $( printf "%03d " "${lbc_spec_fhrs[@]}" ) ) - suffix=".nemsio" - fns=( "${fcst_hhh[@]/%/$suffix}" ) - - if [ "${MACHINE}" = "JET" ]; then - prefix="${yy}${ddd}${hh}00.gfs.t${hh}z.atmf" - else - prefix="gfs.t${hh}z.atmf" - fi - fns_on_disk=( "${fns[@]/#/$prefix}" ) - - prefix="gfs.t${hh}z.atmf" - fns_in_arcv=( "${fns[@]/#/$prefix}" ) - - elif [ "${fv3gfs_file_fmt}" = "grib2" ]; then - - fcst_hhh=( $( printf "%03d " "${lbc_spec_fhrs[@]}" ) ) - prefix="gfs.t${hh}z.pgrb2.0p25.f" - fns_on_disk=( "${fcst_hhh[@]/#/$prefix}" ) - fns_in_arcv=( "${fcst_hhh[@]/#/$prefix}" ) - - elif [ "${fv3gfs_file_fmt}" = "netcdf" ]; then - - fcst_hhh=( $( printf "%03d " "${lbc_spec_fhrs[@]}" ) ) - suffix=".nc" - fns=( "${fcst_hhh[@]/%/$suffix}" ) - - if [ "${MACHINE}" = "JET" ]; then - prefix="${yy}${ddd}${hh}00.gfs.t${hh}z.atmf" - else - prefix="gfs.t${hh}z.atmf" - fi - fns_on_disk=( "${fns[@]/#/$prefix}" ) - - prefix="gfs.t${hh}z.atmf" - fns_in_arcv=( "${fns[@]/#/$prefix}" ) - - fi - ;; - - "RAP") -# -# Note that this is GSL RAPX data, not operational NCEP RAP data. An option for the latter -# may be added in the future. -# - fcst_hh=( $( printf "%02d " "${lbc_spec_fhrs[@]}" ) ) - - if [ "${MACHINE}" = "JET" ]; then - prefix="wrfnat_130_" - suffix=".grib2" - else - prefix="${yy}${ddd}${hh}${mn}" - suffix="${fcst_mn}" - fi - fns_on_disk=( "${fcst_hh[@]/#/$prefix}" ) - fns_on_disk=( "${fns_on_disk[@]/%/$suffix}" ) - - prefix="${yy}${ddd}${hh}${mn}" - fns_in_arcv=( "${fcst_hh[@]/#/$prefix}" ) - suffix="${fcst_mn}" - fns_in_arcv=( "${fns_in_arcv[@]/%/$suffix}" ) - ;; - - "HRRR") -# -# Note that this is GSL HRRRX data, not operational NCEP HRRR data. An option for the latter -# may be added in the future. -# - fcst_hh=( $( printf "%02d " "${lbc_spec_fhrs[@]}" ) ) - - if [ "${MACHINE}" = "JET" ]; then - prefix="wrfnat_hrconus_" - suffix=".grib2" - else - prefix="${yy}${ddd}${hh}${mn}" - suffix="${fcst_mn}" - fi - fns_on_disk=( "${fcst_hh[@]/#/$prefix}" ) - fns_on_disk=( "${fns_on_disk[@]/%/$suffix}" ) - - prefix="${yy}${ddd}${hh}${mn}" - fns_in_arcv=( "${fcst_hh[@]/#/$prefix}" ) - suffix="${fcst_mn}" - fns_in_arcv=( "${fns_in_arcv[@]/%/$suffix}" ) - ;; - - "NAM") - fcst_hhh=( $( printf "%03d " "${lbc_spec_fhrs[@]}" ) ) - prefix="nam.t${hh}z.bgrdsf" - fns=( "${fcst_hhh[@]/#/$prefix}" ) - suffix="" - fns_on_disk=( "${fns[@]/%/$suffix}" ) - fns_in_arcv=( "${fns[@]/%/$suffix}" ) - ;; - - *) - print_err_msg_exit "\ -The external model file names have not yet been specified for this com- -bination of external model (extrn_mdl_name) and analysis or forecast -(anl_or_fcst): - extrn_mdl_name = \"${extrn_mdl_name}\" - anl_or_fcst = \"${anl_or_fcst}\"" - ;; - - esac - ;; - - esac -# -#----------------------------------------------------------------------- -# -# Set the system directory (i.e. a directory on disk) in which the external -# model output files for the specified cycle date (cdate) may be located. -# Note that this will be used by the calling script only if the output -# files for the specified cdate actually exist at this location. Otherwise, -# the files will be searched for on the mass store (HPSS). -# -#----------------------------------------------------------------------- -# - if [ "${anl_or_fcst}" = "ANL" ]; then - sysbasedir="${EXTRN_MDL_SYSBASEDIR_ICS}" - elif [ "${anl_or_fcst}" = "FCST" ]; then - sysbasedir="${EXTRN_MDL_SYSBASEDIR_LBCS}" - fi - - case "${extrn_mdl_name}" in - -# -# It is not clear which, if any, systems the (old) spectral GFS model is -# available on, so set sysdir for this external model to a null string. -# - "GSMGFS") - case "$MACHINE" in - "WCOSS_CRAY") - sysdir="" - ;; - "WCOSS_DELL_P3") - sysdir="" - ;; - "HERA") - sysdir="" - ;; - "ORION") - sysdir="$sysbasedir" - ;; - "JET") - sysdir="" - ;; - "ODIN") - sysdir="$sysbasedir" - ;; - "CHEYENNE") - sysdir="" - ;; - "STAMPEDE") - sysdir="$sysbasedir" - ;; - *) - print_err_msg_exit "\ -The system directory in which to look for external model output files -has not been specified for this external model and machine combination: - extrn_mdl_name = \"${extrn_mdl_name}\" - MACHINE = \"$MACHINE\"" - ;; - esac - ;; - - - "FV3GFS") - case "$MACHINE" in - "WCOSS_CRAY") - sysdir="$sysbasedir/gfs.${yyyymmdd}/${hh}/atmos" - ;; - "WCOSS_DELL_P3") - sysdir="$sysbasedir/gfs.${yyyymmdd}/${hh}/atmos" - ;; - "HERA") - sysdir="$sysbasedir/gfs.${yyyymmdd}/${hh}/atmos" - ;; - "ORION") - sysdir="$sysbasedir" - ;; - "JET") - sysdir="$sysbasedir" - ;; - "ODIN") - sysdir="$sysbasedir/${yyyymmdd}" - ;; - "CHEYENNE") - sysdir="$sysbasedir/gfs.${yyyymmdd}/${hh}" - ;; - "STAMPEDE") - sysdir="$sysbasedir" - ;; - *) - print_err_msg_exit "\ -The system directory in which to look for external model output files -has not been specified for this external model and machine combination: - extrn_mdl_name = \"${extrn_mdl_name}\" - MACHINE = \"$MACHINE\"" - ;; - esac - ;; - - - "RAP") - case "$MACHINE" in - "WCOSS_CRAY") - sysdir="$sysbasedir" - ;; - "WCOSS_DELL_P3") - sysdir="$sysbasedir" - ;; - "HERA") - sysdir="$sysbasedir" - ;; - "ORION") - sysdir="$sysbasedir" - ;; - "JET") - sysdir="$sysbasedir/${yyyymmdd}${hh}/postprd" - ;; - "ODIN") - sysdir="$sysbasedir" - ;; - "CHEYENNE") - sysdir="$sysbasedir" - ;; - *) - print_err_msg_exit "\ -The system directory in which to look for external model output files -has not been specified for this external model and machine combination: - extrn_mdl_name = \"${extrn_mdl_name}\" - MACHINE = \"$MACHINE\"" - ;; - esac - ;; - - - "HRRR") - case "$MACHINE" in - "WCOSS_CRAY") - sysdir="$sysbasedir" - ;; - "WCOSS_DELL_P3") - sysdir="$sysbasedir" - ;; - "HERA") - sysdir="$sysbasedir" - ;; - "ORION") - sysdir="$sysbasedir" - ;; - "JET") - sysdir="$sysbasedir/${yyyymmdd}${hh}/postprd" - ;; - "ODIN") - sysdir="$sysbasedir" - ;; - "CHEYENNE") - sysdir="$sysbasedir" - ;; - *) - print_err_msg_exit "\ -The system directory in which to look for external model output files -has not been specified for this external model and machine combination: - extrn_mdl_name = \"${extrn_mdl_name}\" - MACHINE = \"$MACHINE\"" - ;; - esac - ;; - - "NAM") - case "$MACHINE" in - "WCOSS_CRAY") - sysdir="$sysbasedir" - ;; - "WCOSS_DELL_P3") - sysdir="$sysbasedir" - ;; - "HERA") - sysdir="$sysbasedir" - ;; - "ORION") - sysdir="$sysbasedir" - ;; - "JET") - sysdir="$sysbasedir" - ;; - "ODIN") - sysdir="$sysbasedir" - ;; - "CHEYENNE") - sysdir="$sysbasedir" - ;; - *) - print_err_msg_exit "\ -The system directory in which to look for external model output files -has not been specified for this external model and machine combination: - extrn_mdl_name = \"${extrn_mdl_name}\" - MACHINE = \"$MACHINE\"" - ;; - esac - ;; - - - *) - print_err_msg_exit "\ -The system directory in which to look for external model output files -has not been specified for this external model: - extrn_mdl_name = \"${extrn_mdl_name}\"" - - esac -# -#----------------------------------------------------------------------- -# -# Set parameters associated with the mass store (HPSS) for the specified -# cycle date (cdate). These consist of: -# -# 1) The type of the archive file (e.g. tar, zip, etc). -# 2) The name of the archive file. -# 3) The full path in HPSS to the archive file. -# 4) The relative directory in the archive file in which the model output -# files are located. -# -# Note that these will be used by the calling script only if the archive -# file for the specified cdate actually exists on HPSS. -# -#----------------------------------------------------------------------- -# - case "${extrn_mdl_name}" in - - "GSMGFS") - arcv_dir="/NCEPPROD/hpssprod/runhistory/rh${yyyy}/${yyyy}${mm}/${yyyymmdd}" - arcv_fmt="tar" - arcv_fns="gpfs_hps_nco_ops_com_gfs_prod_gfs.${cdate}." - if [ "${anl_or_fcst}" = "ANL" ]; then - arcv_fns="${arcv_fns}anl" - arcvrel_dir="." - elif [ "${anl_or_fcst}" = "FCST" ]; then - arcv_fns="${arcv_fns}sigma" - arcvrel_dir="/gpfs/hps/nco/ops/com/gfs/prod/gfs.${yyyymmdd}" - fi - arcv_fns="${arcv_fns}.${arcv_fmt}" - arcv_fps="${arcv_dir}/${arcv_fns}" - ;; - - "FV3GFS") - - if [ "${cdate_FV3LAM}" -lt "2019061200" ]; then - arcv_dir="/NCEPDEV/emc-global/5year/emc.glopara/WCOSS_C/Q2FY19/prfv3rt3/${cdate_FV3LAM}" - arcv_fns="" - elif [ "${cdate_FV3LAM}" -ge "2019061200" ] && \ - [ "${cdate_FV3LAM}" -lt "2020022600" ]; then - arcv_dir="/NCEPPROD/hpssprod/runhistory/rh${yyyy}/${yyyy}${mm}/${yyyymmdd}" - arcv_fns="gpfs_dell1_nco_ops_com_gfs_prod_gfs.${yyyymmdd}_${hh}." - elif [ "${cdate_FV3LAM}" -ge "2020022600" ]; then - arcv_dir="/NCEPPROD/hpssprod/runhistory/rh${yyyy}/${yyyy}${mm}/${yyyymmdd}" - arcv_fns="com_gfs_prod_gfs.${yyyymmdd}_${hh}." - fi - - if [ "${fv3gfs_file_fmt}" = "nemsio" ]; then - - if [ "${anl_or_fcst}" = "ANL" ]; then - arcv_fns="${arcv_fns}gfs_nemsioa" - elif [ "${anl_or_fcst}" = "FCST" ]; then - last_fhr_in_nemsioa="39" - first_lbc_fhr="${lbc_spec_fhrs[0]}" - last_lbc_fhr="${lbc_spec_fhrs[-1]}" - if [ "${last_lbc_fhr}" -le "${last_fhr_in_nemsioa}" ]; then - arcv_fns="${arcv_fns}gfs_nemsioa" - elif [ "${first_lbc_fhr}" -gt "${last_fhr_in_nemsioa}" ]; then - arcv_fns="${arcv_fns}gfs_nemsiob" - else - arcv_fns=( "${arcv_fns}gfs_nemsioa" "${arcv_fns}gfs_nemsiob" ) - fi - fi - - elif [ "${fv3gfs_file_fmt}" = "grib2" ]; then - - arcv_fns="${arcv_fns}gfs_pgrb2" - - elif [ "${fv3gfs_file_fmt}" = "netcdf" ]; then - - if [ "${anl_or_fcst}" = "ANL" ]; then - arcv_fns="${arcv_fns}gfs_nca" - elif [ "${anl_or_fcst}" = "FCST" ]; then - last_fhr_in_netcdfa="39" - first_lbc_fhr="${lbc_spec_fhrs[0]}" - last_lbc_fhr="${lbc_spec_fhrs[-1]}" - if [ "${last_lbc_fhr}" -le "${last_fhr_in_netcdfa}" ]; then - arcv_fns="${arcv_fns}gfs_nca" - elif [ "${first_lbc_fhr}" -gt "${last_fhr_in_netcdfa}" ]; then - arcv_fns="${arcv_fns}gfs_ncb" - else - arcv_fns=( "${arcv_fns}gfs_nca" "${arcv_fns}gfs_ncb" ) - fi - fi - - fi - - arcv_fmt="tar" - - slash_atmos_or_null="" - if [ "${cdate_FV3LAM}" -ge "2021032100" ]; then - slash_atmos_or_null="/atmos" - fi - arcvrel_dir="./gfs.${yyyymmdd}/${hh}${slash_atmos_or_null}" - - is_array arcv_fns - if [ "$?" = "0" ]; then - suffix=".${arcv_fmt}" - arcv_fns=( "${arcv_fns[@]/%/$suffix}" ) - prefix="${arcv_dir}/" - arcv_fps=( "${arcv_fns[@]/#/$prefix}" ) - else - arcv_fns="${arcv_fns}.${arcv_fmt}" - arcv_fps="${arcv_dir}/${arcv_fns}" - fi - ;; - - - "RAP") -# -# Note that this is GSL RAPX data, not operational NCEP RAP data. An option for the latter -# may be added in the future. -# -# The zip archive files for RAPX are named such that the forecast files -# for odd-numbered starting hours (e.g. 01, 03, ..., 23) are stored -# together with the forecast files for the corresponding preceding even- -# numbered starting hours (e.g. 00, 02, ..., 22, respectively), in an -# archive file whose name contains only the even-numbered hour. Thus, -# in forming the name of the archive file, if the starting hour (hh) is -# odd, we reduce it by one to get the corresponding even-numbered hour -# and use that to form the archive file name. -# - hh_orig=$hh -# Convert hh to a decimal (i.e. base-10) number. We need this because -# if it starts with a 0 (e.g. 00, 01, ..., 09), bash will treat it as an -# octal number, and 08 and 09 are illegal ocatal numbers for which the -# arithmetic operations below will fail. - hh=$((10#$hh)) - if [ $(($hh%2)) = 1 ]; then - hh=$((hh-1)) - fi -# Now that the arithmetic is done, recast hh as a two-digit string because -# that is needed in constructing the names below. - hh=$( printf "%02d\n" $hh ) - - arcv_dir="/BMC/fdr/Permanent/${yyyy}/${mm}/${dd}/data/fsl/rap/full/wrfnat" - arcv_fmt="zip" - arcv_fns="${yyyy}${mm}${dd}${hh}00.${arcv_fmt}" - arcv_fps="${arcv_dir}/${arcv_fns}" - arcvrel_dir="" -# -# Reset hh to its original value in case it is used again later below. -# - hh=${hh_orig} - ;; - - "HRRR") -# -# Note that this is GSL HRRRX data, not operational NCEP HRRR data. An option for the latter -# may be added in the future. -# - arcv_dir="/BMC/fdr/Permanent/${yyyy}/${mm}/${dd}/data/fsl/hrrr/conus/wrfnat" - arcv_fmt="zip" - arcv_fns="${yyyy}${mm}${dd}${hh}00.${arcv_fmt}" - arcv_fps="${arcv_dir}/${arcv_fns}" - arcvrel_dir="" - ;; - - "NAM") - arcv_dir="/NCEPPROD/hpssprod/runhistory/rh${yyyy}/${yyyy}${mm}/${yyyymmdd}" - arcv_fmt="tar" - arcv_fns="com_nam_prod_nam.${yyyy}${mm}${dd}${hh}.bgrid.${arcv_fmt}" - arcv_fps="${arcv_dir}/${arcv_fns}" - arcvrel_dir="" - ;; - - *) - print_err_msg_exit "\ -Archive file information has not been specified for this external model: - extrn_mdl_name = \"${extrn_mdl_name}\"" - ;; - - esac -# -# Depending on the experiment configuration, the above code may set -# arcv_fns and arcv_fps to either scalars or arrays. If they are not -# arrays, recast them as arrays because that is what is expected in the -# code below. -# - is_array arcv_fns || arcv_fns=( "${arcv_fns}" ) - is_array arcv_fps || arcv_fps=( "${arcv_fps}" ) -# -#----------------------------------------------------------------------- -# -# Use the eval function to set the output variables. Note that each of -# these is set only if the corresponding input variable specifying the -# name to use for the output variable is not empty. -# -#----------------------------------------------------------------------- -# - if [ ! -z "${varname_extrn_mdl_cdate}" ]; then - eval ${varname_extrn_mdl_cdate}="${cdate}" - fi - - if [ ! -z "${varname_extrn_mdl_lbc_spec_fhrs}" ]; then - lbc_spec_fhrs_str="( "$( printf "\"%s\" " "${lbc_spec_fhrs[@]}" )")" - eval ${varname_extrn_mdl_lbc_spec_fhrs}=${lbc_spec_fhrs_str} - fi - - if [ ! -z "${varname_extrn_mdl_fns_on_disk}" ]; then - fns_on_disk_str="( "$( printf "\"%s\" " "${fns_on_disk[@]}" )")" - eval ${varname_extrn_mdl_fns_on_disk}=${fns_on_disk_str} - fi - - if [ ! -z "${varname_extrn_mdl_fns_in_arcv}" ]; then - fns_in_arcv_str="( "$( printf "\"%s\" " "${fns_in_arcv[@]}" )")" - eval ${varname_extrn_mdl_fns_in_arcv}=${fns_in_arcv_str} - fi - - if [ ! -z "${varname_extrn_mdl_sysdir}" ]; then - eval ${varname_extrn_mdl_sysdir}="${sysdir}" - fi - - if [ ! -z "${varname_extrn_mdl_arcv_fmt}" ]; then - eval ${varname_extrn_mdl_arcv_fmt}="${arcv_fmt}" - fi - - if [ ! -z "${varname_extrn_mdl_arcv_fns}" ]; then - arcv_fns_str="( "$( printf "\"%s\" " "${arcv_fns[@]}" )")" - eval ${varname_extrn_mdl_arcv_fns}=${arcv_fns_str} - fi - - if [ ! -z "${varname_extrn_mdl_arcv_fps}" ]; then - arcv_fps_str="( "$( printf "\"%s\" " "${arcv_fps[@]}" )")" - eval ${varname_extrn_mdl_arcv_fps}=${arcv_fps_str} - fi - - if [ ! -z "${varname_extrn_mdl_arcvrel_dir}" ]; then - eval ${varname_extrn_mdl_arcvrel_dir}="${arcvrel_dir}" - fi -# -#----------------------------------------------------------------------- -# -# Restore the shell options saved at the beginning of this script/function. -# -#----------------------------------------------------------------------- -# - { restore_shell_opts; } > /dev/null 2>&1 -} diff --git a/ush/link_fix.sh b/ush/link_fix.sh index e9a641d96..71f163e00 100755 --- a/ush/link_fix.sh +++ b/ush/link_fix.sh @@ -16,7 +16,7 @@ function link_fix() { # #----------------------------------------------------------------------- # - { save_shell_opts; set -u -x; } > /dev/null 2>&1 + { save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/ush/load_modules_run_task.sh b/ush/load_modules_run_task.sh index 862bf9338..672adc363 100755 --- a/ush/load_modules_run_task.sh +++ b/ush/load_modules_run_task.sh @@ -155,7 +155,7 @@ jjob_fp="$2" machine=${MACHINE,,} env_fn="build_${machine}_${COMPILER}.env" env_fp="${SR_WX_APP_TOP_DIR}/env/${env_fn}" -source "${env_fp}" || print_err_msg_exit "\ +. "${env_fp}" || print_err_msg_exit "\ Sourcing platform- and compiler-specific environment file (env_fp) for the workflow task specified by task_name failed: task_name = \"${task_name}\" diff --git a/ush/make_grid_mosaic_file.sh b/ush/make_grid_mosaic_file.sh index 0026cb86a..871bff76c 100644 --- a/ush/make_grid_mosaic_file.sh +++ b/ush/make_grid_mosaic_file.sh @@ -17,7 +17,7 @@ function make_grid_mosaic_file() { # #----------------------------------------------------------------------- # - { save_shell_opts; set -u -x; } > /dev/null 2>&1 + { save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/ush/set_FV3nml_sfc_climo_filenames.sh b/ush/set_FV3nml_sfc_climo_filenames.sh index 2d44a9d0c..06c54c2ed 100644 --- a/ush/set_FV3nml_sfc_climo_filenames.sh +++ b/ush/set_FV3nml_sfc_climo_filenames.sh @@ -20,7 +20,7 @@ function set_FV3nml_sfc_climo_filenames() { # #----------------------------------------------------------------------- # - { save_shell_opts; set -u -x; } > /dev/null 2>&1 + { save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/ush/set_FV3nml_stoch_params.sh b/ush/set_FV3nml_stoch_params.sh index a6d4c6dfd..372869c99 100644 --- a/ush/set_FV3nml_stoch_params.sh +++ b/ush/set_FV3nml_stoch_params.sh @@ -22,7 +22,7 @@ function set_FV3nml_stoch_params() { # #----------------------------------------------------------------------- # - { save_shell_opts; set -u -x; } > /dev/null 2>&1 + { save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/ush/set_extrn_mdl_params.sh b/ush/set_extrn_mdl_params.sh deleted file mode 100644 index 72855d9fb..000000000 --- a/ush/set_extrn_mdl_params.sh +++ /dev/null @@ -1,379 +0,0 @@ -# -#----------------------------------------------------------------------- -# -# This file defines and then calls a function that sets parameters -# associated with the external model used for initial conditions (ICs) -# and the one used for lateral boundary conditions (LBCs). -# -#----------------------------------------------------------------------- -# -function set_extrn_mdl_params() { -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# -local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) -local scrfunc_fn=$( basename "${scrfunc_fp}" ) -local scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Get the name of this function. -# -#----------------------------------------------------------------------- -# -local func_name="${FUNCNAME[0]}" -# -#----------------------------------------------------------------------- -# -# Set the system directory (i.e. location on disk, not on HPSS) in which -# the files generated by the external model specified by EXTRN_MDL_NAME_ICS -# that are necessary for generating initial condition (IC) and surface -# files for the FV3SAR are stored (usually for a limited time, e.g. for -# the GFS external model, 2 weeks on WCOSS and 2 days on hera). If for -# a given cycle these files are available in this system directory, they -# will be copied over to a subdirectory under the cycle directory. If -# these files are not available in the system directory, then we search -# for them elsewhere, e.g. in the mass store (HPSS). -# -#----------------------------------------------------------------------- -# -if [ "${RUN_ENVIR}" = "nco" ]; then - - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-$COMINgfs}" - -else - - case "${EXTRN_MDL_NAME_ICS}" in - - "GSMGFS") - case "$MACHINE" in - "WCOSS_CRAY") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-}" - ;; - "WCOSS_DELL_P3") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-}" - ;; - "HERA") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-}" - ;; - "ORION") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-}" - ;; - "JET") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-}" - ;; - "ODIN") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-/scratch/ywang/EPIC/GDAS/2019053000_mem001}" - ;; - "CHEYENNE") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-/glade/p/ral/jntp/UFS_CAM/COMGFS}" - ;; - "STAMPEDE") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-/scratch/00315/tg455890/GDAS/20190530/2019053000_mem001}" - ;; - esac - ;; - - "FV3GFS") - case "$MACHINE" in - "WCOSS_CRAY") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-/gpfs/dell1/nco/ops/com/gfs/prod}" - ;; - "WCOSS_DELL_P3") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-/gpfs/dell1/nco/ops/com/gfs/prod}" - ;; - "HERA") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-/scratch1/NCEPDEV/rstprod/com/gfs/prod}" - ;; - "ORION") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-}" - ;; - "JET") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-/public/data/grids/gfs/nemsio}" - ;; - "ODIN") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-/scratch/ywang/test_runs/FV3_regional/gfs}" - ;; - "STAMPEDE") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-/scratch/00315/tg455890/GDAS/20190530/2019053000_mem001}" - ;; - "CHEYENNE") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-/glade/p/ral/jntp/UFS_CAM/COMGFS}" - ;; - esac - ;; - - "RAP") - case "$MACHINE" in - "WCOSS_CRAY") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-/gpfs/hps/nco/ops/com/rap/prod}" - ;; - "WCOSS_DELL_P3") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-/gpfs/hps/nco/ops/com/rap/prod}" - ;; - "HERA") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-/scratch2/BMC/public/data/gsd/rap/full/wrfnat}" - ;; - "ORION") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-}" - ;; - "JET") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-/misc/whome/rtrr/rap}" - ;; - "CHEYENNE") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-dummy_value}" - ;; - esac - ;; - - "HRRR") - case "$MACHINE" in - "WCOSS_CRAY") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-/gpfs/hps/nco/ops/com/hrrr/prod}" - ;; - "WCOSS_DELL_P3") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-/gpfs/hps/nco/ops/com/hrrr/prod}" - ;; - "HERA") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-/scratch2/BMC/public/data/gsd/hrrr/conus/wrfnat}" - ;; - "ORION") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-}" - ;; - "JET") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-/misc/whome/rtrr/hrrr}" - ;; - "CHEYENNE") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-dummy_value}" - ;; - esac - ;; - - "NAM") - case "$MACHINE" in - "WCOSS_CRAY") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-/gpfs/dell1/nco/ops/com/nam/prod}" - ;; - "WCOSS_DELL_P3") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-/gpfs/dell1/nco/ops/com/nam/prod}" - ;; - "HERA") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-dummy_value}" - ;; - "CHEYENNE") - EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS:-dummy_value}" - ;; - esac - ;; - - esac - -fi -# -# If EXTRN_MDL_SYSBASEDIR_ICS has not been set (not even to a null string), -# print out an error message and exit. -# -if [ -z "${EXTRN_MDL_SYSBASEDIR_ICS+x}" ]; then - print_err_msg_exit "\ -The variable EXTRN_MDL_SYSBASEDIR_ICS specifying the system directory -in which to look for the files generated by the external model for ICs -has not been set for the current combination of machine (MACHINE) and -external model (EXTRN_MDL_NAME_ICS): - MACHINE = \"$MACHINE\" - EXTRN_MDL_NAME_ICS = \"${EXTRN_MDL_NAME_ICS}\"" -fi -# -#----------------------------------------------------------------------- -# -# Set EXTRN_MDL_LBCS_OFFSET_HRS, which is the number of hours to shift -# the starting time of the external model that provides lateral boundary -# conditions. -# -#----------------------------------------------------------------------- -# -case "${EXTRN_MDL_NAME_LBCS}" in - "GSMGFS") - EXTRN_MDL_LBCS_OFFSET_HRS="0" - ;; - "FV3GFS") - EXTRN_MDL_LBCS_OFFSET_HRS="0" - ;; - "RAP") - EXTRN_MDL_LBCS_OFFSET_HRS="3" - ;; - "HRRR") - EXTRN_MDL_LBCS_OFFSET_HRS="0" - ;; - "NAM") - EXTRN_MDL_LBCS_OFFSET_HRS="0" - ;; -esac -# -#----------------------------------------------------------------------- -# -# Set the system directory (i.e. location on disk, not on HPSS) in which -# the files generated by the external model specified by EXTRN_MDL_NAME_LBCS -# that are necessary for generating lateral boundary condition (LBC) files -# for the FV3SAR are stored (usually for a limited time, e.g. for the GFS -# external model, 2 weeks on WCOSS and 2 days on hera). If for a given -# cycle these files are available in this system directory, they will be -# copied over to a subdirectory under the cycle directory. If these files -# are not available in the system directory, then we search for them -# elsewhere, e.g. in the mass store (HPSS). -# -#----------------------------------------------------------------------- -# -if [ "${RUN_ENVIR}" = "nco" ]; then - - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-$COMINgfs}" - -else - - case "${EXTRN_MDL_NAME_LBCS}" in - - "GSMGFS") - case "$MACHINE" in - "WCOSS_CRAY") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-}" - ;; - "WCOSS_DELL_P3") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-}" - ;; - "HERA") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-}" - ;; - "ORION") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-}" - ;; - "JET") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-}" - ;; - "ODIN") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-/scratch/ywang/EPIC/GDAS/2019053000_mem001}" - ;; - "CHEYENNE") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-/glade/p/ral/jntp/UFS_CAM/COMGFS}" - ;; - "STAMPEDE") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-/scratch/00315/tg455890/GDAS/20190530/2019053000_mem001}" - ;; - esac - ;; - - "FV3GFS") - case "$MACHINE" in - "WCOSS_CRAY") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-/gpfs/dell1/nco/ops/com/gfs/prod}" - ;; - "WCOSS_DELL_P3") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-/gpfs/dell1/nco/ops/com/gfs/prod}" - ;; - "HERA") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-/scratch1/NCEPDEV/rstprod/com/gfs/prod}" - ;; - "ORION") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-}" - ;; - "JET") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-/public/data/grids/gfs/nemsio}" - ;; - "ODIN") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-/scratch/ywang/test_runs/FV3_regional/gfs}" - ;; - "CHEYENNE") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-/glade/p/ral/jntp/UFS_CAM/COMGFS}" - ;; - "STAMPEDE") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-/scratch/00315/tg455890/GDAS/20190530/2019053000_mem001}" - ;; - esac - ;; - - "RAP") - case "$MACHINE" in - "WCOSS_CRAY") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-/gpfs/hps/nco/ops/com/rap/prod}" - ;; - "WCOSS_DELL_P3") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-/gpfs/hps/nco/ops/com/rap/prod}" - ;; - "HERA") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-/scratch2/BMC/public/data/gsd/rap/full/wrfnat}" - ;; - "ORION") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-}" - ;; - "JET") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-/misc/whome/rtrr/rap}" - ;; - "CHEYENNE") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-dummy_value}" - ;; - esac - ;; - - "HRRR") - case "$MACHINE" in - "WCOSS_CRAY") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-/gpfs/hps/nco/ops/com/hrrr/prod}" - ;; - "WCOSS_DELL_P3") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-/gpfs/hps/nco/ops/com/hrrr/prod}" - ;; - "HERA") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-/scratch2/BMC/public/data/gsd/hrrr/conus/wrfnat}" - ;; - "ORION") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-}" - ;; - "JET") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-/misc/whome/rtrr/hrrr}" - ;; - "CHEYENNE") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-dummy_value}" - ;; - esac - ;; - - "NAM") - case "$MACHINE" in - "HERA") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-dummy_value}" - ;; - "CHEYENNE") - EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS:-dummy_value}" - ;; - esac - ;; - - esac - -fi -# -# If EXTRN_MDL_SYSBASEDIR_LBCS has not been set (not even to a null string), -# print out an error message and exit. -# -if [ -z "${EXTRN_MDL_SYSBASEDIR_LBCS+x}" ]; then - print_err_msg_exit "\ -The variable EXTRN_MDL_SYSBASEDIR_LBCS specifying the system directory -in which to look for the files generated by the external model for LBCs -has not been set for the current combination of machine (MACHINE) and -external model (EXTRN_MDL_NAME_LBCS): - MACHINE = \"$MACHINE\" - EXTRN_MDL_NAME_LBCS = \"${EXTRN_MDL_NAME_LBCS}\"" -fi -} -# -#----------------------------------------------------------------------- -# -# Call the function defined above. -# -#----------------------------------------------------------------------- -# -set_extrn_mdl_params diff --git a/ush/set_ozone_param.sh b/ush/set_ozone_param.sh index 5fb90f839..3844bbb52 100644 --- a/ush/set_ozone_param.sh +++ b/ush/set_ozone_param.sh @@ -34,7 +34,7 @@ function set_ozone_param() { # #----------------------------------------------------------------------- # - { save_shell_opts; set -u -x; } > /dev/null 2>&1 + { save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/ush/set_thompson_mp_fix_files.sh b/ush/set_thompson_mp_fix_files.sh index 558333435..6f9ebf2b7 100644 --- a/ush/set_thompson_mp_fix_files.sh +++ b/ush/set_thompson_mp_fix_files.sh @@ -22,7 +22,7 @@ function set_thompson_mp_fix_files() { # #----------------------------------------------------------------------- # - { save_shell_opts; set -u -x; } > /dev/null 2>&1 + { save_shell_opts; set -u +x; } > /dev/null 2>&1 # #----------------------------------------------------------------------- # diff --git a/ush/setup.sh b/ush/setup.sh index c147f25d4..98ee37c3f 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -64,6 +64,7 @@ cd_vrfy ${scrfunc_dir} . ./set_ozone_param.sh . ./set_thompson_mp_fix_files.sh . ./check_ruc_lsm.sh +. ./extrn_mdl/set_extrn_mdl_default_basedir.sh # #----------------------------------------------------------------------- # @@ -131,245 +132,95 @@ check_var_valid_value "RUN_ENVIR" "valid_vals_RUN_ENVIR" # #----------------------------------------------------------------------- # -# Make sure that VERBOSE is set to a valid value. -# -#----------------------------------------------------------------------- -# -check_var_valid_value "VERBOSE" "valid_vals_VERBOSE" -# -# Set VERBOSE to either "TRUE" or "FALSE" so we don't have to consider -# other valid values later on. -# -VERBOSE=${VERBOSE^^} -if [ "$VERBOSE" = "TRUE" ] || \ - [ "$VERBOSE" = "YES" ]; then - VERBOSE="TRUE" -elif [ "$VERBOSE" = "FALSE" ] || \ - [ "$VERBOSE" = "NO" ]; then - VERBOSE="FALSE" -fi -# -#----------------------------------------------------------------------- -# -# Make sure that USE_CRON_TO_RELAUNCH is set to a valid value. -# -#----------------------------------------------------------------------- -# -check_var_valid_value "USE_CRON_TO_RELAUNCH" "valid_vals_USE_CRON_TO_RELAUNCH" -# -# Set USE_CRON_TO_RELAUNCH to either "TRUE" or "FALSE" so we don't have to consider -# other valid values later on. -# -USE_CRON_TO_RELAUNCH=${USE_CRON_TO_RELAUNCH^^} -if [ "${USE_CRON_TO_RELAUNCH}" = "TRUE" ] || \ - [ "${USE_CRON_TO_RELAUNCH}" = "YES" ]; then - USE_CRON_TO_RELAUNCH="TRUE" -elif [ "${USE_CRON_TO_RELAUNCH}" = "FALSE" ] || \ - [ "${USE_CRON_TO_RELAUNCH}" = "NO" ]; then - USE_CRON_TO_RELAUNCH="FALSE" -fi -# -#----------------------------------------------------------------------- -# -# Make sure that RUN_TASK_MAKE_GRID is set to a valid value. -# -#----------------------------------------------------------------------- -# -check_var_valid_value "RUN_TASK_MAKE_GRID" "valid_vals_RUN_TASK_MAKE_GRID" -# -# Set RUN_TASK_MAKE_GRID to either "TRUE" or "FALSE" so we don't have to -# consider other valid values later on. -# -RUN_TASK_MAKE_GRID=${RUN_TASK_MAKE_GRID^^} -if [ "${RUN_TASK_MAKE_GRID}" = "TRUE" ] || \ - [ "${RUN_TASK_MAKE_GRID}" = "YES" ]; then - RUN_TASK_MAKE_GRID="TRUE" -elif [ "${RUN_TASK_MAKE_GRID}" = "FALSE" ] || \ - [ "${RUN_TASK_MAKE_GRID}" = "NO" ]; then - RUN_TASK_MAKE_GRID="FALSE" -fi -# -#----------------------------------------------------------------------- -# -# Make sure that RUN_TASK_MAKE_OROG is set to a valid value. -# -#----------------------------------------------------------------------- -# -check_var_valid_value "RUN_TASK_MAKE_OROG" "valid_vals_RUN_TASK_MAKE_OROG" -# -# Set RUN_TASK_MAKE_OROG to either "TRUE" or "FALSE" so we don't have to -# consider other valid values later on. -# -RUN_TASK_MAKE_OROG=${RUN_TASK_MAKE_OROG^^} -if [ "${RUN_TASK_MAKE_OROG}" = "TRUE" ] || \ - [ "${RUN_TASK_MAKE_OROG}" = "YES" ]; then - RUN_TASK_MAKE_OROG="TRUE" -elif [ "${RUN_TASK_MAKE_OROG}" = "FALSE" ] || \ - [ "${RUN_TASK_MAKE_OROG}" = "NO" ]; then - RUN_TASK_MAKE_OROG="FALSE" -fi -# -#----------------------------------------------------------------------- -# -# Make sure that RUN_TASK_MAKE_SFC_CLIMO is set to a valid value. -# -#----------------------------------------------------------------------- -# -check_var_valid_value \ - "RUN_TASK_MAKE_SFC_CLIMO" "valid_vals_RUN_TASK_MAKE_SFC_CLIMO" -# -# Set RUN_TASK_MAKE_SFC_CLIMO to either "TRUE" or "FALSE" so we don't -# have to consider other valid values later on. -# -RUN_TASK_MAKE_SFC_CLIMO=${RUN_TASK_MAKE_SFC_CLIMO^^} -if [ "${RUN_TASK_MAKE_SFC_CLIMO}" = "TRUE" ] || \ - [ "${RUN_TASK_MAKE_SFC_CLIMO}" = "YES" ]; then - RUN_TASK_MAKE_SFC_CLIMO="TRUE" -elif [ "${RUN_TASK_MAKE_SFC_CLIMO}" = "FALSE" ] || \ - [ "${RUN_TASK_MAKE_SFC_CLIMO}" = "NO" ]; then - RUN_TASK_MAKE_SFC_CLIMO="FALSE" -fi -# -#----------------------------------------------------------------------- -# -# Make sure that RUN_TASK_RUN_POST is set to a valid value. -# -#----------------------------------------------------------------------- -# -check_var_valid_value \ - "RUN_TASK_RUN_POST" "valid_vals_RUN_TASK_RUN_POST" -# -# Set RUN_TASK_RUN_POST to either "TRUE" or "FALSE" so we don't -# have to consider other valid values later on. -# -RUN_TASK_RUN_POST=${RUN_TASK_RUN_POST^^} -if [ "${RUN_TASK_RUN_POST}" = "TRUE" ] || \ - [ "${RUN_TASK_RUN_POST}" = "YES" ]; then - RUN_TASK_RUN_POST="TRUE" -elif [ "${RUN_TASK_RUN_POST}" = "FALSE" ] || \ - [ "${RUN_TASK_RUN_POST}" = "NO" ]; then - RUN_TASK_RUN_POST="FALSE" -fi -# -#----------------------------------------------------------------------- -# -# Make sure that RUN_TASK_VX_GRIDSTAT is set to a valid value. -# -#----------------------------------------------------------------------- -# -check_var_valid_value "RUN_TASK_VX_GRIDSTAT" "valid_vals_RUN_TASK_VX_GRIDSTAT" -# -# Set RUN_TASK_VX_GRIDSTAT to either "TRUE" or "FALSE" so we don't have to -# consider other valid values later on. -# -RUN_TASK_VX_GRIDSTAT=${RUN_TASK_VX_GRIDSTAT^^} -if [ "${RUN_TASK_VX_GRIDSTAT}" = "TRUE" ] || \ - [ "${RUN_TASK_VX_GRIDSTAT}" = "YES" ]; then - RUN_TASK_VX_GRIDSTAT="TRUE" -elif [ "${RUN_TASK_VX_GRIDSTAT}" = "FALSE" ] || \ - [ "${RUN_TASK_VX_GRIDSTAT}" = "NO" ]; then - RUN_TASK_VX_GRIDSTAT="FALSE" -fi -# -#----------------------------------------------------------------------- -# -# Make sure that RUN_TASK_VX_POINTSTAT is set to a valid value. -# -#----------------------------------------------------------------------- -# -check_var_valid_value "RUN_TASK_VX_POINTSTAT" "valid_vals_RUN_TASK_VX_POINTSTAT" -# -# Set RUN_TASK_VX_POINTSTAT to either "TRUE" or "FALSE" so we don't have to -# consider other valid values later on. -# -RUN_TASK_VX_POINTSTAT=${RUN_TASK_VX_POINTSTAT^^} -if [ "${RUN_TASK_VX_POINTSTAT}" = "TRUE" ] || \ - [ "${RUN_TASK_VX_POINTSTAT}" = "YES" ]; then - RUN_TASK_VX_POINTSTAT="TRUE" -elif [ "${RUN_TASK_VX_POINTSTAT}" = "FALSE" ] || \ - [ "${RUN_TASK_VX_POINTSTAT}" = "NO" ]; then - RUN_TASK_VX_POINTSTAT="FALSE" -fi -# -#----------------------------------------------------------------------- -# -# Make sure that DO_SHUM is set to a valid value. -# -#----------------------------------------------------------------------- -# -check_var_valid_value "DO_SHUM" "valid_vals_DO_SHUM" -# -# Set DO_SHUM to either "TRUE" or "FALSE" so we don't +# Make sure that the various boolean experiment variables are set to +# valid values. Then reset them to either "TRUE" or "FALSE" so we don't # have to consider other valid values later on. # -DO_SHUM=${DO_SHUM^^} -if [ "${DO_SHUM}" = "TRUE" ] || \ - [ "${DO_SHUM}" = "YES" ]; then - DO_SHUM="TRUE" -elif [ "${DO_SHUM}" = "FALSE" ] || \ - [ "${DO_SHUM}" = "NO" ]; then - DO_SHUM="FALSE" -fi -# -#----------------------------------------------------------------------- -# -# Make sure that DO_SPPT is set to a valid value. -# -#----------------------------------------------------------------------- -# -check_var_valid_value "DO_SPPT" "valid_vals_DO_SPPT" -# -# Set DO_SPPT to either "TRUE" or "FALSE" so we don't -# have to consider other valid values later on. -# -DO_SPPT=${DO_SPPT^^} -if [ "${DO_SPPT}" = "TRUE" ] || \ - [ "${DO_SPPT}" = "YES" ]; then - DO_SPPT="TRUE" -elif [ "${DO_SPPT}" = "FALSE" ] || \ - [ "${DO_SPPT}" = "NO" ]; then - DO_SPPT="FALSE" -fi -# #----------------------------------------------------------------------- # -# Make sure that DO_SKEB is set to a valid value. -# -#----------------------------------------------------------------------- -# -check_var_valid_value "DO_SKEB" "valid_vals_DO_SKEB" -# -# Set DO_SKEB to either "TRUE" or "FALSE" so we don't -# have to consider other valid values later on. -# -DO_SKEB=${DO_SKEB^^} -if [ "${DO_SKEB}" = "TRUE" ] || \ - [ "${DO_SKEB}" = "YES" ]; then - DO_SKEB="TRUE" -elif [ "${DO_SKEB}" = "FALSE" ] || \ - [ "${DO_SKEB}" = "NO" ]; then - DO_SKEB="FALSE" -fi -# -#----------------------------------------------------------------------- -# -# Make sure that DO_SPP is set to a valid value. -# -#----------------------------------------------------------------------- -# -check_var_valid_value "DO_SPP" "valid_vals_DO_SPP" -# -# Set DO_SPP to either "TRUE" or "FALSE" so we don't -# have to consider other valid values later on. -# -DO_SPP=${DO_SPP^^} -if [ "${DO_SPP}" = "TRUE" ] || \ - [ "${DO_SPP}" = "YES" ]; then - DO_SPP="TRUE" -elif [ "${DO_SPP}" = "FALSE" ] || \ - [ "${DO_SPP}" = "NO" ]; then - DO_SPP="FALSE" -fi +check_var_valid_value "VERBOSE" "valid_vals_BOOLEAN" +set_boolean_to_TRUE_or_FALSE boolean_value="$VERBOSE" \ + outvarname_boolean="VERBOSE" + +check_var_valid_value "USE_CRON_TO_RELAUNCH" "valid_vals_BOOLEAN" +set_boolean_to_TRUE_or_FALSE boolean_value="${USE_CRON_TO_RELAUNCH}" \ + outvarname_boolean="USE_CRON_TO_RELAUNCH" + +check_var_valid_value "RUN_TASK_MAKE_GRID" "valid_vals_BOOLEAN" +set_boolean_to_TRUE_or_FALSE boolean_value="${RUN_TASK_MAKE_GRID}" \ + outvarname_boolean="RUN_TASK_MAKE_GRID" + +check_var_valid_value "RUN_TASK_MAKE_OROG" "valid_vals_BOOLEAN" +set_boolean_to_TRUE_or_FALSE boolean_value="${RUN_TASK_MAKE_OROG}" \ + outvarname_boolean="RUN_TASK_MAKE_OROG" + +check_var_valid_value "RUN_TASK_MAKE_SFC_CLIMO" "valid_vals_BOOLEAN" +set_boolean_to_TRUE_or_FALSE boolean_value="${RUN_TASK_MAKE_SFC_CLIMO}" \ + outvarname_boolean="RUN_TASK_MAKE_SFC_CLIMO" + +check_var_valid_value "RUN_TASK_GET_EXTRN_ICS" "valid_vals_BOOLEAN" +set_boolean_to_TRUE_or_FALSE boolean_value="${RUN_TASK_GET_EXTRN_ICS}" \ + outvarname_boolean="RUN_TASK_GET_EXTRN_ICS" + +check_var_valid_value "RUN_TASK_GET_EXTRN_LBCS" "valid_vals_BOOLEAN" +set_boolean_to_TRUE_or_FALSE boolean_value="${RUN_TASK_GET_EXTRN_LBCS}" \ + outvarname_boolean="RUN_TASK_GET_EXTRN_LBCS" + +check_var_valid_value "RUN_TASK_RUN_POST" "valid_vals_BOOLEAN" +set_boolean_to_TRUE_or_FALSE boolean_value="${RUN_TASK_RUN_POST}" \ + outvarname_boolean="RUN_TASK_RUN_POST" + +check_var_valid_value "RUN_TASK_VX_GRIDSTAT" "valid_vals_BOOLEAN" +set_boolean_to_TRUE_or_FALSE boolean_value="${RUN_TASK_VX_GRIDSTAT}" \ + outvarname_boolean="RUN_TASK_VX_GRIDSTAT" + +check_var_valid_value "RUN_TASK_VX_POINTSTAT" "valid_vals_BOOLEAN" +set_boolean_to_TRUE_or_FALSE boolean_value="${RUN_TASK_VX_POINTSTAT}" \ + outvarname_boolean="RUN_TASK_VX_POINTSTAT" + +check_var_valid_value "DO_SHUM" "valid_vals_BOOLEAN" +set_boolean_to_TRUE_or_FALSE boolean_value="${DO_SHUM}" \ + outvarname_boolean="DO_SHUM" + +check_var_valid_value "DO_SPPT" "valid_vals_BOOLEAN" +set_boolean_to_TRUE_or_FALSE boolean_value="${DO_SPPT}" \ + outvarname_boolean="DO_SPPT" + +check_var_valid_value "DO_SKEB" "valid_vals_BOOLEAN" +set_boolean_to_TRUE_or_FALSE boolean_value="${DO_SKEB}" \ + outvarname_boolean="DO_SKEB" + +check_var_valid_value "DO_SPP" "valid_vals_BOOLEAN" +set_boolean_to_TRUE_or_FALSE boolean_value="${DO_SPP}" \ + outvarname_boolean="DO_SPP" + +check_var_valid_value "USE_FVCOM" "valid_vals_BOOLEAN" +set_boolean_to_TRUE_or_FALSE boolean_value="${USE_FVCOM}" \ + outvarname_boolean="USE_FVCOM" + +check_var_valid_value "SUB_HOURLY_POST" "valid_vals_BOOLEAN" +set_boolean_to_TRUE_or_FALSE boolean_value="${SUB_HOURLY_POST}" \ + outvarname_boolean="SUB_HOURLY_POST" + +check_var_valid_value "USE_CUSTOM_POST_CONFIG_FILE" "valid_vals_BOOLEAN" +set_boolean_to_TRUE_or_FALSE boolean_value="${USE_CUSTOM_POST_CONFIG_FILE}" \ + outvarname_boolean="USE_CUSTOM_POST_CONFIG_FILE" + +check_var_valid_value "DO_ENSEMBLE" "valid_vals_BOOLEAN" +set_boolean_to_TRUE_or_FALSE boolean_value="${DO_ENSEMBLE}" \ + outvarname_boolean="DO_ENSEMBLE" + +check_var_valid_value "WRITE_DOPOST" "valid_vals_BOOLEAN" +set_boolean_to_TRUE_or_FALSE boolean_value="${WRITE_DOPOST}" \ + outvarname_boolean="WRITE_DOPOST" + +check_var_valid_value "QUILTING" "valid_vals_BOOLEAN" +set_boolean_to_TRUE_or_FALSE boolean_value="$QUILTING" \ + outvarname_boolean="QUILTING" + +check_var_valid_value "PRINT_ESMF" "valid_vals_BOOLEAN" +set_boolean_to_TRUE_or_FALSE boolean_value="${PRINT_ESMF}" \ + outvarname_boolean="PRINT_ESMF" # #----------------------------------------------------------------------- # @@ -405,48 +256,6 @@ fi # #----------------------------------------------------------------------- # -# Make sure that USE_FVCOM is set to a valid value and assign directory -# and file names. -# -#----------------------------------------------------------------------- -# -check_var_valid_value "USE_FVCOM" "valid_vals_USE_FVCOM" -# -# Set USE_FVCOM to either "TRUE" or "FALSE" so we don't have to consider -# other valid values later on. -# -USE_FVCOM=${USE_FVCOM^^} -if [ "$USE_FVCOM" = "TRUE" ] || \ - [ "$USE_FVCOM" = "YES" ]; then - USE_FVCOM="TRUE" -elif [ "$USE_FVCOM" = "FALSE" ] || \ - [ "$USE_FVCOM" = "NO" ]; then - USE_FVCOM="FALSE" -fi -# -#----------------------------------------------------------------------- -# -# Make sure that SUB_HOURLY_POST and DT_SUBHOURLY_POST_MNTS are set to -# valid values. -# -#----------------------------------------------------------------------- -# -check_var_valid_value "SUB_HOURLY_POST" "valid_vals_SUB_HOURLY_POST" -# -# Set SUB_HOURLY_POST to either "TRUE" or "FALSE" so we don't have to consider -# other valid values later on. -# -SUB_HOURLY_POST=${SUB_HOURLY_POST^^} -if [ "${SUB_HOURLY_POST}" = "TRUE" ] || \ - [ "${SUB_HOURLY_POST}" = "YES" ]; then - SUB_HOURLY_POST="TRUE" -elif [ "${SUB_HOURLY_POST}" = "FALSE" ] || \ - [ "${SUB_HOURLY_POST}" = "NO" ]; then - SUB_HOURLY_POST="FALSE" -fi -# -#----------------------------------------------------------------------- -# # Make sure that DOT_OR_USCORE is set to a valid value. # #----------------------------------------------------------------------- @@ -998,27 +807,6 @@ fi # #----------------------------------------------------------------------- # -# Make sure that USE_CUSTOM_POST_CONFIG_FILE is set to a valid value. -# -#----------------------------------------------------------------------- -# -check_var_valid_value \ - "USE_CUSTOM_POST_CONFIG_FILE" "valid_vals_USE_CUSTOM_POST_CONFIG_FILE" -# -# Set USE_CUSTOM_POST_CONFIG_FILE to either "TRUE" or "FALSE" so we don't -# have to consider other valid values later on. -# -USE_CUSTOM_POST_CONFIG_FILE=${USE_CUSTOM_POST_CONFIG_FILE^^} -if [ "$USE_CUSTOM_POST_CONFIG_FILE" = "TRUE" ] || \ - [ "$USE_CUSTOM_POST_CONFIG_FILE" = "YES" ]; then - USE_CUSTOM_POST_CONFIG_FILE="TRUE" -elif [ "$USE_CUSTOM_POST_CONFIG_FILE" = "FALSE" ] || \ - [ "$USE_CUSTOM_POST_CONFIG_FILE" = "NO" ]; then - USE_CUSTOM_POST_CONFIG_FILE="FALSE" -fi -# -#----------------------------------------------------------------------- -# # If using a custom post configuration file, make sure that it exists. # #----------------------------------------------------------------------- @@ -1077,8 +865,8 @@ fi # #----------------------------------------------------------------------- # -LBC_SPEC_FCST_HRS=($( seq ${LBC_SPEC_INTVL_HRS} ${LBC_SPEC_INTVL_HRS} \ - ${FCST_LEN_HRS} )) +LBC_SPEC_FHRS=($( seq ${LBC_SPEC_INTVL_HRS} ${LBC_SPEC_INTVL_HRS} \ + ${FCST_LEN_HRS} )) # #----------------------------------------------------------------------- # @@ -1525,71 +1313,67 @@ NEMS_CONFIG_FP="${EXPTDIR}/${NEMS_CONFIG_FN}" # #----------------------------------------------------------------------- # -# Make sure that USE_USER_STAGED_EXTRN_FILES is set to a valid value. +# Make sure that all the elements of EXTRN_MDL_DATA_SOURCES have valid +# values. # #----------------------------------------------------------------------- # -check_var_valid_value "USE_USER_STAGED_EXTRN_FILES" "valid_vals_USE_USER_STAGED_EXTRN_FILES" +for (( i=0; i<${#EXTRN_MDL_DATA_SOURCES[@]}; i++ )); do + check_var_valid_value "EXTRN_MDL_DATA_SOURCES[$i]" \ + "valid_vals_EXTRN_MDL_DATA_SOURCES" +done # -# Set USE_USER_STAGED_EXTRN_FILES to either "TRUE" or "FALSE" so we don't -# have to consider other valid values later on. +#----------------------------------------------------------------------- # -USE_USER_STAGED_EXTRN_FILES=${USE_USER_STAGED_EXTRN_FILES^^} -if [ "${USE_USER_STAGED_EXTRN_FILES}" = "YES" ]; then - USE_USER_STAGED_EXTRN_FILES="TRUE" -elif [ "${USE_USER_STAGED_EXTRN_FILES}" = "NO" ]; then - USE_USER_STAGED_EXTRN_FILES="FALSE" -fi +# Make sure that EXTRN_MDL_DIR_FILE_LAYOUT is set to a valid value. +# +#----------------------------------------------------------------------- +# +check_var_valid_value "EXTRN_MDL_DIR_FILE_LAYOUT" \ + "valid_vals_EXTRN_MDL_DIR_FILE_LAYOUT" # #----------------------------------------------------------------------- # -# If USE_USER_STAGED_EXTRN_FILES is set to TRUE, make sure that the user- -# specified directories under which the external model files should be -# located actually exist. +# In NCO mode, EXTRN_MDL_DIR_FILE_LAYOUT must be set to "native_to_extrn_mdl". +# Reset it if necessary. # #----------------------------------------------------------------------- # -if [ "${USE_USER_STAGED_EXTRN_FILES}" = "TRUE" ]; then +if [ "${RUN_ENVIR}" = "nco" ]; then - if [ ! -d "${EXTRN_MDL_SOURCE_BASEDIR_ICS}" ]; then - print_err_msg_exit "\ -The directory (EXTRN_MDL_SOURCE_BASEDIR_ICS) in which the user-staged -external model files for generating ICs should be located does not exist: - EXTRN_MDL_SOURCE_BASEDIR_ICS = \"${EXTRN_MDL_SOURCE_BASEDIR_ICS}\"" - fi + if [ "${EXTRN_MDL_DIR_FILE_LAYOUT}" = "user_spec" ]; then + + msg=" +When RUN_ENVIR is set to \"nco\", the workflow assumes that the external +model's directory structure and file naming convention are the ones native +to the external model (as opposed to being user-specified), i.e. that +EXTRN_MDL_DIR_FILE_LAYOUT is set to \"native_to_extrn_mdl\". However, the +current value of EXTRN_MDL_DIR_FILE_LAYOUT is: + EXTRN_MDL_DIR_FILE_LAYOUT = \"${EXTRN_MDL_DIR_FILE_LAYOUT}\" +Resetting EXTRN_MDL_DIR_FILE_LAYOUT to \"native_to_extrn_mdl\". Reset +value is:" + + EXTRN_MDL_DIR_FILE_LAYOUT="native_to_extrn_mdl" + + msg="$msg"" + EXTRN_MDL_DIR_FILE_LAYOUT = \"${EXTRN_MDL_DIR_FILE_LAYOUT}\" +" + + print_info_msg "$msg" - if [ ! -d "${EXTRN_MDL_SOURCE_BASEDIR_LBCS}" ]; then - print_err_msg_exit "\ -The directory (EXTRN_MDL_SOURCE_BASEDIR_LBCS) in which the user-staged -external model files for generating LBCs should be located does not exist: - EXTRN_MDL_SOURCE_BASEDIR_LBCS = \"${EXTRN_MDL_SOURCE_BASEDIR_LBCS}\"" fi fi # #----------------------------------------------------------------------- # -# Make sure that DO_ENSEMBLE is set to a valid value. Then set the names -# of the ensemble members. These will be used to set the ensemble member -# directories. Also, set the full path to the FV3 namelist file corresponding -# to each ensemble member. +# If doing ensemble forecasts, set the names of the ensemble members. +# These will be used to set the ensemble member directories. Also, set +# the full path to the FV3 namelist file corresponding to each ensemble +# member. # #----------------------------------------------------------------------- # -check_var_valid_value "DO_ENSEMBLE" "valid_vals_DO_ENSEMBLE" -# -# Set DO_ENSEMBLE to either "TRUE" or "FALSE" so we don't have to consider -# other valid values later on. -# -DO_ENSEMBLE=${DO_ENSEMBLE^^} -if [ "$DO_ENSEMBLE" = "TRUE" ] || \ - [ "$DO_ENSEMBLE" = "YES" ]; then - DO_ENSEMBLE="TRUE" -elif [ "$DO_ENSEMBLE" = "FALSE" ] || \ - [ "$DO_ENSEMBLE" = "NO" ]; then - DO_ENSEMBLE="FALSE" -fi - NDIGITS_ENSMEM_NAMES="0" ENSMEM_NAMES=("") FV3_NML_ENSMEM_FPS=("") @@ -1951,13 +1735,81 @@ fi # #----------------------------------------------------------------------- # -# Set cycle-independent parameters associated with the external models -# from which we will obtain the ICs and LBCs. +# If EXTRN_MDL_BASEDIRS_ICS is defined as a scalar, then redefine it as +# an array because it is assumed to be an array in the workflow scripts. +# Then get the default value of the external model system base directory +# to use for generating ICs (extrn_mdl_default_basedir) and, if it is +# not an empty string, append it to EXTRN_MDL_BASEDIRS_ICS. [Note that +# this default value depends on both the external model and the machine +# (as well as the experiment mode, i.e. NCO or community, so it may not +# be defined for all such combinations. In such cases, it will get set +# below to a null string.] +# +#----------------------------------------------------------------------- +# +if ! is_array "EXTRN_MDL_BASEDIRS_ICS"; then + EXTRN_MDL_BASEDIRS_ICS=( "${EXTRN_MDL_BASEDIRS_ICS}" ) +fi + +set_extrn_mdl_default_basedir \ + extrn_mdl_name="${EXTRN_MDL_NAME_ICS}" \ + outvarname_extrn_mdl_default_basedir="extrn_mdl_default_basedir" + +if [ ! -z "${extrn_mdl_default_basedir}" ]; then + EXTRN_MDL_BASEDIRS_ICS+=( "${extrn_mdl_default_basedir}" ) +fi +# +# For clarity, remove those elements of EXTRN_MDL_BASEDIRS_ICS that are +# null strings (e.g. left over from the default value). +# +tmp=() +for (( i=0; i<${#EXTRN_MDL_BASEDIRS_ICS[@]}; i++ )); do + basedir="${EXTRN_MDL_BASEDIRS_ICS[$i]}" + if [ ! -z "$basedir" ]; then + tmp+=( "$basedir" ) + fi +done + +if [ "${#tmp[@]}" -ne "0" ]; then + EXTRN_MDL_BASEDIRS_ICS=( "${tmp[@]}" ) +else + EXTRN_MDL_BASEDIRS_ICS=( "" ) +fi +# +#----------------------------------------------------------------------- +# +# Same as above but for LBCs. # #----------------------------------------------------------------------- # -. ./set_extrn_mdl_params.sh +if ! is_array "EXTRN_MDL_BASEDIRS_LBCS"; then + EXTRN_MDL_BASEDIRS_LBCS=( "${EXTRN_MDL_BASEDIRS_LBCS}" ) +fi + +set_extrn_mdl_default_basedir \ + extrn_mdl_name="${EXTRN_MDL_NAME_LBCS}" \ + outvarname_extrn_mdl_default_basedir="extrn_mdl_default_basedir" +if [ ! -z "${extrn_mdl_default_basedir}" ]; then + EXTRN_MDL_BASEDIRS_LBCS+=( "${extrn_mdl_default_basedir}" ) +fi +# +# For clarity, remove those elements of EXTRN_MDL_BASEDIRS_LBCS that are +# null strings (e.g. left over from the default value). +# +tmp=() +for (( i=0; i<${#EXTRN_MDL_BASEDIRS_LBCS[@]}; i++ )); do + basedir="${EXTRN_MDL_BASEDIRS_LBCS[$i]}" + if [ ! -z "$basedir" ]; then + tmp+=( "$basedir" ) + fi +done + +if [ "${#tmp[@]}" -ne "0" ]; then + EXTRN_MDL_BASEDIRS_LBCS=( "${tmp[@]}" ) +else + EXTRN_MDL_BASEDIRS_LBCS=( "" ) +fi # #----------------------------------------------------------------------- # @@ -2029,6 +1881,11 @@ if [ "${GRID_GEN_METHOD}" = "GFDLgrid" ]; then output_varname_jstart_of_t7_with_halo_on_t6sg="JSTART_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG" \ output_varname_jend_of_t7_with_halo_on_t6sg="JEND_OF_RGNL_DOM_WITH_WIDE_HALO_ON_T6SG" # +# Set to default values those grid parameters that do not apply to a +# GFDLgrid type of grid. +# + PAZI="" +# #----------------------------------------------------------------------- # # Now consider Jim Purser's map projection/grid generation method. @@ -2068,7 +1925,6 @@ fi #----------------------------------------------------------------------- # mkdir_vrfy -p "$EXPTDIR" - # #----------------------------------------------------------------------- # @@ -2186,72 +2042,20 @@ fi # #----------------------------------------------------------------------- # -# Make sure that WRITE_DOPOST is set to a valid value. +# If doing inline post (i.e. if WRITE_DOPOST is set to "TRUE"), turn off +# the RUN_POST_TN task(s) since post will now be performed as part of +# the RUN_FCST_TN task. Also, the workflow is not set up to do inline +# post along with sub-hourly post, so if sub_hourly post is turned on, +# quit with an appropriate error message. # #----------------------------------------------------------------------- # -check_var_valid_value "WRITE_DOPOST" "valid_vals_WRITE_DOPOST" -# -# Set WRITE_DOPOST to either "TRUE" or "FALSE" so we don't have to consider -# other valid values later on. -# -WRITE_DOPOST=${WRITE_DOPOST^^} -if [ "$WRITE_DOPOST" = "TRUE" ] || \ - [ "$WRITE_DOPOST" = "YES" ]; then - WRITE_DOPOST="TRUE" - -# Turn off run_post +if [ "${WRITE_DOPOST}" = "TRUE" ]; then RUN_TASK_RUN_POST="FALSE" - -# Check if SUB_HOURLY_POST is on if [ "${SUB_HOURLY_POST}" = "TRUE" ]; then print_err_msg_exit "\ SUB_HOURLY_POST is NOT available with Inline Post yet." fi - -elif [ "$WRITE_DOPOST" = "FALSE" ] || \ - [ "$WRITE_DOPOST" = "NO" ]; then - WRITE_DOPOST="FALSE" -fi -# -#----------------------------------------------------------------------- -# -# Make sure that QUILTING is set to a valid value. -# -#----------------------------------------------------------------------- -# -check_var_valid_value "QUILTING" "valid_vals_QUILTING" -# -# Set QUILTING to either "TRUE" or "FALSE" so we don't have to consider -# other valid values later on. -# -QUILTING=${QUILTING^^} -if [ "$QUILTING" = "TRUE" ] || \ - [ "$QUILTING" = "YES" ]; then - QUILTING="TRUE" -elif [ "$QUILTING" = "FALSE" ] || \ - [ "$QUILTING" = "NO" ]; then - QUILTING="FALSE" -fi -# -#----------------------------------------------------------------------- -# -# Make sure that PRINT_ESMF is set to a valid value. -# -#----------------------------------------------------------------------- -# -check_var_valid_value "PRINT_ESMF" "valid_vals_PRINT_ESMF" -# -# Set PRINT_ESMF to either "TRUE" or "FALSE" so we don't have to consider -# other valid values later on. -# -PRINT_ESMF=${PRINT_ESMF^^} -if [ "${PRINT_ESMF}" = "TRUE" ] || \ - [ "${PRINT_ESMF}" = "YES" ]; then - PRINT_ESMF="TRUE" -elif [ "${PRINT_ESMF}" = "FALSE" ] || \ - [ "${PRINT_ESMF}" = "NO" ]; then - PRINT_ESMF="FALSE" fi # #----------------------------------------------------------------------- @@ -2310,7 +2114,6 @@ fi #----------------------------------------------------------------------- # NNODES_RUN_FCST=$(( (PE_MEMBER01 + PPN_RUN_FCST - 1)/PPN_RUN_FCST )) - # #----------------------------------------------------------------------- # @@ -2358,6 +2161,121 @@ set_thompson_mp_fix_files \ # #----------------------------------------------------------------------- # +# If the first element of EXTRN_MDL_DATA_SOURCES is set to "nomads", try +# fetching the files from NOMADS (and placing them in the appropriate +# location under the experiment directory. If successful, reset the +# flags RUN_TASK_GET_EXTRN_ICS and/or RUN_TASK_GET_EXTRN_LBCS. +# +# Note that as of 20210822, the stand-alone script that is called below +# (get_FV3GFS_grib2_files_from_NOMADS.sh) can only fetch grib2-formatted +# files from the FV3GFS external model. Other models and/or file formats +# may be added later, in which case the code below must be modified. +# +# Also, note the following in the way get_FV3GFS_grib2_files_from_NOMADS.sh +# is called: +# +# 1) This script has to be called from within a subsell [i.e. using +# $( ... )] because it reloads the default experiment configuration +# file, and that would overwrite the experiment variables with default +# values if it were done in the current shell. +# +# 2) Normally, the stdout of a subshell goes into the variable on the +# left-hand-side of the equal sign (in this case "tmp") while the +# stderr goes to the stderr of the parent shell (which is just the +# screen). Here, however, we want both to go to the screen so the +# user can see what the script is doing. The "1>&2" inside the +# subshell redirects stdout from the subshell to go to the same +# location as the stderr (i.e. the screen). This causes nothing to +# go into the variable "tmp" and all messages that are meant to go +# to the screen in the called script to in fact appear on the screen. +# +#----------------------------------------------------------------------- +# +if [ "${EXTRN_MDL_DATA_SOURCES[0]}" = "nomads" ]; then + + all_cdates_str="( "$( printf "\"%s\" " "${ALL_CDATES[@]}" )")" + lbc_spec_fhrs_str="( "$( printf "\"%s\" " "${LBC_SPEC_FHRS[@]}" )")" +# +# First, consider ICs. +# + if [ "${EXTRN_MDL_NAME_ICS}" = "FV3GFS" ] && \ + [ "${FV3GFS_FILE_FMT_ICS}" = "grib2" ]; then + + print_info_msg " +Attempting to fetch external model files for ICs from NOMADS for: + EXTRN_MDL_NAME_ICS = \"${EXTRN_MDL_NAME_ICS}\" + FV3GFS_FILE_FMT_ICS = \"${FV3GFS_FILE_FMT_ICS}\"" +# +# Call the stand-alone script that gets FV3GFS grib2 files from NOMADS. +# See explanation above for details of what is happening here. +# + tmp=$( \ + $USHDIR/get_FV3GFS_grib2_files_from_NOMADS.sh \ + machine="$MACHINE" \ + all_cdates="${all_cdates_str}" \ + file_types="ANL" \ + data_basedir="$EXPTDIR" \ + data_relsubdir="${EXTRN_MDL_NAME_ICS}/for_ICS" 1>&2 \ + ) + + if [ "$?" -eq "0" ]; then + print_info_msg " +Successfully fetched external model files for ICs from NOMADS. Resetting +RUN_TASK_GET_EXTRN_ICS to \"FALSE\" since the \"${GET_EXTRN_ICS_TN}\" task +no longer needs to be run..." + RUN_TASK_GET_EXTRN_ICS="FALSE" + else + print_info_msg " +Fetching of external model files for ICs from NOMADS failed. Another +attempt will be made by the \"${GET_EXTRN_ICS_TN}\" workflow task, possibly +from a source other than NOMADS." + fi + + fi +# +# Now consider LBCs. +# + if [ "${EXTRN_MDL_NAME_LBCS}" = "FV3GFS" ] && \ + [ "${FV3GFS_FILE_FMT_LBCS}" = "grib2" ]; then + + print_info_msg " +Attempting to fetch external model files for LBCs from NOMADS for: + EXTRN_MDL_NAME_LBCS = \"${EXTRN_MDL_NAME_LBCS}\" + FV3GFS_FILE_FMT_LBCS = \"${FV3GFS_FILE_FMT_LBCS}\" +" +# +# Call the stand-alone script that gets FV3GFS grib2 files from NOMADS. +# See explanation above for details of what is happening here. +# + tmp=$( \ + $USHDIR/get_FV3GFS_grib2_files_from_NOMADS.sh \ + machine="$MACHINE" \ + all_cdates="${all_cdates_str}" \ + file_types="FCST" \ + data_basedir="$EXPTDIR" \ + data_relsubdir="${EXTRN_MDL_NAME_LBCS}/for_LBCS" \ + lbc_spec_fhrs="${lbc_spec_fhrs_str}" 1>&2 \ + ) + + if [ "$?" -eq "0" ]; then + print_info_msg " +Successfully fetched external model files for ICs from NOMADS. Resetting +RUN_TASK_GET_EXTRN_LBCS to \"FALSE\" since the \"${GET_EXTRN_LBCS_TN}\" task +no longer needs to be run..." + RUN_TASK_GET_EXTRN_LBCS="FALSE" + else + print_info_msg " +Fetching of external model files for LBCs from NOMADS failed. Another +attempt will be made by the \"${GET_EXTRN_LBCS_TN}\" workflow task, possibly +from a source other than NOMADS." + fi + + fi + +fi +# +#----------------------------------------------------------------------- +# # Generate the shell script that will appear in the experiment directory # (EXPTDIR) and will contain definitions of variables needed by the va- # rious scripts in the workflow. We refer to this as the experiment/ @@ -2385,7 +2303,7 @@ set_thompson_mp_fix_files \ # #----------------------------------------------------------------------- # -GLOBAL_VAR_DEFNS_FP="$EXPTDIR/$GLOBAL_VAR_DEFNS_FN" +GLOBAL_VAR_DEFNS_FP="$EXPTDIR/${GLOBAL_VAR_DEFNS_FN}" cp_vrfy $USHDIR/${EXPT_DEFAULT_CONFIG_FN} ${GLOBAL_VAR_DEFNS_FP} # #----------------------------------------------------------------------- @@ -2877,45 +2795,12 @@ OZONE_PARAM="${OZONE_PARAM}" # #----------------------------------------------------------------------- # -# If USE_USER_STAGED_EXTRN_FILES is set to "FALSE", this is the system -# directory in which the workflow scripts will look for the files generated -# by the external model specified in EXTRN_MDL_NAME_ICS. These files will -# be used to generate the input initial condition and surface files for -# the FV3-LAM. -# -#----------------------------------------------------------------------- -# -EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SYSBASEDIR_ICS}" -# -#----------------------------------------------------------------------- -# -# If USE_USER_STAGED_EXTRN_FILES is set to "FALSE", this is the system -# directory in which the workflow scripts will look for the files generated -# by the external model specified in EXTRN_MDL_NAME_LBCS. These files -# will be used to generate the input lateral boundary condition files for -# the FV3-LAM. -# -#----------------------------------------------------------------------- -# -EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SYSBASEDIR_LBCS}" -# -#----------------------------------------------------------------------- -# -# Shift back in time (in units of hours) of the starting time of the ex- -# ternal model specified in EXTRN_MDL_NAME_LBCS. -# -#----------------------------------------------------------------------- -# -EXTRN_MDL_LBCS_OFFSET_HRS="${EXTRN_MDL_LBCS_OFFSET_HRS}" -# -#----------------------------------------------------------------------- -# # Boundary condition update times (in units of forecast hours). Note that -# LBC_SPEC_FCST_HRS is an array, even if it has only one element. +# LBC_SPEC_FHRS is an array, even if it has only one element. # #----------------------------------------------------------------------- # -LBC_SPEC_FCST_HRS=(${LBC_SPEC_FCST_HRS[@]}) +LBC_SPEC_FHRS=(${LBC_SPEC_FHRS[@]}) # #----------------------------------------------------------------------- # diff --git a/ush/source_util_funcs.sh b/ush/source_util_funcs.sh index bc6a2f083..8113cb147 100644 --- a/ush/source_util_funcs.sh +++ b/ush/source_util_funcs.sh @@ -189,6 +189,24 @@ function source_util_funcs() { #----------------------------------------------------------------------- # . ${bashutils_dir}/create_symlink_to_file.sh +# +#----------------------------------------------------------------------- +# +# Source the file containing the function that parses a given cycle date +# and returns its parts. +# +#----------------------------------------------------------------------- +# + . ${bashutils_dir}/parse_cdate.sh +# +#----------------------------------------------------------------------- +# +# Source the file containing the function that ensures that a Boolean +# variable is set to either "TRUE" or "FALSE". +# +#----------------------------------------------------------------------- +# + . ${bashutils_dir}/set_boolean_to_TRUE_or_FALSE.sh } source_util_funcs diff --git a/ush/templates/FV3LAM_wflow.xml b/ush/templates/FV3LAM_wflow.xml index 519855266..85f0ceffb 100644 --- a/ush/templates/FV3LAM_wflow.xml +++ b/ush/templates/FV3LAM_wflow.xml @@ -54,6 +54,8 @@ Flags that specify whether to run the preprocessing and/or verification tasks. + + &LOGDIR;/&MAKE_GRID_TN;_task_complete.txt @@ -367,7 +376,10 @@ MODULES_RUN_TASK_FP script. - + + + &RUN_TASK_GET_EXTRN_LBCS;FALSE + &LOGDIR;/&MAKE_GRID_TN;_task_complete.txt diff --git a/ush/valid_param_vals.sh b/ush/valid_param_vals.sh index 96998bc50..ff8092d4f 100644 --- a/ush/valid_param_vals.sh +++ b/ush/valid_param_vals.sh @@ -1,8 +1,8 @@ # # Define valid values for various global experiment/workflow variables. # +valid_vals_BOOLEAN=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_RUN_ENVIR=("nco" "community") -valid_vals_VERBOSE=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_MACHINE=("WCOSS_CRAY" "WCOSS_DELL_P3" "HERA" "ORION" "JET" "ODIN" "CHEYENNE" "STAMPEDE") valid_vals_SCHED=("slurm" "pbspro" "lsf" "lsfcray" "none") valid_vals_FCST_MODEL=("ufs-weather-model" "fv3gfs_aqm") @@ -41,34 +41,14 @@ valid_vals_CCPP_PHYS_SUITE=( \ valid_vals_GFDLgrid_RES=("48" "96" "192" "384" "768" "1152" "3072") valid_vals_EXTRN_MDL_NAME_ICS=("GSMGFS" "FV3GFS" "RAP" "HRRR" "NAM") valid_vals_EXTRN_MDL_NAME_LBCS=("GSMGFS" "FV3GFS" "RAP" "HRRR" "NAM") -valid_vals_USE_USER_STAGED_EXTRN_FILES=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_FV3GFS_FILE_FMT_ICS=("nemsio" "grib2" "netcdf") valid_vals_FV3GFS_FILE_FMT_LBCS=("nemsio" "grib2" "netcdf") valid_vals_GRID_GEN_METHOD=("GFDLgrid" "ESGgrid") -valid_vals_PREEXISTING_DIR_METHOD=("delete" "rename" "quit") +valid_vals_PREEXISTING_DIR_METHOD=("delete" "rename" "quit" "none") valid_vals_GTYPE=("regional") valid_vals_WRTCMP_output_grid=("rotated_latlon" "lambert_conformal" "regional_latlon") -valid_vals_RUN_TASK_MAKE_GRID=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") -valid_vals_RUN_TASK_MAKE_OROG=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") -valid_vals_RUN_TASK_MAKE_SFC_CLIMO=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") -valid_vals_RUN_TASK_RUN_POST=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") -valid_vals_RUN_TASK_VX_GRIDSTAT=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") -valid_vals_RUN_TASK_VX_POINTSTAT=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") -valid_vals_WRITE_DOPOST=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") -valid_vals_QUILTING=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") -valid_vals_PRINT_ESMF=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") -valid_vals_USE_CRON_TO_RELAUNCH=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_DOT_OR_USCORE=("." "_") -valid_vals_NOMADS=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") -valid_vals_NOMADS_file_type=("GRIB2" "grib2" "NEMSIO" "nemsio") -valid_vals_DO_ENSEMBLE=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") -valid_vals_USE_CUSTOM_POST_CONFIG_FILE=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") -valid_vals_DO_SHUM=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") -valid_vals_DO_SPPT=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") -valid_vals_DO_SPP=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") -valid_vals_DO_SKEB=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") -valid_vals_USE_ZMTNBLCK=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") -valid_vals_USE_FVCOM=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_COMPILER=("intel" "gnu") -valid_vals_SUB_HOURLY_POST=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_DT_SUBHOURLY_POST_MNTS=("1" "01" "2" "02" "3" "03" "4" "04" "5" "05" "6" "06" "10" "12" "15" "20" "30") +valid_vals_EXTRN_MDL_DATA_SOURCES=("disk" "noaa_hpss" "nomads") +valid_vals_EXTRN_MDL_DIR_FILE_LAYOUT=("native_to_extrn_mdl" "user_spec")