From 4715f05e9b5aa8106dd127dc7eabb7eea33333db Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Wed, 2 Feb 2022 02:14:42 +0000 Subject: [PATCH 01/29] WIP --- jobs/JREGIONAL_GET_EXTRN_MDL_FILES | 283 ++++----- scripts/exregional_get_extrn_mdl_files.sh | 675 +--------------------- ush/config_defaults.sh | 4 +- ush/templates/FV3LAM_wflow.xml | 2 - 4 files changed, 148 insertions(+), 816 deletions(-) diff --git a/jobs/JREGIONAL_GET_EXTRN_MDL_FILES b/jobs/JREGIONAL_GET_EXTRN_MDL_FILES index a452062c5..2a3ec0b07 100755 --- a/jobs/JREGIONAL_GET_EXTRN_MDL_FILES +++ b/jobs/JREGIONAL_GET_EXTRN_MDL_FILES @@ -33,15 +33,6 @@ # #----------------------------------------------------------------------- # -# Source the file defining the function that will be used to set various -# external-model-associated variables. -# -#----------------------------------------------------------------------- -# -. $USHDIR/get_extrn_mdl_file_dir_info.sh -# -#----------------------------------------------------------------------- -# # Save current shell options (in a global array). Then set new options # for this script/function. # @@ -72,86 +63,12 @@ print_info_msg " Entering script: \"${scrfunc_fn}\" In directory: \"${scrfunc_dir}\" -This is the J-job script for the task that copies/fetches to a local -directory (either from disk or HPSS) the external model files from which -initial or boundary condition files for the FV3 will be generated. +This is the J-job script for the task that copies/fetches to a local +directory (either from disk, HPSS, or URL) the external model files from +which initial or boundary condition files for the FV3 will be generated. ========================================================================" -# -#----------------------------------------------------------------------- -# -# Check whether output files from the specified external model (EXTRN_MDL_NAME) -# are available on the specified cycle date and time (CDATE). -# -#----------------------------------------------------------------------- -# -case ${EXTRN_MDL_NAME} in -"GSMGFS") -# The transition date from the GSMGFS to the FV3GFS was 2019061212, i.e. -# this was the first official forecast with the FV3GFS. So we set the -# last CDATE for the GSMGFS to the one 6 hours before this. - cdate_max="2019061206" - if [ "$CDATE" -gt "$cdate_max" ]; then - print_err_msg_exit "\ -Output from the specified external model (EXTRN_MDL_NAME) is not availa- -ble for the specified cycle date and time (CDATE) because the latter is -later than the last forecast date and time (cdate_max) with this model: - EXTRN_MDL_NAME = \"${EXTRN_MDL_NAME}\" - cdate_max = \"${cdate_max}\" - CDATE = \"${CDATE}\"" - fi - ;; - -"FV3GFS") -# The transition date from the GSMGFS to the FV3GFS was 2019061212, i.e. -# this was the first official forecast with the FV3GFS. However, paral- -# lel runs with the FV3GFS go back to 2018121500. So we set the first -# CDATE for the FV3GFS to this date and time. -# CDATE_min="2019061212" - CDATE_min="2018121500" - if [ "$CDATE" -lt "$CDATE_min" ]; then - print_err_msg_exit "\ -Output from the specified external model (EXTRN_MDL_NAME) is not availa- -ble for the specified cycle date and time (CDATE) because the latter is -earlier than the implementation date of this model: - EXTRN_MDL_NAME = \"${EXTRN_MDL_NAME}\" - CDATE_min = \"${CDATE_min}\" - CDATE = \"${CDATE}\"" - fi - ;; -"RAP") -# Examination of the HPSS archives shows that the RAPX data goes back to -# July 01, 2015. - CDATE_min="2015070100" - if [ "$CDATE" -lt "$CDATE_min" ]; then - print_err_msg_exit "\ -Output from the specified external model (EXTRN_MDL_NAME) is not availa- -ble for the specified cycle date and time (CDATE) because the latter is -earlier than the implementation date of this model: - EXTRN_MDL_NAME = \"${EXTRN_MDL_NAME}\" - CDATE_min = \"${CDATE_min}\" - CDATE = \"${CDATE}\"" - fi - ;; - -"HRRR") -# From the HRRR home page (https://rapidrefresh.noaa.gov/hrrr), the -# implementation of the first version of the operational HRRR was -# September 30, 2014. - CDATE_min="2014103000" - if [ "$CDATE" -lt "$CDATE_min" ]; then - print_err_msg_exit "\ -Output from the specified external model (EXTRN_MDL_NAME) is not availa- -ble for the specified cycle date and time (CDATE) because the latter is -earlier than the implementation date of this model: - EXTRN_MDL_NAME = \"${EXTRN_MDL_NAME}\" - CDATE_min = \"${CDATE_min}\" - CDATE = \"${CDATE}\"" - fi - ;; - -esac # #----------------------------------------------------------------------- # @@ -169,7 +86,7 @@ check_var_valid_value "ICS_OR_LBCS" "valid_vals_ICS_OR_LBCS" # # Set parameters for grabbing either the initial conditions from analysis or # forecast files of external models, or the lateral boundary conditions -# from external models. The script has been called to do the work for +# from external models. This script has been called to do the work for # one or the other. # #----------------------------------------------------------------------- @@ -182,88 +99,138 @@ if [ "${ICS_OR_LBCS}" = "ICS" ]; then anl_or_fcst="FCST" time_offset_hrs=${EXTRN_MDL_ICS_OFFSET_HRS:-0} fi + extrn_mdl_name=${EXTRN_MDL_NAME_ICS} + fcst_hrs=${time_offset_hrs} + file_names=${EXTRN_MDL_FILES_ICS[@]} + if [ ${extrn_mdl_name} = FV3GFS ] ; then + file_type=$FV3GFS_FILE_FMT_ICS + fi + input_file_path=${EXTRN_MDL_SOURCE_BASEDIR_ICS:-$EXTRN_MDL_SYSBASEDIR_ICS} + elif [ "${ICS_OR_LBCS}" = "LBCS" ]; then anl_or_fcst="FCST" time_offset_hrs=${EXTRN_MDL_LBCS_OFFSET_HRS:-0} + extrn_mdl_name=${EXTRN_MDL_NAME_LBCS} + fcst_hrs="${time_offset_hrs} ${FCST_LEN_HRS} ${LBC_SPEC_INTVL_HRS}" + file_names=${EXTRN_MDL_FILES_LBCS[@]} + if [ ${extrn_mdl_name} = FV3GFS ] ; then + file_type=$FV3GFS_FILE_FMT_LBCS + fi + input_file_path=${EXTRN_MDL_SOURCE_BASEDIR_LBCS:-$EXTRN_MDL_SYSBASEDIR_LBCS} fi + # #----------------------------------------------------------------------- # -# Create the directory where the exetrnal model files should be stored -# -#----------------------------------------------------------------------- -# -extrn_mdl_staging_dir="${CYCLE_DIR}/${EXTRN_MDL_NAME}/for_${ICS_OR_LBCS}" -mkdir_vrfy -p "${extrn_mdl_staging_dir}" -cd_vrfy "${extrn_mdl_staging_dir}" -# -#----------------------------------------------------------------------- -# -# Call the function that sets various external-model-associated variables. -# See the function defintion file for the definitions of these variables. -# -#----------------------------------------------------------------------- -# -get_extrn_mdl_file_dir_info \ - extrn_mdl_name="${EXTRN_MDL_NAME}" \ - anl_or_fcst="${anl_or_fcst}" \ - cdate_FV3LAM="${CDATE}" \ - time_offset_hrs="${time_offset_hrs}" \ - varname_extrn_mdl_cdate="extrn_mdl_cdate" \ - varname_extrn_mdl_lbc_spec_fhrs="extrn_mdl_lbc_spec_fhrs" \ - varname_extrn_mdl_fns_on_disk="extrn_mdl_fns_on_disk" \ - varname_extrn_mdl_fns_in_arcv="extrn_mdl_fns_in_arcv" \ - varname_extrn_mdl_sysdir="extrn_mdl_sysdir" \ - varname_extrn_mdl_arcv_fmt="extrn_mdl_arcv_fmt" \ - varname_extrn_mdl_arcv_fns="extrn_mdl_arcv_fns" \ - varname_extrn_mdl_arcv_fps="extrn_mdl_arcv_fps" \ - varname_extrn_mdl_arcvrel_dir="extrn_mdl_arcvrel_dir" || \ -print_err_msg_exit "\ -Call to function get_extrn_mdl_file_dir_info failed." -# -#----------------------------------------------------------------------- -# -# Set the directory in which to check for the external model files (which -# we refer to here as the "source" directory) to the default one set above -# for the current machine and external model. +# Set the external model start time # #----------------------------------------------------------------------- # -extrn_mdl_source_dir="${extrn_mdl_sysdir}" + +hh=${CDATE:8:2} +yyyymmdd=${CDATE:0:8} +extrn_mdl_cdate=$( $DATE_UTIL --utc --date "${yyyymmdd} ${hh} UTC - ${time_offset_hrs} hours" "+%Y%m%d%H" ) + # #----------------------------------------------------------------------- # -# If the user has specified that the external model files to be used for -# generating ICs or LBCs are staged, then reset extrn_mdl_source_dir to -# the user-specified directory in which these files are staged, and reset -# extrn_mdl_fns_on_disk to the user-specified array containing the names -# of the files. +# Check whether output files from the specified external model +# (extrn_mdl_name) are available on the specified cycle date and time +# (extrn_mdl_cdate). # #----------------------------------------------------------------------- # -if [ "${USE_USER_STAGED_EXTRN_FILES}" = "TRUE" ]; then - if [ "${ICS_OR_LBCS}" = "ICS" ]; then - extrn_mdl_source_dir="${EXTRN_MDL_SOURCE_BASEDIR_ICS}/$CDATE" - extrn_mdl_fns_on_disk=( $( printf "%s " "${EXTRN_MDL_FILES_ICS[@]}" )) - elif [ "${ICS_OR_LBCS}" = "LBCS" ]; then - extrn_mdl_source_dir="${EXTRN_MDL_SOURCE_BASEDIR_LBCS}/$CDATE" - extrn_mdl_fns_on_disk=( $( printf "%s " "${EXTRN_MDL_FILES_LBCS[@]}" )) +function data_unavailble() { + + local name cdate end_date min_max + + name=$1 + cdate=$2 + end_date=$3 + min_max=$4 + + if [ ${min_max} = max ]; then + msg="\ +Output from the specified external model (extrn_mdl_name) is not availa- +ble for the specified cycle date and time (extrn_mdl_cdate) because the latter is +later than the last forecast date and time (cdate_max) with this model: + extrn_mdl_name = \"${name}\" + cdate_max = \"${end_date}\" + extrn_mdl_cdate = \"${cdate}\"" + + elif [ ${min_max} = min ]; then + msg="\ +Output from the specified external model (extrn_mdl_name) is not availa- +ble for the specified cycle date and time (extrn_mdl_cdate) because the latter is +earlier than the implementation date of this model: + extrn_mdl_name = \"${name}\" + CDATE_min = \"${end_date}\" + extrn_mdl_cdate = \"${cdate}\"" fi - if [ ! -d "${extrn_mdl_source_dir}" ]; then + echo ${msg} +} + + +case ${extrn_mdl_name} in + +"GSMGFS") +# The transition date from the GSMGFS to the FV3GFS was 2019061212, i.e. +# this was the first official forecast with the FV3GFS. So we set the +# last CDATE for the GSMGFS to the one 6 hours before this. + cdate_max="2019061206" + if [ "$extrn_mdl_cdate" -gt "$cdate_max" ]; then print_err_msg_exit "\ -The directory extrn_mdl_source_dir containing the user-staged external -model files does not exist: - extrn_mdl_source_dir = \"${extrn_mdl_source_dir}\" -Please ensure that the directory specified by extrn_mdl_source_dir exists -and that all the files specified in the array extrn_mdl_fns_on_disk exist -within it: - extrn_mdl_source_dir = \"${extrn_mdl_source_dir}\" - extrn_mdl_fns_on_disk = ( $( printf "\"%s\" " "${extrn_mdl_fns_on_disk[@]}" ))" + $(data_unavailble $extrn_mdl_name $extrn_mdl_cdate $cdate_max max)" fi + ;; -fi +"FV3GFS") +# The transition date from the GSMGFS to the FV3GFS was 2019061212, i.e. +# this was the first official forecast with the FV3GFS. However, paral- +# lel runs with the FV3GFS go back to 2018121500. So we set the first +# extrn_mdl_cdate for the FV3GFS to this date and time. +# CDATE_min="2019061212" + CDATE_min="2018121500" + if [ "$extrn_mdl_cdate" -lt "$CDATE_min" ]; then + print_err_msg_exit "\ + $(data_unavailble $extrn_mdl_name $extrn_mdl_cdate $cdate_min min)" + fi + ;; + +"RAP") +# Examination of the HPSS archives shows that the RAPX data goes back to +# July 01, 2015. + CDATE_min="2015070100" + if [ "$extrn_mdl_cdate" -lt "$CDATE_min" ]; then + print_err_msg_exit "\ + $(data_unavailble $extrn_mdl_name $extrn_mdl_cdate $cdate_min min)" + fi + ;; + +"HRRR") +# From the HRRR home page (https://rapidrefresh.noaa.gov/hrrr), the +# implementation of the first version of the operational HRRR was +# September 30, 2014. + CDATE_min="2014103000" + if [ "$extrn_mdl_cdate" -lt "$CDATE_min" ]; then + print_err_msg_exit "\ + $(data_unavailble $extrn_mdl_name $extrn_mdl_cdate $cdate_min min)" + fi + ;; + +esac +# +#----------------------------------------------------------------------- +# +# Create the directory where the exetrnal model files should be stored +# +#----------------------------------------------------------------------- +# +extrn_mdl_staging_dir="${CYCLE_DIR}/${extrn_mdl_name}/for_${ICS_OR_LBCS}" +mkdir_vrfy -p "${extrn_mdl_staging_dir}" +cd_vrfy "${extrn_mdl_staging_dir}" # #----------------------------------------------------------------------- # @@ -271,25 +238,15 @@ fi # #----------------------------------------------------------------------- # -extrn_mdl_lbc_spec_fhrs_str="( "$( printf "\"%s\" " "${extrn_mdl_lbc_spec_fhrs[@]}" )")" -extrn_mdl_fns_on_disk_str="( "$( printf "\"%s\" " "${extrn_mdl_fns_on_disk[@]}" )")" -extrn_mdl_fns_in_arcv_str="( "$( printf "\"%s\" " "${extrn_mdl_fns_in_arcv[@]}" )")" -extrn_mdl_arcv_fns_str="( "$( printf "\"%s\" " "${extrn_mdl_arcv_fns[@]}" )")" -extrn_mdl_arcv_fps_str="( "$( printf "\"%s\" " "${extrn_mdl_arcv_fps[@]}" )")" - $SCRIPTSDIR/exregional_get_extrn_mdl_files.sh \ - ics_or_lbcs="${ICS_OR_LBCS}" \ - use_user_staged_extrn_files="${USE_USER_STAGED_EXTRN_FILES}" \ + anl_or_fcst="${anl_or_fcst}" \ extrn_mdl_cdate="${extrn_mdl_cdate}" \ - extrn_mdl_lbc_spec_fhrs="${extrn_mdl_lbc_spec_fhrs_str}" \ - extrn_mdl_fns_on_disk="${extrn_mdl_fns_on_disk_str}" \ - extrn_mdl_fns_in_arcv="${extrn_mdl_fns_in_arcv_str}" \ - extrn_mdl_source_dir="${extrn_mdl_source_dir}" \ + extrn_mdl_name="${extrn_mdl_name}" \ extrn_mdl_staging_dir="${extrn_mdl_staging_dir}" \ - extrn_mdl_arcv_fmt="${extrn_mdl_arcv_fmt}" \ - extrn_mdl_arcv_fns="${extrn_mdl_arcv_fns_str}" \ - extrn_mdl_arcv_fps="${extrn_mdl_arcv_fps_str}" \ - extrn_mdl_arcvrel_dir="${extrn_mdl_arcvrel_dir}" || \ + fcst_hrs="${fcst_hrs}" \ + file_names="${file_names}" \ + file_type="${file_type}" \ + input_file_path="${input_file_path}" || print_err_msg_exit "\ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." # diff --git a/scripts/exregional_get_extrn_mdl_files.sh b/scripts/exregional_get_extrn_mdl_files.sh index ac5127eb8..1b151e436 100755 --- a/scripts/exregional_get_extrn_mdl_files.sh +++ b/scripts/exregional_get_extrn_mdl_files.sh @@ -56,18 +56,14 @@ boundary condition files for the FV3 will be generated. #----------------------------------------------------------------------- # valid_args=( \ -"ics_or_lbcs" \ -"use_user_staged_extrn_files" \ +"anl_or_fcst" \ "extrn_mdl_cdate" \ -"extrn_mdl_lbc_spec_fhrs" \ -"extrn_mdl_fns_on_disk" \ -"extrn_mdl_fns_in_arcv" \ -"extrn_mdl_source_dir" \ +"extrn_mdl_name" \ "extrn_mdl_staging_dir" \ -"extrn_mdl_arcv_fmt" \ -"extrn_mdl_arcv_fns" \ -"extrn_mdl_arcv_fps" \ -"extrn_mdl_arcvrel_dir" \ +"fcst_hrs" \ +"file_names" \ +"file_type" \ +"input_file_path" \ ) process_args valid_args "$@" # @@ -80,665 +76,46 @@ process_args valid_args "$@" #----------------------------------------------------------------------- # print_input_args valid_args -# -#----------------------------------------------------------------------- -# -# Set num_files_to_copy to the number of external model files that need -# to be copied or linked to from/at a location on disk. Then set -# extrn_mdl_fps_on_disk to the full paths of the external model files -# on disk. -# -#----------------------------------------------------------------------- -# -num_files_to_copy="${#extrn_mdl_fns_on_disk[@]}" -prefix="${extrn_mdl_source_dir}/" -extrn_mdl_fps_on_disk=( "${extrn_mdl_fns_on_disk[@]/#/$prefix}" ) -# -#----------------------------------------------------------------------- -# -# Loop through the list of external model files and check whether they -# all exist on disk. The counter num_files_found_on_disk keeps track of -# the number of external model files that were actually found on disk in -# the directory specified by extrn_mdl_source_dir. -# -# If the location extrn_mdl_source_dir is a user-specified directory -# (i.e. if use_user_staged_extrn_files is set to "TRUE"), then if/when we -# encounter the first file that does not exist, we exit the script with -# an error message. If extrn_mdl_source_dir is a system directory (i.e. -# if use_user_staged_extrn_files is not set to "TRUE"), then if/when we -# encounter the first file that does not exist or exists but is younger -# than a certain age, we break out of the loop and try to fetch all the -# necessary external model files from HPSS. The age cutoff is to ensure -# that files are not still being written to. -# -#----------------------------------------------------------------------- -# -num_files_found_on_disk="0" -min_age="5" # Minimum file age, in minutes. -for fp in "${extrn_mdl_fps_on_disk[@]}"; do - # - # If the external model file exists, then... - # - if [ -f "$fp" ]; then - # - # Increment the counter that keeps track of the number of external - # model files found on disk and print out an informational message. - # - num_files_found_on_disk=$(( num_files_found_on_disk+1 )) - print_info_msg " -File fp exists on disk: - fp = \"$fp\"" - # - # If we are NOT searching for user-staged external model files, then - # we also check that the current file is at least min_age minutes old. - # If not, we try searching for all the external model files on HPSS. - # - if [ "${use_user_staged_extrn_files}" != "TRUE" ]; then - if [ $( find "$fp" -mmin +${min_age} ) ]; then - - print_info_msg " -File fp is older than the minimum required age of min_age minutes: - fp = \"$fp\" - min_age = ${min_age} minutes" - - else - - print_info_msg " -File fp is NOT older than the minumum required age of min_age minutes: - fp = \"$fp\" - min_age = ${min_age} minutes -Will try fetching all external model files from HPSS. Not checking -presence and age of remaining external model files on disk." - break - - fi - - fi - # - # If the external model file does not exist, then... - # - else - # - # If an external model file is not found and we are searching for it - # in a user-specified directory, print out an error message and exit. - # - if [ "${use_user_staged_extrn_files}" = "TRUE" ]; then - - print_err_msg_exit "\ -File fp does NOT exist on disk: - fp = \"$fp\" -Please ensure that the directory specified by extrn_mdl_source_dir exists -and that all the files specified in the array extrn_mdl_fns_on_disk exist -within it: - extrn_mdl_source_dir = \"${extrn_mdl_source_dir}\" - extrn_mdl_fns_on_disk = ( $( printf "\"%s\" " "${extrn_mdl_fns_on_disk[@]}" ))" - # - # If an external model file is not found and we are searching for it - # in a system directory, give up on the system directory and try instead - # to get all the external model files from HPSS. - # - else - - print_info_msg " -File fp does NOT exist on disk: - fp = \"$fp\" -Will try fetching all external model files from HPSS. Not checking -presence and age of remaining external model files on disk." - break - - fi - - fi - -done # #----------------------------------------------------------------------- # -# Set the variable (data_src) that determines the source of the external -# model files (either disk or HPSS). +# Set up optional flags for calling retrieve_data.py # #----------------------------------------------------------------------- # -if [ "${num_files_found_on_disk}" -eq "${num_files_to_copy}" ]; then - data_src="disk" -else - data_src="HPSS" -fi +additional_flags="" -if [ ${NOMADS} == "TRUE" ]; then - data_src="online" +if [ -n ${file_names} ] ; then + additional_flags="$additional_flags + --file_names ${file_names}" fi -# -#----------------------------------------------------------------------- -# -# If the source of the external model files is "disk", copy the files -# from the source directory on disk to a staging directory. -# -#----------------------------------------------------------------------- -# -extrn_mdl_fns_on_disk_str="( "$( printf "\"%s\" " "${extrn_mdl_fns_on_disk[@]}" )")" - -if [ "${data_src}" = "disk" ]; then - - if [ "${RUN_ENVIR}" = "nco" ]; then - - print_info_msg " -Creating links in staging directory (extrn_mdl_staging_dir) to external -model files on disk (extrn_mdl_fns_on_disk) in the source directory -(extrn_mdl_source_dir): - extrn_mdl_staging_dir = \"${extrn_mdl_staging_dir}\" - extrn_mdl_source_dir = \"${extrn_mdl_source_dir}\" - extrn_mdl_fns_on_disk = ${extrn_mdl_fns_on_disk_str}" - - ln_vrfy -sf -t ${extrn_mdl_staging_dir} ${extrn_mdl_fps_on_disk[@]} - - else - - # - # If the external model files are user-staged, then simply link to - # them. Otherwise, if they are on the system disk, copy them to the - # staging directory. - # - if [ "${use_user_staged_extrn_files}" = "TRUE" ]; then - print_info_msg " -Creating symlinks in the staging directory (extrn_mdl_staging_dir) to the -external model files on disk (extrn_mdl_fns_on_disk) in the source directory -(extrn_mdl_source_dir): - extrn_mdl_source_dir = \"${extrn_mdl_source_dir}\" - extrn_mdl_fns_on_disk = ${extrn_mdl_fns_on_disk_str} - extrn_mdl_staging_dir = \"${extrn_mdl_staging_dir}\"" - ln_vrfy -sf -t ${extrn_mdl_staging_dir} ${extrn_mdl_fps_on_disk[@]} - else - print_info_msg " -Copying external model files on disk (extrn_mdl_fns_on_disk) from source -directory (extrn_mdl_source_dir) to staging directory (extrn_mdl_staging_dir): - extrn_mdl_source_dir = \"${extrn_mdl_source_dir}\" - extrn_mdl_fns_on_disk = ${extrn_mdl_fns_on_disk_str} - extrn_mdl_staging_dir = \"${extrn_mdl_staging_dir}\"" - cp_vrfy ${extrn_mdl_fps_on_disk[@]} ${extrn_mdl_staging_dir} - fi - - fi -# -#----------------------------------------------------------------------- -# -# Print message indicating successful completion of script. -# -#----------------------------------------------------------------------- -# - if [ "${ics_or_lbcs}" = "ICS" ]; then - - print_info_msg " -======================================================================== -Successfully copied or linked to external model files on disk needed for -generating initial conditions and surface fields for the FV3 forecast!!! - -Exiting script: \"${scrfunc_fn}\" -In directory: \"${scrfunc_dir}\" -========================================================================" - - elif [ "${ics_or_lbcs}" = "LBCS" ]; then - - print_info_msg " -======================================================================== -Successfully copied or linked to external model files on disk needed for -generating lateral boundary conditions for the FV3 forecast!!! - -Exiting script: \"${scrfunc_fn}\" -In directory: \"${scrfunc_dir}\" -========================================================================" - - fi -# -#----------------------------------------------------------------------- -# -# If the source of the external model files is "HPSS", fetch them from -# HPSS. -# -#----------------------------------------------------------------------- -# -elif [ "${data_src}" = "HPSS" ]; then -# -#----------------------------------------------------------------------- -# -# Set extrn_mdl_fps_in_arcv to the full paths within the archive files of -# the external model files. -# -#----------------------------------------------------------------------- -# - prefix=${extrn_mdl_arcvrel_dir:+${extrn_mdl_arcvrel_dir}/} - extrn_mdl_fps_in_arcv=( "${extrn_mdl_fns_in_arcv[@]/#/$prefix}" ) - - extrn_mdl_fps_in_arcv_str="( "$( printf "\"%s\" " "${extrn_mdl_fps_in_arcv[@]}" )")" - extrn_mdl_arcv_fps_str="( "$( printf "\"%s\" " "${extrn_mdl_arcv_fps[@]}" )")" - - print_info_msg " -Fetching external model files from HPSS. The full paths to these files -in the archive file(s) (extrn_mdl_fps_in_arcv), the archive files on HPSS -in which these files are stored (extrn_mdl_arcv_fps), and the staging -directory to which they will be copied (extrn_mdl_staging_dir) are: - extrn_mdl_fps_in_arcv = ${extrn_mdl_fps_in_arcv_str} - extrn_mdl_arcv_fps = ${extrn_mdl_arcv_fps_str} - extrn_mdl_staging_dir = \"${extrn_mdl_staging_dir}\"" -# -#----------------------------------------------------------------------- -# -# Get the number of archive files to consider. -# -#----------------------------------------------------------------------- -# - num_arcv_files="${#extrn_mdl_arcv_fps[@]}" -# -#----------------------------------------------------------------------- -# -# Consider the case of the archive file to be fetched from HPSS being in -# tar format. -# -#----------------------------------------------------------------------- -# - if [ "${extrn_mdl_arcv_fmt}" = "tar" ]; then -# -#----------------------------------------------------------------------- -# -# Loop through the set of archive files specified in extrn_mdl_arcv_fps -# and extract a subset of the specified external model files from each. -# -#----------------------------------------------------------------------- -# - num_files_to_extract="${#extrn_mdl_fps_in_arcv[@]}" - - for (( narcv=0; narcv<${num_arcv_files}; narcv++ )); do - - narcv_formatted=$( printf "%02d" $narcv ) - arcv_fp="${extrn_mdl_arcv_fps[$narcv]}" -# -# Before trying to extract (a subset of) the external model files from -# the current tar archive file (which is on HPSS), create a list of those -# external model files that are stored in the current tar archive file. -# For this purpose, we first use the "htar -tvf" command to list all the -# external model files that are in the current archive file and store the -# result in a log file. (This command also indirectly checks whether the -# archive file exists on HPSS.) We then grep this log file for each -# external model file and create a list containing only those external -# model files that exist in the current archive. -# -# Note that the "htar -tvf" command will fail if the tar archive file -# itself doesn't exist on HPSS, but it won't fail if any of the external -# model file names passed to it don't exist in the archive file. In the -# latter case, the missing files' names simply won't appear in the log -# file. -# - htar_log_fn="log.htar_tvf.${narcv_formatted}" - htar -tvf ${arcv_fp} ${extrn_mdl_fps_in_arcv[@]} >& ${htar_log_fn} || \ - print_err_msg_exit "\ -htar file list operation (\"htar -tvf ...\") failed. Check the log file -htar_log_fn in the staging directory (extrn_mdl_staging_di)r for details: - extrn_mdl_staging_dir = \"${extrn_mdl_staging_dir}\" - htar_log_fn = \"${htar_log_fn}\"" - - i=0 - files_in_crnt_arcv=() - for (( nfile=0; nfile<${num_files_to_extract}; nfile++ )); do - extrn_mdl_fp="${extrn_mdl_fps_in_arcv[$nfile]}" -# grep -n ${extrn_mdl_fp} ${htar_log_fn} 2>&1 && { \ - grep -n ${extrn_mdl_fp} ${htar_log_fn} > /dev/null 2>&1 && { \ - files_in_crnt_arcv[$i]="${extrn_mdl_fp}"; \ - i=$((i+1)); \ - } - done -# -# If none of the external model files were found in the current archive -# file, print out an error message and exit. -# - num_files_in_crnt_arcv=${#files_in_crnt_arcv[@]} - if [ ${num_files_in_crnt_arcv} -eq 0 ]; then - extrn_mdl_fps_in_arcv_str="( "$( printf "\"%s\" " "${extrn_mdl_fps_in_arcv[@]}" )")" - print_err_msg_exit "\ -The current archive file (arcv_fp) does not contain any of the external -model files listed in extrn_mdl_fps_in_arcv: - arcv_fp = \"${arcv_fp}\" - extrn_mdl_fps_in_arcv = ${extrn_mdl_fps_in_arcv_str} -The archive file should contain at least one external model file; otherwise, -it would not be needed." - fi -# -# Extract from the current tar archive file on HPSS all the external model -# files that exist in that archive file. Also, save the output of the -# "htar -xvf" command in a log file for debugging (if necessary). -# - htar_log_fn="log.htar_xvf.${narcv_formatted}" - htar -xvf ${arcv_fp} ${files_in_crnt_arcv[@]} >& ${htar_log_fn} || \ - print_err_msg_exit "\ -htar file extract operation (\"htar -xvf ...\") failed. Check the log -file htar_log_fn in the staging directory (extrn_mdl_staging_dir) for -details: - extrn_mdl_staging_dir = \"${extrn_mdl_staging_dir}\" - htar_log_fn = \"${htar_log_fn}\"" -# -# Note that the htar file extract operation above may return with a 0 -# exit code (success) even if one or more (or all) external model files -# that it is supposed to contain were not extracted. The names of those -# files that were not extracted will not be listed in the log file. Thus, -# we now check whether the log file contains the name of each external -# model file that should have been extracted. If any are missing, we -# print out a message and exit the script because initial condition and -# surface field files needed by FV3 cannot be generated without all the -# external model files. -# - for fp in "${files_in_crnt_arcv[@]}"; do -# -# If the file path is absolute (i.e. starts with a "/"), then drop the -# leading "/" because htar strips it before writing the file path to the -# log file. -# - fp=${fp#/} - - grep -n "${fp}" "${htar_log_fn}" > /dev/null 2>&1 || \ - print_err_msg_exit "\ -External model file fp not extracted from tar archive file arcv_fp: - arcv_fp = \"${arcv_fp}\" - fp = \"$fp\" -Check the log file htar_log_fn in the staging directory (extrn_mdl_staging_dir) -for details: - extrn_mdl_staging_dir = \"${extrn_mdl_staging_dir}\" - htar_log_fn = \"${htar_log_fn}\"" - done - - done -# -#----------------------------------------------------------------------- -# -# For each external model file that was supposed to have been extracted -# from the set of specified archive files, loop through the extraction -# log files and check that it appears exactly once in one of the log files. -# If it doesn't appear at all, then it means that file was not extracted, -# and if it appears more than once, then something else is wrong. In -# either case, print out an error message and exit. -# -#----------------------------------------------------------------------- -# - for (( nfile=0; nfile<${num_files_to_extract}; nfile++ )); do - extrn_mdl_fp="${extrn_mdl_fps_in_arcv[$nfile]}" -# -# If the file path is absolute (i.e. starts with a "/"), then drop the -# leading "/" because htar strips it before writing the file path to the -# log file. -# - extrn_mdl_fp=${extrn_mdl_fp#/} - - num_occurs=0 - for (( narcv=0; narcv<${num_arcv_files}; narcv++ )); do - narcv_formatted=$( printf "%02d" $narcv ) - htar_log_fn="log.htar_xvf.${narcv_formatted}" - grep -n ${extrn_mdl_fp} ${htar_log_fn} > /dev/null 2>&1 && { \ - num_occurs=$((num_occurs+1)); \ - } - done - - if [ ${num_occurs} -eq 0 ]; then - print_err_msg_exit "\ -The current external model file (extrn_mdl_fp) does not appear in any of -the archive extraction log files: - extrn_mdl_fp = \"${extrn_mdl_fp}\" -Thus, it was not extracted, likely because it doesn't exist in any of the -archive files." - elif [ ${num_occurs} -gt 1 ]; then - print_err_msg_exit "\ -The current external model file (extrn_mdl_fp) appears more than once in -the archive extraction log files: - extrn_mdl_fp = \"${extrn_mdl_fp}\" -The number of times it occurs in the log files is: - num_occurs = ${num_occurs} -Thus, it was extracted from more than one archive file, with the last one -that was extracted overwriting all previous ones. This should normally -not happen." - fi - - done -# -#----------------------------------------------------------------------- -# -# If extrn_mdl_arcvrel_dir is not set to the current directory (i.e. it -# is not equal to "."), then the htar command will have created the -# subdirectory "./${extrn_mdl_arcvrel_dir}" under the current directory -# and placed the extracted files there. In that case, we move these -# extracted files back to the current directory and then remove the -# subdirectory created by htar. -# -#----------------------------------------------------------------------- -# - if [ "${extrn_mdl_arcvrel_dir}" != "." ]; then -# -# The code below works if extrn_mdl_arcvrel_dir starts with a "/" or a -# "./", which are the only case encountered thus far. The code may have -# to be modified to accomodate other cases. -# - if [ "${extrn_mdl_arcvrel_dir:0:1}" = "/" ] || \ - [ "${extrn_mdl_arcvrel_dir:0:2}" = "./" ]; then -# -# Strip the "/" or "./" from the beginning of extrn_mdl_arcvrel_dir to -# obtain the relative directory from which to move the extracted files -# to the current directory. Then move the files. -# - rel_dir=$( printf "%s" "${extrn_mdl_arcvrel_dir}" | \ - $SED -r 's%^(\/|\.\/)([^/]*)(.*)%\2\3%' ) - mv_vrfy ${rel_dir}/* . -# -# Get the first subdirectory in rel_dir, i.e. the subdirectory before the -# first forward slash. This is the subdirectory that we want to remove -# since it no longer contains any files (only subdirectories). Then remove -# it. -# - subdir_to_remove=$( printf "%s" "${rel_dir}" | \ - $SED -r 's%^([^/]*)(.*)%\1%' ) - rm_vrfy -rf ./${subdir_to_remove} -# -# If extrn_mdl_arcvrel_dir does not start with a "/" (and it is not -# equal to "."), then print out an error message and exit. -# - else - - print_err_msg_exit "\ -The archive-relative directory specified by extrn_mdl_arcvrel_dir [i.e. -the directory \"within\" the tar file(s) listed in extrn_mdl_arcv_fps] is -not the current directory (i.e. it is not \".\"), and it does not start -with a \"/\" or a \"./\": - extrn_mdl_arcvrel_dir = \"${extrn_mdl_arcvrel_dir}\" - extrn_mdl_arcv_fps = ${extrn_mdl_arcv_fps_str} -This script must be modified to account for this case." - - fi - - fi -# -#----------------------------------------------------------------------- -# -# Consider the case of the archive file to be fetched from HPSS being in -# zip format. -# -#----------------------------------------------------------------------- -# - elif [ "${extrn_mdl_arcv_fmt}" = "zip" ]; then -# -#----------------------------------------------------------------------- -# -# For archive files that are in "zip" format files, the array extrn_mdl_arcv_fps -# containing the list of archive files should contain only one element, -# i.e. there should be only one archive file to consider. Check for this. -# If this ever changes (e.g. due to the way an external model that uses -# the "zip" format archives its output files on HPSS), the code below must -# be modified to loop over all archive files. -# -#----------------------------------------------------------------------- -# - if [ "${num_arcv_files}" -gt 1 ]; then - print_err_msg_exit "\ -Currently, this script is coded to handle only one archive file if the -archive file format is specified to be \"zip\", but the number of archive -files (num_arcv_files) passed to this script is greater than 1: - extrn_mdl_arcv_fmt = \"${extrn_mdl_arcv_fmt}\" - num_arcv_files = ${num_arcv_files} -Please modify the script to handle more than one \"zip\" archive file. -Note that code already exists in this script that can handle multiple -archive files if the archive file format is specified to be \"tar\", so -that can be used as a guide for the \"zip\" case." - else - arcv_fn="${extrn_mdl_arcv_fns[0]}" - arcv_fp="${extrn_mdl_arcv_fps[0]}" - fi -# -#----------------------------------------------------------------------- -# -# Fetch the zip archive file from HPSS. -# -#----------------------------------------------------------------------- -# - hsi_log_fn="log.hsi_get" - hsi get "${arcv_fp}" >& ${hsi_log_fn} || \ - print_err_msg_exit "\ -hsi file get operation (\"hsi get ...\") failed. Check the log file -hsi_log_fn in the staging directory (extrn_mdl_staging_dir) for details: - extrn_mdl_staging_dir = \"${extrn_mdl_staging_dir}\" - hsi_log_fn = \"${hsi_log_fn}\"" -# -#----------------------------------------------------------------------- -# -# List the contents of the zip archive file and save the result in a log -# file. -# -#----------------------------------------------------------------------- -# - unzip_log_fn="log.unzip_lv" - unzip -l -v ${arcv_fn} >& ${unzip_log_fn} || \ - print_err_msg_exit "\ -unzip operation to list the contents of the zip archive file arcv_fn in -the staging directory (extrn_mdl_staging_dir) failed. Check the log -file unzip_log_fn in that directory for details: - arcv_fn = \"${arcv_fn}\" - extrn_mdl_staging_dir = \"${extrn_mdl_staging_dir}\" - unzip_log_fn = \"${unzip_log_fn}\"" -# -#----------------------------------------------------------------------- -# -# Check that the log file from the unzip command above contains the name -# of each external model file. If any are missing, then the corresponding -# files are not in the zip file and thus cannot be extracted. In that -# case, print out a message and exit the script because initial condition -# and surface field files for the FV3-LAM cannot be generated without all -# the external model files. -# -#----------------------------------------------------------------------- -# - for fp in "${extrn_mdl_fps_in_arcv[@]}"; do - grep -n "${fp}" "${unzip_log_fn}" > /dev/null 2>&1 || \ - print_err_msg_exit "\ -External model file fp does not exist in the zip archive file arcv_fn in -the staging directory (extrn_mdl_staging_dir). Check the log file -unzip_log_fn in that directory for the contents of the zip archive: - extrn_mdl_staging_dir = \"${extrn_mdl_staging_dir}\" - arcv_fn = \"${arcv_fn}\" - fp = \"$fp\" - unzip_log_fn = \"${unzip_log_fn}\"" - done -# -#----------------------------------------------------------------------- -# -# Extract the external model files from the zip file on HPSS. Note that -# the -o flag to unzip is needed to overwrite existing files. Otherwise, -# unzip will wait for user input as to whether the existing files should -# be overwritten. -# -#----------------------------------------------------------------------- -# - unzip_log_fn="log.unzip" - unzip -o "${arcv_fn}" ${extrn_mdl_fps_in_arcv[@]} >& ${unzip_log_fn} || \ - print_err_msg_exit "\ -unzip file extract operation (\"unzip -o ...\") failed. Check the log -file unzip_log_fn in the staging directory (extrn_mdl_staging_dir) for -details: - extrn_mdl_staging_dir = \"${extrn_mdl_staging_dir}\" - unzip_log_fn = \"${unzip_log_fn}\"" -# -# NOTE: -# If extrn_mdl_arcvrel_dir is not empty, the unzip command above will -# create a subdirectory under extrn_mdl_staging_dir and place the external -# model files there. We have not encountered this for the RAP and HRRR -# models, but it may happen for other models in the future. In that case, -# extra code must be included here to move the external model files from -# the subdirectory up to extrn_mdl_staging_dir and then the subdirectory -# (analogous to what is done above for the case of extrn_mdl_arcv_fmt set -# to "tar". -# - - fi -# -#----------------------------------------------------------------------- -# -# Print message indicating successful completion of script. -# -#----------------------------------------------------------------------- -# - if [ "${ics_or_lbcs}" = "ICS" ]; then - - print_info_msg " -======================================================================== -External model files needed for generating initial condition and surface -fields for the FV3-LAM successfully fetched from HPSS!!! - -Exiting script: \"${scrfunc_fn}\" -In directory: \"${scrfunc_dir}\" -========================================================================" - - elif [ "${ics_or_lbcs}" = "LBCS" ]; then - - print_info_msg " -======================================================================== -External model files needed for generating lateral boundary conditions -on the halo of the FV3-LAM's regional grid successfully fetched from -HPSS!!! - -Exiting script: \"${scrfunc_fn}\" -In directory: \"${scrfunc_dir}\" -========================================================================" - - fi - -elif [ "${data_src}" = "online" ]; then - print_info_msg " -======================================================================== -getting data from online nomads data sources -========================================================================" +if [ -n ${file_type} ] ; then + additional_flags="$additional_flags + --file_type ${file_type}" +fi # #----------------------------------------------------------------------- # -# Set extrn_mdl_fps to the full paths within the archive files of the -# external model output files. +# Call ush script to retrieve files # #----------------------------------------------------------------------- # - prefix=${extrn_mdl_arcvrel_dir:+${extrn_mdl_arcvrel_dir}/} - extrn_mdl_fps=( "${extrn_mdl_fns_on_disk[@]/#/$prefix}" ) - - extrn_mdl_fps_str="( "$( printf "\"%s\" " "${extrn_mdl_fps[@]}" )")" - - print_info_msg " -Getting external model files from nomads: - extrn_mdl_fps= ${extrn_mdl_fps_str}" - num_files_to_extract="${#extrn_mdl_fps[@]}" - wget_LOG_FN="log.wget.txt" - for (( nfile=0; nfile<${num_files_to_extract}; nfile++ )); do - cp ../../../${extrn_mdl_fps[$nfile]} . || \ - print_err_msg_exit "\ - onlie file ${extrn_mdl_fps[$nfile]} not found." - done +${USHDIR}/retrieve_data.py \ + --anl_or_fcst ${anl_or_fcst} \ + --config ${USHDIR}/templates/data_locations.yml \ + --cycle_date ${extrn_mdl_cdate} \ + --data_stores disk hpss aws \ + --external_model ${EXTRN_MDL_NAME} \ + --fcst_hrs fcst_hrs \ + --output_path ${extrn_mdl_staging_dir} \ + --input_file_path ${input_file_path} \ + ${additional_flags} - -fi # #----------------------------------------------------------------------- # diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index e7860e021..eb6fa9775 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -707,9 +707,9 @@ EXTRN_MDL_SYSBASEDIR_LBCS='' #----------------------------------------------------------------------- # USE_USER_STAGED_EXTRN_FILES="FALSE" -EXTRN_MDL_SOURCE_BASEDIR_ICS="/base/dir/containing/user/staged/extrn/mdl/files/for/ICs" +EXTRN_MDL_SOURCE_BASEDIR_ICS="" EXTRN_MDL_FILES_ICS=( "ICS_file1" "ICS_file2" "..." ) -EXTRN_MDL_SOURCE_BASEDIR_LBCS="/base/dir/containing/user/staged/extrn/mdl/files/for/LBCs" +EXTRN_MDL_SOURCE_BASEDIR_LBCS="" EXTRN_MDL_FILES_LBCS=( "LBCS_file1" "LBCS_file2" "..." ) # #----------------------------------------------------------------------- diff --git a/ush/templates/FV3LAM_wflow.xml b/ush/templates/FV3LAM_wflow.xml index 05b97e7e8..28447060f 100644 --- a/ush/templates/FV3LAM_wflow.xml +++ b/ush/templates/FV3LAM_wflow.xml @@ -281,7 +281,6 @@ MODULES_RUN_TASK_FP script. PDY@Y@m@d CDATE@Y@m@d@H CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H - EXTRN_MDL_NAME{{ extrn_mdl_name_ics }} ICS_OR_LBCSICS @@ -311,7 +310,6 @@ MODULES_RUN_TASK_FP script. PDY@Y@m@d CDATE@Y@m@d@H CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H - EXTRN_MDL_NAME{{ extrn_mdl_name_lbcs }} ICS_OR_LBCSLBCS From 7a352990f67fb2f464b446c9a66cc80446486b25 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Fri, 11 Feb 2022 21:18:50 +0000 Subject: [PATCH 02/29] Remove unused file. --- ush/get_extrn_mdl_file_dir_info.sh | 681 ----------------------------- 1 file changed, 681 deletions(-) delete mode 100755 ush/get_extrn_mdl_file_dir_info.sh diff --git a/ush/get_extrn_mdl_file_dir_info.sh b/ush/get_extrn_mdl_file_dir_info.sh deleted file mode 100755 index 30a6e2d88..000000000 --- a/ush/get_extrn_mdl_file_dir_info.sh +++ /dev/null @@ -1,681 +0,0 @@ -# -#----------------------------------------------------------------------- -# -# Source the variable definitions file and the bash utility functions. -# -#----------------------------------------------------------------------- -# -. ${GLOBAL_VAR_DEFNS_FP} -. $USHDIR/source_util_funcs.sh -# -#----------------------------------------------------------------------- -# -# This file defines a function that is used to obtain information (e.g. -# output file names, system and mass store file and/or directory names) -# for a specified external model, analysis or forecast, and cycle date. -# See the usage statement below for this function should be called and -# the definitions of the input parameters. -# -#----------------------------------------------------------------------- -# - -usage () { - -echo " -Incorrect number of arguments specified: - - Function name: \"${func_name}\" - Number of arguments specified: $# - -Usage: - - ${func_name} \ - extrn_mdl_name \ - anl_or_fcst \ - cdate_FV3LAM \ - time_offset_hrs \ - varname_extrn_mdl_cdate \ - varname_extrn_mdl_lbc_spec_fhrs \ - varname_extrn_mdl_fns \ - varname_extrn_mdl_sysdir \ - varname_extrn_mdl_arcv_fmt \ - varname_extrn_mdl_arcv_fns \ - varname_extrn_mdl_arcv_fps \ - varname_extrn_mdl_arcvrel_dir - -where the arguments are defined as follows: - - extrn_mdl_name: - Name of the external model, i.e. the name of the model providing the - fields from which files containing initial conditions, surface fields, - and/or lateral boundary conditions for the FV3-LAM will be generated. - - anl_or_fcst: - Flag that specifies whether the external model files we are interested - in obtaining are analysis or forecast files. - - cdate_FV3LAM: - The cycle date and time (hours only) for which we want to obtain file - and directory information. This has the form YYYYMMDDHH, where YYYY - is the four-digit starting year of the cycle, MM is the two-digit - month, DD is the two-digit day of the month, and HH is the two-digit - hour of day. - - time_offset_hrs: - The number of hours by which to shift back in time the start time of - the external model forecast from the specified cycle start time of the - FV3-LAM (cdate_FV3LAM). When getting directory and file information on - external model analysis files, this is normally set to 0. When get- - ting directory and file information on external model forecast files, - this may be set to a nonzero value to obtain information for an exter- - nal model run that started time_offset_hrs hours before cdate_FV3LAM - (instead of exactly at cdate_FV3LAM). Note that in this case, the - forecast hours (relative to the external model run's start time) at - which the lateral boundary conditions will be updated must be shifted - forward by time_offset_hrs hours relative to those for the FV3-LAM in - order to make up for the backward-in-time shift in the starting time - of the external model. - - varname_extrn_mdl_cdate: - Name of the global variable that will contain the starting date and - hour of the external model run. - - varname_extrn_mdl_lbc_spec_fhrs: - Name of the global variable that will contain the forecast hours (re- - lative to the starting time of the external model run, which is earli- - er than that of the FV3-LAM by time_offset_hrs hours) at which lateral - boundary condition (LBC) output files are obtained from the external - model (and will be used to update the LBCs of the FV3-LAM). - - varname_extrn_mdl_fns_on_disk: - Name of the global variable that will contain the expected names of - the external model output files on disk. - - varname_extrn_mdl_fns_in_arcv: - Name of the global variable that will contain the expected names of - the external model output files on NOAA HPSS. - - varname_extrn_mdl_sysdir: - Name of the global variable that will contain the system directory in - which the externaml model output files may be stored. - - varname_extrn_mdl_arcv_fmt: - Name of the global variable that will contain the format of the ar- - chive file on HPSS in which the externaml model output files may be - stored. - - varname_extrn_mdl_arcv_fns: - Name of the global variable that will contain the name of the archive - file on HPSS in which the externaml model output files may be stored. - - varname_extrn_mdl_arcv_fps: - Name of the global variable that will contain the full path to the ar- - chive file on HPSS in which the externaml model output files may be - stored. - - varname_extrn_mdl_arcvrel_dir: - Name of the global variable that will contain the archive-relative di- - rectory, i.e. the directory \"inside\" the archive file in which the ex- - ternal model output files may be stored. -" -} - -function quit_unless_user_spec_data() { - if [ "${USE_USER_STAGED_EXTRN_FILES}" != "TRUE" ]; then - print_err_msg_exit "\ -The system directory in which to look for external model output files -has not been specified for this external model and machine combination: - extrn_mdl_name = \"${extrn_mdl_name}\" - MACHINE = \"$MACHINE\"" - fi -} - -function get_extrn_mdl_file_dir_info() { - - { save_shell_opts; set -u +x; } > /dev/null 2>&1 - - local func_name="${FUNCNAME[0]}" - # - #----------------------------------------------------------------------- - # - # Specify the set of valid argument names for this script/function. Then - # process the arguments provided to this script/function (which should - # consist of a set of name-value pairs of the form arg1="value1", etc). - # - #----------------------------------------------------------------------- - # - local valid_args=( \ - "extrn_mdl_name" \ - "anl_or_fcst" \ - "cdate_FV3LAM" \ - "time_offset_hrs" \ - "varname_extrn_mdl_cdate" \ - "varname_extrn_mdl_lbc_spec_fhrs" \ - "varname_extrn_mdl_fns_on_disk" \ - "varname_extrn_mdl_fns_in_arcv" \ - "varname_extrn_mdl_sysdir" \ - "varname_extrn_mdl_arcv_fmt" \ - "varname_extrn_mdl_arcv_fns" \ - "varname_extrn_mdl_arcv_fps" \ - "varname_extrn_mdl_arcvrel_dir" \ - ) - process_args valid_args "$@" - - if [ "$#" -ne "13" ]; then - print_err_msg_exit $(usage) - fi - - # - #----------------------------------------------------------------------- - # - # For debugging purposes, print out values of arguments passed to this - # script/function. Note that these will be printed out only if VERBOSE - # is set to TRUE. - # - #----------------------------------------------------------------------- - # - print_input_args valid_args - - # - #----------------------------------------------------------------------- - # - # Declare additional local variables. - # - #----------------------------------------------------------------------- - # - local yyyy yy mm dd hh mn yyyymmdd ddd \ - lbc_spec_fhrs i num_fhrs \ - fcst_hhh fcst_hh fcst_mn \ - prefix suffix fns fns_on_disk fns_in_arcv \ - sysbasedir sysdir \ - arcv_dir arcv_fmt arcv_fns arcv_fps arcvrel_dir - - anl_or_fcst=$(echo_uppercase $anl_or_fcst) - valid_vals_anl_or_fcst=( "ANL" "FCST" ) - check_var_valid_value "anl_or_fcst" "valid_vals_anl_or_fcst" - # - #----------------------------------------------------------------------- - # - # Set cdate to the start time for the external model being used. - # - #----------------------------------------------------------------------- - # - hh=${cdate_FV3LAM:8:2} - yyyymmdd=${cdate_FV3LAM:0:8} - - # Adjust time for offset - cdate=$( $DATE_UTIL --utc --date "${yyyymmdd} ${hh} UTC - ${time_offset_hrs} hours" "+%Y%m%d%H" ) - - yyyy=${cdate:0:4} - yy=${yyyy:2:4} - mm=${cdate:4:2} - dd=${cdate:6:2} - hh=${cdate:8:2} - mn="00" - yyyymmdd=${cdate:0:8} - # ddd is the Julian day -- not 3 digit day of month - ddd=$( $DATE_UTIL --utc --date "${yyyy}-${mm}-${dd} ${hh}:${mn} UTC" "+%j" ) - # - #----------------------------------------------------------------------- - # - # Initialize lbc_spec_fhrs array. Skip the initial time, since it is - # handled separately. - # - #----------------------------------------------------------------------- - # - lbc_spec_fhrs=( "" ) - - if [ "${anl_or_fcst}" = "FCST" ]; then - - lbc_spec_fhrs=( "${LBC_SPEC_FCST_HRS[@]}" ) - - num_fhrs=${#lbc_spec_fhrs[@]} - for (( i=0; i<=$((num_fhrs-1)); i++ )); do - # Add in offset to account for shift in initial time - lbc_spec_fhrs[$i]=$(( ${lbc_spec_fhrs[$i]} + time_offset_hrs )) - done - - fi - # - #----------------------------------------------------------------------- - # - # The model may be started with a variety of file types from FV3GFS. - # Set that file type now - # - #----------------------------------------------------------------------- - # - - if [ "${anl_or_fcst}" = "ANL" ]; then - fv3gfs_file_fmt="${FV3GFS_FILE_FMT_ICS}" - elif [ "${anl_or_fcst}" = "FCST" ]; then - fv3gfs_file_fmt="${FV3GFS_FILE_FMT_LBCS}" - fi - - # - #----------------------------------------------------------------------- - # - # Generate an array of file names expected from the external model - # Assume that filenames in archive and on disk are the same, unless - # otherwise specified (primarily on Jet). - # - #----------------------------------------------------------------------- - # - declare -a fns_on_disk - declare -a fns_in_arcv - case "${anl_or_fcst}" in - - "ANL") - - fcst_hh="00" - fcst_mn="00" - - case "${extrn_mdl_name}" in - - "GSMGFS") - fns_in_arcv=("gfs.t${hh}z.atmanl.nemsio" "gfs.t${hh}z.sfcanl.nemsio") - ;; - - "FV3GFS") - case "${fv3gfs_file_fmt}" in - "nemsio") - fns_in_arcv=("gfs.t${hh}z.atmanl.nemsio" "gfs.t${hh}z.sfcanl.nemsio") - - # File names are prefixed with a date time on Jet - if [ "${MACHINE}" = "JET" ]; then - prefix="${yy}${ddd}${hh}00" - fns_on_disk=( ${fns_in_arcv[@]/#/$prefix}) - fi - ;; - "grib2") - fns_in_arcv=( "gfs.t${hh}z.pgrb2.0p25.f000" ) - ;; - "netcdf") - fns_in_arcv=("gfs.t${hh}z.atmanl.nc" "gfs.t${hh}z.sfcanl.nc") - # File names are prefixed with a date time on Jet - if [ "${MACHINE}" = "JET" ]; then - prefix="${yy}${ddd}${hh}00" - fns_on_disk=( ${fns_in_arcv[@]/#/$prefix}) - fi - ;; - esac - ;; - - "RAP") - ;& # Fall through. RAP and HRRR follow same naming rules - - "HRRR") - fns_in_arcv=( "${yy}${ddd}${hh}${mn}${fcst_hh}${fcst_mn}" ) - if [ "${MACHINE}" = "JET" ]; then - fns_on_disk=( "${yy}${ddd}${hh}${mn}${fcst_mn}${fcst_hh}${fcst_mn}" ) - fi - ;; - - "NAM") - fns=( "" ) - fns_in_arcv=( "nam.t${hh}z.bgrdsf${fcst_hh}.tm00" ) - ;; - - *) - if [ "${USE_USER_STAGED_EXTRN_FILES}" != "TRUE" ]; then - print_err_msg_exit "\ -The external model file names (either on disk or in archive files) have -not yet been specified for this combination of external model (extrn_mdl_name) -and analysis or forecast (anl_or_fcst): - extrn_mdl_name = \"${extrn_mdl_name}\" - anl_or_fcst = \"${anl_or_fcst}\"" - fi - ;; - - esac # End external model case for ANL files - ;; - - "FCST") - fcst_mn="00" - fcst_hhh=( $( printf "%03d " "${lbc_spec_fhrs[@]}" ) ) - fcst_hh=( $( printf "%02d " "${lbc_spec_fhrs[@]}" ) ) - - case "${extrn_mdl_name}" in - - "GSMGFS") - fn_tmpl="gfs.t${hh}z.atmfFHR3.nemsio" - ;; - - "FV3GFS") - - if [ "${fv3gfs_file_fmt}" = "nemsio" ]; then - fn_tmpl="gfs.t${hh}z.atmfFHR3.nemsio" - if [ "${MACHINE}" = "JET" ]; then - disk_tmpl="${yy}${ddd}${hh}00.gfs.t${hh}z.atmfFHR3.nemsio" - for fhr in ${fcst_hhh[@]} ; do - fns_on_disk+=(${disk_tmpl/FHR3/$fhr}) - done - fi - elif [ "${fv3gfs_file_fmt}" = "grib2" ]; then - fn_tmpl="gfs.t${hh}z.pgrb2.0p25.fFHR3" - elif [ "${fv3gfs_file_fmt}" = "netcdf" ]; then - fn_tmpl="gfs.t${hh}z.atmfFHR3.nc" - if [ "${MACHINE}" = "JET" ]; then - disk_tmpl="${yy}${ddd}${hh}00.gfs.t${hh}z.atmfFHR3.nc" - for fhr in ${fcst_hhh[@]} ; do - fns_on_disk+=(${disk_tmpl/FHR3/$fhr}) - done - fi - fi - ;; - - "RAP") - ;& # Fall through since RAP and HRRR are named the same - - "HRRR") - fn_tmpl="${yy}${ddd}${hh}00FHR200" - if [ "${MACHINE}" = "JET" ]; then - disk_tmpl="${yy}${ddd}${hh}0000FHR2" - for fhr in ${fcst_hhh[@]} ; do - fns_on_disk+=(${disk_tmpl/FHR3/$fhr}) - done - fi - ;; - - "NAM") - fn_tmpl="nam.t${hh}z.bgrdsfFHR3" - ;; - - *) - if [ "${USE_USER_STAGED_EXTRN_FILES}" != "TRUE" ]; then - print_err_msg_exit "\ -The external model file names have not yet been specified for this com- -bination of external model (extrn_mdl_name) and analysis or forecast -(anl_or_fcst): - extrn_mdl_name = \"${extrn_mdl_name}\" - anl_or_fcst = \"${anl_or_fcst}\"" - fi - ;; - - esac # End external model case for FCST files - ;; - esac # End ANL FCST case - - # - # Expand the archive file names for all forecast hours - # - if [ ${anl_or_fcst} = FCST ] ; then - if [[ $fn_tmpl =~ FHR3 ]] ; then - fhrs=( $( printf "%03d " "${lbc_spec_fhrs[@]}" ) ) - tmpl=FHR3 - elif [[ ${fn_tmpl} =~ FHR2 ]] ; then - fhrs=( $( printf "%02d " "${lbc_spec_fhrs[@]}" ) ) - tmpl=FHR2 - else - print_err_msg_exit "\ - Forecast file name templates are expected to contain a template - string, either FHR2 or FHR3" - fi - for fhr in ${fhrs[@]}; do - fns_in_arcv+=(${fn_tmpl/$tmpl/$fhr}) - done - fi - - # Make sure all filenames variables are set. - if [ -z $fns_in_arcv ] ; then - print_err_msg_exit "\ - The script has not set \$fns_in_arcv properly" - fi - - if [ -z ${fns_on_disk:-} ] ; then - fns_on_disk=(${fns_in_arcv[@]}) - fi - # - #----------------------------------------------------------------------- - # - # Set the system directory (i.e. a directory on disk) in which the external - # model output files for the specified cycle date (cdate) may be located. - # Note that this will be used by the calling script only if the output - # files for the specified cdate actually exist at this location. Otherwise, - # the files will be searched for on the mass store (HPSS). - # - #----------------------------------------------------------------------- - # - if [ "${anl_or_fcst}" = "ANL" ]; then - sysdir=$(eval echo ${EXTRN_MDL_SYSBASEDIR_ICS}) - elif [ "${anl_or_fcst}" = "FCST" ]; then - sysdir=$(eval echo ${EXTRN_MDL_SYSBASEDIR_LBCS}) - fi - - # - #----------------------------------------------------------------------- - # - # Set parameters associated with the mass store (HPSS) for the specified - # cycle date (cdate). These consist of: - # - # 1) The type of the archive file (e.g. tar, zip, etc). - # 2) The name of the archive file. - # 3) The full path in HPSS to the archive file. - # 4) The relative directory in the archive file in which the model output - # files are located. - # - # Note that these will be used by the calling script only if the archive - # file for the specified cdate actually exists on HPSS. - # - #----------------------------------------------------------------------- - # - case "${extrn_mdl_name}" in - - "GSMGFS") - arcv_dir="/NCEPPROD/hpssprod/runhistory/rh${yyyy}/${yyyy}${mm}/${yyyymmdd}" - arcv_fmt="tar" - arcv_fns="gpfs_hps_nco_ops_com_gfs_prod_gfs.${cdate}." - if [ "${anl_or_fcst}" = "ANL" ]; then - arcv_fns="${arcv_fns}anl" - arcvrel_dir="." - elif [ "${anl_or_fcst}" = "FCST" ]; then - arcv_fns="${arcv_fns}sigma" - arcvrel_dir="/gpfs/hps/nco/ops/com/gfs/prod/gfs.${yyyymmdd}" - fi - arcv_fns="${arcv_fns}.${arcv_fmt}" - arcv_fps="${arcv_dir}/${arcv_fns}" - ;; - - "FV3GFS") - - if [ "${cdate_FV3LAM}" -lt "2019061200" ]; then - arcv_dir="/NCEPDEV/emc-global/5year/emc.glopara/WCOSS_C/Q2FY19/prfv3rt3/${cdate_FV3LAM}" - arcv_fns="" - elif [ "${cdate_FV3LAM}" -ge "2019061200" ] && \ - [ "${cdate_FV3LAM}" -lt "2020022600" ]; then - arcv_dir="/NCEPPROD/hpssprod/runhistory/rh${yyyy}/${yyyy}${mm}/${yyyymmdd}" - arcv_fns="gpfs_dell1_nco_ops_com_gfs_prod_gfs.${yyyymmdd}_${hh}." - elif [ "${cdate_FV3LAM}" -ge "2020022600" ]; then - arcv_dir="/NCEPPROD/hpssprod/runhistory/rh${yyyy}/${yyyy}${mm}/${yyyymmdd}" - arcv_fns="com_gfs_prod_gfs.${yyyymmdd}_${hh}." - fi - - if [ "${fv3gfs_file_fmt}" = "nemsio" ]; then - - if [ "${anl_or_fcst}" = "ANL" ]; then - arcv_fns="${arcv_fns}gfs_nemsioa" - elif [ "${anl_or_fcst}" = "FCST" ]; then - last_fhr_in_nemsioa="39" - first_lbc_fhr="${lbc_spec_fhrs[0]}" - last_lbc_fhr="${lbc_spec_fhrs[-1]}" - if [ "${last_lbc_fhr}" -le "${last_fhr_in_nemsioa}" ]; then - arcv_fns="${arcv_fns}gfs_nemsioa" - elif [ "${first_lbc_fhr}" -gt "${last_fhr_in_nemsioa}" ]; then - arcv_fns="${arcv_fns}gfs_nemsiob" - else - arcv_fns=( "${arcv_fns}gfs_nemsioa" "${arcv_fns}gfs_nemsiob" ) - fi - fi - - elif [ "${fv3gfs_file_fmt}" = "grib2" ]; then - - arcv_fns="${arcv_fns}gfs_pgrb2" - - elif [ "${fv3gfs_file_fmt}" = "netcdf" ]; then - - if [ "${anl_or_fcst}" = "ANL" ]; then - arcv_fns="${arcv_fns}gfs_nca" - elif [ "${anl_or_fcst}" = "FCST" ]; then - last_fhr_in_netcdfa="39" - first_lbc_fhr="${lbc_spec_fhrs[0]}" - last_lbc_fhr="${lbc_spec_fhrs[-1]}" - if [ "${last_lbc_fhr}" -le "${last_fhr_in_netcdfa}" ]; then - arcv_fns="${arcv_fns}gfs_nca" - elif [ "${first_lbc_fhr}" -gt "${last_fhr_in_netcdfa}" ]; then - arcv_fns="${arcv_fns}gfs_ncb" - else - arcv_fns=( "${arcv_fns}gfs_nca" "${arcv_fns}gfs_ncb" ) - fi - fi - - fi - - arcv_fmt="tar" - - slash_atmos_or_null="" - if [ "${cdate_FV3LAM}" -ge "2021032100" ]; then - slash_atmos_or_null="/atmos" - fi - arcvrel_dir="./gfs.${yyyymmdd}/${hh}${slash_atmos_or_null}" - - is_array arcv_fns - if [ "$?" = "0" ]; then - suffix=".${arcv_fmt}" - arcv_fns=( "${arcv_fns[@]/%/$suffix}" ) - prefix="${arcv_dir}/" - arcv_fps=( "${arcv_fns[@]/#/$prefix}" ) - else - arcv_fns="${arcv_fns}.${arcv_fmt}" - arcv_fps="${arcv_dir}/${arcv_fns}" - fi - ;; - - - "RAP") - # - # Note that this is GSL RAPX data, not operational NCEP RAP data. - # An option for the latter may be added in the future. - # - # The zip archive files for RAPX are named such that the forecast - # files for odd-numbered starting hours (e.g. 01, 03, ..., 23) are - # stored together with the forecast files for the corresponding - # preceding even numbered starting hours (e.g. 00, 02, ..., 22, - # respectively), in an archive file whose name contains only the - # even-numbered hour. Thus, in forming the name of the archive - # file, if the starting hour (hh) is odd, we reduce it by one to get - # the corresponding even-numbered hour and use that to form the - # archive file name. - # - # Convert hh to a decimal (i.e. base-10) number to ovoid octal - # interpretation in bash. - - hh_orig=$hh - hh=$((10#$hh)) - if [ $(($hh%2)) = 1 ]; then - hh=$((hh-1)) - fi - # Archive files use 2-digit forecast hour - hh=$( printf "%02d\n" $hh ) - - arcv_dir="/BMC/fdr/Permanent/${yyyy}/${mm}/${dd}/data/fsl/rap/full/wrfnat" - arcv_fmt="zip" - arcv_fns="${yyyy}${mm}${dd}${hh}00.${arcv_fmt}" - arcv_fps="${arcv_dir}/${arcv_fns}" - arcvrel_dir="" - - # Reset hh to its original value - hh=${hh_orig} - ;; - - "HRRR") - # - # Note that this is GSL HRRRX data, not operational NCEP HRRR data. - # An option for the latter may be added in the future. - # - arcv_dir="/BMC/fdr/Permanent/${yyyy}/${mm}/${dd}/data/fsl/hrrr/conus/wrfnat" - arcv_fmt="zip" - arcv_fns="${yyyy}${mm}${dd}${hh}00.${arcv_fmt}" - arcv_fps="${arcv_dir}/${arcv_fns}" - arcvrel_dir="" - ;; - - "NAM") - arcv_dir="/NCEPPROD/hpssprod/runhistory/rh${yyyy}/${yyyy}${mm}/${yyyymmdd}" - arcv_fmt="tar" - arcv_fns="com_nam_prod_nam.${yyyy}${mm}${dd}${hh}.bgrid.${arcv_fmt}" - arcv_fps="${arcv_dir}/${arcv_fns}" - arcvrel_dir="" - ;; - - *) - print_err_msg_exit "\ -Archive file information has not been specified for this external model: - extrn_mdl_name = \"${extrn_mdl_name}\"" - ;; - - esac - # - # Depending on the experiment configuration, the above code may set - # arcv_fns and arcv_fps to either scalars or arrays. If they are not - # arrays, recast them as arrays because that is what is expected in - # the code below. - # - is_array arcv_fns || arcv_fns=( "${arcv_fns}" ) - is_array arcv_fps || arcv_fps=( "${arcv_fps}" ) - # - #----------------------------------------------------------------------- - # - # Use the eval function to set the output variables. Note that each - # of these is set only if the corresponding input variable specifying - # the name to use for the output variable is not empty. - # - #----------------------------------------------------------------------- - # - if [ ! -z "${varname_extrn_mdl_cdate}" ]; then - eval ${varname_extrn_mdl_cdate}="${cdate}" - fi - - if [ ! -z "${varname_extrn_mdl_lbc_spec_fhrs}" ]; then - lbc_spec_fhrs_str="( "$( printf "\"%s\" " "${lbc_spec_fhrs[@]}" )")" - eval ${varname_extrn_mdl_lbc_spec_fhrs}=${lbc_spec_fhrs_str} - fi - - if [ ! -z "${varname_extrn_mdl_fns_on_disk}" ]; then - fns_on_disk_str="( "$( printf "\"%s\" " "${fns_on_disk[@]}" )")" - eval ${varname_extrn_mdl_fns_on_disk}=${fns_on_disk_str} - fi - - if [ ! -z "${varname_extrn_mdl_fns_in_arcv}" ]; then - fns_in_arcv_str="( "$( printf "\"%s\" " "${fns_in_arcv[@]}" )")" - eval ${varname_extrn_mdl_fns_in_arcv}=${fns_in_arcv_str} - fi - - if [ ! -z "${varname_extrn_mdl_sysdir}" ]; then - eval ${varname_extrn_mdl_sysdir}="${sysdir}" - fi - - if [ ! -z "${varname_extrn_mdl_arcv_fmt}" ]; then - eval ${varname_extrn_mdl_arcv_fmt}="${arcv_fmt}" - fi - - if [ ! -z "${varname_extrn_mdl_arcv_fns}" ]; then - arcv_fns_str="( "$( printf "\"%s\" " "${arcv_fns[@]}" )")" - eval ${varname_extrn_mdl_arcv_fns}=${arcv_fns_str} - fi - - if [ ! -z "${varname_extrn_mdl_arcv_fps}" ]; then - arcv_fps_str="( "$( printf "\"%s\" " "${arcv_fps[@]}" )")" - eval ${varname_extrn_mdl_arcv_fps}=${arcv_fps_str} - fi - - if [ ! -z "${varname_extrn_mdl_arcvrel_dir}" ]; then - eval ${varname_extrn_mdl_arcvrel_dir}="${arcvrel_dir}" - fi - # - #----------------------------------------------------------------------- - # - # Restore the shell options saved at the beginning of this script/function. - # - #----------------------------------------------------------------------- - # - { restore_shell_opts; } > /dev/null 2>&1 -} From 76ee57296f92668454b8775f52fc3d33f523d170 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Fri, 11 Feb 2022 21:19:22 +0000 Subject: [PATCH 03/29] Updating jet module files. --- modulefiles/tasks/jet/get_extrn_ics.local | 1 + modulefiles/tasks/jet/get_extrn_lbcs.local | 1 + modulefiles/tasks/jet/make_grid.local | 4 +--- modulefiles/tasks/jet/make_ics.local | 5 +---- modulefiles/tasks/jet/make_lbcs.local | 5 +---- modulefiles/tasks/jet/miniconda_regional_workflow | 2 ++ modulefiles/tasks/jet/run_fcst.local | 5 +---- 7 files changed, 8 insertions(+), 15 deletions(-) create mode 100644 modulefiles/tasks/jet/miniconda_regional_workflow diff --git a/modulefiles/tasks/jet/get_extrn_ics.local b/modulefiles/tasks/jet/get_extrn_ics.local index 9935033fd..4b0b48cc0 100644 --- a/modulefiles/tasks/jet/get_extrn_ics.local +++ b/modulefiles/tasks/jet/get_extrn_ics.local @@ -6,3 +6,4 @@ module purge module load hpss +module load miniconda_regional_workflow diff --git a/modulefiles/tasks/jet/get_extrn_lbcs.local b/modulefiles/tasks/jet/get_extrn_lbcs.local index 1919f3355..477dfb2e4 100644 --- a/modulefiles/tasks/jet/get_extrn_lbcs.local +++ b/modulefiles/tasks/jet/get_extrn_lbcs.local @@ -6,3 +6,4 @@ module purge module load hpss +module load miniconda_regional_workflow diff --git a/modulefiles/tasks/jet/make_grid.local b/modulefiles/tasks/jet/make_grid.local index 011a832c9..92505cf09 100644 --- a/modulefiles/tasks/jet/make_grid.local +++ b/modulefiles/tasks/jet/make_grid.local @@ -1,5 +1,3 @@ #%Module -module use -a /contrib/miniconda3/modulefiles -module load miniconda3 -setenv SRW_ENV regional_workflow +module load miniconda_regional_workflow diff --git a/modulefiles/tasks/jet/make_ics.local b/modulefiles/tasks/jet/make_ics.local index c3c0f61e5..7d7f9a313 100644 --- a/modulefiles/tasks/jet/make_ics.local +++ b/modulefiles/tasks/jet/make_ics.local @@ -1,7 +1,4 @@ #%Module module load wgrib2/2.0.8 -module use -a /contrib/miniconda3/modulefiles -module load miniconda3 - -setenv SRW_ENV regional_workflow +module load miniconda_regional_workflow diff --git a/modulefiles/tasks/jet/make_lbcs.local b/modulefiles/tasks/jet/make_lbcs.local index c3c0f61e5..7d7f9a313 100644 --- a/modulefiles/tasks/jet/make_lbcs.local +++ b/modulefiles/tasks/jet/make_lbcs.local @@ -1,7 +1,4 @@ #%Module module load wgrib2/2.0.8 -module use -a /contrib/miniconda3/modulefiles -module load miniconda3 - -setenv SRW_ENV regional_workflow +module load miniconda_regional_workflow diff --git a/modulefiles/tasks/jet/miniconda_regional_workflow b/modulefiles/tasks/jet/miniconda_regional_workflow new file mode 100644 index 000000000..61a3a7725 --- /dev/null +++ b/modulefiles/tasks/jet/miniconda_regional_workflow @@ -0,0 +1,2 @@ +#%Module +module load miniconda_regional_workflow diff --git a/modulefiles/tasks/jet/run_fcst.local b/modulefiles/tasks/jet/run_fcst.local index 011a832c9..61a3a7725 100644 --- a/modulefiles/tasks/jet/run_fcst.local +++ b/modulefiles/tasks/jet/run_fcst.local @@ -1,5 +1,2 @@ #%Module -module use -a /contrib/miniconda3/modulefiles -module load miniconda3 - -setenv SRW_ENV regional_workflow +module load miniconda_regional_workflow From c63d92bfdf8613213c214ea4eb93fb6837c286d2 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Fri, 11 Feb 2022 21:20:24 +0000 Subject: [PATCH 04/29] Updating hera module files. --- modulefiles/tasks/hera/get_extrn_ics.local | 1 + modulefiles/tasks/hera/get_extrn_lbcs.local | 1 + modulefiles/tasks/hera/make_grid.local | 4 +--- modulefiles/tasks/hera/make_ics.local | 5 ++--- modulefiles/tasks/hera/make_lbcs.local | 5 ++--- modulefiles/tasks/hera/miniconda_regional_workflow | 2 ++ modulefiles/tasks/hera/run_fcst.local | 5 +---- 7 files changed, 10 insertions(+), 13 deletions(-) create mode 100644 modulefiles/tasks/hera/miniconda_regional_workflow diff --git a/modulefiles/tasks/hera/get_extrn_ics.local b/modulefiles/tasks/hera/get_extrn_ics.local index 9935033fd..4b0b48cc0 100644 --- a/modulefiles/tasks/hera/get_extrn_ics.local +++ b/modulefiles/tasks/hera/get_extrn_ics.local @@ -6,3 +6,4 @@ module purge module load hpss +module load miniconda_regional_workflow diff --git a/modulefiles/tasks/hera/get_extrn_lbcs.local b/modulefiles/tasks/hera/get_extrn_lbcs.local index 1919f3355..477dfb2e4 100644 --- a/modulefiles/tasks/hera/get_extrn_lbcs.local +++ b/modulefiles/tasks/hera/get_extrn_lbcs.local @@ -6,3 +6,4 @@ module purge module load hpss +module load miniconda_regional_workflow diff --git a/modulefiles/tasks/hera/make_grid.local b/modulefiles/tasks/hera/make_grid.local index 011a832c9..92505cf09 100644 --- a/modulefiles/tasks/hera/make_grid.local +++ b/modulefiles/tasks/hera/make_grid.local @@ -1,5 +1,3 @@ #%Module -module use -a /contrib/miniconda3/modulefiles -module load miniconda3 -setenv SRW_ENV regional_workflow +module load miniconda_regional_workflow diff --git a/modulefiles/tasks/hera/make_ics.local b/modulefiles/tasks/hera/make_ics.local index 011a832c9..7d7f9a313 100644 --- a/modulefiles/tasks/hera/make_ics.local +++ b/modulefiles/tasks/hera/make_ics.local @@ -1,5 +1,4 @@ #%Module -module use -a /contrib/miniconda3/modulefiles -module load miniconda3 +module load wgrib2/2.0.8 -setenv SRW_ENV regional_workflow +module load miniconda_regional_workflow diff --git a/modulefiles/tasks/hera/make_lbcs.local b/modulefiles/tasks/hera/make_lbcs.local index 011a832c9..7d7f9a313 100644 --- a/modulefiles/tasks/hera/make_lbcs.local +++ b/modulefiles/tasks/hera/make_lbcs.local @@ -1,5 +1,4 @@ #%Module -module use -a /contrib/miniconda3/modulefiles -module load miniconda3 +module load wgrib2/2.0.8 -setenv SRW_ENV regional_workflow +module load miniconda_regional_workflow diff --git a/modulefiles/tasks/hera/miniconda_regional_workflow b/modulefiles/tasks/hera/miniconda_regional_workflow new file mode 100644 index 000000000..61a3a7725 --- /dev/null +++ b/modulefiles/tasks/hera/miniconda_regional_workflow @@ -0,0 +1,2 @@ +#%Module +module load miniconda_regional_workflow diff --git a/modulefiles/tasks/hera/run_fcst.local b/modulefiles/tasks/hera/run_fcst.local index 011a832c9..61a3a7725 100644 --- a/modulefiles/tasks/hera/run_fcst.local +++ b/modulefiles/tasks/hera/run_fcst.local @@ -1,5 +1,2 @@ #%Module -module use -a /contrib/miniconda3/modulefiles -module load miniconda3 - -setenv SRW_ENV regional_workflow +module load miniconda_regional_workflow From 101d39a61edfec7d090d2ca4b669ab2b9456fbc6 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Fri, 11 Feb 2022 21:24:21 +0000 Subject: [PATCH 05/29] Updating cheyenne module files. --- modulefiles/tasks/cheyenne/get_extrn_ics | 5 ++--- modulefiles/tasks/cheyenne/get_extrn_lbcs | 5 ++--- modulefiles/tasks/cheyenne/make_grid.local | 9 +-------- modulefiles/tasks/cheyenne/make_ics.local | 9 +-------- modulefiles/tasks/cheyenne/make_lbcs.local | 9 +-------- modulefiles/tasks/cheyenne/pylib_regional_workflow | 2 ++ modulefiles/tasks/cheyenne/run_fcst.local | 9 +-------- modulefiles/tasks/cheyenne/run_vx.local | 9 +-------- 8 files changed, 11 insertions(+), 46 deletions(-) create mode 100644 modulefiles/tasks/cheyenne/pylib_regional_workflow diff --git a/modulefiles/tasks/cheyenne/get_extrn_ics b/modulefiles/tasks/cheyenne/get_extrn_ics index 58f82dca1..9f7a4e4f7 100644 --- a/modulefiles/tasks/cheyenne/get_extrn_ics +++ b/modulefiles/tasks/cheyenne/get_extrn_ics @@ -1,5 +1,4 @@ -#%Module##################################################### -## Module file intentionally blank for Cheyenne -############################################################# +#%Module +module load pylib_regional_workflow diff --git a/modulefiles/tasks/cheyenne/get_extrn_lbcs b/modulefiles/tasks/cheyenne/get_extrn_lbcs index 58f82dca1..9f7a4e4f7 100644 --- a/modulefiles/tasks/cheyenne/get_extrn_lbcs +++ b/modulefiles/tasks/cheyenne/get_extrn_lbcs @@ -1,5 +1,4 @@ -#%Module##################################################### -## Module file intentionally blank for Cheyenne -############################################################# +#%Module +module load pylib_regional_workflow diff --git a/modulefiles/tasks/cheyenne/make_grid.local b/modulefiles/tasks/cheyenne/make_grid.local index 4232b8659..2f92a39e5 100644 --- a/modulefiles/tasks/cheyenne/make_grid.local +++ b/modulefiles/tasks/cheyenne/make_grid.local @@ -1,9 +1,2 @@ #%Module -if [module-info mode load] { - system "ncar_pylib /glade/p/ral/jntp/UFS_CAM/ncar_pylib_20200427" -} - -if [module-info mode remove] { - system "deactivate" -} - +module load pylib_regional_workflow diff --git a/modulefiles/tasks/cheyenne/make_ics.local b/modulefiles/tasks/cheyenne/make_ics.local index 4232b8659..2f92a39e5 100644 --- a/modulefiles/tasks/cheyenne/make_ics.local +++ b/modulefiles/tasks/cheyenne/make_ics.local @@ -1,9 +1,2 @@ #%Module -if [module-info mode load] { - system "ncar_pylib /glade/p/ral/jntp/UFS_CAM/ncar_pylib_20200427" -} - -if [module-info mode remove] { - system "deactivate" -} - +module load pylib_regional_workflow diff --git a/modulefiles/tasks/cheyenne/make_lbcs.local b/modulefiles/tasks/cheyenne/make_lbcs.local index 4232b8659..2f92a39e5 100644 --- a/modulefiles/tasks/cheyenne/make_lbcs.local +++ b/modulefiles/tasks/cheyenne/make_lbcs.local @@ -1,9 +1,2 @@ #%Module -if [module-info mode load] { - system "ncar_pylib /glade/p/ral/jntp/UFS_CAM/ncar_pylib_20200427" -} - -if [module-info mode remove] { - system "deactivate" -} - +module load pylib_regional_workflow diff --git a/modulefiles/tasks/cheyenne/pylib_regional_workflow b/modulefiles/tasks/cheyenne/pylib_regional_workflow new file mode 100644 index 000000000..2f92a39e5 --- /dev/null +++ b/modulefiles/tasks/cheyenne/pylib_regional_workflow @@ -0,0 +1,2 @@ +#%Module +module load pylib_regional_workflow diff --git a/modulefiles/tasks/cheyenne/run_fcst.local b/modulefiles/tasks/cheyenne/run_fcst.local index 4232b8659..2f92a39e5 100644 --- a/modulefiles/tasks/cheyenne/run_fcst.local +++ b/modulefiles/tasks/cheyenne/run_fcst.local @@ -1,9 +1,2 @@ #%Module -if [module-info mode load] { - system "ncar_pylib /glade/p/ral/jntp/UFS_CAM/ncar_pylib_20200427" -} - -if [module-info mode remove] { - system "deactivate" -} - +module load pylib_regional_workflow diff --git a/modulefiles/tasks/cheyenne/run_vx.local b/modulefiles/tasks/cheyenne/run_vx.local index 234d79908..8fb3be36d 100644 --- a/modulefiles/tasks/cheyenne/run_vx.local +++ b/modulefiles/tasks/cheyenne/run_vx.local @@ -1,11 +1,4 @@ #%Module - -if [module-info mode load] { - system "ncar_pylib /glade/p/ral/jntp/UFS_SRW_app/ncar_pylib_20200427" -} - -if [module-info mode remove] { - system "deactivate" -} +module load pylib_regional_workflow module use /glade/p/ral/jntp/MET/MET_releases/modulefiles module load met/10.0.0 From 187aca97a4be212c36cd05b66c10571df6e8a970 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Fri, 11 Feb 2022 21:26:55 +0000 Subject: [PATCH 06/29] Updating orion module files. --- modulefiles/tasks/orion/get_extrn_ics.local | 1 + modulefiles/tasks/orion/get_extrn_lbcs.local | 1 + modulefiles/tasks/orion/make_grid.local | 5 +---- modulefiles/tasks/orion/make_ics.local | 5 +---- modulefiles/tasks/orion/make_lbcs.local | 5 +---- modulefiles/tasks/orion/miniconda_regional_workflow | 5 +++++ modulefiles/tasks/orion/run_fcst.local | 5 +---- 7 files changed, 11 insertions(+), 16 deletions(-) create mode 100644 modulefiles/tasks/orion/miniconda_regional_workflow diff --git a/modulefiles/tasks/orion/get_extrn_ics.local b/modulefiles/tasks/orion/get_extrn_ics.local index a9d5b4412..655aa30a2 100644 --- a/modulefiles/tasks/orion/get_extrn_ics.local +++ b/modulefiles/tasks/orion/get_extrn_ics.local @@ -3,4 +3,5 @@ ############################################################# module purge +module load miniconda_regional_workflow diff --git a/modulefiles/tasks/orion/get_extrn_lbcs.local b/modulefiles/tasks/orion/get_extrn_lbcs.local index 09f37151a..a05a15faa 100644 --- a/modulefiles/tasks/orion/get_extrn_lbcs.local +++ b/modulefiles/tasks/orion/get_extrn_lbcs.local @@ -4,3 +4,4 @@ module purge +module load miniconda_regional_workflow diff --git a/modulefiles/tasks/orion/make_grid.local b/modulefiles/tasks/orion/make_grid.local index 4de2a79ca..92505cf09 100644 --- a/modulefiles/tasks/orion/make_grid.local +++ b/modulefiles/tasks/orion/make_grid.local @@ -1,6 +1,3 @@ #%Module -module use -a /apps/contrib/miniconda3-noaa-gsl/modulefiles -module load miniconda3/3.8 - -setenv SRW_ENV regional_workflow +module load miniconda_regional_workflow diff --git a/modulefiles/tasks/orion/make_ics.local b/modulefiles/tasks/orion/make_ics.local index 3703a9ba1..61a3a7725 100644 --- a/modulefiles/tasks/orion/make_ics.local +++ b/modulefiles/tasks/orion/make_ics.local @@ -1,5 +1,2 @@ #%Module -module use -a /apps/contrib/miniconda3-noaa-gsl/modulefiles -module load miniconda3/3.8 - -setenv SRW_ENV regional_workflow +module load miniconda_regional_workflow diff --git a/modulefiles/tasks/orion/make_lbcs.local b/modulefiles/tasks/orion/make_lbcs.local index 3703a9ba1..61a3a7725 100644 --- a/modulefiles/tasks/orion/make_lbcs.local +++ b/modulefiles/tasks/orion/make_lbcs.local @@ -1,5 +1,2 @@ #%Module -module use -a /apps/contrib/miniconda3-noaa-gsl/modulefiles -module load miniconda3/3.8 - -setenv SRW_ENV regional_workflow +module load miniconda_regional_workflow diff --git a/modulefiles/tasks/orion/miniconda_regional_workflow b/modulefiles/tasks/orion/miniconda_regional_workflow new file mode 100644 index 000000000..3703a9ba1 --- /dev/null +++ b/modulefiles/tasks/orion/miniconda_regional_workflow @@ -0,0 +1,5 @@ +#%Module +module use -a /apps/contrib/miniconda3-noaa-gsl/modulefiles +module load miniconda3/3.8 + +setenv SRW_ENV regional_workflow diff --git a/modulefiles/tasks/orion/run_fcst.local b/modulefiles/tasks/orion/run_fcst.local index 3703a9ba1..61a3a7725 100644 --- a/modulefiles/tasks/orion/run_fcst.local +++ b/modulefiles/tasks/orion/run_fcst.local @@ -1,5 +1,2 @@ #%Module -module use -a /apps/contrib/miniconda3-noaa-gsl/modulefiles -module load miniconda3/3.8 - -setenv SRW_ENV regional_workflow +module load miniconda_regional_workflow From e79bc212c40a2d7ae4f0ec8f8445183ab7c71421 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Fri, 11 Feb 2022 21:29:50 +0000 Subject: [PATCH 07/29] Call retrieve_data.py at run time. --- jobs/JREGIONAL_GET_EXTRN_MDL_FILES | 21 +--- scripts/exregional_get_extrn_mdl_files.sh | 122 ++++++++++------------ scripts/exregional_make_ics.sh | 2 +- scripts/exregional_make_lbcs.sh | 6 +- 4 files changed, 61 insertions(+), 90 deletions(-) diff --git a/jobs/JREGIONAL_GET_EXTRN_MDL_FILES b/jobs/JREGIONAL_GET_EXTRN_MDL_FILES index 2a3ec0b07..ef717e350 100755 --- a/jobs/JREGIONAL_GET_EXTRN_MDL_FILES +++ b/jobs/JREGIONAL_GET_EXTRN_MDL_FILES @@ -93,30 +93,15 @@ check_var_valid_value "ICS_OR_LBCS" "valid_vals_ICS_OR_LBCS" # if [ "${ICS_OR_LBCS}" = "ICS" ]; then if [ ${EXTRN_MDL_ICS_OFFSET_HRS} -eq 0 ] ; then - anl_or_fcst="ANL" time_offset_hrs=0 else - anl_or_fcst="FCST" time_offset_hrs=${EXTRN_MDL_ICS_OFFSET_HRS:-0} fi extrn_mdl_name=${EXTRN_MDL_NAME_ICS} - fcst_hrs=${time_offset_hrs} - file_names=${EXTRN_MDL_FILES_ICS[@]} - if [ ${extrn_mdl_name} = FV3GFS ] ; then - file_type=$FV3GFS_FILE_FMT_ICS - fi - input_file_path=${EXTRN_MDL_SOURCE_BASEDIR_ICS:-$EXTRN_MDL_SYSBASEDIR_ICS} elif [ "${ICS_OR_LBCS}" = "LBCS" ]; then - anl_or_fcst="FCST" time_offset_hrs=${EXTRN_MDL_LBCS_OFFSET_HRS:-0} extrn_mdl_name=${EXTRN_MDL_NAME_LBCS} - fcst_hrs="${time_offset_hrs} ${FCST_LEN_HRS} ${LBC_SPEC_INTVL_HRS}" - file_names=${EXTRN_MDL_FILES_LBCS[@]} - if [ ${extrn_mdl_name} = FV3GFS ] ; then - file_type=$FV3GFS_FILE_FMT_LBCS - fi - input_file_path=${EXTRN_MDL_SOURCE_BASEDIR_LBCS:-$EXTRN_MDL_SYSBASEDIR_LBCS} fi # @@ -239,14 +224,10 @@ cd_vrfy "${extrn_mdl_staging_dir}" #----------------------------------------------------------------------- # $SCRIPTSDIR/exregional_get_extrn_mdl_files.sh \ - anl_or_fcst="${anl_or_fcst}" \ extrn_mdl_cdate="${extrn_mdl_cdate}" \ extrn_mdl_name="${extrn_mdl_name}" \ extrn_mdl_staging_dir="${extrn_mdl_staging_dir}" \ - fcst_hrs="${fcst_hrs}" \ - file_names="${file_names}" \ - file_type="${file_type}" \ - input_file_path="${input_file_path}" || + time_offset_hrs=${time_offset_hrs} || print_err_msg_exit "\ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." # diff --git a/scripts/exregional_get_extrn_mdl_files.sh b/scripts/exregional_get_extrn_mdl_files.sh index 1b151e436..67f11c09e 100755 --- a/scripts/exregional_get_extrn_mdl_files.sh +++ b/scripts/exregional_get_extrn_mdl_files.sh @@ -56,14 +56,10 @@ boundary condition files for the FV3 will be generated. #----------------------------------------------------------------------- # valid_args=( \ -"anl_or_fcst" \ "extrn_mdl_cdate" \ "extrn_mdl_name" \ "extrn_mdl_staging_dir" \ -"fcst_hrs" \ -"file_names" \ -"file_type" \ -"input_file_path" \ +"time_offset_hrs" \ ) process_args valid_args "$@" # @@ -78,6 +74,39 @@ process_args valid_args "$@" print_input_args valid_args +# +#----------------------------------------------------------------------- +# +# Set up variables for call to retrieve_data.py +# +#----------------------------------------------------------------------- +# +set -x +if [ "${ICS_OR_LBCS}" = "ICS" ]; then + if [ ${time_offset_hrs} -eq 0 ] ; then + anl_or_fcst="anl" + else + anl_or_fcst="fcst" + fi + fcst_hrs=${time_offset_hrs} + file_names=${EXTRN_MDL_FILES_ICS[@]} + if [ ${extrn_mdl_name} = FV3GFS ] ; then + file_type=$FV3GFS_FILE_FMT_ICS + fi + input_file_path=${EXTRN_MDL_SOURCE_BASEDIR_ICS:-$EXTRN_MDL_SYSBASEDIR_ICS} + +elif [ "${ICS_OR_LBCS}" = "LBCS" ]; then + anl_or_fcst="fcst" + first_time=$((time_offset_hrs + LBC_SPEC_INTVL_HRS)) + last_time=$((time_offset_hrs + FCST_LEN_HRS)) + fcst_hrs="${first_time} ${last_time} ${LBC_SPEC_INTVL_HRS}" + file_names=${EXTRN_MDL_FILES_LBCS[@]} + if [ ${extrn_mdl_name} = FV3GFS ] ; then + file_type=$FV3GFS_FILE_FMT_LBCS + fi + input_file_path=${EXTRN_MDL_SOURCE_BASEDIR_LBCS:-$EXTRN_MDL_SYSBASEDIR_LBCS} +fi + # #----------------------------------------------------------------------- # @@ -87,16 +116,19 @@ print_input_args valid_args # additional_flags="" -if [ -n ${file_names} ] ; then - additional_flags="$additional_flags - --file_names ${file_names}" -fi +if [ -n ${file_names:-} ] ; then + + if [ -n "${file_type:-}" ] ; then + additional_flags="$additional_flags \ + --file_type ${file_type}" + fi -if [ -n ${file_type} ] ; then - additional_flags="$additional_flags - --file_type ${file_type}" + # Set up the yaml string with a list + additional_flags="$additional_flags \ + --file_templates ${file_names[@]}" fi +set +x # #----------------------------------------------------------------------- # @@ -104,68 +136,26 @@ fi # #----------------------------------------------------------------------- # - -${USHDIR}/retrieve_data.py \ +cmd=" +python3 -u ${USHDIR}/retrieve_data.py \ + --debug \ --anl_or_fcst ${anl_or_fcst} \ --config ${USHDIR}/templates/data_locations.yml \ --cycle_date ${extrn_mdl_cdate} \ --data_stores disk hpss aws \ - --external_model ${EXTRN_MDL_NAME} \ - --fcst_hrs fcst_hrs \ + --external_model ${extrn_mdl_name} \ + --fcst_hrs ${fcst_hrs[@]} \ --output_path ${extrn_mdl_staging_dir} \ --input_file_path ${input_file_path} \ - ${additional_flags} + --summary_file ${EXTRN_MDL_VAR_DEFNS_FN} \ + $additional_flags" -# -#----------------------------------------------------------------------- -# -# Create a variable definitions file (a shell script) and save in it the -# values of several external-model-associated variables generated in this -# script that will be needed by downstream workflow tasks. -# -#----------------------------------------------------------------------- -# -if [ "${ics_or_lbcs}" = "ICS" ]; then - extrn_mdl_var_defns_fn="${EXTRN_MDL_ICS_VAR_DEFNS_FN}" -elif [ "${ics_or_lbcs}" = "LBCS" ]; then - extrn_mdl_var_defns_fn="${EXTRN_MDL_LBCS_VAR_DEFNS_FN}" -fi -extrn_mdl_var_defns_fp="${extrn_mdl_staging_dir}/${extrn_mdl_var_defns_fn}" -check_for_preexist_dir_file "${extrn_mdl_var_defns_fp}" "delete" - -if [ "${data_src}" = "disk" ]; then - extrn_mdl_fns_str="( "$( printf "\"%s\" " "${extrn_mdl_fns_on_disk[@]}" )")" -elif [ "${data_src}" = "HPSS" ]; then - extrn_mdl_fns_str="( "$( printf "\"%s\" " "${extrn_mdl_fns_in_arcv[@]}" )")" -elif [ "${data_src}" = "online" ]; then - extrn_mdl_fns_str="( "$( printf "\"%s\" " "${extrn_mdl_fns_on_disk[@]}" )")" -fi - -settings="\ -DATA_SRC=\"${data_src}\" -EXTRN_MDL_CDATE=\"${extrn_mdl_cdate}\" -EXTRN_MDL_STAGING_DIR=\"${extrn_mdl_staging_dir}\" -EXTRN_MDL_FNS=${extrn_mdl_fns_str}" -# -# If the external model files obtained above were for generating LBCS (as -# opposed to ICs), then add to the external model variable definitions -# file the array variable EXTRN_MDL_LBC_SPEC_FHRS containing the forecast -# hours at which the lateral boundary conditions are specified. -# -if [ "${ics_or_lbcs}" = "LBCS" ]; then - extrn_mdl_lbc_spec_fhrs_str="( "$( printf "\"%s\" " "${extrn_mdl_lbc_spec_fhrs[@]}" )")" - settings="$settings -EXTRN_MDL_LBC_SPEC_FHRS=${extrn_mdl_lbc_spec_fhrs_str}" -fi +$cmd || print_err_msg_exit "\ +Call to retrieve_data.py failed with a non-zero exit status. -{ cat << EOM >> ${extrn_mdl_var_defns_fp} -$settings -EOM -} || print_err_msg_exit "\ -Heredoc (cat) command to create a variable definitions file associated -with the external model from which to generate ${ics_or_lbcs} returned with a -nonzero status. The full path to this variable definitions file is: - extrn_mdl_var_defns_fp = \"${extrn_mdl_var_defns_fp}\"" +The command was: +${cmd} +" # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index 7161f16aa..9e7dd3ca0 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -126,7 +126,7 @@ fi #----------------------------------------------------------------------- # extrn_mdl_staging_dir="${CYCLE_DIR}/${EXTRN_MDL_NAME_ICS}/for_ICS" -extrn_mdl_var_defns_fp="${extrn_mdl_staging_dir}/${EXTRN_MDL_ICS_VAR_DEFNS_FN}" +extrn_mdl_var_defns_fp="${extrn_mdl_staging_dir}/${EXTRN_MDL_VAR_DEFNS_FN}" . ${extrn_mdl_var_defns_fp} # #----------------------------------------------------------------------- diff --git a/scripts/exregional_make_lbcs.sh b/scripts/exregional_make_lbcs.sh index f123dfc40..cc41c4948 100755 --- a/scripts/exregional_make_lbcs.sh +++ b/scripts/exregional_make_lbcs.sh @@ -124,7 +124,7 @@ fi #----------------------------------------------------------------------- # extrn_mdl_staging_dir="${CYCLE_DIR}/${EXTRN_MDL_NAME_LBCS}/for_LBCS" -extrn_mdl_var_defns_fp="${extrn_mdl_staging_dir}/${EXTRN_MDL_LBCS_VAR_DEFNS_FN}" +extrn_mdl_var_defns_fp="${extrn_mdl_staging_dir}/${EXTRN_MDL_VAR_DEFNS_FN}" . ${extrn_mdl_var_defns_fp} # #----------------------------------------------------------------------- @@ -375,12 +375,12 @@ fi # #----------------------------------------------------------------------- # -num_fhrs="${#EXTRN_MDL_LBC_SPEC_FHRS[@]}" +num_fhrs="${#EXTRN_MDL_FHRS[@]}" for (( i=0; i<${num_fhrs}; i++ )); do # # Get the forecast hour of the external model. # - fhr="${EXTRN_MDL_LBC_SPEC_FHRS[$i]}" + fhr="${EXTRN_MDL_FHRS[$i]}" # # Set external model output file name and file type/format. Note that # these are now inputs into chgres_cube. From c908fb23bfe85653165b1abd0ff5e4f6ed083ecd Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Fri, 11 Feb 2022 21:30:40 +0000 Subject: [PATCH 08/29] Update test script to define input location. --- tests/WE2E/run_WE2E_tests.sh | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/tests/WE2E/run_WE2E_tests.sh b/tests/WE2E/run_WE2E_tests.sh index 776c80d28..863be5786 100755 --- a/tests/WE2E/run_WE2E_tests.sh +++ b/tests/WE2E/run_WE2E_tests.sh @@ -921,10 +921,7 @@ machine (MACHINE): elif [ "${FV3GFS_FILE_FMT_ICS}" = "grib2" ]; then EXTRN_MDL_FILES_ICS=( "gfs.pgrb2.0p25.f000" ) fi - elif [ "${EXTRN_MDL_NAME_ICS}" = "HRRR" ] || \ - [ "${EXTRN_MDL_NAME_ICS}" = "RAP" ]; then - EXTRN_MDL_FILES_ICS=( "${EXTRN_MDL_NAME_ICS,,}.out.for_f000" ) - elif [ "${EXTRN_MDL_NAME_ICS}" = "NAM" ]; then + else EXTRN_MDL_FILES_ICS=( "${EXTRN_MDL_NAME_ICS,,}.out.for_f000" ) fi @@ -951,16 +948,12 @@ boundary conditions specification interval (LBC_SPEC_INTVL_HRS): if [ "${EXTRN_MDL_NAME_LBCS}" = "FV3GFS" ] || \ [ "${EXTRN_MDL_NAME_LBCS}" = "GSMGFS" ]; then if [ "${FV3GFS_FILE_FMT_LBCS}" = "nemsio" ]; then - EXTRN_MDL_FILES_LBCS=( "${EXTRN_MDL_FILES_LBCS[@]/#/gfs.atmf}" ) - EXTRN_MDL_FILES_LBCS=( "${EXTRN_MDL_FILES_LBCS[@]/%/.nemsio}" ) + EXTRN_MDL_FILES_LBCS='gfs.atmf{fcst_hr:03d}.nemsio' elif [ "${FV3GFS_FILE_FMT_LBCS}" = "grib2" ]; then - EXTRN_MDL_FILES_LBCS=( "${EXTRN_MDL_FILES_LBCS[@]/#/gfs.pgrb2.0p25.f}" ) + EXTRN_MDL_FILES_LBCS='gfs.pgrb2.0p25.f{fcst_hr:03d}' fi - elif [ "${EXTRN_MDL_NAME_LBCS}" = "HRRR" ] || \ - [ "${EXTRN_MDL_NAME_LBCS}" = "RAP" ]; then - EXTRN_MDL_FILES_LBCS=( "${EXTRN_MDL_FILES_LBCS[@]/#/${EXTRN_MDL_NAME_LBCS,,}.out.for_f}" ) - elif [ "${EXTRN_MDL_NAME_LBCS}" = "NAM" ]; then - EXTRN_MDL_FILES_LBCS=( "${EXTRN_MDL_FILES_LBCS[@]/#/${EXTRN_MDL_NAME_LBCS,,}.out.for_f}" ) + else + EXTRN_MDL_FILES_LBCS='{yy}{jjj}{hh}{fcst_hr:04d}00' fi expt_config_str=${expt_config_str}" @@ -968,10 +961,10 @@ boundary conditions specification interval (LBC_SPEC_INTVL_HRS): # Locations and names of user-staged external model files for generating # ICs and LBCs. # -EXTRN_MDL_SOURCE_BASEDIR_ICS=\"${EXTRN_MDL_SOURCE_BASEDIR_ICS}\" -EXTRN_MDL_FILES_ICS=( $( printf "\"%s\" " "${EXTRN_MDL_FILES_ICS[@]}" )) -EXTRN_MDL_SOURCE_BASEDIR_LBCS=\"${EXTRN_MDL_SOURCE_BASEDIR_LBCS}\" -EXTRN_MDL_FILES_LBCS=( $( printf "\"%s\" " "${EXTRN_MDL_FILES_LBCS[@]}" ))" +EXTRN_MDL_SOURCE_BASEDIR_ICS=${EXTRN_MDL_SOURCE_BASEDIR_ICS}/\${DATE_FIRST_CYCL}\${CYCL_HRS[0]} +EXTRN_MDL_FILES_ICS=\"${EXTRN_MDL_FILES_ICS}\" +EXTRN_MDL_SOURCE_BASEDIR_LBCS=${EXTRN_MDL_SOURCE_BASEDIR_LBCS}/\${DATE_FIRST_CYCL}\${CYCL_HRS[0]}/for_LBCS +EXTRN_MDL_FILES_LBCS=\"${EXTRN_MDL_FILES_LBCS}\"" fi # From 3a91165a911fc9ce22b7e2366efc8572daccc0f4 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Fri, 11 Feb 2022 21:33:19 +0000 Subject: [PATCH 09/29] Updating the defaults. --- ush/config_defaults.sh | 32 ++++++++++++-------------------- 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index eb6fa9775..1758ca820 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -377,19 +377,12 @@ DOT_OR_USCORE="_" # to each workflow task) in order to make all the experiment variables # available in those scripts. # -# EXTRN_MDL_ICS_VAR_DEFNS_FN: -# Name of file (a shell script) containing the defintions of variables -# associated with the external model from which ICs are generated. This -# file is created by the GET_EXTRN_ICS_TN task because the values of the -# variables it contains are not known before this task runs. The file is -# then sourced by the MAKE_ICS_TN task. -# -# EXTRN_MDL_LBCS_VAR_DEFNS_FN: -# Name of file (a shell script) containing the defintions of variables -# associated with the external model from which LBCs are generated. This -# file is created by the GET_EXTRN_LBCS_TN task because the values of the -# variables it contains are not known before this task runs. The file is -# then sourced by the MAKE_ICS_TN task. +# EXTRN_MDL_VAR_DEFNS_FN: +# Name of file (a shell script) containing the defintions of variables +# associated with the external model from which ICs or LBCs are generated. This +# file is created by the GET_EXTRN_*_TN task because the values of the variables +# it contains are not known before this task runs. The file is then sourced by +# the MAKE_ICS_TN and MAKE_LBCS_TN tasks. # # WFLOW_LAUNCH_SCRIPT_FN: # Name of the script that can be used to (re)launch the experiment's rocoto @@ -419,8 +412,7 @@ FCST_MODEL="ufs-weather-model" WFLOW_XML_FN="FV3LAM_wflow.xml" GLOBAL_VAR_DEFNS_FN="var_defns.sh" -EXTRN_MDL_ICS_VAR_DEFNS_FN="extrn_mdl_ics_var_defns.sh" -EXTRN_MDL_LBCS_VAR_DEFNS_FN="extrn_mdl_lbcs_var_defns.sh" +EXTRN_MDL_VAR_DEFNS_FN="extrn_mdl_var_defns.sh" WFLOW_LAUNCH_SCRIPT_FN="launch_FV3LAM_wflow.sh" WFLOW_LAUNCH_LOG_FN="log.launch_FV3LAM_wflow" # @@ -694,9 +686,9 @@ EXTRN_MDL_SYSBASEDIR_LBCS='' # set to "FALSE". # # EXTRN_MDL_FILES_ICS: -# Array containing the names of the files to search for in the directory -# specified by EXTRN_MDL_SOURCE_BASEDIR_ICS. This variable is not used -# if USE_USER_STAGED_EXTRN_FILES is set to "FALSE". +# Array containing templates of the names of the files to search for in the +# directory specified by EXTRN_MDL_SOURCE_BASEDIR_ICS. This variable is not +# used if USE_USER_STAGED_EXTRN_FILES is set to "FALSE". # # EXTRN_MDL_SOURCE_BASEDIR_LBCS: # Analogous to EXTRN_MDL_SOURCE_BASEDIR_ICS but for LBCs instead of ICs. @@ -708,9 +700,9 @@ EXTRN_MDL_SYSBASEDIR_LBCS='' # USE_USER_STAGED_EXTRN_FILES="FALSE" EXTRN_MDL_SOURCE_BASEDIR_ICS="" -EXTRN_MDL_FILES_ICS=( "ICS_file1" "ICS_file2" "..." ) +EXTRN_MDL_FILES_ICS="" EXTRN_MDL_SOURCE_BASEDIR_LBCS="" -EXTRN_MDL_FILES_LBCS=( "LBCS_file1" "LBCS_file2" "..." ) +EXTRN_MDL_FILES_LBCS="" # #----------------------------------------------------------------------- # From 7faa5e73f14b401f38c8e1ae6ea2b3b24df46f19 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Fri, 11 Feb 2022 21:33:59 +0000 Subject: [PATCH 10/29] Update retrieve_data.py Have CLI accept a list of file name templates. Have the script write out a summary file necessry for downstream tasks. --- ush/retrieve_data.py | 107 ++++++++++++++++++++++++++++++------------- 1 file changed, 76 insertions(+), 31 deletions(-) mode change 100644 => 100755 ush/retrieve_data.py diff --git a/ush/retrieve_data.py b/ush/retrieve_data.py old mode 100644 new mode 100755 index 48210c0d5..30ae524cd --- a/ush/retrieve_data.py +++ b/ush/retrieve_data.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 # pylint: disable=logging-fstring-interpolation ''' This script helps users pull data from known data streams, including @@ -31,6 +32,8 @@ import shutil import subprocess import sys +from textwrap import dedent + import yaml @@ -184,7 +187,7 @@ def fhr_list(args): Must ensure that the list contains integers. ''' - args = args if isinstance(args, list) else [args] + args = args if isinstance(args, list) else list(args) arg_len = len(args) if arg_len in (2, 3): args[1] += 1 @@ -271,7 +274,7 @@ def find_archive_files(paths, file_names, cycle_date): return '', 0 -def get_requested_files(cla, file_names, input_loc, method='disk'): +def get_requested_files(cla, file_templates, input_loc, method='disk'): ''' This function copies files from disk locations or downloads files from a url, depending on the option specified for @@ -282,12 +285,13 @@ def get_requested_files(cla, file_names, input_loc, method='disk'): Arguments: - cla Namespace object containing command line arguments - file_names Dict of file names by file type and kind - input_loc A string containing a single data location, either a url - or disk path. - method Choice of disk or download to indicate protocol for - retrieval + cla Namespace object containing command line arguments + file_templates Dict of file names by file type and kind, or a list of file + templates + input_loc A string containing a single data location, either a url + or disk path. + method Choice of disk or download to indicate protocol for + retrieval Returns unavailable a dict whose keys are "method" and whose values are a @@ -296,12 +300,15 @@ def get_requested_files(cla, file_names, input_loc, method='disk'): unavailable = {} - if cla.file_type is not None: - file_names = file_names[cla.file_type] - file_names = file_names[cla.anl_or_fcst] + if isinstance(file_templates, dict): + if cla.file_type is not None: + file_templates = file_templates[cla.file_type] + file_templates = file_templates[cla.anl_or_fcst] + + logging.info(f'Getting files named like {file_templates}') - file_names = file_names if isinstance(file_names, list) else \ - [file_names] + file_templates = file_templates if isinstance(file_templates, list) else \ + [file_templates] target_path = fill_template(cla.output_path, cla.cycle_date) @@ -310,8 +317,9 @@ def get_requested_files(cla, file_names, input_loc, method='disk'): os.chdir(target_path) unavailable = {} for fcst_hr in cla.fcst_hrs: - for file_name in file_names: - loc = os.path.join(input_loc, file_name) + for file_template in file_templates: + loc = os.path.join(input_loc, file_template) + logging.debug(f'Full file path: {loc}') loc = fill_template(loc, cla.cycle_date, fcst_hr) if method == 'disk': @@ -505,6 +513,29 @@ def setup_logging(debug=False): +def write_summary_file(cla, data_store, file_templates): + + ''' Given the command line arguments and the data store from which the data + was retrieved, write a bash summary file that is needed by the workflow + elements downstream. ''' + + for tmpl in file_templates: + files = [fill_template(tmpl, cla.cycle_date, fh) for fh in cla.fcst_hrs] + + summary_fp = os.path.join(cla.output_path, cla.summary_file) + logging.info(f'Writing a summary file to {summary_fp}') + file_contents = dedent(f''' + DATA_SRC={data_store} + EXTRN_MDL_CDATE={cla.cycle_date.strftime('%Y%m%d%H')} + EXTRN_MDL_STAGING_DIR={cla.output_path} + EXTRN_MDL_FNS=( {' '.join(files)} ) + EXTRN_MDL_FHRS=( {' '.join([str(i) for i in cla.fcst_hrs])} ) + ''') + logging.info(f'Contents: {file_contents}') + with open(summary_fp, "w") as summary: + summary.write(file_contents) + + def to_datetime(arg): ''' Return a datetime object give a string like YYYYMMDDHH. ''' @@ -521,8 +552,6 @@ def main(cla): paths in priority order. ''' - setup_logging(cla.debug) - known_data_info = cla.config.get(cla.external_model) if known_data_info is None: msg = ('No data stores have been defined for', @@ -535,15 +564,15 @@ def main(cla): store_specs = known_data_info.get(data_store, {}) if data_store == 'disk': - file_names = cla.file_names if cla.file_names else \ + file_templates = cla.file_templates if cla.file_templates else \ known_data_info.get('hpss', {}).get('file_names') - logging.debug(f'User supplied file names are: {file_names}') - if not file_names: - msg = ('No file name found. They must be provided \ + logging.debug(f'User supplied file names are: {file_templates}') + if not file_templates: + msg = ('No file naming convention found. They must be provided \ either on the command line or on in a config file.') raise argparse.ArgumentTypeError(msg) unavailable = get_requested_files(cla, - file_names=file_names, + file_templates=file_templates, input_loc=cla.input_file_path, method='disk', ) @@ -553,14 +582,14 @@ def main(cla): raise KeyError(msg) if store_specs.get('protocol') == 'download': - file_names = store_specs.get('file_names') - if not file_names: - msg = ('No file name found. They must be provided \ + file_templates = store_specs.get('file_names') + if not file_templates: + msg = ('No file name naming convention found. They must be provided \ either on the command line or on in a config file.') raise argparse.ArgumentTypeError(msg) unavailable = get_requested_files(cla, - file_names=file_names, + file_templates=file_templates, input_loc=store_specs['url'], method='download', ) @@ -570,6 +599,9 @@ def main(cla): if not unavailable: # All files are found. Stop looking! + # Write a variable definitions file for the data, if requested + if cla.summary_file: + write_summary_file(cla, data_store, file_templates) break logging.warning(f'Requested files are unavialable from {data_store}') @@ -657,11 +689,11 @@ def parse_args(): help='Print debug messages', ) parser.add_argument( - '--file_names', - help='A YAML-formatted string that indicates the naming \ + '--file_templates', + help='One or more file template strings defining the naming \ convention the be used for the files retrieved from disk. If \ not provided, the default names from hpss are used.', - type=load_str, + nargs='*', ) parser.add_argument( '--file_type', @@ -672,9 +704,13 @@ def parse_args(): '--input_file_path', help='A path to data stored on disk. The path may contain \ Python templates. File names may be supplied using the \ - --file_names flag, or the default naming convention will be \ + --file_templates flag, or the default naming convention will be \ taken from the --config file.', - nargs='*', + ) + parser.add_argument( + '--summary_file', + help='Name of the summary file to be written to the output \ + directory', ) return parser.parse_args() @@ -684,6 +720,15 @@ def parse_args(): CLA.output_path = path_exists(CLA.output_path) CLA.fcst_hrs = fhr_list(CLA.fcst_hrs) + + setup_logging(CLA.debug) + print(f"Running script retrieve_data.py with args:\n", + f"{('-' * 80)}\n{('-' * 80)}") + for name, val in CLA.__dict__.items(): + if name not in ['config']: + print(f"{name:>15s}: {val}") + print(f"{('-' * 80)}\n{('-' * 80)}") + if 'disk' in CLA.data_stores: # Make sure a path was provided. if not CLA.input_file_path: From 20aca9aa336ba022bcb89f548d7de6254196aef7 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Thu, 17 Feb 2022 16:47:15 +0000 Subject: [PATCH 11/29] Updating common tasks for each machine. --- modulefiles/tasks/cheyenne/pylib_regional_workflow | 9 ++++++++- modulefiles/tasks/hera/miniconda_regional_workflow | 5 ++++- modulefiles/tasks/jet/miniconda_regional_workflow | 5 ++++- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/modulefiles/tasks/cheyenne/pylib_regional_workflow b/modulefiles/tasks/cheyenne/pylib_regional_workflow index 2f92a39e5..4232b8659 100644 --- a/modulefiles/tasks/cheyenne/pylib_regional_workflow +++ b/modulefiles/tasks/cheyenne/pylib_regional_workflow @@ -1,2 +1,9 @@ #%Module -module load pylib_regional_workflow +if [module-info mode load] { + system "ncar_pylib /glade/p/ral/jntp/UFS_CAM/ncar_pylib_20200427" +} + +if [module-info mode remove] { + system "deactivate" +} + diff --git a/modulefiles/tasks/hera/miniconda_regional_workflow b/modulefiles/tasks/hera/miniconda_regional_workflow index 61a3a7725..011a832c9 100644 --- a/modulefiles/tasks/hera/miniconda_regional_workflow +++ b/modulefiles/tasks/hera/miniconda_regional_workflow @@ -1,2 +1,5 @@ #%Module -module load miniconda_regional_workflow +module use -a /contrib/miniconda3/modulefiles +module load miniconda3 + +setenv SRW_ENV regional_workflow diff --git a/modulefiles/tasks/jet/miniconda_regional_workflow b/modulefiles/tasks/jet/miniconda_regional_workflow index 61a3a7725..011a832c9 100644 --- a/modulefiles/tasks/jet/miniconda_regional_workflow +++ b/modulefiles/tasks/jet/miniconda_regional_workflow @@ -1,2 +1,5 @@ #%Module -module load miniconda_regional_workflow +module use -a /contrib/miniconda3/modulefiles +module load miniconda3 + +setenv SRW_ENV regional_workflow From ed57e5665790766ed28ed1b918cb54c85e0d0401 Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Sat, 19 Feb 2022 16:06:45 +0000 Subject: [PATCH 12/29] Adding support for GSMGFS --- ush/templates/data_locations.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/ush/templates/data_locations.yml b/ush/templates/data_locations.yml index 23354e171..bf78e7ac6 100644 --- a/ush/templates/data_locations.yml +++ b/ush/templates/data_locations.yml @@ -103,6 +103,28 @@ FV3GFS: file_names: <<: *gfs_file_names +GSMGFS: + hpss: + protocol: htar + archive_path: + - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyymm}/{yyyymmdd} + archive_internal_dir: + anl: + - ./ + fcst: + - /gpfs/hps/nco/ops/com/gfs/prod/gfs.{yyyymmdd} + archive_file_names: + anl: + - gpfs_hps_nco_ops_com_gfs_prod_gfs.{yyyymmddhh}.anl.tar + fcst: + - gpfs_hps_nco_ops_com_gfs_prod_gfs.{yyyymmddhh}.sigma.tar + file_names: + anl: + - gfs.t{hh}z.atmanl.nemsio + - gfs.t{hh}z.sfcanl.nemsio + fcst: + - gfs.t{hh}z.atmf{fcst_hr:03d}.nemsio + RAP: hpss: protocol: htar From f37e6d77a0390d61bb91b729361ae53f54dddbaa Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Sat, 19 Feb 2022 16:07:21 +0000 Subject: [PATCH 13/29] retrieve_data updates. Fix summary file output Fix functionality for using disk without known location. --- ush/retrieve_data.py | 42 ++++++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/ush/retrieve_data.py b/ush/retrieve_data.py index 30ae524cd..8ee2ba6b3 100755 --- a/ush/retrieve_data.py +++ b/ush/retrieve_data.py @@ -46,7 +46,7 @@ def clean_up_output_dir(expected_subdir, local_archive, output_path, source_path unavailable = {} # Check to make sure the files exist on disk for file_path in source_paths: - local_file_path = os.path.join(output_path,file_path) + local_file_path = os.path.join(output_path, file_path.lstrip("/")) if not os.path.exists(local_file_path): logging.info(f'File does not exist: {local_file_path}') unavailable['hpss'] = source_paths @@ -415,8 +415,14 @@ def hpss_requested_files(cla, store_specs): file_names = file_names[cla.file_type] file_names = file_names[cla.anl_or_fcst] + logging.info(f'Files in archive are named: {file_names}') + logging.debug(f'Grabbing archive number {which_archive} in list.') - archive_internal_dir = store_specs.get('archive_internal_dir', [''])[which_archive] + archive_internal_dir = store_specs.get('archive_internal_dir', ['']) + if isinstance(archive_internal_dir, dict): + archive_internal_dir = archive_internal_dir.get(cla.anl_or_fcst, ['']) + + archive_internal_dir = archive_internal_dir[which_archive] archive_internal_dir = fill_template(archive_internal_dir, cla.cycle_date) @@ -519,8 +525,9 @@ def write_summary_file(cla, data_store, file_templates): was retrieved, write a bash summary file that is needed by the workflow elements downstream. ''' + files = [] for tmpl in file_templates: - files = [fill_template(tmpl, cla.cycle_date, fh) for fh in cla.fcst_hrs] + files.extend([fill_template(tmpl, cla.cycle_date, fh) for fh in cla.fcst_hrs]) summary_fp = os.path.join(cla.output_path, cla.summary_file) logging.info(f'Writing a summary file to {summary_fp}') @@ -552,14 +559,18 @@ def main(cla): paths in priority order. ''' - known_data_info = cla.config.get(cla.external_model) - if known_data_info is None: - msg = ('No data stores have been defined for', - f'{cla.external_model}!') - raise KeyError(msg) + data_stores = cla.data_stores + known_data_info = cla.config.get(cla.external_model, {}) + if not known_data_info: + msg = dedent(f'''No data stores have been defined for + {cla.external_model}!''') + if cla.input_file_path is None: + data_stores = ['disk'] + raise KeyError(msg) + logging.info(msg + ' Only checking provided disk location.') unavailable = {} - for data_store in cla.data_stores: + for data_store in data_stores: logging.info(f'Checking {data_store} for {cla.external_model}') store_specs = known_data_info.get(data_store, {}) @@ -581,19 +592,18 @@ def main(cla): msg = (f'No information is available for {data_store}.') raise KeyError(msg) - if store_specs.get('protocol') == 'download': - file_templates = store_specs.get('file_names') - if not file_templates: - msg = ('No file name naming convention found. They must be provided \ - either on the command line or on in a config file.') - raise argparse.ArgumentTypeError(msg) + file_templates = store_specs.get('file_names') + if not file_templates: + msg = ('No file name naming convention found. They must be provided \ + either on the command line or on in a config file.') + raise argparse.ArgumentTypeError(msg) + if store_specs.get('protocol') == 'download': unavailable = get_requested_files(cla, file_templates=file_templates, input_loc=store_specs['url'], method='download', ) - if store_specs.get('protocol') == 'htar': unavailable = hpss_requested_files(cla, store_specs) From 99387be98886770d06a30282f7bca3e6f236ad22 Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Sat, 19 Feb 2022 16:09:33 +0000 Subject: [PATCH 14/29] Separate logic for additional args. --- scripts/exregional_get_extrn_mdl_files.sh | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/scripts/exregional_get_extrn_mdl_files.sh b/scripts/exregional_get_extrn_mdl_files.sh index 67f11c09e..419ce9c82 100755 --- a/scripts/exregional_get_extrn_mdl_files.sh +++ b/scripts/exregional_get_extrn_mdl_files.sh @@ -116,14 +116,13 @@ fi # additional_flags="" -if [ -n ${file_names:-} ] ; then - if [ -n "${file_type:-}" ] ; then - additional_flags="$additional_flags \ - --file_type ${file_type}" - fi +if [ -n "${file_type:-}" ] ; then + additional_flags="$additional_flags \ + --file_type ${file_type}" +fi - # Set up the yaml string with a list +if [ -n "${file_names:-}" ] ; then additional_flags="$additional_flags \ --file_templates ${file_names[@]}" fi From b92b7e40131217dc97ea24d655d3a09e95b6ccea Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Sat, 19 Feb 2022 16:10:16 +0000 Subject: [PATCH 15/29] Configure for correct disk locations. --- tests/WE2E/run_WE2E_tests.sh | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/tests/WE2E/run_WE2E_tests.sh b/tests/WE2E/run_WE2E_tests.sh index 863be5786..fb8361885 100755 --- a/tests/WE2E/run_WE2E_tests.sh +++ b/tests/WE2E/run_WE2E_tests.sh @@ -912,7 +912,9 @@ machine (MACHINE): fi EXTRN_MDL_SOURCE_BASEDIR_ICS="${extrn_mdl_source_basedir}/${EXTRN_MDL_NAME_ICS}" if [ "${EXTRN_MDL_NAME_ICS}" = "FV3GFS" ] && [ "$MACHINE" = "HERA" ]; then - EXTRN_MDL_SOURCE_BASEDIR_ICS="${EXTRN_MDL_SOURCE_BASEDIR_ICS}/${FV3GFS_FILE_FMT_ICS}" + EXTRN_MDL_SOURCE_BASEDIR_ICS="${EXTRN_MDL_SOURCE_BASEDIR_ICS}/${FV3GFS_FILE_FMT_ICS}/\${DATE_FIRST_CYCL}\${CYCL_HRS[0]}" + else + EXTRN_MDL_SOURCE_BASEDIR_ICS="${EXTRN_MDL_SOURCE_BASEDIR_ICS}/\${DATE_FIRST_CYCL}\${CYCL_HRS[0]}" fi if [ "${EXTRN_MDL_NAME_ICS}" = "FV3GFS" ] || \ [ "${EXTRN_MDL_NAME_ICS}" = "GSMGFS" ]; then @@ -927,7 +929,11 @@ machine (MACHINE): EXTRN_MDL_SOURCE_BASEDIR_LBCS="${extrn_mdl_source_basedir}/${EXTRN_MDL_NAME_LBCS}" if [ "${EXTRN_MDL_NAME_LBCS}" = "FV3GFS" ] && [ "$MACHINE" = "HERA" ]; then - EXTRN_MDL_SOURCE_BASEDIR_LBCS="${EXTRN_MDL_SOURCE_BASEDIR_LBCS}/${FV3GFS_FILE_FMT_LBCS}" + EXTRN_MDL_SOURCE_BASEDIR_LBCS="${EXTRN_MDL_SOURCE_BASEDIR_LBCS}/${FV3GFS_FILE_FMT_LBCS}/\${DATE_FIRST_CYCL}\${CYCL_HRS[0]}" + elif [ "${EXTRN_MDL_NAME_LBCS}" = "GSMGFS" ] ; then + EXTRN_MDL_SOURCE_BASEDIR_LBCS="${EXTRN_MDL_SOURCE_BASEDIR_LBCS}/\${DATE_FIRST_CYCL}\${CYCL_HRS[0]}" + else + EXTRN_MDL_SOURCE_BASEDIR_LBCS="${EXTRN_MDL_SOURCE_BASEDIR_LBCS}/\${DATE_FIRST_CYCL}\${CYCL_HRS[0]}/for_LBCS" fi # # Make sure that the forecast length is evenly divisible by the interval @@ -943,28 +949,26 @@ boundary conditions specification interval (LBC_SPEC_INTVL_HRS): LBC_SPEC_INTVL_HRS = ${LBC_SPEC_INTVL_HRS} rem = FCST_LEN_HRS%%LBC_SPEC_INTVL_HRS = $rem" fi - lbc_spec_times_hrs=( $( seq "${LBC_SPEC_INTVL_HRS}" "${LBC_SPEC_INTVL_HRS}" "${FCST_LEN_HRS}" ) ) - EXTRN_MDL_FILES_LBCS=( $( printf "%03d " "${lbc_spec_times_hrs[@]}" ) ) if [ "${EXTRN_MDL_NAME_LBCS}" = "FV3GFS" ] || \ [ "${EXTRN_MDL_NAME_LBCS}" = "GSMGFS" ]; then if [ "${FV3GFS_FILE_FMT_LBCS}" = "nemsio" ]; then - EXTRN_MDL_FILES_LBCS='gfs.atmf{fcst_hr:03d}.nemsio' + EXTRN_MDL_FILES_LBCS=( 'gfs.atmf{fcst_hr:03d}.nemsio' ) elif [ "${FV3GFS_FILE_FMT_LBCS}" = "grib2" ]; then - EXTRN_MDL_FILES_LBCS='gfs.pgrb2.0p25.f{fcst_hr:03d}' + EXTRN_MDL_FILES_LBCS=( 'gfs.pgrb2.0p25.f{fcst_hr:03d}' ) fi else - EXTRN_MDL_FILES_LBCS='{yy}{jjj}{hh}{fcst_hr:04d}00' + EXTRN_MDL_FILES_LBCS=( '{yy}{jjj}{hh}{fcst_hr:04d}00' ) fi - expt_config_str=${expt_config_str}" + expt_config_str="${expt_config_str} # # Locations and names of user-staged external model files for generating # ICs and LBCs. # -EXTRN_MDL_SOURCE_BASEDIR_ICS=${EXTRN_MDL_SOURCE_BASEDIR_ICS}/\${DATE_FIRST_CYCL}\${CYCL_HRS[0]} -EXTRN_MDL_FILES_ICS=\"${EXTRN_MDL_FILES_ICS}\" -EXTRN_MDL_SOURCE_BASEDIR_LBCS=${EXTRN_MDL_SOURCE_BASEDIR_LBCS}/\${DATE_FIRST_CYCL}\${CYCL_HRS[0]}/for_LBCS -EXTRN_MDL_FILES_LBCS=\"${EXTRN_MDL_FILES_LBCS}\"" +EXTRN_MDL_SOURCE_BASEDIR_ICS=${EXTRN_MDL_SOURCE_BASEDIR_ICS} +EXTRN_MDL_FILES_ICS=( ${EXTRN_MDL_FILES_ICS[@]} ) +EXTRN_MDL_SOURCE_BASEDIR_LBCS=${EXTRN_MDL_SOURCE_BASEDIR_LBCS} +EXTRN_MDL_FILES_LBCS=( ${EXTRN_MDL_FILES_LBCS[@]} )" fi # From 95ab89cf06e74aeeeada31154658c71f80f2b774 Mon Sep 17 00:00:00 2001 From: Christina Holt Date: Tue, 22 Feb 2022 22:51:09 +0000 Subject: [PATCH 16/29] Clean up unused code. Reduce duplication. --- ush/retrieve_data.py | 88 +++++++++++--------------------------------- 1 file changed, 21 insertions(+), 67 deletions(-) diff --git a/ush/retrieve_data.py b/ush/retrieve_data.py index 8ee2ba6b3..718c182d6 100755 --- a/ush/retrieve_data.py +++ b/ush/retrieve_data.py @@ -129,46 +129,6 @@ def download_file(url): return True -def download_requested_files(cla, data_store, store_specs): - - ''' This function interacts with the "download" protocol in a - provided data store specs file to download a set of files requested - by the user. It calls download_file for each individual file that - should be downloaded. ''' - - base_urls = store_specs['url'] - base_urls = base_urls if isinstance(base_urls, list) else [base_urls] - - file_names = store_specs.get('file_names', {}) - if cla.file_type is not None: - file_names = file_names[cla.file_type] - file_names = file_names[cla.anl_or_fcst] - target_path = fill_template(cla.output_path, - cla.cycle_date) - - logging.info(f'Downloaded files will be placed here: \n {target_path}') - orig_path = os.getcwd() - os.chdir(target_path) - unavailable = {} - for base_url in base_urls: - for fcst_hr in cla.fcst_hrs: - for file_name in file_names: - url = os.path.join(base_url, file_name) - url = fill_template(url, cla.cycle_date, fcst_hr) - downloaded = download_file(url) - if not downloaded: - - if unavailable.get(data_store) is None: - unavailable[data_store] = [] - unavailable[data_store].append(target_path) - os.chdir(orig_path) - # Returning here assumes that if the first file - # isn't found, none of the others will be. Don't - # waste time timing out on every requested file. - return unavailable - os.chdir(orig_path) - return unavailable - def fhr_list(args): ''' @@ -286,8 +246,7 @@ def get_requested_files(cla, file_templates, input_loc, method='disk'): Arguments: cla Namespace object containing command line arguments - file_templates Dict of file names by file type and kind, or a list of file - templates + file_templates a list of file templates input_loc A string containing a single data location, either a url or disk path. method Choice of disk or download to indicate protocol for @@ -300,11 +259,6 @@ def get_requested_files(cla, file_templates, input_loc, method='disk'): unavailable = {} - if isinstance(file_templates, dict): - if cla.file_type is not None: - file_templates = file_templates[cla.file_type] - file_templates = file_templates[cla.anl_or_fcst] - logging.info(f'Getting files named like {file_templates}') file_templates = file_templates if isinstance(file_templates, list) else \ @@ -366,7 +320,7 @@ def hsi_single_file(file_path, mode='ls'): return file_path -def hpss_requested_files(cla, store_specs): +def hpss_requested_files(cla, file_names, store_specs): ''' This function interacts with the "hpss" protocol in a provided data store specs file to download a set of files requested @@ -409,12 +363,6 @@ def hpss_requested_files(cla, store_specs): unavailable['archive'] = list(zip(archive_paths, archive_file_names)) return unavailable - # Use the found archive file path to get the necessary files - file_names = store_specs.get('file_names', {}) - if cla.file_type is not None: - file_names = file_names[cla.file_type] - file_names = file_names[cla.anl_or_fcst] - logging.info(f'Files in archive are named: {file_names}') logging.debug(f'Grabbing archive number {which_archive} in list.') @@ -592,20 +540,26 @@ def main(cla): msg = (f'No information is available for {data_store}.') raise KeyError(msg) - file_templates = store_specs.get('file_names') - if not file_templates: - msg = ('No file name naming convention found. They must be provided \ - either on the command line or on in a config file.') - raise argparse.ArgumentTypeError(msg) + else: - if store_specs.get('protocol') == 'download': - unavailable = get_requested_files(cla, - file_templates=file_templates, - input_loc=store_specs['url'], - method='download', - ) - if store_specs.get('protocol') == 'htar': - unavailable = hpss_requested_files(cla, store_specs) + file_templates = store_specs.get('file_names') + if isinstance(file_templates, dict): + if cla.file_type is not None: + file_templates = file_templates[cla.file_type] + file_templates = file_templates[cla.anl_or_fcst] + if not file_templates: + msg = ('No file name naming convention found. They must be provided \ + either on the command line or on in a config file.') + raise argparse.ArgumentTypeError(msg) + + if store_specs.get('protocol') == 'download': + unavailable = get_requested_files(cla, + file_templates=file_templates, + input_loc=store_specs['url'], + method='download', + ) + if store_specs.get('protocol') == 'htar': + unavailable = hpss_requested_files(cla, file_templates, store_specs) if not unavailable: # All files are found. Stop looking! From e6cb24a1cc4f518a8048647d3c01b352aea54792 Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Thu, 17 Mar 2022 16:19:48 +0000 Subject: [PATCH 17/29] Adding examples to config_defaults. --- ush/config_defaults.sh | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index 1758ca820..3c8d23c70 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -686,9 +686,18 @@ EXTRN_MDL_SYSBASEDIR_LBCS='' # set to "FALSE". # # EXTRN_MDL_FILES_ICS: -# Array containing templates of the names of the files to search for in the -# directory specified by EXTRN_MDL_SOURCE_BASEDIR_ICS. This variable is not -# used if USE_USER_STAGED_EXTRN_FILES is set to "FALSE". +# Array containing templates of the names of the files to search for in +# the directory specified by EXTRN_MDL_SOURCE_BASEDIR_ICS. This +# variable is not used if USE_USER_STAGED_EXTRN_FILES is set to "FALSE". +# A single template should be used for each model file type that is +# meant to be used. You may use any of the Python-style templates +# allowed in the ush/retrieve_data.py script. To see the full list of +# supported templates, run that script with a -h option. Here is an example of +# setting FV3GFS nemsio input files: +# EXTRN_MDL_FILES_ICS=( gfs.t{hh}z.atmf{fcst_hr:03d}.nemsio \ +# gfs.t{hh}z.sfcf{fcst_hr:03d}.nemsio ) +# Or for FV3GFS grib files: +# EXTRN_MDL_FILES_ICS=( gfs.t{hh}z.pgrb2.0p25.f{fcst_hr:03d} ) # # EXTRN_MDL_SOURCE_BASEDIR_LBCS: # Analogous to EXTRN_MDL_SOURCE_BASEDIR_ICS but for LBCs instead of ICs. From ebca7830f8c50d572119fba2566fb7b88f2ef40d Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Thu, 17 Mar 2022 20:41:44 +0000 Subject: [PATCH 18/29] Addressing failure reported by Jeff B. --- scripts/exregional_get_extrn_mdl_files.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/exregional_get_extrn_mdl_files.sh b/scripts/exregional_get_extrn_mdl_files.sh index 419ce9c82..23c61cf5c 100755 --- a/scripts/exregional_get_extrn_mdl_files.sh +++ b/scripts/exregional_get_extrn_mdl_files.sh @@ -107,6 +107,7 @@ elif [ "${ICS_OR_LBCS}" = "LBCS" ]; then input_file_path=${EXTRN_MDL_SOURCE_BASEDIR_LBCS:-$EXTRN_MDL_SYSBASEDIR_LBCS} fi +input_file_path=$(eval echo ${input_file_path}) # #----------------------------------------------------------------------- # From 0d38bd6567c818aa7be673567e502ee6344584a5 Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Thu, 17 Mar 2022 20:42:00 +0000 Subject: [PATCH 19/29] Addressing poor spelling. --- jobs/JREGIONAL_GET_EXTRN_MDL_FILES | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/jobs/JREGIONAL_GET_EXTRN_MDL_FILES b/jobs/JREGIONAL_GET_EXTRN_MDL_FILES index ef717e350..3b369749d 100755 --- a/jobs/JREGIONAL_GET_EXTRN_MDL_FILES +++ b/jobs/JREGIONAL_GET_EXTRN_MDL_FILES @@ -126,7 +126,7 @@ extrn_mdl_cdate=$( $DATE_UTIL --utc --date "${yyyymmdd} ${hh} UTC - ${time_offse #----------------------------------------------------------------------- # -function data_unavailble() { +function data_unavailable() { local name cdate end_date min_max @@ -167,7 +167,7 @@ case ${extrn_mdl_name} in cdate_max="2019061206" if [ "$extrn_mdl_cdate" -gt "$cdate_max" ]; then print_err_msg_exit "\ - $(data_unavailble $extrn_mdl_name $extrn_mdl_cdate $cdate_max max)" + $(data_unavailable $extrn_mdl_name $extrn_mdl_cdate $cdate_max max)" fi ;; @@ -180,7 +180,7 @@ case ${extrn_mdl_name} in CDATE_min="2018121500" if [ "$extrn_mdl_cdate" -lt "$CDATE_min" ]; then print_err_msg_exit "\ - $(data_unavailble $extrn_mdl_name $extrn_mdl_cdate $cdate_min min)" + $(data_unavailable $extrn_mdl_name $extrn_mdl_cdate $cdate_min min)" fi ;; @@ -190,7 +190,7 @@ case ${extrn_mdl_name} in CDATE_min="2015070100" if [ "$extrn_mdl_cdate" -lt "$CDATE_min" ]; then print_err_msg_exit "\ - $(data_unavailble $extrn_mdl_name $extrn_mdl_cdate $cdate_min min)" + $(data_unavailable $extrn_mdl_name $extrn_mdl_cdate $cdate_min min)" fi ;; @@ -201,7 +201,7 @@ case ${extrn_mdl_name} in CDATE_min="2014103000" if [ "$extrn_mdl_cdate" -lt "$CDATE_min" ]; then print_err_msg_exit "\ - $(data_unavailble $extrn_mdl_name $extrn_mdl_cdate $cdate_min min)" + $(data_unavailable $extrn_mdl_name $extrn_mdl_cdate $cdate_min min)" fi ;; From 653f2b6e72178b26020346ea838ded30de7586ee Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Thu, 17 Mar 2022 20:49:58 +0000 Subject: [PATCH 20/29] Define the supported time values for a template. --- scripts/exregional_get_extrn_mdl_files.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/scripts/exregional_get_extrn_mdl_files.sh b/scripts/exregional_get_extrn_mdl_files.sh index 23c61cf5c..c204b17f6 100755 --- a/scripts/exregional_get_extrn_mdl_files.sh +++ b/scripts/exregional_get_extrn_mdl_files.sh @@ -107,6 +107,16 @@ elif [ "${ICS_OR_LBCS}" = "LBCS" ]; then input_file_path=${EXTRN_MDL_SOURCE_BASEDIR_LBCS:-$EXTRN_MDL_SYSBASEDIR_LBCS} fi + +yyyymmddhh=${extrn_mdl_cdate:0:10} +yyyy=${yyyymmddhh:0:4} +yyyymm=${yyyymmddhh:0:6} +yyyymmdd=${yyyymmddhh:0:8} +mm=${yyyymmddhh:4:2} +dd=${yyyymmddhh:6:2} +hh=${yyyymmddhh:8:2} + + input_file_path=$(eval echo ${input_file_path}) # #----------------------------------------------------------------------- From b6c59b7d0769353aabfdb52e70321b6791b16646 Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Thu, 17 Mar 2022 22:26:29 +0000 Subject: [PATCH 21/29] Handle disk, but no file name template. --- ush/retrieve_data.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ush/retrieve_data.py b/ush/retrieve_data.py index 718c182d6..c61c018c1 100755 --- a/ush/retrieve_data.py +++ b/ush/retrieve_data.py @@ -525,6 +525,10 @@ def main(cla): if data_store == 'disk': file_templates = cla.file_templates if cla.file_templates else \ known_data_info.get('hpss', {}).get('file_names') + if isinstance(file_templates, dict): + if cla.file_type is not None: + file_templates = file_templates[cla.file_type] + file_templates = file_templates[cla.anl_or_fcst] logging.debug(f'User supplied file names are: {file_templates}') if not file_templates: msg = ('No file naming convention found. They must be provided \ From c43bae378a9c0cea9ea67013ec3526192ab05067 Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Mon, 21 Mar 2022 16:45:31 +0000 Subject: [PATCH 22/29] Fix run time failures for data ingest. --- scripts/exregional_get_extrn_mdl_files.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/exregional_get_extrn_mdl_files.sh b/scripts/exregional_get_extrn_mdl_files.sh index c204b17f6..1b7de2a03 100755 --- a/scripts/exregional_get_extrn_mdl_files.sh +++ b/scripts/exregional_get_extrn_mdl_files.sh @@ -138,7 +138,11 @@ if [ -n "${file_names:-}" ] ; then --file_templates ${file_names[@]}" fi -set +x +if [ -n "${input_file_path:-}" ] ; then + additional_flags="$additional_flags \ + --input_file_path ${input_file_path}" +fi + # #----------------------------------------------------------------------- # @@ -156,7 +160,6 @@ python3 -u ${USHDIR}/retrieve_data.py \ --external_model ${extrn_mdl_name} \ --fcst_hrs ${fcst_hrs[@]} \ --output_path ${extrn_mdl_staging_dir} \ - --input_file_path ${input_file_path} \ --summary_file ${EXTRN_MDL_VAR_DEFNS_FN} \ $additional_flags" From aec1c6fc376fe5e3e18c5a447812e6fb2a2d15f4 Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Mon, 21 Mar 2022 17:17:26 +0000 Subject: [PATCH 23/29] Addressing some review comments. --- jobs/JREGIONAL_GET_EXTRN_MDL_FILES | 14 ++++------- modulefiles/tasks/hera/make_ics.local | 1 - modulefiles/tasks/hera/make_lbcs.local | 2 -- modulefiles/tasks/jet/make_ics.local | 2 -- modulefiles/tasks/jet/make_lbcs.local | 2 -- scripts/exregional_get_extrn_mdl_files.sh | 7 +++--- ush/templates/data_locations.yml | 30 ----------------------- 7 files changed, 9 insertions(+), 49 deletions(-) diff --git a/jobs/JREGIONAL_GET_EXTRN_MDL_FILES b/jobs/JREGIONAL_GET_EXTRN_MDL_FILES index 3b369749d..31c296386 100755 --- a/jobs/JREGIONAL_GET_EXTRN_MDL_FILES +++ b/jobs/JREGIONAL_GET_EXTRN_MDL_FILES @@ -63,9 +63,9 @@ print_info_msg " Entering script: \"${scrfunc_fn}\" In directory: \"${scrfunc_dir}\" -This is the J-job script for the task that copies/fetches to a local -directory (either from disk, HPSS, or URL) the external model files from -which initial or boundary condition files for the FV3 will be generated. +This is the J-job script for the task that copies or fetches external +model files from disk, HPSS, or URL, and stages them for downstream use +to generate initial or lateral boundary conditions for the FV3 model. ========================================================================" @@ -92,11 +92,7 @@ check_var_valid_value "ICS_OR_LBCS" "valid_vals_ICS_OR_LBCS" #----------------------------------------------------------------------- # if [ "${ICS_OR_LBCS}" = "ICS" ]; then - if [ ${EXTRN_MDL_ICS_OFFSET_HRS} -eq 0 ] ; then - time_offset_hrs=0 - else - time_offset_hrs=${EXTRN_MDL_ICS_OFFSET_HRS:-0} - fi + time_offset_hrs=${EXTRN_MDL_ICS_OFFSET_HRS:-0} extrn_mdl_name=${EXTRN_MDL_NAME_ICS} elif [ "${ICS_OR_LBCS}" = "LBCS" ]; then @@ -141,7 +137,7 @@ Output from the specified external model (extrn_mdl_name) is not availa- ble for the specified cycle date and time (extrn_mdl_cdate) because the latter is later than the last forecast date and time (cdate_max) with this model: extrn_mdl_name = \"${name}\" - cdate_max = \"${end_date}\" + CDATE_max = \"${end_date}\" extrn_mdl_cdate = \"${cdate}\"" elif [ ${min_max} = min ]; then diff --git a/modulefiles/tasks/hera/make_ics.local b/modulefiles/tasks/hera/make_ics.local index 7d7f9a313..92505cf09 100644 --- a/modulefiles/tasks/hera/make_ics.local +++ b/modulefiles/tasks/hera/make_ics.local @@ -1,4 +1,3 @@ #%Module -module load wgrib2/2.0.8 module load miniconda_regional_workflow diff --git a/modulefiles/tasks/hera/make_lbcs.local b/modulefiles/tasks/hera/make_lbcs.local index 7d7f9a313..61a3a7725 100644 --- a/modulefiles/tasks/hera/make_lbcs.local +++ b/modulefiles/tasks/hera/make_lbcs.local @@ -1,4 +1,2 @@ #%Module -module load wgrib2/2.0.8 - module load miniconda_regional_workflow diff --git a/modulefiles/tasks/jet/make_ics.local b/modulefiles/tasks/jet/make_ics.local index 7d7f9a313..61a3a7725 100644 --- a/modulefiles/tasks/jet/make_ics.local +++ b/modulefiles/tasks/jet/make_ics.local @@ -1,4 +1,2 @@ #%Module -module load wgrib2/2.0.8 - module load miniconda_regional_workflow diff --git a/modulefiles/tasks/jet/make_lbcs.local b/modulefiles/tasks/jet/make_lbcs.local index 7d7f9a313..61a3a7725 100644 --- a/modulefiles/tasks/jet/make_lbcs.local +++ b/modulefiles/tasks/jet/make_lbcs.local @@ -1,4 +1,2 @@ #%Module -module load wgrib2/2.0.8 - module load miniconda_regional_workflow diff --git a/scripts/exregional_get_extrn_mdl_files.sh b/scripts/exregional_get_extrn_mdl_files.sh index 1b7de2a03..2e2bf6fa4 100755 --- a/scripts/exregional_get_extrn_mdl_files.sh +++ b/scripts/exregional_get_extrn_mdl_files.sh @@ -42,9 +42,10 @@ print_info_msg " Entering script: \"${scrfunc_fn}\" In directory: \"${scrfunc_dir}\" -This is the ex-script for the task that copies/fetches to a local directory -either from disk or HPSS) the external model files from which initial or -boundary condition files for the FV3 will be generated. +This is the ex-script for the task that copies or fetches external model +input data from disk, HPSS, or a URL, and stages them to the +workflow-specified location so that they may be used to generate initial +or lateral boundary conditions for the FV3. ========================================================================" # #----------------------------------------------------------------------- diff --git a/ush/templates/data_locations.yml b/ush/templates/data_locations.yml index bf78e7ac6..6393c593c 100644 --- a/ush/templates/data_locations.yml +++ b/ush/templates/data_locations.yml @@ -147,22 +147,6 @@ RAP: file_names: <<: *rap_file_names -RAPx: - hpss: - protocol: htar - archive_format: zip - archive_path: - - /BMC/fdr/Permanent/{yyyy}/{mm}/{dd}/data/fsl/rap/full/wrfnat - archive_file_names: - # RAPx bins two cycles togehter, and named by the lower even value - # of the cycle hour. - - '{yyyymmdd}{hh_even}00.zip' - file_names: - anl: - - '{yy}{jjj}{hh}00{fcst_hr:02d}00' - fcst: - - '{yy}{jjj}{hh}00{fcst_hr:02d}00' - HRRR: hpss: protocol: htar @@ -185,20 +169,6 @@ HRRR: file_names: <<: *hrrr_file_names -HRRRx: - hpss: - protocol: htar - archive_format: zip - archive_path: - - /BMC/fdr/Permanent/{yyyy}/{mm}/{dd}/data/fsl/hrrr/conus/wrfnat - archive_file_names: - - '{yyyymmddhh}00.zip' - file_names: - anl: - - '{yy}{jjj}{hh}00{fcst_hr:02d}00' - fcst: - - '{yy}{jjj}{hh}00{fcst_hr:02d}00' - NAM: hpss: protocol: htar From 18aedb48f0ae3dbbe6b8f6aad830874a89ea4847 Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Fri, 25 Mar 2022 14:47:39 +0000 Subject: [PATCH 24/29] Handle case when no path is defined for platform --- scripts/exregional_get_extrn_mdl_files.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/exregional_get_extrn_mdl_files.sh b/scripts/exregional_get_extrn_mdl_files.sh index 2e2bf6fa4..fc02e3b99 100755 --- a/scripts/exregional_get_extrn_mdl_files.sh +++ b/scripts/exregional_get_extrn_mdl_files.sh @@ -108,6 +108,7 @@ elif [ "${ICS_OR_LBCS}" = "LBCS" ]; then input_file_path=${EXTRN_MDL_SOURCE_BASEDIR_LBCS:-$EXTRN_MDL_SYSBASEDIR_LBCS} fi +data_stores="hpss aws" yyyymmddhh=${extrn_mdl_cdate:0:10} yyyy=${yyyymmddhh:0:4} @@ -140,6 +141,7 @@ if [ -n "${file_names:-}" ] ; then fi if [ -n "${input_file_path:-}" ] ; then + data_stores="disk hpss aws" additional_flags="$additional_flags \ --input_file_path ${input_file_path}" fi @@ -157,7 +159,7 @@ python3 -u ${USHDIR}/retrieve_data.py \ --anl_or_fcst ${anl_or_fcst} \ --config ${USHDIR}/templates/data_locations.yml \ --cycle_date ${extrn_mdl_cdate} \ - --data_stores disk hpss aws \ + --data_stores ${data_stores} \ --external_model ${extrn_mdl_name} \ --fcst_hrs ${fcst_hrs[@]} \ --output_path ${extrn_mdl_staging_dir} \ From ce63b1978afed7e0c25bd1c46416bc0131d72177 Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Mon, 28 Mar 2022 17:13:24 +0000 Subject: [PATCH 25/29] Include additional ops data paths. Use wrfprs. --- ush/templates/data_locations.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/ush/templates/data_locations.yml b/ush/templates/data_locations.yml index 5963d311c..c84579785 100644 --- a/ush/templates/data_locations.yml +++ b/ush/templates/data_locations.yml @@ -131,10 +131,13 @@ RAP: archive_format: tar archive_path: - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyymm}/{yyyymmdd} + - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyymm}/{yyyymmdd} archive_internal_dir: - ./ + - ./ archive_file_names: # RAP forecasts are binned into 6 hour tar files. + - gpfs_hps_nco_ops_com_rap_prod_rap.{yyyymmdd}{bin6}.wrf.tar - com_rap_prod_rap.{yyyymmdd}{bin6}.wrf.tar file_names: &rap_file_names anl: @@ -153,16 +156,19 @@ HRRR: archive_format: tar archive_path: - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyymm}/{yyyymmdd} + - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyymm}/{yyyymmdd} archive_internal_dir: - ./ + - ./ archive_file_names: # HRRR forecasts are binned into 6 hour tar files. - - com_hrrr_prod_hrrr.{yyyymmdd}_conus{bin6}.wrfnatdng.tar + - gpfs_hps_nco_ops_com_hrrr_prod_hrrr.{yyyymmdd}_conus{bin6}.wrf.tar + - com_hrrr_prod_hrrr.{yyyymmdd}_conus{bin6}.wrf.tar file_names: &hrrr_file_names anl: - - hrrr.t{hh}z.wrfnatf{fcst_hr:02d}.grib2 + - hrrr.t{hh}z.wrfprsf{fcst_hr:02d}.grib2 fcst: - - hrrr.t{hh}z.wrfnatf{fcst_hr:02d}.grib2 + - hrrr.t{hh}z.wrfprsf{fcst_hr:02d}.grib2 aws: protocol: download url: https://noaa-hrrr-bdp-pds.s3.amazonaws.com/hrrr.{yyyymmdd}/conus/ From 8a4d4b6f3bb252e58146f32f26a1ccd4d4f65950 Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Fri, 8 Apr 2022 14:40:07 +0000 Subject: [PATCH 26/29] Try multiple archive internal directories. --- ush/retrieve_data.py | 100 +++++++++++++++++++++++-------------------- 1 file changed, 53 insertions(+), 47 deletions(-) diff --git a/ush/retrieve_data.py b/ush/retrieve_data.py index c61c018c1..86cb87e07 100755 --- a/ush/retrieve_data.py +++ b/ush/retrieve_data.py @@ -354,9 +354,9 @@ def hpss_requested_files(cla, file_names, store_specs): f' {list(zip(archive_paths, archive_file_names))}') existing_archive, which_archive = find_archive_files(archive_paths, - archive_file_names, - cla.cycle_date, - ) + archive_file_names, + cla.cycle_date, + ) if not existing_archive: logging.warning('No archive files were found!') @@ -365,53 +365,59 @@ def hpss_requested_files(cla, file_names, store_specs): logging.info(f'Files in archive are named: {file_names}') - logging.debug(f'Grabbing archive number {which_archive} in list.') - archive_internal_dir = store_specs.get('archive_internal_dir', ['']) - if isinstance(archive_internal_dir, dict): - archive_internal_dir = archive_internal_dir.get(cla.anl_or_fcst, ['']) + archive_internal_dirs = store_specs.get('archive_internal_dir', ['']) + if isinstance(archive_internal_dirs, dict): + archive_internal_dirs = archive_internal_dirs.get(cla.anl_or_fcst, ['']) + + # which_archive matters for choosing the correct file names within, + # but we can safely just try all options for the + # archive_internal_dir + logging.debug(f'Checking archive number {which_archive} in list.') + + for archive_internal_dir_tmpl in archive_internal_dirs: + archive_internal_dir = fill_template(archive_internal_dir_tmpl, + cla.cycle_date) + + output_path = fill_template(cla.output_path, cla.cycle_date) + logging.info(f'Will place files in {os.path.abspath(output_path)}') + orig_path = os.getcwd() + os.chdir(output_path) + logging.debug(f'CWD: {os.getcwd()}') + + source_paths = [] + for fcst_hr in cla.fcst_hrs: + for file_name in file_names: + source_paths.append(fill_template( + os.path.join(archive_internal_dir, file_name), + cla.cycle_date, + fcst_hr, + )) + + if store_specs.get('archive_format', 'tar') == 'zip': + # Get the entire file from HPSS + existing_archive = hsi_single_file(existing_archive, mode='get') + + # Grab only the necessary files from the archive + cmd = f'unzip -o {os.path.basename(existing_archive)} {" ".join(source_paths)}' - archive_internal_dir = archive_internal_dir[which_archive] - archive_internal_dir = fill_template(archive_internal_dir, - cla.cycle_date) - - output_path = fill_template(cla.output_path, cla.cycle_date) - logging.info(f'Will place files in {os.path.abspath(output_path)}') - orig_path = os.getcwd() - os.chdir(output_path) - logging.debug(f'CWD: {os.getcwd()}') - - source_paths = [] - for fcst_hr in cla.fcst_hrs: - for file_name in file_names: - source_paths.append(fill_template( - os.path.join(archive_internal_dir, file_name), - cla.cycle_date, - fcst_hr, - )) - - if store_specs.get('archive_format', 'tar') == 'zip': - # Get the entire file from HPSS - existing_archive = hsi_single_file(existing_archive, mode='get') - - # Grab only the necessary files from the archive - cmd = f'unzip -o {os.path.basename(existing_archive)} {" ".join(source_paths)}' + else: + cmd = f'htar -xvf {existing_archive} {" ".join(source_paths)}' - else: - cmd = f'htar -xvf {existing_archive} {" ".join(source_paths)}' + logging.info(f'Running command \n {cmd}') + subprocess.run(cmd, + check=True, + shell=True, + ) - logging.info(f'Running command \n {cmd}') - subprocess.run(cmd, - check=True, - shell=True, - ) - - # Check that files exist and Remove any data transfer artifacts. - unavailable = clean_up_output_dir( - expected_subdir=archive_internal_dir, - local_archive=os.path.basename(existing_archive), - output_path=output_path, - source_paths=source_paths, - ) + # Check that files exist and Remove any data transfer artifacts. + unavailable = clean_up_output_dir( + expected_subdir=archive_internal_dir, + local_archive=os.path.basename(existing_archive), + output_path=output_path, + source_paths=source_paths, + ) + if not unavailable: + return unavailable os.chdir(orig_path) From 31f6050c9f15f9581ecc67509a188e9c5845c4ef Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Fri, 8 Apr 2022 14:40:37 +0000 Subject: [PATCH 27/29] Add netcdf files to FV3GFS. --- ush/templates/data_locations.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ush/templates/data_locations.yml b/ush/templates/data_locations.yml index c84579785..d82eeec67 100644 --- a/ush/templates/data_locations.yml +++ b/ush/templates/data_locations.yml @@ -65,6 +65,13 @@ FV3GFS: fcst: - gfs.t{hh}z.atmf{fcst_hr:03d}.nemsio - gfs.t{hh}z.sfcf{fcst_hr:03d}.nemsio + netcdf: + anl: + - gfs.t{hh}z.atmanl.nc + - gfs.t{hh}z.sfcanl.nc + fcst: + - gfs.t{hh}z.atmf{fcst_hr:03d}.nc + - gfs.t{hh}z.sfcf{fcst_hr:03d}.nc hpss: protocol: htar archive_path: From 29f977faf3d1b8783f217931e082901fbc4f9b82 Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Fri, 8 Apr 2022 15:08:29 +0000 Subject: [PATCH 28/29] Fix botched merge. --- tests/WE2E/run_WE2E_tests.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/WE2E/run_WE2E_tests.sh b/tests/WE2E/run_WE2E_tests.sh index 747fc9dfa..856ea6f8a 100755 --- a/tests/WE2E/run_WE2E_tests.sh +++ b/tests/WE2E/run_WE2E_tests.sh @@ -940,9 +940,11 @@ external model files should be located has not been specified for this machine (MACHINE): MACHINE= \"${MACHINE}\"" fi - EXTRN_MDL_SOURCE_BASEDIR_ICS="${EXTRN_MDL_SOURCE_BASEDIR_ICS}/\${DATE_FIRST_CYCL}\${CYCL_HRS[0]}" + EXTRN_MDL_SOURCE_BASEDIR_ICS="${extrn_mdl_source_basedir}/${EXTRN_MDL_NAME_ICS}" if [ "${EXTRN_MDL_NAME_ICS}" = "FV3GFS" ]; then EXTRN_MDL_SOURCE_BASEDIR_ICS="${EXTRN_MDL_SOURCE_BASEDIR_ICS}/${FV3GFS_FILE_FMT_ICS}/\${DATE_FIRST_CYCL}\${CYCL_HRS[0]}" + else + EXTRN_MDL_SOURCE_BASEDIR_ICS="${EXTRN_MDL_SOURCE_BASEDIR_ICS}/\${DATE_FIRST_CYCL}\${CYCL_HRS[0]}" fi if [ "${EXTRN_MDL_NAME_ICS}" = "FV3GFS" ] || \ [ "${EXTRN_MDL_NAME_ICS}" = "GSMGFS" ]; then @@ -955,11 +957,13 @@ machine (MACHINE): EXTRN_MDL_FILES_ICS=( "${EXTRN_MDL_NAME_ICS,,}.out.for_f000" ) fi - EXTRN_MDL_SOURCE_BASEDIR_LBCS="${EXTRN_MDL_SOURCE_BASEDIR_LBCS}/\${DATE_FIRST_CYCL}\${CYCL_HRS[0]}/for_LBCS" + EXTRN_MDL_SOURCE_BASEDIR_LBCS="${extrn_mdl_source_basedir}/${EXTRN_MDL_NAME_LBCS}" if [ "${EXTRN_MDL_NAME_LBCS}" = "FV3GFS" ] ; then EXTRN_MDL_SOURCE_BASEDIR_LBCS="${EXTRN_MDL_SOURCE_BASEDIR_LBCS}/${FV3GFS_FILE_FMT_LBCS}/\${DATE_FIRST_CYCL}\${CYCL_HRS[0]}" elif [ "${EXTRN_MDL_NAME_LBCS}" = "GSMGFS" ] ; then EXTRN_MDL_SOURCE_BASEDIR_LBCS="${EXTRN_MDL_SOURCE_BASEDIR_LBCS}/\${DATE_FIRST_CYCL}\${CYCL_HRS[0]}" + else + EXTRN_MDL_SOURCE_BASEDIR_LBCS="${EXTRN_MDL_SOURCE_BASEDIR_LBCS}/\${DATE_FIRST_CYCL}\${CYCL_HRS[0]}/for_LBCS" fi # # Make sure that the forecast length is evenly divisible by the interval From 2cf85d5b0e2f84cd107a84605fdb3ebb044166e9 Mon Sep 17 00:00:00 2001 From: "Christina.Holt" Date: Tue, 12 Apr 2022 17:39:40 +0000 Subject: [PATCH 29/29] Fix another merge botch. --- tests/WE2E/run_WE2E_tests.sh | 25 +++---------------------- 1 file changed, 3 insertions(+), 22 deletions(-) diff --git a/tests/WE2E/run_WE2E_tests.sh b/tests/WE2E/run_WE2E_tests.sh index 856ea6f8a..f3040208e 100755 --- a/tests/WE2E/run_WE2E_tests.sh +++ b/tests/WE2E/run_WE2E_tests.sh @@ -941,21 +941,13 @@ machine (MACHINE): MACHINE= \"${MACHINE}\"" fi EXTRN_MDL_SOURCE_BASEDIR_ICS="${extrn_mdl_source_basedir}/${EXTRN_MDL_NAME_ICS}" - if [ "${EXTRN_MDL_NAME_ICS}" = "FV3GFS" ]; then + if [ "${EXTRN_MDL_NAME_ICS}" = "FV3GFS" ] ; then EXTRN_MDL_SOURCE_BASEDIR_ICS="${EXTRN_MDL_SOURCE_BASEDIR_ICS}/${FV3GFS_FILE_FMT_ICS}/\${DATE_FIRST_CYCL}\${CYCL_HRS[0]}" + elif [ "${EXTRN_MDL_NAME_ICS}" = "NAM" ] ; then + EXTRN_MDL_SOURCE_BASEDIR_ICS="${EXTRN_MDL_SOURCE_BASEDIR_ICS}/\${DATE_FIRST_CYCL}\${CYCL_HRS[0]}/for_ICS" else EXTRN_MDL_SOURCE_BASEDIR_ICS="${EXTRN_MDL_SOURCE_BASEDIR_ICS}/\${DATE_FIRST_CYCL}\${CYCL_HRS[0]}" fi - if [ "${EXTRN_MDL_NAME_ICS}" = "FV3GFS" ] || \ - [ "${EXTRN_MDL_NAME_ICS}" = "GSMGFS" ]; then - if [ "${FV3GFS_FILE_FMT_ICS}" = "nemsio" ]; then - EXTRN_MDL_FILES_ICS=( "gfs.atmanl.nemsio" "gfs.sfcanl.nemsio" ) - elif [ "${FV3GFS_FILE_FMT_ICS}" = "grib2" ]; then - EXTRN_MDL_FILES_ICS=( "gfs.pgrb2.0p25.f000" ) - fi - else - EXTRN_MDL_FILES_ICS=( "${EXTRN_MDL_NAME_ICS,,}.out.for_f000" ) - fi EXTRN_MDL_SOURCE_BASEDIR_LBCS="${extrn_mdl_source_basedir}/${EXTRN_MDL_NAME_LBCS}" if [ "${EXTRN_MDL_NAME_LBCS}" = "FV3GFS" ] ; then @@ -979,17 +971,6 @@ boundary conditions specification interval (LBC_SPEC_INTVL_HRS): LBC_SPEC_INTVL_HRS = ${LBC_SPEC_INTVL_HRS} rem = FCST_LEN_HRS%%LBC_SPEC_INTVL_HRS = $rem" fi - if [ "${EXTRN_MDL_NAME_LBCS}" = "FV3GFS" ] || \ - [ "${EXTRN_MDL_NAME_LBCS}" = "GSMGFS" ]; then - if [ "${FV3GFS_FILE_FMT_LBCS}" = "nemsio" ]; then - EXTRN_MDL_FILES_LBCS=( 'gfs.atmf{fcst_hr:03d}.nemsio' ) - elif [ "${FV3GFS_FILE_FMT_LBCS}" = "grib2" ]; then - EXTRN_MDL_FILES_LBCS=( 'gfs.pgrb2.0p25.f{fcst_hr:03d}' ) - fi - else - EXTRN_MDL_FILES_LBCS=( '{yy}{jjj}{hh}{fcst_hr:04d}00' ) - fi - expt_config_str="${expt_config_str} # # Locations and names of user-staged external model files for generating