diff --git a/jobs/JREGIONAL_GET_EXTRN_MDL_FILES b/jobs/JREGIONAL_GET_EXTRN_MDL_FILES index a452062c5..31c296386 100755 --- a/jobs/JREGIONAL_GET_EXTRN_MDL_FILES +++ b/jobs/JREGIONAL_GET_EXTRN_MDL_FILES @@ -33,15 +33,6 @@ # #----------------------------------------------------------------------- # -# Source the file defining the function that will be used to set various -# external-model-associated variables. -# -#----------------------------------------------------------------------- -# -. $USHDIR/get_extrn_mdl_file_dir_info.sh -# -#----------------------------------------------------------------------- -# # Save current shell options (in a global array). Then set new options # for this script/function. # @@ -72,33 +63,107 @@ print_info_msg " Entering script: \"${scrfunc_fn}\" In directory: \"${scrfunc_dir}\" -This is the J-job script for the task that copies/fetches to a local -directory (either from disk or HPSS) the external model files from which -initial or boundary condition files for the FV3 will be generated. +This is the J-job script for the task that copies or fetches external +model files from disk, HPSS, or URL, and stages them for downstream use +to generate initial or lateral boundary conditions for the FV3 model. ========================================================================" + + +# +#----------------------------------------------------------------------- +# +# Check whether the environment variable ICS_OR_LBCS is set to a valid +# value. This variable specifies whether we are getting the external +# model files for the purpose of generating initial conditions (ICs) or +# lateral boundary condtions (LBCs) for the forecast model. +# +#----------------------------------------------------------------------- +# +valid_vals_ICS_OR_LBCS=( "ICS" "LBCS" ) +check_var_valid_value "ICS_OR_LBCS" "valid_vals_ICS_OR_LBCS" +# +#----------------------------------------------------------------------- +# +# Set parameters for grabbing either the initial conditions from analysis or +# forecast files of external models, or the lateral boundary conditions +# from external models. This script has been called to do the work for +# one or the other. +# +#----------------------------------------------------------------------- +# +if [ "${ICS_OR_LBCS}" = "ICS" ]; then + time_offset_hrs=${EXTRN_MDL_ICS_OFFSET_HRS:-0} + extrn_mdl_name=${EXTRN_MDL_NAME_ICS} + +elif [ "${ICS_OR_LBCS}" = "LBCS" ]; then + time_offset_hrs=${EXTRN_MDL_LBCS_OFFSET_HRS:-0} + extrn_mdl_name=${EXTRN_MDL_NAME_LBCS} +fi + +# +#----------------------------------------------------------------------- +# +# Set the external model start time +# +#----------------------------------------------------------------------- +# + +hh=${CDATE:8:2} +yyyymmdd=${CDATE:0:8} +extrn_mdl_cdate=$( $DATE_UTIL --utc --date "${yyyymmdd} ${hh} UTC - ${time_offset_hrs} hours" "+%Y%m%d%H" ) + # #----------------------------------------------------------------------- # -# Check whether output files from the specified external model (EXTRN_MDL_NAME) -# are available on the specified cycle date and time (CDATE). +# Check whether output files from the specified external model +# (extrn_mdl_name) are available on the specified cycle date and time +# (extrn_mdl_cdate). # #----------------------------------------------------------------------- # -case ${EXTRN_MDL_NAME} in + +function data_unavailable() { + + local name cdate end_date min_max + + name=$1 + cdate=$2 + end_date=$3 + min_max=$4 + + if [ ${min_max} = max ]; then + msg="\ +Output from the specified external model (extrn_mdl_name) is not availa- +ble for the specified cycle date and time (extrn_mdl_cdate) because the latter is +later than the last forecast date and time (cdate_max) with this model: + extrn_mdl_name = \"${name}\" + CDATE_max = \"${end_date}\" + extrn_mdl_cdate = \"${cdate}\"" + + elif [ ${min_max} = min ]; then + msg="\ +Output from the specified external model (extrn_mdl_name) is not availa- +ble for the specified cycle date and time (extrn_mdl_cdate) because the latter is +earlier than the implementation date of this model: + extrn_mdl_name = \"${name}\" + CDATE_min = \"${end_date}\" + extrn_mdl_cdate = \"${cdate}\"" + fi + + echo ${msg} +} + + +case ${extrn_mdl_name} in "GSMGFS") # The transition date from the GSMGFS to the FV3GFS was 2019061212, i.e. # this was the first official forecast with the FV3GFS. So we set the # last CDATE for the GSMGFS to the one 6 hours before this. cdate_max="2019061206" - if [ "$CDATE" -gt "$cdate_max" ]; then + if [ "$extrn_mdl_cdate" -gt "$cdate_max" ]; then print_err_msg_exit "\ -Output from the specified external model (EXTRN_MDL_NAME) is not availa- -ble for the specified cycle date and time (CDATE) because the latter is -later than the last forecast date and time (cdate_max) with this model: - EXTRN_MDL_NAME = \"${EXTRN_MDL_NAME}\" - cdate_max = \"${cdate_max}\" - CDATE = \"${CDATE}\"" + $(data_unavailable $extrn_mdl_name $extrn_mdl_cdate $cdate_max max)" fi ;; @@ -106,17 +171,12 @@ later than the last forecast date and time (cdate_max) with this model: # The transition date from the GSMGFS to the FV3GFS was 2019061212, i.e. # this was the first official forecast with the FV3GFS. However, paral- # lel runs with the FV3GFS go back to 2018121500. So we set the first -# CDATE for the FV3GFS to this date and time. +# extrn_mdl_cdate for the FV3GFS to this date and time. # CDATE_min="2019061212" CDATE_min="2018121500" - if [ "$CDATE" -lt "$CDATE_min" ]; then + if [ "$extrn_mdl_cdate" -lt "$CDATE_min" ]; then print_err_msg_exit "\ -Output from the specified external model (EXTRN_MDL_NAME) is not availa- -ble for the specified cycle date and time (CDATE) because the latter is -earlier than the implementation date of this model: - EXTRN_MDL_NAME = \"${EXTRN_MDL_NAME}\" - CDATE_min = \"${CDATE_min}\" - CDATE = \"${CDATE}\"" + $(data_unavailable $extrn_mdl_name $extrn_mdl_cdate $cdate_min min)" fi ;; @@ -124,14 +184,9 @@ earlier than the implementation date of this model: # Examination of the HPSS archives shows that the RAPX data goes back to # July 01, 2015. CDATE_min="2015070100" - if [ "$CDATE" -lt "$CDATE_min" ]; then + if [ "$extrn_mdl_cdate" -lt "$CDATE_min" ]; then print_err_msg_exit "\ -Output from the specified external model (EXTRN_MDL_NAME) is not availa- -ble for the specified cycle date and time (CDATE) because the latter is -earlier than the implementation date of this model: - EXTRN_MDL_NAME = \"${EXTRN_MDL_NAME}\" - CDATE_min = \"${CDATE_min}\" - CDATE = \"${CDATE}\"" + $(data_unavailable $extrn_mdl_name $extrn_mdl_cdate $cdate_min min)" fi ;; @@ -140,14 +195,9 @@ earlier than the implementation date of this model: # implementation of the first version of the operational HRRR was # September 30, 2014. CDATE_min="2014103000" - if [ "$CDATE" -lt "$CDATE_min" ]; then + if [ "$extrn_mdl_cdate" -lt "$CDATE_min" ]; then print_err_msg_exit "\ -Output from the specified external model (EXTRN_MDL_NAME) is not availa- -ble for the specified cycle date and time (CDATE) because the latter is -earlier than the implementation date of this model: - EXTRN_MDL_NAME = \"${EXTRN_MDL_NAME}\" - CDATE_min = \"${CDATE_min}\" - CDATE = \"${CDATE}\"" + $(data_unavailable $extrn_mdl_name $extrn_mdl_cdate $cdate_min min)" fi ;; @@ -155,141 +205,25 @@ esac # #----------------------------------------------------------------------- # -# Check whether the environment variable ICS_OR_LBCS is set to a valid -# value. This variable specifies whether we are getting the external -# model files for the purpose of generating initial conditions (ICs) or -# lateral boundary condtions (LBCs) for the forecast model. -# -#----------------------------------------------------------------------- -# -valid_vals_ICS_OR_LBCS=( "ICS" "LBCS" ) -check_var_valid_value "ICS_OR_LBCS" "valid_vals_ICS_OR_LBCS" -# -#----------------------------------------------------------------------- -# -# Set parameters for grabbing either the initial conditions from analysis or -# forecast files of external models, or the lateral boundary conditions -# from external models. The script has been called to do the work for -# one or the other. -# -#----------------------------------------------------------------------- -# -if [ "${ICS_OR_LBCS}" = "ICS" ]; then - if [ ${EXTRN_MDL_ICS_OFFSET_HRS} -eq 0 ] ; then - anl_or_fcst="ANL" - time_offset_hrs=0 - else - anl_or_fcst="FCST" - time_offset_hrs=${EXTRN_MDL_ICS_OFFSET_HRS:-0} - fi -elif [ "${ICS_OR_LBCS}" = "LBCS" ]; then - anl_or_fcst="FCST" - time_offset_hrs=${EXTRN_MDL_LBCS_OFFSET_HRS:-0} -fi -# -#----------------------------------------------------------------------- -# # Create the directory where the exetrnal model files should be stored # #----------------------------------------------------------------------- # -extrn_mdl_staging_dir="${CYCLE_DIR}/${EXTRN_MDL_NAME}/for_${ICS_OR_LBCS}" +extrn_mdl_staging_dir="${CYCLE_DIR}/${extrn_mdl_name}/for_${ICS_OR_LBCS}" mkdir_vrfy -p "${extrn_mdl_staging_dir}" cd_vrfy "${extrn_mdl_staging_dir}" # #----------------------------------------------------------------------- # -# Call the function that sets various external-model-associated variables. -# See the function defintion file for the definitions of these variables. -# -#----------------------------------------------------------------------- -# -get_extrn_mdl_file_dir_info \ - extrn_mdl_name="${EXTRN_MDL_NAME}" \ - anl_or_fcst="${anl_or_fcst}" \ - cdate_FV3LAM="${CDATE}" \ - time_offset_hrs="${time_offset_hrs}" \ - varname_extrn_mdl_cdate="extrn_mdl_cdate" \ - varname_extrn_mdl_lbc_spec_fhrs="extrn_mdl_lbc_spec_fhrs" \ - varname_extrn_mdl_fns_on_disk="extrn_mdl_fns_on_disk" \ - varname_extrn_mdl_fns_in_arcv="extrn_mdl_fns_in_arcv" \ - varname_extrn_mdl_sysdir="extrn_mdl_sysdir" \ - varname_extrn_mdl_arcv_fmt="extrn_mdl_arcv_fmt" \ - varname_extrn_mdl_arcv_fns="extrn_mdl_arcv_fns" \ - varname_extrn_mdl_arcv_fps="extrn_mdl_arcv_fps" \ - varname_extrn_mdl_arcvrel_dir="extrn_mdl_arcvrel_dir" || \ -print_err_msg_exit "\ -Call to function get_extrn_mdl_file_dir_info failed." -# -#----------------------------------------------------------------------- -# -# Set the directory in which to check for the external model files (which -# we refer to here as the "source" directory) to the default one set above -# for the current machine and external model. -# -#----------------------------------------------------------------------- -# -extrn_mdl_source_dir="${extrn_mdl_sysdir}" -# -#----------------------------------------------------------------------- -# -# If the user has specified that the external model files to be used for -# generating ICs or LBCs are staged, then reset extrn_mdl_source_dir to -# the user-specified directory in which these files are staged, and reset -# extrn_mdl_fns_on_disk to the user-specified array containing the names -# of the files. -# -#----------------------------------------------------------------------- -# -if [ "${USE_USER_STAGED_EXTRN_FILES}" = "TRUE" ]; then - - if [ "${ICS_OR_LBCS}" = "ICS" ]; then - extrn_mdl_source_dir="${EXTRN_MDL_SOURCE_BASEDIR_ICS}/$CDATE" - extrn_mdl_fns_on_disk=( $( printf "%s " "${EXTRN_MDL_FILES_ICS[@]}" )) - elif [ "${ICS_OR_LBCS}" = "LBCS" ]; then - extrn_mdl_source_dir="${EXTRN_MDL_SOURCE_BASEDIR_LBCS}/$CDATE" - extrn_mdl_fns_on_disk=( $( printf "%s " "${EXTRN_MDL_FILES_LBCS[@]}" )) - fi - - if [ ! -d "${extrn_mdl_source_dir}" ]; then - print_err_msg_exit "\ -The directory extrn_mdl_source_dir containing the user-staged external -model files does not exist: - extrn_mdl_source_dir = \"${extrn_mdl_source_dir}\" -Please ensure that the directory specified by extrn_mdl_source_dir exists -and that all the files specified in the array extrn_mdl_fns_on_disk exist -within it: - extrn_mdl_source_dir = \"${extrn_mdl_source_dir}\" - extrn_mdl_fns_on_disk = ( $( printf "\"%s\" " "${extrn_mdl_fns_on_disk[@]}" ))" - fi - -fi -# -#----------------------------------------------------------------------- -# # Call the ex-script for this J-job and pass to it the necessary variables. # #----------------------------------------------------------------------- # -extrn_mdl_lbc_spec_fhrs_str="( "$( printf "\"%s\" " "${extrn_mdl_lbc_spec_fhrs[@]}" )")" -extrn_mdl_fns_on_disk_str="( "$( printf "\"%s\" " "${extrn_mdl_fns_on_disk[@]}" )")" -extrn_mdl_fns_in_arcv_str="( "$( printf "\"%s\" " "${extrn_mdl_fns_in_arcv[@]}" )")" -extrn_mdl_arcv_fns_str="( "$( printf "\"%s\" " "${extrn_mdl_arcv_fns[@]}" )")" -extrn_mdl_arcv_fps_str="( "$( printf "\"%s\" " "${extrn_mdl_arcv_fps[@]}" )")" - $SCRIPTSDIR/exregional_get_extrn_mdl_files.sh \ - ics_or_lbcs="${ICS_OR_LBCS}" \ - use_user_staged_extrn_files="${USE_USER_STAGED_EXTRN_FILES}" \ extrn_mdl_cdate="${extrn_mdl_cdate}" \ - extrn_mdl_lbc_spec_fhrs="${extrn_mdl_lbc_spec_fhrs_str}" \ - extrn_mdl_fns_on_disk="${extrn_mdl_fns_on_disk_str}" \ - extrn_mdl_fns_in_arcv="${extrn_mdl_fns_in_arcv_str}" \ - extrn_mdl_source_dir="${extrn_mdl_source_dir}" \ + extrn_mdl_name="${extrn_mdl_name}" \ extrn_mdl_staging_dir="${extrn_mdl_staging_dir}" \ - extrn_mdl_arcv_fmt="${extrn_mdl_arcv_fmt}" \ - extrn_mdl_arcv_fns="${extrn_mdl_arcv_fns_str}" \ - extrn_mdl_arcv_fps="${extrn_mdl_arcv_fps_str}" \ - extrn_mdl_arcvrel_dir="${extrn_mdl_arcvrel_dir}" || \ + time_offset_hrs=${time_offset_hrs} || print_err_msg_exit "\ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." # diff --git a/modulefiles/tasks/cheyenne/get_extrn_ics b/modulefiles/tasks/cheyenne/get_extrn_ics index 58f82dca1..9f7a4e4f7 100644 --- a/modulefiles/tasks/cheyenne/get_extrn_ics +++ b/modulefiles/tasks/cheyenne/get_extrn_ics @@ -1,5 +1,4 @@ -#%Module##################################################### -## Module file intentionally blank for Cheyenne -############################################################# +#%Module +module load pylib_regional_workflow diff --git a/modulefiles/tasks/cheyenne/get_extrn_lbcs b/modulefiles/tasks/cheyenne/get_extrn_lbcs index 58f82dca1..9f7a4e4f7 100644 --- a/modulefiles/tasks/cheyenne/get_extrn_lbcs +++ b/modulefiles/tasks/cheyenne/get_extrn_lbcs @@ -1,5 +1,4 @@ -#%Module##################################################### -## Module file intentionally blank for Cheyenne -############################################################# +#%Module +module load pylib_regional_workflow diff --git a/modulefiles/tasks/cheyenne/make_grid.local b/modulefiles/tasks/cheyenne/make_grid.local index 4232b8659..2f92a39e5 100644 --- a/modulefiles/tasks/cheyenne/make_grid.local +++ b/modulefiles/tasks/cheyenne/make_grid.local @@ -1,9 +1,2 @@ #%Module -if [module-info mode load] { - system "ncar_pylib /glade/p/ral/jntp/UFS_CAM/ncar_pylib_20200427" -} - -if [module-info mode remove] { - system "deactivate" -} - +module load pylib_regional_workflow diff --git a/modulefiles/tasks/cheyenne/make_ics.local b/modulefiles/tasks/cheyenne/make_ics.local index 4232b8659..2f92a39e5 100644 --- a/modulefiles/tasks/cheyenne/make_ics.local +++ b/modulefiles/tasks/cheyenne/make_ics.local @@ -1,9 +1,2 @@ #%Module -if [module-info mode load] { - system "ncar_pylib /glade/p/ral/jntp/UFS_CAM/ncar_pylib_20200427" -} - -if [module-info mode remove] { - system "deactivate" -} - +module load pylib_regional_workflow diff --git a/modulefiles/tasks/cheyenne/make_lbcs.local b/modulefiles/tasks/cheyenne/make_lbcs.local index 4232b8659..2f92a39e5 100644 --- a/modulefiles/tasks/cheyenne/make_lbcs.local +++ b/modulefiles/tasks/cheyenne/make_lbcs.local @@ -1,9 +1,2 @@ #%Module -if [module-info mode load] { - system "ncar_pylib /glade/p/ral/jntp/UFS_CAM/ncar_pylib_20200427" -} - -if [module-info mode remove] { - system "deactivate" -} - +module load pylib_regional_workflow diff --git a/modulefiles/tasks/cheyenne/pylib_regional_workflow b/modulefiles/tasks/cheyenne/pylib_regional_workflow new file mode 100644 index 000000000..4232b8659 --- /dev/null +++ b/modulefiles/tasks/cheyenne/pylib_regional_workflow @@ -0,0 +1,9 @@ +#%Module +if [module-info mode load] { + system "ncar_pylib /glade/p/ral/jntp/UFS_CAM/ncar_pylib_20200427" +} + +if [module-info mode remove] { + system "deactivate" +} + diff --git a/modulefiles/tasks/cheyenne/run_fcst.local b/modulefiles/tasks/cheyenne/run_fcst.local index 4232b8659..2f92a39e5 100644 --- a/modulefiles/tasks/cheyenne/run_fcst.local +++ b/modulefiles/tasks/cheyenne/run_fcst.local @@ -1,9 +1,2 @@ #%Module -if [module-info mode load] { - system "ncar_pylib /glade/p/ral/jntp/UFS_CAM/ncar_pylib_20200427" -} - -if [module-info mode remove] { - system "deactivate" -} - +module load pylib_regional_workflow diff --git a/modulefiles/tasks/cheyenne/run_vx.local b/modulefiles/tasks/cheyenne/run_vx.local index 234d79908..8fb3be36d 100644 --- a/modulefiles/tasks/cheyenne/run_vx.local +++ b/modulefiles/tasks/cheyenne/run_vx.local @@ -1,11 +1,4 @@ #%Module - -if [module-info mode load] { - system "ncar_pylib /glade/p/ral/jntp/UFS_SRW_app/ncar_pylib_20200427" -} - -if [module-info mode remove] { - system "deactivate" -} +module load pylib_regional_workflow module use /glade/p/ral/jntp/MET/MET_releases/modulefiles module load met/10.0.0 diff --git a/modulefiles/tasks/hera/get_extrn_ics.local b/modulefiles/tasks/hera/get_extrn_ics.local index 9935033fd..4b0b48cc0 100644 --- a/modulefiles/tasks/hera/get_extrn_ics.local +++ b/modulefiles/tasks/hera/get_extrn_ics.local @@ -6,3 +6,4 @@ module purge module load hpss +module load miniconda_regional_workflow diff --git a/modulefiles/tasks/hera/get_extrn_lbcs.local b/modulefiles/tasks/hera/get_extrn_lbcs.local index 1919f3355..477dfb2e4 100644 --- a/modulefiles/tasks/hera/get_extrn_lbcs.local +++ b/modulefiles/tasks/hera/get_extrn_lbcs.local @@ -6,3 +6,4 @@ module purge module load hpss +module load miniconda_regional_workflow diff --git a/modulefiles/tasks/hera/make_grid.local b/modulefiles/tasks/hera/make_grid.local index 011a832c9..92505cf09 100644 --- a/modulefiles/tasks/hera/make_grid.local +++ b/modulefiles/tasks/hera/make_grid.local @@ -1,5 +1,3 @@ #%Module -module use -a /contrib/miniconda3/modulefiles -module load miniconda3 -setenv SRW_ENV regional_workflow +module load miniconda_regional_workflow diff --git a/modulefiles/tasks/hera/make_ics.local b/modulefiles/tasks/hera/make_ics.local index 011a832c9..92505cf09 100644 --- a/modulefiles/tasks/hera/make_ics.local +++ b/modulefiles/tasks/hera/make_ics.local @@ -1,5 +1,3 @@ #%Module -module use -a /contrib/miniconda3/modulefiles -module load miniconda3 -setenv SRW_ENV regional_workflow +module load miniconda_regional_workflow diff --git a/modulefiles/tasks/hera/make_lbcs.local b/modulefiles/tasks/hera/make_lbcs.local index 011a832c9..61a3a7725 100644 --- a/modulefiles/tasks/hera/make_lbcs.local +++ b/modulefiles/tasks/hera/make_lbcs.local @@ -1,5 +1,2 @@ #%Module -module use -a /contrib/miniconda3/modulefiles -module load miniconda3 - -setenv SRW_ENV regional_workflow +module load miniconda_regional_workflow diff --git a/modulefiles/tasks/hera/miniconda_regional_workflow b/modulefiles/tasks/hera/miniconda_regional_workflow new file mode 100644 index 000000000..011a832c9 --- /dev/null +++ b/modulefiles/tasks/hera/miniconda_regional_workflow @@ -0,0 +1,5 @@ +#%Module +module use -a /contrib/miniconda3/modulefiles +module load miniconda3 + +setenv SRW_ENV regional_workflow diff --git a/modulefiles/tasks/hera/run_fcst.local b/modulefiles/tasks/hera/run_fcst.local index 011a832c9..61a3a7725 100644 --- a/modulefiles/tasks/hera/run_fcst.local +++ b/modulefiles/tasks/hera/run_fcst.local @@ -1,5 +1,2 @@ #%Module -module use -a /contrib/miniconda3/modulefiles -module load miniconda3 - -setenv SRW_ENV regional_workflow +module load miniconda_regional_workflow diff --git a/modulefiles/tasks/jet/get_extrn_ics.local b/modulefiles/tasks/jet/get_extrn_ics.local index 9935033fd..4b0b48cc0 100644 --- a/modulefiles/tasks/jet/get_extrn_ics.local +++ b/modulefiles/tasks/jet/get_extrn_ics.local @@ -6,3 +6,4 @@ module purge module load hpss +module load miniconda_regional_workflow diff --git a/modulefiles/tasks/jet/get_extrn_lbcs.local b/modulefiles/tasks/jet/get_extrn_lbcs.local index 1919f3355..477dfb2e4 100644 --- a/modulefiles/tasks/jet/get_extrn_lbcs.local +++ b/modulefiles/tasks/jet/get_extrn_lbcs.local @@ -6,3 +6,4 @@ module purge module load hpss +module load miniconda_regional_workflow diff --git a/modulefiles/tasks/jet/make_grid.local b/modulefiles/tasks/jet/make_grid.local index 011a832c9..92505cf09 100644 --- a/modulefiles/tasks/jet/make_grid.local +++ b/modulefiles/tasks/jet/make_grid.local @@ -1,5 +1,3 @@ #%Module -module use -a /contrib/miniconda3/modulefiles -module load miniconda3 -setenv SRW_ENV regional_workflow +module load miniconda_regional_workflow diff --git a/modulefiles/tasks/jet/make_ics.local b/modulefiles/tasks/jet/make_ics.local index c3c0f61e5..61a3a7725 100644 --- a/modulefiles/tasks/jet/make_ics.local +++ b/modulefiles/tasks/jet/make_ics.local @@ -1,7 +1,2 @@ #%Module -module load wgrib2/2.0.8 - -module use -a /contrib/miniconda3/modulefiles -module load miniconda3 - -setenv SRW_ENV regional_workflow +module load miniconda_regional_workflow diff --git a/modulefiles/tasks/jet/make_lbcs.local b/modulefiles/tasks/jet/make_lbcs.local index c3c0f61e5..61a3a7725 100644 --- a/modulefiles/tasks/jet/make_lbcs.local +++ b/modulefiles/tasks/jet/make_lbcs.local @@ -1,7 +1,2 @@ #%Module -module load wgrib2/2.0.8 - -module use -a /contrib/miniconda3/modulefiles -module load miniconda3 - -setenv SRW_ENV regional_workflow +module load miniconda_regional_workflow diff --git a/modulefiles/tasks/jet/miniconda_regional_workflow b/modulefiles/tasks/jet/miniconda_regional_workflow new file mode 100644 index 000000000..011a832c9 --- /dev/null +++ b/modulefiles/tasks/jet/miniconda_regional_workflow @@ -0,0 +1,5 @@ +#%Module +module use -a /contrib/miniconda3/modulefiles +module load miniconda3 + +setenv SRW_ENV regional_workflow diff --git a/modulefiles/tasks/jet/run_fcst.local b/modulefiles/tasks/jet/run_fcst.local index 011a832c9..61a3a7725 100644 --- a/modulefiles/tasks/jet/run_fcst.local +++ b/modulefiles/tasks/jet/run_fcst.local @@ -1,5 +1,2 @@ #%Module -module use -a /contrib/miniconda3/modulefiles -module load miniconda3 - -setenv SRW_ENV regional_workflow +module load miniconda_regional_workflow diff --git a/modulefiles/tasks/orion/get_extrn_ics.local b/modulefiles/tasks/orion/get_extrn_ics.local index a9d5b4412..655aa30a2 100644 --- a/modulefiles/tasks/orion/get_extrn_ics.local +++ b/modulefiles/tasks/orion/get_extrn_ics.local @@ -3,4 +3,5 @@ ############################################################# module purge +module load miniconda_regional_workflow diff --git a/modulefiles/tasks/orion/get_extrn_lbcs.local b/modulefiles/tasks/orion/get_extrn_lbcs.local index 09f37151a..a05a15faa 100644 --- a/modulefiles/tasks/orion/get_extrn_lbcs.local +++ b/modulefiles/tasks/orion/get_extrn_lbcs.local @@ -4,3 +4,4 @@ module purge +module load miniconda_regional_workflow diff --git a/modulefiles/tasks/orion/make_grid.local b/modulefiles/tasks/orion/make_grid.local index 4de2a79ca..92505cf09 100644 --- a/modulefiles/tasks/orion/make_grid.local +++ b/modulefiles/tasks/orion/make_grid.local @@ -1,6 +1,3 @@ #%Module -module use -a /apps/contrib/miniconda3-noaa-gsl/modulefiles -module load miniconda3/3.8 - -setenv SRW_ENV regional_workflow +module load miniconda_regional_workflow diff --git a/modulefiles/tasks/orion/make_ics.local b/modulefiles/tasks/orion/make_ics.local index 3703a9ba1..61a3a7725 100644 --- a/modulefiles/tasks/orion/make_ics.local +++ b/modulefiles/tasks/orion/make_ics.local @@ -1,5 +1,2 @@ #%Module -module use -a /apps/contrib/miniconda3-noaa-gsl/modulefiles -module load miniconda3/3.8 - -setenv SRW_ENV regional_workflow +module load miniconda_regional_workflow diff --git a/modulefiles/tasks/orion/make_lbcs.local b/modulefiles/tasks/orion/make_lbcs.local index 3703a9ba1..61a3a7725 100644 --- a/modulefiles/tasks/orion/make_lbcs.local +++ b/modulefiles/tasks/orion/make_lbcs.local @@ -1,5 +1,2 @@ #%Module -module use -a /apps/contrib/miniconda3-noaa-gsl/modulefiles -module load miniconda3/3.8 - -setenv SRW_ENV regional_workflow +module load miniconda_regional_workflow diff --git a/modulefiles/tasks/orion/miniconda_regional_workflow b/modulefiles/tasks/orion/miniconda_regional_workflow new file mode 100644 index 000000000..3703a9ba1 --- /dev/null +++ b/modulefiles/tasks/orion/miniconda_regional_workflow @@ -0,0 +1,5 @@ +#%Module +module use -a /apps/contrib/miniconda3-noaa-gsl/modulefiles +module load miniconda3/3.8 + +setenv SRW_ENV regional_workflow diff --git a/modulefiles/tasks/orion/run_fcst.local b/modulefiles/tasks/orion/run_fcst.local index 3703a9ba1..61a3a7725 100644 --- a/modulefiles/tasks/orion/run_fcst.local +++ b/modulefiles/tasks/orion/run_fcst.local @@ -1,5 +1,2 @@ #%Module -module use -a /apps/contrib/miniconda3-noaa-gsl/modulefiles -module load miniconda3/3.8 - -setenv SRW_ENV regional_workflow +module load miniconda_regional_workflow diff --git a/scripts/exregional_get_extrn_mdl_files.sh b/scripts/exregional_get_extrn_mdl_files.sh index ac5127eb8..fc02e3b99 100755 --- a/scripts/exregional_get_extrn_mdl_files.sh +++ b/scripts/exregional_get_extrn_mdl_files.sh @@ -42,9 +42,10 @@ print_info_msg " Entering script: \"${scrfunc_fn}\" In directory: \"${scrfunc_dir}\" -This is the ex-script for the task that copies/fetches to a local directory -either from disk or HPSS) the external model files from which initial or -boundary condition files for the FV3 will be generated. +This is the ex-script for the task that copies or fetches external model +input data from disk, HPSS, or a URL, and stages them to the +workflow-specified location so that they may be used to generate initial +or lateral boundary conditions for the FV3. ========================================================================" # #----------------------------------------------------------------------- @@ -56,18 +57,10 @@ boundary condition files for the FV3 will be generated. #----------------------------------------------------------------------- # valid_args=( \ -"ics_or_lbcs" \ -"use_user_staged_extrn_files" \ "extrn_mdl_cdate" \ -"extrn_mdl_lbc_spec_fhrs" \ -"extrn_mdl_fns_on_disk" \ -"extrn_mdl_fns_in_arcv" \ -"extrn_mdl_source_dir" \ +"extrn_mdl_name" \ "extrn_mdl_staging_dir" \ -"extrn_mdl_arcv_fmt" \ -"extrn_mdl_arcv_fns" \ -"extrn_mdl_arcv_fps" \ -"extrn_mdl_arcvrel_dir" \ +"time_offset_hrs" \ ) process_args valid_args "$@" # @@ -80,715 +73,105 @@ process_args valid_args "$@" #----------------------------------------------------------------------- # print_input_args valid_args -# -#----------------------------------------------------------------------- -# -# Set num_files_to_copy to the number of external model files that need -# to be copied or linked to from/at a location on disk. Then set -# extrn_mdl_fps_on_disk to the full paths of the external model files -# on disk. -# -#----------------------------------------------------------------------- -# -num_files_to_copy="${#extrn_mdl_fns_on_disk[@]}" -prefix="${extrn_mdl_source_dir}/" -extrn_mdl_fps_on_disk=( "${extrn_mdl_fns_on_disk[@]/#/$prefix}" ) -# -#----------------------------------------------------------------------- -# -# Loop through the list of external model files and check whether they -# all exist on disk. The counter num_files_found_on_disk keeps track of -# the number of external model files that were actually found on disk in -# the directory specified by extrn_mdl_source_dir. -# -# If the location extrn_mdl_source_dir is a user-specified directory -# (i.e. if use_user_staged_extrn_files is set to "TRUE"), then if/when we -# encounter the first file that does not exist, we exit the script with -# an error message. If extrn_mdl_source_dir is a system directory (i.e. -# if use_user_staged_extrn_files is not set to "TRUE"), then if/when we -# encounter the first file that does not exist or exists but is younger -# than a certain age, we break out of the loop and try to fetch all the -# necessary external model files from HPSS. The age cutoff is to ensure -# that files are not still being written to. -# -#----------------------------------------------------------------------- -# -num_files_found_on_disk="0" -min_age="5" # Minimum file age, in minutes. - -for fp in "${extrn_mdl_fps_on_disk[@]}"; do - # - # If the external model file exists, then... - # - if [ -f "$fp" ]; then - # - # Increment the counter that keeps track of the number of external - # model files found on disk and print out an informational message. - # - num_files_found_on_disk=$(( num_files_found_on_disk+1 )) - print_info_msg " -File fp exists on disk: - fp = \"$fp\"" - # - # If we are NOT searching for user-staged external model files, then - # we also check that the current file is at least min_age minutes old. - # If not, we try searching for all the external model files on HPSS. - # - if [ "${use_user_staged_extrn_files}" != "TRUE" ]; then - - if [ $( find "$fp" -mmin +${min_age} ) ]; then - - print_info_msg " -File fp is older than the minimum required age of min_age minutes: - fp = \"$fp\" - min_age = ${min_age} minutes" - - else - - print_info_msg " -File fp is NOT older than the minumum required age of min_age minutes: - fp = \"$fp\" - min_age = ${min_age} minutes -Will try fetching all external model files from HPSS. Not checking -presence and age of remaining external model files on disk." - break - - fi - - fi - # - # If the external model file does not exist, then... - # - else - # - # If an external model file is not found and we are searching for it - # in a user-specified directory, print out an error message and exit. - # - if [ "${use_user_staged_extrn_files}" = "TRUE" ]; then - - print_err_msg_exit "\ -File fp does NOT exist on disk: - fp = \"$fp\" -Please ensure that the directory specified by extrn_mdl_source_dir exists -and that all the files specified in the array extrn_mdl_fns_on_disk exist -within it: - extrn_mdl_source_dir = \"${extrn_mdl_source_dir}\" - extrn_mdl_fns_on_disk = ( $( printf "\"%s\" " "${extrn_mdl_fns_on_disk[@]}" ))" - # - # If an external model file is not found and we are searching for it - # in a system directory, give up on the system directory and try instead - # to get all the external model files from HPSS. - # - else - - print_info_msg " -File fp does NOT exist on disk: - fp = \"$fp\" -Will try fetching all external model files from HPSS. Not checking -presence and age of remaining external model files on disk." - break - - fi - fi - -done -# -#----------------------------------------------------------------------- -# -# Set the variable (data_src) that determines the source of the external -# model files (either disk or HPSS). -# -#----------------------------------------------------------------------- -# -if [ "${num_files_found_on_disk}" -eq "${num_files_to_copy}" ]; then - data_src="disk" -else - data_src="HPSS" -fi -if [ ${NOMADS} == "TRUE" ]; then - data_src="online" -fi # #----------------------------------------------------------------------- # -# If the source of the external model files is "disk", copy the files -# from the source directory on disk to a staging directory. +# Set up variables for call to retrieve_data.py # #----------------------------------------------------------------------- # -extrn_mdl_fns_on_disk_str="( "$( printf "\"%s\" " "${extrn_mdl_fns_on_disk[@]}" )")" - -if [ "${data_src}" = "disk" ]; then - - if [ "${RUN_ENVIR}" = "nco" ]; then - - print_info_msg " -Creating links in staging directory (extrn_mdl_staging_dir) to external -model files on disk (extrn_mdl_fns_on_disk) in the source directory -(extrn_mdl_source_dir): - extrn_mdl_staging_dir = \"${extrn_mdl_staging_dir}\" - extrn_mdl_source_dir = \"${extrn_mdl_source_dir}\" - extrn_mdl_fns_on_disk = ${extrn_mdl_fns_on_disk_str}" - - ln_vrfy -sf -t ${extrn_mdl_staging_dir} ${extrn_mdl_fps_on_disk[@]} - +set -x +if [ "${ICS_OR_LBCS}" = "ICS" ]; then + if [ ${time_offset_hrs} -eq 0 ] ; then + anl_or_fcst="anl" else - - # - # If the external model files are user-staged, then simply link to - # them. Otherwise, if they are on the system disk, copy them to the - # staging directory. - # - if [ "${use_user_staged_extrn_files}" = "TRUE" ]; then - print_info_msg " -Creating symlinks in the staging directory (extrn_mdl_staging_dir) to the -external model files on disk (extrn_mdl_fns_on_disk) in the source directory -(extrn_mdl_source_dir): - extrn_mdl_source_dir = \"${extrn_mdl_source_dir}\" - extrn_mdl_fns_on_disk = ${extrn_mdl_fns_on_disk_str} - extrn_mdl_staging_dir = \"${extrn_mdl_staging_dir}\"" - ln_vrfy -sf -t ${extrn_mdl_staging_dir} ${extrn_mdl_fps_on_disk[@]} - else - print_info_msg " -Copying external model files on disk (extrn_mdl_fns_on_disk) from source -directory (extrn_mdl_source_dir) to staging directory (extrn_mdl_staging_dir): - extrn_mdl_source_dir = \"${extrn_mdl_source_dir}\" - extrn_mdl_fns_on_disk = ${extrn_mdl_fns_on_disk_str} - extrn_mdl_staging_dir = \"${extrn_mdl_staging_dir}\"" - cp_vrfy ${extrn_mdl_fps_on_disk[@]} ${extrn_mdl_staging_dir} - fi - + anl_or_fcst="fcst" fi -# -#----------------------------------------------------------------------- -# -# Print message indicating successful completion of script. -# -#----------------------------------------------------------------------- -# - if [ "${ics_or_lbcs}" = "ICS" ]; then - - print_info_msg " -======================================================================== -Successfully copied or linked to external model files on disk needed for -generating initial conditions and surface fields for the FV3 forecast!!! - -Exiting script: \"${scrfunc_fn}\" -In directory: \"${scrfunc_dir}\" -========================================================================" - - elif [ "${ics_or_lbcs}" = "LBCS" ]; then - - print_info_msg " -======================================================================== -Successfully copied or linked to external model files on disk needed for -generating lateral boundary conditions for the FV3 forecast!!! - -Exiting script: \"${scrfunc_fn}\" -In directory: \"${scrfunc_dir}\" -========================================================================" - + fcst_hrs=${time_offset_hrs} + file_names=${EXTRN_MDL_FILES_ICS[@]} + if [ ${extrn_mdl_name} = FV3GFS ] ; then + file_type=$FV3GFS_FILE_FMT_ICS fi -# -#----------------------------------------------------------------------- -# -# If the source of the external model files is "HPSS", fetch them from -# HPSS. -# -#----------------------------------------------------------------------- -# -elif [ "${data_src}" = "HPSS" ]; then -# -#----------------------------------------------------------------------- -# -# Set extrn_mdl_fps_in_arcv to the full paths within the archive files of -# the external model files. -# -#----------------------------------------------------------------------- -# - prefix=${extrn_mdl_arcvrel_dir:+${extrn_mdl_arcvrel_dir}/} - extrn_mdl_fps_in_arcv=( "${extrn_mdl_fns_in_arcv[@]/#/$prefix}" ) - - extrn_mdl_fps_in_arcv_str="( "$( printf "\"%s\" " "${extrn_mdl_fps_in_arcv[@]}" )")" - extrn_mdl_arcv_fps_str="( "$( printf "\"%s\" " "${extrn_mdl_arcv_fps[@]}" )")" - - print_info_msg " -Fetching external model files from HPSS. The full paths to these files -in the archive file(s) (extrn_mdl_fps_in_arcv), the archive files on HPSS -in which these files are stored (extrn_mdl_arcv_fps), and the staging -directory to which they will be copied (extrn_mdl_staging_dir) are: - extrn_mdl_fps_in_arcv = ${extrn_mdl_fps_in_arcv_str} - extrn_mdl_arcv_fps = ${extrn_mdl_arcv_fps_str} - extrn_mdl_staging_dir = \"${extrn_mdl_staging_dir}\"" -# -#----------------------------------------------------------------------- -# -# Get the number of archive files to consider. -# -#----------------------------------------------------------------------- -# - num_arcv_files="${#extrn_mdl_arcv_fps[@]}" -# -#----------------------------------------------------------------------- -# -# Consider the case of the archive file to be fetched from HPSS being in -# tar format. -# -#----------------------------------------------------------------------- -# - if [ "${extrn_mdl_arcv_fmt}" = "tar" ]; then -# -#----------------------------------------------------------------------- -# -# Loop through the set of archive files specified in extrn_mdl_arcv_fps -# and extract a subset of the specified external model files from each. -# -#----------------------------------------------------------------------- -# - num_files_to_extract="${#extrn_mdl_fps_in_arcv[@]}" - - for (( narcv=0; narcv<${num_arcv_files}; narcv++ )); do - - narcv_formatted=$( printf "%02d" $narcv ) - arcv_fp="${extrn_mdl_arcv_fps[$narcv]}" -# -# Before trying to extract (a subset of) the external model files from -# the current tar archive file (which is on HPSS), create a list of those -# external model files that are stored in the current tar archive file. -# For this purpose, we first use the "htar -tvf" command to list all the -# external model files that are in the current archive file and store the -# result in a log file. (This command also indirectly checks whether the -# archive file exists on HPSS.) We then grep this log file for each -# external model file and create a list containing only those external -# model files that exist in the current archive. -# -# Note that the "htar -tvf" command will fail if the tar archive file -# itself doesn't exist on HPSS, but it won't fail if any of the external -# model file names passed to it don't exist in the archive file. In the -# latter case, the missing files' names simply won't appear in the log -# file. -# - htar_log_fn="log.htar_tvf.${narcv_formatted}" - htar -tvf ${arcv_fp} ${extrn_mdl_fps_in_arcv[@]} >& ${htar_log_fn} || \ - print_err_msg_exit "\ -htar file list operation (\"htar -tvf ...\") failed. Check the log file -htar_log_fn in the staging directory (extrn_mdl_staging_di)r for details: - extrn_mdl_staging_dir = \"${extrn_mdl_staging_dir}\" - htar_log_fn = \"${htar_log_fn}\"" - - i=0 - files_in_crnt_arcv=() - for (( nfile=0; nfile<${num_files_to_extract}; nfile++ )); do - extrn_mdl_fp="${extrn_mdl_fps_in_arcv[$nfile]}" -# grep -n ${extrn_mdl_fp} ${htar_log_fn} 2>&1 && { \ - grep -n ${extrn_mdl_fp} ${htar_log_fn} > /dev/null 2>&1 && { \ - files_in_crnt_arcv[$i]="${extrn_mdl_fp}"; \ - i=$((i+1)); \ - } - done -# -# If none of the external model files were found in the current archive -# file, print out an error message and exit. -# - num_files_in_crnt_arcv=${#files_in_crnt_arcv[@]} - if [ ${num_files_in_crnt_arcv} -eq 0 ]; then - extrn_mdl_fps_in_arcv_str="( "$( printf "\"%s\" " "${extrn_mdl_fps_in_arcv[@]}" )")" - print_err_msg_exit "\ -The current archive file (arcv_fp) does not contain any of the external -model files listed in extrn_mdl_fps_in_arcv: - arcv_fp = \"${arcv_fp}\" - extrn_mdl_fps_in_arcv = ${extrn_mdl_fps_in_arcv_str} -The archive file should contain at least one external model file; otherwise, -it would not be needed." - fi -# -# Extract from the current tar archive file on HPSS all the external model -# files that exist in that archive file. Also, save the output of the -# "htar -xvf" command in a log file for debugging (if necessary). -# - htar_log_fn="log.htar_xvf.${narcv_formatted}" - htar -xvf ${arcv_fp} ${files_in_crnt_arcv[@]} >& ${htar_log_fn} || \ - print_err_msg_exit "\ -htar file extract operation (\"htar -xvf ...\") failed. Check the log -file htar_log_fn in the staging directory (extrn_mdl_staging_dir) for -details: - extrn_mdl_staging_dir = \"${extrn_mdl_staging_dir}\" - htar_log_fn = \"${htar_log_fn}\"" -# -# Note that the htar file extract operation above may return with a 0 -# exit code (success) even if one or more (or all) external model files -# that it is supposed to contain were not extracted. The names of those -# files that were not extracted will not be listed in the log file. Thus, -# we now check whether the log file contains the name of each external -# model file that should have been extracted. If any are missing, we -# print out a message and exit the script because initial condition and -# surface field files needed by FV3 cannot be generated without all the -# external model files. -# - for fp in "${files_in_crnt_arcv[@]}"; do -# -# If the file path is absolute (i.e. starts with a "/"), then drop the -# leading "/" because htar strips it before writing the file path to the -# log file. -# - fp=${fp#/} - - grep -n "${fp}" "${htar_log_fn}" > /dev/null 2>&1 || \ - print_err_msg_exit "\ -External model file fp not extracted from tar archive file arcv_fp: - arcv_fp = \"${arcv_fp}\" - fp = \"$fp\" -Check the log file htar_log_fn in the staging directory (extrn_mdl_staging_dir) -for details: - extrn_mdl_staging_dir = \"${extrn_mdl_staging_dir}\" - htar_log_fn = \"${htar_log_fn}\"" - - done - - done -# -#----------------------------------------------------------------------- -# -# For each external model file that was supposed to have been extracted -# from the set of specified archive files, loop through the extraction -# log files and check that it appears exactly once in one of the log files. -# If it doesn't appear at all, then it means that file was not extracted, -# and if it appears more than once, then something else is wrong. In -# either case, print out an error message and exit. -# -#----------------------------------------------------------------------- -# - for (( nfile=0; nfile<${num_files_to_extract}; nfile++ )); do - extrn_mdl_fp="${extrn_mdl_fps_in_arcv[$nfile]}" -# -# If the file path is absolute (i.e. starts with a "/"), then drop the -# leading "/" because htar strips it before writing the file path to the -# log file. -# - extrn_mdl_fp=${extrn_mdl_fp#/} - - num_occurs=0 - for (( narcv=0; narcv<${num_arcv_files}; narcv++ )); do - narcv_formatted=$( printf "%02d" $narcv ) - htar_log_fn="log.htar_xvf.${narcv_formatted}" - grep -n ${extrn_mdl_fp} ${htar_log_fn} > /dev/null 2>&1 && { \ - num_occurs=$((num_occurs+1)); \ - } - done - - if [ ${num_occurs} -eq 0 ]; then - print_err_msg_exit "\ -The current external model file (extrn_mdl_fp) does not appear in any of -the archive extraction log files: - extrn_mdl_fp = \"${extrn_mdl_fp}\" -Thus, it was not extracted, likely because it doesn't exist in any of the -archive files." - elif [ ${num_occurs} -gt 1 ]; then - print_err_msg_exit "\ -The current external model file (extrn_mdl_fp) appears more than once in -the archive extraction log files: - extrn_mdl_fp = \"${extrn_mdl_fp}\" -The number of times it occurs in the log files is: - num_occurs = ${num_occurs} -Thus, it was extracted from more than one archive file, with the last one -that was extracted overwriting all previous ones. This should normally -not happen." - fi - - done -# -#----------------------------------------------------------------------- -# -# If extrn_mdl_arcvrel_dir is not set to the current directory (i.e. it -# is not equal to "."), then the htar command will have created the -# subdirectory "./${extrn_mdl_arcvrel_dir}" under the current directory -# and placed the extracted files there. In that case, we move these -# extracted files back to the current directory and then remove the -# subdirectory created by htar. -# -#----------------------------------------------------------------------- -# - if [ "${extrn_mdl_arcvrel_dir}" != "." ]; then -# -# The code below works if extrn_mdl_arcvrel_dir starts with a "/" or a -# "./", which are the only case encountered thus far. The code may have -# to be modified to accomodate other cases. -# - if [ "${extrn_mdl_arcvrel_dir:0:1}" = "/" ] || \ - [ "${extrn_mdl_arcvrel_dir:0:2}" = "./" ]; then -# -# Strip the "/" or "./" from the beginning of extrn_mdl_arcvrel_dir to -# obtain the relative directory from which to move the extracted files -# to the current directory. Then move the files. -# - rel_dir=$( printf "%s" "${extrn_mdl_arcvrel_dir}" | \ - $SED -r 's%^(\/|\.\/)([^/]*)(.*)%\2\3%' ) - mv_vrfy ${rel_dir}/* . -# -# Get the first subdirectory in rel_dir, i.e. the subdirectory before the -# first forward slash. This is the subdirectory that we want to remove -# since it no longer contains any files (only subdirectories). Then remove -# it. -# - subdir_to_remove=$( printf "%s" "${rel_dir}" | \ - $SED -r 's%^([^/]*)(.*)%\1%' ) - rm_vrfy -rf ./${subdir_to_remove} -# -# If extrn_mdl_arcvrel_dir does not start with a "/" (and it is not -# equal to "."), then print out an error message and exit. -# - else - - print_err_msg_exit "\ -The archive-relative directory specified by extrn_mdl_arcvrel_dir [i.e. -the directory \"within\" the tar file(s) listed in extrn_mdl_arcv_fps] is -not the current directory (i.e. it is not \".\"), and it does not start -with a \"/\" or a \"./\": - extrn_mdl_arcvrel_dir = \"${extrn_mdl_arcvrel_dir}\" - extrn_mdl_arcv_fps = ${extrn_mdl_arcv_fps_str} -This script must be modified to account for this case." - - fi - - fi -# -#----------------------------------------------------------------------- -# -# Consider the case of the archive file to be fetched from HPSS being in -# zip format. -# -#----------------------------------------------------------------------- -# - elif [ "${extrn_mdl_arcv_fmt}" = "zip" ]; then -# -#----------------------------------------------------------------------- -# -# For archive files that are in "zip" format files, the array extrn_mdl_arcv_fps -# containing the list of archive files should contain only one element, -# i.e. there should be only one archive file to consider. Check for this. -# If this ever changes (e.g. due to the way an external model that uses -# the "zip" format archives its output files on HPSS), the code below must -# be modified to loop over all archive files. -# -#----------------------------------------------------------------------- -# - if [ "${num_arcv_files}" -gt 1 ]; then - print_err_msg_exit "\ -Currently, this script is coded to handle only one archive file if the -archive file format is specified to be \"zip\", but the number of archive -files (num_arcv_files) passed to this script is greater than 1: - extrn_mdl_arcv_fmt = \"${extrn_mdl_arcv_fmt}\" - num_arcv_files = ${num_arcv_files} -Please modify the script to handle more than one \"zip\" archive file. -Note that code already exists in this script that can handle multiple -archive files if the archive file format is specified to be \"tar\", so -that can be used as a guide for the \"zip\" case." - else - arcv_fn="${extrn_mdl_arcv_fns[0]}" - arcv_fp="${extrn_mdl_arcv_fps[0]}" - fi -# -#----------------------------------------------------------------------- -# -# Fetch the zip archive file from HPSS. -# -#----------------------------------------------------------------------- -# - hsi_log_fn="log.hsi_get" - hsi get "${arcv_fp}" >& ${hsi_log_fn} || \ - print_err_msg_exit "\ -hsi file get operation (\"hsi get ...\") failed. Check the log file -hsi_log_fn in the staging directory (extrn_mdl_staging_dir) for details: - extrn_mdl_staging_dir = \"${extrn_mdl_staging_dir}\" - hsi_log_fn = \"${hsi_log_fn}\"" -# -#----------------------------------------------------------------------- -# -# List the contents of the zip archive file and save the result in a log -# file. -# -#----------------------------------------------------------------------- -# - unzip_log_fn="log.unzip_lv" - unzip -l -v ${arcv_fn} >& ${unzip_log_fn} || \ - print_err_msg_exit "\ -unzip operation to list the contents of the zip archive file arcv_fn in -the staging directory (extrn_mdl_staging_dir) failed. Check the log -file unzip_log_fn in that directory for details: - arcv_fn = \"${arcv_fn}\" - extrn_mdl_staging_dir = \"${extrn_mdl_staging_dir}\" - unzip_log_fn = \"${unzip_log_fn}\"" -# -#----------------------------------------------------------------------- -# -# Check that the log file from the unzip command above contains the name -# of each external model file. If any are missing, then the corresponding -# files are not in the zip file and thus cannot be extracted. In that -# case, print out a message and exit the script because initial condition -# and surface field files for the FV3-LAM cannot be generated without all -# the external model files. -# -#----------------------------------------------------------------------- -# - for fp in "${extrn_mdl_fps_in_arcv[@]}"; do - grep -n "${fp}" "${unzip_log_fn}" > /dev/null 2>&1 || \ - print_err_msg_exit "\ -External model file fp does not exist in the zip archive file arcv_fn in -the staging directory (extrn_mdl_staging_dir). Check the log file -unzip_log_fn in that directory for the contents of the zip archive: - extrn_mdl_staging_dir = \"${extrn_mdl_staging_dir}\" - arcv_fn = \"${arcv_fn}\" - fp = \"$fp\" - unzip_log_fn = \"${unzip_log_fn}\"" - done -# -#----------------------------------------------------------------------- -# -# Extract the external model files from the zip file on HPSS. Note that -# the -o flag to unzip is needed to overwrite existing files. Otherwise, -# unzip will wait for user input as to whether the existing files should -# be overwritten. -# -#----------------------------------------------------------------------- -# - unzip_log_fn="log.unzip" - unzip -o "${arcv_fn}" ${extrn_mdl_fps_in_arcv[@]} >& ${unzip_log_fn} || \ - print_err_msg_exit "\ -unzip file extract operation (\"unzip -o ...\") failed. Check the log -file unzip_log_fn in the staging directory (extrn_mdl_staging_dir) for -details: - extrn_mdl_staging_dir = \"${extrn_mdl_staging_dir}\" - unzip_log_fn = \"${unzip_log_fn}\"" -# -# NOTE: -# If extrn_mdl_arcvrel_dir is not empty, the unzip command above will -# create a subdirectory under extrn_mdl_staging_dir and place the external -# model files there. We have not encountered this for the RAP and HRRR -# models, but it may happen for other models in the future. In that case, -# extra code must be included here to move the external model files from -# the subdirectory up to extrn_mdl_staging_dir and then the subdirectory -# (analogous to what is done above for the case of extrn_mdl_arcv_fmt set -# to "tar". -# - + input_file_path=${EXTRN_MDL_SOURCE_BASEDIR_ICS:-$EXTRN_MDL_SYSBASEDIR_ICS} + +elif [ "${ICS_OR_LBCS}" = "LBCS" ]; then + anl_or_fcst="fcst" + first_time=$((time_offset_hrs + LBC_SPEC_INTVL_HRS)) + last_time=$((time_offset_hrs + FCST_LEN_HRS)) + fcst_hrs="${first_time} ${last_time} ${LBC_SPEC_INTVL_HRS}" + file_names=${EXTRN_MDL_FILES_LBCS[@]} + if [ ${extrn_mdl_name} = FV3GFS ] ; then + file_type=$FV3GFS_FILE_FMT_LBCS fi -# -#----------------------------------------------------------------------- -# -# Print message indicating successful completion of script. -# -#----------------------------------------------------------------------- -# - if [ "${ics_or_lbcs}" = "ICS" ]; then - - print_info_msg " -======================================================================== -External model files needed for generating initial condition and surface -fields for the FV3-LAM successfully fetched from HPSS!!! - -Exiting script: \"${scrfunc_fn}\" -In directory: \"${scrfunc_dir}\" -========================================================================" - - elif [ "${ics_or_lbcs}" = "LBCS" ]; then + input_file_path=${EXTRN_MDL_SOURCE_BASEDIR_LBCS:-$EXTRN_MDL_SYSBASEDIR_LBCS} +fi - print_info_msg " -======================================================================== -External model files needed for generating lateral boundary conditions -on the halo of the FV3-LAM's regional grid successfully fetched from -HPSS!!! +data_stores="hpss aws" -Exiting script: \"${scrfunc_fn}\" -In directory: \"${scrfunc_dir}\" -========================================================================" +yyyymmddhh=${extrn_mdl_cdate:0:10} +yyyy=${yyyymmddhh:0:4} +yyyymm=${yyyymmddhh:0:6} +yyyymmdd=${yyyymmddhh:0:8} +mm=${yyyymmddhh:4:2} +dd=${yyyymmddhh:6:2} +hh=${yyyymmddhh:8:2} - fi - -elif [ "${data_src}" = "online" ]; then - print_info_msg " -======================================================================== -getting data from online nomads data sources -========================================================================" +input_file_path=$(eval echo ${input_file_path}) # #----------------------------------------------------------------------- # -# Set extrn_mdl_fps to the full paths within the archive files of the -# external model output files. +# Set up optional flags for calling retrieve_data.py # #----------------------------------------------------------------------- # - prefix=${extrn_mdl_arcvrel_dir:+${extrn_mdl_arcvrel_dir}/} - extrn_mdl_fps=( "${extrn_mdl_fns_on_disk[@]/#/$prefix}" ) +additional_flags="" - extrn_mdl_fps_str="( "$( printf "\"%s\" " "${extrn_mdl_fps[@]}" )")" - print_info_msg " -Getting external model files from nomads: - extrn_mdl_fps= ${extrn_mdl_fps_str}" - - num_files_to_extract="${#extrn_mdl_fps[@]}" - wget_LOG_FN="log.wget.txt" - for (( nfile=0; nfile<${num_files_to_extract}; nfile++ )); do - cp ../../../${extrn_mdl_fps[$nfile]} . || \ - print_err_msg_exit "\ - onlie file ${extrn_mdl_fps[$nfile]} not found." - done +if [ -n "${file_type:-}" ] ; then + additional_flags="$additional_flags \ + --file_type ${file_type}" +fi +if [ -n "${file_names:-}" ] ; then + additional_flags="$additional_flags \ + --file_templates ${file_names[@]}" +fi +if [ -n "${input_file_path:-}" ] ; then + data_stores="disk hpss aws" + additional_flags="$additional_flags \ + --input_file_path ${input_file_path}" fi + # #----------------------------------------------------------------------- # -# Create a variable definitions file (a shell script) and save in it the -# values of several external-model-associated variables generated in this -# script that will be needed by downstream workflow tasks. +# Call ush script to retrieve files # #----------------------------------------------------------------------- # -if [ "${ics_or_lbcs}" = "ICS" ]; then - extrn_mdl_var_defns_fn="${EXTRN_MDL_ICS_VAR_DEFNS_FN}" -elif [ "${ics_or_lbcs}" = "LBCS" ]; then - extrn_mdl_var_defns_fn="${EXTRN_MDL_LBCS_VAR_DEFNS_FN}" -fi -extrn_mdl_var_defns_fp="${extrn_mdl_staging_dir}/${extrn_mdl_var_defns_fn}" -check_for_preexist_dir_file "${extrn_mdl_var_defns_fp}" "delete" +cmd=" +python3 -u ${USHDIR}/retrieve_data.py \ + --debug \ + --anl_or_fcst ${anl_or_fcst} \ + --config ${USHDIR}/templates/data_locations.yml \ + --cycle_date ${extrn_mdl_cdate} \ + --data_stores ${data_stores} \ + --external_model ${extrn_mdl_name} \ + --fcst_hrs ${fcst_hrs[@]} \ + --output_path ${extrn_mdl_staging_dir} \ + --summary_file ${EXTRN_MDL_VAR_DEFNS_FN} \ + $additional_flags" -if [ "${data_src}" = "disk" ]; then - extrn_mdl_fns_str="( "$( printf "\"%s\" " "${extrn_mdl_fns_on_disk[@]}" )")" -elif [ "${data_src}" = "HPSS" ]; then - extrn_mdl_fns_str="( "$( printf "\"%s\" " "${extrn_mdl_fns_in_arcv[@]}" )")" -elif [ "${data_src}" = "online" ]; then - extrn_mdl_fns_str="( "$( printf "\"%s\" " "${extrn_mdl_fns_on_disk[@]}" )")" -fi - -settings="\ -DATA_SRC=\"${data_src}\" -EXTRN_MDL_CDATE=\"${extrn_mdl_cdate}\" -EXTRN_MDL_STAGING_DIR=\"${extrn_mdl_staging_dir}\" -EXTRN_MDL_FNS=${extrn_mdl_fns_str}" -# -# If the external model files obtained above were for generating LBCS (as -# opposed to ICs), then add to the external model variable definitions -# file the array variable EXTRN_MDL_LBC_SPEC_FHRS containing the forecast -# hours at which the lateral boundary conditions are specified. -# -if [ "${ics_or_lbcs}" = "LBCS" ]; then - extrn_mdl_lbc_spec_fhrs_str="( "$( printf "\"%s\" " "${extrn_mdl_lbc_spec_fhrs[@]}" )")" - settings="$settings -EXTRN_MDL_LBC_SPEC_FHRS=${extrn_mdl_lbc_spec_fhrs_str}" -fi +$cmd || print_err_msg_exit "\ +Call to retrieve_data.py failed with a non-zero exit status. -{ cat << EOM >> ${extrn_mdl_var_defns_fp} -$settings -EOM -} || print_err_msg_exit "\ -Heredoc (cat) command to create a variable definitions file associated -with the external model from which to generate ${ics_or_lbcs} returned with a -nonzero status. The full path to this variable definitions file is: - extrn_mdl_var_defns_fp = \"${extrn_mdl_var_defns_fp}\"" +The command was: +${cmd} +" # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index 8a4b83445..fcfefd646 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -111,7 +111,7 @@ fi #----------------------------------------------------------------------- # extrn_mdl_staging_dir="${CYCLE_DIR}/${EXTRN_MDL_NAME_ICS}/for_ICS" -extrn_mdl_var_defns_fp="${extrn_mdl_staging_dir}/${EXTRN_MDL_ICS_VAR_DEFNS_FN}" +extrn_mdl_var_defns_fp="${extrn_mdl_staging_dir}/${EXTRN_MDL_VAR_DEFNS_FN}" . ${extrn_mdl_var_defns_fp} # #----------------------------------------------------------------------- diff --git a/scripts/exregional_make_lbcs.sh b/scripts/exregional_make_lbcs.sh index e1d40fb88..70ba51176 100755 --- a/scripts/exregional_make_lbcs.sh +++ b/scripts/exregional_make_lbcs.sh @@ -109,7 +109,7 @@ fi #----------------------------------------------------------------------- # extrn_mdl_staging_dir="${CYCLE_DIR}/${EXTRN_MDL_NAME_LBCS}/for_LBCS" -extrn_mdl_var_defns_fp="${extrn_mdl_staging_dir}/${EXTRN_MDL_LBCS_VAR_DEFNS_FN}" +extrn_mdl_var_defns_fp="${extrn_mdl_staging_dir}/${EXTRN_MDL_VAR_DEFNS_FN}" . ${extrn_mdl_var_defns_fp} # #----------------------------------------------------------------------- @@ -358,12 +358,12 @@ fi # #----------------------------------------------------------------------- # -num_fhrs="${#EXTRN_MDL_LBC_SPEC_FHRS[@]}" +num_fhrs="${#EXTRN_MDL_FHRS[@]}" for (( i=0; i<${num_fhrs}; i++ )); do # # Get the forecast hour of the external model. # - fhr="${EXTRN_MDL_LBC_SPEC_FHRS[$i]}" + fhr="${EXTRN_MDL_FHRS[$i]}" # # Set external model output file name and file type/format. Note that # these are now inputs into chgres_cube. diff --git a/tests/WE2E/run_WE2E_tests.sh b/tests/WE2E/run_WE2E_tests.sh index c162a5100..f3040208e 100755 --- a/tests/WE2E/run_WE2E_tests.sh +++ b/tests/WE2E/run_WE2E_tests.sh @@ -941,26 +941,21 @@ machine (MACHINE): MACHINE= \"${MACHINE}\"" fi EXTRN_MDL_SOURCE_BASEDIR_ICS="${extrn_mdl_source_basedir}/${EXTRN_MDL_NAME_ICS}" - if [ "${EXTRN_MDL_NAME_ICS}" = "FV3GFS" ]; then - EXTRN_MDL_SOURCE_BASEDIR_ICS="${EXTRN_MDL_SOURCE_BASEDIR_ICS}/${FV3GFS_FILE_FMT_ICS}" - fi - if [ "${EXTRN_MDL_NAME_ICS}" = "FV3GFS" ] || \ - [ "${EXTRN_MDL_NAME_ICS}" = "GSMGFS" ]; then - if [ "${FV3GFS_FILE_FMT_ICS}" = "nemsio" ]; then - EXTRN_MDL_FILES_ICS=( "gfs.atmanl.nemsio" "gfs.sfcanl.nemsio" ) - elif [ "${FV3GFS_FILE_FMT_ICS}" = "grib2" ]; then - EXTRN_MDL_FILES_ICS=( "gfs.pgrb2.0p25.f000" ) - fi - elif [ "${EXTRN_MDL_NAME_ICS}" = "HRRR" ] || \ - [ "${EXTRN_MDL_NAME_ICS}" = "RAP" ]; then - EXTRN_MDL_FILES_ICS=( "${EXTRN_MDL_NAME_ICS,,}.out.for_f000" ) - elif [ "${EXTRN_MDL_NAME_ICS}" = "NAM" ]; then - EXTRN_MDL_FILES_ICS=( "${EXTRN_MDL_NAME_ICS,,}.out.for_f000" ) + if [ "${EXTRN_MDL_NAME_ICS}" = "FV3GFS" ] ; then + EXTRN_MDL_SOURCE_BASEDIR_ICS="${EXTRN_MDL_SOURCE_BASEDIR_ICS}/${FV3GFS_FILE_FMT_ICS}/\${DATE_FIRST_CYCL}\${CYCL_HRS[0]}" + elif [ "${EXTRN_MDL_NAME_ICS}" = "NAM" ] ; then + EXTRN_MDL_SOURCE_BASEDIR_ICS="${EXTRN_MDL_SOURCE_BASEDIR_ICS}/\${DATE_FIRST_CYCL}\${CYCL_HRS[0]}/for_ICS" + else + EXTRN_MDL_SOURCE_BASEDIR_ICS="${EXTRN_MDL_SOURCE_BASEDIR_ICS}/\${DATE_FIRST_CYCL}\${CYCL_HRS[0]}" fi EXTRN_MDL_SOURCE_BASEDIR_LBCS="${extrn_mdl_source_basedir}/${EXTRN_MDL_NAME_LBCS}" - if [ "${EXTRN_MDL_NAME_LBCS}" = "FV3GFS" ]; then - EXTRN_MDL_SOURCE_BASEDIR_LBCS="${EXTRN_MDL_SOURCE_BASEDIR_LBCS}/${FV3GFS_FILE_FMT_LBCS}" + if [ "${EXTRN_MDL_NAME_LBCS}" = "FV3GFS" ] ; then + EXTRN_MDL_SOURCE_BASEDIR_LBCS="${EXTRN_MDL_SOURCE_BASEDIR_LBCS}/${FV3GFS_FILE_FMT_LBCS}/\${DATE_FIRST_CYCL}\${CYCL_HRS[0]}" + elif [ "${EXTRN_MDL_NAME_LBCS}" = "GSMGFS" ] ; then + EXTRN_MDL_SOURCE_BASEDIR_LBCS="${EXTRN_MDL_SOURCE_BASEDIR_LBCS}/\${DATE_FIRST_CYCL}\${CYCL_HRS[0]}" + else + EXTRN_MDL_SOURCE_BASEDIR_LBCS="${EXTRN_MDL_SOURCE_BASEDIR_LBCS}/\${DATE_FIRST_CYCL}\${CYCL_HRS[0]}/for_LBCS" fi # # Make sure that the forecast length is evenly divisible by the interval @@ -976,32 +971,15 @@ boundary conditions specification interval (LBC_SPEC_INTVL_HRS): LBC_SPEC_INTVL_HRS = ${LBC_SPEC_INTVL_HRS} rem = FCST_LEN_HRS%%LBC_SPEC_INTVL_HRS = $rem" fi - lbc_spec_times_hrs=( $( seq "${LBC_SPEC_INTVL_HRS}" "${LBC_SPEC_INTVL_HRS}" "${FCST_LEN_HRS}" ) ) - EXTRN_MDL_FILES_LBCS=( $( printf "%03d " "${lbc_spec_times_hrs[@]}" ) ) - if [ "${EXTRN_MDL_NAME_LBCS}" = "FV3GFS" ] || \ - [ "${EXTRN_MDL_NAME_LBCS}" = "GSMGFS" ]; then - if [ "${FV3GFS_FILE_FMT_LBCS}" = "nemsio" ]; then - EXTRN_MDL_FILES_LBCS=( "${EXTRN_MDL_FILES_LBCS[@]/#/gfs.atmf}" ) - EXTRN_MDL_FILES_LBCS=( "${EXTRN_MDL_FILES_LBCS[@]/%/.nemsio}" ) - elif [ "${FV3GFS_FILE_FMT_LBCS}" = "grib2" ]; then - EXTRN_MDL_FILES_LBCS=( "${EXTRN_MDL_FILES_LBCS[@]/#/gfs.pgrb2.0p25.f}" ) - fi - elif [ "${EXTRN_MDL_NAME_LBCS}" = "HRRR" ] || \ - [ "${EXTRN_MDL_NAME_LBCS}" = "RAP" ]; then - EXTRN_MDL_FILES_LBCS=( "${EXTRN_MDL_FILES_LBCS[@]/#/${EXTRN_MDL_NAME_LBCS,,}.out.for_f}" ) - elif [ "${EXTRN_MDL_NAME_LBCS}" = "NAM" ]; then - EXTRN_MDL_FILES_LBCS=( "${EXTRN_MDL_FILES_LBCS[@]/#/${EXTRN_MDL_NAME_LBCS,,}.out.for_f}" ) - fi - - expt_config_str=${expt_config_str}" + expt_config_str="${expt_config_str} # # Locations and names of user-staged external model files for generating # ICs and LBCs. # -EXTRN_MDL_SOURCE_BASEDIR_ICS=\"${EXTRN_MDL_SOURCE_BASEDIR_ICS}\" -EXTRN_MDL_FILES_ICS=( $( printf "\"%s\" " "${EXTRN_MDL_FILES_ICS[@]}" )) -EXTRN_MDL_SOURCE_BASEDIR_LBCS=\"${EXTRN_MDL_SOURCE_BASEDIR_LBCS}\" -EXTRN_MDL_FILES_LBCS=( $( printf "\"%s\" " "${EXTRN_MDL_FILES_LBCS[@]}" ))" +EXTRN_MDL_SOURCE_BASEDIR_ICS=${EXTRN_MDL_SOURCE_BASEDIR_ICS} +EXTRN_MDL_FILES_ICS=( ${EXTRN_MDL_FILES_ICS[@]} ) +EXTRN_MDL_SOURCE_BASEDIR_LBCS=${EXTRN_MDL_SOURCE_BASEDIR_LBCS} +EXTRN_MDL_FILES_LBCS=( ${EXTRN_MDL_FILES_LBCS[@]} )" fi # diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index be96316d7..145d397ff 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -387,19 +387,12 @@ DOT_OR_USCORE="_" # to each workflow task) in order to make all the experiment variables # available in those scripts. # -# EXTRN_MDL_ICS_VAR_DEFNS_FN: -# Name of file (a shell script) containing the defintions of variables -# associated with the external model from which ICs are generated. This -# file is created by the GET_EXTRN_ICS_TN task because the values of the -# variables it contains are not known before this task runs. The file is -# then sourced by the MAKE_ICS_TN task. -# -# EXTRN_MDL_LBCS_VAR_DEFNS_FN: -# Name of file (a shell script) containing the defintions of variables -# associated with the external model from which LBCs are generated. This -# file is created by the GET_EXTRN_LBCS_TN task because the values of the -# variables it contains are not known before this task runs. The file is -# then sourced by the MAKE_ICS_TN task. +# EXTRN_MDL_VAR_DEFNS_FN: +# Name of file (a shell script) containing the defintions of variables +# associated with the external model from which ICs or LBCs are generated. This +# file is created by the GET_EXTRN_*_TN task because the values of the variables +# it contains are not known before this task runs. The file is then sourced by +# the MAKE_ICS_TN and MAKE_LBCS_TN tasks. # # WFLOW_LAUNCH_SCRIPT_FN: # Name of the script that can be used to (re)launch the experiment's rocoto @@ -429,8 +422,7 @@ NEMS_CONFIG_TMPL_FN="" FCST_MODEL="ufs-weather-model" WFLOW_XML_FN="FV3LAM_wflow.xml" GLOBAL_VAR_DEFNS_FN="var_defns.sh" -EXTRN_MDL_ICS_VAR_DEFNS_FN="extrn_mdl_ics_var_defns.sh" -EXTRN_MDL_LBCS_VAR_DEFNS_FN="extrn_mdl_lbcs_var_defns.sh" +EXTRN_MDL_VAR_DEFNS_FN="extrn_mdl_var_defns.sh" WFLOW_LAUNCH_SCRIPT_FN="launch_FV3LAM_wflow.sh" WFLOW_LAUNCH_LOG_FN="log.launch_FV3LAM_wflow" # @@ -704,9 +696,18 @@ EXTRN_MDL_SYSBASEDIR_LBCS='' # set to "FALSE". # # EXTRN_MDL_FILES_ICS: -# Array containing the names of the files to search for in the directory -# specified by EXTRN_MDL_SOURCE_BASEDIR_ICS. This variable is not used -# if USE_USER_STAGED_EXTRN_FILES is set to "FALSE". +# Array containing templates of the names of the files to search for in +# the directory specified by EXTRN_MDL_SOURCE_BASEDIR_ICS. This +# variable is not used if USE_USER_STAGED_EXTRN_FILES is set to "FALSE". +# A single template should be used for each model file type that is +# meant to be used. You may use any of the Python-style templates +# allowed in the ush/retrieve_data.py script. To see the full list of +# supported templates, run that script with a -h option. Here is an example of +# setting FV3GFS nemsio input files: +# EXTRN_MDL_FILES_ICS=( gfs.t{hh}z.atmf{fcst_hr:03d}.nemsio \ +# gfs.t{hh}z.sfcf{fcst_hr:03d}.nemsio ) +# Or for FV3GFS grib files: +# EXTRN_MDL_FILES_ICS=( gfs.t{hh}z.pgrb2.0p25.f{fcst_hr:03d} ) # # EXTRN_MDL_SOURCE_BASEDIR_LBCS: # Analogous to EXTRN_MDL_SOURCE_BASEDIR_ICS but for LBCs instead of ICs. @@ -717,10 +718,10 @@ EXTRN_MDL_SYSBASEDIR_LBCS='' #----------------------------------------------------------------------- # USE_USER_STAGED_EXTRN_FILES="FALSE" -EXTRN_MDL_SOURCE_BASEDIR_ICS="/base/dir/containing/user/staged/extrn/mdl/files/for/ICs" -EXTRN_MDL_FILES_ICS=( "ICS_file1" "ICS_file2" "..." ) -EXTRN_MDL_SOURCE_BASEDIR_LBCS="/base/dir/containing/user/staged/extrn/mdl/files/for/LBCs" -EXTRN_MDL_FILES_LBCS=( "LBCS_file1" "LBCS_file2" "..." ) +EXTRN_MDL_SOURCE_BASEDIR_ICS="" +EXTRN_MDL_FILES_ICS="" +EXTRN_MDL_SOURCE_BASEDIR_LBCS="" +EXTRN_MDL_FILES_LBCS="" # #----------------------------------------------------------------------- # diff --git a/ush/get_extrn_mdl_file_dir_info.sh b/ush/get_extrn_mdl_file_dir_info.sh deleted file mode 100755 index aab64c2dc..000000000 --- a/ush/get_extrn_mdl_file_dir_info.sh +++ /dev/null @@ -1,684 +0,0 @@ -# -#----------------------------------------------------------------------- -# -# Source the variable definitions file and the bash utility functions. -# -#----------------------------------------------------------------------- -# -. ${GLOBAL_VAR_DEFNS_FP} -. $USHDIR/source_util_funcs.sh -# -#----------------------------------------------------------------------- -# -# This file defines a function that is used to obtain information (e.g. -# output file names, system and mass store file and/or directory names) -# for a specified external model, analysis or forecast, and cycle date. -# See the usage statement below for this function should be called and -# the definitions of the input parameters. -# -#----------------------------------------------------------------------- -# - -usage () { - -echo " -Incorrect number of arguments specified: - - Function name: \"${func_name}\" - Number of arguments specified: $# - -Usage: - - ${func_name} \ - extrn_mdl_name \ - anl_or_fcst \ - cdate_FV3LAM \ - time_offset_hrs \ - varname_extrn_mdl_cdate \ - varname_extrn_mdl_lbc_spec_fhrs \ - varname_extrn_mdl_fns \ - varname_extrn_mdl_sysdir \ - varname_extrn_mdl_arcv_fmt \ - varname_extrn_mdl_arcv_fns \ - varname_extrn_mdl_arcv_fps \ - varname_extrn_mdl_arcvrel_dir - -where the arguments are defined as follows: - - extrn_mdl_name: - Name of the external model, i.e. the name of the model providing the - fields from which files containing initial conditions, surface fields, - and/or lateral boundary conditions for the FV3-LAM will be generated. - - anl_or_fcst: - Flag that specifies whether the external model files we are interested - in obtaining are analysis or forecast files. - - cdate_FV3LAM: - The cycle date and time (hours only) for which we want to obtain file - and directory information. This has the form YYYYMMDDHH, where YYYY - is the four-digit starting year of the cycle, MM is the two-digit - month, DD is the two-digit day of the month, and HH is the two-digit - hour of day. - - time_offset_hrs: - The number of hours by which to shift back in time the start time of - the external model forecast from the specified cycle start time of the - FV3-LAM (cdate_FV3LAM). When getting directory and file information on - external model analysis files, this is normally set to 0. When get- - ting directory and file information on external model forecast files, - this may be set to a nonzero value to obtain information for an exter- - nal model run that started time_offset_hrs hours before cdate_FV3LAM - (instead of exactly at cdate_FV3LAM). Note that in this case, the - forecast hours (relative to the external model run's start time) at - which the lateral boundary conditions will be updated must be shifted - forward by time_offset_hrs hours relative to those for the FV3-LAM in - order to make up for the backward-in-time shift in the starting time - of the external model. - - varname_extrn_mdl_cdate: - Name of the global variable that will contain the starting date and - hour of the external model run. - - varname_extrn_mdl_lbc_spec_fhrs: - Name of the global variable that will contain the forecast hours (re- - lative to the starting time of the external model run, which is earli- - er than that of the FV3-LAM by time_offset_hrs hours) at which lateral - boundary condition (LBC) output files are obtained from the external - model (and will be used to update the LBCs of the FV3-LAM). - - varname_extrn_mdl_fns_on_disk: - Name of the global variable that will contain the expected names of - the external model output files on disk. - - varname_extrn_mdl_fns_in_arcv: - Name of the global variable that will contain the expected names of - the external model output files on NOAA HPSS. - - varname_extrn_mdl_sysdir: - Name of the global variable that will contain the system directory in - which the externaml model output files may be stored. - - varname_extrn_mdl_arcv_fmt: - Name of the global variable that will contain the format of the ar- - chive file on HPSS in which the externaml model output files may be - stored. - - varname_extrn_mdl_arcv_fns: - Name of the global variable that will contain the name of the archive - file on HPSS in which the externaml model output files may be stored. - - varname_extrn_mdl_arcv_fps: - Name of the global variable that will contain the full path to the ar- - chive file on HPSS in which the externaml model output files may be - stored. - - varname_extrn_mdl_arcvrel_dir: - Name of the global variable that will contain the archive-relative di- - rectory, i.e. the directory \"inside\" the archive file in which the ex- - ternal model output files may be stored. -" -} - -function quit_unless_user_spec_data() { - if [ "${USE_USER_STAGED_EXTRN_FILES}" != "TRUE" ]; then - print_err_msg_exit "\ -The system directory in which to look for external model output files -has not been specified for this external model and machine combination: - extrn_mdl_name = \"${extrn_mdl_name}\" - MACHINE = \"$MACHINE\"" - fi -} - -function get_extrn_mdl_file_dir_info() { - - { save_shell_opts; set -u +x; } > /dev/null 2>&1 - - local func_name="${FUNCNAME[0]}" - # - #----------------------------------------------------------------------- - # - # Specify the set of valid argument names for this script/function. Then - # process the arguments provided to this script/function (which should - # consist of a set of name-value pairs of the form arg1="value1", etc). - # - #----------------------------------------------------------------------- - # - local valid_args=( \ - "extrn_mdl_name" \ - "anl_or_fcst" \ - "cdate_FV3LAM" \ - "time_offset_hrs" \ - "varname_extrn_mdl_cdate" \ - "varname_extrn_mdl_lbc_spec_fhrs" \ - "varname_extrn_mdl_fns_on_disk" \ - "varname_extrn_mdl_fns_in_arcv" \ - "varname_extrn_mdl_sysdir" \ - "varname_extrn_mdl_arcv_fmt" \ - "varname_extrn_mdl_arcv_fns" \ - "varname_extrn_mdl_arcv_fps" \ - "varname_extrn_mdl_arcvrel_dir" \ - ) - process_args valid_args "$@" - - if [ "$#" -ne "13" ]; then - print_err_msg_exit $(usage) - fi - - # - #----------------------------------------------------------------------- - # - # For debugging purposes, print out values of arguments passed to this - # script/function. Note that these will be printed out only if VERBOSE - # is set to TRUE. - # - #----------------------------------------------------------------------- - # - print_input_args valid_args - - # - #----------------------------------------------------------------------- - # - # Declare additional local variables. - # - #----------------------------------------------------------------------- - # - local yyyy yy mm dd hh mn yyyymmdd ddd \ - lbc_spec_fhrs i num_fhrs \ - fcst_hhh fcst_hh fcst_mn \ - prefix suffix fns fns_on_disk fns_in_arcv \ - sysbasedir sysdir \ - arcv_dir arcv_fmt arcv_fns arcv_fps arcvrel_dir - - anl_or_fcst=$(echo_uppercase $anl_or_fcst) - valid_vals_anl_or_fcst=( "ANL" "FCST" ) - check_var_valid_value "anl_or_fcst" "valid_vals_anl_or_fcst" - # - #----------------------------------------------------------------------- - # - # Set cdate to the start time for the external model being used. - # - #----------------------------------------------------------------------- - # - hh=${cdate_FV3LAM:8:2} - yyyymmdd=${cdate_FV3LAM:0:8} - - # Adjust time for offset - cdate=$( $DATE_UTIL --utc --date "${yyyymmdd} ${hh} UTC - ${time_offset_hrs} hours" "+%Y%m%d%H" ) - - yyyy=${cdate:0:4} - yy=${yyyy:2:4} - mm=${cdate:4:2} - dd=${cdate:6:2} - hh=${cdate:8:2} - mn="00" - yyyymmdd=${cdate:0:8} - # ddd is the Julian day -- not 3 digit day of month - ddd=$( $DATE_UTIL --utc --date "${yyyy}-${mm}-${dd} ${hh}:${mn} UTC" "+%j" ) - # - #----------------------------------------------------------------------- - # - # Initialize lbc_spec_fhrs array. Skip the initial time, since it is - # handled separately. - # - #----------------------------------------------------------------------- - # - lbc_spec_fhrs=( "" ) - - if [ "${anl_or_fcst}" = "FCST" ]; then - - lbc_spec_fhrs=( "${LBC_SPEC_FCST_HRS[@]}" ) - - num_fhrs=${#lbc_spec_fhrs[@]} - for (( i=0; i<=$((num_fhrs-1)); i++ )); do - # Add in offset to account for shift in initial time - lbc_spec_fhrs[$i]=$(( ${lbc_spec_fhrs[$i]} + time_offset_hrs )) - done - - fi - # - #----------------------------------------------------------------------- - # - # The model may be started with a variety of file types from FV3GFS. - # Set that file type now - # - #----------------------------------------------------------------------- - # - - if [ "${anl_or_fcst}" = "ANL" ]; then - fv3gfs_file_fmt="${FV3GFS_FILE_FMT_ICS}" - elif [ "${anl_or_fcst}" = "FCST" ]; then - fv3gfs_file_fmt="${FV3GFS_FILE_FMT_LBCS}" - fi - - # - #----------------------------------------------------------------------- - # - # Generate an array of file names expected from the external model - # Assume that filenames in archive and on disk are the same, unless - # otherwise specified (primarily on Jet). - # - #----------------------------------------------------------------------- - # - declare -a fns_on_disk - declare -a fns_in_arcv - case "${anl_or_fcst}" in - - "ANL") - - fcst_hh="00" - fcst_mn="00" - - case "${extrn_mdl_name}" in - - "GSMGFS") - fns_in_arcv=("gfs.t${hh}z.atmanl.nemsio" "gfs.t${hh}z.sfcanl.nemsio") - ;; - - "FV3GFS") - case "${fv3gfs_file_fmt}" in - "nemsio") - fns_in_arcv=("gfs.t${hh}z.atmanl.nemsio" "gfs.t${hh}z.sfcanl.nemsio") - - # File names are prefixed with a date time on Jet - if [ "${MACHINE}" = "JET" ]; then - prefix="${yy}${ddd}${hh}00" - fns_on_disk=( ${fns_in_arcv[@]/#/$prefix}) - fi - ;; - "grib2") - fns_in_arcv=( "gfs.t${hh}z.pgrb2.0p25.f000" ) - ;; - "netcdf") - fns_in_arcv=("gfs.t${hh}z.atmanl.nc" "gfs.t${hh}z.sfcanl.nc") - # File names are prefixed with a date time on Jet - if [ "${MACHINE}" = "JET" ]; then - prefix="${yy}${ddd}${hh}00" - fns_on_disk=( ${fns_in_arcv[@]/#/$prefix}) - fi - ;; - esac - ;; - - "RAP") - ;& # Fall through. RAP and HRRR follow same naming rules - - "HRRR") - fns_in_arcv=( "${yy}${ddd}${hh}${mn}${fcst_hh}${fcst_mn}" ) - if [ "${MACHINE}" = "JET" ]; then - fns_on_disk=( "${yy}${ddd}${hh}${mn}${fcst_mn}${fcst_hh}${fcst_mn}" ) - fi - ;; - - "NAM") - fns=( "" ) - fns_in_arcv=( "nam.t${hh}z.awphys${fcst_hh}.tm00.grib2" ) - ;; - - *) - if [ "${USE_USER_STAGED_EXTRN_FILES}" != "TRUE" ]; then - print_err_msg_exit "\ -The external model file names (either on disk or in archive files) have -not yet been specified for this combination of external model (extrn_mdl_name) -and analysis or forecast (anl_or_fcst): - extrn_mdl_name = \"${extrn_mdl_name}\" - anl_or_fcst = \"${anl_or_fcst}\"" - fi - ;; - - esac # End external model case for ANL files - ;; - - "FCST") - fcst_mn="00" - fcst_hhh=( $( printf "%03d " "${lbc_spec_fhrs[@]}" ) ) - fcst_hh=( $( printf "%02d " "${lbc_spec_fhrs[@]}" ) ) - - case "${extrn_mdl_name}" in - - "GSMGFS") - fn_tmpl="gfs.t${hh}z.atmfFHR3.nemsio" - ;; - - "FV3GFS") - - if [ "${fv3gfs_file_fmt}" = "nemsio" ]; then - fn_tmpl="gfs.t${hh}z.atmfFHR3.nemsio" - if [ "${MACHINE}" = "JET" ]; then - disk_tmpl="${yy}${ddd}${hh}00.gfs.t${hh}z.atmfFHR3.nemsio" - for fhr in ${fcst_hhh[@]} ; do - fns_on_disk+=(${disk_tmpl/FHR3/$fhr}) - done - fi - elif [ "${fv3gfs_file_fmt}" = "grib2" ]; then - fn_tmpl="gfs.t${hh}z.pgrb2.0p25.fFHR3" - elif [ "${fv3gfs_file_fmt}" = "netcdf" ]; then - fn_tmpl="gfs.t${hh}z.atmfFHR3.nc" - if [ "${MACHINE}" = "JET" ]; then - disk_tmpl="${yy}${ddd}${hh}00.gfs.t${hh}z.atmfFHR3.nc" - for fhr in ${fcst_hhh[@]} ; do - fns_on_disk+=(${disk_tmpl/FHR3/$fhr}) - done - fi - fi - ;; - - "RAP") - ;& # Fall through since RAP and HRRR are named the same - - "HRRR") - fn_tmpl="${yy}${ddd}${hh}00FHR200" - if [ "${MACHINE}" = "JET" ]; then - disk_tmpl="${yy}${ddd}${hh}0000FHR2" - for fhr in ${fcst_hhh[@]} ; do - fns_on_disk+=(${disk_tmpl/FHR3/$fhr}) - done - fi - ;; - - "NAM") - fn_tmpl="nam.t${hh}z.awphysFHR2.tm00.grib2" - ;; - - *) - if [ "${USE_USER_STAGED_EXTRN_FILES}" != "TRUE" ]; then - print_err_msg_exit "\ -The external model file names have not yet been specified for this com- -bination of external model (extrn_mdl_name) and analysis or forecast -(anl_or_fcst): - extrn_mdl_name = \"${extrn_mdl_name}\" - anl_or_fcst = \"${anl_or_fcst}\"" - fi - ;; - - esac # End external model case for FCST files - ;; - esac # End ANL FCST case - - # - # Expand the archive file names for all forecast hours - # - if [ ${anl_or_fcst} = FCST ] ; then - if [[ $fn_tmpl =~ FHR3 ]] ; then - fhrs=( $( printf "%03d " "${lbc_spec_fhrs[@]}" ) ) - tmpl=FHR3 - elif [[ ${fn_tmpl} =~ FHR2 ]] ; then - fhrs=( $( printf "%02d " "${lbc_spec_fhrs[@]}" ) ) - tmpl=FHR2 - else - print_err_msg_exit "\ - Forecast file name templates are expected to contain a template - string, either FHR2 or FHR3" - fi - for fhr in ${fhrs[@]}; do - fns_in_arcv+=(${fn_tmpl/$tmpl/$fhr}) - done - fi - - # Make sure all filenames variables are set. - if [ -z $fns_in_arcv ] ; then - print_err_msg_exit "\ - The script has not set \$fns_in_arcv properly" - fi - - if [ -z ${fns_on_disk:-} ] ; then - fns_on_disk=(${fns_in_arcv[@]}) - fi - # - #----------------------------------------------------------------------- - # - # Set the system directory (i.e. a directory on disk) in which the external - # model output files for the specified cycle date (cdate) may be located. - # Note that this will be used by the calling script only if the output - # files for the specified cdate actually exist at this location. Otherwise, - # the files will be searched for on the mass store (HPSS). - # - #----------------------------------------------------------------------- - # - if [ "${anl_or_fcst}" = "ANL" ]; then - sysdir=$(eval echo ${EXTRN_MDL_SYSBASEDIR_ICS}) - elif [ "${anl_or_fcst}" = "FCST" ]; then - sysdir=$(eval echo ${EXTRN_MDL_SYSBASEDIR_LBCS}) - fi - - # - #----------------------------------------------------------------------- - # - # Set parameters associated with the mass store (HPSS) for the specified - # cycle date (cdate). These consist of: - # - # 1) The type of the archive file (e.g. tar, zip, etc). - # 2) The name of the archive file. - # 3) The full path in HPSS to the archive file. - # 4) The relative directory in the archive file in which the model output - # files are located. - # - # Note that these will be used by the calling script only if the archive - # file for the specified cdate actually exists on HPSS. - # - #----------------------------------------------------------------------- - # - case "${extrn_mdl_name}" in - - "GSMGFS") - arcv_dir="/NCEPPROD/hpssprod/runhistory/rh${yyyy}/${yyyy}${mm}/${yyyymmdd}" - arcv_fmt="tar" - arcv_fns="gpfs_hps_nco_ops_com_gfs_prod_gfs.${cdate}." - if [ "${anl_or_fcst}" = "ANL" ]; then - arcv_fns="${arcv_fns}anl" - arcvrel_dir="." - elif [ "${anl_or_fcst}" = "FCST" ]; then - arcv_fns="${arcv_fns}sigma" - arcvrel_dir="/gpfs/hps/nco/ops/com/gfs/prod/gfs.${yyyymmdd}" - fi - arcv_fns="${arcv_fns}.${arcv_fmt}" - arcv_fps="${arcv_dir}/${arcv_fns}" - ;; - - "FV3GFS") - - if [ "${cdate_FV3LAM}" -lt "2019061200" ]; then - arcv_dir="/NCEPDEV/emc-global/5year/emc.glopara/WCOSS_C/Q2FY19/prfv3rt3/${cdate_FV3LAM}" - arcv_fns="" - elif [ "${cdate_FV3LAM}" -ge "2019061200" ] && \ - [ "${cdate_FV3LAM}" -lt "2020022600" ]; then - arcv_dir="/NCEPPROD/hpssprod/runhistory/rh${yyyy}/${yyyy}${mm}/${yyyymmdd}" - arcv_fns="gpfs_dell1_nco_ops_com_gfs_prod_gfs.${yyyymmdd}_${hh}." - elif [ "${cdate_FV3LAM}" -ge "2020022600" ]; then - arcv_dir="/NCEPPROD/hpssprod/runhistory/rh${yyyy}/${yyyy}${mm}/${yyyymmdd}" - arcv_fns="com_gfs_prod_gfs.${yyyymmdd}_${hh}." - fi - - if [ "${fv3gfs_file_fmt}" = "nemsio" ]; then - - if [ "${anl_or_fcst}" = "ANL" ]; then - arcv_fns="${arcv_fns}gfs_nemsioa" - elif [ "${anl_or_fcst}" = "FCST" ]; then - last_fhr_in_nemsioa="39" - first_lbc_fhr="${lbc_spec_fhrs[0]}" - last_lbc_fhr="${lbc_spec_fhrs[-1]}" - if [ "${last_lbc_fhr}" -le "${last_fhr_in_nemsioa}" ]; then - arcv_fns="${arcv_fns}gfs_nemsioa" - elif [ "${first_lbc_fhr}" -gt "${last_fhr_in_nemsioa}" ]; then - arcv_fns="${arcv_fns}gfs_nemsiob" - else - arcv_fns=( "${arcv_fns}gfs_nemsioa" "${arcv_fns}gfs_nemsiob" ) - fi - fi - - elif [ "${fv3gfs_file_fmt}" = "grib2" ]; then - - arcv_fns="${arcv_fns}gfs_pgrb2" - - elif [ "${fv3gfs_file_fmt}" = "netcdf" ]; then - - if [ "${anl_or_fcst}" = "ANL" ]; then - arcv_fns="${arcv_fns}gfs_nca" - elif [ "${anl_or_fcst}" = "FCST" ]; then - last_fhr_in_netcdfa="39" - first_lbc_fhr="${lbc_spec_fhrs[0]}" - last_lbc_fhr="${lbc_spec_fhrs[-1]}" - if [ "${last_lbc_fhr}" -le "${last_fhr_in_netcdfa}" ]; then - arcv_fns="${arcv_fns}gfs_nca" - elif [ "${first_lbc_fhr}" -gt "${last_fhr_in_netcdfa}" ]; then - arcv_fns="${arcv_fns}gfs_ncb" - else - arcv_fns=( "${arcv_fns}gfs_nca" "${arcv_fns}gfs_ncb" ) - fi - fi - - fi - - arcv_fmt="tar" - - slash_atmos_or_null="" - if [ "${cdate_FV3LAM}" -ge "2021032100" ]; then - slash_atmos_or_null="/atmos" - fi - arcvrel_dir="./gfs.${yyyymmdd}/${hh}${slash_atmos_or_null}" - - is_array arcv_fns - if [ "$?" = "0" ]; then - suffix=".${arcv_fmt}" - arcv_fns=( "${arcv_fns[@]/%/$suffix}" ) - prefix="${arcv_dir}/" - arcv_fps=( "${arcv_fns[@]/#/$prefix}" ) - else - arcv_fns="${arcv_fns}.${arcv_fmt}" - arcv_fps="${arcv_dir}/${arcv_fns}" - fi - ;; - - - "RAP") - # - # Note that this is GSL RAPX data, not operational NCEP RAP data. - # An option for the latter may be added in the future. - # - # The zip archive files for RAPX are named such that the forecast - # files for odd-numbered starting hours (e.g. 01, 03, ..., 23) are - # stored together with the forecast files for the corresponding - # preceding even numbered starting hours (e.g. 00, 02, ..., 22, - # respectively), in an archive file whose name contains only the - # even-numbered hour. Thus, in forming the name of the archive - # file, if the starting hour (hh) is odd, we reduce it by one to get - # the corresponding even-numbered hour and use that to form the - # archive file name. - # - # Convert hh to a decimal (i.e. base-10) number to ovoid octal - # interpretation in bash. - - hh_orig=$hh - hh=$((10#$hh)) - if [ $(($hh%2)) = 1 ]; then - hh=$((hh-1)) - fi - # Archive files use 2-digit forecast hour - hh=$( printf "%02d\n" $hh ) - - arcv_dir="/BMC/fdr/Permanent/${yyyy}/${mm}/${dd}/data/fsl/rap/full/wrfnat" - arcv_fmt="zip" - arcv_fns="${yyyy}${mm}${dd}${hh}00.${arcv_fmt}" - arcv_fps="${arcv_dir}/${arcv_fns}" - arcvrel_dir="" - - # Reset hh to its original value - hh=${hh_orig} - ;; - - "HRRR") - # - # Note that this is GSL HRRRX data, not operational NCEP HRRR data. - # An option for the latter may be added in the future. - # - arcv_dir="/BMC/fdr/Permanent/${yyyy}/${mm}/${dd}/data/fsl/hrrr/conus/wrfnat" - arcv_fmt="zip" - arcv_fns="${yyyy}${mm}${dd}${hh}00.${arcv_fmt}" - arcv_fps="${arcv_dir}/${arcv_fns}" - arcvrel_dir="" - ;; - - "NAM") - # - # 12-km CONUS 218 grid. - # - arcv_dir="/NCEPPROD/hpssprod/runhistory/rh${yyyy}/${yyyy}${mm}/${yyyymmdd}" - arcv_fmt="tar" - arcv_fns="com_nam_prod_nam.${yyyy}${mm}${dd}${hh}.awphys.${arcv_fmt}" - arcv_fps="${arcv_dir}/${arcv_fns}" - arcvrel_dir="." - ;; - - *) - print_err_msg_exit "\ -Archive file information has not been specified for this external model: - extrn_mdl_name = \"${extrn_mdl_name}\"" - ;; - - esac - # - # Depending on the experiment configuration, the above code may set - # arcv_fns and arcv_fps to either scalars or arrays. If they are not - # arrays, recast them as arrays because that is what is expected in - # the code below. - # - is_array arcv_fns || arcv_fns=( "${arcv_fns}" ) - is_array arcv_fps || arcv_fps=( "${arcv_fps}" ) - # - #----------------------------------------------------------------------- - # - # Use the eval function to set the output variables. Note that each - # of these is set only if the corresponding input variable specifying - # the name to use for the output variable is not empty. - # - #----------------------------------------------------------------------- - # - if [ ! -z "${varname_extrn_mdl_cdate}" ]; then - eval ${varname_extrn_mdl_cdate}="${cdate}" - fi - - if [ ! -z "${varname_extrn_mdl_lbc_spec_fhrs}" ]; then - lbc_spec_fhrs_str="( "$( printf "\"%s\" " "${lbc_spec_fhrs[@]}" )")" - eval ${varname_extrn_mdl_lbc_spec_fhrs}=${lbc_spec_fhrs_str} - fi - - if [ ! -z "${varname_extrn_mdl_fns_on_disk}" ]; then - fns_on_disk_str="( "$( printf "\"%s\" " "${fns_on_disk[@]}" )")" - eval ${varname_extrn_mdl_fns_on_disk}=${fns_on_disk_str} - fi - - if [ ! -z "${varname_extrn_mdl_fns_in_arcv}" ]; then - fns_in_arcv_str="( "$( printf "\"%s\" " "${fns_in_arcv[@]}" )")" - eval ${varname_extrn_mdl_fns_in_arcv}=${fns_in_arcv_str} - fi - - if [ ! -z "${varname_extrn_mdl_sysdir}" ]; then - eval ${varname_extrn_mdl_sysdir}="${sysdir}" - fi - - if [ ! -z "${varname_extrn_mdl_arcv_fmt}" ]; then - eval ${varname_extrn_mdl_arcv_fmt}="${arcv_fmt}" - fi - - if [ ! -z "${varname_extrn_mdl_arcv_fns}" ]; then - arcv_fns_str="( "$( printf "\"%s\" " "${arcv_fns[@]}" )")" - eval ${varname_extrn_mdl_arcv_fns}=${arcv_fns_str} - fi - - if [ ! -z "${varname_extrn_mdl_arcv_fps}" ]; then - arcv_fps_str="( "$( printf "\"%s\" " "${arcv_fps[@]}" )")" - eval ${varname_extrn_mdl_arcv_fps}=${arcv_fps_str} - fi - - if [ ! -z "${varname_extrn_mdl_arcvrel_dir}" ]; then - eval ${varname_extrn_mdl_arcvrel_dir}="${arcvrel_dir}" - fi - # - #----------------------------------------------------------------------- - # - # Restore the shell options saved at the beginning of this script/function. - # - #----------------------------------------------------------------------- - # - { restore_shell_opts; } > /dev/null 2>&1 -} diff --git a/ush/retrieve_data.py b/ush/retrieve_data.py old mode 100644 new mode 100755 index 48210c0d5..86cb87e07 --- a/ush/retrieve_data.py +++ b/ush/retrieve_data.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 # pylint: disable=logging-fstring-interpolation ''' This script helps users pull data from known data streams, including @@ -31,6 +32,8 @@ import shutil import subprocess import sys +from textwrap import dedent + import yaml @@ -43,7 +46,7 @@ def clean_up_output_dir(expected_subdir, local_archive, output_path, source_path unavailable = {} # Check to make sure the files exist on disk for file_path in source_paths: - local_file_path = os.path.join(output_path,file_path) + local_file_path = os.path.join(output_path, file_path.lstrip("/")) if not os.path.exists(local_file_path): logging.info(f'File does not exist: {local_file_path}') unavailable['hpss'] = source_paths @@ -126,46 +129,6 @@ def download_file(url): return True -def download_requested_files(cla, data_store, store_specs): - - ''' This function interacts with the "download" protocol in a - provided data store specs file to download a set of files requested - by the user. It calls download_file for each individual file that - should be downloaded. ''' - - base_urls = store_specs['url'] - base_urls = base_urls if isinstance(base_urls, list) else [base_urls] - - file_names = store_specs.get('file_names', {}) - if cla.file_type is not None: - file_names = file_names[cla.file_type] - file_names = file_names[cla.anl_or_fcst] - target_path = fill_template(cla.output_path, - cla.cycle_date) - - logging.info(f'Downloaded files will be placed here: \n {target_path}') - orig_path = os.getcwd() - os.chdir(target_path) - unavailable = {} - for base_url in base_urls: - for fcst_hr in cla.fcst_hrs: - for file_name in file_names: - url = os.path.join(base_url, file_name) - url = fill_template(url, cla.cycle_date, fcst_hr) - downloaded = download_file(url) - if not downloaded: - - if unavailable.get(data_store) is None: - unavailable[data_store] = [] - unavailable[data_store].append(target_path) - os.chdir(orig_path) - # Returning here assumes that if the first file - # isn't found, none of the others will be. Don't - # waste time timing out on every requested file. - return unavailable - os.chdir(orig_path) - return unavailable - def fhr_list(args): ''' @@ -184,7 +147,7 @@ def fhr_list(args): Must ensure that the list contains integers. ''' - args = args if isinstance(args, list) else [args] + args = args if isinstance(args, list) else list(args) arg_len = len(args) if arg_len in (2, 3): args[1] += 1 @@ -271,7 +234,7 @@ def find_archive_files(paths, file_names, cycle_date): return '', 0 -def get_requested_files(cla, file_names, input_loc, method='disk'): +def get_requested_files(cla, file_templates, input_loc, method='disk'): ''' This function copies files from disk locations or downloads files from a url, depending on the option specified for @@ -282,12 +245,12 @@ def get_requested_files(cla, file_names, input_loc, method='disk'): Arguments: - cla Namespace object containing command line arguments - file_names Dict of file names by file type and kind - input_loc A string containing a single data location, either a url - or disk path. - method Choice of disk or download to indicate protocol for - retrieval + cla Namespace object containing command line arguments + file_templates a list of file templates + input_loc A string containing a single data location, either a url + or disk path. + method Choice of disk or download to indicate protocol for + retrieval Returns unavailable a dict whose keys are "method" and whose values are a @@ -296,12 +259,10 @@ def get_requested_files(cla, file_names, input_loc, method='disk'): unavailable = {} - if cla.file_type is not None: - file_names = file_names[cla.file_type] - file_names = file_names[cla.anl_or_fcst] + logging.info(f'Getting files named like {file_templates}') - file_names = file_names if isinstance(file_names, list) else \ - [file_names] + file_templates = file_templates if isinstance(file_templates, list) else \ + [file_templates] target_path = fill_template(cla.output_path, cla.cycle_date) @@ -310,8 +271,9 @@ def get_requested_files(cla, file_names, input_loc, method='disk'): os.chdir(target_path) unavailable = {} for fcst_hr in cla.fcst_hrs: - for file_name in file_names: - loc = os.path.join(input_loc, file_name) + for file_template in file_templates: + loc = os.path.join(input_loc, file_template) + logging.debug(f'Full file path: {loc}') loc = fill_template(loc, cla.cycle_date, fcst_hr) if method == 'disk': @@ -358,7 +320,7 @@ def hsi_single_file(file_path, mode='ls'): return file_path -def hpss_requested_files(cla, store_specs): +def hpss_requested_files(cla, file_names, store_specs): ''' This function interacts with the "hpss" protocol in a provided data store specs file to download a set of files requested @@ -392,64 +354,70 @@ def hpss_requested_files(cla, store_specs): f' {list(zip(archive_paths, archive_file_names))}') existing_archive, which_archive = find_archive_files(archive_paths, - archive_file_names, - cla.cycle_date, - ) + archive_file_names, + cla.cycle_date, + ) if not existing_archive: logging.warning('No archive files were found!') unavailable['archive'] = list(zip(archive_paths, archive_file_names)) return unavailable - # Use the found archive file path to get the necessary files - file_names = store_specs.get('file_names', {}) - if cla.file_type is not None: - file_names = file_names[cla.file_type] - file_names = file_names[cla.anl_or_fcst] + logging.info(f'Files in archive are named: {file_names}') - logging.debug(f'Grabbing archive number {which_archive} in list.') - archive_internal_dir = store_specs.get('archive_internal_dir', [''])[which_archive] - archive_internal_dir = fill_template(archive_internal_dir, - cla.cycle_date) + archive_internal_dirs = store_specs.get('archive_internal_dir', ['']) + if isinstance(archive_internal_dirs, dict): + archive_internal_dirs = archive_internal_dirs.get(cla.anl_or_fcst, ['']) - output_path = fill_template(cla.output_path, cla.cycle_date) - logging.info(f'Will place files in {os.path.abspath(output_path)}') - orig_path = os.getcwd() - os.chdir(output_path) - logging.debug(f'CWD: {os.getcwd()}') + # which_archive matters for choosing the correct file names within, + # but we can safely just try all options for the + # archive_internal_dir + logging.debug(f'Checking archive number {which_archive} in list.') - source_paths = [] - for fcst_hr in cla.fcst_hrs: - for file_name in file_names: - source_paths.append(fill_template( - os.path.join(archive_internal_dir, file_name), - cla.cycle_date, - fcst_hr, - )) + for archive_internal_dir_tmpl in archive_internal_dirs: + archive_internal_dir = fill_template(archive_internal_dir_tmpl, + cla.cycle_date) - if store_specs.get('archive_format', 'tar') == 'zip': - # Get the entire file from HPSS - existing_archive = hsi_single_file(existing_archive, mode='get') + output_path = fill_template(cla.output_path, cla.cycle_date) + logging.info(f'Will place files in {os.path.abspath(output_path)}') + orig_path = os.getcwd() + os.chdir(output_path) + logging.debug(f'CWD: {os.getcwd()}') - # Grab only the necessary files from the archive - cmd = f'unzip -o {os.path.basename(existing_archive)} {" ".join(source_paths)}' + source_paths = [] + for fcst_hr in cla.fcst_hrs: + for file_name in file_names: + source_paths.append(fill_template( + os.path.join(archive_internal_dir, file_name), + cla.cycle_date, + fcst_hr, + )) - else: - cmd = f'htar -xvf {existing_archive} {" ".join(source_paths)}' + if store_specs.get('archive_format', 'tar') == 'zip': + # Get the entire file from HPSS + existing_archive = hsi_single_file(existing_archive, mode='get') - logging.info(f'Running command \n {cmd}') - subprocess.run(cmd, - check=True, - shell=True, - ) - - # Check that files exist and Remove any data transfer artifacts. - unavailable = clean_up_output_dir( - expected_subdir=archive_internal_dir, - local_archive=os.path.basename(existing_archive), - output_path=output_path, - source_paths=source_paths, - ) + # Grab only the necessary files from the archive + cmd = f'unzip -o {os.path.basename(existing_archive)} {" ".join(source_paths)}' + + else: + cmd = f'htar -xvf {existing_archive} {" ".join(source_paths)}' + + logging.info(f'Running command \n {cmd}') + subprocess.run(cmd, + check=True, + shell=True, + ) + + # Check that files exist and Remove any data transfer artifacts. + unavailable = clean_up_output_dir( + expected_subdir=archive_internal_dir, + local_archive=os.path.basename(existing_archive), + output_path=output_path, + source_paths=source_paths, + ) + if not unavailable: + return unavailable os.chdir(orig_path) @@ -505,6 +473,30 @@ def setup_logging(debug=False): +def write_summary_file(cla, data_store, file_templates): + + ''' Given the command line arguments and the data store from which the data + was retrieved, write a bash summary file that is needed by the workflow + elements downstream. ''' + + files = [] + for tmpl in file_templates: + files.extend([fill_template(tmpl, cla.cycle_date, fh) for fh in cla.fcst_hrs]) + + summary_fp = os.path.join(cla.output_path, cla.summary_file) + logging.info(f'Writing a summary file to {summary_fp}') + file_contents = dedent(f''' + DATA_SRC={data_store} + EXTRN_MDL_CDATE={cla.cycle_date.strftime('%Y%m%d%H')} + EXTRN_MDL_STAGING_DIR={cla.output_path} + EXTRN_MDL_FNS=( {' '.join(files)} ) + EXTRN_MDL_FHRS=( {' '.join([str(i) for i in cla.fcst_hrs])} ) + ''') + logging.info(f'Contents: {file_contents}') + with open(summary_fp, "w") as summary: + summary.write(file_contents) + + def to_datetime(arg): ''' Return a datetime object give a string like YYYYMMDDHH. ''' @@ -521,29 +513,35 @@ def main(cla): paths in priority order. ''' - setup_logging(cla.debug) - - known_data_info = cla.config.get(cla.external_model) - if known_data_info is None: - msg = ('No data stores have been defined for', - f'{cla.external_model}!') - raise KeyError(msg) + data_stores = cla.data_stores + known_data_info = cla.config.get(cla.external_model, {}) + if not known_data_info: + msg = dedent(f'''No data stores have been defined for + {cla.external_model}!''') + if cla.input_file_path is None: + data_stores = ['disk'] + raise KeyError(msg) + logging.info(msg + ' Only checking provided disk location.') unavailable = {} - for data_store in cla.data_stores: + for data_store in data_stores: logging.info(f'Checking {data_store} for {cla.external_model}') store_specs = known_data_info.get(data_store, {}) if data_store == 'disk': - file_names = cla.file_names if cla.file_names else \ + file_templates = cla.file_templates if cla.file_templates else \ known_data_info.get('hpss', {}).get('file_names') - logging.debug(f'User supplied file names are: {file_names}') - if not file_names: - msg = ('No file name found. They must be provided \ + if isinstance(file_templates, dict): + if cla.file_type is not None: + file_templates = file_templates[cla.file_type] + file_templates = file_templates[cla.anl_or_fcst] + logging.debug(f'User supplied file names are: {file_templates}') + if not file_templates: + msg = ('No file naming convention found. They must be provided \ either on the command line or on in a config file.') raise argparse.ArgumentTypeError(msg) unavailable = get_requested_files(cla, - file_names=file_names, + file_templates=file_templates, input_loc=cla.input_file_path, method='disk', ) @@ -552,24 +550,32 @@ def main(cla): msg = (f'No information is available for {data_store}.') raise KeyError(msg) - if store_specs.get('protocol') == 'download': - file_names = store_specs.get('file_names') - if not file_names: - msg = ('No file name found. They must be provided \ + else: + + file_templates = store_specs.get('file_names') + if isinstance(file_templates, dict): + if cla.file_type is not None: + file_templates = file_templates[cla.file_type] + file_templates = file_templates[cla.anl_or_fcst] + if not file_templates: + msg = ('No file name naming convention found. They must be provided \ either on the command line or on in a config file.') raise argparse.ArgumentTypeError(msg) - unavailable = get_requested_files(cla, - file_names=file_names, - input_loc=store_specs['url'], - method='download', - ) - - if store_specs.get('protocol') == 'htar': - unavailable = hpss_requested_files(cla, store_specs) + if store_specs.get('protocol') == 'download': + unavailable = get_requested_files(cla, + file_templates=file_templates, + input_loc=store_specs['url'], + method='download', + ) + if store_specs.get('protocol') == 'htar': + unavailable = hpss_requested_files(cla, file_templates, store_specs) if not unavailable: # All files are found. Stop looking! + # Write a variable definitions file for the data, if requested + if cla.summary_file: + write_summary_file(cla, data_store, file_templates) break logging.warning(f'Requested files are unavialable from {data_store}') @@ -657,11 +663,11 @@ def parse_args(): help='Print debug messages', ) parser.add_argument( - '--file_names', - help='A YAML-formatted string that indicates the naming \ + '--file_templates', + help='One or more file template strings defining the naming \ convention the be used for the files retrieved from disk. If \ not provided, the default names from hpss are used.', - type=load_str, + nargs='*', ) parser.add_argument( '--file_type', @@ -672,9 +678,13 @@ def parse_args(): '--input_file_path', help='A path to data stored on disk. The path may contain \ Python templates. File names may be supplied using the \ - --file_names flag, or the default naming convention will be \ + --file_templates flag, or the default naming convention will be \ taken from the --config file.', - nargs='*', + ) + parser.add_argument( + '--summary_file', + help='Name of the summary file to be written to the output \ + directory', ) return parser.parse_args() @@ -684,6 +694,15 @@ def parse_args(): CLA.output_path = path_exists(CLA.output_path) CLA.fcst_hrs = fhr_list(CLA.fcst_hrs) + + setup_logging(CLA.debug) + print(f"Running script retrieve_data.py with args:\n", + f"{('-' * 80)}\n{('-' * 80)}") + for name, val in CLA.__dict__.items(): + if name not in ['config']: + print(f"{name:>15s}: {val}") + print(f"{('-' * 80)}\n{('-' * 80)}") + if 'disk' in CLA.data_stores: # Make sure a path was provided. if not CLA.input_file_path: diff --git a/ush/templates/FV3LAM_wflow.xml b/ush/templates/FV3LAM_wflow.xml index 48e53d81a..6cdaff347 100644 --- a/ush/templates/FV3LAM_wflow.xml +++ b/ush/templates/FV3LAM_wflow.xml @@ -278,7 +278,6 @@ MODULES_RUN_TASK_FP script. PDY@Y@m@d CDATE@Y@m@d@H CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H - EXTRN_MDL_NAME{{ extrn_mdl_name_ics }} ICS_OR_LBCSICS @@ -305,7 +304,6 @@ MODULES_RUN_TASK_FP script. PDY@Y@m@d CDATE@Y@m@d@H CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H - EXTRN_MDL_NAME{{ extrn_mdl_name_lbcs }} ICS_OR_LBCSLBCS diff --git a/ush/templates/data_locations.yml b/ush/templates/data_locations.yml index c3efd9a29..d82eeec67 100644 --- a/ush/templates/data_locations.yml +++ b/ush/templates/data_locations.yml @@ -65,6 +65,13 @@ FV3GFS: fcst: - gfs.t{hh}z.atmf{fcst_hr:03d}.nemsio - gfs.t{hh}z.sfcf{fcst_hr:03d}.nemsio + netcdf: + anl: + - gfs.t{hh}z.atmanl.nc + - gfs.t{hh}z.sfcanl.nc + fcst: + - gfs.t{hh}z.atmf{fcst_hr:03d}.nc + - gfs.t{hh}z.sfcf{fcst_hr:03d}.nc hpss: protocol: htar archive_path: @@ -103,16 +110,41 @@ FV3GFS: file_names: <<: *gfs_file_names +GSMGFS: + hpss: + protocol: htar + archive_path: + - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyymm}/{yyyymmdd} + archive_internal_dir: + anl: + - ./ + fcst: + - /gpfs/hps/nco/ops/com/gfs/prod/gfs.{yyyymmdd} + archive_file_names: + anl: + - gpfs_hps_nco_ops_com_gfs_prod_gfs.{yyyymmddhh}.anl.tar + fcst: + - gpfs_hps_nco_ops_com_gfs_prod_gfs.{yyyymmddhh}.sigma.tar + file_names: + anl: + - gfs.t{hh}z.atmanl.nemsio + - gfs.t{hh}z.sfcanl.nemsio + fcst: + - gfs.t{hh}z.atmf{fcst_hr:03d}.nemsio + RAP: hpss: protocol: htar archive_format: tar archive_path: - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyymm}/{yyyymmdd} + - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyymm}/{yyyymmdd} archive_internal_dir: - ./ + - ./ archive_file_names: # RAP forecasts are binned into 6 hour tar files. + - gpfs_hps_nco_ops_com_rap_prod_rap.{yyyymmdd}{bin6}.wrf.tar - com_rap_prod_rap.{yyyymmdd}{bin6}.wrf.tar file_names: &rap_file_names anl: @@ -125,58 +157,31 @@ RAP: file_names: <<: *rap_file_names -RAPx: - hpss: - protocol: htar - archive_format: zip - archive_path: - - /BMC/fdr/Permanent/{yyyy}/{mm}/{dd}/data/fsl/rap/full/wrfnat - archive_file_names: - # RAPx bins two cycles togehter, and named by the lower even value - # of the cycle hour. - - '{yyyymmdd}{hh_even}00.zip' - file_names: - anl: - - '{yy}{jjj}{hh}00{fcst_hr:02d}00' - fcst: - - '{yy}{jjj}{hh}00{fcst_hr:02d}00' - HRRR: hpss: protocol: htar archive_format: tar archive_path: - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyymm}/{yyyymmdd} + - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyymm}/{yyyymmdd} archive_internal_dir: - ./ + - ./ archive_file_names: # HRRR forecasts are binned into 6 hour tar files. - - com_hrrr_prod_hrrr.{yyyymmdd}_conus{bin6}.wrfnatdng.tar + - gpfs_hps_nco_ops_com_hrrr_prod_hrrr.{yyyymmdd}_conus{bin6}.wrf.tar + - com_hrrr_prod_hrrr.{yyyymmdd}_conus{bin6}.wrf.tar file_names: &hrrr_file_names anl: - - hrrr.t{hh}z.wrfnatf{fcst_hr:02d}.grib2 + - hrrr.t{hh}z.wrfprsf{fcst_hr:02d}.grib2 fcst: - - hrrr.t{hh}z.wrfnatf{fcst_hr:02d}.grib2 + - hrrr.t{hh}z.wrfprsf{fcst_hr:02d}.grib2 aws: protocol: download url: https://noaa-hrrr-bdp-pds.s3.amazonaws.com/hrrr.{yyyymmdd}/conus/ file_names: <<: *hrrr_file_names -HRRRx: - hpss: - protocol: htar - archive_format: zip - archive_path: - - /BMC/fdr/Permanent/{yyyy}/{mm}/{dd}/data/fsl/hrrr/conus/wrfnat - archive_file_names: - - '{yyyymmddhh}00.zip' - file_names: - anl: - - '{yy}{jjj}{hh}00{fcst_hr:02d}00' - fcst: - - '{yy}{jjj}{hh}00{fcst_hr:02d}00' - NAM: hpss: protocol: htar