diff --git a/jobs/JREGIONAL_GET_OBS_CCPA b/jobs/JREGIONAL_GET_OBS_CCPA deleted file mode 100755 index 6666c26432..0000000000 --- a/jobs/JREGIONAL_GET_OBS_CCPA +++ /dev/null @@ -1,91 +0,0 @@ -#!/bin/bash - -# -#----------------------------------------------------------------------- -# -# This script pulls CCPA observation data for comparison to the model for -# the requested accumulations. Supported accumulations: 01h, 03h, and 06h. -# NOTE: Accumulation is currently hardcoded to 01h. -# The verification uses MET/pcp-combine to sum 01h files into -# desired accumulations. -# -#----------------------------------------------------------------------- -# - -# -#----------------------------------------------------------------------- -# -# Source the variable definitions file and the bash utility functions. -# -#----------------------------------------------------------------------- -# -. $USHdir/source_util_funcs.sh -source_config_for_task " " ${GLOBAL_VAR_DEFNS_FP} -. $USHdir/job_preamble.sh -# -#----------------------------------------------------------------------- -# -# Save current shell options (in a global array). Then set new options -# for this script/function. -# -#----------------------------------------------------------------------- -# -{ save_shell_opts; . $USHdir/preamble.sh; } > /dev/null 2>&1 -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# -scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) -scrfunc_fn=$( basename "${scrfunc_fp}" ) -scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Print message indicating entry into script. -# -#----------------------------------------------------------------------- -# -print_info_msg " -======================================================================== -Entering script: \"${scrfunc_fn}\" -In directory: \"${scrfunc_dir}\" - -This is the J-job script for the task that pulls CCPA observation data -for verification purposes. -========================================================================" - -# -# -#----------------------------------------------------------------------- -# -# Call the ex-script for this J-job and pass to it the necessary varia- -# bles. -# -#----------------------------------------------------------------------- -# -$SCRIPTSdir/exregional_get_obs_ccpa.sh || \ -print_err_msg_exit "\ -Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." -# -#----------------------------------------------------------------------- -# -# Run job postamble. -# -#----------------------------------------------------------------------- -# -job_postamble -# -#----------------------------------------------------------------------- -# -# Restore the shell options saved at the beginning of this script/func- -# tion. -# -#----------------------------------------------------------------------- -# -{ restore_shell_opts; } > /dev/null 2>&1 - diff --git a/jobs/JREGIONAL_GET_OBS_MRMS b/jobs/JREGIONAL_GET_OBS_MRMS deleted file mode 100755 index 780da368e5..0000000000 --- a/jobs/JREGIONAL_GET_OBS_MRMS +++ /dev/null @@ -1,92 +0,0 @@ -#!/bin/bash - -# -#----------------------------------------------------------------------- -# -# This script pulls MRMS observation data for comparison to the model. -# -#----------------------------------------------------------------------- -# - -# -#----------------------------------------------------------------------- -# -# Source the variable definitions file and the bash utility functions. -# -#----------------------------------------------------------------------- -# -. $USHdir/source_util_funcs.sh -source_config_for_task " " ${GLOBAL_VAR_DEFNS_FP} -. $USHdir/job_preamble.sh -# -#----------------------------------------------------------------------- -# -# Save current shell options (in a global array). Then set new options -# for this script/function. -# -#----------------------------------------------------------------------- -# -{ save_shell_opts; . $USHdir/preamble.sh; } > /dev/null 2>&1 -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# -scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) -scrfunc_fn=$( basename "${scrfunc_fp}" ) -scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Print message indicating entry into script. -# -#----------------------------------------------------------------------- -# -print_info_msg " -======================================================================== -Entering script: \"${scrfunc_fn}\" -In directory: \"${scrfunc_dir}\" - -This is the J-job script for the task that pulls MRMS observation data -for verification purposes. -========================================================================" - -# -# -#----------------------------------------------------------------------- -# -# Call the ex-script for this J-job and pass to it the necessary varia- -# bles. -# -#----------------------------------------------------------------------- -# -echo "VAR=${VAR}" -for field in ${VAR[@]}; do - export field - echo "Field=${field}" - $SCRIPTSdir/exregional_get_obs_mrms.sh || \ - print_err_msg_exit "\ - Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." -done -# -#----------------------------------------------------------------------- -# -# Run job postamble. -# -#----------------------------------------------------------------------- -# -job_postamble -# -#----------------------------------------------------------------------- -# -# Restore the shell options saved at the beginning of this script/func- -# tion. -# -#----------------------------------------------------------------------- -# -{ restore_shell_opts; } > /dev/null 2>&1 - diff --git a/jobs/JREGIONAL_GET_OBS_NOHRSC b/jobs/JREGIONAL_GET_OBS_NOHRSC deleted file mode 100755 index cc892951fd..0000000000 --- a/jobs/JREGIONAL_GET_OBS_NOHRSC +++ /dev/null @@ -1,91 +0,0 @@ -#!/bin/bash - -# -#----------------------------------------------------------------------- -# -# This script pulls NOHRSC observation data for comparison to the model for -# the requested accumulations. Supported accumulations: 01h, 03h, and 06h. -# NOTE: Accumulation is currently hardcoded to 06h. -# The verification uses MET/pcp-combine to sum 06h files into -# desired accumulations. -# -#----------------------------------------------------------------------- -# - -# -#----------------------------------------------------------------------- -# -# Source the variable definitions file and the bash utility functions. -# -#----------------------------------------------------------------------- -# -. $USHdir/source_util_funcs.sh -source_config_for_task " " ${GLOBAL_VAR_DEFNS_FP} -. $USHdir/job_preamble.sh -# -#----------------------------------------------------------------------- -# -# Save current shell options (in a global array). Then set new options -# for this script/function. -# -#----------------------------------------------------------------------- -# -{ save_shell_opts; . $USHdir/preamble.sh; } > /dev/null 2>&1 -# -#----------------------------------------------------------------------- -# -# Get the full path to the file in which this script/function is located -# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in -# which the file is located (scrfunc_dir). -# -#----------------------------------------------------------------------- -# -scrfunc_fp=$( $READLINK -f "${BASH_SOURCE[0]}" ) -scrfunc_fn=$( basename "${scrfunc_fp}" ) -scrfunc_dir=$( dirname "${scrfunc_fp}" ) -# -#----------------------------------------------------------------------- -# -# Print message indicating entry into script. -# -#----------------------------------------------------------------------- -# -print_info_msg " -======================================================================== -Entering script: \"${scrfunc_fn}\" -In directory: \"${scrfunc_dir}\" - -This is the J-job script for the task that pulls NOHRSC observation data -for verification purposes. -========================================================================" - -# -# -#----------------------------------------------------------------------- -# -# Call the ex-script for this J-job and pass to it the necessary varia- -# bles. -# -#----------------------------------------------------------------------- -# -$SCRIPTSdir/exregional_get_obs_nohrsc.sh || \ -print_err_msg_exit "\ -Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." -# -#----------------------------------------------------------------------- -# -# Run job postamble. -# -#----------------------------------------------------------------------- -# -job_postamble -# -#----------------------------------------------------------------------- -# -# Restore the shell options saved at the beginning of this script/func- -# tion. -# -#----------------------------------------------------------------------- -# -{ restore_shell_opts; } > /dev/null 2>&1 - diff --git a/jobs/JREGIONAL_GET_OBS_NDAS b/jobs/JREGIONAL_GET_VERIF_OBS similarity index 90% rename from jobs/JREGIONAL_GET_OBS_NDAS rename to jobs/JREGIONAL_GET_VERIF_OBS index 0594f32f0d..3820a739db 100755 --- a/jobs/JREGIONAL_GET_OBS_NDAS +++ b/jobs/JREGIONAL_GET_VERIF_OBS @@ -1,9 +1,9 @@ -#!/bin/bash +#!/usr/bin/env bash # #----------------------------------------------------------------------- # -# This script pulls NDAS observation data for comparison to the model. +# This script checks, pulls, and stages observation data for model verification. # #----------------------------------------------------------------------- # @@ -51,8 +51,8 @@ print_info_msg " Entering script: \"${scrfunc_fn}\" In directory: \"${scrfunc_dir}\" -This is the J-job script for the task that pulls NDAS observation data -for verification purposes. +This is the J-job script for the task that checks, pulls, and stages +observation data for verification purposes. ========================================================================" # @@ -64,7 +64,7 @@ for verification purposes. # #----------------------------------------------------------------------- # -$SCRIPTSdir/exregional_get_obs_ndas.sh || \ +$SCRIPTSdir/exregional_get_verif_obs.sh || \ print_err_msg_exit "\ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." # diff --git a/parm/data_locations.yml b/parm/data_locations.yml index a81a9de67b..68f43ba2d0 100644 --- a/parm/data_locations.yml +++ b/parm/data_locations.yml @@ -318,7 +318,7 @@ GFS_obs: NSSL_mrms: - hpss: + hpss: &MRMS_hpss protocol: htar archive_format: tar archive_path: @@ -337,7 +337,6 @@ NSSL_mrms: obs: - "*MergedReflectivityQC_*_{yyyymmdd}-{hh}{min}*.grib2*" - RAP_obs: hpss: protocol: htar @@ -426,6 +425,76 @@ RAP_obs: - rap.t{hh}z.sevasr.tm00.bufr_d - rap.t{hh}z.ssmisu.tm00.bufr_d +CCPA_obs: + hpss: + protocol: htar + archive_format: tar + archive_path: + - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyy}{mm}/{yyyy}{mm}{dd} + - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyy}{mm}/{yyyy}{mm}{dd} + - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyy}{mm}/{yyyy}{mm}{dd} + - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyy}{mm}/{yyyy}{mm}{dd} + archive_internal_dir: + - "./00" + - "./06" + - "./12" + - "./18" + archive_file_names: + - "com2_ccpa_prod_ccpa.{yyyy}{mm}{dd}.tar" + - "gpfs_dell1_nco_ops_com_ccpa_prod_ccpa.{yyyy}{mm}{dd}.tar" + - "com_ccpa_prod_ccpa.{yyyy}{mm}{dd}.tar" + - "com_ccpa_v4.2_ccpa.{yyyy}{mm}{dd}.tar" + file_names: + obs: + - "ccpa.t{hh}z.01h.hrap.conus.gb2" + +MRMS_obs: + hpss: + <<: *MRMS_hpss + file_names: + obs: + - "MergedReflectivityQCComposite_00.50_{yyyymmdd}-*.grib2.gz" + - "EchoTop_18_00.50_{yyyymmdd}-*.grib2.gz" + archive_internal_dir: + - /dcom/us007003/ldmdata/obs/upperair/mrms/conus/MergedReflectivityQCComposite + - /dcom/us007003/ldmdata/obs/upperair/mrms/conus/EchoTop + - ./upperair/mrms/conus/MergedReflectivityQCComposite + - ./upperair/mrms/conus/EchoTop + +NDAS_obs: + hpss: + protocol: htar + archive_format: tar + archive_path: + - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyy}{mm}/{yyyy}{mm}{dd} + - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyy}{mm}/{yyyy}{mm}{dd} + - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyy}{mm}/{yyyy}{mm}{dd} + - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyy}{mm}/{yyyy}{mm}{dd} + archive_file_names: + - "com2_nam_prod_nam.{yyyy}{mm}{dd}{hh}.bufr.tar" + - "gpfs_dell1_nco_ops_com_nam_prod_nam.{yyyy}{mm}{dd}{hh}.bufr.tar" + - "com_nam_prod_nam.{yyyy}{mm}{dd}{hh}.bufr.tar" + - "com_obsproc_v1.1_nam.{yyyy}{mm}{dd}{hh}.bufr.tar" + file_names: + obs: + - "./nam.t{hh}z.prepbufr.tm*.nr" + +NOHRSC_obs: + hpss: + protocol: htar + archive_format: tar + archive_path: + - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyy}{mm}/{yyyy}{mm}{dd} + - /NCEPPROD/hpssprod/runhistory/rh{yyyy}/{yyyy}{mm}/{yyyy}{mm}{dd} + archive_file_names: + - "dcom_prod_{yyyy}{mm}{dd}.tar" + - "dcom_{yyyy}{mm}{dd}.tar" + file_names: + obs: + - "sfav2_CONUS_*h_{yyyy}{mm}{dd}{hh}_grid184.grb2" + archive_internal_dir: + - ./wgrbbul/nohrsc_snowfall/ + ########################### ########################### ####### Fix Files ######### diff --git a/parm/wflow/verify_pre.yaml b/parm/wflow/verify_pre.yaml index 22096b328f..e46474c361 100644 --- a/parm/wflow/verify_pre.yaml +++ b/parm/wflow/verify_pre.yaml @@ -24,11 +24,12 @@ default_task_verify_pre: &default_task_verify_pre task_get_obs_ccpa: <<: *default_task_verify_pre - command: '&LOAD_MODULES_RUN_TASK_FP; "get_obs" "&JOBSdir;/JREGIONAL_GET_OBS_CCPA"' + command: '&LOAD_MODULES_RUN_TASK_FP; "get_obs" "&JOBSdir;/JREGIONAL_GET_VERIF_OBS"' envars: <<: *default_vars ACCUM_HH: '01' OBS_DIR: '&CCPA_OBS_DIR;' + OBTYPE: 'CCPA' FHR: '{% for h in range(0, workflow.FCST_LEN_HRS+1) %}{{ " %02d" % h }}{% endfor %}' partition: '{% if platform.get("PARTITION_HPSS") %}&PARTITION_HPSS;{% else %}None{% endif %}' queue: "&QUEUE_HPSS;" @@ -36,10 +37,11 @@ task_get_obs_ccpa: task_get_obs_nohrsc: <<: *default_task_verify_pre - command: '&LOAD_MODULES_RUN_TASK_FP; "get_obs" "&JOBSdir;/JREGIONAL_GET_OBS_NOHRSC"' + command: '&LOAD_MODULES_RUN_TASK_FP; "get_obs" "&JOBSdir;/JREGIONAL_GET_VERIF_OBS"' envars: <<: *default_vars OBS_DIR: '&NOHRSC_OBS_DIR;' + OBTYPE: 'NOHRSC' FHR: '{% for h in range(0, workflow.FCST_LEN_HRS+1) %}{{ " %02d" % h }}{% endfor %}' partition: '{% if platform.get("PARTITION_HPSS") %}&PARTITION_HPSS;{% else %}None{% endif %}' queue: "&QUEUE_HPSS;" @@ -47,10 +49,11 @@ task_get_obs_nohrsc: task_get_obs_mrms: <<: *default_task_verify_pre - command: '&LOAD_MODULES_RUN_TASK_FP; "get_obs" "&JOBSdir;/JREGIONAL_GET_OBS_MRMS"' + command: '&LOAD_MODULES_RUN_TASK_FP; "get_obs" "&JOBSdir;/JREGIONAL_GET_VERIF_OBS"' envars: <<: *default_vars OBS_DIR: '&MRMS_OBS_DIR;' + OBTYPE: 'MRMS' VAR: 'REFC RETOP' FHR: '{% for h in range(0, workflow.FCST_LEN_HRS+1) %}{{ " %02d" % h }}{% endfor %}' partition: '{% if platform.get("PARTITION_HPSS") %}&PARTITION_HPSS;{% else %}None{% endif %}' @@ -62,8 +65,9 @@ task_get_obs_ndas: envars: <<: *default_vars OBS_DIR: '&NDAS_OBS_DIR;' + OBTYPE: 'NDAS' FHR: '{% for h in range(0, workflow.FCST_LEN_HRS+1) %}{{ " %02d" % h }}{% endfor %}' - command: '&LOAD_MODULES_RUN_TASK_FP; "get_obs" "&JOBSdir;/JREGIONAL_GET_OBS_NDAS"' + command: '&LOAD_MODULES_RUN_TASK_FP; "get_obs" "&JOBSdir;/JREGIONAL_GET_VERIF_OBS"' queue: "&QUEUE_HPSS;" partition: '{% if platform.get("PARTITION_HPSS") %}&PARTITION_HPSS;{% else %}None{% endif %}' walltime: 02:00:00 diff --git a/scripts/exregional_get_extrn_mdl_files.sh b/scripts/exregional_get_extrn_mdl_files.sh index 6152a62716..77cbc28be1 100755 --- a/scripts/exregional_get_extrn_mdl_files.sh +++ b/scripts/exregional_get_extrn_mdl_files.sh @@ -173,6 +173,9 @@ fi # #----------------------------------------------------------------------- # + +mkdir -p ${EXTRN_MDL_STAGING_DIR}${mem_dir} + if [ $RUN_ENVIR = "nco" ]; then EXTRN_DEFNS="${NET}.${cycle}.${EXTRN_MDL_NAME}.${ICS_OR_LBCS}.${EXTRN_MDL_VAR_DEFNS_FN}.sh" else diff --git a/scripts/exregional_get_obs_ccpa.sh b/scripts/exregional_get_obs_ccpa.sh deleted file mode 100755 index 3ab24a2e3a..0000000000 --- a/scripts/exregional_get_obs_ccpa.sh +++ /dev/null @@ -1,271 +0,0 @@ -#!/bin/bash - -# -#----------------------------------------------------------------------- -# -# Source the variable definitions file and the bash utility functions. -# -#----------------------------------------------------------------------- -# -. $USHdir/source_util_funcs.sh -source_config_for_task " " ${GLOBAL_VAR_DEFNS_FP} -# -#----------------------------------------------------------------------- -# -# Save current shell options (in a global array). Then set new options -# for this script/function. -# -#----------------------------------------------------------------------- -# -{ save_shell_opts; . $USHdir/preamble.sh; } > /dev/null 2>&1 -# -#----------------------------------------------------------------------- -# -# This script reorganizes the CCPA data into a more intuitive structure: -# A valid YYYYMMDD directory is created, and all files for the valid day are placed within the directory. -# Supported accumulations: 01h, 03h, and 06h. NOTE: Accumulation is currently hardcoded to 01h. -# The verification uses MET/pcp-combine to sum 01h files into desired accumulations. -# -#----------------------------------------------------------------------- -# - -# Top-level CCPA directory -ccpa_dir=${OBS_DIR}/.. -if [[ ! -d "$ccpa_dir" ]]; then - mkdir_vrfy -p $ccpa_dir -fi - -# CCPA data from HPSS -ccpa_raw=$ccpa_dir/raw -if [[ ! -d "$ccpa_raw" ]]; then - mkdir_vrfy -p $ccpa_raw -fi - -# Reorganized CCPA location -ccpa_proc=$ccpa_dir/proc -if [[ ! -d "$ccpa_proc" ]]; then - mkdir_vrfy -p $ccpa_proc -fi - -# Accumulation is for accumulation of CCPA data to pull (hardcoded to 01h, see note above.) -#accum=${ACCUM} -accum=01 - -# Initialization -yyyymmdd=${PDY} -hh=${cyc} - -init=${CDATE}${hh} - -fhr_last=`echo ${FHR} | awk '{ print $NF }'` - -# Forecast length -fcst_length=${fhr_last} - -current_fcst=$accum -while [[ ${current_fcst} -le ${fcst_length} ]]; do - # Calculate valid date info - fcst_sec=`expr ${current_fcst} \* 3600` # convert forecast lead hour to seconds - yyyy=`echo ${init} | cut -c1-4` # year (YYYY) of initialization time - mm=`echo ${init} | cut -c5-6` # month (MM) of initialization time - dd=`echo ${init} | cut -c7-8` # day (DD) of initialization time - hh=`echo ${init} | cut -c9-10` # hour (HH) of initialization time - init_ut=`$DATE_UTIL -ud ''${yyyy}-${mm}-${dd}' UTC '${hh}':00:00' +%s` # convert initialization time to universal time - vdate_ut=`expr ${init_ut} + ${fcst_sec}` # calculate current forecast time in universal time - vdate=`$DATE_UTIL -ud '1970-01-01 UTC '${vdate_ut}' seconds' +%Y%m%d%H` # convert universal time to standard time - vyyyymmdd=`echo ${vdate} | cut -c1-8` # forecast time (YYYYMMDD) - vyyyy=`echo ${vdate} | cut -c1-4` # year (YYYY) of valid time - vmm=`echo ${vdate} | cut -c5-6` # month (MM) of valid time - vdd=`echo ${vdate} | cut -c7-8` # day (DD) of valid time - vhh=`echo ${vdate} | cut -c9-10` # forecast hour (HH) - - vhh_noZero=$((10#${vhh})) - - # Calculate valid date - 1 day - vdate_ut_m1=`expr ${vdate_ut} - 86400` - vdate_m1=`$DATE_UTIL -ud '1970-01-01 UTC '${vdate_ut_m1}' seconds' +%Y%m%d%H` - vyyyymmdd_m1=`echo ${vdate_m1} | cut -c1-8` - vyyyy_m1=`echo ${vdate_m1} | cut -c1-4` - vmm_m1=`echo ${vdate_m1} | cut -c5-6` - vdd_m1=`echo ${vdate_m1} | cut -c7-8` - vhh_m1=`echo ${vdate_m1} | cut -c9-10` - - # Calculate valid date + 1 day - vdate_ut_p1=`expr ${vdate_ut} + 86400` - vdate_p1=`$DATE_UTIL -ud '1970-01-01 UTC '${vdate_ut_p1}' seconds' +%Y%m%d%H` - vyyyymmdd_p1=`echo ${vdate_p1} | cut -c1-8` - vyyyy_p1=`echo ${vdate_p1} | cut -c1-4` - vmm_p1=`echo ${vdate_p1} | cut -c5-6` - vdd_p1=`echo ${vdate_p1} | cut -c7-8` - vhh_p1=`echo ${vdate_p1} | cut -c9-10` - - # Create necessary raw and prop directories - if [[ ! -d "$ccpa_raw/${vyyyymmdd}" ]]; then - mkdir_vrfy -p $ccpa_raw/${vyyyymmdd} - fi - - if [[ ! -d "$ccpa_raw/${vyyyymmdd_m1}" ]]; then - mkdir_vrfy -p $ccpa_raw/${vyyyymmdd_m1} - fi - - if [[ ! -d "$ccpa_raw/${vyyyymmdd_p1}" ]]; then - mkdir_vrfy -p $ccpa_raw/${vyyyymmdd_p1} - fi - - if [[ ! -d "$ccpa_proc/${vyyyymmdd}" ]]; then - mkdir_vrfy -p $ccpa_proc/${vyyyymmdd} - fi - - # Name of CCPA tar file on HPSS is dependent on date. Logic accounts for files from 2019 until Sept. 2020. - if [[ ${vyyyymmdd} -ge 20190101 && ${vyyyymmdd} -lt 20190812 ]]; then - TarFile="/NCEPPROD/hpssprod/runhistory/rh${vyyyy}/${vyyyy}${vmm}/${vyyyy}${vmm}${vdd}/com2_ccpa_prod_ccpa.${vyyyy}${vmm}${vdd}.tar" - fi - - if [[ ${vyyyymmdd_m1} -ge 20190101 && ${vyyyymmdd_m1} -lt 20190812 ]]; then - TarFile_m1="/NCEPPROD/hpssprod/runhistory/rh${vyyyy_m1}/${vyyyy_m1}${vmm_m1}/${vyyyy_m1}${vmm_m1}${vdd_m1}/com2_ccpa_prod_ccpa.${vyyyy_m1}${vmm_m1}${vdd_m1}.tar" - fi - - if [[ ${vyyyymmdd_p1} -ge 20190101 && ${vyyyymmdd_p1} -lt 20190812 ]]; then - TarFile_p1="/NCEPPROD/hpssprod/runhistory/rh${vyyyy_p1}/${vyyyy_p1}${vmm_p1}/${vyyyy_p1}${vmm_p1}${vdd_p1}/com2_ccpa_prod_ccpa.${vyyyy_p1}${vmm_p1}${vdd_p1}.tar" - fi - - if [[ ${vyyyymmdd} -ge 20190812 && ${vyyyymmdd} -le 20200217 ]]; then - TarFile="/NCEPPROD/hpssprod/runhistory/rh${vyyyy}/${vyyyy}${vmm}/${vyyyy}${vmm}${vdd}/gpfs_dell1_nco_ops_com_ccpa_prod_ccpa.${vyyyy}${vmm}${vdd}.tar" - fi - - if [[ ${vyyyymmdd_m1} -ge 20190812 && ${vyyyymmdd_m1} -le 20200217 ]]; then - TarFile_m1="/NCEPPROD/hpssprod/runhistory/rh${vyyyy_m1}/${vyyyy_m1}${vmm_m1}/${vyyyy_m1}${vmm_m1}${vdd_m1}/gpfs_dell1_nco_ops_com_ccpa_prod_ccpa.${vyyyy_m1}${vmm_m1}${vdd_m1}.tar" - fi - - if [[ ${vyyyymmdd_p1} -ge 20190812 && ${vyyyymmdd_p1} -le 20200217 ]]; then - TarFile_p1="/NCEPPROD/hpssprod/runhistory/rh${vyyyy_p1}/${vyyyy_p1}${vmm_p1}/${vyyyy_p1}${vmm_p1}${vdd_p1}/gpfs_dell1_nco_ops_com_ccpa_prod_ccpa.${vyyyy_p1}${vmm_p1}${vdd_p1}.tar" - fi - - if [[ ${vyyyymmdd} -gt 20200217 && ${vyyyymmdd} -le 20220618 ]]; then - TarFile="/NCEPPROD/hpssprod/runhistory/rh${vyyyy}/${vyyyy}${vmm}/${vyyyy}${vmm}${vdd}/com_ccpa_prod_ccpa.${vyyyy}${vmm}${vdd}.tar" - fi - - if [[ ${vyyyymmdd_m1} -gt 20200217 && ${vyyyymmdd_p1} -le 20220618 ]]; then - TarFile_m1="/NCEPPROD/hpssprod/runhistory/rh${vyyyy_m1}/${vyyyy_m1}${vmm_m1}/${vyyyy_m1}${vmm_m1}${vdd_m1}/com_ccpa_prod_ccpa.${vyyyy_m1}${vmm_m1}${vdd_m1}.tar" - fi - - if [[ ${vyyyymmdd_p1} -gt 20200217 && ${vyyyymmdd_p1} -le 20220618 ]]; then - TarFile_p1="/NCEPPROD/hpssprod/runhistory/rh${vyyyy_p1}/${vyyyy_p1}${vmm_p1}/${vyyyy_p1}${vmm_p1}${vdd_p1}/com_ccpa_prod_ccpa.${vyyyy_p1}${vmm_p1}${vdd_p1}.tar" - fi - - if [[ ${vyyyymmdd} -ge 20220619 ]]; then - TarFile="/NCEPPROD/hpssprod/runhistory/rh${vyyyy}/${vyyyy}${vmm}/${vyyyy}${vmm}${vdd}/com_ccpa_v4.2_ccpa.${vyyyy}${vmm}${vdd}.tar" - fi - - if [[ ${vyyyymmdd_m1} -ge 20220619 ]]; then - TarFile_m1="/NCEPPROD/hpssprod/runhistory/rh${vyyyy_m1}/${vyyyy_m1}${vmm_m1}/${vyyyy_m1}${vmm_m1}${vdd_m1}/com_ccpa_v4.2_ccpa.${vyyyy_m1}${vmm_m1}${vdd_m1}.tar" - fi - - if [[ ${vyyyymmdd_p1} -ge 20220619 ]]; then - TarFile_p1="/NCEPPROD/hpssprod/runhistory/rh${vyyyy_p1}/${vyyyy_p1}${vmm_p1}/${vyyyy_p1}${vmm_p1}${vdd_p1}/com_ccpa_v4.2_ccpa.${vyyyy_p1}${vmm_p1}${vdd_p1}.tar" - fi - - # Check if file exists on disk; if not, pull it. - ccpa_file="$ccpa_proc/${vyyyymmdd}/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2" - echo "CCPA FILE:${ccpa_file}" - if [[ ! -f "${ccpa_file}" ]]; then - if [[ ${accum} == "01" ]]; then - # Check if valid hour is 00 - if [[ ${vhh_noZero} -ge 19 && ${vhh_noZero} -le 23 ]]; then - cd_vrfy $ccpa_raw/${vyyyymmdd_p1} - # Pull CCPA data from HPSS - TarCommand="htar -xvf ${TarFile_p1} \`htar -tf ${TarFile_p1} | egrep \"ccpa.t${vhh}z.${accum}h.hrap.conus.gb2\" | awk '{print \$7}'\`" - echo "CALLING: ${TarCommand}" - htar -xvf ${TarFile_p1} `htar -tf ${TarFile_p1} | egrep "ccpa.t${vhh}z.${accum}h.hrap.conus.gb2" | awk '{print \$7}'` - else - cd_vrfy $ccpa_raw/${vyyyymmdd} - # Pull CCPA data from HPSS - TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"ccpa.t${vhh}z.${accum}h.hrap.conus.gb2\" | awk '{print \$7}'\`" - echo "CALLING: ${TarCommand}" - htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "ccpa.t${vhh}z.${accum}h.hrap.conus.gb2" | awk '{print \$7}'` - fi - - # One hour CCPA files have incorrect metadeta in the files under the "00" directory from 20180718 to 20210504. - # After data is pulled, reorganize into correct valid yyyymmdd structure. - if [[ ${vhh_noZero} -ge 1 && ${vhh_noZero} -le 6 ]]; then - cp_vrfy $ccpa_raw/${vyyyymmdd}/06/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} - elif [[ ${vhh_noZero} -ge 7 && ${vhh_noZero} -le 12 ]]; then - cp_vrfy $ccpa_raw/${vyyyymmdd}/12/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} - elif [[ ${vhh_noZero} -ge 13 && ${vhh_noZero} -le 18 ]]; then - cp_vrfy $ccpa_raw/${vyyyymmdd}/18/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} - elif [[ ${vhh_noZero} -ge 19 && ${vhh_noZero} -le 23 ]]; then - if [[ ${vyyyymmdd} -ge 20180718 && ${vyyyymmdd} -le 20210504 ]]; then - wgrib2 $ccpa_raw/${vyyyymmdd_p1}/00/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 -set_date -24hr -grib $ccpa_proc/${vyyyymmdd}/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 -s - else - cp_vrfy $ccpa_raw/${vyyyymmdd_p1}/00/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} - fi - elif [[ ${vhh_noZero} -eq 0 ]]; then - if [[ ${vyyyymmdd} -ge 20180718 && ${vyyyymmdd} -le 20210504 ]]; then - wgrib2 $ccpa_raw/${vyyyymmdd}/00/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 -set_date -24hr -grib $ccpa_proc/${vyyyymmdd}/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 -s - else - cp_vrfy $ccpa_raw/${vyyyymmdd}/00/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} - fi - fi - - elif [[ ${accum} == "03" ]]; then - # Check if valid hour is 21 - if [[ ${vhh_noZero} -ne 21 ]]; then - cd_vrfy $ccpa_raw/${vyyyymmdd} - # Pull CCPA data from HPSS - TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"ccpa.t${vhh}z.${accum}h.hrap.conus.gb2\" | awk '{print \$7}'\`" - echo "CALLING: ${TarCommand}" - htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "ccpa.t${vhh}z.${accum}h.hrap.conus.gb2" | awk '{print \$7}'` - elif [[ ${vhh_noZero} -eq 21 ]]; then - cd_vrfy $ccpa_raw/${vyyyymmdd_p1} - # Pull CCPA data from HPSS - TarCommand="htar -xvf ${TarFile_p1} \`htar -tf ${TarFile_p1} | egrep \"ccpa.t${vhh}z.${accum}h.hrap.conus.gb2\" | awk '{print \$7}'\`" - echo "CALLING: ${TarCommand}" - htar -xvf ${TarFile_p1} `htar -tf ${TarFile_p1} | egrep "ccpa.t${vhh}z.${accum}h.hrap.conus.gb2" | awk '{print \$7}'` - fi - - if [[ ${vhh_noZero} -eq 0 ]]; then - cp_vrfy $ccpa_raw/${vyyyymmdd}/00/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} - elif [[ ${vhh_noZero} -eq 3 || ${vhh_noZero} -eq 6 ]]; then - cp_vrfy $ccpa_raw/${vyyyymmdd}/06/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} - elif [[ ${vhh_noZero} -eq 9 || ${vhh_noZero} -eq 12 ]]; then - cp_vrfy $ccpa_raw/${vyyyymmdd}/12/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} - elif [[ ${vhh_noZero} -eq 15 || ${vhh_noZero} -eq 18 ]]; then - cp_vrfy $ccpa_raw/${vyyyymmdd}/18/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} - elif [[ ${vhh_noZero} -eq 21 ]]; then - cp_vrfy $ccpa_raw/${vyyyymmdd_p1}/00/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} - fi - - elif [[ ${accum} == "06" ]]; then - cd_vrfy $ccpa_raw/${vyyyymmdd} - # Pull CCPA data from HPSS - TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"ccpa.t${vhh}z.${accum}h.hrap.conus.gb2\" | awk '{print \$7}'\`" - echo "CALLING: ${TarCommand}" - htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "ccpa.t${vhh}z.${accum}h.hrap.conus.gb2" | awk '{print \$7}'` - - if [[ ${vhh_noZero} -eq 0 ]]; then - cp_vrfy $ccpa_raw/${vyyyymmdd}/00/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} - elif [[ ${vhh_noZero} -eq 6 ]]; then - cp_vrfy $ccpa_raw/${vyyyymmdd}/06/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} - elif [[ ${vhh_noZero} -eq 12 ]]; then - cp_vrfy $ccpa_raw/${vyyyymmdd}/12/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} - elif [[ ${vhh_noZero} -eq 18 ]]; then - cp_vrfy $ccpa_raw/${vyyyymmdd}/18/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} - fi - fi - fi - - # Increment to next forecast hour - current_fcst=$((${current_fcst} + ${accum})) - echo "Current fcst hr=${current_fcst}" - -done -# -#----------------------------------------------------------------------- -# -# Restore the shell options saved at the beginning of this script/func- -# tion. -# -#----------------------------------------------------------------------- -# -{ restore_shell_opts; } > /dev/null 2>&1 - diff --git a/scripts/exregional_get_obs_mrms.sh b/scripts/exregional_get_obs_mrms.sh deleted file mode 100755 index 69fcae4aa7..0000000000 --- a/scripts/exregional_get_obs_mrms.sh +++ /dev/null @@ -1,177 +0,0 @@ -#!/bin/bash - -# -#----------------------------------------------------------------------- -# -# Source the variable definitions file and the bash utility functions. -# -#----------------------------------------------------------------------- -# -. $USHdir/source_util_funcs.sh -source_config_for_task " " ${GLOBAL_VAR_DEFNS_FP} -# -#----------------------------------------------------------------------- -# -# Save current shell options (in a global array). Then set new options -# for this script/function. -# -#----------------------------------------------------------------------- -# -{ save_shell_opts; . $USHdir/preamble.sh; } > /dev/null 2>&1 -# -#----------------------------------------------------------------------- -# -# This script pulls MRMS data from the NOAA HPSS -# Top-level MRMS directory -# -#----------------------------------------------------------------------- -# -set +e - -mrms_dir=${OBS_DIR}/.. -if [[ ! -d "$mrms_dir" ]]; then - mkdir_vrfy -p $mrms_dir -fi - -# MRMS data from HPSS -mrms_raw=$mrms_dir/raw -if [[ ! -d "$mrms_raw" ]]; then - mkdir_vrfy -p $mrms_raw -fi - -# Reorganized MRMS location -mrms_proc=$mrms_dir/proc -if [[ ! -d "$mrms_proc" ]]; then - mkdir_vrfy -p $mrms_proc -fi - -# Initialization -yyyymmdd=${PDY} -hh=${cyc} - -start_valid=${CDATE}${hh} - -fhr_last=`echo ${FHR} | awk '{ print $NF }'` - -# Forecast length -fcst_length=${fhr_last} - -s_yyyy=`echo ${start_valid} | cut -c1-4` # year (YYYY) of start time -s_mm=`echo ${start_valid} | cut -c5-6` # month (MM) of start time -s_dd=`echo ${start_valid} | cut -c7-8` # day (DD) of start time -s_hh=`echo ${start_valid} | cut -c9-10` # hour (HH) of start time -start_valid_ut=`$DATE_UTIL -ud ''${s_yyyy}-${s_mm}-${s_dd}' UTC '${s_hh}':00:00' +%s` # convert start time to universal time - -end_fcst_sec=$(( ${fcst_length} * 3600 )) # convert last forecast lead hour to seconds -end_valid_ut=$(( ${start_valid_ut} + ${end_fcst_sec} )) # calculate current forecast time in universal time - -cur_ut=${start_valid_ut} -current_fcst=0 -fcst_sec=$(( ${current_fcst} * 3600 )) # convert forecast lead hour to seconds - -while [[ ${cur_ut} -le ${end_valid_ut} ]]; do - cur_time=`$DATE_UTIL -ud '1970-01-01 UTC '${cur_ut}' seconds' +%Y%m%d%H` # convert universal time to standard time - echo "cur_time=${cur_time}" - - # Calculate valid date info - vyyyy=`echo ${cur_time} | cut -c1-4` # year (YYYY) of time - vmm=`echo ${cur_time} | cut -c5-6` # month (MM) of time - vdd=`echo ${cur_time} | cut -c7-8` # day (DD) of time - vhh=`echo ${cur_time} | cut -c9-10` # hour (HH) of time - vyyyymmdd=`echo ${cur_time} | cut -c1-8` # YYYYMMDD of time - vinit_ut=`$DATE_UTIL -ud ''${vyyyy}-${vmm}-${vdd}' UTC '${vhh}':00:00' +%s` # convert time to universal time - - # Create necessary raw and proc directories - if [[ ! -d "$mrms_raw/${vyyyymmdd}" ]]; then - mkdir_vrfy -p $mrms_raw/${vyyyymmdd} - fi - - # Set field of interest from the MRMS products, including name and level information. - if [ "${field}" = "REFC" ]; then - #field_base_name="MergedReflectivityQComposite" - #level="_00.00_" - field_base_name="MergedReflectivityQCComposite" - level="_00.50_" - elif [ "${field}" = "RETOP" ]; then - field_base_name="EchoTop" - level="_18_00.50_" - else - echo "Field is not set to REFC or RETOP" - fi - - # Check if file exists on disk; if not, pull it. - mrms_file="$mrms_proc/${vyyyymmdd}/${field_base_name}${level}${vyyyy}${vmm}${vdd}-${vhh}0000.grib2" - echo "MRMS FILE:${mrms_file}" - - if [[ ! -f "${mrms_file}" ]]; then - cd_vrfy $mrms_raw/${vyyyymmdd} - - # Name of MRMS tar file on HPSS is dependent on date. Logic accounts for files from 2019 until Sept. 2020. - if [[ ${vyyyymmdd} -ge 20190101 && ${vyyyymmdd} -lt 20200303 ]]; then - CheckFile=`hsi "ls -1 /NCEPPROD/hpssprod/runhistory/rh${vyyyy}/${vyyyy}${vmm}/${vyyyy}${vmm}${vdd}/ldmdata.gyre.${vyyyy}${vmm}${vdd}.tar"` - Status=$? - if [[ ${Status} == 0 ]]; then - TarFile="/NCEPPROD/hpssprod/runhistory/rh${vyyyy}/${vyyyy}${vmm}/${vyyyy}${vmm}${vdd}/ldmdata.gyre.${vyyyy}${vmm}${vdd}.tar" - else - CheckFile=`hsi "ls -1 /NCEPPROD/hpssprod/runhistory/rh${vyyyy}/${vyyyy}${vmm}/${vyyyy}${vmm}${vdd}/ldmdata.tide.${vyyyy}${vmm}${vdd}.tar" >& /dev/null` - Status=$? - if [[ ${Status} == 0 ]]; then - TarFile="/NCEPPROD/hpssprod/runhistory/rh${vyyyy}/${vyyyy}${vmm}/${vyyyy}${vmm}${vdd}/ldmdata.tide.${vyyyy}${vmm}${vdd}.tar" - else - print_err_msg_exit "MRMS data not available for ${vyyyy}${vmm}${vdd}!" - fi - fi - fi - - if [[ ${vyyyymmdd} -ge 20200303 && ${vyyyymmdd} -lt 20220628 ]]; then - TarFile="/NCEPPROD/hpssprod/runhistory/rh${vyyyy}/${vyyyy}${vmm}/${vyyyy}${vmm}${vdd}/dcom_prod_ldmdata_obs.tar" - fi - - if [[ ${vyyyymmdd} -ge 20220628 ]]; then - TarFile="/NCEPPROD/hpssprod/runhistory/rh${vyyyy}/${vyyyy}${vmm}/${vyyyy}${vmm}${vdd}/dcom_ldmdata_obs.tar" - fi - - echo "TAR FILE:${TarFile}" - - TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"${field_base_name}${level}${vyyyy}${vmm}${vdd}-[0-9][0-9][0-9][0-9][0-9][0-9].grib2.gz\" | awk '{print \$7}'\`" - htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "${field_base_name}${level}${vyyyy}${vmm}${vdd}-[0-9][0-9][0-9][0-9][0-9][0-9].grib2.gz" | awk '{print \$7}'` - Status=$? - - if [[ ${Status} != 0 ]]; then - CurDate="${vyyyy}${vmm}${vdd}" - print_err_msg_exit "Bad return status (${Status}) for date \"${CurDate}\".\ -Did you forget to run \"module load hpss\"?\ - COMMAND: ${TarCommand}" - else - if [[ ! -d "$mrms_proc/${vyyyymmdd}" ]]; then - mkdir_vrfy -p $mrms_proc/${vyyyymmdd} - fi - - hour=0 - while [[ ${hour} -le 23 ]]; do - echo "hour=${hour}" - python ${USHdir}/mrms_pull_topofhour.py ${vyyyy}${vmm}${vdd}${hour} ${mrms_proc} ${mrms_raw} ${field_base_name} ${level} - hour=$((${hour} + 1)) # hourly increment - done - fi - - else - echo "mrms_file exists: \"$mrms_proc/${vyyyymmdd}/${field_base_name}${level}${vyyyy}${vmm}${vdd}-${vhh}0000.grib2\" No work to be done." - fi - - # Increment - current_fcst=$((${current_fcst} + 1)) # hourly increment - fcst_sec=$(( ${current_fcst} * 3600 )) # convert forecast lead hour to seconds - cur_ut=$(( ${start_valid_ut} + ${fcst_sec} )) - -done -# -#----------------------------------------------------------------------- -# -# Restore the shell options saved at the beginning of this script/func- -# tion. -# -#----------------------------------------------------------------------- -# -{ restore_shell_opts; } > /dev/null 2>&1 - diff --git a/scripts/exregional_get_obs_ndas.sh b/scripts/exregional_get_obs_ndas.sh deleted file mode 100755 index 25f6c2ae95..0000000000 --- a/scripts/exregional_get_obs_ndas.sh +++ /dev/null @@ -1,148 +0,0 @@ -#!/bin/bash - -# -#----------------------------------------------------------------------- -# -# Source the variable definitions file and the bash utility functions. -# -#----------------------------------------------------------------------- -# -. $USHdir/source_util_funcs.sh -source_config_for_task " " ${GLOBAL_VAR_DEFNS_FP} -# -#----------------------------------------------------------------------- -# -# Save current shell options (in a global array). Then set new options -# for this script/function. -# -#----------------------------------------------------------------------- -# -{ save_shell_opts; . $USHdir/preamble.sh; } > /dev/null 2>&1 -# -#----------------------------------------------------------------------- -# -# This script reorganizes the NDAS data into a more intuitive structure: -# A valid YYYYMMDD directory is created, and all files for the valid day are placed within the directory. -# -#----------------------------------------------------------------------- -# -# Top-level NDAS directory -ndas_dir=${OBS_DIR}/.. -if [[ ! -d "$ndas_dir" ]]; then - mkdir_vrfy -p $ndas_dir -fi - -# NDAS data from HPSS -ndas_raw=$ndas_dir/raw -if [[ ! -d "$ndas_raw" ]]; then - mkdir_vrfy -p $ndas_raw -fi - -# Reorganized NDAS location -ndas_proc=$ndas_dir/proc -if [[ ! -d "$ndas_proc" ]]; then - mkdir_vrfy -p $ndas_proc -fi - -# Initialization -yyyymmdd=${PDY} -hh=${cyc} - -init=${CDATE}${hh} - -# Forecast length -fhr_last=`echo ${FHR} | awk '{ print $NF }'` - -fcst_length=${fhr_last} - -current_fcst=00 -while [[ ${current_fcst} -le ${fcst_length} ]]; do - fcst_sec=$(( ${current_fcst} * 3600 )) # convert forecast lead hour to seconds - yyyy=${init:0:4} # year (YYYY) of initialization time - mm=${init:4:2} # month (MM) of initialization time - dd=${init:6:2} # day (DD) of initialization time - hh=${init:8:2} # hour (HH) of initialization time - init_ut=`$DATE_UTIL -ud ''${yyyy}-${mm}-${dd}' UTC '${hh}':00:00' +%s` # convert initialization time to universal time - vdate_ut=$(( ${init_ut} + ${fcst_sec} )) # calculate current forecast time in universal time - vdate=`$DATE_UTIL -ud '1970-01-01 UTC '${vdate_ut}' seconds' +%Y%m%d%H` # convert universal time to standard time - vyyyymmdd=${vdate:0:8} # forecast time (YYYYMMDD) - vyyyy=${vdate:0:4} # year (YYYY) of valid time - vmm=${vdate:4:2} # month (MM) of valid time - vdd=${vdate:6:2} # day (DD) of valid time - vhh=${vdate:8:2} # forecast hour (HH) - vhh_noZero=$(( ${vhh} + 0 )) - -echo "yyyy mm dd hh= $yyyy $mm $dd $hh" -echo "vyyyy vmm vdd vhh= $vyyyy $vmm $vdd $vhh" -echo "vhh_noZero=$vhh_noZero" - - - - # Check if file exists on disk - ndas_file="$ndas_proc/prepbufr.ndas.${vyyyymmdd}${vhh}" - echo "NDAS PB FILE:${ndas_file}" - - if [[ ! -f "${ndas_file}" ]]; then - if [[ ! -d "$ndas_raw/${vyyyymmdd}${vhh}" ]]; then - mkdir_vrfy -p $ndas_raw/${vyyyymmdd}${vhh} - fi - cd_vrfy $ndas_raw/${vyyyymmdd}${vhh} - - # Name of NDAS tar file on HPSS is dependent on date. Logic accounts for files from 2019 until July 2020. - if [[ ${vyyyymmdd} -ge 20190101 && ${vyyyymmdd} -le 20190820 ]]; then - TarFile="/NCEPPROD/hpssprod/runhistory/rh${vyyyy}/${vyyyy}${vmm}/${vyyyy}${vmm}${vdd}/com2_nam_prod_nam.${vyyyy}${vmm}${vdd}${vhh}.bufr.tar" - TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"prepbufr.tm[0-9][0-9].nr\" | awk '{print \$7}'\`" - echo "CALLING: ${TarCommand}" - htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "prepbufr.tm[0-9][0-9].nr" | awk '{print \$7}'` - elif [[ ${vyyyymmdd} -ge 20190821 && ${vyyyymmdd} -le 20200226 ]]; then - TarFile="/NCEPPROD/hpssprod/runhistory/rh${vyyyy}/${vyyyy}${vmm}/${vyyyy}${vmm}${vdd}/gpfs_dell1_nco_ops_com_nam_prod_nam.${vyyyy}${vmm}${vdd}${vhh}.bufr.tar" - TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"prepbufr.tm[0-9][0-9].nr\" | awk '{print \$7}'\`" - echo "CALLING: ${TarCommand}" - htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "prepbufr.tm[0-9][0-9].nr" | awk '{print \$7}'` - elif [[ ${vyyyymmdd} -gt 20200226 && ${vyyyymmdd} -le 20220627 ]]; then - TarFile="/NCEPPROD/hpssprod/runhistory/rh${vyyyy}/${vyyyy}${vmm}/${vyyyy}${vmm}${vdd}/com_nam_prod_nam.${vyyyy}${vmm}${vdd}${vhh}.bufr.tar" - TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"prepbufr.tm[0-9][0-9].nr\" | awk '{print \$7}'\`" - echo "CALLING: ${TarCommand}" - htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "prepbufr.tm[0-9][0-9].nr" | awk '{print \$7}'` - elif [[ ${vyyyymmdd} -gt 20220627 && ${vyyyymmdd} -le 20221129 ]]; then - TarFile="/NCEPPROD/hpssprod/runhistory/rh${vyyyy}/${vyyyy}${vmm}/${vyyyy}${vmm}${vdd}/com_obsproc_v1.0_nam.${vyyyy}${vmm}${vdd}${vhh}.bufr.tar" - TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"prepbufr.tm[0-9][0-9].nr\" | awk '{print \$7}'\`" - echo "CALLING: ${TarCommand}" - htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "prepbufr.tm[0-9][0-9].nr" | awk '{print \$7}'` - else - TarFile="/NCEPPROD/hpssprod/runhistory/rh${vyyyy}/${vyyyy}${vmm}/${vyyyy}${vmm}${vdd}/com_obsproc_v1.1_nam.${vyyyy}${vmm}${vdd}${vhh}.bufr.tar" - TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"prepbufr.tm[0-9][0-9].nr\" | awk '{print \$7}'\`" - echo "CALLING: ${TarCommand}" - htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "prepbufr.tm[0-9][0-9].nr" | awk '{print \$7}'` - fi - - if [[ ! -d "$ndas_proc" ]]; then - mkdir_vrfy -p $ndas_proc - fi - - if [[ ${vhh_noZero} -eq 0 || ${vhh} -eq 6 || ${vhh} -eq 12 || ${vhh} -eq 18 ]]; then - # copy files from the previous 6 hours - for tm in $(seq 0 5); do - vdate_ut_tm=$(( ${vdate_ut} - $tm * 3600 )) - vdate_tm=$($DATE_UTIL -ud '1970-01-01 UTC '${vdate_ut_tm}' seconds' +%Y%m%d%H) - vyyyymmddhh_tm=${vdate_tm:0:10} - tm2=$(echo $tm | awk '{printf "%02d\n", $0;}') - - cp_vrfy $ndas_raw/${vyyyymmdd}${vhh}/nam.t${vhh}z.prepbufr.tm${tm2}.nr $ndas_proc/prepbufr.ndas.${vyyyymmddhh_tm} - done - fi - fi - current_fcst=$((${current_fcst} + 6)) - echo "new fcst=${current_fcst}" - -done -# -#----------------------------------------------------------------------- -# -# Restore the shell options saved at the beginning of this script/func- -# tion. -# -#----------------------------------------------------------------------- -# -{ restore_shell_opts; } > /dev/null 2>&1 - diff --git a/scripts/exregional_get_obs_nohrsc.sh b/scripts/exregional_get_obs_nohrsc.sh deleted file mode 100755 index 12ec224fce..0000000000 --- a/scripts/exregional_get_obs_nohrsc.sh +++ /dev/null @@ -1,133 +0,0 @@ -#!/bin/bash - -# -#----------------------------------------------------------------------- -# -# Source the variable definitions file and the bash utility functions. -# -#----------------------------------------------------------------------- -# -. $USHdir/source_util_funcs.sh -source_config_for_task " " ${GLOBAL_VAR_DEFNS_FP} -# -#----------------------------------------------------------------------- -# -# Save current shell options (in a global array). Then set new options -# for this script/function. -# -#----------------------------------------------------------------------- -# -{ save_shell_opts; . $USHdir/preamble.sh; } > /dev/null 2>&1 -# -#----------------------------------------------------------------------- -# -# This script retrieves and organizes the NOHRSC data into a more intuitive structure: -# A valid YYYYMMDD directory is created, and all files for the valid day are placed within the directory. -# NOTE: Accumulation is currently hardcoded to 06h and 24h which are available every 6 and 12 hours, respectively. -# -#----------------------------------------------------------------------- -# - -# Top-level NOHRSC directory -nohrsc_dir=${OBS_DIR}/.. -if [[ ! -d "$nohrsc_dir" ]]; then - mkdir_vrfy -p $nohrsc_dir -fi - -# NOHRSC data from HPSS -nohrsc_raw=$nohrsc_dir/raw -if [[ ! -d "$nohrsc_raw" ]]; then - mkdir_vrfy -p $nohrsc_raw -fi - -# Reorganized NOHRSC location -nohrsc_proc=$nohrsc_dir/proc -if [[ ! -d "$nohrsc_proc" ]]; then - mkdir_vrfy -p $nohrsc_proc -fi - -# Accumulation is for accumulation of NOHRSC data to pull (hardcoded to 06h, see note above.) -#accum=${ACCUM} -accum=06 - -# Initialization -yyyymmdd=${PDY} -hh=${cyc} - -init=${CDATE}${hh} - -fhr_last=`echo ${FHR} | awk '{ print $NF }'` - -# Forecast length -fcst_length=${fhr_last} - -current_fcst=$accum -while [[ ${current_fcst} -le ${fcst_length} ]]; do - # Calculate valid date info - fcst_sec=`expr ${current_fcst} \* 3600` # convert forecast lead hour to seconds - yyyy=`echo ${init} | cut -c1-4` # year (YYYY) of initialization time - mm=`echo ${init} | cut -c5-6` # month (MM) of initialization time - dd=`echo ${init} | cut -c7-8` # day (DD) of initialization time - hh=`echo ${init} | cut -c9-10` # hour (HH) of initialization time - init_ut=`$DATE_UTIL -ud ''${yyyy}-${mm}-${dd}' UTC '${hh}':00:00' +%s` # convert initialization time to universal time - vdate_ut=`expr ${init_ut} + ${fcst_sec}` # calculate current forecast time in universal time - vdate=`$DATE_UTIL -ud '1970-01-01 UTC '${vdate_ut}' seconds' +%Y%m%d%H` # convert universal time to standard time - vyyyymmdd=`echo ${vdate} | cut -c1-8` # forecast time (YYYYMMDD) - vyyyy=`echo ${vdate} | cut -c1-4` # year (YYYY) of valid time - vmm=`echo ${vdate} | cut -c5-6` # month (MM) of valid time - vdd=`echo ${vdate} | cut -c7-8` # day (DD) of valid time - vhh=`echo ${vdate} | cut -c9-10` # forecast hour (HH) - - # Create necessary raw and prop directories - if [[ ! -d "$nohrsc_raw/${vyyyymmdd}" ]]; then - mkdir_vrfy -p $nohrsc_raw/${vyyyymmdd} - fi - - if [[ ! -d "$nohrsc_proc/${vyyyymmdd}" ]]; then - mkdir_vrfy -p $nohrsc_proc/${vyyyymmdd} - fi - - # Name of NOHRSC tar file on HPSS is for files after Mar. 2020. - TarFile="/NCEPPROD/hpssprod/runhistory/rh${vyyyy}/${vyyyy}${vmm}/${vyyyy}${vmm}${vdd}/dcom_prod_${vyyyy}${vmm}${vdd}.tar" - - accum_noZero=$(( 10#$accum )) - nohrsc_file="$nohrsc_proc/${vyyyymmdd}/sfav2_CONUS_06h_${vyyyymmdd}${vhh}_grid184.grb2" - echo "NOHRSC FILE:${nohrsc_file}" - if [[ ! -f "${nohrsc_file}" ]]; then - - if (( ${current_fcst} % 6 == 0 )) ; then - cd_vrfy $nohrsc_raw/${vyyyymmdd} - # Pull NOHRSC data from HPSS - TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"sfav2_CONUS_6h_${vyyyymmdd}${vhh}_grid184.grb2\" | awk '{print \$7}'\`" - echo "CALLING: ${TarCommand}" - htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "sfav2_CONUS_6h_${vyyyymmdd}${vhh}_grid184.grb2" | awk '{print \$7}'` - - cp_vrfy $nohrsc_raw/${vyyyymmdd}/wgrbbul/nohrsc_snowfall/sfav2_CONUS_6h_${vyyyymmdd}${vhh}_grid184.grb2 $nohrsc_proc/${vyyyymmdd}/sfav2_CONUS_06h_${vyyyymmdd}${vhh}_grid184.grb2 - fi - - if (( ${current_fcst} % 12 == 0 )) && (( ${current_fcst} >= 24 )) ; then - cd_vrfy $nohrsc_raw/${vyyyymmdd} - # Pull NOHRSC data from HPSS - TarCommand="htar -xvf ${TarFile} \`htar -tf ${TarFile} | egrep \"sfav2_CONUS_24h_${vyyyymmdd}${vhh}_grid184.grb2\" | awk '{print \$7}'\`" - echo "CALLING: ${TarCommand}" - htar -xvf ${TarFile} `htar -tf ${TarFile} | egrep "sfav2_CONUS_24h_${vyyyymmdd}${vhh}_grid184.grb2" | awk '{print \$7}'` - - cp_vrfy $nohrsc_raw/${vyyyymmdd}/wgrbbul/nohrsc_snowfall/sfav2_CONUS_24h_${vyyyymmdd}${vhh}_grid184.grb2 $nohrsc_proc/${vyyyymmdd}/sfav2_CONUS_24h_${vyyyymmdd}${vhh}_grid184.grb2 - fi - fi - - # Increment to next forecast hour - current_fcst=$((${current_fcst} + 06)) - echo "Current fcst hr=${current_fcst}" - -done -# -#----------------------------------------------------------------------- -# -# Restore the shell options saved at the beginning of this script/func- -# tion. -# -#----------------------------------------------------------------------- -# -{ restore_shell_opts; } > /dev/null 2>&1 - diff --git a/scripts/exregional_get_verif_obs.sh b/scripts/exregional_get_verif_obs.sh new file mode 100755 index 0000000000..f2202ac813 --- /dev/null +++ b/scripts/exregional_get_verif_obs.sh @@ -0,0 +1,456 @@ +#!/usr/bin/env bash + +# +#----------------------------------------------------------------------- +# +# Source the variable definitions file and the bash utility functions. +# +#----------------------------------------------------------------------- +# +. $USHdir/source_util_funcs.sh +source_config_for_task " " ${GLOBAL_VAR_DEFNS_FP} +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# +{ save_shell_opts; . $USHdir/preamble.sh; } > /dev/null 2>&1 +set -x +# +#----------------------------------------------------------------------- +# +# This script performs several important tasks for preparing data for +# verification tasks. Depending on the value of the environment variable +# OBTYPE=(CCPA|MRMS|NDAS|NOHRSC), the script will prepare that particular data +# set. +# +# If data is not available on disk (in the location specified by +# CCPA_OBS_DIR, MRMS_OBS_DIR, NDAS_OBS_DIR, or NOHRSC_OBS_DIR respectively), +# the script attempts to retrieve the data from HPSS using the retrieve_data.py +# script. Depending on the data set, there are a few strange quirks and/or +# bugs in the way data is organized; see in-line comments for details. +# +# +# CCPA +# ---------- +# If data is available on disk, it must be in the following +# directory structure and file name conventions expected by verification +# tasks: +# +# {CCPA_OBS_DIR}/{YYYYMMDD}/ccpa.t{HH}z.01h.hrap.conus.gb2 +# +# If data is retrieved from HPSS, it will automatically staged by this +# this script. +# +# Notes about the data and how it's used for verification: +# +# 1. Accumulation is currently hardcoded to 01h. The verification will +# use MET/pcp-combine to sum 01h files into desired accumulations. +# +# 2. There is a problem with the valid time in the metadata for files +# valid from 19 - 00 UTC (or files under the '00' directory). This is +# accounted for in this script for data retrieved from HPSS, but if you +# have manually staged data on disk you should be sure this is accouned +# for. See in-line comments below for details. +# +# +# MRMS +# ---------- +# If data is available on disk, it must be in the following +# directory structure and file name conventions expected by verification +# tasks: +# +# {MRMS_OBS_DIR}/{YYYYMMDD}/[PREFIX]{YYYYMMDD}-{HH}0000.grib2, +# +# Where [PREFIX] is MergedReflectivityQCComposite_00.50_ for reflectivity +# data and EchoTop_18_00.50_ for echo top data. If data is not available +# at the top of the hour, you should rename the file closest in time to +# your hour(s) of interest to the above naming format. A script +# "ush/mrms_pull_topofhour.py" is provided for this purpose. +# +# If data is retrieved from HPSS, it will automatically staged by this +# this script. +# +# +# NDAS +# ---------- +# If data is available on disk, it must be in the following +# directory structure and file name conventions expected by verification +# tasks: +# +# {NDAS_OBS_DIR}/{YYYYMMDD}/prepbufr.ndas.{YYYYMMDDHH} +# +# Note that data retrieved from HPSS and other sources may be in a +# different format: nam.t{hh}z.prepbufr.tm{prevhour}.nr, where hh is +# either 00, 06, 12, or 18, and prevhour is the number of hours prior to +# hh (00 through 05). If using custom staged data, you will have to +# rename the files accordingly. +# +# If data is retrieved from HPSS, it will automatically staged by this +# this script. +# +# +# NOHRSC +# ---------- +# If data is available on disk, it must be in the following +# directory structure and file name conventions expected by verification +# tasks: +# +# {NOHRSC_OBS_DIR}/{YYYYMMDD}/sfav2_CONUS_{AA}h_{YYYYMMDD}{HH}_grid184.grb2 +# +# where AA is the 2-digit accumulation duration in hours: 06 or 24 +# +# METplus is configured to verify snowfall using 06- and 24-h accumulated +# snowfall from 6- and 12-hourly NOHRSC files, respectively. +# +# If data is retrieved from HPSS, it will automatically staged by this +# this script. + +#----------------------------------------------------------------------- +# Create and enter top-level obs directory (so temporary data from HPSS won't collide with other tasks) +mkdir -p ${OBS_DIR} +cd ${OBS_DIR} + +# Set log file for retrieving obs +logfile=retrieve_data.log + +# PDY and cyc are defined in rocoto XML...they are the yyyymmdd and hh for initial forecast hour respectively +iyyyy=$(echo ${PDY} | cut -c1-4) +imm=$(echo ${PDY} | cut -c5-6) +idd=$(echo ${PDY} | cut -c7-8) +ihh=${cyc} + +# Unix date utility needs dates in yyyy-mm-dd hh:mm:ss format +unix_init_DATE="${iyyyy}-${imm}-${idd} ${ihh}:00:00" + +# This awk expression gets the last item of the list $FHR +fcst_length=$(echo ${FHR} | awk '{ print $NF }') + +current_fcst=01 +while [[ ${current_fcst} -le ${fcst_length} ]]; do + #remove leading zero from current_fcst because bash treats numbers with leading zeros as octal *sigh* + current_fcst=$((10#${current_fcst})) + # Calculate valid date info using date utility + vdate=$($DATE_UTIL -d "${unix_init_DATE} ${current_fcst} hours" +%Y%m%d%H) + unix_vdate=$($DATE_UTIL -d "${unix_init_DATE} ${current_fcst} hours" "+%Y-%m-%d %H:00:00") + vyyyymmdd=$(echo ${vdate} | cut -c1-8) + vhh=$(echo ${vdate} | cut -c9-10) + + # Calculate valid date + 1 day; this is needed because (for some ungodly reason) CCPA files for 19-23z + # are stored in the *next* day's 00h directory + vdate_p1=$($DATE_UTIL -d "${unix_init_DATE} ${current_fcst} hours 1 day" +%Y%m%d%H) + vyyyymmdd_p1=$(echo ${vdate_p1} | cut -c1-8) + + #remove leading zero again, this time keep original + vhh_noZero=$((10#${vhh})) + + # Retrieve CCPA observations + if [[ ${OBTYPE} == "CCPA" ]]; then + + # Staging location for raw CCPA data from HPSS + ccpa_raw=${OBS_DIR}/raw + + # Reorganized CCPA location + ccpa_proc=${OBS_DIR} + + # Accumulation is for accumulation of CCPA data to pull (hardcoded to 01h, see note above.) + accum=01 + + # Check if file exists on disk; if not, pull it. + ccpa_file="$ccpa_proc/${vyyyymmdd}/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2" + echo "CCPA FILE:${ccpa_file}" + if [[ ! -f "${ccpa_file}" ]]; then + # Create necessary raw and prop directories + if [[ ! -d "$ccpa_raw/${vyyyymmdd}" ]]; then + mkdir -p $ccpa_raw/${vyyyymmdd} + fi + if [[ ! -d "$ccpa_raw/${vyyyymmdd_p1}" ]]; then + mkdir -p $ccpa_raw/${vyyyymmdd_p1} + fi + if [[ ! -d "$ccpa_proc/${vyyyymmdd}" ]]; then + mkdir -p $ccpa_proc/${vyyyymmdd} + fi + # Check if valid hour is 00 + if [[ ${vhh_noZero} -ge 19 && ${vhh_noZero} -le 23 ]]; then + # Pull CCPA data from HPSS + cmd=" + python3 -u ${USHdir}/retrieve_data.py \ + --debug \ + --file_set obs \ + --config ${PARMdir}/data_locations.yml \ + --cycle_date ${vyyyymmdd_p1}${vhh} \ + --data_stores hpss \ + --data_type CCPA_obs \ + --output_path $ccpa_raw/${vyyyymmdd_p1} \ + --summary_file ${logfile}" + + echo "CALLING: ${cmd}" + $cmd || print_err_msg_exit "\ + Could not retrieve CCPA data from HPSS + + The following command exited with a non-zero exit status: + ${cmd} +" + + else + # Pull CCPA data from HPSS + cmd=" + python3 -u ${USHdir}/retrieve_data.py \ + --debug \ + --file_set obs \ + --config ${PARMdir}/data_locations.yml \ + --cycle_date ${vyyyymmdd}${vhh} \ + --data_stores hpss \ + --data_type CCPA_obs \ + --output_path $ccpa_raw/${vyyyymmdd} \ + --summary_file ${logfile}" + + echo "CALLING: ${cmd}" + $cmd || print_err_msg_exit "\ + Could not retrieve CCPA data from HPSS + + The following command exited with a non-zero exit status: + ${cmd} +" + fi + + # One hour CCPA files have incorrect metadata in the files under the "00" directory from 20180718 to 20210504. + # After data is pulled, reorganize into correct valid yyyymmdd structure. + if [[ ${vhh_noZero} -ge 1 && ${vhh_noZero} -le 6 ]]; then + cp $ccpa_raw/${vyyyymmdd}/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} + elif [[ ${vhh_noZero} -ge 7 && ${vhh_noZero} -le 12 ]]; then + cp $ccpa_raw/${vyyyymmdd}/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} + elif [[ ${vhh_noZero} -ge 13 && ${vhh_noZero} -le 18 ]]; then + cp $ccpa_raw/${vyyyymmdd}/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} + elif [[ ${vhh_noZero} -ge 19 && ${vhh_noZero} -le 23 ]]; then + if [[ ${vyyyymmdd} -ge 20180718 && ${vyyyymmdd} -le 20210504 ]]; then + wgrib2 $ccpa_raw/${vyyyymmdd_p1}/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 -set_date -24hr -grib $ccpa_proc/${vyyyymmdd}/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 -s + else + cp $ccpa_raw/${vyyyymmdd_p1}/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} + fi + elif [[ ${vhh_noZero} -eq 0 ]]; then + # One hour CCPA files on HPSS have incorrect metadata in the files under the "00" directory from 20180718 to 20210504. + if [[ ${vyyyymmdd} -ge 20180718 && ${vyyyymmdd} -le 20210504 ]]; then + wgrib2 $ccpa_raw/${vyyyymmdd}/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 -set_date -24hr -grib $ccpa_proc/${vyyyymmdd}/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 -s + else + cp $ccpa_raw/${vyyyymmdd}/ccpa.t${vhh}z.${accum}h.hrap.conus.gb2 $ccpa_proc/${vyyyymmdd} + fi + fi + + else + echo "File already exists on disk; will not retrieve" + fi + # Retrieve MRMS observations + elif [[ ${OBTYPE} == "MRMS" ]]; then + # Top-level MRMS directory + # raw MRMS data from HPSS + mrms_raw=${OBS_DIR}/raw + + # Reorganized MRMS location + mrms_proc=${OBS_DIR} + + # For each field (REFC and RETOP), check if file exists on disk; if not, pull it. + for field in ${VAR[@]}; do + if [ "${field}" = "REFC" ]; then + field_base_name="MergedReflectivityQCComposite" + level="_00.50_" + elif [ "${field}" = "RETOP" ]; then + field_base_name="EchoTop" + level="_18_00.50_" + else + echo "Invalid field: ${field}" + print_err_msg_exit "\ + Invalid field specified: ${field} + + Valid options are 'REFC', 'RETOP'. +" + fi + + mrms_file="$mrms_proc/${vyyyymmdd}/${field_base_name}${level}${vyyyymmdd}-${vhh}0000.grib2" + echo "For field ${field}, looking for MRMS FILE: ${mrms_file}" + + if [[ ! -f "${mrms_file}" ]]; then + # Create directories if necessary + if [[ ! -d "$mrms_raw/${vyyyymmdd}" ]]; then + mkdir -p $mrms_raw/${vyyyymmdd} + fi + if [[ ! -d "$mrms_proc/${vyyyymmdd}" ]]; then + mkdir -p $mrms_proc/${vyyyymmdd} + fi + + + # Pull MRMS data from HPSS + cmd=" + python3 -u ${USHdir}/retrieve_data.py \ + --debug \ + --file_set obs \ + --config ${PARMdir}/data_locations.yml \ + --cycle_date ${vyyyymmdd}${vhh} \ + --data_stores hpss \ + --data_type MRMS_obs \ + --output_path $mrms_raw/${vyyyymmdd} \ + --summary_file ${logfile}" + + echo "CALLING: ${cmd}" + + $cmd || print_err_msg_exit "\ + Could not retrieve MRMS data from HPSS + + The following command exited with a non-zero exit status: + ${cmd} +" + + hour=0 + while [[ ${hour} -le 23 ]]; do + HH=$(printf "%02d" $hour) + echo "hour=${hour}" + python ${USHdir}/mrms_pull_topofhour.py --valid_time ${vyyyymmdd}${HH} --outdir ${mrms_proc} --source ${mrms_raw} --product ${field_base_name} + hour=$((${hour} + 1)) # hourly increment + done + + else + echo "mrms_file exists: \"$mrms_proc/${vyyyymmdd}/${field_base_name}${level}${vyyyymmdd}-${vhh}0000.grib2\" No work to be done." + fi + done + + # Retrieve NDAS observations + elif [[ ${OBTYPE} == "NDAS" ]]; then + # raw NDAS data from HPSS + ndas_raw=${OBS_DIR}/raw + + # Reorganized NDAS location + ndas_proc=${OBS_DIR} + + # Check if file exists on disk; NDAS data is available in 6-hourly combined prepbufr files + # If forecast ends on an off-hour (i.e., not 00z, 06z, 12z, or 18z), the last few hours of data may not be retrieved + if [[ ${vhh_noZero} -eq 0 || ${vhh_noZero} -eq 6 || ${vhh_noZero} -eq 12 || ${vhh_noZero} -eq 18 ]]; then + ndas_file="$ndas_proc/prepbufr.ndas.${vyyyymmdd}${vhh}" + echo "NDAS PB FILE:${ndas_file}" + + if [[ ! -f "${ndas_file}" ]]; then + if [[ ! -d "$ndas_raw/${vyyyymmdd}${vhh}" ]]; then + mkdir -p $ndas_raw/${vyyyymmdd}${vhh} + fi + + # Pull NDAS data from HPSS + cmd=" + python3 -u ${USHdir}/retrieve_data.py \ + --debug \ + --file_set obs \ + --config ${PARMdir}/data_locations.yml \ + --cycle_date ${vyyyymmdd}${vhh} \ + --data_stores hpss \ + --data_type NDAS_obs \ + --output_path $ndas_raw/${vyyyymmdd}${vhh} \ + --summary_file ${logfile}" + + echo "CALLING: ${cmd}" + + $cmd || print_err_msg_exit "\ + Could not retrieve NDAS data from HPSS + + The following command exited with a non-zero exit status: + ${cmd} +" + + if [[ ! -d "$ndas_proc" ]]; then + mkdir -p $ndas_proc + fi + + # copy files from the previous 6 hours + for tm in $(seq 0 5); do + vyyyymmddhh_tm=$($DATE_UTIL -d "${unix_vdate} ${tm} hours ago" +%Y%m%d%H) + tm2=$(echo $tm | awk '{printf "%02d\n", $0;}') + + cp $ndas_raw/${vyyyymmdd}${vhh}/nam.t${vhh}z.prepbufr.tm${tm2}.nr $ndas_proc/prepbufr.ndas.${vyyyymmddhh_tm} + done + else + echo "NDAS file exists: ${ndas_file}" + echo "Will not retrieve from HPSS" + fi + fi + + # Retrieve NOHRSC observations + elif [[ ${OBTYPE} == "NOHRSC" ]]; then + + # Reorganized NOHRSC location (no need for raw data dir) + nohrsc_proc=${OBS_DIR} + + nohrsc06h_file="$nohrsc_proc/${vyyyymmdd}/sfav2_CONUS_06h_${vyyyymmdd}${vhh}_grid184.grb2" + nohrsc24h_file="$nohrsc_proc/${vyyyymmdd}/sfav2_CONUS_24h_${vyyyymmdd}${vhh}_grid184.grb2" + retrieve=0 + # If 24-hour files should be available (at 00z and 12z) then look for both files + # Otherwise just look for 6hr file + if (( ${current_fcst} % 12 == 0 )) && (( ${current_fcst} >= 24 )) ; then + if [ ! -f "${nohrsc06h_file}" ] || [ ! -f "${nohrsc24h_file}" ] ; then + retrieve=1 + else + echo "NOHRSC 6h accumulation file exists: ${nohrsc06h_file}" + echo "NOHRSC 24h accumulation file exists: ${nohrsc24h_file}" + fi + elif (( ${current_fcst} % 6 == 0 )) ; then + if [ ! -f "${nohrsc06h_file}" ]; then + retrieve=1 + else + echo "NOHRSC 6h accumulation file exists: ${nohrsc06h_file}" + fi + fi + if [ $retrieve == 1 ]; then + if [[ ! -d "$nohrsc_proc/${vyyyymmdd}" ]]; then + mkdir -p $nohrsc_proc/${vyyyymmdd} + fi + + # Pull NOHRSC data from HPSS; script will retrieve all files so only call once + cmd=" + python3 -u ${USHdir}/retrieve_data.py \ + --debug \ + --file_set obs \ + --config ${PARMdir}/data_locations.yml \ + --cycle_date ${vyyyymmdd}${vhh} \ + --data_stores hpss \ + --data_type NOHRSC_obs \ + --output_path $nohrsc_proc/${vyyyymmdd} \ + --summary_file ${logfile}" + + echo "CALLING: ${cmd}" + + $cmd || print_err_msg_exit "\ + Could not retrieve NOHRSC data from HPSS + + The following command exited with a non-zero exit status: + ${cmd} +" + # 6-hour forecast needs to be renamed + mv $nohrsc_proc/${vyyyymmdd}/sfav2_CONUS_6h_${vyyyymmdd}${vhh}_grid184.grb2 ${nohrsc06h_file} + else + echo "Will not retrieve from HPSS" + fi + + else + print_err_msg_exit "\ + Invalid OBTYPE specified for script; valid options are CCPA, MRMS, NDAS, and NOHRSC + " + fi # Increment to next forecast hour + # Increment to next forecast hour + echo "Finished fcst hr=${current_fcst}" + current_fcst=$((${current_fcst} + 1)) + +done + + +# Clean up raw, unprocessed observation files +rm -rf ${OBS_DIR}/raw + +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/func- +# tion. +# +#----------------------------------------------------------------------- +# +{ restore_shell_opts; } > /dev/null 2>&1 + diff --git a/tests/WE2E/machine_suites/comprehensive b/tests/WE2E/machine_suites/comprehensive index 8619223cc8..eac4a30081 100644 --- a/tests/WE2E/machine_suites/comprehensive +++ b/tests/WE2E/machine_suites/comprehensive @@ -1,5 +1,7 @@ +2020_CAD community custom_ESGgrid +custom_ESGgrid_Great_Lakes_snow_8km custom_GFDLgrid__GFDLgrid_USE_NUM_CELLS_IN_FILENAMES_eq_FALSE custom_GFDLgrid deactivate_tasks @@ -54,9 +56,9 @@ grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_RAP_suite_WoFS_v0 grid_SUBCONUS_Ind_3km_ics_NAM_lbcs_NAM_suite_GFS_v16 grid_SUBCONUS_Ind_3km_ics_RAP_lbcs_RAP_suite_RRFS_v1beta_plot GST_release_public_v1 -MET_verification_only_vx MET_ensemble_verification_only_vx MET_ensemble_verification_only_vx_time_lag +MET_verification_only_vx nco nco_ensemble nco_grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 diff --git a/tests/WE2E/machine_suites/comprehensive.cheyenne b/tests/WE2E/machine_suites/comprehensive.cheyenne index fa893b5298..16a9cb81dd 100644 --- a/tests/WE2E/machine_suites/comprehensive.cheyenne +++ b/tests/WE2E/machine_suites/comprehensive.cheyenne @@ -44,6 +44,7 @@ grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_RAP_suite_WoFS_v0 grid_SUBCONUS_Ind_3km_ics_NAM_lbcs_NAM_suite_GFS_v16 grid_SUBCONUS_Ind_3km_ics_RAP_lbcs_RAP_suite_RRFS_v1beta_plot GST_release_public_v1 +MET_ensemble_verification_only_vx MET_verification_only_vx nco nco_ensemble diff --git a/tests/WE2E/machine_suites/comprehensive.orion b/tests/WE2E/machine_suites/comprehensive.orion index a547d0b8b8..a93d437de1 100644 --- a/tests/WE2E/machine_suites/comprehensive.orion +++ b/tests/WE2E/machine_suites/comprehensive.orion @@ -1,3 +1,4 @@ +2020_CAD community custom_ESGgrid custom_GFDLgrid__GFDLgrid_USE_NUM_CELLS_IN_FILENAMES_eq_FALSE @@ -46,6 +47,7 @@ grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_RAP_suite_WoFS_v0 grid_SUBCONUS_Ind_3km_ics_NAM_lbcs_NAM_suite_GFS_v16 grid_SUBCONUS_Ind_3km_ics_RAP_lbcs_RAP_suite_RRFS_v1beta_plot GST_release_public_v1 +MET_ensemble_verification_only_vx MET_verification_only_vx nco nco_ensemble @@ -56,4 +58,3 @@ nco_grid_RRFS_CONUScompact_25km_ics_HRRR_lbcs_RAP_suite_HRRR pregen_grid_orog_sfc_climo specify_EXTRN_MDL_SYSBASEDIR_ICS_LBCS specify_template_filenames -2020_CAD diff --git a/tests/WE2E/machine_suites/coverage.cheyenne.gnu b/tests/WE2E/machine_suites/coverage.cheyenne.gnu index 79927f7e81..76d407dc80 100644 --- a/tests/WE2E/machine_suites/coverage.cheyenne.gnu +++ b/tests/WE2E/machine_suites/coverage.cheyenne.gnu @@ -6,4 +6,5 @@ grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_FV3GFS_suite_HRRR grid_RRFS_CONUScompact_25km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 grid_RRFS_NA_13km_ics_FV3GFS_lbcs_FV3GFS_suite_RAP grid_SUBCONUS_Ind_3km_ics_NAM_lbcs_NAM_suite_GFS_v16 +MET_verification_only_vx specify_EXTRN_MDL_SYSBASEDIR_ICS_LBCS diff --git a/tests/WE2E/machine_suites/coverage.hera.gnu.com b/tests/WE2E/machine_suites/coverage.hera.gnu.com index bbff016f1e..9186c24d56 100644 --- a/tests/WE2E/machine_suites/coverage.hera.gnu.com +++ b/tests/WE2E/machine_suites/coverage.hera.gnu.com @@ -7,5 +7,5 @@ quilting_false grid_SUBCONUS_Ind_3km_ics_HRRR_lbcs_RAP_suite_WoFS_v0 GST_release_public_v1 MET_verification_only_vx -MET_ensemble_verification_only_vx_time_lag +#MET_ensemble_verification_only_vx_time_lag Removed temporarily due to HPSS permissions issue nco_grid_RRFS_CONUS_13km_ics_FV3GFS_lbcs_FV3GFS_suite_GFS_v16 diff --git a/tests/WE2E/machine_suites/coverage.jet b/tests/WE2E/machine_suites/coverage.jet index 75e0261d38..05e7cf8452 100644 --- a/tests/WE2E/machine_suites/coverage.jet +++ b/tests/WE2E/machine_suites/coverage.jet @@ -1,5 +1,6 @@ community custom_ESGgrid +#custom_ESGgrid_Great_Lakes_snow_8km Removed temporarily due to HPSS permissions issue custom_GFDLgrid get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_nemsio_2021032018 get_from_HPSS_ics_FV3GFS_lbcs_FV3GFS_fmt_netcdf_2022060112_48h diff --git a/tests/WE2E/run_WE2E_tests.py b/tests/WE2E/run_WE2E_tests.py index 3ef3649efd..8a4c913219 100755 --- a/tests/WE2E/run_WE2E_tests.py +++ b/tests/WE2E/run_WE2E_tests.py @@ -188,7 +188,17 @@ def run_we2e_tests(homedir, args) -> None: config_defaults,"lbcs") if 'verification' in test_cfg: - logging.debug(test_cfg['verification']) + # This section checks if we are doing verification on a machine with staged verification + # obs. If so, and if the config file does not explicitly set the observation locations, + # fill these in with defaults from the machine files + obs_vars = ['CCPA_OBS_DIR','MRMS_OBS_DIR','NDAS_OBS_DIR'] + if 'platform' not in test_cfg: + test_cfg['platform'] = {} + for obvar in obs_vars: + mach_path = machine_defaults['platform'].get('TEST_'+obvar) + if not test_cfg['platform'].get(obvar) and mach_path: + logging.debug(f'Setting CCPA_OBS_DIR = {mach_path} from machine file') + test_cfg['platform'][obvar] = mach_path if 'cpl_aqm_parm' in test_cfg: test_aqm_input_basedir = machine_defaults['platform']['TEST_AQM_INPUT_BASEDIR'] diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_RAP.yaml b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_RAP.yaml index 6082ecbd20..1e6c5ea724 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_RAP.yaml +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_FV3GFS_lbcs_RAP_suite_RAP.yaml @@ -21,9 +21,6 @@ rocoto: metatask_run_ensemble: task_run_fcst_mem#mem#: walltime: 01:00:00 - task_get_obs_ccpa: - task_get_obs_mrms: - task_get_obs_ndas: task_get_extrn_ics: EXTRN_MDL_NAME_ICS: FV3GFS FV3GFS_FILE_FMT_ICS: grib2 diff --git a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GFS_v16.yaml b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GFS_v16.yaml index 25099ce9b8..23ad640390 100644 --- a/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GFS_v16.yaml +++ b/tests/WE2E/test_configs/grids_extrn_mdls_suites_community/config.grid_RRFS_CONUS_25km_ics_NAM_lbcs_NAM_suite_GFS_v16.yaml @@ -20,9 +20,6 @@ rocoto: metatask_run_ensemble: task_run_fcst_mem#mem#: walltime: 01:00:00 - task_get_obs_ccpa: - task_get_obs_mrms: - task_get_obs_ndas: task_get_extrn_ics: EXTRN_MDL_NAME_ICS: NAM USE_USER_STAGED_EXTRN_FILES: true diff --git a/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx.yaml b/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx.yaml index ff7b05a906..0c7088f7f8 100644 --- a/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx.yaml +++ b/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx.yaml @@ -18,9 +18,6 @@ workflow: rocoto: tasks: taskgroups: '{{ ["parm/wflow/verify_pre.yaml", "parm/wflow/verify_det.yaml", "parm/wflow/verify_ens.yaml"]|include }}' - task_get_obs_ccpa: - task_get_obs_mrms: - task_get_obs_ndas: global: DO_ENSEMBLE: true diff --git a/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx_time_lag.yaml b/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx_time_lag.yaml index b38eef188b..f7d82cb8cd 100644 --- a/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx_time_lag.yaml +++ b/tests/WE2E/test_configs/verification/config.MET_ensemble_verification_only_vx_time_lag.yaml @@ -19,9 +19,6 @@ workflow: rocoto: tasks: taskgroups: '{{ ["parm/wflow/verify_pre.yaml", "parm/wflow/verify_det.yaml", "parm/wflow/verify_ens.yaml"]|include }}' - task_get_obs_ccpa: - task_get_obs_mrms: - task_get_obs_ndas: nco: NET_default: 'RRFSE_CONUS' @@ -31,6 +28,11 @@ global: NUM_ENS_MEMBERS: 2 ENS_TIME_LAG_HRS: '[ 0, 12 ]' +platform: + CCPA_OBS_DIR: '{{ workflow.EXPTDIR }}/obs_data/ccpa/proc' + MRMS_OBS_DIR: '{{ workflow.EXPTDIR }}/obs_data/mrms/proc' + NDAS_OBS_DIR: '{{ workflow.EXPTDIR }}/obs_data/ndas/proc' + verification: VX_FCST_MODEL_NAME: FV3_GFS_v15p2_CONUS_25km VX_FCST_INPUT_BASEDIR: '{{ platform.get("TEST_VX_FCST_INPUT_BASEDIR") }}' diff --git a/tests/WE2E/test_configs/verification/config.MET_verification_only_vx.yaml b/tests/WE2E/test_configs/verification/config.MET_verification_only_vx.yaml index 1fc04a622a..9bc5ec505a 100644 --- a/tests/WE2E/test_configs/verification/config.MET_verification_only_vx.yaml +++ b/tests/WE2E/test_configs/verification/config.MET_verification_only_vx.yaml @@ -18,13 +18,9 @@ workflow: # # This test assumes that the post-processed forecast files are staged # (i.e., not generated by running the forecast model). -# It also assumes staged observations, so turn off those tasks. rocoto: tasks: taskgroups: '{{ ["parm/wflow/verify_pre.yaml", "parm/wflow/verify_det.yaml"]|include }}' - task_get_obs_ccpa: - task_get_obs_mrms: - task_get_obs_ndas: verification: VX_FCST_MODEL_NAME: FV3_GFS_v15p2_CONUS_25km diff --git a/tests/WE2E/test_configs/verification/config.custom_ESGgrid_Great_Lakes_snow_8km.yaml b/tests/WE2E/test_configs/verification/config.custom_ESGgrid_Great_Lakes_snow_8km.yaml new file mode 100644 index 0000000000..64d9e02d9e --- /dev/null +++ b/tests/WE2E/test_configs/verification/config.custom_ESGgrid_Great_Lakes_snow_8km.yaml @@ -0,0 +1,68 @@ +metadata: + description: |- + This test checks the capability of the workflow to run verification tasks + on a user-specified grid. This also serves as a winter snowstorm case for + checking snowfall verification statistics, and a test for custom domains + with staged netCDF data. +user: + RUN_ENVIR: community +workflow: + CCPP_PHYS_SUITE: FV3_RAP + GRID_GEN_METHOD: ESGgrid + DATE_FIRST_CYCL: '2023021700' + DATE_LAST_CYCL: '2023021700' + FCST_LEN_HRS: 6 + PREEXISTING_DIR_METHOD: rename +rocoto: + tasks: + taskgroups: '{{ ["parm/wflow/prep.yaml", "parm/wflow/coldstart.yaml", "parm/wflow/post.yaml", "parm/wflow/verify_pre.yaml", "parm/wflow/verify_det.yaml"]|include }}' + metatask_run_ensemble: + task_run_fcst_mem#mem#: + walltime: 01:00:00 +task_make_grid: + ESGgrid_LON_CTR: -82.0 + ESGgrid_LAT_CTR: 47.0 + ESGgrid_DELX: 8000.0 + ESGgrid_DELY: 8000.0 + ESGgrid_NX: 300 + ESGgrid_NY: 200 + ESGgrid_WIDE_HALO_WIDTH: 6 + ESGgrid_PAZI: 0.0 +task_get_extrn_ics: + EXTRN_MDL_NAME_ICS: FV3GFS + FV3GFS_FILE_FMT_ICS: netcdf + USE_USER_STAGED_EXTRN_FILES: true +task_get_extrn_lbcs: + EXTRN_MDL_NAME_LBCS: FV3GFS + FV3GFS_FILE_FMT_LBCS: netcdf + LBC_SPEC_INTVL_HRS: 6 + USE_USER_STAGED_EXTRN_FILES: true +task_run_fcst: + DT_ATMOS: 90 + LAYOUT_X: 12 + LAYOUT_Y: 12 + BLOCKSIZE: 32 + QUILTING: true + WRTCMP_write_groups: 1 + WRTCMP_write_tasks_per_group: '{{ LAYOUT_Y }}' + WRTCMP_output_grid: lambert_conformal + WRTCMP_cen_lon: '{{ task_make_grid.ESGgrid_LON_CTR }}' + WRTCMP_cen_lat: '{{ task_make_grid.ESGgrid_LAT_CTR }}' + WRTCMP_lon_lwr_left: -96.1 + WRTCMP_lat_lwr_left: 38.6 + WRTCMP_stdlat1: '{{ task_make_grid.ESGgrid_LAT_CTR }}' + WRTCMP_stdlat2: '{{ task_make_grid.ESGgrid_LAT_CTR }}' + WRTCMP_nx: 299 + WRTCMP_ny: 195 + WRTCMP_dx: '{{ task_make_grid.ESGgrid_DELX }}' + WRTCMP_dy: '{{ task_make_grid.ESGgrid_DELY }}' +task_run_post: + POST_OUTPUT_DOMAIN_NAME: custom_ESGgrid_Michigan_Ontario +verification: + VX_FCST_MODEL_NAME: Michigan_Ontario_snow_8km + VX_FIELDS: [ "APCP", "REFC", "RETOP", "SFC", "UPA", "ASNOW" ] +platform: + CCPA_OBS_DIR: '{{ workflow.EXPTDIR }}/CCPA_obs' + MRMS_OBS_DIR: '{{ workflow.EXPTDIR }}/MRMS_obs' + NDAS_OBS_DIR: '{{ workflow.EXPTDIR }}/NDAS_obs' + NOHRSC_OBS_DIR: '{{ workflow.EXPTDIR }}/NOHRSC_obs' diff --git a/tests/test_python/test_retrieve_data.py b/tests/test_python/test_retrieve_data.py index e35838c0dd..1d54e0904c 100644 --- a/tests/test_python/test_retrieve_data.py +++ b/tests/test_python/test_retrieve_data.py @@ -168,7 +168,7 @@ def test_gdas_ics_from_aws(self): with tempfile.TemporaryDirectory(dir=self.path) as tmp_dir: os.chdir(tmp_dir) - out_path_tmpl = os.path.join(tmp_dir, "mem{{mem:03d}}") + out_path_tmpl = os.path.join(tmp_dir, "mem{mem:03d}") # fmt: off args = [ diff --git a/ush/config_defaults.yaml b/ush/config_defaults.yaml index 7308b84af1..f80d4d7887 100644 --- a/ush/config_defaults.yaml +++ b/ush/config_defaults.yaml @@ -234,118 +234,48 @@ platform: # Set METplus parameters. Definitions: # # CCPA_OBS_DIR: - # User-specified location of top-level directory where CCPA hourly - # precipitation files used by METplus are located. This parameter - # needs to be set for both user-provided observations and for - # observations that are retrieved from the NOAA HPSS (if the user has - # access) via the get_obs_ccpa task (activated in workflow by - # including the parm/wflow/verify.yaml task set in the wflow config). - # In the case of pulling observations directly from NOAA HPSS, the - # data retrieved will be placed in this directory. Please note, this - # path must be defind as /full-path-to-obs/ccpa/proc. METplus is - # configured to verify 01-, 03-, 06-, and 24-h accumulated - # precipitation using hourly CCPA files. - # METplus configuration files require the use of predetermined - # directory structure and file names. Therefore, if the CCPA files are - # user provided, they need to follow the anticipated naming structure: - # - # {YYYYMMDD}/ccpa.t{HH}z.01h.hrap.conus.gb2, - # - # where YYYY is the 4-digit valid year, MM the 2-digit valid month, DD - # the 2-digit valid day of the month, and HH the 2-digit valid hour of - # the day. In addition, a caveat is noted for using hourly CCPA data. - # There is a problem with the valid time in the metadata for files - # valid from 19 - 00 UTC (or files under the '00' directory). The - # script to pull the CCPA data from the NOAA HPSS has an example of - # how to account for this as well as organizing the data into a more - # intuitive format: scripts/exregional_get_ccpa_files.sh. When a fix - # is provided, it will be accounted for in the - # exregional_get_ccpa_files.sh script. - # - # NOHRSC_OBS_DIR: - # User-specified location of top-level directory where NOHRSC 6- and 24-h hour - # snowfall accumulation files used by METplus are located. This parameter - # needs to be set for both user-provided observations and for - # observations that are retrieved from the NOAA HPSS (if the user has - # access) via the get_obs_nohrsc task (activated in workflow by - # including the parm/wflow/verify_pre.yaml task set in the wflow config). - # In the case of pulling observations directly from NOAA HPSS, the - # data retrieved will be placed in this directory. Please note, this - # path must be defind as /full-path-to-obs/nohrsc/proc. METplus is - # configured to verify 06- and 24-h accumulated snowfall - # precipitation using 6- and 12-hourly NOHRSC files, respectively. - # METplus configuration files require the use of predetermined - # directory structure and file names. Therefore, if the NOHRSC files are - # user provided, they need to follow the anticipated naming structure: - # - # {YYYYMMDD}/sfav2_CONUS_{AA}h_{YYYYMMDD}{HH}_grid184.grb2 - # - # where AA is the 2-digit accumulation duration, YYYY is the 4-digit - # valid year, MM the 2-digit valid month, DD the 2-digit valid day - # of the month, and HH the 2-digit valid hour of the day. + # User-specified location of the directory where CCPA hourly + # precipitation files used by METplus are located (or, if + # retrieved by the workflow, where they will be placed). See comments + # in file scripts/exregional_get_verif_obs.sh for more details about + # files and directory structure, as well as important caveats about + # errors in the metadata and file names. + # NOTE: Do not set this to the same path as other *_OBS_DIR variables, + # otherwise unexpected results and data loss may occur # # MRMS_OBS_DIR: - # User-specified location of top-level directory where MRMS composite - # reflectivity files used by METplus are located. This parameter - # needs to be set for both user-provided observations and for - # observations that are retrieved from the NOAA HPSS (if the user has - # access) via the get_obs_mrms task (activated in workflow by - # including the parm/wflow/verify.yaml task set in the wflow config). - # In the case of pulling observations directly from NOAA HPSS, the - # data retrieved will be placed in this directory. Please note, this - # path must be defind as /full-path-to-obs/mrms/proc. - # METplus configuration files require the use of predetermined - # directory structure and file names. Therefore, if the MRMS files are - # user provided, they need to follow the anticipated naming structure: - # - # {YYYYMMDD}/MergedReflectivityQCComposite_00.50_{YYYYMMDD}-{HH}{mm}{SS}.grib2, - # - # where YYYY is the 4-digit valid year, MM the 2-digit valid month, DD - # the 2-digit valid day of the month, HH the 2-digit valid hour of the - # day, mm the 2-digit valid minutes of the hour, and SS is the - # two-digit valid seconds of the hour. In addition, METplus is - # configured to look for a MRMS composite reflectivity file for the - # valid time of the forecast being verified; since MRMS composite - # reflectivity files do not always exactly match the valid time, a - # script, within the main script to retrieve MRMS data from the NOAA - # HPSS, is used to identify and rename the MRMS composite reflectivity - # file to match the valid time of the forecast. The script to pull - # the MRMS data from the NOAA HPSS has an example of the expected file - # naming structure: scripts/exregional_get_mrms_files.sh. This script - # calls the script used to identify the MRMS file closest to the valid - # time: ush/mrms_pull_topofhour.py. + # User-specified location of the directory where MRMS composite + # reflectivity and echo top files used by METplus are located (or, if + # retrieved by the workflow, where they will be placed). See comments + # in the scripts/exregional_get_verif_obs.sh for more details about + # files and directory structure. + # NOTE: Do not set this to the same path as other *_OBS_DIR variables, + # otherwise unexpected results and data loss may occur # # NDAS_OBS_DIR: # User-specified location of top-level directory where NDAS prepbufr - # files used by METplus are located. This parameter needs to be set for - # both user-provided observations and for observations that are - # retrieved from the NOAA HPSS (if the user has access) via the - # get_obs_ndas task (activated in workflow by includeing the - # parm/wflow/verify.yaml task set in the wflow config).  - # In the case of pulling observations directly from NOAA HPSS, the - # data retrieved will be placed in this directory. Please note, this - # path must be defind as /full-path-to-obs/ndas/proc. METplus is - # configured to verify near-surface variables hourly and upper-air - # variables at times valid at 00 and 12 UTC with NDAS prepbufr files. - # METplus configuration files require the use of predetermined file - # names. Therefore, if the NDAS files are user provided, they need to - # follow the anticipated naming structure: - # - # prepbufr.ndas.{YYYYMMDDHH}, - # - # where YYYY is the 4-digit valid year, MM the 2-digit valid month, DD - # the 2-digit valid day of the month, and HH the 2-digit valid hour of - # the day. The script to pull the NDAS data from the NOAA HPSS has an - # example of how to rename the NDAS data into a more intuitive format - # with the valid time listed in the file name: - # scripts/exregional_get_ndas_files.sh - # - #----------------------------------------------------------------------- - # - CCPA_OBS_DIR: "" - NOHRSC_OBS_DIR: "" - MRMS_OBS_DIR: "" - NDAS_OBS_DIR: "" + # files used by METplus are located (or, if retrieved by the workflow, + # where they will be placed). See comments in file + # scripts/exregional_get_verif_obs.sh for more details about files + # and directory structure. + # NOTE: Do not set this to the same path as other *_OBS_DIR variables, + # otherwise unexpected results and data loss may occur + # + # NOHRSC_OBS_DIR: + # User-specified location of top-level directory where NOHRSC 6- and + # 24-hour snowfall accumulation files used by METplus are located (or, + # if retrieved by the workflow, where they will be placed). See comments + # in file scripts/exregional_get_verif_obs.sh for more details about + # files and directory structure + # NOTE: Do not set this to the same path as other *_OBS_DIR variables, + # otherwise unexpected results and data loss may occur + # + #----------------------------------------------------------------------- + # + CCPA_OBS_DIR: "{{ workflow.EXPTDIR }}/obs_data/ccpa/proc" + NOHRSC_OBS_DIR: "{{ workflow.EXPTDIR }}/obs_data/nohrsc/proc" + MRMS_OBS_DIR: "{{ workflow.EXPTDIR }}/obs_data/mrms/proc" + NDAS_OBS_DIR: "{{ workflow.EXPTDIR }}/obs_data/ndas/proc" # #----------------------------------------------------------------------- # diff --git a/ush/machine/cheyenne.yaml b/ush/machine/cheyenne.yaml index 30d523e688..c4aa39d816 100644 --- a/ush/machine/cheyenne.yaml +++ b/ush/machine/cheyenne.yaml @@ -2,9 +2,9 @@ platform: WORKFLOW_MANAGER: rocoto NCORES_PER_NODE: 36 SCHED: pbspro - CCPA_OBS_DIR: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/obs_data/ccpa/proc - MRMS_OBS_DIR: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/obs_data/mrms/proc - NDAS_OBS_DIR: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/obs_data/ndas/proc + TEST_CCPA_OBS_DIR: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/obs_data/ccpa/proc + TEST_MRMS_OBS_DIR: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/obs_data/mrms/proc + TEST_NDAS_OBS_DIR: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/obs_data/ndas/proc DOMAIN_PREGEN_BASEDIR: /glade/work/epicufsrt/contrib/UFS_SRW_data/develop/FV3LAM_pregen QUEUE_DEFAULT: regular QUEUE_FCST: regular diff --git a/ush/machine/gaea.yaml b/ush/machine/gaea.yaml index a170bc2a6b..1f4410cf98 100644 --- a/ush/machine/gaea.yaml +++ b/ush/machine/gaea.yaml @@ -2,9 +2,9 @@ platform: WORKFLOW_MANAGER: rocoto NCORES_PER_NODE: 32 SCHED: slurm - CCPA_OBS_DIR: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/obs_data/ccpa/proc - MRMS_OBS_DIR: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/obs_data/mrms/proc - NDAS_OBS_DIR: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/obs_data/ndas/proc + TEST_CCPA_OBS_DIR: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/obs_data/ccpa/proc + TEST_MRMS_OBS_DIR: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/obs_data/mrms/proc + TEST_NDAS_OBS_DIR: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/obs_data/ndas/proc DOMAIN_PREGEN_BASEDIR: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/FV3LAM_pregen QUEUE_DEFAULT: normal QUEUE_FCST: normal diff --git a/ush/machine/hera.yaml b/ush/machine/hera.yaml index b12853ef00..5640917721 100644 --- a/ush/machine/hera.yaml +++ b/ush/machine/hera.yaml @@ -2,9 +2,9 @@ platform: WORKFLOW_MANAGER: rocoto NCORES_PER_NODE: 40 SCHED: slurm - CCPA_OBS_DIR: /scratch1/NCEPDEV/nems/role.epic/UFS_SRW_data/develop/obs_data/ccpa/proc - MRMS_OBS_DIR: /scratch1/NCEPDEV/nems/role.epic/UFS_SRW_data/develop/obs_data/mrms/proc - NDAS_OBS_DIR: /scratch1/NCEPDEV/nems/role.epic/UFS_SRW_data/develop/obs_data/ndas/proc + TEST_CCPA_OBS_DIR: /scratch1/NCEPDEV/nems/role.epic/UFS_SRW_data/develop/obs_data/ccpa/proc + TEST_MRMS_OBS_DIR: /scratch1/NCEPDEV/nems/role.epic/UFS_SRW_data/develop/obs_data/mrms/proc + TEST_NDAS_OBS_DIR: /scratch1/NCEPDEV/nems/role.epic/UFS_SRW_data/develop/obs_data/ndas/proc DOMAIN_PREGEN_BASEDIR: /scratch1/NCEPDEV/nems/role.epic/UFS_SRW_data/develop/FV3LAM_pregen PARTITION_DEFAULT: hera QUEUE_DEFAULT: batch diff --git a/ush/machine/jet.yaml b/ush/machine/jet.yaml index 3ab36517d4..a110f02614 100644 --- a/ush/machine/jet.yaml +++ b/ush/machine/jet.yaml @@ -2,9 +2,9 @@ platform: WORKFLOW_MANAGER: rocoto NCORES_PER_NODE: 24 SCHED: slurm - CCPA_OBS_DIR: /mnt/lfs4/HFIP/hfv3gfs/role.epic/UFS_SRW_data/develop/obs_data/ccpa/proc - MRMS_OBS_DIR: /mnt/lfs4/HFIP/hfv3gfs/role.epic/UFS_SRW_data/develop/obs_data/mrms/proc - NDAS_OBS_DIR: /mnt/lfs4/HFIP/hfv3gfs/role.epic/UFS_SRW_data/develop/obs_data/ndas/proc + TEST_CCPA_OBS_DIR: /mnt/lfs4/HFIP/hfv3gfs/role.epic/UFS_SRW_data/develop/obs_data/ccpa/proc + TEST_MRMS_OBS_DIR: /mnt/lfs4/HFIP/hfv3gfs/role.epic/UFS_SRW_data/develop/obs_data/mrms/proc + TEST_NDAS_OBS_DIR: /mnt/lfs4/HFIP/hfv3gfs/role.epic/UFS_SRW_data/develop/obs_data/ndas/proc DOMAIN_PREGEN_BASEDIR: /mnt/lfs4/HFIP/hfv3gfs/role.epic/UFS_SRW_data/develop/FV3LAM_pregen PARTITION_DEFAULT: sjet,vjet,kjet,xjet QUEUE_DEFAULT: batch diff --git a/ush/machine/linux.yaml b/ush/machine/linux.yaml index bc4c28c97f..7cedb8fc19 100644 --- a/ush/machine/linux.yaml +++ b/ush/machine/linux.yaml @@ -3,9 +3,9 @@ platform: NCORES_PER_NODE: 8 TASKTHROTTLE: 1 SCHED: slurm - CCPA_OBS_DIR: /home/username/DATA/UFS/obs_data/ccpa/proc - MRMS_OBS_DIR: /home/username/DATA/UFS/obs_data/mrms/proc - NDAS_OBS_DIR: /home/username/DATA/UFS/obs_data/ndas/proc + TEST_CCPA_OBS_DIR: /Users/username/DATA/UFS/obs_data/ccpa/proc + TEST_MRMS_OBS_DIR: /Users/username/DATA/UFS/obs_data/mrms/proc + TEST_NDAS_OBS_DIR: /Users/username/DATA/UFS/obs_data/ndas/proc DOMAIN_PREGEN_BASEDIR: /home/username/DATA/UFS/FV3LAM_pregen RUN_CMD_FCST: mpirun -n ${PE_MEMBER01} RUN_CMD_POST: mpirun diff --git a/ush/machine/macos.yaml b/ush/machine/macos.yaml index 6f6e3c2121..f6d5e2a952 100644 --- a/ush/machine/macos.yaml +++ b/ush/machine/macos.yaml @@ -3,9 +3,9 @@ platform: NCORES_PER_NODE: 8 TASKTHROTTLE: 1 SCHED: slurm - CCPA_OBS_DIR: /Users/username/DATA/UFS/obs_data/ccpa/proc - MRMS_OBS_DIR: /Users/username/DATA/UFS/obs_data/mrms/proc - NDAS_OBS_DIR: /Users/username/DATA/UFS/obs_data/ndas/proc + TEST_CCPA_OBS_DIR: /Users/username/DATA/UFS/obs_data/ccpa/proc + TEST_MRMS_OBS_DIR: /Users/username/DATA/UFS/obs_data/mrms/proc + TEST_NDAS_OBS_DIR: /Users/username/DATA/UFS/obs_data/ndas/proc DOMAIN_PREGEN_BASEDIR: /Users/username/DATA/UFS/FV3LAM_pregen RUN_CMD_FCST: mpirun -n ${PE_MEMBER01} RUN_CMD_POST: mpirun diff --git a/ush/machine/noaacloud.yaml b/ush/machine/noaacloud.yaml index 6b16023914..317f69a5d5 100644 --- a/ush/machine/noaacloud.yaml +++ b/ush/machine/noaacloud.yaml @@ -2,9 +2,9 @@ platform: WORKFLOW_MANAGER: rocoto NCORES_PER_NODE: 36 SCHED: slurm - CCPA_OBS_DIR: /contrib/EPIC/UFS_SRW_data/develop/obs_data/ccpa/proc - MRMS_OBS_DIR: /contrib/EPIC/UFS_SRW_data/develop/obs_data/mrms/proc - NDAS_OBS_DIR: /contrib/EPIC/UFS_SRW_data/develop/obs_data/ndas/proc + TEST_CCPA_OBS_DIR: /contrib/EPIC/UFS_SRW_data/develop/obs_data/ccpa/proc + TEST_MRMS_OBS_DIR: /contrib/EPIC/UFS_SRW_data/develop/obs_data/mrms/proc + TEST_NDAS_OBS_DIR: /contrib/EPIC/UFS_SRW_data/develop/obs_data/ndas/proc REMOVE_MEMORY: True RUN_CMD_FCST: mpiexec -np ${PE_MEMBER01} RUN_CMD_POST: mpiexec -np $nprocs diff --git a/ush/machine/orion.yaml b/ush/machine/orion.yaml index 21a1dffaf8..f203e3af46 100644 --- a/ush/machine/orion.yaml +++ b/ush/machine/orion.yaml @@ -2,9 +2,9 @@ platform: WORKFLOW_MANAGER: rocoto NCORES_PER_NODE: 40 SCHED: slurm - CCPA_OBS_DIR: /work/noaa/epic-ps/role-epic-ps/UFS_SRW_data/develop/obs_data/ccpa/proc - MRMS_OBS_DIR: /work/noaa/epic-ps/role-epic-ps/UFS_SRW_data/develop/obs_data/mrms/proc - NDAS_OBS_DIR: /work/noaa/epic-ps/role-epic-ps/UFS_SRW_data/develop/obs_data/ndas/proc + TEST_CCPA_OBS_DIR: /work/noaa/epic-ps/role-epic-ps/UFS_SRW_data/develop/obs_data/ccpa/proc + TEST_MRMS_OBS_DIR: /work/noaa/epic-ps/role-epic-ps/UFS_SRW_data/develop/obs_data/mrms/proc + TEST_NDAS_OBS_DIR: /work/noaa/epic-ps/role-epic-ps/UFS_SRW_data/develop/obs_data/ndas/proc DOMAIN_PREGEN_BASEDIR: /work/noaa/epic-ps/role-epic-ps/UFS_SRW_data/develop/FV3LAM_pregen PARTITION_DEFAULT: orion QUEUE_DEFAULT: batch diff --git a/ush/machine/wcoss2.yaml b/ush/machine/wcoss2.yaml index d9d67856e7..c9db727f3c 100644 --- a/ush/machine/wcoss2.yaml +++ b/ush/machine/wcoss2.yaml @@ -2,9 +2,9 @@ platform: WORKFLOW_MANAGER: rocoto NCORES_PER_NODE: 128 SCHED: pbspro - CCPA_OBS_DIR: /lfs/h2/emc/lam/noscrub/UFS_SRW_App/develop/obs_data/ccpa/proc - MRMS_OBS_DIR: /lfs/h2/emc/lam/noscrub/UFS_SRW_App/develop/obs_data/mrms/proc - NDAS_OBS_DIR: /lfs/h2/emc/lam/noscrub/UFS_SRW_App/develop/obs_data/ndas/proc + TEST_CCPA_OBS_DIR: /lfs/h2/emc/lam/noscrub/UFS_SRW_App/develop/obs_data/ccpa/proc + TEST_MRMS_OBS_DIR: /lfs/h2/emc/lam/noscrub/UFS_SRW_App/develop/obs_data/mrms/proc + TEST_NDAS_OBS_DIR: /lfs/h2/emc/lam/noscrub/UFS_SRW_App/develop/obs_data/ndas/proc METPLUS_PATH: /apps/ops/para/libs/intel/19.1.3.304/metplus/4.1.1 MET_ROOT: /apps/ops/para/libs/intel/19.1.3.304/met/10.1.1 DOMAIN_PREGEN_BASEDIR: /lfs/h2/emc/lam/noscrub/UFS_SRW_App/develop/FV3LAM_pregen diff --git a/ush/mrms_pull_topofhour.py b/ush/mrms_pull_topofhour.py index 7fc98f42c6..310c5d97f9 100644 --- a/ush/mrms_pull_topofhour.py +++ b/ush/mrms_pull_topofhour.py @@ -1,77 +1,63 @@ import sys, os, shutil, subprocess import datetime -import re, csv, glob +import glob +import argparse import bisect -import numpy as np +import shutil +import gzip + +def main(): + + #Parse input arguments + parser = argparse.ArgumentParser() + parser.add_argument('-v', '--valid_time', type=str, required=True, + help='Valid time (in string format YYYYMMDDHH) to find MRMS data for') + parser.add_argument('-o', '--outdir', type=str, required=True, + help='Destination directory for extracted MRMS data; data will be placed in `dest/YYYYMMDD`') + parser.add_argument('-s', '--source', type=str, required=True, + help='Source directory where zipped MRMS data is found') + parser.add_argument('-p', '--product', type=str, required=True, choices=['MergedReflectivityQCComposite', 'EchoTop'], + help='Name of MRMS product') + parser.add_argument('-l', '--level', type=str, help='MRMS product level', + choices=['_00.50_','_18_00.50_']) + parser.add_argument('-d', '--debug', action='store_true', help='Add additional debug output') + args = parser.parse_args() + + # Level is determined by MRMS product; set if not provided + if args.level is None: + if args.product == "MergedReflectivityQCComposite": + args.level = "_00.50_" + elif args.product == "EchoTop": + args.level = "_18_00.50_" + else: + raise Exception("This should never have happened") -if __name__ == "__main__": # Copy and unzip MRMS files that are closest to top of hour # Done every hour on a 20-minute lag - # Include option to define valid time on command line - # Used to backfill verification - # try: - valid_time = str(sys.argv[1]) - - YYYY = int(valid_time[0:4]) - MM = int(valid_time[4:6]) - DD = int(valid_time[6:8]) - HH = int(valid_time[8:19]) + YYYY = int(args.valid_time[0:4]) + MM = int(args.valid_time[4:6]) + DD = int(args.valid_time[6:8]) + HH = int(args.valid_time[8:19]) valid = datetime.datetime(YYYY, MM, DD, HH, 0, 0) + valid_str = valid.strftime("%Y%m%d") - # except IndexError: - # valid_time = None - - # Default to current hour if not defined on command line - # if valid_time is None: - # now = datetime.datetime.utcnow() - # YYYY = int(now.strftime('%Y')) - # MM = int(now.strftime('%m')) - # DD = int(now.strftime('%d')) - # HH = int(now.strftime('%H')) - - # valid = datetime.datetime(YYYY,MM,DD,HH,0,0) - # valid_time = valid.strftime('%Y%m%d%H') - - print("Pulling " + valid_time + " MRMS data") + print(f"Pulling {args.valid_time} MRMS data") # Set up working directory - DATA_HEAD = str(sys.argv[2]) - MRMS_PROD_DIR = str(sys.argv[3]) - MRMS_PRODUCT = str(sys.argv[4]) - level = str(sys.argv[5]) - VALID_DIR = os.path.join(DATA_HEAD, valid.strftime("%Y%m%d")) - if not os.path.exists(VALID_DIR): - os.makedirs(VALID_DIR) - os.chdir(DATA_HEAD) + dest_dir = os.path.join(args.outdir, valid_str) + if not os.path.exists(dest_dir): + os.makedirs(dest_dir) # Sort list of files for each MRMS product - print(valid.strftime("%Y%m%d")) - if valid.strftime("%Y%m%d") < "20200303": - search_path = ( - MRMS_PROD_DIR - + "/" - + valid.strftime("%Y%m%d") - + "/dcom/us007003/ldmdata/obs/upperair/mrms/conus/" - + MRMS_PRODUCT - + "/" - + MRMS_PRODUCT - + "*.gz" - ) - elif valid.strftime("%Y%m%d") >= "20200303": - search_path = ( - MRMS_PROD_DIR - + "/" - + valid.strftime("%Y%m%d") - + "/upperair/mrms/conus/" - + MRMS_PRODUCT - + "/" - + MRMS_PRODUCT - + "*.gz" - ) + if args.debug: + print(f"Valid date: {valid_str}") + search_path = f"{args.source}/{valid_str}/{args.product}*.gz" file_list = [f for f in glob.glob(search_path)] + if args.debug: + print(f"Files found: \n{file_list}") time_list = [file_list[x][-24:-9] for x in range(len(file_list))] int_list = [ int(time_list[x][0:8] + time_list[x][9:15]) for x in range(len(time_list)) @@ -88,76 +74,23 @@ ) # Check to make sure closest file is within +/- 15 mins of top of the hour - # Copy and rename the file for future ease difference = abs(closest_timestamp - valid) if difference.total_seconds() <= 900: - filename1 = ( - MRMS_PRODUCT - + level - + closest_timestamp.strftime("%Y%m%d-%H%M%S") - + ".grib2.gz" - ) - filename2 = MRMS_PRODUCT + level + valid.strftime("%Y%m%d-%H") + "0000.grib2.gz" - - if valid.strftime("%Y%m%d") < "20200303": - print( - "cp " - + MRMS_PROD_DIR - + "/" - + valid.strftime("%Y%m%d") - + "/dcom/us007003/ldmdata/obs/upperair/mrms/conus/" - + MRMS_PRODUCT - + "/" - + filename1 - + " " - + VALID_DIR - + "/" - + filename2 - ) - - os.system( - "cp " - + MRMS_PROD_DIR - + "/" - + valid.strftime("%Y%m%d") - + "/dcom/us007003/ldmdata/obs/upperair/mrms/conus/" - + MRMS_PRODUCT - + "/" - + filename1 - + " " - + VALID_DIR - + "/" - + filename2 - ) - os.system("gunzip " + VALID_DIR + "/" + filename2) - elif valid.strftime("%Y%m%d") >= "20200303": - print( - "cp " - + MRMS_PROD_DIR - + "/" - + valid.strftime("%Y%m%d") - + "/upperair/mrms/conus/" - + MRMS_PRODUCT - + "/" - + filename1 - + " " - + VALID_DIR - + "/" - + filename2 - ) - - os.system( - "cp " - + MRMS_PROD_DIR - + "/" - + valid.strftime("%Y%m%d") - + "/upperair/mrms/conus/" - + MRMS_PRODUCT - + "/" - + filename1 - + " " - + VALID_DIR - + "/" - + filename2 - ) - os.system("gunzip " + VALID_DIR + "/" + filename2) + filename1 = f"{args.product}{args.level}{closest_timestamp.strftime('%Y%m%d-%H%M%S')}.grib2.gz" + filename2 = f"{args.product}{args.level}{valid.strftime('%Y%m%d-%H')}0000.grib2" + origfile = os.path.join(args.source, valid_str, filename1) + target = os.path.join(dest_dir, filename2) + + if args.debug: + print(f"Unzipping file {origfile} to {target}") + + + # Unzip file to target location + with gzip.open(origfile, 'rb') as f_in: + with open(target, 'wb') as f_out: + shutil.copyfileobj(f_in, f_out) + else: + raise FileNotFoundError(f"Did not find a valid file within 15 minutes of {valid}") + +if __name__ == "__main__": + main() diff --git a/ush/retrieve_data.py b/ush/retrieve_data.py index 91a8e9223b..8f71e73d45 100755 --- a/ush/retrieve_data.py +++ b/ush/retrieve_data.py @@ -50,12 +50,14 @@ def clean_up_output_dir(expected_subdir, local_archive, output_path, source_path unavailable = {} expand_source_paths = [] + logging.debug(f"Cleaning up local paths: {source_paths}") for p in source_paths: - expand_source_paths.extend(glob.glob(p)) + expand_source_paths.extend(glob.glob(p.lstrip("/"))) # Check to make sure the files exist on disk for file_path in expand_source_paths: local_file_path = os.path.join(os.getcwd(), file_path.lstrip("/")) + logging.debug(f"Moving {local_file_path} to {output_path}") if not os.path.exists(local_file_path): logging.info(f"File does not exist: {local_file_path}") unavailable["hpss"] = expand_source_paths @@ -966,22 +968,6 @@ def parse_args(argv): ) parser.add_argument( "--data_type", - choices=( - "FV3GFS", - "GFS_obs", - "GDAS", - "GEFS", - "GSMGFS", - "HRRR", - "NAM", - "NSSL_mrms", - "RAP", - "RAPx", - "RAP_obs", - "HRRRx", - "GSI-FIX", - "UFS-CASE-STUDY" - ), help="External model label. This input is case-sensitive", required=True, )