diff --git a/dev/ecf/README.md b/dev/ecf/README.md index 3eebdbf..507c4ce 100644 --- a/dev/ecf/README.md +++ b/dev/ecf/README.md @@ -13,7 +13,7 @@ This will create a suite definition file called `wafs.def` in the `ecf/def` dire ## Starting `ecflow_server` `ecflow_server` can only be started on dedicated ecflow server nodes. On WCOSS2, the ecflow server nodes for development are: - `cdecflow01`, `cdecflow02` (cactus) -- `ddcflow01`, `ddcflow02` (dogwood) +- `ddecflow01`, `ddecflow02` (dogwood) `ssh` to one of the above ecflow server nodes (e.g. `cdecflow01`). ```bash @@ -53,7 +53,7 @@ export ECF_HOST="cdecflow01" # This is the hostname on which the `ecflow_server Check to ensure the `ecflow_client` can ping the `ecflow_server`: ```bash -`ecflow_client --ping` +ecflow_client --ping ``` If this is successful, one can launch the `ecflow_ui` and place it in the background. @@ -74,5 +74,18 @@ To begin running the suite, use the following command: ecflow_client --begin wafs ``` +## Some useful commands +`ecflow_client --stats` To check the status of ecflow_server: HALTED/RUNNING + +`ecflow_client --restart` To start ecflow_server if it is HALTED + +`ecflow_client --delete=/yoursuite` To delete a suite which was previously loaded + +`ecflow_stop.sh -p NNNNN` To stop ecflow_server, where NNNNN is a port number assigned to the ecflow_server + +`qstat -u $USER` To check any job is running, including triggered by a loaded suite + +`ecflow_client --replace=/yoursuite1 $PWD/yoursuite2.def` To load a new suite if not loaded, otherwise replace the suite + ## Additional Resources For more information on using ecflow, refer to the official [documentation](https://ecflow.readthedocs.io/en/latest/overview.html) diff --git a/dev/ecf/run.ecflow_pdy.sh b/dev/ecf/run.ecflow_pdy.sh new file mode 100755 index 0000000..236a531 --- /dev/null +++ b/dev/ecf/run.ecflow_pdy.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +# Instructions: https://github.com/NOAA-EMC/WAFS/tree/release/wafs.v7/dev/ecf +# +# Two requirements to run this script +# 1. ecflow_server has been started +# 2. ecflow_server is started on either cdecflow01 or ddecflow01 +# +# What the script will do for a PDYcyc: +# 1. Follow the instructions from dev/ecf/setup_ecf.sh and creates a suite definition file for a PDYcyc +# 2. Load the suite def file to the ecflow_server which was started already +# 3. Begin ecflow for a PDYcyc + +set -eu + +echo "Usage: ./setup_ecf.sh [PDYcyc]" + +PDYcyc=${1:-$(date --utc "+%Y%m%d")00} + +# Get the root of the script +readonly DIR_ROOT=$(cd "$(dirname "$(readlink -f -n "${BASH_SOURCE[0]}")")/../.." && pwd -P) +cd $DIR_ROOT/dev/ecf +# create a suite def file in real time +./setup_ecf.sh -d $PDYcyc -x xPDY + +suitename="wafsxPDY" + +# set ECF_HOST according to which WCOSS2 machine +if [[ $(hostname) =~ ^[d][login|dxfer] ]] ; then + export ECF_HOST="ddecflow01" +elif [[ $(hostname) =~ ^[c][login|dxfer] ]] ; then + export ECF_HOST="cdecflow01" +fi +echo $ECF_HOST +module load ecflow + +# Make sure ecflow_server is not halted +ecflow_client --restart + +cd $DIR_ROOT/ecf/def + +# echo "yes" | ecflow_client --delete=/$suitename +# ecflow_client --load $PWD/$suitename.def + +# Replace: it can either load a new suite def or +# replace with a new suite def UNLESS a job is active for the current suite +ecflow_client --replace=/$suitename $PWD/$suitename.def + +ecflow_client --begin $suitename diff --git a/docs/Release_Notes.md b/docs/Release_Notes.md index 8c496f4..796284f 100644 --- a/docs/Release_Notes.md +++ b/docs/Release_Notes.md @@ -85,6 +85,7 @@ Additionally there are other changes: 2. In exwafs_grib2_0p25.sh, only include fields at the extra levels when forecast hour is between 06 and 36 per AWC request 3. In ush/mkwfsgbl.sh: change input dependency from GFS pgrb2.1p00 to master file. 4. Remove files under ush/ folder: wafs_blending.sh wafs_grib2.regrid.sh wafs_intdsk.sh +5. In exwafs_grib2_0p25_blending.sh, remove the condition of sending UK unblended data if US unblended data is missing. It won't happen because the job itself won't get triggered if US unblended data is missing Fix Changes ----------- @@ -146,6 +147,8 @@ Product Changes * Files to be retired * `gfs.tCCz.wafs_icao.grb2fFFF` * wafs.tCCz.master.fFFF.grib2 where FFF is from 001 to 005 +* File changes + * For WAFS blending when UK data is missing at multiple forecast hours, multiple files wafs.tCCz.fFFF.wafs_blend_0p25_usonly.emailbody for each forecast hour will replace one single file gfs.tCCz.wafs_blend_0p25_usonly.emailbody for the whole cycle. * Filename changes * Renamed according to EE2 implementation standards * Exceptions: files sent to UK keep the original names except forecast hour is changed to 3 digits diff --git a/scripts/exwafs_grib2_0p25_blending.sh b/scripts/exwafs_grib2_0p25_blending.sh index 440af58..4df8173 100755 --- a/scripts/exwafs_grib2_0p25_blending.sh +++ b/scripts/exwafs_grib2_0p25_blending.sh @@ -19,25 +19,16 @@ # - Parallel run for each forecast hour, changed from sequential run. # - Fix bugzilla 1593: Improve email notification for missing UK WAFS data. # - Extend waiting time window from 15 to 25 minutes +# - usonly.emailbody is differentiated for each forecast hour with missing UK data. +# - Remove the condition of sending UK unblended data if US unblended data is missing. It won't happen because the job itself won't get triggered if US unblended data is missing. ################################################################################ set -x cd "${DATA}" || err_exit "FATAL ERROR: Could not 'cd ${DATA}'; ABORT!" -SEND_US_WAFS="NO" -SEND_UK_WAFS="NO" -########################## -# look for US WAFS data -########################## -if [[ ! -s "${COMINus}/WAFS_0p25_unblended_${PDY}${cyc}f${fhr}.grib2" ]]; then - echo "Not found US WAFS GRIB2 file '${COMINus}/WAFS_0p25_unblended_${PDY}${cyc}f${fhr}.grib2', continue ..." - SEND_UK_WAFS="YES" -fi +SEND_UNBLENDED_US_WAFS="NO" -########################## -# look for UK WAFS data. -########################## ############################################### # Specify Timeout Behavior for WAFS blending ############################################### @@ -47,6 +38,9 @@ SLEEP_TIME=${SLEEP_TIME:-1500} SLEEP_INT=${SLEEP_INT:-10} SLEEP_LOOP_MAX=$((SLEEP_TIME / SLEEP_INT)) +########################## +# look for UK WAFS data. +########################## for ((ic = 1; ic <= SLEEP_LOOP_MAX; ic++)); do # Three(3) unblended UK files for each cycle+fhour: icing, turb, cb ukfiles=$(find "${COMINuk}" -name "egrr_wafshzds_unblended_*_0p25_${PDY:0:4}-${PDY:4:2}-${PDY:6:2}T${cyc}:00Z_t${fhr}.grib2" | wc -l) @@ -61,38 +55,23 @@ for ((ic = 1; ic <= SLEEP_LOOP_MAX; ic++)); do ukfile="egrr_wafshzds_unblended_${prod}_0p25_${PDY:0:4}-${PDY:4:2}-${PDY:6:2}T${cyc}:00Z_t${fhr}.grib2" if [[ ! -f "${COMINuk}/${ukfile}" ]]; then echo "WARNING: UK WAFS GRIB2 file '${ukfile}' not found after waiting over ${SLEEP_TIME} seconds" - echo "${ukfile}" >>missing_uk_files + echo "${COMINuk}/${ukfile}" >>missing_uk_files fi done echo "WARNING: UK WAFS GRIB2 unblended data is not completely available, no blending" - SEND_US_WAFS="YES" + SEND_UNBLENDED_US_WAFS="YES" break else sleep "${SLEEP_INT}" fi done -########################## -# If both UK and US data are missing. -########################## -if [[ "${SEND_US_WAFS}" == "YES" ]] && [[ "${SEND_UK_WAFS}" == "YES" ]]; then - SEND_US_WAFS=NO - SEND_UK_WAFS=NO - export err=1 - err_exit "FATAL ERROR: Both US and UK data are missing, no blended products for '${PDY}${cyc}f${fhr}'" -fi - ########################## # Blending or unblended ########################## -if [[ "${SEND_US_WAFS}" == "YES" ]]; then +if [[ "${SEND_UNBLENDED_US_WAFS}" == "YES" ]]; then echo "turning back on dbn alert for unblended US WAFS product" -elif [[ "${SEND_UK_WAFS}" == "YES" ]]; then - echo "turning back on dbn alert for unblended UK WAFS product" - # retrieve UK products - # Three(3) unblended UK files for each cycle+fhour: icing, turb, cb - cat "${COMINuk}/egrr_wafshzds_unblended_"*"_0p25_${PDY:0:4}-${PDY:4:2}-${PDY:6:2}T${cyc}:00Z_t${fhr}.grib2" >"EGRR_WAFS_0p25_unblended_${PDY}_${cyc}z_t${fhr}.grib2" else # retrieve UK products # Three(3) unblended UK files for each cycle+fhour: icing, turb, cb @@ -112,7 +91,7 @@ else err=$? if ((err != 0)); then echo "WARNING: WAFS blending 0p25 program failed at '${PDY}${cyc}f${fhr}'. Turning back on dbn alert for unblended US WAFS product" - SEND_US_WAFS="YES" + SEND_UNBLENDED_US_WAFS="YES" fi fi @@ -124,9 +103,9 @@ fi if [[ "${envir}" != "prod" ]]; then maillist="nco.spa@noaa.gov" fi -maillist=${maillist:-"nco.spa@noaa.gov,ncep.sos@noaa.gov"} +maillist=${maillist:-"nco.spa@noaa.gov,ncep.sos@noaa.gov,nco.hpc.dataflow@noaa.gov"} -if [[ "${SEND_US_WAFS}" == "YES" ]]; then +if [[ "${SEND_UNBLENDED_US_WAFS}" == "YES" ]]; then # checking any US WAFS product was sent due to No UK WAFS GRIB2 file or WAFS blending program # (Alert once for each forecast hour) @@ -139,19 +118,21 @@ if [[ "${SEND_US_WAFS}" == "YES" ]]; then "${DBNROOT}/bin/dbn_alert" NTC_LOW WAFS "${job}" "${COMOUTwmo}/iscs_0p25_admin_msg" fi - subject="WARNING! Missing UK data for WAFS GRIB2 0P25 blending, ${PDY} t${cyc}z f${fhr} ${job}" - echo "*************************************************************" >mailmsg - echo "*** WARNING! Missing UK data for WAFS GRIB2 0P25 blending ***" >>mailmsg - echo "*************************************************************" >>mailmsg - echo "Missing data at ${COMINuk}:" >>mailmsg - cat missing_uk_files >>mailmsg - echo >>mailmsg - echo "Send alert message to AWC ...... " >>mailmsg - echo >>mailmsg - cat mailmsg >"${COMOUT}/${RUN}.t${cyc}z.f${fhr}.wafs_blend_0p25_usonly.emailbody" - cat "${COMOUT}/${RUN}.t${cyc}z.f${fhr}.wafs_blend_0p25_usonly.emailbody" | mail.py -s "${subject}" "${maillist}" -v fi + subject="WARNING! Missing UK data for WAFS GRIB2 0P25 blending, ${PDY} t${cyc}z f${fhr} ${job}" + echo "*************************************************************" >mailmsg + echo "*** WARNING! Missing UK data for WAFS GRIB2 0P25 blending ***" >>mailmsg + echo "*************************************************************" >>mailmsg + echo >>mailmsg + echo "Send alert message to AWC ...... " >>mailmsg + echo >>mailmsg + echo "One or more UK WAFS GRIB2 files not found:" >>mailmsg + cat missing_uk_files >>mailmsg + echo "Skipping t${cyc}z f${fhr}..." >>mailmsg + cat mailmsg >"${COMOUT}/${RUN}.t${cyc}z.f${fhr}.wafs_blend_0p25_usonly.emailbody" + cat "${COMOUT}/${RUN}.t${cyc}z.f${fhr}.wafs_blend_0p25_usonly.emailbody" | mail.py -s "${subject}" "${maillist}" -v + # Distribute US WAFS unblend Data to NCEP FTP Server (WOC) and TOC echo "altering the unblended US WAFS products:" echo " - ${COMINus}/WAFS_0p25_unblended_${PDY}${cyc}f${fhr}.grib2" @@ -162,39 +143,6 @@ if [[ "${SEND_US_WAFS}" == "YES" ]]; then "${DBNROOT}/bin/dbn_alert" MODEL WAFS_0P25_UBL_GB2_WIDX "${job}" "${COMINus}/WAFS_0p25_unblended_${PDY}${cyc}f${fhr}.grib2.idx" fi -elif [[ "${SEND_UK_WAFS}" == "YES" ]]; then - - # checking any UK WAFS product was sent due to No US WAFS GRIB2 file - # (Alert once for each forecast hour) - if [[ ! -f ${COMOUTwmo}/wifs_0p25_admin_msg ]]; then - echo "WARNING: Missing US data for WAFS GRIB2 0P25 blending. Send alert message to AWC ......" - make_NTC_file.pl NOXX10 KKCI "${PDY}${cyc}" NONE "${FIXwafs}/wafs/wafs_blending_0p25_admin_msg" "${COMOUTwmo}/wifs_0p25_admin_msg" - make_NTC_file.pl NOXX10 KWBC "${PDY}${cyc}" NONE "${FIXwafs}/wafs/wafs_blending_0p25_admin_msg" "${COMOUTwmo}/iscs_0p25_admin_msg" - if [[ "${SENDDBN_NTC}" == "YES" ]]; then - "${DBNROOT}/bin/dbn_alert" NTC_LOW WAFS "${job}" "${COMOUTwmo}/wifs_0p25_admin_msg" - "${DBNROOT}/bin/dbn_alert" NTC_LOW WAFS "${job}" "${COMOUTwmo}/iscs_0p25_admin_msg" - fi - - export subject="WARNING! Missing US data for WAFS GRIB2 0P25 blending, ${PDY} t${cyc}z ${fhr} ${job}" - echo "*************************************************************" >mailmsg - echo "*** WARNING! Missing US data for WAFS GRIB2 0P25 blending ***" >>mailmsg - echo "*************************************************************" >>mailmsg - echo "Missing data at ${COMINus}:" >>mailmsg - echo "WAFS_0p25_unblended_${PDY}${cyc}f${fhr}.grib2" >>mailmsg - echo >>mailmsg - echo "Send alert message to AWC ...... " >>mailmsg - echo >>mailmsg - cat mailmsg >"${COMOUT}/${RUN}.t${cyc}z.f${fhr}.wafs_blend_0p25_ukonly.emailbody" - cat "${COMOUT}/${RUN}.t${cyc}z.f${fhr}.wafs_blend_0p25_ukonly.emailbody" | mail.py -s "${subject}" "${maillist}" -v - - fi - # Distribute UK WAFS unblend Data to NCEP FTP Server (WOC) and TOC - echo "altering the unblended UK WAFS products - EGRR_WAFS_0p25_unblended_${PDY}_${cyc}z_t${fhr}.grib2" - - if [[ "${SENDDBN}" == "YES" ]]; then - "${DBNROOT}/bin/dbn_alert" MODEL WAFS_UKMET_0P25_UBL_GB2 "${job}" "EGRR_WAFS_0p25_unblended_${PDY}_${cyc}z_t${fhr}.grib2" - fi - else # Distribute US WAFS unblend Data to NCEP FTP Server (WOC) and TOC if [[ "${SENDCOM}" == "YES" ]]; then