[develop] Add informative error messages to all err_exit calls

dev/jobs/JGDAS_AERO_ANALYSIS_GENERATE_BMATRIX

@@ -41,7 +41,7 @@ EXSCRIPT=${GDASAEROBMATPY:-${SCRglobal}/exgdas_aero_analysis_generate_bmatrix.py
 ${EXSCRIPT} && true
 export err=$?
 if [[ ${err} -ne 0 ]]; then
-    err_exit
+    err_exit "Failed to generate bmatrix for aerosol analysis"
 fi
 
 ##############################################
@@ -61,7 +61,7 @@ fi
 cd "${DATAROOT}" && true
 export err=$?
 if [[ ${err} -ne 0 ]]; then
-    err_exit
+    err_exit "Failed to change directory to ${DATAROOT}"
 fi
 if [[ "${KEEPDATA}" == "NO" ]]; then
     rm -rf "${DATA}"

dev/jobs/JGDAS_ATMOS_CHGRES_FORENKF

@@ -36,7 +36,7 @@ declare -rx COMIN_ATMOS_HISTORY_MEM="${ROTDIR}/${RUN}.${PDY}/${cyc}/mem001/model
 ${CHGRESFCSTSH:-${SCRglobal}/exgdas_atmos_chgres_forenkf.sh} && true
 export err=$?
 if [[ ${err} -ne 0 ]]; then
-    err_exit
+    err_exit "Failed to run chgres for the ensemble"
 fi
 
 ##############################################

dev/jobs/JGDAS_ATMOS_GEMPAK

@@ -42,18 +42,18 @@ done
 
 export err=$?
 if [[ ${err} -ne 0 ]]; then
-    err_exit
+    err_exit "Failed to generate AWIPS products at 1 degree resolution for forecast hour ${FHR3}"
 fi
 
 ########################################################
-# Execute the script for quater-degree grib
+# Execute the script for quarter-degree grib
 ########################################################
 
 "${SCRglobal}/exgdas_atmos_nawips.sh" "0p25" "${FHR3}" "GDAS_GEMPAK" "${COMOUT_ATMOS_GEMPAK_0p25}" && true
 
 export err=$?
 if [[ ${err} -ne 0 ]]; then
-    err_exit
+    err_exit "Failed to generate AWIPS products at 0.25 degree resolution for forecast hour ${FHR3}"
 fi
 
 ############################################

dev/jobs/JGDAS_ATMOS_GEMPAK_META_NCDC

@@ -66,26 +66,27 @@ export pgmout=OUTPUT.$$
 "${HOMEglobal}/gempak/ush/gdas_meta_na.sh" && true
 export err=$?
 if [[ ${err} -ne 0 ]]; then
-    err_exit
+    err_exit "Failed to generate North American meta gempak products"
 fi
 
 "${HOMEglobal}/gempak/ush/gdas_meta_loop.sh" && true
 export err=$?
 if [[ ${err} -ne 0 ]]; then
-    err_exit
+    err_exit "Failed to generate looped image meta gempak products"
 fi
 
 if [[ "${cyc}" == '06' ]]; then
+    # TODO remove ECMWF and UKMET meta generation once the products are no longer needed
     "${HOMEglobal}/gempak/ush/gdas_ecmwf_meta_ver.sh" && true
     export err=$?
     if [[ ${err} -ne 0 ]]; then
-        err_exit
+        err_exit "Failed to generate ECMWF meta gempak products"
     fi
 
     "${HOMEglobal}/gempak/ush/gdas_ukmet_meta_ver.sh" && true
     export err=$?
     if [[ ${err} -ne 0 ]]; then
-        err_exit
+        err_exit "Failed to generate UKMET meta gempak products"
     fi
 fi
 ########################################################
@@ -101,7 +102,7 @@ if ((cyc % 12 == 0)); then
 fi
 export err=$?
 if [[ ${err} -ne 0 ]]; then
-    err_exit
+    err_exit "Failed to generate NCDC GIFs from GDAS GEMPAK products"
 fi
 ########################################################
 

dev/jobs/JGDAS_ATMOS_VERFOZN

@@ -42,7 +42,7 @@ if [[ ! -d ${TANKverf_ozn} ]]; then mkdir -p "${TANKverf_ozn}"; fi
 "${SCRglobal}/exgdas_atmos_verfozn.sh" && true
 export err=$?
 if [[ ${err} -ne 0 ]]; then
-    err_exit
+    err_exit "Failed to generate ozone monitor products"
 fi
 
 ################################

dev/jobs/JGDAS_ATMOS_VERFRAD

@@ -45,7 +45,7 @@ if [[ ! -d ${TANKverf_rad} ]]; then mkdir -p "${TANKverf_rad}"; fi
 "${SCRglobal}/exgdas_atmos_verfrad.sh" && true
 export err=$?
 if [[ ${err} -ne 0 ]]; then
-    err_exit
+    err_exit "Failed to generate radiance monitor products"
 fi
 
 ################################

dev/jobs/JGDAS_ENKF_POST

@@ -33,7 +33,7 @@ export LEVS=$((LEVS - 1))
 ${ENKFPOSTSH:-${SCRglobal}/exgdas_enkf_post.sh} && true
 export err=$?
 if [[ ${err} -ne 0 ]]; then
-    err_exit
+    err_exit "Failed to run ensemble post processing"
 fi
 
 ##############################################

dev/jobs/JGFS_ATMOS_AWIPS_20KM_1P0DEG

@@ -42,7 +42,7 @@ export pgmout=OUTPUT.$$
 "${SCRglobal}/exgfs_atmos_awips_20km_1p0deg.sh" "${fcsthr}" && true
 export err=$?
 if [[ ${err} -ne 0 ]]; then
-    err_exit
+    err_exit "Failed to generate AWIPS products at 20km resolution for forecast hour ${fcsthr}"
 fi
 ########################################################
 

dev/jobs/JGFS_ATMOS_CYCLONE_GENESIS

@@ -51,7 +51,7 @@ source "${USHglobal}/unset_strict.sh"
 export err=$?
 source "${USHglobal}/set_strict.sh"
 if [[ ${err} -ne 0 ]]; then
-    err_exit
+    err_exit "Failed to run the tropical cyclone genesis script"
 fi
 
 ##############################################

dev/jobs/JGFS_ATMOS_CYCLONE_TRACKER

@@ -60,14 +60,14 @@ export loopnum=1
 "${USHens_tracker}/data_check_gfs.sh"
 export err=$?
 if [[ ${err} -ne 0 ]]; then
-    err_exit
+    err_exit "Data check failed for GFS input data. Cannot run the tracker script."
 fi
 #------------------------------------------------
 
 "${USHens_tracker}/extrkr_gfs.sh" "${loopnum}" "${cmodel}" "${PDY}${cyc}" "${pert}" "${DATA}"
 export err=$?
 if [[ ${err} -ne 0 ]]; then
-    err_exit
+    err_exit "Tropical cyclone tracking failed"
 fi
 
 #  cat ${DATA}/trak.avnx.atcfunix.${PDY}${cyc} | \

dev/jobs/JGFS_ATMOS_FBWIND

@@ -36,7 +36,7 @@ fi
 "${SCRglobal}/exgfs_atmos_fbwind.sh" && true
 export err=$?
 if [[ ${err} -ne 0 ]]; then
-    err_exit
+    err_exit "Failed to generate Pacific WMO wind products"
 fi
 ########################################################
 

dev/jobs/JGFS_ATMOS_GEMPAK

@@ -56,7 +56,7 @@ if [[ "${fhr}" -ge "${fhmin}" ]] && [[ "${fhr}" -le "${fhmax}" ]]; then
         "${SCRglobal}/exgfs_atmos_nawips.sh" "1p00" "${FHR3}" "GFS_GEMPAK" "${COMOUT_ATMOS_GEMPAK_1p00}" && true
         export err=$?
         if [[ ${err} -ne 0 ]]; then
-            err_exit
+            err_exit "Failed to generate 1 degree GEMPAK products for forecast hour ${FHR3}"
         fi
     fi
 fi
@@ -68,7 +68,7 @@ if [[ "${fhr}" -ge "${fhmin}" ]] && [[ "${fhr}" -le "${fhmax}" ]]; then
         "${SCRglobal}/exgfs_atmos_nawips.sh" "1p00" "${FHR3}" "GFS_GEMPAK" "${COMOUT_ATMOS_GEMPAK_1p00}" && true
         export err=$?
         if [[ ${err} -ne 0 ]]; then
-            err_exit
+            err_exit "Failed to generate 1 degree GEMPAK products for forecast hour ${FHR3}"
         fi
     fi
 fi
@@ -83,7 +83,7 @@ if [[ "${fhr}" -ge "${fhmin}" ]] && [[ "${fhr}" -le "${fhmax}" ]]; then
         "${SCRglobal}/exgfs_atmos_nawips.sh" "0p50" "${FHR3}" "GFS_GEMPAK" "${COMOUT_ATMOS_GEMPAK_0p50}" && true
         export err=$?
         if [[ ${err} -ne 0 ]]; then
-            err_exit
+            err_exit "Failed to generate 0.5 degree GEMPAK products for forecast hour ${FHR3}"
         fi
     fi
 fi
@@ -95,7 +95,7 @@ if [[ "${fhr}" -ge "${fhmin}" ]] && [[ "${fhr}" -le "${fhmax}" ]]; then
         "${SCRglobal}/exgfs_atmos_nawips.sh" "0p50" "${FHR3}" "GFS_GEMPAK" "${COMOUT_ATMOS_GEMPAK_0p50}" && true
         export err=$?
         if [[ ${err} -ne 0 ]]; then
-            err_exit
+            err_exit "Failed to generate 0.5 degree GEMPAK products for forecast hour ${FHR3}"
         fi
     fi
 fi
@@ -107,7 +107,7 @@ if [[ "${fhr}" -ge "${fhmin}" ]] && [[ "${fhr}" -le "${fhmax}" ]]; then
         "${SCRglobal}/exgfs_atmos_nawips.sh" "0p50" "${FHR3}" "GFS_GEMPAK" "${COMOUT_ATMOS_GEMPAK_0p50}" && true
         export err=$?
         if [[ ${err} -ne 0 ]]; then
-            err_exit
+            err_exit "Failed to generate 0.5 degree GEMPAK products for forecast hour ${FHR3}"
         fi
     fi
 fi
@@ -122,7 +122,7 @@ if [[ "${fhr}" -ge "${fhmin}" ]] && [[ "${fhr}" -le "${fhmax}" ]]; then
         "${SCRglobal}/exgfs_atmos_nawips.sh" "0p25" "${FHR3}" "GFS_GEMPAK" "${COMOUT_ATMOS_GEMPAK_0p25}" && true
         export err=$?
         if [[ ${err} -ne 0 ]]; then
-            err_exit
+            err_exit "Failed to generate 0.25 degree GEMPAK products for forecast hour ${FHR3}"
         fi
     fi
 fi
@@ -134,7 +134,7 @@ if [[ "${fhr}" -ge "${fhmin}" ]] && [[ "${fhr}" -le "${fhmax}" ]]; then
         "${SCRglobal}/exgfs_atmos_nawips.sh" "0p25" "${FHR3}" "GFS_GEMPAK" "${COMOUT_ATMOS_GEMPAK_0p25}" && true
         export err=$?
         if [[ ${err} -ne 0 ]]; then
-            err_exit
+            err_exit "Failed to generate 0.25 degree GEMPAK products for forecast hour ${FHR3}"
         fi
     fi
 fi
@@ -146,7 +146,7 @@ if [[ "${fhr}" -ge "${fhmin}" ]] && [[ "${fhr}" -le "${fhmax}" ]]; then
         "${SCRglobal}/exgfs_atmos_nawips.sh" "0p25" "${FHR3}" "GFS_GEMPAK" "${COMOUT_ATMOS_GEMPAK_0p25}" && true
         export err=$?
         if [[ ${err} -ne 0 ]]; then
-            err_exit
+            err_exit "Failed to generate 0.25 degree GEMPAK products for forecast hour ${FHR3}"
         fi
     fi
 fi
@@ -161,19 +161,19 @@ if [[ "${fhr}" -ge "${fhmin}" ]] && [[ "${fhr}" -le "${fhmax}" ]]; then
         "${SCRglobal}/exgfs_atmos_nawips.sh" "35km_pac" "${FHR3}" "GFS_GEMPAK_WWB" "${COMOUT_ATMOS_GEMPAK_35km_pac}" && true
         export err=$?
         if [[ ${err} -ne 0 ]]; then
-            err_exit
+            err_exit "Failed to generate 35km Pacific GEMPAK products for forecast hour ${FHR3}"
         fi
 
         "${SCRglobal}/exgfs_atmos_nawips.sh" "35km_atl" "${FHR3}" "GFS_GEMPAK_WWB" "${COMOUT_ATMOS_GEMPAK_35km_atl}" && true
         export err=$?
         if [[ ${err} -ne 0 ]]; then
-            err_exit
+            err_exit "Failed to generate 35km Atlantic GEMPAK products for forecast hour ${FHR3}"
         fi
 
         "${SCRglobal}/exgfs_atmos_nawips.sh" "40km" "${FHR3}" "GFS_GEMPAK_WWB" "${COMOUT_ATMOS_GEMPAK_40km}" && true
         export err=$?
         if [[ ${err} -ne 0 ]]; then
-            err_exit
+            err_exit "Failed to generate 40km global ocean GEMPAK products for forecast hour ${FHR3}"
         fi
     fi
 fi

dev/jobs/JGFS_ATMOS_GEMPAK_META

@@ -65,7 +65,7 @@ mkdir -m 775 -p "${COMOUT_ATMOS_GEMPAK_META}"
 "${SCRglobal}/exgfs_atmos_gempak_meta.sh" && true
 export err=$?
 if [[ ${err} -ne 0 ]]; then
-    err_exit
+    err_exit "Failed to generate at least one GFS GEMPAK meta product"
 fi
 ########################################################
 

dev/jobs/JGFS_ATMOS_GEMPAK_NCDC_UPAPGIF

@@ -57,7 +57,7 @@ export pgmout=OUTPUT.$$
 "${SCRglobal}/exgfs_atmos_gempak_gif_ncdc_skew_t.sh"
 export err=$?
 if [[ ${err} -ne 0 ]]; then
-    err_exit
+    err_exit "Failed to generate NCDC skew-T GIFs from GFS GEMPAK products"
 fi
 ########################################################
 

dev/jobs/JGFS_ATMOS_GEMPAK_PGRB2_SPEC

@@ -58,7 +58,7 @@ export EXT=".grd221"
 
 export err=$?
 if [[ ${err} -ne 0 ]]; then
-    err_exit
+    err_exit "Failed to simulate GOES data from GFS GEMPAK products for forecast hour ${FHR3}"
 fi
 
 cd "${DATA}" || exit 1

dev/jobs/JGFS_ATMOS_PGRB2_SPEC_NPOESS

@@ -36,7 +36,7 @@ mkdir -m 775 -p "${COMOUT_ATMOS_GOES}"
 "${SCRglobal}/exgfs_atmos_grib2_special_npoess.sh" && true
 export err=$?
 if [[ ${err} -ne 0 ]]; then
-    err_exit
+    err_exit "Failed to simulate NPOESS data from GFS GRIB2 products"
 fi
 #############################################################
 

dev/jobs/JGFS_ATMOS_POSTSND

@@ -43,7 +43,7 @@ fi
 "${SCRglobal}/exgfs_atmos_postsnd.sh" && true
 export err=$?
 if [[ ${err} -ne 0 ]]; then
-    err_exit
+    err_exit "Failed to generate GFS BUFR soundings and/or associated WMO products"
 fi
 
 ##############################################

dev/jobs/JGLOBAL_AERO_ANALYSIS_FINALIZE

@@ -42,7 +42,7 @@ EXSCRIPT=${GDASAEROFINALPY:-${SCRglobal}/exglobal_aero_analysis_finalize.py}
 ${EXSCRIPT} && true
 export err=$?
 if [[ ${err} -ne 0 ]]; then
-    err_exit
+    err_exit "Failed to finalize aerosol analysis"
 fi
 
 ##############################################
@@ -62,7 +62,7 @@ fi
 cd "${DATAROOT}" && true
 export err=$?
 if [[ ${err} -ne 0 ]]; then
-    err_exit
+    err_exit "Failed to navigate to DATAROOT!"
 fi
 if [[ "${KEEPDATA}" == "NO" ]]; then
     rm -rf "${DATA}"

dev/jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE

@@ -52,7 +52,7 @@ EXSCRIPT=${GDASAEROINITPY:-${SCRglobal}/exglobal_aero_analysis_initialize.py}
 ${EXSCRIPT} && true
 export err=$?
 if [[ ${err} -ne 0 ]]; then
-    err_exit
+    err_exit "Failed to initialize aerosol analysis"
 fi
 
 ##############################################

dev/jobs/JGLOBAL_AERO_ANALYSIS_VARIATIONAL

@@ -34,7 +34,7 @@ EXSCRIPT=${GDASAEROVARSH:-${SCRglobal}/exglobal_aero_analysis_variational.py}
 ${EXSCRIPT} && true
 export err=$?
 if [[ ${err} -ne 0 ]]; then
-    err_exit
+    err_exit "Failed to run variational aerosol analysis"
 fi
 
 ##############################################

dev/jobs/JGLOBAL_ANALYSIS_STATS

@@ -58,7 +58,7 @@ EXSCRIPT=${ANLSTATSPY:-${SCRglobal}/exglobal_analysis_stats.py}
 ${EXSCRIPT} && true
 export err=$?
 if [[ ${err} -ne 0 ]]; then
-    err_exit
+    err_exit "Failed to generate analysis statistics for ${RUN} ${PDY}${cyc}"
 fi
 
 ##############################################

dev/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE

@@ -42,7 +42,7 @@ EXSCRIPT=${GDASATMENSFINALPY:-${SCRglobal}/exglobal_atmens_analysis_finalize.py}
 ${EXSCRIPT} && true
 export err=$?
 if [[ ${err} -ne 0 ]]; then
-    err_exit
+    err_exit "Failed to finalize the atmospheric ensemble analysis"
 fi
 
 ##############################################

dev/jobs/JGLOBAL_ATMENS_ANALYSIS_FV3_INCREMENT

@@ -34,7 +34,7 @@ EXSCRIPT=${GDASATMENSRUNSH:-${SCRglobal}/exglobal_atmens_analysis_fv3_increment.
 ${EXSCRIPT} && true
 export err=$?
 if [[ ${err} -ne 0 ]]; then
-    err_exit
+    err_exit "Failed to generate the atmospheric ensemble analysis FV3 increment"
 fi
 
 ##############################################

dev/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE

@@ -47,7 +47,7 @@ EXSCRIPT=${GDASATMENSINITPY:-${SCRglobal}/exglobal_atmens_analysis_initialize.py
 ${EXSCRIPT} && true
 export err=$?
 if [[ ${err} -ne 0 ]]; then
-    err_exit
+    err_exit "Failed to initialize the atmospheric ensemble analysis"
 fi
 
 ##############################################

dev/jobs/JGLOBAL_ATMENS_ANALYSIS_LETKF

@@ -33,7 +33,7 @@ EXSCRIPT=${GDASATMENSRUNSH:-${SCRglobal}/exglobal_atmens_analysis_letkf.py}
 ${EXSCRIPT} && true
 export err=$?
 if [[ ${err} -ne 0 ]]; then
-    err_exit
+    err_exit "Failed to run the atmospheric ensemble LETKF analysis"
 fi
 
 ##############################################

dev/jobs/JGLOBAL_ATMENS_ANALYSIS_OBS

@@ -33,7 +33,7 @@ EXSCRIPT=${GDASATMENSOBSSH:-${SCRglobal}/exglobal_atmens_analysis_obs.py}
 ${EXSCRIPT} && true
 export err=$?
 if [[ ${err} -ne 0 ]]; then
-    err_exit
+    err_exit "Failed to complete the observer for the atmospheric ensemble analysis"
 fi
 
 ##############################################

dev/jobs/JGLOBAL_ATMENS_ANALYSIS_SOL

@@ -34,7 +34,7 @@ EXSCRIPT=${GDASATMENSSOLSH:-${SCRglobal}/exglobal_atmens_analysis_sol.py}
 ${EXSCRIPT} && true
 export err=$?
 if [[ ${err} -ne 0 ]]; then
-    err_exit
+    err_exit "Failed to complete the atmospheric ensemble analysis solver"
 fi
 
 ##############################################