diff --git a/dev/jobs/JGLOBAL_ATMOS_ANALYSIS b/dev/jobs/JGLOBAL_ATMOS_ANALYSIS index 88dc666d4b6..a28a0a91216 100755 --- a/dev/jobs/JGLOBAL_ATMOS_ANALYSIS +++ b/dev/jobs/JGLOBAL_ATMOS_ANALYSIS @@ -36,6 +36,7 @@ RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ MEMDIR='ensstat' RUN=${GDUMP_ENS} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ COMIN_ATMOS_HISTORY_ENS_PREV:COM_ATMOS_HISTORY_TMPL +# shellcheck disable=SC2153 mkdir -p "${COMOUT_ATMOS_ANALYSIS}" export ATMGES="${COMIN_ATMOS_HISTORY_PREV}/${GPREFIX}atm.f006.nc" @@ -45,6 +46,14 @@ if [[ ! -f "${ATMGES}" ]]; then err_exit "FILE MISSING: ATMGES == ${ATMGES}" fi +export GENDIAG="${GENDIAG:-YES}" + +# Create a DATAROOT directory variable for the gsidiags if GENDIAG is YES +if [[ "${GENDIAG}" == "YES" ]]; then + pCOMOUT_ATMOS_ANALYSIS="$(dataroot_com_path "${COMOUT_ATMOS_ANALYSIS}")" + export pCOMOUT_ATMOS_ANALYSIS +fi + # Get LEVS export LEVS LEVS=$(${NCLEN} "${ATMGES}" pfull) && true diff --git a/dev/jobs/JGLOBAL_ATMOS_ANALYSIS_DIAG b/dev/jobs/JGLOBAL_ATMOS_ANALYSIS_DIAG index 553a58e1160..0f7eabd9033 100755 --- a/dev/jobs/JGLOBAL_ATMOS_ANALYSIS_DIAG +++ b/dev/jobs/JGLOBAL_ATMOS_ANALYSIS_DIAG @@ -26,6 +26,11 @@ YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL mkdir -m 775 -p "${COMOUT_ATMOS_ANALYSIS}" +# Create a DATAROOT directory variable where the gsidiags live +# shellcheck disable=SC2153 +pCOMIN_ATMOS_ANALYSIS="$(dataroot_com_path "${COMIN_ATMOS_ANALYSIS}")" +export pCOMIN_ATMOS_ANALYSIS + ############################################################### # Run relevant script ${ANALDIAGSH:-${SCRgfs}/exglobal_diag.sh} && true diff --git a/dev/jobs/JGLOBAL_ENKF_DIAG b/dev/jobs/JGLOBAL_ENKF_DIAG index fa494dfb847..5e92c7ac56c 100755 --- a/dev/jobs/JGLOBAL_ENKF_DIAG +++ b/dev/jobs/JGLOBAL_ENKF_DIAG @@ -29,6 +29,11 @@ MEMDIR="ensstat" YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ COMIN_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL \ COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL +# Create a DATAROOT directory variable where the gsidiags live +# shellcheck disable=SC2153 +pCOMIN_ATMOS_ANALYSIS="$(dataroot_com_path "${COMIN_ATMOS_ANALYSIS}")" +export pCOMIN_ATMOS_ANALYSIS + #RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ # COMIN_OBS_PREV:COM_OBS_TMPL \ # COMIN_ATMOS_ANALYSIS_DET_PREV:COM_ATMOS_ANALYSIS_TMPL diff --git a/dev/jobs/JGLOBAL_ENKF_SELECT_OBS b/dev/jobs/JGLOBAL_ENKF_SELECT_OBS index 69ae44e23d8..35f10c50167 100755 --- a/dev/jobs/JGLOBAL_ENKF_SELECT_OBS +++ b/dev/jobs/JGLOBAL_ENKF_SELECT_OBS @@ -28,6 +28,7 @@ RUN=${RUN/enkf/} YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ COMIN_OBS:COM_OBS_TMPL MEMDIR='ensstat' YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL +# shellcheck disable=SC2153 declare -rx COMOUT_ATMOS_ANALYSIS_ENS="${COMOUT_ATMOS_ANALYSIS}" RUN=${RUN/enkf/} YMD=${PDY} HH=${cyc} declare_from_tmpl -r \ @@ -40,6 +41,7 @@ MEMDIR='ensstat' RUN=${GDUMP_ENS} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ RUN="${GDUMP}" YMD=${gPDY} HH=${gcyc} declare_from_tmpl -r \ COMIN_ATMOS_ANALYSIS_DET_PREV:COM_ATMOS_ANALYSIS_TMPL +# shellcheck disable=SC2153 mkdir -p "${COMOUT_ATMOS_ANALYSIS}" export ATMGES_ENSMEAN="${COMIN_ATMOS_HISTORY_PREV}/${GPREFIX}${GSUFFIX}atm.f006.nc" @@ -48,6 +50,14 @@ if [[ ! -f ${ATMGES_ENSMEAN} ]]; then err_exit "FILE MISSING: ATMGES_ENSMEAN == ${ATMGES_ENSMEAN}" fi +export GENDIAG="${GENDIAG:-YES}" + +# Create a DATAROOT directory variable for the gsidiags if GENDIAG is YES +if [[ "${GENDIAG}" == "YES" ]]; then + pCOMOUT_ATMOS_ANALYSIS="$(dataroot_com_path "${COMOUT_ATMOS_ANALYSIS}")" + export pCOMOUT_ATMOS_ANALYSIS +fi + LEVS=$(${NCLEN} "${ATMGES}" pfull) && true export err=$? set_strict diff --git a/dev/scripts/exglobal_atmos_analysis.sh b/dev/scripts/exglobal_atmos_analysis.sh index 5dcff0873ce..eb2d0e557ab 100755 --- a/dev/scripts/exglobal_atmos_analysis.sh +++ b/dev/scripts/exglobal_atmos_analysis.sh @@ -413,6 +413,16 @@ else echo "not using correlated obs error" fi +# If GENDIAG is selected, verify that pCOMOUT_ATMOS_ANALYSIS is set +if [[ "${GENDIAG}" == "YES" ]]; then + if [[ -z "${pCOMOUT_ATMOS_ANALYSIS}" ]]; then + export err=1 + err_exit "pCOMOUT_ATMOS_ANALYSIS must be set when GENDIAG=YES" + fi + # Make the gsidiags directory to house the GSI diagnostic data + GSIDIAGDIR=${GSIDIAGDIR:-"${pCOMOUT_ATMOS_ANALYSIS}/gsidiags"} +fi + ############################################################## # CRTM Spectral and Transmittance coefficients mkdir -p crtm_coeffs @@ -895,19 +905,32 @@ if [[ "${SENDECF}" == "YES" && "${RUN}" != "enkf" ]]; then ecflow_client --event release_fcst fi -# Diagnostic files -# if requested, GSI diagnostic file directories for use later +echo "${rCDUMP} ${PDY}${cyc} atminc done at $(date)" > "${COMOUT_ATMOS_ANALYSIS}/${APREFIX}increment.done.txt" + if [[ "${GENDIAG}" == "YES" ]]; then - tar -cvf gsidiags.tar dir.???? - export err=$? - if [[ ${err} -ne 0 ]]; then - err_exit "Failed to tar GSI diagnostic directories!" + # Move the gsidiags dir.* directories to pCOMOUT_ATMOS_ANALYSIS for diagnostic jobs + # First, check that the directories exist (we need at least one, so stop after the first match) + count_dirs=$(find . -maxdepth 1 -type d -name 'dir.????' -printf "." -quit | wc -c) + if [[ ${count_dirs:-0} -gt 0 ]]; then + mkdir -p "${GSIDIAGDIR}" + err=$? + + if [[ ! -d "${GSIDIAGDIR}" || ${err} -ne 0 ]]; then + err_exit "Failed to create gsidiags directory at ${GSIDIAGDIR}" + fi + + for dir in dir.????; do + mv "${dir}" "${GSIDIAGDIR}/" + export err=$? + if [[ ${err} -ne 0 ]]; then + err_exit "Failed to move ${dir} to ${GSIDIAGDIR}/" + fi + done + else + echo "WARNING: No gsidiags dir.* directories found to move." fi - cpfs gsidiags.tar "${COMOUT_ATMOS_ANALYSIS}/${APREFIX}gsidiags${DIAG_SUFFIX:-}.tar" fi -echo "${rCDUMP} ${PDY}${cyc} atminc done at $(date)" > "${COMOUT_ATMOS_ANALYSIS}/${APREFIX}increment.done.txt" - ################################################################################ exit "${err}" diff --git a/dev/scripts/exglobal_diag.sh b/dev/scripts/exglobal_diag.sh index 4d9aa5de94e..006515dc554 100755 --- a/dev/scripts/exglobal_diag.sh +++ b/dev/scripts/exglobal_diag.sh @@ -39,7 +39,7 @@ rm -f "${RADSTAT}" "${PCPSTAT}" "${CNVSTAT}" "${OZNSTAT}" # Obs diag GENDIAG=${GENDIAG:-"YES"} -GSIDIAG=${GSIDIAG:-"${COMIN_ATMOS_ANALYSIS}/${APREFIX}gsidiags${DIAG_SUFFIX:-}.tar"} +GSIDIAGDIR=${GSIDIAGDIR:-"${pCOMIN_ATMOS_ANALYSIS}/gsidiags"} USE_BUILD_GSINFO=${USE_BUILD_GSINFO:-"NO"} DIAG_COMPRESS=${DIAG_COMPRESS:-"YES"} if [[ "${DIAG_COMPRESS:-}" == "YES" ]]; then @@ -55,15 +55,30 @@ if [[ "${GENDIAG}" != "YES" ]]; then exit 0 fi +# Check that the gsidiags directory exists +if [[ ! -d "${GSIDIAGDIR}" ]]; then + export err=1 + err_exit "gsidiags directory ${GSIDIAGDIR} does not exist" +fi + ################################################################################ -# Copy gsidiags.tar file from COMIN to DATA and untar -cpreq "${GSIDIAG}" ./gsidiags.tar -tar -xvf gsidiags.tar -export err=$? -if [[ ${err} -ne 0 ]]; then - err_exit "Unable to unpack gsidiags.tar file!" +# Link to the gsidiags directory if it is populated +count_dirs=$(find "${GSIDIAGDIR}" -maxdepth 1 -type d -name "dir.*" | wc -l) +if [[ ${count_dirs} -eq 0 ]]; then + export err=1 + err_exit "No gsidiags directories found in ${GSIDIAGDIR}" fi +# Continue if there is at least one file to process +# Note -quit stops find after the first match +count_files=$(find "${GSIDIAGDIR}"/dir.* -maxdepth 1 -type f -printf '.' -quit | wc -c) +if [[ ${count_files} -eq 0 ]]; then + echo "WARNING: No diagnostic files found to process!" + exit 0 +fi + +${NLN} "${GSIDIAGDIR}/"dir.* . + # Set up lists and variables for various types of diagnostic files. ntype=3 @@ -153,7 +168,8 @@ for loop in ${loops}; do if [[ "${dtype}" == *"avhrr"* ]]; then # recast dtype in avhrr form dtype="avhrr${dtype##avhrr[23]}" fi - count=$(find ./ -path "./dir.*/${dtype}_${loop}*" -type f -printf "." | wc -c) + # -L to follow symlinks, as dir.* are symlinks to gsidiags/dir.* + count=$(find -L ./dir.* -path "./dir.*/${dtype}_${loop}*" -type f -printf "." | wc -c) if [[ ${count} -eq 0 ]]; then continue fi