Rework gsi diagnostic file handling through DATAROOT storage

dev/jobs/JGLOBAL_ATMOS_ANALYSIS

@@ -36,6 +36,7 @@ RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \
 MEMDIR='ensstat' RUN=${GDUMP_ENS} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \
     COMIN_ATMOS_HISTORY_ENS_PREV:COM_ATMOS_HISTORY_TMPL
 
+# shellcheck disable=SC2153
 mkdir -p "${COMOUT_ATMOS_ANALYSIS}"
 
 export ATMGES="${COMIN_ATMOS_HISTORY_PREV}/${GPREFIX}atm.f006.nc"
@@ -45,6 +46,14 @@ if [[ ! -f "${ATMGES}" ]]; then
     err_exit "FILE MISSING: ATMGES == ${ATMGES}"
 fi
 
+export GENDIAG="${GENDIAG:-YES}"
+
+# Create a DATAROOT directory variable for the gsidiags if GENDIAG is YES
+if [[ "${GENDIAG}" == "YES" ]]; then
+    pCOMOUT_ATMOS_ANALYSIS="$(dataroot_com_path "${COMOUT_ATMOS_ANALYSIS}")"
+    export pCOMOUT_ATMOS_ANALYSIS
+fi
+
 # Get LEVS
 export LEVS
 LEVS=$(${NCLEN} "${ATMGES}" pfull) && true

dev/jobs/JGLOBAL_ATMOS_ANALYSIS_DIAG

@@ -26,6 +26,11 @@ YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
     COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL
 mkdir -m 775 -p "${COMOUT_ATMOS_ANALYSIS}"
 
+# Create a DATAROOT directory variable where the gsidiags live
+# shellcheck disable=SC2153
+pCOMIN_ATMOS_ANALYSIS="$(dataroot_com_path "${COMIN_ATMOS_ANALYSIS}")"
+export pCOMIN_ATMOS_ANALYSIS
+
 ###############################################################
 # Run relevant script
 ${ANALDIAGSH:-${SCRgfs}/exglobal_diag.sh} && true

dev/jobs/JGLOBAL_ENKF_DIAG

@@ -29,6 +29,11 @@ MEMDIR="ensstat" YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
     COMIN_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL \
     COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL
 
+# Create a DATAROOT directory variable where the gsidiags live
+# shellcheck disable=SC2153
+pCOMIN_ATMOS_ANALYSIS="$(dataroot_com_path "${COMIN_ATMOS_ANALYSIS}")"
+export pCOMIN_ATMOS_ANALYSIS
+
 #RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \
 #    COMIN_OBS_PREV:COM_OBS_TMPL \
 #    COMIN_ATMOS_ANALYSIS_DET_PREV:COM_ATMOS_ANALYSIS_TMPL

dev/jobs/JGLOBAL_ENKF_SELECT_OBS

@@ -28,6 +28,7 @@ RUN=${RUN/enkf/} YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
     COMIN_OBS:COM_OBS_TMPL
 MEMDIR='ensstat' YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
     COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL
+# shellcheck disable=SC2153
 declare -rx COMOUT_ATMOS_ANALYSIS_ENS="${COMOUT_ATMOS_ANALYSIS}"
 
 RUN=${RUN/enkf/} YMD=${PDY} HH=${cyc} declare_from_tmpl -r \
@@ -40,6 +41,7 @@ MEMDIR='ensstat' RUN=${GDUMP_ENS} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \
 RUN="${GDUMP}" YMD=${gPDY} HH=${gcyc} declare_from_tmpl -r \
     COMIN_ATMOS_ANALYSIS_DET_PREV:COM_ATMOS_ANALYSIS_TMPL
 
+# shellcheck disable=SC2153
 mkdir -p "${COMOUT_ATMOS_ANALYSIS}"
 
 export ATMGES_ENSMEAN="${COMIN_ATMOS_HISTORY_PREV}/${GPREFIX}${GSUFFIX}atm.f006.nc"
@@ -48,6 +50,14 @@ if [[ ! -f ${ATMGES_ENSMEAN} ]]; then
     err_exit "FILE MISSING: ATMGES_ENSMEAN == ${ATMGES_ENSMEAN}"
 fi
 
+export GENDIAG="${GENDIAG:-YES}"
+
+# Create a DATAROOT directory variable for the gsidiags if GENDIAG is YES
+if [[ "${GENDIAG}" == "YES" ]]; then
+    pCOMOUT_ATMOS_ANALYSIS="$(dataroot_com_path "${COMOUT_ATMOS_ANALYSIS}")"
+    export pCOMOUT_ATMOS_ANALYSIS
+fi
+
 LEVS=$(${NCLEN} "${ATMGES}" pfull) && true
 export err=$?
 set_strict

dev/scripts/exglobal_atmos_analysis.sh

@@ -413,6 +413,16 @@ else
     echo "not using correlated obs error"
 fi
 
+# If GENDIAG is selected, verify that pCOMOUT_ATMOS_ANALYSIS is set
+if [[ "${GENDIAG}" == "YES" ]]; then
+    if [[ -z "${pCOMOUT_ATMOS_ANALYSIS}" ]]; then
+        export err=1
+        err_exit "pCOMOUT_ATMOS_ANALYSIS must be set when GENDIAG=YES"
+    fi
+    # Make the gsidiags directory to house the GSI diagnostic data
+    GSIDIAGDIR=${GSIDIAGDIR:-"${pCOMOUT_ATMOS_ANALYSIS}/gsidiags"}
+fi
+
 ##############################################################
 # CRTM Spectral and Transmittance coefficients
 mkdir -p crtm_coeffs
@@ -895,19 +905,32 @@ if [[ "${SENDECF}" == "YES" && "${RUN}" != "enkf" ]]; then
     ecflow_client --event release_fcst
 fi
 
-# Diagnostic files
-# if requested, GSI diagnostic file directories for use later
+echo "${rCDUMP} ${PDY}${cyc} atminc done at $(date)" > "${COMOUT_ATMOS_ANALYSIS}/${APREFIX}increment.done.txt"
+
 if [[ "${GENDIAG}" == "YES" ]]; then
-    tar -cvf gsidiags.tar dir.????
-    export err=$?
-    if [[ ${err} -ne 0 ]]; then
-        err_exit "Failed to tar GSI diagnostic directories!"
+    # Move the gsidiags dir.* directories to pCOMOUT_ATMOS_ANALYSIS for diagnostic jobs
+    # First, check that the directories exist (we need at least one, so stop after the first match)
+    count_dirs=$(find . -maxdepth 1 -type d -name 'dir.????' -printf "." -quit | wc -c)
+    if [[ ${count_dirs:-0} -gt 0 ]]; then
+        mkdir -p "${GSIDIAGDIR}"
+        err=$?
+
+        if [[ ! -d "${GSIDIAGDIR}" || ${err} -ne 0 ]]; then
+            err_exit "Failed to create gsidiags directory at ${GSIDIAGDIR}"
+        fi
+
+        for dir in dir.????; do
+            mv "${dir}" "${GSIDIAGDIR}/"
+            export err=$?
+            if [[ ${err} -ne 0 ]]; then
+                err_exit "Failed to move ${dir} to ${GSIDIAGDIR}/"
+            fi
+        done
+    else
+        echo "WARNING: No gsidiags dir.* directories found to move."
     fi
-    cpfs gsidiags.tar "${COMOUT_ATMOS_ANALYSIS}/${APREFIX}gsidiags${DIAG_SUFFIX:-}.tar"
 fi
 
-echo "${rCDUMP} ${PDY}${cyc} atminc done at $(date)" > "${COMOUT_ATMOS_ANALYSIS}/${APREFIX}increment.done.txt"
-
 ################################################################################
 
 exit "${err}"

dev/scripts/exglobal_diag.sh

@@ -39,7 +39,7 @@ rm -f "${RADSTAT}" "${PCPSTAT}" "${CNVSTAT}" "${OZNSTAT}"
 
 # Obs diag
 GENDIAG=${GENDIAG:-"YES"}
-GSIDIAG=${GSIDIAG:-"${COMIN_ATMOS_ANALYSIS}/${APREFIX}gsidiags${DIAG_SUFFIX:-}.tar"}
+GSIDIAGDIR=${GSIDIAGDIR:-"${pCOMIN_ATMOS_ANALYSIS}/gsidiags"}
 USE_BUILD_GSINFO=${USE_BUILD_GSINFO:-"NO"}
 DIAG_COMPRESS=${DIAG_COMPRESS:-"YES"}
 if [[ "${DIAG_COMPRESS:-}" == "YES" ]]; then
@@ -55,15 +55,30 @@ if [[ "${GENDIAG}" != "YES" ]]; then
     exit 0
 fi
 
+# Check that the gsidiags directory exists
+if [[ ! -d "${GSIDIAGDIR}" ]]; then
+    export err=1
+    err_exit "gsidiags directory ${GSIDIAGDIR} does not exist"
+fi
+
 ################################################################################
-# Copy gsidiags.tar file from COMIN to DATA and untar
-cpreq "${GSIDIAG}" ./gsidiags.tar
-tar -xvf gsidiags.tar
-export err=$?
-if [[ ${err} -ne 0 ]]; then
-    err_exit "Unable to unpack gsidiags.tar file!"
+# Link to the gsidiags directory if it is populated
+count_dirs=$(find "${GSIDIAGDIR}" -maxdepth 1 -type d -name "dir.*" | wc -l)
+if [[ ${count_dirs} -eq 0 ]]; then
+    export err=1
+    err_exit "No gsidiags directories found in ${GSIDIAGDIR}"
 fi
 
+# Continue if there is at least one file to process
+# Note -quit stops find after the first match
+count_files=$(find "${GSIDIAGDIR}"/dir.* -maxdepth 1 -type f -printf '.' -quit | wc -c)
+if [[ ${count_files} -eq 0 ]]; then
+    echo "WARNING: No diagnostic files found to process!"
+    exit 0
+fi
+
+${NLN} "${GSIDIAGDIR}/"dir.* .
+
 # Set up lists and variables for various types of diagnostic files.
 ntype=3
 
@@ -153,7 +168,8 @@ for loop in ${loops}; do
             if [[ "${dtype}" == *"avhrr"* ]]; then # recast dtype in avhrr form
                 dtype="avhrr${dtype##avhrr[23]}"
             fi
-            count=$(find ./ -path "./dir.*/${dtype}_${loop}*" -type f -printf "." | wc -c)
+            # -L to follow symlinks, as dir.* are symlinks to gsidiags/dir.*
+            count=$(find -L ./dir.* -path "./dir.*/${dtype}_${loop}*" -type f -printf "." | wc -c)
             if [[ ${count} -eq 0 ]]; then
                 continue
             fi