From def4c7c01f18dc3a676d9c6ac8632b04e2787b84 Mon Sep 17 00:00:00 2001 From: David Huber Date: Tue, 27 Jan 2026 09:57:52 -0500 Subject: [PATCH 01/14] Add the dataroot_com_path function --- ush/bash_utils.sh | 42 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/ush/bash_utils.sh b/ush/bash_utils.sh index 48007623351..9bec838a63f 100755 --- a/ush/bash_utils.sh +++ b/ush/bash_utils.sh @@ -36,7 +36,9 @@ function declare_from_tmpl() { # MEMDIR='mem001' YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ # COMOUT_ATMOS_HISTORY:COM_ATMOS_HISTORY_TMPL # - if [[ ${DEBUG_WORKFLOW:-"NO"} == "NO" ]]; then set +x; fi + if [[ ${DEBUG_WORKFLOW:-"NO"} == "NO" ]]; then + set +x + fi local opts="-g" local OPTIND=1 while getopts "rx" option; do @@ -110,7 +112,45 @@ function wait_for_file() { return 1 } +# This utility is to be used to create a COM structure in the DATAROOT +# It will replace the root path (up to $ROTDIR) with $DATAROOT +# Use realpath --relative-to to get the relative path from $ROTDIR to the target file +# and then prepend $DATAROOT to that path to get the new target path +function dataroot_com_path() { + # + # Generate a COM path in the DATAROOT based on an existing COM path. + # + # This function takes an existing COM path and generates a corresponding + # path in the DATAROOT by replacing the root directory with DATAROOT. + # + # Syntax: + # dataroot_com_path original_com_path + # + # original_com_path: The original COM path to be transformed. + # + # Example: + # # Declare COMOUT_ATMOS_ANALYSIS using template + # YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ + # COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL + # # Get the DATAROOT version of the COM path + # pCOMOUT_ATMOS_ANALYSIS=$(dataroot_com_path "${COMOUT_ATMOS_ANALYSIS}") + # echo "New COM path in DATAROOT: ${pCOMOUT_ATMOS_ANALYSIS}" + # + local original_com_path=${1:?"dataroot_com_path() requires an original COM path"} + + if [[ -z "${ROTDIR:-}" || -z "${DATAROOT:-}" ]]; then + echo "FATAL ERROR in dataroot_com_path: ROTDIR and DATAROOT must be defined!" + exit 2 + fi + + local relative_path=$(realpath --relative-to="${ROTDIR}" "${original_com_path}") + local new_com_path="${DATAROOT}/${relative_path}" + + echo "${new_com_path}" +} + # shellcheck disable= declare -xf declare_from_tmpl declare -xf wait_for_file +declare -xf dataroot_com_path From 4c2db2734ba9672a2ccb9eb7dad273171dfd40e3 Mon Sep 17 00:00:00 2001 From: David Huber Date: Tue, 27 Jan 2026 10:51:56 -0500 Subject: [PATCH 02/14] Declare on separate lines from command --- ush/bash_utils.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/ush/bash_utils.sh b/ush/bash_utils.sh index 9bec838a63f..964a4f59004 100755 --- a/ush/bash_utils.sh +++ b/ush/bash_utils.sh @@ -136,14 +136,22 @@ function dataroot_com_path() { # pCOMOUT_ATMOS_ANALYSIS=$(dataroot_com_path "${COMOUT_ATMOS_ANALYSIS}") # echo "New COM path in DATAROOT: ${pCOMOUT_ATMOS_ANALYSIS}" # - local original_com_path=${1:?"dataroot_com_path() requires an original COM path"} + + if [[ $# -ne 1 ]]; then + echo "FATAL ERROR in dataroot_com_path: Incorrect number of arguments!" + echo "Usage: dataroot_com_path original_com_path" + exit 2 + fi + + local original_com_path=${1} if [[ -z "${ROTDIR:-}" || -z "${DATAROOT:-}" ]]; then echo "FATAL ERROR in dataroot_com_path: ROTDIR and DATAROOT must be defined!" exit 2 fi - local relative_path=$(realpath --relative-to="${ROTDIR}" "${original_com_path}") + local relative_path + relative_path=$(realpath --relative-to="${ROTDIR}" "${original_com_path}") local new_com_path="${DATAROOT}/${relative_path}" echo "${new_com_path}" From 4074fb731964098ffd39c2e55352af56d8967283 Mon Sep 17 00:00:00 2001 From: David Huber Date: Tue, 27 Jan 2026 16:18:12 -0500 Subject: [PATCH 03/14] Use DATAROOT for gsidiags --- dev/jobs/JGLOBAL_ATMOS_ANALYSIS | 7 ++++++ dev/jobs/JGLOBAL_ATMOS_ANALYSIS_DIAG | 5 +++++ dev/scripts/exglobal_atmos_analysis.sh | 31 ++++++++++++++++---------- dev/scripts/exglobal_diag.sh | 14 ++++++------ 4 files changed, 38 insertions(+), 19 deletions(-) diff --git a/dev/jobs/JGLOBAL_ATMOS_ANALYSIS b/dev/jobs/JGLOBAL_ATMOS_ANALYSIS index 88dc666d4b6..6cb509d6b2a 100755 --- a/dev/jobs/JGLOBAL_ATMOS_ANALYSIS +++ b/dev/jobs/JGLOBAL_ATMOS_ANALYSIS @@ -45,6 +45,13 @@ if [[ ! -f "${ATMGES}" ]]; then err_exit "FILE MISSING: ATMGES == ${ATMGES}" fi +export GENDIAG="${GENDIAG:-YES}" + +# Create a DATAROOT directory variable for the gsidiags if GENDIAG is YES +if [[ "${GENDIAG}" == "YES" ]]; then + export pCOMOUT_ATMOS_ANALYSIS="$(dataroot_com_path "${COMOUT_ATMOS_ANALYSIS}")" +fi + # Get LEVS export LEVS LEVS=$(${NCLEN} "${ATMGES}" pfull) && true diff --git a/dev/jobs/JGLOBAL_ATMOS_ANALYSIS_DIAG b/dev/jobs/JGLOBAL_ATMOS_ANALYSIS_DIAG index 553a58e1160..23e26dd7ce0 100755 --- a/dev/jobs/JGLOBAL_ATMOS_ANALYSIS_DIAG +++ b/dev/jobs/JGLOBAL_ATMOS_ANALYSIS_DIAG @@ -26,6 +26,11 @@ YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL mkdir -m 775 -p "${COMOUT_ATMOS_ANALYSIS}" +# Create a DATAROOT directory variable where the gsidiags live +if [[ "${GENDIAG}" == "YES" ]]; then + export pCOMIN_ATMOS_ANALYSIS="$(dataroot_com_path "${COMIN_ATMOS_ANALYSIS}")" +fi + ############################################################### # Run relevant script ${ANALDIAGSH:-${SCRgfs}/exglobal_diag.sh} && true diff --git a/dev/scripts/exglobal_atmos_analysis.sh b/dev/scripts/exglobal_atmos_analysis.sh index 7ce7ede9db7..b556cb3bd6b 100755 --- a/dev/scripts/exglobal_atmos_analysis.sh +++ b/dev/scripts/exglobal_atmos_analysis.sh @@ -198,7 +198,6 @@ RUN_SELECT=${RUN_SELECT:-"NO"} USE_SELECT=${USE_SELECT:-"NO"} USE_RADSTAT=${USE_RADSTAT:-"YES"} SELECT_OBS=${SELECT_OBS:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}obsinput.tar} -GENDIAG=${GENDIAG:-"YES"} # Set script / GSI control parameters DOHYBVAR=${DOHYBVAR:-"NO"} @@ -882,6 +881,25 @@ if [[ "${SENDDBN}" == "YES" ]]; then fi fi +if [[ "${GENDIAGS}" == "YES" ]]; then + # Move the gsidiags dir.* directories to pCOMOUT_ATMOS_ANALYSIS for diagnostic jobs + gsidiags_dir="${pCOMOUT_ATMOS_ANALYSIS}/gsidiags" + # First, check that the directories exist + count_dirs=$(find . -maxdepth 1 -type d -name 'dir.????' | wc -l) + if [[ ${count_dirs:-0} -gt 0 ]]; then + mkdir -p "${gsidiags_dir}" + for dir in dir.????; do + mv "${dir}" "${gsidiags_dir}/" + export err=$? + if [[ ${err} -ne 0 ]]; then + err_exit "Failed to move ${dir} to ${gsidiags_dir}/" + fi + done + else + echo "WARNING: No gsidiags dir.* directories found to move." + fi +fi + ################################################################################ # Postprocessing cd "${DATA}" || exit 1 @@ -895,17 +913,6 @@ if [[ "${SENDECF}" == "YES" && "${RUN}" != "enkf" ]]; then ecflow_client --event release_fcst fi -# Diagnostic files -# if requested, GSI diagnostic file directories for use later -if [[ "${GENDIAG}" == "YES" ]]; then - tar -cvf gsidiags.tar dir.???? - export err=$? - if [[ ${err} -ne 0 ]]; then - err_exit "Failed to tar GSI diagnostic directories!" - fi - cpfs gsidiags.tar "${COMOUT_ATMOS_ANALYSIS}/${APREFIX}gsidiags${DIAG_SUFFIX:-}.tar" -fi - echo "${rCDUMP} ${PDY}${cyc} atminc done at $(date)" > "${COMOUT_ATMOS_ANALYSIS}/${APREFIX}increment.done.txt" ################################################################################ diff --git a/dev/scripts/exglobal_diag.sh b/dev/scripts/exglobal_diag.sh index 4d9aa5de94e..2a5cc42e273 100755 --- a/dev/scripts/exglobal_diag.sh +++ b/dev/scripts/exglobal_diag.sh @@ -39,7 +39,7 @@ rm -f "${RADSTAT}" "${PCPSTAT}" "${CNVSTAT}" "${OZNSTAT}" # Obs diag GENDIAG=${GENDIAG:-"YES"} -GSIDIAG=${GSIDIAG:-"${COMIN_ATMOS_ANALYSIS}/${APREFIX}gsidiags${DIAG_SUFFIX:-}.tar"} +GSIDIAGDIR=${GSIDIAGDIR:-"${pCOMIN_ATMOS_ANALYSIS}/gsidiags"} USE_BUILD_GSINFO=${USE_BUILD_GSINFO:-"NO"} DIAG_COMPRESS=${DIAG_COMPRESS:-"YES"} if [[ "${DIAG_COMPRESS:-}" == "YES" ]]; then @@ -56,13 +56,13 @@ if [[ "${GENDIAG}" != "YES" ]]; then fi ################################################################################ -# Copy gsidiags.tar file from COMIN to DATA and untar -cpreq "${GSIDIAG}" ./gsidiags.tar -tar -xvf gsidiags.tar -export err=$? -if [[ ${err} -ne 0 ]]; then - err_exit "Unable to unpack gsidiags.tar file!" +# Link to the gsidiags directory if it is populated +count_dirs=$(find "${GSIDIAGDIR}" -maxdepth 1 -type d -name "dir.*" | wc -l) +if [[ ${count_dirs} -eq 0 ]]; then + export err=1 + err_exit "No gsidiags directories found in ${GSIDIAGDIR}" fi +${NLN} -sf "${GSIDIAGDIR}" . # Set up lists and variables for various types of diagnostic files. ntype=3 From 9b3dc8941df169b03405eaaaa6609876e39284b3 Mon Sep 17 00:00:00 2001 From: David Huber Date: Wed, 28 Jan 2026 14:15:24 -0500 Subject: [PATCH 04/14] Fix tyop in GENDIAG boolean --- dev/scripts/exglobal_atmos_analysis.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/scripts/exglobal_atmos_analysis.sh b/dev/scripts/exglobal_atmos_analysis.sh index b556cb3bd6b..7dabfd4facd 100755 --- a/dev/scripts/exglobal_atmos_analysis.sh +++ b/dev/scripts/exglobal_atmos_analysis.sh @@ -881,7 +881,7 @@ if [[ "${SENDDBN}" == "YES" ]]; then fi fi -if [[ "${GENDIAGS}" == "YES" ]]; then +if [[ "${GENDIAG}" == "YES" ]]; then # Move the gsidiags dir.* directories to pCOMOUT_ATMOS_ANALYSIS for diagnostic jobs gsidiags_dir="${pCOMOUT_ATMOS_ANALYSIS}/gsidiags" # First, check that the directories exist From 784c76fa828af7918a737fe0bd82aa54cf3bf150 Mon Sep 17 00:00:00 2001 From: David Huber Date: Wed, 28 Jan 2026 14:20:13 -0500 Subject: [PATCH 05/14] Use comroot instead of rotdir --- ush/bash_utils.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ush/bash_utils.sh b/ush/bash_utils.sh index 964a4f59004..d3155e4dfcb 100755 --- a/ush/bash_utils.sh +++ b/ush/bash_utils.sh @@ -113,8 +113,8 @@ function wait_for_file() { } # This utility is to be used to create a COM structure in the DATAROOT -# It will replace the root path (up to $ROTDIR) with $DATAROOT -# Use realpath --relative-to to get the relative path from $ROTDIR to the target file +# It will replace the root path (up to $COMROOT) with $DATAROOT +# Use realpath --relative-to to get the relative path from $COMROOT to the target file # and then prepend $DATAROOT to that path to get the new target path function dataroot_com_path() { # @@ -145,13 +145,13 @@ function dataroot_com_path() { local original_com_path=${1} - if [[ -z "${ROTDIR:-}" || -z "${DATAROOT:-}" ]]; then - echo "FATAL ERROR in dataroot_com_path: ROTDIR and DATAROOT must be defined!" + if [[ -z "${COMROOT:-}" || -z "${DATAROOT:-}" ]]; then + echo "FATAL ERROR in dataroot_com_path: COMROOT and DATAROOT must be defined!" exit 2 fi local relative_path - relative_path=$(realpath --relative-to="${ROTDIR}" "${original_com_path}") + relative_path=$(realpath --relative-to="${COMROOT}" "${original_com_path}") local new_com_path="${DATAROOT}/${relative_path}" echo "${new_com_path}" From 8dcbe683c889267524d4a67b961b070fbe6e3b6f Mon Sep 17 00:00:00 2001 From: David Huber Date: Wed, 28 Jan 2026 15:06:23 -0500 Subject: [PATCH 06/14] Fix typos --- dev/jobs/JGLOBAL_ATMOS_ANALYSIS_DIAG | 4 +--- dev/scripts/exglobal_diag.sh | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/dev/jobs/JGLOBAL_ATMOS_ANALYSIS_DIAG b/dev/jobs/JGLOBAL_ATMOS_ANALYSIS_DIAG index 23e26dd7ce0..39a8e8b3dff 100755 --- a/dev/jobs/JGLOBAL_ATMOS_ANALYSIS_DIAG +++ b/dev/jobs/JGLOBAL_ATMOS_ANALYSIS_DIAG @@ -27,9 +27,7 @@ YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ mkdir -m 775 -p "${COMOUT_ATMOS_ANALYSIS}" # Create a DATAROOT directory variable where the gsidiags live -if [[ "${GENDIAG}" == "YES" ]]; then - export pCOMIN_ATMOS_ANALYSIS="$(dataroot_com_path "${COMIN_ATMOS_ANALYSIS}")" -fi +export pCOMIN_ATMOS_ANALYSIS="$(dataroot_com_path "${COMIN_ATMOS_ANALYSIS}")" ############################################################### # Run relevant script diff --git a/dev/scripts/exglobal_diag.sh b/dev/scripts/exglobal_diag.sh index 2a5cc42e273..99bcfa9537f 100755 --- a/dev/scripts/exglobal_diag.sh +++ b/dev/scripts/exglobal_diag.sh @@ -62,7 +62,7 @@ if [[ ${count_dirs} -eq 0 ]]; then export err=1 err_exit "No gsidiags directories found in ${GSIDIAGDIR}" fi -${NLN} -sf "${GSIDIAGDIR}" . +${NLN} "${GSIDIAGDIR}/"dir.* . # Set up lists and variables for various types of diagnostic files. ntype=3 From 5e9d5d6b62ef9525c6df80d0600316679b4590c2 Mon Sep 17 00:00:00 2001 From: David Huber Date: Wed, 28 Jan 2026 16:09:38 -0500 Subject: [PATCH 07/14] Follow symbolic links --- dev/scripts/exglobal_diag.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dev/scripts/exglobal_diag.sh b/dev/scripts/exglobal_diag.sh index 99bcfa9537f..dbc513228ba 100755 --- a/dev/scripts/exglobal_diag.sh +++ b/dev/scripts/exglobal_diag.sh @@ -153,7 +153,8 @@ for loop in ${loops}; do if [[ "${dtype}" == *"avhrr"* ]]; then # recast dtype in avhrr form dtype="avhrr${dtype##avhrr[23]}" fi - count=$(find ./ -path "./dir.*/${dtype}_${loop}*" -type f -printf "." | wc -c) + # -L to follow symlinks, as dir.* are symlinks to gsidiags/dir.* + count=$(find -L ./dir.* -path "./dir.*/${dtype}_${loop}*" -type f -printf "." | wc -c) if [[ ${count} -eq 0 ]]; then continue fi From 3a75c981c326301f14c26e38cd240a1e1155d8ae Mon Sep 17 00:00:00 2001 From: David Huber Date: Wed, 28 Jan 2026 16:21:38 -0500 Subject: [PATCH 08/14] Move diagnostic output after releasing the forecast --- dev/scripts/exglobal_atmos_analysis.sh | 30 +++++++++++++------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/dev/scripts/exglobal_atmos_analysis.sh b/dev/scripts/exglobal_atmos_analysis.sh index 7dabfd4facd..9ea71d4d4f5 100755 --- a/dev/scripts/exglobal_atmos_analysis.sh +++ b/dev/scripts/exglobal_atmos_analysis.sh @@ -881,6 +881,21 @@ if [[ "${SENDDBN}" == "YES" ]]; then fi fi +################################################################################ +# Postprocessing +cd "${DATA}" || exit 1 + +############################################################## +# Add this statement to release the forecast job once the +# atmopsheric analysis and updated surface RESTARTS are +# available. Do not release forecast when RUN=enkf +############################################################## +if [[ "${SENDECF}" == "YES" && "${RUN}" != "enkf" ]]; then + ecflow_client --event release_fcst +fi + +echo "${rCDUMP} ${PDY}${cyc} atminc done at $(date)" > "${COMOUT_ATMOS_ANALYSIS}/${APREFIX}increment.done.txt" + if [[ "${GENDIAG}" == "YES" ]]; then # Move the gsidiags dir.* directories to pCOMOUT_ATMOS_ANALYSIS for diagnostic jobs gsidiags_dir="${pCOMOUT_ATMOS_ANALYSIS}/gsidiags" @@ -900,21 +915,6 @@ if [[ "${GENDIAG}" == "YES" ]]; then fi fi -################################################################################ -# Postprocessing -cd "${DATA}" || exit 1 - -############################################################## -# Add this statement to release the forecast job once the -# atmopsheric analysis and updated surface RESTARTS are -# available. Do not release forecast when RUN=enkf -############################################################## -if [[ "${SENDECF}" == "YES" && "${RUN}" != "enkf" ]]; then - ecflow_client --event release_fcst -fi - -echo "${rCDUMP} ${PDY}${cyc} atminc done at $(date)" > "${COMOUT_ATMOS_ANALYSIS}/${APREFIX}increment.done.txt" - ################################################################################ exit "${err}" From 14459e192196d31a682a57c184a2f0620e89dd19 Mon Sep 17 00:00:00 2001 From: David Huber Date: Wed, 28 Jan 2026 16:29:17 -0500 Subject: [PATCH 09/14] Address shellcheck issues --- dev/jobs/JGLOBAL_ATMOS_ANALYSIS | 4 +++- dev/jobs/JGLOBAL_ATMOS_ANALYSIS_DIAG | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/dev/jobs/JGLOBAL_ATMOS_ANALYSIS b/dev/jobs/JGLOBAL_ATMOS_ANALYSIS index 6cb509d6b2a..a28a0a91216 100755 --- a/dev/jobs/JGLOBAL_ATMOS_ANALYSIS +++ b/dev/jobs/JGLOBAL_ATMOS_ANALYSIS @@ -36,6 +36,7 @@ RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ MEMDIR='ensstat' RUN=${GDUMP_ENS} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ COMIN_ATMOS_HISTORY_ENS_PREV:COM_ATMOS_HISTORY_TMPL +# shellcheck disable=SC2153 mkdir -p "${COMOUT_ATMOS_ANALYSIS}" export ATMGES="${COMIN_ATMOS_HISTORY_PREV}/${GPREFIX}atm.f006.nc" @@ -49,7 +50,8 @@ export GENDIAG="${GENDIAG:-YES}" # Create a DATAROOT directory variable for the gsidiags if GENDIAG is YES if [[ "${GENDIAG}" == "YES" ]]; then - export pCOMOUT_ATMOS_ANALYSIS="$(dataroot_com_path "${COMOUT_ATMOS_ANALYSIS}")" + pCOMOUT_ATMOS_ANALYSIS="$(dataroot_com_path "${COMOUT_ATMOS_ANALYSIS}")" + export pCOMOUT_ATMOS_ANALYSIS fi # Get LEVS diff --git a/dev/jobs/JGLOBAL_ATMOS_ANALYSIS_DIAG b/dev/jobs/JGLOBAL_ATMOS_ANALYSIS_DIAG index 39a8e8b3dff..0f7eabd9033 100755 --- a/dev/jobs/JGLOBAL_ATMOS_ANALYSIS_DIAG +++ b/dev/jobs/JGLOBAL_ATMOS_ANALYSIS_DIAG @@ -27,7 +27,9 @@ YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ mkdir -m 775 -p "${COMOUT_ATMOS_ANALYSIS}" # Create a DATAROOT directory variable where the gsidiags live -export pCOMIN_ATMOS_ANALYSIS="$(dataroot_com_path "${COMIN_ATMOS_ANALYSIS}")" +# shellcheck disable=SC2153 +pCOMIN_ATMOS_ANALYSIS="$(dataroot_com_path "${COMIN_ATMOS_ANALYSIS}")" +export pCOMIN_ATMOS_ANALYSIS ############################################################### # Run relevant script From d75e34761d02971d7b08eb7d28d426acdaeff7f8 Mon Sep 17 00:00:00 2001 From: David Huber Date: Fri, 30 Jan 2026 14:03:57 -0500 Subject: [PATCH 10/14] Fix eobs and make diag variable names more consistent --- dev/jobs/JGLOBAL_ENKF_SELECT_OBS | 8 ++++++++ dev/scripts/exglobal_atmos_analysis.sh | 28 ++++++++++++++++++++------ dev/scripts/exglobal_diag.sh | 15 ++++++++++++++ 3 files changed, 45 insertions(+), 6 deletions(-) diff --git a/dev/jobs/JGLOBAL_ENKF_SELECT_OBS b/dev/jobs/JGLOBAL_ENKF_SELECT_OBS index af264f7d9c6..50ccef6ee7f 100755 --- a/dev/jobs/JGLOBAL_ENKF_SELECT_OBS +++ b/dev/jobs/JGLOBAL_ENKF_SELECT_OBS @@ -48,6 +48,14 @@ if [[ ! -f ${ATMGES_ENSMEAN} ]]; then err_exit "FILE MISSING: ATMGES_ENSMEAN == ${ATMGES_ENSMEAN}" fi +export GENDIAG="${GENDIAG:-YES}" + +# Create a DATAROOT directory variable for the gsidiags if GENDIAG is YES +if [[ "${GENDIAG}" == "YES" ]]; then + pCOMOUT_ATMOS_ANALYSIS="$(dataroot_com_path "${COMOUT_ATMOS_ANALYSIS}")" + export pCOMOUT_ATMOS_ANALYSIS +fi + LEVS=$(${NCLEN} "${ATMGES}" pfull) && true export err=$? set_strict diff --git a/dev/scripts/exglobal_atmos_analysis.sh b/dev/scripts/exglobal_atmos_analysis.sh index 9ea71d4d4f5..5652c452996 100755 --- a/dev/scripts/exglobal_atmos_analysis.sh +++ b/dev/scripts/exglobal_atmos_analysis.sh @@ -198,6 +198,7 @@ RUN_SELECT=${RUN_SELECT:-"NO"} USE_SELECT=${USE_SELECT:-"NO"} USE_RADSTAT=${USE_RADSTAT:-"YES"} SELECT_OBS=${SELECT_OBS:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}obsinput.tar} +GENDIAG=${GENDIAG:-"YES"} # Set script / GSI control parameters DOHYBVAR=${DOHYBVAR:-"NO"} @@ -412,6 +413,16 @@ else echo "not using correlated obs error" fi +# If GENDIAG is selected, verify that pCOMOUT_ATMOS_ANALYSIS is set +if [[ "${GENDIAG}" == "YES" ]]; then + if [[ -z "${pCOMOUT_ATMOS_ANALYSIS}" ]]; then + export err=1 + err_exit "pCOMOUT_ATMOS_ANALYSIS must be set when GENDIAG=YES" + fi + # Make the gsidiags directory to house the GSI diagnostic data + GSIDIAGDIR=${GSIDIAGDIR:-"${pCOMOUT_ATMOS_ANALYSIS}/gsidiags"} +fi + ############################################################## # CRTM Spectral and Transmittance coefficients mkdir -p crtm_coeffs @@ -898,16 +909,21 @@ echo "${rCDUMP} ${PDY}${cyc} atminc done at $(date)" > "${COMOUT_ATMOS_ANALYSIS} if [[ "${GENDIAG}" == "YES" ]]; then # Move the gsidiags dir.* directories to pCOMOUT_ATMOS_ANALYSIS for diagnostic jobs - gsidiags_dir="${pCOMOUT_ATMOS_ANALYSIS}/gsidiags" - # First, check that the directories exist - count_dirs=$(find . -maxdepth 1 -type d -name 'dir.????' | wc -l) + # First, check that the directories exist (we need at least one, so stop after the first match) + count_dirs=$(find . -maxdepth 1 -type d -name 'dir.????' -quit -printf "." | wc -c) if [[ ${count_dirs:-0} -gt 0 ]]; then - mkdir -p "${gsidiags_dir}" + mkdir -p "${GSIDIAGDIR}" + err=$? + + if [[ ! -d "${GSIDIAGDIR}" || ${err} -ne 0 ]]; then + err_exit "Failed to create gsidiags directory at ${GSIDIAGDIR}" + fi + for dir in dir.????; do - mv "${dir}" "${gsidiags_dir}/" + mv "${dir}" "${GSIDIAGDIR}/" export err=$? if [[ ${err} -ne 0 ]]; then - err_exit "Failed to move ${dir} to ${gsidiags_dir}/" + err_exit "Failed to move ${dir} to ${GSIDIAGDIR}/" fi done else diff --git a/dev/scripts/exglobal_diag.sh b/dev/scripts/exglobal_diag.sh index dbc513228ba..8fcdc8fae03 100755 --- a/dev/scripts/exglobal_diag.sh +++ b/dev/scripts/exglobal_diag.sh @@ -55,6 +55,12 @@ if [[ "${GENDIAG}" != "YES" ]]; then exit 0 fi +# Check that the gsidiags directory exists +if [[ ! -d "${GSIDIAGDIR}" ]]; then + export err=1 + err_exit "gsidiags directory ${GSIDIAGDIR} does not exist" +fi + ################################################################################ # Link to the gsidiags directory if it is populated count_dirs=$(find "${GSIDIAGDIR}" -maxdepth 1 -type d -name "dir.*" | wc -l) @@ -62,6 +68,15 @@ if [[ ${count_dirs} -eq 0 ]]; then export err=1 err_exit "No gsidiags directories found in ${GSIDIAGDIR}" fi + +# Continue if there is at least one file to process +# Note -quit stops find after the first match +count_files=$(find "${GSIDIAGDIR}"/dir.* -maxdepth 1 -type f -quit -printf '.' | wc -c) +if [[ ${count_files} -eq 0 ]]; then + echo "WARNING: No diagnostic files found to process!" + exit 0 +fi + ${NLN} "${GSIDIAGDIR}/"dir.* . # Set up lists and variables for various types of diagnostic files. From 888b0ed74dad7ddbb195cee57fd48cf1179c587b Mon Sep 17 00:00:00 2001 From: David Huber Date: Fri, 30 Jan 2026 14:26:54 -0500 Subject: [PATCH 11/14] Ignore sc2153 message for COM variable --- dev/jobs/JGLOBAL_ENKF_SELECT_OBS | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/jobs/JGLOBAL_ENKF_SELECT_OBS b/dev/jobs/JGLOBAL_ENKF_SELECT_OBS index 50ccef6ee7f..0436c5bf868 100755 --- a/dev/jobs/JGLOBAL_ENKF_SELECT_OBS +++ b/dev/jobs/JGLOBAL_ENKF_SELECT_OBS @@ -40,6 +40,7 @@ MEMDIR='ensstat' RUN=${GDUMP_ENS} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ RUN="${GDUMP}" YMD=${gPDY} HH=${gcyc} declare_from_tmpl -r \ COMIN_ATMOS_ANALYSIS_DET_PREV:COM_ATMOS_ANALYSIS_TMPL +# shellcheck disable=SC2153 mkdir -p "${COMOUT_ATMOS_ANALYSIS}" export ATMGES_ENSMEAN="${COMIN_ATMOS_HISTORY_PREV}/${GPREFIX}${GSUFFIX}atm.f006.nc" From 8dbcd6000aa181456b6ea7026bb25ce999e46494 Mon Sep 17 00:00:00 2001 From: David Huber Date: Fri, 30 Jan 2026 14:39:30 -0500 Subject: [PATCH 12/14] Ignore shellcheck 2153 --- dev/jobs/JGLOBAL_ENKF_SELECT_OBS | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/jobs/JGLOBAL_ENKF_SELECT_OBS b/dev/jobs/JGLOBAL_ENKF_SELECT_OBS index 0436c5bf868..48e96539c6e 100755 --- a/dev/jobs/JGLOBAL_ENKF_SELECT_OBS +++ b/dev/jobs/JGLOBAL_ENKF_SELECT_OBS @@ -28,6 +28,7 @@ RUN=${RUN/enkf/} YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ COMIN_OBS:COM_OBS_TMPL MEMDIR='ensstat' YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL +# shellcheck disable=SC2153 declare -rx COMOUT_ATMOS_ANALYSIS_ENS="${COMOUT_ATMOS_ANALYSIS}" RUN=${RUN/enkf/} YMD=${PDY} HH=${cyc} declare_from_tmpl -r \ From c1eac3fe58cc914cd2b6f3dc0920a0da79bbec79 Mon Sep 17 00:00:00 2001 From: DavidHuber-NOAA Date: Wed, 4 Feb 2026 12:22:35 +0000 Subject: [PATCH 13/14] Add missing pCOMIN declaration --- dev/jobs/JGLOBAL_ENKF_DIAG | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dev/jobs/JGLOBAL_ENKF_DIAG b/dev/jobs/JGLOBAL_ENKF_DIAG index fa494dfb847..5e92c7ac56c 100755 --- a/dev/jobs/JGLOBAL_ENKF_DIAG +++ b/dev/jobs/JGLOBAL_ENKF_DIAG @@ -29,6 +29,11 @@ MEMDIR="ensstat" YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ COMIN_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL \ COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL +# Create a DATAROOT directory variable where the gsidiags live +# shellcheck disable=SC2153 +pCOMIN_ATMOS_ANALYSIS="$(dataroot_com_path "${COMIN_ATMOS_ANALYSIS}")" +export pCOMIN_ATMOS_ANALYSIS + #RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ # COMIN_OBS_PREV:COM_OBS_TMPL \ # COMIN_ATMOS_ANALYSIS_DET_PREV:COM_ATMOS_ANALYSIS_TMPL From aa6873704642a699b5c9b4481a69a1f24aef75cd Mon Sep 17 00:00:00 2001 From: DavidHuber-NOAA Date: Wed, 4 Feb 2026 12:23:01 +0000 Subject: [PATCH 14/14] Fix find order --- dev/scripts/exglobal_atmos_analysis.sh | 2 +- dev/scripts/exglobal_diag.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/scripts/exglobal_atmos_analysis.sh b/dev/scripts/exglobal_atmos_analysis.sh index 5652c452996..59879131f81 100755 --- a/dev/scripts/exglobal_atmos_analysis.sh +++ b/dev/scripts/exglobal_atmos_analysis.sh @@ -910,7 +910,7 @@ echo "${rCDUMP} ${PDY}${cyc} atminc done at $(date)" > "${COMOUT_ATMOS_ANALYSIS} if [[ "${GENDIAG}" == "YES" ]]; then # Move the gsidiags dir.* directories to pCOMOUT_ATMOS_ANALYSIS for diagnostic jobs # First, check that the directories exist (we need at least one, so stop after the first match) - count_dirs=$(find . -maxdepth 1 -type d -name 'dir.????' -quit -printf "." | wc -c) + count_dirs=$(find . -maxdepth 1 -type d -name 'dir.????' -printf "." -quit | wc -c) if [[ ${count_dirs:-0} -gt 0 ]]; then mkdir -p "${GSIDIAGDIR}" err=$? diff --git a/dev/scripts/exglobal_diag.sh b/dev/scripts/exglobal_diag.sh index 8fcdc8fae03..006515dc554 100755 --- a/dev/scripts/exglobal_diag.sh +++ b/dev/scripts/exglobal_diag.sh @@ -71,7 +71,7 @@ fi # Continue if there is at least one file to process # Note -quit stops find after the first match -count_files=$(find "${GSIDIAGDIR}"/dir.* -maxdepth 1 -type f -quit -printf '.' | wc -c) +count_files=$(find "${GSIDIAGDIR}"/dir.* -maxdepth 1 -type f -printf '.' -quit | wc -c) if [[ ${count_files} -eq 0 ]]; then echo "WARNING: No diagnostic files found to process!" exit 0