Refactor cleanup task and add checks for gempak files

dev/parm/config/gfs/config.cleanup

@@ -9,22 +9,30 @@ source "${EXPDIR}/config.resources" cleanup
 export CLEANUP_COM="YES"   # NO=retain ROTDIR.  YES default in cleanup.sh
 
 #--starting and ending hours of previous cycles to be removed from rotating directory
-export RMOLDSTD=144
-export RMOLDEND=24
+# Selectively remove files between SELECTIVE_RM_MIN and SELECTIVE_RM_MIN hours old, based on exclude_string
+# Remove all RTOFS files older than RTOFS_CLEANUP_MAX hours
+# Remove all files, except GEMPAK files, older than SELECTIVE_RM_MAX hours, based on gempak_exclude_string
+# Remove all files older than the max of all SELECTIVE_RM* variables
+# Retain all files newer than SELECTIVE_RM_MIN hours
 
-if [[ "${DO_GEMPAK}" == "YES" && "${RUN}" == "gfs" ]]; then
-    export RMOLDSTD=346
-    export RMOLDEND=222
+export SELECTIVE_CLEANUP_MAX=144
+export SELECTIVE_CLEANUP_MIN=24
+export GEMPAK_CLEANUP_MAX=240
+export RTOFS_CLEANUP_MAX=48
+if [[ ${SELECTIVE_CLEANUP_MIN} -gt ${SELECTIVE_CLEANUP_MAX} ]]; then
+    echo "FATAL ERROR: Invalid selective cleanup times: "
+    echo "             SELECTIVE_CLEANUP_MIN=${SELECTIVE_CLEANUP_MIN} > SELECTIVE_CLEANUP_MAX=${SELECTIVE_CLEANUP_MAX}"
+    exit 1
 fi
 
 # Specify the list of files to exclude from the first stage of cleanup
 # Because arrays cannot be exported, list is a single string of comma-
 # separated values. This string is split to form an array at runtime.
+export exclude_string=""
 case ${RUN} in
-	gdas | gfs)	exclude_string="*prepbufr*, *cnvstat.tar*, *analysis.atm.a*.nc" ;;
-	enkf*)      exclude_string="*f006.ens*" ;;
+	gdas | gfs)	exclude_string+="*prepbufr*, *cnvstat.tar*, *analysis.atm.a*.nc" ;;
+	enkf*)      exclude_string+="*f006.ens*" ;;
 	*)			exclude_string="" ;;
 esac
-export exclude_string
 
 echo "END: config.cleanup"

jobs/JGFS_ATMOS_POSTSND

@@ -12,6 +12,7 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "postsnd" -c "base postsnd"
 
 YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
     COMIN_ATMOS_HISTORY:COM_ATMOS_HISTORY_TMPL \
+    COMIN_ATMOS_BUFR:COM_ATMOS_BUFR_TMPL \
     COMOUT_ATMOS_BUFR:COM_ATMOS_BUFR_TMPL \
     COMOUT_ATMOS_WMO:COM_ATMOS_WMO_TMPL \
     COMOUT_ATMOS_GEMPAK:COM_ATMOS_GEMPAK_TMPL

scripts/exgfs_atmos_postsnd.sh

@@ -148,12 +148,14 @@ else
 fi
 ${runscript} "${fhr}" "${fhr_p}" "${FINT}" "${F00FLAG}" "${DATA}"
 
-##############################################################
-# Tar and gzip the individual bufr files and send them to /com
-##############################################################
-cd "${COMOUT_ATMOS_BUFR}" || exit 2
-tar -cf - . | /usr/bin/gzip > "${RUN}.${cycle}.bufrsnd.tar.gz"
-cd "${DATA}" || exit 2
+############################################
+# Tar and gzip the bufr files created so far
+############################################
+find "${COMIN_ATMOS_BUFR}/bufr.*" -printf '%f\n' > "${DATA}/all_bufr_files.txt"
+
+tar -czf "${RUN}.${cycle}.bufrsnd.tar.gz" -C "${COMIN_ATMOS_BUFR}" -T "${DATA}/all_bufr_files.txt"
+
+cpfs "${RUN}.${cycle}.bufrsnd.tar.gz" "${COMOUT_ATMOS_BUFR}/"
 
 ########################################
 # Send the single tar file to OSO
@@ -183,6 +185,7 @@ chmod +x cmdfile
 
 ${APRUN_POSTSNDCFP} cmdfile
 
+
 ########################################
 # Read BUFR output and transfer into
 # GEMPAK surface and sounding data files

scripts/exglobal_cleanup.sh

@@ -6,7 +6,9 @@ echo "Begin Cleanup ${DATAROOT}!"
 # Remove DATAoutput from the forecast model run
 # TODO: Handle this better
 DATAfcst="${DATAROOT}/${RUN}fcst.${PDY:-}${cyc}"
-if [[ -d "${DATAfcst}" ]]; then rm -rf "${DATAfcst}"; fi
+if [[ -d "${DATAfcst}" ]];
+    then rm -rf "${DATAfcst}";
+fi
 #DATAefcs="${DATAROOT}/${RUN}efcs???${PDY:-}${cyc}"
 rm -rf "${DATAROOT}/${RUN}efcs"*"${PDY:-}${cyc}"
 ###############################################################
@@ -15,69 +17,93 @@ if [[ "${CLEANUP_COM:-YES}" == NO ]] ; then
     exit 0
 fi
 
+SELECTIVE_CLEANUP_MIN=${SELECTIVE_CLEANUP_MIN:-24}
+SELECTIVE_CLEANUP_MAX=${SELECTIVE_CLEANUP_MAX:-120}
+RTOFS_CLEANUP_MAX=${RTOFS_CLEANUP_MAX:-48}
+GEMPAK_CLEANUP_MAX=${GEMPAK_CLEANUP_MAX:-240}
 ###############################################################
 # Clean up previous cycles; various depths
 
 # Step back every assim_freq hours and remove old rotating directories
 # for successful cycles (defaults from 24h to 120h).
 # Retain files needed by Fit2Obs
-last_date=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} -${RMOLDEND:-24} hours")
-first_date=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} -${RMOLDSTD:-120} hours")
-last_rtofs=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} -${RMOLDRTOFS:-48} hours")
+first_selective_date=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} -${SELECTIVE_CLEANUP_MAX} hours")
+last_rtofs_date=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} -${RTOFS_CLEANUP_MAX} hours")
+last_gempak_date=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} -${GEMPAK_CLEANUP_MAX} hours")
+exclude_string="${exclude_string:-}"
+
+# Find the last date among all cleanup targets
+max_cleanup_max="${SELECTIVE_CLEANUP_MAX:-120}"
+for cleanup_max in "${RTOFS_CLEANUP_MAX}" "${GEMPAK_CLEANUP_MAX}"; do
+    if [[ ${cleanup_max} -gt ${max_cleanup_max} ]]; then
+        max_cleanup_max=${cleanup_max}
+    fi
+done
+
+last_date=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} -${max_cleanup_max} hours")
+
 function remove_files() {
     local directory=$1
     shift
     if [[ ! -d ${directory} ]]; then
         echo "No directory ${directory} to remove files from, skiping"
         return
     fi
-    local find_exclude_string=""
-    for exclude in "$@"; do
-        find_exclude_string+="${find_exclude_string} -name ${exclude} -or"
+    # Find all files and links in the directory and store as an arry
+    # Run find only once for efficiency
+    flist=()
+    mapfile -t flist < <(find "${directory}" -type f -or -type l) || true
+
+    # Now remove those files that match the exclude patterns
+    for exclude_pattern in "$@"; do
+        # Use a temporary array to hold files that do not match the exclude pattern
+        temp_flist=()
+        for file in "${flist[@]}"; do
+            # shellcheck disable=SC2053
+            if [[ ! $(basename "${file}") == ${exclude_pattern} ]]; then
+                temp_flist+=("${file}")
+            fi
+        done
+        flist=("${temp_flist[@]}")
     done
-    # Chop off any trailing or
-    find_exclude_string="${find_exclude_string[*]/%-or}"
-    # Remove all regular files that do not match
-    # shellcheck disable=SC2086
-    if [[ -n "${find_exclude_string}" ]]; then
-    # String is non-empty → use exclusion
-       find "${directory}" -type f -not \( ${find_exclude_string} \) -ignore_readdir_race -delete
-    else
-    # String is empty → no exclusion
-       find "${directory}" -type f -ignore_readdir_race -delete
-    fi
 
-    # Remove all symlinks that do not match
-    # shellcheck disable=SC2086
-    if [[ -n "${find_exclude_string}" ]]; then
-    # String is non-empty → use exclusion
-       find "${directory}" -type l -not \( ${find_exclude_string} \) -ignore_readdir_race -delete
-    else
-    # String is empty → no exclusion
-       find "${directory}" -type l -ignore_readdir_race -delete
-    fi
+    # Delete all files in flist.
+    for file in "${flist[@]}"; do
+        rm -f "${file}"
+    done
 
     # Remove any empty directories
     find "${directory}" -type d -empty -delete
 }
 
-for (( current_date=first_date; current_date <= last_date; \
+# Now start removing old COM files/directories
+for (( current_date=first_selective_date; current_date <= last_date; \
   current_date=$(date --utc +%Y%m%d%H -d "${current_date:0:8} ${current_date:8:2} +${assim_freq} hours") )); do
     current_PDY="${current_date:0:8}"
     current_cyc="${current_date:8:2}"
-    rtofs_dir="${ROTDIR}/rtofs.${current_PDY}"
     rocotolog="${EXPDIR}/logs/${current_date}.log"
+
+    # Extend the exclude list for gempak files if needed
+    if [[ "${RUN}" == "gfs" && ${current_date} -lt ${last_gempak_date} && "${DO_GEMPAK}" == "YES" ]]; then
+        # Provide the gempak exclude pattern(s)
+        exclude_string+=", *gfs_1p00_*"
+    fi
+
+    # Check if the cycle completed successfully by looking at the rocoto log
     if [[ -f "${rocotolog}" ]]; then
         # TODO: This needs to be revamped to not look at the rocoto log.
-        # shellcheck disable=SC2312
-        if [[ $(tail -n 1 "${rocotolog}") =~ "This cycle is complete: Success" ]]; then
+        tail_log=$(tail -n 1 "${rocotolog}") || true
+        # Test if the last line of rocotolog indicates success
+        if [[ ${tail_log} =~ "This cycle is complete: Success" ]]; then
             YMD="${current_PDY}" HH="${current_cyc}" declare_from_tmpl \
                 COMOUT_TOP:COM_TOP_TMPL
             if [[ -d "${COMOUT_TOP}" ]]; then
-                IFS=", " read -r -a exclude_list <<< "${exclude_string:-}"
+                IFS=", " read -r -a exclude_list <<< "${exclude_string}"
                 remove_files "${COMOUT_TOP}" "${exclude_list[@]:-}"
             fi
-            if [[ -d "${rtofs_dir}" ]] && (( current_date < last_rtofs )); then rm -rf "${rtofs_dir}" ; fi
+            # Remove all rtofs directories in each RUN older than last_rtofs_date
+            rtofs_dir="${ROTDIR}/rtofs.${current_PDY}"
+            if [[ -d "${rtofs_dir}" ]] && (( current_date < last_rtofs_date )); then rm -rf "${rtofs_dir}" ; fi
         fi
     fi
 done
@@ -108,13 +134,16 @@ if [[ "${RUN}" == "gfs" ]]; then
 fi
 
 # Remove $RUN.$rPDY for the older of GDATE or RDATE
-GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} -${RMOLDSTD:-120} hours")
+GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} -${max_cleanup_max} hours")
 RDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} -${FHMAX_GFS} hours")
 if (( GDATE < RDATE )); then
     RDATE=${GDATE}
 fi
+
 deletion_target="${ROTDIR}/${RUN}.${RDATE:0:8}"
-if [[ -d ${deletion_target} ]]; then rm -rf "${deletion_target}"; fi
+if [[ -d "${deletion_target}" ]]; then
+    rm -rf "${deletion_target}"
+fi
 
 # sync and wait to avoid filesystem synchronization issues
 sync && sleep 1

ush/gfs_sndp.sh

@@ -10,7 +10,7 @@
   #  Create "collectives" consisting of groupings of the soundings
   #  into files designated by geographical region.   Each input
   #  file gfs_collective*.list (1-9) contains the list of stations to
-  #  put in a particular collective output file. 
+  #  put in a particular collective output file.
 export m=$1
 mkdir $DATA/$m
 cd $DATA/$m
@@ -19,18 +19,21 @@ cd $DATA/$m
     file_list=gfs_collective${m}.list
 
     if [ $m -le 2 ]
-    then 
+    then
       WMOHEAD=JUSA4$m
     elif [ $m -le 6 ]
-    then 
+    then
       WMOHEAD=JUSB4$m
-    else 
+    else
       WMOHEAD=JUSX4$m
     fi
 
     for stn in $(cat $file_list)
     do
-       cpreq "${COMOUT_ATMOS_BUFR}/bufr.${stn}.${PDY}${cyc}" "${DATA}/${m}/bufrin"
+       bufr_target="bufr.${stn}.${PDY}${cyc}"
+       cpreq "${COMIN_ATMOS_BUFR}/${bufr_target}" "${DATA}/${m}/bufrin"
+       # Also write the filename to a list for files to tar
+       echo "${bufr_target}" >> "${DATA}/${m}/bufr_snd_files.list"
        export pgm=tocsbufr.x
        #. prep_step
        export FORT11=$DATA/${m}/bufrin
@@ -59,4 +62,3 @@ EOF
 				   "${COMOUT_ATMOS_WMO}/gfs_collective${m}.postsnd_${cyc}"
     fi
     cpfs "${DATA}/${m}/gfs_collective${m}.fil" "${COMOUT_ATMOS_BUFR}/."
-

ush/preamble.sh

@@ -33,8 +33,9 @@ declare -x PS4='+ $(basename ${BASH_SOURCE[0]:-${FUNCNAME[0]:-"Unknown"}})[${LIN
 
 set_strict() {
     if [[ ${STRICT:-"YES"} == "YES" ]]; then
-        # Exit on error and undefined variable
-        set -eu
+        # Exit on error or undefined variable
+        # TODO: Also error in a pipeline (e.g. if and command in "cmd | cmd2" fails)
+        set -eu # -o pipefail
     fi
 }