From e0a632f1d34c78f4ae498cc279f5061ddfac900d Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Thu, 5 Sep 2024 15:20:04 +0000
Subject: [PATCH 1/3] Wait to delete DATAROOT until the end of the cleanup job

---
 scripts/exglobal_cleanup.sh | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/scripts/exglobal_cleanup.sh b/scripts/exglobal_cleanup.sh
index 75b1f927bc2..85573751493 100755
--- a/scripts/exglobal_cleanup.sh
+++ b/scripts/exglobal_cleanup.sh
@@ -11,14 +11,6 @@ DATAfcst="${DATAROOT}/${RUN}fcst.${PDY:-}${cyc}"
 if [[ -d "${DATAfcst}" ]]; then rm -rf "${DATAfcst}"; fi
 #DATAefcs="${DATAROOT}/${RUN}efcs???${PDY:-}${cyc}"
 rm -rf "${DATAROOT}/${RUN}efcs"*"${PDY:-}${cyc}"
-
-# In XML, DATAROOT is defined as:
-#DATAROOT="${STMP}/RUNDIRS/${PSLOT}/${RUN}.${PDY}${cyc}"
-# cleanup is only executed after the entire cycle is successfully completed.
-# removing DATAROOT should be possible if that is the case.
-rm -rf "${DATAROOT}"
-
-echo "Cleanup ${DATAROOT} completed!"
 ###############################################################
 
 if [[ "${CLEANUP_COM:-YES}" == NO ]] ; then
@@ -113,3 +105,13 @@ if (( GDATE < RDATE )); then
 fi
 deletion_target="${ROTDIR}/${RUN}.${RDATE:0:8}"
 if [[ -d ${deletion_target} ]]; then rm -rf "${deletion_target}"; fi
+
+# Finally, delete DATAROOT.
+# This will also delete the working directory, so save it until the end.
+# In XML, DATAROOT is defined as:
+#DATAROOT="${STMP}/RUNDIRS/${PSLOT}/${RUN}.${PDY}${cyc}"
+# cleanup is only executed after the entire cycle is successfully completed.
+# removing DATAROOT should be possible if that is the case.
+rm -rf "${DATAROOT}"
+
+echo "Cleanup ${DATAROOT} completed!"

From b502db38c0007f92cfe198af5fb84e21d5f40eeb Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Thu, 5 Sep 2024 16:23:25 +0000
Subject: [PATCH 2/3] Add -ignore_readdir_race to find to prevent errors if a
 file has already been deleted

---
 scripts/exglobal_cleanup.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/exglobal_cleanup.sh b/scripts/exglobal_cleanup.sh
index 85573751493..e95110434ae 100755
--- a/scripts/exglobal_cleanup.sh
+++ b/scripts/exglobal_cleanup.sh
@@ -41,10 +41,10 @@ function remove_files() {
     find_exclude_string="${find_exclude_string[*]/%-or}"
     # Remove all regular files that do not match
     # shellcheck disable=SC2086
-    find "${directory}" -type f -not \( ${find_exclude_string} \) -delete
+    find "${directory}" -type f -not \( ${find_exclude_string} \) -ignore_readdir_race -delete
     # Remove all symlinks that do not match
     # shellcheck disable=SC2086
-    find "${directory}" -type l -not \( ${find_exclude_string} \) -delete
+    find "${directory}" -type l -not \( ${find_exclude_string} \) -ignore_readdir_race -delete
     # Remove any empty directories
     find "${directory}" -type d -empty -delete
 }

From 36e664d149d178ea83afe9a24ba4ebbe05a855e6 Mon Sep 17 00:00:00 2001
From: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com>
Date: Fri, 6 Sep 2024 07:55:00 -0400
Subject: [PATCH 3/3] Update scripts/exglobal_cleanup.sh

Co-authored-by: Rahul Mahajan <aerorahul@users.noreply.github.com>
---
 scripts/exglobal_cleanup.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scripts/exglobal_cleanup.sh b/scripts/exglobal_cleanup.sh
index e95110434ae..73637a0d558 100755
--- a/scripts/exglobal_cleanup.sh
+++ b/scripts/exglobal_cleanup.sh
@@ -106,6 +106,9 @@ fi
 deletion_target="${ROTDIR}/${RUN}.${RDATE:0:8}"
 if [[ -d ${deletion_target} ]]; then rm -rf "${deletion_target}"; fi
 
+# sync and wait to avoid filesystem synchronization issues
+sync && sleep 1
+
 # Finally, delete DATAROOT.
 # This will also delete the working directory, so save it until the end.
 # In XML, DATAROOT is defined as: