Change the "overwrite" option to "upgrade" for preexisting directory

ush/bash_utils/check_for_preexist_dir_file.sh

@@ -63,7 +63,7 @@ where the arguments are defined as follows:
 
   method:
   String specifying the action to take if a preexisting version of 
-  dir_or_file is found.  Valid values are \"delete\", \"overwrite\", \"rename\", and \"quit\".
+  dir_or_file is found.  Valid values are \"delete\", \"upgrade\", \"rename\", and \"quit\".
 "
 
   fi
@@ -84,7 +84,7 @@ where the arguments are defined as follows:
 #
 #-----------------------------------------------------------------------
 #
-  local valid_vals_method=( "delete" "overwrite" "rename" "quit" )
+  local valid_vals_method=( "delete" "upgrade" "rename" "quit" )
   check_var_valid_value "method" "valid_vals_method"
 #
 #-----------------------------------------------------------------------
@@ -112,15 +112,42 @@ where the arguments are defined as follows:
 #
 #-----------------------------------------------------------------------
 #
-# If method is set to "overwrite", just overwrite existing directory 
-# file. Useful to keep ongoing run uninterrupted in some situations:
-#   rocotoco *db files and dependent previous cycle dirs stay
+# If method is set to "upgrade", 
+#  keep preexisting directory intact except that
+#  when preexisting directory is $EXPDIR, do the following:
+#    save all old files to a subdirecotry oldxxx/ and then
+#    populate new files into the $EXPDIR directory
+# This is useful to keep ongoing runs uninterrupted:
+#  rocotoco *db files and previous cycles will stay and hence
+#    1. no need to manually cp/mv *db files and previous cycles back
+#    2. no need to manually restart related rocoto tasks failed during
+#         the workflow generation process
+# This may best suit for incremental system upgrades.
+#
+# Alternatively, one can always elect to use the "rename" option 
+#   and then manually do the above aftermath
 #
 #-----------------------------------------------------------------------
 #
-    "overwrite")
+    "upgrade")
+      if [[ "${dir_or_file}" == "${EXPTDIR}" ]]; then
+        local i=1
+        local old_indx=$( printf "%03d" "$i" )
+        local old_dir_or_file="${dir_or_file}/old${old_indx}"
+        while [ -d "${old_dir_or_file}" ]; do
+          i=$[$i+1]
+          old_indx=$( printf "%03d" "$i" )
+          old_dir_or_file="${dir_or_file}/old${old_indx}"
+        done
+
+        print_info_msg "$VERBOSE" "
+  Specified directory or file (dir_or_file) already exists:
+    dir_or_file = \"${dir_or_file}\"
+  Moving (renaming) preexisting directory or file to:
+    old_dir_or_file = \"${old_dir_or_file}\""
 
-      continue
+        rsync_vrfy -a --exclude "old*" "${dir_or_file}/" "${old_dir_or_file}"
+      fi
       ;;
 #
 #-----------------------------------------------------------------------

ush/bash_utils/filesys_cmds_vrfy.sh

@@ -234,6 +234,12 @@ function cp_vrfy() {
   { restore_shell_opts; } > /dev/null 2>&1
 }
 
+function rsync_vrfy() {
+  { save_shell_opts; set -u +x; } > /dev/null 2>&1
+  filesys_cmd_vrfy "rsync" "$@"
+  { restore_shell_opts; } > /dev/null 2>&1
+}
+
 function mv_vrfy() {
   { save_shell_opts; set -u +x; } > /dev/null 2>&1
   filesys_cmd_vrfy "mv" "$@"

ush/config.sh.3DRTMA_dev1

@@ -24,7 +24,7 @@ FIX_CRTM=/home/rtrr/FIX_EXEC_MODULE/crtm/CRTM_v2.3.0
 VERBOSE="TRUE"
 
 RUN_ENVIR="nco"
-PREEXISTING_DIR_METHOD="overwrite"
+PREEXISTING_DIR_METHOD="upgrade"
 
 PREDEF_GRID_NAME=RRFS_CONUS_3km
 

ush/config.sh.RRFS_AK_dev1

@@ -54,7 +54,7 @@ if [[ $DO_RETRO == "true" ]] ; then
     BOUNDARY_CYCLEDEF="00 00-02,04-08,10-14,16-20,22,23 ${CYCLEDAY} ${CYCLEMONTH} 2021 *"
     BOUNDARY_LONG_CYCLEDEF="00 03,09,15,21 ${CYCLEDAY} ${CYCLEMONTH} 2021 *"
   else
-    PREEXISTING_DIR_METHOD="overwrite"
+    PREEXISTING_DIR_METHOD="upgrade"
     PREP_COLDSTART_CYCLEDEF="00 09,21 ${CYCLEDAY} ${CYCLEMONTH} 2021 *"
     PREP_WARMSTART_CYCLEDEF="00 00-08,10-20,22,23 ${CYCLEDAY} ${CYCLEMONTH} 2021 *"
     ANALYSIS_CYCLEDEF="00 00-23 ${CYCLEDAY} ${CYCLEMONTH} 2021 *"

ush/config.sh.RRFS_NA_13km

@@ -54,7 +54,7 @@ if [[ $DO_RETRO == "true" ]] ; then
     INITIAL_CYCLEDEF="00 03,15 ${CYCLEDAY} ${CYCLEMONTH} 2021 *"
     BOUNDARY_CYCLEDEF="00 00,06,12,18 ${CYCLEDAY} ${CYCLEMONTH} 2021 *"
   else
-    PREEXISTING_DIR_METHOD="overwrite"
+    PREEXISTING_DIR_METHOD="upgrade"
     PREP_COLDSTART_CYCLEDEF="00 03,15 ${CYCLEDAY} ${CYCLEMONTH} 2021 *"
     PREP_WARMSTART_CYCLEDEF="00 00-02,04-14,16-23 ${CYCLEDAY} ${CYCLEMONTH} 2021 *"
     ANALYSIS_CYCLEDEF="00 00-23/01 ${CYCLEDAY} ${CYCLEMONTH} 2021 *"

ush/config.sh.RRFS_NA_3km

@@ -55,7 +55,7 @@ if [[ $DO_RETRO == "true" ]] ; then
     INITIAL_CYCLEDEF="00 00,12 ${CYCLEDAY} ${CYCLEMONTH} 2021 *"
     BOUNDARY_CYCLEDEF="00 00,12 ${CYCLEDAY} ${CYCLEMONTH} 2021 *"
   else
-    PREEXISTING_DIR_METHOD="overwrite"
+    PREEXISTING_DIR_METHOD="upgrade"
     PREP_COLDSTART_CYCLEDEF="00 00,12 ${CYCLEDAY} ${CYCLEMONTH} 2021 *"
     #PREP_WARMSTART_CYCLEDEF="00 00-02,04-14,16-23 ${CYCLEDAY} ${CYCLEMONTH} 2021 *" <!-- placeholder; future capability -->
     #ANALYSIS_CYCLEDEF="00 00-23/01 ${CYCLEDAY} ${CYCLEMONTH} 2021 *"                <!-- placeholder; future capability -->

ush/config.sh.RRFS_dev1

@@ -71,7 +71,7 @@ if [[ $DO_RETRO == "true" ]] ; then
     BOUNDARY_CYCLEDEF="00 00-02/01,04-08/01,10-14/01,16-20/01,22,23 ${CYCLEDAY} ${CYCLEMONTH} 2021 *"
     BOUNDARY_LONG_CYCLEDEF="00 03,09,15,21 ${CYCLEDAY} ${CYCLEMONTH} 2021 *"
   else
-    PREEXISTING_DIR_METHOD="overwrite"
+    PREEXISTING_DIR_METHOD="upgrade"
     PREP_COLDSTART_CYCLEDEF="00 10,22 ${CYCLEDAY} ${CYCLEMONTH} 2021 *"
     PREP_WARMSTART_CYCLEDEF="00 00-09/01,11-21/01,23 ${CYCLEDAY} ${CYCLEMONTH} 2021 *"
     ANALYSIS_CYCLEDEF="00 00-23/01 ${CYCLEDAY} ${CYCLEMONTH} 2021 *"

ush/config.sh.RRFS_wcoss

@@ -53,7 +53,7 @@ if [[ $DO_RETRO == "true" ]] ; then
     INITIAL_CYCLEDEF="${DATE_FIRST_CYCL}0300 ${DATE_LAST_CYCL}2300 12:00:00"
     BOUNDARY_CYCLEDEF="${DATE_FIRST_CYCL}0000 ${DATE_LAST_CYCL}2300 06:00:00"
   else
-    PREEXISTING_DIR_METHOD="overwrite"
+    PREEXISTING_DIR_METHOD="upgrade"
     PREP_COLDSTART_CYCLEDEF="00 03,15 ${CYCLEDAY} ${CYCLEMONTH} 2021 *"
     PREP_WARMSTART_CYCLEDEF="00 00-02,04-14,16-23 ${CYCLEDAY} ${CYCLEMONTH} 2021 *"
     ANALYSIS_CYCLEDEF="00 00-23/01 ${CYCLEDAY} ${CYCLEMONTH} 2021 *"

ush/config_defaults.sh

@@ -1137,15 +1137,16 @@ PREDEF_GRID_NAME=""
 # use to deal with preexisting directories [e.g ones generated by previous
 # calls to the experiment generation script using the same experiment name
 # (EXPT_SUBDIR) as the current experiment].  This variable must be set to
-# one of "delete", "overwrite", "rename", and "quit".  The resulting behavior
+# one of "delete", "upgrade", "rename", and "quit".  The resulting behavior
 # for each of these values is as follows:
 #
 # * "delete":
 #   The preexisting directory is deleted and a new directory (having the
 #   same name as the original preexisting directory) is created.
 #
-# * "overwrite":
-#   overwrite preexisting directory
+# * "upgrade":
+#   save a copy and then upgrade the preexisting $EXPDIR directory
+#   keep intact for other preexisting directories
 #
 # * "rename":
 #   The preexisting directory is renamed and a new directory (having the

ush/source_util_funcs.sh

@@ -76,7 +76,7 @@ function source_util_funcs() {
 #
 # Source the file containing the function that checks for preexisting 
 # directories or files and handles them according to a specified method
-# (which can be one of "delete", "overwrite", "rename", and "quit").
+# (which can be one of "delete", "upgrade", "rename", and "quit").
 #
 #-----------------------------------------------------------------------
 #

ush/valid_param_vals.sh

@@ -42,7 +42,7 @@ valid_vals_USE_USER_STAGED_EXTRN_FILES=("TRUE" "true" "YES" "yes" "FALSE" "false
 valid_vals_FV3GFS_FILE_FMT_ICS=("nemsio" "grib2")
 valid_vals_FV3GFS_FILE_FMT_LBCS=("nemsio" "grib2")
 valid_vals_GRID_GEN_METHOD=("GFDLgrid" "ESGgrid")
-valid_vals_PREEXISTING_DIR_METHOD=("delete" "overwrite" "rename" "quit")
+valid_vals_PREEXISTING_DIR_METHOD=("delete" "upgrade" "rename" "quit")
 valid_vals_GTYPE=("regional")
 valid_vals_WRTCMP_output_grid=("rotated_latlon" "lambert_conformal" "regional_latlon")
 valid_vals_RUN_TASK_MAKE_GRID=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no")