diff --git a/ush/bash_utils/check_for_preexist_dir_file.sh b/ush/bash_utils/check_for_preexist_dir_file.sh index cf09f48d9..59231af9c 100644 --- a/ush/bash_utils/check_for_preexist_dir_file.sh +++ b/ush/bash_utils/check_for_preexist_dir_file.sh @@ -63,7 +63,7 @@ where the arguments are defined as follows: method: String specifying the action to take if a preexisting version of - dir_or_file is found. Valid values are \"delete\", \"overwrite\", \"rename\", and \"quit\". + dir_or_file is found. Valid values are \"delete\", \"upgrade\", \"rename\", and \"quit\". " fi @@ -84,7 +84,7 @@ where the arguments are defined as follows: # #----------------------------------------------------------------------- # - local valid_vals_method=( "delete" "overwrite" "rename" "quit" ) + local valid_vals_method=( "delete" "upgrade" "rename" "quit" ) check_var_valid_value "method" "valid_vals_method" # #----------------------------------------------------------------------- @@ -112,15 +112,42 @@ where the arguments are defined as follows: # #----------------------------------------------------------------------- # -# If method is set to "overwrite", just overwrite existing directory -# file. Useful to keep ongoing run uninterrupted in some situations: -# rocotoco *db files and dependent previous cycle dirs stay +# If method is set to "upgrade", +# keep preexisting directory intact except that +# when preexisting directory is $EXPDIR, do the following: +# save all old files to a subdirecotry oldxxx/ and then +# populate new files into the $EXPDIR directory +# This is useful to keep ongoing runs uninterrupted: +# rocotoco *db files and previous cycles will stay and hence +# 1. no need to manually cp/mv *db files and previous cycles back +# 2. no need to manually restart related rocoto tasks failed during +# the workflow generation process +# This may best suit for incremental system upgrades. +# +# Alternatively, one can always elect to use the "rename" option +# and then manually do the above aftermath # #----------------------------------------------------------------------- # - "overwrite") + "upgrade") + if [[ "${dir_or_file}" == "${EXPTDIR}" ]]; then + local i=1 + local old_indx=$( printf "%03d" "$i" ) + local old_dir_or_file="${dir_or_file}/old${old_indx}" + while [ -d "${old_dir_or_file}" ]; do + i=$[$i+1] + old_indx=$( printf "%03d" "$i" ) + old_dir_or_file="${dir_or_file}/old${old_indx}" + done + + print_info_msg "$VERBOSE" " + Specified directory or file (dir_or_file) already exists: + dir_or_file = \"${dir_or_file}\" + Moving (renaming) preexisting directory or file to: + old_dir_or_file = \"${old_dir_or_file}\"" - continue + rsync_vrfy -a --exclude "old*" "${dir_or_file}/" "${old_dir_or_file}" + fi ;; # #----------------------------------------------------------------------- diff --git a/ush/bash_utils/filesys_cmds_vrfy.sh b/ush/bash_utils/filesys_cmds_vrfy.sh index d11ec4720..d3aed0451 100644 --- a/ush/bash_utils/filesys_cmds_vrfy.sh +++ b/ush/bash_utils/filesys_cmds_vrfy.sh @@ -234,6 +234,12 @@ function cp_vrfy() { { restore_shell_opts; } > /dev/null 2>&1 } +function rsync_vrfy() { + { save_shell_opts; set -u +x; } > /dev/null 2>&1 + filesys_cmd_vrfy "rsync" "$@" + { restore_shell_opts; } > /dev/null 2>&1 +} + function mv_vrfy() { { save_shell_opts; set -u +x; } > /dev/null 2>&1 filesys_cmd_vrfy "mv" "$@" diff --git a/ush/config.sh.3DRTMA_dev1 b/ush/config.sh.3DRTMA_dev1 index 87a6ee7d5..39349d3e6 100644 --- a/ush/config.sh.3DRTMA_dev1 +++ b/ush/config.sh.3DRTMA_dev1 @@ -24,7 +24,7 @@ FIX_CRTM=/home/rtrr/FIX_EXEC_MODULE/crtm/CRTM_v2.3.0 VERBOSE="TRUE" RUN_ENVIR="nco" -PREEXISTING_DIR_METHOD="overwrite" +PREEXISTING_DIR_METHOD="upgrade" PREDEF_GRID_NAME=RRFS_CONUS_3km diff --git a/ush/config.sh.RRFS_AK_dev1 b/ush/config.sh.RRFS_AK_dev1 index ef87cb820..e14aaf658 100644 --- a/ush/config.sh.RRFS_AK_dev1 +++ b/ush/config.sh.RRFS_AK_dev1 @@ -54,7 +54,7 @@ if [[ $DO_RETRO == "true" ]] ; then BOUNDARY_CYCLEDEF="00 00-02,04-08,10-14,16-20,22,23 ${CYCLEDAY} ${CYCLEMONTH} 2021 *" BOUNDARY_LONG_CYCLEDEF="00 03,09,15,21 ${CYCLEDAY} ${CYCLEMONTH} 2021 *" else - PREEXISTING_DIR_METHOD="overwrite" + PREEXISTING_DIR_METHOD="upgrade" PREP_COLDSTART_CYCLEDEF="00 09,21 ${CYCLEDAY} ${CYCLEMONTH} 2021 *" PREP_WARMSTART_CYCLEDEF="00 00-08,10-20,22,23 ${CYCLEDAY} ${CYCLEMONTH} 2021 *" ANALYSIS_CYCLEDEF="00 00-23 ${CYCLEDAY} ${CYCLEMONTH} 2021 *" diff --git a/ush/config.sh.RRFS_NA_13km b/ush/config.sh.RRFS_NA_13km index 280cb686c..a37e7286d 100644 --- a/ush/config.sh.RRFS_NA_13km +++ b/ush/config.sh.RRFS_NA_13km @@ -54,7 +54,7 @@ if [[ $DO_RETRO == "true" ]] ; then INITIAL_CYCLEDEF="00 03,15 ${CYCLEDAY} ${CYCLEMONTH} 2021 *" BOUNDARY_CYCLEDEF="00 00,06,12,18 ${CYCLEDAY} ${CYCLEMONTH} 2021 *" else - PREEXISTING_DIR_METHOD="overwrite" + PREEXISTING_DIR_METHOD="upgrade" PREP_COLDSTART_CYCLEDEF="00 03,15 ${CYCLEDAY} ${CYCLEMONTH} 2021 *" PREP_WARMSTART_CYCLEDEF="00 00-02,04-14,16-23 ${CYCLEDAY} ${CYCLEMONTH} 2021 *" ANALYSIS_CYCLEDEF="00 00-23/01 ${CYCLEDAY} ${CYCLEMONTH} 2021 *" diff --git a/ush/config.sh.RRFS_NA_3km b/ush/config.sh.RRFS_NA_3km index 40b654b61..ab093ebd5 100644 --- a/ush/config.sh.RRFS_NA_3km +++ b/ush/config.sh.RRFS_NA_3km @@ -55,7 +55,7 @@ if [[ $DO_RETRO == "true" ]] ; then INITIAL_CYCLEDEF="00 00,12 ${CYCLEDAY} ${CYCLEMONTH} 2021 *" BOUNDARY_CYCLEDEF="00 00,12 ${CYCLEDAY} ${CYCLEMONTH} 2021 *" else - PREEXISTING_DIR_METHOD="overwrite" + PREEXISTING_DIR_METHOD="upgrade" PREP_COLDSTART_CYCLEDEF="00 00,12 ${CYCLEDAY} ${CYCLEMONTH} 2021 *" #PREP_WARMSTART_CYCLEDEF="00 00-02,04-14,16-23 ${CYCLEDAY} ${CYCLEMONTH} 2021 *" #ANALYSIS_CYCLEDEF="00 00-23/01 ${CYCLEDAY} ${CYCLEMONTH} 2021 *" diff --git a/ush/config.sh.RRFS_dev1 b/ush/config.sh.RRFS_dev1 index ac287936c..b409057b4 100644 --- a/ush/config.sh.RRFS_dev1 +++ b/ush/config.sh.RRFS_dev1 @@ -71,7 +71,7 @@ if [[ $DO_RETRO == "true" ]] ; then BOUNDARY_CYCLEDEF="00 00-02/01,04-08/01,10-14/01,16-20/01,22,23 ${CYCLEDAY} ${CYCLEMONTH} 2021 *" BOUNDARY_LONG_CYCLEDEF="00 03,09,15,21 ${CYCLEDAY} ${CYCLEMONTH} 2021 *" else - PREEXISTING_DIR_METHOD="overwrite" + PREEXISTING_DIR_METHOD="upgrade" PREP_COLDSTART_CYCLEDEF="00 10,22 ${CYCLEDAY} ${CYCLEMONTH} 2021 *" PREP_WARMSTART_CYCLEDEF="00 00-09/01,11-21/01,23 ${CYCLEDAY} ${CYCLEMONTH} 2021 *" ANALYSIS_CYCLEDEF="00 00-23/01 ${CYCLEDAY} ${CYCLEMONTH} 2021 *" diff --git a/ush/config.sh.RRFS_wcoss b/ush/config.sh.RRFS_wcoss index 8d61b4199..8b8f097d5 100644 --- a/ush/config.sh.RRFS_wcoss +++ b/ush/config.sh.RRFS_wcoss @@ -53,7 +53,7 @@ if [[ $DO_RETRO == "true" ]] ; then INITIAL_CYCLEDEF="${DATE_FIRST_CYCL}0300 ${DATE_LAST_CYCL}2300 12:00:00" BOUNDARY_CYCLEDEF="${DATE_FIRST_CYCL}0000 ${DATE_LAST_CYCL}2300 06:00:00" else - PREEXISTING_DIR_METHOD="overwrite" + PREEXISTING_DIR_METHOD="upgrade" PREP_COLDSTART_CYCLEDEF="00 03,15 ${CYCLEDAY} ${CYCLEMONTH} 2021 *" PREP_WARMSTART_CYCLEDEF="00 00-02,04-14,16-23 ${CYCLEDAY} ${CYCLEMONTH} 2021 *" ANALYSIS_CYCLEDEF="00 00-23/01 ${CYCLEDAY} ${CYCLEMONTH} 2021 *" diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index baaf9ae86..ddac2fe7a 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -1137,15 +1137,16 @@ PREDEF_GRID_NAME="" # use to deal with preexisting directories [e.g ones generated by previous # calls to the experiment generation script using the same experiment name # (EXPT_SUBDIR) as the current experiment]. This variable must be set to -# one of "delete", "overwrite", "rename", and "quit". The resulting behavior +# one of "delete", "upgrade", "rename", and "quit". The resulting behavior # for each of these values is as follows: # # * "delete": # The preexisting directory is deleted and a new directory (having the # same name as the original preexisting directory) is created. # -# * "overwrite": -# overwrite preexisting directory +# * "upgrade": +# save a copy and then upgrade the preexisting $EXPDIR directory +# keep intact for other preexisting directories # # * "rename": # The preexisting directory is renamed and a new directory (having the diff --git a/ush/source_util_funcs.sh b/ush/source_util_funcs.sh index 3e5912c67..3779869f1 100644 --- a/ush/source_util_funcs.sh +++ b/ush/source_util_funcs.sh @@ -76,7 +76,7 @@ function source_util_funcs() { # # Source the file containing the function that checks for preexisting # directories or files and handles them according to a specified method -# (which can be one of "delete", "overwrite", "rename", and "quit"). +# (which can be one of "delete", "upgrade", "rename", and "quit"). # #----------------------------------------------------------------------- # diff --git a/ush/valid_param_vals.sh b/ush/valid_param_vals.sh index c5714d035..666e0b2f9 100644 --- a/ush/valid_param_vals.sh +++ b/ush/valid_param_vals.sh @@ -42,7 +42,7 @@ valid_vals_USE_USER_STAGED_EXTRN_FILES=("TRUE" "true" "YES" "yes" "FALSE" "false valid_vals_FV3GFS_FILE_FMT_ICS=("nemsio" "grib2") valid_vals_FV3GFS_FILE_FMT_LBCS=("nemsio" "grib2") valid_vals_GRID_GEN_METHOD=("GFDLgrid" "ESGgrid") -valid_vals_PREEXISTING_DIR_METHOD=("delete" "overwrite" "rename" "quit") +valid_vals_PREEXISTING_DIR_METHOD=("delete" "upgrade" "rename" "quit") valid_vals_GTYPE=("regional") valid_vals_WRTCMP_output_grid=("rotated_latlon" "lambert_conformal" "regional_latlon") valid_vals_RUN_TASK_MAKE_GRID=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no")