From d0c5482f1e6ef5331abe526ec58c51ed502c97a3 Mon Sep 17 00:00:00 2001 From: gerard ketefian Date: Wed, 14 Oct 2020 17:54:13 -0500 Subject: [PATCH] Add and call a function that checks for use of Thompson microphysics parameterization in the SDF and if so, adjusts certain workflow arrays to contain the names and other associated values of the fixed files needed by this parameterization so that those files are automatically copied and/or linked to. --- scripts/exregional_run_fcst.sh | 155 ++++++++++++++-------------- ush/generate_FV3LAM_wflow.sh | 19 ---- ush/set_ozone_param.sh | 4 +- ush/set_thompson_mp_fix_files.sh | 168 +++++++++++++++++++++++++++++++ ush/setup.sh | 15 ++- 5 files changed, 261 insertions(+), 100 deletions(-) create mode 100644 ush/set_thompson_mp_fix_files.sh diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 8c86f4db1..cc92781fc 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -88,57 +88,63 @@ print_input_args valid_args #----------------------------------------------------------------------- # case $MACHINE in -# -"WCOSS_CRAY") -# - ulimit -s unlimited - ulimit -a - APRUN="aprun -b -j1 -n${PE_MEMBER01} -N24 -d1 -cc depth" - ;; -# -"WCOSS_DELL_P3") - ulimit -s unlimited - ulimit -a - APRUN="mpirun -l -np ${PE_MEMBER01}" - ;; -# -"HERA") - ulimit -s unlimited - ulimit -a - APRUN="srun" - OMP_NUM_THREADS=4 - ;; -# -"JET") - ulimit -s unlimited - ulimit -a - APRUN="srun" - OMP_NUM_THREADS=4 - ;; -# -"ODIN") -# - module list - - ulimit -s unlimited - ulimit -a - APRUN="srun -n ${PE_MEMBER01}" - ;; -# -"CHEYENNE") -# - module list - nprocs=$(( NNODES_RUN_FCST*PPN_RUN_FCST )) - APRUN="mpirun -np $nprocs" - ;; -# - -"STAMPEDE") -# - module list - APRUN="ibrun -np ${PE_MEMBER01}" - ;; + "WCOSS_CRAY") + ulimit -s unlimited + ulimit -a + APRUN="aprun -b -j1 -n${PE_MEMBER01} -N24 -d1 -cc depth" + ;; + + "WCOSS_DELL_P3") + ulimit -s unlimited + ulimit -a + APRUN="mpirun -l -np ${PE_MEMBER01}" + ;; + + "HERA") + ulimit -s unlimited + ulimit -a + APRUN="srun" + OMP_NUM_THREADS=4 + ;; + + "ORION") + ulimit -s unlimited + ulimit -a + APRUN="srun" + ;; + + "JET") + ulimit -s unlimited + ulimit -a + APRUN="srun" + OMP_NUM_THREADS=4 + ;; + + "ODIN") + module list + ulimit -s unlimited + ulimit -a + APRUN="srun -n ${PE_MEMBER01}" + ;; + + "CHEYENNE") + module list + nprocs=$(( NNODES_RUN_FCST*PPN_RUN_FCST )) + APRUN="mpirun -np $nprocs" + ;; + + "STAMPEDE") + module list + APRUN="ibrun -np ${PE_MEMBER01}" + ;; + + *) + print_err_msg_exit "\ +Run command has not been specified for this machine: + MACHINE = \"$MACHINE\" + APRUN = \"$APRUN\"" + ;; esac # @@ -152,13 +158,13 @@ run_dir="${cycle_dir}${slash_ensmem_subdir}" # #----------------------------------------------------------------------- # -# Create links in the INPUT subdirectory of the current run directory to +# Create links in the INPUT subdirectory of the current run directory to # the grid and (filtered) orography files. # #----------------------------------------------------------------------- # print_info_msg "$VERBOSE" " -Creating links in the INPUT subdirectory of the current run directory to +Creating links in the INPUT subdirectory of the current run directory to the grid and (filtered) orography files ..." @@ -249,14 +255,14 @@ Cannot create symlink because target does not exist: fi # # If using the FV3_RRFS_v1beta physics suite, there are two files (that -# contain statistics of the orography) that are needed by the drag +# contain statistics of the orography) that are needed by the drag # parameterization in that suite. Below, symlinks to these are created # in the run directory. Note that the symlinks must have specific names -# that the FV3 model is hardcoded to recognize ("${CRES}_" and "halo0" +# that the FV3 model is hardcoded to recognize ("${CRES}_" and "halo0" # must be stripped from the file names). We use those below. # if [ "${CCPP_PHYS_SUITE}" = "FV3_RRFS_v1beta" ]; then -# Symlink to orographic statistics fields file with "${CRES}_" and "halo0" +# Symlink to orographic statistics fields file with "${CRES}_" and "halo0" # stripped from name. target="${FIXLAM}/${CRES}${DOT_OR_USCORE}oro_data_ls.tile${TILE_RGNL}.halo${NH0}.nc" symlink="oro_data_ls.nc" @@ -312,7 +318,7 @@ fi # gfs_bndy*.nc # gfs_ctrl.nc # -# Some of these files (gfs_ctrl.nc, gfs_bndy*.nc) already exist, but +# Some of these files (gfs_ctrl.nc, gfs_bndy*.nc) already exist, but # others do not. Thus, create links with these names to the appropriate # files (in this case the initial condition and surface files only). # @@ -353,9 +359,9 @@ fi #----------------------------------------------------------------------- # # Create links in the current run directory to fixed (i.e. static) files -# in the FIXam directory. These links have names that are set to the +# in the FIXam directory. These links have names that are set to the # names of files that the forecast model expects to exist in the current -# working directory when the forecast model executable is called (and +# working directory when the forecast model executable is called (and # that is just the run directory). # #----------------------------------------------------------------------- @@ -363,7 +369,7 @@ fi cd_vrfy ${run_dir} print_info_msg "$VERBOSE" " -Creating links in the current run directory (run_dir) to fixed (i.e. +Creating links in the current run directory (run_dir) to fixed (i.e. static) files in the FIXam directory: FIXam = \"${FIXam}\" run_dir = \"${run_dir}\"" @@ -397,8 +403,8 @@ done # #----------------------------------------------------------------------- # -# If running this cycle/ensemble member combination more than once (e.g. -# using rocotoboot), remove any time stamp file that may exist from the +# If running this cycle/ensemble member combination more than once (e.g. +# using rocotoboot), remove any time stamp file that may exist from the # previous attempt. # #----------------------------------------------------------------------- @@ -409,13 +415,13 @@ rm_vrfy -f time_stamp.out #----------------------------------------------------------------------- # # Create links in the current run directory to cycle-independent (and -# ensemble-member-independent) model input files in the main experiment +# ensemble-member-independent) model input files in the main experiment # directory. # #----------------------------------------------------------------------- # print_info_msg "$VERBOSE" " -Creating links in the current run directory to cycle-independent model +Creating links in the current run directory to cycle-independent model input files in the main experiment directory..." relative_or_null="" @@ -432,13 +438,6 @@ if [ "${DO_ENSEMBLE}" = TRUE ]; then else ln_vrfy -sf ${relative_or_null} ${FV3_NML_FP} ${run_dir} fi - -ln_vrfy -sf ${relative_or_null} ${CCPP_PHYS_SUITE_FP} ${run_dir} -if [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_v0" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_RRFS_v1beta" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_SAR" ]; then - ln_vrfy -sf ${relative_or_null} $EXPTDIR/CCN_ACTIVATE.BIN ${run_dir} -fi # #----------------------------------------------------------------------- # @@ -451,17 +450,17 @@ create_model_configure_file \ cdate="$cdate" \ nthreads=${OMP_NUM_THREADS:-1} \ run_dir="${run_dir}" || print_err_msg_exit "\ -Call to function to create a model configuration file for the current +Call to function to create a model configuration file for the current cycle's (cdate) run directory (run_dir) failed: cdate = \"${cdate}\" run_dir = \"${run_dir}\"" # #----------------------------------------------------------------------- # -# If running ensemble forecasts, create a link to the cycle-specific -# diagnostic tables file in the cycle directory. Note that this link -# should not be made if not running ensemble forecasts because in that -# case, the cycle directory is the run directory (and we would be creating +# If running ensemble forecasts, create a link to the cycle-specific +# diagnostic tables file in the cycle directory. Note that this link +# should not be made if not running ensemble forecasts because in that +# case, the cycle directory is the run directory (and we would be creating # a symlink with the name of a file that already exists). # #----------------------------------------------------------------------- @@ -486,15 +485,15 @@ export OMP_STACKSIZE=1024m #----------------------------------------------------------------------- # # Run the FV3-LAM model. Note that we have to launch the forecast from -# the current cycle's directory because the FV3 executable will look for -# input files in the current directory. Since those files have been +# the current cycle's directory because the FV3 executable will look for +# input files in the current directory. Since those files have been # staged in the cycle directory, the current directory must be the cycle # directory (which it already is). # #----------------------------------------------------------------------- # $APRUN ${FV3_EXEC_FP} || print_err_msg_exit "\ -Call to executable to run FV3-LAM forecast returned with nonzero exit +Call to executable to run FV3-LAM forecast returned with nonzero exit code." # #----------------------------------------------------------------------- diff --git a/ush/generate_FV3LAM_wflow.sh b/ush/generate_FV3LAM_wflow.sh index 02140cfe3..7391b3456 100755 --- a/ush/generate_FV3LAM_wflow.sh +++ b/ush/generate_FV3LAM_wflow.sh @@ -518,25 +518,6 @@ print_info_msg "$VERBOSE" " Copying the CCPP physics suite definition XML file from its location in the forecast model directory sturcture to the experiment directory..." cp_vrfy "${CCPP_PHYS_SUITE_IN_CCPP_FP}" "${CCPP_PHYS_SUITE_FP}" -# -# If using a physics suite that includes the Thompson microphysics -# parameterization, copy the fixed file containing cloud condensation -# nuclei (CCN) data that is needed by that parameterization. -# -if [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_v0" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_RRFS_v1beta" ] || \ - [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_SAR" ]; then - print_info_msg "$VERBOSE" " -Copying the fixed file containing cloud condensation nuclei (CCN) data -(needed by the Thompson microphysics parameterization) to the experiment -directory..." - cp_vrfy "${FIXgsm}/CCN_ACTIVATE.BIN" "$EXPTDIR" -fi - - - - - # #----------------------------------------------------------------------- # diff --git a/ush/set_ozone_param.sh b/ush/set_ozone_param.sh index df2dfa704..5fb90f839 100644 --- a/ush/set_ozone_param.sh +++ b/ush/set_ozone_param.sh @@ -203,9 +203,9 @@ After setting the file name of the ozone production/loss file in the FIXgsm directory (based on the ozone parameterization specified in the CCPP suite definition file), the array specifying the mapping between the symlinks that need to be created in the cycle directories and the -files in the FIXgsm directory is: +files in the FIXam directory is: -CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING = ( \\ + CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING = ( \\ " msg="$msg"$( printf "\"%s\" \\\\\n" "${CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING[@]}" ) msg="$msg"$( printf "\n)" ) diff --git a/ush/set_thompson_mp_fix_files.sh b/ush/set_thompson_mp_fix_files.sh new file mode 100644 index 000000000..a2bd16c48 --- /dev/null +++ b/ush/set_thompson_mp_fix_files.sh @@ -0,0 +1,168 @@ +# +#----------------------------------------------------------------------- +# +# This file defins a function that first checks whether the Thompson +# microphysics parameterization is being called by the selected physics +# suite. If not, it does nothing else. If so, it modifies the workflow +# arrays FIXgsm_FILES_TO_COPY_TO_FIXam and CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING +# to ensure that fixed files needed by this parameterization are copied +# to the FIXam directory and that appropriate symlinks to these files +# are created in the run directories. +# +#----------------------------------------------------------------------- +# +function set_thompson_mp_fix_files() { +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# + { save_shell_opts; set -u -x; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Get the full path to the file in which this script/function is located +# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in +# which the file is located (scrfunc_dir). +# +#----------------------------------------------------------------------- +# + local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) + local scrfunc_fn=$( basename "${scrfunc_fp}" ) + local scrfunc_dir=$( dirname "${scrfunc_fp}" ) +# +#----------------------------------------------------------------------- +# +# Get the name of this function. +# +#----------------------------------------------------------------------- +# + local func_name="${FUNCNAME[0]}" +# +#----------------------------------------------------------------------- +# +# Specify the set of valid argument names for this script/function. Then +# process the arguments provided to this script/function (which should +# consist of a set of name-value pairs of the form arg1="value1", etc). +# +#----------------------------------------------------------------------- +# + local valid_args=( \ + "ccpp_phys_suite_fp" \ + ) + process_args valid_args "$@" +# +#----------------------------------------------------------------------- +# +# For debugging purposes, print out values of arguments passed to this +# script. Note that these will be printed out only if VERBOSE is set to +# TRUE. +# +#----------------------------------------------------------------------- +# + print_input_args valid_args +# +#----------------------------------------------------------------------- +# +# Declare local variables. +# +#----------------------------------------------------------------------- +# + local thompson_mp_name \ + regex_search \ + thompson_mp_name_or_null \ + thompson_mp_is_used \ + thompson_mp_fix_files \ + num_files \ + mapping \ + msg +# +#----------------------------------------------------------------------- +# +# Check the suite definition file to see whether the Thompson microphysics +# parameterization is being used. +# +#----------------------------------------------------------------------- +# + thompson_mp_name="mp_thompson" + regex_search="^[ ]*(${thompson_mp_name})<\/scheme>[ ]*$" + thompson_mp_name_or_null=$( sed -r -n -e "s/${regex_search}/\1/p" "${ccpp_phys_suite_fp}" ) + + if [ "${thompson_mp_name_or_null}" = "${thompson_mp_name}" ]; then + thompson_mp_is_used="TRUE" + elif [ -z "${thompson_mp_name_or_null}" ]; then + thompson_mp_is_used="FALSE" + else + print_err_msg_exit "\ +Unexpected value returned for thompson_mp_name_or_null: + thompson_mp_name_or_null = \"${thompson_mp_name_or_null}\" +This variable should be set to either \"${thompson_mp_name}\" or an empty +string." + fi +# +#----------------------------------------------------------------------- +# +# If the Thompson microphysics parameterization is being used, then +# append the names of the fixed files needed by this parameterization to +# the workflow array FIXgsm_FILES_TO_COPY_TO_FIXam, and append to the +# workflow array CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING the mappings +# between these files and the names of the corresponding symlinks that +# need to be created in the run directories. +# +#----------------------------------------------------------------------- +# + if [ "${thompson_mp_is_used}" = "TRUE" ]; then + + thompson_mp_fix_files=( \ + "CCN_ACTIVATE.BIN" \ + "freezeH2O.dat" \ + "qr_acr_qg.dat" \ + "qr_acr_qs.dat" \ + ) + + FIXgsm_FILES_TO_COPY_TO_FIXam+=( "${thompson_mp_fix_files[@]}" ) + + num_files=${#thompson_mp_fix_files[@]} + for (( i=0; i<${num_files}; i++ )); do + mapping="${thompson_mp_fix_files[i]} | ${thompson_mp_fix_files[i]}" + CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING+=( "${mapping}" ) + done + + msg=" +Since the Thompson microphysics parameterization is being used by this +physics suite (CCPP_PHYS_SUITE), the names of the fixed files needed by +this scheme have been appended to the array FIXgsm_FILES_TO_COPY_TO_FIXam, +and the mappings between these files and the symlinks that need to be +created in the cycle directories have been appended to the array +CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING. After these modifications, the +values of these parameters are as follows: + + CCPP_PHYS_SUITE = \"${CCPP_PHYS_SUITE}\" + + FIXgsm_FILES_TO_COPY_TO_FIXam = ( \\ +" + msg="$msg"$( printf "\"%s\" \\\\\n" "${FIXgsm_FILES_TO_COPY_TO_FIXam[@]}" ) + msg="$msg"$( printf "\n)" ) + msg="$msg + + CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING = ( \\ +" + msg="$msg"$( printf "\"%s\" \\\\\n" "${CYCLEDIR_LINKS_TO_FIXam_FILES_MAPPING[@]}" ) + msg="$msg"$( printf "\n)" ) + print_info_msg "$msg" + + fi +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/function. +# +#----------------------------------------------------------------------- +# + { restore_shell_opts; } > /dev/null 2>&1 + +} + diff --git a/ush/setup.sh b/ush/setup.sh index d25fdd6c0..058973d5d 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -62,6 +62,7 @@ cd_vrfy ${scrfunc_dir} . ./set_gridparams_ESGgrid.sh . ./link_fix.sh . ./set_ozone_param.sh +. ./set_thompson_mp_fix_files.sh # #----------------------------------------------------------------------- # @@ -2116,7 +2117,19 @@ mkdir_vrfy -p "$EXPTDIR" - +# +#----------------------------------------------------------------------- +# +# Call the function that, if the Thompson microphysics parameterization +# is being called by the physics suite, modifies certain workflow arrays +# to ensure that fixed files needed by this parameterization are copied +# to the FIXam directory and appropriate symlinks to them are created in +# the run directories. +# +#----------------------------------------------------------------------- +# +set_thompson_mp_fix_files \ + ccpp_phys_suite_fp="${CCPP_PHYS_SUITE_IN_CCPP_FP}"