diff --git a/jobs/JREGIONAL_GET_EXTRN_MDL_FILES b/jobs/JREGIONAL_GET_EXTRN_MDL_FILES index 48baa2691..94eb6dfc6 100755 --- a/jobs/JREGIONAL_GET_EXTRN_MDL_FILES +++ b/jobs/JREGIONAL_GET_EXTRN_MDL_FILES @@ -192,7 +192,7 @@ fi # #----------------------------------------------------------------------- # -extrn_mdl_staging_dir="${CYCLE_DIR}/${EXTRN_MDL_NAME}/${ICS_OR_LBCS}" +extrn_mdl_staging_dir="${CYCLE_DIR}/${EXTRN_MDL_NAME}/for_${ICS_OR_LBCS}" mkdir_vrfy -p "${extrn_mdl_staging_dir}" cd_vrfy "${extrn_mdl_staging_dir}" # diff --git a/jobs/JREGIONAL_MAKE_ICS b/jobs/JREGIONAL_MAKE_ICS index 8a4627f71..2ddf94bac 100755 --- a/jobs/JREGIONAL_MAKE_ICS +++ b/jobs/JREGIONAL_MAKE_ICS @@ -106,7 +106,7 @@ Directory in which the wgrib2 executable is located could not be found: # #----------------------------------------------------------------------- # -ics_dir="${CYCLE_DIR}/INPUT" +ics_dir="${CYCLE_DIR}${SLASH_ENSMEM_SUBDIR}/INPUT" mkdir_vrfy -p "${ics_dir}" # #----------------------------------------------------------------------- diff --git a/jobs/JREGIONAL_MAKE_LBCS b/jobs/JREGIONAL_MAKE_LBCS index a2d5ef6da..49923a0db 100755 --- a/jobs/JREGIONAL_MAKE_LBCS +++ b/jobs/JREGIONAL_MAKE_LBCS @@ -107,7 +107,7 @@ Directory in which the wgrib2 executable is located could not be found: # #----------------------------------------------------------------------- # -lbcs_dir="${CYCLE_DIR}/INPUT" +lbcs_dir="${CYCLE_DIR}${SLASH_ENSMEM_SUBDIR}/INPUT" mkdir_vrfy -p "${lbcs_dir}" # #----------------------------------------------------------------------- diff --git a/jobs/JREGIONAL_RUN_FCST b/jobs/JREGIONAL_RUN_FCST index 39e5c17b2..5ab82f99d 100755 --- a/jobs/JREGIONAL_RUN_FCST +++ b/jobs/JREGIONAL_RUN_FCST @@ -58,12 +58,13 @@ the specified cycle. # #----------------------------------------------------------------------- # -# Create the INPUT and RESTART directories under the cycle directory. +# Create the INPUT and RESTART directories under the run directory. # #----------------------------------------------------------------------- # -mkdir_vrfy -p ${CYCLE_DIR}/INPUT -mkdir_vrfy -p ${CYCLE_DIR}/RESTART +run_dir="${CYCLE_DIR}${SLASH_ENSMEM_SUBDIR}" +mkdir_vrfy -p ${run_dir}/INPUT +mkdir_vrfy -p ${run_dir}/RESTART # #----------------------------------------------------------------------- # @@ -73,7 +74,9 @@ mkdir_vrfy -p ${CYCLE_DIR}/RESTART #----------------------------------------------------------------------- # $SCRIPTSDIR/exregional_run_fcst.sh \ - CYCLE_DIR="${CYCLE_DIR}" || \ + cycle_dir="${CYCLE_DIR}" \ + ensmem_indx="${ENSMEM_INDX}" \ + slash_ensmem_subdir="${SLASH_ENSMEM_SUBDIR}" || \ print_err_msg_exit "\ Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." # diff --git a/jobs/JREGIONAL_RUN_POST b/jobs/JREGIONAL_RUN_POST index e6f354eda..28df0dc03 100755 --- a/jobs/JREGIONAL_RUN_POST +++ b/jobs/JREGIONAL_RUN_POST @@ -57,13 +57,21 @@ on the output files corresponding to a specified forecast hour. # #----------------------------------------------------------------------- # +# Set the run directory. +# +#----------------------------------------------------------------------- +# +run_dir="${CYCLE_DIR}${SLASH_ENSMEM_SUBDIR}" +# +#----------------------------------------------------------------------- +# # If it doesn't already exist, create the directory (postprd_dir) in which # to store post-processing output. (Note that postprd_dir may already # have been created by this post-processing script run for a different -# forecast hour.) Also, create a temporary work directory (fhr_dir) for -# the current forecast hour being processed. fhr_dir will be deleted -# later after the processing for the current forecast hour is complete. -# Then change location to fhr_dir. +# forecast hour of the same cycle and/or ensemble member.) Also, create +# a temporary work directory (fhr_dir) for the current forecast hour. +# fhr_dir will be deleted later after the processing for the current +# forecast hour is complete. Then change location to fhr_dir. # # Note that there may be a preexisting version of fhr_dir from previous # runs of this script for the current forecast hour (and current cycle), @@ -73,10 +81,10 @@ on the output files corresponding to a specified forecast hour. #----------------------------------------------------------------------- # if [ "${RUN_ENVIR}" = "nco" ]; then - COMOUT="${COMOUT_BASEDIR}/$RUN.$PDY/$cyc" + COMOUT="${COMOUT_BASEDIR}/$RUN.$PDY/$cyc${SLASH_ENSMEM_SUBDIR}" postprd_dir="$COMOUT" else - postprd_dir="${CYCLE_DIR}/postprd" + postprd_dir="${run_dir}/postprd" fi mkdir_vrfy -p "${postprd_dir}" @@ -118,7 +126,8 @@ fi #----------------------------------------------------------------------- # $SCRIPTSDIR/exregional_run_post.sh \ - cycle_dir="${CYCLE_DIR}" \ + cdate="${CDATE}" \ + run_dir="${run_dir}" \ postprd_dir="${postprd_dir}" \ fhr_dir="${fhr_dir}" \ fhr="${fhr}" || \ diff --git a/modulefiles/tasks/wcoss_cray/run_fcst.local b/modulefiles/tasks/wcoss_cray/run_fcst.local new file mode 100644 index 000000000..1dfb151a1 --- /dev/null +++ b/modulefiles/tasks/wcoss_cray/run_fcst.local @@ -0,0 +1,2 @@ +module unload python/2.7.14 +module load python/3.6.3 diff --git a/modulefiles/tasks/wcoss_dell_p3/run_fcst.local b/modulefiles/tasks/wcoss_dell_p3/run_fcst.local new file mode 100644 index 000000000..1dfb151a1 --- /dev/null +++ b/modulefiles/tasks/wcoss_dell_p3/run_fcst.local @@ -0,0 +1,2 @@ +module unload python/2.7.14 +module load python/3.6.3 diff --git a/scripts/exregional_make_grid.sh b/scripts/exregional_make_grid.sh index 48cfb49a2..cfe7ea562 100755 --- a/scripts/exregional_make_grid.sh +++ b/scripts/exregional_make_grid.sh @@ -19,6 +19,8 @@ . $USHDIR/make_grid_mosaic_file.sh . $USHDIR/link_fix.sh . $USHDIR/set_FV3nml_sfc_climo_filenames.sh +. $USHDIR/set_FV3nml_stoch_params.sh +. $USHDIR/create_diag_table_files.sh # #----------------------------------------------------------------------- # @@ -647,21 +649,46 @@ failed." # #----------------------------------------------------------------------- # -# Call a function to set the values of those variables in the forecast -# model's namelist file that specify the paths to the surface climatology -# files. These files will either already be avaialable in a user-specified -# directory (SFC_CLIMO_DIR) or will be generated by the MAKE_SFC_CLIMO_TN -# task. They (or symlinks to them) will be placed (or wll already exist) -# in the FIXsar directory. +# Call a function (set_FV3nml_sfc_climo_filenames) to set the values of +# those variables in the forecast model's namelist file that specify the +# paths to the surface climatology files. These files will either already +# be avaialable in a user-specified directory (SFC_CLIMO_DIR) or will be +# generated by the MAKE_SFC_CLIMO_TN task. They (or symlinks to them) +# will be placed (or wll already exist) in the FIXsar directory. # -# Note this action has to be performed here (instead of earlier during -# experiment generation) because the surface climatology file names depend -# on the grid resolution variable CRES, and that may not be available -# until the above steps in this script have been performed. +# Also, if running ensemble forecasts, call a function (set_FV3nml_stoch_params) +# to create a new FV3 namelist file for each ensemble member that contains +# a unique set of stochastic parameters (i.e. relative to the namelist +# files of the other members). +# +# Note that unless RUN_TASK_MAKE_GRID is set to "FALSE", the call to +# set_FV3nml_sfc_climo_filenames has to be performed here instead of +# earlier during experiment generation because the surface climatology +# file names depend on the grid resolution variable CRES, and that may +# not be available until the above steps in this script have been performed. +# +# Similarly, unless RUN_TASK_MAKE_GRID is set to "FALSE", the call to +# set_FV3nml_stoch_params must be performed here because it uses the +# namelist file generated by the call to set_FV3nml_sfc_climo_filenames +# as a starting point (base) and modifies it to add the stochastic +# parameters. Thus, the changes made by set_FV3nml_sfc_climo_filenames +# must already be in the base namelist file. # #----------------------------------------------------------------------- # -set_FV3nml_sfc_climo_filenames +set_FV3nml_sfc_climo_filenames || print_err_msg_exit "\ +Call to function to set surface climatology file names in the FV3 namelist +file failed." + +if [ "${DO_ENSEMBLE}" = TRUE ]; then + set_FV3nml_stoch_params || print_err_msg_exit "\ +Call to function to set stochastic parameters in the FV3 namelist files +for the various ensemble members failed." +fi + +create_diag_table_files || print_err_msg_exit "\ +Call to function to create a diagnostics table file under each cycle +directory failed." # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_make_ics.sh b/scripts/exregional_make_ics.sh index cb7e15c41..0da673482 100755 --- a/scripts/exregional_make_ics.sh +++ b/scripts/exregional_make_ics.sh @@ -21,7 +21,7 @@ # #----------------------------------------------------------------------- # -# Get the full path to the file in which this script/function is located +# Get the full path to the file in which this script/function is located # (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in # which the file is located (scrfunc_dir). # @@ -42,15 +42,15 @@ print_info_msg " Entering script: \"${scrfunc_fn}\" In directory: \"${scrfunc_dir}\" -This is the ex-script for the task that generates initial condition -(IC), surface, and zeroth hour lateral boundary condition (LBC0) files +This is the ex-script for the task that generates initial condition +(IC), surface, and zeroth hour lateral boundary condition (LBC0) files for FV3 (in NetCDF format). ========================================================================" # #----------------------------------------------------------------------- # -# Specify the set of valid argument names for this script/function. Then -# process the arguments provided to this script/function (which should +# Specify the set of valid argument names for this script/function. Then +# process the arguments provided to this script/function (which should # consist of a set of name-value pairs of the form arg1="value1", etc). # #----------------------------------------------------------------------- @@ -71,17 +71,17 @@ process_args valid_args "$@" #----------------------------------------------------------------------- # print_input_args valid_args -# -#----------------------------------------------------------------------- -# -# Source the file containing definitions of variables associated with the +# +#----------------------------------------------------------------------- +# +# Source the file containing definitions of variables associated with the # external model for ICs. -# -#----------------------------------------------------------------------- -# -extrn_mdl_staging_dir="${CYCLE_DIR}/${EXTRN_MDL_NAME_ICS}/ICS" -extrn_mdl_var_defns_fp="${extrn_mdl_staging_dir}/${EXTRN_MDL_ICS_VAR_DEFNS_FN}" -. ${extrn_mdl_var_defns_fp} +# +#----------------------------------------------------------------------- +# +extrn_mdl_staging_dir="${CYCLE_DIR}/${EXTRN_MDL_NAME_ICS}/for_ICS" +extrn_mdl_var_defns_fp="${extrn_mdl_staging_dir}/${EXTRN_MDL_ICS_VAR_DEFNS_FN}" +. ${extrn_mdl_var_defns_fp} # #----------------------------------------------------------------------- # @@ -121,7 +121,7 @@ case "${CCPP_PHYS_SUITE}" in ;; *) print_err_msg_exit "\ -Physics-suite-dependent namelist variables have not yet been specified +Physics-suite-dependent namelist variables have not yet been specified for this physics suite: CCPP_PHYS_SUITE = \"${CCPP_PHYS_SUITE}\"" ;; @@ -136,7 +136,7 @@ esac # subset of these all variables are set (since some may be irrelevant). # # external_model: -# Name of the external model from which we are obtaining the fields +# Name of the external model from which we are obtaining the fields # needed to generate the ICs. # # fn_atm_nemsio: @@ -150,22 +150,22 @@ esac # input_type: # The "type" of input being provided to chgres. This contains a combi- # nation of information on the external model, external model file for- -# mat, and maybe other parameters. For clarity, it would be best to +# mat, and maybe other parameters. For clarity, it would be best to # eliminate this variable in chgres and replace with with 2 or 3 others # (e.g. extrn_mdl, extrn_mdl_file_format, etc). -# +# # tracers_input: # List of atmospheric tracers to read in from the external model file # containing these tracers. # # tracers: -# Names to use in the output NetCDF file for the atmospheric tracers +# Names to use in the output NetCDF file for the atmospheric tracers # specified in tracers_input. With the possible exception of GSD phys- # ics, the elements of this array should have a one-to-one correspond- # ence with the elements in tracers_input, e.g. if the third element of # tracers_input is the name of the O3 mixing ratio, then the third ele- # ment of tracers should be the name to use for the O3 mixing ratio in -# the output file. For GSD physics, three additional tracers -- ice, +# the output file. For GSD physics, three additional tracers -- ice, # rain, and water number concentrations -- may be specified at the end # of tracers, and these will be calculated by chgres. # @@ -178,19 +178,19 @@ esac # The number of soil layers to include in the output NetCDF file. # # replace_FIELD, where FIELD="vgtyp", "sotyp", or "vgfrc": -# Logical variable indicating whether or not to obtain the field in +# Logical variable indicating whether or not to obtain the field in # question from climatology instead of the external model. The field in # question is one of vegetation type (FIELD="vgtyp"), soil type (FIELD= # "sotyp"), and vegetation fraction (FIELD="vgfrc"). If replace_FIELD # is set to ".true.", then the field is obtained from climatology (re- # gardless of whether or not it exists in an external model file). If -# it is set to ".false.", then the field is obtained from the external +# it is set to ".false.", then the field is obtained from the external # model. If the external model file does not provide this field, then # chgres prints out an error message and stops. # # tg3_from_soil: # Logical variable indicating whether or not to set the tg3 soil tempe- # Needs to be verified. -# rature field to the temperature of the deepest soil layer. +# rature field to the temperature of the deepest soil layer. # #----------------------------------------------------------------------- # @@ -202,30 +202,30 @@ esac # # "spfh","o3mr","clwmr" # -# Note also that these are hardcoded in the code (file input_data.F90, +# Note also that these are hardcoded in the code (file input_data.F90, # subroutine read_input_atm_gfs_spectral_file), so that subroutine will # break if tracers_input(:) is not specified as above. # -# Note that there are other fields too ["hgt" (surface height (togography?)), +# Note that there are other fields too ["hgt" (surface height (togography?)), # pres (surface pressure), ugrd, vgrd, and tmp (temperature)] in the atmanl file, but those # are not considered tracers (they're categorized as dynamics variables, # I guess). # -# Another note: The way things are set up now, tracers_input(:) and +# Another note: The way things are set up now, tracers_input(:) and # tracers(:) are assumed to have the same number of elements (just the # atmospheric tracer names in the input and output files may be differ- # ent). There needs to be a check for this in the chgres_cube code!! -# If there was a varmap table that specifies how to handle missing +# If there was a varmap table that specifies how to handle missing # fields, that would solve this problem. # -# Also, it seems like the order of tracers in tracers_input(:) and -# tracers(:) must match, e.g. if ozone mixing ratio is 3rd in +# Also, it seems like the order of tracers in tracers_input(:) and +# tracers(:) must match, e.g. if ozone mixing ratio is 3rd in # tracers_input(:), it must also be 3rd in tracers(:). How can this be checked? # -# NOTE: Really should use a varmap table for GFS, just like we do for +# NOTE: Really should use a varmap table for GFS, just like we do for # RAP/HRRR. # -# A non-prognostic variable that appears in the field_table for GSD physics +# A non-prognostic variable that appears in the field_table for GSD physics # is cld_amt. Why is that in the field_table at all (since it is a non- # prognostic field), and how should we handle it here?? @@ -266,7 +266,7 @@ case "${EXTRN_MDL_NAME_ICS}" in tracers_input="[\"spfh\",\"clwmr\",\"o3mr\"]" tracers="[\"sphum\",\"liq_wat\",\"o3mr\"]" - + internal_GSD=False numsoil_out="4" replace_vgtyp=True @@ -293,7 +293,7 @@ case "${EXTRN_MDL_NAME_ICS}" in # # If CCPP is being used, then the list of atmospheric tracers to include # in the output file depends on the physics suite. Hopefully, this me- -# thod of specifying output tracers will be replaced with a variable +# thod of specifying output tracers will be replaced with a variable # table (which should be specific to each combination of external model, # external model file type, and physics suite). # @@ -430,7 +430,7 @@ HRRRX grib2 files created after about \"${cdate_min_HRRRX}\"..." *) print_err_msg_exit "\ -External-model-dependent namelist variables have not yet been specified +External-model-dependent namelist variables have not yet been specified for this external model: EXTRN_MDL_NAME_ICS = \"${EXTRN_MDL_NAME_ICS}\"" ;; @@ -454,14 +454,14 @@ hh="${EXTRN_MDL_CDATE:8:2}" #----------------------------------------------------------------------- # exec_fn="chgres_cube.exe" -exec_fp="$EXECDIR/${exec_fn}" -if [ ! -f "${exec_fp}" ]; then - print_err_msg_exit "\ +exec_fp="$EXECDIR/${exec_fn}" +if [ ! -f "${exec_fp}" ]; then + print_err_msg_exit "\ The executable (exec_fp) for generating initial conditions on the FV3SAR -native grid does not exist: - exec_fp = \"${exec_fp}\" -Please ensure that you've built this executable." -fi +native grid does not exist: + exec_fp = \"${exec_fp}\" +Please ensure that you've built this executable." +fi # #----------------------------------------------------------------------- # @@ -472,17 +472,17 @@ fi # For GFS physics, the character arrays tracers_input(:) and tracers(:) # must be specified in the namelist file. tracers_input(:) contains the # tracer name to look for in the external model file(s), while tracers(:) -# contains the names to use for the tracers in the output NetCDF files -# that chgres creates (that will be read in by FV3). Since when FV3 +# contains the names to use for the tracers in the output NetCDF files +# that chgres creates (that will be read in by FV3). Since when FV3 # reads these NetCDF files it looks for atmospheric traces as specified -# in the file field_table, tracers(:) should be set to the names in +# in the file field_table, tracers(:) should be set to the names in # field_table. # # NOTE: This process should be automated where the set of elements that # tracers(:) should be set to is obtained from reading in field_table. # # To know how to set tracers_input(:), you have to know the names of the -# variables in the input atmospheric nemsio file (usually this file is +# variables in the input atmospheric nemsio file (usually this file is # named gfs.t00z.atmanl.nemsio). # # It is not quite clear how these should be specified. Here are a list @@ -561,8 +561,8 @@ settings=" nml_fn="fort.41" ${USHDIR}/set_namelist.py -q -u "$settings" -o ${nml_fn} || \ print_err_msg_exit "\ -Call to python script set_namelist.py to set the variables in the namelist -file read in by the ${exec_fn} executable failed. Parameters passed to +Call to python script set_namelist.py to set the variables in the namelist +file read in by the ${exec_fn} executable failed. Parameters passed to this script are: Name of output namelist file: nml_fn = \"${nml_fn}\" @@ -580,13 +580,13 @@ $settings" # Often when the chgres_cube.exe run fails, it still returns a zero re- # turn code, so the failure isn't picked up the the logical OR (||) be- # low. That should be fixed. This might be due to the APRUN command - -# maybe that is returning a zero exit code even though the exit code +# maybe that is returning a zero exit code even though the exit code # of chgres_cube is nonzero. # A similar thing happens in the forecast task. # ${APRUN} ${exec_fp} || \ print_err_msg_exit "\ -Call to executable (exec_fp) to generate surface and initial conditions +Call to executable (exec_fp) to generate surface and initial conditions (ICs) files for the FV3SAR failed: exec_fp = \"${exec_fp}\" The external model from which the ICs files are to be generated is: @@ -598,7 +598,7 @@ located in the following directory: #----------------------------------------------------------------------- # # Move initial condition, surface, control, and 0-th hour lateral bound- -# ary files to ICs_BCs directory. +# ary files to ICs_BCs directory. # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_make_lbcs.sh b/scripts/exregional_make_lbcs.sh index f4dc6bef5..d9a837439 100755 --- a/scripts/exregional_make_lbcs.sh +++ b/scripts/exregional_make_lbcs.sh @@ -49,8 +49,8 @@ hour zero). # #----------------------------------------------------------------------- # -# Specify the set of valid argument names for this script/function. Then -# process the arguments provided to this script/function (which should +# Specify the set of valid argument names for this script/function. Then +# process the arguments provided to this script/function (which should # consist of a set of name-value pairs of the form arg1="value1", etc). # #----------------------------------------------------------------------- @@ -79,7 +79,7 @@ print_input_args valid_args # #----------------------------------------------------------------------- # -extrn_mdl_staging_dir="${CYCLE_DIR}/${EXTRN_MDL_NAME_LBCS}/LBCS" +extrn_mdl_staging_dir="${CYCLE_DIR}/${EXTRN_MDL_NAME_LBCS}/for_LBCS" extrn_mdl_var_defns_fp="${extrn_mdl_staging_dir}/${EXTRN_MDL_LBCS_VAR_DEFNS_FN}" . ${extrn_mdl_var_defns_fp} # @@ -417,8 +417,8 @@ list file has not specified for this external model: ;; esac # -# Get the starting date (year, month, and day together), month, day, and -# hour of the the external model forecast. Then add the forecast hour +# Get the starting date (year, month, and day together), month, day, and +# hour of the the external model forecast. Then add the forecast hour # to it to get a date and time corresponding to the current forecast time. # yyyymmdd="${EXTRN_MDL_CDATE:0:8}" @@ -428,7 +428,7 @@ list file has not specified for this external model: cdate_crnt_fhr=$( date --utc --date "${yyyymmdd} ${hh} UTC + ${fhr} hours" "+%Y%m%d%H" ) # -# Get the month, day, and hour corresponding to the current forecast time +# Get the month, day, and hour corresponding to the current forecast time # of the the external model. # mm="${cdate_crnt_fhr:4:2}" @@ -509,15 +509,15 @@ $settings" # ${APRUN} ${exec_fp} || \ print_err_msg_exit "\ -Call to executable (exec_fp) to generate lateral boundary conditions (LBCs) +Call to executable (exec_fp) to generate lateral boundary conditions (LBCs) file for the FV3SAR for forecast hour fhr failed: exec_fp = \"${exec_fp}\" fhr = \"$fhr\" -The external model from which the LBCs files are to be generated is: - EXTRN_MDL_NAME_LBCS = \"${EXTRN_MDL_NAME_LBCS}\" -The external model files that are inputs to the executable (exec_fp) are -located in the following directory: - extrn_mdl_staging_dir = \"${extrn_mdl_staging_dir}\"" +The external model from which the LBCs files are to be generated is: + EXTRN_MDL_NAME_LBCS = \"${EXTRN_MDL_NAME_LBCS}\" +The external model files that are inputs to the executable (exec_fp) are +located in the following directory: + extrn_mdl_staging_dir = \"${extrn_mdl_staging_dir}\"" # # Move LBCs file for the current lateral boundary update time to the LBCs # work directory. Note that we rename the file by including in its name diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 10b7fed07..a6b51d735 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -55,7 +55,11 @@ specified cycle. # #----------------------------------------------------------------------- # -valid_args=( "CYCLE_DIR" ) +valid_args=( \ +"cycle_dir" \ +"ensmem_indx" \ +"slash_ensmem_subdir" \ +) process_args valid_args "$@" # #----------------------------------------------------------------------- @@ -133,20 +137,28 @@ esac # #----------------------------------------------------------------------- # -# Create links in the INPUT subdirectory of the current cycle's run di- -# rectory to the grid and (filtered) orography files. +# Set the forecast run directory. +# +#----------------------------------------------------------------------- +# +run_dir="${cycle_dir}${slash_ensmem_subdir}" +# +#----------------------------------------------------------------------- +# +# Create links in the INPUT subdirectory of the current run directory to +# the grid and (filtered) orography files. # #----------------------------------------------------------------------- # print_info_msg "$VERBOSE" " -Creating links in the INPUT subdirectory of the current cycle's run di- -rectory to the grid and (filtered) orography files ..." +Creating links in the INPUT subdirectory of the current run directory to +the grid and (filtered) orography files ..." # Create links to fix files in the FIXsar directory. -cd_vrfy ${CYCLE_DIR}/INPUT +cd_vrfy ${run_dir}/INPUT relative_or_null="" if [ "${RUN_TASK_MAKE_GRID}" = "TRUE" ]; then @@ -162,7 +174,7 @@ if [ -f "${target}" ]; then else print_err_msg_exit "\ Cannot create symlink because target does not exist: - target = \"$target}\"" + target = \"$target\"" fi ## Symlink to halo-3 grid file with "halo3" stripped from name. @@ -186,7 +198,7 @@ if [ -f "${target}" ]; then else print_err_msg_exit "\ Cannot create symlink because target does not exist: - target = \"$target}\"" + target = \"$target\"" fi # Symlink to halo-4 grid file with "${CRES}_" stripped from name. @@ -208,7 +220,7 @@ if [ -f "${target}" ]; then else print_err_msg_exit "\ Cannot create symlink because target does not exist: - target = \"$target}\"" + target = \"$target\"" fi @@ -226,7 +238,7 @@ if [ -f "${target}" ]; then else print_err_msg_exit "\ Cannot create symlink because target does not exist: - target = \"$target}\"" + target = \"$target\"" fi # @@ -249,7 +261,7 @@ if [ -f "${target}" ]; then else print_err_msg_exit "\ Cannot create symlink because target does not exist: - target = \"$target}\"" + target = \"$target\"" fi @@ -272,9 +284,11 @@ fi # print_info_msg "$VERBOSE" " Creating links with names that FV3 looks for in the INPUT subdirectory -of the current cycle's run directory (CYCLE_DIR)..." +of the current run directory (run_dir), where + run_dir = \"${run_dir}\" +..." -cd_vrfy ${CYCLE_DIR}/INPUT +cd_vrfy ${run_dir}/INPUT #ln_vrfy -sf gfs_data.tile${TILE_RGNL}.halo${NH0}.nc gfs_data.nc #ln_vrfy -sf sfc_data.tile${TILE_RGNL}.halo${NH0}.nc sfc_data.nc @@ -287,7 +301,7 @@ if [ -f "${target}" ]; then else print_err_msg_exit "\ Cannot create symlink because target does not exist: - target = \"$target}\"" + target = \"$target\"" fi target="sfc_data.tile${TILE_RGNL}.halo${NH0}.nc" @@ -297,26 +311,26 @@ if [ -f "${target}" ]; then else print_err_msg_exit "\ Cannot create symlink because target does not exist: - target = \"$target}\"" + target = \"$target\"" fi # #----------------------------------------------------------------------- # -# Create links in the current cycle directory to fixed (i.e. static) files +# Create links in the current run directory to fixed (i.e. static) files # in the FIXam directory. These links have names that are set to the # names of files that the forecast model expects to exist in the current # working directory when the forecast model executable is called (and -# that is just the cycle directory). +# that is just the run directory). # #----------------------------------------------------------------------- # -cd_vrfy ${CYCLE_DIR} +cd_vrfy ${run_dir} print_info_msg "$VERBOSE" " -Creating links in the current cycle directory (CYCLE_DIR) to fixed (i.e. +Creating links in the current run directory (run_dir) to fixed (i.e. static) files in the FIXam directory: FIXam = \"${FIXam}\" - CYCLE_DIR = \"${CYCLE_DIR}\"" + run_dir = \"${run_dir}\"" relative_or_null="" if [ "${RUN_ENVIR}" != "nco" ]; then @@ -333,200 +347,86 @@ for (( i=0; i<${num_symlinks}; i++ )); do target=$( printf "%s\n" "$mapping" | \ sed -n -r -e "s/${regex_search}/\2/p" ) - symlink="${CYCLE_DIR}/$symlink" + symlink="${run_dir}/$symlink" target="$FIXam/$target" if [ -f "${target}" ]; then ln_vrfy -sf ${relative_or_null} $target $symlink else print_err_msg_exit "\ Cannot create symlink because target does not exist: - target = \"$target}\"" + target = \"$target\"" fi done # #----------------------------------------------------------------------- # -# If running this cycle more than once (e.g. using rocotoboot), remove -# any time stamp file that may exist from the previous attempt. +# If running this cycle/ensemble member combination more than once (e.g. +# using rocotoboot), remove any time stamp file that may exist from the +# previous attempt. # #----------------------------------------------------------------------- # -cd_vrfy ${CYCLE_DIR} +cd_vrfy ${run_dir} rm_vrfy -f time_stamp.out # #----------------------------------------------------------------------- # -# Create links in the current cycle's run directory to cycle-independent -# model input files in the main experiment directory. +# Create links in the current run directory to cycle-independent (and +# ensemble-member-independent) model input files in the main experiment +# directory. # #----------------------------------------------------------------------- # print_info_msg "$VERBOSE" " -Creating links in the current cycle's run directory to cycle-independent -model input files in the main experiment directory..." +Creating links in the current run directory to cycle-independent model +input files in the main experiment directory..." relative_or_null="" if [ "${RUN_ENVIR}" != "nco" ]; then relative_or_null="--relative" fi -ln_vrfy -sf ${relative_or_null} ${DATA_TABLE_FP} ${CYCLE_DIR} -ln_vrfy -sf ${relative_or_null} ${FIELD_TABLE_FP} ${CYCLE_DIR} -ln_vrfy -sf ${relative_or_null} ${FV3_NML_FP} ${CYCLE_DIR} -ln_vrfy -sf ${relative_or_null} ${NEMS_CONFIG_FP} ${CYCLE_DIR} +ln_vrfy -sf ${relative_or_null} ${DATA_TABLE_FP} ${run_dir} +ln_vrfy -sf ${relative_or_null} ${FIELD_TABLE_FP} ${run_dir} +ln_vrfy -sf ${relative_or_null} ${NEMS_CONFIG_FP} ${run_dir} + +if [ "${DO_ENSEMBLE}" = TRUE ]; then + ln_vrfy -sf ${relative_or_null} "${FV3_NML_ENSMEM_FPS[$(( 10#${ensmem_indx}-1 ))]}" ${run_dir}/${FV3_NML_FN} +else + ln_vrfy -sf ${relative_or_null} ${FV3_NML_FP} ${run_dir} +fi if [ "${USE_CCPP}" = "TRUE" ]; then - ln_vrfy -sf ${relative_or_null} ${CCPP_PHYS_SUITE_FP} ${CYCLE_DIR} + ln_vrfy -sf ${relative_or_null} ${CCPP_PHYS_SUITE_FP} ${run_dir} if [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_v0" ] || \ [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_SAR_v1" ] || \ [ "${CCPP_PHYS_SUITE}" = "FV3_RRFS_v0" ] || \ [ "${CCPP_PHYS_SUITE}" = "FV3_GSD_SAR" ]; then - ln_vrfy -sf ${relative_or_null} $EXPTDIR/CCN_ACTIVATE.BIN ${CYCLE_DIR} + ln_vrfy -sf ${relative_or_null} $EXPTDIR/CCN_ACTIVATE.BIN ${run_dir} fi fi # #----------------------------------------------------------------------- # -# Copy templates of cycle-dependent model input files from the templates -# directory to the current cycle's run directory. +# If running enemble forecasts, create links to the cycle-specific +# diagnostic tables file and model configuration file in the cycle +# directory. Note that these links should not be made if not running +# ensemble forecasts because in that case, the cycle directory is the +# run directory (and we would be creating a symlink with the name of a +# file that already exists). # #----------------------------------------------------------------------- # -print_info_msg "$VERBOSE" " -Copying cycle-dependent model input files from the templates directory -to the current cycle's run directory..." - -print_info_msg "$VERBOSE" " - Copying the template model configuration file to the current cycle's - run directory..." -model_config_fp="${CYCLE_DIR}/${MODEL_CONFIG_FN}" -cp_vrfy "${MODEL_CONFIG_TMPL_FP}" "${model_config_fp}" -# -#----------------------------------------------------------------------- -# -# Extract from CDATE the starting year, month, day, and hour of the -# forecast. These are needed below for various operations. -# -#----------------------------------------------------------------------- -# -YYYY=${CDATE:0:4} -MM=${CDATE:4:2} -DD=${CDATE:6:2} -HH=${CDATE:8:2} -# -#----------------------------------------------------------------------- -# -# Set parameters in the diagnostics table file. -# -#----------------------------------------------------------------------- -# - -diag_table_fp="${CYCLE_DIR}/${DIAG_TABLE_FN}" - -print_info_msg "$VERBOSE" " - Using the template diagnostics table file: - - diag_table_tmpl_fp = ${DIAG_TABLE_TMPL_FP} - - to create: - - diag_table_fp = \"${diag_table_fp}\"" - -settings=" - starttime: !datetime ${CDATE} - cres: ${CRES} -" - -$USHDIR/fill_jinja_template.py -q -u "${settings}" -t "${DIAG_TABLE_TMPL_FP}" -o "${diag_table_fp}" - -if [[ $? -ne 0 ]] ; then - echo " - !!!!!!!!!!!!!!!!! - - fill_jinja_template.py failed! - - !!!!!!!!!!!!!!!!! - " - - exit 1 -fi - - -# -#----------------------------------------------------------------------- -# -# Set parameters in the model configuration file. -# -#----------------------------------------------------------------------- -# -print_info_msg "$VERBOSE" " -Setting parameters in file: - model_config_fp = \"${model_config_fp}\"" - -dot_quilting_dot="."${QUILTING,,}"." -dot_print_esmf_dot="."${PRINT_ESMF,,}"." - -set_file_param "${model_config_fp}" "PE_MEMBER01" "${PE_MEMBER01}" -set_file_param "${model_config_fp}" "dt_atmos" "${DT_ATMOS}" -set_file_param "${model_config_fp}" "start_year" "$YYYY" -set_file_param "${model_config_fp}" "start_month" "$MM" -set_file_param "${model_config_fp}" "start_day" "$DD" -set_file_param "${model_config_fp}" "start_hour" "$HH" -set_file_param "${model_config_fp}" "nhours_fcst" "${FCST_LEN_HRS}" -set_file_param "${model_config_fp}" "ncores_per_node" "${NCORES_PER_NODE}" -set_file_param "${model_config_fp}" "quilting" "${dot_quilting_dot}" -set_file_param "${model_config_fp}" "print_esmf" "${dot_print_esmf_dot}" -# -#----------------------------------------------------------------------- -# -# If the write component is to be used, then a set of parameters, in- -# cluding those that define the write component's output grid, need to -# be specified in the model configuration file (model_config_fp). This -# is done by appending a template file (in which some write-component -# parameters are set to actual values while others are set to placehol- -# ders) to model_config_fp and then replacing the placeholder values in -# the (new) model_config_fp file with actual values. The full path of -# this template file is specified in the variable WRTCMP_PA RAMS_TEMP- -# LATE_FP. -# -#----------------------------------------------------------------------- -# -if [ "$QUILTING" = "TRUE" ]; then - - cat ${WRTCMP_PARAMS_TMPL_FP} >> ${model_config_fp} - - set_file_param "${model_config_fp}" "write_groups" "$WRTCMP_write_groups" - set_file_param "${model_config_fp}" "write_tasks_per_group" "$WRTCMP_write_tasks_per_group" - - set_file_param "${model_config_fp}" "output_grid" "\'$WRTCMP_output_grid\'" - set_file_param "${model_config_fp}" "cen_lon" "$WRTCMP_cen_lon" - set_file_param "${model_config_fp}" "cen_lat" "$WRTCMP_cen_lat" - set_file_param "${model_config_fp}" "lon1" "$WRTCMP_lon_lwr_left" - set_file_param "${model_config_fp}" "lat1" "$WRTCMP_lat_lwr_left" - - if [ "${WRTCMP_output_grid}" = "rotated_latlon" ]; then - set_file_param "${model_config_fp}" "lon2" "$WRTCMP_lon_upr_rght" - set_file_param "${model_config_fp}" "lat2" "$WRTCMP_lat_upr_rght" - set_file_param "${model_config_fp}" "dlon" "$WRTCMP_dlon" - set_file_param "${model_config_fp}" "dlat" "$WRTCMP_dlat" - elif [ "${WRTCMP_output_grid}" = "lambert_conformal" ]; then - set_file_param "${model_config_fp}" "stdlat1" "$WRTCMP_stdlat1" - set_file_param "${model_config_fp}" "stdlat2" "$WRTCMP_stdlat2" - set_file_param "${model_config_fp}" "nx" "$WRTCMP_nx" - set_file_param "${model_config_fp}" "ny" "$WRTCMP_ny" - set_file_param "${model_config_fp}" "dx" "$WRTCMP_dx" - set_file_param "${model_config_fp}" "dy" "$WRTCMP_dy" - elif [ "${WRTCMP_output_grid}" = "regional_latlon" ]; then - set_file_param "${model_config_fp}" "lon2" "$WRTCMP_lon_upr_rght" - set_file_param "${model_config_fp}" "lat2" "$WRTCMP_lat_upr_rght" - set_file_param "${model_config_fp}" "dlon" "$WRTCMP_dlon" - set_file_param "${model_config_fp}" "dlat" "$WRTCMP_dlat" - fi - +if [ "${DO_ENSEMBLE}" = "TRUE" ]; then + relative_or_null="--relative" + diag_table_fp="${cycle_dir}/${DIAG_TABLE_FN}" + ln_vrfy -sf ${relative_or_null} ${diag_table_fp} ${run_dir} + model_config_fp="${cycle_dir}/${MODEL_CONFIG_FN}" + ln_vrfy -sf ${relative_or_null} ${model_config_fp} ${run_dir} fi # #----------------------------------------------------------------------- diff --git a/scripts/exregional_run_post.sh b/scripts/exregional_run_post.sh index 60758381e..36992736c 100755 --- a/scripts/exregional_run_post.sh +++ b/scripts/exregional_run_post.sh @@ -56,7 +56,8 @@ the output files corresponding to a specified forecast hour. #----------------------------------------------------------------------- # valid_args=( \ -"cycle_dir" \ +"cdate" \ +"run_dir" \ "postprd_dir" \ "fhr_dir" \ "fhr" \ @@ -150,12 +151,12 @@ cp_vrfy ${EXECDIR}/ncep_post . #----------------------------------------------------------------------- # # Get the cycle date and hour (in formats of yyyymmdd and hh, respect- -# ively) from CDATE. +# ively) from cdate. # #----------------------------------------------------------------------- # -yyyymmdd=${CDATE:0:8} -hh=${CDATE:8:2} +yyyymmdd=${cdate:0:8} +hh=${cdate:8:2} cyc=$hh tmmark="tm$hh" # @@ -166,21 +167,20 @@ tmmark="tm$hh" # #----------------------------------------------------------------------- # -dyn_file="${cycle_dir}/dynf0${fhr}.nc" -phy_file="${cycle_dir}/phyf0${fhr}.nc" +dyn_file="${run_dir}/dynf0${fhr}.nc" +phy_file="${run_dir}/phyf0${fhr}.nc" -#POST_TIME=$( ${NDATE} +${fhr} ${CDATE} ) -POST_TIME=$( date --utc --date "${yyyymmdd} ${hh} UTC + ${fhr} hours" "+%Y%m%d%H" ) -POST_YYYY=${POST_TIME:0:4} -POST_MM=${POST_TIME:4:2} -POST_DD=${POST_TIME:6:2} -POST_HH=${POST_TIME:8:2} +post_time=$( date --utc --date "${yyyymmdd} ${hh} UTC + ${fhr} hours" "+%Y%m%d%H" ) +post_yyyy=${post_time:0:4} +post_mm=${post_time:4:2} +post_dd=${post_time:6:2} +post_hh=${post_time:8:2} cat > itag < /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Get the full path to the file in which this script/function is located +# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in +# which the file is located (scrfunc_dir). +# +#----------------------------------------------------------------------- +# + local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) + local scrfunc_fn=$( basename "${scrfunc_fp}" ) + local scrfunc_dir=$( dirname "${scrfunc_fp}" ) +# +#----------------------------------------------------------------------- +# +# Get the name of this function. +# +#----------------------------------------------------------------------- +# + local func_name="${FUNCNAME[0]}" +# +#----------------------------------------------------------------------- +# +# Specify the set of valid argument names for this script/function. Then +# process the arguments provided to this script/function (which should +# consist of a set of name-value pairs of the form arg1="value1", etc). +# +#----------------------------------------------------------------------- +# + local valid_args=() + process_args valid_args "$@" +# +#----------------------------------------------------------------------- +# +# For debugging purposes, print out values of arguments passed to this +# script. Note that these will be printed out only if VERBOSE is set to +# TRUE. +# +#----------------------------------------------------------------------- +# + print_input_args valid_args +# +#----------------------------------------------------------------------- +# +# Declare local variables. +# +#----------------------------------------------------------------------- +# + local i \ + cdate \ + cycle_dir \ + diag_table_fp \ + settings +# +#----------------------------------------------------------------------- +# +# Create a diagnostics table file within each cycle directory. +# +#----------------------------------------------------------------------- +# + print_info_msg "$VERBOSE" " +Creating a diagnostics table file (\"${DIAG_TABLE_FN}\") within each cycle +directory..." + + for (( i=0; i<${NUM_CYCLES}; i++ )); do + + cdate="${ALL_CDATES[$i]}" + cycle_dir="${CYCLE_BASEDIR}/$cdate" + + diag_table_fp="${cycle_dir}/${DIAG_TABLE_FN}" + print_info_msg "$VERBOSE" " +Using the template diagnostics table file: + + diag_table_tmpl_fp = ${DIAG_TABLE_TMPL_FP} + +to create: + + diag_table_fp = \"${diag_table_fp}\"" + + settings=" + starttime: !datetime ${ALL_CDATES[$i]} + cres: ${CRES} +" + + $USHDIR/fill_jinja_template.py -q -u "${settings}" -t "${DIAG_TABLE_TMPL_FP}" -o "${diag_table_fp}" || \ + print_err_msg_exit " +!!!!!!!!!!!!!!!!! + +fill_jinja_template.py failed! + +!!!!!!!!!!!!!!!!! +" + + done +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/function. +# +#----------------------------------------------------------------------- +# + { restore_shell_opts; } > /dev/null 2>&1 + +} + diff --git a/ush/create_model_config_files.sh b/ush/create_model_config_files.sh new file mode 100644 index 000000000..d2f9fd357 --- /dev/null +++ b/ush/create_model_config_files.sh @@ -0,0 +1,182 @@ +# +#----------------------------------------------------------------------- +# +# This file defines a function that creates a model configuration file +# for each cycle to be run. +# +#----------------------------------------------------------------------- +# +function create_model_config_files() { +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# + { save_shell_opts; set -u -x; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Get the full path to the file in which this script/function is located +# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in +# which the file is located (scrfunc_dir). +# +#----------------------------------------------------------------------- +# + local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) + local scrfunc_fn=$( basename "${scrfunc_fp}" ) + local scrfunc_dir=$( dirname "${scrfunc_fp}" ) +# +#----------------------------------------------------------------------- +# +# Get the name of this function. +# +#----------------------------------------------------------------------- +# + local func_name="${FUNCNAME[0]}" +# +#----------------------------------------------------------------------- +# +# Specify the set of valid argument names for this script/function. Then +# process the arguments provided to this script/function (which should +# consist of a set of name-value pairs of the form arg1="value1", etc). +# +#----------------------------------------------------------------------- +# + local valid_args=() + process_args valid_args "$@" +# +#----------------------------------------------------------------------- +# +# For debugging purposes, print out values of arguments passed to this +# script. Note that these will be printed out only if VERBOSE is set to +# TRUE. +# +#----------------------------------------------------------------------- +# + print_input_args valid_args +# +#----------------------------------------------------------------------- +# +# Declare local variables. +# +#----------------------------------------------------------------------- +# + local i \ + cdate \ + cycle_dir \ + model_config_fp \ + yyyy \ + mm \ + dd \ + hh \ + mm \ + dot_quilting_dot \ + dot_print_esmf_dot +# +#----------------------------------------------------------------------- +# +# Create a model configuration file within each cycle directory. +# +#----------------------------------------------------------------------- +# + print_info_msg "$VERBOSE" " +Creating a model configuration file (\"${MODEL_CONFIG_FN}\") within each +cycle directory..." + + for (( i=0; i<${NUM_CYCLES}; i++ )); do + + cdate="${ALL_CDATES[$i]}" + cycle_dir="${CYCLE_BASEDIR}/$cdate" +# +# Copy template of cycle-dependent model configure files from the templates +# directory to the current cycle directory. +# + model_config_fp="${cycle_dir}/${MODEL_CONFIG_FN}" + cp_vrfy "${MODEL_CONFIG_TMPL_FP}" "${model_config_fp}" +# +# Extract from cdate the starting year, month, day, and hour of the forecast. +# + yyyy=${cdate:0:4} + mm=${cdate:4:2} + dd=${cdate:6:2} + hh=${cdate:8:2} +# +# Set parameters in the model configure file. +# + dot_quilting_dot="."${QUILTING,,}"." + dot_print_esmf_dot="."${PRINT_ESMF,,}"." + + set_file_param "${model_config_fp}" "PE_MEMBER01" "${PE_MEMBER01}" + set_file_param "${model_config_fp}" "dt_atmos" "${DT_ATMOS}" + set_file_param "${model_config_fp}" "start_year" "$yyyy" + set_file_param "${model_config_fp}" "start_month" "$mm" + set_file_param "${model_config_fp}" "start_day" "$dd" + set_file_param "${model_config_fp}" "start_hour" "$hh" + set_file_param "${model_config_fp}" "nhours_fcst" "${FCST_LEN_HRS}" + set_file_param "${model_config_fp}" "ncores_per_node" "${NCORES_PER_NODE}" + set_file_param "${model_config_fp}" "quilting" "${dot_quilting_dot}" + set_file_param "${model_config_fp}" "print_esmf" "${dot_print_esmf_dot}" +# +#----------------------------------------------------------------------- +# +# If the write component is to be used, then a set of parameters, in- +# cluding those that define the write component's output grid, need to +# be specified in the model configuration file (model_config_fp). This +# is done by appending a template file (in which some write-component +# parameters are set to actual values while others are set to placehol- +# ders) to model_config_fp and then replacing the placeholder values in +# the (new) model_config_fp file with actual values. The full path of +# this template file is specified in the variable WRTCMP_PA RAMS_TEMP- +# LATE_FP. +# +#----------------------------------------------------------------------- +# + if [ "$QUILTING" = "TRUE" ]; then + + cat ${WRTCMP_PARAMS_TMPL_FP} >> ${model_config_fp} + + set_file_param "${model_config_fp}" "write_groups" "$WRTCMP_write_groups" + set_file_param "${model_config_fp}" "write_tasks_per_group" "$WRTCMP_write_tasks_per_group" + + set_file_param "${model_config_fp}" "output_grid" "\'$WRTCMP_output_grid\'" + set_file_param "${model_config_fp}" "cen_lon" "$WRTCMP_cen_lon" + set_file_param "${model_config_fp}" "cen_lat" "$WRTCMP_cen_lat" + set_file_param "${model_config_fp}" "lon1" "$WRTCMP_lon_lwr_left" + set_file_param "${model_config_fp}" "lat1" "$WRTCMP_lat_lwr_left" + + if [ "${WRTCMP_output_grid}" = "rotated_latlon" ]; then + set_file_param "${model_config_fp}" "lon2" "$WRTCMP_lon_upr_rght" + set_file_param "${model_config_fp}" "lat2" "$WRTCMP_lat_upr_rght" + set_file_param "${model_config_fp}" "dlon" "$WRTCMP_dlon" + set_file_param "${model_config_fp}" "dlat" "$WRTCMP_dlat" + elif [ "${WRTCMP_output_grid}" = "lambert_conformal" ]; then + set_file_param "${model_config_fp}" "stdlat1" "$WRTCMP_stdlat1" + set_file_param "${model_config_fp}" "stdlat2" "$WRTCMP_stdlat2" + set_file_param "${model_config_fp}" "nx" "$WRTCMP_nx" + set_file_param "${model_config_fp}" "ny" "$WRTCMP_ny" + set_file_param "${model_config_fp}" "dx" "$WRTCMP_dx" + set_file_param "${model_config_fp}" "dy" "$WRTCMP_dy" + elif [ "${WRTCMP_output_grid}" = "regional_latlon" ]; then + set_file_param "${model_config_fp}" "lon2" "$WRTCMP_lon_upr_rght" + set_file_param "${model_config_fp}" "lat2" "$WRTCMP_lat_upr_rght" + set_file_param "${model_config_fp}" "dlon" "$WRTCMP_dlon" + set_file_param "${model_config_fp}" "dlat" "$WRTCMP_dlat" + fi + + fi + + done +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/function. +# +#----------------------------------------------------------------------- +# + { restore_shell_opts; } > /dev/null 2>&1 + +} + diff --git a/ush/generate_FV3SAR_wflow.sh b/ush/generate_FV3SAR_wflow.sh index 46a8273ec..974033159 100755 --- a/ush/generate_FV3SAR_wflow.sh +++ b/ush/generate_FV3SAR_wflow.sh @@ -47,6 +47,9 @@ ushdir="${scrfunc_dir}" # . $ushdir/source_util_funcs.sh . $ushdir/set_FV3nml_sfc_climo_filenames.sh +. $ushdir/set_FV3nml_stoch_params.sh +. $ushdir/create_model_config_files.sh +. $ushdir/create_diag_table_files.sh # #----------------------------------------------------------------------- # @@ -90,6 +93,15 @@ WFLOW_XML_FP="$EXPTDIR/${WFLOW_XML_FN}" # #----------------------------------------------------------------------- # +ensmem_indx_name="\"\"" +uscore_ensmem_name="\"\"" +slash_ensmem_subdir="\"\"" +if [ "${DO_ENSEMBLE}" = "TRUE" ]; then + ensmem_indx_name="mem" + uscore_ensmem_name="_mem#${ensmem_indx_name}#" + slash_ensmem_subdir="/mem#${ensmem_indx_name}#" +fi + settings="\ # # Parameters needed by the job scheduler. @@ -177,13 +189,25 @@ settings="\ # # Parameters that determine the set of cycles to run. # - 'date_first_cycl': !datetime ${DATE_FIRST_CYCL}${CYCL_HRS[0]} - 'date_last_cycl': !datetime ${DATE_LAST_CYCL}${CYCL_HRS[0]} + 'date_first_cycl': ${DATE_FIRST_CYCL} + 'date_last_cycl': ${DATE_LAST_CYCL} + 'cdate_first_cycl': !datetime ${DATE_FIRST_CYCL}${CYCL_HRS[0]} + 'cycl_hrs': [ $( printf "\'%s\', " "${CYCL_HRS[@]}" ) ] 'cycl_freq': !!str 24:00:00 # # Forecast length (same for all cycles). # - 'fcst_len_hrs': ${FCST_LEN_HRS}" + 'fcst_len_hrs': ${FCST_LEN_HRS} +# +# Ensemble-related parameters. +# + 'do_ensemble': ${DO_ENSEMBLE} + 'num_ens_members': ${NUM_ENS_MEMBERS} + 'ndigits_ensmem_names': !!str ${NDIGITS_ENSMEM_NAMES} + 'ensmem_indx_name': ${ensmem_indx_name} + 'uscore_ensmem_name': ${uscore_ensmem_name} + 'slash_ensmem_subdir': ${slash_ensmem_subdir} +" # End of "settings" variable. print_info_msg $VERBOSE " The variable \"settings\" specifying values of the rococo XML variables @@ -213,18 +237,42 @@ are: Namelist settings specified on command line: settings = $settings" +# +#----------------------------------------------------------------------- +# +# Create the cycle directories. +# +#----------------------------------------------------------------------- +# +print_info_msg "$VERBOSE" " +Creating the cycle directories..." +for (( i=0; i<${NUM_CYCLES}; i++ )); do + cdate="${ALL_CDATES[$i]}" + cycle_dir="${CYCLE_BASEDIR}/$cdate" + mkdir_vrfy -p "${cycle_dir}" +done +# +#----------------------------------------------------------------------- +# +# Call the function that creates the model configuration file within each +# cycle directory. # #----------------------------------------------------------------------- # -# For select workflow tasks, copy module files from the various cloned -# external repositories to the appropriate subdirectory under the -# workflow directory tree. In principle, this is better than having -# hard-coded module files for tasks because the copied module files will -# always be up to date. However, it does require that these module files -# in the external repositories be coded correctly, e.g. that they really -# be lua module files and not contain any shell commands -# (like "export SOME_VARIABLE"). +create_model_config_files || print_err_msg_exit "\ +Call to function to create a model configuration file under each cycle +directory failed." +# +#----------------------------------------------------------------------- +# +# For select workflow tasks, copy module files from the various cloned +# external repositories to the appropriate subdirectory under the workflow +# directory tree. In principle, this is better than having hard-coded +# module files for tasks because the copied module files will always be +# up-to-date. However, it does require that these module files in the +# external repositories be coded correctly, e.g. that they really be lua +# module files and not contain any shell commands (like "export SOME_VARIABLE"). # #----------------------------------------------------------------------- # @@ -236,11 +284,16 @@ cp_vrfy -f "${UFS_UTILS_DIR}/modulefiles/fv3gfs/orog.$machine" "${MAKE_OROG_TN}" cp_vrfy -f "${UFS_UTILS_DIR}/modulefiles/modulefile.sfc_climo_gen.$machine" "${MAKE_SFC_CLIMO_TN}" cp_vrfy -f "${CHGRES_DIR}/modulefiles/chgres_cube.$machine" "${MAKE_ICS_TN}" cp_vrfy -f "${CHGRES_DIR}/modulefiles/chgres_cube.$machine" "${MAKE_LBCS_TN}" -cp_vrfy -f "${UFS_WTHR_MDL_DIR}/modulefiles/$machine.intel/fv3" "${RUN_FCST_TN}" +if [ $MACHINE = "WCOSS_CRAY" -o $MACHINE = "WCOSS_DELL_P3" ] ; then + cp_vrfy -f "${UFS_WTHR_MDL_DIR}/modulefiles/$machine/fv3" "${RUN_FCST_TN}" +else + cp_vrfy -f "${UFS_WTHR_MDL_DIR}/modulefiles/$machine.intel/fv3" "${RUN_FCST_TN}" +fi task_names=( "${MAKE_GRID_TN}" "${MAKE_OROG_TN}" "${MAKE_SFC_CLIMO_TN}" "${MAKE_ICS_TN}" "${MAKE_LBCS_TN}" "${RUN_FCST_TN}" ) # -# Only some platforms build EMC_post using modules, and some machines require a different EMC_post modulefile name. +# Only some platforms build EMC_post using modules, and some machines +# require a different EMC_post modulefile name. # if [ "${MACHINE}" = "CHEYENNE" ]; then print_info_msg "No post modulefile needed for ${MACHINE}" @@ -427,7 +480,7 @@ fi # # This if-statement is a temporary fix that makes corrections to the suite # definition file for the "FV3_GFS_2017_gfdlmp_regional" physics suite -# that EMC uses. +# that EMC uses. # # IMPORTANT: # This if-statement must be removed once these corrections are made to @@ -613,6 +666,11 @@ settings="\ settings="$settings 'namsfc': {" +dummy_run_dir="$EXPTDIR/any_cyc" +if [ "${DO_ENSEMBLE}" = "TRUE" ]; then + dummy_run_dir="${dummy_run_dir}/any_ensmem" +fi + regex_search="^[ ]*([^| ]+)[ ]*[|][ ]*([^| ]+)[ ]*$" num_nml_vars=${#FV3_NML_VARNAME_TO_FIXam_FILES_MAPPING[@]} for (( i=0; i<${num_nml_vars}; i++ )); do @@ -632,8 +690,7 @@ for (( i=0; i<${num_nml_vars}; i++ )); do # the experiment directory). # if [ "${RUN_ENVIR}" != "nco" ]; then - fp=$( realpath --canonicalize-missing \ - --relative-to="$EXPTDIR/any_cycle_dir" "$fp" ) + fp=$( realpath --canonicalize-missing --relative-to="${dummy_run_dir}" "$fp" ) fi fi # @@ -646,19 +703,11 @@ for (( i=0; i<${num_nml_vars}; i++ )); do done # -# Add to "settings" several namelist variable name-and-value pairs that -# are constant. These should probably be added to the base namelist file -# (FV3_NML_BASE_FP) and this step removed. +# Add the closing curly bracket to "settings". # settings="$settings -# 'FNZORC': \"igbp\", -# 'FNTSFA': \"\", -# 'FNACNA': \"\", -# 'FNSNOA': \"\", }" -# -# For debugging purposes, print out what "settings" has been set to. -# + print_info_msg $VERBOSE " The variable \"settings\" specifying values of the namelist variables has been set as follows: @@ -669,17 +718,17 @@ $settings" #----------------------------------------------------------------------- # # Call the set_namelist.py script to create a new FV3 namelist file (full -# path specified by FV3_NML_FP) using the file FV3_NML_BASE_FP as the base -# (i.e. starting) namelist file, with physics-suite-dependent modifications -# to the base file specified in the yaml configuration file FV3_NML_YAML_CONFIG_FP -# (for the physics suite specified by CCPP_PHYS_SUITE), and with additional -# physics-suite-independent modificaitons specified in the variable -# "settings" set above. +# path specified by FV3_NML_FP) using the file FV3_NML_BASE_SUITE_FP as +# the base (i.e. starting) namelist file, with physics-suite-dependent +# modifications to the base file specified in the yaml configuration file +# FV3_NML_YAML_CONFIG_FP (for the physics suite specified by CCPP_PHYS_SUITE), +# and with additional physics-suite-independent modificaitons specified +# in the variable "settings" set above. # #----------------------------------------------------------------------- # $USHDIR/set_namelist.py -q \ - -n ${FV3_NML_BASE_FP} \ + -n ${FV3_NML_BASE_SUITE_FP} \ -c ${FV3_NML_YAML_CONFIG_FP} ${CCPP_PHYS_SUITE} \ -u "$settings" \ -o ${FV3_NML_FP} || \ @@ -687,7 +736,7 @@ $USHDIR/set_namelist.py -q \ Call to python script set_namelist.py to generate an FV3 namelist file failed. Parameters passed to this script are: Full path to base namelist file: - FV3_NML_BASE_FP = \"${FV3_NML_BASE_FP}\" + FV3_NML_BASE_SUITE_FP = \"${FV3_NML_BASE_SUITE_FP}\" Full path to yaml configuration file for various physics suites: FV3_NML_YAML_CONFIG_FP = \"${FV3_NML_YAML_CONFIG_FP}\" Physics suite to extract from yaml configuration file: @@ -710,7 +759,21 @@ $settings" # configurations is not known until the grid is created. # if [ "${RUN_TASK_MAKE_GRID}" = "FALSE" ]; then - set_FV3nml_sfc_climo_filenames + + set_FV3nml_sfc_climo_filenames || print_err_msg_exit "\ +Call to function to set surface climatology file names in the FV3 namelist +file failed." + + if [ "${DO_ENSEMBLE}" = TRUE ]; then + set_FV3nml_stoch_params || print_err_msg_exit "\ +Call to function to set stochastic parameters in the FV3 namelist files +for the various ensemble members failed." + fi + + create_diag_table_files || print_err_msg_exit "\ +Call to function to create a diagnostics table file under each cycle +directory failed." + fi # #----------------------------------------------------------------------- diff --git a/ush/set_FV3nml_sfc_climo_filenames.sh b/ush/set_FV3nml_sfc_climo_filenames.sh index 9239c9910..d79a333cb 100644 --- a/ush/set_FV3nml_sfc_climo_filenames.sh +++ b/ush/set_FV3nml_sfc_climo_filenames.sh @@ -1,10 +1,10 @@ # #----------------------------------------------------------------------- # -# This file defines a function that sets the values of the variables in +# This file defines a function that sets the values of the variables in # the forecast model's namelist file that specify the paths to the surface -# climatology files on the FV3SAR native grid (which are either pregenerated -# or created by the MAKE_SFC_CLIMO_TN task). Note that the workflow +# climatology files on the FV3SAR native grid (which are either pregenerated +# or created by the MAKE_SFC_CLIMO_TN task). Note that the workflow # generation scripts create symlinks to these surface climatology files # in the FIXsar directory, and the values in the namelist file that get # set by this function are relative or full paths to these links. @@ -121,20 +121,25 @@ regex_search="^[ ]*([^| ]+)[ ]*[|][ ]*([^| ]+)[ ]*$" #suffix="tile${TILE_RGNL}.halo4.nc" suffix="tileX.nc" # -# Create a multiline variable that consists of a yaml-compliant string -# specifying the values that the namelist variables that specify the -# surface climatology file paths need to be set to (one namelist variable -# per line, plus a header and footer). Below, this variable will be +# Create a multiline variable that consists of a yaml-compliant string +# specifying the values that the namelist variables that specify the +# surface climatology file paths need to be set to (one namelist variable +# per line, plus a header and footer). Below, this variable will be # passed to a python script that will create the namelist file. # -# Note that the array FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING contains -# the mapping between the namelist variables and the surface climatology +# Note that the array FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING contains +# the mapping between the namelist variables and the surface climatology # fields. Here, we loop through this array and process each element to # construct each line of "settings". # settings="\ 'namsfc': {" +dummy_run_dir="$EXPTDIR/any_cyc" +if [ "${DO_ENSEMBLE}" = "TRUE" ]; then + dummy_run_dir="${dummy_run_dir}/any_ensmem" +fi + num_nml_vars=${#FV3_NML_VARNAME_TO_SFC_CLIMO_FIELD_MAPPING[@]} for (( i=0; i<${num_nml_vars}; i++ )); do @@ -158,12 +163,11 @@ for (( i=0; i<${num_nml_vars}; i++ )); do # the experiment directory). # if [ "${RUN_ENVIR}" != "nco" ]; then - fp=$( realpath --canonicalize-missing \ - --relative-to="$EXPTDIR/any_cycle_dir" "$fp" ) + fp=$( realpath --canonicalize-missing --relative-to="${dummy_run_dir}" "$fp" ) fi # # Add a line to the variable "settings" that specifies (in a yaml-compliant -# format) the name of the current namelist variable and the value it should +# format) the name of the current namelist variable and the value it should # be set to. # settings="$settings @@ -177,7 +181,7 @@ settings="$settings # For debugging purposes, print out what "settings" has been set to. # print_info_msg $VERBOSE " -The variable \"settings\" specifying values of the namelist variables +The variable \"settings\" specifying values of the namelist variables has been set as follows: settings = @@ -187,7 +191,7 @@ $settings" # # Rename the FV3 namelist file for the experiment by appending the string # ".base" to its name. The call to the set_namelist.py script below will -# use this file as the base (i.e. starting) namelist file, and it will +# use this file as the base (i.e. starting) namelist file, and it will # modify it as specified by the varaible "settings" above, saving the # result in a new FV3 namelist file for the experiment. Once this is # done, we remove the base namelist file since it is no longer needed. @@ -203,7 +207,7 @@ $USHDIR/set_namelist.py -q \ -o ${FV3_NML_FP} || \ print_err_msg_exit "\ Call to python script set_namelist.py to set the variables in the FV3 -namelist file that specify the paths to the surface climatology files +namelist file that specify the paths to the surface climatology files failed. Parameters passed to this script are: Full path to base namelist file: fv3_nml_base_fp = \"${fv3_nml_base_fp}\" diff --git a/ush/set_FV3nml_stoch_params.sh b/ush/set_FV3nml_stoch_params.sh new file mode 100644 index 000000000..98e1c3469 --- /dev/null +++ b/ush/set_FV3nml_stoch_params.sh @@ -0,0 +1,158 @@ +# +#----------------------------------------------------------------------- +# +# This file defines a function that, for an ensemble-enabled experiment +# (i.e. for an experiment for which the workflow configuration variable +# DO_ENSEMBLE has been set to "TRUE"), adds to a base FV3 namelist file +# a set of stochastic "seed" parameters that is unique to each ensemble +# member to generate a new namelist file for each member. The namelist +# files of any two ensemble members differ only in their stochastic "seed" +# parameter values. Each such member-specific namelist file is placed at +# the top level of the experiment directory. (Then, during the RUN_FCST_TN +# step of the workflow, links are created from the run directories to these +# member-specific namelist files.) +# +#----------------------------------------------------------------------- +# +function set_FV3nml_stoch_params() { +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# + { save_shell_opts; set -u -x; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Get the full path to the file in which this script/function is located +# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in +# which the file is located (scrfunc_dir). +# +#----------------------------------------------------------------------- +# + local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) + local scrfunc_fn=$( basename "${scrfunc_fp}" ) + local scrfunc_dir=$( dirname "${scrfunc_fp}" ) +# +#----------------------------------------------------------------------- +# +# Get the name of this function. +# +#----------------------------------------------------------------------- +# + local func_name="${FUNCNAME[0]}" +# +#----------------------------------------------------------------------- +# +# Specify the set of valid argument names for this script/function. Then +# process the arguments provided to this script/function (which should +# consist of a set of name-value pairs of the form arg1="value1", etc). +# +#----------------------------------------------------------------------- +# + local valid_args=() + process_args valid_args "$@" +# +#----------------------------------------------------------------------- +# +# For debugging purposes, print out values of arguments passed to this +# script. Note that these will be printed out only if VERBOSE is set to +# TRUE. +# +#----------------------------------------------------------------------- +# + print_input_args valid_args +# +#----------------------------------------------------------------------- +# +# Declare local variables. +# +#----------------------------------------------------------------------- +# + local cdate \ + i \ + ip1 \ + fv3_nml_ens_fp \ + iseed_shum \ + iseed_skeb \ + iseed_sppt \ + settings +# +#----------------------------------------------------------------------- +# +# At this point, there should exist a namelist file with full path as +# specified in the workflow variable FV3_NML_FP. This is the namelist +# file for a non-ensmble-enabled experiment. This file will be used below +# as the base namelist file to which we will add the stochastic "seed" +# parameters to obtain the final namelist file for each ensemble member. +# To clarify that this namelist file is not the final namelist file that +# FV3 will read in, we now rename it to the name specified in FV3_NML_BASE_ENS_FP. +# +#----------------------------------------------------------------------- +# +mv_vrfy "${FV3_NML_FP}" "${FV3_NML_BASE_ENS_FP}" +# +#----------------------------------------------------------------------- +# +# Select a cdate (date and hour, in the 10-digit format YYYYMMDDHH) to +# use in the formula for generating the stochastic seed values below. +# Here, we form cdate the starting date and time of the first forecast. +# +#----------------------------------------------------------------------- +# +cdate="${DATE_FIRST_CYCL}${CYCL_HRS[0]}" +# +#----------------------------------------------------------------------- +# +# Now loop through the ensemble members and generate a namelist file for +# each one. +# +#----------------------------------------------------------------------- +# +for (( i=0; i<${NUM_ENS_MEMBERS}; i++ )); do + + fv3_nml_ensmem_fp="${FV3_NML_ENSMEM_FPS[$i]}" + + ip1=$(( i+1 )) + iseed_shum=$(( cdate*1000 + ip1*10 + 2 )) + iseed_skeb=$(( cdate*1000 + ip1*10 + 3 )) + iseed_sppt=$(( cdate*1000 + ip1*10 + 1 )) + + settings="\ +'nam_stochy': { + 'iseed_shum': ${iseed_shum}, + 'iseed_skeb': ${iseed_skeb}, + 'iseed_sppt': ${iseed_sppt}, + }" + + $USHDIR/set_namelist.py -q \ + -n ${FV3_NML_BASE_ENS_FP} \ + -u "$settings" \ + -o ${fv3_nml_ensmem_fp} || \ + print_err_msg_exit "\ +Call to python script set_namelist.py to set the variables in the FV3 +namelist file that specify the paths to the surface climatology files +failed. Parameters passed to this script are: + Full path to base namelist file: + FV3_NML_BASE_ENS_FP = \"${FV3_NML_BASE_ENS_FP}\" + Full path to output namelist file: + fv3_nml_ensmem_fp = \"${fv3_nml_ensmem_fp}\" + Namelist settings specified on command line (these have highest precedence): + settings = +$settings" + +done +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/function. +# +#----------------------------------------------------------------------- +# + { restore_shell_opts; } > /dev/null 2>&1 + +} + diff --git a/ush/set_cycle_dates.sh b/ush/set_cycle_dates.sh new file mode 100644 index 000000000..b8d406c12 --- /dev/null +++ b/ush/set_cycle_dates.sh @@ -0,0 +1,130 @@ +# +#----------------------------------------------------------------------- +# +# This file defines a function that, given the starting date (date_start, +# in the form YYYYMMDD), the ending date (date_end, in the form YYYYMMDD), +# and an array containing the cycle hours for each day (whose elements +# have the form HH), returns an array of cycle date-hours whose elements +# have the form YYYYMMDD. Here, YYYY is a four-digit year, MM is a two- +# digit month, DD is a two-digit day of the month, and HH is a two-digit +# hour of the day. +# +#----------------------------------------------------------------------- +# +function set_cycle_dates() { +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# + { save_shell_opts; set -u +x; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Get the full path to the file in which this script/function is located +# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in +# which the file is located (scrfunc_dir). +# +#----------------------------------------------------------------------- +# + local scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) + local scrfunc_fn=$( basename "${scrfunc_fp}" ) + local scrfunc_dir=$( dirname "${scrfunc_fp}" ) +# +#----------------------------------------------------------------------- +# +# Get the name of this function. +# +#----------------------------------------------------------------------- +# + local func_name="${FUNCNAME[0]}" +# +#----------------------------------------------------------------------- +# +# Specify the set of valid argument names for this script/function. Then +# process the arguments provided to this script/function (which should +# consist of a set of name-value pairs of the form arg1="value1", etc). +# +#----------------------------------------------------------------------- +# + local valid_args=( \ +"date_start" \ +"date_end" \ +"cycle_hrs" \ +"output_varname_all_cdates" \ + ) + process_args valid_args "$@" +# +#----------------------------------------------------------------------- +# +# For debugging purposes, print out values of arguments passed to this +# script. Note that these will be printed out only if VERBOSE is set to +# TRUE. +# +#----------------------------------------------------------------------- +# + print_input_args valid_args +# +#----------------------------------------------------------------------- +# +# Declare local variables. +# +#----------------------------------------------------------------------- +# + local all_cdates date_crnt +# +#----------------------------------------------------------------------- +# +# Ensure that the ending date is at or after the starting date. +# +#----------------------------------------------------------------------- +# + if [ "${date_end}" -lt "${date_start}" ]; then + print_err_msg_exit "\ +End date (date_end) must be at or after start date (date_start): + date_start = \"${date_start}\" + date_end = \"${date_end}\"" + fi +# +#----------------------------------------------------------------------- +# +# In the following "while" loop, we begin with the starting date and +# increment by 1 day each time through the loop until we reach the ending +# date. For each date, we obtain an intermediate array of cdates (whose +# elements have the format YYYYMMDDHH) by prepending to the elements of +# cycle_hrs the current date. (Thus, this array has the same number of +# elements as cycle_hrs.) We then append this intermediate array to the +# final array that will contain all cdates (i.e. over all days and cycle +# hours). +# +#----------------------------------------------------------------------- +# + all_cdates=() + date_crnt="${date_start}" + while [ "${date_crnt}" -le "${date_end}" ]; do + all_cdates+=( $( printf "%s " ${cycle_hrs[@]/#/${date_crnt}} ) ) + date_crnt=$( date -d "${date_crnt} + 1 days" +%Y%m%d ) + done +# +#----------------------------------------------------------------------- +# +# Set output variables. +# +#----------------------------------------------------------------------- +# + all_cdates_str="( "$( printf "\"%s\" " "${all_cdates[@]}" )")" + eval ${output_varname_all_cdates}=${all_cdates_str} +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/function. +# +#----------------------------------------------------------------------- +# + { restore_shell_opts; } > /dev/null 2>&1 + +} + diff --git a/ush/setup.sh b/ush/setup.sh index 57b015b93..21ef37ece 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -57,6 +57,7 @@ cd_vrfy ${scrfunc_dir} # #----------------------------------------------------------------------- # +. ./set_cycle_dates.sh . ./set_gridparams_GFDLgrid.sh . ./set_gridparams_JPgrid.sh . ./link_fix.sh @@ -620,15 +621,21 @@ done # #----------------------------------------------------------------------- # -# Extract from CDATE the starting year, month, day, and hour of the -# forecast. These are needed below for various operations. +# Call a function to generate the array ALL_CDATES containing the cycle +# dates/hours for which to run forecasts. The elements of this array +# will have the form YYYYMMDDHH. They are the starting dates/times of +# the forecasts that will be run in the experiment. Then set NUM_CYCLES +# to the number of elements in this array. # #----------------------------------------------------------------------- # -YYYY_FIRST_CYCL=${DATE_FIRST_CYCL:0:4} -MM_FIRST_CYCL=${DATE_FIRST_CYCL:4:2} -DD_FIRST_CYCL=${DATE_FIRST_CYCL:6:2} -HH_FIRST_CYCL=${CYCL_HRS[0]} +set_cycle_dates \ + date_start="${DATE_FIRST_CYCL}" \ + date_end="${DATE_LAST_CYCL}" \ + cycle_hrs="${CYCL_HRS_str}" \ + output_varname_all_cdates="ALL_CDATES" + +NUM_CYCLES="${#ALL_CDATES[@]}" # #----------------------------------------------------------------------- # @@ -859,12 +866,12 @@ fi # #----------------------------------------------------------------------- # -FCST_LEN_HRS_MAX="999" -if [ "${FCST_LEN_HRS}" -gt "${FCST_LEN_HRS_MAX}" ]; then +fcst_len_hrs_max="999" +if [ "${FCST_LEN_HRS}" -gt "${fcst_len_hrs_max}" ]; then print_err_msg_exit "\ Forecast length is greater than maximum allowed length: FCST_LEN_HRS = ${FCST_LEN_HRS} - FCST_LEN_HRS_MAX = ${FCST_LEN_HRS_MAX}" + fcst_len_hrs_max = ${fcst_len_hrs_max}" fi # #----------------------------------------------------------------------- @@ -1129,8 +1136,9 @@ NEMS_CONFIG_TMPL_FN="${NEMS_CONFIG_FN}" DATA_TABLE_TMPL_FP="${TEMPLATE_DIR}/${DATA_TABLE_TMPL_FN}" DIAG_TABLE_TMPL_FP="${TEMPLATE_DIR}/${DIAG_TABLE_TMPL_FN}" FIELD_TABLE_TMPL_FP="${TEMPLATE_DIR}/${FIELD_TABLE_TMPL_FN}" -FV3_NML_BASE_FP="${TEMPLATE_DIR}/${FV3_NML_BASE_FN}" +FV3_NML_BASE_SUITE_FP="${TEMPLATE_DIR}/${FV3_NML_BASE_SUITE_FN}" FV3_NML_YAML_CONFIG_FP="${TEMPLATE_DIR}/${FV3_NML_YAML_CONFIG_FN}" +FV3_NML_BASE_ENS_FP="${EXPTDIR}/${FV3_NML_BASE_ENS_FN}" MODEL_CONFIG_TMPL_FP="${TEMPLATE_DIR}/${MODEL_CONFIG_TMPL_FN}" NEMS_CONFIG_TMPL_FP="${TEMPLATE_DIR}/${NEMS_CONFIG_TMPL_FN}" # @@ -1198,12 +1206,41 @@ fi # DATA_TABLE_FP="${EXPTDIR}/${DATA_TABLE_FN}" FIELD_TABLE_FP="${EXPTDIR}/${FIELD_TABLE_FN}" -FV3_NML_FN="${FV3_NML_BASE_FN%.*}" +FV3_NML_FN="${FV3_NML_BASE_SUITE_FN%.*}" FV3_NML_FP="${EXPTDIR}/${FV3_NML_FN}" NEMS_CONFIG_FP="${EXPTDIR}/${NEMS_CONFIG_FN}" # #----------------------------------------------------------------------- # +# Make sure that DO_ENSEMBLE is set to a valid value. Then set the names +# of the ensemble members. These will be used to set the ensemble member +# directories. Also, set the full path to the FV3 namelist file corresponding +# to each ensemble member. +# +#----------------------------------------------------------------------- +# +check_var_valid_value "DO_ENSEMBLE" "valid_vals_DO_ENSEMBLE" + +NDIGITS_ENSMEM_NAMES="0" +ENSMEM_NAMES=("") +FV3_NML_ENSMEM_FPS=("") +if [ "${DO_ENSEMBLE}" = TRUE ]; then + NDIGITS_ENSMEM_NAMES="${#NUM_ENS_MEMBERS}" +# Strip away all leading zeros in NUM_ENS_MEMBERS by converting it to a +# decimal (leading zeros will cause bash to interpret the number as an +# octal). Note that the variable definitions file will therefore contain +# the version of NUM_ENS_MEMBERS with any leading zeros stripped away. + NUM_ENS_MEMBERS="$((10#${NUM_ENS_MEMBERS}))" + fmt="%0${NDIGITS_ENSMEM_NAMES}d" + for (( i=0; i<${NUM_ENS_MEMBERS}; i++ )); do + ip1=$( printf "$fmt" $((i+1)) ) + ENSMEM_NAMES[$i]="mem${ip1}" + FV3_NML_ENSMEM_FPS[$i]="$EXPTDIR/${FV3_NML_FN}_${ENSMEM_NAMES[$i]}" + done +fi +# +#----------------------------------------------------------------------- +# # Set the full path to the forecast model executable. # #----------------------------------------------------------------------- @@ -2005,12 +2042,6 @@ NNODES_RUN_FCST=$(( (PE_MEMBER01 + PPN_RUN_FCST - 1)/PPN_RUN_FCST )) #----------------------------------------------------------------------- # check_var_valid_value "OZONE_PARAM_NO_CCPP" "valid_vals_OZONE_PARAM_NO_CCPP" - - - - - - # #----------------------------------------------------------------------- # @@ -2026,9 +2057,6 @@ mkdir_vrfy -p "$EXPTDIR" - - - # #----------------------------------------------------------------------- # @@ -2345,7 +2373,7 @@ var_name = \"${var_name}\"" else # var_value=$(printf "%s" "\\\\\\n") var_value="\\\\\n" - for (( i=0; i<${num_elems}; i++)); do + for (( i=0; i<${num_elems}; i++ )); do # var_value=$(printf "%s\"%s\" %s" "${var_value}" "${array[$i]}" "\\\\\\n") var_value="${var_value}\"${array[$i]}\" \\\\\n" # var_value="${var_value}\"${array[$i]}\" " @@ -2467,6 +2495,10 @@ CYCLE_BASEDIR="${CYCLE_BASEDIR}" GRID_DIR="${GRID_DIR}" OROG_DIR="${OROG_DIR}" SFC_CLIMO_DIR="${SFC_CLIMO_DIR}" + +NDIGITS_ENSMEM_NAMES="${NDIGITS_ENSMEM_NAMES}" +ENSMEM_NAMES=( $( printf "\"%s\" " "${ENSMEM_NAMES[@]}" )) +FV3_NML_ENSMEM_FPS=( $( printf "\"%s\" " "${FV3_NML_ENSMEM_FPS[@]}" )) # #----------------------------------------------------------------------- # @@ -2488,8 +2520,9 @@ NEMS_CONFIG_TMPL_FN="${NEMS_CONFIG_TMPL_FN}" DATA_TABLE_TMPL_FP="${DATA_TABLE_TMPL_FP}" DIAG_TABLE_TMPL_FP="${DIAG_TABLE_TMPL_FP}" FIELD_TABLE_TMPL_FP="${FIELD_TABLE_TMPL_FP}" -FV3_NML_BASE_FP="${FV3_NML_BASE_FP}" +FV3_NML_BASE_SUITE_FP="${FV3_NML_BASE_SUITE_FP}" FV3_NML_YAML_CONFIG_FP="${FV3_NML_YAML_CONFIG_FP}" +FV3_NML_BASE_ENS_FP="${FV3_NML_BASE_ENS_FP}" MODEL_CONFIG_TMPL_FP="${MODEL_CONFIG_TMPL_FP}" NEMS_CONFIG_TMPL_FP="${NEMS_CONFIG_TMPL_FP}" @@ -2661,6 +2694,18 @@ LBC_SPEC_FCST_HRS=(${LBC_SPEC_FCST_HRS[@]}) # #----------------------------------------------------------------------- # +# The number of cycles for which to make forecasts and the list of starting +# dates/hours of these cycles. +# +#----------------------------------------------------------------------- +# +NUM_CYCLES="${NUM_CYCLES}" +ALL_CDATES=( \\ +$( printf "\"%s\" \\\\\n" "${ALL_CDATES[@]}" ) +) +# +#----------------------------------------------------------------------- +# # Computational parameters. # #----------------------------------------------------------------------- diff --git a/ush/templates/FV3.input.yml b/ush/templates/FV3.input.yml index 6d08c3b0d..de54d5dfa 100644 --- a/ush/templates/FV3.input.yml +++ b/ush/templates/FV3.input.yml @@ -356,7 +356,6 @@ FV3_CPT_v0: domains_stack_size: 1800200 fv_core_nml: <<: *gfs_v15_fv_core - consv_te: 1.0 dnats: 0 fv_sg_adj: 450 k_split: 6 diff --git a/ush/templates/FV3SAR_wflow.xml b/ush/templates/FV3SAR_wflow.xml index 2219c8f16..d0d9d2299 100644 --- a/ush/templates/FV3SAR_wflow.xml +++ b/ush/templates/FV3SAR_wflow.xml @@ -70,10 +70,15 @@ tasks; and the "FCST" type is used for the RUN_FCST_TN task. ]> - - {# Double quotes are required inside the strftime! Expect an error from reading the template if using single quotes. #} - {{ date_first_cycl.strftime("%M %H %d %m %Y *") }} - {{ date_first_cycl.strftime("%Y%m%d%H%M") }} {{ date_last_cycl.strftime("%Y%m%d%H%M") }} {{ cycl_freq }} +{# Double quotes are required inside the strftime! Expect an error from reading the template if using single quotes. #} + {{ cdate_first_cycl.strftime("%M %H %d %m %Y *") }} +{% for c in cycl_hrs %} + + {%- set cdate_first=date_first_cycl ~ c ~ "00" -%} + {%- set cdate_last=date_last_cycl ~ c ~ "00" -%} + {{- cdate_first ~ " " ~ cdate_last ~ " " ~ cycl_freq -}} + +{%- endfor %} &LOGDIR;/FV3SAR_wflow.log @@ -146,7 +151,7 @@ MODULES_RUN_TASK_FP script. - &LOGDIR;/make_grid_task_complete.txt + &LOGDIR;/&MAKE_GRID_TN;_task_complete.txt &RUN_TASK_MAKE_GRID;FALSE @@ -237,20 +242,31 @@ MODULES_RUN_TASK_FP script. ************************************************************************ ************************************************************************ --> - +{%- if do_ensemble %} + + + +{%- for m in range(1, num_ens_members+1) -%} + {%- set fmtstr=" %0"~ndigits_ensmem_names~"d" -%} + {{- fmtstr%m -}} +{%- endfor %} +{%- endif %} + + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&MAKE_ICS_TN;" "&JOBSDIR;/JREGIONAL_MAKE_ICS" {{ nnodes_make_ics }}:ppn={{ ppn_make_ics }} {{ wtime_make_ics }} &NCORES_PER_NODE; - &MAKE_ICS_TN; - &LOGDIR;/&MAKE_ICS_TN;_@Y@m@d@H.log + &MAKE_ICS_TN;{{ uscore_ensmem_name }} + &LOGDIR;/&MAKE_ICS_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; PDY@Y@m@d CDATE@Y@m@d@H CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} @@ -278,20 +294,21 @@ MODULES_RUN_TASK_FP script. ************************************************************************ ************************************************************************ --> - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&MAKE_LBCS_TN;" "&JOBSDIR;/JREGIONAL_MAKE_LBCS" {{ nnodes_make_lbcs }}:ppn={{ ppn_make_lbcs }} {{ wtime_make_lbcs }} &NCORES_PER_NODE; - &MAKE_LBCS_TN; - &LOGDIR;/&MAKE_LBCS_TN;_@Y@m@d@H.log + &MAKE_LBCS_TN;{{ uscore_ensmem_name }} + &LOGDIR;/&MAKE_LBCS_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; PDY@Y@m@d CDATE@Y@m@d@H CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} @@ -319,25 +336,27 @@ MODULES_RUN_TASK_FP script. ************************************************************************ ************************************************************************ --> - + &RSRV_FCST; &LOAD_MODULES_RUN_TASK_FP; "&RUN_FCST_TN;" "&JOBSDIR;/JREGIONAL_RUN_FCST" {{ nnodes_run_fcst }}:ppn={{ ppn_run_fcst }} {{ wtime_run_fcst }} &NCORES_PER_NODE; - &RUN_FCST_TN; - &LOGDIR;/&RUN_FCST_TN;_@Y@m@d@H.log + &RUN_FCST_TN;{{ uscore_ensmem_name }} + &LOGDIR;/&RUN_FCST_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; PDY@Y@m@d CDATE@Y@m@d@H CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H - + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + - - + + @@ -346,31 +365,32 @@ MODULES_RUN_TASK_FP script. ************************************************************************ ************************************************************************ --> - + {% for h in range(0, fcst_len_hrs+1) %}{{ " %02d" % h }}{% endfor %} - + &RSRV_DEFAULT; &LOAD_MODULES_RUN_TASK_FP; "&RUN_POST_TN;" "&JOBSDIR;/JREGIONAL_RUN_POST" {{ nnodes_run_post }}:ppn={{ ppn_run_post }} {{ wtime_run_post }} &NCORES_PER_NODE; - &RUN_POST_TN;_#fhr# - &LOGDIR;/&RUN_POST_TN;_#fhr#_@Y@m@d@H.log + &RUN_POST_TN;{{ uscore_ensmem_name }}_f#fhr# + &LOGDIR;/&RUN_POST_TN;{{ uscore_ensmem_name }}_#fhr#_@Y@m@d@H.log GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; PDY@Y@m@d CDATE@Y@m@d@H CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} cyc@H fhr#fhr# - &CYCLE_BASEDIR;/@Y@m@d@H/dynf0#fhr#.nc - &CYCLE_BASEDIR;/@Y@m@d@H/phyf0#fhr#.nc + &CYCLE_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/dynf0#fhr#.nc + &CYCLE_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/phyf0#fhr#.nc @@ -378,4 +398,8 @@ MODULES_RUN_TASK_FP script. +{% if do_ensemble %} + +{% endif %} + diff --git a/ush/valid_param_vals.sh b/ush/valid_param_vals.sh index a4378f03d..2d83d4784 100644 --- a/ush/valid_param_vals.sh +++ b/ush/valid_param_vals.sh @@ -47,6 +47,7 @@ valid_vals_QUILTING=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_PRINT_ESMF=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_USE_CRON_TO_RELAUNCH=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_DOT_OR_USCORE=("." "_") +valid_vals_DO_ENSEMBLE=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_DO_SHUM=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_DO_SPPT=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_DO_SKEB=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no")