From 2c828050ed811f0b4785032ec075a8f4181fb8d3 Mon Sep 17 00:00:00 2001 From: chunhuazhou Date: Thu, 1 Sep 2022 21:28:28 +0000 Subject: [PATCH 1/8] Point to hourly GEFS data for the ensemble LBCs --- ush/config.sh.RRFS_CONUS_3km_ens.2022HWT.CYCL | 12 ++++++------ ush/set_extrn_mdl_params.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ush/config.sh.RRFS_CONUS_3km_ens.2022HWT.CYCL b/ush/config.sh.RRFS_CONUS_3km_ens.2022HWT.CYCL index a14dcef1b..14984c2f2 100644 --- a/ush/config.sh.RRFS_CONUS_3km_ens.2022HWT.CYCL +++ b/ush/config.sh.RRFS_CONUS_3km_ens.2022HWT.CYCL @@ -80,8 +80,8 @@ BOUNDARY_LEN_HRS="6" # avaialble retro period: # 20210511-20210531; 20210718-20210801 -DATE_FIRST_CYCL="20220620" -DATE_LAST_CYCL="20220830" +DATE_FIRST_CYCL="20220901" +DATE_LAST_CYCL="20221031" CYCL_HRS=( "00" "12" ) CYCL_HRS=( "18" ) CYCL_HRS_SPINSTART=("03" "15") @@ -96,9 +96,9 @@ INITIAL_CYCLEDEF="${DATE_FIRST_CYCL}1800 ${DATE_LAST_CYCL}2300 24:00:00" BOUNDARY_CYCLEDEF="${DATE_FIRST_CYCL}1800 ${DATE_LAST_CYCL}2300 24:00:00" #PROD_CYCLEDEF="${DATE_FIRST_CYCL}2100 ${DATE_LAST_CYCL}2300 01:00:00" #POSTPROC_CYCLEDEF="${DATE_FIRST_CYCL}2100 ${DATE_LAST_CYCL}2300 01:00:00" -PROD_CYCLEDEF="00 18-23 * 6-8 2022 *" -RECENTER_CYCLEDEF="00 19 * 6-8 2022 *" -POSTPROC_CYCLEDEF="00 18-23 * 6-8 2022 *" +PROD_CYCLEDEF="00 18-23 * 9-10 2022 *" +RECENTER_CYCLEDEF="00 19 * 9-10 2022 *" +POSTPROC_CYCLEDEF="00 18-23 * 9-10 2022 *" #POSTPROC_LONG_CYCLEDEF="${DATE_FIRST_CYCL}0900 ${DATE_LAST_CYCL}2300 03:00:00" ARCHIVE_CYCLEDEF="${DATE_FIRST_CYCL}1500 ${DATE_LAST_CYCL}2300 24:00:00" if [[ $DO_SPINUP == "TRUE" ]] ; then @@ -159,7 +159,7 @@ WTIME_RUN_FCST="00:30:00" WTIME_MAKE_LBCS="02:00:00" EXTRN_MDL_NAME_ICS="GEFS" -EXTRN_MDL_NAME_LBCS="FV3GFS" +EXTRN_MDL_NAME_LBCS="GEFS" FV3GFS_FILE_FMT_ICS="grib2" FV3GFS_FILE_FMT_LBCS="grib2" diff --git a/ush/set_extrn_mdl_params.sh b/ush/set_extrn_mdl_params.sh index 12ced2700..f5bd447cd 100644 --- a/ush/set_extrn_mdl_params.sh +++ b/ush/set_extrn_mdl_params.sh @@ -325,7 +325,7 @@ else "GEFS") case "$MACHINE" in "JET") - EXTRN_MDL_SYSBASEDIR_LBCS="/mnt/lfs4/BMC/public/data/grids/gens/pgrb2b" + EXTRN_MDL_SYSBASEDIR_LBCS="/mnt/lfs4/BMC/wrfruc/RRFSE/data/gens" ;; esac ;; From 0cd33d5586611c1c25a160ba7a1dd14833bb6b3c Mon Sep 17 00:00:00 2001 From: chunhuazhou Date: Thu, 22 Sep 2022 22:47:41 +0000 Subject: [PATCH 2/8] Add user-defined GEFS ICs/LBCs path --- ush/set_extrn_mdl_params.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ush/set_extrn_mdl_params.sh b/ush/set_extrn_mdl_params.sh index f5bd447cd..f2b11efa1 100644 --- a/ush/set_extrn_mdl_params.sh +++ b/ush/set_extrn_mdl_params.sh @@ -126,7 +126,7 @@ else EXTRN_MDL_SYSBASEDIR_ICS="" ;; "JET") - EXTRN_MDL_SYSBASEDIR_ICS="/mnt/lfs4/BMC/public/data/grids/gens/pgrb2b" + EXTRN_MDL_SYSBASEDIR_ICS="${EXTRN_MDL_SOURCE_BASEDIR_ICS:-/mnt/lfs4/BMC/public/data/grids/gens/pgrb2b}" ;; esac ;; @@ -325,7 +325,7 @@ else "GEFS") case "$MACHINE" in "JET") - EXTRN_MDL_SYSBASEDIR_LBCS="/mnt/lfs4/BMC/wrfruc/RRFSE/data/gens" + EXTRN_MDL_SYSBASEDIR_LBCS="${EXTRN_MDL_SOURCE_BASEDIR_LBCS:-/mnt/lfs4/BMC/public/data/grids/gens/pgrb2b}" ;; esac ;; From 9ab7daeb04bdebe1d3bb370a71cfda247e6e3e21 Mon Sep 17 00:00:00 2001 From: chunhuazhou Date: Thu, 22 Sep 2022 22:55:34 +0000 Subject: [PATCH 3/8] Changing config.sh for running ensemble initialization at start of the cycle, and specifying cycles to run SPP --- ush/config.sh.RRFS_CONUS_3km_ens.2022HWT.CYCL | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/ush/config.sh.RRFS_CONUS_3km_ens.2022HWT.CYCL b/ush/config.sh.RRFS_CONUS_3km_ens.2022HWT.CYCL index 14984c2f2..ebf5a315f 100644 --- a/ush/config.sh.RRFS_CONUS_3km_ens.2022HWT.CYCL +++ b/ush/config.sh.RRFS_CONUS_3km_ens.2022HWT.CYCL @@ -80,25 +80,29 @@ BOUNDARY_LEN_HRS="6" # avaialble retro period: # 20210511-20210531; 20210718-20210801 -DATE_FIRST_CYCL="20220901" +DATE_FIRST_CYCL="20220922" DATE_LAST_CYCL="20221031" CYCL_HRS=( "00" "12" ) CYCL_HRS=( "18" ) CYCL_HRS_SPINSTART=("03" "15") CYCL_HRS_PRODSTART=("09" "21") CYCL_HRS_PRODSTART=("18") -CYCL_HRS_RECENTER=("19") +CYCL_HRS_ENSINIT=("18") +CYCL_HRS_SPP=("00") +#CYCL_HRS_RECENTER=("19") #CYCLEMONTH="5" #CYCLEDAY="11-12" PREEXISTING_DIR_METHOD="upgrade" # "rename" INITIAL_CYCLEDEF="${DATE_FIRST_CYCL}1800 ${DATE_LAST_CYCL}2300 24:00:00" -BOUNDARY_CYCLEDEF="${DATE_FIRST_CYCL}1800 ${DATE_LAST_CYCL}2300 24:00:00" +#BOUNDARY_CYCLEDEF="${DATE_FIRST_CYCL}1800 ${DATE_LAST_CYCL}2300 24:00:00" +BOUNDARY_CYCLEDEF="00 00,18 * 9-10 2022 *" #PROD_CYCLEDEF="${DATE_FIRST_CYCL}2100 ${DATE_LAST_CYCL}2300 01:00:00" #POSTPROC_CYCLEDEF="${DATE_FIRST_CYCL}2100 ${DATE_LAST_CYCL}2300 01:00:00" -PROD_CYCLEDEF="00 18-23 * 9-10 2022 *" -RECENTER_CYCLEDEF="00 19 * 9-10 2022 *" -POSTPROC_CYCLEDEF="00 18-23 * 9-10 2022 *" +ENSINIT_CYCLEDEF="00 18 * 9-10 2022 *" +PROD_CYCLEDEF="00 00,18-23 * 9-10 2022 *" +#RECENTER_CYCLEDEF="00 19 * 9-10 2022 *" +POSTPROC_CYCLEDEF="00 00,18-23 * 9-10 2022 *" #POSTPROC_LONG_CYCLEDEF="${DATE_FIRST_CYCL}0900 ${DATE_LAST_CYCL}2300 03:00:00" ARCHIVE_CYCLEDEF="${DATE_FIRST_CYCL}1500 ${DATE_LAST_CYCL}2300 24:00:00" if [[ $DO_SPINUP == "TRUE" ]] ; then @@ -160,6 +164,8 @@ WTIME_MAKE_LBCS="02:00:00" EXTRN_MDL_NAME_ICS="GEFS" EXTRN_MDL_NAME_LBCS="GEFS" +EXTRN_MDL_SOURCE_BASEDIR_ICS="/mnt/lfs4/BMC/wrfruc/RRFSE/data/gens" +EXTRN_MDL_SOURCE_BASEDIR_LBCS="/mnt/lfs4/BMC/wrfruc/RRFSE/data/gens" FV3GFS_FILE_FMT_ICS="grib2" FV3GFS_FILE_FMT_LBCS="grib2" @@ -208,14 +214,15 @@ if [[ ${DO_ENSEMBLE} == "TRUE" ]]; then # DO_ENSCONTROL="TRUE" DO_GSIOBSERVER="TRUE" DO_ENKFUPDATE="TRUE" - DO_RECENTER="TRUE" +# DO_RECENTER="TRUE" DO_ENS_GRAPHICS="TRUE" DO_ENKF_RADAR_REF="TRUE" DO_ENSPOST="FALSE" + DO_ENSINIT="TRUE" RADAR_REF_THINNING="2" ARCHIVEDIR="/5year/BMC/wrfruc/rrfs_ens" NNODES_RUN_RECENTER="3" - PPN_RUN_RECENTER="31" + PPN_RUN_RECENTER="$(( ${NUM_ENS_MEMBERS} + 1 ))" CLEAN_OLDFCST_HRS="48" CLEAN_OLDSTMPPOST_HRS="48" cld_bld_hgt=0.0 @@ -232,7 +239,7 @@ COMINgfs="" STMP="/mnt/lfs4/BMC/wrfruc/RRFSE/NCO_dirs/stmp" # Path to directory STMP that mostly contains input files. PTMP="/mnt/lfs4/BMC/wrfruc/RRFSE/NCO_dirs/ptmp" # Path to directory STMP that mostly contains input files. NWGES="/mnt/lfs4/BMC/wrfruc/RRFSE/NCO_dirs/nwges" # Path to directory NWGES that save boundary, cold initial, restart files -if [[ ${DO_RECENTER} == "TRUE" ]]; then +if [[ ${DO_RECENTER} == "TRUE" || ${DO_ENSINIT} == "TRUE" ]]; then ENSCTRL_STMP="/mnt/lfs4/BMC/nrtrr/NCO_dirs/stmp" # Path to directory STMP that mostly contains control input files for ensemble recentering. ENSCTRL_PTMP="/mnt/lfs4/BMC/nrtrr/NCO_dirs/ptmp" # Path to directory STMP that mostly contains control input files for ensemble recentering. ENSCTRL_NWGES="/mnt/lfs4/BMC/nrtrr/NCO_dirs/nwges" # Path to directory STMP that mostly contains control input files for ensemble recentering. From 9b76fdffec474d2aaf276c4873b2eda6015845bb Mon Sep 17 00:00:00 2001 From: chunhuazhou Date: Thu, 22 Sep 2022 23:45:29 +0000 Subject: [PATCH 4/8] Changes for running ensemble initialization at 1st time step and specifying hours to turn SPP on --- jobs/JREGIONAL_RUN_FCST | 2 + jobs/JREGIONAL_RUN_PREPSTART | 9 + jobs/JREGIONAL_RUN_PREPSTART_ENSINIT | 119 +++++++ jobs/JREGIONAL_SAVE_RESTART | 3 + scripts/exregional_run_fcst.sh | 12 +- scripts/exregional_run_prepstart_ensinit.sh | 324 ++++++++++++++++++ scripts/exregional_save_restart.sh | 2 +- ush/config.sh.RRFS_CONUS_3km_ens.2022HWT.CYCL | 11 +- ush/config_defaults.sh | 34 +- ush/create_model_configure_file.sh | 14 + ush/generate_FV3LAM_wflow.sh | 8 + ush/templates/FV3LAM_wflow.xml | 176 +++++++++- ush/valid_param_vals.sh | 1 + 13 files changed, 702 insertions(+), 13 deletions(-) create mode 100755 jobs/JREGIONAL_RUN_PREPSTART_ENSINIT create mode 100755 scripts/exregional_run_prepstart_ensinit.sh diff --git a/jobs/JREGIONAL_RUN_FCST b/jobs/JREGIONAL_RUN_FCST index 831da9a62..2bddff2a2 100755 --- a/jobs/JREGIONAL_RUN_FCST +++ b/jobs/JREGIONAL_RUN_FCST @@ -68,6 +68,8 @@ the specified cycle. CYCLE_TYPE=${CYCLE_TYPE:-prod} if [ ${CYCLE_TYPE} == "spinup" ]; then run_dir="${CYCLE_DIR}${SLASH_ENSMEM_SUBDIR}/fcst_fv3lam_spinup" +elif [ ${CYCLE_TYPE} == "ensinit" ]; then + run_dir="${CYCLE_DIR}${SLASH_ENSMEM_SUBDIR}/fcst_fv3lam_ensinit" else run_dir="${CYCLE_DIR}${SLASH_ENSMEM_SUBDIR}/fcst_fv3lam" fi diff --git a/jobs/JREGIONAL_RUN_PREPSTART b/jobs/JREGIONAL_RUN_PREPSTART index f80d82760..cbf144564 100755 --- a/jobs/JREGIONAL_RUN_PREPSTART +++ b/jobs/JREGIONAL_RUN_PREPSTART @@ -72,6 +72,15 @@ if [ ${CYCLE_TYPE} == "spinup" ]; then else modelinputdir=${CYCLE_DIR}${SLASH_ENSMEM_SUBDIR}/fcst_fv3lam/INPUT fi + +if [ ${DO_ENSINIT} == "TRUE" ]; then + for cyc_start in "${CYCL_HRS_ENSINIT[@]}"; do + if [ ${HH} -eq ${cyc_start} ]; then + modelinputdir=${CYCLE_DIR}${SLASH_ENSMEM_SUBDIR}/fcst_fv3lam_ensinit/INPUT + fi + done +fi + rm -fr ${modelinputdir} mkdir_vrfy -p ${modelinputdir} # diff --git a/jobs/JREGIONAL_RUN_PREPSTART_ENSINIT b/jobs/JREGIONAL_RUN_PREPSTART_ENSINIT new file mode 100755 index 000000000..0391eb99e --- /dev/null +++ b/jobs/JREGIONAL_RUN_PREPSTART_ENSINIT @@ -0,0 +1,119 @@ +#!/bin/bash + +# +#----------------------------------------------------------------------- +# +# This J-JOB script stages initial and boundary conditions for +# the analysis and model of the cycle it is running. The initial +# could come from external model or previous cycles forecast. +# The boundary could come from previous many cycles. +# +#----------------------------------------------------------------------- + +# +#----------------------------------------------------------------------- +# +# Source the variable definitions file and the bash utility functions. +# +#----------------------------------------------------------------------- +# +. ${GLOBAL_VAR_DEFNS_FP} +. $USHDIR/source_util_funcs.sh +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# +{ save_shell_opts; set -u +x; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Get the full path to the file in which this script/function is located +# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in +# which the file is located (scrfunc_dir). +# +#----------------------------------------------------------------------- +# +scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fn=$( basename "${scrfunc_fp}" ) +scrfunc_dir=$( dirname "${scrfunc_fp}" ) +# +#----------------------------------------------------------------------- +# +# Print message indicating entry into script. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +Entering script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" + +This is the J-job script for the task that runs a analysis with FV3 for +the specified cycle. +========================================================================" +# +#----------------------------------------------------------------------- +# +# Create the working directory under the cycle directory. +# +#----------------------------------------------------------------------- +# +# CYCLE_TYPE is to indicate which type of cycle this job for: +# spinup: this job is for spin-up cycles +# others(prod): this job is for product cycle +# +CYCLE_TYPE=${CYCLE_TYPE:-prod} +if [ ${CYCLE_TYPE} == "spinup" ]; then + modelinputdir=${CYCLE_DIR}${SLASH_ENSMEM_SUBDIR}/fcst_fv3lam_spinup/INPUT +else + modelinputdir=${CYCLE_DIR}${SLASH_ENSMEM_SUBDIR}/fcst_fv3lam/INPUT +fi +rm -fr ${modelinputdir} +mkdir_vrfy -p ${modelinputdir} +# +# +# +gridspec_dir=${NWGES_BASEDIR}/grid_spec +# +#----------------------------------------------------------------------- +# +# Call the ex-script for this J-job and pass to it the necessary varia- +# bles. +# +#----------------------------------------------------------------------- +# +$SCRIPTSDIR/exregional_run_prepstart_ensinit.sh \ + cycle_dir="${CYCLE_DIR}" \ + cycle_type="${CYCLE_TYPE}" \ + gridspec_dir="${gridspec_dir}" \ + modelinputdir="${modelinputdir}" \ + lbcs_root="${LBCS_ROOT}" \ + fg_root="${FG_ROOT}" || \ +print_err_msg_exit "\ +Call to ex-script corresponding to J-job \"${scrfunc_fn}\" failed." +# +#----------------------------------------------------------------------- +# +# Print exit message. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +Exiting script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" +========================================================================" +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/func- +# tion. +# +#----------------------------------------------------------------------- +# +{ restore_shell_opts; } > /dev/null 2>&1 + diff --git a/jobs/JREGIONAL_SAVE_RESTART b/jobs/JREGIONAL_SAVE_RESTART index 38bad90af..7edae6592 100755 --- a/jobs/JREGIONAL_SAVE_RESTART +++ b/jobs/JREGIONAL_SAVE_RESTART @@ -69,6 +69,9 @@ CYCLE_TYPE=${CYCLE_TYPE:-prod} if [ ${CYCLE_TYPE} == "spinup" ]; then run_dir="${CYCLE_DIR}${SLASH_ENSMEM_SUBDIR}/fcst_fv3lam_spinup" nwges_dir="${NWGES_DIR}${SLASH_ENSMEM_SUBDIR}/fcst_fv3lam_spinup" +elif [ ${CYCLE_TYPE} == "ensinit" ]; then + run_dir="${CYCLE_DIR}${SLASH_ENSMEM_SUBDIR}/fcst_fv3lam_ensinit" + nwges_dir="${NWGES_DIR}${SLASH_ENSMEM_SUBDIR}/fcst_fv3lam_ensinit" else run_dir="${CYCLE_DIR}${SLASH_ENSMEM_SUBDIR}/fcst_fv3lam" nwges_dir="${NWGES_DIR}${SLASH_ENSMEM_SUBDIR}/fcst_fv3lam" diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index d409099c0..d6112fd24 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -502,15 +502,23 @@ else fi fi +for cyc_start in "${CYCL_HRS_SPP[@]}"; do + hh=${cdate:8:2} + if [ ${hh} -eq ${cyc_start} ]; then + DO_SPP = TRUE + fi +done + if [ "${DO_ENSEMBLE}" = TRUE ] && ([ "${DO_SPP}" = TRUE ] || [ "${DO_SPPT}" = TRUE ] || [ "${DO_SHUM}" = TRUE ] \ [ "${DO_SKEB}" = TRUE ] || [ "${DO_LSM_SPP}" = TRUE ]); then cp ${run_dir}/input.nml ${run_dir}/input.nml_base set_FV3nml_ens_stoch_seeds cdate="$cdate" || print_err_msg_exit "\ - Call to function to create the ensemble-based namelist for the current - cycle's (cdate) run directory (run_dir) failed: + Call to function to create the ensemble-based namelist for the current + cycle's (cdate) run directory (run_dir) failed: cdate = \"${cdate}\" run_dir = \"${run_dir}\"" fi + # #----------------------------------------------------------------------- # diff --git a/scripts/exregional_run_prepstart_ensinit.sh b/scripts/exregional_run_prepstart_ensinit.sh new file mode 100755 index 000000000..ddc95478d --- /dev/null +++ b/scripts/exregional_run_prepstart_ensinit.sh @@ -0,0 +1,324 @@ +#!/bin/bash + +# +#----------------------------------------------------------------------- +# +# Source the variable definitions file and the bash utility functions. +# +#----------------------------------------------------------------------- +# +. ${GLOBAL_VAR_DEFNS_FP} +. $USHDIR/source_util_funcs.sh +# +#----------------------------------------------------------------------- +# +# Save current shell options (in a global array). Then set new options +# for this script/function. +# +#----------------------------------------------------------------------- +# +{ save_shell_opts; set -u -x; } > /dev/null 2>&1 +# +#----------------------------------------------------------------------- +# +# Get the full path to the file in which this script/function is located +# (scrfunc_fp), the name of that file (scrfunc_fn), and the directory in +# which the file is located (scrfunc_dir). +# +#----------------------------------------------------------------------- +# +scrfunc_fp=$( readlink -f "${BASH_SOURCE[0]}" ) +scrfunc_fn=$( basename "${scrfunc_fp}" ) +scrfunc_dir=$( dirname "${scrfunc_fp}" ) +# +#----------------------------------------------------------------------- +# +# Print message indicating entry into script. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +Entering script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" + +This is the ex-script for the task that runs a analysis with FV3 for the +specified cycle. +========================================================================" +# +#----------------------------------------------------------------------- +# +# Specify the set of valid argument names for this script/function. +# Then process the arguments provided to this script/function (which +# should consist of a set of name-value pairs of the form arg1="value1", +# etc). +# +#----------------------------------------------------------------------- +# +valid_args=( "cycle_dir" "cycle_type" "gridspec_dir" "modelinputdir" "lbcs_root" "fg_root") +process_args valid_args "$@" +# +#----------------------------------------------------------------------- +# +# For debugging purposes, print out values of arguments passed to this +# script. Note that these will be printed out only if VERBOSE is set to +# TRUE. +# +#----------------------------------------------------------------------- +# +print_input_args valid_args +# +#----------------------------------------------------------------------- +# +# Load modules. +# +#----------------------------------------------------------------------- +# +case $MACHINE in + + "WCOSS_CRAY") + export ntasks=1 + export ptile=1 + export threads=1 + APRUN="aprun -j 1 -n${ntasks} -N${ptile} -d${threads} -cc depth" + ;; + + "WCOSS_DELL_P3") + APRUN="mpirun" + ;; + + "HERA") + APRUN="srun" + ;; + + "ORION") + APRUN="srun" + ;; + + "JET") + APRUN="srun" + ;; + + "ODIN") + APRUN="srun -n 1" + ;; + + "CHEYENNE") + module list + APRUN="mpirun -np 1" + ;; + + "STAMPEDE") + APRUN="ibrun -n 1" + ;; + + *) + print_err_msg_exit "\ +Run command has not been specified for this machine: + MACHINE = \"$MACHINE\" + APRUN = \"$APRUN\"" + ;; + +esac + +# +#----------------------------------------------------------------------- +# +# Load modules. +# +#----------------------------------------------------------------------- +# +#----------------------------------------------------------------------- +# +# Extract from CDATE the starting year, month, day, and hour of the +# forecast. These are needed below for various operations. +# +#----------------------------------------------------------------------- +# +START_DATE=$(echo "${CDATE}" | sed 's/\([[:digit:]]\{2\}\)$/ \1/') + +YYYYMMDDHH=$(date +%Y%m%d%H -d "${START_DATE}") + +YYYY=${YYYYMMDDHH:0:4} +MM=${YYYYMMDDHH:4:2} +DD=${YYYYMMDDHH:6:2} +HH=${YYYYMMDDHH:8:2} +YYYYMMDD=${YYYYMMDDHH:0:8} + +current_time=$(date "+%T") + +# +#----------------------------------------------------------------------- +# +# go to INPUT directory. +# prepare initial conditions for +# cold start if BKTYPE=1 +# warm start if BKTYPE=0 +# spinupcyc + warm start if BKTYPE=2 +# the previous 6 cycles are searched to find the restart files +# valid at this time from the closet previous cycle. +# +#----------------------------------------------------------------------- + +cd_vrfy ${modelinputdir} + +# Setup the INPUT directory for warm start cycles, which can be spin-up cycle or product cycle. +# +# First decide the source of the first guess (fg_restart_dirname) depending on cycle_type and BKTYPE: +# 1. If cycle is spinup cycle (cycle_type == spinup) or it is the product start cycle (BKTYPE==2), +# looking for the first guess from spinup forecast (fcst_fv3lam_spinup) +# 2. Others, looking for the first guess from product forecast (fcst_fv3lam) +# + + fg_restart_dirname=fcst_fv3lam_ensinit + bkpath=${fg_root}/${YYYYMMDDHH}${SLASH_ENSMEM_SUBDIR}/${fg_restart_dirname}/RESTART # cycling, use background from RESTART + +# +# the restart file from FV3 has a name like: ${YYYYMMDD}.${HH}0000.fv_core.res.tile1.nc +# But the restart files for the forecast length has a name like: fv_core.res.tile1.nc +# So the defination of restart_prefix needs a "." at the end. +# + restart_prefix="${YYYYMMDD}.${HH}0000." + filelistn="fv_core.res.tile1.nc fv_srf_wnd.res.tile1.nc fv_tracer.res.tile1.nc phy_data.nc sfc_data.nc" + checkfile=${bkpath}/${restart_prefix}coupler.res + n_iolayouty=$(($IO_LAYOUT_Y-1)) + list_iolayout=$(seq 0 $n_iolayouty) + if [ -r "${checkfile}" ] ; then + cp_vrfy ${bkpath}/${restart_prefix}coupler.res bk_coupler.res + cp_vrfy ${bkpath}/${restart_prefix}fv_core.res.nc fv_core.res.nc + if [ "${IO_LAYOUT_Y}" == "1" ]; then + for file in ${filelistn}; do + cp_vrfy ${bkpath}/${restart_prefix}${file} ${file} + ln_vrfy -s ${bkpath}/${restart_prefix}${file} bk_${file} + done + else + for file in ${filelistn}; do + for ii in $list_iolayout + do + iii=$(printf %4.4i $ii) + cp_vrfy ${bkpath}/${restart_prefix}${file}.${iii} ${file}.${iii} + ln_vrfy -s ${bkpath}/${restart_prefix}${file}.${iii} bk_${file}.${iii} + done + done + fi + cp_vrfy ${fg_root}/${YYYYMMDDHH}${SLASH_ENSMEM_SUBDIR}/${fg_restart_dirname}/INPUT/gfs_ctrl.nc gfs_ctrl.nc + if [ ${SAVE_CYCLE_LOG} == "TRUE" ] ; then + echo "${YYYYMMDDHH}(${cycle_type}): warm start at ${current_time} from ${checkfile} " >> ${EXPTDIR}/log.cycles + fi +# +# remove checksum from restart files. Checksum will cause trouble if model initializes from analysis +# + if [ "${IO_LAYOUT_Y}" == "1" ]; then + for file in ${filelistn}; do + ncatted -a checksum,,d,, ${file} + done + else + for file in ${filelistn}; do + for ii in $list_iolayout + do + iii=$(printf %4.4i $ii) + ncatted -a checksum,,d,, ${file}.${iii} + done + done + fi + ncatted -a checksum,,d,, fv_core.res.nc + +# generate coupler.res with right date + head -2 bk_coupler.res > coupler.res + head -2 bk_coupler.res | tail -1 >> coupler.res + else + print_err_msg_exit "Error: cannot find background: ${checkfile}" + fi + +# +#----------------------------------------------------------------------- +# +# go to INPUT directory. +# prepare boundary conditions: +# the previous 12 cycles are searched to find the boundary files +# that can cover the forecast length. +# The 0-h boundary is copied and others are linked. +# +#----------------------------------------------------------------------- + + num_fhrs=( "${#FCST_LEN_HRS_CYCLES[@]}" ) + ihh=$( expr ${HH} + 0 ) + if [ ${num_fhrs} -gt ${ihh} ]; then + FCST_LEN_HRS_thiscycle=${FCST_LEN_HRS_CYCLES[${ihh}]} + else + FCST_LEN_HRS_thiscycle=${FCST_LEN_HRS} + fi + if [ ${cycle_type} == "spinup" ]; then + FCST_LEN_HRS_thiscycle=${FCST_LEN_HRS_SPINUP} + fi + print_info_msg "$VERBOSE" " The forecast length for cycle (\"${HH}\") is + ( \"${FCST_LEN_HRS_thiscycle}\") " + +# let us figure out which boundary file is available + bndy_prefix=gfs_bndy.tile7 + n=${EXTRN_MDL_LBCS_SEARCH_OFFSET_HRS} + end_search_hr=$(( 12 + ${EXTRN_MDL_LBCS_SEARCH_OFFSET_HRS} )) + YYYYMMDDHHmInterv=$(date +%Y%m%d%H -d "${START_DATE} ${n} hours ago") + lbcs_path=${lbcs_root}/${YYYYMMDDHHmInterv}${SLASH_ENSMEM_SUBDIR}/lbcs + while [[ $n -le ${end_search_hr} ]] ; do + last_bdy_time=$(( n + ${FCST_LEN_HRS_thiscycle} )) + last_bdy=$(printf %3.3i $last_bdy_time) + checkfile=${lbcs_path}/${bndy_prefix}.${last_bdy}.nc + if [ -r "${checkfile}" ]; then + print_info_msg "$VERBOSE" "Found ${checkfile}; Use it as boundary for forecast " + break + else + n=$((n + 1)) + YYYYMMDDHHmInterv=$(date +%Y%m%d%H -d "${START_DATE} ${n} hours ago") + lbcs_path=${lbcs_root}/${YYYYMMDDHHmInterv}${SLASH_ENSMEM_SUBDIR}/lbcs + fi + done +# + relative_or_null="--relative" + nb=1 + if [ -r "${checkfile}" ]; then + while [ $nb -le ${FCST_LEN_HRS_thiscycle} ] + do + bdy_time=$(( ${n} + ${nb} )) + this_bdy=$(printf %3.3i $bdy_time) + local_bdy=$(printf %3.3i $nb) + + if [ -f "${lbcs_path}/${bndy_prefix}.${this_bdy}.nc" ]; then + ln_vrfy -sf ${relative_or_null} ${lbcs_path}/${bndy_prefix}.${this_bdy}.nc ${bndy_prefix}.${local_bdy}.nc + fi + + nb=$((nb + 1)) + done +# check 0-h boundary condition + if [ ! -f "${bndy_prefix}.000.nc" ]; then + this_bdy=$(printf %3.3i ${n}) + cp_vrfy ${lbcs_path}/${bndy_prefix}.${this_bdy}.nc ${bndy_prefix}.000.nc + fi + else + print_err_msg_exit "Error: cannot find boundary file: ${checkfile}" + fi + +# +#----------------------------------------------------------------------- +# +# Print message indicating successful completion of script. +# +#----------------------------------------------------------------------- +# +print_info_msg " +======================================================================== +Prepare start completed successfully!!! + +Exiting script: \"${scrfunc_fn}\" +In directory: \"${scrfunc_dir}\" +========================================================================" +# +#----------------------------------------------------------------------- +# +# Restore the shell options saved at the beginning of this script/func- +# tion. +# +#----------------------------------------------------------------------- +# +{ restore_shell_opts; } > /dev/null 2>&1 + diff --git a/scripts/exregional_save_restart.sh b/scripts/exregional_save_restart.sh index e399297ed..964719234 100755 --- a/scripts/exregional_save_restart.sh +++ b/scripts/exregional_save_restart.sh @@ -166,7 +166,7 @@ else print_info_msg "$VERBOSE" " The forecast length for cycle (\"${hh}\") is ( \"${FCST_LEN_HRS_thiscycle}\") " - if [ -r "$run_dir/RESTART/coupler.res" ] && [ ${fhr} -eq ${FCST_LEN_HRS_thiscycle} ] ; then + if [ -r "$run_dir/RESTART/coupler.res" ] && ([ ${fhr} -eq ${FCST_LEN_HRS_thiscycle} ] || [ ${cycle_type} == "ensinit" ]) ; then if [ "${IO_LAYOUT_Y}" == "1" ]; then for file in ${filelistn}; do mv_vrfy $run_dir/RESTART/${file} ${nwges_dir}/RESTART/${restart_prefix}.${file} diff --git a/ush/config.sh.RRFS_CONUS_3km_ens.2022HWT.CYCL b/ush/config.sh.RRFS_CONUS_3km_ens.2022HWT.CYCL index ebf5a315f..8773e1668 100644 --- a/ush/config.sh.RRFS_CONUS_3km_ens.2022HWT.CYCL +++ b/ush/config.sh.RRFS_CONUS_3km_ens.2022HWT.CYCL @@ -76,7 +76,7 @@ CCPP_PHYS_SUITE="FV3_HRRR" EXTRN_MDL_ICS_OFFSET_HRS="30" LBC_SPEC_INTVL_HRS="1" EXTRN_MDL_LBCS_OFFSET_HRS="6" -BOUNDARY_LEN_HRS="6" +BOUNDARY_LEN_HRS="36" # avaialble retro period: # 20210511-20210531; 20210718-20210801 @@ -103,7 +103,7 @@ ENSINIT_CYCLEDEF="00 18 * 9-10 2022 *" PROD_CYCLEDEF="00 00,18-23 * 9-10 2022 *" #RECENTER_CYCLEDEF="00 19 * 9-10 2022 *" POSTPROC_CYCLEDEF="00 00,18-23 * 9-10 2022 *" -#POSTPROC_LONG_CYCLEDEF="${DATE_FIRST_CYCL}0900 ${DATE_LAST_CYCL}2300 03:00:00" +POSTPROC_LONG_CYCLEDEF="${DATE_FIRST_CYCL}0000 ${DATE_LAST_CYCL}2300 24:00:00" ARCHIVE_CYCLEDEF="${DATE_FIRST_CYCL}1500 ${DATE_LAST_CYCL}2300 24:00:00" if [[ $DO_SPINUP == "TRUE" ]] ; then SPINUP_CYCLEDEF="00 03-08,15-20 ${CYCLEDAY} ${CYCLEMONTH} 2021 *" @@ -112,17 +112,20 @@ fi FCST_LEN_HRS="1" FCST_LEN_HRS_SPINUP="1" POSTPROC_LEN_HRS="1" -POSTPROC_LONG_LEN_HRS="2" +POSTPROC_LONG_LEN_HRS="36" #FCST_LEN_HRS_CYCLES=(48 18 18 18 18 18 48 18 18 18 18 18 48 18 18 18 18 18 48 18 18 18 18 18) for i in {0..23}; do FCST_LEN_HRS_CYCLES[$i]=1; done for i in {0..23..3}; do FCST_LEN_HRS_CYCLES[$i]=1; done for i in 0; do FCST_LEN_HRS_CYCLES[$i]=36; done DA_CYCLE_INTERV="1" RESTART_INTERVAL="1" +netcdf_diag=.true. +binary_diag=.false. #SST_update_hour=01 #GVF_update_hour=04 +DT_ATMOS=60 NCORES_RUN_ANAL=360 NCORES_RUN_OBSERVER=80 HYBENSMEM_NMIN=66 @@ -159,7 +162,7 @@ if [[ -n $RESERVATION ]] ; then PPN_RUN_POST="40" fi -WTIME_RUN_FCST="00:30:00" +WTIME_RUN_FCST="05:30:00" WTIME_MAKE_LBCS="02:00:00" EXTRN_MDL_NAME_ICS="GEFS" diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index 9837a4d98..b144c569a 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -482,6 +482,21 @@ WFLOW_LAUNCH_LOG_FN="log.launch_FV3LAM_wflow" # CYCL_HRS_RECENTER: # An array containing the hours of the day at which the ensemble recenter is on # +# CYCL_HRS_ENSINIT: +# An array containing the hours of the day at which the ensemble initialization is on +# +# CYCL_HRS_SPP: +# An array containing the hours of the day at which the SPP is on +# +# CYCL_HRS_SPPT: +# An array containing the hours of the day at which the SPPT is on +# +# CYCL_HRS_SHUM: +# An array containing the hours of the day at which the SHUM is on +# +# CYCL_HRS_SKEB: +# An array containing the hours of the day at which the SKEB is on +# # BOUNDARY_LEN_HRS # The length of boundary condition for normal forecast, in integer hours. # @@ -522,6 +537,11 @@ CYCL_HRS=( "HH1" "HH2" ) CYCL_HRS_SPINSTART=( "HH1" "HH2" ) CYCL_HRS_PRODSTART=( "HH1" "HH2" ) CYCL_HRS_RECENTER=( "HH1" "HH2" ) +CYCL_HRS_ENSINIT=( "HH1" "HH2" ) +CYCL_HRS_SPP=( "HH1" "HH2" ) +CYCL_HRS_SPPT=( "HH1" "HH2" ) +CYCL_HRS_SHUM=( "HH1" "HH2" ) +CYCL_HRS_SKEB=( "HH1" "HH2" ) BOUNDARY_LEN_HRS="0" BOUNDARY_LONG_LEN_HRS="0" POSTPROC_LEN_HRS="1" @@ -565,6 +585,11 @@ CYCL_HRS_HYB_FV3LAM_ENS=( "99" ) # cycle definition for "boundary_long" group # This group runs: get_extrn_lbcs_long,make_lbcs # +# ENSINIT_CYCLEDEF: +# cycle definition for ensemble initialization cycle group +# This group runs run_fcst for 1 timestep to get restart files at +# the beginning of the cycles and to be recentered at the control analysis +# # SPINUP_CYCLEDEF: # cycle definition for spin-up cycle group # This group runs: anal_gsi_input_spinup and data process, run_fcst_spinup, run_post_spinup @@ -597,6 +622,7 @@ AT_START_CYCLEDEF="00 01 01 01 2100 *" INITIAL_CYCLEDEF="00 01 01 01 2100 *" BOUNDARY_CYCLEDEF="00 01 01 01 2100 *" BOUNDARY_LONG_CYCLEDEF="00 01 01 01 2100 *" +ENSINIT_CYCLEDEF="00 01 01 01 2100 *" SPINUP_CYCLEDEF="00 01 01 01 2100 *" PROD_CYCLEDEF="00 01 01 01 2100 *" RECENTER_CYCLEDEF="00 01 01 01 2100 *" @@ -1153,8 +1179,6 @@ ESGgrid_WIDE_HALO_WIDTH="" # NSOUT_MIN # setup frequency of writing out forecast files in minutes # -#----------------------------------------------------------------------- -# DT_ATMOS="" LAYOUT_X="" LAYOUT_Y="" @@ -1587,6 +1611,7 @@ PREP_START_TN="prep_start" PREP_CYC_SPINUP_TN="prep_cyc_spinup" PREP_CYC_PROD_TN="prep_cyc_prod" PREP_CYC_ENSMEAN_TN="prep_cyc_ensmean" +PREP_CYC_ENSINIT_TN="prep_cyc_ensinit" PREP_CYC_TN="prep_cyc" PROCESS_RADAR_REF_TN="process_radarref" PROCESS_LIGHTNING_TN="process_lightning" @@ -1871,6 +1896,10 @@ TILE_SETS="full" # # DO_ENSPOST: # Flag to turn on/off python ensemble postprocessing for WPC testbeds. +# +# DO_ENSINIT: +# Decide whether or not to do ensemble initialization by running 1 timestep ensemble +# forecast and recentering on the deterministic analysis #----------------------------------------------------------------------- # DO_ENSEMBLE="FALSE" @@ -1883,6 +1912,7 @@ DO_ENVAR_RADAR_REF="FALSE" DO_RECENTER="FALSE" DO_ENS_GRAPHICS="FALSE" DO_ENSPOST="FALSE" +DO_ENSINIT="FALSE" # #----------------------------------------------------------------------- # diff --git a/ush/create_model_configure_file.sh b/ush/create_model_configure_file.sh index c821aa923..ae1c6539b 100644 --- a/ush/create_model_configure_file.sh +++ b/ush/create_model_configure_file.sh @@ -117,6 +117,20 @@ run directory (run_dir): if [ ${cycle_type} == "spinup" ]; then FCST_LEN_HRS_thiscycle=${FCST_LEN_HRS_SPINUP} fi + + if [ "${CYCLE_TYPE}" == "ensinit" ]; then + for cyc_start in "${CYCL_HRS_ENSINIT[@]}"; do + if [ ${hh} -eq ${cyc_start} ]; then + FCST_LEN_HRS_thiscycle=$( expr ${DT_ATMOS}/3600 | bc -l ) + FCST_LEN_HRS_thiscycle=$( printf "%.5f\n" ${FCST_LEN_HRS_thiscycle} ) + NSOUT=1 + RESTART_INTERVAL=0 + print_info_msg "DT_ATMOS ${DT_ATMOS} FCST_LEN_HRS_thiscycle ${FCST_LEN_HRS_thiscycle} \ + NSOUT $NSOUT \ + RESTART_INTERVAL ${RESTART_INTERVAL} " + fi + done + fi # #----------------------------------------------------------------------- diff --git a/ush/generate_FV3LAM_wflow.sh b/ush/generate_FV3LAM_wflow.sh index 0bfe43cb3..258ce9e75 100755 --- a/ush/generate_FV3LAM_wflow.sh +++ b/ush/generate_FV3LAM_wflow.sh @@ -212,6 +212,7 @@ settings="\ 'prep_cyc_spinup': ${PREP_CYC_SPINUP_TN} 'prep_cyc_prod': ${PREP_CYC_PROD_TN} 'prep_cyc_ensmean': ${PREP_CYC_ENSMEAN_TN} + 'prep_cyc_ensinit': ${PREP_CYC_ENSINIT_TN} 'prep_cyc': ${PREP_CYC_TN} 'process_radarref': ${PROCESS_RADAR_REF_TN} 'process_lightning': ${PROCESS_LIGHTNING_TN} @@ -411,6 +412,11 @@ settings="\ 'cycl_hrs_spinstart': [ $( printf "\'%s\', " "${CYCL_HRS_SPINSTART[@]}" ) ] 'cycl_hrs_prodstart': [ $( printf "\'%s\', " "${CYCL_HRS_PRODSTART[@]}" ) ] 'cycl_hrs_recenter': [ $( printf "\'%s\', " "${CYCL_HRS_RECENTER[@]}" ) ] + 'cycl_hrs_ensinit': [ $( printf "\'%s\', " "${CYCL_HRS_ENSINIT[@]}" ) ] + 'cycl_hrs_spp': [ $( printf "\'%s\', " "${CYCL_HRS_SPP[@]}" ) ] + 'cycl_hrs_sppt': [ $( printf "\'%s\', " "${CYCL_HRS_SPPT[@]}" ) ] + 'cycl_hrs_shum': [ $( printf "\'%s\', " "${CYCL_HRS_SHUM[@]}" ) ] + 'cycl_hrs_skeb': [ $( printf "\'%s\', " "${CYCL_HRS_SKEB[@]}" ) ] 'cycl_hrs_hyb_fv3lam_ens': [ $( printf "\'%s\', " "${CYCL_HRS_HYB_FV3LAM_ENS[@]}" ) ] 'restart_hrs_prod': ${RESTART_INTERVAL} 'cycl_freq': !!str 12:00:00 @@ -418,6 +424,7 @@ settings="\ 'initial_cycledef': ${INITIAL_CYCLEDEF} 'boundary_cycledef': ${BOUNDARY_CYCLEDEF} 'boundary_long_cycledef': ${BOUNDARY_LONG_CYCLEDEF} + 'ensinit_cycledef': ${ENSINIT_CYCLEDEF} 'spinup_cycledef': ${SPINUP_CYCLEDEF} 'prod_cycledef': ${PROD_CYCLEDEF} 'recenter_cycledef': ${RECENTER_CYCLEDEF} @@ -451,6 +458,7 @@ settings="\ 'do_bufrsnd': ${DO_BUFRSND} 'do_ens_graphics': ${DO_ENS_GRAPHICS} 'do_enspost': ${DO_ENSPOST} + 'do_ensinit': ${DO_ENSINIT} # # data assimilation related parameters. # diff --git a/ush/templates/FV3LAM_wflow.xml b/ush/templates/FV3LAM_wflow.xml index e2d49a2a2..3d8d82f19 100644 --- a/ush/templates/FV3LAM_wflow.xml +++ b/ush/templates/FV3LAM_wflow.xml @@ -47,6 +47,7 @@ Workflow task names. + @@ -218,6 +219,7 @@ tasks; and the "FCST" type is used for the RUN_FCST_TN task. {{ boundary_long_cycledef }} {{ spinup_cycledef }} + {{ ensinit_cycledef }} {{ prod_cycledef }} {{ recenter_cycledef }} @@ -719,11 +721,11 @@ MODULES_RUN_TASK_FP script. {%- endif %} + - &RSRV_DEFAULT; @@ -739,6 +741,7 @@ MODULES_RUN_TASK_FP script. GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; PDY@Y@m@d CDATE@Y@m@d@H + HH@H CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H FG_ROOT&FG_ROOT; LBCS_ROOT&FG_ROOT; @@ -751,9 +754,15 @@ MODULES_RUN_TASK_FP script. - {%- for h in cycl_hrs_prodstart %} - {{ h }}@H - {%- endfor %} + {%- if do_ensinit %} + {%- for h in cycl_hrs_ensinit %} + {{ h }}@H + {%- endfor %} + {%- else %} + {%- for h in cycl_hrs_prodstart %} + {{ h }}@H + {%- endfor %} + {%- endif %} {%- if do_spinup %} {%- if do_retro %} @@ -825,6 +834,156 @@ MODULES_RUN_TASK_FP script. +{% if do_ensinit and do_ensemble -%} + + + + &RSRV_FCST; + &WALL_LIMIT_FCST; + + &LOAD_MODULES_RUN_TASK_FP; "&RUN_FCST_TN;" "&JOBSDIR;/JREGIONAL_RUN_FCST" + {% if machine in ["JET", "HERA"] %} + {{ ncores_run_fcst }} + {{ native_run_fcst }} &RRFS_RESERVE; + {% else %} + {{ nnodes_run_fcst }}:ppn={{ ppn_run_fcst }} + &NCORES_PER_NODE; + {% endif %} + {{ wtime_run_fcst }} + &TAG;_&RUN_FCST_TN;_ensinit{{ uscore_ensmem_name }} + &LOGDIR;/&RUN_FCST_TN;_ensinit{{ uscore_ensmem_name }}_@Y@m@d@H.log + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + PDY@Y@m@d + CDATE@Y@m@d@H + CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + CYCLE_TYPEensinit + NWGES_BASEDIR&NWGES_BASEDIR; + + + + + {%- for h in cycl_hrs_ensinit %} + {{ h }}@H + {%- endfor %} + + + + + + + + + + + &RSRV_DEFAULT; + &WALL_LIMIT_SAVE_RESTART; + &LOAD_MODULES_RUN_TASK_FP; "&SAVE_RESTART_TN;" "&JOBSDIR;/JREGIONAL_SAVE_RESTART" + {{ nnodes_save_restart }}:ppn={{ ppn_save_restart }} + {{ wtime_save_restart }} + &NCORES_PER_NODE; + &TAG;_&SAVE_RESTART_TN;_ensinit{{ uscore_ensmem_name }} + &LOGDIR;/&SAVE_RESTART_TN;_ensinit{{ uscore_ensmem_name }}_@Y@m@d@H.log + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + PDY@Y@m@d + CDATE@Y@m@d@H + CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H + NWGES_DIR&NWGES_BASEDIR;/@Y@m@d@H + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + cyc@H + fhr0 + CYCLE_TYPEensinit + + + + @Y@m@d@H@M00 + &CYCLE_BASEDIR;/@Y@m@d@H{{ slash_ensmem_subdir }}/fcst_fv3lam_ensinit/RESTART/coupler.res + + + + + + + + + &RSRV_DEFAULT; + &WALL_LIMIT_PRE; + + &LOAD_MODULES_RUN_TASK_FP; "&RUN_PREPSTART_TN;" "&JOBSDIR;/JREGIONAL_RUN_PREPSTART_ENSINIT" + {{ nnodes_run_prepstart }}:ppn={{ ppn_run_prepstart }} + &NCORES_PER_NODE; + {{ wtime_run_prepstart }} + &TAG;_&PREP_CYC_ENSINIT_TN;{{ uscore_ensmem_name }} + &LOGDIR;/&PREP_CYC_ENSINIT_TN;{{ uscore_ensmem_name }}_@Y@m@d@H.log + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + PDY@Y@m@d + CDATE@Y@m@d@H + CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H + FG_ROOT&FG_ROOT; + LBCS_ROOT&FG_ROOT; + CYCLE_TYPEprod + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + NWGES_BASEDIR&NWGES_BASEDIR; + + + + + + + + + + + &RSRV_ENKF; + &WALL_LIMIT_RECENTER; + &LOAD_MODULES_RUN_TASK_FP; "&RUN_RECENTER_TN;" &JOBSDIR;/JREGIONAL_RUN_RECENTER + + {{ nnodes_run_recenter }}:ppn={{ ppn_run_recenter }} + &NCORES_PER_NODE; + {{ wtime_run_recenter }} + &TAG;_&RUN_RECENTER_TN; + &LOGDIR;/&RUN_RECENTER_TN;_@Y@m@d@H.log + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + + nens{{ num_ens_members }} + CDATE@Y@m@d@H + CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H + CYCLE_ROOT&CYCLE_BASEDIR; + ENSCTRL_CYCLE_DIR&ENSCTRL_CYCLE_BASEDIR;/@Y@m@d@H + ENSCTRL_CYCLE_ROOT&ENSCTRL_CYCLE_BASEDIR; + NWGES_DIR&NWGES_BASEDIR;/@Y@m@d@H + + + + {%- for m in range(1, num_ens_members+1) %} + + {%- endfor %} + + &ENSCTRL_CYCLE_BASEDIR;/@Y@m@d@H/nonvar_cldanl/nonvarcldanl_complete.txt + &ENSCTRL_CYCLE_BASEDIR;/@Y@m@d@H/nonvar_cldanl_spinup/nonvarcldanl_complete.txt + + + + + +{%- endif %} + @@ -1244,6 +1403,15 @@ MODULES_RUN_TASK_FP script. {%- else %} {%- endif %} + {%- elif do_ensinit and do_ensemble %} + + + {%- for h in cycl_hrs_ensinit %} + {{ h }}@H + {%- endfor %} + + + {%- elif do_enkfupdate or do_enkf_radar_ref %} diff --git a/ush/valid_param_vals.sh b/ush/valid_param_vals.sh index 34fd610c9..cade406a0 100644 --- a/ush/valid_param_vals.sh +++ b/ush/valid_param_vals.sh @@ -68,6 +68,7 @@ valid_vals_DO_RECENTER=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_DO_BUFRSND=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_DO_RETRO=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_DO_SPINUP=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") +valid_vals_DO_ENSINIT=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_LBCS_ICS_ONLY=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_DO_NONVAR_CLDANAL=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_DO_JEDI_ENVAR_IODA=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") From b1d08dedf56f980bac830c6574d2ade017cd7440 Mon Sep 17 00:00:00 2001 From: chunhuazhou Date: Fri, 23 Sep 2022 21:56:55 +0000 Subject: [PATCH 5/8] Add tasks to run ensemble free forecast at a reduced ensemble size --- scripts/exregional_run_fcst.sh | 246 +++++++++++++++++- ush/config.sh.RRFS_CONUS_3km_ens.2022HWT.CYCL | 10 +- ush/config_defaults.sh | 47 +++- ush/generate_FV3LAM_wflow.sh | 96 +------ ush/setup.sh | 11 +- ush/templates/FV3LAM_wflow.xml | 117 ++++++++- ush/valid_param_vals.sh | 1 + 7 files changed, 422 insertions(+), 106 deletions(-) diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index d6112fd24..c938227d8 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -502,13 +502,257 @@ else fi fi +# +#----------------------------------------------------------------------- +# +# Set the stochatics physics related namelist on the fly for the current +# forecast cycle +# +#----------------------------------------------------------------------- +# for cyc_start in "${CYCL_HRS_SPP[@]}"; do hh=${cdate:8:2} if [ ${hh} -eq ${cyc_start} ]; then - DO_SPP = TRUE + DO_SPP=TRUE + fi +done +for cyc_start in "${CYCL_HRS_SPPT[@]}"; do + hh=${cdate:8:2} + if [ ${hh} -eq ${cyc_start} ]; then + DO_SPPT=TRUE fi done +for cyc_start in "${CYCL_HRS_SHUM[@]}"; do + hh=${cdate:8:2} + if [ ${hh} -eq ${cyc_start} ]; then + DO_SHUM=TRUE + fi +done +for cyc_start in "${CYCL_HRS_SKEB[@]}"; do + hh=${cdate:8:2} + if [ ${hh} -eq ${cyc_start} ]; then + DO_SKEB=TRUE + fi +done +for cyc_start in "${CYCL_HRS_LSM_SPP[@]}"; do + hh=${cdate:8:2} + if [ ${hh} -eq ${cyc_start} ]; then + DO_LSM_SPP=TRUE + fi +done +# +#----------------------------------------------------------------------- +# +# If running with SPP in MYNN PBL, MYNN SFC, GSL GWD, Thompson MP, or +# RRTMG, count the number of entries in SPP_VAR_LIST to correctly set +# N_VAR_SPP, otherwise set it to zero. +# +#----------------------------------------------------------------------- +# +N_VAR_SPP=0 +if [ "${DO_SPP}" = "TRUE" ]; then + N_VAR_SPP=${#SPP_VAR_LIST[@]} +fi +# +#----------------------------------------------------------------------- +# +# If running with Noah or RUC-LSM SPP, count the number of entries in +# LSM_SPP_VAR_LIST to correctly set N_VAR_LNDP, otherwise set it to zero. +# Also set LNDP_TYPE to 2 for LSM SPP, otherwise set it to zero. Finally, +# initialize an "FHCYC_LSM_SPP" variable to 0 and set it to 999 if LSM SPP +# is turned on. This requirement is necessary since LSM SPP cannot run with +# FHCYC=0 at the moment, but FHCYC cannot be set to anything less than the +# length of the forecast either. A bug fix will be submitted to +# ufs-weather-model soon, at which point, this requirement can be removed +# from regional_workflow. +# +#----------------------------------------------------------------------- +# +N_VAR_LNDP=0 +LNDP_TYPE=0 +FHCYC_LSM_SPP_OR_NOT=0 +if [ "${DO_LSM_SPP}" = "TRUE" ]; then + N_VAR_LNDP=${#LSM_SPP_VAR_LIST[@]} + LNDP_TYPE=2 + FHCYC_LSM_SPP_OR_NOT=999 +fi +# +#----------------------------------------------------------------------- +# +# If running with SPP, confirm that each SPP-related namelist value +# contains the same number of entries as N_VAR_SPP (set above to be equal +# to the number of entries in SPP_VAR_LIST). +# +#----------------------------------------------------------------------- +# +if [ "${DO_SPP}" = "TRUE" ]; then + if [ "${#SPP_MAG_LIST[@]}" != "${N_VAR_SPP}" ] || \ + [ "${#SPP_LSCALE[@]}" != "${N_VAR_SPP}" ] || \ + [ "${#SPP_TSCALE[@]}" != "${N_VAR_SPP}" ] || \ + [ "${#SPP_SIGTOP1[@]}" != "${N_VAR_SPP}" ] || \ + [ "${#SPP_SIGTOP2[@]}" != "${N_VAR_SPP}" ] || \ + [ "${#SPP_STDDEV_CUTOFF[@]}" != "${N_VAR_SPP}" ] || \ + [ "${#ISEED_SPP[@]}" != "${N_VAR_SPP}" ]; then + print_err_msg_exit "\ +All MYNN PBL, MYNN SFC, GSL GWD, Thompson MP, or RRTMG SPP-related namelist +variables set in ${CONFIG_FN} must be equal in number of entries to what is +found in SPP_VAR_LIST: + Number of entries in SPP_VAR_LIST = \"${#SPP_VAR_LIST[@]}\"" + fi +fi +# +#----------------------------------------------------------------------- +# +# If running with LSM SPP, confirm that each LSM SPP-related namelist +# value contains the same number of entries as N_VAR_LNDP (set above to +# be equal to the number of entries in LSM_SPP_VAR_LIST). +# +#----------------------------------------------------------------------- +# +if [ "${DO_LSM_SPP}" = "TRUE" ]; then + if [ "${#LSM_SPP_MAG_LIST[@]}" != "${N_VAR_LNDP}" ] || \ + [ "${#LSM_SPP_LSCALE[@]}" != "${N_VAR_LNDP}" ] || \ + [ "${#LSM_SPP_TSCALE[@]}" != "${N_VAR_LNDP}" ]; then + print_err_msg_exit "\ +All Noah or RUC-LSM SPP-related namelist variables (except ISEED_LSM_SPP) +set in ${CONFIG_FN} must be equal in number of entries to what is found in +SPP_VAR_LIST: + Number of entries in SPP_VAR_LIST = \"${#LSM_SPP_VAR_LIST[@]}\"" + fi +fi + +# +#----------------------------------------------------------------------- +# +# Add stochatics related namelist +# +settings="\ +'gfs_physics_nml': { + 'do_shum': ${DO_SHUM}, + 'do_sppt': ${DO_SPPT}, + 'do_skeb': ${DO_SKEB}, + 'do_spp': ${DO_SPP}, + 'n_var_spp': ${N_VAR_SPP}, + 'n_var_lndp': ${N_VAR_LNDP}, + 'lndp_type': ${LNDP_TYPE}, + 'lndp_each_step': ${LSM_SPP_EACH_STEP}, + 'fhcyc': ${FHCYC_LSM_SPP_OR_NOT}, + }" + +settings="$settings +'nam_stochy': {" +if [ "${DO_SPPT}" = "TRUE" ]; then + settings="$settings + 'iseed_sppt': ${ISEED_SPPT}, + 'new_lscale': ${NEW_LSCALE}, + 'sppt': ${SPPT_MAG}, + 'sppt_logit': ${SPPT_LOGIT}, + 'sppt_lscale': ${SPPT_LSCALE}, + 'sppt_sfclimit': ${SPPT_SFCLIMIT}, + 'sppt_tau': ${SPPT_TSCALE}, + 'spptint': ${SPPT_INT}, + 'use_zmtnblck': ${USE_ZMTNBLCK}," +fi +if [ "${DO_SHUM}" = "TRUE" ]; then + settings="$settings + 'iseed_shum': ${ISEED_SHUM}, + 'new_lscale': ${NEW_LSCALE}, + 'shum': ${SHUM_MAG}, + 'shum_lscale': ${SHUM_LSCALE}, + 'shum_tau': ${SHUM_TSCALE}, + 'shumint': ${SHUM_INT}," +fi + +if [ "${DO_SKEB}" = "TRUE" ]; then + settings="$settings + 'iseed_skeb': ${ISEED_SKEB}, + 'new_lscale': ${NEW_LSCALE}, + 'skeb': ${SKEB_MAG}, + 'skeb_lscale': ${SKEB_LSCALE}, + 'skebnorm': ${SKEBNORM}, + 'skeb_tau': ${SKEB_TSCALE}, + 'skebint': ${SKEB_INT}, + 'skeb_vdof': ${SKEB_VDOF}," +fi + +if [ "${DO_SPP}" = "TRUE" ] || [ "${DO_LSM_SPP}" = "TRUE" ]; then + settings="$settings + 'new_lscale': ${NEW_LSCALE}," +fi +settings="$settings + }" +# +# Add the relevant SPP namelist variables to "settings" when running with +# SPP turned on. Otherwise only include an empty "nam_sppperts" stanza. +# +settings="$settings +'nam_sppperts': {" +if [ "${DO_SPP}" = "TRUE" ]; then + settings="$settings + 'iseed_spp': [ $( printf "%s, " "${ISEED_SPP[@]}" ) ], + 'spp_lscale': [ $( printf "%s, " "${SPP_LSCALE[@]}" ) ], + 'spp_prt_list': [ $( printf "%s, " "${SPP_MAG_LIST[@]}" ) ], + 'spp_sigtop1': [ $( printf "%s, " "${SPP_SIGTOP1[@]}" ) ], + 'spp_sigtop2': [ $( printf "%s, " "${SPP_SIGTOP2[@]}" ) ], + 'spp_stddev_cutoff': [ $( printf "%s, " "${SPP_STDDEV_CUTOFF[@]}" ) ], + 'spp_tau': [ $( printf "%s, " "${SPP_TSCALE[@]}" ) ], + 'spp_var_list': [ $( printf "%s, " "${SPP_VAR_LIST[@]}" ) ]," +fi +settings="$settings + }" +# +# Add the relevant LSM SPP namelist variables to "settings" when running with +# LSM SPP turned on. +# +settings="$settings +'nam_sfcperts': {" +if [ "${DO_LSM_SPP}" = "TRUE" ]; then + settings="$settings + 'lndp_type': ${LNDP_TYPE}, + 'lndp_tau': [ $( printf "%s, " "${LSM_SPP_TSCALE[@]}" ) ], + 'lndp_lscale': [ $( printf "%s, " "${LSM_SPP_LSCALE[@]}" ) ], + 'iseed_lndp': [ $( printf "%s, " "${ISEED_LSM_SPP[@]}" ) ], + 'lndp_var_list': [ $( printf "%s, " "${LSM_SPP_VAR_LIST[@]}" ) ], + 'lndp_prt_list': [ $( printf "%s, " "${LSM_SPP_MAG_LIST[@]}" ) ]," +fi +settings="$settings + }" +print_info_msg $VERBOSE " +The variable \"settings\" specifying values of the namelist variables +has been set as follows: + +settings = +$settings" +# +#----------------------------------------------------------------------- +# +# Call the set_namelist.py script to create a new FV3 namelist file +# using the file input.nml_temp as the base (i.e. starting) namelist file +# and with additional physics-suite-independent modificaitons specified +# in the variable "settings" set above. +# +#----------------------------------------------------------------------- +# +mv ${run_dir}/input.nml ${run_dir}/input.nml_temp +$USHDIR/set_namelist.py -q \ + -n ${run_dir}/input.nml_temp \ + -u "$settings" \ + -o ${run_dir}/input.nml || \ + print_err_msg_exit "\ +Call to python script set_namelist.py to generate an FV3 namelist file +failed. Parameters passed to this script are: + Full path to base namelist file: + FV3_NML_BASE_SUITE_FP = \"${run_dir}/input.nml_temp\" + Full path to output namelist file: + FV3_NML_FP = \"${run_dir}/input.nml \" + Namelist settings specified on command line: + settings = +$settings" + +# +#----------------------------------------------------------------------- +# if [ "${DO_ENSEMBLE}" = TRUE ] && ([ "${DO_SPP}" = TRUE ] || [ "${DO_SPPT}" = TRUE ] || [ "${DO_SHUM}" = TRUE ] \ [ "${DO_SKEB}" = TRUE ] || [ "${DO_LSM_SPP}" = TRUE ]); then cp ${run_dir}/input.nml ${run_dir}/input.nml_base diff --git a/ush/config.sh.RRFS_CONUS_3km_ens.2022HWT.CYCL b/ush/config.sh.RRFS_CONUS_3km_ens.2022HWT.CYCL index 8773e1668..9798eb1d9 100644 --- a/ush/config.sh.RRFS_CONUS_3km_ens.2022HWT.CYCL +++ b/ush/config.sh.RRFS_CONUS_3km_ens.2022HWT.CYCL @@ -1,13 +1,13 @@ MACHINE="jet" ACCOUNT="nrtrr" RESERVATION="rrfsens" -RESERVATION_POST="rrfsens" +RESERVATION_POST="rrfsdet" EXPT_BASEDIR="/misc/whome/wrfruc/RRFSE/" EXPT_SUBDIR="RRFS_CONUS_3km_ens" if [[ -n $RESERVATION ]] ; then ACCOUNT=rtwrfruc - SERVICE_ACCOUNT=rtwrfruc + SERVICE_ACCOUNT=rtrr PARTITION_DEFAULT=kjet PARTITION_FCST=kjet PARTITION_GRAPHICS=kjet @@ -80,7 +80,7 @@ BOUNDARY_LEN_HRS="36" # avaialble retro period: # 20210511-20210531; 20210718-20210801 -DATE_FIRST_CYCL="20220922" +DATE_FIRST_CYCL="20220923" DATE_LAST_CYCL="20221031" CYCL_HRS=( "00" "12" ) CYCL_HRS=( "18" ) @@ -101,6 +101,8 @@ BOUNDARY_CYCLEDEF="00 00,18 * 9-10 2022 *" #POSTPROC_CYCLEDEF="${DATE_FIRST_CYCL}2100 ${DATE_LAST_CYCL}2300 01:00:00" ENSINIT_CYCLEDEF="00 18 * 9-10 2022 *" PROD_CYCLEDEF="00 00,18-23 * 9-10 2022 *" +FCST_CYCLEDEF="00 18-23 * 9-10 2022 *" +FCST_LONG_CYCLEDEF="00 00 * 9-10 2022 *" #RECENTER_CYCLEDEF="00 19 * 9-10 2022 *" POSTPROC_CYCLEDEF="00 00,18-23 * 9-10 2022 *" POSTPROC_LONG_CYCLEDEF="${DATE_FIRST_CYCL}0000 ${DATE_LAST_CYCL}2300 24:00:00" @@ -211,6 +213,8 @@ fi if [[ ${DO_ENSEMBLE} == "TRUE" ]]; then NUM_ENS_MEMBERS=30 + DO_ENSEMBLE_REDUCED="TRUE" + NUM_ENS_MEMBERS_REDUCED=9 # DO_SPPT=TRUE # DO_SPP=TRUE # SPPT_MAG=0.5 diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index b144c569a..bbf10591e 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -486,16 +486,19 @@ WFLOW_LAUNCH_LOG_FN="log.launch_FV3LAM_wflow" # An array containing the hours of the day at which the ensemble initialization is on # # CYCL_HRS_SPP: -# An array containing the hours of the day at which the SPP is on +# An array containing the hours of the day at which SPP is turned on # # CYCL_HRS_SPPT: -# An array containing the hours of the day at which the SPPT is on +# An array containing the hours of the day at which SPPT is turned on # # CYCL_HRS_SHUM: -# An array containing the hours of the day at which the SHUM is on +# An array containing the hours of the day at which SHUM is turned on # # CYCL_HRS_SKEB: -# An array containing the hours of the day at which the SKEB is on +# An array containing the hours of the day at which SKEB is turned on +# +# CYCL_HRS_LSM_SPP: +# An array containing the hours of the day at which LSM_SPP is turned on # # BOUNDARY_LEN_HRS # The length of boundary condition for normal forecast, in integer hours. @@ -542,6 +545,7 @@ CYCL_HRS_SPP=( "HH1" "HH2" ) CYCL_HRS_SPPT=( "HH1" "HH2" ) CYCL_HRS_SHUM=( "HH1" "HH2" ) CYCL_HRS_SKEB=( "HH1" "HH2" ) +CYCL_HRS_LSM_SPP=( "HH1" "HH2" ) BOUNDARY_LEN_HRS="0" BOUNDARY_LONG_LEN_HRS="0" POSTPROC_LEN_HRS="1" @@ -596,7 +600,16 @@ CYCL_HRS_HYB_FV3LAM_ENS=( "99" ) # # PROD_CYCLEDEF: # cycle definition for product cycle group -# This group runs: anal_gsi_input and data process, run_fcst, python_skewt, run_clean +# This group runs: anal_gsi_input and data process, python_skewt, run_clean +# +# FCST_CYCLEDEF: +# cycle definition for forecast cycle group +# This group runs: run_fcst +# +# FCST_LONG_CYCLEDEF: +# cycle definition for free forecast cycle group, when DO_ENSEMBLE=TRUE and +# DO_ENSEMBLE_REDUCED=TRUE, running free forecast with a reduced ensemble size +# This group runs: run_fcst # # RECENTER_CYCLEDEF: # cycle definition for recenter cycle group @@ -625,6 +638,8 @@ BOUNDARY_LONG_CYCLEDEF="00 01 01 01 2100 *" ENSINIT_CYCLEDEF="00 01 01 01 2100 *" SPINUP_CYCLEDEF="00 01 01 01 2100 *" PROD_CYCLEDEF="00 01 01 01 2100 *" +FCST_CYCLEDEF="00 01 01 01 2100 *" +FCST_LONG_CYCLEDEF="00 01 01 01 2100 *" RECENTER_CYCLEDEF="00 01 01 01 2100 *" POSTPROC_CYCLEDEF="00 01 01 01 2100 *" POSTPROC_LONG_CYCLEDEF="00 01 01 01 2100 *" @@ -1859,6 +1874,12 @@ TILE_SETS="full" # forecasts are run for each cycle, each with a different set of stochastic # seed values. Otherwise, a single forecast is run for each cycle. # +# DO_ENSEMBLE_REDUCED: +# Flag that determines whether to run a subset of ensemble forecasts (for +# each set of specified cycles). If this is set to "TRUE", NUM_ENS_MEMBERS_REDUCED +# forecasts are run for each specified cycle, each with a different set of stochastic +# seed values. +# # NUM_ENS_MEMBERS: # The number of ensemble members to run if DO_ENSEMBLE is set to "TRUE". # This variable also controls the naming of the ensemble member directories. @@ -1871,6 +1892,20 @@ TILE_SETS="full" # strip away those leading zeros. Thus, in the variable definitions file # (GLOBAL_VAR_DEFNS_FN), this variable appear with its leading zeros # stripped. This variable is not used if DO_ENSEMBLE is not set to "TRUE". +# +# NUM_ENS_MEMBERS_REDUCED: +# The number of ensemble members to run if DO_ENSEMBLE_REDUCED is set to "TRUE", +# and if the free forecast has a reduced ensemble size. +# This variable also controls the naming of the ensemble member directories. +# For example, if this is set to "8", the member directories will be named +# mem1, mem2, ..., mem8. If it is set to "08" (note the leading zero), +# the member directories will be named mem01, mem02, ..., mem08. Note, +# however, that after reading in the number of characters in this string +# (in order to determine how many leading zeros, if any, should be placed +# in the names of the member directories), the workflow generation scripts +# strip away those leading zeros. Thus, in the variable definitions file +# (GLOBAL_VAR_DEFNS_FN), this variable appear with its leading zeros +# stripped. This variable is not used if DO_ENSEMBLE is not set to "TRUE". # # DO_ENSCONTROL: # In ensemble mode, whether or not to run member 1 as control member @@ -1903,7 +1938,9 @@ TILE_SETS="full" #----------------------------------------------------------------------- # DO_ENSEMBLE="FALSE" +DO_ENSEMBLE_REDUCED="FALSE" NUM_ENS_MEMBERS="1" +NUM_ENS_MEMBERS_REDUCED="1" DO_ENSCONTROL="FALSE" DO_GSIOBSERVER="FALSE" DO_ENKFUPDATE="FALSE" diff --git a/ush/generate_FV3LAM_wflow.sh b/ush/generate_FV3LAM_wflow.sh index 258ce9e75..a772542b1 100755 --- a/ush/generate_FV3LAM_wflow.sh +++ b/ush/generate_FV3LAM_wflow.sh @@ -417,6 +417,7 @@ settings="\ 'cycl_hrs_sppt': [ $( printf "\'%s\', " "${CYCL_HRS_SPPT[@]}" ) ] 'cycl_hrs_shum': [ $( printf "\'%s\', " "${CYCL_HRS_SHUM[@]}" ) ] 'cycl_hrs_skeb': [ $( printf "\'%s\', " "${CYCL_HRS_SKEB[@]}" ) ] + 'cycl_hrs_lsm_spp': [ $( printf "\'%s\', " "${CYCL_HRS_LSM_SPP[@]}" ) ] 'cycl_hrs_hyb_fv3lam_ens': [ $( printf "\'%s\', " "${CYCL_HRS_HYB_FV3LAM_ENS[@]}" ) ] 'restart_hrs_prod': ${RESTART_INTERVAL} 'cycl_freq': !!str 12:00:00 @@ -427,6 +428,8 @@ settings="\ 'ensinit_cycledef': ${ENSINIT_CYCLEDEF} 'spinup_cycledef': ${SPINUP_CYCLEDEF} 'prod_cycledef': ${PROD_CYCLEDEF} + 'fcst_cycledef': ${FCST_CYCLEDEF} + 'fcst_long_cycledef': ${FCST_LONG_CYCLEDEF} 'recenter_cycledef': ${RECENTER_CYCLEDEF} 'postproc_long_cycledef': ${POSTPROC_LONG_CYCLEDEF} 'archive_cycledef': ${ARCHIVE_CYCLEDEF} @@ -444,7 +447,9 @@ settings="\ # Ensemble-related parameters. # 'do_ensemble': ${DO_ENSEMBLE} + 'do_ensemble_reduced': ${DO_ENSEMBLE_REDUCED} 'num_ens_members': ${NUM_ENS_MEMBERS} + 'num_ens_members_reduced': ${NUM_ENS_MEMBERS_REDUCED} 'ndigits_ensmem_names': !!str ${NDIGITS_ENSMEM_NAMES} 'ensmem_indx_name': ${ensmem_indx_name} 'uscore_ensmem_name': ${uscore_ensmem_name} @@ -901,97 +906,6 @@ done settings="$settings }" # -# Add the relevant tendency-based stochastic physics namelist variables to -# "settings" when running with SPPT, SHUM, or SKEB turned on. If running -# with SPP or LSM SPP, set the "new_lscale" variable. Otherwise only -# include an empty "nam_stochy" stanza. -# -settings="$settings -'nam_stochy': {" -if [ "${DO_SPPT}" = "TRUE" ]; then - settings="$settings - 'iseed_sppt': ${ISEED_SPPT}, - 'new_lscale': ${NEW_LSCALE}, - 'sppt': ${SPPT_MAG}, - 'sppt_logit': ${SPPT_LOGIT}, - 'sppt_lscale': ${SPPT_LSCALE}, - 'sppt_sfclimit': ${SPPT_SFCLIMIT}, - 'sppt_tau': ${SPPT_TSCALE}, - 'spptint': ${SPPT_INT}, - 'use_zmtnblck': ${USE_ZMTNBLCK}," -fi - -if [ "${DO_SHUM}" = "TRUE" ]; then - settings="$settings - 'iseed_shum': ${ISEED_SHUM}, - 'new_lscale': ${NEW_LSCALE}, - 'shum': ${SHUM_MAG}, - 'shum_lscale': ${SHUM_LSCALE}, - 'shum_tau': ${SHUM_TSCALE}, - 'shumint': ${SHUM_INT}," -fi - -if [ "${DO_SKEB}" = "TRUE" ]; then - settings="$settings - 'iseed_skeb': ${ISEED_SKEB}, - 'new_lscale': ${NEW_LSCALE}, - 'skeb': ${SKEB_MAG}, - 'skeb_lscale': ${SKEB_LSCALE}, - 'skebnorm': ${SKEBNORM}, - 'skeb_tau': ${SKEB_TSCALE}, - 'skebint': ${SKEB_INT}, - 'skeb_vdof': ${SKEB_VDOF}," -fi - -if [ "${DO_SPP}" = "TRUE" ] || [ "${DO_LSM_SPP}" = "TRUE" ]; then - settings="$settings - 'new_lscale': ${NEW_LSCALE}," -fi -settings="$settings - }" -# -# Add the relevant SPP namelist variables to "settings" when running with -# SPP turned on. Otherwise only include an empty "nam_sppperts" stanza. -# -settings="$settings -'nam_sppperts': {" -if [ "${DO_SPP}" = "TRUE" ]; then - settings="$settings - 'iseed_spp': [ $( printf "%s, " "${ISEED_SPP[@]}" ) ], - 'spp_lscale': [ $( printf "%s, " "${SPP_LSCALE[@]}" ) ], - 'spp_prt_list': [ $( printf "%s, " "${SPP_MAG_LIST[@]}" ) ], - 'spp_sigtop1': [ $( printf "%s, " "${SPP_SIGTOP1[@]}" ) ], - 'spp_sigtop2': [ $( printf "%s, " "${SPP_SIGTOP2[@]}" ) ], - 'spp_stddev_cutoff': [ $( printf "%s, " "${SPP_STDDEV_CUTOFF[@]}" ) ], - 'spp_tau': [ $( printf "%s, " "${SPP_TSCALE[@]}" ) ], - 'spp_var_list': [ $( printf "%s, " "${SPP_VAR_LIST[@]}" ) ]," -fi -settings="$settings - }" -# -# Add the relevant LSM SPP namelist variables to "settings" when running with -# LSM SPP turned on. -# -settings="$settings -'nam_sfcperts': {" -if [ "${DO_LSM_SPP}" = "TRUE" ]; then - settings="$settings - 'lndp_type': ${LNDP_TYPE}, - 'lndp_tau': [ $( printf "%s, " "${LSM_SPP_TSCALE[@]}" ) ], - 'lndp_lscale': [ $( printf "%s, " "${LSM_SPP_LSCALE[@]}" ) ], - 'iseed_lndp': [ $( printf "%s, " "${ISEED_LSM_SPP[@]}" ) ], - 'lndp_var_list': [ $( printf "%s, " "${LSM_SPP_VAR_LIST[@]}" ) ], - 'lndp_prt_list': [ $( printf "%s, " "${LSM_SPP_MAG_LIST[@]}" ) ]," -fi -settings="$settings - }" -print_info_msg $VERBOSE " -The variable \"settings\" specifying values of the namelist variables -has been set as follows: - -settings = -$settings" -# #----------------------------------------------------------------------- # # Call the set_namelist.py script to create a new FV3 namelist file (full diff --git a/ush/setup.sh b/ush/setup.sh index 4798ebb72..e45ea89d2 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -1622,7 +1622,7 @@ fi # #----------------------------------------------------------------------- # -# Make sure that DO_ENSEMBLE is set to a valid value. Then set the names +# Make sure that DO_ENSEMBLE and DO_ENSEMBLE_REDUCED is set to a valid value. Then set the names # of the ensemble members. These will be used to set the ensemble member # directories. Also, set the full path to the FV3 namelist file corresponding # to each ensemble member. @@ -1630,6 +1630,7 @@ fi #----------------------------------------------------------------------- # check_var_valid_value "DO_ENSEMBLE" "valid_vals_DO_ENSEMBLE" +check_var_valid_value "DO_ENSEMBLE_REDUCED" "valid_vals_DO_ENSEMBLE_REDUCED" # # Set DO_ENSEMBLE to either "TRUE" or "FALSE" so we don't have to consider # other valid values later on. @@ -1642,6 +1643,14 @@ elif [ "$DO_ENSEMBLE" = "FALSE" ] || \ [ "$DO_ENSEMBLE" = "NO" ]; then DO_ENSEMBLE="FALSE" fi +DO_ENSEMBLE_REDUCED=${DO_ENSEMBLE_REDUCED^^} +if [ "$DO_ENSEMBLE_REDUCED" = "TRUE" ] || \ + [ "$DO_ENSEMBLE_REDUCED" = "YES" ]; then + DO_ENSEMBLE_REDUCED="TRUE" +elif [ "$DO_ENSEMBLE_REDUCED" = "FALSE" ] || \ + [ "$DO_ENSEMBLE_REDUCED" = "NO" ]; then + DO_ENSEMBLE_REDUCED="FALSE" +fi NDIGITS_ENSMEM_NAMES="0" ENSMEM_NAMES=("") FV3_NML_ENSMEM_FPS=("") diff --git a/ush/templates/FV3LAM_wflow.xml b/ush/templates/FV3LAM_wflow.xml index 3d8d82f19..5eb46f114 100644 --- a/ush/templates/FV3LAM_wflow.xml +++ b/ush/templates/FV3LAM_wflow.xml @@ -121,7 +121,7 @@ tasks; and the "FCST" type is used for the RUN_FCST_TN task. &ACCOUNT;&QUEUE_DEFAULT;"> {%- endif %} {%- if partition_hpss is not none %} -&SERVICE_ACCOUNT;&QUEUE_HPSS;{{ partition_hpss }}&RRFS_RESERVE;"> +&SERVICE_ACCOUNT;&QUEUE_HPSS;{{ partition_hpss }}&RRFS_POST_RESERVE;"> {%- else %} &SERVICE_ACCOUNT;&QUEUE_HPSS;"> {%- endif %} @@ -221,6 +221,8 @@ tasks; and the "FCST" type is used for the RUN_FCST_TN task. {{ spinup_cycledef }} {{ ensinit_cycledef }} {{ prod_cycledef }} + {{ fcst_cycledef }} + {{ fcst_long_cycledef }} {{ recenter_cycledef }} {{ prod_cycledef }} @@ -852,7 +854,7 @@ MODULES_RUN_TASK_FP script. {{ nnodes_run_fcst }}:ppn={{ ppn_run_fcst }} &NCORES_PER_NODE; {% endif %} - {{ wtime_run_fcst }} + 00:30:00 &TAG;_&RUN_FCST_TN;_ensinit{{ uscore_ensmem_name }} &LOGDIR;/&RUN_FCST_TN;_ensinit{{ uscore_ensmem_name }}_@Y@m@d@H.log @@ -1361,7 +1363,11 @@ MODULES_RUN_TASK_FP script. ************************************************************************ ************************************************************************ --> + {%- if do_ensemble and do_ensemble_reduced %} + + {%- else %} + {%- endif %} &RSRV_FCST; &WALL_LIMIT_FCST; @@ -1403,7 +1409,9 @@ MODULES_RUN_TASK_FP script. {%- else %} {%- endif %} - {%- elif do_ensinit and do_ensemble %} + {%- elif do_enkfupdate or do_enkf_radar_ref %} + + {%- if do_ensinit and do_ensemble %} {%- for h in cycl_hrs_ensinit %} @@ -1412,8 +1420,7 @@ MODULES_RUN_TASK_FP script. - {%- elif do_enkfupdate or do_enkf_radar_ref %} - + {%- else %} {%- for h in cycl_hrs_prodstart %} @@ -1422,6 +1429,7 @@ MODULES_RUN_TASK_FP script. + {%- endif %} {%- if do_recenter %} @@ -2220,4 +2228,103 @@ MODULES_RUN_TASK_FP script. {%- endif %} + + + {%- if do_ensemble and do_ensemble_reduced %} + + + {%- for m in range(1, num_ens_members_reduced+1) -%} + {%- set fmtstr=" %0"~ndigits_ensmem_names~"d" -%} + {{- fmtstr%m -}} + {%- endfor %} + + + + &RSRV_FCST; + &WALL_LIMIT_FCST; + + &LOAD_MODULES_RUN_TASK_FP; "&RUN_FCST_TN;" "&JOBSDIR;/JREGIONAL_RUN_FCST" + {% if machine in ["JET", "HERA"] %} + {{ ncores_run_fcst }} + {{ native_run_fcst }} &RRFS_RESERVE; + {% else %} + {{ nnodes_run_fcst }}:ppn={{ ppn_run_fcst }} + &NCORES_PER_NODE; + {% endif %} + {{ wtime_run_fcst }} + &TAG;_&RUN_FCST_TN;_prod{{ uscore_ensmem_name }} + &LOGDIR;/&RUN_FCST_TN;_prod{{ uscore_ensmem_name }}_@Y@m@d@H.log + + GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP; + PDY@Y@m@d + CDATE@Y@m@d@H + CYCLE_DIR&CYCLE_BASEDIR;/@Y@m@d@H + SLASH_ENSMEM_SUBDIR{{ slash_ensmem_subdir }} + ENSMEM_INDX#{{ ensmem_indx_name }}# + CYCLE_TYPEprod + NWGES_BASEDIR&NWGES_BASEDIR; + + + {%- if do_enkfupdate or do_enkf_radar_ref %} + + {%- if do_ensinit and do_ensemble %} + + + {%- for h in cycl_hrs_ensinit %} + {{ h }}@H + {%- endfor %} + + + + {%- else %} + + + {%- for h in cycl_hrs_prodstart %} + {{ h }}@H + {%- endfor %} + + + + {%- endif %} + {%- if do_recenter %} + + + + {%- for h in cycl_hrs_recenter %} + {{ h }}@H + {%- endfor %} + + &CYCLE_BASEDIR;/@Y@m@d@H/recenter/fcst_fv3lam/INPUT/recenter_complete.txt + + + {%- for h in cycl_hrs_recenter %} + {{ h }}@H + {%- endfor %} + {%- for h in cycl_hrs_prodstart %} + {{ h }}@H + {%- endfor %} + + + + {%- elif do_nonvar_cldanal%} + + {%- elif do_enkf_radar_ref %} + + {%- else %} + + {%- endif %} + + {%- else %} + + {%- endif %} + + + + + + {%- endif %} + diff --git a/ush/valid_param_vals.sh b/ush/valid_param_vals.sh index cade406a0..d65e921be 100644 --- a/ush/valid_param_vals.sh +++ b/ush/valid_param_vals.sh @@ -58,6 +58,7 @@ valid_vals_DOT_OR_USCORE=("." "_") valid_vals_NOMADS=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_NOMADS_file_type=("GRIB2" "grib2" "NEMSIO" "nemsio") valid_vals_DO_ENSEMBLE=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") +valid_vals_DO_ENSEMBLE_REDUCED=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_DO_ENS_GRAPHICS=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_DO_ENSPOST=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_DO_DACYCLE=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") From 9a47832bf50b91465cf45db0e86380e2b3c3a9b4 Mon Sep 17 00:00:00 2001 From: chunhua zhou Date: Sun, 25 Sep 2022 16:55:45 -0600 Subject: [PATCH 6/8] Update config.sh.RRFS_CONUS_3km_ens.2022HWT.CYCL --- ush/config.sh.RRFS_CONUS_3km_ens.2022HWT.CYCL | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/config.sh.RRFS_CONUS_3km_ens.2022HWT.CYCL b/ush/config.sh.RRFS_CONUS_3km_ens.2022HWT.CYCL index 9798eb1d9..8fa4531cb 100644 --- a/ush/config.sh.RRFS_CONUS_3km_ens.2022HWT.CYCL +++ b/ush/config.sh.RRFS_CONUS_3km_ens.2022HWT.CYCL @@ -224,7 +224,7 @@ if [[ ${DO_ENSEMBLE} == "TRUE" ]]; then # DO_RECENTER="TRUE" DO_ENS_GRAPHICS="TRUE" DO_ENKF_RADAR_REF="TRUE" - DO_ENSPOST="FALSE" + DO_ENSPOST="TRUE" DO_ENSINIT="TRUE" RADAR_REF_THINNING="2" ARCHIVEDIR="/5year/BMC/wrfruc/rrfs_ens" From 82470090bec5a37ab81903e67207aeb19416b94c Mon Sep 17 00:00:00 2001 From: chunhua zhou Date: Wed, 28 Sep 2022 09:41:49 -0600 Subject: [PATCH 7/8] Update exregional_run_fcst.sh --- scripts/exregional_run_fcst.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index c938227d8..46069957c 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -510,32 +510,28 @@ fi # #----------------------------------------------------------------------- # +hh=${cdate:8:2} for cyc_start in "${CYCL_HRS_SPP[@]}"; do - hh=${cdate:8:2} if [ ${hh} -eq ${cyc_start} ]; then DO_SPP=TRUE fi done for cyc_start in "${CYCL_HRS_SPPT[@]}"; do - hh=${cdate:8:2} if [ ${hh} -eq ${cyc_start} ]; then DO_SPPT=TRUE fi done for cyc_start in "${CYCL_HRS_SHUM[@]}"; do - hh=${cdate:8:2} if [ ${hh} -eq ${cyc_start} ]; then DO_SHUM=TRUE fi done for cyc_start in "${CYCL_HRS_SKEB[@]}"; do - hh=${cdate:8:2} if [ ${hh} -eq ${cyc_start} ]; then DO_SKEB=TRUE fi done for cyc_start in "${CYCL_HRS_LSM_SPP[@]}"; do - hh=${cdate:8:2} if [ ${hh} -eq ${cyc_start} ]; then DO_LSM_SPP=TRUE fi From b38689d5f888342f6b39d81516e34d495127b93e Mon Sep 17 00:00:00 2001 From: chunhua zhou Date: Wed, 28 Sep 2022 14:26:27 -0600 Subject: [PATCH 8/8] Update exregional_run_fcst.sh --- scripts/exregional_run_fcst.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh index 46069957c..063dd4b20 100755 --- a/scripts/exregional_run_fcst.sh +++ b/scripts/exregional_run_fcst.sh @@ -505,7 +505,7 @@ fi # #----------------------------------------------------------------------- # -# Set the stochatics physics related namelist on the fly for the current +# Set the stochastic physics related namelist on the fly for the current # forecast cycle # #-----------------------------------------------------------------------