NOAA-GSL · christinaholtNOAA · Apr 15, 2021 · Mar 29, 2021 · Mar 29, 2021 · Mar 29, 2021
diff --git a/modulefiles/tasks/linux/make_grid.local b/modulefiles/tasks/linux/make_grid.local
@@ -1,5 +1,5 @@
 #%Module
-module use -a /contrib/miniconda3/modulefiles
+module use -a /contrib/apps/miniconda3/modulefiles
 module load miniconda3
 if [module-info mode load] {
   system "conda activate regional_workflow"

diff --git a/modulefiles/tasks/linux/make_ics.local b/modulefiles/tasks/linux/make_ics.local
@@ -1,5 +1,5 @@
 #%Module
-module use -a /contrib/miniconda3/modulefiles
+module use -a /contrib/apps/miniconda3/modulefiles
 module load miniconda3
 if [module-info mode load] {
   system "conda activate regional_workflow"

diff --git a/modulefiles/tasks/linux/make_lbcs.local b/modulefiles/tasks/linux/make_lbcs.local
@@ -1,5 +1,5 @@
 #%Module
-module use -a /contrib/miniconda3/modulefiles
+module use -a /contrib/apps/miniconda3/modulefiles
 module load miniconda3
 if [module-info mode load] {
   system "conda activate regional_workflow"

diff --git a/modulefiles/tasks/linux/run_fcst.local b/modulefiles/tasks/linux/run_fcst.local
@@ -1,5 +1,5 @@
 #%Module
-module use -a /contrib/miniconda3/modulefiles
+module use -a /contrib/apps/miniconda3/modulefiles
 module load miniconda3
 if [module-info mode load] {
   system "conda activate regional_workflow"

diff --git a/scripts/exregional_run_fcst.sh b/scripts/exregional_run_fcst.sh
@@ -419,7 +419,7 @@ if [ "${RUN_ENVIR}" != "nco" ] && [ "${MACHINE}" != "WCOSS_CRAY" ] ; then
 fi
 
 ln_vrfy -sf ${relative_or_null} ${DATA_TABLE_FP} ${run_dir}
-ln_vrfy -sf ${relative_or_null} ${FIELD_TABLE_FP} ${run_dir}
+ln_vrfy -sf ${relative_or_null} "${FIELD_TABLE_FP[$(( 10#${ensmem_indx}-1 ))]}" ${run_dir}/${FIELD_TABLE_FN}
 ln_vrfy -sf ${relative_or_null} ${NEMS_CONFIG_FP} ${run_dir}
 
 if [ "${DO_ENSEMBLE}" = TRUE ]; then
@@ -476,6 +476,42 @@ if [ "${DO_ENSEMBLE}" = "TRUE" ]; then
   diag_table_fp="${cycle_dir}/${DIAG_TABLE_FN}"
   ln_vrfy -sf ${relative_or_null} ${diag_table_fp} ${run_dir}
 fi
+
+#
+#-----------------------------------------------------------------------
+#
+# Copy and create files for inline post
+#
+#-----------------------------------------------------------------------
+#
+
+cat > itag <<EOF
+
+ &NAMPGB
+ KPO=47,PO=1000.,975.,950.,925.,900.,875.,850.,825.,800.,775.,750.,725.,700.,675.,650.,625.,600.,575.,550.,525.,500.,475.,450.,425.,400.,375.,350.,325.,300.,275.,250.,225.,200.,175.,150.,125.,100.,70.,50.,30.,20.,10.,7.,5.,3.,2.,1.,
+   /
+EOF
+
+if [ ${USE_CUSTOM_POST_CONFIG_FILE} = "TRUE" ]; then
+  post_config_fp="${CUSTOM_POST_CONFIG_FP}"
+  print_info_msg "
+====================================================================
+Copying the user-defined post flat file specified by
+CUSTOM_POST_CONFIG_FP to the forecast run directory (fhr_dir):
+  CUSTOM_POST_CONFIG_FP = \"${CUSTOM_POST_CONFIG_FP}\"
+===================================================================="
+else
+  post_config_fp="${UFS_WTHR_MDL_DIR}/tests/parm/postxconfig-NT-fv3lam.txt"
+  print_info_msg "
+====================================================================
+Copying the default post flat file specified by post_config_fp to the
+forecast run directory (fhr_dir): post_config_fp = \"${post_config_fp}\"
+===================================================================="
+fi
+cp_vrfy ${post_config_fp} ./postxconfig-NT.txt
+cp_vrfy ${post_config_fp} ./postxconfig-NT_FH00.txt
+cp_vrfy ${UFS_WTHR_MDL_DIR}/tests/parm/params_grib2_tbl_new ./params_grib2_tbl_new
+
 #
 #-----------------------------------------------------------------------
 #
@@ -485,7 +521,6 @@ fi
 #
 export KMP_AFFINITY=scatter
 export OMP_NUM_THREADS=${OMP_NUM_THREADS:-1} #Needs to be 1 for dynamic build of CCPP with GFDL fast physics, was 2 before.
-export OMP_STACKSIZE=1024m
 
 #
 #-----------------------------------------------------------------------

diff --git a/ush/README.HWT_ENS b/ush/README.HWT_ENS
@@ -0,0 +1,57 @@
+
+
+# HWT Ensemble
+
+## Description
+
+The ensemble for the HWT Spring Experiment 2021 is a 9 member forecast
+ensemble run to 60 hours at 00 UTC daily. It is run on the North
+American (NA) 3km domain, and mixes in ICS/LBCS external models and
+pertubations, multiple physics packages, and multiple stochastic physics
+approaches to acheive a more desirable spread in forcasts. The sections
+below provide details of the variations among members.
+
+
+## ICS/LBCS
+The ICS and LBCS come from both the GFS and GEFS using 6 hour forecasts
+from 18 UTC as follows:
+
+  - Members 1-3 have GFS ICS/LBCS
+  - Members 4-6 have GEFS Member 01 ICS/LBCS
+  - Members 7-9 have GEFS Member 02 ICS/LBCS
+
+Below, these sets will be referred to as "sub-ensembles".
+
+## Physics
+
+Each sub-ensemble will run with varying physics in each member. Here is
+an example of the GFS sub-ensemble, and the same pattern applies to
+each sub-ensemble:
+
+  - Member 1: FV3_GFS_v15_thompson_mynn CCPP Suite
+  - Member 2: GFSv16 (without parameterized convection)
+  - Member 3: ???
+
+
+## Stochastic perturbations
+
+Each sub-ensemble will be treated with the same stochastic
+perturbations for all members:
+
+  - Members 1-3: None
+  - Members 4-6: SPPT
+  - Members 7-9: SPPT/SHUM/SKEB
+
+# Configuring the ensemble
+
+The configuration layer doesn't currently handle this type of mixed
+ensemble. Instead there is a manual method to do this work:
+
+1. Configure each sub-ensemble using different RUN and NET variables
+(RRFS_NA_GFS, RRFS_NA_GEFS01, RRFS_NA_GEFS02)
+
+  - Each configuration stage will generate 3 namelists with varying
+  seed values for stochastic physics. 
+2. Turn on each rocotorun job in the crontab.
+
+
diff --git a/ush/config.sh.gefs01 b/ush/config.sh.gefs01
@@ -12,8 +12,8 @@ RUN_CMD_POST="srun --mpi=pmi2"
 
 NCORES_PER_NODE="36"
 PPN_RUN_FCST="18"
-LAYOUT_X="36"
-LAYOUT_Y="18"
+LAYOUT_X="18"
+LAYOUT_Y="36"
 WRTCMP_write_groups="1"
 WRTCMP_write_tasks_per_group="72"
 
@@ -28,8 +28,8 @@ LMOD_PATH="/apps/lmod/lmod/init/sh"
 BUILD_ENV_FN="build_aws_intel.env"
 
 # Copied directly from setup.sh HERA settings
-DO_ENSEMBLE="FALSE"
-NUM_ENS_MEMBERS="1"
+DO_ENSEMBLE="TRUE"
+NUM_ENS_MEMBERS="2"
 NDIGITS_ENSMEM_NAMES="2"
 
 # Get external model files staged on disk
@@ -53,6 +53,7 @@ PREEXISTING_DIR_METHOD="rename"
 QUILTING="TRUE"
 
 CCPP_PHYS_SUITE="FV3_GSD_SAR"
+SUITES=( FV3_GFS_v15_thompson_mynn rrfs_gfsv16 )
 FCST_LEN_HRS="3"
 LBC_SPEC_INTVL_HRS="3"
 
@@ -73,3 +74,4 @@ FIXgsm=${FIXgsm:-"/lustre/NCEPDEV/global/glopara/fix/fix_am"}
 TOPO_DIR=${TOPO_DIR:-"/lustre/NCEPDEV/global/glopara/fix/fix_orog"}
 SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/lustre/NCEPDEV/global/glopara/fix/fix_sfc_climo"}
 FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/lustre/NCEPDEV/global/glopara/fix"}
+
diff --git a/ush/config.sh.gfs b/ush/config.sh.gfs
@@ -12,8 +12,8 @@ RUN_CMD_POST="srun --mpi=pmi2"
 
 NCORES_PER_NODE="36"
 PPN_RUN_FCST="18"
-LAYOUT_X="36"
-LAYOUT_Y="18"
+LAYOUT_X="18"
+LAYOUT_Y="36"
 WRTCMP_write_groups="1"
 WRTCMP_write_tasks_per_group="72"
 
@@ -28,8 +28,8 @@ LMOD_PATH="/apps/lmod/lmod/init/sh"
 BUILD_ENV_FN="build_aws_intel.env"
 
 # Copied directly from setup.sh HERA settings
-DO_ENSEMBLE="FALSE"
-NUM_ENS_MEMBERS="1"
+DO_ENSEMBLE="TRUE"
+NUM_ENS_MEMBERS="2"
 NDIGITS_ENSMEM_NAMES="2"
 
 # Get external model files staged on disk
@@ -54,6 +54,7 @@ PREEXISTING_DIR_METHOD="rename"
 QUILTING="TRUE"
 
 CCPP_PHYS_SUITE="FV3_GSD_SAR"
+SUITES=( FV3_GFS_v15_thompson_mynn rrfs_gfsv16 )
 FCST_LEN_HRS="3"
 LBC_SPEC_INTVL_HRS="3"
 

diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh
@@ -539,6 +539,7 @@ EXTRN_MDL_FILES_LBCS=( "LBCS_file1" "LBCS_file2" "..." )
 #-----------------------------------------------------------------------
 #
 CCPP_PHYS_SUITE="FV3_GFS_v15p2"
+SUITES=( $CCPP_PHYS_SUITE )
 #
 #-----------------------------------------------------------------------
 #

diff --git a/ush/create_diag_table_file.sh b/ush/create_diag_table_file.sh
@@ -67,6 +67,7 @@ function create_diag_table_file() {
 #-----------------------------------------------------------------------
 #
   local i \
+        ensmem \
         diag_table_fp \
         settings
 #
@@ -76,6 +77,9 @@ function create_diag_table_file() {
 #
 #-----------------------------------------------------------------------
 #
+
+ensmem=$(( ${ENSMEM_INDX} - 1 ))
+
   print_info_msg "$VERBOSE" "                                            
 Creating a diagnostics table file (\"${DIAG_TABLE_FN}\")  in the
 specified run
@@ -89,7 +93,7 @@ directory (run_dir):
   print_info_msg "$VERBOSE" "
 Using the template diagnostics table file:
 
-  diag_table_tmpl_fp = ${DIAG_TABLE_TMPL_FP}
+  diag_table_tmpl_fp = ${DIAG_TABLE_TMPL_FP[$ensmem]}
 
 to create:
 
@@ -100,7 +104,7 @@ to create:
   cres: ${CRES}
 "
 
-  $USHDIR/fill_jinja_template.py -q -u "${settings}" -t "${DIAG_TABLE_TMPL_FP}" -o "${diag_table_fp}" || \
+  $USHDIR/fill_jinja_template.py -q -u "${settings}" -t "${DIAG_TABLE_TMPL_FP[$ensmem]}" -o "${diag_table_fp}" || \
   print_err_msg_exit "
 !!!!!!!!!!!!!!!!!