From 6ab44fef78f33785268492f54ee1b2c0f8a17242 Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Fri, 22 Jan 2021 14:44:59 +0000
Subject: [PATCH 001/125] Issue #164 - update machine-setup.sh for Hera stack,
 update checkout/build/linking for UPP tag that supports hpc-stack, remove
 post patch in HERA.env

---
 Externals.cfg           |  3 ++-
 env/HERA.env            |  8 --------
 sorc/build_ncep_post.sh |  4 ++--
 sorc/checkout.sh        |  4 +++-
 sorc/link_fv3gfs.sh     |  2 +-
 sorc/machine-setup.sh   | 11 ++++++-----
 6 files changed, 14 insertions(+), 18 deletions(-)

diff --git a/Externals.cfg b/Externals.cfg
index 584f02174c2..d4f57c32991 100644
--- a/Externals.cfg
+++ b/Externals.cfg
@@ -22,7 +22,8 @@ protocol = git
 required = True
 
 [EMC_post]
-tag = upp_gfsv16_release.v1.1.1
+#No externals setting = .gitmodules will be invoked for CMakeModules and comupp/src/lib/crtm2 submodules
+tag = upp_v10.0.3
 local_path = sorc/gfs_post.fd
 repo_url = https://github.com/NOAA-EMC/EMC_post.git
 protocol = git
diff --git a/env/HERA.env b/env/HERA.env
index f71531c1ed2..156873e891e 100755
--- a/env/HERA.env
+++ b/env/HERA.env
@@ -160,14 +160,6 @@ elif [ $step = "efcs" ]; then
 
 elif [ $step = "post" ]; then
 
-    source /apps/lmod/lmod/init/sh
-    module unload esmf/8.0.1_ParallelNetCDF.release
-    module unload netcdf_parallel/4.7.4.release
-    module unload hdf5_parallel/1.10.6.release
-    module use -a /scratch2/NCEPDEV/nwprod/NCEPLIBS/modulefiles
-    module load hdf5_parallel/1.10.6
-    module load netcdf_parallel/4.7.4
-
     nth_max=$(($npe_node_max / $npe_node_post))
 
     export NTHREADS_NP=${nth_np:-1}
diff --git a/sorc/build_ncep_post.sh b/sorc/build_ncep_post.sh
index 5bbc2f680af..95aa0caedd7 100755
--- a/sorc/build_ncep_post.sh
+++ b/sorc/build_ncep_post.sh
@@ -16,5 +16,5 @@ if [ ! -d "../exec" ]; then
   mkdir ../exec
 fi
 
-cd gfs_post.fd/sorc
-sh build_ncep_post.sh
+cd gfs_post.fd/tests
+./compile_upp.sh
diff --git a/sorc/checkout.sh b/sorc/checkout.sh
index 9c9ed202965..0eb3160ddef 100755
--- a/sorc/checkout.sh
+++ b/sorc/checkout.sh
@@ -81,7 +81,9 @@ if [[ ! -d gfs_post.fd ]] ; then
     rm -f ${topdir}/checkout-gfs_post.log
     git clone https://github.com/NOAA-EMC/EMC_post.git gfs_post.fd >> ${topdir}/checkout-gfs_post.log 2>&1
     cd gfs_post.fd
-    git checkout upp_gfsv16_release.v1.1.1
+    git checkout upp_v10.0.3
+    git submodule update --init CMakeModules
+
     ################################################################################
     # checkout_gtg
     ## yes: The gtg code at NCAR private repository is available for ops. GFS only.
diff --git a/sorc/link_fv3gfs.sh b/sorc/link_fv3gfs.sh
index e5f0f08ad44..008f6e5d0c6 100755
--- a/sorc/link_fv3gfs.sh
+++ b/sorc/link_fv3gfs.sh
@@ -206,7 +206,7 @@ if [ -d ../sorc/fv3gfs.fd/WW3/exec ]; then # Wave execs
 fi
 
 [[ -s gfs_ncep_post ]] && rm -f gfs_ncep_post
-$LINK ../sorc/gfs_post.fd/exec/ncep_post gfs_ncep_post
+$LINK ../sorc/gfs_post.fd/exec/upp.x gfs_ncep_post
 
 if [ -d ${pwd}/gfs_wafs.fd ]; then 
     for wafsexe in \
diff --git a/sorc/machine-setup.sh b/sorc/machine-setup.sh
index c39cbfb2722..b934f361b8b 100644
--- a/sorc/machine-setup.sh
+++ b/sorc/machine-setup.sh
@@ -44,12 +44,13 @@ elif [[ -d /scratch1 ]] ; then
         source /apps/lmod/lmod/init/$__ms_shell
     fi
     target=hera
+
     module purge
-    module load intel
-    module load impi
-    export NCEPLIBS=/scratch2/NCEPDEV/nwprod/NCEPLIBS
-    module use $NCEPLIBS/modulefiles
-    #export WRFPATH=$NCEPLIBS/wrf.shared.new/v1.1.1/src
+    module use /scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack
+    module load hpc/1.1.0
+    module load hpc-intel/18.0.5.274
+    module load hpc-impi/2018.0.4
+
     export myFC=mpiifort
     export FCOMP=mpiifort
 

From 40184c2a21a22d9ac432a02e1e5bdc4ee18f0887 Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Fri, 22 Jan 2021 16:36:46 +0000
Subject: [PATCH 002/125] Issue #164 - remove build_libs.sh, update
 module_base.hera for hpc-stack, plus updates for building
 enkf_chgres_recenter, gfs_bufr, and tocsbufr with hpc-stack

---
 modulefiles/fv3gfs/enkf_chgres_recenter.hera | 16 +++--
 modulefiles/gfs_bufr.hera                    | 31 +++++-----
 modulefiles/module_base.hera                 | 27 +++++----
 sorc/build_libs.sh                           | 61 --------------------
 sorc/gfs_bufr.fd/makefile_module             | 10 ++--
 sorc/tocsbufr.fd/makefile_module             |  2 +-
 6 files changed, 46 insertions(+), 101 deletions(-)
 delete mode 100755 sorc/build_libs.sh

diff --git a/modulefiles/fv3gfs/enkf_chgres_recenter.hera b/modulefiles/fv3gfs/enkf_chgres_recenter.hera
index ed374938283..c5fe744ca16 100644
--- a/modulefiles/fv3gfs/enkf_chgres_recenter.hera
+++ b/modulefiles/fv3gfs/enkf_chgres_recenter.hera
@@ -2,11 +2,15 @@
 ## enkf_chgres_recenter component - hera
 #############################################################
 
-module use -a /scratch2/NCEPDEV/nwprod/NCEPLIBS/modulefiles
-module load nemsio/2.2.3
-module load bacio/2.0.2
-module load w3nco/2.0.6
-module load ip/3.0.1
-module load sp/2.0.2
+module use /scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack
+module load hpc/1.1.0
+module load hpc-intel/18.0.5.274
+module load hpc-impi/2018.0.4
+
+module load nemsio/2.5.2
+module load bacio/2.4.1
+module load w3nco/2.4.1
+module load ip/3.3.3
+module load sp/2.3.3
 
 export FC=ifort
diff --git a/modulefiles/gfs_bufr.hera b/modulefiles/gfs_bufr.hera
index 5d4ef5f7acf..3dc2d1c988d 100644
--- a/modulefiles/gfs_bufr.hera
+++ b/modulefiles/gfs_bufr.hera
@@ -2,24 +2,23 @@
 ## bufrsnd
 #############################################################
 
-# Loading Intel Compiler Suite
-module load intel
-module load impi
-module use -a /scratch2/NCEPDEV/nwprod/NCEPLIBS/modulefiles
-# Loading nceplibs modules
-module load sigio/2.1.1
-module load bacio/2.0.3
-module load w3nco/2.0.6
-module load bufr/11.3.0
-module load nemsio/2.2.3
-module load intelpython/3.6.8
-module load w3emc/2.3.1
+#Load from hpc-stack
+module use /scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack
+module load hpc/1.1.0
+module load hpc-intel/18.0.5.274
+module load hpc-impi/2018.0.4
+
+module load sigio/2.3.2
+module load bacio/2.4.1
+module load w3nco/2.4.1
+module load bufr/11.4.0
+module load nemsio/2.5.2
+module load w3emc/2.7.3
 
-module use /scratch2/NCEPDEV/nwprod/NCEPLIBS/modulefiles
-module load hdf5_parallel/1.10.6
-module load netcdf_parallel/4.7.4
-#module load w3emc_para/2.4.0          
+module load netcdf/4.7.4
+module load hdf5/1.10.6
 
+module load intelpython/3.6.8
 
 export myFC=$FCOMP
 export myFCFLAGS="-O3 -convert big_endian -traceback -g -fp-model source -qopenmp"
diff --git a/modulefiles/module_base.hera b/modulefiles/module_base.hera
index adb8370a29f..ddec91b55b1 100644
--- a/modulefiles/module_base.hera
+++ b/modulefiles/module_base.hera
@@ -3,24 +3,27 @@
 ##      FV3GFS prerequisites
 ##
 
-module load intel/18.0.5.274
-module load impi/2018.0.4
+module use /scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack
+module load hpc/1.1.0
+module load hpc-intel/18.0.5.274
+module load hpc-impi/2018.0.4
+
 module load wgrib2/2.0.8
 module load hpss/hpss
-module load nco/4.7.0
+module load nco/4.9.1
 module load gempak/7.4.2
 
-#Load from official NCEPLIBS
-module use -a /scratch2/NCEPDEV/nwprod/NCEPLIBS/modulefiles
-module load prod_util/1.1.0
-module load grib_util/1.1.1
-module load g2tmpl/1.6.0
+#Load from hpc-stack
+module load prod_util/1.2.2
+module load grib_util/1.2.2
+module load g2tmpl/1.9.1
 module load crtm/2.3.0
 
-module use /scratch1/NCEPDEV/nems/emc.nemspara/soft/modulefiles/
-module load netcdf_parallel/4.7.4.release
-module load hdf5_parallel/1.10.6.release
-module load esmf/8.0.1_ParallelNetCDF.release
+setenv CRTM_FIX /scratch2/NCEPDEV/nwprod/NCEPLIBS/fix/crtm_v2.3.0
+
+module load netcdf/4.7.4
+module load hdf5/1.10.6
+module load esmf/8_1_0_beta_snapshot_27
 
 # python
 module use -a /contrib/anaconda/modulefiles
diff --git a/sorc/build_libs.sh b/sorc/build_libs.sh
deleted file mode 100755
index 31e158c1fcd..00000000000
--- a/sorc/build_libs.sh
+++ /dev/null
@@ -1,61 +0,0 @@
-#!/bin/sh
-set -eux
-
-source ./machine-setup.sh > /dev/null 2>&1
-cwd=`pwd`
-
-#
-# Clean lib directory (libraries, modulefiles and incmod)
-#
-cd $cwd/lib
-rm -rf libpng* libjasper*  lib*.a incmod/* modulefiles/*
-
-#
-# Optional install, first independent libraries:
-#
-
-# --- first choose BUFR version (different on different machine)
-# --- hopefully this will lead to single version in the future
-if [ $target = wcoss_cray ]; then
-  bufr_ver=v11.0.1
-fi
-# --------------------------------------------------------------
-
-for lib in \
-    bacio_v2.0.2         \
-    bufr_${bufr_ver}     \
-    crtm_v2.0.6          \
-    g2tmpl_v1.3.0        \
-    gfsio_v1.1.0         \
-    ip_v2.0.0            \
-    ip_v3.0.0            \
-    jasper-1.900.1       \
-    landsfcutil_v2.1.0   \
-    nemsio_v2.2.3        \
-    png-1.2.44           \
-    sfcio_v1.0.0         \
-    sigio_v2.0.1         \
-    sp_v2.0.2            \
-    w3nco_v2.0.6         \
-    z-1.2.6
-do
- cd $cwd/lib/sorc/$lib
- ./build.sh
- cd $cwd
-done
-
-#
-# Optional install, now libraries depending on previously installed libs:
-#
-for lib in \
-    w3emc_v2.2.0         \
-    nemsiogfs_v2.0.1     \
-    g2c_v1.5.0           \
-    g2_v3.1.0
-do
- cd $cwd/lib/sorc/$lib
- ./build.sh
- cd $cwd
-done
-
-exit
diff --git a/sorc/gfs_bufr.fd/makefile_module b/sorc/gfs_bufr.fd/makefile_module
index c640d0a9278..d9d5374a7a2 100755
--- a/sorc/gfs_bufr.fd/makefile_module
+++ b/sorc/gfs_bufr.fd/makefile_module
@@ -8,20 +8,20 @@
 FC = $(myFC) $(myFCFLAGS)
 CPP = $(myCPP) $(myCPPFLAGS)
 
-FFLAGS   = -I$(NETCDF_INC) \
+FFLAGS   = -I$(NETCDF_INCLUDES) \
     	   -I$(NEMSIO_INC) \
-           -I$(SIGIO_INC4) \
+           -I$(SIGIO_INC) \
            -I$(W3EMC_INC4)
 
-LIBS =    -L$(NETCDF_LIB) -lnetcdf -lnetcdff \
-          -L$(HDF5_LIB) -lhdf5_hl -lhdf5 \
+LIBS =    -L$(NETCDF_LIBRARIES) -lnetcdff -lnetcdf \
+          -L$(HDF5_LIBRARIES) -lhdf5_hl -lhdf5 -lz \
 	       $(NEMSIO_LIB) \
            $(W3EMC_LIB4) \
            $(W3NCO_LIB4) \
            $(BUFR_LIB4) \
            $(BACIO_LIB4) \
            $(SP_LIB4) \
-           $(SIGIO_LIB4)
+           $(SIGIO_LIB)
 
 SRCM    = gfsbufr.f
 OBJS    = physcons.o funcphys.o meteorg.o bfrhdr.o newsig1.o terp3.o\
diff --git a/sorc/tocsbufr.fd/makefile_module b/sorc/tocsbufr.fd/makefile_module
index b6310ff7659..06f5ba7092b 100755
--- a/sorc/tocsbufr.fd/makefile_module
+++ b/sorc/tocsbufr.fd/makefile_module
@@ -47,7 +47,7 @@ LIBS     = $(W3EMC_LIB4) \
            $(BUFR_LIB4) \
            $(BACIO_LIB4) \
            $(SP_LIB4) \
-           $(SIGIO_LIB4)
+           $(SIGIO_LIB)
 CMD =		../../exec/tocsbufr
 PROFLIB =	-lprof
 

From e09cb35b1ad654715ec9341f92ac98a2a9813730 Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Mon, 25 Jan 2021 19:18:12 +0000
Subject: [PATCH 003/125] Issue #164 - update GLDAS tag in Externals.cfg and
 stack updates for fbwndgfs and grib_util builds

---
 Externals.cfg                                 |  2 +-
 modulefiles/gfs_fbwndgfs.hera                 | 16 ++++++----
 modulefiles/modulefile.grib_util.hera         | 24 ++++++++-------
 util/modulefiles/gfs_util.hera                | 29 +++++++++----------
 .../overgridid.fd/compile_overgridid_wcoss.sh |  2 +-
 5 files changed, 40 insertions(+), 33 deletions(-)

diff --git a/Externals.cfg b/Externals.cfg
index 8759cd79952..d846088fff3 100644
--- a/Externals.cfg
+++ b/Externals.cfg
@@ -15,7 +15,7 @@ protocol = git
 required = True
 
 [GLDAS]
-tag = gldas_gfsv16_release.v1.12.0
+tag = gldas_gfsv16_release.v1.13.0
 local_path = sorc/gldas.fd
 repo_url = https://github.com/NOAA-EMC/GLDAS.git
 protocol = git
diff --git a/modulefiles/gfs_fbwndgfs.hera b/modulefiles/gfs_fbwndgfs.hera
index 60a8ccad54c..b1374464e7c 100644
--- a/modulefiles/gfs_fbwndgfs.hera
+++ b/modulefiles/gfs_fbwndgfs.hera
@@ -2,9 +2,13 @@
 ## Module file for fbwndgfs 
 #############################################################
 
-module use -a /scratch2/NCEPDEV/nwprod/NCEPLIBS/modulefiles
-module load bacio/2.0.3
-module load w3emc/2.3.1
-module load w3nco/2.0.6
-module load ip/3.0.1
-module load sp/2.0.2
+module use /scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack
+module load hpc/1.1.0
+module load hpc-intel/18.0.5.274
+module load hpc-impi/2018.0.4
+
+module load bacio/2.4.1
+module load w3emc/2.7.3
+module load w3nco/2.4.1
+module load ip/3.3.3
+module load sp/2.3.3
diff --git a/modulefiles/modulefile.grib_util.hera b/modulefiles/modulefile.grib_util.hera
index b912ecc12d0..2b853d2663d 100644
--- a/modulefiles/modulefile.grib_util.hera
+++ b/modulefiles/modulefile.grib_util.hera
@@ -1,17 +1,21 @@
 #%Module######################################################################
 #  Module file for grib_util
 ##############################################################################
-module use -a /scratch2/NCEPDEV/nwprod/NCEPLIBS/modulefiles
-module load intel/18.0.5.274 
-module load bacio/2.0.3
-module load w3emc/2.3.1
-module load w3nco/2.0.6
-module load ip/3.0.1
-module load sp/2.0.2
 
-module load jasper/1.900.1
-module load png/1.2.44
-module load z/1.2.11
+module use /scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack
+module load hpc/1.1.0
+module load hpc-intel/18.0.5.274
+module load hpc-impi/2018.0.4
+
+module load bacio/2.4.1
+module load w3emc/2.7.3
+module load w3nco/2.4.1
+module load ip/3.3.3
+module load sp/2.3.3
+
+module load jasper/2.0.22
+module load png/1.6.35
+module load zlib/1.2.11
 
 export FCMP=ifort
 export CCMP=icc
diff --git a/util/modulefiles/gfs_util.hera b/util/modulefiles/gfs_util.hera
index cf880d66b66..ac8a7d941cd 100644
--- a/util/modulefiles/gfs_util.hera
+++ b/util/modulefiles/gfs_util.hera
@@ -5,25 +5,24 @@
 #  Loading required system modules
 #
 
-module use -a /scratch2/NCEPDEV/nwprod/NCEPLIBS/modulefiles
-# Loading Intel Compiler Suite
-module load intel/18.0.5.274
+module use /scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack
+module load hpc/1.1.0
+module load hpc-intel/18.0.5.274
+module load hpc-impi/2018.0.4
 
-module load bacio/2.0.3
-module load w3emc/2.3.1
-module load w3nco/2.0.6
-module load ip/3.0.1
-module load sp/2.0.2
-module load bufr/11.3.0
+module load bacio/2.4.1
+module load w3emc/2.7.3
+module load w3nco/2.4.1
+module load ip/3.3.3
+module load sp/2.3.3
+module load bufr/11.4.0
 
-module load jasper/1.900.1
-module load png/1.2.44
-module load z/1.2.11
+module load jasper/2.0.22
+module load png/1.6.35
+module load zlib/1.2.11
 
 module load ncl/6.5.0
-module load gempak/7.4.2 
-
-module load impi/2018.0.4
+module load gempak/7.4.2
 
 export GEMINC=/apps/gempak/7.4.2/gempak/include
 export GEMOLB=/apps/gempak/7.4.2/os/linux64/lib
diff --git a/util/sorc/overgridid.fd/compile_overgridid_wcoss.sh b/util/sorc/overgridid.fd/compile_overgridid_wcoss.sh
index f6800a1943b..ccce82b0e52 100755
--- a/util/sorc/overgridid.fd/compile_overgridid_wcoss.sh
+++ b/util/sorc/overgridid.fd/compile_overgridid_wcoss.sh
@@ -12,7 +12,7 @@ cwd=`pwd`
 
 if [ "$target" = "wcoss_dell_p3" ] || [ "$target" = "wcoss_cray" ] || [ "$target" = "hera" ] ; then
    echo " "
-   echo " You are on WCOSS:  $target "
+   echo " You are on $target "
    echo " "
 elif [ "$target" = "wcoss" ] ; then
    echo " "

From bc273e98a506d4e2724d4c5c35718f74776ce9d7 Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Mon, 25 Jan 2021 19:50:49 +0000
Subject: [PATCH 004/125] Issue #164 - updates to partial build and stack
 updates for tropcy build

---
 .../modulefile.storm_reloc_v6.0.0.hera        | 28 +++++++++--------
 sorc/build_all.sh                             | 30 ++++++-------------
 sorc/build_tropcy_NEMS.sh                     |  8 +++--
 sorc/fv3gfs_build.cfg                         |  1 -
 sorc/partial_build.sh                         |  2 +-
 5 files changed, 31 insertions(+), 38 deletions(-)

diff --git a/modulefiles/modulefile.storm_reloc_v6.0.0.hera b/modulefiles/modulefile.storm_reloc_v6.0.0.hera
index 36f96e01b90..489c9449f16 100644
--- a/modulefiles/modulefile.storm_reloc_v6.0.0.hera
+++ b/modulefiles/modulefile.storm_reloc_v6.0.0.hera
@@ -2,17 +2,21 @@
 ## Module file for storm_reloc_v6.0.0 - Hera
 #############################################################
 
-module use -a /scratch2/NCEPDEV/nwprod/NCEPLIBS/modulefiles
-module load w3emc/2.3.0
-module load w3nco/2.0.6
-module load bacio/2.0.2
-module load sp/2.0.2
-module load nemsio/2.2.3
-module load nemsiogfs/2.2.1
-module load sigio/2.1.1
-module load jasper/1.900.1
-module load png/1.2.44
-module load z/1.2.11
-module load g2/3.1.0
+module use /scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack
+module load hpc/1.1.0
+module load hpc-intel/18.0.5.274
+module load hpc-impi/2018.0.4
+
+module load w3emc/2.7.3
+module load w3nco/2.4.1
+module load bacio/2.4.1
+module load sp/2.3.3
+module load nemsio/2.5.2
+module load nemsiogfs/2.5.3
+module load sigio/2.3.2
+module load jasper/2.0.22
+module load png/1.6.35
+module load zlib/1.2.11
+module load g2/3.4.1
 
 set FC mpiifort
diff --git a/sorc/build_all.sh b/sorc/build_all.sh
index 4f8cfae730b..398abd00899 100755
--- a/sorc/build_all.sh
+++ b/sorc/build_all.sh
@@ -58,15 +58,6 @@ source ./machine-setup.sh > /dev/null 2>&1
 ERRSCRIPT=${ERRSCRIPT:-'eval [[ $err = 0 ]]'}
 err=0
 
-#------------------------------------
-# build libraries first
-#------------------------------------
-$Build_libs && {
-echo " .... Library build not currently supported .... "
-#echo " .... Building libraries .... "
-#./build_libs.sh > $logs_dir/build_libs.log 2>&1
-}
-
 #------------------------------------
 # build fv3
 #------------------------------------
@@ -269,19 +260,16 @@ fi
 #------------------------------------
 # build gfs_util       
 #------------------------------------
-# Only build on WCOSS
-if [ $target = wcoss -o $target = wcoss_cray -o $target = wcoss_dell_p3 ]; then
- $Build_gfs_util && {
- echo " .... Building gfs_util .... "
- ./build_gfs_util.sh > $logs_dir/build_gfs_util.log 2>&1
- rc=$?
- if [[ $rc -ne 0 ]] ; then
-     echo "Fatal error in building gfs_util."
-     echo "The log file is in $logs_dir/build_gfs_util.log"
- fi
- ((err+=$rc))
- }
+$Build_gfs_util && {
+echo " .... Building gfs_util .... "
+./build_gfs_util.sh > $logs_dir/build_gfs_util.log 2>&1
+rc=$?
+if [[ $rc -ne 0 ]] ; then
+    echo "Fatal error in building gfs_util."
+    echo "The log file is in $logs_dir/build_gfs_util.log"
 fi
+((err+=$rc))
+}
 
 #------------------------------------
 # Exception Handling
diff --git a/sorc/build_tropcy_NEMS.sh b/sorc/build_tropcy_NEMS.sh
index 77a69dc95e0..4a04a61828a 100755
--- a/sorc/build_tropcy_NEMS.sh
+++ b/sorc/build_tropcy_NEMS.sh
@@ -30,7 +30,9 @@ fi
 
 source ../modulefiles/modulefile.storm_reloc_v6.0.0.$target
 export FC=mpiifort
-export JASPER_LIB=${JASPER_LIB:-$JASPER_LIBRARY_DIRS/libjasper.a}
+export JASPER_LIB=${JASPER_LIB:-${JASPER_LIBRARIES}/libjasper.a}
+export PNG_LIB=${PNG_LIB:-${PNG_ROOT}/lib64/libpng.a}
+export Z_LIB=${Z_LIB:-${ZLIB_LIBRARIES}/libz.a}
 
 export INC="${G2_INCd} -I${NEMSIO_INC}"
 export LIBS="${W3EMC_LIBd} ${W3NCO_LIBd} ${BACIO_LIB4} ${G2_LIBd} ${PNG_LIB} ${JASPER_LIB} ${Z_LIB}"
@@ -38,8 +40,8 @@ export LIBS_SUP="${W3EMC_LIBd} ${W3NCO_LIBd}"
 echo lset
 echo lset
  export LIBS_REL="${W3NCO_LIB4}"
-export LIBS_REL="${NEMSIOGFS_LIB} ${NEMSIO_LIB} ${LIBS_REL} ${SIGIO_LIB4} ${BACIO_LIB4} ${SP_LIBd}"
-export LIBS_SIG="${SIGIO_INC4}"
+export LIBS_REL="${NEMSIOGFS_LIB} ${NEMSIO_LIB} ${LIBS_REL} ${SIGIO_LIB} ${BACIO_LIB4} ${SP_LIBd}"
+export LIBS_SIG="${SIGIO_INC}"
 export LIBS_SYN_GET="${W3NCO_LIB4}"
 export LIBS_SYN_MAK="${W3NCO_LIB4} ${BACIO_LIB4}"
 export LIBS_SYN_QCT="${W3NCO_LIB8}"
diff --git a/sorc/fv3gfs_build.cfg b/sorc/fv3gfs_build.cfg
index 130c6dde03d..0bc7895b006 100644
--- a/sorc/fv3gfs_build.cfg
+++ b/sorc/fv3gfs_build.cfg
@@ -1,7 +1,6 @@
 #
 # ***** configuration of fv3gfs build *****
 
- Building libraries (libs) ............................. no
  Building fv3gfs (fv3gfs) .............................. yes
  Building gsi (gsi) .................................... yes
  Building gldas (gldas) ................................ yes
diff --git a/sorc/partial_build.sh b/sorc/partial_build.sh
index 264d3c1410a..4b911bc94ec 100755
--- a/sorc/partial_build.sh
+++ b/sorc/partial_build.sh
@@ -1,7 +1,7 @@
 #
 # define the array of the name of build program
 #
- declare -a Build_prg=("Build_libs" "Build_fv3gfs" \
+ declare -a Build_prg=("Build_fv3gfs" \
                        "Build_gsi" \
                        "Build_gldas" \
                        "Build_ncep_post" \

From cf4a226caa40c132a3c944fbf9585db0ac93427e Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Mon, 25 Jan 2021 19:57:07 +0000
Subject: [PATCH 005/125] Issue #164 - stack updates for regrid_nemsio build

---
 modulefiles/modulefile.regrid_nemsio.hera | 25 +++++++++++++----------
 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/modulefiles/modulefile.regrid_nemsio.hera b/modulefiles/modulefile.regrid_nemsio.hera
index 6280a02436c..d33110842a6 100644
--- a/modulefiles/modulefile.regrid_nemsio.hera
+++ b/modulefiles/modulefile.regrid_nemsio.hera
@@ -2,16 +2,19 @@
 ## Module file for regrid_nemsio
 #############################################################
 
-module use -a /scratch2/NCEPDEV/nwprod/NCEPLIBS/modulefiles
-module load netcdf/4.7.0
-module load hdf5/1.10.5
-module load bacio/2.0.2
-module load nemsio/2.2.3
-module load w3nco/2.0.6
-module load sp/2.0.2
+module use /scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack
+module load hpc/1.1.0
+module load hpc-intel/18.0.5.274
+module load hpc-impi/2018.0.4
+
+module load netcdf/4.7.4
+module load hdf5/1.10.6
+module load bacio/2.4.1
+module load nemsio/2.5.2
+module load w3nco/2.4.1
+module load sp/2.3.3
 
 export FCMP="$FCOMP" 
-export  NETCDF_LDFLAGS_F="-L${NETCDF}/lib -lnetcdf -lnetcdff -lnetcdf  -L${HDF5}/lib -lhdf5 -lhdf5_hl -lhdf5 -lz"
-export  NETCDF_LDFLAGS="-L${NETCDF}/lib -lnetcdf -lnetcdff -lnetcdf  -L${HDF5}/lib -lhdf5 -lhdf5_hl -lhdf5 -lz"
-export NETCDF_INCLUDE=-I$NETCDF/include
-#export FCMP="mpif90 -f90=ifort"
+export NETCDF_LDFLAGS_F="-L${NETCDF_LIBRARIES} -lnetcdf -lnetcdff -lnetcdf  -L${HDF5_LIBRARIES} -lhdf5 -lhdf5_hl -lhdf5 -lz"
+export NETCDF_LDFLAGS="-L${NETCDF_LIBRARIES} -lnetcdf -lnetcdff -lnetcdf  -L${HDF5_LIBRARIES} -lhdf5 -lhdf5_hl -lhdf5 -lz"
+export NETCDF_INCLUDE="-I${NETCDF_INCLUDES}"

From aecf55db6c3e591e704be9d7404b18ed97d41add Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Mon, 25 Jan 2021 20:04:32 +0000
Subject: [PATCH 006/125] Issue #164 - stack updates for UFS_UTILS build

---
 sorc/build_ufs_utils.sh | 4 ++--
 sorc/checkout.sh        | 4 ++--
 sorc/link_fv3gfs.sh     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/sorc/build_ufs_utils.sh b/sorc/build_ufs_utils.sh
index 292145d7645..0a22135a155 100755
--- a/sorc/build_ufs_utils.sh
+++ b/sorc/build_ufs_utils.sh
@@ -7,9 +7,9 @@ cwd=`pwd`
 if [ $target = wcoss_dell_p3 ]; then target=dell; fi
 if [ $target = wcoss_cray ]; then target=cray; fi
 
-cd ufs_utils.fd/sorc
+cd ufs_utils.fd
 
-./build_all_ufs_utils.sh
+./build_all.sh
 
 exit
 
diff --git a/sorc/checkout.sh b/sorc/checkout.sh
index be70217ca89..b5da9b5f74c 100755
--- a/sorc/checkout.sh
+++ b/sorc/checkout.sh
@@ -68,9 +68,9 @@ fi
 echo ufs_utils checkout ...
 if [[ ! -d ufs_utils.fd ]] ; then
     rm -f ${topdir}/checkout-ufs_utils.log
-    git clone https://github.com/NOAA-EMC/UFS_UTILS.git ufs_utils.fd >> ${topdir}/checkout-ufs_utils.fd.log 2>&1
+    git clone --recursive https://github.com/NOAA-EMC/UFS_UTILS.git ufs_utils.fd >> ${topdir}/checkout-ufs_utils.fd.log 2>&1
     cd ufs_utils.fd
-    git checkout ops-gfsv16.0.0
+    #git checkout ops-gfsv16.0.0
     cd ${topdir}
 else
     echo 'Skip.  Directory ufs_utils.fd already exists.'
diff --git a/sorc/link_fv3gfs.sh b/sorc/link_fv3gfs.sh
index 008f6e5d0c6..15fe5225227 100755
--- a/sorc/link_fv3gfs.sh
+++ b/sorc/link_fv3gfs.sh
@@ -49,7 +49,7 @@ for dir in fix_am fix_fv3_gmted2010 fix_gldas fix_orog fix_verif fix_wave_gfs ;
 done
 
 if [ -d ${pwd}/ufs_utils.fd ]; then
-  cd ${pwd}/ufs_utils.fd/sorc
+  cd ${pwd}/ufs_utils.fd/fix
   ./link_fixdirs.sh $RUN_ENVIR $machine
 fi
 

From a7391d70716c5a543ec57cf74f2c5cf133c15330 Mon Sep 17 00:00:00 2001
From: "kate.friedman" <kate.friedman@noaa.gov>
Date: Thu, 28 Jan 2021 19:04:52 +0000
Subject: [PATCH 007/125] Update free-forecast mode chgres jobs for chgres_cube

Update free-forecast mode to interface with UFS_UTILS
gdas_init utility scripts. Update getic job to use gdas_init get
scripts to pull ICs off HPSS for GFS versions 13 and later.
Rename fv3ic job to "init" and update it to interface with gdas_init
run scripts to run chgres_cube and produce GFSv16 ICs.

Update job dependencies to detect need to run chgres jobs and hold
forecast jobs until ICs are generated or present.

Further updates coming for this task. Tested on WCOSS-Dell, need to
test elsewhere still. Will disable getic job on Orion.

Refs: #1, #178
---
 env/HERA.env                          |   6 +-
 env/JET.env                           |   6 +-
 env/ORION.env                         |   6 +-
 env/WCOSS_C.env                       |   6 +-
 env/WCOSS_DELL_P3.env                 |   6 +-
 jobs/rocoto/fv3ic.sh                  |  69 ---------
 jobs/rocoto/getic.sh                  | 212 +++++++-------------------
 jobs/rocoto/init.sh                   |  81 ++++++++++
 parm/config/config.getic              |  10 +-
 parm/config/config.init               |  19 +++
 parm/config/config.resources          |   7 +
 sorc/build_ufs_utils.sh               |   4 +-
 sorc/link_fv3gfs.sh                   |   2 +-
 ush/rocoto/setup_workflow_fcstonly.py |  59 ++-----
 14 files changed, 193 insertions(+), 300 deletions(-)
 delete mode 100755 jobs/rocoto/fv3ic.sh
 create mode 100755 jobs/rocoto/init.sh
 create mode 100755 parm/config/config.init

diff --git a/env/HERA.env b/env/HERA.env
index f71531c1ed2..4290bdb259b 100755
--- a/env/HERA.env
+++ b/env/HERA.env
@@ -214,11 +214,9 @@ elif [ $step = "epos" ]; then
     [[ $NTHREADS_EPOS -gt $nth_max ]] && export NTHREADS_EPOS=$nth_max
     export APRUN_EPOS="$launcher"
 
-elif [ $step = "fv3ic" ]; then
+elif [ $step = "init" ]; then
 
-    export NTHREADS_CHGRES=${nth_chgres:-$npe_node_max}
-    [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max
-    export APRUN_CHGRES="time"
+    export APRUN="$launcher"
 
 elif [ $step = "postsnd" ]; then
 
diff --git a/env/JET.env b/env/JET.env
index ef37e468df0..5bbb903bd11 100755
--- a/env/JET.env
+++ b/env/JET.env
@@ -179,11 +179,9 @@ elif [ $step = "epos" ]; then
     [[ $NTHREADS_EPOS -gt $nth_max ]] && export NTHREADS_EPOS=$nth_max
     export APRUN_EPOS="$launcher ${npe_epos:-$PBS_NP}"
 
-elif [ $step = "fv3ic" ]; then
+elif [ $step = "init" ]; then
 
-    export NTHREADS_CHGRES=${nth_chgres:-$npe_node_max}
-    [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max
-    export APRUN_CHGRES="time"
+    export APRUN="$launcher"
 
 elif [ $step = "postsnd" ]; then
 
diff --git a/env/ORION.env b/env/ORION.env
index de989331ff1..69721c21edc 100755
--- a/env/ORION.env
+++ b/env/ORION.env
@@ -202,11 +202,9 @@ elif [ $step = "epos" ]; then
     [[ $NTHREADS_EPOS -gt $nth_max ]] && export NTHREADS_EPOS=$nth_max
     export APRUN_EPOS="$launcher"
 
-elif [ $step = "fv3ic" ]; then
+elif [ $step = "init" ]; then
 
-    export NTHREADS_CHGRES=${nth_chgres:-$npe_node_max}
-    [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max
-    export APRUN_CHGRES="time"
+    export APRUN="$launcher"
 
 elif [ $step = "postsnd" ]; then
 
diff --git a/env/WCOSS_C.env b/env/WCOSS_C.env
index 4885a966a05..bd88a44f31a 100755
--- a/env/WCOSS_C.env
+++ b/env/WCOSS_C.env
@@ -198,11 +198,9 @@ elif [ $step = "epos" ]; then
     [[ $NTHREADS_EPOS -gt $nth_max ]] && export NTHREADS_EPOS=$nth_max
     export APRUN_EPOS="$launcher -j 1 -n $npe_epos -N $npe_node_epos -d $NTHREADS_EPOS -cc depth"
 
-elif [ $step = "fv3ic" ]; then
+elif [ $step = "init" ]; then
 
-    export NTHREADS_CHGRES=${nth_chgres:-$npe_node_max}
-    [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max
-    export APRUN_CHGRES="$launcher -j 1 -n 1 -N 1 -d $NTHREADS_CHGRES -cc depth"
+    export APRUN="$launcher"
 
 elif [ $step = "vrfy" ]; then
 
diff --git a/env/WCOSS_DELL_P3.env b/env/WCOSS_DELL_P3.env
index a028826b931..e2a6989f1d7 100755
--- a/env/WCOSS_DELL_P3.env
+++ b/env/WCOSS_DELL_P3.env
@@ -204,11 +204,9 @@ elif [ $step = "epos" ]; then
     [[ $NTHREADS_EPOS -gt $nth_max ]] && export NTHREADS_EPOS=$nth_max
     export APRUN_EPOS="$launcher ${npe_epos:-$PBS_NP}"
 
-elif [ $step = "fv3ic" ]; then
+elif [ $step = "init" ]; then
 
-    export NTHREADS_CHGRES=${nth_chgres:-$npe_node_max}
-    [[ $NTHREADS_CHGRES -gt $npe_node_max ]] && export NTHREADS_CHGRES=$npe_node_max
-    export APRUN_CHGRES="time"
+    export APRUN="mpirun"
 
 elif [ $step = "postsnd" ]; then
 
diff --git a/jobs/rocoto/fv3ic.sh b/jobs/rocoto/fv3ic.sh
deleted file mode 100755
index b5c2e40b484..00000000000
--- a/jobs/rocoto/fv3ic.sh
+++ /dev/null
@@ -1,69 +0,0 @@
-#!/bin/ksh -x
-
-###############################################################
-## Abstract:
-## Create FV3 initial conditions from GFS intitial conditions
-## RUN_ENVIR : runtime environment (emc | nco)
-## HOMEgfs   : /full/path/to/workflow
-## EXPDIR : /full/path/to/config/files
-## CDATE  : current date (YYYYMMDDHH)
-## CDUMP  : cycle name (gdas / gfs)
-## PDY    : current date (YYYYMMDD)
-## cyc    : current cycle (HH)
-###############################################################
-
-###############################################################
-# Source FV3GFS workflow modules
-. $HOMEgfs/ush/load_fv3gfs_modules.sh
-status=$?
-[[ $status -ne 0 ]] && exit $status
-
-###############################################################
-# Source relevant configs
-configs="base fv3ic"
-for config in $configs; do
-    . $EXPDIR/config.${config}
-    status=$?
-    [[ $status -ne 0 ]] && exit $status
-done
-
-###############################################################
-# Source machine runtime environment
-. $BASE_ENV/${machine}.env fv3ic
-status=$?
-[[ $status -ne 0 ]] && exit $status
-
-# Set component
-export COMPONENT=${COMPONENT:-atmos}
-
-# Temporary runtime directory
-export DATA="$RUNDIR/$CDATE/$CDUMP/fv3ic$$"
-[[ -d $DATA ]] && rm -rf $DATA
-
-# Input GFS initial condition directory
-export INIDIR="$ICSDIR/$CDATE/$CDUMP/$CDUMP.$PDY/$cyc"
-
-# Output FV3 initial condition directory
-export OUTDIR="$ICSDIR/$CDATE/$CDUMP/$CASE/INPUT"
-
-export OMP_NUM_THREADS_CH=$NTHREADS_CHGRES
-export APRUNC=$APRUN_CHGRES
-
-# Call global_chgres_driver.sh
-$HOMEgfs/ush/global_chgres_driver.sh
-status=$?
-if [ $status -ne 0 ]; then
-    echo "global_chgres_driver.sh returned with a non-zero exit code, ABORT!"
-    exit $status
-fi
-
-# Stage the FV3 initial conditions to ROTDIR
-COMOUT="$ROTDIR/$CDUMP.$PDY/$cyc/$COMPONENT"
-[[ ! -d $COMOUT ]] && mkdir -p $COMOUT
-cd $COMOUT || exit 99
-rm -rf INPUT
-$NLN $OUTDIR .
-
-###############################################################
-# Exit cleanly
-exit 0
diff --git a/jobs/rocoto/getic.sh b/jobs/rocoto/getic.sh
index 120e3b9454f..fd07a9ead4f 100755
--- a/jobs/rocoto/getic.sh
+++ b/jobs/rocoto/getic.sh
@@ -36,166 +36,68 @@ status=$?
 ###############################################################
 # Set script and dependency variables
 
-yyyy=$(echo $CDATE | cut -c1-4)
+yy=$(echo $CDATE | cut -c1-4)
 mm=$(echo $CDATE | cut -c5-6)
 dd=$(echo $CDATE | cut -c7-8)
-cyc=${cyc:-$(echo $CDATE | cut -c9-10)}
-
-export COMPONENT=${COMPONENT:-atmos}
-
-###############################################################
-
-target_dir=$ICSDIR/$CDATE/$CDUMP
-mkdir -p $target_dir
-cd $target_dir
-
-# Initialize return code to 0
-rc=1
-
-if [ $ics_from = "opsgfs" ]; then
-
-    # Location of production tarballs on HPSS
-    hpssdir="/NCEPPROD/hpssprod/runhistory/rh$yyyy/$yyyy$mm/$PDY"
-
-    # Handle nemsio and pre-nemsio GFS filenames
-    if [ $CDATE -le "2019061118" ]; then #GFSv14
-        # Add CDUMP.PDY/CYC to target_dir
-        target_dir=$ICSDIR/$CDATE/$CDUMP/${CDUMP}.$yyyy$mm$dd/$cyc
-        mkdir -p $target_dir
-        cd $target_dir
-
-        nfanal=4
-        fanal[1]="./${CDUMP}.t${cyc}z.atmanl.nemsio"
-        fanal[2]="./${CDUMP}.t${cyc}z.sfcanl.nemsio"
-        fanal[3]="./${CDUMP}.t${cyc}z.nstanl.nemsio"
-        fanal[4]="./${CDUMP}.t${cyc}z.pgrbanl"
-        flanal="${fanal[1]} ${fanal[2]} ${fanal[3]} ${fanal[4]}"
-        tarpref="gpfs_hps_nco_ops_com"
-        if [ $CDUMP = "gdas" ]; then
-            tarball="$hpssdir/${tarpref}_gfs_prod_${CDUMP}.${CDATE}.tar"
-        elif [ $CDUMP = "gfs" ]; then
-            tarball="$hpssdir/${tarpref}_gfs_prod_${CDUMP}.${CDATE}.anl.tar"
-        fi
-    else #GFSv15
-        nfanal=2
-        fanal[1]="./${CDUMP}.$yyyy$mm$dd/$cyc/${CDUMP}.t${cyc}z.atmanl.nemsio"
-        fanal[2]="./${CDUMP}.$yyyy$mm$dd/$cyc/${CDUMP}.t${cyc}z.sfcanl.nemsio"
-        flanal="${fanal[1]} ${fanal[2]}"
-        if [ $CDATE -ge "2020022600" ]; then 
-          tarpref="com"
-        else 
-          tarpref="gpfs_dell1_nco_ops_com"
-        fi
-        if [ $CDUMP = "gdas" ]; then
-            tarball="$hpssdir/${tarpref}_gfs_prod_${CDUMP}.${yyyy}${mm}${dd}_${cyc}.${CDUMP}_nemsio.tar"
-        elif [ $CDUMP = "gfs" ]; then
-            tarball="$hpssdir/${tarpref}_gfs_prod_${CDUMP}.${yyyy}${mm}${dd}_${cyc}.${CDUMP}_nemsioa.tar"
-        fi
-    fi
-
-    # First check the COMROOT for files, if present copy over
-    if [ $machine = "WCOSS_C" ]; then
-
-        # Need COMROOT
-        module load prod_envir/1.1.0 >> /dev/null 2>&1
-
-        comdir="$COMROOT/$CDUMP/prod/$CDUMP.$PDY"
-        rc=0
-        for i in `seq 1 $nfanal`; do
-            if [ -f $comdir/${fanal[i]} ]; then
-                $NCP $comdir/${fanal[i]} ${fanal[i]}
-            else
-                rb=1 ; ((rc+=rb))
-            fi
-        done
-
-    fi
-
-    # Get initial conditions from HPSS
-    if [ $rc -ne 0 ]; then
-
-        # check if the tarball exists
-        hsi ls -l $tarball
-        rc=$?
-        if [ $rc -ne 0 ]; then
-            echo "$tarball does not exist and should, ABORT!"
-            exit $rc
-        fi
-        # get the tarball
-        htar -xvf $tarball $flanal
-        rc=$?
-        if [ $rc -ne 0 ]; then
-            echo "untarring $tarball failed, ABORT!"
-            exit $rc
-        fi
-
-        # Move the files to legacy EMC filenames
-        if [ $CDATE -le "2019061118" ]; then #GFSv14
-           for i in `seq 1 $nfanal`; do
-             $NMV ${fanal[i]} ${flanal[i]}
-           done
-        fi
-
-    fi
-
-    # If found, exit out
-    if [ $rc -ne 0 ]; then
-        echo "Unable to obtain operational GFS initial conditions, ABORT!"
-        exit 1
-    fi
-
-elif [ $ics_from = "pargfs" ]; then
-
-    # Add CDUMP.PDY/CYC to target_dir
-    target_dir=$ICSDIR/$CDATE/$CDUMP/${CDUMP}.$yyyy$mm$dd/$cyc
-    mkdir -p $target_dir
-    cd $target_dir
-
-    # Filenames in parallel
-    nfanal=4
-    fanal[1]="gfnanl.${CDUMP}.$CDATE"
-    fanal[2]="sfnanl.${CDUMP}.$CDATE"
-    fanal[3]="nsnanl.${CDUMP}.$CDATE"
-    fanal[4]="pgbanl.${CDUMP}.$CDATE"
-    flanal="${fanal[1]} ${fanal[2]} ${fanal[3]} ${fanal[4]}"
-
-    # Get initial conditions from HPSS from retrospective parallel
-    tarball="$HPSS_PAR_PATH/${CDATE}${CDUMP}.tar"
-
-    # check if the tarball exists
-    hsi ls -l $tarball
-    rc=$?
-    if [ $rc -ne 0 ]; then
-        echo "$tarball does not exist and should, ABORT!"
-        exit $rc
-    fi
-    # get the tarball
-    htar -xvf $tarball $flanal
-    rc=$?
-    if [ $rc -ne 0 ]; then
-        echo "untarring $tarball failed, ABORT!"
-        exit $rc
-    fi
-
-    # If found, exit out
-    if [ $rc -ne 0 ]; then
-        echo "Unable to obtain parallel GFS initial conditions, ABORT!"
-        exit 1
-    fi
-
-else
+hh=${cyc:-$(echo $CDATE | cut -c9-10)}
+
+EXTRACT_DIR=${PTMP}/gdas.init_${CDATE}/input
+OUTDIR=${PTMP}/gdas.init_${CDATE}/output
+PRODHPSSDIR=/NCEPPROD/hpssprod/runhistory/rh${yy}/${yy}${mm}/${yy}${mm}${dd}
+
+COMPONENT="atmos"
+
+gfs_ver=v16
+GETICSH=${GDASINIT_DIR}/get_v16.data.sh
+
+# No ENKF data prior to 2012/05/21/00z
+if [ $yy$mm$dd$hh -lt 2012052100 ]; then
+  set +x
+  echo FATAL ERROR: SCRIPTS DO NOT SUPPORT OLD GFS DATA
+  exit 2
+elif [ $yy$mm$dd$hh -lt 2016051000 ]; then
+  gfs_ver=v12
+  GETICSH=${GDASINIT_DIR}/get_pre-v14.data.sh
+elif [ $yy$mm$dd$hh -lt 2017072000 ]; then
+  gfs_ver=v13
+  GETICSH=${GDASINIT_DIR}/get_pre-v14.data.sh
+elif [ $yy$mm$dd$hh -lt 2019061200 ]; then
+  gfs_ver=v14
+  GETICSH=${GDASINIT_DIR}/get_${gfs_ver}.data.sh
+elif [ $yy$mm$dd$hh -lt 2021020300 ]; then
+  gfs_ver=v15
+  GETICSH=${GDASINIT_DIR}/get_${gfs_ver}.data.sh
+fi
 
-    echo "ics_from = $ics_from is not supported, ABORT!"
-    exit 1
+export EXTRACT_DIR yy mm dd hh UFS_DIR OUTDIR CRES_HIRES CRES_ENKF
+export LEVS gfs_ver
 
-fi
-###############################################################
+# Run get data script
+if [ ! -d $EXTRACT_DIR ]; then mkdir -p $EXTRACT_DIR ; fi
+sh ${GETICSH} ${CDUMP}
+status=$?
+[[ $status -ne 0 ]] && exit $status
 
-# Copy pgbanl file to COMROT for verification - GFSv14 only
-if [ $CDATE -le "2019061118" ]; then #GFSv14
-  COMROT=$ROTDIR/${CDUMP}.$PDY/$cyc/$COMPONENT
-  [[ ! -d $COMROT ]] && mkdir -p $COMROT
-  $NCP ${fanal[4]} $COMROT/${CDUMP}.t${cyc}z.pgrbanl
+# Copy pgbanl file to ROTDIR for verification/archival - v14+
+cd $EXTRACT_DIR
+OUTDIR2=${ROTDIR}/gfs.${yy}${mm}${dd}/${hh}/${COMPONENT}
+if [ ! -d ${OUTDIR2} ]; then mkdir -p ${OUTDIR2} ; fi
+if [ $gfs_ver = v14 ]; then
+  for grid in 0p25 0p50 1p00
+  do
+    tarball=gpfs_hps_nco_ops_com_gfs_prod_gfs.${yy}${mm}${dd}_${hh}.pgrb2_${grid}.tar
+    file=gfs.t${hh}z.pgrb2.${grid}.anl
+    htar -xvf ${PRODHPSSDIR}/${tarball} ./gfs.${yy}${mm}${dd}/${hh}/${file}
+    mv ${EXTRACT_DIR}/gfs.${yy}${mm}${dd}/${hh}/${file} ${OUTDIR2}/${file}
+  done
+elif [ $gfs_ver = v15 ]; then
+  tarball=com_gfs_prod_gfs.${yy}${mm}${dd}_${hh}.gfs_pgrb2.tar  
+  for grid in 0p25 0p50 1p00
+  do
+    file=gfs.t${hh}z.pgrb2.${grid}.anl
+    htar -xvf ${PRODHPSSDIR}/${tarball} ./gfs.${yy}${mm}${dd}/${hh}/${file}
+    mv ${EXTRACT_DIR}/gfs.${yy}${mm}${dd}/${hh}/${file} ${OUTDIR2}/${file}
+  done
 fi
 
 ###############################################################
diff --git a/jobs/rocoto/init.sh b/jobs/rocoto/init.sh
new file mode 100755
index 00000000000..c36addb2ed5
--- /dev/null
+++ b/jobs/rocoto/init.sh
@@ -0,0 +1,81 @@
+#!/bin/ksh -x
+
+###############################################################
+## Abstract:
+## Get GFS intitial conditions
+## RUN_ENVIR : runtime environment (emc | nco)
+## HOMEgfs   : /full/path/to/workflow
+## EXPDIR : /full/path/to/config/files
+## CDATE  : current date (YYYYMMDDHH)
+## CDUMP  : cycle name (gdas / gfs)
+## PDY    : current date (YYYYMMDD)
+## cyc    : current cycle (HH)
+###############################################################
+
+###############################################################
+# Source FV3GFS workflow modules
+. $HOMEgfs/ush/load_fv3gfs_modules.sh
+status=$?
+[[ $status -ne 0 ]] && exit $status
+
+###############################################################
+# Source relevant configs
+configs="base init"
+for config in $configs; do
+    . $EXPDIR/config.${config}
+    status=$?
+    [[ $status -ne 0 ]] && exit $status
+done
+
+###############################################################
+# Source machine runtime environment
+. $BASE_ENV/${machine}.env init
+status=$?
+[[ $status -ne 0 ]] && exit $status
+
+###############################################################
+# Set script and dependency variables
+
+yy=$(echo $CDATE | cut -c1-4)
+mm=$(echo $CDATE | cut -c5-6)
+dd=$(echo $CDATE | cut -c7-8)
+hh=${cyc:-$(echo $CDATE | cut -c9-10)}
+
+EXTRACT_DIR=${PTMP}/gdas.init_${CDATE}/input
+WORKDIR=${PTMP}/gdas.init_${CDATE}/output
+OUTDIR=${ROTDIR}
+
+gfs_ver=v16
+RUNICSH=${GDASINIT_DIR}/run_v16.chgres.sh
+
+# No ENKF data prior to 2012/05/21/00z
+if [ $yy$mm$dd$hh -lt 2012052100 ]; then
+  set +x
+  echo FATAL ERROR: SCRIPTS DO NOT SUPPORT OLD GFS DATA
+  exit 2
+elif [ $yy$mm$dd$hh -lt 2016051000 ]; then
+  gfs_ver=v12
+  RUNICSH=${GDASINIT_DIR}/run_pre-v14.chgres.sh
+elif [ $yy$mm$dd$hh -lt 2017072000 ]; then
+  gfs_ver=v13
+  RUNICSH=${GDASINIT_DIR}/run_pre-v14.chgres.sh
+elif [ $yy$mm$dd$hh -lt 2019061200 ]; then
+  gfs_ver=v14
+  RUNICSH=${GDASINIT_DIR}/run_${gfs_ver}.chgres.sh
+elif [ $yy$mm$dd$hh -lt 2021020300 ]; then
+  gfs_ver=v15
+  RUNICSH=${GDASINIT_DIR}/run_${gfs_ver}.chgres.gfs.sh
+fi
+
+export EXTRACT_DIR yy mm dd hh UFS_DIR OUTDIR CRES_HIRES CRES_ENKF
+export LEVS gfs_ver
+
+# Run chgres_cube
+if [ ! -d $OUTDIR ]; then mkdir -p $OUTDIR ; fi
+sh ${RUNICSH} ${CDUMP}
+status=$?
+[[ $status -ne 0 ]] && exit $status
+
+###############################################################
+# Exit out cleanly
+exit 0
diff --git a/parm/config/config.getic b/parm/config/config.getic
index a7544542641..64f7159cc98 100755
--- a/parm/config/config.getic
+++ b/parm/config/config.getic
@@ -8,13 +8,9 @@ echo "BEGIN: config.getic"
 # Get task specific resources
 . $EXPDIR/config.resources getic
 
-# We should just be supporting the OPSGFS only
-export ics_from="opsgfs" # initial conditions from opsgfs or pargfs
+UFS_DIR=${HOMEgfs}/sorc/ufs_utils.fd
+GDASINIT_DIR=${UFS_DIR}/util/gdas_init
 
-# Provide a parallel experiment name and path to HPSS archive
-if [ $ics_from = "pargfs" ]; then
-    export parexp="prnemsrn"
-    export HPSS_PAR_PATH="/5year/NCEPDEV/emc-global/emc.glopara/WCOSS_C/$parexp"
-fi
+EXTRACT_DATA="yes"
 
 echo "END: config.getic"
diff --git a/parm/config/config.init b/parm/config/config.init
new file mode 100755
index 00000000000..f9efbde888b
--- /dev/null
+++ b/parm/config/config.init
@@ -0,0 +1,19 @@
+#!/bin/ksh -x
+
+########## config.init ##########
+# Prepare initial conditions 
+
+echo "BEGIN: config.init"
+
+# Get task specific resources
+. $EXPDIR/config.resources init
+
+UFS_DIR=${HOMEgfs}/sorc/ufs_utils.fd
+GDASINIT_DIR=${UFS_DIR}/util/gdas_init
+
+RUN_CHGRES="yes"
+
+CRES_HIRES=$CASE
+CRES_ENKF=$CASE_ENKF
+
+echo "END: config.init"
diff --git a/parm/config/config.resources b/parm/config/config.resources
index 3d996d96b65..d62424aa324 100755
--- a/parm/config/config.resources
+++ b/parm/config/config.resources
@@ -262,6 +262,13 @@ elif [ $step = "echgres" ]; then
     export nth_echgres=$npe_node_max
     export npe_node_echgres=1
 
+elif [ $step = "init" ]; then
+
+    export wtime_init="00:15:00"
+    export npe_init=24
+    export nth_init=1
+    export npe_node_init=6
+
 elif [ $step = "arch" -o $step = "earc" -o $step = "getic" ]; then
 
     eval "export wtime_$step='06:00:00'"
diff --git a/sorc/build_ufs_utils.sh b/sorc/build_ufs_utils.sh
index 292145d7645..0a22135a155 100755
--- a/sorc/build_ufs_utils.sh
+++ b/sorc/build_ufs_utils.sh
@@ -7,9 +7,9 @@ cwd=`pwd`
 if [ $target = wcoss_dell_p3 ]; then target=dell; fi
 if [ $target = wcoss_cray ]; then target=cray; fi
 
-cd ufs_utils.fd/sorc
+cd ufs_utils.fd
 
-./build_all_ufs_utils.sh
+./build_all.sh
 
 exit
 
diff --git a/sorc/link_fv3gfs.sh b/sorc/link_fv3gfs.sh
index e5f0f08ad44..403c7364cae 100755
--- a/sorc/link_fv3gfs.sh
+++ b/sorc/link_fv3gfs.sh
@@ -49,7 +49,7 @@ for dir in fix_am fix_fv3_gmted2010 fix_gldas fix_orog fix_verif fix_wave_gfs ;
 done
 
 if [ -d ${pwd}/ufs_utils.fd ]; then
-  cd ${pwd}/ufs_utils.fd/sorc
+  cd ${pwd}/ufs_utils.fd/fix
   ./link_fixdirs.sh $RUN_ENVIR $machine
 fi
 
diff --git a/ush/rocoto/setup_workflow_fcstonly.py b/ush/rocoto/setup_workflow_fcstonly.py
index c7021f904e5..795602204eb 100755
--- a/ush/rocoto/setup_workflow_fcstonly.py
+++ b/ush/rocoto/setup_workflow_fcstonly.py
@@ -27,8 +27,7 @@
 import rocoto
 import workflow_utils as wfu
 
-#taskplan = ['getic', 'fv3ic', 'waveinit', 'waveprep', 'fcst', 'post', 'wavepostsbs', 'wavegempak', 'waveawipsbulls', 'waveawipsgridded', 'wavepost', 'wavestat', 'wafs', 'wafsgrib2', 'wafsblending', 'wafsgcip', 'wafsgrib20p25', 'wafsblending0p25', 'vrfy', 'metp', 'arch']
-taskplan = ['getic', 'fv3ic', 'waveinit', 'waveprep', 'fcst', 'post', 'wavepostsbs', 'wavepostbndpnt', 'wavepostpnt', 'wavegempak', 'waveawipsbulls', 'waveawipsgridded', 'wafs', 'wafsgrib2', 'wafsblending', 'wafsgcip', 'wafsgrib20p25', 'wafsblending0p25', 'vrfy', 'metp', 'arch']
+taskplan = ['getic', 'init', 'waveinit', 'waveprep', 'fcst', 'post', 'wavepostsbs', 'wavepostbndpnt', 'wavepostpnt', 'wavegempak', 'waveawipsbulls', 'waveawipsgridded', 'wafs', 'wafsgrib2', 'wafsblending', 'wafsgcip', 'wafsgrib20p25', 'wafsblending0p25', 'vrfy', 'metp', 'arch']
 
 def main():
     parser = ArgumentParser(description='Setup XML workflow and CRONTAB for a forecast only experiment.', formatter_class=ArgumentDefaultsHelpFormatter)
@@ -244,73 +243,41 @@ def get_workflow(dict_configs, cdump='gdas'):
 
     tasks = []
 
-    # getics
+    # getic
     deps = []
-    data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CDUMP;.@Y@m@d/@H/siganl.&CDUMP;.@Y@m@d@H'
+    data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/INPUT/sfc_data.tile6.nc'
     dep_dict = {'type':'data', 'data':data}
     deps.append(rocoto.add_dependency(dep_dict))
-    data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CDUMP;.@Y@m@d/@H/&CDUMP;.t@Hz.sanl'
-    dep_dict = {'type':'data', 'data':data}
-    deps.append(rocoto.add_dependency(dep_dict))
-    data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CDUMP;.@Y@m@d/@H/gfnanl.&CDUMP;.@Y@m@d@H'
-    dep_dict = {'type':'data', 'data':data}
-    deps.append(rocoto.add_dependency(dep_dict))
-    data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CDUMP;.@Y@m@d/@H/&CDUMP;.t@Hz.atmanl.nemsio'
-    dep_dict = {'type':'data', 'data':data}
-    deps.append(rocoto.add_dependency(dep_dict))
-    dependencies = rocoto.create_dependency(dep_condition='or', dep=deps)
-
-    deps = []
-    data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CASE;/INPUT/gfs_data.tile6.nc'
-    dep_dict = {'type':'data', 'data':data}
-    deps.append(rocoto.add_dependency(dep_dict))
-    data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CASE;/INPUT/sfc_data.tile6.nc'
+    data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/RESTART/@Y@m@d.@H0000.sfcanl_data.tile6.nc'
     dep_dict = {'type':'data', 'data':data}
     deps.append(rocoto.add_dependency(dep_dict))
-    deps = rocoto.create_dependency(dep_condition='and', dep=deps)
-    dependencies2 = rocoto.create_dependency(dep_condition='not', dep=deps)
-
-    deps = []
-    deps.append(dependencies)
-    deps.append(dependencies2)
-    dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
+    dependencies = rocoto.create_dependency(dep_condition='nor', dep=deps)
 
     task = wfu.create_wf_task('getic', cdump=cdump, envar=envars, dependency=dependencies)
     tasks.append(task)
     tasks.append('\n')
 
-    # chgres fv3ic
+    # init - chgres_cube
     deps = []
-    data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CDUMP;.@Y@m@d/@H/siganl.&CDUMP;.@Y@m@d@H'
+    data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/INPUT/sfc_data.tile6.nc'
     dep_dict = {'type':'data', 'data':data}
     deps.append(rocoto.add_dependency(dep_dict))
-    data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CDUMP;.@Y@m@d/@H/&CDUMP;.t@Hz.sanl'
-    dep_dict = {'type':'data', 'data':data}
-    deps.append(rocoto.add_dependency(dep_dict))
-    data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CDUMP;.@Y@m@d/@H/gfnanl.&CDUMP;.@Y@m@d@H'
-    dep_dict = {'type':'data', 'data':data}
-    deps.append(rocoto.add_dependency(dep_dict))
-    data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CDUMP;.@Y@m@d/@H/&CDUMP;.t@Hz.atmanl.nemsio'
+    data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/RESTART/@Y@m@d.@H0000.sfcanl_data.tile6.nc'
     dep_dict = {'type':'data', 'data':data}
     deps.append(rocoto.add_dependency(dep_dict))
-    dependencies = rocoto.create_dependency(dep_condition='or', dep=deps)
+    dependencies = rocoto.create_dependency(dep_condition='nor', dep=deps)
 
     deps = []
-    data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CASE;/INPUT/gfs_data.tile6.nc'
-    dep_dict = {'type':'data', 'data':data}
-    deps.append(rocoto.add_dependency(dep_dict))
-    data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CASE;/INPUT/sfc_data.tile6.nc'
-    dep_dict = {'type':'data', 'data':data}
+    dep_dict = {'type': 'task', 'name': '%sgetic' % cdump}
     deps.append(rocoto.add_dependency(dep_dict))
-    deps = rocoto.create_dependency(dep_condition='and', dep=deps)
-    dependencies2 = rocoto.create_dependency(dep_condition='not', dep=deps)
+    dependencies2 = rocoto.create_dependency(dep=deps)
 
     deps = []
     deps.append(dependencies)
     deps.append(dependencies2)
     dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
 
-    task = wfu.create_wf_task('fv3ic', cdump=cdump, envar=envars, dependency=dependencies)
+    task = wfu.create_wf_task('init', cdump=cdump, envar=envars, dependency=dependencies)
     tasks.append(task)
     tasks.append('\n')
 
@@ -332,7 +299,7 @@ def get_workflow(dict_configs, cdump='gdas'):
 
     # fcst
     deps = []
-    data = '&ICSDIR;/@Y@m@d@H/&CDUMP;/&CASE;/INPUT/sfc_data.tile6.nc'
+    data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/INPUT/sfc_data.tile6.nc'
     dep_dict = {'type':'data', 'data':data}
     deps.append(rocoto.add_dependency(dep_dict))
     data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/RESTART/@Y@m@d.@H0000.sfcanl_data.tile6.nc'

From b251164cb3adfab274d7f7ea15c30eaff5eabf79 Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Fri, 29 Jan 2021 19:45:44 +0000
Subject: [PATCH 008/125] Add missing component subfolder to fcst dependency

The free-forecast mode fcst job data dependency file path was missing
the component "atmos" subfolder. Added and tested in experiment on
Hera.

Refs: #178
---
 ush/rocoto/setup_workflow_fcstonly.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ush/rocoto/setup_workflow_fcstonly.py b/ush/rocoto/setup_workflow_fcstonly.py
index 795602204eb..1bd02a8504a 100755
--- a/ush/rocoto/setup_workflow_fcstonly.py
+++ b/ush/rocoto/setup_workflow_fcstonly.py
@@ -299,10 +299,10 @@ def get_workflow(dict_configs, cdump='gdas'):
 
     # fcst
     deps = []
-    data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/INPUT/sfc_data.tile6.nc'
+    data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/INPUT/sfc_data.tile6.nc'
     dep_dict = {'type':'data', 'data':data}
     deps.append(rocoto.add_dependency(dep_dict))
-    data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/RESTART/@Y@m@d.@H0000.sfcanl_data.tile6.nc'
+    data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/RESTART/@Y@m@d.@H0000.sfcanl_data.tile6.nc'
     dep_dict = {'type':'data', 'data':data}
     deps.append(rocoto.add_dependency(dep_dict))
     dependencies = rocoto.create_dependency(dep_condition='or', dep=deps)

From af971d76b2c62aa27d7e989877805a1950e52982 Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Mon, 1 Feb 2021 17:40:21 +0000
Subject: [PATCH 009/125] Updates for hpc-stack

- Remove unneeded ignores from .gitignore
- Update Externals.cfg and checkout.sh for component stack versions
- Update link_fv3gfs.sh for changed component links
- Further Hera module updates for hpc-stack

Refs: #164
---
 .gitignore                                    | 11 ++-------
 Externals.cfg                                 |  6 ++---
 modulefiles/fv3gfs/enkf_chgres_recenter.hera  |  3 +++
 .../fv3gfs/enkf_chgres_recenter_nc.hera       | 21 +++++++++--------
 modulefiles/fv3gfs/gaussian_sfcanl.hera       | 23 ++++++++++---------
 modulefiles/module_base.hera                  |  2 +-
 modulefiles/modulefile.fv3nc2nemsio.hera      | 16 ++++++++-----
 sorc/build_fv3.sh                             |  9 ++++----
 sorc/checkout.sh                              | 14 +++--------
 sorc/link_fv3gfs.sh                           |  9 ++++----
 10 files changed, 54 insertions(+), 60 deletions(-)

diff --git a/.gitignore b/.gitignore
index fb64c9dbc48..35a72621fae 100644
--- a/.gitignore
+++ b/.gitignore
@@ -31,6 +31,7 @@ sorc/logs
 sorc/fv3gfs.fd
 sorc/gfs_post.fd
 sorc/gfs_wafs.fd
+sorc/gldas.fd
 sorc/gsi.fd
 sorc/ufs_utils.fd
 sorc/verif-global.fd
@@ -42,14 +43,12 @@ sorc/calc_increment_ens.fd
 sorc/calc_increment_ens_ncio.fd
 sorc/emcsfc_ice_blend.fd
 sorc/emcsfc_snow2mdl.fd
-sorc/filter_topo.fd
 sorc/fregrid.fd
 sorc/gdas2gldas.fd
 sorc/getsfcensmeanp.fd
 sorc/getsigensmeanp_smooth.fd
 sorc/getsigensstatp.fd
 sorc/gfs_ncep_post.fd
-sorc/gldas.fd
 sorc/gldas2gdas.fd
 sorc/gldas_forcing.fd
 sorc/gldas_model.fd
@@ -62,13 +61,8 @@ sorc/global_gsi.fd
 sorc/interp_inc.fd
 sorc/make_hgrid.fd
 sorc/make_solo_mosaic.fd
-sorc/mkgfsnemsioctl.fd
 sorc/ncdiag_cat.fd
-sorc/nemsio_chgdate.fd
-sorc/nemsio_get.fd
-sorc/nemsio_read.fd
 sorc/nst_tf_chg.fd
-sorc/orog.fd
 sorc/oznmon_horiz.fd
 sorc/oznmon_time.fd
 sorc/radmon_angle.fd
@@ -76,7 +70,6 @@ sorc/radmon_bcoef.fd
 sorc/radmon_bcor.fd
 sorc/radmon_time.fd
 sorc/recentersigp.fd
-sorc/shave.fd
 sorc/wafs_awc_wafavn.fd
 sorc/wafs_blending.fd
 sorc/wafs_cnvgrib2.fd
@@ -140,10 +133,10 @@ scripts/exglobal_diag.sh
 # ush symlinks
 ush/calcanl_gfs.py
 ush/calcinc_gfs.py
+ush/chgres_cube.sh
 ush/emcsfc_ice_blend.sh
 ush/emcsfc_snow.sh
 ush/fix_precip.sh
-ush/fv3gfs_chgres.sh
 ush/fv3gfs_downstream_nems.sh
 ush/fv3gfs_driver_grid.sh
 ush/fv3gfs_dwn_nems.sh
diff --git a/Externals.cfg b/Externals.cfg
index d846088fff3..da08a0c762c 100644
--- a/Externals.cfg
+++ b/Externals.cfg
@@ -1,7 +1,7 @@
 # External sub-modules of global-workflow
 
 [FV3GFS]
-tag = GFS.v16.0.14
+hash = 06bc77ba3640358b54b6ff03f542e2ce0d3242be
 local_path = sorc/fv3gfs.fd
 repo_url = https://github.com/ufs-community/ufs-weather-model.git
 protocol = git
@@ -15,7 +15,7 @@ protocol = git
 required = True
 
 [GLDAS]
-tag = gldas_gfsv16_release.v1.13.0
+tag = gldas_gfsv16_release.v1.14.0
 local_path = sorc/gldas.fd
 repo_url = https://github.com/NOAA-EMC/GLDAS.git
 protocol = git
@@ -30,7 +30,7 @@ protocol = git
 required = True
 
 [UFS_UTILS]
-tag = ops-gfsv16.0.0
+#tag = ops-gfsv16.0.0
 local_path = sorc/ufs_utils.fd
 repo_url = https://github.com/NOAA-EMC/UFS_UTILS.git
 protocol = git
diff --git a/modulefiles/fv3gfs/enkf_chgres_recenter.hera b/modulefiles/fv3gfs/enkf_chgres_recenter.hera
index c5fe744ca16..d4e14803f80 100644
--- a/modulefiles/fv3gfs/enkf_chgres_recenter.hera
+++ b/modulefiles/fv3gfs/enkf_chgres_recenter.hera
@@ -13,4 +13,7 @@ module load w3nco/2.4.1
 module load ip/3.3.3
 module load sp/2.3.3
 
+module load hdf5/1.10.6
+module load netcdf/4.7.4
+
 export FC=ifort
diff --git a/modulefiles/fv3gfs/enkf_chgres_recenter_nc.hera b/modulefiles/fv3gfs/enkf_chgres_recenter_nc.hera
index a3bc77c352a..b12233fb1ac 100644
--- a/modulefiles/fv3gfs/enkf_chgres_recenter_nc.hera
+++ b/modulefiles/fv3gfs/enkf_chgres_recenter_nc.hera
@@ -1,15 +1,18 @@
 #%Module#####################################################
-## enkf_chgres_recenter component - hera
+## enkf_chgres_recenter_nc component - hera
 #############################################################
 
-module use -a /scratch2/NCEPDEV/nwprod/NCEPLIBS/modulefiles
-module load bacio/2.0.2
-module load w3nco/2.0.6
-module load ip/3.0.1
-module load sp/2.0.2
+module use /scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack
+module load hpc/1.1.0
+module load hpc-intel/18.0.5.274
+module load hpc-impi/2018.0.4
 
-module use /scratch1/NCEPDEV/nems/emc.nemspara/soft/modulefiles/
-module load netcdf_parallel/4.7.4.release
-module load hdf5_parallel/1.10.6.release
+module load bacio/2.4.1
+module load w3nco/2.4.1
+module load ip/3.3.3
+module load sp/2.3.3
+
+module load hdf5/1.10.6
+module load netcdf/4.7.4
 
 export FC=ifort
diff --git a/modulefiles/fv3gfs/gaussian_sfcanl.hera b/modulefiles/fv3gfs/gaussian_sfcanl.hera
index 6663b6cb55a..e86cd855409 100644
--- a/modulefiles/fv3gfs/gaussian_sfcanl.hera
+++ b/modulefiles/fv3gfs/gaussian_sfcanl.hera
@@ -2,17 +2,18 @@
 ## gaussian_sfcanl build module for Hera
 #############################################################
 
-module use -a /scratch2/NCEPDEV/nwprod/NCEPLIBS/modulefiles
-module load w3nco/2.0.6
-module load bacio/2.0.2
-module load nemsio/2.2.3
-module load sp/2.0.2
-module use /scratch1/NCEPDEV/nems/emc.nemspara/soft/modulefiles/
-module load netcdf_parallel/4.7.4.release
-module load hdf5_parallel/1.10.6.release
+module use /scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack
+module load hpc/1.1.0
+module load hpc-intel/18.0.5.274
+module load hpc-impi/2018.0.4
 
+module load w3nco/2.4.1
+module load bacio/2.4.1
+module load nemsio/2.5.2
+module load sp/2.3.3
 
-export NETCDF_INCLUDE="-I${NETCDF}/include"
-export NETCDF_LDFLAGS_F="-L${NETCDF}/lib -lnetcdf -lnetcdff -L${HDF5}/lib -lhdf5 -lhdf5_fortran"
+module load netcdf/4.7.4
+module load hdf5/1.10.6
 
-#export FCOMP=$FCOMP
+export NETCDF_INCLUDE="-I${NETCDF}/include"
+export NETCDF_LDFLAGS_F="-L${NETCDF}/lib -lnetcdf -lnetcdff -L${HDF5_LIBRARIES} -lhdf5 -lhdf5_fortran"
diff --git a/modulefiles/module_base.hera b/modulefiles/module_base.hera
index ddec91b55b1..ae4bd4ac2a0 100644
--- a/modulefiles/module_base.hera
+++ b/modulefiles/module_base.hera
@@ -17,8 +17,8 @@ module load gempak/7.4.2
 module load prod_util/1.2.2
 module load grib_util/1.2.2
 module load g2tmpl/1.9.1
-module load crtm/2.3.0
 
+module load crtm/2.3.0
 setenv CRTM_FIX /scratch2/NCEPDEV/nwprod/NCEPLIBS/fix/crtm_v2.3.0
 
 module load netcdf/4.7.4
diff --git a/modulefiles/modulefile.fv3nc2nemsio.hera b/modulefiles/modulefile.fv3nc2nemsio.hera
index c18bab3def7..91e896ce861 100644
--- a/modulefiles/modulefile.fv3nc2nemsio.hera
+++ b/modulefiles/modulefile.fv3nc2nemsio.hera
@@ -2,12 +2,16 @@
 ## Module file for fv3nc2nemsio
 #############################################################
 
-module use -a /scratch2/NCEPDEV/nwprod/NCEPLIBS/modulefiles
-module load netcdf/4.7.0
-module load hdf5/1.10.5
-module load bacio/2.0.2
-module load nemsio/2.2.3
-module load w3nco/2.0.6
+module use /scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack
+module load hpc/1.1.0
+module load hpc-intel/18.0.5.274
+module load hpc-impi/2018.0.4
+
+module load netcdf/4.7.4
+module load hdf5/1.10.6
+module load bacio/2.4.1
+module load nemsio/2.5.2
+module load w3nco/2.4.1
 
 export FCMP="ifort"
 export FFLAGS="-g -O2 -traceback"
diff --git a/sorc/build_fv3.sh b/sorc/build_fv3.sh
index e812f7e0cac..a4f99fa411b 100755
--- a/sorc/build_fv3.sh
+++ b/sorc/build_fv3.sh
@@ -23,10 +23,9 @@ cd fv3gfs.fd/
 FV3=$( pwd -P )/FV3
 cd tests/
 
-if [ ${RUN_CCPP:-${1:-"NO"}} = "NO" ]; then
- ./compile.sh "$FV3" "$target" "WW3=Y 32BIT=Y" 1
- mv -f fv3_1.exe ../NEMS/exe/global_fv3gfs.x
-else
+# GFSv16 with waves
+# ./compile.sh "$FV3" "$target" "WW3=Y 32BIT=Y" 1
+# mv -f fv3_1.exe ../NEMS/exe/global_fv3gfs.x
+
  ./compile.sh "$target" "CCPP=Y 32BIT=Y SUITES=FV3_GFS_v15,FV3_GFS_v16beta" 2 NO NO
  mv -f fv3_2.exe ../NEMS/exe/global_fv3gfs.x
-fi
diff --git a/sorc/checkout.sh b/sorc/checkout.sh
index b5da9b5f74c..e2d2916e46b 100755
--- a/sorc/checkout.sh
+++ b/sorc/checkout.sh
@@ -2,17 +2,13 @@
 #set -xue
 set -x
 
-while getopts "oc" option;
+while getopts "o" option;
 do
  case $option in
   o)
    echo "Received -o flag for optional checkout of GTG, will check out GTG with EMC_post"
    checkout_gtg="YES"
    ;;
-  c)
-   echo "Received -c flag, check out ufs-weather-model develop branch with CCPP physics"  
-   run_ccpp="YES"
-   ;;
   :)
    echo "option -$OPTARG needs an argument"
    ;;
@@ -31,11 +27,7 @@ if [[ ! -d fv3gfs.fd ]] ; then
     rm -f ${topdir}/checkout-fv3gfs.log
     git clone https://github.com/ufs-community/ufs-weather-model fv3gfs.fd >> ${topdir}/checkout-fv3gfs.log 2>&1
     cd fv3gfs.fd
-    if [ ${run_ccpp:-"NO"} = "NO" ]; then
-     git checkout GFS.v16.0.14
-    else
-     git checkout b771e5be7e35eaea5ee7f762d644afccab019ed3
-    fi
+    git checkout 06bc77ba3640358b54b6ff03f542e2ce0d3242be
     git submodule update --init --recursive
     cd ${topdir}
 else
@@ -59,7 +51,7 @@ if [[ ! -d gldas.fd ]] ; then
     rm -f ${topdir}/checkout-gldas.log
     git clone https://github.com/NOAA-EMC/GLDAS.git gldas.fd >> ${topdir}/checkout-gldas.fd.log 2>&1
     cd gldas.fd
-    git checkout gldas_gfsv16_release.v1.13.0
+    git checkout gldas_gfsv16_release.v1.14.0
     cd ${topdir}
 else
     echo 'Skip.  Directory gldas.fd already exists.'
diff --git a/sorc/link_fv3gfs.sh b/sorc/link_fv3gfs.sh
index 15fe5225227..6c611b77a96 100755
--- a/sorc/link_fv3gfs.sh
+++ b/sorc/link_fv3gfs.sh
@@ -78,7 +78,7 @@ cd ${pwd}/../ush                ||exit 8
     done
     for file in emcsfc_ice_blend.sh  fv3gfs_driver_grid.sh  fv3gfs_make_orog.sh  global_cycle_driver.sh \
         emcsfc_snow.sh  fv3gfs_filter_topo.sh  global_chgres_driver.sh  global_cycle.sh \
-        fv3gfs_chgres.sh  fv3gfs_make_grid.sh  global_chgres.sh  ; do
+        chgres_cube.sh  fv3gfs_make_grid.sh  global_chgres.sh  ; do
         $LINK ../sorc/ufs_utils.fd/ush/$file                  .
     done
     for file in gldas_archive.sh  gldas_forcing.sh gldas_get_data.sh  gldas_process_data.sh gldas_liscrd.sh  gldas_post.sh ; do
@@ -295,13 +295,12 @@ cd ${pwd}/../sorc   ||   exit 8
 
     $SLINK gfs_post.fd/sorc/ncep_post.fd                                                   gfs_ncep_post.fd
 
-    $SLINK ufs_utils.fd/sorc/fre-nctools.fd/tools/shave.fd                                 shave.fd
-    for prog in filter_topo fregrid make_hgrid make_solo_mosaic ; do
+    for prog in fregrid make_hgrid make_solo_mosaic ; do
         $SLINK ufs_utils.fd/sorc/fre-nctools.fd/tools/$prog                                ${prog}.fd                                
     done
-    for prog in  global_cycle.fd   nemsio_read.fd  nemsio_chgdate.fd \
+    for prog in  global_cycle.fd \
         emcsfc_ice_blend.fd  nst_tf_chg.fd \
-        emcsfc_snow2mdl.fd   global_chgres.fd  nemsio_get.fd    orog.fd ;do
+        emcsfc_snow2mdl.fd   global_chgres.fd ;do
         $SLINK ufs_utils.fd/sorc/$prog                                                     $prog
     done
 

From abdc1334e3c58d40977b20fb7acabd4c14a80cf6 Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Wed, 3 Feb 2021 19:28:44 +0000
Subject: [PATCH 010/125] Further init updates for free-forecast mode

- move COMPONENT setting from getic to init job
- cleanup how getic job sets tarball paths and pulls pgbanl files
- move second step of pgbanl pull to init job for consistency
- add MODE variable to config.base and its definition to setup scripts
- add MODE setting to config.base if-block that adjusts IAU variables
- add missing COMPONENT subfolder to data dependencies for getic and
init jobs in setup_workflow_fcstonly.py
- remove unneeded commented out wavestat job from setup_workflow_fcstonly.py

Refs: #178
---
 jobs/rocoto/getic.sh                  | 22 +++++-----------------
 jobs/rocoto/init.sh                   | 15 +++++++++++++++
 parm/config/config.base.emc.dyn       |  5 ++++-
 ush/rocoto/setup_expt.py              |  1 +
 ush/rocoto/setup_expt_fcstonly.py     |  1 +
 ush/rocoto/setup_workflow_fcstonly.py | 18 ++++--------------
 6 files changed, 30 insertions(+), 32 deletions(-)

diff --git a/jobs/rocoto/getic.sh b/jobs/rocoto/getic.sh
index fd07a9ead4f..b3cb741b33e 100755
--- a/jobs/rocoto/getic.sh
+++ b/jobs/rocoto/getic.sh
@@ -45,8 +45,6 @@ EXTRACT_DIR=${PTMP}/gdas.init_${CDATE}/input
 OUTDIR=${PTMP}/gdas.init_${CDATE}/output
 PRODHPSSDIR=/NCEPPROD/hpssprod/runhistory/rh${yy}/${yy}${mm}/${yy}${mm}${dd}
 
-COMPONENT="atmos"
-
 gfs_ver=v16
 GETICSH=${GDASINIT_DIR}/get_v16.data.sh
 
@@ -64,9 +62,11 @@ elif [ $yy$mm$dd$hh -lt 2017072000 ]; then
 elif [ $yy$mm$dd$hh -lt 2019061200 ]; then
   gfs_ver=v14
   GETICSH=${GDASINIT_DIR}/get_${gfs_ver}.data.sh
+  tarball=gpfs_hps_nco_ops_com_gfs_prod_gfs.${yy}${mm}${dd}_${hh}.pgrb2_${grid}.tar
 elif [ $yy$mm$dd$hh -lt 2021020300 ]; then
   gfs_ver=v15
   GETICSH=${GDASINIT_DIR}/get_${gfs_ver}.data.sh
+  tarball=com_gfs_prod_gfs.${yy}${mm}${dd}_${hh}.gfs_pgrb2.tar
 fi
 
 export EXTRACT_DIR yy mm dd hh UFS_DIR OUTDIR CRES_HIRES CRES_ENKF
@@ -78,25 +78,13 @@ sh ${GETICSH} ${CDUMP}
 status=$?
 [[ $status -ne 0 ]] && exit $status
 
-# Copy pgbanl file to ROTDIR for verification/archival - v14+
-cd $EXTRACT_DIR
-OUTDIR2=${ROTDIR}/gfs.${yy}${mm}${dd}/${hh}/${COMPONENT}
-if [ ! -d ${OUTDIR2} ]; then mkdir -p ${OUTDIR2} ; fi
-if [ $gfs_ver = v14 ]; then
-  for grid in 0p25 0p50 1p00
-  do
-    tarball=gpfs_hps_nco_ops_com_gfs_prod_gfs.${yy}${mm}${dd}_${hh}.pgrb2_${grid}.tar
-    file=gfs.t${hh}z.pgrb2.${grid}.anl
-    htar -xvf ${PRODHPSSDIR}/${tarball} ./gfs.${yy}${mm}${dd}/${hh}/${file}
-    mv ${EXTRACT_DIR}/gfs.${yy}${mm}${dd}/${hh}/${file} ${OUTDIR2}/${file}
-  done
-elif [ $gfs_ver = v15 ]; then
-  tarball=com_gfs_prod_gfs.${yy}${mm}${dd}_${hh}.gfs_pgrb2.tar  
+# Pull pgbanl file for verification/archival - v14+
+if [ $gfs_ver = v14 -o $gfs_ver = v15 ]; then
+  cd $EXTRACT_DIR
   for grid in 0p25 0p50 1p00
   do
     file=gfs.t${hh}z.pgrb2.${grid}.anl
     htar -xvf ${PRODHPSSDIR}/${tarball} ./gfs.${yy}${mm}${dd}/${hh}/${file}
-    mv ${EXTRACT_DIR}/gfs.${yy}${mm}${dd}/${hh}/${file} ${OUTDIR2}/${file}
   done
 fi
 
diff --git a/jobs/rocoto/init.sh b/jobs/rocoto/init.sh
index c36addb2ed5..330846290ee 100755
--- a/jobs/rocoto/init.sh
+++ b/jobs/rocoto/init.sh
@@ -45,6 +45,8 @@ EXTRACT_DIR=${PTMP}/gdas.init_${CDATE}/input
 WORKDIR=${PTMP}/gdas.init_${CDATE}/output
 OUTDIR=${ROTDIR}
 
+COMPONENT="atmos"
+
 gfs_ver=v16
 RUNICSH=${GDASINIT_DIR}/run_v16.chgres.sh
 
@@ -76,6 +78,19 @@ sh ${RUNICSH} ${CDUMP}
 status=$?
 [[ $status -ne 0 ]] && exit $status
 
+# Copy pgbanl file to ROTDIR for verification/archival - v14+
+if [ $gfs_ver = v14 -o $gfs_ver = v15 ]; then
+  cd $EXTRACT_DIR
+  for grid in 0p25 0p50 1p00
+  do
+    file=gfs.t${hh}z.pgrb2.${grid}.anl
+    mv ${EXTRACT_DIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${file} ${OUTDIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${COMPONENT}/${file}
+  done
+fi
+
+# Clean up EXTRACT_DIR
+rm -rf $EXTRACT_DIR
+
 ###############################################################
 # Exit out cleanly
 exit 0
diff --git a/parm/config/config.base.emc.dyn b/parm/config/config.base.emc.dyn
index 37669926eed..dd3459ce5d3 100755
--- a/parm/config/config.base.emc.dyn
+++ b/parm/config/config.base.emc.dyn
@@ -58,6 +58,8 @@ export WAFSF="NO"          # WAFS products
 #  use RUNMOS flag (currently in config.vrfy)
 export REALTIME="YES"
 
+# Experiment mode (cycled or free-forecast)
+export MODE="@MODE@" # cycled/free
 
 ####################################################
 # DO NOT ADD MACHINE DEPENDENT STUFF BELOW THIS LINE
@@ -209,7 +211,8 @@ export IAU_OFFSET=6
 export DOIAU_ENKF=${DOIAU:-"YES"}   # Enable 4DIAU for EnKF ensemble
 export IAUFHRS_ENKF="3,6,9"
 export IAU_DELTHRS_ENKF=6
-if [[ "$SDATE" = "$CDATE" && $EXP_WARM_START = ".false." ]] || [[ "$DOIAU" = "NO" ]] ; then # Cold starting or IAU off
+# Check if cycle is cold starting, DOIAU off, or free-forecast mode
+if [[ "$MODE" = "cycled" && "$SDATE" = "$CDATE" && $EXP_WARM_START = ".false." ]] || [[ "$DOIAU" = "NO" ]] || [[ "$MODE" = "free" && $EXP_WARM_START = ".false." ]] ; then
   export IAU_OFFSET=0
   export IAU_FHROT=0
 fi
diff --git a/ush/rocoto/setup_expt.py b/ush/rocoto/setup_expt.py
index afd3b028304..903b91eaf57 100755
--- a/ush/rocoto/setup_expt.py
+++ b/ush/rocoto/setup_expt.py
@@ -105,6 +105,7 @@ def edit_baseconfig():
                     .replace('@QUEUE_SERVICE@', queue_service) \
                     .replace('@PARTITION_BATCH@', partition_batch) \
                     .replace('@EXP_WARM_START@', exp_warm_start) \
+                    .replace('@MODE@', 'cycled') \
                     .replace('@CHGRP_RSTPROD@', chgrp_rstprod) \
                     .replace('@CHGRP_CMD@', chgrp_cmd) \
                     .replace('@HPSSARCH@', hpssarch) \
diff --git a/ush/rocoto/setup_expt_fcstonly.py b/ush/rocoto/setup_expt_fcstonly.py
index 8a7ffb58dcf..c95d88cea0e 100755
--- a/ush/rocoto/setup_expt_fcstonly.py
+++ b/ush/rocoto/setup_expt_fcstonly.py
@@ -79,6 +79,7 @@ def edit_baseconfig():
                     .replace('@QUEUE_SERVICE@', queue_service) \
                     .replace('@PARTITION_BATCH@', partition_batch) \
                     .replace('@EXP_WARM_START@', exp_warm_start) \
+                    .replace('@MODE@', 'free') \
                     .replace('@CHGRP_RSTPROD@', chgrp_rstprod) \
                     .replace('@CHGRP_CMD@', chgrp_cmd) \
                     .replace('@HPSSARCH@', hpssarch) \
diff --git a/ush/rocoto/setup_workflow_fcstonly.py b/ush/rocoto/setup_workflow_fcstonly.py
index 1bd02a8504a..7e6a7779afb 100755
--- a/ush/rocoto/setup_workflow_fcstonly.py
+++ b/ush/rocoto/setup_workflow_fcstonly.py
@@ -245,10 +245,10 @@ def get_workflow(dict_configs, cdump='gdas'):
 
     # getic
     deps = []
-    data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/INPUT/sfc_data.tile6.nc'
+    data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/INPUT/sfc_data.tile6.nc'
     dep_dict = {'type':'data', 'data':data}
     deps.append(rocoto.add_dependency(dep_dict))
-    data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/RESTART/@Y@m@d.@H0000.sfcanl_data.tile6.nc'
+    data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/RESTART/@Y@m@d.@H0000.sfcanl_data.tile6.nc'
     dep_dict = {'type':'data', 'data':data}
     deps.append(rocoto.add_dependency(dep_dict))
     dependencies = rocoto.create_dependency(dep_condition='nor', dep=deps)
@@ -259,10 +259,10 @@ def get_workflow(dict_configs, cdump='gdas'):
 
     # init - chgres_cube
     deps = []
-    data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/INPUT/sfc_data.tile6.nc'
+    data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/INPUT/sfc_data.tile6.nc'
     dep_dict = {'type':'data', 'data':data}
     deps.append(rocoto.add_dependency(dep_dict))
-    data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/RESTART/@Y@m@d.@H0000.sfcanl_data.tile6.nc'
+    data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/RESTART/@Y@m@d.@H0000.sfcanl_data.tile6.nc'
     dep_dict = {'type':'data', 'data':data}
     deps.append(rocoto.add_dependency(dep_dict))
     dependencies = rocoto.create_dependency(dep_condition='nor', dep=deps)
@@ -380,16 +380,6 @@ def get_workflow(dict_configs, cdump='gdas'):
         tasks.append(task)
         tasks.append('\n')
 
-    # wavestat
-    #if do_wave in ['Y', 'YES'] and do_wave_cdump in ['GFS', 'BOTH']:
-    #    deps = []
-    #    dep_dict = {'type':'task', 'name':'%swavepost' % cdump}
-    #    deps.append(rocoto.add_dependency(dep_dict))
-    #    dependencies = rocoto.create_dependency(dep=deps)
-    #    task = wfu.create_wf_task('wavestat', cdump=cdump, envar=envars, dependency=dependencies)
-    #    tasks.append(task)
-    #    tasks.append('\n')
-
     # wavegempak
     if do_wave in ['Y', 'YES'] and do_gempak in ['Y', 'YES']:
         deps = []

From ed74d83b64cbc98a36698dfb39b98fc860892ea4 Mon Sep 17 00:00:00 2001
From: "kate.friedman" <kate.friedman@noaa.gov>
Date: Thu, 4 Feb 2021 13:28:59 -0600
Subject: [PATCH 011/125] Adjustments for Orion, dependencies, and using DATA
 folder

- update getic and init jobs to use DATA folder and ROTDIR
- move pgb copy back to getic job since it now dumps into ROTDIR
- update setup_workflow_fcstonly.py to turn off getic on Orion and
  adjust dependencies for init for input files from supported versions

Refs: #178
---
 jobs/rocoto/getic.sh                  | 10 ++++--
 jobs/rocoto/init.sh                   | 25 ++++++--------
 ush/rocoto/setup_workflow_fcstonly.py | 47 +++++++++++++++------------
 3 files changed, 44 insertions(+), 38 deletions(-)

diff --git a/jobs/rocoto/getic.sh b/jobs/rocoto/getic.sh
index b3cb741b33e..f5ff1c96980 100755
--- a/jobs/rocoto/getic.sh
+++ b/jobs/rocoto/getic.sh
@@ -41,8 +41,11 @@ mm=$(echo $CDATE | cut -c5-6)
 dd=$(echo $CDATE | cut -c7-8)
 hh=${cyc:-$(echo $CDATE | cut -c9-10)}
 
-EXTRACT_DIR=${PTMP}/gdas.init_${CDATE}/input
-OUTDIR=${PTMP}/gdas.init_${CDATE}/output
+export DATA=${DATA:-${DATAROOT}/init}
+
+EXTRACT_DIR=${EXTRACT_DIR:-$ROTDIR}
+WORKDIR=${WORKDIR:-$DATA}
+OUTDIR=${OUTDIR:-$ROTDIR}
 PRODHPSSDIR=/NCEPPROD/hpssprod/runhistory/rh${yy}/${yy}${mm}/${yy}${mm}${dd}
 
 gfs_ver=v16
@@ -84,7 +87,8 @@ if [ $gfs_ver = v14 -o $gfs_ver = v15 ]; then
   for grid in 0p25 0p50 1p00
   do
     file=gfs.t${hh}z.pgrb2.${grid}.anl
-    htar -xvf ${PRODHPSSDIR}/${tarball} ./gfs.${yy}${mm}${dd}/${hh}/${file}
+    htar -xvf ${PRODHPSSDIR}/${tarball} ./${CDUMP}.${yy}${mm}${dd}/${hh}/${file}
+    mv ${EXTRACT_DIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${file} ${OUTDIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${COMPONENT}/${file}
   done
 fi
 
diff --git a/jobs/rocoto/init.sh b/jobs/rocoto/init.sh
index 330846290ee..5b4ee7dedd6 100755
--- a/jobs/rocoto/init.sh
+++ b/jobs/rocoto/init.sh
@@ -41,9 +41,11 @@ mm=$(echo $CDATE | cut -c5-6)
 dd=$(echo $CDATE | cut -c7-8)
 hh=${cyc:-$(echo $CDATE | cut -c9-10)}
 
-EXTRACT_DIR=${PTMP}/gdas.init_${CDATE}/input
-WORKDIR=${PTMP}/gdas.init_${CDATE}/output
-OUTDIR=${ROTDIR}
+export DATA=${DATA:-${DATAROOT}/init}
+
+EXTRACT_DIR=${EXTRACT_DIR:-$ROTDIR}
+WORKDIR=${WORKDIR:-$DATA}
+OUTDIR=${OUTDIR:-$ROTDIR}
 
 COMPONENT="atmos"
 
@@ -78,18 +80,11 @@ sh ${RUNICSH} ${CDUMP}
 status=$?
 [[ $status -ne 0 ]] && exit $status
 
-# Copy pgbanl file to ROTDIR for verification/archival - v14+
-if [ $gfs_ver = v14 -o $gfs_ver = v15 ]; then
-  cd $EXTRACT_DIR
-  for grid in 0p25 0p50 1p00
-  do
-    file=gfs.t${hh}z.pgrb2.${grid}.anl
-    mv ${EXTRACT_DIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${file} ${OUTDIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${COMPONENT}/${file}
-  done
-fi
-
-# Clean up EXTRACT_DIR
-rm -rf $EXTRACT_DIR
+##########################################
+# Remove the Temporary working directory
+##########################################
+cd $DATAROOT
+[[ $KEEPDATA = "NO" ]] && rm -rf $DATA
 
 ###############################################################
 # Exit out cleanly
diff --git a/ush/rocoto/setup_workflow_fcstonly.py b/ush/rocoto/setup_workflow_fcstonly.py
index 7e6a7779afb..9a1ff4e6287 100755
--- a/ush/rocoto/setup_workflow_fcstonly.py
+++ b/ush/rocoto/setup_workflow_fcstonly.py
@@ -234,6 +234,7 @@ def get_workflow(dict_configs, cdump='gdas'):
     envars.append(rocoto.create_envar(name='cyc', value='<cyclestr>@H</cyclestr>'))
 
     base = dict_configs['base']
+    machine = base.get('machine', wfu.detectMachine())
     do_wave = base.get('DO_WAVE', 'NO').upper()
     do_wave_cdump = base.get('WAVE_CDUMP', 'BOTH').upper()
     do_gempak = base.get('DO_GEMPAK', 'NO').upper()
@@ -244,38 +245,44 @@ def get_workflow(dict_configs, cdump='gdas'):
     tasks = []
 
     # getic
+    if machine not in ['ORION']:
+      deps = []
+      data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/INPUT/sfc_data.tile6.nc'
+      dep_dict = {'type':'data', 'data':data}
+      deps.append(rocoto.add_dependency(dep_dict))
+      data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/RESTART/@Y@m@d.@H0000.sfcanl_data.tile6.nc'
+      dep_dict = {'type':'data', 'data':data}
+      deps.append(rocoto.add_dependency(dep_dict))
+      dependencies = rocoto.create_dependency(dep_condition='nor', dep=deps)
+
+      task = wfu.create_wf_task('getic', cdump=cdump, envar=envars, dependency=dependencies)
+      tasks.append(task)
+      tasks.append('\n')
+
+    # init
     deps = []
-    data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/INPUT/sfc_data.tile6.nc'
+    data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/gfs.t@Hz.sanl'
     dep_dict = {'type':'data', 'data':data}
     deps.append(rocoto.add_dependency(dep_dict))
-    data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/RESTART/@Y@m@d.@H0000.sfcanl_data.tile6.nc'
-    dep_dict = {'type':'data', 'data':data}
-    deps.append(rocoto.add_dependency(dep_dict))
-    dependencies = rocoto.create_dependency(dep_condition='nor', dep=deps)
-
-    task = wfu.create_wf_task('getic', cdump=cdump, envar=envars, dependency=dependencies)
-    tasks.append(task)
-    tasks.append('\n')
-
-    # init - chgres_cube
-    deps = []
-    data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/INPUT/sfc_data.tile6.nc'
+    data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/gfs.t@Hz.atmanl.nemsio'
     dep_dict = {'type':'data', 'data':data}
     deps.append(rocoto.add_dependency(dep_dict))
     data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/RESTART/@Y@m@d.@H0000.sfcanl_data.tile6.nc'
     dep_dict = {'type':'data', 'data':data}
     deps.append(rocoto.add_dependency(dep_dict))
-    dependencies = rocoto.create_dependency(dep_condition='nor', dep=deps)
+    dependencies = rocoto.create_dependency(dep_condition='or', dep=deps)
 
-    deps = []
-    dep_dict = {'type': 'task', 'name': '%sgetic' % cdump}
-    deps.append(rocoto.add_dependency(dep_dict))
-    dependencies2 = rocoto.create_dependency(dep=deps)
+    if machine not in ['ORION']:
+      deps = []
+      dep_dict = {'type': 'task', 'name': '%sgetic' % cdump}
+      deps.append(rocoto.add_dependency(dep_dict))
+      dependencies2 = rocoto.create_dependency(dep=deps)
 
     deps = []
     deps.append(dependencies)
-    deps.append(dependencies2)
-    dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
+    if machine not in ['ORION']:
+      deps.append(dependencies2)
+      dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
 
     task = wfu.create_wf_task('init', cdump=cdump, envar=envars, dependency=dependencies)
     tasks.append(task)

From 7a34968e3a66fc31127de1450d5ce194d9b9681b Mon Sep 17 00:00:00 2001
From: "kate.friedman" <kate.friedman@noaa.gov>
Date: Thu, 4 Feb 2021 14:32:30 -0600
Subject: [PATCH 012/125] Small adjustments from testing

- add needed exports in getic.sh
- add check to init for whether it needs to run
- add check to init for copying pgbanl files when getic doesn't run
- remove atmos subfolder from init dependencies for older versions

Refs: #178
---
 jobs/rocoto/getic.sh                  |  6 ++---
 jobs/rocoto/init.sh                   | 33 ++++++++++++++++++++-------
 ush/rocoto/setup_workflow_fcstonly.py |  4 ++--
 3 files changed, 30 insertions(+), 13 deletions(-)

diff --git a/jobs/rocoto/getic.sh b/jobs/rocoto/getic.sh
index f5ff1c96980..c6798008922 100755
--- a/jobs/rocoto/getic.sh
+++ b/jobs/rocoto/getic.sh
@@ -43,9 +43,9 @@ hh=${cyc:-$(echo $CDATE | cut -c9-10)}
 
 export DATA=${DATA:-${DATAROOT}/init}
 
-EXTRACT_DIR=${EXTRACT_DIR:-$ROTDIR}
-WORKDIR=${WORKDIR:-$DATA}
-OUTDIR=${OUTDIR:-$ROTDIR}
+export EXTRACT_DIR=${EXTRACT_DIR:-$ROTDIR}
+export WORKDIR=${WORKDIR:-$DATA}
+export OUTDIR=${OUTDIR:-$ROTDIR}
 PRODHPSSDIR=/NCEPPROD/hpssprod/runhistory/rh${yy}/${yy}${mm}/${yy}${mm}${dd}
 
 gfs_ver=v16
diff --git a/jobs/rocoto/init.sh b/jobs/rocoto/init.sh
index 5b4ee7dedd6..8a1078f2fbf 100755
--- a/jobs/rocoto/init.sh
+++ b/jobs/rocoto/init.sh
@@ -43,9 +43,9 @@ hh=${cyc:-$(echo $CDATE | cut -c9-10)}
 
 export DATA=${DATA:-${DATAROOT}/init}
 
-EXTRACT_DIR=${EXTRACT_DIR:-$ROTDIR}
-WORKDIR=${WORKDIR:-$DATA}
-OUTDIR=${OUTDIR:-$ROTDIR}
+export EXTRACT_DIR=${EXTRACT_DIR:-$ROTDIR}
+export WORKDIR=${WORKDIR:-$DATA}
+export OUTDIR=${OUTDIR:-$ROTDIR}
 
 COMPONENT="atmos"
 
@@ -74,11 +74,28 @@ fi
 export EXTRACT_DIR yy mm dd hh UFS_DIR OUTDIR CRES_HIRES CRES_ENKF
 export LEVS gfs_ver
 
-# Run chgres_cube
-if [ ! -d $OUTDIR ]; then mkdir -p $OUTDIR ; fi
-sh ${RUNICSH} ${CDUMP}
-status=$?
-[[ $status -ne 0 ]] && exit $status
+# Check if init is needed and run if so
+if [[ $gfs_ver = "v16" && $EXP_WARM_START = ".true." && $CASE = "C768" ]]; then
+  echo "Detected v16 C768 warm starts, will not run init. Exiting..."
+else
+  # Run chgres_cube
+  if [ ! -d $OUTDIR ]; then mkdir -p $OUTDIR ; fi
+  sh ${RUNICSH} ${CDUMP}
+  status=$?
+  [[ $status -ne 0 ]] && exit $status
+fi
+
+# Check for pgbanl files and move
+if [ $gfs_ver = v14 -o $gfs_ver = v15 ]; then
+  cd $EXTRACT_DIR
+  for grid in 0p25 0p50 1p00
+  do
+    file=gfs.t${hh}z.pgrb2.${grid}.anl
+    if [ -f ${EXTRACT_DIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${file} ]; then
+      mv ${EXTRACT_DIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${file} ${OUTDIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${COMPONENT}/${file}
+    fi
+  done
+fi
 
 ##########################################
 # Remove the Temporary working directory
diff --git a/ush/rocoto/setup_workflow_fcstonly.py b/ush/rocoto/setup_workflow_fcstonly.py
index 9a1ff4e6287..f1f27c672e0 100755
--- a/ush/rocoto/setup_workflow_fcstonly.py
+++ b/ush/rocoto/setup_workflow_fcstonly.py
@@ -261,10 +261,10 @@ def get_workflow(dict_configs, cdump='gdas'):
 
     # init
     deps = []
-    data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/gfs.t@Hz.sanl'
+    data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/gfs.t@Hz.sanl'
     dep_dict = {'type':'data', 'data':data}
     deps.append(rocoto.add_dependency(dep_dict))
-    data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/gfs.t@Hz.atmanl.nemsio'
+    data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/gfs.t@Hz.atmanl.nemsio'
     dep_dict = {'type':'data', 'data':data}
     deps.append(rocoto.add_dependency(dep_dict))
     data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/RESTART/@Y@m@d.@H0000.sfcanl_data.tile6.nc'

From 613a181c040682dfb41997c64c2d220a33eeecbe Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Thu, 4 Feb 2021 20:57:07 +0000
Subject: [PATCH 013/125] Adjusting from testing

- add COMPONENT setting back into getic.sh
- add gfsinit as dependency to gfswaveprep since UFS_UTILS gdas_init
scripts remove the atmos folder

Refs: #178
---
 jobs/rocoto/getic.sh                  | 2 ++
 ush/rocoto/setup_workflow_fcstonly.py | 4 +++-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/jobs/rocoto/getic.sh b/jobs/rocoto/getic.sh
index c6798008922..0f3557346a1 100755
--- a/jobs/rocoto/getic.sh
+++ b/jobs/rocoto/getic.sh
@@ -48,6 +48,8 @@ export WORKDIR=${WORKDIR:-$DATA}
 export OUTDIR=${OUTDIR:-$ROTDIR}
 PRODHPSSDIR=/NCEPPROD/hpssprod/runhistory/rh${yy}/${yy}${mm}/${yy}${mm}${dd}
 
+COMPONENT="atmos"
+
 gfs_ver=v16
 GETICSH=${GDASINIT_DIR}/get_v16.data.sh
 
diff --git a/ush/rocoto/setup_workflow_fcstonly.py b/ush/rocoto/setup_workflow_fcstonly.py
index f1f27c672e0..49c004e917c 100755
--- a/ush/rocoto/setup_workflow_fcstonly.py
+++ b/ush/rocoto/setup_workflow_fcstonly.py
@@ -299,7 +299,9 @@ def get_workflow(dict_configs, cdump='gdas'):
         deps = []
         dep_dict = {'type': 'task', 'name': '%swaveinit' % cdump}
         deps.append(rocoto.add_dependency(dep_dict))
-        dependencies = rocoto.create_dependency(dep=deps)
+        dep_dict = {'type': 'task', 'name': '%sinit' % cdump}
+        deps.append(rocoto.add_dependency(dep_dict))
+        dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
         task = wfu.create_wf_task('waveprep', cdump=cdump, envar=envars, dependency=dependencies)
         tasks.append(task)
         tasks.append('\n')

From 77d7408b759a761d1697ad672dd4feae14d9c5b1 Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Tue, 9 Feb 2021 18:49:05 +0000
Subject: [PATCH 014/125] Move GFSv16 version date and set C96 to use 4 threads

- change config.fv3 to use 4 threads for C96 (too few nodes with 1 thread)
- update GFSv16 version if-block date in getic and init scripts
to use updated/tentative implementation date; update when finalized

Refs: #178
---
 jobs/rocoto/getic.sh   | 2 +-
 jobs/rocoto/init.sh    | 2 +-
 parm/config/config.fv3 | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/jobs/rocoto/getic.sh b/jobs/rocoto/getic.sh
index 0f3557346a1..02e5e1faf22 100755
--- a/jobs/rocoto/getic.sh
+++ b/jobs/rocoto/getic.sh
@@ -68,7 +68,7 @@ elif [ $yy$mm$dd$hh -lt 2019061200 ]; then
   gfs_ver=v14
   GETICSH=${GDASINIT_DIR}/get_${gfs_ver}.data.sh
   tarball=gpfs_hps_nco_ops_com_gfs_prod_gfs.${yy}${mm}${dd}_${hh}.pgrb2_${grid}.tar
-elif [ $yy$mm$dd$hh -lt 2021020300 ]; then
+elif [ $yy$mm$dd$hh -lt 2021031700 ]; then
   gfs_ver=v15
   GETICSH=${GDASINIT_DIR}/get_${gfs_ver}.data.sh
   tarball=com_gfs_prod_gfs.${yy}${mm}${dd}_${hh}.gfs_pgrb2.tar
diff --git a/jobs/rocoto/init.sh b/jobs/rocoto/init.sh
index 8a1078f2fbf..a385557ae7e 100755
--- a/jobs/rocoto/init.sh
+++ b/jobs/rocoto/init.sh
@@ -66,7 +66,7 @@ elif [ $yy$mm$dd$hh -lt 2017072000 ]; then
 elif [ $yy$mm$dd$hh -lt 2019061200 ]; then
   gfs_ver=v14
   RUNICSH=${GDASINIT_DIR}/run_${gfs_ver}.chgres.sh
-elif [ $yy$mm$dd$hh -lt 2021020300 ]; then
+elif [ $yy$mm$dd$hh -lt 2021031700 ]; then
   gfs_ver=v15
   RUNICSH=${GDASINIT_DIR}/run_${gfs_ver}.chgres.gfs.sh
 fi
diff --git a/parm/config/config.fv3 b/parm/config/config.fv3
index 3fdfc468a2f..5fbe168916d 100755
--- a/parm/config/config.fv3
+++ b/parm/config/config.fv3
@@ -59,7 +59,7 @@ case $case_in in
         export layout_y_gfs=4
         export npe_wav=140
         export npe_wav_gfs=140
-        export nth_fv3=1
+        export nth_fv3=4
         export cdmbgwd="0.14,1.8,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
         export WRITE_GROUP=1
         export WRTTASK_PER_GROUP=$npe_node_max

From 10ed5f1152c385a4d19864ac5924b77dd66dc75e Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Tue, 9 Feb 2021 19:08:38 +0000
Subject: [PATCH 015/125] Add wave jobs to arch job dependencies

Refs: #178
---
 ush/rocoto/setup_workflow_fcstonly.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/ush/rocoto/setup_workflow_fcstonly.py b/ush/rocoto/setup_workflow_fcstonly.py
index 49c004e917c..589ee4a569b 100755
--- a/ush/rocoto/setup_workflow_fcstonly.py
+++ b/ush/rocoto/setup_workflow_fcstonly.py
@@ -627,6 +627,13 @@ def get_workflow(dict_configs, cdump='gdas'):
     deps.append(rocoto.add_dependency(dep_dict))
     dep_dict = {'type':'streq', 'left':'&ARCHIVE_TO_HPSS;', 'right':'YES'}
     deps.append(rocoto.add_dependency(dep_dict))
+    if do_wave in ['Y', 'YES']:
+      dep_dict = {'type': 'task', 'name': '%swavepostsbs' % cdump}
+      deps.append(rocoto.add_dependency(dep_dict))
+      dep_dict = {'type': 'task', 'name': '%swavepostpnt' % cdump}
+      deps.append(rocoto.add_dependency(dep_dict))
+      dep_dict = {'type': 'task', 'name': '%swavepostbndpnt' % cdump}
+      deps.append(rocoto.add_dependency(dep_dict))
     dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
     task = wfu.create_wf_task('arch', cdump=cdump, envar=envars, dependency=dependencies, final=True)
     tasks.append(task)

From 181606755de789d1764d1f21a5a2e72ea302da34 Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Fri, 12 Feb 2021 14:19:51 +0000
Subject: [PATCH 016/125] Updates for archival, NSST, and v16 retros

- add variables: MODE to dilineate free-forecast and cycled
modes; RETRO variable to toggle between v16 ops and retro inputs
- update arch step to archive INPUT files instead of RESTART files
when in free (free-forecast) mode; other small fixes to arch step
- set NSST to spinup before availability date
- added support for running off v16 retrospective inputs
- fix missing bufrsnd job in free-forecast mode
- update resource settings for C48 and C96; further refinement coming

Refs: #178
---
 jobs/rocoto/arch.sh                   |  8 +++-
 jobs/rocoto/getic.sh                  | 47 +++++++++++--------
 jobs/rocoto/init.sh                   | 41 ++++++++++-------
 parm/config/config.base.emc.dyn       |  1 +
 parm/config/config.fv3                | 30 ++++++------
 parm/config/config.getic              |  2 +
 parm/config/config.init               |  2 +-
 parm/config/config.nsst               |  6 +--
 ush/hpssarch_gen.sh                   | 66 +++++++++++++++++++--------
 ush/rocoto/setup_workflow_fcstonly.py | 45 +++++++++++++++++-
 10 files changed, 170 insertions(+), 78 deletions(-)

diff --git a/jobs/rocoto/arch.sh b/jobs/rocoto/arch.sh
index ee3af795c26..85a9e6406aa 100755
--- a/jobs/rocoto/arch.sh
+++ b/jobs/rocoto/arch.sh
@@ -181,7 +181,13 @@ if [ $CDUMP = "gfs" ]; then
 
     #for targrp in gfs_flux gfs_netcdf/nemsio gfs_pgrb2b; do
     if [ ${SAVEFCSTNEMSIO:-"YES"} = "YES" ]; then
-        for targrp in gfs_flux gfs_${format}a gfs_${format}b gfs_pgrb2b; do
+        if [ $MODE = "cycled" ]; then
+          targrp_list=`gfs_flux gfs_${format}a gfs_${format}b gfs_pgrb2b`
+        elif [ $MODE = "free" ]; then
+          targrp_list=`gfs_flux gfs_${format}b gfs_pgrb2b`
+        fi
+        #for targrp in gfs_flux gfs_${format}a gfs_${format}b gfs_pgrb2b; do
+        for targrp in $targrp_list; do
             htar -P -cvf $ATARDIR/$CDATE/${targrp}.tar `cat $ARCH_LIST/${targrp}.txt`
             status=$?
             if [ $status -ne 0  -a $CDATE -ge $firstday ]; then
diff --git a/jobs/rocoto/getic.sh b/jobs/rocoto/getic.sh
index 02e5e1faf22..8a8dfee7004 100755
--- a/jobs/rocoto/getic.sh
+++ b/jobs/rocoto/getic.sh
@@ -20,7 +20,7 @@ status=$?
 
 ###############################################################
 # Source relevant configs
-configs="base getic"
+configs="base init"
 for config in $configs; do
     . $EXPDIR/config.${config}
     status=$?
@@ -53,25 +53,32 @@ COMPONENT="atmos"
 gfs_ver=v16
 GETICSH=${GDASINIT_DIR}/get_v16.data.sh
 
-# No ENKF data prior to 2012/05/21/00z
-if [ $yy$mm$dd$hh -lt 2012052100 ]; then
-  set +x
-  echo FATAL ERROR: SCRIPTS DO NOT SUPPORT OLD GFS DATA
-  exit 2
-elif [ $yy$mm$dd$hh -lt 2016051000 ]; then
-  gfs_ver=v12
-  GETICSH=${GDASINIT_DIR}/get_pre-v14.data.sh
-elif [ $yy$mm$dd$hh -lt 2017072000 ]; then
-  gfs_ver=v13
-  GETICSH=${GDASINIT_DIR}/get_pre-v14.data.sh
-elif [ $yy$mm$dd$hh -lt 2019061200 ]; then
-  gfs_ver=v14
-  GETICSH=${GDASINIT_DIR}/get_${gfs_ver}.data.sh
-  tarball=gpfs_hps_nco_ops_com_gfs_prod_gfs.${yy}${mm}${dd}_${hh}.pgrb2_${grid}.tar
-elif [ $yy$mm$dd$hh -lt 2021031700 ]; then
-  gfs_ver=v15
-  GETICSH=${GDASINIT_DIR}/get_${gfs_ver}.data.sh
-  tarball=com_gfs_prod_gfs.${yy}${mm}${dd}_${hh}.gfs_pgrb2.tar
+RETRO=${RETRO:-"NO"}
+if [ $RETRO = "YES" ]; then
+  GETICSH=${GDASINIT_DIR}/get_v16retro.data.sh
+fi
+
+if [ $RETRO = "NO" ]; then # Operational input
+  # No ENKF data prior to 2012/05/21/00z
+  if [ $yy$mm$dd$hh -lt 2012052100 ]; then
+    set +x
+    echo FATAL ERROR: SCRIPTS DO NOT SUPPORT OLD GFS DATA
+    exit 2
+  elif [ $yy$mm$dd$hh -lt 2016051000 ]; then
+    gfs_ver=v12
+    GETICSH=${GDASINIT_DIR}/get_pre-v14.data.sh
+  elif [ $yy$mm$dd$hh -lt 2017072000 ]; then
+    gfs_ver=v13
+    GETICSH=${GDASINIT_DIR}/get_pre-v14.data.sh
+  elif [ $yy$mm$dd$hh -lt 2019061200 ]; then
+    gfs_ver=v14
+    GETICSH=${GDASINIT_DIR}/get_${gfs_ver}.data.sh
+    tarball=gpfs_hps_nco_ops_com_gfs_prod_gfs.${yy}${mm}${dd}_${hh}.pgrb2_${grid}.tar
+  elif [ $yy$mm$dd$hh -lt 2021031700 ]; then
+    gfs_ver=v15
+    GETICSH=${GDASINIT_DIR}/get_${gfs_ver}.data.sh
+    tarball=com_gfs_prod_gfs.${yy}${mm}${dd}_${hh}.gfs_pgrb2.tar
+  fi
 fi
 
 export EXTRACT_DIR yy mm dd hh UFS_DIR OUTDIR CRES_HIRES CRES_ENKF
diff --git a/jobs/rocoto/init.sh b/jobs/rocoto/init.sh
index a385557ae7e..b944216a838 100755
--- a/jobs/rocoto/init.sh
+++ b/jobs/rocoto/init.sh
@@ -52,23 +52,30 @@ COMPONENT="atmos"
 gfs_ver=v16
 RUNICSH=${GDASINIT_DIR}/run_v16.chgres.sh
 
-# No ENKF data prior to 2012/05/21/00z
-if [ $yy$mm$dd$hh -lt 2012052100 ]; then
-  set +x
-  echo FATAL ERROR: SCRIPTS DO NOT SUPPORT OLD GFS DATA
-  exit 2
-elif [ $yy$mm$dd$hh -lt 2016051000 ]; then
-  gfs_ver=v12
-  RUNICSH=${GDASINIT_DIR}/run_pre-v14.chgres.sh
-elif [ $yy$mm$dd$hh -lt 2017072000 ]; then
-  gfs_ver=v13
-  RUNICSH=${GDASINIT_DIR}/run_pre-v14.chgres.sh
-elif [ $yy$mm$dd$hh -lt 2019061200 ]; then
-  gfs_ver=v14
-  RUNICSH=${GDASINIT_DIR}/run_${gfs_ver}.chgres.sh
-elif [ $yy$mm$dd$hh -lt 2021031700 ]; then
-  gfs_ver=v15
-  RUNICSH=${GDASINIT_DIR}/run_${gfs_ver}.chgres.gfs.sh
+RETRO=${RETRO:-"NO"}
+if [ $RETRO = "YES" ]; then
+  GETICSH=${GDASINIT_DIR}/run_v16retro.chgres.sh
+fi
+
+if [ $RETRO = "NO" ]; then # Operational input
+  # No ENKF data prior to 2012/05/21/00z
+  if [ $yy$mm$dd$hh -lt 2012052100 ]; then
+    set +x
+    echo FATAL ERROR: SCRIPTS DO NOT SUPPORT OLD GFS DATA
+    exit 2
+  elif [ $yy$mm$dd$hh -lt 2016051000 ]; then
+    gfs_ver=v12
+    RUNICSH=${GDASINIT_DIR}/run_pre-v14.chgres.sh
+  elif [ $yy$mm$dd$hh -lt 2017072000 ]; then
+    gfs_ver=v13
+    RUNICSH=${GDASINIT_DIR}/run_pre-v14.chgres.sh
+  elif [ $yy$mm$dd$hh -lt 2019061200 ]; then
+    gfs_ver=v14
+    RUNICSH=${GDASINIT_DIR}/run_${gfs_ver}.chgres.sh
+  elif [ $yy$mm$dd$hh -lt 2021031700 ]; then
+    gfs_ver=v15
+    RUNICSH=${GDASINIT_DIR}/run_${gfs_ver}.chgres.gfs.sh
+  fi
 fi
 
 export EXTRACT_DIR yy mm dd hh UFS_DIR OUTDIR CRES_HIRES CRES_ENKF
diff --git a/parm/config/config.base.emc.dyn b/parm/config/config.base.emc.dyn
index dd3459ce5d3..5634905a2f7 100755
--- a/parm/config/config.base.emc.dyn
+++ b/parm/config/config.base.emc.dyn
@@ -226,6 +226,7 @@ export gldas_cyc=00
 
 # run wave component
 export DO_WAVE="YES"
+if [[ "$SDATE" -lt "2019020100" ]]; then DO_WAVE="NO" ; fi # no rtofs in GDA
 export WAVE_CDUMP="both"
 
 # Microphysics Options: 99-ZhaoCarr, 8-Thompson; 6-WSM6, 10-MG, 11-GFDL
diff --git a/parm/config/config.fv3 b/parm/config/config.fv3
index 5fbe168916d..410a5ed4a7d 100755
--- a/parm/config/config.fv3
+++ b/parm/config/config.fv3
@@ -36,33 +36,33 @@ fi
 # (Standard) Model resolution dependent variables
 case $case_in in
     "C48")
-        export DELTIM=450
-        export layout_x=2
-        export layout_y=4
-        export layout_x_gfs=2
-        export layout_y_gfs=4
-        export npe_wav=140
-        export npe_wav_gfs=140
+        export DELTIM=1200
+        export layout_x=3
+        export layout_y=2
+        export layout_x_gfs=3
+        export layout_y_gfs=2
+        export npe_wav=16
+        export npe_wav_gfs=16
         export nth_fv3=1
         export cdmbgwd="0.071,2.1,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
         export WRITE_GROUP=1
         export WRTTASK_PER_GROUP=$npe_node_max
         export WRITE_GROUP_GFS=1
         export WRTTASK_PER_GROUP_GFS=$npe_node_max
-        export WRTIOBUF="4M"
+        export WRTIOBUF="1M"
         ;;
     "C96")
-        export DELTIM=450
-        export layout_x=4
+        export DELTIM=900
+        export layout_x=6
         export layout_y=4
-        export layout_x_gfs=4
+        export layout_x_gfs=6
         export layout_y_gfs=4
-        export npe_wav=140
-        export npe_wav_gfs=140
-        export nth_fv3=4
+        export npe_wav=40
+        export npe_wav_gfs=40
+        export nth_fv3=1
         export cdmbgwd="0.14,1.8,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
         export WRITE_GROUP=1
-        export WRTTASK_PER_GROUP=$npe_node_max
+        export WRTTASK_PER_GROUP=6
         export WRITE_GROUP_GFS=1
         export WRTTASK_PER_GROUP_GFS=$npe_node_max
         export WRTIOBUF="4M"
diff --git a/parm/config/config.getic b/parm/config/config.getic
index 64f7159cc98..52498a01d3d 100755
--- a/parm/config/config.getic
+++ b/parm/config/config.getic
@@ -13,4 +13,6 @@ GDASINIT_DIR=${UFS_DIR}/util/gdas_init
 
 EXTRACT_DATA="yes"
 
+RETRO="NO" # YES = Pull v16 inputs from retrospective parallels; NO = use operational inputs
+
 echo "END: config.getic"
diff --git a/parm/config/config.init b/parm/config/config.init
index f9efbde888b..a473d9747a5 100755
--- a/parm/config/config.init
+++ b/parm/config/config.init
@@ -11,7 +11,7 @@ echo "BEGIN: config.init"
 UFS_DIR=${HOMEgfs}/sorc/ufs_utils.fd
 GDASINIT_DIR=${UFS_DIR}/util/gdas_init
 
-RUN_CHGRES="yes"
+RETRO="NO" # YES = Pull v16 inputs from retrospective parallels; NO = use operational inputs
 
 CRES_HIRES=$CASE
 CRES_ENKF=$CASE_ENKF
diff --git a/parm/config/config.nsst b/parm/config/config.nsst
index a59ca19495f..ef8767834d1 100755
--- a/parm/config/config.nsst
+++ b/parm/config/config.nsst
@@ -12,9 +12,9 @@ export NST_MODEL=2
 
 # nstf_name(2) : NST_SPINUP : 0 = OFF, 1 = ON,
 export NST_SPINUP=0
-#if [[ "$CDATE" = $SDATE ]]; then
-#    export NST_SPINUP=1
-#fi
+if [[ "$CDATE" -lt "2017072000" ]]; then
+    export NST_SPINUP=1
+fi
 
 # nstf_name(3) : NST_RESV (Reserved, NSST Analysis) : 0 = OFF, 1 = ON
 export NST_RESV=0
diff --git a/ush/hpssarch_gen.sh b/ush/hpssarch_gen.sh
index 1f4c8ed542b..4e65d7dbdf7 100755
--- a/ush/hpssarch_gen.sh
+++ b/ush/hpssarch_gen.sh
@@ -41,17 +41,23 @@ if [ $type = "gfs" ]; then
   rm -f gfsb.txt
   rm -f gfs_pgrb2b.txt
   rm -f gfs_flux.txt
-  rm -f gfs_${format}a.txt
-  rm -f gfs_${format}b.txt
   rm -f gfs_restarta.txt
   touch gfsa.txt
   touch gfsb.txt
   touch gfs_pgrb2b.txt
   touch gfs_flux.txt
-  touch gfs_${format}a.txt
-  touch gfs_${format}b.txt
   touch gfs_restarta.txt
 
+  if [ $MODE = "cycled" ]; then
+    rm -f gfs_${format}a.txt
+    touch gfs_${format}a.txt
+  fi
+
+  if [ $OUTPUT_HISTORY = ".true." ]; then
+    rm -f gfs_${format}b.txt
+    touch gfs_${format}b.txt
+  fi
+
   if [ $DO_DOWN = "YES" ]; then
     rm -f gfs_downstream.txt
     touch gfs_downstream.txt
@@ -69,11 +75,13 @@ if [ $type = "gfs" ]; then
   echo  "${dirname}${head}pgrb2b.0p50.anl.idx              " >>gfs_pgrb2b.txt
 
   echo  "./logs/${CDATE}/gfs*.log                          " >>gfsa.txt
-  echo  "${dirname}${head}gsistat                          " >>gfsa.txt
-  echo  "${dirname}${head}nsstbufr                         " >>gfsa.txt
-  echo  "${dirname}${head}prepbufr                         " >>gfsa.txt
-  echo  "${dirname}${head}prepbufr_pre-qc                  " >>gfsa.txt
-  echo  "${dirname}${head}prepbufr.acft_profiles           " >>gfsa.txt
+  if [ $MODE = "cycled" ]; then
+    echo  "${dirname}${head}gsistat                          " >>gfsa.txt
+    echo  "${dirname}${head}nsstbufr                         " >>gfsa.txt
+    echo  "${dirname}${head}prepbufr                         " >>gfsa.txt
+    echo  "${dirname}${head}prepbufr_pre-qc                  " >>gfsa.txt
+    echo  "${dirname}${head}prepbufr.acft_profiles           " >>gfsa.txt
+  fi
   echo  "${dirname}${head}pgrb2.0p25.anl                   " >>gfsa.txt
   echo  "${dirname}${head}pgrb2.0p25.anl.idx               " >>gfsa.txt
   echo  "${dirname}avno.t${cyc}z.cyclone.trackatcfunix     " >>gfsa.txt
@@ -145,11 +153,13 @@ if [ $type = "gfs" ]; then
 
 
   #..................
-  echo  "${dirname}${head}atmanl${SUFFIX}            " >>gfs_${format}a.txt
-  echo  "${dirname}${head}sfcanl${SUFFIX}            " >>gfs_${format}a.txt
-  echo  "${dirname}${head}atmi*.nc                   " >>gfs_${format}a.txt
-  echo  "${dirname}${head}dtfanl.nc                  " >>gfs_${format}a.txt
-  echo  "${dirname}${head}loginc.txt                 " >>gfs_${format}a.txt
+  if [ $MODE = "cycled" ]; then
+    echo  "${dirname}${head}atmanl${SUFFIX}            " >>gfs_${format}a.txt
+    echo  "${dirname}${head}sfcanl${SUFFIX}            " >>gfs_${format}a.txt
+    echo  "${dirname}${head}atmi*.nc                   " >>gfs_${format}a.txt
+    echo  "${dirname}${head}dtfanl.nc                  " >>gfs_${format}a.txt
+    echo  "${dirname}${head}loginc.txt                 " >>gfs_${format}a.txt
+  fi
 
   #..................
   if [ $OUTPUT_HISTORY = ".true." ]; then
@@ -163,12 +173,28 @@ if [ $type = "gfs" ]; then
   fi
 
   #..................
-  echo  "${dirname}RESTART/*0000.sfcanl_data.tile1.nc  " >>gfs_restarta.txt
-  echo  "${dirname}RESTART/*0000.sfcanl_data.tile2.nc  " >>gfs_restarta.txt
-  echo  "${dirname}RESTART/*0000.sfcanl_data.tile3.nc  " >>gfs_restarta.txt
-  echo  "${dirname}RESTART/*0000.sfcanl_data.tile4.nc  " >>gfs_restarta.txt
-  echo  "${dirname}RESTART/*0000.sfcanl_data.tile5.nc  " >>gfs_restarta.txt
-  echo  "${dirname}RESTART/*0000.sfcanl_data.tile6.nc  " >>gfs_restarta.txt
+  if [ $MODE = "cycled" ]; then
+    echo  "${dirname}RESTART/*0000.sfcanl_data.tile1.nc  " >>gfs_restarta.txt
+    echo  "${dirname}RESTART/*0000.sfcanl_data.tile2.nc  " >>gfs_restarta.txt
+    echo  "${dirname}RESTART/*0000.sfcanl_data.tile3.nc  " >>gfs_restarta.txt
+    echo  "${dirname}RESTART/*0000.sfcanl_data.tile4.nc  " >>gfs_restarta.txt
+    echo  "${dirname}RESTART/*0000.sfcanl_data.tile5.nc  " >>gfs_restarta.txt
+    echo  "${dirname}RESTART/*0000.sfcanl_data.tile6.nc  " >>gfs_restarta.txt
+  elif [ $MODE = "free" ]; then
+    echo  "${dirname}INPUT/gfs_ctrl.nc        " >>gfs_restarta.txt
+    echo  "${dirname}INPUT/gfs_data.tile1.nc  " >>gfs_restarta.txt
+    echo  "${dirname}INPUT/gfs_data.tile2.nc  " >>gfs_restarta.txt
+    echo  "${dirname}INPUT/gfs_data.tile3.nc  " >>gfs_restarta.txt
+    echo  "${dirname}INPUT/gfs_data.tile4.nc  " >>gfs_restarta.txt
+    echo  "${dirname}INPUT/gfs_data.tile5.nc  " >>gfs_restarta.txt
+    echo  "${dirname}INPUT/gfs_data.tile6.nc  " >>gfs_restarta.txt
+    echo  "${dirname}INPUT/sfc_data.tile1.nc  " >>gfs_restarta.txt
+    echo  "${dirname}INPUT/sfc_data.tile2.nc  " >>gfs_restarta.txt
+    echo  "${dirname}INPUT/sfc_data.tile3.nc  " >>gfs_restarta.txt
+    echo  "${dirname}INPUT/sfc_data.tile4.nc  " >>gfs_restarta.txt
+    echo  "${dirname}INPUT/sfc_data.tile5.nc  " >>gfs_restarta.txt
+    echo  "${dirname}INPUT/sfc_data.tile6.nc  " >>gfs_restarta.txt
+  fi
 
   #..................
   if [ $DO_WAVE = "YES" ]; then
diff --git a/ush/rocoto/setup_workflow_fcstonly.py b/ush/rocoto/setup_workflow_fcstonly.py
index 589ee4a569b..767984eb079 100755
--- a/ush/rocoto/setup_workflow_fcstonly.py
+++ b/ush/rocoto/setup_workflow_fcstonly.py
@@ -27,7 +27,7 @@
 import rocoto
 import workflow_utils as wfu
 
-taskplan = ['getic', 'init', 'waveinit', 'waveprep', 'fcst', 'post', 'wavepostsbs', 'wavepostbndpnt', 'wavepostpnt', 'wavegempak', 'waveawipsbulls', 'waveawipsgridded', 'wafs', 'wafsgrib2', 'wafsblending', 'wafsgcip', 'wafsgrib20p25', 'wafsblending0p25', 'vrfy', 'metp', 'arch']
+taskplan = ['getic', 'init', 'waveinit', 'waveprep', 'fcst', 'post', 'wavepostsbs', 'wavepostbndpnt', 'wavepostpnt', 'wavegempak', 'waveawipsbulls', 'waveawipsgridded', 'wafs', 'wafsgrib2', 'wafsblending', 'wafsgcip', 'wafsgrib20p25', 'wafsblending0p25', 'postsnd', 'gempak', 'awips', 'vrfy', 'metp', 'arch']
 
 def main():
     parser = ArgumentParser(description='Setup XML workflow and CRONTAB for a forecast only experiment.', formatter_class=ArgumentDefaultsHelpFormatter)
@@ -158,6 +158,7 @@ def get_resources(dict_configs, cdump='gdas'):
     scheduler = wfu.get_scheduler(machine)
 
     do_wave = base.get('DO_WAVE', 'NO').upper()
+    do_bufrsnd = base.get('DO_BUFRSND', 'NO').upper()
     do_gempak = base.get('DO_GEMPAK', 'NO').upper()
     do_awips = base.get('DO_AWIPS', 'NO').upper()
     do_metp = base.get('DO_METP', 'NO').upper()
@@ -237,6 +238,7 @@ def get_workflow(dict_configs, cdump='gdas'):
     machine = base.get('machine', wfu.detectMachine())
     do_wave = base.get('DO_WAVE', 'NO').upper()
     do_wave_cdump = base.get('WAVE_CDUMP', 'BOTH').upper()
+    do_bufrsnd = base.get('DO_BUFRSND', 'NO').upper()
     do_gempak = base.get('DO_GEMPAK', 'NO').upper()
     do_awips = base.get('DO_AWIPS', 'NO').upper()
     do_wafs = base.get('WAFSF', 'NO').upper()
@@ -267,6 +269,9 @@ def get_workflow(dict_configs, cdump='gdas'):
     data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/gfs.t@Hz.atmanl.nemsio'
     dep_dict = {'type':'data', 'data':data}
     deps.append(rocoto.add_dependency(dep_dict))
+    data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/gfs.t@Hz.atmanl.nc'
+    dep_dict = {'type':'data', 'data':data}
+    deps.append(rocoto.add_dependency(dep_dict))
     data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/RESTART/@Y@m@d.@H0000.sfcanl_data.tile6.nc'
     dep_dict = {'type':'data', 'data':data}
     deps.append(rocoto.add_dependency(dep_dict))
@@ -593,6 +598,44 @@ def get_workflow(dict_configs, cdump='gdas'):
         tasks.append(task)
         tasks.append('\n')
 
+    #postsnd
+    if do_bufrsnd in ['Y', 'YES']:
+        deps = []
+        dep_dict = {'type': 'task', 'name': '%sfcst' % cdump}
+        deps.append(rocoto.add_dependency(dep_dict))
+        dependencies = rocoto.create_dependency(dep=deps)
+        task = wfu.create_wf_task('postsnd', cdump=cdump, envar=envars, dependency=dependencies)
+        tasks.append(task)
+        tasks.append('\n')
+
+    # awips
+    if do_awips in ['Y', 'YES']:
+        deps = []
+        dep_dict = {'type': 'metatask', 'name': '%spost' % cdump}
+        deps.append(rocoto.add_dependency(dep_dict))
+        dependencies = rocoto.create_dependency(dep=deps)
+        fhrgrp = rocoto.create_envar(name='FHRGRP', value='#grp#')
+        fhrlst = rocoto.create_envar(name='FHRLST', value='#lst#')
+        ROTDIR = rocoto.create_envar(name='ROTDIR', value='&ROTDIR;')
+        awipsenvars = envars + [fhrgrp] + [fhrlst] + [ROTDIR]
+        varname1, varname2, varname3 = 'grp', 'dep', 'lst'
+        varval1, varval2, varval3 = get_awipsgroups(dict_configs['awips'], cdump=cdump)
+        vardict = {varname2: varval2, varname3: varval3}
+        task = wfu.create_wf_task('awips', cdump=cdump, envar=awipsenvars, dependency=dependencies,
+                                  metatask='awips', varname=varname1, varval=varval1, vardict=vardict)
+        tasks.append(task)
+        tasks.append('\n')
+
+    # gempak
+    if do_gempak in ['Y', 'YES']:
+        deps = []
+        dep_dict = {'type': 'metatask', 'name': '%spost' % cdump}
+        deps.append(rocoto.add_dependency(dep_dict))
+        dependencies = rocoto.create_dependency(dep=deps)
+        task = wfu.create_wf_task('gempak', cdump=cdump, envar=envars, dependency=dependencies)
+        tasks.append(task)
+        tasks.append('\n')
+
     # vrfy
     deps = []
     dep_dict = {'type':'metatask', 'name':'%spost' % cdump}

From d41796ae9d7024624078c5972aed032eea25ca65 Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Tue, 23 Feb 2021 18:37:10 +0000
Subject: [PATCH 017/125] Updates from testing on Hera

- add WGRIB2 setenv to module_base.hera; wgrib2 was unknown in post jobs without
- set RUN_CCPP default to YES; should be running with CCPP now, consider removing variable

Refs: #164
---
 modulefiles/module_base.hera    | 4 +++-
 parm/config/config.base.emc.dyn | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/modulefiles/module_base.hera b/modulefiles/module_base.hera
index ae4bd4ac2a0..e98363d92f4 100644
--- a/modulefiles/module_base.hera
+++ b/modulefiles/module_base.hera
@@ -8,7 +8,6 @@ module load hpc/1.1.0
 module load hpc-intel/18.0.5.274
 module load hpc-impi/2018.0.4
 
-module load wgrib2/2.0.8
 module load hpss/hpss
 module load nco/4.9.1
 module load gempak/7.4.2
@@ -25,6 +24,9 @@ module load netcdf/4.7.4
 module load hdf5/1.10.6
 module load esmf/8_1_0_beta_snapshot_27
 
+module load wgrib2/2.0.8
+setenv WGRIB2 wgrib2
+
 # python
 module use -a /contrib/anaconda/modulefiles
 module load anaconda/2.3.0
diff --git a/parm/config/config.base.emc.dyn b/parm/config/config.base.emc.dyn
index 37669926eed..8c79e743aa6 100755
--- a/parm/config/config.base.emc.dyn
+++ b/parm/config/config.base.emc.dyn
@@ -140,7 +140,7 @@ export CASE="@CASECTL@"
 export CASE_ENKF="@CASEENS@"
 
 # Run with CCPP physics
-export RUN_CCPP="NO"
+export RUN_CCPP="YES"
 export CCPP_SUITE="FV3_GFS_v16beta"
 
 # Surface cycle update frequency

From 95ec834eb3192b12cae979ef5696b9670cc84b72 Mon Sep 17 00:00:00 2001
From: "kate.friedman" <kate.friedman@noaa.gov>
Date: Thu, 4 Mar 2021 14:35:43 +0000
Subject: [PATCH 018/125] hpc-stack updates for WCOSS-Dell

- update OznMon, Radmon modulefiles
- update module_base.wcoss_dell_p3
- remove WW3 execs from link_fv3gfs.sh; will come from new build script
- update wcoss_dell_p3 section of machine-setup.sh for hpc-stack
- update checkout.sh for hpc-stack supported components (some test versions)

Refs: #164
---
 modulefiles/OznMonBuild.wcoss_dell_p3 |  5 ++---
 modulefiles/RadMonBuild.wcoss_dell_p3 |  4 ++--
 modulefiles/module_base.wcoss_dell_p3 | 25 ++++++++++++++++---------
 parm/config/config.base.emc.dyn       |  4 ++--
 sorc/checkout.sh                      | 12 ++++++++----
 sorc/link_fv3gfs.sh                   |  6 ------
 sorc/machine-setup.sh                 |  5 +++++
 7 files changed, 35 insertions(+), 26 deletions(-)

diff --git a/modulefiles/OznMonBuild.wcoss_dell_p3 b/modulefiles/OznMonBuild.wcoss_dell_p3
index efb31a7da31..dd561c24d4b 100644
--- a/modulefiles/OznMonBuild.wcoss_dell_p3
+++ b/modulefiles/OznMonBuild.wcoss_dell_p3
@@ -10,6 +10,5 @@ export FFLAGS="-O3 -fp-model strict -convert big_endian -assume byterecl"
 
 export D_FFLAGS="-O3 -fp-model strict -convert big_endian -assume byterecl -debug"
 
-module load  ips/18.0.1.163
-module load w3nco/2.0.6
-
+module load hpc-ips/18.0.1.163
+module load w3nco/2.4.1
diff --git a/modulefiles/RadMonBuild.wcoss_dell_p3 b/modulefiles/RadMonBuild.wcoss_dell_p3
index 9cece6e1166..bdf9f7ac042 100644
--- a/modulefiles/RadMonBuild.wcoss_dell_p3
+++ b/modulefiles/RadMonBuild.wcoss_dell_p3
@@ -11,5 +11,5 @@ export FFLAGS="-O3 -fp-model strict -D_REAL8_ -traceback -convert big_endian -as
 export CHECK_LIBS="/gpfs/hps/nco/ops/nwprod/spa_util/check_libs.bash"
 export D_FFLAGS="-O3 -fp-model strict -D_REAL8_ -traceback -convert big_endian -assume byterecl -debug"
 
-module load ips/18.0.1.163
-module load w3nco/2.0.6
+module load hpc-ips/18.0.1.163
+module load w3nco/2.4.1
diff --git a/modulefiles/module_base.wcoss_dell_p3 b/modulefiles/module_base.wcoss_dell_p3
index f3a76c9a226..e9807c11989 100644
--- a/modulefiles/module_base.wcoss_dell_p3
+++ b/modulefiles/module_base.wcoss_dell_p3
@@ -4,29 +4,35 @@
 
 # From default environment
 
-module load ips/18.0.1.163
-module load impi/18.0.1    
+module use /usrx/local/nceplibs/dev/hpc-stack/libs/hpc-stack/modulefiles/stack
+module load hpc/1.1.0
+module load hpc-ips/18.0.1.163
+module load hpc-impi/18.0.1
+
 module load lsf/10.1       
 module load EnvVars/1.0.3
 
 module load HPSS/5.0.2.5
-module load prod_util/1.1.4
+
 module load prod_envir/1.1.0
-module unload grib_util/1.0.6   
-module load grib_util/1.1.0
+module load prod_util/1.2.2
+module load grib_util/1.2.2
+module load g2tmpl/1.9.1
+
+module load wgrib2/2.0.8
+setenv WGRIB2 wgrib2
 
 module load NCO/4.7.0   
 module load CFP/2.0.2
 setenv USE_CFP YES
 module load pm5
 
-module load NetCDF-parallel/4.7.4
-module load ESMF/8.0.1
-module load HDF5-parallel/1.10.6
+module load netcdf/4.7.4
+module load esmf/8_1_0_beta_snapshot_27
+module load hdf5/1.10.6
 
 module use -a /gpfs/dell1/nco/ops/nwprod/modulefiles/
 module load gempak/7.3.3
-module load g2tmpl/1.6.0
 
 # Load for WAFS
 module load bufr_dumplist/2.0.0
@@ -38,6 +44,7 @@ module load python/3.6.3
 
 # Load crtm
 module load crtm/2.3.0
+setenv CRTM_FIX /gpfs/dell1/nco/ops/nwprod/lib/crtm/v2.3.0/fix
 
 # waveprep
 module load cdo/1.9.8
diff --git a/parm/config/config.base.emc.dyn b/parm/config/config.base.emc.dyn
index 8c79e743aa6..84255366a71 100755
--- a/parm/config/config.base.emc.dyn
+++ b/parm/config/config.base.emc.dyn
@@ -69,8 +69,8 @@ export REALTIME="YES"
 export FIXgsi="$HOMEgfs/fix/fix_gsi"
 export HOMEfv3gfs="$HOMEgfs/sorc/fv3gfs.fd"
 export HOMEpost="$HOMEgfs"
-export HOMEobsproc_prep="$BASE_GIT/obsproc/gfsv16b/obsproc_prep_RB-5.4.0"
-export HOMEobsproc_network="$BASE_GIT/obsproc/gfsv16b/obsproc_global_RB-3.4.0"
+export HOMEobsproc_prep="$BASE_GIT/obsproc/obsproc_prep_RB-5.4.0_hpc-stack"
+export HOMEobsproc_network="$BASE_GIT/obsproc/obsproc_global_RB-3.4.0_hpc-stack"
 export HOMEobsproc_global=$HOMEobsproc_network
 export BASE_VERIF="$BASE_GIT/verif/global/tags/vsdb"
 
diff --git a/sorc/checkout.sh b/sorc/checkout.sh
index 8eb26f0dbbe..21f5de156c0 100755
--- a/sorc/checkout.sh
+++ b/sorc/checkout.sh
@@ -37,9 +37,11 @@ fi
 echo gsi checkout ...
 if [[ ! -d gsi.fd ]] ; then
     rm -f ${topdir}/checkout-gsi.log
-    git clone --recursive https://github.com/NOAA-EMC/GSI.git gsi.fd >> ${topdir}/checkout-gsi.log 2>&1
+    #git clone --recursive https://github.com/NOAA-EMC/GSI.git gsi.fd >> ${topdir}/checkout-gsi.log 2>&1
+    git clone --recursive https://github.com/RussTreadon-NOAA/GSI.git gsi.fd >> ${topdir}/checkout-gsi.log 2>&1
     cd gsi.fd
-    git checkout gfsda.v16.0.0
+    #git checkout gfsda.v16.0.0
+    git checkout feature/hpc-stack
     git submodule update
     cd ${topdir}
 else
@@ -60,9 +62,11 @@ fi
 echo ufs_utils checkout ...
 if [[ ! -d ufs_utils.fd ]] ; then
     rm -f ${topdir}/checkout-ufs_utils.log
-    git clone --recursive https://github.com/NOAA-EMC/UFS_UTILS.git ufs_utils.fd >> ${topdir}/checkout-ufs_utils.fd.log 2>&1
+    #git clone --recursive https://github.com/NOAA-EMC/UFS_UTILS.git ufs_utils.fd >> ${topdir}/checkout-ufs_utils.fd.log 2>&1
+    git clone --recursive https://github.com/GeorgeGayno-NOAA/UFS_UTILS.git ufs_utils.fd >> ${topdir}/checkout-ufs_utils.fd.log 2>&1
     cd ufs_utils.fd
     #git checkout ops-gfsv16.0.0
+    git checkout feature/stack
     cd ${topdir}
 else
     echo 'Skip.  Directory ufs_utils.fd already exists.'
@@ -73,7 +77,7 @@ if [[ ! -d gfs_post.fd ]] ; then
     rm -f ${topdir}/checkout-gfs_post.log
     git clone https://github.com/NOAA-EMC/EMC_post.git gfs_post.fd >> ${topdir}/checkout-gfs_post.log 2>&1
     cd gfs_post.fd
-    git checkout upp_v10.0.3
+    git checkout 33eb60e5b486d9ff670c76b799f886b7f47c2022
     git submodule update --init CMakeModules
 
     ################################################################################
diff --git a/sorc/link_fv3gfs.sh b/sorc/link_fv3gfs.sh
index 01aaaa68e06..db79e2e7538 100755
--- a/sorc/link_fv3gfs.sh
+++ b/sorc/link_fv3gfs.sh
@@ -213,12 +213,6 @@ done
 
 [[ -s global_fv3gfs.x ]] && rm -f global_fv3gfs.x
 $LINK ../sorc/fv3gfs.fd/NEMS/exe/global_fv3gfs.x .
-if [ -d ../sorc/fv3gfs.fd/WW3/exec ]; then # Wave execs
-  for waveexe in ww3_gint ww3_grib ww3_grid ww3_multi ww3_ounf ww3_ounp ww3_outf ww3_outp ww3_prep ww3_prnc; do
-    [[ -s $waveexe ]] && rm -f $waveexe
-    $LINK ../sorc/fv3gfs.fd/WW3/exec/$waveexe .
-  done
-fi
 
 [[ -s gfs_ncep_post ]] && rm -f gfs_ncep_post
 $LINK ../sorc/gfs_post.fd/exec/upp.x gfs_ncep_post
diff --git a/sorc/machine-setup.sh b/sorc/machine-setup.sh
index b934f361b8b..b2bdb85c2e0 100644
--- a/sorc/machine-setup.sh
+++ b/sorc/machine-setup.sh
@@ -95,7 +95,12 @@ elif [[ -L /usrx && "$( readlink /usrx 2> /dev/null )" =~ dell ]] ; then
 	source /usrx/local/prod/lmod/lmod/init/$__ms_shell
     fi
     target=wcoss_dell_p3
+
     module purge 
+    module use module use /usrx/local/nceplibs/dev/hpc-stack/libs/hpc-stack/modulefiles/stack
+    module load hpc/1.1.0
+    module load hpc-ips/18.0.1.163
+    module load hpc-impi/18.0.1
 
 ##---------------------------------------------------------------------------
 

From fd50508f84d3ec721747a069412c929b7269a602 Mon Sep 17 00:00:00 2001
From: "kate.friedman" <kate.friedman@noaa.gov>
Date: Thu, 4 Mar 2021 08:43:37 -0600
Subject: [PATCH 019/125] hpc-stack updates for Orion

- update OznMon and RadMon modulefiles
- update module_base.orion
- update Orion if-block in machine-setup.sh
- remove duplicate module use for WCOSS-Dell in machine-setup.sh

Refs: #164
---
 modulefiles/OznMonBuild.orion |  7 ++----
 modulefiles/RadMonBuild.orion |  6 ++----
 modulefiles/module_base.orion | 40 +++++++++++++++++------------------
 sorc/machine-setup.sh         | 13 ++++++------
 4 files changed, 30 insertions(+), 36 deletions(-)

diff --git a/modulefiles/OznMonBuild.orion b/modulefiles/OznMonBuild.orion
index efac4df0576..f88717d1817 100644
--- a/modulefiles/OznMonBuild.orion
+++ b/modulefiles/OznMonBuild.orion
@@ -10,8 +10,5 @@ export FFLAGS="-O3 -fp-model strict -convert big_endian -assume byterecl"
 
 export D_FFLAGS="-O3 -fp-model strict -convert big_endian -assume byterecl -debug"
 
-module load intel/2018.4
-#module load w3nco/2.0.6
-module use -a /work/noaa/noaatest/NCEPLIBS/lib/modulefiles
-module load w3nco-intel-sandybridge/2.0.6
-
+module load hpc-intel/2018.4
+module load w3nco/2.4.1
diff --git a/modulefiles/RadMonBuild.orion b/modulefiles/RadMonBuild.orion
index 932bb6f95ba..411f2d8c8d8 100644
--- a/modulefiles/RadMonBuild.orion
+++ b/modulefiles/RadMonBuild.orion
@@ -11,7 +11,5 @@ export FFLAGS="-O3 -fp-model strict -D_REAL8_ -traceback -convert big_endian -as
 export CHECK_LIBS="/gpfs/hps/nco/ops/nwprod/spa_util/check_libs.bash"
 export D_FFLAGS="-O3 -fp-model strict -D_REAL8_ -traceback -convert big_endian -assume byterecl -debug"
 
-module load intel/2018.4
-#module load w3nco/2.0.6
-module use -a /work/noaa/noaatest/NCEPLIBS/lib/modulefiles
-module load w3nco-intel-sandybridge/2.0.6
+module load hpc-intel/2018.4
+module load w3nco/2.4.1
diff --git a/modulefiles/module_base.orion b/modulefiles/module_base.orion
index facf50d4009..0161f153fd9 100755
--- a/modulefiles/module_base.orion
+++ b/modulefiles/module_base.orion
@@ -3,37 +3,35 @@
 ##      FV3GFS prerequisites
 ##
 
-module load intel/2018.4
-module load impi/2018.4
+module use /apps/contrib/NCEP/libs/hpc-stack/modulefiles/stack
+module load hpc/1.1.0
+module load hpc-intel/2018.4
+module load hpc-impi/2018.4
+
 module load wgrib/2.0.8
+setenv WGRIB2 wgrib2
+
 #module load hpss/hpss
 module load nco/4.8.1
-module load cdo/1.9.5
-
 module load gempak/7.5.1
 
-#Load from official NCEPLIBS
-module use /apps/contrib/NCEPLIBS/orion/modulefiles
-module load grib_util/1.2.0
-module load prod_util/1.2.0
-module load g2tmpl/1.6.0
+module load grib_util/1.2.2
+module load prod_util/1.2.2
+module load g2tmpl/1.9.1
+
 module load crtm/2.3.0
-#module load netcdf_parallel/4.7.4
-module load hdf5_parallel/1.10.6
-
-## load ESMF library for above compiler / MPI combination
-### use pre-compiled EMSF library for above compiler / MPI combination
-###
-module use /apps/contrib/NCEPLIBS/lib/modulefiles
-module load netcdfp/4.7.4 
-#module load hdf5_parallel/1.10.6
-module load esmflocal/8.0.1.08bs
-module load post-intel-sandybridge/8.0.5
+setenv CRTM_FIX /apps/contrib/NCEPLIBS/orion/fix/crtm_v2.3.0
+
+module load hdf5/1.10.6
+module load netcdf/4.7.4
+module load esmf/8_1_0_beta_snapshot_27
 
 module load contrib
-module load rocoto/1.3.2
+module load rocoto/1.3.3
 module load slurm/19.05.3-2
 
 # Python
 module load python/3.7.5
 
+# waveprep
+module load cdo/1.9.5
diff --git a/sorc/machine-setup.sh b/sorc/machine-setup.sh
index b2bdb85c2e0..9ce289a6b02 100644
--- a/sorc/machine-setup.sh
+++ b/sorc/machine-setup.sh
@@ -27,12 +27,13 @@ if [[ -d /work ]] ; then
         source /apps/lmod/lmod/init/$__ms_shell
     fi
     target=orion
+
     module purge
-    module load intel/2018.4
-    module load impi/2018.4
-    export NCEPLIBS=/apps/contrib/NCEPLIBS/orion
-    export WRFPATH=$NCEPLIBS/wrf.shared.new/v1.1.1/src
-    module use $NCEPLIBS/modulefiles
+    module use /apps/contrib/NCEP/libs/hpc-stack/modulefiles/stack
+    module load hpc/1.1.0
+    module load hpc-intel/2018.4
+    module load hpc-impi/2018.4
+
     export myFC=mpiifort
     export FCOMP=mpiifort
 
@@ -97,7 +98,7 @@ elif [[ -L /usrx && "$( readlink /usrx 2> /dev/null )" =~ dell ]] ; then
     target=wcoss_dell_p3
 
     module purge 
-    module use module use /usrx/local/nceplibs/dev/hpc-stack/libs/hpc-stack/modulefiles/stack
+    module use /usrx/local/nceplibs/dev/hpc-stack/libs/hpc-stack/modulefiles/stack
     module load hpc/1.1.0
     module load hpc-ips/18.0.1.163
     module load hpc-impi/18.0.1

From d32199cd58329b4b2c9c5c3e84e7d8d0801305e0 Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Thu, 4 Mar 2021 17:43:58 +0000
Subject: [PATCH 020/125] Move getic/init settings from scripts to configs

- consolidate RETRO variable in config.getic
- move configuration settings from getic/init scripts to configs
- add dependency to setup_workflow_fcstonly.py
- add getic and init to task list at top of config.resources

Refs: #178
---
 jobs/rocoto/getic.sh                  | 54 +++++----------------------
 jobs/rocoto/init.sh                   |  2 +-
 parm/config/config.getic              | 36 ++++++++++++++++--
 parm/config/config.init               | 35 ++++++++++++++---
 parm/config/config.resources          |  1 +
 ush/rocoto/setup_workflow_fcstonly.py |  3 ++
 6 files changed, 77 insertions(+), 54 deletions(-)

diff --git a/jobs/rocoto/getic.sh b/jobs/rocoto/getic.sh
index 8a8dfee7004..6523acce5cf 100755
--- a/jobs/rocoto/getic.sh
+++ b/jobs/rocoto/getic.sh
@@ -20,7 +20,7 @@ status=$?
 
 ###############################################################
 # Source relevant configs
-configs="base init"
+configs="base getic init"
 for config in $configs; do
     . $EXPDIR/config.${config}
     status=$?
@@ -36,53 +36,19 @@ status=$?
 ###############################################################
 # Set script and dependency variables
 
-yy=$(echo $CDATE | cut -c1-4)
-mm=$(echo $CDATE | cut -c5-6)
-dd=$(echo $CDATE | cut -c7-8)
-hh=${cyc:-$(echo $CDATE | cut -c9-10)}
+export yy=$(echo $CDATE | cut -c1-4)
+export mm=$(echo $CDATE | cut -c5-6)
+export dd=$(echo $CDATE | cut -c7-8)
+export hh=${cyc:-$(echo $CDATE | cut -c9-10)}
 
 export DATA=${DATA:-${DATAROOT}/init}
-
 export EXTRACT_DIR=${EXTRACT_DIR:-$ROTDIR}
 export WORKDIR=${WORKDIR:-$DATA}
 export OUTDIR=${OUTDIR:-$ROTDIR}
-PRODHPSSDIR=/NCEPPROD/hpssprod/runhistory/rh${yy}/${yy}${mm}/${yy}${mm}${dd}
-
-COMPONENT="atmos"
-
-gfs_ver=v16
-GETICSH=${GDASINIT_DIR}/get_v16.data.sh
-
-RETRO=${RETRO:-"NO"}
-if [ $RETRO = "YES" ]; then
-  GETICSH=${GDASINIT_DIR}/get_v16retro.data.sh
-fi
-
-if [ $RETRO = "NO" ]; then # Operational input
-  # No ENKF data prior to 2012/05/21/00z
-  if [ $yy$mm$dd$hh -lt 2012052100 ]; then
-    set +x
-    echo FATAL ERROR: SCRIPTS DO NOT SUPPORT OLD GFS DATA
-    exit 2
-  elif [ $yy$mm$dd$hh -lt 2016051000 ]; then
-    gfs_ver=v12
-    GETICSH=${GDASINIT_DIR}/get_pre-v14.data.sh
-  elif [ $yy$mm$dd$hh -lt 2017072000 ]; then
-    gfs_ver=v13
-    GETICSH=${GDASINIT_DIR}/get_pre-v14.data.sh
-  elif [ $yy$mm$dd$hh -lt 2019061200 ]; then
-    gfs_ver=v14
-    GETICSH=${GDASINIT_DIR}/get_${gfs_ver}.data.sh
-    tarball=gpfs_hps_nco_ops_com_gfs_prod_gfs.${yy}${mm}${dd}_${hh}.pgrb2_${grid}.tar
-  elif [ $yy$mm$dd$hh -lt 2021031700 ]; then
-    gfs_ver=v15
-    GETICSH=${GDASINIT_DIR}/get_${gfs_ver}.data.sh
-    tarball=com_gfs_prod_gfs.${yy}${mm}${dd}_${hh}.gfs_pgrb2.tar
-  fi
-fi
-
-export EXTRACT_DIR yy mm dd hh UFS_DIR OUTDIR CRES_HIRES CRES_ENKF
-export LEVS gfs_ver
+export PRODHPSSDIR=${PRODHPSSDIR:-/NCEPPROD/hpssprod/runhistory}
+export COMPONENT="atmos"
+export gfs_ver=${gfs_ver:-v16}
+export GETICSH=${GETICSH:-${GDASINIT_DIR}/get_v16.data.sh}
 
 # Run get data script
 if [ ! -d $EXTRACT_DIR ]; then mkdir -p $EXTRACT_DIR ; fi
@@ -96,7 +62,7 @@ if [ $gfs_ver = v14 -o $gfs_ver = v15 ]; then
   for grid in 0p25 0p50 1p00
   do
     file=gfs.t${hh}z.pgrb2.${grid}.anl
-    htar -xvf ${PRODHPSSDIR}/${tarball} ./${CDUMP}.${yy}${mm}${dd}/${hh}/${file}
+    htar -xvf ${PRODHPSSDIR}/rh${yy}/${yy}${mm}/${yy}${mm}${dd}/${tarball} ./${CDUMP}.${yy}${mm}${dd}/${hh}/${file}
     mv ${EXTRACT_DIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${file} ${OUTDIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${COMPONENT}/${file}
   done
 fi
diff --git a/jobs/rocoto/init.sh b/jobs/rocoto/init.sh
index b944216a838..1671bbb1057 100755
--- a/jobs/rocoto/init.sh
+++ b/jobs/rocoto/init.sh
@@ -20,7 +20,7 @@ status=$?
 
 ###############################################################
 # Source relevant configs
-configs="base init"
+configs="base getic init"
 for config in $configs; do
     . $EXPDIR/config.${config}
     status=$?
diff --git a/parm/config/config.getic b/parm/config/config.getic
index 52498a01d3d..ec396849ae7 100755
--- a/parm/config/config.getic
+++ b/parm/config/config.getic
@@ -8,11 +8,39 @@ echo "BEGIN: config.getic"
 # Get task specific resources
 . $EXPDIR/config.resources getic
 
-UFS_DIR=${HOMEgfs}/sorc/ufs_utils.fd
-GDASINIT_DIR=${UFS_DIR}/util/gdas_init
+export RETRO="NO" # YES = Pull v16 inputs from retrospective parallels; NO = use operational inputs
+export gfs_ver="v16" # Default = v16
 
-EXTRACT_DATA="yes"
+export UFS_DIR=${HOMEgfs}/sorc/ufs_utils.fd
+export GDASINIT_DIR=${UFS_DIR}/util/gdas_init
 
-RETRO="NO" # YES = Pull v16 inputs from retrospective parallels; NO = use operational inputs
+export PRODHPSSDIR=/NCEPPROD/hpssprod/runhistory
+export GETICSH=${GDASINIT_DIR}/get_v16.data.sh
+
+if [ ${RETRO:-"NO"} = "YES" ]; then
+  export GETICSH=${GDASINIT_DIR}/get_v16retro.data.sh
+fi
+
+if [ ${RETRO:-"NO"} = "NO" ]; then # Operational input
+  # No ENKF data prior to 2012/05/21/00z
+  if [[ "$CDATE" -lt "2012052100" ]]; then
+    set +x
+    echo FATAL ERROR: SCRIPTS DO NOT SUPPORT OLD GFS DATA
+  elif [[ "$CDATE" -lt "2016051000" ]]; then
+    export gfs_ver=v12
+    export GETICSH=${GDASINIT_DIR}/get_pre-v14.data.sh
+  elif [[ "$CDATE" -lt "2017072000" ]]; then
+    export gfs_ver=v13
+    export GETICSH=${GDASINIT_DIR}/get_pre-v14.data.sh
+  elif [[ "$CDATE" -lt "2019061200" ]]; then
+    export gfs_ver=v14
+    export GETICSH=${GDASINIT_DIR}/get_${gfs_ver}.data.sh
+    export tarball=gpfs_hps_nco_ops_com_gfs_prod_gfs.${PDY}_${cyc}.pgrb2_${grid}.tar
+  elif [[ "$CDATE" -lt "2021031700" ]]; then
+    export gfs_ver=v15
+    export GETICSH=${GDASINIT_DIR}/get_${gfs_ver}.data.sh
+    export tarball=com_gfs_prod_gfs.${PDY}_${cyc}.gfs_pgrb2.tar
+  fi
+fi
 
 echo "END: config.getic"
diff --git a/parm/config/config.init b/parm/config/config.init
index a473d9747a5..212fa422df3 100755
--- a/parm/config/config.init
+++ b/parm/config/config.init
@@ -8,12 +8,37 @@ echo "BEGIN: config.init"
 # Get task specific resources
 . $EXPDIR/config.resources init
 
-UFS_DIR=${HOMEgfs}/sorc/ufs_utils.fd
-GDASINIT_DIR=${UFS_DIR}/util/gdas_init
+# Get task specific resources
+. $EXPDIR/config.getic
+
+export UFS_DIR=${HOMEgfs}/sorc/ufs_utils.fd
+export GDASINIT_DIR=${UFS_DIR}/util/gdas_init
+
+export CRES_HIRES=$CASE
+export CRES_ENKF=$CASE_ENKF
 
-RETRO="NO" # YES = Pull v16 inputs from retrospective parallels; NO = use operational inputs
+if [ ${RETRO:-"NO"} = "YES" ]; then
+  export GETICSH=${GDASINIT_DIR}/run_v16retro.chgres.sh
+fi
 
-CRES_HIRES=$CASE
-CRES_ENKF=$CASE_ENKF
+if [ ${RETRO:-"NO"} = "NO" ]; then # Operational input
+  # No ENKF data prior to 2012/05/21/00z
+  if [[ "$CDATE" -lt "2012052100" ]]; then
+    set +x
+    echo FATAL ERROR: SCRIPTS DO NOT SUPPORT OLD GFS DATA
+  elif [[ "$CDATE" -lt "2016051000" ]]; then
+    export gfs_ver=v12
+    export RUNICSH=${GDASINIT_DIR}/run_pre-v14.chgres.sh
+  elif [[ "$CDATE" -lt "2017072000" ]]; then
+    export gfs_ver=v13
+    export RUNICSH=${GDASINIT_DIR}/run_pre-v14.chgres.sh
+  elif [[ "$CDATE" -lt "2019061200" ]]; then
+    export gfs_ver=v14
+    export RUNICSH=${GDASINIT_DIR}/run_${gfs_ver}.chgres.sh
+  elif [[ "$CDATE" -lt "2021031700" ]]; then
+    export gfs_ver=v15
+    export RUNICSH=${GDASINIT_DIR}/run_${gfs_ver}.chgres.gfs.sh
+  fi
+fi
 
 echo "END: config.init"
diff --git a/parm/config/config.resources b/parm/config/config.resources
index d62424aa324..ac081db9768 100755
--- a/parm/config/config.resources
+++ b/parm/config/config.resources
@@ -8,6 +8,7 @@ if [ $# -ne 1 ]; then
 
     echo "Must specify an input task argument to set resource variables!"
     echo "argument can be any one of the following:"
+    echo "getic init"
     echo "anal analcalc analdiag gldas fcst post vrfy metp arch echgres"
     echo "eobs ediag eomg eupd ecen esfc efcs epos earc"
     echo "waveinit waveprep wavepostsbs wavepostbndpnt wavepostpnt"
diff --git a/ush/rocoto/setup_workflow_fcstonly.py b/ush/rocoto/setup_workflow_fcstonly.py
index 767984eb079..195734e1af0 100755
--- a/ush/rocoto/setup_workflow_fcstonly.py
+++ b/ush/rocoto/setup_workflow_fcstonly.py
@@ -272,6 +272,9 @@ def get_workflow(dict_configs, cdump='gdas'):
     data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/gfs.t@Hz.atmanl.nc'
     dep_dict = {'type':'data', 'data':data}
     deps.append(rocoto.add_dependency(dep_dict))
+    data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/gfs.t@Hz.atmanl.nc'
+    dep_dict = {'type':'data', 'data':data}
+    deps.append(rocoto.add_dependency(dep_dict))
     data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/RESTART/@Y@m@d.@H0000.sfcanl_data.tile6.nc'
     dep_dict = {'type':'data', 'data':data}
     deps.append(rocoto.add_dependency(dep_dict))

From be0ed38a53d2422923ccd5380d610785e61deafa Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Thu, 4 Mar 2021 18:06:24 +0000
Subject: [PATCH 021/125] Consolidate/cleanup init config and script

Refs: #178
---
 jobs/rocoto/init.sh     | 46 +++++++----------------------------------
 parm/config/config.init |  2 +-
 2 files changed, 8 insertions(+), 40 deletions(-)

diff --git a/jobs/rocoto/init.sh b/jobs/rocoto/init.sh
index 1671bbb1057..2fb81b268b6 100755
--- a/jobs/rocoto/init.sh
+++ b/jobs/rocoto/init.sh
@@ -36,50 +36,18 @@ status=$?
 ###############################################################
 # Set script and dependency variables
 
-yy=$(echo $CDATE | cut -c1-4)
-mm=$(echo $CDATE | cut -c5-6)
-dd=$(echo $CDATE | cut -c7-8)
-hh=${cyc:-$(echo $CDATE | cut -c9-10)}
+export yy=$(echo $CDATE | cut -c1-4)
+export mm=$(echo $CDATE | cut -c5-6)
+export dd=$(echo $CDATE | cut -c7-8)
+export hh=${cyc:-$(echo $CDATE | cut -c9-10)}
 
 export DATA=${DATA:-${DATAROOT}/init}
-
 export EXTRACT_DIR=${EXTRACT_DIR:-$ROTDIR}
 export WORKDIR=${WORKDIR:-$DATA}
 export OUTDIR=${OUTDIR:-$ROTDIR}
-
-COMPONENT="atmos"
-
-gfs_ver=v16
-RUNICSH=${GDASINIT_DIR}/run_v16.chgres.sh
-
-RETRO=${RETRO:-"NO"}
-if [ $RETRO = "YES" ]; then
-  GETICSH=${GDASINIT_DIR}/run_v16retro.chgres.sh
-fi
-
-if [ $RETRO = "NO" ]; then # Operational input
-  # No ENKF data prior to 2012/05/21/00z
-  if [ $yy$mm$dd$hh -lt 2012052100 ]; then
-    set +x
-    echo FATAL ERROR: SCRIPTS DO NOT SUPPORT OLD GFS DATA
-    exit 2
-  elif [ $yy$mm$dd$hh -lt 2016051000 ]; then
-    gfs_ver=v12
-    RUNICSH=${GDASINIT_DIR}/run_pre-v14.chgres.sh
-  elif [ $yy$mm$dd$hh -lt 2017072000 ]; then
-    gfs_ver=v13
-    RUNICSH=${GDASINIT_DIR}/run_pre-v14.chgres.sh
-  elif [ $yy$mm$dd$hh -lt 2019061200 ]; then
-    gfs_ver=v14
-    RUNICSH=${GDASINIT_DIR}/run_${gfs_ver}.chgres.sh
-  elif [ $yy$mm$dd$hh -lt 2021031700 ]; then
-    gfs_ver=v15
-    RUNICSH=${GDASINIT_DIR}/run_${gfs_ver}.chgres.gfs.sh
-  fi
-fi
-
-export EXTRACT_DIR yy mm dd hh UFS_DIR OUTDIR CRES_HIRES CRES_ENKF
-export LEVS gfs_ver
+export COMPONENT="atmos"
+export gfs_ver=${gfs_ver:-v16}
+export RUNICSH=${RUNICSH:-${GDASINIT_DIR}/run_v16.chgres.sh}
 
 # Check if init is needed and run if so
 if [[ $gfs_ver = "v16" && $EXP_WARM_START = ".true." && $CASE = "C768" ]]; then
diff --git a/parm/config/config.init b/parm/config/config.init
index 212fa422df3..a9932028a56 100755
--- a/parm/config/config.init
+++ b/parm/config/config.init
@@ -18,7 +18,7 @@ export CRES_HIRES=$CASE
 export CRES_ENKF=$CASE_ENKF
 
 if [ ${RETRO:-"NO"} = "YES" ]; then
-  export GETICSH=${GDASINIT_DIR}/run_v16retro.chgres.sh
+  export RUNICSH=${GDASINIT_DIR}/run_v16retro.chgres.sh
 fi
 
 if [ ${RETRO:-"NO"} = "NO" ]; then # Operational input

From efdf917a843cc301a2fe7d5386aea1e29dd02c46 Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Fri, 5 Mar 2021 21:29:22 +0000
Subject: [PATCH 022/125] Update in wave prep script and add ability to pull
 warm starts

- Update getic job to detect warm start config info and pull warm start RESTART
files off HPSS and place in ROTDIR; just for retro ICs, will add ops after implementation
- Add check to JGLOBAL_WAVE_PREP for prior cycle rtofs ROTDIR symlink;
normally cycled mode would have already created this during prior cycle
so added check for use in free-forecast mode

Refs: #178
---
 jobs/JGLOBAL_WAVE_PREP   |  4 ++++
 jobs/rocoto/getic.sh     | 36 +++++++++++++++++++++++++++++++++---
 parm/config/config.getic | 17 +++++++++++++----
 3 files changed, 50 insertions(+), 7 deletions(-)

diff --git a/jobs/JGLOBAL_WAVE_PREP b/jobs/JGLOBAL_WAVE_PREP
index 4fe921c9aba..138a3bf64d8 100755
--- a/jobs/JGLOBAL_WAVE_PREP
+++ b/jobs/JGLOBAL_WAVE_PREP
@@ -77,6 +77,10 @@ else
   if [ ! -L $ROTDIR/${WAVECUR_DID}.${RPDY} ]; then # Check if symlink already exists in ROTDIR
     $NLN $DMPDIR/${WAVECUR_DID}.${RPDY} $ROTDIR/${WAVECUR_DID}.${RPDY}
   fi
+  BRPDY=`$NDATE -24 ${RPDY}00 | cut -c1-8`
+  if [ ! -L $ROTDIR/${WAVECUR_DID}.${BRPDY} ]; then # Check if symlink already exists in ROTDIR
+    $NLN $DMPDIR/${WAVECUR_DID}.${BRPDY} $ROTDIR/${WAVECUR_DID}.${BRPDY}
+  fi
   if [ ! -L $ROTDIR/${CDUMP}.${PDY}/${cyc}/atmos/${WAVICEFILE} ]; then # Check if symlink already exists in ROTDIR
     $NLN $DMPDIR/$CDUMP.${PDY}/$cyc/${WAVICEFILE} $ROTDIR/$CDUMP.${PDY}/$cyc/atmos/${WAVICEFILE}
   fi
diff --git a/jobs/rocoto/getic.sh b/jobs/rocoto/getic.sh
index 6523acce5cf..a890b0de200 100755
--- a/jobs/rocoto/getic.sh
+++ b/jobs/rocoto/getic.sh
@@ -52,9 +52,39 @@ export GETICSH=${GETICSH:-${GDASINIT_DIR}/get_v16.data.sh}
 
 # Run get data script
 if [ ! -d $EXTRACT_DIR ]; then mkdir -p $EXTRACT_DIR ; fi
-sh ${GETICSH} ${CDUMP}
-status=$?
-[[ $status -ne 0 ]] && exit $status
+
+# Check if init is needed and run if so
+if [[ $gfs_ver = "v16" && $EXP_WARM_START = ".true." && $CASE = "C768" ]]; then
+  # Pull RESTART files off HPSS
+
+  if [ ! -d $ROTDIR ]; then mkdir $ROTDIR ; fi
+  cd $ROTDIR
+
+  if [ ${RETRO:-"NO"} = "YES" ]; then # Retrospective parallel input
+
+    # Pull prior cycle restart files
+    BDATE=`$NDATE -06 ${yy}${mm}${dd}${hh}`
+    htar -xvf ${HPSSDIR}/${BDATE}/gdas_restartb.tar
+    status=$?
+    [[ $status -ne 0 ]] && exit $status
+
+    # Pull current cycle restart files
+    htar -xvf ${HPSSDIR}/${CDATE}/gfs_restarta.tar
+    status=$?
+    [[ $status -ne 0 ]] && exit $status
+
+# else # Opertional input
+#   # ADD AFTER IMPLEMENTATION
+  fi
+
+else
+
+  # Run UFS_UTILS GETICSH
+  sh ${GETICSH} ${CDUMP}
+  status=$?
+  [[ $status -ne 0 ]] && exit $status
+
+fi
 
 # Pull pgbanl file for verification/archival - v14+
 if [ $gfs_ver = v14 -o $gfs_ver = v15 ]; then
diff --git a/parm/config/config.getic b/parm/config/config.getic
index ec396849ae7..b86f9c4aebb 100755
--- a/parm/config/config.getic
+++ b/parm/config/config.getic
@@ -17,11 +17,20 @@ export GDASINIT_DIR=${UFS_DIR}/util/gdas_init
 export PRODHPSSDIR=/NCEPPROD/hpssprod/runhistory
 export GETICSH=${GDASINIT_DIR}/get_v16.data.sh
 
-if [ ${RETRO:-"NO"} = "YES" ]; then
+if [ ${RETRO:-"NO"} = "YES" ]; then # Retrospective parallel input
   export GETICSH=${GDASINIT_DIR}/get_v16retro.data.sh
-fi
-
-if [ ${RETRO:-"NO"} = "NO" ]; then # Operational input
+  if [[ "$CDATE" -lt "2019060106" ]]; then
+    HPSSDIR=/NCEPDEV/emc-global/5year/emc.glopara/WCOSS_D/gfsv16/v16retro0e
+  elif [[ "$CDATE" -lt "2019090100" ]]; then
+    HPSSDIR=/NCEPDEV/emc-global/5year/emc.glopara/WCOSS_D/gfsv16/v16retro1e
+  elif [[ "$CDATE" -lt "2019101706" ]]; then
+    HPSSDIR=/NCEPDEV/emc-global/5year/emc.glopara/WCOSS_D/gfsv16/v16retro2e
+  elif [[ "$CDATE" -lt "2020122206" ]]; then
+    HPSSDIR=/NCEPDEV/emc-global/5year/emc.glopara/WCOSS_D/gfsv16/v16rt2
+  else
+    HPSSDIR=/NCEPDEV/emc-global/5year/emc.gloparadev/WCOSS_D/gfsv16/v16rt2n
+  fi
+elif [ ${RETRO:-"NO"} = "NO" ]; then # Operational input
   # No ENKF data prior to 2012/05/21/00z
   if [[ "$CDATE" -lt "2012052100" ]]; then
     set +x

From 86b0345a550a7128fd6420befcb793a94411e4db Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Mon, 8 Mar 2021 14:09:40 +0000
Subject: [PATCH 023/125] Add pull of IAU increment files off HPSS for
 free-forecast

Refs: #178
---
 jobs/rocoto/getic.sh | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/jobs/rocoto/getic.sh b/jobs/rocoto/getic.sh
index a890b0de200..f8f06cda999 100755
--- a/jobs/rocoto/getic.sh
+++ b/jobs/rocoto/getic.sh
@@ -73,6 +73,11 @@ if [[ $gfs_ver = "v16" && $EXP_WARM_START = ".true." && $CASE = "C768" ]]; then
     status=$?
     [[ $status -ne 0 ]] && exit $status
 
+    # Pull IAU increment files
+    htar -xvf ${HPSSDIR}/${CDATE}/gfs_netcdfa.tar
+    status=$?
+    [[ $status -ne 0 ]] && exit $status
+
 # else # Opertional input
 #   # ADD AFTER IMPLEMENTATION
   fi

From 364e46ff2acabed7bd9f6cb07ca10aa19033a980 Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Tue, 9 Mar 2021 16:08:57 +0000
Subject: [PATCH 024/125] Add pull of gdas wave restart files in getic.sh if
 DO_WAVE=YES

Refs: #178
---
 jobs/rocoto/getic.sh | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/jobs/rocoto/getic.sh b/jobs/rocoto/getic.sh
index f8f06cda999..33ca3789d99 100755
--- a/jobs/rocoto/getic.sh
+++ b/jobs/rocoto/getic.sh
@@ -68,6 +68,13 @@ if [[ $gfs_ver = "v16" && $EXP_WARM_START = ".true." && $CASE = "C768" ]]; then
     status=$?
     [[ $status -ne 0 ]] && exit $status
 
+    if [ $DO_WAVE = "YES" ]; then
+      # Pull prior cycle wave restart files
+      htar -xvf ${HPSSDIR}/${BDATE}/gdaswave_restart.tar
+      status=$?
+      [[ $status -ne 0 ]] && exit $status
+    fi
+
     # Pull current cycle restart files
     htar -xvf ${HPSSDIR}/${CDATE}/gfs_restarta.tar
     status=$?

From e6a9e576744a1c245148bdc4b967a5674c9416b5 Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Tue, 9 Mar 2021 16:55:41 +0000
Subject: [PATCH 025/125] Remove UFS_UTILS symlinks no longer needed in
 link_fv3gfs.sh

Refs: #164
---
 sorc/link_fv3gfs.sh | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/sorc/link_fv3gfs.sh b/sorc/link_fv3gfs.sh
index db79e2e7538..ef0eabd50c3 100755
--- a/sorc/link_fv3gfs.sh
+++ b/sorc/link_fv3gfs.sh
@@ -77,8 +77,8 @@ cd ${pwd}/../ush                ||exit 8
         $LINK ../sorc/gfs_post.fd/ush/$file                  .
     done
     for file in emcsfc_ice_blend.sh  fv3gfs_driver_grid.sh  fv3gfs_make_orog.sh  global_cycle_driver.sh \
-        emcsfc_snow.sh  fv3gfs_filter_topo.sh  global_chgres_driver.sh  global_cycle.sh \
-        chgres_cube.sh  fv3gfs_make_grid.sh  global_chgres.sh  ; do
+        emcsfc_snow.sh  fv3gfs_filter_topo.sh  global_cycle.sh \
+        chgres_cube.sh  fv3gfs_make_grid.sh ; do
         $LINK ../sorc/ufs_utils.fd/ush/$file                  .
     done
     for file in gldas_archive.sh  gldas_forcing.sh gldas_get_data.sh  gldas_process_data.sh gldas_liscrd.sh  gldas_post.sh ; do
@@ -307,9 +307,7 @@ cd ${pwd}/../sorc   ||   exit 8
     for prog in fregrid make_hgrid make_solo_mosaic ; do
         $SLINK ufs_utils.fd/sorc/fre-nctools.fd/tools/$prog                                ${prog}.fd                                
     done
-    for prog in  global_cycle.fd \
-        emcsfc_ice_blend.fd  nst_tf_chg.fd \
-        emcsfc_snow2mdl.fd   global_chgres.fd ;do
+    for prog in global_cycle.fd emcsfc_ice_blend.fd emcsfc_snow2mdl.fd ;do
         $SLINK ufs_utils.fd/sorc/$prog                                                     $prog
     done
 

From 1285d803fd402945ebc03eb42eb8bfa3d2aa10da Mon Sep 17 00:00:00 2001
From: "kate.friedman" <kate.friedman@noaa.gov>
Date: Tue, 9 Mar 2021 10:59:09 -0600
Subject: [PATCH 026/125] Update GSI checkout to clone GSI-master @ 9c1fc15

Refs: #164
---
 sorc/checkout.sh | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/sorc/checkout.sh b/sorc/checkout.sh
index 25a1a3c385f..05dc37fb9f0 100755
--- a/sorc/checkout.sh
+++ b/sorc/checkout.sh
@@ -37,11 +37,9 @@ fi
 echo gsi checkout ...
 if [[ ! -d gsi.fd ]] ; then
     rm -f ${topdir}/checkout-gsi.log
-    #git clone --recursive https://github.com/NOAA-EMC/GSI.git gsi.fd >> ${topdir}/checkout-gsi.log 2>&1
-    git clone --recursive https://github.com/RussTreadon-NOAA/GSI.git gsi.fd >> ${topdir}/checkout-gsi.log 2>&1
+    git clone --recursive https://github.com/NOAA-EMC/GSI.git gsi.fd >> ${topdir}/checkout-gsi.log 2>&1
     cd gsi.fd
-    #git checkout gfsda.v16.0.0
-    git checkout feature/hpc-stack
+    git checkout 9c1fc15d42573b398037319bbf8d5143ad126fb6
     git submodule update
     cd ${topdir}
 else

From 2f6d7ab06b3be6eaedb3406efca77ded61ef7aca Mon Sep 17 00:00:00 2001
From: "kate.friedman" <kate.friedman@noaa.gov>
Date: Tue, 9 Mar 2021 17:41:14 +0000
Subject: [PATCH 027/125] Module base updates for WCOSS-Dell, Hera, and Orion

- add zlib module load
- reorder wgrib2, hdf5, and netcdf module loads to accommodate module dependencies

Refs: #164
---
 modulefiles/module_base.hera          |  3 ++-
 modulefiles/module_base.orion         |  7 ++++---
 modulefiles/module_base.wcoss_dell_p3 | 11 ++++++-----
 3 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/modulefiles/module_base.hera b/modulefiles/module_base.hera
index e98363d92f4..6b21d485d56 100644
--- a/modulefiles/module_base.hera
+++ b/modulefiles/module_base.hera
@@ -20,8 +20,9 @@ module load g2tmpl/1.9.1
 module load crtm/2.3.0
 setenv CRTM_FIX /scratch2/NCEPDEV/nwprod/NCEPLIBS/fix/crtm_v2.3.0
 
-module load netcdf/4.7.4
+module load zlib/1.2.11
 module load hdf5/1.10.6
+module load netcdf/4.7.4
 module load esmf/8_1_0_beta_snapshot_27
 
 module load wgrib2/2.0.8
diff --git a/modulefiles/module_base.orion b/modulefiles/module_base.orion
index 0161f153fd9..2110560cdbb 100755
--- a/modulefiles/module_base.orion
+++ b/modulefiles/module_base.orion
@@ -8,9 +8,6 @@ module load hpc/1.1.0
 module load hpc-intel/2018.4
 module load hpc-impi/2018.4
 
-module load wgrib/2.0.8
-setenv WGRIB2 wgrib2
-
 #module load hpss/hpss
 module load nco/4.8.1
 module load gempak/7.5.1
@@ -22,10 +19,14 @@ module load g2tmpl/1.9.1
 module load crtm/2.3.0
 setenv CRTM_FIX /apps/contrib/NCEPLIBS/orion/fix/crtm_v2.3.0
 
+module load zlib/1.2.11
 module load hdf5/1.10.6
 module load netcdf/4.7.4
 module load esmf/8_1_0_beta_snapshot_27
 
+module load wgrib/2.0.8
+setenv WGRIB2 wgrib2
+
 module load contrib
 module load rocoto/1.3.3
 module load slurm/19.05.3-2
diff --git a/modulefiles/module_base.wcoss_dell_p3 b/modulefiles/module_base.wcoss_dell_p3
index e9807c11989..5f00bc7be82 100644
--- a/modulefiles/module_base.wcoss_dell_p3
+++ b/modulefiles/module_base.wcoss_dell_p3
@@ -19,17 +19,18 @@ module load prod_util/1.2.2
 module load grib_util/1.2.2
 module load g2tmpl/1.9.1
 
-module load wgrib2/2.0.8
-setenv WGRIB2 wgrib2
-
 module load NCO/4.7.0   
 module load CFP/2.0.2
 setenv USE_CFP YES
-module load pm5
+#module load pm5
 
+module load zlib/1.2.11
+module load hdf5/1.10.6
 module load netcdf/4.7.4
 module load esmf/8_1_0_beta_snapshot_27
-module load hdf5/1.10.6
+
+module load wgrib2/2.0.8
+setenv WGRIB2 wgrib2
 
 module use -a /gpfs/dell1/nco/ops/nwprod/modulefiles/
 module load gempak/7.3.3

From fcd720edef1fe242c27d543f445bb3b4ee4fe9ec Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Tue, 9 Mar 2021 17:54:51 +0000
Subject: [PATCH 028/125] Remove unneeded UFS_UTILS symlinks in link_fv3gfs.sh

Refs: #178
---
 sorc/link_fv3gfs.sh | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/sorc/link_fv3gfs.sh b/sorc/link_fv3gfs.sh
index 876bf9456a4..451b174692e 100755
--- a/sorc/link_fv3gfs.sh
+++ b/sorc/link_fv3gfs.sh
@@ -77,8 +77,7 @@ cd ${pwd}/../ush                ||exit 8
         $LINK ../sorc/gfs_post.fd/ush/$file                  .
     done
     for file in emcsfc_ice_blend.sh  fv3gfs_driver_grid.sh  fv3gfs_make_orog.sh  global_cycle_driver.sh \
-        emcsfc_snow.sh  fv3gfs_filter_topo.sh  global_chgres_driver.sh  global_cycle.sh \
-        fv3gfs_chgres.sh  fv3gfs_make_grid.sh  global_chgres.sh  ; do
+        emcsfc_snow.sh  fv3gfs_filter_topo.sh  global_cycle.sh fv3gfs_make_grid.sh  ; do
         $LINK ../sorc/ufs_utils.fd/ush/$file                  .
     done
     for file in gldas_archive.sh  gldas_forcing.sh gldas_get_data.sh  gldas_process_data.sh gldas_liscrd.sh  gldas_post.sh ; do
@@ -310,13 +309,10 @@ cd ${pwd}/../sorc   ||   exit 8
 
     $SLINK gfs_post.fd/sorc/ncep_post.fd                                                   gfs_ncep_post.fd
 
-    $SLINK ufs_utils.fd/sorc/fre-nctools.fd/tools/shave.fd                                 shave.fd
-    for prog in filter_topo fregrid make_hgrid make_solo_mosaic ; do
+    for prog in fregrid make_hgrid make_solo_mosaic ; do
         $SLINK ufs_utils.fd/sorc/fre-nctools.fd/tools/$prog                                ${prog}.fd                                
     done
-    for prog in  global_cycle.fd   nemsio_read.fd  nemsio_chgdate.fd \
-        emcsfc_ice_blend.fd  nst_tf_chg.fd \
-        emcsfc_snow2mdl.fd   global_chgres.fd  nemsio_get.fd    orog.fd ;do
+    for prog in global_cycle.fd emcsfc_ice_blend.fd emcsfc_snow2mdl.fd ; do
         $SLINK ufs_utils.fd/sorc/$prog                                                     $prog
     done
 

From 4f7243bc68164faf0008893467c5a2ea53d216ae Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Tue, 9 Mar 2021 18:07:23 +0000
Subject: [PATCH 029/125] Remove global_chgres exec from link_fv3gfs.sh; no
 longer exists from UFS_UTILS

Refs: #178
---
 sorc/link_fv3gfs.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sorc/link_fv3gfs.sh b/sorc/link_fv3gfs.sh
index 451b174692e..f79ccc01b38 100755
--- a/sorc/link_fv3gfs.sh
+++ b/sorc/link_fv3gfs.sh
@@ -233,7 +233,7 @@ if [ -d ${pwd}/gfs_wafs.fd ]; then
 fi
 
 for ufs_utilsexe in \
-     emcsfc_ice_blend  emcsfc_snow2mdl  global_chgres  global_cycle ; do
+     emcsfc_ice_blend  emcsfc_snow2mdl  global_cycle ; do
     [[ -s $ufs_utilsexe ]] && rm -f $ufs_utilsexe
     $LINK ../sorc/ufs_utils.fd/exec/$ufs_utilsexe .
 done

From 5c77500bc4742a840a26f05aedffe2aca3e15ec7 Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Mon, 22 Mar 2021 15:44:40 +0000
Subject: [PATCH 030/125] Add exit in init.sh when C768 v16 warm starts
 detected

Exit init.sh and init job when high res C768 v16 warm starts are detected
and running chgres_cube is not needed.

Refs: #178
---
 jobs/rocoto/init.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/jobs/rocoto/init.sh b/jobs/rocoto/init.sh
index 2fb81b268b6..f7bc323aaf9 100755
--- a/jobs/rocoto/init.sh
+++ b/jobs/rocoto/init.sh
@@ -52,6 +52,7 @@ export RUNICSH=${RUNICSH:-${GDASINIT_DIR}/run_v16.chgres.sh}
 # Check if init is needed and run if so
 if [[ $gfs_ver = "v16" && $EXP_WARM_START = ".true." && $CASE = "C768" ]]; then
   echo "Detected v16 C768 warm starts, will not run init. Exiting..."
+  exit 0
 else
   # Run chgres_cube
   if [ ! -d $OUTDIR ]; then mkdir -p $OUTDIR ; fi

From ab488f1aeb383d84a67b37e47a888d58662cf82f Mon Sep 17 00:00:00 2001
From: "kate.friedman" <kate.friedman@noaa.gov>
Date: Tue, 6 Apr 2021 15:33:11 +0000
Subject: [PATCH 031/125] Update OznMonBuild, RadMonBuild and module_base for
 hpc-stack on WCOSS-Dell

Refs: #164
---
 modulefiles/OznMonBuild.wcoss_dell_p3 |  3 +++
 modulefiles/RadMonBuild.wcoss_dell_p3 |  3 +++
 modulefiles/module_base.wcoss_dell_p3 | 18 ++++++------------
 3 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/modulefiles/OznMonBuild.wcoss_dell_p3 b/modulefiles/OznMonBuild.wcoss_dell_p3
index dd561c24d4b..6b4193792b7 100644
--- a/modulefiles/OznMonBuild.wcoss_dell_p3
+++ b/modulefiles/OznMonBuild.wcoss_dell_p3
@@ -10,5 +10,8 @@ export FFLAGS="-O3 -fp-model strict -convert big_endian -assume byterecl"
 
 export D_FFLAGS="-O3 -fp-model strict -convert big_endian -assume byterecl -debug"
 
+module use /usrx/local/nceplibs/dev/hpc-stack/libs/hpc-stack/modulefiles/stack
+module load hpc/1.1.0
 module load hpc-ips/18.0.1.163
+module load hpc-impi/18.0.1
 module load w3nco/2.4.1
diff --git a/modulefiles/RadMonBuild.wcoss_dell_p3 b/modulefiles/RadMonBuild.wcoss_dell_p3
index bdf9f7ac042..1c85e93ded0 100644
--- a/modulefiles/RadMonBuild.wcoss_dell_p3
+++ b/modulefiles/RadMonBuild.wcoss_dell_p3
@@ -11,5 +11,8 @@ export FFLAGS="-O3 -fp-model strict -D_REAL8_ -traceback -convert big_endian -as
 export CHECK_LIBS="/gpfs/hps/nco/ops/nwprod/spa_util/check_libs.bash"
 export D_FFLAGS="-O3 -fp-model strict -D_REAL8_ -traceback -convert big_endian -assume byterecl -debug"
 
+module use /usrx/local/nceplibs/dev/hpc-stack/libs/hpc-stack/modulefiles/stack
+module load hpc/1.1.0
 module load hpc-ips/18.0.1.163
+module load hpc-impi/18.0.1
 module load w3nco/2.4.1
diff --git a/modulefiles/module_base.wcoss_dell_p3 b/modulefiles/module_base.wcoss_dell_p3
index 5f00bc7be82..9316dc73c88 100644
--- a/modulefiles/module_base.wcoss_dell_p3
+++ b/modulefiles/module_base.wcoss_dell_p3
@@ -11,27 +11,25 @@ module load hpc-impi/18.0.1
 
 module load lsf/10.1       
 module load EnvVars/1.0.3
-
 module load HPSS/5.0.2.5
 
-module load prod_envir/1.1.0
 module load prod_util/1.2.2
+module load prod_envir/1.1.0
 module load grib_util/1.2.2
 module load g2tmpl/1.9.1
 
+module load crtm/2.3.0
+setenv CRTM_FIX /gpfs/dell1/nco/ops/nwprod/lib/crtm/v2.3.0/fix
+
 module load NCO/4.7.0   
 module load CFP/2.0.2
 setenv USE_CFP YES
-#module load pm5
+module load pm5
 
-module load zlib/1.2.11
-module load hdf5/1.10.6
 module load netcdf/4.7.4
+module load hdf5/1.10.6
 module load esmf/8_1_0_beta_snapshot_27
 
-module load wgrib2/2.0.8
-setenv WGRIB2 wgrib2
-
 module use -a /gpfs/dell1/nco/ops/nwprod/modulefiles/
 module load gempak/7.3.3
 
@@ -43,9 +41,5 @@ module load util_shared/1.1.2
 # python
 module load python/3.6.3
 
-# Load crtm
-module load crtm/2.3.0
-setenv CRTM_FIX /gpfs/dell1/nco/ops/nwprod/lib/crtm/v2.3.0/fix
-
 # waveprep
 module load cdo/1.9.8

From b55ecf8e5ae761ddc8e3897411aceb4e0529e106 Mon Sep 17 00:00:00 2001
From: "kate.friedman" <kate.friedman@noaa.gov>
Date: Tue, 6 Apr 2021 15:34:05 +0000
Subject: [PATCH 032/125] Remove Tide/Gyre reference from machine-setup.sh

Refs: #164
---
 sorc/machine-setup.sh | 11 -----------
 1 file changed, 11 deletions(-)

diff --git a/sorc/machine-setup.sh b/sorc/machine-setup.sh
index 9ce289a6b02..a5407ec3c4a 100644
--- a/sorc/machine-setup.sh
+++ b/sorc/machine-setup.sh
@@ -103,17 +103,6 @@ elif [[ -L /usrx && "$( readlink /usrx 2> /dev/null )" =~ dell ]] ; then
     module load hpc-ips/18.0.1.163
     module load hpc-impi/18.0.1
 
-##---------------------------------------------------------------------------
-
-elif [[ -d /dcom && -d /hwrf ]] ; then
-    # We are on NOAA Tide or Gyre
-    if ( ! eval module help > /dev/null 2>&1 ) ; then
-	echo load the module command 1>&2
-        source /usrx/local/Modules/default/init/$__ms_shell
-    fi
-    target=wcoss
-    module purge
-
 ##---------------------------------------------------------------------------
 elif [[ -d /glade ]] ; then
     # We are on NCAR Yellowstone

From 48ea7abd9fe4cfd0e774c92383948c9bb477a905 Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Mon, 12 Apr 2021 17:10:59 +0000
Subject: [PATCH 033/125] Low resolution resource adjustments

- C96, C192, and C384 config.fv3 resource updates for successful jobs
- provided by Jun Wang and Fanglin Yang
- tested on Hera

Refs: #178
---
 parm/config/config.fv3 | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/parm/config/config.fv3 b/parm/config/config.fv3
index eca11a0f815..1c4025f84c6 100755
--- a/parm/config/config.fv3
+++ b/parm/config/config.fv3
@@ -53,7 +53,7 @@ case $case_in in
         export WRTIOBUF="1M"
         ;;
     "C96")
-        export DELTIM=900
+        export DELTIM=720
         export layout_x=6
         export layout_y=4
         export layout_x_gfs=6
@@ -64,10 +64,11 @@ case $case_in in
         export nth_fv3_gfs=1
         export cdmbgwd="0.14,1.8,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
         export WRITE_GROUP=1
-        export WRTTASK_PER_GROUP=6
+        export WRTTASK_PER_GROUP=$npe_node_max
         export WRITE_GROUP_GFS=1
         export WRTTASK_PER_GROUP_GFS=$npe_node_max
         export WRTIOBUF="4M"
+        export n_split=6
         ;;
     "C192")
         export DELTIM=450
@@ -88,14 +89,14 @@ case $case_in in
         ;;
     "C384")
         export DELTIM=240
-        export layout_x=8
+        export layout_x=6
         export layout_y=8
         export layout_x_gfs=6
-        export layout_y_gfs=6
+        export layout_y_gfs=8
         export npe_wav=140
         export npe_wav_gfs=140
-        export nth_fv3=1
-        export nth_fv3_gfs=1
+        export nth_fv3=2
+        export nth_fv3_gfs=2
         export cdmbgwd="1.1,0.72,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
         export WRITE_GROUP=1
         export WRTTASK_PER_GROUP=$npe_node_max

From 6b6b9ed3949c2e3ca3d697ffd3bb3a65ec728744 Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Mon, 12 Apr 2021 17:13:06 +0000
Subject: [PATCH 034/125] getic and init job updates for v16 in ops

- update GFSv16 implementation cycle in config if-blocks to be 2021032100
- add note about missing ops data on HPSS for 2021032106 cycle
- add run_v16.chgres.sh as default RUNICSH script

Refs: #178
---
 parm/config/config.getic | 10 +++++++++-
 parm/config/config.init  | 13 +++++++++++--
 2 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/parm/config/config.getic b/parm/config/config.getic
index b86f9c4aebb..90b9660716b 100755
--- a/parm/config/config.getic
+++ b/parm/config/config.getic
@@ -45,10 +45,18 @@ elif [ ${RETRO:-"NO"} = "NO" ]; then # Operational input
     export gfs_ver=v14
     export GETICSH=${GDASINIT_DIR}/get_${gfs_ver}.data.sh
     export tarball=gpfs_hps_nco_ops_com_gfs_prod_gfs.${PDY}_${cyc}.pgrb2_${grid}.tar
-  elif [[ "$CDATE" -lt "2021031700" ]]; then
+  elif [[ "$CDATE" -lt "2021032100" ]]; then
     export gfs_ver=v15
     export GETICSH=${GDASINIT_DIR}/get_${gfs_ver}.data.sh
     export tarball=com_gfs_prod_gfs.${PDY}_${cyc}.gfs_pgrb2.tar
+  elif [[ "$CDATE" -lt "2021032106" ]]; then
+    # The way the v16 switch over was done, there is no complete
+    # set of v16 or v15 data for 2021032100.  And although
+    # v16 was officially implemented 2021032212, the v16 prod
+    # tarballs were archived starting 2021032106.
+    set +x
+    echo FATAL ERROR: NO V15 OR V16 DATA FOR 2021032100
+    exit 1
   fi
 fi
 
diff --git a/parm/config/config.init b/parm/config/config.init
index a9932028a56..d1d504c7c06 100755
--- a/parm/config/config.init
+++ b/parm/config/config.init
@@ -17,7 +17,8 @@ export GDASINIT_DIR=${UFS_DIR}/util/gdas_init
 export CRES_HIRES=$CASE
 export CRES_ENKF=$CASE_ENKF
 
-if [ ${RETRO:-"NO"} = "YES" ]; then
+export RUNICSH=${GDASINIT_DIR}/run_v16.chgres.sh
+if [ ${RETRO:-"NO"} = "YES" || $CDUMP = "gdas" ]; then
   export RUNICSH=${GDASINIT_DIR}/run_v16retro.chgres.sh
 fi
 
@@ -35,9 +36,17 @@ if [ ${RETRO:-"NO"} = "NO" ]; then # Operational input
   elif [[ "$CDATE" -lt "2019061200" ]]; then
     export gfs_ver=v14
     export RUNICSH=${GDASINIT_DIR}/run_${gfs_ver}.chgres.sh
-  elif [[ "$CDATE" -lt "2021031700" ]]; then
+  elif [[ "$CDATE" -lt "2021032100" ]]; then
     export gfs_ver=v15
     export RUNICSH=${GDASINIT_DIR}/run_${gfs_ver}.chgres.gfs.sh
+  elif [[ "$CDATE" -lt "2021032106" ]]; then
+    # The way the v16 switch over was done, there is no complete
+    # set of v16 or v15 data for 2021032100.  And although
+    # v16 was officially implemented 2021032212, the v16 prod
+    # tarballs were archived starting 2021032106.
+    set +x
+    echo FATAL ERROR: NO V15 OR V16 DATA FOR 2021032100
+    exit 1
   fi
 fi
 

From 35c16e3aafc3addd3d24225880b5319332cca97a Mon Sep 17 00:00:00 2001
From: "kate.friedman" <kate.friedman@noaa.gov>
Date: Thu, 22 Apr 2021 12:12:16 +0000
Subject: [PATCH 035/125] Add wgrib2 module load to module_base.wcoss_dell_p3

- add wgrib2/2.0.8 module load
- add setting of WGRIB2 to equal wgrib2

Refs: #164
---
 modulefiles/module_base.wcoss_dell_p3 | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/modulefiles/module_base.wcoss_dell_p3 b/modulefiles/module_base.wcoss_dell_p3
index 9316dc73c88..602f816d71a 100644
--- a/modulefiles/module_base.wcoss_dell_p3
+++ b/modulefiles/module_base.wcoss_dell_p3
@@ -26,6 +26,9 @@ module load CFP/2.0.2
 setenv USE_CFP YES
 module load pm5
 
+module load wgrib2/2.0.8
+setenv WGRIB2 wgrib2
+
 module load netcdf/4.7.4
 module load hdf5/1.10.6
 module load esmf/8_1_0_beta_snapshot_27

From 4c64dce093fb3925b40b2a3f0b57fd196b14d92c Mon Sep 17 00:00:00 2001
From: "kate.friedman" <kate.friedman@noaa.gov>
Date: Thu, 22 Apr 2021 12:14:21 +0000
Subject: [PATCH 036/125] Reduce npe_wav_gfs from 440 to 140 to dev users

Refs: #164
---
 parm/config/config.fv3 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/parm/config/config.fv3 b/parm/config/config.fv3
index e70304bfa9f..662ae01c5e5 100755
--- a/parm/config/config.fv3
+++ b/parm/config/config.fv3
@@ -110,7 +110,7 @@ case $case_in in
         export layout_x_gfs=16
         export layout_y_gfs=16
         export npe_wav=140
-        export npe_wav_gfs=440
+        export npe_wav_gfs=140
         export nth_fv3=4
         export nth_fv3_gfs=7
         export cdmbgwd="4.0,0.15,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling

From 9d733f87203ebd2298d22c0ac9491520bae60983 Mon Sep 17 00:00:00 2001
From: "kate.friedman" <kate.friedman@noaa.gov>
Date: Thu, 22 Apr 2021 12:15:13 +0000
Subject: [PATCH 037/125] Add support for separate gfs thread values

- nth_fv3_gfs was added to configs, add support to setup scripts for resource calculations

Refs: #164
---
 ush/rocoto/workflow_utils.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/ush/rocoto/workflow_utils.py b/ush/rocoto/workflow_utils.py
index e8bb16ddd1c..e32e451406f 100755
--- a/ush/rocoto/workflow_utils.py
+++ b/ush/rocoto/workflow_utils.py
@@ -295,7 +295,10 @@ def get_resources(machine, cfg, task, reservation, cdump='gdas'):
         ppn = cfg['npe_node_%s' % ltask]
 
     if machine in [ 'WCOSS_DELL_P3', 'HERA', 'ORION']:
-        threads = cfg['nth_%s' % ltask]
+        if cdump in ['gfs'] and 'nth_%s_gfs' % task in cfg.keys():
+            threads = cfg['nth_%s_gfs' % ltask]
+        else:
+            threads = cfg['nth_%s' % ltask]
 
     nodes = np.int(np.ceil(np.float(tasks) / np.float(ppn)))
 

From 3cc02d507df48e2b9f99163d737123e5b5c716c0 Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Thu, 22 Apr 2021 12:52:46 +0000
Subject: [PATCH 038/125] Update GLDAS and UFS_UTILS checkouts

- update GLDAS checkout to gldas_gfsv16_release.v1.15.0 tag; corrects build on Orion
- update UFS_UTILS checkout to 4f44bf89 hash of develop (supports stack)

Refs: #164
---
 sorc/checkout.sh | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/sorc/checkout.sh b/sorc/checkout.sh
index 05dc37fb9f0..f53ccead4c3 100755
--- a/sorc/checkout.sh
+++ b/sorc/checkout.sh
@@ -51,7 +51,7 @@ if [[ ! -d gldas.fd ]] ; then
     rm -f ${topdir}/checkout-gldas.log
     git clone https://github.com/NOAA-EMC/GLDAS.git gldas.fd >> ${topdir}/checkout-gldas.fd.log 2>&1
     cd gldas.fd
-    git checkout gldas_gfsv16_release.v1.14.0
+    git checkout gldas_gfsv16_release.v1.15.0
     cd ${topdir}
 else
     echo 'Skip.  Directory gldas.fd already exists.'
@@ -60,11 +60,9 @@ fi
 echo ufs_utils checkout ...
 if [[ ! -d ufs_utils.fd ]] ; then
     rm -f ${topdir}/checkout-ufs_utils.log
-    #git clone --recursive https://github.com/NOAA-EMC/UFS_UTILS.git ufs_utils.fd >> ${topdir}/checkout-ufs_utils.fd.log 2>&1
-    git clone --recursive https://github.com/GeorgeGayno-NOAA/UFS_UTILS.git ufs_utils.fd >> ${topdir}/checkout-ufs_utils.fd.log 2>&1
+    git clone --recursive https://github.com/NOAA-EMC/UFS_UTILS.git ufs_utils.fd >> ${topdir}/checkout-ufs_utils.fd.log 2>&1
     cd ufs_utils.fd
-    #git checkout ops-gfsv16.0.0
-    git checkout feature/stack
+    git checkout 4f44bf8987ef16ca97a7c19364c9d3cf17e87f4e
     cd ${topdir}
 else
     echo 'Skip.  Directory ufs_utils.fd already exists.'

From 305bf5e0fb3c665cf71f10023feed1a121be329b Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Thu, 22 Apr 2021 13:01:04 +0000
Subject: [PATCH 039/125] Update Externals.cfg to match checkout.sh versions

Refs: #164
---
 Externals.cfg | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/Externals.cfg b/Externals.cfg
index 7df6e13e109..8e7ebf51ee3 100644
--- a/Externals.cfg
+++ b/Externals.cfg
@@ -8,14 +8,14 @@ protocol = git
 required = True
 
 [GSI]
-tag = gfsda.v16.0.0
+hash = 9c1fc15d42573b398037319bbf8d5143ad126fb6
 local_path = sorc/gsi.fd
 repo_url = https://github.com/NOAA-EMC/GSI.git
 protocol = git
 required = True
 
 [GLDAS]
-tag = gldas_gfsv16_release.v1.14.0
+tag = gldas_gfsv16_release.v1.15.0
 local_path = sorc/gldas.fd
 repo_url = https://github.com/NOAA-EMC/GLDAS.git
 protocol = git
@@ -23,28 +23,28 @@ required = True
 
 [EMC_post]
 #No externals setting = .gitmodules will be invoked for CMakeModules and comupp/src/lib/crtm2 submodules
-tag = upp_v10.0.3
+hash = 33eb60e5b486d9ff670c76b799f886b7f47c2022
 local_path = sorc/gfs_post.fd
 repo_url = https://github.com/NOAA-EMC/EMC_post.git
 protocol = git
 required = True
 
 [UFS_UTILS]
-#tag = ops-gfsv16.0.0
+hash = 4f44bf8987ef16ca97a7c19364c9d3cf17e87f4e
 local_path = sorc/ufs_utils.fd
 repo_url = https://github.com/NOAA-EMC/UFS_UTILS.git
 protocol = git
 required = True
 
 [EMC_verif-global]
-tag = verif_global_v1.13.4
+tag = verif_global_v1.13.5
 local_path = sorc/verif-global.fd
 repo_url = https://github.com/NOAA-EMC/EMC_verif-global.git
 protocol = git
 required = True
 
 [EMC_gfs_wafs]
-tag = gfs_wafs.v6.0.18
+tag = gfs_wafs.v6.0.19
 local_path = sorc/gfs_wafs.fd
 repo_url = https://github.com/NOAA-EMC/EMC_gfs_wafs.git
 protocol = git

From 2b703d335b8e3e97c9764bd60d87188ed11938fd Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Thu, 22 Apr 2021 13:17:28 +0000
Subject: [PATCH 040/125] Set WAFS checkout to be optional with -o flag

- add checkout_wafs to -o flag settings

Refs: #164
---
 sorc/checkout.sh | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/sorc/checkout.sh b/sorc/checkout.sh
index f53ccead4c3..e37dacc743f 100755
--- a/sorc/checkout.sh
+++ b/sorc/checkout.sh
@@ -6,8 +6,9 @@ while getopts "o" option;
 do
  case $option in
   o)
-   echo "Received -o flag for optional checkout of GTG, will check out GTG with EMC_post"
+   echo "Received -o flag for optional checkout of operational-only codes"
    checkout_gtg="YES"
+   checkout_wafs="YES"
    ;;
   :)
    echo "option -$OPTARG needs an argument"
@@ -93,15 +94,18 @@ else
     echo 'Skip.  Directory gfs_post.fd already exists.'
 fi
 
-echo EMC_gfs_wafs checkout ...
-if [[ ! -d gfs_wafs.fd ]] ; then
+checkout_wafs=${checkout_wafs:-"NO"}
+if [[ ${checkout_wafs} == "YES" ]] ; then
+  echo EMC_gfs_wafs checkout ...
+  if [[ ! -d gfs_wafs.fd ]] ; then
     rm -f ${topdir}/checkout-gfs_wafs.log
     git clone --recursive https://github.com/NOAA-EMC/EMC_gfs_wafs.git gfs_wafs.fd >> ${topdir}/checkout-gfs_wafs.log 2>&1
     cd gfs_wafs.fd
     git checkout gfs_wafs.v6.0.19
     cd ${topdir}
-else
+  else
     echo 'Skip.  Directory gfs_wafs.fd already exists.'
+  fi
 fi
 
 echo EMC_verif-global checkout ...

From b6c392c7569ed909d6b0b27c0f6a4f828b7acdfd Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Thu, 22 Apr 2021 13:40:25 +0000
Subject: [PATCH 041/125] Set DO_WAVE=NO in config.base

- set default DO_WAVE setting to NO in config.base.emc.dyn
- turn off waves until hpc-stack for WW3 execs build is ready

Refs: #164
---
 parm/config/config.base.emc.dyn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/parm/config/config.base.emc.dyn b/parm/config/config.base.emc.dyn
index 2e20112badb..46da367a0f5 100755
--- a/parm/config/config.base.emc.dyn
+++ b/parm/config/config.base.emc.dyn
@@ -222,7 +222,7 @@ export DO_GLDAS="YES"
 export gldas_cyc=00
 
 # run wave component
-export DO_WAVE="YES"
+export DO_WAVE="NO"
 export WAVE_CDUMP="both"
 
 # Microphysics Options: 99-ZhaoCarr, 8-Thompson; 6-WSM6, 10-MG, 11-GFDL

From 00c18b5e341d601fd83da2f9df2ac928e8dcf7c8 Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Thu, 22 Apr 2021 13:44:13 +0000
Subject: [PATCH 042/125] Update FIX_DIR paths in link_fv3gfs.sh

- change platform FIX_DIR paths from fix_nco_gfsv16 set to developmental fix set for v17+

Refs: #164
---
 sorc/link_fv3gfs.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/sorc/link_fv3gfs.sh b/sorc/link_fv3gfs.sh
index c2bad5402a4..6c1f870ebc2 100755
--- a/sorc/link_fv3gfs.sh
+++ b/sorc/link_fv3gfs.sh
@@ -31,13 +31,13 @@ pwd=$(pwd -P)
 #--model fix fields
 #------------------------------
 if [ $machine == "cray" ]; then
-    FIX_DIR="/gpfs/hps3/emc/global/noscrub/emc.glopara/git/fv3gfs/fix_nco_gfsv16"
+    FIX_DIR="/gpfs/hps3/emc/global/noscrub/emc.glopara/git/fv3gfs/fix"
 elif [ $machine = "dell" ]; then
-    FIX_DIR="/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix_nco_gfsv16"
+    FIX_DIR="/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/fv3gfs/fix"
 elif [ $machine = "hera" ]; then
-    FIX_DIR="/scratch1/NCEPDEV/global/glopara/fix_nco_gfsv16"
+    FIX_DIR="/scratch1/NCEPDEV/global/glopara/fix"
 elif [ $machine = "orion" ]; then
-    FIX_DIR="/work/noaa/global/glopara/fix_nco_gfsv16"
+    FIX_DIR="/work/noaa/global/glopara/fix"
 fi
 cd ${pwd}/../fix                ||exit 8
 for dir in fix_am fix_fv3_gmted2010 fix_gldas fix_orog fix_verif fix_wave_gfs ; do

From ece41632a76e697184a46bb2a428c35ccdf08305 Mon Sep 17 00:00:00 2001
From: "kate.friedman" <kate.friedman@noaa.gov>
Date: Thu, 22 Apr 2021 13:58:22 +0000
Subject: [PATCH 043/125] Add separate gfs resource variables to fcst in
 config.resources

Refs: #164
---
 parm/config/config.resources | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/parm/config/config.resources b/parm/config/config.resources
index b3d81e39b9d..492b978acf4 100755
--- a/parm/config/config.resources
+++ b/parm/config/config.resources
@@ -168,8 +168,10 @@ elif [ $step = "fcst" ]; then
     export npe_fcst=$(echo "$layout_x * $layout_y * 6" | bc)
     export npe_fcst_gfs=$(echo "$layout_x_gfs * $layout_y_gfs * 6" | bc)
     export nth_fcst=${nth_fv3:-2}
+    export nth_fcst_gfs=${nth_fv3_gfs:-2}
     if [[ "$CASE" == "C384" ]]; then nth_fcst=4; fi
     export npe_node_fcst=$(echo "$npe_node_max / $nth_fcst" | bc)
+    export npe_node_fcst_gfs=$(echo "$npe_node_max / $nth_fcst_gfs" | bc)
     if [[ "$machine" == "WCOSS_C" ]]; then export memory_fcst="1024M"; fi
 
 elif [ $step = "post" ]; then

From 15c6619c6e211eb67ca6aa5c995e4a3bf4cf586f Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Thu, 22 Apr 2021 14:03:46 +0000
Subject: [PATCH 044/125] Adjust C768 fcst thread value for reduced node usage

- change C768 nth_fv3_gfs value from 7 to 2

Refs: #164
---
 parm/config/config.fv3 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/parm/config/config.fv3 b/parm/config/config.fv3
index 662ae01c5e5..fb4998c3ac0 100755
--- a/parm/config/config.fv3
+++ b/parm/config/config.fv3
@@ -112,7 +112,7 @@ case $case_in in
         export npe_wav=140
         export npe_wav_gfs=140
         export nth_fv3=4
-        export nth_fv3_gfs=7
+        export nth_fv3_gfs=2
         export cdmbgwd="4.0,0.15,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
         export WRITE_GROUP=2
         export WRTTASK_PER_GROUP=$(echo "2*$npe_node_max" |bc)

From dfec5816add3c56721360a418ec9f3459cfd4d8c Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Thu, 22 Apr 2021 15:27:56 +0000
Subject: [PATCH 045/125] Update UFS_UTILS hash to
 9d9dd23e37a51fcc5ff6b7499b834c85ab32e5f3

Refs: #164
---
 sorc/checkout.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sorc/checkout.sh b/sorc/checkout.sh
index e37dacc743f..757ea92b959 100755
--- a/sorc/checkout.sh
+++ b/sorc/checkout.sh
@@ -63,7 +63,7 @@ if [[ ! -d ufs_utils.fd ]] ; then
     rm -f ${topdir}/checkout-ufs_utils.log
     git clone --recursive https://github.com/NOAA-EMC/UFS_UTILS.git ufs_utils.fd >> ${topdir}/checkout-ufs_utils.fd.log 2>&1
     cd ufs_utils.fd
-    git checkout 4f44bf8987ef16ca97a7c19364c9d3cf17e87f4e
+    git checkout 9d9dd23e37a51fcc5ff6b7499b834c85ab32e5f3
     cd ${topdir}
 else
     echo 'Skip.  Directory ufs_utils.fd already exists.'

From 1d0e0c29979101b451f1fda265af3f9a716bb9de Mon Sep 17 00:00:00 2001
From: "kate.friedman" <kate.friedman@noaa.gov>
Date: Tue, 27 Apr 2021 19:33:09 +0000
Subject: [PATCH 046/125] Reverting resource related changes

- will commit resource adjustments separately

Refs: #164
---
 parm/config/config.fv3       | 4 ++--
 parm/config/config.resources | 2 --
 ush/rocoto/workflow_utils.py | 5 +----
 3 files changed, 3 insertions(+), 8 deletions(-)

diff --git a/parm/config/config.fv3 b/parm/config/config.fv3
index fb4998c3ac0..e70304bfa9f 100755
--- a/parm/config/config.fv3
+++ b/parm/config/config.fv3
@@ -110,9 +110,9 @@ case $case_in in
         export layout_x_gfs=16
         export layout_y_gfs=16
         export npe_wav=140
-        export npe_wav_gfs=140
+        export npe_wav_gfs=440
         export nth_fv3=4
-        export nth_fv3_gfs=2
+        export nth_fv3_gfs=7
         export cdmbgwd="4.0,0.15,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
         export WRITE_GROUP=2
         export WRTTASK_PER_GROUP=$(echo "2*$npe_node_max" |bc)
diff --git a/parm/config/config.resources b/parm/config/config.resources
index 492b978acf4..b3d81e39b9d 100755
--- a/parm/config/config.resources
+++ b/parm/config/config.resources
@@ -168,10 +168,8 @@ elif [ $step = "fcst" ]; then
     export npe_fcst=$(echo "$layout_x * $layout_y * 6" | bc)
     export npe_fcst_gfs=$(echo "$layout_x_gfs * $layout_y_gfs * 6" | bc)
     export nth_fcst=${nth_fv3:-2}
-    export nth_fcst_gfs=${nth_fv3_gfs:-2}
     if [[ "$CASE" == "C384" ]]; then nth_fcst=4; fi
     export npe_node_fcst=$(echo "$npe_node_max / $nth_fcst" | bc)
-    export npe_node_fcst_gfs=$(echo "$npe_node_max / $nth_fcst_gfs" | bc)
     if [[ "$machine" == "WCOSS_C" ]]; then export memory_fcst="1024M"; fi
 
 elif [ $step = "post" ]; then
diff --git a/ush/rocoto/workflow_utils.py b/ush/rocoto/workflow_utils.py
index e32e451406f..e8bb16ddd1c 100755
--- a/ush/rocoto/workflow_utils.py
+++ b/ush/rocoto/workflow_utils.py
@@ -295,10 +295,7 @@ def get_resources(machine, cfg, task, reservation, cdump='gdas'):
         ppn = cfg['npe_node_%s' % ltask]
 
     if machine in [ 'WCOSS_DELL_P3', 'HERA', 'ORION']:
-        if cdump in ['gfs'] and 'nth_%s_gfs' % task in cfg.keys():
-            threads = cfg['nth_%s_gfs' % ltask]
-        else:
-            threads = cfg['nth_%s' % ltask]
+        threads = cfg['nth_%s' % ltask]
 
     nodes = np.int(np.ceil(np.float(tasks) / np.float(ppn)))
 

From fb47a5a0f7391f670461b00eaf324c3b0102ef1f Mon Sep 17 00:00:00 2001
From: "kate.friedman" <kate.friedman@noaa.gov>
Date: Wed, 28 Apr 2021 17:50:15 +0000
Subject: [PATCH 047/125] Update TC_tracker version in config.vrfy

- new TC_tracker.v1.1.15.4 was made available by Jiayi Peng
- new version supports hpc-stack on supported platforms
- installed locally on WCOSS-Dells, Hera, and Orion

Refs: #164
---
 parm/config/config.vrfy | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/parm/config/config.vrfy b/parm/config/config.vrfy
index ea79d76d948..fb307abb7a6 100755
--- a/parm/config/config.vrfy
+++ b/parm/config/config.vrfy
@@ -142,14 +142,8 @@ fi
 # Cyclone genesis and cyclone track verification
 #-------------------------------------------------
 
-export ens_tracker_ver=v1.1.15.1
-if [ $machine = "WCOSS_DELL_P3" ] ; then
-    export ens_tracker_ver=v1.1.15.3
-fi
-export HOMEens_tracker=$BASE_GIT/tracker/ens_tracker.${ens_tracker_ver}
-if [ $machine = "ORION" ] ; then
-    export HOMEens_tracker=$BASE_GIT/tracker/TC_tracker.v1.1.15.2
-fi
+export ens_tracker_ver=v1.1.15.4
+export HOMEens_tracker=$BASE_GIT/TC_tracker/TC_tracker.${ens_tracker_ver}
 
 if [ "$VRFYTRAK" = "YES" ]; then
 

From 709095709ca60498d05ea0f519b494501e1ecdf2 Mon Sep 17 00:00:00 2001
From: "kate.friedman" <kate.friedman@noaa.gov>
Date: Mon, 3 May 2021 15:11:51 +0000
Subject: [PATCH 048/125] Update util_shared to 1.3.0 on WCOSS-Dell

- version changed in production for v16.0.9 footer fix

Refs: #164
---
 modulefiles/module_base.wcoss_dell_p3 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modulefiles/module_base.wcoss_dell_p3 b/modulefiles/module_base.wcoss_dell_p3
index 602f816d71a..f1c6cd7189b 100644
--- a/modulefiles/module_base.wcoss_dell_p3
+++ b/modulefiles/module_base.wcoss_dell_p3
@@ -17,6 +17,7 @@ module load prod_util/1.2.2
 module load prod_envir/1.1.0
 module load grib_util/1.2.2
 module load g2tmpl/1.9.1
+module load util_shared/1.3.0
 
 module load crtm/2.3.0
 setenv CRTM_FIX /gpfs/dell1/nco/ops/nwprod/lib/crtm/v2.3.0/fix
@@ -39,7 +40,6 @@ module load gempak/7.3.3
 # Load for WAFS
 module load bufr_dumplist/2.0.0
 module load dumpjb/5.1.0
-module load util_shared/1.1.2
 
 # python
 module load python/3.6.3

From 6ce83363726ece6dbd34719430dd44f5e63bbc3c Mon Sep 17 00:00:00 2001
From: "kate.friedman" <kate.friedman@noaa.gov>
Date: Mon, 3 May 2021 15:14:54 +0000
Subject: [PATCH 049/125] Component updates for UPP, WAFS, UFS_UTILS

- new tag in checkout for UFS_UTILS to use hpc/1.1.0
- new hash in checkout for UPP/EMC_post to support GTG and GTG code copy fix
- new hash in checkout for WAFS/EMC_gfs_wafs to support hpc-stack
- build script update for WAFS (cmake/stack)
- WAFS GCIP config update to fix runtime error

Refs: #164
---
 parm/config/config.wafsgcip | 2 ++
 sorc/build_gfs_wafs.sh      | 2 +-
 sorc/checkout.sh            | 8 ++++----
 3 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/parm/config/config.wafsgcip b/parm/config/config.wafsgcip
index 5c166a541b6..793dae1694c 100755
--- a/parm/config/config.wafsgcip
+++ b/parm/config/config.wafsgcip
@@ -11,5 +11,7 @@ export COMIN=$COMINatmos
 export COMINgfs=$COMIN
 export COMOUT=$COMOUTatmos
 export SENDCOM="YES"
+export COMLISTROOT="$UTILROOT/save/config"
+export COMDATEROOT="N/A"
 
 echo "END: config.wafsgcip"
diff --git a/sorc/build_gfs_wafs.sh b/sorc/build_gfs_wafs.sh
index ceff4eae670..921d97f1173 100755
--- a/sorc/build_gfs_wafs.sh
+++ b/sorc/build_gfs_wafs.sh
@@ -17,6 +17,6 @@ if [ ! -d "../exec" ]; then
 fi
 
 cd gfs_wafs.fd/sorc
-sh build_wafs.sh
+./build_wafs_cmake.sh
 
 exit
diff --git a/sorc/checkout.sh b/sorc/checkout.sh
index 757ea92b959..d499a23271e 100755
--- a/sorc/checkout.sh
+++ b/sorc/checkout.sh
@@ -63,7 +63,7 @@ if [[ ! -d ufs_utils.fd ]] ; then
     rm -f ${topdir}/checkout-ufs_utils.log
     git clone --recursive https://github.com/NOAA-EMC/UFS_UTILS.git ufs_utils.fd >> ${topdir}/checkout-ufs_utils.fd.log 2>&1
     cd ufs_utils.fd
-    git checkout 9d9dd23e37a51fcc5ff6b7499b834c85ab32e5f3
+    git checkout ufs_utils_1_3_0
     cd ${topdir}
 else
     echo 'Skip.  Directory ufs_utils.fd already exists.'
@@ -74,7 +74,7 @@ if [[ ! -d gfs_post.fd ]] ; then
     rm -f ${topdir}/checkout-gfs_post.log
     git clone https://github.com/NOAA-EMC/EMC_post.git gfs_post.fd >> ${topdir}/checkout-gfs_post.log 2>&1
     cd gfs_post.fd
-    git checkout 33eb60e5b486d9ff670c76b799f886b7f47c2022
+    git checkout ff42e0227d6100285d4179a2572b700fd5a959cb
     git submodule update --init CMakeModules
 
     ################################################################################
@@ -86,7 +86,7 @@ if [[ ! -d gfs_post.fd ]] ; then
     checkout_gtg=${checkout_gtg:-"NO"}
     if [[ ${checkout_gtg} == "YES" ]] ; then
       ./manage_externals/checkout_externals
-      cp sorc/post_gtg.fd/*f90 sorc/ncep_post.fd/.
+      cp sorc/post_gtg.fd/*F90 sorc/ncep_post.fd/.
       cp sorc/post_gtg.fd/gtg.config.gfs parm/gtg.config.gfs
     fi
     cd ${topdir}
@@ -101,7 +101,7 @@ if [[ ${checkout_wafs} == "YES" ]] ; then
     rm -f ${topdir}/checkout-gfs_wafs.log
     git clone --recursive https://github.com/NOAA-EMC/EMC_gfs_wafs.git gfs_wafs.fd >> ${topdir}/checkout-gfs_wafs.log 2>&1
     cd gfs_wafs.fd
-    git checkout gfs_wafs.v6.0.19
+    git checkout c2a29a67d9432b4d6fba99eac7797b81d05202b6
     cd ${topdir}
   else
     echo 'Skip.  Directory gfs_wafs.fd already exists.'

From 858f216330d7ab71137afe6331f6576fec413aee Mon Sep 17 00:00:00 2001
From: "kate.friedman" <kate.friedman@noaa.gov>
Date: Mon, 3 May 2021 15:19:56 +0000
Subject: [PATCH 050/125] Update EMC_post, UFS_UTILS, EMC_gfs_wafs versions in
 Externals.cfg

Refs: #164
---
 Externals.cfg | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Externals.cfg b/Externals.cfg
index 8e7ebf51ee3..ee6e42267ab 100644
--- a/Externals.cfg
+++ b/Externals.cfg
@@ -23,14 +23,14 @@ required = True
 
 [EMC_post]
 #No externals setting = .gitmodules will be invoked for CMakeModules and comupp/src/lib/crtm2 submodules
-hash = 33eb60e5b486d9ff670c76b799f886b7f47c2022
+hash = ff42e0227d6100285d4179a2572b700fd5a959cb
 local_path = sorc/gfs_post.fd
 repo_url = https://github.com/NOAA-EMC/EMC_post.git
 protocol = git
 required = True
 
 [UFS_UTILS]
-hash = 4f44bf8987ef16ca97a7c19364c9d3cf17e87f4e
+tag = ufs_utils_1_3_0
 local_path = sorc/ufs_utils.fd
 repo_url = https://github.com/NOAA-EMC/UFS_UTILS.git
 protocol = git
@@ -44,7 +44,7 @@ protocol = git
 required = True
 
 [EMC_gfs_wafs]
-tag = gfs_wafs.v6.0.19
+hash = c2a29a67d9432b4d6fba99eac7797b81d05202b6
 local_path = sorc/gfs_wafs.fd
 repo_url = https://github.com/NOAA-EMC/EMC_gfs_wafs.git
 protocol = git

From eb1613764794a35d165e55423682ed592b2aaacb Mon Sep 17 00:00:00 2001
From: "kate.friedman" <kate.friedman@noaa.gov>
Date: Wed, 5 May 2021 14:22:06 +0000
Subject: [PATCH 051/125] ufs-weather-model hash update for WW3 support with
 hpc-stack

- update ufs-weather-model hash to 554aedcd63e4a7c5012570406132eaf76e249ca9
- update build_fv3.sh for new compile arguments for CCPP/WW3 build

Refs: #164
---
 sorc/build_fv3.sh | 12 +++++++-----
 sorc/checkout.sh  |  2 +-
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/sorc/build_fv3.sh b/sorc/build_fv3.sh
index a4f99fa411b..ca699fc73ed 100755
--- a/sorc/build_fv3.sh
+++ b/sorc/build_fv3.sh
@@ -23,9 +23,11 @@ cd fv3gfs.fd/
 FV3=$( pwd -P )/FV3
 cd tests/
 
-# GFSv16 with waves
-# ./compile.sh "$FV3" "$target" "WW3=Y 32BIT=Y" 1
-# mv -f fv3_1.exe ../NEMS/exe/global_fv3gfs.x
+# Standalone FV3 (no WW3)
+#./compile.sh "$target" "APP=ATM 32BIT=Y SUITES=FV3_GFS_v16" 2 NO NO
 
- ./compile.sh "$target" "CCPP=Y 32BIT=Y SUITES=FV3_GFS_v15,FV3_GFS_v16beta" 2 NO NO
- mv -f fv3_2.exe ../NEMS/exe/global_fv3gfs.x
+# FV3 + WW3
+./compile.sh "$target" "APP=ATMW 32BIT=Y SUITES=FV3_GFS_v16" 2 NO NO
+
+if [ ! -d ../NEMS/exe ]; then mkdir ../NEMS/exe ; fi
+mv -f fv3_2.exe ../NEMS/exe/global_fv3gfs.x
diff --git a/sorc/checkout.sh b/sorc/checkout.sh
index d499a23271e..13f98b5b48f 100755
--- a/sorc/checkout.sh
+++ b/sorc/checkout.sh
@@ -28,7 +28,7 @@ if [[ ! -d fv3gfs.fd ]] ; then
     rm -f ${topdir}/checkout-fv3gfs.log
     git clone https://github.com/ufs-community/ufs-weather-model fv3gfs.fd >> ${topdir}/checkout-fv3gfs.log 2>&1
     cd fv3gfs.fd
-    git checkout 06bc77ba3640358b54b6ff03f542e2ce0d3242be
+    git checkout 554aedcd63e4a7c5012570406132eaf76e249ca9
     git submodule update --init --recursive
     cd ${topdir}
 else

From 214431a2c04f6e685e966810621fc2fae771d2f5 Mon Sep 17 00:00:00 2001
From: "kate.friedman" <kate.friedman@noaa.gov>
Date: Wed, 5 May 2021 14:26:00 +0000
Subject: [PATCH 052/125] Update WAFS execs to include .x

- update WAFS exec names in link_fv3gfs.sh to include ".x" extension that was added with new version

Refs: #164
---
 sorc/link_fv3gfs.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/sorc/link_fv3gfs.sh b/sorc/link_fv3gfs.sh
index 6c1f870ebc2..a8605ffd407 100755
--- a/sorc/link_fv3gfs.sh
+++ b/sorc/link_fv3gfs.sh
@@ -219,9 +219,9 @@ $LINK ../sorc/gfs_post.fd/exec/upp.x gfs_ncep_post
 
 if [ -d ${pwd}/gfs_wafs.fd ]; then 
     for wafsexe in \
-          wafs_awc_wafavn  wafs_blending  wafs_blending_0p25 \
-          wafs_cnvgrib2  wafs_gcip  wafs_grib2_0p25 \
-          wafs_makewafs  wafs_setmissing; do
+          wafs_awc_wafavn.x  wafs_blending.x  wafs_blending_0p25.x \
+          wafs_cnvgrib2.x  wafs_gcip.x  wafs_grib2_0p25.x \
+          wafs_makewafs.x  wafs_setmissing.x ; do
         [[ -s $wafsexe ]] && rm -f $wafsexe
         $LINK ../sorc/gfs_wafs.fd/exec/$wafsexe .
     done

From 1aea3ee393c99d4f76556fbbdbebc43f629558c2 Mon Sep 17 00:00:00 2001
From: "kate.friedman" <kate.friedman@noaa.gov>
Date: Wed, 5 May 2021 14:30:06 +0000
Subject: [PATCH 053/125] Add build script and modulefiles for WW3 build

- WW3 execs no longer built by FV3 build, need separate build now
- add new build_ww3prepost.sh script for building WW3 execs in ufs-weather-model checkout
- add modulefiles for WCOSS-Dell, Hera, and Orion

Refs: #164
---
 modulefiles/modulefile.ww3.hera          | 30 +++++++++
 modulefiles/modulefile.ww3.orion         | 31 +++++++++
 modulefiles/modulefile.ww3.wcoss_dell_p3 | 26 ++++++++
 sorc/build_ww3prepost.sh                 | 84 ++++++++++++++++++++++++
 4 files changed, 171 insertions(+)
 create mode 100644 modulefiles/modulefile.ww3.hera
 create mode 100644 modulefiles/modulefile.ww3.orion
 create mode 100644 modulefiles/modulefile.ww3.wcoss_dell_p3
 create mode 100755 sorc/build_ww3prepost.sh

diff --git a/modulefiles/modulefile.ww3.hera b/modulefiles/modulefile.ww3.hera
new file mode 100644
index 00000000000..a58a482c34e
--- /dev/null
+++ b/modulefiles/modulefile.ww3.hera
@@ -0,0 +1,30 @@
+#%Module######################################################################
+##
+##      S2S prerequisites
+##
+
+module use /scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack
+
+module load hpc/1.1.0
+module load hpc-intel/18.0.5.274
+module load hpc-impi/2018.0.4
+
+module load jasper/2.0.25
+module load zlib/1.2.11
+module load png/1.6.35
+
+module load hdf5/1.10.6
+module load netcdf/4.7.4
+module load esmf/8_1_0_beta_snapshot_36
+
+module load bacio/2.4.1
+module load crtm/2.3.0
+module load g2/3.4.1
+module load g2tmpl/1.9.1
+module load ip/3.3.3
+module load nceppost/dceca26
+module load nemsio/2.5.2
+module load sp/2.3.3
+module load w3emc/2.7.3
+module load w3nco/2.4.1
+module load wgrib2/2.0.8
diff --git a/modulefiles/modulefile.ww3.orion b/modulefiles/modulefile.ww3.orion
new file mode 100644
index 00000000000..19c370de43e
--- /dev/null
+++ b/modulefiles/modulefile.ww3.orion
@@ -0,0 +1,31 @@
+#%Module######################################################################
+# module for ww3 before base uses hpc-stack
+module load contrib noaatools
+
+module load cmake/3.17.3
+
+module use /apps/contrib/NCEP/libs/hpc-stack/modulefiles/stack
+
+module load hpc/1.1.0
+module load hpc-intel/2018.4
+module load hpc-impi/2018.4
+
+module load jasper/2.0.25
+module load zlib/1.2.11
+module load png/1.6.35
+
+module load hdf5/1.10.6
+module load netcdf/4.7.4
+module load esmf/8_1_0_beta_snapshot_27
+
+module load bacio/2.4.1
+module load crtm/2.3.0
+module load g2/3.4.1
+module load g2tmpl/1.9.1
+module load ip/3.3.3
+module load nceppost/dceca26
+module load nemsio/2.5.2
+module load sp/2.3.3
+module load w3emc/2.7.3
+module load w3nco/2.4.1
+module load wgrib2/2.0.8
diff --git a/modulefiles/modulefile.ww3.wcoss_dell_p3 b/modulefiles/modulefile.ww3.wcoss_dell_p3
new file mode 100644
index 00000000000..16784b7d1d2
--- /dev/null
+++ b/modulefiles/modulefile.ww3.wcoss_dell_p3
@@ -0,0 +1,26 @@
+#%Module######################################################################
+
+module use /usrx/local/nceplibs/dev/hpc-stack/libs/hpc-stack/modulefiles/stack
+module load hpc/1.1.0
+module load hpc-ips/18.0.1.163
+module load hpc-impi/18.0.1
+
+module load jasper/2.0.25
+module load zlib/1.2.11
+module load png/1.6.35
+
+module load hdf5/1.10.6
+module load netcdf/4.7.4
+module load esmf/8_1_0_beta_snapshot_36
+
+module load bacio/2.4.1
+module load crtm/2.3.0
+module load g2/3.4.1
+module load g2tmpl/1.9.1
+module load ip/3.3.3
+module load nceppost/dceca26
+module load nemsio/2.5.2
+module load sp/2.3.3
+module load w3emc/2.7.3
+module load w3nco/2.4.1
+module load wgrib2/2.0.8
diff --git a/sorc/build_ww3prepost.sh b/sorc/build_ww3prepost.sh
new file mode 100755
index 00000000000..0ce39088e4d
--- /dev/null
+++ b/sorc/build_ww3prepost.sh
@@ -0,0 +1,84 @@
+#!/bin/sh
+set -x
+
+# Check final exec folder exists
+if [ ! -d "../exec" ]; then
+  mkdir ../exec
+fi
+
+finalexecdir=$( pwd -P )/../exec
+
+set +x
+source ./machine-setup.sh > /dev/null 2>&1
+
+source ../modulefiles/modulefile.ww3.$target
+set -x 
+
+cd fv3gfs.fd/WW3
+export WW3_DIR=$( pwd -P )/model
+export WW3_BINDIR="${WW3_DIR}/bin"
+export WW3_TMPDIR=${WW3_DIR}/tmp
+export WW3_EXEDIR=${WW3_DIR}/exe
+export WW3_COMP=$target 
+export WW3_CC=gcc
+export WW3_F90=gfortran
+export SWITCHFILE="${WW3_DIR}/esmf/switch"
+
+export WWATCH3_ENV=${WW3_BINDIR}/wwatch3.env
+export PNG_LIB=${PNG_LIB:-$PNG_ROOT/lib64/libpng.a}
+export Z_LIB=${Z_LIB:-$ZLIB_ROOT/lib/libz.a}
+export JASPER_LIB=${JASPER_LIB:-$JASPER_ROOT/lib64/libjasper.a}
+export WWATCH3_NETCDF=NC4
+export NETCDF_CONFIG=$NETCDF_ROOT/bin/nc-config
+
+rm  $WWATCH3_ENV
+echo '#'                                              > $WWATCH3_ENV
+echo '# ---------------------------------------'      >> $WWATCH3_ENV
+echo '# Environment variables for wavewatch III'      >> $WWATCH3_ENV
+echo '# ---------------------------------------'      >> $WWATCH3_ENV
+echo '#'                                              >> $WWATCH3_ENV
+echo "WWATCH3_LPR      $PRINTER"                      >> $WWATCH3_ENV
+echo "WWATCH3_F90      $WW3_F90"                      >> $WWATCH3_ENV
+echo "WWATCH3_CC       $WW3_CC"                       >> $WWATCH3_ENV
+echo "WWATCH3_DIR      $WW3_DIR"                      >> $WWATCH3_ENV
+echo "WWATCH3_TMP      $WW3_TMPDIR"                   >> $WWATCH3_ENV
+echo 'WWATCH3_SOURCE   yes'                           >> $WWATCH3_ENV
+echo 'WWATCH3_LIST     yes'                           >> $WWATCH3_ENV
+echo ''                                               >> $WWATCH3_ENV
+
+${WW3_BINDIR}/w3_clean -m 
+${WW3_BINDIR}/w3_setup -q -c $WW3_COMP $WW3_DIR
+
+echo $(cat ${SWITCHFILE}) > ${WW3_BINDIR}/tempswitch
+
+sed -e "s/DIST/SHRD/g"\
+    -e "s/OMPG/ /g"\
+    -e "s/OMPH/ /g"\
+    -e "s/MPIT/ /g"\
+    -e "s/MPI/ /g"\
+    -e "s/PDLIB/ /g"\
+       ${WW3_BINDIR}/tempswitch > ${WW3_BINDIR}/switch
+
+#Build exes for prep jobs: 
+${WW3_BINDIR}/w3_make ww3_grid 
+
+#Build exes for post jobs (except grib)"
+${WW3_BINDIR}/w3_make ww3_outp 
+
+#Update switch for grib: 
+echo $(cat ${SWITCHFILE}) > ${WW3_BINDIR}/tempswitch
+
+sed -e "s/DIST/SHRD/g"\
+    -e "s/OMPG/ /g"\
+    -e "s/OMPH/ /g"\
+    -e "s/MPIT/ /g"\
+    -e "s/MPI/ /g"\
+    -e "s/PDLIB/ /g"\
+    -e "s/NOGRB/NCEP2 NCO/g"\
+       ${WW3_BINDIR}/tempswitch > ${WW3_BINDIR}/switch
+#Build exe for grib
+${WW3_BINDIR}/w3_make ww3_grib
+
+cp $WW3_EXEDIR/ww3_* $finalexecdir/
+${WW3_BINDIR}/w3_clean -c
+rm ${WW3_BINDIR}/tempswitch 

From aa1d1e9ca1d8521ef22e1bdb654d4a70e7b1e29e Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Mon, 10 May 2021 14:27:13 +0000
Subject: [PATCH 054/125] Change CCPP_SUITE from FV3_GFS_v16beta to FV3_GFS_v16

- newer ufs-weather-model hash supports FV3_GFS_v16 suite

Refs: #164
---
 parm/config/config.base.emc.dyn    | 2 +-
 parm/config/config.base.nco.static | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/parm/config/config.base.emc.dyn b/parm/config/config.base.emc.dyn
index 46da367a0f5..59de56b8a82 100755
--- a/parm/config/config.base.emc.dyn
+++ b/parm/config/config.base.emc.dyn
@@ -141,7 +141,7 @@ export CASE_ENKF="@CASEENS@"
 
 # Run with CCPP physics
 export RUN_CCPP="YES"
-export CCPP_SUITE="FV3_GFS_v16beta"
+export CCPP_SUITE="FV3_GFS_v16"
 
 # Surface cycle update frequency
 if [[ "$CDUMP" == "gdas" ]] ; then
diff --git a/parm/config/config.base.nco.static b/parm/config/config.base.nco.static
index 7f5788dddc5..f6ed8dc293b 100755
--- a/parm/config/config.base.nco.static
+++ b/parm/config/config.base.nco.static
@@ -125,8 +125,8 @@ export CASE="C768"
 export CASE_ENKF="C384"
 
 # Run with CCPP physics
-export RUN_CCPP="NO"
-export CCPP_SUITE="FV3_GFS_v16beta"
+export RUN_CCPP="YES"
+export CCPP_SUITE="FV3_GFS_v16"
 
 # Surface cycle update frequency
 if [[ "$CDUMP" == "gdas" ]] ; then

From c6e2a96e86611cce00455609d9be803d580e7bc0 Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Mon, 10 May 2021 14:30:13 +0000
Subject: [PATCH 055/125] Change fix file linking to scan for possible
 subfolders in set

- instead of providing subfolder names, scan for subfolders in FIX_DIR path
and then create symlinks based on existing subfolders in set
- will not have to keep updating list of subfolders now

Refs: #164
---
 sorc/link_fv3gfs.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/sorc/link_fv3gfs.sh b/sorc/link_fv3gfs.sh
index a8605ffd407..4a10a53eecb 100755
--- a/sorc/link_fv3gfs.sh
+++ b/sorc/link_fv3gfs.sh
@@ -40,7 +40,8 @@ elif [ $machine = "orion" ]; then
     FIX_DIR="/work/noaa/global/glopara/fix"
 fi
 cd ${pwd}/../fix                ||exit 8
-for dir in fix_am fix_fv3_gmted2010 fix_gldas fix_orog fix_verif fix_wave_gfs ; do
+dirs=`ls $FIX_DIR`
+for dir in $dirs ; do
     if [ -d $dir ]; then
       [[ $RUN_ENVIR = nco ]] && chmod -R 755 $dir
       rm -rf $dir

From d46e8cf49aff0dbf72e457f1177e10e6f50252a6 Mon Sep 17 00:00:00 2001
From: Jessica Meixner <Jessica.Meixner@noaa.gov>
Date: Mon, 10 May 2021 15:31:56 +0000
Subject: [PATCH 056/125] update for ww3_pre/post build: -- remove ww3 specific
 module files -- updated module base files to be in sync with
 ufs-weather-model    except that jasper is a more recent version which is
 needed for ww3_grib -- added extra ww3 execs needed for this branch -- added
 ww3prepost build to build_all and partial build

---
 modulefiles/module_base.hera             | 20 ++++++++++++---
 modulefiles/modulefile.ww3.hera          | 30 -----------------------
 modulefiles/modulefile.ww3.orion         | 31 ------------------------
 modulefiles/modulefile.ww3.wcoss_dell_p3 | 26 --------------------
 sorc/build_all.sh                        | 14 +++++++++++
 sorc/build_ww3prepost.sh                 | 16 ++++++++++--
 sorc/fv3gfs_build.cfg                    |  1 +
 sorc/partial_build.sh                    |  1 +
 8 files changed, 47 insertions(+), 92 deletions(-)
 delete mode 100644 modulefiles/modulefile.ww3.hera
 delete mode 100644 modulefiles/modulefile.ww3.orion
 delete mode 100644 modulefiles/modulefile.ww3.wcoss_dell_p3

diff --git a/modulefiles/module_base.hera b/modulefiles/module_base.hera
index 6b21d485d56..adf76e47675 100644
--- a/modulefiles/module_base.hera
+++ b/modulefiles/module_base.hera
@@ -15,15 +15,28 @@ module load gempak/7.4.2
 #Load from hpc-stack
 module load prod_util/1.2.2
 module load grib_util/1.2.2
-module load g2tmpl/1.9.1
 
 module load crtm/2.3.0
 setenv CRTM_FIX /scratch2/NCEPDEV/nwprod/NCEPLIBS/fix/crtm_v2.3.0
 
+module load jasper/2.0.25
 module load zlib/1.2.11
+module load png/1.6.35
+
 module load hdf5/1.10.6
 module load netcdf/4.7.4
-module load esmf/8_1_0_beta_snapshot_27
+module load pio/2.5.2
+module load esmf/8_1_0_beta_snapshot_47
+
+module load bacio/2.4.1
+module load g2/3.4.1
+module load g2tmpl/1.9.1
+module load ip/3.3.3
+module load nemsio/2.5.2
+module load sp/2.3.3
+module load w3emc/2.7.3
+module load w3nco/2.4.1
+module load upp/10.0.6
 
 module load wgrib2/2.0.8
 setenv WGRIB2 wgrib2
@@ -33,4 +46,5 @@ module use -a /contrib/anaconda/modulefiles
 module load anaconda/2.3.0
 
 # waveprep
-module load cdo/1.9.5
+module load cdo/1.9.5 
+
diff --git a/modulefiles/modulefile.ww3.hera b/modulefiles/modulefile.ww3.hera
deleted file mode 100644
index a58a482c34e..00000000000
--- a/modulefiles/modulefile.ww3.hera
+++ /dev/null
@@ -1,30 +0,0 @@
-#%Module######################################################################
-##
-##      S2S prerequisites
-##
-
-module use /scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack
-
-module load hpc/1.1.0
-module load hpc-intel/18.0.5.274
-module load hpc-impi/2018.0.4
-
-module load jasper/2.0.25
-module load zlib/1.2.11
-module load png/1.6.35
-
-module load hdf5/1.10.6
-module load netcdf/4.7.4
-module load esmf/8_1_0_beta_snapshot_36
-
-module load bacio/2.4.1
-module load crtm/2.3.0
-module load g2/3.4.1
-module load g2tmpl/1.9.1
-module load ip/3.3.3
-module load nceppost/dceca26
-module load nemsio/2.5.2
-module load sp/2.3.3
-module load w3emc/2.7.3
-module load w3nco/2.4.1
-module load wgrib2/2.0.8
diff --git a/modulefiles/modulefile.ww3.orion b/modulefiles/modulefile.ww3.orion
deleted file mode 100644
index 19c370de43e..00000000000
--- a/modulefiles/modulefile.ww3.orion
+++ /dev/null
@@ -1,31 +0,0 @@
-#%Module######################################################################
-# module for ww3 before base uses hpc-stack
-module load contrib noaatools
-
-module load cmake/3.17.3
-
-module use /apps/contrib/NCEP/libs/hpc-stack/modulefiles/stack
-
-module load hpc/1.1.0
-module load hpc-intel/2018.4
-module load hpc-impi/2018.4
-
-module load jasper/2.0.25
-module load zlib/1.2.11
-module load png/1.6.35
-
-module load hdf5/1.10.6
-module load netcdf/4.7.4
-module load esmf/8_1_0_beta_snapshot_27
-
-module load bacio/2.4.1
-module load crtm/2.3.0
-module load g2/3.4.1
-module load g2tmpl/1.9.1
-module load ip/3.3.3
-module load nceppost/dceca26
-module load nemsio/2.5.2
-module load sp/2.3.3
-module load w3emc/2.7.3
-module load w3nco/2.4.1
-module load wgrib2/2.0.8
diff --git a/modulefiles/modulefile.ww3.wcoss_dell_p3 b/modulefiles/modulefile.ww3.wcoss_dell_p3
deleted file mode 100644
index 16784b7d1d2..00000000000
--- a/modulefiles/modulefile.ww3.wcoss_dell_p3
+++ /dev/null
@@ -1,26 +0,0 @@
-#%Module######################################################################
-
-module use /usrx/local/nceplibs/dev/hpc-stack/libs/hpc-stack/modulefiles/stack
-module load hpc/1.1.0
-module load hpc-ips/18.0.1.163
-module load hpc-impi/18.0.1
-
-module load jasper/2.0.25
-module load zlib/1.2.11
-module load png/1.6.35
-
-module load hdf5/1.10.6
-module load netcdf/4.7.4
-module load esmf/8_1_0_beta_snapshot_36
-
-module load bacio/2.4.1
-module load crtm/2.3.0
-module load g2/3.4.1
-module load g2tmpl/1.9.1
-module load ip/3.3.3
-module load nceppost/dceca26
-module load nemsio/2.5.2
-module load sp/2.3.3
-module load w3emc/2.7.3
-module load w3nco/2.4.1
-module load wgrib2/2.0.8
diff --git a/sorc/build_all.sh b/sorc/build_all.sh
index 2d11fd04c26..952bf7a97dd 100755
--- a/sorc/build_all.sh
+++ b/sorc/build_all.sh
@@ -71,6 +71,20 @@ fi
 ((err+=$rc))
 }
 
+#------------------------------------
+# build WW3 pre & post execs 
+#------------------------------------
+$Build_ww3_prepost && {
+echo " .... Building WW3 pre and post execs .... "
+./build_ww3prepost.sh > $logs_dir/build_ww3_prepost.log 2>&1
+rc=$?
+if [[ $rc -ne 0 ]] ; then
+    echo "Fatal error in building WW3 pre/post processing."
+    echo "The log file is in $logs_dir/build_ww3_prepost.log"
+fi
+((err+=$rc))
+}
+
 #------------------------------------
 # build gsi
 #------------------------------------
diff --git a/sorc/build_ww3prepost.sh b/sorc/build_ww3prepost.sh
index 0ce39088e4d..06e28a38045 100755
--- a/sorc/build_ww3prepost.sh
+++ b/sorc/build_ww3prepost.sh
@@ -10,10 +10,15 @@ finalexecdir=$( pwd -P )/../exec
 
 set +x
 source ./machine-setup.sh > /dev/null 2>&1
-
-source ../modulefiles/modulefile.ww3.$target
+source ../modulefiles/module_base.$target
 set -x 
 
+
+if [ $target = hera ]; then target=hera.intel ; fi
+if [ $target = orion ]; then target=orion.intel ; fi
+if [ $target = stampede ]; then target=stampede.intel ; fi
+
+#cd ufs_coupled.fd/WW3
 cd fv3gfs.fd/WW3
 export WW3_DIR=$( pwd -P )/model
 export WW3_BINDIR="${WW3_DIR}/bin"
@@ -61,9 +66,16 @@ sed -e "s/DIST/SHRD/g"\
 
 #Build exes for prep jobs: 
 ${WW3_BINDIR}/w3_make ww3_grid 
+${WW3_BINDIR}/w3_make ww3_prep
+${WW3_BINDIR}/w3_make ww3_prnc
 
 #Build exes for post jobs (except grib)"
 ${WW3_BINDIR}/w3_make ww3_outp 
+${WW3_BINDIR}/w3_make ww3_outf
+${WW3_BINDIR}/w3_make ww3_outp
+${WW3_BINDIR}/w3_make ww3_gint
+${WW3_BINDIR}/w3_make ww3_ounf
+${WW3_BINDIR}/w3_make ww3_ounp
 
 #Update switch for grib: 
 echo $(cat ${SWITCHFILE}) > ${WW3_BINDIR}/tempswitch
diff --git a/sorc/fv3gfs_build.cfg b/sorc/fv3gfs_build.cfg
index 161a21325ef..601b13713a7 100644
--- a/sorc/fv3gfs_build.cfg
+++ b/sorc/fv3gfs_build.cfg
@@ -2,6 +2,7 @@
 # ***** configuration of global-workflow build *****
 
  Building fv3gfs (fv3gfs) .............................. yes
+ Building ww3_prepost (ww3_prepost) .................... yes
  Building gsi (gsi) .................................... yes
  Building gldas (gldas) ................................ yes
  Building ncep_post (ncep_post) ........................ yes
diff --git a/sorc/partial_build.sh b/sorc/partial_build.sh
index 691a3c7b3a1..3be40418805 100755
--- a/sorc/partial_build.sh
+++ b/sorc/partial_build.sh
@@ -2,6 +2,7 @@
 # define the array of the name of build program
 #
  declare -a Build_prg=("Build_fv3gfs" \
+                       "Build_ww3_prepost" \
                        "Build_gsi" \
                        "Build_gldas" \
                        "Build_ncep_post" \

From 9cf615d11b096ea4843f4e5938a6a65f3500df49 Mon Sep 17 00:00:00 2001
From: "kate.friedman" <kate.friedman@noaa.gov>
Date: Thu, 20 May 2021 20:10:18 +0000
Subject: [PATCH 057/125] Modulefile updates for testing with waves on

- updated ww3 and base modulefiles to add modules and update some
- added new: pio/2.5.2i, fms/2020.04.03
- updated esmf module to esmf/8_1_1 and nceppost/dceca26 to upp/10.0.6

Refs: #164
---
 modulefiles/module_base.hera             |  3 ++-
 modulefiles/module_base.orion            | 18 ++++++++++++++++--
 modulefiles/module_base.wcoss_dell_p3    | 19 +++++++++++++++++--
 modulefiles/modulefile.ww3.hera          |  6 ++++--
 modulefiles/modulefile.ww3.orion         |  6 ++++--
 modulefiles/modulefile.ww3.wcoss_dell_p3 |  6 ++++--
 6 files changed, 47 insertions(+), 11 deletions(-)

diff --git a/modulefiles/module_base.hera b/modulefiles/module_base.hera
index adf76e47675..3b885306a1a 100644
--- a/modulefiles/module_base.hera
+++ b/modulefiles/module_base.hera
@@ -26,7 +26,8 @@ module load png/1.6.35
 module load hdf5/1.10.6
 module load netcdf/4.7.4
 module load pio/2.5.2
-module load esmf/8_1_0_beta_snapshot_47
+module load esmf/8_1_1
+module load fms/2020.04.03
 
 module load bacio/2.4.1
 module load g2/3.4.1
diff --git a/modulefiles/module_base.orion b/modulefiles/module_base.orion
index 2110560cdbb..dacf9e1e847 100755
--- a/modulefiles/module_base.orion
+++ b/modulefiles/module_base.orion
@@ -14,15 +14,29 @@ module load gempak/7.5.1
 
 module load grib_util/1.2.2
 module load prod_util/1.2.2
-module load g2tmpl/1.9.1
 
 module load crtm/2.3.0
 setenv CRTM_FIX /apps/contrib/NCEPLIBS/orion/fix/crtm_v2.3.0
 
+module load jasper/2.0.25
 module load zlib/1.2.11
+module load png/1.6.35
+
 module load hdf5/1.10.6
 module load netcdf/4.7.4
-module load esmf/8_1_0_beta_snapshot_27
+module load pio/2.5.2
+module load esmf/8_1_1
+module load fms/2020.04.03
+
+module load bacio/2.4.1
+module load g2/3.4.1
+module load g2tmpl/1.9.1
+module load ip/3.3.3
+module load nemsio/2.5.2
+module load sp/2.3.3
+module load w3emc/2.7.3
+module load w3nco/2.4.1
+module load upp/10.0.6
 
 module load wgrib/2.0.8
 setenv WGRIB2 wgrib2
diff --git a/modulefiles/module_base.wcoss_dell_p3 b/modulefiles/module_base.wcoss_dell_p3
index 9eec3351447..9945cbbefd3 100755
--- a/modulefiles/module_base.wcoss_dell_p3
+++ b/modulefiles/module_base.wcoss_dell_p3
@@ -16,7 +16,6 @@ module load HPSS/5.0.2.5
 module load prod_util/1.2.2
 module load prod_envir/1.1.0
 module load grib_util/1.2.2
-module load g2tmpl/1.9.1
 module load util_shared/1.3.0
 
 module load crtm/2.3.0
@@ -27,9 +26,25 @@ module load CFP/2.0.2
 setenv USE_CFP YES
 module load pm5
 
+module load jasper/2.0.25
+module load zlib/1.2.11
+module load png/1.6.35
+
 module load hdf5/1.10.6
 module load netcdf/4.7.4
-module load esmf/8_1_0_beta_snapshot_27
+module load pio/2.5.2
+module load esmf/8_1_1
+module load fms/2020.04.03
+
+module load bacio/2.4.1
+module load g2/3.4.1
+module load g2tmpl/1.9.1
+module load ip/3.3.3
+module load nemsio/2.5.2
+module load sp/2.3.3
+module load w3emc/2.7.3
+module load w3nco/2.4.1
+module load upp/10.0.6
 
 module load wgrib2/2.0.8
 setenv WGRIB2 wgrib2
diff --git a/modulefiles/modulefile.ww3.hera b/modulefiles/modulefile.ww3.hera
index a58a482c34e..fd3f0830315 100644
--- a/modulefiles/modulefile.ww3.hera
+++ b/modulefiles/modulefile.ww3.hera
@@ -15,16 +15,18 @@ module load png/1.6.35
 
 module load hdf5/1.10.6
 module load netcdf/4.7.4
-module load esmf/8_1_0_beta_snapshot_36
+module load pio/2.5.2
+module load esmf/8_1_1
+module load fms/2020.04.03
 
 module load bacio/2.4.1
 module load crtm/2.3.0
 module load g2/3.4.1
 module load g2tmpl/1.9.1
 module load ip/3.3.3
-module load nceppost/dceca26
 module load nemsio/2.5.2
 module load sp/2.3.3
 module load w3emc/2.7.3
 module load w3nco/2.4.1
 module load wgrib2/2.0.8
+module load upp/10.0.6
diff --git a/modulefiles/modulefile.ww3.orion b/modulefiles/modulefile.ww3.orion
index 19c370de43e..1bc98a32f79 100644
--- a/modulefiles/modulefile.ww3.orion
+++ b/modulefiles/modulefile.ww3.orion
@@ -16,16 +16,18 @@ module load png/1.6.35
 
 module load hdf5/1.10.6
 module load netcdf/4.7.4
-module load esmf/8_1_0_beta_snapshot_27
+module load pio/2.5.2
+module load esmf/8_1_1
+module load fms/2020.04.03
 
 module load bacio/2.4.1
 module load crtm/2.3.0
 module load g2/3.4.1
 module load g2tmpl/1.9.1
 module load ip/3.3.3
-module load nceppost/dceca26
 module load nemsio/2.5.2
 module load sp/2.3.3
 module load w3emc/2.7.3
 module load w3nco/2.4.1
 module load wgrib2/2.0.8
+module load upp/10.0.6
diff --git a/modulefiles/modulefile.ww3.wcoss_dell_p3 b/modulefiles/modulefile.ww3.wcoss_dell_p3
index 16784b7d1d2..20e84f704fc 100644
--- a/modulefiles/modulefile.ww3.wcoss_dell_p3
+++ b/modulefiles/modulefile.ww3.wcoss_dell_p3
@@ -11,16 +11,18 @@ module load png/1.6.35
 
 module load hdf5/1.10.6
 module load netcdf/4.7.4
-module load esmf/8_1_0_beta_snapshot_36
+module load pio/2.5.2
+module load esmf/8_1_1
+module load fms/2020.04.03
 
 module load bacio/2.4.1
 module load crtm/2.3.0
 module load g2/3.4.1
 module load g2tmpl/1.9.1
 module load ip/3.3.3
-module load nceppost/dceca26
 module load nemsio/2.5.2
 module load sp/2.3.3
 module load w3emc/2.7.3
 module load w3nco/2.4.1
 module load wgrib2/2.0.8
+module load upp/10.0.6

From 151e0b56b6e38f40449c9eb0807ffa92b6aa46f9 Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Tue, 25 May 2021 20:41:32 +0000
Subject: [PATCH 058/125] Correct if-statement syntax in config.init

Refs: #178
---
 parm/config/config.init | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/parm/config/config.init b/parm/config/config.init
index d1d504c7c06..eeb04a7d409 100755
--- a/parm/config/config.init
+++ b/parm/config/config.init
@@ -18,7 +18,7 @@ export CRES_HIRES=$CASE
 export CRES_ENKF=$CASE_ENKF
 
 export RUNICSH=${GDASINIT_DIR}/run_v16.chgres.sh
-if [ ${RETRO:-"NO"} = "YES" || $CDUMP = "gdas" ]; then
+if [ "${RETRO:-"NO"}" = "YES" ] || [ "$CDUMP" = "gdas" ]; then
   export RUNICSH=${GDASINIT_DIR}/run_v16retro.chgres.sh
 fi
 

From 824ff7254fba04a0b3726316d300f9fa6282c937 Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Tue, 25 May 2021 20:42:25 +0000
Subject: [PATCH 059/125] Change UFS_UTILS version to feature branch for
 testing

Refs: #178
---
 sorc/checkout.sh | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/sorc/checkout.sh b/sorc/checkout.sh
index 62d6fcd6d04..34c80c36dcf 100755
--- a/sorc/checkout.sh
+++ b/sorc/checkout.sh
@@ -68,9 +68,11 @@ fi
 echo ufs_utils checkout ...
 if [[ ! -d ufs_utils.fd ]] ; then
     rm -f ${topdir}/checkout-ufs_utils.log
-    git clone https://github.com/NOAA-EMC/UFS_UTILS.git ufs_utils.fd >> ${topdir}/checkout-ufs_utils.fd.log 2>&1
+    #git clone https://github.com/NOAA-EMC/UFS_UTILS.git ufs_utils.fd >> ${topdir}/checkout-ufs_utils.fd.log 2>&1
+    git clone --recursive https://github.com/GeorgeGayno-NOAA/UFS_UTILS.git ufs_utils.fd >> ${topdir}/checkout-ufs_utils.fd.log 2>&1
     cd ufs_utils.fd
-    git checkout ops-gfsv16.0.0
+    #git checkout ops-gfsv16.0.0
+    git checkout feature/gdas_init.v16
     cd ${topdir}
 else
     echo 'Skip.  Directory ufs_utils.fd already exists.'

From 499f217f92b5bb0fad9e71de221937d9973046ce Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Tue, 1 Jun 2021 15:49:41 +0000
Subject: [PATCH 060/125] Resolve bug in wavepostpnt jobs when DOBLL_WAV=NO

- change DOBNDPNT_WAV=YES and CFP_MP=YES if-block to only prepare cmdtarfile for CFP
when DOBLL_WAV=YES to avoid scenario where only a single command is added to cmdtarfile

Refs: #178
---
 scripts/exgfs_wave_post_pnt.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/exgfs_wave_post_pnt.sh b/scripts/exgfs_wave_post_pnt.sh
index 45f74796dd1..4ee72c3c2d7 100755
--- a/scripts/exgfs_wave_post_pnt.sh
+++ b/scripts/exgfs_wave_post_pnt.sh
@@ -624,7 +624,7 @@
 
   if [ ${CFP_MP:-"NO"} = "YES" ]; then nm=0; fi
 
-  if [ ${CFP_MP:-"NO"} = "YES" ]; then
+  if [ ${CFP_MP:-"NO"} = "YES" ] && [ "$DOBLL_WAV" = "YES" ]; then
     if [ "$DOBNDPNT_WAV" = YES ]; then
       if [ "$DOSPC_WAV" = YES ]; then
         echo "$nm $USHwave/wave_tar.sh $WAV_MOD_TAG ibp $Nb > ${WAV_MOD_TAG}_spec_tar.out 2>&1 "   >> cmdtarfile

From 127723adfde2b7b7c79bd4d346cafcb1f265dc1c Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Tue, 1 Jun 2021 16:44:37 +0000
Subject: [PATCH 061/125] getic job updates to add OPS_RES variable and pull
 from operational tarballs for warm starting

- add OPS_RES variable to define current operational resolution; use variable where needed now
- updated getic job script to use OPS_RES variable in if-block check
- updated getic job script with new operational tarballs to pull for warm starting high res runs

Refs: #178
---
 jobs/rocoto/getic.sh            | 21 ++++++++++++++++-----
 parm/config/config.base.emc.dyn |  1 +
 2 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/jobs/rocoto/getic.sh b/jobs/rocoto/getic.sh
index 33ca3789d99..89c207a281e 100755
--- a/jobs/rocoto/getic.sh
+++ b/jobs/rocoto/getic.sh
@@ -40,6 +40,11 @@ export yy=$(echo $CDATE | cut -c1-4)
 export mm=$(echo $CDATE | cut -c5-6)
 export dd=$(echo $CDATE | cut -c7-8)
 export hh=${cyc:-$(echo $CDATE | cut -c9-10)}
+export GDATE=$($NDATE -${assim_freq:-"06"} $CDATE)
+export gyy=$(echo $GDATE | cut -c1-4)
+export gmm=$(echo $GDATE | cut -c5-6)
+export gdd=$(echo $GDATE | cut -c7-8)
+export ghh=$(echo $GDATE | cut -c9-10)
 
 export DATA=${DATA:-${DATAROOT}/init}
 export EXTRACT_DIR=${EXTRACT_DIR:-$ROTDIR}
@@ -47,16 +52,17 @@ export WORKDIR=${WORKDIR:-$DATA}
 export OUTDIR=${OUTDIR:-$ROTDIR}
 export PRODHPSSDIR=${PRODHPSSDIR:-/NCEPPROD/hpssprod/runhistory}
 export COMPONENT="atmos"
-export gfs_ver=${gfs_ver:-v16}
+export gfs_ver=${gfs_ver:-"v16"}
+export OPS_RES=${OPS_RES:-"C768"}
 export GETICSH=${GETICSH:-${GDASINIT_DIR}/get_v16.data.sh}
 
 # Run get data script
 if [ ! -d $EXTRACT_DIR ]; then mkdir -p $EXTRACT_DIR ; fi
 
 # Check if init is needed and run if so
-if [[ $gfs_ver = "v16" && $EXP_WARM_START = ".true." && $CASE = "C768" ]]; then
-  # Pull RESTART files off HPSS
+if [[ $gfs_ver = "v16" && $EXP_WARM_START = ".true." && $CASE = $OPS_RES ]]; then
 
+  # Pull RESTART files off HPSS
   if [ ! -d $ROTDIR ]; then mkdir $ROTDIR ; fi
   cd $ROTDIR
 
@@ -85,8 +91,13 @@ if [[ $gfs_ver = "v16" && $EXP_WARM_START = ".true." && $CASE = "C768" ]]; then
     status=$?
     [[ $status -ne 0 ]] && exit $status
 
-# else # Opertional input
-#   # ADD AFTER IMPLEMENTATION
+  else # Opertional input - warm starts
+
+    # Pull CDATE gfs restart tarball
+    htar -xvf ${PRODHPSSDIR}/rh${yy}/${yy}${mm}/${yy}${mm}${dd}/com_gfs_prod_gfs.${yy}${mm}${dd}_${hh}.gfs_restart.tar
+    # Pull GDATE gdas restart tarball
+    htar -xvf ${PRODHPSSDIR}/rh${gyy}/${gyy}${gmm}/${gyy}${gmm}${gdd}/com_gfs_prod_gfs.${gyy}${gmm}${gdd}_${ghh}.gfs_restart.tar
+
   fi
 
 else
diff --git a/parm/config/config.base.emc.dyn b/parm/config/config.base.emc.dyn
index 26be95adc60..3016b983e42 100755
--- a/parm/config/config.base.emc.dyn
+++ b/parm/config/config.base.emc.dyn
@@ -140,6 +140,7 @@ export DBNROOT=${DBNROOT:-${UTILROOT}/fakedbn}
 export LEVS=128
 export CASE="@CASECTL@"
 export CASE_ENKF="@CASEENS@"
+export OPS_RES="C768"
 
 # Run with CCPP physics
 export RUN_CCPP="YES"

From e62cefb3943a87a90fd40234f08845534df818a1 Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Tue, 1 Jun 2021 16:50:28 +0000
Subject: [PATCH 062/125] Correct gdas operational tarball name in getic job
 script

Refs: #178
---
 jobs/rocoto/getic.sh | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/jobs/rocoto/getic.sh b/jobs/rocoto/getic.sh
index 89c207a281e..f62b4494a5f 100755
--- a/jobs/rocoto/getic.sh
+++ b/jobs/rocoto/getic.sh
@@ -96,8 +96,7 @@ if [[ $gfs_ver = "v16" && $EXP_WARM_START = ".true." && $CASE = $OPS_RES ]]; the
     # Pull CDATE gfs restart tarball
     htar -xvf ${PRODHPSSDIR}/rh${yy}/${yy}${mm}/${yy}${mm}${dd}/com_gfs_prod_gfs.${yy}${mm}${dd}_${hh}.gfs_restart.tar
     # Pull GDATE gdas restart tarball
-    htar -xvf ${PRODHPSSDIR}/rh${gyy}/${gyy}${gmm}/${gyy}${gmm}${gdd}/com_gfs_prod_gfs.${gyy}${gmm}${gdd}_${ghh}.gfs_restart.tar
-
+    htar -xvf ${PRODHPSSDIR}/rh${gyy}/${gyy}${gmm}/${gyy}${gmm}${gdd}/com_gfs_prod_gdas.${gyy}${gmm}${gdd}_${ghh}.gdas_restart.tar
   fi
 
 else

From f0afb91bae738413539ee2f1fdc63b79e93b3e52 Mon Sep 17 00:00:00 2001
From: "kate.friedman" <kate.friedman@noaa.gov>
Date: Wed, 2 Jun 2021 14:22:43 +0000
Subject: [PATCH 063/125] Change BDATE to GDATE in getic.sh

Refs: #178
---
 jobs/rocoto/getic.sh | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/jobs/rocoto/getic.sh b/jobs/rocoto/getic.sh
index f62b4494a5f..d17089e095d 100755
--- a/jobs/rocoto/getic.sh
+++ b/jobs/rocoto/getic.sh
@@ -69,14 +69,13 @@ if [[ $gfs_ver = "v16" && $EXP_WARM_START = ".true." && $CASE = $OPS_RES ]]; the
   if [ ${RETRO:-"NO"} = "YES" ]; then # Retrospective parallel input
 
     # Pull prior cycle restart files
-    BDATE=`$NDATE -06 ${yy}${mm}${dd}${hh}`
-    htar -xvf ${HPSSDIR}/${BDATE}/gdas_restartb.tar
+    htar -xvf ${HPSSDIR}/${GDATE}/gdas_restartb.tar
     status=$?
     [[ $status -ne 0 ]] && exit $status
 
     if [ $DO_WAVE = "YES" ]; then
       # Pull prior cycle wave restart files
-      htar -xvf ${HPSSDIR}/${BDATE}/gdaswave_restart.tar
+      htar -xvf ${HPSSDIR}/${GDATE}/gdaswave_restart.tar
       status=$?
       [[ $status -ne 0 ]] && exit $status
     fi

From e2ac5872ca482eb335bc6294e00594dc3a806718 Mon Sep 17 00:00:00 2001
From: "kate.friedman" <kate.friedman@noaa.gov>
Date: Wed, 2 Jun 2021 14:23:35 +0000
Subject: [PATCH 064/125] Update WCOSS-Dell section of machine-setup.sh

- needed cmake module loaded for building
- added cmake module load and prereq stack loads

Refs: #178
---
 sorc/machine-setup.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/sorc/machine-setup.sh b/sorc/machine-setup.sh
index c39cbfb2722..f0f5586c235 100644
--- a/sorc/machine-setup.sh
+++ b/sorc/machine-setup.sh
@@ -95,6 +95,9 @@ elif [[ -L /usrx && "$( readlink /usrx 2> /dev/null )" =~ dell ]] ; then
     fi
     target=wcoss_dell_p3
     module purge 
+    module use /gpfs/dell2/usrx/local/nceplibs/dev/hpc-stack/libs/hpc-stack/modulefiles
+    module load hpc/1.1.0
+    module load cmake/3.20.0
 
 ##---------------------------------------------------------------------------
 

From 771782f109de15f9f0214bcceb852b51bb4b0e02 Mon Sep 17 00:00:00 2001
From: "kate.friedman" <kate.friedman@noaa.gov>
Date: Thu, 3 Jun 2021 13:09:42 -0500
Subject: [PATCH 065/125] Replace Orion checks with hpssarch checks for getic
 job

- add hpssarch variable to setup_workflow_fcstonly.py
- replace Orion checks with hpssarch=YES checks

Refs: #178
---
 ush/rocoto/setup_workflow_fcstonly.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/ush/rocoto/setup_workflow_fcstonly.py b/ush/rocoto/setup_workflow_fcstonly.py
index 079a4ad3eca..66f553b8be7 100755
--- a/ush/rocoto/setup_workflow_fcstonly.py
+++ b/ush/rocoto/setup_workflow_fcstonly.py
@@ -91,6 +91,7 @@ def get_definitions(base):
 
     machine = base.get('machine', wfu.detectMachine())
     scheduler = wfu.get_scheduler(machine)
+    hpssarch = base.get('HPSSARCH', 'NO').upper()
 
     strings = []
 
@@ -131,7 +132,7 @@ def get_definitions(base):
     strings.append('\t<!ENTITY SCHEDULER  "%s">\n' % scheduler)
     strings.append('\n')
     strings.append('\t<!-- Toggle HPSS archiving -->\n')
-    strings.append('\t<!ENTITY ARCHIVE_TO_HPSS "YES">\n')
+    strings.append('\t<!ENTITY ARCHIVE_TO_HPSS "%s">\n' % base['HPSSARCH'])
     strings.append('\n')
     strings.append('\t<!-- ROCOTO parameters that control workflow -->\n')
     strings.append('\t<!ENTITY CYCLETHROTTLE "2">\n')
@@ -236,6 +237,7 @@ def get_workflow(dict_configs, cdump='gdas'):
 
     base = dict_configs['base']
     machine = base.get('machine', wfu.detectMachine())
+    hpssarch = base.get('HPSSARCH', 'NO').upper()
     do_wave = base.get('DO_WAVE', 'NO').upper()
     do_wave_cdump = base.get('WAVE_CDUMP', 'BOTH').upper()
     do_bufrsnd = base.get('DO_BUFRSND', 'NO').upper()
@@ -247,7 +249,7 @@ def get_workflow(dict_configs, cdump='gdas'):
     tasks = []
 
     # getic
-    if machine not in ['ORION']:
+    if hpssarch in ['YES']:
       deps = []
       data = '&ROTDIR;/&CDUMP;.@Y@m@d/@H/atmos/INPUT/sfc_data.tile6.nc'
       dep_dict = {'type':'data', 'data':data}
@@ -280,7 +282,7 @@ def get_workflow(dict_configs, cdump='gdas'):
     deps.append(rocoto.add_dependency(dep_dict))
     dependencies = rocoto.create_dependency(dep_condition='or', dep=deps)
 
-    if machine not in ['ORION']:
+    if hpssarch in ['YES']:
       deps = []
       dep_dict = {'type': 'task', 'name': '%sgetic' % cdump}
       deps.append(rocoto.add_dependency(dep_dict))
@@ -288,7 +290,7 @@ def get_workflow(dict_configs, cdump='gdas'):
 
     deps = []
     deps.append(dependencies)
-    if machine not in ['ORION']:
+    if hpssarch in ['YES']:
       deps.append(dependencies2)
       dependencies = rocoto.create_dependency(dep_condition='and', dep=deps)
 

From bfb0a33305a46d87f62c4a5c8bd7799a60362678 Mon Sep 17 00:00:00 2001
From: "kate.friedman" <kate.friedman@noaa.gov>
Date: Thu, 3 Jun 2021 13:13:08 -0500
Subject: [PATCH 066/125] Remove RUN_CCPP option, force CCPP now

- remove RUN_CCPP case option from build_all.sh
- remove RUN_CCPP option in build_fv3.sh, force CCPP build now
- remove pre-CCPP GFS ops tag checkout option in checkout.sh, forcing CCPP hash checkout now

Refs: #178
---
 sorc/build_all.sh | 11 -----------
 sorc/build_fv3.sh | 13 ++++++-------
 sorc/checkout.sh  | 10 +---------
 3 files changed, 7 insertions(+), 27 deletions(-)

diff --git a/sorc/build_all.sh b/sorc/build_all.sh
index acba8d6a2cc..6c027efc792 100755
--- a/sorc/build_all.sh
+++ b/sorc/build_all.sh
@@ -9,16 +9,6 @@ set +x
 #                   Anything other than "true"  will use libraries locally.
 #------------------------------------
 
-while getopts "c" option;
-do
- case $option in
-  c)
-   echo "Received -c flag, check out ufs-weather-model develop branch with CCPP physics"
-   RUN_CCPP="YES"
-   ;;
- esac
-done
-
 export USE_PREINST_LIBS="true"
 
 #------------------------------------
@@ -61,7 +51,6 @@ err=0
 #------------------------------------
 $Build_fv3gfs && {
 echo " .... Building fv3 .... "
-export RUN_CCPP=${RUN_CCPP:-"NO"}
 ./build_fv3.sh > $logs_dir/build_fv3.log 2>&1
 rc=$?
 if [[ $rc -ne 0 ]] ; then
diff --git a/sorc/build_fv3.sh b/sorc/build_fv3.sh
index 2c3efdadec3..376967f3740 100755
--- a/sorc/build_fv3.sh
+++ b/sorc/build_fv3.sh
@@ -23,10 +23,9 @@ cd fv3gfs.fd/
 FV3=$( pwd -P )/FV3
 cd tests/
 if [ ! -d ../NEMS/exe ]; then mkdir ../NEMS/exe ; fi
-if [ ${RUN_CCPP:-${1:-"YES"}} = "NO" ]; then
- ./compile.sh "$FV3" "$target" "WW3=Y 32BIT=Y" 1
- mv -f fv3_1.exe ../NEMS/exe/global_fv3gfs.x
-else
- ./compile.sh "$target" "APP=ATM CCPP=Y 32BIT=Y SUITES=FV3_GFS_v16,FV3_GFS_v16_RRTMGP" 2 YES NO
- mv -f fv3_2.exe ../NEMS/exe/global_fv3gfs.x
-fi
+
+#./compile.sh "$FV3" "$target" "WW3=Y 32BIT=Y" 1
+#mv -f fv3_1.exe ../NEMS/exe/global_fv3gfs.x
+
+./compile.sh "$target" "APP=ATM CCPP=Y 32BIT=Y SUITES=FV3_GFS_v16,FV3_GFS_v16_RRTMGP" 2 YES NO
+mv -f fv3_2.exe ../NEMS/exe/global_fv3gfs.x
diff --git a/sorc/checkout.sh b/sorc/checkout.sh
index ed3b6b1c563..eda2d007355 100755
--- a/sorc/checkout.sh
+++ b/sorc/checkout.sh
@@ -9,10 +9,6 @@ do
    echo "Received -o flag for optional checkout of GTG, will check out GTG with EMC_post"
    checkout_gtg="YES"
    ;;
-  c)
-   echo "Received -c flag, check out ufs-weather-model develop branch with CCPP physics"  
-   run_ccpp="YES"
-   ;;
   :)
    echo "option -$OPTARG needs an argument"
    ;;
@@ -31,11 +27,7 @@ if [[ ! -d fv3gfs.fd ]] ; then
     rm -f ${topdir}/checkout-fv3gfs.log
     git clone https://github.com/ufs-community/ufs-weather-model fv3gfs.fd >> ${topdir}/checkout-fv3gfs.log 2>&1
     cd fv3gfs.fd
-    if [ ${run_ccpp:-"NO"} = "NO" ]; then
-     git checkout GFS.v16.0.16
-    else
-     git checkout 9350745855aebe0790813e0ed2ba5ad680e3f75c
-    fi
+    git checkout 9350745855aebe0790813e0ed2ba5ad680e3f75c
     git submodule update --init --recursive
     cd ${topdir}
 else

From fe448ff29aedca6a40a748ee47ae001b1639188c Mon Sep 17 00:00:00 2001
From: "kate.friedman" <kate.friedman@noaa.gov>
Date: Thu, 3 Jun 2021 19:32:15 +0000
Subject: [PATCH 067/125] Set DO_WAVE to NO

- turning off waves by default in config.base

Refs: #178
---
 parm/config/config.base.emc.dyn | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/parm/config/config.base.emc.dyn b/parm/config/config.base.emc.dyn
index 3016b983e42..e68d0c0b0ab 100755
--- a/parm/config/config.base.emc.dyn
+++ b/parm/config/config.base.emc.dyn
@@ -226,7 +226,7 @@ export DO_GLDAS="YES"
 export gldas_cyc=00
 
 # run wave component
-export DO_WAVE="YES"
+export DO_WAVE="NO"
 if [[ "$SDATE" -lt "2019020100" ]]; then DO_WAVE="NO" ; fi # no rtofs in GDA
 export WAVE_CDUMP="both"
 

From 1cdc2a44c6d29cfb8f06a50238ea29d28642c1f1 Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Tue, 8 Jun 2021 15:56:46 +0000
Subject: [PATCH 068/125] Add cmake module load to Hera block in
 machine-setup.sh

Refs: #178
---
 sorc/machine-setup.sh | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/sorc/machine-setup.sh b/sorc/machine-setup.sh
index f0f5586c235..5f28d6abaa9 100644
--- a/sorc/machine-setup.sh
+++ b/sorc/machine-setup.sh
@@ -45,11 +45,10 @@ elif [[ -d /scratch1 ]] ; then
     fi
     target=hera
     module purge
-    module load intel
-    module load impi
-    export NCEPLIBS=/scratch2/NCEPDEV/nwprod/NCEPLIBS
-    module use $NCEPLIBS/modulefiles
-    #export WRFPATH=$NCEPLIBS/wrf.shared.new/v1.1.1/src
+    module use /scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack
+    module load hpc/1.1.0
+    module load cmake/3.20.0
+
     export myFC=mpiifort
     export FCOMP=mpiifort
 

From 898a43fb74005e969ecb5baf28ecf2d51f7c5dda Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Tue, 8 Jun 2021 15:57:32 +0000
Subject: [PATCH 069/125] Adjust getic/init job scripts

- adjust EXTRACT_DIR, DATA, and ROTDIR usage in getic and init jobs
- remove duplicate pgb file pull from init job

Refs: #178
---
 jobs/rocoto/getic.sh | 25 ++++++++++++++++---------
 jobs/rocoto/init.sh  | 12 ------------
 2 files changed, 16 insertions(+), 21 deletions(-)

diff --git a/jobs/rocoto/getic.sh b/jobs/rocoto/getic.sh
index d17089e095d..9f0f9277348 100755
--- a/jobs/rocoto/getic.sh
+++ b/jobs/rocoto/getic.sh
@@ -46,26 +46,23 @@ export gmm=$(echo $GDATE | cut -c5-6)
 export gdd=$(echo $GDATE | cut -c7-8)
 export ghh=$(echo $GDATE | cut -c9-10)
 
-export DATA=${DATA:-${DATAROOT}/init}
-export EXTRACT_DIR=${EXTRACT_DIR:-$ROTDIR}
-export WORKDIR=${WORKDIR:-$DATA}
-export OUTDIR=${OUTDIR:-$ROTDIR}
+export DATA=${DATA:-${DATAROOT}/getic}
+export EXTRACT_DIR=${DATA:-$EXTRACT_DIR}
 export PRODHPSSDIR=${PRODHPSSDIR:-/NCEPPROD/hpssprod/runhistory}
 export COMPONENT="atmos"
 export gfs_ver=${gfs_ver:-"v16"}
 export OPS_RES=${OPS_RES:-"C768"}
 export GETICSH=${GETICSH:-${GDASINIT_DIR}/get_v16.data.sh}
 
-# Run get data script
+# Create ROTDIR/EXTRACT_DIR
+if [ ! -d $ROTDIR ]; then mkdir -p $ROTDIR ; fi
 if [ ! -d $EXTRACT_DIR ]; then mkdir -p $EXTRACT_DIR ; fi
+cd $EXTRACT_DIR
 
 # Check if init is needed and run if so
 if [[ $gfs_ver = "v16" && $EXP_WARM_START = ".true." && $CASE = $OPS_RES ]]; then
 
   # Pull RESTART files off HPSS
-  if [ ! -d $ROTDIR ]; then mkdir $ROTDIR ; fi
-  cd $ROTDIR
-
   if [ ${RETRO:-"NO"} = "YES" ]; then # Retrospective parallel input
 
     # Pull prior cycle restart files
@@ -107,6 +104,10 @@ else
 
 fi
 
+# Move extracted data to ROTDIR
+if [ ! -d ${ROTDIR}/${CDUMP}.${yy}${mm}${dd}/${hh} ]; then mkdir -p ${ROTDIR}/${CDUMP}.${yy}${mm}${dd}/${hh} ; fi
+mv ${EXTRACT_DIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/* ${ROTDIR}/${CDUMP}.${yy}${mm}${dd}/${hh}
+
 # Pull pgbanl file for verification/archival - v14+
 if [ $gfs_ver = v14 -o $gfs_ver = v15 ]; then
   cd $EXTRACT_DIR
@@ -114,10 +115,16 @@ if [ $gfs_ver = v14 -o $gfs_ver = v15 ]; then
   do
     file=gfs.t${hh}z.pgrb2.${grid}.anl
     htar -xvf ${PRODHPSSDIR}/rh${yy}/${yy}${mm}/${yy}${mm}${dd}/${tarball} ./${CDUMP}.${yy}${mm}${dd}/${hh}/${file}
-    mv ${EXTRACT_DIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${file} ${OUTDIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${COMPONENT}/${file}
+    mv ${EXTRACT_DIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${file} ${ROTDIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${COMPONENT}/${file}
   done
 fi
 
+##########################################
+# Remove the Temporary working directory
+##########################################
+cd $DATAROOT
+[[ $KEEPDATA = "NO" ]] && rm -rf $DATA
+
 ###############################################################
 # Exit out cleanly
 exit 0
diff --git a/jobs/rocoto/init.sh b/jobs/rocoto/init.sh
index f7bc323aaf9..6b70842821f 100755
--- a/jobs/rocoto/init.sh
+++ b/jobs/rocoto/init.sh
@@ -61,18 +61,6 @@ else
   [[ $status -ne 0 ]] && exit $status
 fi
 
-# Check for pgbanl files and move
-if [ $gfs_ver = v14 -o $gfs_ver = v15 ]; then
-  cd $EXTRACT_DIR
-  for grid in 0p25 0p50 1p00
-  do
-    file=gfs.t${hh}z.pgrb2.${grid}.anl
-    if [ -f ${EXTRACT_DIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${file} ]; then
-      mv ${EXTRACT_DIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${file} ${OUTDIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${COMPONENT}/${file}
-    fi
-  done
-fi
-
 ##########################################
 # Remove the Temporary working directory
 ##########################################

From b7da671df64636fc38e2a33ade6eefbe9f60ee18 Mon Sep 17 00:00:00 2001
From: Raffaele Montuoro <raffaele.montuoro@noaa.gov>
Date: Tue, 8 Jun 2021 16:12:30 +0000
Subject: [PATCH 070/125] Checkout and build ufs-weather-model app integrating
 prognostic aerosols into the UFS, replacing original FV3-GOCART system.

A forked repository is temporarily used to download
the updated ufs-weather-model app.
---
 sorc/build_ufs_coupled.sh | 7 ++++---
 sorc/checkout.sh          | 4 ++--
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/sorc/build_ufs_coupled.sh b/sorc/build_ufs_coupled.sh
index e18e6fd3e88..0d6be6a0e19 100755
--- a/sorc/build_ufs_coupled.sh
+++ b/sorc/build_ufs_coupled.sh
@@ -13,15 +13,16 @@ if [ $target = hera ]; then target=hera.intel ; fi
 if [ $target = orion ]; then target=orion.intel ; fi
 if [ $target = stampede ]; then target=stampede.intel ; fi
 
-MOD_PATH=$cwd/ufs_coupled.fd/modulefiles/$target
+MOD_PATH=$cwd/ufs_coupled.fd/modulefiles
 
 module purge 
 module use $MOD_PATH 
-module load fv3 
+module load ufs_${target}
 cd ufs_coupled.fd/
 if [[ -d build ]]; then rm -Rf build; fi
 if [[ -d GOCART ]]; then
-  ./build.sh
+  module load ufs_aerosols_${target}
+  CMAKE_FLAGS="-DAPP=ATMAERO" CCPP_SUITES="FV3_GFS_v16" ./build.sh
 else
   CMAKE_FLAGS="-DS2S=ON -DWW3=ON" CCPP_SUITES="FV3_GFS_v15p2_coupled,FV3_GFS_v16_coupled,FV3_GFS_v16" ./build.sh
 fi
diff --git a/sorc/checkout.sh b/sorc/checkout.sh
index 796ccd79121..fc35a12e375 100755
--- a/sorc/checkout.sh
+++ b/sorc/checkout.sh
@@ -46,9 +46,9 @@ if [ ${COUPLED:-"NO"} = "NO" ]; then
 else 
   if [[ ! -d ufs_coupled.fd ]] ; then
     if [ "${AEROSOL}" = "YES" ] ; then
-      git clone https://github.com/NOAA-EMC/FV3-GOCART ufs_coupled.fd >> ${topdir}/checkout-ufs_coupled.log 2>&1
+      git clone https://github.com/rmontuoro/ufs-weather-model.git ufs_coupled.fd >> ${topdir}/checkout-ufs_coupled.log 2>&1
       cd ufs_coupled.fd
-      git checkout develop
+      git checkout feature/gocart
     else
       git clone https://github.com/ufs-community/ufs-weather-model ufs_coupled.fd >> ${topdir}/checkout-ufs_coupled.log 2>&1
       cd ufs_coupled.fd

From 5c8e0781b058f600c7c8826566ff3e67ee577310 Mon Sep 17 00:00:00 2001
From: Raffaele Montuoro <raffaele.montuoro@noaa.gov>
Date: Tue, 8 Jun 2021 16:28:55 +0000
Subject: [PATCH 071/125] Use proper modulefiles for updated ufs-weather-model
 app with prognostic aerosols.

---
 ush/load_fv3gfs_modules.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/ush/load_fv3gfs_modules.sh b/ush/load_fv3gfs_modules.sh
index 2cb57f0a04f..19c7a2664ec 100755
--- a/ush/load_fv3gfs_modules.sh
+++ b/ush/load_fv3gfs_modules.sh
@@ -24,10 +24,10 @@ elif [[ -d /lfs3 ]] ; then
 	module load module_base.jet 
 elif [[ -d /scratch1 ]] ; then
 	# We are on NOAA Hera
-	module load module_base.hera
+	module use "$HOMEgfs/sorc/ufs_coupled.fd/modulefiles"
+	module load ufs_hera.intel
 	if [[ -d $HOMEgfs/sorc/ufs_coupled.fd/GOCART ]] ; then
-		module use "$HOMEgfs/sorc/ufs_coupled.fd/modulefiles/hera.intel"
-		module load fv3
+		module load ufs_aerosols_hera.intel
 	fi
 elif [[ -d /work ]] ; then
 	# We are on MSU Orion

From e2f52cce042cd08b7b76cf23ecda231ce2d41303 Mon Sep 17 00:00:00 2001
From: Raffaele Montuoro <raffaele.montuoro@noaa.gov>
Date: Tue, 8 Jun 2021 16:30:26 +0000
Subject: [PATCH 072/125] Update parsing function for FV3 namelist to work with
 latest FV3 revision and include settings for FV3_GFS_v16 and aerosol
 scavenging factors.

---
 ush/parsing_namelists_FV3.sh | 33 +++++++++++++++++++++++++--------
 1 file changed, 25 insertions(+), 8 deletions(-)

diff --git a/ush/parsing_namelists_FV3.sh b/ush/parsing_namelists_FV3.sh
index ddcca3a2412..0053478275d 100755
--- a/ush/parsing_namelists_FV3.sh
+++ b/ush/parsing_namelists_FV3.sh
@@ -191,13 +191,14 @@ cat > input.nml <<EOF
   imp_physics  = ${imp_physics:-"99"}	! CROW configured
 EOF
 
-
-if [ $CCPP_SUITE = "FV3_GFS_v15p2_coupled" ]; then
+case "${CCPP_SUITE:-}" in
+  "FV3_GFS_v15p2_coupled")
   cat >> input.nml << EOF
   oz_phys      = .false.
   oz_phys_2015 = .true.
 EOF
-elif [ $CCPP_SUITE = "FV3_GSD_v0" ]; then
+  ;;
+  "FV3_GSD_v0")
   cat >> input.nml << EOF
   iovr         = ${iovr:-"3"}
   ltaerosol    = ${ltaerosol:-".F."}    ! In config.fcst
@@ -215,7 +216,8 @@ elif [ $CCPP_SUITE = "FV3_GSD_v0" ]; then
   min_lakeice  = ${min_lakeice:-"0.15"}
   min_seaice   = ${min_seaice:-"0.15"}
 EOF
-elif [ $CCPP_SUITE = "FV3_GFS_v16_coupled" ]; then
+ ;;
+  "FV3_GFS_v16_coupled")
   cat >> input.nml << EOF
   iovr         = ${iovr:-"3"}
   ltaerosol    = ${ltaerosol:-".false."}
@@ -233,12 +235,25 @@ elif [ $CCPP_SUITE = "FV3_GFS_v16_coupled" ]; then
   min_lakeice  = ${min_lakeice:-"0.15"}
   min_seaice   = ${min_seaice:-"0.15"}
 EOF
-else
+  ;;
+  "FV3_GFS_v16")
+  cat >> input.nml << EOF
+  iovr         = ${iovr:-"3"}
+  ltaerosol    = ${ltaerosol:-".false."}
+  oz_phys      = ${oz_phys:-".false."}
+  oz_phys_2015 = ${oz_phys_2015:-".true."}
+  lsoil        = ${lsoil:-"4"}
+  icloud_bl    = ${icloud_bl:-"1"}
+  min_lakeice  = ${min_lakeice:-"0.15"}
+  min_seaice   = ${min_seaice:-"0.15"}
+EOF
+  ;;
+  *)
   cat >> input.nml << EOF
-  iovr_lw      = ${iovr_lw:-"3"}
-  iovr_sw      = ${iovr_sw:-"3"}
+  iovr         = ${iovr:-"3"}
 EOF
-fi
+  ;;
+esac
 
 cat >> input.nml <<EOF
   pdfcld       = ${pdfcld:-".false."}
@@ -301,10 +316,12 @@ cat >> input.nml <<EOF
   do_shum      = ${DO_SHUM:-".false."}
   do_skeb      = ${DO_SKEB:-".false."}
   frac_grid    = ${FRAC_GRID:-".false."}
+  fscav_aero   = ${fscav_aero:-'*:0.0'}
 EOF
 
 if [ $cpl = .true. ]; then
   cat >> input.nml << EOF
+  cplchm       = ${cplchem:-".false."}
   cplflx       = $cplflx
   cplwav2atm   = ${cplwav2atm}          ! CROW configured
 EOF

From ff971a2974e9e08723c875c18879fef11c8ad674 Mon Sep 17 00:00:00 2001
From: Raffaele Montuoro <raffaele.montuoro@noaa.gov>
Date: Tue, 8 Jun 2021 16:34:48 +0000
Subject: [PATCH 073/125] Add aerosol scavenging factors (fscav_aero) to FV3
 settings. This item is implemented as a YAML list of strings, which allows to
 set scavenging factors for each tracer.

---
 workflow/defaults/fv3_gfs.yaml | 1 +
 workflow/schema/fv3.yaml       | 4 ++++
 2 files changed, 5 insertions(+)

diff --git a/workflow/defaults/fv3_gfs.yaml b/workflow/defaults/fv3_gfs.yaml
index 8b7a7af7f4b..53dd7d436df 100644
--- a/workflow/defaults/fv3_gfs.yaml
+++ b/workflow/defaults/fv3_gfs.yaml
@@ -82,6 +82,7 @@ fv3_gfs_defaults: &fv3_gfs_defaults
 # WRTTASK_PER_GROUP: layout.WGRP_NTASKS
 # WRTIOBUF: layout.WRTIOBUF
 
+  fscav_aero: [ '*:0.0' ]
   new_o3force: yes
   h2o_phys: yes
   do_vort_damp: yes
diff --git a/workflow/schema/fv3.yaml b/workflow/schema/fv3.yaml
index 61aaba6eb3f..a8bce77d52f 100644
--- a/workflow/schema/fv3.yaml
+++ b/workflow/schema/fv3.yaml
@@ -32,6 +32,10 @@ fv3_settings_template: !Template &fv3_settings_template
   agrid_vel_rst:
     type: bool
     description: "write velocity restarts on A grid?"
+  fscav_aero:
+    type: string list
+    description: aerosol convective scavenging coefficients
+    optional: true
 
 #  cal_pre:            { type: bool }
 #  do_sat_adjust:      { type: bool }

From d16630f3b34671e2350e6a88b9df749e4fd58ece Mon Sep 17 00:00:00 2001
From: Raffaele Montuoro <raffaele.montuoro@noaa.gov>
Date: Tue, 8 Jun 2021 16:39:55 +0000
Subject: [PATCH 074/125] Properly set NST_SPINUP flag in FV3 namelist.

---
 workflow/config/nsst.yaml   | 2 +-
 workflow/defaults/nsst.yaml | 2 +-
 workflow/schema/nsst.yaml   | 8 ++++----
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/workflow/config/nsst.yaml b/workflow/config/nsst.yaml
index d37114b33d8..52a8d6a2d8a 100644
--- a/workflow/config/nsst.yaml
+++ b/workflow/config/nsst.yaml
@@ -6,7 +6,7 @@ config_nsst:
   filename: config.nsst
   nst_spinup_logic: !FirstTrue
     - when: !calc doc.nsst.get("NST_SPINUP","")
-      do: !expand "export NST_SPINUP=$NST_SPINUP"
+      do: !expand "export NST_SPINUP={doc.nsst.NST_SPINUP}"
     - otherwise: |
         export NST_SPINUP=0
         #if [[ "$CDATE" = $SDATE ]]; then
diff --git a/workflow/defaults/nsst.yaml b/workflow/defaults/nsst.yaml
index 1ac69093986..f426364e9dc 100644
--- a/workflow/defaults/nsst.yaml
+++ b/workflow/defaults/nsst.yaml
@@ -1,8 +1,8 @@
 nsst_defaults: &nsst_defaults
    NST_MODEL: 2
+   NST_SPINUP: 0
    NST_RESV: 0
    ZSEA1: 0
    ZSEA2: 0
    NST_GSI: 3
    NSTINFO: 0
-   NST_SPINUP: 0
diff --git a/workflow/schema/nsst.yaml b/workflow/schema/nsst.yaml
index 1efcc8eb05a..7ffd0b319bc 100644
--- a/workflow/schema/nsst.yaml
+++ b/workflow/schema/nsst.yaml
@@ -5,6 +5,10 @@ nsst_schema: &nsst_schema !Template
     description: >-
       nstf_name(1) : NST_MODEL (NSST Model) : 0 = OFF, 
       1 = ON but uncoupled, 2 = ON and coupled
+  NST_SPINUP:
+    type: int
+    allowed: [ 0, 1]
+    optional: yes
   NST_RESV:
     type: int
     allowed: [ 0, 1 ]
@@ -24,7 +28,3 @@ nsst_schema: &nsst_schema !Template
   NSTINFO:
     type: int
     description: number of elements added in obs. data array
-  NST_SPINUP:
-    type: int
-    allowed: [ 0, 1]
-    optional: yes

From c26a0dd7256ce7993e15a3c19bf77049309ef7c6 Mon Sep 17 00:00:00 2001
From: Raffaele Montuoro <raffaele.montuoro@noaa.gov>
Date: Tue, 8 Jun 2021 16:44:03 +0000
Subject: [PATCH 075/125] Restore `cplchem` as the flag for coupled aerosols,
 removing all references to `cplgocart`.

---
 scripts/exglobal_forecast.sh    | 8 +-------
 ush/cplvalidate.sh              | 2 +-
 ush/forecast_postdet.sh         | 8 ++------
 workflow/config/base.yaml       | 4 ++--
 workflow/config/fcst.yaml       | 6 ++++--
 workflow/defaults/chem.yaml     | 6 +++---
 workflow/defaults/settings.yaml | 3 ---
 workflow/schema/chem.yaml       | 2 +-
 8 files changed, 14 insertions(+), 25 deletions(-)

diff --git a/scripts/exglobal_forecast.sh b/scripts/exglobal_forecast.sh
index 9efed80ef46..fdfebc350a6 100755
--- a/scripts/exglobal_forecast.sh
+++ b/scripts/exglobal_forecast.sh
@@ -102,7 +102,6 @@ cpl=${cpl:-.false.}
 cplflx=${cplflx:-.false.} # default off,import from outside source
 cplwav=${cplwav:-.false.} # ? how to control 1-way/2-way?
 cplchem=${cplchem:-.false.} # Chemistry model
-cplgocart=${cplgocart:-.false.} # Chemistry model
 cplice=${cplice:-.false.} # ICE model
 
 OCNTIM=${OCNTIM:-3600}
@@ -134,7 +133,6 @@ esac
 [[ $cplflx = .true. ]] && MOM6_predet
 #[[ $cplwav = .true. ]] && WW3_predet #no WW3_predet at this time
 [[ $cplice = .true. ]] && CICE_predet
-[[ $cplchem = .true. ]] && GSD_predet
 
 case $RUN in
 	'gfs') FV3_GFS_det;;
@@ -144,7 +142,6 @@ esac				#no run type determination for data atmosphere
 [[ $cplflx = .true. ]] && MOM6_det
 [[ $cplwav = .true. ]] && WW3_det
 [[ $cplice = .true. ]] && CICE_det
-[[ $cplchem = .true. ]] && GSD_det
 
 echo "MAIN: RUN Type Determined"
 
@@ -159,7 +156,6 @@ esac				#no post determination set up for data atmosphere
 [[ $cplflx = .true. ]] && MOM6_postdet
 [[ $cplwav = .true. ]] && WW3_postdet
 [[ $cplice = .true. ]] && CICE_postdet
-[[ $cplchem = .true. ]] && GSD_postdet
 echo "MAIN: Post-determination set up of run type finished"
 
 echo "MAIN: Writing name lists and model configuration"
@@ -172,8 +168,7 @@ esac				#no namelist for data atmosphere
 [[ $cplflx = .true. ]] && MOM6_nml
 [[ $cplwav = .true. ]] && WW3_nml
 [[ $cplice = .true. ]] && CICE_nml
-[[ $cplchem = .true. ]] && GSD_nml
-[[ $cplgocart = .true. ]] && GOCART_rc
+[[ $cplchem = .true. ]] && GOCART_rc
 
 case $RUN in
 	'data') DATM_model_configure;;
@@ -216,7 +211,6 @@ if [ $machine != 'sandbox' ]; then
 	[[ $cplflx = .true. ]] && MOM6_out
 	[[ $cplwav = .true. ]] && WW3_out
 	[[ $cplice = .true. ]] && CICE_out
-	[[ $cplchem = .true. ]] && GSD_out
 	[[ $esmf_profile = .true. ]] && CPL_out
 else
 	echo "MAIN: Running on sandbox mode, no output linking"
diff --git a/ush/cplvalidate.sh b/ush/cplvalidate.sh
index ec481cd8f23..46f861dbe12 100755
--- a/ush/cplvalidate.sh
+++ b/ush/cplvalidate.sh
@@ -29,7 +29,7 @@ case $confignamevarfornems in
 	*) echo "SUB cplvalidate: Combination not supported" 
 		exit 1 ;;
 esac
-control=$cpl$cplflx$cplice$cplwav$cplgocart
+control=$cpl$cplflx$cplice$cplwav$cplchem
 #echo $control
 if [ $control != $combination ]; then
 	echo "SUB cplvalidate: inconsistent cpl setting!"
diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh
index 7bdde50324f..4acbd1c2fba 100755
--- a/ush/forecast_postdet.sh
+++ b/ush/forecast_postdet.sh
@@ -498,11 +498,7 @@ FV3_GFS_nml(){
 		echo "MAIN: !!!Sandbox mode, writing to current directory!!!"
 	fi
 	# Call child scripts in current script directory
-	if [ $cplgocart = .true. ]; then
-		source $SCRIPTDIR/parsing_namelists_FV3_GOCART.sh
-	else
-		source $SCRIPTDIR/parsing_namelists_FV3.sh
-	fi
+	source $SCRIPTDIR/parsing_namelists_FV3.sh
 	FV3_namelists
 	echo SUB ${FUNCNAME[0]}: FV3 name lists and model configure file created
 }
@@ -868,7 +864,7 @@ GSD_in() {
 
 GSD_nml() {
 	echo "SUB ${FUNCNAME[0]}: Creating name list for GSD"
-	sh parsing_namelists_GSD.sh
+	# generate GSD-specific input
 }
 
 GSD_out() {
diff --git a/workflow/config/base.yaml b/workflow/config/base.yaml
index 92c24578579..24be461e5f5 100644
--- a/workflow/config/base.yaml
+++ b/workflow/config/base.yaml
@@ -26,13 +26,13 @@ config_base:
     # Coupling settings
     export cpl="{doc.settings.cpl}"
     export cplflx="{doc.settings.cplflx}"
-    export cplgocart="{doc.settings.cplgocart}"
+    export cplchem="{doc.settings.cplchem}"
     export cplice="{doc.settings.cplice}"
     export cplwav="{doc.settings.cplwav}"
     export cplwav2atm="{doc.settings.cplwav2atm}"
 
     # Temporarily override settings for aerosol coupling
-    if [ $cplgocart = ".true." ]; then
+    if [ $cplchem = ".true." ]; then
         export cplflx=".false."
         export cplice=".false."
         export cplocn=".false."
diff --git a/workflow/config/fcst.yaml b/workflow/config/fcst.yaml
index d96dcac912a..7f1175b7325 100644
--- a/workflow/config/fcst.yaml
+++ b/workflow/config/fcst.yaml
@@ -5,7 +5,7 @@
 config_fcst:
   filename: config.fcst
   FCSTEXEC: !FirstTrue
-    - when: !calc ( doc.settings.cplflx==".true." or doc.settings.cplgocart==".true." )
+    - when: !calc ( doc.settings.cplflx==".true." or doc.settings.cplchem==".true." )
       do: "ufs_model"
     - otherwise: "global_fv3gfs.x"
   more_exports_for_microphys: !FirstTrue
@@ -123,6 +123,8 @@ config_fcst:
     export dddmp={doc.fv3_gfs_settings.dddmp}
 
     # gfs_physics_nml section
+    export fscav_aero="{tools.fort(doc.fv3_gfs_settings.fscav_aero)}"
+
     export h2o_phys="{tools.fort(doc.fv3_gfs_settings.h2o_phys)}"
 
     export ncld={doc.fv3_gfs_settings.phy_dependent_var.ncld}
@@ -225,7 +227,7 @@ config_fcst:
 
     fi
 
-    if [ $cplgocart = .true. ]; then # temporary settings for aerosol coupling
+    if [ $cplchem = .true. ]; then # temporary settings for aerosol coupling
         export DIAG_TABLE="$HOMEgfs/parm/parm_fv3diag/diag_table_aer"
         export FIELD_TABLE="$HOMEgfs/parm/parm_fv3diag/field_table_aer"
         export CHM_CFGDIR="{doc.chem_settings.CFGDIR}"
diff --git a/workflow/defaults/chem.yaml b/workflow/defaults/chem.yaml
index 58f45cd8aab..f26ab117202 100644
--- a/workflow/defaults/chem.yaml
+++ b/workflow/defaults/chem.yaml
@@ -1,13 +1,13 @@
 # default settings for chemistry model core
 
 chem_defaults: &chem_defaults
-  MOD: gsdchem
+  MOD: gocart
   ntdiag: 0
   CFGDIR: !FirstTrue
-    - when: !calc (doc.settings.cplgocart != ".true.")
+    - when: !calc (doc.settings.cplchem != ".true.")
       do: "not_set"
     - otherwise:
   INPDIR: !FirstTrue
-    - when: !calc (doc.settings.cplgocart != ".true.")
+    - when: !calc (doc.settings.cplchem != ".true.")
       do: "not_set"
     - otherwise:
diff --git a/workflow/defaults/settings.yaml b/workflow/defaults/settings.yaml
index f6e52d3742c..be713b9478d 100644
--- a/workflow/defaults/settings.yaml
+++ b/workflow/defaults/settings.yaml
@@ -55,7 +55,6 @@ default_settings: &default_settings
   cplwav: .false.
   cplwav2atm: .false.
   cplchem: .false.
-  cplgocart: .false.
   cpl: !FirstTrue
     - when: !calc doc.settings.cplflx==".true."
       do: .true.
@@ -65,8 +64,6 @@ default_settings: &default_settings
       do: .true.
     - when: !calc doc.settings.cplchem==".true."
       do: .true.
-    - when: !calc doc.settings.cplgocart==".true."
-      do: .true.
     - otherwise: .false.
 
   ic_source: coupled_ic  # choices: coupled_ic, fv3_ic; default: coupled_ic. This option controls which
diff --git a/workflow/schema/chem.yaml b/workflow/schema/chem.yaml
index 64b10900a33..eb49adf9f53 100644
--- a/workflow/schema/chem.yaml
+++ b/workflow/schema/chem.yaml
@@ -5,7 +5,7 @@
 chem_settings_template: !Template &chem_settings_template
   MOD:
     type: string
-    allowed: [ gsdchem ]
+    allowed: [ gocart ]
     description: "model selection for aerosols"
   CFGDIR:
     type: string

From 9661d1eaade20a015b8268a4ba20b1147ba2897d Mon Sep 17 00:00:00 2001
From: Raffaele Montuoro <raffaele.montuoro@noaa.gov>
Date: Tue, 8 Jun 2021 16:46:23 +0000
Subject: [PATCH 076/125] - Set NEMS identifier for coupled aerosol component
 to 'gocart'.

- Copy NUOPC field dictionary to run directory by default.
---
 ush/nems_configure.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/ush/nems_configure.sh b/ush/nems_configure.sh
index 8e446271458..ed7bc1d50de 100755
--- a/ush/nems_configure.sh
+++ b/ush/nems_configure.sh
@@ -36,7 +36,7 @@ ATM_model=${ATM_model:-'fv3'}
 OCN_model=${OCN_model:-'mom6'}
 ICE_model=${ICE_model:-'cice'}
 WAV_model=${WAV_model:-'ww3'}
-CHM_model=${CHM_model:-'gsdchem'}
+CHM_model=${CHM_model:-'gocart'}
 
 ATMPETS=${ATMPETS:-8}
 MEDPETS=${MEDPETS:-8} 
@@ -90,7 +90,7 @@ if [ $cplice = .true. ]; then
         sed -i -e "s;@\[MESHICE\];$MESHICE;g" tmp1
         sed -i -e "s;@\[FHMAX\];$FHMAX_GFS;g" tmp1
 fi
-if [ $cplchem = .true. -o $cplgocart = .true. ]; then
+if [ $cplchem = .true. ]; then
 	sed -i -e "s;@\[chm_model\];$CHM_model;g" tmp1
 	sed -i -e "s;@\[chm_petlist_bounds\];$chm_petlist_bounds;g" tmp1
 	sed -i -e "s;@\[coupling_interval_fast_sec\];$CPL_FAST;g" tmp1
@@ -155,10 +155,10 @@ EOF
 
 echo "$(cat med_modelio.nml)"
 
-cp $HOMEgfs/sorc/ufs_coupled.fd/CMEPS-interface/CMEPS/mediator/fd_nems.yaml fd_nems.yaml
-
 fi 
 
+cp $HOMEgfs/sorc/ufs_coupled.fd/tests/parm/fd_nems.yaml fd_nems.yaml
+
 echo "SUB ${FUNCNAME[0]}: Nems configured for $confignamevarfornems"
 
 }

From 52dbd17e193944b35a7836dc89b0510652a2a789 Mon Sep 17 00:00:00 2001
From: Raffaele Montuoro <raffaele.montuoro@noaa.gov>
Date: Tue, 8 Jun 2021 16:48:37 +0000
Subject: [PATCH 077/125] Update test cases and input files for coupled
 aerosols runs.

---
 parm/parm_fv3diag/diag_table_aer           |   6 +-
 parm/parm_fv3diag/field_table_aer          | 257 +++++++++++----------
 workflow/cases/aerosol_firex_forecast.yaml |  12 +-
 workflow/cases/aerosol_free_forecast.yaml  |   9 +-
 4 files changed, 150 insertions(+), 134 deletions(-)

diff --git a/parm/parm_fv3diag/diag_table_aer b/parm/parm_fv3diag/diag_table_aer
index 2a7ba17e0db..f2617c1c560 100644
--- a/parm/parm_fv3diag/diag_table_aer
+++ b/parm/parm_fv3diag/diag_table_aer
@@ -123,7 +123,7 @@
 # chemical tracers advected by FV3
 ###
 "gfs_dyn",     "so2",         "so2",       "fv3_history",    "all",  .false.,  "none",  2
-"gfs_dyn",     "sulf",        "sulf",      "fv3_history",    "all",  .false.,  "none",  2
+"gfs_dyn",     "so4",         "so4",       "fv3_history",    "all",  .false.,  "none",  2
 "gfs_dyn",     "dms",         "dms",       "fv3_history",    "all",  .false.,  "none",  2
 "gfs_dyn",     "msa",         "msa",       "fv3_history",    "all",  .false.,  "none",  2
 "gfs_dyn",     "bc1",         "bc1",       "fv3_history",    "all",  .false.,  "none",  2
@@ -145,8 +145,8 @@
 "gfs_dyn",     "no3an1",      "no3an1",    "fv3_history",    "all",  .false.,  "none",  2
 "gfs_dyn",     "no3an2",      "no3an2",    "fv3_history",    "all",  .false.,  "none",  2
 "gfs_dyn",     "no3an3",      "no3an3",    "fv3_history",    "all",  .false.,  "none",  2
-"gfs_dyn",     "pp25",        "pp25",      "fv3_history",    "all",  .false.,  "none",  2
-"gfs_dyn",     "pp10",        "pp10",      "fv3_history",    "all",  .false.,  "none",  2
+"gfs_dyn",     "pm25",        "pm25",      "fv3_history",    "all",  .false.,  "none",  2
+"gfs_dyn",     "pm10",        "pm10",      "fv3_history",    "all",  .false.,  "none",  2
 
 "gfs_phys",  "ALBDO_ave",     "albdo_ave",   "fv3_history2d",         "all",  .false.,  "none",  2
 "gfs_phys",  "cnvprcp_ave",   "cprat_ave",   "fv3_history2d",         "all",  .false.,  "none",  2
diff --git a/parm/parm_fv3diag/field_table_aer b/parm/parm_fv3diag/field_table_aer
index 0bfa937980d..c31d4b157ad 100644
--- a/parm/parm_fv3diag/field_table_aer
+++ b/parm/parm_fv3diag/field_table_aer
@@ -1,158 +1,169 @@
 # added by FRE: sphum must be present in atmos
 # specific humidity for moist runs
  "TRACER", "atmos_mod", "sphum"
-           "longname",     "specific humidity"
-           "units",        "kg/kg"
-       "profile_type", "fixed", "surface_value=1.e30" /
+           "longname",  "specific humidity"
+           "units",     "kg/kg"
+       "profile_type",  "fixed", "surface_value=1.e30" /
 # prognostic cloud water mixing ratio
  "TRACER", "atmos_mod", "liq_wat"
-           "longname",     "cloud water mixing ratio"
-           "units",        "kg/kg"
-       "profile_type", "fixed", "surface_value=1.e30" /
+           "longname",  "cloud water mixing ratio"
+           "units",     "kg/kg"
+       "profile_type",  "fixed", "surface_value=1.e30" /
  "TRACER", "atmos_mod", "rainwat"
-           "longname",     "rain mixing ratio"
-           "units",        "kg/kg"
-       "profile_type", "fixed", "surface_value=1.e30" /
+           "longname",  "rain mixing ratio"
+           "units",     "kg/kg"
+       "profile_type",  "fixed", "surface_value=1.e30" /
  "TRACER", "atmos_mod", "ice_wat"
-           "longname",     "cloud ice mixing ratio"
-           "units",        "kg/kg"
-       "profile_type", "fixed", "surface_value=1.e30" /
+           "longname",  "cloud ice mixing ratio"
+           "units",     "kg/kg"
+       "profile_type",  "fixed", "surface_value=1.e30" /
  "TRACER", "atmos_mod", "snowwat"
-           "longname",     "snow mixing ratio"
-           "units",        "kg/kg"
-       "profile_type", "fixed", "surface_value=1.e30" /
+           "longname",  "snow mixing ratio"
+           "units",     "kg/kg"
+       "profile_type",  "fixed", "surface_value=1.e30" /
  "TRACER", "atmos_mod", "graupel"
-           "longname",     "graupel mixing ratio"
-           "units",        "kg/kg"
-       "profile_type", "fixed", "surface_value=1.e30" /
+           "longname",  "graupel mixing ratio"
+           "units",     "kg/kg"
+       "profile_type",  "fixed", "surface_value=1.e30" /
 # prognostic ozone mixing ratio tracer
  "TRACER", "atmos_mod", "o3mr"
-           "longname",     "ozone mixing ratio"
-           "units",        "kg/kg"
-       "profile_type", "fixed", "surface_value=1.e30" /
-# prognostic so2 mixing ratio tracer
+           "longname",  "ozone mixing ratio"
+           "units",     "kg/kg"
+       "profile_type",  "fixed", "surface_value=1.e30" /
+# prognostic subgrid scale turbulent kinetic energy
+ "TRACER", "atmos_mod", "sgs_tke"
+           "longname",  "subgrid scale turbulent kinetic energy"
+           "units",     "m2/s2"
+       "profile_type",  "fixed", "surface_value=1.e30" /
+# prognostic chemistry tracers
  "TRACER", "atmos_mod", "so2"
-           "longname",     "so2 mixing ratio"
-           "units",        "ppm"
-       "profile_type", "fixed", "surface_value=5.e-6" /
-# prognostic sulfate mixing ratio tracer
- "TRACER", "atmos_mod", "sulf"
-           "longname",     "sulfate mixing ratio"
-           "units",        "ug/kg"
-       "profile_type", "fixed", "surface_value=3.e-6" /
-# prognostic DMS mixing ratio tracer
+           "longname",  "so2 mixing ratio"
+           "units",     "ppm"
+       "tracer_usage",  "chemistry"
+       "profile_type",  "fixed", "surface_value=5.e-6" /
+ "TRACER", "atmos_mod", "so4"
+           "longname",  "sulfate mixing ratio"
+           "units",     "ug/kg"
+       "tracer_usage",  "chemistry"
+       "profile_type",  "fixed", "surface_value=3.e-6" /
  "TRACER", "atmos_mod", "dms"
-           "longname",     "DMS mixing ratio"
-           "units",        "ppm"
-       "profile_type", "fixed", "surface_value=1.e-7" /
-# prognostic msa mixing ratio tracer
+           "longname",  "DMS mixing ratio"
+           "units",     "ppm"
+       "tracer_usage",  "chemistry"
+       "profile_type",  "fixed", "surface_value=1.e-7" /
  "TRACER", "atmos_mod", "msa"
-           "longname",     "msa mixing ratio"
-           "units",        "ppm"
-       "profile_type", "fixed", "surface_value=1.e-7" /
-# prognostic bc1 mixing ratio tracer
+           "longname",  "msa mixing ratio"
+           "units",     "ppm"
+       "tracer_usage",  "chemistry"
+       "profile_type",  "fixed", "surface_value=1.e-7" /
  "TRACER", "atmos_mod", "bc1"
-           "longname",     "hydrophobic black carbon mixing ratio"
-           "units",        "ug/kg"
-       "profile_type", "fixed", "surface_value=1.e-7" /
-# prognostic bc2 mixing ratio tracer
+           "longname",  "hydrophobic black carbon mixing ratio"
+           "units",     "ug/kg"
+       "tracer_usage",  "chemistry"
+       "profile_type",  "fixed", "surface_value=1.e-7" /
  "TRACER", "atmos_mod", "bc2"
-           "longname",     "hydrophillic black carbon mixing ratio"
-           "units",        "ug/kg"
-       "profile_type", "fixed", "surface_value=1.e-7" /
-# prognostic oc1 mixing ratio tracer
+           "longname",  "hydrophillic black carbon mixing ratio"
+           "units",     "ug/kg"
+       "tracer_usage",  "chemistry"
+       "profile_type",  "fixed", "surface_value=1.e-7" /
  "TRACER", "atmos_mod", "oc1"
-           "longname",     "hydrophobic organic carbon mixing ratio"
-           "units",        "ug/kg"
-       "profile_type", "fixed", "surface_value=1.e-7" /
-# prognostic oc2 mixing ratio tracer
+           "longname",  "hydrophobic organic carbon mixing ratio"
+           "units",     "ug/kg"
+       "tracer_usage",  "chemistry"
+       "profile_type",  "fixed", "surface_value=1.e-7" /
  "TRACER", "atmos_mod", "oc2"
-           "longname",     "hydrophillic organic carbon mixing ratio"
-           "units",        "ug/kg"
-       "profile_type", "fixed", "surface_value=1.e-7" /
-# prognostic dust1 mixing ratio tracer
+           "longname",  "hydrophillic organic carbon mixing ratio"
+           "units",     "ug/kg"
+       "tracer_usage",  "chemistry"
+       "profile_type",  "fixed", "surface_value=1.e-7" /
  "TRACER", "atmos_mod", "dust1"
-           "longname",     "fine dust1 mixing ratio"
-           "units",        "ug/kg"
-       "profile_type", "fixed", "surface_value=1.e-7" /
-# prognostic dust2 mixing ratio tracer
+           "longname",  "fine dust1 mixing ratio"
+           "units",     "ug/kg"
+       "tracer_usage",  "chemistry"
+       "profile_type",  "fixed", "surface_value=1.e-7" /
  "TRACER", "atmos_mod", "dust2"
-           "longname",     "fine dust2 mixing ratio"
-           "units",        "ug/kg"
-       "profile_type", "fixed", "surface_value=1.e-7" /
-# prognostic dust3 mixing ratio tracer
+           "longname",  "fine dust2 mixing ratio"
+           "units",     "ug/kg"
+       "tracer_usage",  "chemistry"
+       "profile_type",  "fixed", "surface_value=1.e-7" /
  "TRACER", "atmos_mod", "dust3"
-           "longname",     "coarse dust3 mixing ratio"
-           "units",        "ug/kg"
-       "profile_type", "fixed", "surface_value=1.e-7" /
-# prognostic dust4 mixing ratio tracer
+           "longname",  "coarse dust3 mixing ratio"
+           "units",     "ug/kg"
+       "tracer_usage",  "chemistry"
+       "profile_type",  "fixed", "surface_value=1.e-7" /
  "TRACER", "atmos_mod", "dust4"
-           "longname",     "coarse dust4 mixing ratio"
-           "units",        "ug/kg"
-       "profile_type", "fixed", "surface_value=1.e-7" /
-# prognostic dust5 mixing ratio tracer
+           "longname",  "coarse dust4 mixing ratio"
+           "units",     "ug/kg"
+       "tracer_usage",  "chemistry"
+       "profile_type",  "fixed", "surface_value=1.e-7" /
  "TRACER", "atmos_mod", "dust5"
-           "longname",     "coarse dust5 mixing ratio"
-           "units",        "ug/kg"
-       "profile_type", "fixed", "surface_value=1.e-7" /
-# prognostic seas1 mixing ratio tracer
+           "longname",  "coarse dust5 mixing ratio"
+           "units",     "ug/kg"
+       "tracer_usage",  "chemistry"
+       "profile_type",  "fixed", "surface_value=1.e-7" /
  "TRACER", "atmos_mod", "seas1"
-           "longname",     "seasalt1 mixing ratio"
-           "units",        "ug/kg"
-       "profile_type", "fixed", "surface_value=1.e-7" /
-# prognostic seas2 mixing ratio tracer
+           "longname",  "seasalt1 mixing ratio"
+           "units",     "ug/kg"
+       "tracer_usage",  "chemistry"
+       "profile_type",  "fixed", "surface_value=1.e-7" /
  "TRACER", "atmos_mod", "seas2"
-           "longname",     "seasalt2 mixing ratio"
-           "units",        "ug/kg"
-       "profile_type", "fixed", "surface_value=1.e-7" /
-# prognostic seas3 mixing ratio tracer
+           "longname",  "seasalt2 mixing ratio"
+           "units",     "ug/kg"
+       "tracer_usage",  "chemistry"
+       "profile_type",  "fixed", "surface_value=1.e-7" /
  "TRACER", "atmos_mod", "seas3"
-           "longname",     "seasalt3 mixing ratio"
-           "units",        "ug/kg"
-       "profile_type", "fixed", "surface_value=1.e-7" /
-# prognostic seas4 mixing ratio tracer
+           "longname",  "seasalt3 mixing ratio"
+           "units",     "ug/kg"
+       "tracer_usage",  "chemistry"
+       "profile_type",  "fixed", "surface_value=1.e-7" /
  "TRACER", "atmos_mod", "seas4"
-           "longname",     "seasalt4 mixing ratio"
-           "units",        "ug/kg"
-       "profile_type", "fixed", "surface_value=1.e-7" /
-# prognostic seas5 mixing ratio tracer
+           "longname",  "seasalt4 mixing ratio"
+           "units",     "ug/kg"
+       "tracer_usage",  "chemistry"
+       "profile_type",  "fixed", "surface_value=1.e-7" /
  "TRACER", "atmos_mod", "seas5"
-           "longname",     "seasalt5 mixing ratio"
-           "units",        "ug/kg"
-       "profile_type", "fixed", "surface_value=1.e-7" /
-# prognostic NI mixing ratio tracer
+           "longname",  "seasalt5 mixing ratio"
+           "units",     "ug/kg"
+       "tracer_usage",  "chemistry"
+       "profile_type",  "fixed", "surface_value=1.e-7" /
  "TRACER", "atmos_mod", "nh3"
-           "longname",     "primary NH3 mixing ratio"
-           "units",        "ug/kg"
-       "profile_type", "fixed", "surface_value=1.e+0" /
+           "longname",  "primary NH3 mixing ratio"
+           "units",     "ug/kg"
+       "tracer_usage",  "chemistry"
+       "profile_type",  "fixed", "surface_value=1.e+0" /
  "TRACER", "atmos_mod", "nh4a"
-           "longname",     "primary NH4a mixing ratio"
-           "units",        "ug/kg"
-       "profile_type", "fixed", "surface_value=1.e+0" /
+           "longname",  "primary NH4a mixing ratio"
+           "units",     "ug/kg"
+       "tracer_usage",  "chemistry"
+       "profile_type",  "fixed", "surface_value=1.e+0" /
  "TRACER", "atmos_mod", "no3an1"
-           "longname",     "primary NO3an1 mixing ratio"
-           "units",        "ug/kg"
-       "profile_type", "fixed", "surface_value=1.e+0" /
+           "longname",  "primary NO3an1 mixing ratio"
+           "units",     "ug/kg"
+       "tracer_usage",  "chemistry"
+       "profile_type",  "fixed", "surface_value=1.e+0" /
  "TRACER", "atmos_mod", "no3an2"
-           "longname",     "primary NO3an2 mixing ratio"
-           "units",        "ug/kg"
-       "profile_type", "fixed", "surface_value=1.e+0" /
+           "longname",  "primary NO3an2 mixing ratio"
+           "units",     "ug/kg"
+       "tracer_usage",  "chemistry"
+       "profile_type",  "fixed", "surface_value=1.e+0" /
  "TRACER", "atmos_mod", "no3an3"
-           "longname",     "primary NO3an3 mixing ratio"
-           "units",        "ug/kg"
-       "profile_type", "fixed", "surface_value=1.e+0" /
-# diagnostic pm2.5 tracer
- "TRACER", "atmos_mod", "pp25"
-           "longname",     "primary PM25 mixing ratio"
-           "units",        "ug/m3"
-       "profile_type", "fixed", "surface_value=1.e+0" /
-# diagnostic pm10 tracer
- "TRACER", "atmos_mod", "pp10"
-           "longname",     "primary PM10 mixing ratio"
-           "units",        "ug/m3"
-       "profile_type", "fixed", "surface_value=1.e+0" /
+           "longname",  "primary NO3an3 mixing ratio"
+           "units",     "ug/kg"
+       "tracer_usage",  "chemistry"
+       "profile_type",  "fixed", "surface_value=1.e+0" /
+# diagnostic chemistry tracers
+ "TRACER", "atmos_mod", "pm25"
+           "longname",  "primary PM25 mixing ratio"
+           "units",     "ug/m3"
+       "tracer_usage",  "chemistry", "type=diagnostic"
+       "profile_type",  "fixed", "surface_value=1.e+0" /
+ "TRACER", "atmos_mod", "pm10"
+           "longname",  "primary PM10 mixing ratio"
+           "units",     "ug/m3"
+       "tracer_usage",  "chemistry", "type=diagnostic"
+       "profile_type",  "fixed", "surface_value=1.e+0" /
 # non-prognostic cloud amount
  "TRACER", "atmos_mod", "cld_amt"
-           "longname",     "cloud amount"
-           "units",        "1"
-       "profile_type", "fixed", "surface_value=1.e30" /
+           "longname",  "cloud amount"
+           "units",     "1"
+       "profile_type",  "fixed", "surface_value=1.e30" /
diff --git a/workflow/cases/aerosol_firex_forecast.yaml b/workflow/cases/aerosol_firex_forecast.yaml
index 4a881d25929..8feea0d05a4 100644
--- a/workflow/cases/aerosol_firex_forecast.yaml
+++ b/workflow/cases/aerosol_firex_forecast.yaml
@@ -5,8 +5,8 @@ case:
   places:
     workflow_file: layout/free_forecast_gfs.yaml
     BASE_CPLIC: /scratch1/NCEPDEV/nems/Raffaele.Montuoro/data
-    OROFIX:     /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200914/INTEL/FV3_input_data_c384/INPUT
-    GRDFIX:     /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200914/INTEL/FV3_input_data_c384/INPUT
+    OROFIX:     /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/input-data-20210528/FV3_input_data384/INPUT
+    GRDFIX:     /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/input-data-20210528/FV3_input_data384/INPUT
 
   settings: 
     SDATE: 2019-07-01t00:00:00
@@ -15,7 +15,7 @@ case:
 
     gfs_cyc: 1
 
-    cplgocart: .true.
+    cplchem: .true.
     print_esmf: .true.
     nems_temp: 'atm_aer'
 
@@ -28,7 +28,8 @@ case:
     KEEPDATA: Yes
 
   nsst:
-    NST_MODEL: 0
+    NST_MODEL:  2
+    NST_SPINUP: 1
 
   output_settings:
     FHOUT_GFS: 6
@@ -49,9 +50,10 @@ case:
        WGRP_NTASKS: 24
        WRTIOBUF: "32M"
     CPL_ATMIC: FV3ICS
+    CCPP_SUITE: FV3_GFS_v16
 
   chem_settings:
-    MOD: gsdchem
+    MOD: gocart
     ntdiag: 2
     CFGDIR: /scratch1/NCEPDEV/nems/Raffaele.Montuoro/data/NASA/rc.firex
     INPDIR: /scratch1/NCEPDEV/nems/Raffaele.Montuoro/data/NASA/ExtData
diff --git a/workflow/cases/aerosol_free_forecast.yaml b/workflow/cases/aerosol_free_forecast.yaml
index 28923dd2c7a..9d4c77bed9d 100644
--- a/workflow/cases/aerosol_free_forecast.yaml
+++ b/workflow/cases/aerosol_free_forecast.yaml
@@ -7,7 +7,7 @@ case:
     EDATE: 2013-04-01t00:00:00
     max_job_tries: 1
 
-    cplgocart: .true.
+    cplchem: .true.
     print_esmf: .true.
     nems_temp: 'atm_aer'
     nems_temp_cold: 'atm_aer'
@@ -21,7 +21,8 @@ case:
     KEEPDATA: Yes
 
   nsst:
-    NST_MODEL: 0
+    NST_MODEL:  2
+    NST_SPINUP: 1
 
   output_settings:
     FHOUT_GFS: 6
@@ -41,9 +42,11 @@ case:
        WGRP: 1
        WGRP_NTASKS: 24
        WRTIOBUF: "32M"
+    CCPP_SUITE: FV3_GFS_v16
+    fscav_aero: [ '*:0.6', 'seas1:1.0', 'seas2:1.0' ]
 
   chem_settings:
-    MOD: gsdchem
+    MOD: gocart
     ntdiag: 2
     # On Hera
     CFGDIR: /scratch1/NCEPDEV/nems/Raffaele.Montuoro/data/NASA/rc

From 4c858c0f059abd0ff4c937c7bab692d7b13a8163 Mon Sep 17 00:00:00 2001
From: "kate.friedman" <kate.friedman@noaa.gov>
Date: Tue, 8 Jun 2021 13:05:08 -0500
Subject: [PATCH 078/125] Adjust comments in getic job script

Refs: #178
---
 jobs/rocoto/getic.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/jobs/rocoto/getic.sh b/jobs/rocoto/getic.sh
index 9f0f9277348..f6692591794 100755
--- a/jobs/rocoto/getic.sh
+++ b/jobs/rocoto/getic.sh
@@ -59,8 +59,8 @@ if [ ! -d $ROTDIR ]; then mkdir -p $ROTDIR ; fi
 if [ ! -d $EXTRACT_DIR ]; then mkdir -p $EXTRACT_DIR ; fi
 cd $EXTRACT_DIR
 
-# Check if init is needed and run if so
-if [[ $gfs_ver = "v16" && $EXP_WARM_START = ".true." && $CASE = $OPS_RES ]]; then
+# Check version, cold/warm start, and resolution
+if [[ $gfs_ver = "v16" && $EXP_WARM_START = ".true." && $CASE = $OPS_RES ]]; then # Pull warm start ICs - no chgres
 
   # Pull RESTART files off HPSS
   if [ ${RETRO:-"NO"} = "YES" ]; then # Retrospective parallel input
@@ -95,7 +95,7 @@ if [[ $gfs_ver = "v16" && $EXP_WARM_START = ".true." && $CASE = $OPS_RES ]]; the
     htar -xvf ${PRODHPSSDIR}/rh${gyy}/${gyy}${gmm}/${gyy}${gmm}${gdd}/com_gfs_prod_gdas.${gyy}${gmm}${gdd}_${ghh}.gdas_restart.tar
   fi
 
-else
+else # Pull chgres cube inputs for cold start IC generation
 
   # Run UFS_UTILS GETICSH
   sh ${GETICSH} ${CDUMP}

From b52d2f0b82e0e376905c64daa593362396f8da1b Mon Sep 17 00:00:00 2001
From: "kate.friedman" <kate.friedman@noaa.gov>
Date: Tue, 8 Jun 2021 13:06:30 -0500
Subject: [PATCH 079/125] Update UFS_UTILS checkout to ufs_utils_1_4_0 tag

- new ufs_utils_1_4_0 tag includes updates to support this branch
- From tag release notes:
  - Update the GDAS initialization utility to ingest GFS v16 data.
  - Update to process either the GFS or GDAS CDUMPs.

Refs: #178
---
 sorc/checkout.sh | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/sorc/checkout.sh b/sorc/checkout.sh
index eda2d007355..9ac4b97ea83 100755
--- a/sorc/checkout.sh
+++ b/sorc/checkout.sh
@@ -60,11 +60,9 @@ fi
 echo ufs_utils checkout ...
 if [[ ! -d ufs_utils.fd ]] ; then
     rm -f ${topdir}/checkout-ufs_utils.log
-    #git clone https://github.com/NOAA-EMC/UFS_UTILS.git ufs_utils.fd >> ${topdir}/checkout-ufs_utils.fd.log 2>&1
-    git clone --recursive https://github.com/GeorgeGayno-NOAA/UFS_UTILS.git ufs_utils.fd >> ${topdir}/checkout-ufs_utils.fd.log 2>&1
+    git clone https://github.com/NOAA-EMC/UFS_UTILS.git ufs_utils.fd >> ${topdir}/checkout-ufs_utils.fd.log 2>&1
     cd ufs_utils.fd
-    #git checkout ops-gfsv16.0.0
-    git checkout feature/gdas_init.v16
+    git checkout ufs_utils_1_4_0
     cd ${topdir}
 else
     echo 'Skip.  Directory ufs_utils.fd already exists.'

From bc7c20a880bf29a303ff3ac9446c7628c9f73d0d Mon Sep 17 00:00:00 2001
From: Raffaele Montuoro <raffaele.montuoro@noaa.gov>
Date: Tue, 8 Jun 2021 22:16:57 +0000
Subject: [PATCH 080/125] Add scavenging factors to aerosol test cases.

---
 workflow/cases/aerosol_firex_forecast.yaml | 1 +
 workflow/cases/aerosol_free_forecast.yaml  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/workflow/cases/aerosol_firex_forecast.yaml b/workflow/cases/aerosol_firex_forecast.yaml
index 8feea0d05a4..d5d4b5ab27e 100644
--- a/workflow/cases/aerosol_firex_forecast.yaml
+++ b/workflow/cases/aerosol_firex_forecast.yaml
@@ -51,6 +51,7 @@ case:
        WRTIOBUF: "32M"
     CPL_ATMIC: FV3ICS
     CCPP_SUITE: FV3_GFS_v16
+    fscav_aero: [ '*:0.6', 'seas1:1.0', 'seas2:1.0', 'seas3:1.0', 'seas4:1.0', 'seas5:1.0' ]
 
   chem_settings:
     MOD: gocart
diff --git a/workflow/cases/aerosol_free_forecast.yaml b/workflow/cases/aerosol_free_forecast.yaml
index 9d4c77bed9d..13547e87068 100644
--- a/workflow/cases/aerosol_free_forecast.yaml
+++ b/workflow/cases/aerosol_free_forecast.yaml
@@ -43,7 +43,7 @@ case:
        WGRP_NTASKS: 24
        WRTIOBUF: "32M"
     CCPP_SUITE: FV3_GFS_v16
-    fscav_aero: [ '*:0.6', 'seas1:1.0', 'seas2:1.0' ]
+    fscav_aero: [ '*:0.6', 'seas1:1.0', 'seas2:1.0', 'seas3:1.0', 'seas4:1.0', 'seas5:1.0' ]
 
   chem_settings:
     MOD: gocart

From 18970c716d79922fa02b86165996bb47fb42d3fa Mon Sep 17 00:00:00 2001
From: "kate.friedman" <kate.friedman@noaa.gov>
Date: Wed, 9 Jun 2021 12:04:01 -0500
Subject: [PATCH 081/125] Disconnect archive job in workflow from HPSS access
 check

- remove dependency on HPSS access to check for adding gdas[gfs]arch job to workflow
- add HPSSARCH variable to ARCHIVE_TO_HPSS definition in cycled workflow, similar to prior addition to free-forecast script

Refs: #178
---
 ush/rocoto/setup_workflow.py          | 5 ++---
 ush/rocoto/setup_workflow_fcstonly.py | 2 --
 2 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/ush/rocoto/setup_workflow.py b/ush/rocoto/setup_workflow.py
index 4a4bfa6bb82..f20a09a2668 100755
--- a/ush/rocoto/setup_workflow.py
+++ b/ush/rocoto/setup_workflow.py
@@ -161,6 +161,7 @@ def get_definitions(base):
 
     machine = base.get('machine', wfu.detectMachine())
     scheduler = wfu.get_scheduler(machine)
+    hpssarch = base.get('HPSSARCH', 'NO').upper()
 
     strings = []
 
@@ -198,7 +199,7 @@ def get_definitions(base):
     strings.append('\t<!ENTITY SCHEDULER  "%s">\n' % scheduler)
     strings.append('\n')
     strings.append('\t<!-- Toggle HPSS archiving -->\n')
-    strings.append('\t<!ENTITY ARCHIVE_TO_HPSS "YES">\n')
+    strings.append('\t<!ENTITY ARCHIVE_TO_HPSS "%s">\n' % base['HPSSARCH'])
     strings.append('\n')
     strings.append('\t<!-- ROCOTO parameters that control workflow -->\n')
     strings.append('\t<!ENTITY CYCLETHROTTLE "3">\n')
@@ -918,8 +919,6 @@ def get_gdasgfs_tasks(dict_configs, cdump='gdas'):
     deps = []
     dep_dict = {'type': 'task', 'name': '%svrfy' % cdump}
     deps.append(rocoto.add_dependency(dep_dict))
-    dep_dict = {'type': 'streq', 'left': '&ARCHIVE_TO_HPSS;', 'right': 'YES'}
-    deps.append(rocoto.add_dependency(dep_dict))
     if do_wave in ['Y', 'YES']:
       dep_dict = {'type': 'task', 'name': '%swavepostsbs' % cdump}
       deps.append(rocoto.add_dependency(dep_dict))
diff --git a/ush/rocoto/setup_workflow_fcstonly.py b/ush/rocoto/setup_workflow_fcstonly.py
index 66f553b8be7..f25ec53f1c5 100755
--- a/ush/rocoto/setup_workflow_fcstonly.py
+++ b/ush/rocoto/setup_workflow_fcstonly.py
@@ -685,8 +685,6 @@ def get_workflow(dict_configs, cdump='gdas'):
     deps.append(rocoto.add_dependency(dep_dict))
     dep_dict = {'type':'task', 'name':'%svrfy' % cdump}
     deps.append(rocoto.add_dependency(dep_dict))
-    dep_dict = {'type':'streq', 'left':'&ARCHIVE_TO_HPSS;', 'right':'YES'}
-    deps.append(rocoto.add_dependency(dep_dict))
     if do_wave in ['Y', 'YES']:
       dep_dict = {'type': 'task', 'name': '%swavepostsbs' % cdump}
       deps.append(rocoto.add_dependency(dep_dict))

From f81741bdb3a8cc221f6e0846c706cfe3cc9c4010 Mon Sep 17 00:00:00 2001
From: Raffaele Montuoro <raffaele.montuoro@noaa.gov>
Date: Thu, 10 Jun 2021 03:44:06 +0000
Subject: [PATCH 082/125] Use updated modulefile names only when building the
 ufs-weather-model with prognostic aerosols. This option is recognized by
 means of a hidden file created in the model's repository when running
 checkout.sh with option -a.

---
 sorc/build_ufs_coupled.sh | 10 ++++++----
 sorc/checkout.sh          |  1 +
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/sorc/build_ufs_coupled.sh b/sorc/build_ufs_coupled.sh
index 0d6be6a0e19..66de17a7fda 100755
--- a/sorc/build_ufs_coupled.sh
+++ b/sorc/build_ufs_coupled.sh
@@ -16,13 +16,15 @@ if [ $target = stampede ]; then target=stampede.intel ; fi
 MOD_PATH=$cwd/ufs_coupled.fd/modulefiles
 
 module purge 
-module use $MOD_PATH 
-module load ufs_${target}
+
 cd ufs_coupled.fd/
 if [[ -d build ]]; then rm -Rf build; fi
-if [[ -d GOCART ]]; then
-  module load ufs_aerosols_${target}
+if [[ -e .build_aerosols ]] ; then
+  module use ${MOD_PATH}
+  module load ufs_${target} ufs_aerosols_${target}
   CMAKE_FLAGS="-DAPP=ATMAERO" CCPP_SUITES="FV3_GFS_v16" ./build.sh
 else
+  module use ${MOD_PATH}/${target}
+  module load fv3
   CMAKE_FLAGS="-DS2S=ON -DWW3=ON" CCPP_SUITES="FV3_GFS_v15p2_coupled,FV3_GFS_v16_coupled,FV3_GFS_v16" ./build.sh
 fi
diff --git a/sorc/checkout.sh b/sorc/checkout.sh
index fc35a12e375..10f79ceb849 100755
--- a/sorc/checkout.sh
+++ b/sorc/checkout.sh
@@ -49,6 +49,7 @@ else
       git clone https://github.com/rmontuoro/ufs-weather-model.git ufs_coupled.fd >> ${topdir}/checkout-ufs_coupled.log 2>&1
       cd ufs_coupled.fd
       git checkout feature/gocart
+      echo > .build_aerosols
     else
       git clone https://github.com/ufs-community/ufs-weather-model ufs_coupled.fd >> ${topdir}/checkout-ufs_coupled.log 2>&1
       cd ufs_coupled.fd

From 0307896af1d309718d6b08d89631f8c080c575e7 Mon Sep 17 00:00:00 2001
From: Raffaele Montuoro <raffaele.montuoro@noaa.gov>
Date: Thu, 10 Jun 2021 03:55:52 +0000
Subject: [PATCH 083/125] Load updated modulefiles for coupled aerosols on
 Hera/Intel.

A new shell function (get_platform) is introduced to increase
modularity. This fuction auto-detects the local platform and
returns a string identifier. As a result, load_fv3gfs_modules.sh
is refactored.
---
 ush/get_platform.sh        | 44 ++++++++++++++++++++
 ush/load_fv3gfs_modules.sh | 82 +++++++++++++++++---------------------
 2 files changed, 80 insertions(+), 46 deletions(-)
 create mode 100644 ush/get_platform.sh

diff --git a/ush/get_platform.sh b/ush/get_platform.sh
new file mode 100644
index 00000000000..7a2442ec1c6
--- /dev/null
+++ b/ush/get_platform.sh
@@ -0,0 +1,44 @@
+#!/bin/sh
+
+###############################################################
+# Auto-detect platform and return string identifier
+###############################################################
+
+get_platform() {
+
+  local hname=`hostname -d`
+  local pname="unknown"
+
+  if [[ $hname = 'stampede2.tacc.utexas.edu' ]] ; then
+    # We are on Xsede stampede2
+    pname="stampede"
+  elif [[ $hname = 'hpc.msstate.edu' ]] ; then
+    # We are on MSU Orion
+    pname="orion"
+  elif [[ -d /lfs3 ]] ; then
+    # We are on NOAA Jet
+    pname="jet"
+  elif [[ -d /scratch1 ]] ; then
+    # We are on NOAA Hera
+    pname="hera.intel"
+  elif [[ -d /gpfs/hps && -e /etc/SuSE-release ]] ; then
+    # We are on NOAA Luna or Surge
+    pname="wcoss_c"
+  elif [[ -L /usrx && "$( readlink /usrx 2> /dev/null )" =~ dell ]] ; then
+    # We are on NOAA Mars or Venus
+    pname="wcoss_dell_p3"
+  elif [[ -d /dcom && -d /hwrf ]] ; then
+    # We are on NOAA Tide or Gyre
+    pname="wcoss"
+  elif [[ -d /glade ]] ; then
+    # We are on NCAR Cheyenne
+    pname="cheyenne"
+  elif [[ -d /lustre && -d /ncrc ]] ; then
+    # We are on GAEA.
+    pname="gaea"
+  else
+    echo WARNING: UNKNOWN PLATFORM 
+  fi
+
+  echo "${pname}"
+}
diff --git a/ush/load_fv3gfs_modules.sh b/ush/load_fv3gfs_modules.sh
index 19c7a2664ec..a1e39ec7689 100755
--- a/ush/load_fv3gfs_modules.sh
+++ b/ush/load_fv3gfs_modules.sh
@@ -1,59 +1,49 @@
 #!/bin/sh
-#set -x
 
 ###############################################################
 # Setup runtime environment by loading modules
+###############################################################
+
 ulimit_s=$( ulimit -S -s )
-#ulimit -S -s 10000
 
 # Find module command and purge:
 source "$HOMEgfs/modulefiles/module-setup.sh.inc" 
+source "$HOMEgfs/ush/get_platform.sh"
 
-# Load our modules:
-module use "$HOMEgfs/modulefiles" 
-hname=`hostname -d`
+platform=$( get_platform )
 
-if [[ $hname = 'stampede2.tacc.utexas.edu' ]] ; then
-	# We are on Xsede stampede2
-	module load module_base.stampede
-elif [[ $hname = 'hpc.msstate.edu' ]] ; then
-	# We are on MSU Orion
-	module load module_base.orion
-elif [[ -d /lfs3 ]] ; then
-	# We are on NOAA Jet
-	module load module_base.jet 
-elif [[ -d /scratch1 ]] ; then
-	# We are on NOAA Hera
-	module use "$HOMEgfs/sorc/ufs_coupled.fd/modulefiles"
-	module load ufs_hera.intel
-	if [[ -d $HOMEgfs/sorc/ufs_coupled.fd/GOCART ]] ; then
-		module load ufs_aerosols_hera.intel
-	fi
-elif [[ -d /work ]] ; then
-	# We are on MSU Orion
-	module load module_base.orion
-	if [[ -d $HOMEgfs/sorc/ufs_coupled.fd/GOCART ]] ; then
-		module use "$HOMEgfs/sorc/ufs_coupled.fd/modulefiles/orion.intel"
-		module load fv3
-	fi
-elif [[ -d /gpfs/hps && -e /etc/SuSE-release ]] ; then
-	# We are on NOAA Luna or Surge
-	module load module_base.wcoss_c 
-elif [[ -L /usrx && "$( readlink /usrx 2> /dev/null )" =~ dell ]] ; then
-	# We are on NOAA Mars or Venus
-	module load module_base.wcoss_dell_p3 
-elif [[ -d /dcom && -d /hwrf ]] ; then
-	# We are on NOAA Tide or Gyre
-	module load module_base.wcoss 
-elif [[ -d /glade ]] ; then
-	# We are on NCAR Yellowstone
-	module load module_base.cheyenne 
-elif [[ -d /lustre && -d /ncrc ]] ; then
-	# We are on GAEA.
-	module load module_base.gaea 
-else
-	echo WARNING: UNKNOWN PLATFORM 
-fi
+moduledir=
+modulelist=
+case "${platform}" in
+  hera.intel)
+    moduledir="$HOMEgfs/sorc/ufs_coupled.fd/modulefiles"
+    if [[ -r ${moduledir}/${platform}/fv3 ]] ; then
+      moduledir="$HOMEgfs/sorc/ufs_coupled.fd/modulefiles/${platform}"
+      modulelist=fv3
+    elif [[ -r ${moduledir}/ufs_${platform} ]] ; then
+      modulelist=ufs_${platform}
+      if [[ "$( grep UFS_GOCART ${HOMEgfs}/sorc/ufs_coupled.fd/build/CMakeCache.txt 2>/dev/null | cut -d= -f2 )" = "ON" ]] ; then
+        # add aerosols modulefile
+        modulelist="${modulelist} ufs_aerosols_${platform}"
+      fi
+    else
+      echo "Unable to load modulefiles on ${platform}"
+      exit 1
+    fi
+    ;;
+  unknown)
+    echo "Unknown platform"
+    exit 1
+    ;;
+  *)
+    moduledir="$HOMEgfs/modulefiles"
+    modulelist="module_base.${platform}"
+    ;;
+esac
+
+# Load our modules:
+module use ${moduledir}
+module load ${modulelist}
 
 # Restore stack soft limit:
 ulimit -S -s "$ulimit_s"

From 6b19a39b6f086ff7eb077c60a3acbc40d6dae5a1 Mon Sep 17 00:00:00 2001
From: Raffaele Montuoro <raffaele.montuoro@noaa.gov>
Date: Thu, 10 Jun 2021 04:04:51 +0000
Subject: [PATCH 084/125] Remove GSD-specific functions in forecast postdet.

---
 ush/forecast_postdet.sh | 18 ------------------
 1 file changed, 18 deletions(-)

diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh
index 4acbd1c2fba..545a09d2929 100755
--- a/ush/forecast_postdet.sh
+++ b/ush/forecast_postdet.sh
@@ -856,21 +856,3 @@ GOCART_rc() {
 		[[ $status -ne 0 ]] && exit $status
 	fi
 }
-
-GSD_in() {
-	echo "SUB ${FUNCNAME[0]}: Linking input data for GSD"
-	# soft link commands insert here
-}
-
-GSD_nml() {
-	echo "SUB ${FUNCNAME[0]}: Creating name list for GSD"
-	# generate GSD-specific input
-}
-
-GSD_out() {
-	echo "SUB ${FUNCNAME[0]}: Copying output data for GSD"
-	# soft link commands insert here
-}
-
-
-

From 8a8194f7fe2210de4a70de8322de2f4384d3098a Mon Sep 17 00:00:00 2001
From: Raffaele Montuoro <raffaele.montuoro@noaa.gov>
Date: Thu, 10 Jun 2021 04:05:57 +0000
Subject: [PATCH 085/125] Remove no longer needed parsing script used to
 generate the FV3 namelist file for FV3-GOCART.

---
 ush/parsing_namelists_FV3_GOCART.sh | 364 ----------------------------
 1 file changed, 364 deletions(-)
 delete mode 100755 ush/parsing_namelists_FV3_GOCART.sh

diff --git a/ush/parsing_namelists_FV3_GOCART.sh b/ush/parsing_namelists_FV3_GOCART.sh
deleted file mode 100755
index 0d2390515c3..00000000000
--- a/ush/parsing_namelists_FV3_GOCART.sh
+++ /dev/null
@@ -1,364 +0,0 @@
-#! /bin/sh
-##### 
-## "parsing_namelist_FV3.sh"
-## This script writes namelist for FV3 model
-##
-## This is the child script of ex-global forecast,
-## writing namelist for FV3
-## This script is a direct execution.
-#####
-
-FV3_namelists(){
-
-# copy over the tables
-DIAG_TABLE=${DIAG_TABLE:-$PARM_FV3DIAG/diag_table_aer}
-DATA_TABLE=${DATA_TABLE:-$PARM_FV3DIAG/data_table}
-FIELD_TABLE=${FIELD_TABLE:-$PARM_FV3DIAG/field_table_aer}
-
-# build the diag_table with the experiment name and date stamp
-if [ $DOIAU = "YES" ]; then
-cat > diag_table << EOF
-FV3 Forecast
-${gPDY:0:4} ${gPDY:4:2} ${gPDY:6:2} ${gcyc} 0 0
-EOF
-cat $DIAG_TABLE >> diag_table
-else
-cat > diag_table << EOF
-FV3 Forecast
-${sPDY:0:4} ${sPDY:4:2} ${sPDY:6:2} ${scyc} 0 0
-EOF
-cat $DIAG_TABLE >> diag_table
-fi
-
-$NCP $DATA_TABLE  data_table
-$NCP $FIELD_TABLE field_table
-
-cat > input.nml <<EOF
-&amip_interp_nml
-  interp_oi_sst = .true.
-  use_ncep_sst = .true.
-  use_ncep_ice = .false.
-  no_anom_sst = .false.
-  data_set = 'reynolds_oi'
-  date_out_of_range = 'climo'
-  $amip_interp_nml
-/
-
-&atmos_model_nml
-  blocksize = $blocksize
-  chksum_debug = $chksum_debug
-  dycore_only = $dycore_only
-  fdiag = $FDIAG
-  fhmax = $FHMAX
-  fhout = $FHOUT
-  fhmaxhf = $FHMAX_HF     ! CROW configured
-  fhouthf = $FHOUT_HF     ! CROW configured
-  $atmos_model_nml
-/
-
-&diag_manager_nml
-  prepend_date = .false.
-  $diag_manager_nml
-/
-
-&fms_io_nml
-  checksum_required = .false.
-  max_files_r = 100
-  max_files_w = 100
-  $fms_io_nml
-/
-
-&fms_nml
-  clock_grain = 'ROUTINE'
-  domains_stack_size = ${domains_stack_size:-3000000}
-  print_memory_usage = ${print_memory_usage:-".false."}
-  $fms_nml
-/
-
-&fv_core_nml
-  layout = $layout_x,$layout_y
-  io_layout = $io_layout
-  npx = $npx
-  npy = $npy
-  ntiles = $ntiles
-  npz = $npz
-  grid_type = -1
-  make_nh = $make_nh 
-  fv_debug = ${fv_debug:-".false."}
-  range_warn = ${range_warn:-".false."}
-  reset_eta = .false.
-  n_sponge = ${n_sponge:-"10"}    ! CROW configured
-  nudge_qv = ${nudge_qv:-".true."}
-  tau = ${tau:-10.}
-  rf_cutoff = ${rf_cutoff:-"7.5e2"}
-  d2_bg_k1 = ${d2_bg_k1:-"0.15"}    ! CROW configured
-  d2_bg_k2 = ${d2_bg_k2:-"0.02"}    ! CROW configured
-  kord_tm = ${kord_tm:-"-9"}
-  kord_mt = ${kord_mt:-"9"}
-  kord_wz = ${kord_wz:-"9"}
-  kord_tr = ${kord_tr:-"9"}
-  hydrostatic = $hydrostatic
-  phys_hydrostatic = $phys_hydrostatic
-  use_hydro_pressure = $use_hydro_pressure
-  beta = 0.
-  a_imp = 1.
-  p_fac = 0.1
-  k_split = $k_split
-  n_split = $n_split
-  nwat = ${nwat:-2}		! CROW configured
-  na_init = $na_init
-  d_ext = 0.
-  dnats = ${dnats:-0}		! CROW configured
-  fv_sg_adj = ${fv_sg_adj:-"450"}
-  d2_bg = 0.
-  nord = ${nord:-3}		! CROW configured
-  dddmp = ${dddmp:-0.2}		! CROW configured
-  d4_bg = ${d4_bg:-0.15}	! CROW configured
-  vtdm4 = $vtdm4
-  delt_max = ${delt_max:-"0.002"}
-  ke_bg = 0.
-  do_vort_damp = $do_vort_damp
-  external_ic = $external_ic
-  external_eta = ${external_eta:-.true.}
-  gfs_phil = ${gfs_phil:-".false."}
-  nggps_ic = $nggps_ic
-  mountain = $mountain
-  ncep_ic = $ncep_ic
-  d_con = $d_con
-  hord_mt = $hord_mt		! CROW configured
-  hord_vt = $hord_xx		! CROW configured
-  hord_tm = $hord_xx		! CROW configured
-  hord_dp = -$hord_xx		! CROW configured
-  hord_tr = ${hord_tr:-"8"}	
-  adjust_dry_mass = ${adjust_dry_mass:-".true."}
-  consv_te = $consv_te
-  do_sat_adj = ${do_sat_adj:-".false."}	! CROW configured
-  consv_am = .false.
-  fill = .true.
-  dwind_2d = .false.
-  print_freq = $print_freq
-  warm_start = $warm_start
-  no_dycore = $no_dycore
-  z_tracer = .true.
-  read_increment = $read_increment
-  res_latlon_dynamics = $res_latlon_dynamics
-  $fv_core_nml
-/
-
-&cires_ugwp_nml
-       knob_ugwp_solver  = ${knob_ugwp_solver:-2}
-       knob_ugwp_source  = ${knob_ugwp_source:-1,1,0,0}
-       knob_ugwp_wvspec  = ${knob_ugwp_wvspec:-1,25,25,25}
-       knob_ugwp_azdir   = ${knob_ugwp_azdir:-2,4,4,4}
-       knob_ugwp_stoch   = ${knob_ugwp_stoch:-0,0,0,0}
-       knob_ugwp_effac   = ${knob_ugwp_effac:-1,1,1,1}
-       knob_ugwp_doaxyz  = ${knob_ugwp_doaxyz:-1}
-       knob_ugwp_doheat  = ${knob_ugwp_doheat:-1}
-       knob_ugwp_dokdis  = ${knob_ugwp_dokdis:-1}
-       knob_ugwp_ndx4lh  = ${knob_ugwp_ndx4lh:-1}
-       knob_ugwp_version = ${knob_ugwp_version:-0}
-       launch_level      = ${launch_level:-54}                   
-/
-
-&external_ic_nml
-  filtered_terrain = $filtered_terrain
-  levp = $LEVS
-  gfs_dwinds = $gfs_dwinds
-  checker_tr = .false.
-  nt_checker = 0
-  $external_ic_nml
-/
-
-&gfs_physics_nml
-       fhzero         = 6.
-       ldiag3d        = .false.
-       ldiag_ugwp     = .F.
-       do_ugwp        = .F.
-       do_tofd        = .F.
-       fhcyc          = 24.
-       nst_anl        = .true.
-       use_ufo        = .true.
-       pre_rad        = .false.
-       ncld           = 5
-       imp_physics    = 11
-       pdfcld         = .false.
-       fhswr          = 3600.
-       fhlwr          = 3600.
-       ialb           = 1
-       iems           = 1
-       IAER           = 111
-       ico2           = 2
-       isubc_sw       = 2
-       isubc_lw       = 2
-       isol           = 2
-       lwhtr          = .true.
-       swhtr          = .true.
-       cnvgwd         = .true.
-       shal_cnv       = .true.
-       cal_pre        = .false.
-       redrag         = .true.
-       dspheat        = .true.
-       hybedmf        = .T.
-       satmedmf       = .F.
-       lheatstrg      = .F.
-       lgfdlmprad     = .F.
-       effr_in        = .F.
-       random_clds    = .false.
-       trans_trac     = .true.
-       cnvcld         = .true.
-       imfshalcnv     = 2
-       imfdeepcnv     = 2
-       cdmbgwd        = 3.5,0.25
-       prslrd0        = 0.
-       ivegsrc        = 1
-       isot           = 1
-       lsm            = 1
-       iopt_dveg      = 2
-       iopt_crs       = 1
-       iopt_btr       = 1
-       iopt_run       = 1
-       iopt_sfc       = 1
-       iopt_frz       = 1
-       iopt_inf       = 1
-       iopt_rad       = 1
-       iopt_alb       = 2
-       iopt_snf       = 4
-       iopt_tbot      = 2
-       iopt_stc       = 1
-       debug          = .false.
-       h2o_phys       = .F.
-       nstf_name      = 2,1,1,0,5
-       iau_drymassfixer = .false.
-       cplflx         = .F.
-       cplwav         = .F.
-       cplwav2atm     = .F.
-       cplgocart      = .T.
-       xkzminv        = 0.3
-       xkzm_m         = 1.0
-       xkzm_h         = 1.0
-EOF
-
-cat >> input.nml <<EOF
-  $gfs_physics_nml
-/
-EOF
-
-echo "" >> input.nml
-
-cat >> input.nml <<EOF
-&gfdl_cloud_microphysics_nml
-       sedi_transport = .true.
-       do_sedi_heat = .false.
-       rad_snow = .true.
-       rad_graupel = .true.
-       rad_rain = .true.
-       const_vi = .F.
-       const_vs = .F.
-       const_vg = .F.
-       const_vr = .F.
-       vi_max = 1.
-       vs_max = 2.
-       vg_max = 12.
-       vr_max = 12.
-       qi_lim = 1.
-       prog_ccn = .false.
-       do_qa = .true.
-       fast_sat_adj = .true.
-       tau_l2v = 300.
-       tau_l2v = 225.
-       tau_v2l = 150.
-       tau_g2v = 900.
-       rthresh = 10.e-6  ! This is a key parameter for cloud water
-       dw_land  = 0.16
-       dw_ocean = 0.10
-       ql_gen = 1.0e-3
-       ql_mlt = 1.0e-3
-       qi0_crt = 8.0E-5
-       qs0_crt = 1.0e-3
-       tau_i2s = 1000.
-       c_psaci = 0.05
-       c_pgacs = 0.01
-       rh_inc = 0.30
-       rh_inr = 0.30
-       rh_ins = 0.30
-       ccn_l = 300.
-       ccn_o = 100.
-       c_paut = 0.5
-       c_cracw = 0.8
-       use_ppm = .false.
-       use_ccn = .true.
-       mono_prof = .true.
-       z_slope_liq  = .true.
-       z_slope_ice  = .true.
-       de_ice = .false.
-       fix_negative = .true.
-       icloud_f = 1
-       mp_time = 150.
-  $gfdl_cloud_microphysics_nml
-/
-
-&interpolator_nml
-  interp_method = 'conserve_great_circle'
-  $interpolator_nml
-/
-
-&namsfc
-  FNGLAC   = '${FNGLAC}'
-  FNMXIC   = '${FNMXIC}'
-  FNTSFC   = '${FNTSFC}'
-  FNSNOC   = '${FNSNOC}'
-  FNZORC   = '${FNZORC}'
-  FNALBC   = '${FNALBC}'
-  FNALBC2  = '${FNALBC2}'
-  FNAISC   = '${FNAISC}'
-  FNTG3C   = '${FNTG3C}'
-  FNVEGC   = '${FNVEGC}'
-  FNVETC   = '${FNVETC}'
-  FNSOTC   = '${FNSOTC}'
-  FNSMCC   = '${FNSMCC}'
-  FNMSKH   = '${FNMSKH}'
-  FNTSFA   = '${FNTSFA}'
-  FNACNA   = '${FNACNA}'
-  FNSNOA   = '${FNSNOA}'
-  FNVMNC   = '${FNVMNC}'
-  FNVMXC   = '${FNVMXC}'
-  FNSLPC   = '${FNSLPC}'
-  FNABSC   = '${FNABSC}'
-  LDEBUG = ${LDEBUG:-".false."}
-  FSMCL(2) = ${FSMCL2:-99999}
-  FSMCL(3) = ${FSMCL3:-99999}
-  FSMCL(4) = ${FSMCL4:-99999}
-  LANDICE  = ${landice:-".true."}
-  FTSFS = ${FTSFS:-90}
-  FAISL = ${FAISL:-99999}
-  FAISS = ${FAISS:-99999}
-  FSNOL = ${FSNOL:-99999}
-  FSNOS = ${FSNOS:-99999}
-  FSICL = ${FSICL:-99999}
-  FSICS = ${FSICS:-99999}
-  FTSFL = ${FTSFL:-99999}
-  FVETL = ${FVETL:-99999}
-  FSOTL = ${FSOTL:-99999}
-  FvmnL = ${FvmnL:-99999}
-  FvmxL = ${FvmxL:-99999}
-  FSLPL = ${FSLPL:-99999}
-  FABSL = ${FABSL:-99999}
-  $namsfc_nml
-/
-
-&fv_grid_nml
-  grid_file = 'INPUT/grid_spec.nc'
-  $fv_grid_nml
-/
-EOF
-
-# Add namelist for stochastic physics options
-cat >> input.nml << EOF
-&nam_stochy
-/
-&nam_sfcperts
-/
-EOF
-
-echo "$(cat input.nml)"
-}

From a821a1aef761b386beace4efab476495e209adbb Mon Sep 17 00:00:00 2001
From: Raffaele Montuoro <raffaele.montuoro@noaa.gov>
Date: Thu, 10 Jun 2021 05:32:41 +0000
Subject: [PATCH 086/125] Return proper name for Hera platform and load correct
 modules.

---
 ush/get_platform.sh        |  2 +-
 ush/load_fv3gfs_modules.sh | 16 +++++++---------
 2 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/ush/get_platform.sh b/ush/get_platform.sh
index 7a2442ec1c6..417f6ce920f 100644
--- a/ush/get_platform.sh
+++ b/ush/get_platform.sh
@@ -20,7 +20,7 @@ get_platform() {
     pname="jet"
   elif [[ -d /scratch1 ]] ; then
     # We are on NOAA Hera
-    pname="hera.intel"
+    pname="hera"
   elif [[ -d /gpfs/hps && -e /etc/SuSE-release ]] ; then
     # We are on NOAA Luna or Surge
     pname="wcoss_c"
diff --git a/ush/load_fv3gfs_modules.sh b/ush/load_fv3gfs_modules.sh
index a1e39ec7689..0a5f69e983b 100755
--- a/ush/load_fv3gfs_modules.sh
+++ b/ush/load_fv3gfs_modules.sh
@@ -15,20 +15,18 @@ platform=$( get_platform )
 moduledir=
 modulelist=
 case "${platform}" in
-  hera.intel)
+  hera)
     moduledir="$HOMEgfs/sorc/ufs_coupled.fd/modulefiles"
-    if [[ -r ${moduledir}/${platform}/fv3 ]] ; then
-      moduledir="$HOMEgfs/sorc/ufs_coupled.fd/modulefiles/${platform}"
-      modulelist=fv3
-    elif [[ -r ${moduledir}/ufs_${platform} ]] ; then
-      modulelist=ufs_${platform}
+    target=${platform}.intel
+    if [[ -r ${moduledir}/ufs_${target} ]] ; then
+      modulelist=ufs_${target}
       if [[ "$( grep UFS_GOCART ${HOMEgfs}/sorc/ufs_coupled.fd/build/CMakeCache.txt 2>/dev/null | cut -d= -f2 )" = "ON" ]] ; then
         # add aerosols modulefile
-        modulelist="${modulelist} ufs_aerosols_${platform}"
+        modulelist="${modulelist} ufs_aerosols_${target}"
       fi
     else
-      echo "Unable to load modulefiles on ${platform}"
-      exit 1
+      moduledir="$HOMEgfs/modulefiles"
+      modulelist="module_base.${platform}"
     fi
     ;;
   unknown)

From 30aefb11dd8130f1d4a1e6eb951e1fa400fdc361 Mon Sep 17 00:00:00 2001
From: "kate.friedman" <kate.friedman@noaa.gov>
Date: Thu, 10 Jun 2021 15:43:27 +0000
Subject: [PATCH 087/125] Update/fix pull of v14/v15 pgrb2 anl files in getic
 job

Refs: #178
---
 jobs/rocoto/getic.sh     | 13 ++++++++++---
 parm/config/config.getic |  2 --
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/jobs/rocoto/getic.sh b/jobs/rocoto/getic.sh
index f6692591794..b90412ee3dd 100755
--- a/jobs/rocoto/getic.sh
+++ b/jobs/rocoto/getic.sh
@@ -110,12 +110,19 @@ mv ${EXTRACT_DIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/* ${ROTDIR}/${CDUMP}.${yy}${mm}
 
 # Pull pgbanl file for verification/archival - v14+
 if [ $gfs_ver = v14 -o $gfs_ver = v15 ]; then
-  cd $EXTRACT_DIR
   for grid in 0p25 0p50 1p00
   do
     file=gfs.t${hh}z.pgrb2.${grid}.anl
-    htar -xvf ${PRODHPSSDIR}/rh${yy}/${yy}${mm}/${yy}${mm}${dd}/${tarball} ./${CDUMP}.${yy}${mm}${dd}/${hh}/${file}
-    mv ${EXTRACT_DIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${file} ${ROTDIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${COMPONENT}/${file}
+    if [ $gfs_ver = v14 ]; then
+      cd $ROTDIR/${CDUMP}.${yy}${mm}${dd}/${hh}/${COMPONENT}
+      export tarball="gpfs_hps_nco_ops_com_gfs_prod_gfs.${yy}${mm}${dd}${hh}.pgrb2_${grid}.tar"
+      htar -xvf ${PRODHPSSDIR}/rh${yy}/${yy}${mm}/${yy}${mm}${dd}/${tarball} ./${file}
+    elif [ $gfs_ver = v15 ]; then
+      cd $EXTRACT_DIR
+      export tarball="com_gfs_prod_gfs.${yy}${mm}${dd}_${hh}.gfs_pgrb2.tar"
+      htar -xvf ${PRODHPSSDIR}/rh${yy}/${yy}${mm}/${yy}${mm}${dd}/${tarball} ./${CDUMP}.${yy}${mm}${dd}/${hh}/${file}
+      mv ${EXTRACT_DIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${file} ${ROTDIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${COMPONENT}/${file}
+    fi
   done
 fi
 
diff --git a/parm/config/config.getic b/parm/config/config.getic
index 90b9660716b..ced2c051f00 100755
--- a/parm/config/config.getic
+++ b/parm/config/config.getic
@@ -44,11 +44,9 @@ elif [ ${RETRO:-"NO"} = "NO" ]; then # Operational input
   elif [[ "$CDATE" -lt "2019061200" ]]; then
     export gfs_ver=v14
     export GETICSH=${GDASINIT_DIR}/get_${gfs_ver}.data.sh
-    export tarball=gpfs_hps_nco_ops_com_gfs_prod_gfs.${PDY}_${cyc}.pgrb2_${grid}.tar
   elif [[ "$CDATE" -lt "2021032100" ]]; then
     export gfs_ver=v15
     export GETICSH=${GDASINIT_DIR}/get_${gfs_ver}.data.sh
-    export tarball=com_gfs_prod_gfs.${PDY}_${cyc}.gfs_pgrb2.tar
   elif [[ "$CDATE" -lt "2021032106" ]]; then
     # The way the v16 switch over was done, there is no complete
     # set of v16 or v15 data for 2021032100.  And although

From db18627960d2def7bbfee30994f4be6586babe1c Mon Sep 17 00:00:00 2001
From: "kate.friedman" <kate.friedman@noaa.gov>
Date: Thu, 10 Jun 2021 18:36:10 +0000
Subject: [PATCH 088/125] Increase init job walltime to 30 mins from 15 mins

Refs: #178
---
 parm/config/config.resources | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/parm/config/config.resources b/parm/config/config.resources
index 31cf6f0d659..f404ad8b1b3 100755
--- a/parm/config/config.resources
+++ b/parm/config/config.resources
@@ -273,7 +273,7 @@ elif [ $step = "echgres" ]; then
 
 elif [ $step = "init" ]; then
 
-    export wtime_init="00:15:00"
+    export wtime_init="00:30:00"
     export npe_init=24
     export nth_init=1
     export npe_node_init=6

From 37e7c2e0806678c93a11bae682f48da172940fd6 Mon Sep 17 00:00:00 2001
From: "kate.friedman" <kate.friedman@noaa.gov>
Date: Thu, 10 Jun 2021 18:36:59 +0000
Subject: [PATCH 089/125] Add OPS_RES variable to init script

Refs: #178
---
 jobs/rocoto/init.sh | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/jobs/rocoto/init.sh b/jobs/rocoto/init.sh
index 6b70842821f..d76233f92f5 100755
--- a/jobs/rocoto/init.sh
+++ b/jobs/rocoto/init.sh
@@ -46,12 +46,13 @@ export EXTRACT_DIR=${EXTRACT_DIR:-$ROTDIR}
 export WORKDIR=${WORKDIR:-$DATA}
 export OUTDIR=${OUTDIR:-$ROTDIR}
 export COMPONENT="atmos"
-export gfs_ver=${gfs_ver:-v16}
+export gfs_ver=${gfs_ver:-"v16"}
+export OPS_RES=${OPS_RES:-"C768"}
 export RUNICSH=${RUNICSH:-${GDASINIT_DIR}/run_v16.chgres.sh}
 
 # Check if init is needed and run if so
-if [[ $gfs_ver = "v16" && $EXP_WARM_START = ".true." && $CASE = "C768" ]]; then
-  echo "Detected v16 C768 warm starts, will not run init. Exiting..."
+if [[ $gfs_ver = "v16" && $EXP_WARM_START = ".true." && $CASE = $OPS_RES ]]; then
+  echo "Detected v16 $OPS_RES warm starts, will not run init. Exiting..."
   exit 0
 else
   # Run chgres_cube

From 65ff48e049a651e74be8be121163c20957bf1c44 Mon Sep 17 00:00:00 2001
From: "kate.friedman" <kate.friedman@noaa.gov>
Date: Thu, 10 Jun 2021 18:37:24 +0000
Subject: [PATCH 090/125] Add v16 pgb anl pull to getic script

Refs: #178
---
 jobs/rocoto/getic.sh | 46 ++++++++++++++++++++++++++++++++++++--------
 1 file changed, 38 insertions(+), 8 deletions(-)

diff --git a/jobs/rocoto/getic.sh b/jobs/rocoto/getic.sh
index b90412ee3dd..5c8e3353b9f 100755
--- a/jobs/rocoto/getic.sh
+++ b/jobs/rocoto/getic.sh
@@ -105,26 +105,56 @@ else # Pull chgres cube inputs for cold start IC generation
 fi
 
 # Move extracted data to ROTDIR
-if [ ! -d ${ROTDIR}/${CDUMP}.${yy}${mm}${dd}/${hh} ]; then mkdir -p ${ROTDIR}/${CDUMP}.${yy}${mm}${dd}/${hh} ; fi
-mv ${EXTRACT_DIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/* ${ROTDIR}/${CDUMP}.${yy}${mm}${dd}/${hh}
+if [ ! -d ${ROTDIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${COMPONENT} ]; then mkdir -p ${ROTDIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${COMPONENT} ; fi
+if [ $gfs_ver = v16 -a $RETRO = "YES" ]; then
+  mv ${EXTRACT_DIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/* ${ROTDIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${COMPONENT}
+else
+  mv ${EXTRACT_DIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/* ${ROTDIR}/${CDUMP}.${yy}${mm}${dd}/${hh}
+fi
 
 # Pull pgbanl file for verification/archival - v14+
-if [ $gfs_ver = v14 -o $gfs_ver = v15 ]; then
+if [ $gfs_ver = v14 -o $gfs_ver = v15 -o $gfs_ver = v16 ]; then
   for grid in 0p25 0p50 1p00
   do
     file=gfs.t${hh}z.pgrb2.${grid}.anl
-    if [ $gfs_ver = v14 ]; then
+
+    if [ $gfs_ver = v14 ]; then # v14 production source
+
       cd $ROTDIR/${CDUMP}.${yy}${mm}${dd}/${hh}/${COMPONENT}
       export tarball="gpfs_hps_nco_ops_com_gfs_prod_gfs.${yy}${mm}${dd}${hh}.pgrb2_${grid}.tar"
       htar -xvf ${PRODHPSSDIR}/rh${yy}/${yy}${mm}/${yy}${mm}${dd}/${tarball} ./${file}
-    elif [ $gfs_ver = v15 ]; then
+
+    elif [ $gfs_ver = v15 ]; then # v15 production source
+
       cd $EXTRACT_DIR
       export tarball="com_gfs_prod_gfs.${yy}${mm}${dd}_${hh}.gfs_pgrb2.tar"
       htar -xvf ${PRODHPSSDIR}/rh${yy}/${yy}${mm}/${yy}${mm}${dd}/${tarball} ./${CDUMP}.${yy}${mm}${dd}/${hh}/${file}
       mv ${EXTRACT_DIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${file} ${ROTDIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${COMPONENT}/${file}
-    fi
-  done
-fi
+
+    elif [ $gfs_ver = v16 ]; then # v16 - determine RETRO or production source next
+
+      if [ $RETRO = "YES" ]; then # Retrospective parallel source
+
+        cd $EXTRACT_DIR
+        if [ $grid = "0p25" ]; then # anl file spread across multiple tarballs
+          export tarball="gfsa.tar"
+        elif [ $grid = "0p50" -o $grid = "1p00" ]; then
+          export tarball="gfsb.tar"
+        fi
+        htar -xvf ${HPSSDIR}/${yy}${mm}${dd}${hh}/${tarball} ./${CDUMP}.${yy}${mm}${dd}/${hh}/${file}
+        mv ${EXTRACT_DIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${file} ${ROTDIR}/${CDUMP}.${yy}${mm}${dd}/${hh}/${COMPONENT}/${file}
+
+      else # Production source
+
+        cd $ROTDIR
+        export tarball="com_gfs_prod_gfs.${yy}${mm}${dd}_${hh}.gfs_pgrb2.tar"
+        htar -xvf ${PRODHPSSDIR}/rh${yy}/${yy}${mm}/${yy}${mm}${dd}/${tarball} ./${CDUMP}.${yy}${mm}${dd}/${hh}/atmos/${file}
+
+      fi # RETRO vs production
+
+    fi # Version check
+  done # grid loop
+fi # v14-v16 pgrb anl file pull
 
 ##########################################
 # Remove the Temporary working directory

From c5456b5c6373254769d3e655be4deb34d7805495 Mon Sep 17 00:00:00 2001
From: Raffaele Montuoro <raffaele.montuoro@noaa.gov>
Date: Sun, 13 Jun 2021 02:31:59 +0000
Subject: [PATCH 091/125] Load aerosol-specific modulefiles only for the
 forecast task.

---
 ush/load_fv3gfs_modules.sh             | 19 ++++++++++++++++++-
 workflow/layout/free_forecast_gfs.yaml |  4 ++--
 2 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/ush/load_fv3gfs_modules.sh b/ush/load_fv3gfs_modules.sh
index 0a5f69e983b..d9ae4f0e9ad 100755
--- a/ush/load_fv3gfs_modules.sh
+++ b/ush/load_fv3gfs_modules.sh
@@ -2,6 +2,10 @@
 
 ###############################################################
 # Setup runtime environment by loading modules
+#
+# This script accepts an optional argument for the task type.
+# Acceptable types are: exclusive, forecast, service, shared
+#
 ###############################################################
 
 ulimit_s=$( ulimit -S -s )
@@ -10,6 +14,19 @@ ulimit_s=$( ulimit -S -s )
 source "$HOMEgfs/modulefiles/module-setup.sh.inc" 
 source "$HOMEgfs/ush/get_platform.sh"
 
+tasktype=
+if [ $# -eq 1 ] ; then
+  case "${1}" in
+    exclusive|forecast|service|shared)
+      tasktype="${1}"
+      ;;
+    *)
+      echo "Unsupported task type"
+      exit 1
+      ;;
+  esac
+fi
+
 platform=$( get_platform )
 
 moduledir=
@@ -18,7 +35,7 @@ case "${platform}" in
   hera)
     moduledir="$HOMEgfs/sorc/ufs_coupled.fd/modulefiles"
     target=${platform}.intel
-    if [[ -r ${moduledir}/ufs_${target} ]] ; then
+    if [[ "${tasktype}" = "forecast" && -r ${moduledir}/ufs_${target} ]] ; then
       modulelist=ufs_${target}
       if [[ "$( grep UFS_GOCART ${HOMEgfs}/sorc/ufs_coupled.fd/build/CMakeCache.txt 2>/dev/null | cut -d= -f2 )" = "ON" ]] ; then
         # add aerosols modulefile
diff --git a/workflow/layout/free_forecast_gfs.yaml b/workflow/layout/free_forecast_gfs.yaml
index 2d8f64da654..f45b752263a 100644
--- a/workflow/layout/free_forecast_gfs.yaml
+++ b/workflow/layout/free_forecast_gfs.yaml
@@ -69,7 +69,7 @@ suite: !Cycle
 
     forecast: !Family
       medcold: !Task
-        <<: *exclusive_task_template
+        <<: *forecast_task_template
         Disable: !calc doc.settings.cplflx == '.false.' or doc.settings.onestep == '.true.'
         Trigger: !FirstTrue
            - when: !calc doc.settings.cplwav=='.true.'
@@ -80,7 +80,7 @@ suite: !Cycle
         J_JOB: JGLOBAL_FORECAST_MEDCOLD
 
       highres: !Task
-        <<: *exclusive_task_template
+        <<: *forecast_task_template
         Trigger: !FirstTrue
            - when: !calc doc.settings.cplflx=='.true.' and doc.settings.onestep=='.false.'
              do: !Depend medcold

From bfe150ae29d18c5ddf3353bd9ac773accdf82613 Mon Sep 17 00:00:00 2001
From: Raffaele Montuoro <raffaele.montuoro@noaa.gov>
Date: Mon, 14 Jun 2021 18:57:29 +0000
Subject: [PATCH 092/125] Replace legacy 'cplgocart' flag with 'cplchem'.

---
 workflow/config/fcst.yaml              | 2 +-
 workflow/layout/free_forecast_gfs.yaml | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/workflow/config/fcst.yaml b/workflow/config/fcst.yaml
index bd46c040b73..f424cfc7c11 100644
--- a/workflow/config/fcst.yaml
+++ b/workflow/config/fcst.yaml
@@ -40,7 +40,7 @@ config_fcst:
     - otherwise: !calc doc.partition_common.resources.run_gfsfcst
 
   restart_interval_gfs: !FirstTrue
-    - when: !calc doc.settings.cplgocart == ".false."
+    - when: !calc doc.settings.cplchem == ".false."
       do: !calc doc.fv3_gfs_settings.restart_interval
     - otherwise: !FirstTrue
       - when: !calc doc.fv3_gfs_settings.restart_interval == "0"
diff --git a/workflow/layout/free_forecast_gfs.yaml b/workflow/layout/free_forecast_gfs.yaml
index d6046a95625..624202e7fc1 100644
--- a/workflow/layout/free_forecast_gfs.yaml
+++ b/workflow/layout/free_forecast_gfs.yaml
@@ -47,7 +47,7 @@ suite: !Cycle
     jgfs_init_chem: !Task
       <<: *exclusive_task_template
       RUN: 'gfs'
-      Disable: !calc doc.settings.cplgocart != '.true.'
+      Disable: !calc doc.settings.cplchem != '.true.'
       Trigger: !Depend jgfs_emc_mom6ic & up.gfs.forecast.at('-' + tools.dt_to_HMS(doc.settings.STEP_GFS))
       resources: !calc partition.resources.run_init_chem
       J_JOB: rocoto/init_chem.sh
@@ -99,7 +99,7 @@ suite: !Cycle
              do: !Depend medcold
            - when: !calc doc.settings.cplwav=='.true.'
              do: !Depend up.prep
-           - when: !calc doc.settings.cplgocart=='.true.'
+           - when: !calc doc.settings.cplchem=='.true.'
              do: !Depend up.up.ics.jgfs_init_chem & up.up.ics.jgfs_emc_mom6ic
            - otherwise: !Depend up.up.ics.jgfs_emc_mom6ic
         resources_remap: !JobRequest [ { <<: *remap_resource_template } ]

From 88f0358b7f09713f11cdf9606c2b71a68def94fe Mon Sep 17 00:00:00 2001
From: Raffaele Montuoro <raffaele.montuoro@noaa.gov>
Date: Mon, 14 Jun 2021 18:58:16 +0000
Subject: [PATCH 093/125] Update tracer list.

---
 parm/chem/gocart_tracer.list | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/parm/chem/gocart_tracer.list b/parm/chem/gocart_tracer.list
index 871a41eedb7..0ac80b32ff4 100644
--- a/parm/chem/gocart_tracer.list
+++ b/parm/chem/gocart_tracer.list
@@ -1,9 +1,9 @@
 so2
-sulf
+so4
 dms
 msa
-pp25
-pp10
+pm25
+pm10
 bc1
 bc2
 oc1
@@ -17,4 +17,4 @@ seas1
 seas2
 seas3
 seas4
-seas5
\ No newline at end of file
+seas5

From 2c40cdcdece2c06b35e3d3d9c20dac61596c5bf7 Mon Sep 17 00:00:00 2001
From: Raffaele Montuoro <raffaele.montuoro@noaa.gov>
Date: Mon, 14 Jun 2021 22:45:22 +0000
Subject: [PATCH 094/125] Restore original path to modulefiles in
 module_base.hera.

---
 modulefiles/module_base.hera | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/modulefiles/module_base.hera b/modulefiles/module_base.hera
index aa910eb6dba..f42986068ca 100644
--- a/modulefiles/module_base.hera
+++ b/modulefiles/module_base.hera
@@ -4,8 +4,7 @@
 ##
 
 # Modules availble through hpc-stack
-# module use /scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack
-module use /scratch1/NCEPDEV/nems/Raffaele.Montuoro/dev/nasa/libs/hpc-stack/1.1.0-nasa/modulefiles/stack
+module use /scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack
 
 module load hpc/1.1.0
 module load hpc-intel/18.0.5.274

From 921838feb9f6fd3fc3c69579c677ffdd7175a103 Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Wed, 16 Jun 2021 13:18:35 +0000
Subject: [PATCH 095/125] Remove old compile command from build_fv3.sh

Refs: #178
---
 sorc/build_fv3.sh | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/sorc/build_fv3.sh b/sorc/build_fv3.sh
index 376967f3740..67c28e02ae8 100755
--- a/sorc/build_fv3.sh
+++ b/sorc/build_fv3.sh
@@ -24,8 +24,5 @@ FV3=$( pwd -P )/FV3
 cd tests/
 if [ ! -d ../NEMS/exe ]; then mkdir ../NEMS/exe ; fi
 
-#./compile.sh "$FV3" "$target" "WW3=Y 32BIT=Y" 1
-#mv -f fv3_1.exe ../NEMS/exe/global_fv3gfs.x
-
 ./compile.sh "$target" "APP=ATM CCPP=Y 32BIT=Y SUITES=FV3_GFS_v16,FV3_GFS_v16_RRTMGP" 2 YES NO
 mv -f fv3_2.exe ../NEMS/exe/global_fv3gfs.x

From fba8cef5b870d46df1c665d3e9cabb23904ad3cf Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Wed, 16 Jun 2021 13:57:14 +0000
Subject: [PATCH 096/125] Remove wave restart pull in getic script

Refs: #178
---
 jobs/rocoto/getic.sh | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/jobs/rocoto/getic.sh b/jobs/rocoto/getic.sh
index 5c8e3353b9f..f08a10a8dd4 100755
--- a/jobs/rocoto/getic.sh
+++ b/jobs/rocoto/getic.sh
@@ -70,13 +70,6 @@ if [[ $gfs_ver = "v16" && $EXP_WARM_START = ".true." && $CASE = $OPS_RES ]]; the
     status=$?
     [[ $status -ne 0 ]] && exit $status
 
-    if [ $DO_WAVE = "YES" ]; then
-      # Pull prior cycle wave restart files
-      htar -xvf ${HPSSDIR}/${GDATE}/gdaswave_restart.tar
-      status=$?
-      [[ $status -ne 0 ]] && exit $status
-    fi
-
     # Pull current cycle restart files
     htar -xvf ${HPSSDIR}/${CDATE}/gfs_restarta.tar
     status=$?

From 7e84ea45bc6207572695f13bae2278ed10d659b9 Mon Sep 17 00:00:00 2001
From: Raffaele Montuoro <raffaele.montuoro@noaa.gov>
Date: Tue, 22 Jun 2021 17:01:41 +0000
Subject: [PATCH 097/125] Update aerosol_firex_forecast test case.

---
 workflow/cases/aerosol_firex_forecast.yaml | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/workflow/cases/aerosol_firex_forecast.yaml b/workflow/cases/aerosol_firex_forecast.yaml
index 0b240126482..057b9c86159 100644
--- a/workflow/cases/aerosol_firex_forecast.yaml
+++ b/workflow/cases/aerosol_firex_forecast.yaml
@@ -4,14 +4,12 @@
 case:
   places:
     workflow_file: layout/free_forecast_gfs.yaml
-    BASE_CPLIC: /scratch1/NCEPDEV/nems/Raffaele.Montuoro/data
-    OROFIX:     /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/input-data-20210528/FV3_input_data384/INPUT
-    GRDFIX:     /scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/input-data-20210528/FV3_input_data384/INPUT
+    BASE_CPLIC: /scratch1/NCEPDEV/nems/Raffaele.Montuoro/data/ICS
 
   settings: 
     SDATE: 2019-07-01t00:00:00
-    EDATE: 2019-07-15t00:00:00
-    STEP_GFS: !timedelta "24:00:00"
+    EDATE: 2019-07-02t00:00:00
+    STEP_GFS: !timedelta "06:00:00"
     STEP_DA: !timedelta "00:00:00"
     max_job_tries: 1
 
@@ -34,7 +32,7 @@ case:
   output_settings:
     FHOUT_GFS: 6
     FHMIN_GFS: 0
-    FHMAX_GFS: 24
+    FHMAX_GFS: 6
     FHMAX_HF_GFS: 0
     FHOUT_HF_GFS: -1
 
@@ -45,7 +43,7 @@ case:
     layout:
        x: 6
        y: 8
-       nth: 1
+       nth: 2
        WGRP: 1
        WGRP_NTASKS: 24
        WRTIOBUF: "32M"

From 165c2b716cfe29ba3fb0009e9b39b114316b8e4b Mon Sep 17 00:00:00 2001
From: Jessica Meixner <Jessica.Meixner@noaa.gov>
Date: Wed, 23 Jun 2021 13:41:40 +0000
Subject: [PATCH 098/125] update wave ww3_multi.inp file Issue #325

---
 scripts/exgfs_wave_prep.sh | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/scripts/exgfs_wave_prep.sh b/scripts/exgfs_wave_prep.sh
index 6cae1b2539d..f0533b5f8a3 100755
--- a/scripts/exgfs_wave_prep.sh
+++ b/scripts/exgfs_wave_prep.sh
@@ -888,7 +888,7 @@
   case ${WW3ATMINP} in
     'YES' )
       NFGRIDS=`expr $NFGRIDS + 1`
-      WINDLINE="  '$WAVEWND_FID'  F F T F F F F"
+      WINDLINE="  '$WAVEWND_FID'  F F T F F F F F F"
       WINDFLAG="$WAVEWND_FID"
     ;;
     'CPL' )
@@ -896,7 +896,7 @@
       if [ ${waveesmfGRD} ]
       then
         WINDFLAG="CPL:${waveesmfGRD}"
-        CPLILINE="  '${waveesmfGRD}' F F T F F F F"
+        CPLILINE="  '${waveesmfGRD}' F F T F F F F F F"
       else 
         WINDFLAG="CPL:native"
       fi
@@ -907,7 +907,7 @@
     'YES' ) 
       NFGRIDS=`expr $NFGRIDS + 1`
       ICEIFLAG='T'
-      ICELINE="  '$WAVEICE_FID'  F F F T F F F"
+      ICELINE="  '$WAVEICE_FID'  F F F T F F F F F"
       ICEFLAG="$WAVEICE_FID"
     ;;
     'CPL' )
@@ -915,7 +915,7 @@
       if [ ${waveesmfGRD} ]
       then
         ICEFLAG="CPL:${waveesmfGRD}"
-        CPLILINE="  '${waveesmfGRD}' F F ${WNDIFLAG} T F F F"
+        CPLILINE="  '${waveesmfGRD}' F F ${WNDIFLAG} T F F F F F"
       else 
         ICEFLAG="CPL:native"
       fi
@@ -926,10 +926,10 @@
     'YES' ) 
       if [ "$WAVECUR_FID" != "$WAVEICE_FID" ]; then
         NFGRIDS=`expr $NFGRIDS + 1`
-        CURRLINE="  '$WAVECUR_FID'  F T F F F F F"
+        CURRLINE="  '$WAVECUR_FID'  F T F F F F F F F"
         CURRFLAG="$WAVECUR_FID"
       else # cur fields share the same grid as ice grid
-        ICELINE="  '$WAVEICE_FID'  F T F ${ICEIFLAG} F F F"
+        ICELINE="  '$WAVEICE_FID'  F T F ${ICEIFLAG} F F F F F"
         CURRFLAG="$WAVEICE_FID"
       fi
     ;;
@@ -938,7 +938,7 @@
       if [ ${waveesmfGRD} ]
       then
         CURRFLAG="CPL:${waveesmfGRD}"
-        CPLILINE="  '${waveesmfGRD}' F T ${WNDIFLAG} ${ICEFLAG} F F F"
+        CPLILINE="  '${waveesmfGRD}' F T ${WNDIFLAG} ${ICEFLAG} F F F F F"
       else 
         CURRFLAG="CPL:native"
       fi
@@ -957,7 +957,7 @@
     NMGRIDS=`expr $NMGRIDS + 1`
     gridN=`echo $waveGRDN | awk -v i=$GRDN '{print $i}'`
     gridG=`echo $waveGRDG | awk -v i=$GRDN '{print $i}'`
-    gline="${gline}'${grid}'  'no' 'CURRFLAG' 'WINDFLAG' 'ICEFLAG'  'no' 'no' 'no'  ${gridN} ${gridG}  0.00 1.00  F\n"
+    gline="${gline}'${grid}'  'no' 'CURRFLAG' 'WINDFLAG' 'ICEFLAG'  'no' 'no' 'no' 'no' 'no'  ${gridN} ${gridG}  0.00 1.00  F\n"
   done
   gline="${gline}\$"
   echo $gline

From 6f4a3f26656c9725ef8a2d3eb2dd5b69c59cb157 Mon Sep 17 00:00:00 2001
From: "Walter.Kolczynski" <Walter.Kolczynski@noaa.gov>
Date: Tue, 22 Jun 2021 21:44:53 +0000
Subject: [PATCH 099/125] Update UFS model hash

Updates the UFS model to the current tip of develop.

Refs: #324
---
 sorc/checkout.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sorc/checkout.sh b/sorc/checkout.sh
index f13d99a833f..6be7d40aeca 100755
--- a/sorc/checkout.sh
+++ b/sorc/checkout.sh
@@ -52,7 +52,7 @@ else
     else
       git clone https://github.com/ufs-community/ufs-weather-model ufs_coupled.fd >> ${topdir}/checkout-ufs_coupled.log 2>&1
       cd ufs_coupled.fd
-      git checkout 3e46f5b7050e18884a0bed13691823ad88d443c3
+      git checkout aa2ab44
     fi
     git submodule update --init --recursive
     cd ${topdir} 

From d0de272a3a6f5f19f0c821e5eb299572fc1bf8c8 Mon Sep 17 00:00:00 2001
From: "Walter.Kolczynski" <Walter.Kolczynski@noaa.gov>
Date: Tue, 22 Jun 2021 21:46:06 +0000
Subject: [PATCH 100/125] Update MOM_input with landmask change option

Adds the ALLOW_LANDMASK_CHANGES option to the MOM6 input templates
and allows the setting via MOM6_ALLOW_LANDMASK_CHANGES (default
False).

Refs: #324
---
 parm/mom6/MOM_input_template_025 | 2 ++
 parm/mom6/MOM_input_template_050 | 2 ++
 parm/mom6/MOM_input_template_100 | 2 ++
 ush/parsing_namelists_MOM6.sh    | 6 ++++--
 4 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/parm/mom6/MOM_input_template_025 b/parm/mom6/MOM_input_template_025
index aa1da780298..1340db5b66b 100644
--- a/parm/mom6/MOM_input_template_025
+++ b/parm/mom6/MOM_input_template_025
@@ -148,6 +148,8 @@ TOPO_FILE = "ocean_topog.nc"    ! default = "topog.nc"
                                 ! The file from which the bathymetry is read.
 TOPO_EDITS_FILE = "All_edits.nc" ! default = ""
                                 ! The file from which to read a list of i,j,z topography overrides.
+ALLOW_LANDMASK_CHANGES = MOM6_ALLOW_LANDMASK_CHANGES   ! default = "False"
+                                ! If true, allow topography overrides to change ocean points to land
 MAXIMUM_DEPTH = 6500.0          !   [m]
                                 ! The maximum depth of the ocean.
 MINIMUM_DEPTH = 9.5             !   [m] default = 0.0
diff --git a/parm/mom6/MOM_input_template_050 b/parm/mom6/MOM_input_template_050
index 46780dfc822..ceff6e98d43 100644
--- a/parm/mom6/MOM_input_template_050
+++ b/parm/mom6/MOM_input_template_050
@@ -147,6 +147,8 @@ TOPO_CONFIG = "file"            !
                                 !     USER - call a user modified routine.
 TOPO_FILE = "ocean_topog.nc"    ! default = "topog.nc"
                                 ! The file from which the bathymetry is read.
+ALLOW_LANDMASK_CHANGES = MOM6_ALLOW_LANDMASK_CHANGES   ! default = "False"
+                                ! If true, allow topography overrides to change ocean points to land
 MAXIMUM_DEPTH = 6500.0          !   [m]
                                 ! The maximum depth of the ocean.
 MINIMUM_DEPTH = 9.5             !   [m] default = 0.0
diff --git a/parm/mom6/MOM_input_template_100 b/parm/mom6/MOM_input_template_100
index d5af4cccd21..24ea3c7c47a 100644
--- a/parm/mom6/MOM_input_template_100
+++ b/parm/mom6/MOM_input_template_100
@@ -140,6 +140,8 @@ TOPO_CONFIG = "file"            !
                                 !     USER - call a user modified routine.
 TOPO_EDITS_FILE = "topo_edits_011818.nc" ! default = ""
                                 ! The file from which to read a list of i,j,z topography overrides.
+ALLOW_LANDMASK_CHANGES = MOM6_ALLOW_LANDMASK_CHANGES   ! default = "False"
+                                ! If true, allow topography overrides to change ocean points to land
 MAXIMUM_DEPTH = 6500.0          !   [m]
                                 ! The maximum depth of the ocean.
 MINIMUM_DEPTH = 9.5             !   [m] default = 0.0
diff --git a/ush/parsing_namelists_MOM6.sh b/ush/parsing_namelists_MOM6.sh
index 2af5e1d457d..e7ab58a0ae0 100644
--- a/ush/parsing_namelists_MOM6.sh
+++ b/ush/parsing_namelists_MOM6.sh
@@ -9,8 +9,9 @@ MOM_INPUT=MOM_input_template_$OCNRES
 #TODO: Make these variables configurable 
 
 #Set to False for restart reproducibility   
-MOM6_REPRO_LA='True'
-MOM6_THERMO_SPAN='False'
+MOM6_REPRO_LA=${MOM6_REPRO_LA:-'True'}
+MOM6_THERMO_SPAN=${MOM6_THERMO_SPAN:-'False'}
+MOM6_ALLOW_LANDMASK_CHANGES=${MOM6_ALLOW_LANDMASK_CHANGES:-'False'}
 
 if [ $cplwav = ".true." ] ; then
   MOM6_USE_WAVES='True'
@@ -74,6 +75,7 @@ sed -e "s/DT_THERM_MOM6/$DT_THERM_MOM6/g" \
     -e "s/MOM6_THERMO_SPAN/$MOM6_THERMO_SPAN/g" \
     -e "s/MOM6_REPRO_LA/$MOM6_REPRO_LA/g" \
     -e "s/MOM6_USE_WAVES/$MOM6_USE_WAVES/g" \
+    -e "s/MOM6_ALLOW_LANDMASK_CHANGES/$MOM6_ALLOW_LANDMASK_CHANGES/g" \
     -e "s/NX_GLB/$NX_GLB/g" \
     -e "s/NY_GLB/$NY_GLB/g" \
     -e "s/CHLCLIM/$CHLCLIM/g" $DATA/INPUT/MOM_input_template_$OCNRES > $DATA/INPUT/MOM_input

From 56d559757fc9257db81d0a6e07e1a423f2ce146d Mon Sep 17 00:00:00 2001
From: "Walter.Kolczynski" <Walter.Kolczynski@noaa.gov>
Date: Tue, 22 Jun 2021 22:26:54 +0000
Subject: [PATCH 101/125] Update CICE inputs for new model version

The newest version of CICE replaces istep0 with the normal month and day,
so the namelist is updated to reflect that.

ktherm and tfrz_option are updated to allow alternate settings, following
similar changes to the UFS test harness. The default values are the
previous settings.

Settings for tr_FY, sw_redist, f_mlt_onset, and f_frx_onset are updated
to new values in the UFS test harness.

Additionally, many namelist options have been removed following the same
deletions in the UFS test harness.

Refs: #324
---
 ush/forecast_postdet.sh       |   6 +-
 ush/parsing_namelists_CICE.sh | 261 ++--------------------------------
 2 files changed, 19 insertions(+), 248 deletions(-)

diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh
index 7bdde50324f..7abd83a92c9 100755
--- a/ush/forecast_postdet.sh
+++ b/ush/forecast_postdet.sh
@@ -729,9 +729,11 @@ CICE_postdet() {
 	echo "SUB ${FUNCNAME[0]}: CICE after run type determination"
 
 	year=$(echo $CDATE|cut -c 1-4)
+	month=$(echo $CDATE|cut -c 5-6)
+	day=$(echo $CDATE|cut -c 7-8)
+
 	stepsperhr=$((3600/$ICETIM))
 	nhours=$($NHOUR $CDATE ${year}010100)
-	istep0=$((nhours*stepsperhr))
 	steps=$((nhours*stepsperhr))
 	npt=$((FHMAX*$stepsperhr))      # Need this in order for dump_last to work
 
@@ -746,6 +748,8 @@ CICE_postdet() {
 	cice_hist_avg=${cice_hist_avg:-".true."}
 
 	FRAZIL_FWSALT=${FRAZIL_FWSALT:-".true."}
+	ktherm=${ktherm:-1}
+	tfrz_option=${tfrz_option:-"linear_salt"}
 	tr_pond_lvl=${tr_pond_lvl:-".true."} # Use level melt ponds tr_pond_lvl=true
 
 	# restart_pond_lvl (if tr_pond_lvl=true):
diff --git a/ush/parsing_namelists_CICE.sh b/ush/parsing_namelists_CICE.sh
index 60baa023e39..d19f0c41722 100644
--- a/ush/parsing_namelists_CICE.sh
+++ b/ush/parsing_namelists_CICE.sh
@@ -14,12 +14,13 @@ cat > ice_in <<eof
    days_per_year  = 365
    use_leap_years = .true.
    year_init      = $year
-   istep0         = $steps
+   month_init     = $month
+   day_init       = $day
    dt             = $ICETIM
    npt            = $npt
    ndtd           = 1
    runtype        = '$cmeps_run_type' 
-   runid          = 'cpcice' 
+   runid          = 'cpcice'
    ice_ic         = '$iceic'
    restart        = .true.
    restart_ext    = .false.
@@ -44,7 +45,6 @@ cat > ice_in <<eof
    lonpnt(1)      =   0.
    latpnt(2)      = -65.
    lonpnt(2)      = -45.
-   dbug           = .false.
    histfreq       = 'm','d','h','x','x'
    histfreq_n     =  0 , 0 , 6 , 1 , 1
    hist_avg       = $cice_hist_avg
@@ -61,7 +61,6 @@ cat > ice_in <<eof
    grid_type    = 'tripole'
    grid_file    = '$ice_grid_file'
    kmt_file     = '$ice_kmt_file'
-   use_bathymetry = .false.
    kcatbound    = 0
    ncat         = 5
    nfsd         = 1
@@ -72,17 +71,9 @@ cat > ice_in <<eof
 /
 
 &tracer_nml
-   n_aero       = 0
-   n_zaero      = 0
-   n_algae      = 0
-   n_doc        = 0
-   n_dic        = 0
-   n_don        = 0
-   n_fed        = 0
-   n_fep        = 0
    tr_iage      = .true.
    restart_age  = .false.
-   tr_FY        = .true.
+   tr_FY        = .false.
    restart_FY   = .false.
    tr_lvl       = .true.
    restart_lvl  = .false.
@@ -100,7 +91,7 @@ cat > ice_in <<eof
 
 &thermo_nml
    kitd              = 1
-   ktherm            = 1
+   ktherm            = $ktherm
    conduct           = 'MU71'
    a_rapid_mode      =  0.5e-3
    Rac_rapid_mode    =    10.0
@@ -108,7 +99,6 @@ cat > ice_in <<eof
    dSdt_slow_mode    = -5.0e-8
    phi_c_slow_mode   =    0.05
    phi_i_mushy       =    0.85
-   sw_redist         = .true.
 /
 
 &dynamics_nml
@@ -127,11 +117,10 @@ cat > ice_in <<eof
    Cf              = 17.
    Ktens           = 0.
    e_ratio         = 2.
-   basalstress     = .false.
-   k1              = 8.
    coriolis        = 'latitude'
    kridge          = 1
    ktransport      = 1
+   sw_redist       = .true.
 /
 
 &shortwave_nml
@@ -173,30 +162,8 @@ cat > ice_in <<eof
    fbot_xfer_type  = 'constant'
    update_ocn_f    = $FRAZIL_FWSALT
    l_mpond_fresh   = .false.
-   tfrz_option     = 'linear_salt'
+   tfrz_option     = $tfrz_option
    restart_coszen  = .true.
-   oceanmixed_ice  = .false.
-   wave_spec_type  = 'none'
-   wave_spec_file  = 'unknown_wave_spec_file'
-   nfreq           = 25
-   restore_ice     = .false.
-   restore_ocn     = .false.
-   trestore        =  90
-   precip_units    = 'mm_per_month'
-   default_season  = 'winter'
-   atm_data_type   = 'ncar'
-   ocn_data_type   = 'default'
-   bgc_data_type   = 'default'
-   fe_data_type    = 'default'
-   ice_data_type   = 'default'
-   fyear_init      = 1997
-   ycycle          = 1
-   atm_data_format = 'nc'
-   atm_data_dir    = './INPUT/gx3_forcing_fields.nc'
-   bgc_data_dir    = 'unknown_bgc_data_dir'
-   ocn_data_format = 'bin'
-   ocn_data_dir    = '/unknown_ocn_data_dir'
-   oceanmixed_file = 'unknown_oceanmixed_file'
 /
 
 &domain_nml
@@ -217,152 +184,6 @@ cat > ice_in <<eof
 /
 
 &zbgc_nml
-   tr_brine        = .false.
-   restart_hbrine  = .false.
-   tr_zaero        = .false.
-   modal_aero      = .false.
-   skl_bgc         = .false.
-   z_tracers       = .false.
-   dEdd_algae      = .false.
-   solve_zbgc      = .false.
-   bgc_flux_type   = 'Jin2006'
-   restore_bgc     = .false.
-   restart_bgc     = .false.
-   scale_bgc       = .false.
-   solve_zsal      = .false.
-   restart_zsal    = .false.
-   tr_bgc_Nit      = .true.
-   tr_bgc_C        = .true.
-   tr_bgc_chl      = .false.
-   tr_bgc_Am       = .true.
-   tr_bgc_Sil      = .true.
-   tr_bgc_DMS      = .false.
-   tr_bgc_PON      = .true.
-   tr_bgc_hum      = .true.
-   tr_bgc_DON      = .false.
-   tr_bgc_Fe       = .true. 
-   grid_o          = 0.006
-   grid_o_t        = 0.006
-   l_sk            = 0.024
-   grid_oS         = 0.0
-   l_skS           = 0.028
-   phi_snow        = -0.3
-   initbio_frac    = 0.8
-   frazil_scav     = 0.8  
-   ratio_Si2N_diatoms = 1.8                         
-   ratio_Si2N_sp      = 0.0
-   ratio_Si2N_phaeo   = 0.0
-   ratio_S2N_diatoms  = 0.03  
-   ratio_S2N_sp       = 0.03 
-   ratio_S2N_phaeo    = 0.03
-   ratio_Fe2C_diatoms = 0.0033
-   ratio_Fe2C_sp      = 0.0033
-   ratio_Fe2C_phaeo   = 0.1
-   ratio_Fe2N_diatoms = 0.023 
-   ratio_Fe2N_sp      = 0.023
-   ratio_Fe2N_phaeo   = 0.7
-   ratio_Fe2DON       = 0.023
-   ratio_Fe2DOC_s     = 0.1
-   ratio_Fe2DOC_l     = 0.033
-   fr_resp            = 0.05
-   tau_min            = 5200.0
-   tau_max            = 173000.0
-   algal_vel          = 0.0000000111
-   R_dFe2dust         = 0.035
-   dustFe_sol         = 0.005
-   chlabs_diatoms     = 0.03
-   chlabs_sp          = 0.01
-   chlabs_phaeo       = 0.05
-   alpha2max_low_diatoms = 0.8
-   alpha2max_low_sp      = 0.67
-   alpha2max_low_phaeo   = 0.67
-   beta2max_diatoms   = 0.018
-   beta2max_sp        = 0.0025
-   beta2max_phaeo     = 0.01
-   mu_max_diatoms     = 1.44
-   mu_max_sp          = 0.851
-   mu_max_phaeo       = 0.851
-   grow_Tdep_diatoms  = 0.06
-   grow_Tdep_sp       = 0.06
-   grow_Tdep_phaeo    = 0.06
-   fr_graze_diatoms   = 0.0
-   fr_graze_sp        = 0.1
-   fr_graze_phaeo     = 0.1
-   mort_pre_diatoms   = 0.007
-   mort_pre_sp        = 0.007
-   mort_pre_phaeo     = 0.007
-   mort_Tdep_diatoms  = 0.03
-   mort_Tdep_sp       = 0.03
-   mort_Tdep_phaeo    = 0.03
-   k_exude_diatoms    = 0.0
-   k_exude_sp         = 0.0
-   k_exude_phaeo      = 0.0
-   K_Nit_diatoms      = 1.0
-   K_Nit_sp           = 1.0
-   K_Nit_phaeo        = 1.0
-   K_Am_diatoms       = 0.3
-   K_Am_sp            = 0.3
-   K_Am_phaeo         = 0.3
-   K_Sil_diatoms      = 4.0
-   K_Sil_sp           = 0.0
-   K_Sil_phaeo        = 0.0
-   K_Fe_diatoms       = 1.0
-   K_Fe_sp            = 0.2
-   K_Fe_phaeo         = 0.1
-   f_don_protein      = 0.6
-   kn_bac_protein     = 0.03
-   f_don_Am_protein   = 0.25
-   f_doc_s            = 0.4
-   f_doc_l            = 0.4
-   f_exude_s          = 1.0
-   f_exude_l          = 1.0
-   k_bac_s            = 0.03
-   k_bac_l            = 0.03
-   T_max              = 0.0
-   fsal               = 1.0
-   op_dep_min         = 0.1
-   fr_graze_s         = 0.5
-   fr_graze_e         = 0.5
-   fr_mort2min        = 0.5
-   fr_dFe             = 0.3
-   k_nitrif           = 0.0
-   t_iron_conv        = 3065.0
-   max_loss           = 0.9
-   max_dfe_doc1       = 0.2
-   fr_resp_s          = 0.75
-   y_sk_DMS           = 0.5
-   t_sk_conv          = 3.0
-   t_sk_ox            = 10.0
-   algaltype_diatoms  = 0.0
-   algaltype_sp       = 0.5
-   algaltype_phaeo    = 0.5
-   nitratetype        = -1.0
-   ammoniumtype       = 1.0
-   silicatetype       = -1.0
-   dmspptype          = 0.5
-   dmspdtype          = -1.0
-   humtype            = 1.0
-   doctype_s          = 0.5
-   doctype_l          = 0.5
-   dontype_protein    = 0.5
-   fedtype_1          = 0.5
-   feptype_1          = 0.5
-   zaerotype_bc1      = 1.0
-   zaerotype_bc2      = 1.0
-   zaerotype_dust1    = 1.0
-   zaerotype_dust2    = 1.0
-   zaerotype_dust3    = 1.0
-   zaerotype_dust4    = 1.0
-   ratio_C2N_diatoms  = 7.0
-   ratio_C2N_sp       = 7.0
-   ratio_C2N_phaeo    = 7.0
-   ratio_chl2N_diatoms= 2.1
-   ratio_chl2N_sp     = 1.1
-   ratio_chl2N_phaeo  = 0.84
-   F_abs_chl_diatoms  = 2.0
-   F_abs_chl_sp       = 4.0
-   F_abs_chl_phaeo    = 5.0
-   ratio_C2N_proteins = 7.0
 /
 
 &icefields_nml
@@ -454,8 +275,8 @@ cat > ice_in <<eof
    f_fsurf_ai      = 'x'
    f_fcondtop_ai   = 'x'
    f_fmeltt_ai     = 'x' 
-   f_strairx       = 'mdhxx' 
-   f_strairy       = 'mdhxx' 
+   f_strairx       = 'mdhxx'
+   f_strairy       = 'mdhxx'
    f_strtltx       = 'x' 
    f_strtlty       = 'x' 
    f_strcorx       = 'x' 
@@ -472,14 +293,14 @@ cat > ice_in <<eof
    f_sig1          = 'x' 
    f_sig2          = 'x'
    f_sigP          = 'x' 
-   f_dvidtt        = 'mdhxx' 
-   f_dvidtd        = 'mdhxx' 
+   f_dvidtt        = 'mdhxx'
+   f_dvidtd        = 'mdhxx'
    f_daidtt        = 'mdhxx'
-   f_daidtd        = 'mdhxx' 
+   f_daidtd        = 'mdhxx'
    f_dagedtt       = 'x'
    f_dagedtd       = 'x'
-   f_mlt_onset     = 'mdhxx'
-   f_frz_onset     = 'mdhxx'
+   f_mlt_onset     = 'x'
+   f_frz_onset     = 'x'
    f_hisnap        = 'x'
    f_aisnap        = 'x'
    f_trsig         = 'x'
@@ -538,44 +359,6 @@ cat > ice_in <<eof
 /
 
 &icefields_bgc_nml
-   f_faero_atm    = 'x'
-   f_faero_ocn    = 'x'
-   f_aero         = 'x'
-   f_fbio         = 'x'
-   f_fbio_ai      = 'x'
-   f_zaero        = 'x'
-   f_bgc_S        = 'x'
-   f_bgc_N        = 'x'
-   f_bgc_C        = 'x'
-   f_bgc_DOC      = 'x'
-   f_bgc_DIC      = 'x'
-   f_bgc_chl      = 'x'
-   f_bgc_Nit      = 'x'
-   f_bgc_Am       = 'x'
-   f_bgc_Sil      = 'x'
-   f_bgc_DMSPp    = 'x'
-   f_bgc_DMSPd    = 'x'
-   f_bgc_DMS      = 'x'
-   f_bgc_DON      = 'x'
-   f_bgc_Fe       = 'x'
-   f_bgc_hum      = 'x'
-   f_bgc_PON      = 'x'
-   f_bgc_ml       = 'x'
-   f_upNO         = 'x'
-   f_upNH         = 'x'
-   f_bTin         = 'x'
-   f_bphi         = 'x'
-   f_iDi          = 'x'
-   f_iki          = 'x'
-   f_fbri         = 'x'
-   f_hbri         = 'x'
-   f_zfswin       = 'x'
-   f_bionet       = 'x'
-   f_biosnow      = 'x'
-   f_grownet      = 'x'
-   f_PPnet        = 'x'
-   f_algalpeak    = 'x'
-   f_zbgc_frac    = 'x'
 /
 
 &icefields_drag_nml
@@ -583,22 +366,6 @@ cat > ice_in <<eof
    f_Cdn_atm      = 'x'
    f_Cdn_ocn      = 'x'
 /
-
-&icefields_fsd_nml
-   f_fsdrad       = 'x'
-   f_fsdperim     = 'x'
-   f_afsd         = 'x'
-   f_afsdn        = 'x'
-   f_dafsd_newi   = 'x'
-   f_dafsd_latg   = 'x'
-   f_dafsd_latm   = 'x'
-   f_dafsd_wave   = 'x'
-   f_dafsd_weld   = 'x'
-   f_wave_sig_ht  = 'x'
-   f_aice_ww      = 'x'
-   f_diam_ww      = 'x'
-   f_hice_ww      = 'x'
-/
 eof
 
 } 

From 434d7c6038a42dd5d53ebdd7f19f957f5655016b Mon Sep 17 00:00:00 2001
From: "Walter.Kolczynski" <Walter.Kolczynski@noaa.gov>
Date: Tue, 22 Jun 2021 23:13:36 +0000
Subject: [PATCH 102/125] Update UFS build script

Some changes to the UFS require updates to the build script.

The modulefile organization has been updated to be
modulefiles/ufs_${target} rather than modulefiles/${target}/fv3, so
the module paths in the build script are updated correspondingly.

A cmake switch DAPP is now required in order to build UFS. Non-
aerosol coupled is using S2SW for the time being. This replaces
setting DS2S and DWW3 to YES.

The old v15 suite is removed from the CCPP suite list.

Refs: #324
---
 sorc/build_ufs_coupled.sh | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/sorc/build_ufs_coupled.sh b/sorc/build_ufs_coupled.sh
index e18e6fd3e88..e2176967178 100755
--- a/sorc/build_ufs_coupled.sh
+++ b/sorc/build_ufs_coupled.sh
@@ -13,15 +13,17 @@ if [ $target = hera ]; then target=hera.intel ; fi
 if [ $target = orion ]; then target=orion.intel ; fi
 if [ $target = stampede ]; then target=stampede.intel ; fi
 
-MOD_PATH=$cwd/ufs_coupled.fd/modulefiles/$target
+MOD_PATH=$cwd/ufs_coupled.fd/modulefiles
 
 module purge 
 module use $MOD_PATH 
-module load fv3 
+module load ufs_${target}
 cd ufs_coupled.fd/
 if [[ -d build ]]; then rm -Rf build; fi
 if [[ -d GOCART ]]; then
+  module load ufs_aerosols_${target}
+  CMAKE_FLAGS="-DAPP=ATMAERO" CCPP_SUITES="FV3_GFS_v16" ./build.sh
   ./build.sh
 else
-  CMAKE_FLAGS="-DS2S=ON -DWW3=ON" CCPP_SUITES="FV3_GFS_v15p2_coupled,FV3_GFS_v16_coupled,FV3_GFS_v16" ./build.sh
+  CMAKE_FLAGS="-DAPP=S2SW" CCPP_SUITES="FV3_GFS_v16_coupled,FV3_GFS_v16" ./build.sh
 fi

From e86db2317aab6a98d56d3ae4d63696f82ad765f7 Mon Sep 17 00:00:00 2001
From: "Walter.Kolczynski" <Walter.Kolczynski@noaa.gov>
Date: Tue, 22 Jun 2021 23:24:12 +0000
Subject: [PATCH 103/125] Update source of fd_nems.yaml

The location of the fd_nems.yaml file changed in the UFS repository,
so updated nems_configure to copy from the new location.

Refs: #324
---
 ush/nems_configure.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ush/nems_configure.sh b/ush/nems_configure.sh
index 8e446271458..402e8598d70 100755
--- a/ush/nems_configure.sh
+++ b/ush/nems_configure.sh
@@ -155,9 +155,9 @@ EOF
 
 echo "$(cat med_modelio.nml)"
 
-cp $HOMEgfs/sorc/ufs_coupled.fd/CMEPS-interface/CMEPS/mediator/fd_nems.yaml fd_nems.yaml
+cp $HOMEgfs/sorc/ufs_coupled.fd/tests/parm/fd_nems.yaml fd_nems.yaml
 
-fi 
+fi
 
 echo "SUB ${FUNCNAME[0]}: Nems configured for $confignamevarfornems"
 

From 12050b8edfebd20efaa7c35904e26dbf2f8243b3 Mon Sep 17 00:00:00 2001
From: "Walter.Kolczynski" <Walter.Kolczynski@noaa.gov>
Date: Wed, 23 Jun 2021 00:19:36 +0000
Subject: [PATCH 104/125] Update templates to set ocean mesh file

The latest version of UFS adds a setting to define the ocean mesh file,
so the nems.configure templates are updated to define them. They are
identical to the ice mesh file, so the MESHICE variable is renamed to
MESH_OCN_ICE and used for both. Additionally, the use_mommesh variable
is added and defaults to true.

Refs: #324
---
 ush/forecast_postdet.sh                       | 2 +-
 ush/nems.configure.cpld.IN                    | 4 +++-
 ush/nems.configure.cpld_wave.IN               | 4 +++-
 ush/nems.configure.medcold_atm_ocn_ice.IN     | 3 ++-
 ush/nems.configure.medcold_atm_ocn_ice_wav.IN | 3 ++-
 ush/nems_configure.sh                         | 5 ++++-
 workflow/config/fcst.yaml                     | 2 +-
 7 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh
index 7abd83a92c9..1a56c89754b 100755
--- a/ush/forecast_postdet.sh
+++ b/ush/forecast_postdet.sh
@@ -789,7 +789,7 @@ CICE_postdet() {
 	echo "Link CICE fixed files"
 	$NLN -sf $FIXcice/$ICERES/${ice_grid_file} $DATA/
 	$NLN -sf $FIXcice/$ICERES/${ice_kmt_file} $DATA/
-	$NLN -sf $FIXcice/$ICERES/$MESHICE $DATA/
+	$NLN -sf $FIXcice/$ICERES/$MESH_OCN_ICE $DATA/
 }
 
 CICE_nml() {
diff --git a/ush/nems.configure.cpld.IN b/ush/nems.configure.cpld.IN
index 08d40044c09..69b6c737f3a 100644
--- a/ush/nems.configure.cpld.IN
+++ b/ush/nems.configure.cpld.IN
@@ -31,6 +31,7 @@ OCN_attributes::
   DumpFields = @[DumpFields]
   ProfileMemory = false
   OverwriteSlice = true
+  mesh_ocn = @[MESH_OCN_ICE]
 ::
 
 # ICE #
@@ -41,7 +42,7 @@ ICE_attributes::
   DumpFields = @[DumpFields]
   ProfileMemory = false
   OverwriteSlice = true
-  mesh_ice = @[MESHICE]
+  mesh_ice = @[MESH_OCN_ICE]
   stop_n = @[RESTART_N]
   stop_option = nhours
   stop_ymd = -999
@@ -99,4 +100,5 @@ ALLCOMP_attributes::
       restart_ymd = -999
       dbug_flag = @[cap_dbug_flag]
       use_coldstart = @[use_coldstart]
+      use_mommesh = @[use_mommesh]
 ::
diff --git a/ush/nems.configure.cpld_wave.IN b/ush/nems.configure.cpld_wave.IN
index ee4443f2d90..e4d9eca31d2 100644
--- a/ush/nems.configure.cpld_wave.IN
+++ b/ush/nems.configure.cpld_wave.IN
@@ -31,6 +31,7 @@ OCN_attributes::
   DumpFields = @[DumpFields]
   ProfileMemory = false
   OverwriteSlice = true
+  mesh_ocn = @[MESH_OCN_ICE]
 ::
 
 # ICE #
@@ -41,7 +42,7 @@ ICE_attributes::
   DumpFields = @[DumpFields]
   ProfileMemory = false
   OverwriteSlice = true
-  mesh_ice = @[MESHICE]
+  mesh_ice = @[MESH_OCN_ICE]
   stop_n = @[RESTART_N]
   stop_option = nhours
   stop_ymd = -999
@@ -113,4 +114,5 @@ ALLCOMP_attributes::
       restart_ymd = -999
       dbug_flag = @[cap_dbug_flag]
       use_coldstart = @[use_coldstart]
+      use_mommesh = @[use_mommesh]
 ::
diff --git a/ush/nems.configure.medcold_atm_ocn_ice.IN b/ush/nems.configure.medcold_atm_ocn_ice.IN
index d54f6596a6f..fe4030274e5 100644
--- a/ush/nems.configure.medcold_atm_ocn_ice.IN
+++ b/ush/nems.configure.medcold_atm_ocn_ice.IN
@@ -32,6 +32,7 @@ OCN_attributes::
   ProfileMemory = false
   OverwriteSlice = true
   dbug_flag = 0  
+  mesh_ocn = @[MESH_OCN_ICE]
 ::
 
 # ICE #
@@ -42,7 +43,7 @@ ICE_attributes::
   DumpFields = @[DumpFields]
   ProfileMemory = false
   OverwriteSlice = true
-  mesh_ice = @[MESHICE]
+  mesh_ice = @[MESH_OCN_ICE]
   stop_n = @[FHMAX]
   stop_option = nhours
   stop_ymd = -999
diff --git a/ush/nems.configure.medcold_atm_ocn_ice_wav.IN b/ush/nems.configure.medcold_atm_ocn_ice_wav.IN
index 53cfc8c09d4..fa8ab4e4ff1 100644
--- a/ush/nems.configure.medcold_atm_ocn_ice_wav.IN
+++ b/ush/nems.configure.medcold_atm_ocn_ice_wav.IN
@@ -31,6 +31,7 @@ OCN_attributes::
   DumpFields = @[DumpFields]
   ProfileMemory = false
   OverwriteSlice = true
+  mesh_ocn = @[MESH_OCN_ICE]
 ::
 
 # ICE #
@@ -41,7 +42,7 @@ ICE_attributes::
   DumpFields = @[DumpFields]
   ProfileMemory = false
   OverwriteSlice = true
-  mesh_ice = @[MESHICE]
+  mesh_ice = @[MESH_OCN_ICE]
   stop_n = @[FHMAX]
   stop_option = nhours
   stop_ymd = -999
diff --git a/ush/nems_configure.sh b/ush/nems_configure.sh
index 402e8598d70..58ecef74a52 100755
--- a/ush/nems_configure.sh
+++ b/ush/nems_configure.sh
@@ -45,6 +45,8 @@ ICEPETS=${ICEPETS:-8}
 WAVPETS=${WAVPETS:-8}
 CHMPETS=${CHMPETS:-${ATMPETS}}
 
+USE_MOMMESH=${USE_MOMMESH:-"true"}
+
 rm -f $DATA/nems.configure
 
 med_petlist_bounds=${med_petlist_bounds:-"0 $(( $MEDPETS-1 ))"}
@@ -79,6 +81,7 @@ if [ $cplflx = .true. ]; then
 	sed -i -e "s;@\[coupling_interval_slow_sec\];$CPL_SLOW;g" tmp1
 	sed -i -e "s;@\[coupling_interval_fast_sec\];$CPL_FAST;g" tmp1
         sed -i -e "s;@\[RESTART_N\];$restart_interval_nems;g" tmp1
+  sed -i -e "s;@\[use_mommesh\];$USE_MOMMESH;g" tmp1
 fi
 if [ $cplwav = .true. ]; then
 	sed -i -e "s;@\[wav_model\];ww3;g" tmp1
@@ -87,7 +90,7 @@ fi
 if [ $cplice = .true. ]; then
 	sed -i -e "s;@\[ice_model\];$ICE_model;g" tmp1
 	sed -i -e "s;@\[ice_petlist_bounds\];$ice_petlist_bounds;g" tmp1
-        sed -i -e "s;@\[MESHICE\];$MESHICE;g" tmp1
+        sed -i -e "s;@\[MESH_OCN_ICE\];$MESH_OCN_ICE;g" tmp1
         sed -i -e "s;@\[FHMAX\];$FHMAX_GFS;g" tmp1
 fi
 if [ $cplchem = .true. -o $cplgocart = .true. ]; then
diff --git a/workflow/config/fcst.yaml b/workflow/config/fcst.yaml
index a89a9e2e84d..42ccca3c064 100644
--- a/workflow/config/fcst.yaml
+++ b/workflow/config/fcst.yaml
@@ -110,7 +110,7 @@ config_fcst:
     # CICE parameters
     export NX_GLB="{doc.ice_settings.NX_GLB}"
     export NY_GLB="{doc.ice_settings.NY_GLB}"
-    export MESHICE="mesh.mx{doc.ice_settings.ICERES}.nc"
+    export MESH_OCN_ICE="mesh.mx{doc.ice_settings.ICERES}.nc"
     
     if [[ "$machine" == "WCOSS_C" ]]; then
         export memory_fcst="1024M"

From d4910b5df107c3c311d365113202bbcfa2f29f22 Mon Sep 17 00:00:00 2001
From: "Walter.Kolczynski" <Walter.Kolczynski@noaa.gov>
Date: Wed, 23 Jun 2021 01:29:52 +0000
Subject: [PATCH 105/125] Update Ww3 prepost build

The WW3 pre and post components now need NetCDF to build, so NetCDF is
added back to the modulelist, and previously removed code in the build
script is restored.

Refs: #324
---
 modulefiles/modulefile.ww3.hera          | 2 ++
 modulefiles/modulefile.ww3.orion         | 2 ++
 modulefiles/modulefile.ww3.wcoss_dell_p3 | 2 ++
 sorc/build_ww3prepost.sh                 | 5 +++--
 4 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/modulefiles/modulefile.ww3.hera b/modulefiles/modulefile.ww3.hera
index 59bacd17471..462d7a9dac5 100644
--- a/modulefiles/modulefile.ww3.hera
+++ b/modulefiles/modulefile.ww3.hera
@@ -15,3 +15,5 @@ module load png/1.6.35
 
 module load bacio/2.4.1
 module load g2/3.4.1
+
+module load netcdf/4.7.4
diff --git a/modulefiles/modulefile.ww3.orion b/modulefiles/modulefile.ww3.orion
index 36774389891..1702532b716 100644
--- a/modulefiles/modulefile.ww3.orion
+++ b/modulefiles/modulefile.ww3.orion
@@ -16,3 +16,5 @@ module load png/1.6.35
 
 module load bacio/2.4.1
 module load g2/3.4.1
+
+module load netcdf/4.7.4
diff --git a/modulefiles/modulefile.ww3.wcoss_dell_p3 b/modulefiles/modulefile.ww3.wcoss_dell_p3
index ab43f602bf5..edb57b9fd24 100644
--- a/modulefiles/modulefile.ww3.wcoss_dell_p3
+++ b/modulefiles/modulefile.ww3.wcoss_dell_p3
@@ -11,3 +11,5 @@ module load png/1.6.35
 
 module load bacio/2.4.1
 module load g2/3.4.1
+
+module load netcdf/4.7.4
diff --git a/sorc/build_ww3prepost.sh b/sorc/build_ww3prepost.sh
index 339f050ba61..f17d7bcbeb0 100755
--- a/sorc/build_ww3prepost.sh
+++ b/sorc/build_ww3prepost.sh
@@ -35,8 +35,8 @@ export WWATCH3_ENV=${WW3_BINDIR}/wwatch3.env
 export PNG_LIB=${PNG_LIB:-$PNG_ROOT/lib64/libpng.a}
 export Z_LIB=${Z_LIB:-$ZLIB_ROOT/lib/libz.a}
 export JASPER_LIB=${JASPER_LIB:-$JASPER_ROOT/lib64/libjasper.a}
-# export WWATCH3_NETCDF=NC4
-# export NETCDF_CONFIG=$NETCDF_ROOT/bin/nc-config
+export WWATCH3_NETCDF=NC4
+export NETCDF_CONFIG=$NETCDF_ROOT/bin/nc-config
 export PRINTER=${PRINTER:-""}
 
 rm  $WWATCH3_ENV
@@ -50,6 +50,7 @@ echo "WWATCH3_F90      $WW3_F90"                      >> $WWATCH3_ENV
 echo "WWATCH3_CC       $WW3_CC"                       >> $WWATCH3_ENV
 echo "WWATCH3_DIR      $WW3_DIR"                      >> $WWATCH3_ENV
 echo "WWATCH3_TMP      $WW3_TMPDIR"                   >> $WWATCH3_ENV
+echo "WWATCH3_NETCDF   $WWATCH3_NETCDF"               >> $WWATCH3_ENV
 echo 'WWATCH3_SOURCE   yes'                           >> $WWATCH3_ENV
 echo 'WWATCH3_LIST     yes'                           >> $WWATCH3_ENV
 echo ''                                               >> $WWATCH3_ENV

From 06c5e3cb7bd9d82d4e51a63168465118a34e5882 Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Wed, 23 Jun 2021 19:06:42 +0000
Subject: [PATCH 106/125] Reduce C768 resource settings to fit node limits

- default C768 resources on Hera were 218 nodes which is higher than the 210 node limit per job
- bring C768 settings in config.fv3 down to 148 nodes (on Hera)
- tested new settings in free-forecast mode on Hera

Refs: #353
---
 parm/config/config.fv3 | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/parm/config/config.fv3 b/parm/config/config.fv3
index 1c4025f84c6..b7ecee8ab94 100755
--- a/parm/config/config.fv3
+++ b/parm/config/config.fv3
@@ -109,15 +109,15 @@ case $case_in in
         export layout_x=8
         export layout_y=12
         export layout_x_gfs=16
-        export layout_y_gfs=16
+        export layout_y_gfs=12
         export npe_wav=140
         export npe_wav_gfs=440
         export nth_fv3=4
-        export nth_fv3_gfs=7
+        export nth_fv3_gfs=4
         export cdmbgwd="4.0,0.15,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
         export WRITE_GROUP=2
         export WRTTASK_PER_GROUP=$(echo "2*$npe_node_max" |bc)
-        export WRITE_GROUP_GFS=8
+        export WRITE_GROUP_GFS=4
         export WRTTASK_PER_GROUP_GFS=$(echo "2*$npe_node_max" |bc)
         export WRTIOBUF="32M"
         ;;

From 7ed96c34f3dd6e4c4251e87d9abf62a0dbb413a2 Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Wed, 23 Jun 2021 19:10:17 +0000
Subject: [PATCH 107/125] Add cd to ROTDIR when pulling ops warm starts

- pull v16 warm starts directly into ROTDIR; don't need temporary location to handle subfolder diffs
- resolves issue with GDATE gdas restart files being left in EXTRACT_DIR

Refs: #353
---
 jobs/rocoto/getic.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/jobs/rocoto/getic.sh b/jobs/rocoto/getic.sh
index f08a10a8dd4..a6cb5d83f17 100755
--- a/jobs/rocoto/getic.sh
+++ b/jobs/rocoto/getic.sh
@@ -82,6 +82,7 @@ if [[ $gfs_ver = "v16" && $EXP_WARM_START = ".true." && $CASE = $OPS_RES ]]; the
 
   else # Opertional input - warm starts
 
+    cd $ROTDIR
     # Pull CDATE gfs restart tarball
     htar -xvf ${PRODHPSSDIR}/rh${yy}/${yy}${mm}/${yy}${mm}${dd}/com_gfs_prod_gfs.${yy}${mm}${dd}_${hh}.gfs_restart.tar
     # Pull GDATE gdas restart tarball

From 0f1cea6662695cad9ecf7df8317a36e1ab5c57e1 Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Wed, 23 Jun 2021 19:12:27 +0000
Subject: [PATCH 108/125] Add checks for EXP_WARM_START in config.base and
 config.getic

- add OPS_RES variable to both config.base.emc.dyn and config.getic
- add if-block in both configs to force EXP_WARM_START=true if CASE=OPS_RES and gfs_ver=v16 (current ops)
- check in config.base helps with later IAU checks
- help catch when user doesn't set EXP_WARM_START=true when running v16 C768

Refs: #353
---
 parm/config/config.base.emc.dyn | 8 +++++++-
 parm/config/config.getic        | 5 +++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/parm/config/config.base.emc.dyn b/parm/config/config.base.emc.dyn
index 588cca49233..da2fdc3c248 100755
--- a/parm/config/config.base.emc.dyn
+++ b/parm/config/config.base.emc.dyn
@@ -112,6 +112,13 @@ export ARCDIR="$NOSCRUB/archive/$PSLOT"
 export ICSDIR="@ICSDIR@"
 export ATARDIR="/NCEPDEV/$HPSS_PROJECT/1year/$USER/$machine/scratch/$PSLOT"
 
+# Set operational version variables
+export OPS_RES="C768" # Do not change
+export gfs_ver="v16" # Do not change
+if [[ $gfs_ver = "v16" && $CASE = $OPS_RES ]]; then # Force EXP_WARM_START=true if v16 C768
+  export EXP_WARM_START=".true."
+fi
+
 # Commonly defined parameters in JJOBS
 export envir=${envir:-"prod"}
 export NET="gfs"
@@ -140,7 +147,6 @@ export DBNROOT=${DBNROOT:-${UTILROOT}/fakedbn}
 export LEVS=128
 export CASE="@CASECTL@"
 export CASE_ENKF="@CASEENS@"
-export OPS_RES="C768"
 
 # Run with CCPP physics
 export RUN_CCPP="YES"
diff --git a/parm/config/config.getic b/parm/config/config.getic
index ced2c051f00..bfe6fde4002 100755
--- a/parm/config/config.getic
+++ b/parm/config/config.getic
@@ -10,6 +10,11 @@ echo "BEGIN: config.getic"
 
 export RETRO="NO" # YES = Pull v16 inputs from retrospective parallels; NO = use operational inputs
 export gfs_ver="v16" # Default = v16
+export OPS_RES=${OPS_RES:-"C768"} # Operational resolution
+
+if [[ $gfs_ver = "v16" && $CASE = $OPS_RES ]]; then # Going to be using warm starts
+  export EXP_WARM_START=".true."
+fi
 
 export UFS_DIR=${HOMEgfs}/sorc/ufs_utils.fd
 export GDASINIT_DIR=${UFS_DIR}/util/gdas_init

From 1c03067ca570295b65f967467ef2329fb7999578 Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Wed, 23 Jun 2021 19:33:18 +0000
Subject: [PATCH 109/125] Revert config.base EXP_WARM_START check change

Refs: #353
---
 parm/config/config.base.emc.dyn | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/parm/config/config.base.emc.dyn b/parm/config/config.base.emc.dyn
index da2fdc3c248..3f40715ffa8 100755
--- a/parm/config/config.base.emc.dyn
+++ b/parm/config/config.base.emc.dyn
@@ -112,13 +112,6 @@ export ARCDIR="$NOSCRUB/archive/$PSLOT"
 export ICSDIR="@ICSDIR@"
 export ATARDIR="/NCEPDEV/$HPSS_PROJECT/1year/$USER/$machine/scratch/$PSLOT"
 
-# Set operational version variables
-export OPS_RES="C768" # Do not change
-export gfs_ver="v16" # Do not change
-if [[ $gfs_ver = "v16" && $CASE = $OPS_RES ]]; then # Force EXP_WARM_START=true if v16 C768
-  export EXP_WARM_START=".true."
-fi
-
 # Commonly defined parameters in JJOBS
 export envir=${envir:-"prod"}
 export NET="gfs"
@@ -143,6 +136,9 @@ export SENDDBN_NTC=${SENDDBN_NTC:-"NO"}
 export SENDDBN=${SENDDBN:-"NO"}
 export DBNROOT=${DBNROOT:-${UTILROOT}/fakedbn}
 
+# Set operational resolution
+export OPS_RES="C768" # Do not change
+
 # Resolution specific parameters
 export LEVS=128
 export CASE="@CASECTL@"

From 08f62e45319c9772ef33cdb3c7aa3ec165fede09 Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Wed, 23 Jun 2021 21:21:51 +0000
Subject: [PATCH 110/125] Revert EXP_WARM_START if-block addition in
 config.getic

- need to determine better way to set EXP_WARM_START

Refs: #353
---
 parm/config/config.getic | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/parm/config/config.getic b/parm/config/config.getic
index bfe6fde4002..21c7da40586 100755
--- a/parm/config/config.getic
+++ b/parm/config/config.getic
@@ -12,10 +12,6 @@ export RETRO="NO" # YES = Pull v16 inputs from retrospective parallels; NO = use
 export gfs_ver="v16" # Default = v16
 export OPS_RES=${OPS_RES:-"C768"} # Operational resolution
 
-if [[ $gfs_ver = "v16" && $CASE = $OPS_RES ]]; then # Going to be using warm starts
-  export EXP_WARM_START=".true."
-fi
-
 export UFS_DIR=${HOMEgfs}/sorc/ufs_utils.fd
 export GDASINIT_DIR=${UFS_DIR}/util/gdas_init
 

From 351f5aa04be1fcd40ce31e1af09c6a6b0cb6ffbc Mon Sep 17 00:00:00 2001
From: "Walter.Kolczynski" <Walter.Kolczynski@noaa.gov>
Date: Thu, 24 Jun 2021 03:10:25 +0000
Subject: [PATCH 111/125] Update meditaor name

In the latest UFS, the mediator name was changed from nems to cmeps, so
the nems.configure is updated to match.

Refs: #324
---
 ush/nems_configure.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ush/nems_configure.sh b/ush/nems_configure.sh
index 58ecef74a52..c904948f999 100755
--- a/ush/nems_configure.sh
+++ b/ush/nems_configure.sh
@@ -58,7 +58,7 @@ chm_petlist_bounds=${chm_petlist_bounds:-"0 $(( $CHMPETS-1 ))"}
 
 # Copy the selected template into run directory
 cp $SCRIPTDIR/nems.configure.$confignamevarfornems.IN tmp1
-sed -i -e "s;@\[med_model\];nems;g" tmp1
+sed -i -e "s;@\[med_model\];cmeps;g" tmp1
 sed -i -e "s;@\[atm_model\];$ATM_model;g" tmp1
 sed -i -e "s;@\[med_petlist_bounds\];$med_petlist_bounds;g" tmp1
 sed -i -e "s;@\[atm_petlist_bounds\];$atm_petlist_bounds;g" tmp1

From 88fb063ffd501a4de66f3477e3772294b405384d Mon Sep 17 00:00:00 2001
From: "Walter.Kolczynski" <Walter.Kolczynski@noaa.gov>
Date: Thu, 24 Jun 2021 03:16:19 +0000
Subject: [PATCH 112/125] Update wave IC

Update wave IC to new set compatible with current UFS version.

Refs: #324
---
 workflow/cases/coupled_free_forecast_wave.yaml | 2 +-
 workflow/schema/wave.yaml                      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/workflow/cases/coupled_free_forecast_wave.yaml b/workflow/cases/coupled_free_forecast_wave.yaml
index acf39609daf..90fbd8b46ee 100644
--- a/workflow/cases/coupled_free_forecast_wave.yaml
+++ b/workflow/cases/coupled_free_forecast_wave.yaml
@@ -66,7 +66,7 @@ case:
 
   wave_settings: 
     WAVPETS: 160
-    CPL_WAVIC: CFSRwave20200925
+    CPL_WAVIC: GEFSwave20210623
     waveGRD: 'gwes_30m'
     wavepostGRD: 'gwes_30m'
     waveesmfGRD: ''
diff --git a/workflow/schema/wave.yaml b/workflow/schema/wave.yaml
index 47df41acbf1..1da284f41ec 100644
--- a/workflow/schema/wave.yaml
+++ b/workflow/schema/wave.yaml
@@ -9,4 +9,4 @@ wave_settings_template: !Template &wave_settings_template
     description: "model selection for wave"
   CPL_WAVIC:
     type: string
-    allowed: [ 'CFSR', 'CFSRwave20200925' ]
+    allowed: [ 'CFSR', 'CFSRwave20200925', 'GEFSwave20210623' ]

From 1957bb9d4b5bb81420715ef5f613fc9ab888fce6 Mon Sep 17 00:00:00 2001
From: "Walter.Kolczynski" <Walter.Kolczynski@noaa.gov>
Date: Thu, 24 Jun 2021 05:43:57 +0000
Subject: [PATCH 113/125] Update modules to match UFS

Update module versions to match those in the UFS common file.

Refs: #324
---
 modulefiles/module_base.hera          | 6 +++---
 modulefiles/module_base.orion         | 6 +++---
 modulefiles/module_base.wcoss_dell_p3 | 6 +++---
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/modulefiles/module_base.hera b/modulefiles/module_base.hera
index 5f845a7caab..51eb9c343bc 100644
--- a/modulefiles/module_base.hera
+++ b/modulefiles/module_base.hera
@@ -12,8 +12,8 @@ module load hpc-intel/18.0.5.274
 module load hpc-impi/2018.0.4
 
 # Non-MPI
-module load g2/3.4.1
-module load g2tmpl/1.9.1
+module load g2/3.4.2
+module load g2tmpl/1.10.0
 module load grib_util/1.2.2
 module load crtm/2.3.0
 module load prod_util/1.2.2
@@ -31,7 +31,7 @@ module load png/1.6.35
 # MPI
 module load netcdf/4.7.4
 module load hdf5/1.10.6
-module load esmf/8_1_0_beta_snapshot_27
+module load esmf/8_1_1
 module load w3emc/2.7.3
 module load wgrib2/2.0.8
 setenv "WGRIB2" "wgrib2"
diff --git a/modulefiles/module_base.orion b/modulefiles/module_base.orion
index 9fbc6a081b4..86a8ce419ae 100644
--- a/modulefiles/module_base.orion
+++ b/modulefiles/module_base.orion
@@ -20,8 +20,8 @@ module load hpc-intel/2018.4
 module load hpc-impi/2018.4
 
 # Non-MPI
-module load g2/3.4.1
-module load g2tmpl/1.9.1
+module load g2/3.4.2
+module load g2tmpl/1.10.0
 module load grib_util/1.2.2
 module load crtm 2.3.0
 module load prod_util/1.2.2
@@ -38,7 +38,7 @@ module load png/1.6.35
 # MPI
 module load netcdf/4.7.4
 module load hdf5/1.10.6
-module load esmf/8_1_0_beta_snapshot_27
+module load esmf/8_1_1
 module load w3emc/2.7.3
 module load wgrib2/2.0.8
 setenv WGRIB2 wgrib2
diff --git a/modulefiles/module_base.wcoss_dell_p3 b/modulefiles/module_base.wcoss_dell_p3
index a651cb86c0c..e53199967c8 100755
--- a/modulefiles/module_base.wcoss_dell_p3
+++ b/modulefiles/module_base.wcoss_dell_p3
@@ -11,8 +11,8 @@ module load CFP/2.0.2
 setenv USE_CFP YES
 
 # Non-MPI
-module load g2/3.4.1
-module load g2tmpl/1.9.1
+module load g2/3.4.2
+module load g2tmpl/1.10.0
 module load grib_util/1.2.2
 module load crtm 2.3.0
 module load prod_util/1.2.2
@@ -28,7 +28,7 @@ module load png/1.6.35
 # MPI
 module load netcdf/4.7.4
 module load hdf5/1.10.6
-module load esmf/8_1_0_beta_snapshot_27
+module load esmf/8_1_1
 module load w3emc/2.7.3
 module load wgrib2/2.0.8
 setenv WGRIB2 wgrib2

From 69cf53412a61e7fe0fb49e8c688bc2cbb4ded572 Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Thu, 24 Jun 2021 16:38:25 +0000
Subject: [PATCH 114/125] Set EXP_WARM_START=true in config.base via
 setup_expt_fcsonly.py if resolution is C768

- have free-forecast setup_expt script check if resolution is operational resolution (C768)
  and if so, set EXP_WARM_START=.true.
- forcing EXP_WARM_START=.true. when C768 covers most C768 scenarios regarding cold vs warm starting
- the --start argument still works but new check will override if res=768
- users can still change config.base setting as needed for special scenarios

Refs: #353
---
 ush/rocoto/setup_expt_fcstonly.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/ush/rocoto/setup_expt_fcstonly.py b/ush/rocoto/setup_expt_fcstonly.py
index f556683b934..e8a2d05f763 100755
--- a/ush/rocoto/setup_expt_fcstonly.py
+++ b/ush/rocoto/setup_expt_fcstonly.py
@@ -144,6 +144,10 @@ def edit_baseconfig():
     elif start == 'warm':
       exp_warm_start = '.true.'
 
+    # Check resolution for start setting - assume warm start if ops res
+    if res == 768:
+      exp_warm_start = '.true.'
+
     # Set FDATE (first full cycle)
     fdate = idate + timedelta(hours=6)
 

From 2dcf8f2b1039b0ca1b706d1465e141894209ad98 Mon Sep 17 00:00:00 2001
From: "Walter.Kolczynski" <Walter.Kolczynski@noaa.gov>
Date: Thu, 24 Jun 2021 19:07:57 +0000
Subject: [PATCH 115/125] Update ice configuration to match UFS

There were still a number of settings in the ice input file that
did not match the setting in UFS (and one setting that was misplaced).

Refs: #324
---
 ush/parsing_namelists_CICE.sh | 54 +++++++++++++++++------------------
 1 file changed, 27 insertions(+), 27 deletions(-)

diff --git a/ush/parsing_namelists_CICE.sh b/ush/parsing_namelists_CICE.sh
index d19f0c41722..62cde8494b7 100644
--- a/ush/parsing_namelists_CICE.sh
+++ b/ush/parsing_namelists_CICE.sh
@@ -120,7 +120,6 @@ cat > ice_in <<eof
    coriolis        = 'latitude'
    kridge          = 1
    ktransport      = 1
-   sw_redist       = .true.
 /
 
 &shortwave_nml
@@ -137,6 +136,7 @@ cat > ice_in <<eof
    dT_mlt          = 1.5
    rsnw_mlt        = 1500.
    kalg            = 0.0
+   sw_redist       = .true.
 /
 
 &ponds_nml
@@ -209,11 +209,11 @@ cat > ice_in <<eof
    f_hi            = 'mdhxx'
    f_hs            = 'mdhxx' 
    f_Tsfc          = 'mdhxx' 
-   f_sice          = 'x' 
+   f_sice          = 'mdhxx' 
    f_uvel          = 'mdhxx' 
    f_vvel          = 'mdhxx'
-   f_uatm          = 'x'
-   f_vatm          = 'x'
+   f_uatm          = 'mdhxx'
+   f_vatm          = 'mdhxx'
    f_fswdn         = 'mdhxx' 
    f_flwdn         = 'mdhxx'
    f_snowfrac      = 'x'
@@ -223,18 +223,18 @@ cat > ice_in <<eof
    f_rain_ai       = 'x' 
    f_sst           = 'mdhxx' 
    f_sss           = 'mdhxx' 
-   f_uocn          = 'x' 
-   f_vocn          = 'x' 
-   f_frzmlt        = 'x'
+   f_uocn          = 'mdhxx' 
+   f_vocn          = 'mdhxx' 
+   f_frzmlt        = 'mdhxx'
    f_fswfac        = 'mdhxx'
    f_fswint_ai     = 'x'
    f_fswabs        = 'mdhxx' 
    f_fswabs_ai     = 'x' 
    f_albsni        = 'mdhxx' 
-   f_alvdr         = 'x'
-   f_alidr         = 'x'
-   f_alvdf         = 'x'
-   f_alidf         = 'x'
+   f_alvdr         = 'mdhxx'
+   f_alidr         = 'mdhxx'
+   f_alvdf         = 'mdhxx'
+   f_alidf         = 'mdhxx'
    f_alvdr_ai      = 'x'
    f_alidr_ai      = 'x'
    f_alvdf_ai      = 'x'
@@ -256,19 +256,19 @@ cat > ice_in <<eof
    f_Tref          = 'mdhxx' 
    f_Qref          = 'mdhxx'
    f_congel        = 'mdhxx' 
-   f_frazil        = 'x' 
-   f_snoice        = 'x' 
-   f_dsnow         = 'x' 
-   f_melts         = 'x'
+   f_frazil        = 'mdhxx' 
+   f_snoice        = 'mdhxx' 
+   f_dsnow         = 'mdhxx' 
+   f_melts         = 'mdhxx'
    f_meltt         = 'mdhxx'
    f_meltb         = 'mdhxx'
    f_meltl         = 'mdhxx'
-   f_fresh         = 'x'
+   f_fresh         = 'mdhxx'
    f_fresh_ai      = 'x'
-   f_fsalt         = 'x'
+   f_fsalt         = 'mdhxx'
    f_fsalt_ai      = 'x'
-   f_fbot          = 'x'
-   f_fhocn         = 'x' 
+   f_fbot          = 'mdhxx'
+   f_fhocn         = 'mdhxx' 
    f_fhocn_ai      = 'x' 
    f_fswthru       = 'x' 
    f_fswthru_ai    = 'x' 
@@ -281,22 +281,22 @@ cat > ice_in <<eof
    f_strtlty       = 'x' 
    f_strcorx       = 'x' 
    f_strcory       = 'x' 
-   f_strocnx       = 'x' 
-   f_strocny       = 'x' 
+   f_strocnx       = 'mdhxx' 
+   f_strocny       = 'mdhxx' 
    f_strintx       = 'x' 
    f_strinty       = 'x'
    f_taubx         = 'x'
    f_tauby         = 'x'
    f_strength      = 'x'
-   f_divu          = 'mdhxx'
-   f_shear         = 'mdhxx'
+   f_divu          = 'x'
+   f_shear         = 'x'
    f_sig1          = 'x' 
    f_sig2          = 'x'
    f_sigP          = 'x' 
-   f_dvidtt        = 'mdhxx'
-   f_dvidtd        = 'mdhxx'
-   f_daidtt        = 'mdhxx'
-   f_daidtd        = 'mdhxx'
+   f_dvidtt        = 'x'
+   f_dvidtd        = 'x'
+   f_daidtt        = 'x'
+   f_daidtd        = 'x'
    f_dagedtt       = 'x'
    f_dagedtd       = 'x'
    f_mlt_onset     = 'x'

From 4fc8131f294f731ed38b308cd6fed56908f05856 Mon Sep 17 00:00:00 2001
From: Raffaele Montuoro <raffaele.montuoro@noaa.gov>
Date: Thu, 1 Jul 2021 22:46:28 +0000
Subject: [PATCH 116/125] Add ammonia and nitrates to the list of cycled GOCART
 tracers.

---
 parm/chem/gocart_tracer.list | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/parm/chem/gocart_tracer.list b/parm/chem/gocart_tracer.list
index 0ac80b32ff4..8b0202e2c4e 100644
--- a/parm/chem/gocart_tracer.list
+++ b/parm/chem/gocart_tracer.list
@@ -2,8 +2,6 @@ so2
 so4
 dms
 msa
-pm25
-pm10
 bc1
 bc2
 oc1
@@ -18,3 +16,10 @@ seas2
 seas3
 seas4
 seas5
+nh3
+nh4a
+no3an1
+no3an2
+no3an3
+pm25
+pm10

From acdc6a69194eadcba05c115219dfb5a4afabdd00 Mon Sep 17 00:00:00 2001
From: Raffaele Montuoro <raffaele.montuoro@noaa.gov>
Date: Thu, 1 Jul 2021 22:48:37 +0000
Subject: [PATCH 117/125] Update checkout and build script to use the
 authoritative GitHub repository for the UFS weather model only. This repo now
 contains UFS-Aerosols, the aerosol component based on NASA's GOCART.

Since we are using a single repository for both S2S and
coupled aerosols, the -a option was removed from checkout.sh
and added to build_all.sh. This option is passed to
build_ufs_coupled.sh and partial_build.sh, which have
been streamlined and now accept both short and long options:

build_ufs_coupled.sh:
  -a, --aerosols
  -c, --coupled

partial_build.sh:
  -a, --aerosols
  -c, --coupled
  -h, --help
  -v, --verbose
---
 sorc/build_all.sh         | 17 ++++++++--
 sorc/build_ufs_coupled.sh | 41 ++++++++++++++++--------
 sorc/checkout.sh          | 18 ++---------
 sorc/partial_build.sh     | 67 ++++++++++++++++++---------------------
 4 files changed, 76 insertions(+), 67 deletions(-)

diff --git a/sorc/build_all.sh b/sorc/build_all.sh
index 577f42df052..a47eb8455ab 100755
--- a/sorc/build_all.sh
+++ b/sorc/build_all.sh
@@ -9,14 +9,25 @@ set +x
 #                   Anything other than "true"  will use libraries locally.
 #------------------------------------
 
-while getopts "c" option;
+_build_ufs_options=""
+
+while getopts "ac" option;
 do
  case $option in
+  a)
+   echo "Received -a flag, build ufs-weather-model develop branch with aerosols and CCPP physics"
+   echo "setting coupled=yes and skipping builds not needed for prototype runs"
+   _build_ufs_options=-a
+   RUN_CCPP="YES"
+   COUPLED="YES"
+   break
+   ;;
   c)
    echo "Received -c flag, build ufs-weather-model develop branch with CCPP physics"
    echo "setting coupled=yes and skipping builds not needed for prototype runs"
    RUN_CCPP="YES"
    COUPLED="YES" 
+   break
    ;;
  esac
 done
@@ -49,7 +60,7 @@ source ./machine-setup.sh > /dev/null 2>&1
 #------------------------------------
 # INCLUDE PARTIAL BUILD 
 #------------------------------------
-. ./partial_build.sh
+. ./partial_build.sh $@
 
 #------------------------------------
 # Exception Handling Init
@@ -87,7 +98,7 @@ if [[ $rc -ne 0 ]] ; then
 fi
 ((err+=$rc))
 else 
-./build_ufs_coupled.sh > $logs_dir/build_ufs_coupled.log 2>&1
+./build_ufs_coupled.sh ${_build_ufs_options} > $logs_dir/build_ufs_coupled.log 2>&1
 rc=$?
 if [[ $rc -ne 0 ]] ; then
     echo "Fatal error in building ufs coupled forecast model."
diff --git a/sorc/build_ufs_coupled.sh b/sorc/build_ufs_coupled.sh
index 66de17a7fda..75bec964fa3 100755
--- a/sorc/build_ufs_coupled.sh
+++ b/sorc/build_ufs_coupled.sh
@@ -1,6 +1,25 @@
 #! /usr/bin/env bash
 set -eux
 
+# Build S2S by default
+CMAKE_FLAGS="-DS2S=ON -DWW3=ON"
+CCPP_SUITES="FV3_GFS_v15p2_coupled,FV3_GFS_v16_coupled,FV3_GFS_v16"
+
+while [ ${#} -gt 0 ]
+do
+  case "${1}" in
+    -a|--aerosols)
+      CMAKE_FLAGS="-DAPP=ATMAERO"
+      CCPP_SUITES="FV3_GFS_v16"
+      shift
+      ;;
+    *)
+      echo "Unrecognized option: ${1}"
+      exit 1
+      ;;
+  esac
+done
+
 source ./machine-setup.sh > /dev/null 2>&1
 cwd=`pwd`
 
@@ -9,22 +28,18 @@ if [ ! -d "../exec" ]; then
   mkdir ../exec
 fi
 
-if [ $target = hera ]; then target=hera.intel ; fi
-if [ $target = orion ]; then target=orion.intel ; fi
-if [ $target = stampede ]; then target=stampede.intel ; fi
+# Set target platform
+case "${target}" in
+  hera|orion|stampede)
+    target=${target}.intel
+    ;;
+esac
 
 MOD_PATH=$cwd/ufs_coupled.fd/modulefiles
 
 module purge 
 
 cd ufs_coupled.fd/
-if [[ -d build ]]; then rm -Rf build; fi
-if [[ -e .build_aerosols ]] ; then
-  module use ${MOD_PATH}
-  module load ufs_${target} ufs_aerosols_${target}
-  CMAKE_FLAGS="-DAPP=ATMAERO" CCPP_SUITES="FV3_GFS_v16" ./build.sh
-else
-  module use ${MOD_PATH}/${target}
-  module load fv3
-  CMAKE_FLAGS="-DS2S=ON -DWW3=ON" CCPP_SUITES="FV3_GFS_v15p2_coupled,FV3_GFS_v16_coupled,FV3_GFS_v16" ./build.sh
-fi
+module use ${MOD_PATH}
+module load ufs_${target}
+CMAKE_FLAGS="${CMAKE_FLAGS}" CCPP_SUITES="${CCPP_SUITES}" ./build.sh
diff --git a/sorc/checkout.sh b/sorc/checkout.sh
index 078cd64a887..3ed89732c05 100755
--- a/sorc/checkout.sh
+++ b/sorc/checkout.sh
@@ -13,11 +13,6 @@ do
    echo "Received -c flag, running coupled model" 
    COUPLED="YES"
    ;;
-  a)
-   echo "Received -a flag, running coupled aerosols model"
-   AEROSOL="YES"
-   COUPLED="YES"
-   ;;
   :)
    echo "option -$OPTARG needs an argument"
    ;;
@@ -45,16 +40,9 @@ if [ ${COUPLED:-"NO"} = "NO" ]; then
   fi 
 else 
   if [[ ! -d ufs_coupled.fd ]] ; then
-    if [ "${AEROSOL}" = "YES" ] ; then
-      git clone https://github.com/rmontuoro/ufs-weather-model.git ufs_coupled.fd >> ${topdir}/checkout-ufs_coupled.log 2>&1
-      cd ufs_coupled.fd
-      git checkout feature/gocart
-      echo > .build_aerosols
-    else
-      git clone https://github.com/ufs-community/ufs-weather-model ufs_coupled.fd >> ${topdir}/checkout-ufs_coupled.log 2>&1
-      cd ufs_coupled.fd
-      git checkout 3e46f5b7050e18884a0bed13691823ad88d443c3
-    fi
+    git clone https://github.com/ufs-community/ufs-weather-model ufs_coupled.fd >> ${topdir}/checkout-ufs_coupled.log 2>&1
+    cd ufs_coupled.fd
+    git checkout a842d57f9c352acd3771fbccd8d8eb044558d090
     git submodule update --init --recursive
     cd ${topdir} 
   else
diff --git a/sorc/partial_build.sh b/sorc/partial_build.sh
index 671ca6da2da..24971568fa9 100755
--- a/sorc/partial_build.sh
+++ b/sorc/partial_build.sh
@@ -129,54 +129,49 @@
    fi
  }
 
+
+ usage() {
+   echo "Usage: $0 [ALL|config=config_file|[select=][prog1[,prog2[,...]]]" 2>&1
+ }
+
 #
 # read command line arguments; processing config file
 #
+ declare -a parse_argv=()
+ coupled=false
  verbose=false
- num_arg=$#
- (( num_arg > 1 )) && {
-   [[ ${1,,} == "--verbose" ]] && {
-     verbose=true
-   } || {
-     echo "Usage: $0 [ALL|config=config_file|[select=][prog1[,prog2[,...]]]" 2>&1
-     exit 1
-   }
- }
- (( num_arg == 1 )) && {
-     ( [[ $1 == "-h" ]] || [[ $1 == "--help" ]] ) && {
-       echo "Usage: $0 [ALL|config=config_file|[select=][prog1[,prog2[,...]]]" 2>&1
+ while [ ${#} -ne 0 ]; do
+   case "${1}" in
+     -v|--verbose)
+       verbose=true
+       parse_argv+=( "${1}" )
+       shift
+       ;;
+     -h|--help)
+       usage
        exit 2
-     }
-     ( [[ $1 == "-v" ]] || [[ ${1,,} == "--verbose" ]] || [[ $1 == "-c" ]] ) && {
-       if [[ $1 == "-v" ]]; then
-         verbose=true
-       fi
-       if [[ $1 == "-c" ]]; then
-         coupled=true
-       fi
-       num_arg=0
-     } || {
-       echo "Usage: $0 [ALL|config=config_file|[select=][prog1[,prog2[,...]]]" 2>&1
+       ;;
+     -a|--aerosols|-c|--coupled)
+       coupled=true
+       shift
+       ;;
+     *)
+       usage
        exit 3
-     }
- }
+       ;;
+   esac
+ done
 
- if (( num_arg == 0 )); then
-#
-# set default values for partial build
-#   
-   if [[ $coupled ]]; then 
-     parse_cfg 1 "config=cpl_build.cfg" ${Build_prg[@]}
-   else 
-     parse_cfg 1 "config=fv3gfs_build.cfg" ${Build_prg[@]}
-   fi 
+ if [ "${coupled}" = true ]; then
+   parse_argv+=( "config=cpl_build.cfg" )
  else
+   parse_argv+=( "config=fv3gfs_build.cfg" )
+ fi
 
 #
 # call arguments retriever/config parser
 #
-   parse_cfg $num_arg "$@" ${Build_prg[@]}
- fi
+  parse_cfg ${#parse_argv[@]} "${parse_argv[@]}" ${Build_prg[@]}
 
 #
 # print values of build array

From a51d4ef77e36995e437f481a10cdef4eea14f35b Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Thu, 8 Jul 2021 20:39:17 +0000
Subject: [PATCH 118/125] Add spaces to indent FV3 build section for better
 readability

Refs: #230
---
 sorc/build_all.sh | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/sorc/build_all.sh b/sorc/build_all.sh
index 577f42df052..242e1c002e0 100755
--- a/sorc/build_all.sh
+++ b/sorc/build_all.sh
@@ -78,22 +78,22 @@ fi
 $Build_fv3gfs && {
 echo " .... Building forecast model .... "
 if [ ${COUPLED:-"NO"} = "NO" ]; then 
-export RUN_CCPP=${RUN_CCPP:-"NO"}
-./build_fv3.sh > $logs_dir/build_fv3.log 2>&1
-rc=$?
-if [[ $rc -ne 0 ]] ; then
+  export RUN_CCPP=${RUN_CCPP:-"NO"}
+  ./build_fv3.sh > $logs_dir/build_fv3.log 2>&1
+  rc=$?
+  if [[ $rc -ne 0 ]] ; then
     echo "Fatal error in building fv3."
     echo "The log file is in $logs_dir/build_fv3.log"
-fi
-((err+=$rc))
+  fi
+  ((err+=$rc))
 else 
-./build_ufs_coupled.sh > $logs_dir/build_ufs_coupled.log 2>&1
-rc=$?
-if [[ $rc -ne 0 ]] ; then
+  ./build_ufs_coupled.sh > $logs_dir/build_ufs_coupled.log 2>&1
+  rc=$?
+  if [[ $rc -ne 0 ]] ; then
     echo "Fatal error in building ufs coupled forecast model."
     echo "The log file is in $logs_dir/build_ufs_coupled.log"
-fi
-((err+=$rc))
+  fi
+  ((err+=$rc))
 fi
 }
 

From 5bec2a725271d7423690d1050771eaf83db562d3 Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Thu, 8 Jul 2021 20:41:14 +0000
Subject: [PATCH 119/125] Add FV3_GFS_v16_couplednsst CCPP suite option to UFS
 coupled build and FV3 namelist parser

Refs: #230
---
 sorc/build_ufs_coupled.sh    | 2 +-
 ush/parsing_namelists_FV3.sh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/sorc/build_ufs_coupled.sh b/sorc/build_ufs_coupled.sh
index e2176967178..9d368fff95a 100755
--- a/sorc/build_ufs_coupled.sh
+++ b/sorc/build_ufs_coupled.sh
@@ -25,5 +25,5 @@ if [[ -d GOCART ]]; then
   CMAKE_FLAGS="-DAPP=ATMAERO" CCPP_SUITES="FV3_GFS_v16" ./build.sh
   ./build.sh
 else
-  CMAKE_FLAGS="-DAPP=S2SW" CCPP_SUITES="FV3_GFS_v16_coupled,FV3_GFS_v16" ./build.sh
+  CMAKE_FLAGS="-DAPP=S2SW" CCPP_SUITES="FV3_GFS_v16_coupled,FV3_GFS_v16_couplednsst,FV3_GFS_v16" ./build.sh
 fi
diff --git a/ush/parsing_namelists_FV3.sh b/ush/parsing_namelists_FV3.sh
index ddcca3a2412..b14eec96d51 100755
--- a/ush/parsing_namelists_FV3.sh
+++ b/ush/parsing_namelists_FV3.sh
@@ -215,7 +215,7 @@ elif [ $CCPP_SUITE = "FV3_GSD_v0" ]; then
   min_lakeice  = ${min_lakeice:-"0.15"}
   min_seaice   = ${min_seaice:-"0.15"}
 EOF
-elif [ $CCPP_SUITE = "FV3_GFS_v16_coupled" ]; then
+elif [ $CCPP_SUITE = "FV3_GFS_v16_coupled" -o $CCPP_SUITE = "FV3_GFS_v16_couplednsst" ]; then
   cat >> input.nml << EOF
   iovr         = ${iovr:-"3"}
   ltaerosol    = ${ltaerosol:-".false."}

From bbcad18ba4e91a6c242a0276f7c52525c056f772 Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Thu, 8 Jul 2021 20:42:46 +0000
Subject: [PATCH 120/125] Replace CROW yaml nst_spinup_logic in nsst.yaml with
 NST_SPINUP variable

- removed CROW FirstTrue check when building config.nsst from nsst.yaml
- replace nst_spinup_logic with NST_SPINUP variable that is parsed from variable dictionary
- nst_spinup_logic not needed currently

Refs: #230
---
 workflow/config/nsst.yaml | 10 +---------
 1 file changed, 1 insertion(+), 9 deletions(-)

diff --git a/workflow/config/nsst.yaml b/workflow/config/nsst.yaml
index 6a188a03771..bc965adf4b5 100644
--- a/workflow/config/nsst.yaml
+++ b/workflow/config/nsst.yaml
@@ -4,14 +4,6 @@
 
 config_nsst:
   filename: config.nsst
-  nst_spinup_logic: !FirstTrue
-    - when: !calc doc.nsst.get("NST_SPINUP","")
-      do: !expand "export NST_SPINUP={doc.nsst.NST_SPINUP}"
-    - otherwise: |
-        export NST_SPINUP=0
-        #if [[ "$CDATE" = $SDATE ]]; then
-        #    export NST_SPINUP=1
-        #fi
   content: !expand |
     #!/bin/ksh -x
     
@@ -30,7 +22,7 @@ config_nsst:
     export NST_MODEL={doc.nsst.NST_MODEL}
     
     # nstf_name(2) : NST_SPINUP : 0 = OFF, 1 = ON,
-    {nst_spinup_logic}
+    export NST_SPINUP={doc.nsst.NST_SPINUP}
 
     # nstf_name(3) : NST_RESV (Reserved, NSST Analysis) : 0 = OFF, 1 = ON
     export NST_RESV={doc.nsst.NST_RESV}

From cec61887b6c5635828045b422877de7ae61ca1c0 Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Fri, 9 Jul 2021 13:13:48 +0000
Subject: [PATCH 121/125] Add NSST settings to the
 coupled_free_forecast_wave.yaml case file

- set NST_MODEL=2 (turns it on)
- add additional NSST namelist variables: NST_SPINUP, NST_RESV, ZSEA1, ZSEA2
- set NST_SPINUP=1 (NSST will spinup)
- change CCPP_SUITE from FV3_GFS_v16_coupled to FV3_GFS_v16_couplednsst

Refs: #230
---
 workflow/cases/coupled_free_forecast_wave.yaml | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/workflow/cases/coupled_free_forecast_wave.yaml b/workflow/cases/coupled_free_forecast_wave.yaml
index 90fbd8b46ee..c6329bfd313 100644
--- a/workflow/cases/coupled_free_forecast_wave.yaml
+++ b/workflow/cases/coupled_free_forecast_wave.yaml
@@ -22,7 +22,11 @@ case:
     KEEPDATA: NO
 
   nsst:
-    NST_MODEL: 0
+    NST_MODEL: 2
+    NST_SPINUP: 1
+    NST_RESV: 0
+    ZSEA1: 0
+    ZSEA2: 0
 
   output_settings:
     OCN_INTERVAL: 24
@@ -39,7 +43,7 @@ case:
     min_lakeice: 0.15
     min_seaice: 1.0e-11
     SEEDLET: 10
-    CCPP_SUITE: FV3_GFS_v16_coupled
+    CCPP_SUITE: FV3_GFS_v16_couplednsst
     CPL_ATMIC: CFSRfracL127
     nst_anl: yes
     psm_bc: 1

From d3039974cd7887ed3f984f4e7c586408bbda3678 Mon Sep 17 00:00:00 2001
From: "Walter.Kolczynski" <Walter.Kolczynski@noaa.gov>
Date: Wed, 14 Jul 2021 04:22:59 +0000
Subject: [PATCH 122/125] Update nems.configure to inclue eps_imesh

Adds the eps_imesh setting to the nems.configure files when CICE is
on.

Refs: #324, #333
---
 ush/nems.configure.cpld.IN                    | 1 +
 ush/nems.configure.cpld_wave.IN               | 1 +
 ush/nems.configure.medcold_atm_ocn_ice.IN     | 1 +
 ush/nems.configure.medcold_atm_ocn_ice_wav.IN | 1 +
 ush/nems_configure.sh                         | 2 ++
 5 files changed, 6 insertions(+)

diff --git a/ush/nems.configure.cpld.IN b/ush/nems.configure.cpld.IN
index 69b6c737f3a..8a070787f51 100644
--- a/ush/nems.configure.cpld.IN
+++ b/ush/nems.configure.cpld.IN
@@ -101,4 +101,5 @@ ALLCOMP_attributes::
       dbug_flag = @[cap_dbug_flag]
       use_coldstart = @[use_coldstart]
       use_mommesh = @[use_mommesh]
+      eps_imesh = @[eps_imesh]
 ::
diff --git a/ush/nems.configure.cpld_wave.IN b/ush/nems.configure.cpld_wave.IN
index e4d9eca31d2..2b8a49e56b6 100644
--- a/ush/nems.configure.cpld_wave.IN
+++ b/ush/nems.configure.cpld_wave.IN
@@ -115,4 +115,5 @@ ALLCOMP_attributes::
       dbug_flag = @[cap_dbug_flag]
       use_coldstart = @[use_coldstart]
       use_mommesh = @[use_mommesh]
+      eps_imesh = @[eps_imesh]
 ::
diff --git a/ush/nems.configure.medcold_atm_ocn_ice.IN b/ush/nems.configure.medcold_atm_ocn_ice.IN
index fe4030274e5..80a1527b9aa 100644
--- a/ush/nems.configure.medcold_atm_ocn_ice.IN
+++ b/ush/nems.configure.medcold_atm_ocn_ice.IN
@@ -100,4 +100,5 @@ ALLCOMP_attributes::
       restart_option = nhours
       restart_ymd = -999
       dbug_flag = 0
+      eps_imesh = @[eps_imesh]
 ::
diff --git a/ush/nems.configure.medcold_atm_ocn_ice_wav.IN b/ush/nems.configure.medcold_atm_ocn_ice_wav.IN
index fa8ab4e4ff1..2ef5f706bf3 100644
--- a/ush/nems.configure.medcold_atm_ocn_ice_wav.IN
+++ b/ush/nems.configure.medcold_atm_ocn_ice_wav.IN
@@ -113,4 +113,5 @@ ALLCOMP_attributes::
       restart_option = nhours
       restart_ymd = -999
       dbug_flag = 0
+      eps_imesh = @[eps_imesh]
 ::
diff --git a/ush/nems_configure.sh b/ush/nems_configure.sh
index c904948f999..7916a6a6cc7 100755
--- a/ush/nems_configure.sh
+++ b/ush/nems_configure.sh
@@ -47,6 +47,7 @@ CHMPETS=${CHMPETS:-${ATMPETS}}
 
 USE_MOMMESH=${USE_MOMMESH:-"true"}
 
+
 rm -f $DATA/nems.configure
 
 med_petlist_bounds=${med_petlist_bounds:-"0 $(( $MEDPETS-1 ))"}
@@ -82,6 +83,7 @@ if [ $cplflx = .true. ]; then
 	sed -i -e "s;@\[coupling_interval_fast_sec\];$CPL_FAST;g" tmp1
         sed -i -e "s;@\[RESTART_N\];$restart_interval_nems;g" tmp1
   sed -i -e "s;@\[use_mommesh\];$USE_MOMMESH;g" tmp1
+  sed -i -e "s;@\[eps_imesh\];$ICERESdec;g" tmp1
 fi
 if [ $cplwav = .true. ]; then
 	sed -i -e "s;@\[wav_model\];ww3;g" tmp1

From dc970f9f3644ed67624141d08ca4b9841eccea00 Mon Sep 17 00:00:00 2001
From: "Walter.Kolczynski" <Walter.Kolczynski@noaa.gov>
Date: Wed, 14 Jul 2021 16:21:32 +0000
Subject: [PATCH 123/125] Change forecast module back to module_base

The modulefile in UFS doesn't load some modules needed for the forecast
script at runtime, notably produtil, which provides err_chk. The module
for the forecast job is changed back to module_base for the time being.

Refs: #336
---
 modulefiles/module_base.hera |  2 ++
 ush/load_fv3gfs_modules.sh   | 28 ++--------------------------
 2 files changed, 4 insertions(+), 26 deletions(-)

diff --git a/modulefiles/module_base.hera b/modulefiles/module_base.hera
index a14ab46cb8c..de89fbf9547 100644
--- a/modulefiles/module_base.hera
+++ b/modulefiles/module_base.hera
@@ -9,6 +9,8 @@ module use /scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack
 module load hpc/1.1.0
 module load hpc-intel/18.0.5.274
 module load hpc-impi/2018.0.4
+# use shared memory and OpenFabrics Alliance (OFA) fabric with Intel MPI to circumvent RDMA-related bug in DAPL.
+setenv I_MPI_FABRICS shm:ofa
 
 # Non-MPI
 module load g2/3.4.2
diff --git a/ush/load_fv3gfs_modules.sh b/ush/load_fv3gfs_modules.sh
index d9ae4f0e9ad..521d0eccfb0 100755
--- a/ush/load_fv3gfs_modules.sh
+++ b/ush/load_fv3gfs_modules.sh
@@ -29,32 +29,8 @@ fi
 
 platform=$( get_platform )
 
-moduledir=
-modulelist=
-case "${platform}" in
-  hera)
-    moduledir="$HOMEgfs/sorc/ufs_coupled.fd/modulefiles"
-    target=${platform}.intel
-    if [[ "${tasktype}" = "forecast" && -r ${moduledir}/ufs_${target} ]] ; then
-      modulelist=ufs_${target}
-      if [[ "$( grep UFS_GOCART ${HOMEgfs}/sorc/ufs_coupled.fd/build/CMakeCache.txt 2>/dev/null | cut -d= -f2 )" = "ON" ]] ; then
-        # add aerosols modulefile
-        modulelist="${modulelist} ufs_aerosols_${target}"
-      fi
-    else
-      moduledir="$HOMEgfs/modulefiles"
-      modulelist="module_base.${platform}"
-    fi
-    ;;
-  unknown)
-    echo "Unknown platform"
-    exit 1
-    ;;
-  *)
-    moduledir="$HOMEgfs/modulefiles"
-    modulelist="module_base.${platform}"
-    ;;
-esac
+moduledir="$HOMEgfs/modulefiles"
+modulelist="module_base.${platform}"
 
 # Load our modules:
 module use ${moduledir}

From b34c581c3b19ecdcdb4eff463e855597e5c08d80 Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Wed, 14 Jul 2021 19:15:31 +0000
Subject: [PATCH 124/125] Adjust how EXP_WARM_START is set from
 setup_expt_fcstonly.py step

- remove default value for start (EXP_WARM_START)
- if user provides start flag then use it for EXP_WARM_START
- if user doesn't provide start flag then make a decision for them based on resolution
- if start=None and res=768 then start=warm
- if start=None and res!=768 then start=cold

Refs: #353
---
 ush/rocoto/setup_expt_fcstonly.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/ush/rocoto/setup_expt_fcstonly.py b/ush/rocoto/setup_expt_fcstonly.py
index e8a2d05f763..f9e5b2c4808 100755
--- a/ush/rocoto/setup_expt_fcstonly.py
+++ b/ush/rocoto/setup_expt_fcstonly.py
@@ -118,7 +118,7 @@ def edit_baseconfig():
     parser.add_argument('--configdir', help='full path to directory containing the config files', type=str, required=False, default=None)
     parser.add_argument('--gfs_cyc', help='GFS cycles to run', type=int, choices=[0, 1, 2, 4], default=1, required=False)
     parser.add_argument('--partition', help='partition on machine', type=str, required=False, default=None)
-    parser.add_argument('--start', help='restart mode: warm or cold', type=str, choices=['warm', 'cold'], required=False, default='cold')
+    parser.add_argument('--start', help='restart mode: warm or cold', type=str, choices=['warm', 'cold'], required=False)
 
     args = parser.parse_args()
 
@@ -139,15 +139,16 @@ def edit_baseconfig():
     start = args.start
 
     # Set restart setting in config.base
-    if start == 'cold':
+    if start is None:
+      if res == 768:
+        exp_warm_start = '.true.'
+      else:
+        exp_warm_start = '.false.'
+    elif start == 'cold':
       exp_warm_start = '.false.'
     elif start == 'warm':
       exp_warm_start = '.true.'
 
-    # Check resolution for start setting - assume warm start if ops res
-    if res == 768:
-      exp_warm_start = '.true.'
-
     # Set FDATE (first full cycle)
     fdate = idate + timedelta(hours=6)
 

From 7a166da9f20a59729be39fff953bfe367987f046 Mon Sep 17 00:00:00 2001
From: "Kate.Friedman" <Kate.Friedman@noaa.gov>
Date: Wed, 14 Jul 2021 19:19:45 +0000
Subject: [PATCH 125/125] Reduce C768 npe_wav_gfs from ops 440 to dev 140

Refs: #353
---
 parm/config/config.fv3 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/parm/config/config.fv3 b/parm/config/config.fv3
index b7ecee8ab94..13362980885 100755
--- a/parm/config/config.fv3
+++ b/parm/config/config.fv3
@@ -111,7 +111,7 @@ case $case_in in
         export layout_x_gfs=16
         export layout_y_gfs=12
         export npe_wav=140
-        export npe_wav_gfs=440
+        export npe_wav_gfs=140
         export nth_fv3=4
         export nth_fv3_gfs=4
         export cdmbgwd="4.0,0.15,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling