ufs-community · DeniseWorthen · Jun 3, 2020 · Nov 20, 2019 · Nov 22, 2019 · Nov 22, 2019
diff --git a/.gitmodules b/.gitmodules
@@ -22,6 +22,10 @@
 	path = MOM6
 	url = https://github.com/NOAA-EMC/MOM6-interface
 	branch = develop
+[submodule "CMEPS"]
+	path = CMEPS
+	url = https://github.com/ESCOMP/CMEPS.git
+	branch = master
 [submodule "WW3"]
 	path = WW3
 	url = https://github.com/NOAA-EMC/WW3.git

diff --git a/CMEPS b/CMEPS
diff --git a/NEMS b/NEMS
diff --git a/compsets/cpld_fv3_ccpp_mom6_cice_2d_atm_flux.input b/compsets/cpld_fv3_ccpp_mom6_cice_2d_atm_flux.input
@@ -110,7 +110,9 @@ test cpld_fv3_ccpp_mom6_cice_2d_atm_flux: fv3_ccpp_mom6_cice.exe {
                     DUMPFREQ="@[DUMPFREQ]",
                     DT_THERM="@[DT_THERM_MOM6]",
                     DT_DYNAM="@[DT_DYNAM_MOM6]",
-           MOM6_RIVER_RUNOFF="@[MOM6_RIVER_RUNOFF]")
+           MOM6_RIVER_RUNOFF="@[MOM6_RIVER_RUNOFF]",
+               MOM6_REPRO_LA="@[MOM6_REPRO_LA]",
+            MOM6_THERMO_SPAN="@[MOM6_THERMO_SPAN]")
 
     # Specify output files:
     criteria output {

diff --git a/compsets/cpld_fv3_ccpp_mom6_cice_cold_atm_flux.input b/compsets/cpld_fv3_ccpp_mom6_cice_cold_atm_flux.input
@@ -107,7 +107,9 @@ test cpld_fv3_ccpp_mom6_cice_cold_atm_flux: fv3_ccpp_mom6_cice.exe {
                     DUMPFREQ="@[DUMPFREQ]",
                     DT_THERM="@[DT_THERM_MOM6]",
                     DT_DYNAM="@[DT_DYNAM_MOM6]",
-           MOM6_RIVER_RUNOFF="@[MOM6_RIVER_RUNOFF]")
+           MOM6_RIVER_RUNOFF="@[MOM6_RIVER_RUNOFF]",
+               MOM6_REPRO_LA="@[MOM6_REPRO_LA]",
+            MOM6_THERMO_SPAN="@[MOM6_THERMO_SPAN]")
 
     # Specify output files:
     criteria output {

diff --git a/compsets/fv3mom6cice5_ccpp.input b/compsets/fv3mom6cice5_ccpp.input
@@ -119,8 +119,12 @@ cpl_defaults = {
     ocn_model="mom6"
     ice_model="cice"
     wav_model="ww3"
-# future use: for MOM6 river runoff 
+# MOM6 river runoff 
     MOM6_RIVER_RUNOFF='True'
+# set USE_LA_LI2016 to the current default; this must be set False for restart repro
+    MOM6_REPRO_LA='True'
+# set the THERMO_SPANS_COUPLING to the current default; according to Gustavo and Alper, the correct setting is "False"
+    MOM6_THERMO_SPAN='True'
     NPROC_ICE='48'
     DT_ATMOS='900' #needed for C96 cases
     DT_DYNAM_MOM6='900'
@@ -151,7 +155,7 @@ cpl_defaults = {
 # embeded bash script to create input files at prerun from specification 
 # in compset
 
-embed bash edit_inputs(CDATE,ATMRES,DT_CICE,NPROC_ICE,RUNTYPE,USE_RESTART_TIME,FRAZIL_FWSALT,RUNID,CICE_HIST_AVG,DUMPFREQ_N,DUMPFREQ,DT_THERM,DT_DYNAM,MOM6_RIVER_RUNOFF) [[[ 
+embed bash edit_inputs(CDATE,ATMRES,DT_CICE,NPROC_ICE,RUNTYPE,USE_RESTART_TIME,FRAZIL_FWSALT,RUNID,CICE_HIST_AVG,DUMPFREQ_N,DUMPFREQ,DT_THERM,DT_DYNAM,MOM6_RIVER_RUNOFF, MOM6_REPRO_LA, MOM6_THERMO_SPAN) [[[ 
 
 python <<EOT
 import os
@@ -206,7 +210,9 @@ with open("INPUT/MOM_input_template",'rt') as inf:
    for x in inf.readlines():
     newline=x.replace("DT_THERM_MOM6","@[DT_THERM]") \
              .replace("DT_DYNAM_MOM6","@[DT_DYNAM]") \
-             .replace("MOM6_RIVER_RUNOFF","@[MOM6_RIVER_RUNOFF]")
+             .replace("MOM6_RIVER_RUNOFF","@[MOM6_RIVER_RUNOFF]") \
+             .replace("MOM6_REPRO_LA","@[MOM6_REPRO_LA]") \
+             .replace("MOM6_THERMO_SPAN","@[MOM6_THERMO_SPAN]") 
     outf.write(newline)
 
 #diag_table

diff --git a/conf/configure.fv3_coupled.cheyenne.intel b/conf/configure.fv3_coupled.cheyenne.intel
@@ -198,6 +198,10 @@ endif
 
 LDFLAGS += $(LIBS)
 
+ifeq ($(CMEPS),Y)
+CPPDEFS += -DCMEPS
+endif
+
 ifdef InNemsMakefile
 FFLAGS += $(ESMF_INC)
 CPPFLAGS += -DFV3_CPLD -traditional

diff --git a/conf/configure.fv3_coupled.cheyenne.intel-impi b/conf/configure.fv3_coupled.cheyenne.intel-impi
@@ -121,7 +121,7 @@ endif
 
 FFLAGS_OPT = -O2 -debug minimal -fp-model source -qoverride-limits -qopt-prefetch=3
 FFLAGS_REPRO = -O2 -debug minimal -fp-model consistent -qoverride-limits -g -traceback
-FFLAGS_DEBUG = -g -O0 -check all -check noarg_temp_created -check nopointer -warn -warn noerrors -fp-stack-check -fstack-protector-all -fpe0 -debug -traceback -ftrapuv
+FFLAGS_DEBUG = -g -O0 -check all -check noarg_temp_created -check nopointer -warn -warn noerrors -fp-stack-check -fstack-protector-all -fpe0 -debug -traceback -ftrapuv -link_mpi=dbg
 
 TRANSCENDENTALS := -fast-transcendentals
 FFLAGS_OPENMP = -qopenmp
@@ -192,6 +192,10 @@ CFLAGS += $(SIONLIB_INC)
 FFLAGS += $(SIONLIB_INC)
 endif
 
+ifeq ($(CMEPS),Y)
+CPPDEFS += -DCMEPS
+endif
+
 LDFLAGS += $(LIBS)
 
 ifdef InNemsMakefile

diff --git a/conf/configure.fv3_coupled.gaea.intel b/conf/configure.fv3_coupled.gaea.intel
@@ -94,7 +94,7 @@ endif
 
 FFLAGS_OPT = -O2 -debug minimal -fp-model source -qoverride-limits -qopt-prefetch=3
 FFLAGS_REPRO = -O2 -debug minimal -fp-model consistent -qoverride-limits -g -traceback
-FFLAGS_DEBUG = -g -O0 -check all -check noarg_temp_created -check nopointer -warn -warn noerrors -fp-stack-check -fstack-protector-all -fpe0 -debug -traceback -ftrapuv
+FFLAGS_DEBUG = -g -O0 -check all -check noarg_temp_created -check nopointer -warn -warn noerrors -fp-stack-check -fstack-protector-all -fpe0 -debug -traceback -ftrapuv -link_mpi=dbg        
 
 TRANSCENDENTALS := -fast-transcendentals
 FFLAGS_OPENMP = -qopenmp
@@ -159,6 +159,10 @@ FFLAGS += -I$(PATH_CCPP)/include
 LDFLAGS += -L$(PATH_CCPP)/lib -lccppphys -lccpp $(NCEPLIBS) -lxml2
 endif
 
+ifeq ($(CMEPS),Y)
+CPPDEFS += -DCMEPS
+endif
+
 ifneq ($(findstring netcdf/4,$(LOADEDMODULES)),)
   LIBS += -lnetcdff -lnetcdf -L$(HDF5)/lib -lhdf5_hl -lhdf5 -lz
 else

diff --git a/conf/configure.fv3_coupled.hera.intel b/conf/configure.fv3_coupled.hera.intel
@@ -168,6 +168,10 @@ FFLAGS += -I$(PATH_CCPP)/include
 LDFLAGS += -L$(PATH_CCPP)/lib -lccppphys -lccpp $(NCEPLIBS) -lxml2
 endif
 
+ifeq ($(CMEPS),Y)
+CPPDEFS += -DCMEPS
+endif
+
 LDFLAGS += $(LIBS)
 
 ifdef InNemsMakefile

diff --git a/conf/configure.fv3_coupled.orion.intel b/conf/configure.fv3_coupled.orion.intel
@@ -168,6 +168,10 @@ FFLAGS += -I$(PATH_CCPP)/include
 LDFLAGS += -L$(PATH_CCPP)/lib -lccppphys -lccpp $(NCEPLIBS) -lxml2
 endif
 
+ifeq ($(CMEPS),Y)
+CPPDEFS += -DCMEPS
+endif
+
 LDFLAGS += $(LIBS)
 
 ifdef InNemsMakefile

diff --git a/conf/configure.fv3_coupled.wcoss_cray b/conf/configure.fv3_coupled.wcoss_cray
@@ -151,6 +151,10 @@ FFLAGS += -I$(PATH_CCPP)/include
 LDFLAGS += -L$(PATH_CCPP)/lib -lccppphys -lccpp $(NCEPLIBS) -lxml2
 endif
 
+ifeq ($(CMEPS),Y)
+CPPDEFS += -DCMEPS
+endif
+
 ifneq ($(findstring netcdf/4,$(LOADEDMODULES)),)
   LIBS += -lnetcdff -lnetcdf -lhdf5_hl -lhdf5 -lz
 else

diff --git a/conf/configure.fv3_coupled.wcoss_dell_p3 b/conf/configure.fv3_coupled.wcoss_dell_p3
@@ -143,6 +143,10 @@ FFLAGS += -I$(PATH_CCPP)/include
 LDFLAGS += -L$(PATH_CCPP)/lib -lccppphys -lccpp $(NCEPLIBS) -lxml2
 endif
 
+ifeq ($(CMEPS),Y)
+CPPDEFS += -DCMEPS
+endif
+
 ifneq ($(findstring NetCDF/4,$(LOADEDMODULES)),)
   LIBS += -lnetcdff -lhdf5 $(Z_LIB)
 else

diff --git a/conf/configure.fv3_coupled.wcoss_phase2 b/conf/configure.fv3_coupled.wcoss_phase2
@@ -123,6 +123,10 @@ FFLAGS += $(FFLAGS_VERBOSE)
 LDFLAGS += $(LDFLAGS_VERBOSE)
 endif
 
+ifeq ($(CMEPS),Y)
+CPPDEFS += -DCMEPS
+endif
+
 ifneq ($(findstring NetCDF/4,$(LOADEDMODULES)),)
   LIBS += -lnetcdff -lhdf5 $(Z_LIB)
 else

diff --git a/coupledFV3_CCPP_MOM6_CICE_CMEPS.appBuilder b/coupledFV3_CCPP_MOM6_CICE_CMEPS.appBuilder
@@ -0,0 +1,7 @@
+# Coupled FV3-CCPP-MOM6-CICE-CMEPS
+
+## NEMS Application Builder file
+
+COMPONENTS=( FMS CCPP FV3 MOM6 CICE CMEPS )
+
+FV3_MAKEOPT="CCPP=Y SUITES=FV3_GFS_2017_coupled,FV3_GFS_2017_satmedmf_coupled,FV3_GFS_v15p2_coupled CMEPS=Y"
diff --git a/coupledFV3_CCPP_MOM6_CICE_CMEPS_debug.appBuilder b/coupledFV3_CCPP_MOM6_CICE_CMEPS_debug.appBuilder
@@ -0,0 +1,7 @@
+# Coupled FV3-CCPP-MOM6-CICE-CMEPS debug
+#
+## NEMS Application Builder file
+
+COMPONENTS=( FMS CCPP FV3 MOM6%DEBUG=Y CICE%DEBUG=Y CMEPS%DEBUG=Y)
+
+FV3_MAKEOPT="CCPP=Y DEBUG=Y SUITES=FV3_GFS_2017_coupled,FV3_GFS_2017_satmedmf_coupled,FV3_GFS_v15p2_coupled CMEPS=Y"
diff --git a/coupledFV3_CCPP_MOM6_CICE_WW3.appBuilder b/coupledFV3_CCPP_MOM6_CICE_WW3.appBuilder
@@ -1,4 +1,4 @@
-# Coupled FV3-CCPP-MOM6-CICE
+# Coupled FV3-CCPP-MOM6-CICE-WW3
 #
 ## NEMS Application Builder file
 

diff --git a/coupledFV3_CCPP_MOM6_CICE_WW3_CMEPS.appBuilder b/coupledFV3_CCPP_MOM6_CICE_WW3_CMEPS.appBuilder
@@ -0,0 +1,7 @@
+# Coupled FV3-CCPP-MOM6-CICE-WW3-CMEPS
+
+## NEMS Application Builder file
+
+COMPONENTS=( FMS CCPP FV3 MOM6 CICE CMEPS WW3 )
+
+FV3_MAKEOPT=" CCPP=Y SUITES=FV3_GFS_2017_coupled,FV3_GFS_2017_satmedmf_coupled,FV3_GFS_v15p2_coupled CMEPS=Y"
diff --git a/coupledFV3_CCPP_MOM6_CICE_debug.appBuilder b/coupledFV3_CCPP_MOM6_CICE_debug.appBuilder
@@ -1,4 +1,4 @@
-# Coupled FV3-CCPP-MOM6-CICE
+# Coupled FV3-CCPP-MOM6-CICE debug
 #
 ## NEMS Application Builder file
 

diff --git a/coupledFV3_MOM6_CICE_CMEPS.appBuilder b/coupledFV3_MOM6_CICE_CMEPS.appBuilder
@@ -0,0 +1,6 @@
+# Coupled FV3-MOM6-CICE-CMEPS
+#
+## NEMS Application Builder file
+
+COMPONENTS=( FMS FV3%CMEPS=Y MOM6 CICE CMEPS )
+
diff --git a/modulefiles/cheyenne.intel/fv3_coupled_debug b/modulefiles/cheyenne.intel/fv3_coupled_debug
@@ -0,0 +1,39 @@
+#%Module######################################################################
+##
+##    NEMS FV3 Prerequisites: Cheyenne/Intel
+
+proc ModulesHelp {} {
+  puts stderr "\tcit - loads modules required for building and running FV3 under NEMS on Cheyenne/Intel"
+}
+
+module-whatis "loads NEMS FV3 prerequisites for Cheyenne/Intel"
+
+# NOTE: the "module purge" and loading of the module command are
+# handled by the module-setup.sh (or .csh) script.
+
+##
+## load programming environment
+## this typically includes compiler, MPI and job scheduler
+##
+module load ncarenv/1.3
+module load intel/19.0.2
+module load mpt/2.19
+module load ncarcompilers/0.5.0
+module load netcdf/4.6.3
+
+module use -a /glade/p/ral/jntp/GMTB/tools/modulefiles/intel-19.0.2/mpt-2.19
+
+##
+## use pre-compiled EMSF library for above compiler / MPI combination
+##
+module load esmf/8.0.0-debug
+
+##
+## NCEP libraries (download and build manually for the time being; https://github.com/NCAR/NCEPlibs)
+##
+module load NCEPlibs/9.9.9
+
+##
+## SIONlib library
+##
+module load SIONlib/1.7.4
diff --git a/parm/MOM_input_template b/parm/MOM_input_template
@@ -31,7 +31,7 @@ DT_THERM = DT_THERM_MOM6        !   [s] default = 900.0
                                 ! THERMO_SPANS_COUPLING is true, in which case DT_THERM
                                 ! can be an integer multiple of the coupling timestep.  By
                                 ! default DT_THERM is set to DT.
-THERMO_SPANS_COUPLING = True    !   [Boolean] default = False
+THERMO_SPANS_COUPLING = MOM6_THERMO_SPAN   !   [Boolean] default = False
                                 ! If true, the MOM will take thermodynamic and tracer
                                 ! timesteps that can be longer than the coupling timestep.
                                 ! The actual thermodynamic timestep that is used in this
@@ -773,7 +773,7 @@ EPBL_TRANSITION_SCALE = 0.01    !   [nondim] default = 0.1
                                 ! A scale for the mixing length in the transition layer
                                 ! at the edge of the boundary layer as a fraction of the
                                 ! boundary layer thickness.  The default is 0.1.
-USE_LA_LI2016 = True            !   [nondim] default = False
+USE_LA_LI2016 = MOM6_REPRO_LA   !   [nondim] default = False
                                 ! A logical to use the Li et al. 2016 (submitted) formula to
                                 !  determine the Langmuir number.
 LT_ENHANCE = 3                  !   [nondim] default = 0

diff --git a/parm/fd_nems.yaml b/parm/fd_nems.yaml
@@ -0,0 +1 @@
+../CMEPS/mediator/fd_nems.yaml
diff --git a/parm/ice_in_template b/parm/ice_in_template
@@ -14,9 +14,9 @@
   , use_restart_time = USE_RESTART_TIME
   , restart_format = 'nc'
   , lcdf64         = .false.
-  , restart_dir    = './restart/'
+  , restart_dir    = './RESTART/'
   , restart_file   = 'iced'
-  , pointer_file   = './restart/ice.restart_file'
+  , pointer_file   = './ice.restart_file'
   , dumpfreq       = 'DUMPFREQ'
   , dumpfreq_n     =  DUMPFREQ_N
   , dump_last      = .false.  

diff --git a/parm/input.benchmark.nml.IN b/parm/input.benchmark.nml.IN
@@ -47,7 +47,7 @@
   ntiles   = 6,
   npz    = @[NPZ]
   grid_type = -1
-  make_nh = .true.
+  make_nh = @[MAKE_NH]
   fv_debug = .false.
   range_warn = .false.
   reset_eta = .false.
@@ -71,7 +71,7 @@
   k_split = 2
   n_split = 6
   nwat = 6
-  na_init = 1
+  na_init = @[NA_INIT]
   d_ext = 0.
   dnats = 1
   fv_sg_adj = 450
@@ -83,11 +83,11 @@
   delt_max = 0.002
   ke_bg = 0.
   do_vort_damp = .true.
-  external_ic = .true.
+  external_ic = @[EXTERNAL_IC]
   external_eta = .true.
   gfs_phil = .false.
-  nggps_ic = .true.
-  mountain = .false.
+  nggps_ic = @[NGGPS_IC]
+  mountain = @[MOUNTAIN]
   ncep_ic = .false.
   d_con = 1.
   hord_mt = 5
@@ -102,7 +102,7 @@
   fill = .true.
   dwind_2d = .false.
   print_freq = 6
-  warm_start = .false.
+  warm_start = @[WARM_START]
   no_dycore = .false.
   z_tracer = .true.
   agrid_vel_rst = .true.
@@ -320,7 +320,7 @@
          output_directory = 'MOM6_OUTPUT/',
          input_filename = '@[MOM6_RESTART_SETTING]'
          restart_input_dir = 'INPUT/',
-         restart_output_dir = 'MOM6_RESTART/',
+         restart_output_dir = 'RESTART/',
          parameter_filename = 'INPUT/MOM_input',
                               'INPUT/MOM_override' /
 
diff --git a/parm/input.benchmark_ccpp.nml.IN b/parm/input.benchmark_ccpp.nml.IN
@@ -48,7 +48,7 @@
   ntiles   = 6,
   npz    = @[NPZ]
   grid_type = -1
-  make_nh = .true.
+  make_nh = @[MAKE_NH]
   fv_debug = .false.
   range_warn = .false.
   reset_eta = .false.
@@ -72,7 +72,7 @@
   k_split = 2
   n_split = 6
   nwat = 6
-  na_init = 1
+  na_init = @[NA_INIT]
   d_ext = 0.
   dnats = 1
   fv_sg_adj = 450
@@ -84,11 +84,11 @@
   delt_max = 0.002
   ke_bg = 0.
   do_vort_damp = .true.
-  external_ic = .true.
+  external_ic = @[EXTERNAL_IC]
   external_eta = .true.
   gfs_phil = .false.
-  nggps_ic = .true.
-  mountain = .false.
+  nggps_ic = @[NGGPS_IC]
+  mountain = @[MOUNTAIN]
   ncep_ic = .false.
   d_con = 1.
   hord_mt = 5
@@ -103,7 +103,7 @@
   fill = .true.
   dwind_2d = .false.
   print_freq = 6
-  warm_start = .false.
+  warm_start = @[WARM_START]
   no_dycore = .false.
   z_tracer = .true.
   agrid_vel_rst = .true.
@@ -323,7 +323,7 @@
          output_directory = 'MOM6_OUTPUT/',
          input_filename = '@[MOM6_RESTART_SETTING]'
          restart_input_dir = 'INPUT/',
-         restart_output_dir = 'MOM6_RESTART/',
+         restart_output_dir = 'RESTART/',
          parameter_filename = 'INPUT/MOM_input',
                               'INPUT/MOM_override' /
+2 −4		src/incmake/component_CCPP.mk
+61 −0		src/incmake/component_CICE6.mk
+1 −0		src/incmake/component_CMEPS.mk
+16 −0		src/module_EARTH_GRID_COMP.F90