NCAR · climbfuji · Feb 3, 2020 · Dec 18, 2019 · Dec 27, 2019 · Jan 10, 2020
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1 @@
+coupled*.mk
diff --git a/.gitmodules b/.gitmodules
@@ -4,8 +4,8 @@
 	branch = dtc/develop
 [submodule "NEMS"]
 	path = NEMS
-	url = https://github.com/NCAR/NEMS
-	branch = dtc/develop
+	url = https://github.com/NOAA-EMC/NEMS
+	branch = develop
 [submodule "FMS"]
 	path = FMS
 	url = https://github.com/NOAA-GFDL/FMS

diff --git a/FV3 b/FV3
diff --git a/NEMS b/NEMS
diff --git a/WW3 b/WW3
diff --git a/cmake/configure_cheyenne.intel-impi.cmake b/cmake/configure_cheyenne.intel-impi.cmake
@@ -0,0 +1,35 @@
+message("")
+message("Setting configuration for $ENV{CMAKE_Platform}")
+message("")
+
+get_filename_component (C_COMPILER_NAME ${CMAKE_C_COMPILER} NAME)
+get_filename_component (CXX_COMPILER_NAME ${CMAKE_CXX_COMPILER} NAME)
+get_filename_component (Fortran_COMPILER_NAME ${CMAKE_Fortran_COMPILER} NAME)
+message("C       compiler: ${CMAKE_C_COMPILER_ID} ${CMAKE_C_COMPILER_VERSION} (${C_COMPILER_NAME})")
+message("CXX     compiler: ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION} (${CXX_COMPILER_NAME})")
+message("Fortran compiler: ${CMAKE_Fortran_COMPILER_ID} ${CMAKE_Fortran_COMPILER_VERSION} (${Fortran_COMPILER_NAME})")
+message("")
+
+option(DEBUG   "Enable DEBUG mode" OFF)
+option(REPRO   "Enable REPRO mode" OFF)
+option(VERBOSE "Enable VERBOSE mode" OFF)
+option(32BIT   "Enable 32BIT (single precision arithmetic in dycore)" OFF)
+option(OPENMP  "Enable OpenMP threading" ON)
+option(AVX2    "Enable AVX2 instruction set" ON)
+
+option(INLINE_POST "Enable inline post" OFF)
+
+include( cmake/${CMAKE_Fortran_COMPILER_ID}.cmake )
+
+set(NEMSIO_INC $ENV{NEMSIO_INC})
+set(POST_INC $ENV{POST_INC})
+set(NCEP_LIBS $ENV{POST_LIB} $ENV{NEMSIO_LIB} $ENV{G2_LIB4} $ENV{G2TMPL_LIB} $ENV{BACIO_LIB4} $ENV{SP_LIBd} $ENV{W3EMC_LIBd} $ENV{W3NCO_LIBd} $ENV{CRTM_LIB} $ENV{PNG_LIB} $ENV{JASPER_LIB} $ENV{Z_LIB})
+
+set(ESMF_MOD ${ESMF_F90COMPILEPATHS})
+set(ESMF_LIBS "${ESMF_F90ESMFLINKRPATHS} ${ESMF_F90ESMFLINKPATHS} ${ESMF_F90ESMFLINKLIBS}")
+
+set(NETCDF_INC_DIR $ENV{NETCDF}/include)
+set(NETCDF_LIBDIR $ENV{NETCDF}/lib)
+set(NETCDF_LIBS -L$ENV{NETCDF}/lib -lnetcdff -lnetcdf)
+
+message("")
diff --git a/cmake/configure_gaea.cmake b/cmake/configure_gaea.cmake
@@ -17,6 +17,8 @@ option(32BIT   "Enable 32BIT (single precision arithmetic in dycore)" OFF)
 option(OPENMP  "Enable OpenMP threading" ON)
 option(AVX2    "Enable AVX2 instruction set" ON)
 
+option(INLINE_POST "Enable inline post" OFF)
+
 include( cmake/${CMAKE_Fortran_COMPILER_ID}.cmake )
 
 set(NEMSIO_INC $ENV{NEMSIO_INC})

diff --git a/compsets/all.input b/compsets/all.input
@@ -3,7 +3,7 @@ load 'fv3.input'
 run fv3_control                 @ fv3, standard, baseline, ccpptrans
 run fv3_decomp                  @ fv3, standard
 run fv3_2threads                @ fv3, standard
-run fv3_ccpp_control            @ fv3, standard, ccpptrans, plat==hera.intel
+#run fv3_ccpp_control            @ fv3, standard, ccpptrans, plat==hera.intel
 run fv3_restart                 @ fv3, standard, baseline
 run fv3_read_inc                @ fv3, standard, baseline
 run fv3_gfdlmp                  @ fv3, standard, baseline
@@ -22,7 +22,7 @@ run fv3_wrtGauss_netcdf_esmf    @ fv3, standard, baseline
 run fv3_wrtGauss_netcdf         @ fv3, standard, baseline
 run fv3_wrtGauss_nemsio         @ fv3, standard, baseline
 
-run fv3_wrtGauss_nemsio_c768    @ fv3,           baseline, ww3, plat==hera.intel, plat==wcoss.cray, plat==wcoss_dell_p3
+run fv3_wrtGauss_nemsio_c768    @ fv3,           baseline, ww3, plat==hera.intel
 run fv3_wrtGauss_nemsio_c192    @ fv3, standard, baseline
 run fv3_stochy                  @ fv3, standard, baseline
 run fv3_iau                     @ fv3, standard, baseline

diff --git a/compsets/fv3.input b/compsets/fv3.input
@@ -1764,10 +1764,11 @@ test fv3_wrtGauss_nemsio_c768: fv3_ww3.exe {
     TEST_DESCR="Compare c768 FV3-WW3 Gaussian grid nemsio output results with previous trunk version"
     CNTL_NAME="fv3_wrtGauss_nemsio_c768"
 
-    TASKS='1752'
+    TASKS='1470'
     PPN="@[plat%C768_PPN]"
+    THRD="@[plat%C768_THRD]"
     INPES='16'
-    JNPES='16'
+    JNPES='12'
     NPX='769'
     NPY='769'
     NPZ='64'
@@ -1779,7 +1780,7 @@ test fv3_wrtGauss_nemsio_c768: fv3_ww3.exe {
     FHMAX='06'
     WLCLK='30'
     WRITE_GROUP='3'
-    WRTTASK_PER_GROUP='64'
+    WRTTASK_PER_GROUP='36'
     FDIAG='0,1,2,3,4,5,6,7,8,9,10,11,12'
     OUTPUT_GRID="'gaussian_grid'"
     OUTPUT_FILE="'nemsio'"
@@ -1805,9 +1806,9 @@ test fv3_wrtGauss_nemsio_c768: fv3_ww3.exe {
 
     nems_configure="blocked_atm_wav"
     atm_model='fv3'
-    atm_petlist_bounds="0 1727"
+    atm_petlist_bounds="0 1259"
     wav_model='ww3'
-    wav_petlist_bounds="1728 1751"
+    wav_petlist_bounds="1260 1469"
     coupling_interval_sec=1800.0  # 30min coupling, fv3 time step
 
     # The build variable is the fv3_ww3.exe, which
@@ -1834,6 +1835,7 @@ test fv3_wrtGauss_nemsio_c768: fv3_ww3.exe {
      'nems.configure' <=atparse=   "@[PARMnems]/nems.configure.@[nems_configure].IN"
           'mod_def.*' <=copyfrom=  "@[WW3_input_data]"
       'ww3_multi.inp' <=copyfrom=  "@[WW3_input_data]/@[SYEAR]@[SMONTH]@[SDAY]"
+      'rmp_src*' <=copyfrom=  "@[WW3_input_data]/@[SYEAR]@[SMONTH]@[SDAY]"
     }
 
     # Specify output files:
@@ -1879,7 +1881,9 @@ test fv3_wrtGauss_nemsio_c768: fv3_ww3.exe {
         'RESTART/sfc_data.tile4.nc'             .bitcmp. "@[CNTL]/RESTART/"
         'RESTART/sfc_data.tile5.nc'             .bitcmp. "@[CNTL]/RESTART/"
         'RESTART/sfc_data.tile6.nc'             .bitcmp. "@[CNTL]/RESTART/"
-        'out_grd.glo_30m'                       .bitcmp. "@[CNTL]"
+        'out_grd.glo_10m'                       .bitcmp. "@[CNTL]"
+        'out_grd.ant_9km'                       .bitcmp. "@[CNTL]"
+        'out_grd.aoc_9km'                       .bitcmp. "@[CNTL]"
       # Executable validation.  This makes an MD5 sum of the fv3.exe
       # for comparison against the MD5 sum made in the build job.
       # This is to ensure the executable did not change during the
@@ -1888,7 +1892,7 @@ test fv3_wrtGauss_nemsio_c768: fv3_ww3.exe {
     }
 
     spawn execute {
-        {"@[build%target]", ranks="@[TASKS]", ppn="@[PPN]" }
+        {"@[build%target]", ranks="@[TASKS]", threads="@[THRD]", ppn="@[PPN]" }
     }
 }
 
@@ -2006,6 +2010,7 @@ test fv3_gfdlmprad_32bit_post: fv3_32bit.exe {
          'model_configure' <=atparse=  "@[PARMnems]/model_configure.IN"
                     'itag' <=atparse=  "@[PARMnems]/post_itag"
       'postxconfig-NT.txt' <=atparse=  "@[PARMnems]/postxconfig-NT.txt"
+      'postxconfig-NT_FH00.txt' <=atparse=  "@[PARMnems]/postxconfig-NT_FH00.txt"
     'params_grib2_tbl_new' <=atparse=  "@[PARMnems]/params_grib2_tbl_new"
                        'aerosol.dat' <=copyfrom= "@[FV3_input_data]/INPUT"
         'co2historicaldata_201*.txt' <=copyfrom= "@[FV3_input_data]/INPUT"
@@ -2033,7 +2038,9 @@ test fv3_gfdlmprad_32bit_post: fv3_32bit.exe {
         'phyf024.nemsio'         .bitcmp. "@[CNTL]"
         'dynf000.nemsio'         .bitcmp. "@[CNTL]"
         'dynf024.nemsio'         .bitcmp. "@[CNTL]"
+        'GFSFLX.GrbF00'          .bitcmp. "@[CNTL]"
         'GFSPRS.GrbF00'          .bitcmp. "@[CNTL]"
+        'GFSFLX.GrbF24'          .bitcmp. "@[CNTL]"
         'GFSPRS.GrbF24'          .bitcmp. "@[CNTL]"
         'RESTART/coupler.res'                   .bitcmp. "@[CNTL]/RESTART/"
         'RESTART/fv_core.res.nc'                .bitcmp. "@[CNTL]/RESTART/"

diff --git a/compsets/hera.input b/compsets/hera.input
@@ -16,7 +16,8 @@ platform hera.intel {
 
     cores_per_node=40 # Number of cores per node on compute nodes
     cpus_per_core=2
-    C768_PPN=30
+    C768_PPN=10
+    C768_THRD=1
 
  # hera testing: using longer run time
     DEFAULT_TEST_WALLTIME=3000
@@ -30,8 +31,8 @@ platform hera.intel {
     # INPUTS is the input directory, which should contain fix and parm
     # files, plus any restarts or other inputs.
 
-    BASELINE="/scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/trunk-20191127"
-    BASELINE_TEMPLATE="/scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/trunk-20191127"
+    BASELINE="/scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200115"
+    BASELINE_TEMPLATE="/scratch1/NCEPDEV/nems/emc.nemspara/RT/NEMSfv3gfs/develop-20200115"
     INPUTS="@[BASELINE]"
 
     default_resources={

diff --git a/compsets/platforms.input b/compsets/platforms.input
@@ -26,6 +26,7 @@ platform.defaults={
     walltime="@[DEFAULT_TEST_WALLTIME]"
 
     C768_PPN="@[cores_per_node]"
+    C768_THRD=1
     execution_time_modules=""
 }
 

diff --git a/compsets/wcoss.input b/compsets/wcoss.input
@@ -17,6 +17,8 @@ platform wcoss.phase2 {
     pex='1'
     cores_per_node=24
     cpus_per_core=2
+    C768_PPN=6
+    C768_THRD=1
 
     # WCOSS Phase 2 has been slow of late.
     DEFAULT_TEST_WALLTIME=3000
@@ -26,8 +28,8 @@ platform wcoss.phase2 {
     LONG_TEST_QUEUE='&LONGQ;'
     BUILD_QUEUE='&BUILDQ;'
     MACHINE_ID='wcoss.phase2'
-    BASELINE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/trunk-20191127"
-    BASELINE_TEMPLATE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/trunk-20191127"
+    BASELINE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200115"
+    BASELINE_TEMPLATE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200115"
 
     default_resources={
         TASKS=156
@@ -120,9 +122,12 @@ platform wcoss.phase1 {
     pex='1'
     cores_per_node=16
     cpus_per_core=2
+    C768_PPN=6
+    C768_THRD=1
+
     MACHINE_ID='wcoss.phase1'
-    BASELINE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/trunk-20191127"
-    BASELINE_TEMPLATE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/trunk-20191127"
+    BASELINE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200115"
+    BASELINE_TEMPLATE="/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200115"
 
     # WCOSS Phase 2 has been slow of late.
     DEFAULT_TEST_WALLTIME=2700
@@ -197,25 +202,27 @@ platform wcoss_dell_p3 {
     pex='3'
     cores_per_node=28
     cpus_per_core=2
+    C768_PPN=7
+    C768_THRD=1
 
-    DEFAULT_TEST_WALLTIME=1800
+    DEFAULT_TEST_WALLTIME=3000
 
     MPI='LSF_IMPI'
     MACHINE_ID='wcoss_dell_p3'
     SHORT_TEST_QUEUE='&SHORTQ;'
     LONG_TEST_QUEUE='&LONGQ;'
     BUILD_QUEUE='&BUILDQ;'
 
-    BASELINE="/gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/trunk-20191127"
-    BASELINE_TEMPLATE="/gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/trunk-20191127"
+    BASELINE="/gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200115/"
+    BASELINE_TEMPLATE="/gpfs/dell2/emc/modeling/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200115/"
 
     default_resources={
         TASKS=156
         WRTTASK_PER_GROUP=12
-        TPN=8
+        TPN=6
         INPES=3
         JNPES=8
-        THRD=1
+        THRD=4
         atm_petlist_bounds="0 155"
         wav_petlist_bounds="156 263"
         CPLTASKS=264
@@ -295,6 +302,9 @@ platform wcoss.cray {
     pex='cray'
     cores_per_node=24
     cpus_per_core=2
+    C768_PPN=6
+    C768_THRD=1
+
     MPI='lsf_cray_intel'
     SHORT_TEST_QUEUE='&SHORTQ;'
     LONG_TEST_QUEUE='&LONGQ;'
@@ -304,8 +314,8 @@ platform wcoss.cray {
     BUILD_WALLTIME="3600"
     DEFAULT_TEST_WALLTIME=1800
 
-    BASELINE="/gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/trunk-20191127"
-    BASELINE_TEMPLATE="/gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/trunk-20191127"
+    BASELINE="/gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200115"
+    BASELINE_TEMPLATE="/gpfs/hps3/emc/nems/noscrub/emc.nemspara/RT/NEMSfv3gfs/develop-20200115"
 
     execution_time_modules=[[[
       module load alps

diff --git a/conf/configure.fv3.gaea.intel b/conf/configure.fv3.gaea.intel
@@ -66,6 +66,10 @@ else
 CPPDEFS += -DMOIST_CAPPA -DUSE_COND
 endif
 
+ifeq ($(NAM_phys),Y)
+CPPDEFS += -DNAM_phys
+endif
+
 ifeq ($(32BIT),Y)
 CPPDEFS += -DOVERLOAD_R4 -DOVERLOAD_R8
 FFLAGS += -i4 -real-size 32

diff --git a/conf/configure.fv3.macosx.gnu b/conf/configure.fv3.macosx.gnu
@@ -1,7 +1,7 @@
 ## NEMS configuration file
 ##
 ## Platform: Darwin Mac OS X
-## Compiler: GNU (clang/gfortran) with MPICH
+## Compiler: GNU (gcc/clang+gfortran) with MPICH
 
 SHELL=/bin/sh
 

diff --git a/conf/configure.fv3.wcoss_phase1 b/conf/configure.fv3.wcoss_phase1
@@ -43,7 +43,7 @@ CFLAGS := $(INCLUDE)
 FFLAGS := $(INCLUDE) -fno-alias -auto -safe-cray-ptr -ftz -assume byterecl -nowarn -sox -align array64byte
 
 CPPDEFS += -Duse_libMPI -Duse_netCDF -DSPMD -DUSE_LOG_DIAG_FIELD_INFO -Duse_LARGEFILE -DUSE_GFSL63 -DGFS_PHYS -Duse_WRTCOMP
-CPPDEFS += -DNEW_TAUCTMAX -DINTERNAL_FILE_NML
+CPPDEFS += -DNEW_TAUCTMAX -DINTERNAL_FILE_NML -DNO_INLINE_POST
 
 ifeq ($(HYDRO),Y)
 CPPDEFS +=

diff --git a/conf/configure.fv3.wcoss_phase2 b/conf/configure.fv3.wcoss_phase2
@@ -43,7 +43,7 @@ CFLAGS := $(INCLUDE)
 FFLAGS := $(INCLUDE) -fno-alias -auto -safe-cray-ptr -ftz -assume byterecl -nowarn -sox -align array64byte
 
 CPPDEFS += -Duse_libMPI -Duse_netCDF -DSPMD -DUSE_LOG_DIAG_FIELD_INFO -Duse_LARGEFILE -DUSE_GFSL63 -DGFS_PHYS -Duse_WRTCOMP
-CPPDEFS += -DNEW_TAUCTMAX -DINTERNAL_FILE_NML
+CPPDEFS += -DNEW_TAUCTMAX -DINTERNAL_FILE_NML -DNO_INLINE_POST
 
 ifeq ($(HYDRO),Y)
 CPPDEFS +=
+1 −1		atmos_cubed_sphere
+29 −7		atmos_model.F90
+1 −1		ccpp/framework
+1 −1		ccpp/physics
+12 −6		fv3_cap.F90
+15 −12		gfsphysics/GFS_layer/GFS_physics_driver.F90
+30 −8		gfsphysics/GFS_layer/GFS_radiation_driver.F90
+1 −1		gfsphysics/physics/gfdl_cloud_microphys.F90
+0 −0		gfsphysics/physics/module_sf_noahmp_glacier.f90
+0 −0		gfsphysics/physics/module_sf_noahmplsm.f90
+0 −0		gfsphysics/physics/module_wrf_utl.f90
+5 −1		gfsphysics/physics/moninedmf_hafs.f
+0 −0		gfsphysics/physics/noahmp_tables.f90
+14 −14		gfsphysics/physics/samfdeepcnv.f
+7 −4		gfsphysics/physics/satmedmfvdifq.f
+0 −0		gfsphysics/physics/sfc_noahmp_drv.f
+1 −0		gfsphysics/physics/sflx.f
+3 −3		io/FV3GFS_io.F90
+1 −1		io/module_write_nemsio.F90
+61 −54		io/module_write_netcdf.F90
+42 −22		io/module_wrt_grid_comp.F90
+77 −16		io/post_gfs.F90
+74 −35		module_fcst_grid_comp.F90
+4 −3		module_fv3_config.F90