Bring exchange grid support to UFS

CMEPS-interface/CMakeLists.txt

@@ -1,4 +1,7 @@
-# CMEPS compiler flags
+###############################################################################
+### CMEPS compiler flags
+###############################################################################
+
 if(CMAKE_Fortran_COMPILER_ID MATCHES "GNU")
   set(CMAKE_Fortran_FLAGS "-g -fbacktrace -ffree-line-length-none")
   if(${CMAKE_Fortran_COMPILER_VERSION} VERSION_GREATER_EQUAL 10)
@@ -18,6 +21,10 @@ else()
   message(WARNING "Fortran compiler with ID ${CMAKE_Fortran_COMPILER_ID} will be used with CMake default options")
 endif()
 
+###############################################################################
+### Source files
+###############################################################################
+
 list(APPEND _nems_util_files
   CMEPS/ufs/flux_atmocn_mod.F90
   CMEPS/ufs/glc_elevclass_mod.F90
@@ -62,12 +69,104 @@ list(APPEND _mediator_files
   CMEPS/mediator/med_phases_post_rof_mod.F90
   CMEPS/mediator/med_phases_post_wav_mod.F90)
 
-### Create target library and set PUBLIC interfaces on the library
-add_library(cmeps STATIC ${_nems_util_files} ${_mediator_files})
+###############################################################################
+### Host model for CCPP
+###############################################################################
+
+if(CMEPS_AOFLUX)
+  ### Query CCPP path
+  get_filename_component(FV3_PATH "../FV3" REALPATH)
+
+  ### If FV3 is available build CCPP host model under CMEPS too
+  if(EXISTS ${FV3_PATH})
+    ### Print out information
+    message("FV3_PATH = ${FV3_PATH}")
+    message("Compiling CMEPS with CCPP support.")
+
+    ### Set environment variable for FV3 directory, used by config file
+    set(ENV{FV3_PATH} ${FV3_PATH})
+
+    ### Set suite file that needs to be compiled
+    set(_ccpp_suites_arg "--suites=FV3_sfc_ocean")
+
+    ### Set options for ccpp_prebuild.py
+    if(DEBUG)
+      ### Enable debugging features in auto-generated physics caps
+      set(_ccpp_debug_arg "--debug")
+      ### Enable verbose output from ccpp_prebuild.py
+      set(_ccpp_verbose_arg "--verbose")
+    endif()
+
+    ### Trigger ccpp_prebuild.py script to connect host model with CCPP physics
+    message("Calling CCPP code generator (ccpp_prebuild.py) for suites ${_ccpp_suites_arg} ...")
+    execute_process(
+                    COMMAND ${Python_EXECUTABLE} 
+                    "${FV3_PATH}/ccpp/framework/scripts/ccpp_prebuild.py"
+                    "--config=CMEPS/ufs/ccpp/config/ccpp_prebuild_config.py"
+                    "--builddir=${CMAKE_CURRENT_BINARY_DIR}"
+                    "--namespace=med"
+                    ${_ccpp_suites_arg}
+                    ${_ccpp_debug_arg}
+                    ${_ccpp_verbose_arg}
+                    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+                    OUTPUT_FILE ${CMAKE_CURRENT_BINARY_DIR}/ccpp_prebuild.out
+                    ERROR_FILE  ${CMAKE_CURRENT_BINARY_DIR}/ccpp_prebuild.err
+                    RESULT_VARIABLE return_code)
+
+    ### Check return code from CCPP prebuild.py
+    if(return_code EQUAL 0)
+      message (STATUS "CCPP prebuild step completed successfully")
+    else(return_code EQUAL 0)
+      message (FATAL_ERROR "CCPP prebuild step failed, check ${CMAKE_CURRENT_BINARY_DIR}/ccpp_prebuild.{out,err}")
+    endif(return_code EQUAL 0)
+
+    ### Add extra driver file to the source
+    list(APPEND _nems_util_files
+      CMEPS/ufs/ccpp/driver/med_ccpp_driver.F90
+      CMEPS/ufs/flux_atmocn_ccpp_mod.F90
+      CMEPS/ufs/ufs_io_mod.F90)
+
+    ### Set the sources: type definitions
+    include(${CMAKE_CURRENT_BINARY_DIR}/physics/CCPP_TYPEDEFS.cmake)
+    foreach(typedef_module ${TYPEDEFS})
+      list(APPEND MODULES_F90 ${CMAKE_CURRENT_BINARY_DIR}/${typedef_module})
+    endforeach()
+
+    ### Set the sources: physics schemes
+    include(${CMAKE_CURRENT_BINARY_DIR}/physics/CCPP_SCHEMES.cmake)
+
+    ### Set the sources: static API
+    include(${CMAKE_CURRENT_BINARY_DIR}/physics/CCPP_STATIC_API.cmake)
+
+    ### Set the sources: cap
+    include(${CMAKE_CURRENT_BINARY_DIR}/physics/CCPP_CAPS.cmake)
+
+    ### Generate list of Fortran modules from defined sources
+    foreach(source_f90 ${CAPS} ${API})
+      get_filename_component(tmp_source_f90 ${source_f90} NAME)
+      string(REGEX REPLACE ".F90" ".mod" tmp_module_f90 ${tmp_source_f90})
+      string(TOLOWER ${tmp_module_f90} module_f90)
+      list(APPEND MODULES_F90 ${CMAKE_CURRENT_BINARY_DIR}/${module_f90})
+    endforeach()
+
+    ### Add special CPP flag for CMEPS/CCPP support
+    add_definitions(-DUFS_AOFLUX)
+  endif()
+endif()
+
+###############################################################################
+### Libraries
+###############################################################################
+
+### Create target library
+add_library(cmeps STATIC ${_nems_util_files} ${_mediator_files} ${SCHEMES} ${CAPS} ${API})
+
+### Set PUBLIC interfaces on the library
 set_target_properties(cmeps PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod)
 target_include_directories(cmeps PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/mod>
                                         $<INSTALL_INTERFACE:mod>)
-target_link_libraries(cmeps PUBLIC esmf
+target_link_libraries(cmeps PUBLIC fms
+                                   esmf
                                    PIO::PIO_Fortran)
 if(OpenMP_Fortran_FOUND)
   target_link_libraries(cmeps PRIVATE OpenMP::OpenMP_Fortran)

CMakeLists.txt

@@ -66,6 +66,7 @@ set(MOVING_NEST     OFF CACHE BOOL "Enable moving nest code")
 set(OPENMP          ON  CACHE BOOL "Enable OpenMP threading")
 set(PARALLEL_NETCDF OFF CACHE BOOL "Enable parallel NetCDF")
 set(JEDI_DRIVER     OFF CACHE BOOL "Enable JEDI as top level driver")
+set(CMEPS_AOFLUX    OFF CACHE BOOL "Enable atmosphere-ocean flux calculation in mediator") 
 
 set(CMAKE_Platform $ENV{CMAKE_Platform})
 if(CMAKE_Platform)
@@ -90,6 +91,8 @@ message("MOVING_NEST ...... ${MOVING_NEST}")
 message("OPENMP ........... ${OPENMP}")
 message("PARALLEL_NETCDF .. ${PARALLEL_NETCDF}")
 message("JEDI_DRIVER ...... ${JEDI_DRIVER}")
+message("CMEPS_AOFLUX ..... ${CMEPS_AOFLUX}")
+
 message("")
 
 ###############################################################################

doc/UsersGuide/source/BuildingAndRunning.rst

@@ -146,6 +146,12 @@ For the ufs-weather-model S2S app (atm/ice/ocean) with debugging flags turned on
     export CMAKE_FLAGS="-DAPP=S2S -DDEBUG=ON -DCCPP_SUITES=FV3_GFS_2017_coupled,FV3_GFS_2017_satmedmf_coupled,FV3_GFS_v15p2_coupled,FV3_GFS_v16_coupled,FV3_GFS_v16_couplednsst"
     export BUILD_VERBOSE=1
 
+For the ufs-weather-model S2S app (atm/ice/ocean) with activating CCPP host model under CMEPS and receiving atmosphere-ocean fluxes from mediator:
+
+.. code-block:: console
+
+    export CMAKE_FLAGS="-DAPP=S2S -DCCPP_SUITES=FV3_GFS_v17_coupled_p8_sfcocn -DCMEPS_AOFLUX=ON"
+
 For the ufs-weather-model S2SW app (atm/ice/ocean/wave):
 
 .. code-block:: console

doc/UsersGuide/source/InputNML.inc

@@ -299,3 +299,42 @@ A sample of this namelist is shown below:
    /
 
 The namelist section relating to the FMS diagnostic manager ``&diag_manager_nml`` is described in :numref:`Section %s <DiagManagerNML>`.
+
+---------------
+gfs_physics_nml
+---------------
+
+The namelist section ``&gfs_physics_nml`` contains physics-related information used by the atmosphere model and
+some of the variables are only relevant for specific parameterizations and/or configurations.
+The small set of variables used in ``&gfs_physics_nml`` are described in :numref:`Table %s <gfs_physics_nml>`.
+
+.. _gfs_physics_nml:
+
+.. list-table:: *List of common variables in the *gfs_physics_nml* namelist section.
+   :widths: 10 40 15 15
+   :header-rows: 1
+
+   * - Variable Name
+     - Description
+     - Data Type
+     - Default Value
+   * - cplflx
+     - Flag to activate atmosphere-ocean coupling. If true, turn on receiving exchange fields from other components such as ocean.
+     - logical
+     - .false.
+   * - use_med_flux
+     - Flag to receive atmosphere-ocean fluxes from mediator. If true, atmosphere-ocean fluxes will be received into the CCPP physics and used there, instead of calculating them.
+     - logical
+     - .false.
+
+A sample subset of this namelist is shown below:
+
+.. code-block:: console
+
+   &gfs_physics_nml
+     use_med_flux = .true.
+     cplflx       = .true.
+   /
+
+Additional variables for the ``&gfs_physics_nml`` namelist can be found in the ``FV3/ccpp/data/GFS_typedefs.F90``
+file.

doc/UsersGuide/source/InputsOutputs.rst

@@ -1809,6 +1809,131 @@ For the coupled GOCART in S2SAW application, a sample *nems.configure* is shown
 	      stop_ymd = -999
 	::
 
+For the fully coupled S2S application that receives atmosphere-ocean fluxes from mediator, a sample *nems.configure* is shown below :
+
+.. code-block:: console
+
+        # EARTH #
+        EARTH_component_list: MED ATM CHM OCN ICE WAV
+        EARTH_attributes::
+          Verbosity = 0
+        ::
+
+        # MED #
+        MED_model:                      cmeps
+        MED_petlist_bounds:             0 143
+        ::
+
+        # ATM #
+        ATM_model:                      fv3
+        ATM_petlist_bounds:             0 149
+        ATM_attributes::
+          Verbosity = 0
+          DumpFields = false
+          ProfileMemory = false
+          OverwriteSlice = true
+        ::
+
+        # OCN #
+        OCN_model:                      mom6
+        OCN_petlist_bounds:             150 269
+        OCN_attributes::
+          Verbosity = 0
+          DumpFields = false
+          ProfileMemory = false
+          OverwriteSlice = true
+          mesh_ocn = mesh.mx025.nc
+        ::
+
+        # ICE #
+        ICE_model:                      cice6
+        ICE_petlist_bounds:             270 317
+        ICE_attributes::
+          Verbosity = 0
+          DumpFields = false
+          ProfileMemory = false
+          OverwriteSlice = true
+          mesh_ice = mesh.mx025.nc
+          stop_n = 840 
+          stop_option = nhours
+          stop_ymd = -999
+        ::
+
+        # CMEPS warm run sequence
+        runSeq::
+        @720
+           MED med_phases_prep_ocn_avg
+           MED -> OCN :remapMethod=redist
+           OCN
+           @720
+             MED med_phases_aofluxes_run
+             MED med_phases_prep_atm
+             MED med_phases_prep_ice
+             MED -> ATM :remapMethod=redist
+             MED -> ICE :remapMethod=redist
+             ATM
+             ICE
+             ATM -> MED :remapMethod=redist
+             MED med_phases_post_atm
+             ICE -> MED :remapMethod=redist
+             MED med_phases_post_ice
+             MED med_phases_prep_ocn_accum
+           @
+           OCN -> MED :remapMethod=redist
+           MED med_phases_post_ocn
+           MED med_phases_restart_write
+           MED med_phases_history_write
+        @
+        ::
+
+        # CMEPS variables
+
+        DRIVER_attributes::
+        ::
+
+        MED_attributes::
+              ATM_model = fv3
+              ICE_model = cice6
+              OCN_model = mom6
+              history_n = 3
+              history_option = nhours
+              history_ymd = -999
+              coupling_mode = nems_frac_aoflux
+              history_tile_atm = 96
+              aoflux_grid = 'xgrid'
+              aoflux_code = 'ccpp'
+              aoflux_ccpp_suite = 'FV3_sfc_ocean'
+              ccpp_restart_interval = -1
+              ccpp_ini_mosaic_file = 'INPUT/C96_mosaic.nc'
+              ccpp_input_dir = 'INPUT/'
+              ccpp_ini_file_prefix = 'INPUT/sfc_data.tile'
+              ccpp_nstf_name = 2,1,0,0,0
+              ccpp_ini_read = true
+        ::
+        ALLCOMP_attributes::
+              ScalarFieldCount = 2
+              ScalarFieldIdxGridNX = 1
+              ScalarFieldIdxGridNY = 2
+              ScalarFieldName = cpl_scalars
+              start_type = startup
+              restart_dir = RESTART/
+              case_name = ufs.cpld
+              restart_n = 12
+              restart_option = nhours
+              restart_ymd = -999
+              dbug_flag = 0
+              use_coldstart = false
+              use_mommesh = true
+              eps_imesh = 1.0e-1
+              stop_n = 840
+              stop_option = nhours
+              stop_ymd = -999
+        ::
+
+.. note:: The *aoflux_grid* option is used to select the grid/mesh to perform atmosphere-ocean flux calculation. The possible options are *xgrid* (exchange grid), *agrid* (atmosphere model grid) and *ogrid* (ocean model grid).
+
+.. note:: The *aoflux_code* option is used to define the algorithm that will be used to calculate atmosphere-ocean fluxes. The possible options are *cesm* and *ccpp*. If *ccpp* is selected then the suite file provided in the *aoflux_ccpp_suite* option is used to calculate atmosphere-ocean fluxes through the use of CCPP host model.
+
 ---------------------------------------
 *The SDF (Suite Definition File) file*
 ---------------------------------------