Merge HAFS coupling related developments back to ufs-weather-model

.gitmodules

@@ -20,7 +20,11 @@
   branch = develop
 [submodule "CMEPS"]
   path = CMEPS-interface/CMEPS
-  url = https://github.com/NOAA-EMC/CMEPS.git
+  url = https://github.com/NOAA-EMC/CMEPS
+  branch = emc/develop
+[submodule "HYCOM"]
+  path = HYCOM-interface/HYCOM
+  url = https://github.com/NOAA-EMC/HYCOM-src
   branch = emc/develop
 [submodule "MOM6"]
   path = MOM6-interface/MOM6

CMakeLists.txt

@@ -16,7 +16,7 @@ list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/CMakeModules/Modules)
 ###############################################################################
 
 # Valid applications and choices
-list(APPEND VALID_APPS ATM ATMAERO ATMW S2S S2SW NG-GODAS NG-GODAS-NEMSDATM)
+list(APPEND VALID_APPS ATM ATMAERO ATMW S2S S2SW HAFS HAFSW HAFS-ALL NG-GODAS NG-GODAS-NEMSDATM)
 set(APP NONE CACHE BOOL "Application Name")
 if(NOT (APP IN_LIST VALID_APPS))
   message(FATAL_ERROR "${APP} is not a valid application.\nValid Applications are: ${VALID_APPS}")
@@ -26,6 +26,7 @@ set(FMS             OFF CACHE BOOL "Enable FMS")
 set(FV3             OFF CACHE BOOL "Enable FV3")
 set(UFS_GOCART      OFF CACHE BOOL "Enable GOCART")
 set(MOM6            OFF CACHE BOOL "Enable MOM6")
+set(HYCOM           OFF CACHE BOOL "Enable HYCOM")
 set(CICE6           OFF CACHE BOOL "Enable CICE6")
 set(WW3             OFF CACHE BOOL "Enable WW3")
 set(STOCH_PHYS      OFF CACHE BOOL "Enable Stochastic Physics")
@@ -42,6 +43,7 @@ message("FMS .............. ${FMS}")
 message("FV3 .............. ${FV3}")
 message("GOCART ........... ${UFS_GOCART}")
 message("MOM6 ............. ${MOM6}")
+message("HYCOM ............ ${HYCOM}")
 message("CICE6 ............ ${CICE6}")
 message("WW3 .............. ${WW3}")
 message("STOCH_PHYS ....... ${STOCH_PHYS}")
@@ -181,12 +183,16 @@ if(WW3)
 endif()
 
 ###############################################################################
-### Marine Components [MOM6, CICE6]
+### Marine Components [MOM6, HYCOM, CICE6]
 ###############################################################################
 if(MOM6)
   add_subdirectory(MOM6-interface)
 endif()
 
+if(HYCOM)
+  add_subdirectory(HYCOM-interface)
+endif()
+
 if(CICE6)
   add_subdirectory(CICE-interface)
 endif()
@@ -263,6 +269,12 @@ if(MOM6)
   list(APPEND _ufs_libs_public mom6)
 endif()
 
+if(HYCOM)
+  list(APPEND _ufs_defs_private FRONT_HYCOM=HYCOM_Mod)
+  add_dependencies(ufs hycom)
+  target_link_libraries(ufs PUBLIC hycom)
+endif()
+
 if(CICE6)
   add_dependencies(ufs cice)
   list(APPEND _ufs_defs_private FRONT_CICE6=ice_comp_nuopc)
@@ -278,6 +290,7 @@ endif()
 if(CDEPS)
   add_dependencies(ufs cdeps::cdeps)
   list(APPEND _ufs_defs_private FRONT_CDEPS_DATM=cdeps_datm_comp)
+  list(APPEND _ufs_defs_private FRONT_CDEPS_DOCN=cdeps_docn_comp)
   target_link_libraries(ufs PUBLIC cdeps::cdeps)
 endif()
 

HYCOM-interface/CMakeLists.txt

@@ -0,0 +1,124 @@
+get_filename_component(hycom_dir "${CMAKE_CURRENT_SOURCE_DIR}/HYCOM" ABSOLUTE)
+
+### HYCOM Fortran compiler flags
+if(CMAKE_Fortran_COMPILER_ID MATCHES "GNU")
+  set(CMAKE_Fortran_FLAGS "-g -fbacktrace")
+  set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fdefault-real-8 -fdefault-double-8")
+  set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -Waliasing -fcray-pointer -fconvert=big-endian -ffree-line-length-none -fno-range-check -fbacktrace")
+  set(CMAKE_Fortran_FLAGS_RELEASE "-O3")
+  set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -fcheck=bounds -ffpe-trap=invalid,zero,overflow,underflow" )
+  set(CMAKE_Fortran_LINK_FLAGS "")
+elseif(CMAKE_Fortran_COMPILER_ID MATCHES "Intel")
+  set(CMAKE_Fortran_FLAGS "-g -traceback -xSSE4.2 -mcmodel=small -r8")
+  set(CMAKE_Fortran_FLAGS_RELEASE "-O3 -fp-model source -warn nogeneral")
+  set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -check -check noarg_temp_created -check nopointer -fpe0 -ftrapuv -link_mpi=dbg")
+  set(CMAKE_Fortran_LINK_FLAGS "-V ${CMAKE_Fortran_FLAGS} -static-intel")
+else()
+  message(WARNING "Fortran compiler with ID ${CMAKE_Fortran_COMPILER_ID} will be used with CMake default options")
+endif()
+
+### HYCOM C compiler flags
+if(CMAKE_C_COMPILER_ID MATCHES "GNU")
+  set(CMAKE_C_FLAGS "-g -fbacktrace")
+  set(CMAKE_C_FLAGS_RELEASE "-O3")
+  set(CMAKE_C_FLAGS_DEBUG "-O0")
+  set(CMAKE_C_LINK_FLAGS "")
+elseif(CMAKE_C_COMPILER_ID MATCHES "Intel")
+  set(CMAKE_C_FLAGS "-g -traceback -xSSE4.2 -mcmodel=small")
+  set(CMAKE_C_FLAGS_RELEASE "-O")
+  set(CMAKE_C_FLAGS_DEBUG "-O0 -ftrapuv")
+  set(CMAKE_C_LINK_FLAGS "-V ${CMAKE_C_FLAGS} -static-intel")
+else()
+  message(WARNING "C compiler with ID ${CMAKE_C_COMPILER_ID} will be used with CMake default options")
+endif()
+
+list(APPEND _hycom_defs IA32
+                        REAL8
+                        MPI
+                        SERIAL_IO
+                        ENDIAN_IO
+                        NAN2003
+                        TIME
+                        RELO
+                        EOS_SIG2
+                        EOS_17T
+                        ESPC_COUPLE
+)
+
+# User option to build HYCOM offline executable
+set(HYCOMOFFLINE OFF CACHE BOOL "Build HYCOM offline")
+
+message("Build HYCOM:")
+message("   in:  ${hycom_dir}")
+message("   HYCOMOFFLINE: ${HYCOMOFFLINE}")
+message("")
+
+# Too many files to list, so include them via this file
+include("hycom_files.cmake")
+
+### Use common object library for building target library
+add_library(hycom_obj OBJECT ${_hycom_src_files})
+set_target_properties(hycom_obj PROPERTIES ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/HYCOM)
+set_target_properties(hycom_obj PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod)
+target_include_directories(hycom_obj PRIVATE $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/mod>)
+target_include_directories(hycom_obj PRIVATE $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/HYCOM>)
+target_compile_definitions(hycom_obj PRIVATE "${_hycom_defs}")
+
+
+### Use NUOPC object library for building target library
+add_library(hycom_nuopc_obj OBJECT ${_hycom_nuopc_src_files})
+add_dependencies(hycom_nuopc_obj hycom_obj)
+set_target_properties(hycom_nuopc_obj PROPERTIES ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/HYCOM)
+set_target_properties(hycom_nuopc_obj PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod)
+target_include_directories(hycom_nuopc_obj PRIVATE $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/mod>)
+target_include_directories(hycom_nuopc_obj PRIVATE $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/HYCOM>
+                                                   $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/HYCOM/NUOPC>)
+target_compile_definitions(hycom_nuopc_obj PRIVATE "${_hycom_defs}")
+target_link_libraries(hycom_nuopc_obj PRIVATE hycom_obj
+                                              esmf
+                                              NetCDF::NetCDF_Fortran)
+
+### Create target library and set PUBLIC interfaces on the library
+add_library(hycom STATIC $<TARGET_OBJECTS:hycom_obj>
+                         $<TARGET_OBJECTS:hycom_nuopc_obj>)
+target_include_directories(hycom PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/mod>
+                                        $<INSTALL_INTERFACE:mod>)
+target_include_directories(hycom PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/HYCOM>
+                                        $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/HYCOM/NUOPC>)
+target_compile_definitions(hycom PRIVATE "${_hycom_defs}")
+target_link_libraries(hycom PUBLIC esmf
+                                   NetCDF::NetCDF_Fortran)
+
+### Create HYCOM offline executable
+if(HYCOMOFFLINE)
+  message("Building HYCOM offline executable")
+  add_executable(hycomoffline ${_hycom_offline_src_files})
+  add_dependencies(hycomoffline hycom_obj)
+  set_target_properties(hycomoffline PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod_offline)
+  target_include_directories(hycomoffline PRIVATE $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/mod>)
+  target_include_directories(hycomoffline PRIVATE $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/mod_offline>)
+  target_include_directories(hycomoffline PRIVATE $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/HYCOM>)
+
+  target_link_libraries(hycomoffline PRIVATE hycom_obj)
+endif()
+
+###############################################################################
+### Install
+###############################################################################
+
+install(
+  TARGETS hycom
+  EXPORT  hycom-config
+  RUNTIME DESTINATION bin
+  LIBRARY DESTINATION lib
+  ARCHIVE DESTINATION lib
+  COMPONENT Library)
+
+install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod DESTINATION ${CMAKE_INSTALL_PREFIX}/HYCOM)
+
+install(EXPORT      hycom-config
+        DESTINATION lib/cmake)
+
+if(HYCOMOFFLINE)
+  install(TARGETS hycomoffline RUNTIME DESTINATION bin)
+endif()

HYCOM-interface/hycom_files.cmake

@@ -0,0 +1,68 @@
+list(APPEND _hycom_src_files
+  HYCOM/mod_dimensions.F90
+  HYCOM/mod_xc.F90
+  HYCOM/mod_za.F90
+  HYCOM/mod_cb_arrays.F90
+  HYCOM/mod_pipe.F90
+  HYCOM/mod_incupd.F90
+  HYCOM/mod_floats.F90
+  HYCOM/mod_stokes.F90
+  HYCOM/mod_tides.F90
+  HYCOM/mod_mean.F90
+  HYCOM/mod_archiv.F90
+  HYCOM/mod_tsadvc.F90
+  HYCOM/mod_momtum.F90
+  HYCOM/mod_barotp.F90
+  HYCOM/mod_asselin.F90
+  HYCOM/mod_restart.F90
+  HYCOM/mod_import.F90
+  HYCOM/mod_hycom.F90
+
+  HYCOM/bigrid.F90
+  HYCOM/blkdat.F90
+  HYCOM/cnuity.F90
+  HYCOM/convec.F90
+  HYCOM/diapfl.F90
+  HYCOM/dpthuv.F90
+  HYCOM/dpudpv.F90
+  HYCOM/forfun.F90
+  HYCOM/geopar.F90
+  HYCOM/hybgen.F90
+  HYCOM/icloan.F90
+  HYCOM/inicon.F90
+  HYCOM/inigiss.F90
+  HYCOM/inikpp.F90
+  HYCOM/inimy.F90
+  HYCOM/latbdy.F90
+  HYCOM/matinv.F90
+  HYCOM/mxkprf.F90
+  HYCOM/mxkrt.F90
+  HYCOM/mxkrtm.F90
+  HYCOM/mxpwp.F90
+  HYCOM/overtn.F90
+  HYCOM/poflat.F90
+  HYCOM/prtmsk.F90
+  HYCOM/psmoo.F90
+  HYCOM/thermf.F90
+  HYCOM/trcupd.F90
+  HYCOM/machine.F90
+  HYCOM/wtime.F90
+  HYCOM/machi_c.c
+  HYCOM/isnan.F90
+  HYCOM/s8gefs.F90
+)
+
+list(APPEND _hycom_nuopc_src_files
+  HYCOM/NUOPC/HYCOM_OceanComp.F90
+  HYCOM/NUOPC/HYCOM_ESMF_Extensions.F90
+  HYCOM/NUOPC/hycom_couple.F90
+  HYCOM/NUOPC/read_impexp_config_mod.F90
+  HYCOM/NUOPC/impexpField_cdf_mod.F90
+  HYCOM/NUOPC/export_from_hycom_tiled.F90
+  HYCOM/NUOPC/hycom_read_latlon.F90
+  HYCOM/NUOPC/hycom_nuopc_flags.F90
+)
+
+list(APPEND _hycom_offline_src_files
+  HYCOM/hycom.F90
+)

cmake/configure_apps.cmake

@@ -54,6 +54,22 @@ if(APP MATCHES "^(S2S|S2SW)$")
   endif()
 endif()
 
+if(APP MATCHES "^(HAFS|HAFSW|HAFS-ALL)$")
+  set(CMEPS      ON  CACHE BOOL "Enable CMEPS"               FORCE)
+  if(APP MATCHES "^(HAFS-ALL)$")
+    set(CDEPS    ON  CACHE BOOL "Enable CDEPS"               FORCE)
+    message("Configuring UFS app in HAFS with CDEPS mode")
+  endif()
+  set(FMS        ON  CACHE BOOL "Enable FMS"                 FORCE)
+  set(FV3        ON  CACHE BOOL "Enable FV3"                 FORCE)
+  set(STOCH_PHYS ON  CACHE BOOL "Enable Stochastic Physics"  FORCE)
+  set(HYCOM      ON  CACHE BOOL "Enable HYCOM"               FORCE)
+  if(APP MATCHES "^(HAFSW|HAFS-ALL)$")
+    set(WW3      ON  CACHE BOOL "Enable WAVEWATCH III"       FORCE)
+    message("Configuring UFS app in HAFS with Waves mode")
+  endif()
+endif()
+
 if(APP MATCHES "^(ATMAERO)$")
   set(FMS        ON  CACHE BOOL "Enable FMS"                 FORCE)
   set(FV3        ON  CACHE BOOL "Enable FV3"                 FORCE)

modulefiles/ufs_wcoss_cray

@@ -38,6 +38,7 @@ setenv PNG_ROOT /usrx/local/prod//png/1.2.49/intel/sandybridge
 ## NCEP libraries
 ##
 module use /usrx/local/nceplibs/NCEPLIBS/cmake/install/NCEPLIBS-v1.3.0/modules
+module load pio/2.5.2
 module load bacio/2.4.1
 module load crtm/2.3.0
 module load g2/3.4.1

modulefiles/ufs_wcoss_cray_debug

@@ -38,6 +38,7 @@ setenv PNG_ROOT /usrx/local/prod//png/1.2.49/intel/sandybridge
 ## NCEP libraries
 ##
 module use /usrx/local/nceplibs/NCEPLIBS/cmake/install/NCEPLIBS-v1.3.0/modules
+module load pio/2.5.2
 module load bacio/2.4.1
 module load crtm/2.3.0
 module load g2/3.4.1