Updates to build for JEDI linking/control, add wcoss2

CMakeLists.txt

@@ -32,6 +32,7 @@ set(QUAD_PRECISION  ON  CACHE BOOL "Enable QUAD_PRECISION (for certain grid metr
 set(REPRO           OFF CACHE BOOL "Enable REPRO mode")
 set(WW3             OFF CACHE BOOL "Enable WW3")
 set(S2S             OFF CACHE BOOL "Enable S2S")
+set(JEDI_DRIVER     OFF CACHE BOOL "Enable JEDI as top level driver")
 set(DATM            OFF CACHE BOOL "Enable Data Atmosphere")
 
 ###############################################################################
@@ -132,7 +133,7 @@ endif()
 ### FMS
 ###############################################################################
 include(fms_files.cmake)
-add_library(fms ${fms_src_files})
+add_library(fms ${fms_src_files} ${fms_headers})
 # stupid cmake can not figure out dependency of fft.F90 on fft99.F90 because 'use fft99_mod' is inside ifdefs
 set_property(SOURCE FMS/fft/fft.F90 APPEND_STRING PROPERTY COMPILE_FLAGS "-DSGICRAY=0 -DNAGFFT=0")
 
@@ -153,16 +154,20 @@ target_compile_definitions(fms PRIVATE "${_fms_defs_private}")
 
 target_include_directories(fms PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/FMS/include>
                                       $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/FMS/fms>
+                                      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/FMS/fms2_io/include>
+                                      $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/FMS/mod>
                                       $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/FMS/mpp/include>)
+target_include_directories(fms INTERFACE 
+                                         $<INSTALL_INTERFACE:${CMAKE_INSTALL_PREFIX}/include>
+                                         $<INSTALL_INTERFACE:${CMAKE_INSTALL_PREFIX}/mod>)
 target_link_libraries(fms PUBLIC MPI::MPI_Fortran
                                  NetCDF::NetCDF_Fortran)
 if(OpenMP_Fortran_FOUND)
   target_link_libraries(fms PRIVATE OpenMP::OpenMP_Fortran)
 endif()
 set_target_properties(fms PROPERTIES ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/FMS)
 set_target_properties(fms PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/FMS/mod)
-target_include_directories(fms INTERFACE $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/FMS/mod>
-                                         $<INSTALL_INTERFACE:FMS/mod>)
+set_target_properties(fms PROPERTIES PUBLIC_HEADER "${fms_headers}" )
 
 ###############################################################################
 ### stochastic_physics
@@ -189,6 +194,8 @@ if(WW3)
       set(WW3_COMP "hera")
     elseif(${CMAKE_Platform} STREQUAL "orion.intel")
       set(WW3_COMP "orion")
+    elseif(${CMAKE_Platform} STREQUAL "wcoss2")
+      set(WW3_COMP "wcoss_cray")
     elseif(${CMAKE_Platform} MATCHES "linux*" OR ${CMAKE_Platform} MATCHES "macosx*")
       if (CMAKE_Fortran_COMPILER_ID MATCHES "GNU")
         set(WW3_COMP "gnu")
@@ -259,7 +266,9 @@ else()
   list(APPEND _ufs_defs_private FRONT_FV3=fv3gfs_cap_mod)
   add_dependencies(ufs fv3atm)
 endif()
-
+if(JEDI_DRIVER)
+  list(APPEND _ufs_defs_private JEDI_DRIVER=ON)
+endif()
 set_target_properties(ufs PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod)
 target_include_directories(ufs PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/NEMS/src)
 
@@ -314,7 +323,7 @@ endif()
 target_compile_definitions(ufs_model PRIVATE "${_ufs_model_defs_private}")
 
 if(DATM)
-  target_link_libraries(ufs_model PRIVATE ufs w3nco::w3nco_d)
+  target_link_libraries(ufs_model PUBLIC ufs w3nco::w3nco_d)
 endif()
 target_link_libraries(ufs_model PRIVATE ufs
                                         esmf
@@ -325,17 +334,28 @@ target_link_libraries(ufs_model PRIVATE ufs
 ### Install
 ###############################################################################
 install(
-  TARGETS ufs_model ufs fms
-  EXPORT  ufs-config
-  RUNTIME DESTINATION bin
+  TARGETS fms 
+  EXPORT fms-config
+  LIBRARY DESTINATION lib
+  ARCHIVE DESTINATION lib
+  PUBLIC_HEADER DESTINATION include )
+
+install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/FMS/mod DESTINATION ${CMAKE_INSTALL_PREFIX})
+
+install(EXPORT fms-config
+  DESTINATION lib/cmake
+)
+install(
+  TARGETS ufs 
+  EXPORT ufs-config
   LIBRARY DESTINATION lib
   ARCHIVE DESTINATION lib)
 
 install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod DESTINATION ${CMAKE_INSTALL_PREFIX})
 
-install(EXPORT      ufs-config
-        DESTINATION lib/cmake)
-
+install(EXPORT ufs-config
+  DESTINATION lib/cmake
+)
 ###############################################################################
 ### Done
 ###############################################################################

cmake/Intel.cmake

@@ -21,8 +21,10 @@ elseif(DEBUG)
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -qno-opt-dynamic-align")
 else()
     if(AVX2)
-        set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -xCORE-AVX2 -qno-opt-dynamic-align")
-        set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -xCORE-AVX2 -qno-opt-dynamic-align")
+        #set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -xCORE-AVX2 -qno-opt-dynamic-align")
+        #set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -xCORE-AVX2 -qno-opt-dynamic-align")
+        set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -march=core-avx2 -qno-opt-dynamic-align")
+        set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -march=core-avx2 -qno-opt-dynamic-align")
     elseif(SIMDMULTIARCH)
         set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -axSSE4.2,AVX,CORE-AVX2,CORE-AVX512 -qno-opt-dynamic-align")
         set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -axSSE4.2,AVX,CORE-AVX2,CORE-AVX512 -qno-opt-dynamic-align")

cmake/configure_wcoss2.cmake

@@ -0,0 +1,3 @@
+set(INLINE_POST     ON  CACHE BOOL "Enable inline post" FORCE)
+set(PARALLEL_NETCDF ON  CACHE BOOL "Enable parallel NetCDF" FORCE)
+set(DEBUG_LINKMPI   OFF CACHE BOOL "Enable linkmpi option when DEBUG mode is on" FORCE)

fms_files.cmake

@@ -80,3 +80,8 @@ list(APPEND fms_src_files
     FMS/mpp/nsclock.c
     FMS/mpp/threadloc.c
 )
+
+list( APPEND fms_headers 
+FMS/include/fms_platform.h
+FMS/include/file_version.h
+)

modulefiles/wcoss2/fv3

@@ -0,0 +1,42 @@
+#%Module
+
+proc ModulesHelp {} {
+  puts stderr "\tcit - loads modules required for building and running UFS Model on the Dell side of WCOSS"
+}
+
+module-whatis "loads UFS Model prerequisites on Venus and Mars"
+
+module unload cpe-cray cce
+module load cpe-intel intel
+
+module load cmake/3.17.3
+setenv CMAKE_C_COMPILER cc
+setenv CMAKE_CXX_COMPILER CC
+setenv CMAKE_Fortran_COMPILER ftn
+setenv CMAKE_Platform wcoss2
+
+module use /lfs/h1/emc/nceplibs/noscrub/hpc-stack/test/noaa/modulefiles/stack
+
+module load hpc/1.0.0-beta1
+module load hpc-intel/19.1.1.217
+module load hpc-cray-mpich/8.0.15
+
+module load jasper/2.0.22
+module load zlib/1.2.11
+module load png/1.6.35
+
+module load hdf5/1.10.6
+module load netcdf/4.7.4
+module load pio/2.5.1
+module load esmf/8_1_0_beta_snapshot_27
+
+module load bacio/2.4.1
+module load crtm/2.3.0
+module load g2/3.4.1
+module load g2tmpl/1.9.1
+module load ip/3.3.3
+module load nemsio/2.5.2
+module load sp/2.3.3
+module load w3emc/2.7.3
+module load w3nco/2.4.1
+module load upp/10.0.0

modulefiles/wcoss2/fv3_debug

@@ -0,0 +1,42 @@
+#%Module
+
+proc ModulesHelp {} {
+  puts stderr "\tcit - loads modules required for building and running UFS Model on the Dell side of WCOSS"
+}
+
+module-whatis "loads UFS Model prerequisites on Venus and Mars"
+
+module unload cpe-cray cce
+module load cpe-intel intel
+
+module load cmake/3.17.3
+setenv CMAKE_C_COMPILER cc
+setenv CMAKE_CXX_COMPILER CC
+setenv CMAKE_Fortran_COMPILER ftn
+setenv CMAKE_Platform wcoss2
+
+module use /lfs/h1/emc/nceplibs/noscrub/hpc-stack/test/noaa/modulefiles/stack
+
+module load hpc/1.0.0-beta1
+module load hpc-intel/19.1.1.217
+module load hpc-cray-mpich/8.0.15
+
+module load jasper/2.0.22
+module load zlib/1.2.11
+module load png/1.6.35
+
+module load hdf5/1.10.6
+module load netcdf/4.7.4
+module load pio/2.5.1
+module load esmf/8_1_0_beta_snapshot_27
+
+module load bacio/2.4.1
+module load crtm/2.3.0
+module load g2/3.4.1
+module load g2tmpl/1.9.1
+module load ip/3.3.3
+module load nemsio/2.5.2
+module load sp/2.3.3
+module load w3emc/2.7.3
+module load w3nco/2.4.1
+module load upp/10.0.0