update to ufs-community develop

CICE-interface/cice_files.cmake

@@ -28,6 +28,7 @@ list(APPEND cice_shared_files
   CICE/cicecore/cicedynB/dynamics/ice_dyn_evp.F90
   CICE/cicecore/cicedynB/dynamics/ice_dyn_evp_1d.F90
   CICE/cicecore/cicedynB/dynamics/ice_dyn_shared.F90
+  CICE/cicecore/cicedynB/dynamics/ice_dyn_vp.F90
   CICE/cicecore/cicedynB/dynamics/ice_transport_driver.F90
   CICE/cicecore/cicedynB/dynamics/ice_transport_remap.F90
 

CMakeLists.txt

@@ -32,6 +32,7 @@ set(QUAD_PRECISION  ON  CACHE BOOL "Enable QUAD_PRECISION (for certain grid metr
 set(REPRO           OFF CACHE BOOL "Enable REPRO mode")
 set(WW3             OFF CACHE BOOL "Enable WW3")
 set(S2S             OFF CACHE BOOL "Enable S2S")
+set(JEDI_DRIVER     OFF CACHE BOOL "Enable JEDI as top level driver")
 set(DATM            OFF CACHE BOOL "Enable Data Atmosphere")
 
 ###############################################################################
@@ -132,7 +133,7 @@ endif()
 ### FMS
 ###############################################################################
 include(fms_files.cmake)
-add_library(fms ${fms_src_files})
+add_library(fms ${fms_src_files} ${fms_headers})
 # stupid cmake can not figure out dependency of fft.F90 on fft99.F90 because 'use fft99_mod' is inside ifdefs
 set_property(SOURCE FMS/fft/fft.F90 APPEND_STRING PROPERTY COMPILE_FLAGS "-DSGICRAY=0 -DNAGFFT=0")
 
@@ -153,16 +154,20 @@ target_compile_definitions(fms PRIVATE "${_fms_defs_private}")
 
 target_include_directories(fms PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/FMS/include>
                                       $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/FMS/fms>
+                                      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/FMS/fms2_io/include>
+                                      $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/FMS/mod>
                                       $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/FMS/mpp/include>)
+target_include_directories(fms INTERFACE 
+                                         $<INSTALL_INTERFACE:${CMAKE_INSTALL_PREFIX}/include>
+                                         $<INSTALL_INTERFACE:${CMAKE_INSTALL_PREFIX}/mod>)
 target_link_libraries(fms PUBLIC MPI::MPI_Fortran
                                  NetCDF::NetCDF_Fortran)
 if(OpenMP_Fortran_FOUND)
   target_link_libraries(fms PRIVATE OpenMP::OpenMP_Fortran)
 endif()
 set_target_properties(fms PROPERTIES ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/FMS)
 set_target_properties(fms PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/FMS/mod)
-target_include_directories(fms INTERFACE $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/FMS/mod>
-                                         $<INSTALL_INTERFACE:FMS/mod>)
+set_target_properties(fms PROPERTIES PUBLIC_HEADER "${fms_headers}" )
 
 ###############################################################################
 ### stochastic_physics
@@ -189,6 +194,8 @@ if(WW3)
       set(WW3_COMP "hera")
     elseif(${CMAKE_Platform} STREQUAL "orion.intel")
       set(WW3_COMP "orion")
+    elseif(${CMAKE_Platform} STREQUAL "wcoss2")
+      set(WW3_COMP "wcoss_cray")
     elseif(${CMAKE_Platform} MATCHES "linux*" OR ${CMAKE_Platform} MATCHES "macosx*")
       if (CMAKE_Fortran_COMPILER_ID MATCHES "GNU")
         set(WW3_COMP "gnu")
@@ -259,7 +266,9 @@ else()
   list(APPEND _ufs_defs_private FRONT_FV3=fv3gfs_cap_mod)
   add_dependencies(ufs fv3atm)
 endif()
-
+if(JEDI_DRIVER)
+  list(APPEND _ufs_defs_private JEDI_DRIVER=ON)
+endif()
 set_target_properties(ufs PROPERTIES Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod)
 target_include_directories(ufs PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/NEMS/src)
 
@@ -314,7 +323,7 @@ endif()
 target_compile_definitions(ufs_model PRIVATE "${_ufs_model_defs_private}")
 
 if(DATM)
-  target_link_libraries(ufs_model PRIVATE ufs w3nco::w3nco_d)
+  target_link_libraries(ufs_model PUBLIC ufs w3nco::w3nco_d)
 endif()
 target_link_libraries(ufs_model PRIVATE ufs
                                         esmf
@@ -325,17 +334,28 @@ target_link_libraries(ufs_model PRIVATE ufs
 ### Install
 ###############################################################################
 install(
-  TARGETS ufs_model ufs fms
-  EXPORT  ufs-config
-  RUNTIME DESTINATION bin
+  TARGETS fms 
+  EXPORT fms-config
+  LIBRARY DESTINATION lib
+  ARCHIVE DESTINATION lib
+  PUBLIC_HEADER DESTINATION include )
+
+install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/FMS/mod DESTINATION ${CMAKE_INSTALL_PREFIX})
+
+install(EXPORT fms-config
+  DESTINATION lib/cmake
+)
+install(
+  TARGETS ufs 
+  EXPORT ufs-config
   LIBRARY DESTINATION lib
   ARCHIVE DESTINATION lib)
 
 install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/mod DESTINATION ${CMAKE_INSTALL_PREFIX})
 
-install(EXPORT      ufs-config
-        DESTINATION lib/cmake)
-
+install(EXPORT ufs-config
+  DESTINATION lib/cmake
+)
 ###############################################################################
 ### Done
 ###############################################################################

MOM6-interface/mom6_files.cmake

@@ -24,12 +24,13 @@ list(APPEND mom6_src_files
   MOM6/src/core/MOM_CoriolisAdv.F90
   MOM6/src/core/MOM_PressureForce.F90
   MOM6/src/core/MOM_PressureForce_Montgomery.F90
-  MOM6/src/core/MOM_PressureForce_analytic_FV.F90
+  MOM6/src/core/MOM_PressureForce_FV.F90
   MOM6/src/core/MOM_barotropic.F90
   MOM6/src/core/MOM_boundary_update.F90
   MOM6/src/core/MOM_checksum_packages.F90
   MOM6/src/core/MOM_continuity.F90
   MOM6/src/core/MOM_continuity_PPM.F90
+  MOM6/src/core/MOM_density_integrals.F90
   MOM6/src/core/MOM_dynamics_split_RK2.F90
   MOM6/src/core/MOM_dynamics_unsplit.F90
   MOM6/src/core/MOM_dynamics_unsplit_RK2.F90
@@ -214,12 +215,13 @@ list(APPEND mom6_src_files
   MOM6/src/user/Kelvin_initialization.F90
   MOM6/src/user/MOM_controlled_forcing.F90
   MOM6/src/user/MOM_wave_interface.F90
-  MOM6/src/user/Neverland_initialization.F90
+  MOM6/src/user/Neverworld_initialization.F90
   MOM6/src/user/Phillips_initialization.F90
   MOM6/src/user/RGC_initialization.F90
   MOM6/src/user/Rossby_front_2d_initialization.F90
   MOM6/src/user/SCM_CVMix_tests.F90
   MOM6/src/user/adjustment_initialization.F90
+  MOM6/src/user/basin_builder.F90
   MOM6/src/user/baroclinic_zone_initialization.F90
   MOM6/src/user/benchmark_initialization.F90
   MOM6/src/user/circle_obcs_initialization.F90
@@ -241,11 +243,15 @@ list(APPEND mom6_src_files
   MOM6/src/user/user_revise_forcing.F90
 
   MOM6/src/ocean_data_assim/MOM_oda_driver.F90
-  MOM6/src/ocean_data_assim/core/ocean_da_core.F90
-  MOM6/src/ocean_data_assim/core/ocean_da_types.F90
-  MOM6/src/ocean_data_assim/core/write_ocean_obs.F90
 
-  MOM6/src/ocean_data_assim/geoKdTree/kdtree.f90
+  MOM6/config_src/external/GFDL_ocean_BGC/FMS_coupler_util.F90
+  MOM6/config_src/external/GFDL_ocean_BGC/generic_tracer.F90
+  MOM6/config_src/external/GFDL_ocean_BGC/generic_tracer_utils.F90
+  MOM6/config_src/external/ODA_hooks/kdtree.f90
+  MOM6/config_src/external/ODA_hooks/ocean_da_core.F90
+  MOM6/config_src/external/ODA_hooks/ocean_da_types.F90
+  MOM6/config_src/external/ODA_hooks/write_ocean_obs.F90
+
 )
 
 list(APPEND mom6_nuopc_src_files
@@ -261,7 +267,6 @@ list(APPEND mom6_solo_src_files
   MOM6/config_src/solo_driver/MESO_surface_forcing.F90
   MOM6/config_src/solo_driver/MOM_driver.F90
   MOM6/config_src/solo_driver/MOM_surface_forcing.F90
-  MOM6/config_src/solo_driver/Neverland_surface_forcing.F90
   MOM6/config_src/solo_driver/atmos_ocean_fluxes.F90
   MOM6/config_src/solo_driver/user_surface_forcing.F90
 )

cmake/Intel.cmake

@@ -21,8 +21,10 @@ elseif(DEBUG)
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -qno-opt-dynamic-align")
 else()
     if(AVX2)
-        set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -xCORE-AVX2 -qno-opt-dynamic-align")
-        set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -xCORE-AVX2 -qno-opt-dynamic-align")
+        #set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -xCORE-AVX2 -qno-opt-dynamic-align")
+        #set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -xCORE-AVX2 -qno-opt-dynamic-align")
+        set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -march=core-avx2 -qno-opt-dynamic-align")
+        set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -march=core-avx2 -qno-opt-dynamic-align")
     elseif(SIMDMULTIARCH)
         set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -axSSE4.2,AVX,CORE-AVX2,CORE-AVX512 -qno-opt-dynamic-align")
         set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -axSSE4.2,AVX,CORE-AVX2,CORE-AVX512 -qno-opt-dynamic-align")

cmake/configure_jet.intel.cmake

@@ -1,3 +1,4 @@
-set(INLINE_POST     ON  CACHE BOOL "Enable inline post" FORCE)
+set(INLINE_POST     ON CACHE BOOL "Enable inline post" FORCE)
+set(PARALLEL_NETCDF ON CACHE BOOL "Enable parallel NetCDF" FORCE)
 set(AVX2            OFF CACHE BOOL "Enable AVX2 instruction set" FORCE)
 set(SIMDMULTIARCH   ON  CACHE BOOL "Enable multi-target SIMD instruction sets" FORCE)

cmake/configure_wcoss2.cmake

@@ -0,0 +1,3 @@
+set(INLINE_POST     ON  CACHE BOOL "Enable inline post" FORCE)
+set(PARALLEL_NETCDF ON  CACHE BOOL "Enable parallel NetCDF" FORCE)
+set(DEBUG_LINKMPI   OFF CACHE BOOL "Enable linkmpi option when DEBUG mode is on" FORCE)

fms_files.cmake

@@ -1,4 +1,5 @@
 list(APPEND fms_src_files
+    FMS/affinity/fms_affinity.F90
     FMS/amip_interp/amip_interp.F90
     FMS/astronomy/astronomy.F90
     FMS/axis_utils/axis_utils.F90
@@ -79,3 +80,8 @@ list(APPEND fms_src_files
     FMS/mpp/nsclock.c
     FMS/mpp/threadloc.c
 )
+
+list( APPEND fms_headers 
+FMS/include/fms_platform.h
+FMS/include/file_version.h
+)

modulefiles/jet.intel/fv3

@@ -1,53 +1,43 @@
-#%Module######################################################################
-##
-##      nems prerequisites
-##
+#%Module
 
 proc ModulesHelp {} {
-     puts stderr "The prerequisites for compiling or running FV3 on Jet/Intel"
+  puts stderr "\tcit - loads modules required for building and running UFS Model on Jet/Intel"
 }
 
-module purge
+module-whatis "loads UFS Model prerequisites for Jet/Intel"
 
-##
-## load contrib environment
-## load slurm utils (arbitrary.pl  layout.pl)
-##
-module use -a /contrib/sutils/modulefiles
+module use /contrib/sutils/modulefiles
 module load sutils
 
-module load intel/18.0.5.274
-module load impi/2018.4.274
-
-##
-## use pre-compiled PNG, NetCDF, ESMF and NCEP libraries for above compiler / MPI combination
-##
-module use -a /lfs4/HFIP/hfv3gfs/software/ufs-stack-20200909/intel-18.0.5.274/impi-2018.4.274/modules
-
-module load libpng/1.6.35
-module load netcdf/4.7.4
-module load esmf/8.1.0bs27
-
-module load bacio/2.4.0
-module load crtm/2.3.0
-module load g2/3.4.0
-module load g2tmpl/1.9.0
-module load ip/3.3.0
-module load nceppost/dceca26
-module load nemsio/2.5.1
-module load sp/2.3.0
-module load w3emc/2.7.0
-module load w3nco/2.4.0
-
-module load gfsio/1.4.0
-module load sfcio/1.4.0
-module load sigio/2.3.0
-
-##
-## load cmake
-##
 module load cmake/3.16.1
 setenv CMAKE_C_COMPILER mpiicc
 setenv CMAKE_CXX_COMPILER mpiicpc
 setenv CMAKE_Fortran_COMPILER mpiifort
 setenv CMAKE_Platform jet.intel
+
+module use /lfs4/HFIP/hfv3gfs/nwprod/hpc-stack/libs/modulefiles/stack
+
+# At this time (2020/11/30), this is the pre-release version 1.0.0
+module load hpc/1.1.0
+module load hpc-intel/18.0.5.274
+module load hpc-impi/2018.4.274
+
+module load jasper/2.0.22
+module load zlib/1.2.11
+module load png/1.6.35
+
+module load hdf5/1.10.6
+module load netcdf/4.7.4
+module load pio/2.5.1
+module load esmf/8_1_0_beta_snapshot_27
+
+module load bacio/2.4.1
+module load crtm/2.3.0
+module load g2/3.4.1
+module load g2tmpl/1.9.1
+module load ip/3.3.3
+module load nemsio/2.5.2
+module load sp/2.3.3
+module load w3emc/2.7.3
+module load w3nco/2.4.1
+module load upp/10.0.0

modulefiles/wcoss2/fv3

@@ -0,0 +1,42 @@
+#%Module
+
+proc ModulesHelp {} {
+  puts stderr "\tcit - loads modules required for building and running UFS Model on the Dell side of WCOSS"
+}
+
+module-whatis "loads UFS Model prerequisites on Venus and Mars"
+
+module unload cpe-cray cce
+module load cpe-intel intel
+
+module load cmake/3.17.3
+setenv CMAKE_C_COMPILER cc
+setenv CMAKE_CXX_COMPILER CC
+setenv CMAKE_Fortran_COMPILER ftn
+setenv CMAKE_Platform wcoss2
+
+module use /lfs/h1/emc/nceplibs/noscrub/hpc-stack/test/noaa/modulefiles/stack
+
+module load hpc/1.0.0-beta1
+module load hpc-intel/19.1.1.217
+module load hpc-cray-mpich/8.0.15
+
+module load jasper/2.0.22
+module load zlib/1.2.11
+module load png/1.6.35
+
+module load hdf5/1.10.6
+module load netcdf/4.7.4
+module load pio/2.5.1
+module load esmf/8_1_0_beta_snapshot_27
+
+module load bacio/2.4.1
+module load crtm/2.3.0
+module load g2/3.4.1
+module load g2tmpl/1.9.1
+module load ip/3.3.3
+module load nemsio/2.5.2
+module load sp/2.3.3
+module load w3emc/2.7.3
+module load w3nco/2.4.1
+module load upp/10.0.0

modulefiles/wcoss2/fv3_debug

@@ -0,0 +1,42 @@
+#%Module
+
+proc ModulesHelp {} {
+  puts stderr "\tcit - loads modules required for building and running UFS Model on the Dell side of WCOSS"
+}
+
+module-whatis "loads UFS Model prerequisites on Venus and Mars"
+
+module unload cpe-cray cce
+module load cpe-intel intel
+
+module load cmake/3.17.3
+setenv CMAKE_C_COMPILER cc
+setenv CMAKE_CXX_COMPILER CC
+setenv CMAKE_Fortran_COMPILER ftn
+setenv CMAKE_Platform wcoss2
+
+module use /lfs/h1/emc/nceplibs/noscrub/hpc-stack/test/noaa/modulefiles/stack
+
+module load hpc/1.0.0-beta1
+module load hpc-intel/19.1.1.217
+module load hpc-cray-mpich/8.0.15
+
+module load jasper/2.0.22
+module load zlib/1.2.11
+module load png/1.6.35
+
+module load hdf5/1.10.6
+module load netcdf/4.7.4
+module load pio/2.5.1
+module load esmf/8_1_0_beta_snapshot_27
+
+module load bacio/2.4.1
+module load crtm/2.3.0
+module load g2/3.4.1
+module load g2tmpl/1.9.1
+module load ip/3.3.3
+module load nemsio/2.5.2
+module load sp/2.3.3
+module load w3emc/2.7.3
+module load w3nco/2.4.1
+module load upp/10.0.0