Merge develop to dev/ufs-weather-model

.github/workflows/gnu.yml

@@ -1,13 +1,13 @@
 name: GNU Linux Build
-on: [push, pull_request]
+on: [push, pull_request, workflow_dispatch]
 
 # Cancel in-progress workflows when pushing to a branch
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
 env:
-  cache_key: gnu5
+  cache_key: gnu8
   CC: gcc-10
   FC: gfortran-10
   CXX: g++-10
@@ -19,7 +19,7 @@ env:
 
 jobs:
   setup:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
 
     steps:
       - name: checkout-ww3
@@ -37,19 +37,20 @@ jobs:
             spack
             ~/.spack
             work_oasis3-mct
-          key: spack-${{ runner.os }}-${{ env.cache_key }}-${{ hashFiles('ww3/model/ci/spack.yaml') }}
+          key: spack-${{ runner.os }}-${{ env.cache_key }}-${{ hashFiles('ww3/model/ci/spack_gnu.yaml') }}
 
       # Build WW3 spack environment
       - name: install-dependencies-with-spack
         if: steps.cache-env.outputs.cache-hit != 'true'
         run: |
           # Install NetCDF, ESMF, g2, etc using Spack
+          sudo apt install cmake
           git clone -c feature.manyFiles=true https://github.com/spack/spack.git
           source spack/share/spack/setup-env.sh
-          spack env create ww3-gnu ww3/model/ci/spack.yaml
+          spack env create ww3-gnu ww3/model/ci/spack_gnu.yaml
           spack env activate ww3-gnu
           spack compiler find
-          spack external find
+          spack external find cmake
           spack add mpich@3.4.2
           spack concretize
           spack install --dirty -v
@@ -72,7 +73,7 @@ jobs:
     strategy:
       matrix:
         switch: [Ifremer1, NCEP_st2, NCEP_st4, ite_pdlib, NCEP_st4sbs, NCEP_glwu, OASACM, UKMO, MULTI_ESMF, NUOPC_MESH]
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
 
     steps:
       - name: checkout-ww3
@@ -88,7 +89,7 @@ jobs:
             spack
             ~/.spack
             work_oasis3-mct
-          key: spack-${{ runner.os }}-${{ env.cache_key }}-${{ hashFiles('ww3/model/ci/spack.yaml') }}
+          key: spack-${{ runner.os }}-${{ env.cache_key }}-${{ hashFiles('ww3/model/ci/spack_gnu.yaml') }}
 
       - name: build-ww3
         run: |

.github/workflows/intel.yml

@@ -1,33 +1,27 @@
 name: Intel Linux Build
-on: [push, pull_request]
+on: [push, pull_request, workflow_dispatch]
 
 # Cancel in-progress workflows when pushing to a branch
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
-# Use custom shell with -l so .bash_profile is sourced which loads intel/oneapi/setvars.sh
-# without having to do it in manually every step
-defaults:
-  run:
-    shell: bash -leo pipefail {0}
-
 # Set I_MPI_CC/F90 so Intel MPI wrapper uses icc/ifort instead of gcc/gfortran
 env:
-  cache_key: intel6
+  cache_key: intel7
   CC: icc
   FC: ifort
   CXX: icpc
   I_MPI_CC: icc
   I_MPI_F90: ifort
 
 # Split into a dependency build step, and a WW3 build step which
-# builds multiple switches in a matrix. The setup is run once and
+# builds multiple switches in a matrix. The setup is run once and 
 # the environment is cached so each build of WW3 can share the dependencies.
 
 jobs:
   setup:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04 
 
     steps:
 
@@ -48,7 +42,7 @@ jobs:
             ~/.spack
             work_oasis3-mct
             /opt/intel
-          key: spack-${{ runner.os }}-${{ env.cache_key }}-${{ hashFiles('ww3/model/ci/spack.yaml') }}
+          key: spack-${{ runner.os }}-${{ env.cache_key }}-${{ hashFiles('ww3/model/ci/spack_intel.yaml') }}
 
       - name: install-intel-compilers
         if: steps.cache-env.outputs.cache-hit != 'true'
@@ -57,27 +51,31 @@ jobs:
           sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
           echo "deb https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list
           sudo apt-get update
-          sudo apt-get install intel-oneapi-dev-utilities intel-oneapi-mpi-devel intel-oneapi-openmp intel-oneapi-compiler-fortran intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic
-          echo "source /opt/intel/oneapi/setvars.sh" >> ~/.bash_profile
+          sudo apt-get install intel-oneapi-mpi-devel intel-oneapi-openmp intel-oneapi-compiler-fortran intel-oneapi-compiler-dpcpp-cpp-and-cpp-classic
 
       # Build WW3 spack environment
       - name: install-dependencies-with-spack
         if: steps.cache-env.outputs.cache-hit != 'true'
         run: |
           # Install NetCDF, ESMF, g2, etc using Spack
+          . /opt/intel/oneapi/setvars.sh
+          sudo mv /usr/local /usrlocal_renamed
+          sudo apt install cmake
           git clone -c feature.manyFiles=true https://github.com/spack/spack.git
           source spack/share/spack/setup-env.sh
-          spack env create ww3-intel ww3/model/ci/spack.yaml
+          ln -s $(realpath $(which gcc)) spack/lib/spack/env/intel/gcc # spack/make bug in ESMF
+          spack env create ww3-intel ww3/model/ci/spack_intel.yaml
           spack env activate ww3-intel
           spack compiler find
-          spack external find
+          spack external find cmake
           spack add intel-oneapi-mpi
           spack concretize
-          spack install --dirty -v
+          spack install --dirty -v --fail-fast
 
       - name: build-oasis
         if: steps.cache-env.outputs.cache-hit != 'true'
         run: |
+          . /opt/intel/oneapi/setvars.sh
           source spack/share/spack/setup-env.sh
           spack env activate ww3-intel
           export WWATCH3_DIR=${GITHUB_WORKSPACE}/ww3/model
@@ -93,18 +91,14 @@ jobs:
     strategy:
       matrix:
         switch: [Ifremer1, NCEP_st2, NCEP_st4, ite_pdlib, NCEP_st4sbs, NCEP_glwu, OASACM, UKMO, MULTI_ESMF, NUOPC_MESH]
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04 
 
     steps:
       - name: checkout-ww3
         uses: actions/checkout@v2
         with:
             path: ww3
 
-      - name: install-intel
-        run: |
-          echo "source /opt/intel/oneapi/setvars.sh" >> ~/.bash_profile
-
       - name: cache-env
         id: cache-env
         uses: actions/cache@v2
@@ -114,10 +108,11 @@ jobs:
             ~/.spack
             work_oasis3-mct
             /opt/intel
-          key: spack-${{ runner.os }}-${{ env.cache_key }}-${{ hashFiles('ww3/model/ci/spack.yaml') }}
+          key: spack-${{ runner.os }}-${{ env.cache_key }}-${{ hashFiles('ww3/model/ci/spack_intel.yaml') }}
 
       - name: build-ww3
         run: |
+          . /opt/intel/oneapi/setvars.sh
           source spack/share/spack/setup-env.sh
           spack env activate ww3-intel
           cd ww3

cmake/FindSCOTCH.cmake

@@ -0,0 +1,84 @@
+message(STATUS "Searching for PTSCOTCHparmetis library ...")
+find_library(ptscotchparmetis_lib NAMES libptscotchparmetisv3.a  HINTS ENV SCOTCH_PATH PATH_SUFFIXES lib)
+find_path(ptscotchparmetis_inc parmetis.h HINTS ENV SCOTCH_PATH PATH_SUFFIXES include)
+
+message(STATUS "Searching for SCOTCH library ...")
+find_library(scotch_lib NAMES libscotch.a HINTS ENV SCOTCH_PATH PATH_SUFFIXES lib)
+find_path(scotch_inc scotch.h HINTS ENV SCOTCH_PATH PATH_SUFFIXES include)
+
+message(STATUS "Searching for PTSCOTCH library ...")
+find_library(ptscotch_lib NAMES libptscotch.a HINTS ENV SCOTCH_PATH PATH_SUFFIXES lib)
+find_path(ptscotch_inc ptscotch.h HINTS ENV SCOTCH_PATH PATH_SUFFIXES include)
+
+message(STATUS "Searching for SCOTCHerr library ...")
+find_library(scotcherr_lib NAMES libscotcherr.a HINTS ENV SCOTCH_PATH PATH_SUFFIXES lib)
+find_path(scotcherr_inc scotch.h HINTS ENV SCOTCH_PATH PATH_SUFFIXES include)
+
+message(STATUS "Searching for PTSCOTCHerr library ...")
+find_library(ptscotcherr_lib NAMES libptscotcherr.a HINTS ENV SCOTCH_PATH PATH_SUFFIXES lib)
+find_path(ptscotcherr_inc ptscotch.h HINTS ENV SCOTCH_PATH PATH_SUFFIXES include)
+
+add_library(PTSCOTCHparmetis::PTSCOTCHparmetis STATIC IMPORTED)
+add_library(SCOTCH::SCOTCH STATIC IMPORTED)
+add_library(PTSCOTCH::PTSCOTCH STATIC IMPORTED)
+add_library(SCOTCHerr::SCOTCHerr STATIC IMPORTED)
+add_library(PTSCOTCHerr::PTSCOTCHerr STATIC IMPORTED)
+
+set_target_properties(SCOTCH::SCOTCH PROPERTIES
+  IMPORTED_LOCATION "${scotch_lib}"
+  INTERFACE_INCLUDE_DIRECTORIES "${scotch_inc}")
+
+set_target_properties(SCOTCHerr::SCOTCHerr PROPERTIES
+  IMPORTED_LOCATION "${scotcherr_lib}"
+  INTERFACE_INCLUDE_DIRECTORIES "${scotcherr_inc}")
+
+set_target_properties(PTSCOTCH::PTSCOTCH PROPERTIES
+  IMPORTED_LOCATION "${ptscotch_lib}"
+  INTERFACE_INCLUDE_DIRECTORIES "${ptscotch_inc}")
+
+set_target_properties(PTSCOTCHerr::PTSCOTCHerr PROPERTIES
+  IMPORTED_LOCATION "${ptscotcherr_lib}"
+  INTERFACE_INCLUDE_DIRECTORIES "${ptscotcherr_inc}")
+
+set_target_properties(PTSCOTCHparmetis::PTSCOTCHparmetis PROPERTIES
+  IMPORTED_LOCATION "${ptscotchparmetis_lib}"
+  INTERFACE_INCLUDE_DIRECTORIES "${ptscotchparmetis_inc}")
+
+
+## Interfaces and links
+target_link_libraries(PTSCOTCHparmetis::PTSCOTCHparmetis INTERFACE PTSCOTCH::PTSCOTCH PTSCOTCHerr::PTSCOTCHerr  SCOTCH::SCOTCH)
+
+
+## Finalize find_package
+include(FindPackageHandleStandardArgs)
+
+find_package_handle_standard_args(
+    ${CMAKE_FIND_PACKAGE_NAME}
+    REQUIRED_VARS scotch_lib
+    scotch_inc)
+
+find_package_handle_standard_args(
+    ${CMAKE_FIND_PACKAGE_NAME}
+    REQUIRED_VARS ptscotch_lib
+    ptscotch_inc)
+
+find_package_handle_standard_args(
+    ${CMAKE_FIND_PACKAGE_NAME}
+    REQUIRED_VARS scotcherr_lib
+    scotcherr_inc)
+
+find_package_handle_standard_args(
+    ${CMAKE_FIND_PACKAGE_NAME}
+    REQUIRED_VARS ptscotcherr_lib
+    ptscotcherr_inc)
+
+find_package_handle_standard_args(
+    ${CMAKE_FIND_PACKAGE_NAME}
+    REQUIRED_VARS ptscotchparmetis_lib
+    ptscotchparmetis_inc)
+
+message(STATUS "Found SCOTCH: ${scotch_lib}")
+message(STATUS "Found PTSCOTCH: ${ptscotch_lib}")
+message(STATUS "Found SCOTCHerr: ${scotcherr_lib}")
+message(STATUS "Found PTSCOTCHerr: ${ptscotcherr_lib}")
+message(STATUS "Found PTSCOTCHparmetis: ${ptscotchparmetis_lib}")

model/bin/build_utils.sh

@@ -40,7 +40,7 @@ check_switches()
               dstress s_ice s_is reflection \
               wind windx wcor rwind curr currx mgwind mgprop mggse \
               subsec tdyn dss0 pdif tide refrx ig rotag nnt mprf \
-              cou oasis agcm ogcm igcm trknc setup pdlib memck uost rstwind b4b
+              cou oasis agcm ogcm igcm trknc setup pdlib ddlib memck uost rstwind b4b
   do
 
 # 1.a.1 Group switches by category
@@ -291,6 +291,12 @@ check_switches()
                ID='use pdlib'
                TS='PDLIB'
                OK='PDLIB' ;;
+
+#sort:ddlib:
+      ddlib  ) TY='upto1'
+               ID='domain decomposition library'
+               OK='METIS SCOTCH' ;;
+
 #sort:memck:
       memck  ) TY='upto1'
                ID='check memory use'
@@ -428,6 +434,7 @@ check_switches()
       igcm   ) igcm=$sw ;;
       trknc  ) trknc=$sw ;;
       pdlib  ) pdlib=$sw ;;
+      ddlib  ) ddlib=$sw ;;
       memck  ) memck=$sw ;;
       setup  ) setup=$sw ;;
       uost   ) uost=$sw ;;
@@ -547,6 +554,30 @@ check_switches()
       echo ' ' ; exit 17
   fi
 
+  if [ "$pdlib" = 'PDLIB' ] && [ "$ddlib" != 'SCOTCH' ]
+      then
+         if [ "$ddlib" != 'METIS' ]
+         then
+            echo ' '
+            echo "   *** For PDLIB, we need either SCOTCH or METIS, not both."
+            echo ' ' ; exit 18
+         fi
+  fi
+
+
+  if [ "$ddlib" = 'METIS' ] && [ "$pdlib" != 'PDLIB' ]
+  then
+      echo ' '
+      echo "   *** For METIS, we need to have PDLIB as well."
+      echo ' ' ; exit 19
+  fi
+
+  if [ "$ddlib" = 'SCOTCH' ] && [ "$pdlib" != 'PDLIB' ]
+  then
+      echo ' '
+      echo "   *** For SCOTCH, we need to have PDLIB as well."
+      echo ' ' ; exit 20
+  fi
 
 } #end of check_switches
 

model/bin/link.tmpl

@@ -117,9 +117,23 @@
     libs="$libs `$NETCDF_CONFIG --flibs` `$NETCDF_CONFIG --libs`"
   fi
 
+  # scotch library 
+  if [ "$prog" = 'ww3_shel' ] || [ "$prog" = 'ww3_multi' ] || [ "$prog" = 'ww3_sbs1' ] ; then
+    if [ "$scotch_mod" = 'yes' ] ; then
+      if [ -z "$(env | grep SCOTCH_PATH)" ] ; then
+        echo ''
+        echo "[ERROR] SCOTCH_PATH is not defined"
+        exit 1
+      fi
+      echo "link with scotch" 
+     libs="$libs $SCOTCH_PATH/lib/libptscotchparmetisv3.a $SCOTCH_PATH/lib/libptscotch.a  $SCOTCH_PATH/lib/libptscotcherr.a $SCOTCH_PATH/lib/libscotch.a"
+    fi
+  fi
+
+
   # parmetis library
   if [ "$prog" = 'ww3_shel' ] || [ "$prog" = 'ww3_multi' ] || [ "$prog" = 'ww3_sbs1' ] ; then
-    if [ "$pdlib_mod" = 'yes' ] ; then
+    if [ "$metis_mod" = 'yes' ] ; then
       if [ -z "$(env | grep METIS_PATH)" ] ; then
         echo ''
         echo "[ERROR] METIS_PATH is not defined"

model/bin/switch_Ifremer2_pdlib

@@ -1 +1 @@
-NOGRB PDLIB SCRIP SCRIPNC TRKNC DIST MPI PR3 UQ FLX0 LN1 ST4 STAB0 NL1 BT4 DB1 MLIM TR0 BS0 IC2 IS2 REF1 IG1 WNT2 WNX1 RWND CRT1 CRX1 TIDE O0 O1 O2 O2a O2b O2c O3 O4 O5 O6 O7
+NOGRB PDLIB METIS SCRIP SCRIPNC TRKNC DIST MPI PR3 UQ FLX0 LN1 ST4 STAB0 NL1 BT4 DB1 MLIM TR0 BS0 IC2 IS2 REF1 IG1 WNT2 WNX1 RWND CRT1 CRX1 TIDE O0 O1 O2 O2a O2b O2c O3 O4 O5 O6 O7

model/bin/switch_USACE_1

@@ -1 +1 @@
-NOGRB TRKNC DIST MPI MLIM PR3 UQ FLX1 PDLIB LN1 ST1 STAB0 NL1 BT4 DB0 TR0 BS0 IS0 IC0 REF0 WNT2 WNX1 CRT1 CRX1 O0 O1 O2 O2a O2b O2c O3 O4 O5 O6 O7
+NOGRB TRKNC DIST MPI MLIM PR3 UQ FLX1 PDLIB SCOTCH LN1 ST1 STAB0 NL1 BT4 DB0 TR0 BS0 IS0 IC0 REF0 WNT2 WNX1 CRT1 CRX1 O0 O1 O2 O2a O2b O2c O3 O4 O5 O6 O7

model/bin/switch_USACE_2

@@ -1 +1 @@
-NOGRB PDLIB TRKNC DIST MPI PR3 UQ FLX0 LN1 ST4 STAB0 NL1 BT1 DB1 MLIM TR0 BS0 WNT1 WNX1 IC0 IS0 REF0 RWND CRT1 CRX1 O0 O1 O2 O2a O2b O2c O3 O4 O5 O6 O7
+NOGRB PDLIB SCOTCH TRKNC DIST MPI PR3 UQ FLX0 LN1 ST4 STAB0 NL1 BT1 DB1 MLIM TR0 BS0 WNT1 WNX1 IC0 IS0 REF0 RWND CRT1 CRX1 O0 O1 O2 O2a O2b O2c O3 O4 O5 O6 O7

model/bin/switch_ite_pdlib

@@ -1 +1 @@
-NOGRB PDLIB TRKNC DIST MPI PR3 UQ FLX0 LN1 ST4 STAB0 NL1 BT4 DB1 TR0 BS0 IC0 IS0 REF1 WNT2 WNX1 RWND CRT1 CRX1 TIDE O0 O1 O2 O2a O2b O2c O3 O4 O5 O6 O7
+NOGRB PDLIB SCOTCH TRKNC DIST MPI PR3 UQ FLX0 LN1 ST4 STAB0 NL1 BT4 DB1 TR0 BS0 IC0 IS0 REF1 WNT2 WNX1 RWND CRT1 CRX1 TIDE O0 O1 O2 O2a O2b O2c O3 O4 O5 O6 O7

model/bin/switch_ugdev2

@@ -1 +1 @@
-NOGRB TRKNC DIST MPI SCRIP MLIM PR3 UQ FLX0 PDLIB LN1 ST4 STAB0 NL1 BT4 DB1 TR0 BS0 IS0 IC0 REF0 WNT2 WNX1 RWND CRT1 CRX1 O0 O1 O2 O2a O2b O2c O3 O4 O5 O6 O7
+NOGRB TRKNC DIST MPI SCRIP MLIM PR3 UQ FLX0 PDLIB SCOTCH LN1 ST4 STAB0 NL1 BT4 DB1 TR0 BS0 IS0 IC0 REF0 WNT2 WNX1 RWND CRT1 CRX1 O0 O1 O2 O2a O2b O2c O3 O4 O5 O6 O7

model/bin/w3_automake

@@ -305,6 +305,8 @@
     ./sort_switch -s -r switch
     sed -e 's/DIST/SHRD/g' \
         -e 's/PDLIB //g'    \
+        -e 's/METIS //g'    \
+        -e 's/SCOTCH //g'    \
         -e 's/OMPG //g'    \
         -e 's/PDLIB //g'    \
         -e 's/OMPH //g'    \