diff --git a/.github/actions/setup-spack-stack/action.yaml b/.github/actions/setup-spack-stack/action.yaml index 604930279..304ebb945 100644 --- a/.github/actions/setup-spack-stack/action.yaml +++ b/.github/actions/setup-spack-stack/action.yaml @@ -171,6 +171,14 @@ runs: spack config add "packages:all:compiler:[${{ inputs.compiler }}]" # Set compiler as hard requirement for all packages spack config add "packages:all:require:one_of:['%${{ inputs.compiler }}']" + # DH* Add this in when switching to unified-dev + ## Modify the compiler matrix definition + #if [[ "$RUNNER_OS" == "macOS" ]]; then + # sed -i '' "s/\['\%apple-clang', '\%gcc', '\%intel'\]/\['\%${{ inputs.compiler }}'\]/g" ${{ inputs.path }}/envs/${{ inputs.name }}/spack.yaml + #else + # sed -i "s/\['\%apple-clang', '\%gcc', '\%intel'\]/\['\%${{ inputs.compiler }}'\]/g" ${{ inputs.path }}/envs/${{ inputs.name }}/spack.yaml + #fi + # *DH if [[ "$RUNNER_OS" == "macOS" ]]; then # Turn of SSL for ecflow CI builds for now diff --git a/.github/workflows/macos-dom.yaml b/.github/workflows/macos-dom.yaml index b6bcb52c4..a56e0a21e 100644 --- a/.github/workflows/macos-dom.yaml +++ b/.github/workflows/macos-dom.yaml @@ -13,7 +13,7 @@ on: template: description: 'Base spack.yaml template. Default is a complete JEDI-UFS environment.' required: true - default: 'skylab-dev' + default: 'unified-dev' specs: description: 'Which specs to add to the template. Default is none (empty string).' required: false @@ -38,8 +38,8 @@ jobs: - name: create-env run: | source ./setup.sh - spack stack create env --site macos.default --template ${{ inputs.template || 'skylab-dev' }} --name ${{ inputs.template || 'skylab-dev' }}.macos-dom - spack env activate -d envs/${{ inputs.template || 'skylab-dev' }}.macos-dom + spack stack create env --site macos.default --template ${{ inputs.template || 'unified-dev' }} --name ${{ inputs.template || 'unified-dev' }}.macos-dom + spack env activate -d envs/${{ inputs.template || 'unified-dev' }}.macos-dom spack add ${{ inputs.specs || '' }} spack external find @@ -49,8 +49,7 @@ jobs: PATH="/usr/local/opt/curl/bin:${PATH}" spack external find curl spack compiler find - ## https://github.com/dtcenter/MET/issues/2239 - #spack remove global-workflow-env + sed -i '' "s/\['\%apple-clang', '\%gcc', '\%intel'\]/\['\%apple-clang'\]/g" envs/${{ inputs.template || 'unified-dev' }}.macos-dom/spack.yaml spack concretize spack install --fail-fast diff --git a/.github/workflows/macos.yaml b/.github/workflows/macos.yaml index 90eee46a0..4d151d2ae 100644 --- a/.github/workflows/macos.yaml +++ b/.github/workflows/macos.yaml @@ -62,7 +62,7 @@ jobs: strategy: matrix: compiler-mpi: [ {compiler: apple-clang@13.0.0, mpi: openmpi@4.1.4} ] - template: [skylab-dev, ufs-weather-model, ufs-srw-dev] + template: [skylab-dev, ufs-weather-model, ufs-srw-dev] # [unified-dev] runs-on: macos-11 steps: diff --git a/.github/workflows/ubuntu.yaml b/.github/workflows/ubuntu.yaml index df48eea5b..2c421db28 100644 --- a/.github/workflows/ubuntu.yaml +++ b/.github/workflows/ubuntu.yaml @@ -62,7 +62,7 @@ jobs: strategy: matrix: compiler-mpi: [ {compiler: intel@2021.8.0, mpi: intel-oneapi-mpi@2021.8.0}, {compiler: gcc@9.4.0, mpi: mvapich2@2.3.7}, {compiler: gcc@10.3.0, mpi: mpich@4.0.2} ] - template: [skylab-dev, ufs-weather-model, ufs-srw-dev] + template: [skylab-dev, ufs-weather-model, ufs-srw-dev] # [unified-dev] runs-on: ubuntu-20.04 steps: diff --git a/configs/common/modules.yaml b/configs/common/modules.yaml index 666bcef48..5e1e0492f 100644 --- a/configs/common/modules.yaml +++ b/configs/common/modules.yaml @@ -6,7 +6,7 @@ modules: arch_folder: false tcl: # Core compiler is a dummy that is not used to build the stack, do not change. - # Hopefully nobody is trying to build software with gcc-4.6 in 2022 ... + # Hopefully nobody is trying to build software with gcc-4.6 in 2023 ... core_compilers:: - gcc@4.6 projections: @@ -233,7 +233,7 @@ modules: lmod: # Core compiler is a dummy that is not used to build the stack, do not change. - # Hopefully nobody is trying to build software with gcc-4.6 in 2022 ... + # Hopefully nobody is trying to build software with gcc-4.6 in 2023 ... core_compilers:: - gcc@4.6 projections: diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 830846b12..6839a34d8 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -18,8 +18,6 @@ # Attention - when updating also check orion site config boost: version: [1.78.0] - # Install only the minimum needed (JEDI packages need headers only, - # but ecflow needs several libraries) variants: ~atomic +chrono +date_time +exception +filesystem ~graph ~iostreams ~locale ~log ~math ~mpi ~numpy +pic +program_options +python ~random +regex +serialization ~signals +system +test +thread +timer ~wave cxxstd=14 visibility=hidden bufr: version: [11.7.1] @@ -34,15 +32,13 @@ # Attention - when updating also check the various jcsda-emc-bundles env packages crtm: version: [2.4.0] - diffutils: - version: [3.7] + variants: +fix ecbuild: version: [3.6.5] eccodes: version: [2.27.0] - #variants: +netcdf ecflow: - version: [5.8.3] + version: [5.8.4] variants: +ui eckit: version: [1.20.2] @@ -67,11 +63,6 @@ version: [3.3.10] fiat: version: [1.1.0] - findutils: - version: [4.8.0] - flex: - version: [2.6.4] - # Attention - when updating also check the various jcsda-emc-bundles env packages fms: version: [2022.04] variants: +64bit +quad_precision +gfs_phys +openmp +fpic constants=GFS @@ -80,7 +71,7 @@ g2c: version: [1.6.4] g2tmpl: - version: [1.10.0] + version: [1.10.2] gfsio: version: [1.4.1] gftl-shared: @@ -101,8 +92,8 @@ version: [4.2.15] variants: ~fortran ~netcdf hdf5: - version: [1.12.1] - variants: +hl +fortran +mpi +threadsafe + version: [1.14.0] + variants: +hl +fortran +mpi +threadsafe +szip ip: version: [3.3.3] ip2: @@ -142,12 +133,13 @@ ncio: version: [1.1.2] netcdf-c: - version: [4.8.1] - variants: +dap +mpi +parallel-netcdf + version: [4.9.0] + # If using 4.9.1, turn off byterange variant to fix compile error: ~byterange + variants: +dap +mpi ~parallel-netcdf netcdf-cxx4: version: [4.3.1] netcdf-fortran: - version: [4.5.4] + version: [4.6.0] # ninja - when adding information here, also check Cheyenne # and Discover site configs nlohmann-json: @@ -163,7 +155,7 @@ openmpi: variants: +internal-hwloc +two_level_namespace parallelio: - version: [2.5.7] + version: [2.5.9] variants: +pnetcdf parallel-netcdf: version: [1.12.2] @@ -174,7 +166,6 @@ version: [1.2.2] proj: version: [8.1.0] - # don't really need tiff ... variants: ~tiff python: version: [3.10.8] diff --git a/configs/sites/macos.default/packages.yaml b/configs/sites/macos.default/packages.yaml index ad72fe5a7..c35c0a7d0 100644 --- a/configs/sites/macos.default/packages.yaml +++ b/configs/sites/macos.default/packages.yaml @@ -3,3 +3,5 @@ packages: variants: ~openmp cairo: variants: ~png ~svg + global-workflow-env: + variants: ~ncl diff --git a/configs/sites/orion/compilers.yaml b/configs/sites/orion/compilers.yaml index 39f646399..36d18c38b 100644 --- a/configs/sites/orion/compilers.yaml +++ b/configs/sites/orion/compilers.yaml @@ -17,6 +17,24 @@ compilers: LD_LIBRARY_PATH: '/apps/gcc-10.2.0/gcc-10.2.0/lib64:/apps/gcc-10.2.0/gcc-10.2.0/contrib/lib' CPATH: '/apps/gcc-10.2.0/gcc-10.2.0/include' extra_rpaths: [] +- compiler: + spec: intel@18.0.5 + paths: + cc: /apps/intel-2018/intel-2018.u4/compilers_and_libraries_2018.5.274/linux/bin/intel64/icc + cxx: /apps/intel-2018/intel-2018.u4/compilers_and_libraries_2018.5.274/linux/bin/intel64/icpc + f77: /apps/intel-2018/intel-2018.u4/compilers_and_libraries_2018.5.274/linux/bin/intel64/ifort + fc: /apps/intel-2018/intel-2018.u4/compilers_and_libraries_2018.5.274/linux/bin/intel64/ifort + flags: {} + operating_system: centos7 + target: x86_64 + modules: + - intel/2018.4 + environment: + prepend_path: + PATH: '/apps/gcc-8/gcc-8.3.0/bin' + LD_LIBRARY_PATH: '/apps/gcc-8/gcc-8.3.0/lib64:/apps/gcc-8/gcc-8.3.0//lib:/apps/gcc-8/gcc-8.3.0/mpfr-4.0.2/lib:/apps/gcc-8/gcc-8.3.0/mpc-1.1.0/lib:/apps/gcc-8/gcc-8.3.0/gmp-6.1.2/lib' + CPATH: '/apps/gcc-8/gcc-8.3.0/include' + extra_rpaths: [] - compiler: spec: gcc@10.2.0 paths: diff --git a/configs/sites/orion/packages.yaml b/configs/sites/orion/packages.yaml index 3128ea664..95c6ceb9b 100644 --- a/configs/sites/orion/packages.yaml +++ b/configs/sites/orion/packages.yaml @@ -1,10 +1,8 @@ packages: all: - compiler:: [intel@2022.0.2] - #compiler:: [gcc@10.2.0] + compiler:: [intel@2022.0.2, intel@18.0.5, gcc@10.2.0] providers: - mpi:: [intel-oneapi-mpi@2021.5.1] - #mpi:: [openmpi@4.0.4] + mpi:: [intel-oneapi-mpi@2021.5.1, intel-mpi@2018.5.274, openmpi@4.0.4] ### MPI, Python, MKL mpi: @@ -15,22 +13,12 @@ packages: prefix: /apps/intel-2022.1.2/intel-2022.1.2 modules: - impi/2022.1.2 - - spec: intel-oneapi-mpi@2021.2.0%intel@2021.2.0 - prefix: /apps/intel-2021.2/intel-2021.2 + intel-mpi: + externals: + - spec: intel-mpi@2018.5.274%intel@18.0.5 + prefix: /apps/intel-2018/intel-2018.u4/compilers_and_libraries_2018.5.274/linux/mpi modules: - - impi/2021.2 - #intel-oneapi-tbb: - # externals: - # - spec: intel-oneapi-tbb@2021.2.0%intel@2021.2.0 - # prefix: /apps/intel-2021.2/intel-2021.2 - # modules: - # - intel/2021.2 - #intel-oneapi-mkl: - # externals: - # - spec: intel-oneapi-mkl@2021.2.0%intel@2021.2.0 - # prefix: /apps/intel-2021.2/intel-2021.2 - # modules: - # - intel/2021.2 + - impi/2018.4 openmpi: externals: - spec: openmpi@4.0.4%gcc@10.2.0~cuda~cxx~cxx_exceptions~java~memchecker+pmi~static~wrapper-rpath @@ -82,10 +70,6 @@ packages: prefix: /apps/cmake-3.22.1 modules: - cmake/3.22.1 - - spec: cmake@3.18.1 - prefix: /apps/cmake-3.18.1 - modules: - - cmake/3.18.1 cpio: externals: - spec: cpio@2.11 diff --git a/configs/templates/hpc-dev-v1/spack.yaml b/configs/templates/hpc-dev-v1/spack.yaml deleted file mode 100644 index 8d424b3df..000000000 --- a/configs/templates/hpc-dev-v1/spack.yaml +++ /dev/null @@ -1,243 +0,0 @@ -spack: - view: false - - include: [] - - concretizer: - unify: when_possible - - config: - install_tree: - root: $env/install - - modules: - default: - roots: - lmod: $env/install/modulefiles - tcl: $env/install/modulefiles - lmod: - core_specs: - - jedi-tools-env - - view: false - - packages: - bacio: - version: [2.4.1] - boost: - version: [1.78.0] - variants: ~atomic +chrono +date_time +exception +filesystem ~graph ~iostreams ~locale ~log ~math ~mpi ~numpy +pic +program_options +python ~random +regex +serialization ~signals +system +test +thread +timer ~wave cxxstd=14 visibility=hidden - bufr: - version: [11.7.1] - variants: +python - cmake: - version: [3.22.1] - crtm: - version: [2.3.0] - variants: +fix - ecbuild: - version: [3.6.5] - eccodes: - version: [2.27.0] - eckit: - version: [1.20.2] - variants: linalg=eigen,lapack compression=lz4,bzip2 - ecmwf-atlas: - version: [0.31.1] - variants: +fckit +trans - ectrans: - version: [1.2.0] - variants: ~enable_mkl - eigen: - version: [3.4.0] - esmf: - version: [8.3.0b09] - variants: ~xerces ~pnetcdf +pio snapshot=b09 - fckit: - version: [0.9.5] - variants: +eckit - fftw: - version: [3.3.10] - fiat: - version: [1.1.0] - fms: - version: [2022.04] - variants: +64bit +quad_precision +gfs_phys +openmp +fpic constants=GFS - g2: - version: [3.4.5] - g2c: - version: [1.6.4] - g2tmpl: - version: [1.10.0] - gfsio: - version: [1.4.1] - gftl-shared: - version: [1.5.0] - grib-util: - version: [1.2.3] - gsibec: - version: [1.0.6] - gsl-lite: - version: [0.37.0] - hdf: - version: [4.2.15] - variants: ~fortran ~netcdf - hdf5: - version: [1.12.1] - variants: +hl +fortran +mpi +threadsafe - ip: - version: [3.3.3] - ip2: - version: [1.1.2] - jasper: - version: [2.0.32] - jedi-cmake: - version: [1.4.0] - jpeg: - version: [9.1.0] - landsfcutil: - version: [2.4.1] - libjpeg-turbo: - version: [2.1.0] - libpng: - version: [1.6.37] - mapl: - version: [2.22.0] - met: - version: [10.1.1] - variants: +python +grib2 - metplus: - version: [4.1.1] - nco: - version: [5.0.6] - variants: ~doc - nemsio: - version: [2.5.2] - nemsiogfs: - version: [2.5.3] - nccmp: - version: [1.9.0.1] - ncio: - version: [1.1.2] - netcdf-c: - version: [4.8.1] - variants: +dap +mpi +parallel-netcdf - netcdf-cxx4: - version: [4.3.1] - netcdf-fortran: - version: [4.5.4] - nlohmann-json: - version: [3.10.5] - nlohmann-json-schema-validator: - version: [2.1.0] - odc: - version: [1.4.5] - variants: ~fortran - openblas: - version: [0.3.19] - variants: +noavx512 - parallelio: - version: [2.5.7] - variants: +pnetcdf - parallel-netcdf: - version: [1.12.2] - pkgconf: - buildable: False - prod-util: - version: [1.2.2] - proj: - version: [8.1.0] - variants: ~tiff - python: - version: [3.10.8] - py-cartopy: - variants: +plotting - py-click: - version: [8.0.3] - py-eccodes: - version: [1.4.2] - py-h5py: - version: [3.6.0] - py-netcdf4: - version: [1.5.3] - variants: +mpi - py-numpy: - version: [1.22.3] - variants: +blas +lapack - py-openpyxl: - version: [3.0.3] - py-pandas: - version: [1.4.0] - py-pybind11: - version: [2.8.1] - py-pycodestyle: - version: [2.8.0] - py-pygithub: - version: [1.55] - py-pygrib: - version: [2.1.4] - py-pyhdf: - version: [0.10.4] - py-pyproj: - version: [3.1.0] - py-python-dateutil: - version: [2.8.2] - #py-pythran: - # version: [0.11.0] - py-pyyaml: - version: [6.0] - py-scipy: - version: [1.9.3] - py-shapely: - version: [1.8.0] - sfcio: - version: [1.4.1] - shumlib: - version: [macos_clang_linux_intel_port] - sigio: - version: [2.3.2] - sp: - version: [2.3.3] - udunits: - version: [2.2.28] - upp: - version: [10.0.10] - w3emc: - version: [2.9.2] - w3nco: - version: [2.4.1] - wget: - version: [1.21.2] - wgrib2: - version: [2.0.8] - wrf-io: - version: [1.2.0] - yafyaml: - version: [0.5.1] - zlib: - version: [1.2.13] - - definitions: - - compilers: ['%gcc', '%intel'] - - packages: - - jedi-ewok-env@hpc-dev-v1 - - jedi-fv3-env@hpc-dev-v1 - - jedi-tools-env@hpc-dev-v1 - - jedi-ufs-env@hpc-dev-v1 - - jedi-um-env@hpc-dev-v1 - - global-workflow-env@hpc-dev-v1 - - ufs-weather-model-env@hpc-dev-v1 - - ufs-weather-model-env+debug@hpc-dev-v1 - - upp-env@hpc-dev-v1 - - ww3-env@hpc-dev-v1 - - gsi-env@hpc-dev-v1 - - ufs-utils-env@hpc-dev-v1 - - nceplibs-env@hpc-dev-v1 - - soca-env@hpc-dev-v1 - - specs: - - matrix: - - [$packages] - - [$compilers] - exclude: - - jedi-tools-env%intel diff --git a/configs/templates/unified-dev/spack.yaml b/configs/templates/unified-dev/spack.yaml new file mode 100644 index 000000000..6a4b1a27a --- /dev/null +++ b/configs/templates/unified-dev/spack.yaml @@ -0,0 +1,42 @@ +spack: + concretizer: + unify: when_possible + + view: false + include: [] + + definitions: + - compilers: ['%apple-clang', '%gcc', '%intel'] + - packages: + - global-workflow-env@unified-dev + #- gsi-env@unified-dev + - jedi-ewok-env@unified-dev + - jedi-fv3-env@unified-dev + - jedi-tools-env@unified-dev + - jedi-ufs-env@unified-dev + - jedi-um-env@unified-dev + #- nceplibs-env@unified-dev + - soca-env@unified-dev + - ufs-srw-app-env@unified-dev + #- ufs-utils-env@unified-dev + - ufs-weather-model-env~debug@unified-dev + - ufs-weather-model-env+debug@unified-dev + #- upp-env@unified-dev + #- ww3-env@unified-dev + + specs: + - matrix: + - [$packages] + - [$compilers] + exclude: + # jedi-tools doesn't build with Intel + - jedi-tools-env%intel + # Don't even bother building those with Intel 18, + # not needed. Many of the Python packages don't build. + - jedi-ewok-env%intel@18 + - jedi-fv3-env%intel@18 + - jedi-tools-env%intel@18 + - jedi-ufs-env%intel@18 + - jedi-um-env%intel@18 + - soca-env%intel@18 + diff --git a/doc/source/KnownIssues.rst b/doc/source/KnownIssues.rst index f15b9b467..2969da068 100644 --- a/doc/source/KnownIssues.rst +++ b/doc/source/KnownIssues.rst @@ -69,7 +69,7 @@ UW (Univ. of Wisconsin) S4 macOS ============================== -1. Error `invalid argument '-fgnu89-inline' not allowed with 'C++'` +1. Error ``invalid argument '-fgnu89-inline' not allowed with 'C++'`` This error occurs on macOS Monterey with ``mpich-3.4.3`` installed via Homebrew when trying to build the jedi bundles that use ``ecbuild``. The reason was that the C compiler flag ``-fgnu89-inline`` from ``/usr/local/Cellar/mpich/3.4.3/lib/pkgconfig/mpich.pc`` was added to the C++ compiler flags by ecbuild. The solution was to set ``CC=mpicc FC=mpif90 CXX=mpicxx`` when calling ``ecbuild`` for those bundles. Note that it is recommended to install ``mpich`` or ``openmpi`` with spack-stack, not with Homebrew. @@ -77,10 +77,17 @@ macOS This can happen when multiple versions of Python were installed with Homebrew and ``pip3``/``python3`` point to different versions. Run ``brew doctor`` and check if there are issues with Python not being properly linked. Follow the instructions given by ``brew``, if applicable. -3. Errors handling exceptions on macOS. A large number of errors related to handling exceptions thrown by applications was found when using default builds or Homebrew installations of ``mpich`` or ``openmpi``, which use flat namespaces. With our spack version, ``mpich`` and ``openmpi`` are installed with a ``+two_level_namespace`` option that fixes the problem. +3. Error ``AttributeError: Can't get attribute 'Mark' on . # (copies site-specific, application-specific, and common config files into the environment directory) - spack stack create env --site hera --template ufs-weather-model --name ufs-weather-model.hera + spack stack create env --site hera --template unified-dev --name unified-dev.hera # Activate the newly created environment # Optional: decorate the command line prompt using -p # Note: in some cases, this can mess up long lines in bash # because color codes are not escaped correctly. In this # case, use export SPACK_COLOR='never' first. - spack env activate [-p] envs/jedi-fv3.hera + spack env activate [-p] envs/unified-dev.hera + + # Edit the main config file for the environment and adjust the compiler matrix + # to match the compilers available on your system, or a subset of them (see + # note below for more information). Replace + # definitions: + # - compilers: ['%apple-clang', '%gcc', '%intel'] + # with the appropriate list of compilers for your system and desires, e.g. + # definitions: + # - compilers: ['%gcc', '%intel'] + emacs envs/unified-dev.hera/spack.yaml # Optionally edit config files (spack.yaml, packages.yaml compilers.yaml, site.yaml) - emacs envs/jedi-fv3.hera/spack.yaml - emacs envs/jedi-fv3.hera/common/*.yaml - emacs envs/jedi-fv3.hera/site/*.yaml + emacs envs/unified-dev.hera/common/*.yaml + emacs envs/unified-dev.hera/site/*.yaml # Process/concretize the specs spack concretize diff --git a/spack b/spack index 83743d32a..c891ef769 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 83743d32adb1ea1166248a4f64b93f5d4694c502 +Subproject commit c891ef769f4ead9c8e37ca81eec6cbc60d73a500