diff --git a/.github/workflows/macos-ci-x86_64.yaml b/.github/workflows/macos-ci-x86_64.yaml new file mode 100644 index 000000000..338dd4e6f --- /dev/null +++ b/.github/workflows/macos-ci-x86_64.yaml @@ -0,0 +1,143 @@ +name: macos-ci-x86_64-build +on: + pull_request: + paths-ignore: + - 'doc/**' + - '**.md' + - '.github/ISSUE_TEMPLATE/*' + - '.gitignore' + workflow_dispatch: + # inputs: + # template: + # description: 'Base spack.yaml template. Default is a complete JEDI-UFS environment.' + # required: true + # default: 'skylab-dev' + # specs: + # description: 'Which specs to add to the template. Default is none (empty string).' + # required: false + # default: '' + + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +# Use custom shell with -l so .bash_profile is sourced +defaults: + run: + shell: bash -leo pipefail {0} + +jobs: + macos-ci-x86_64-build: + #if: ${{ github.event.label.name == 'run-macos-dom' }} || ${{ inputs.template }} + runs-on: [macos-ci-x86_64] + + steps: + - name: cleanup + run: | + pwd + ls -lart + rm -fr * + + - name: checkout + uses: actions/checkout@v2 + with: + submodules: true + + - name: create-env + run: | + source ./setup.sh + export ENVDIR=$PWD/envs/${{ inputs.template || 'skylab-dev' }}.macos-ci-x86_64 + spack stack create env --site macos.default --template ${{ inputs.template || 'skylab-dev' }} --name ${ENVDIR} + spack env activate -d ${ENVDIR} + spack add ${{ inputs.specs || '' }} + export SPACK_SYSTEM_CONFIG_PATH="${ENVDIR}/site" + + # Find external packages + spack external find --scope system + spack external find --scope system perl + spack external find --scope system wget + PATH="/usr/local/opt/curl/bin:$PATH" \ + spack external find --scope system curl + PATH="/usr/local/opt/qt5/bin:$PATH" \ + spack external find --scope system qt + spack external find --scope system texlive + # NOT YET + # spack external find --scope system mysql + + # Find compilers + spack compiler find --scope system + + export -n SPACK_SYSTEM_CONFIG_PATH + + # For buildcaches + spack config add config:padded_length:true + + # Set compiler and MPI + spack config add "packages:all:providers:mpi:[openmpi@4.1.4]" + spack config add "packages:all:compiler:[apple-clang@14.0.0]" + sed -i '' "s/\['\%apple-clang', '\%gcc', '\%intel'\]/\['\%apple-clang'\]/g" $ENVDIR/spack.yaml + + spack concretize 2>&1 | tee log.concretize.apple-clang-14.0.0 + + # Add and update source cache + spack mirror add local-source file:///Users/ec2-user/spack-stack/source-cache/ + spack mirror create -a -d /Users/ec2-user/spack-stack/source-cache/ + + # Add binary cache and reindex it + spack mirror add local-binary file:///Users/ec2-user/spack-stack/build-cache/ + spack buildcache update-index -m local-binary + echo "Packages in combined spack build caches:" + spack buildcache list + + # Break installation up in pieces and create build caches in between + # This allows us to "spin up" builds that altogether take longer than + # six hours, and/or fail later in the build process. + + # base-env + echo "base-env ..." + spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.apple-clang-14.0.0.base-env + #set +eo pipefail + spack buildcache create -a -r -u -d /Users/ec2-user/spack-stack/build-cache/ --skip-on-error || true + #echo "Return code: '$?'" + #set -eo pipefail + + ## jedi-base-env + #echo "jedi-base-env ..." + #spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.apple-clang-14.0.0.jedi-base-env + ##set +eo pipefail + #spack buildcache create -a -r -u -d /Users/ec2-user/spack-stack/build-cache/ --skip-on-error || true + ##echo "Return code: '$?'" + ##set -eo pipefail + + # the rest + echo "${{ inputs.template || 'skylab-dev' }} ..." + spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.apple-clang-14.0.0.${{ inputs.template || 'skylab-dev' }} + #set +eo pipefail + spack buildcache create -a -r -u -d /Users/ec2-user/spack-stack/build-cache/ --skip-on-error || true + #echo "Return code: '$?'" + #set -eo pipefail + + # Next steps: synchronize source and build cache to a central/combined mirror? + echo "Next steps ..." + spack clean -a + spack module lmod refresh -y + spack stack setup-meta-modules + spack env deactivate + + - name: test-env + run: | + source /usr/local/opt/lmod/init/profile + export ENVDIR=$PWD/envs/${{ inputs.template || 'skylab-dev' }}.macos-ci-x86_64 + module use $ENVDIR/install/modulefiles/Core + module load stack-apple-clang/14.0.0 + module load stack-openmpi/4.1.4 + module load stack-python/3.10.8 + module load jedi-ufs-env/1.0.0 jedi-ewok-env/1.0.0 soca-env/1.0.0 + + # Skip this for now, copied from other yaml + #- name: upload-mirror + # uses: actions/upload-artifact@v2 + # with: + # name: spack_mirror + # path: cache/source_cache diff --git a/.github/workflows/macos-dom.yaml b/.github/workflows/macos-dom.yaml index a56e0a21e..27b4c1587 100644 --- a/.github/workflows/macos-dom.yaml +++ b/.github/workflows/macos-dom.yaml @@ -27,7 +27,7 @@ defaults: jobs: macos-dom-build: if: ${{ github.event.label.name == 'run-macos-dom' }} || ${{ inputs.template }} - runs-on: [self-hosted, macOS, X64, macos-dom] + runs-on: [macos-dom] steps: - name: checkout diff --git a/configs/sites/narwhal/compilers.yaml b/configs/sites/narwhal/compilers.yaml index 183f65bb1..3bb3464e6 100644 --- a/configs/sites/narwhal/compilers.yaml +++ b/configs/sites/narwhal/compilers.yaml @@ -20,3 +20,22 @@ compilers:: set: CRAYPE_LINK_TYPE: 'dynamic' extra_rpaths: [] + - compiler: + spec: gcc@10.3.0 + paths: + cc: cc + cxx: CC + f77: ftn + fc: ftn + flags: {} + operating_system: sles15 + target: any + modules: + - PrgEnv-gnu/8.3.2 + - gcc/10.3.0 + environment: + prepend_path: + LD_LIBRARY_PATH: '/opt/cray/pe/libsci/22.08.1.1/GNU/9.1/x86_64/lib' + set: + CRAYPE_LINK_TYPE: 'dynamic' + extra_rpaths: [] diff --git a/configs/sites/narwhal/packages.yaml b/configs/sites/narwhal/packages.yaml index c722fba5b..bf015cb12 100644 --- a/configs/sites/narwhal/packages.yaml +++ b/configs/sites/narwhal/packages.yaml @@ -1,6 +1,6 @@ packages: all: - compiler:: [intel@2021.3.0] + compiler:: [intel@2021.3.0, gcc@10.3.0] providers: mpi:: [cray-mpich@8.1.14] @@ -12,6 +12,9 @@ packages: - spec: cray-mpich@8.1.14%intel@2021.3.0 ~wrappers modules: - cray-mpich/8.1.14 + - spec: cray-mpich@8.1.14%gcc@10.3.0 ~wrappers + modules: + - cray-mpich/8.1.14 #intel-oneapi-tbb: # externals: # - spec: intel-oneapi-tbb@2021.3.0 diff --git a/doc/source/Platforms.rst b/doc/source/Platforms.rst index d76881974..dca42361a 100644 --- a/doc/source/Platforms.rst +++ b/doc/source/Platforms.rst @@ -31,7 +31,9 @@ spack-stack-v1 +----------------------------------------------------------+---------------------------+--------------------------------------------------------------------------------------------------------------------+ | NASA Discover GNU | Dom Heinzeller | ``/discover/swdev/jcsda/spack-stack/spack-stack-v1/envs/skylab-3.0.0-gnu-10.1.0/install`` | +----------------------------------------------------------+---------------------------+--------------------------------------------------------------------------------------------------------------------+ -| NAVY HPCMP Narwhal | Dom Heinzeller | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-v1/envs/skylab-3.0.0-intel-2021.4.0/install`` | +| NAVY HPCMP Narwhal Intel | Dom Heinzeller | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-v1/envs/skylab-3.0.0-intel-2021.4.0/install`` | ++----------------------------------------------------------+---------------------------+--------------------------------------------------------------------------------------------------------------------+ +| NAVY HPCMP Narwhal GNU (**TEMPORARY LOCATION**) | Dom Heinzeller | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-test-20230214/envs/skylab-dev-gnu-10.3.0-libsci-22.08.1.1/install``| +----------------------------------------------------------+---------------------------+--------------------------------------------------------------------------------------------------------------------+ | NCAR-Wyoming Casper | Dom Heinzeller | ``/glade/work/jedipara/cheyenne/spack-stack/spack-stack-v1/envs/skylab-3.0.0-intel-19.1.1.217-casper/install`` | +----------------------------------------------------------+---------------------------+--------------------------------------------------------------------------------------------------------------------+ @@ -144,7 +146,7 @@ For ``spack-stack-1.2.0``/``skylab-3.0.0`` with GNU, load the following modules NAVY HPCMP Narwhal ------------------------------ -The following is required for building new spack environments and for using spack to build and run software. +With Intel, the following is required for building new spack environments and for using spack to build and run software: .. code-block:: console @@ -171,6 +173,33 @@ For ``spack-stack-1.2.0``/``skylab-3.0.0`` with Intel, load the following module module load stack-cray-mpich/8.1.14 module load stack-python/3.9.7 +With GNU, the following is required for building new spack environments and for using spack to build and run software: + +.. code-block:: console + + module unload PrgEnv-cray + module load PrgEnv-gnu/8.3.2 + module unload gcc + module load gcc/10.3.0 + module unload cray-mpich + module load cray-mpich/8.1.14 + module unload cray-python + module load cray-python/3.9.7.1 + module unload cray-libsci + module load cray-libsci/22.08.1.1 + + module use /p/app/projects/NEPTUNE/spack-stack/modulefiles + module load ecflow/5.8.4 + +For ``spack-stack-1.2.0``/``skylab-3.0.0`` with GNU, load the following modules after loading the above modules. **Note: temporary location!** + +.. code-block:: console + + module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-test-20230214/envs/skylab-dev-gnu-10.3.0-libsci-22.08.1.1/install/modulefiles/Core + module load stack-gcc/10.3.0 + module load stack-cray-mpich/8.1.14 + module load stack-python/3.9.7 + .. _Platforms_Casper: ------------------- @@ -462,11 +491,11 @@ It is also instructive to peruse the GitHub actions scripts in ``.github/workflo macOS ------------------------------ -On macOS, it is important to use certain Homebrew packages as external packages, because the native macOS packages are incomplete (e.g. missing the development header files): ``curl``, ``python``, ``qt``, etc. The instructions provided in the following have been tested extensively on many macOS installations. +On macOS, it is important to use certain Homebrew packages as external packages, because the native macOS packages are incomplete (e.g. missing the development header files): ``curl``, ``qt``, etc. The instructions provided in the following have been tested extensively on many macOS installations. -The instructions below also assume a clean Homebrew installation with a clean Python installation inside. This means that the Homebrew Python only contains nothing but what gets installed with ``pip install poetry`` (which is a temporary workaround). If this is not the case, users can try to install a separate Python using Miniconda as described in :numref:`Sections %s `. +Unlike in previous versions, the instructions below assume that ``Python`` is built by ``spack``. That means that when using the ``spack`` environments (i.e., loading the modules for building or running code), the ``spack`` installation of ``Python`` with its available ``Python`` modules should be used to ensure consistency. However, a Homebrew ``Python`` installation may still be needed to build new ``spack`` environments. It can also be beneficial for the user to have a version of ``Python`` installed with Homebrew that can be used for virtual environments that are completely independent of any ``spack``-built environment. -Further, it is recommended to not use ``mpich`` or ``openmpi`` installed by Homebrew, because these packages are built using a flat namespace that is incompatible with the JEDI software. The spack-stack installations of ``mpich`` and ``openmpi`` use two-level namespaces as required. +It is recommended to not use ``mpich`` or ``openmpi`` installed by Homebrew, because these packages are built using a flat namespace that is incompatible with the JEDI software. The spack-stack installations of ``mpich`` and ``openmpi`` use two-level namespaces as required. Intel M1 platform notes ----------------------- @@ -654,7 +683,7 @@ Remember to activate the ``lua`` module environment and have MacTeX in your sear unset SPACK_SYSTEM_CONFIG_PATH -6. Set default compiler and MPI library and flag Python as non-buildable (make sure to use the correct ``apple-clang`` version for your system and the desired ``openmpi`` version) +6. Set default compiler and MPI library (make sure to use the correct ``apple-clang`` version for your system and the desired ``openmpi`` version) .. code-block:: console @@ -746,9 +775,6 @@ The following instructions were used to prepare a basic Red Hat 8 system as it i # Python yum -y install python39-devel alternatives --set python3 /usr/bin/python3.9 - python3 -m pip install poetry - # test - successful if no output - python3 -c "import poetry" # Exit root session exit @@ -800,9 +826,6 @@ The following instructions were used to prepare a basic Ubuntu 20.04 system as i # Python apt install -y python3-dev python3-pip - python3 -m pip install poetry - # test - successful if no output - python3 -c "import poetry" # Exit root session exit @@ -850,9 +873,6 @@ The following instructions were used to prepare a basic Ubuntu 22.04 system as i # Python apt install -y python3-dev python3-pip - python3 -m pip install poetry - # test - successful if no output - python3 -c "import poetry" # Exit root session exit @@ -887,7 +907,8 @@ It is recommended to increase the stacksize limit by using ``ulimit -S -s unlimi spack external find --scope system spack external find --scope system perl - spack external find --scope system python + # Don't use any external Python, let spack build it + #spack external find --scope system python spack external find --scope system wget spack external find --scope system texlive # On Ubuntu (but not on Red Hat): @@ -905,23 +926,20 @@ It is recommended to increase the stacksize limit by using ``ulimit -S -s unlimi unset SPACK_SYSTEM_CONFIG_PATH -6. Set default compiler and MPI library and flag Python as non-buildable (make sure to use the correct ``gcc`` version for your system and the desired ``openmpi`` version) +6. Set default compiler and MPI library (make sure to use the correct ``gcc`` version for your system and the desired ``openmpi`` version) .. code-block:: console # Example for Red Hat 8 following the above instructions - spack config add "packages:python:buildable:False" spack config add "packages:all:providers:mpi:[openmpi@4.1.4]" spack config add "packages:all:compiler:[gcc@11.2.1]" # Example for Ubuntu 20.04 following the above instructions - spack config add "packages:python:buildable:False" spack config add "packages:all:providers:mpi:[mpich@4.0.2]" spack config add "packages:all:compiler:[gcc@10.3.0]" # Example for Ubuntu 22.04 following the above instructions sed -i 's/tcl/lmod/g' envs/jedi-ufs.mylinux/site/modules.yaml - spack config add "packages:python:buildable:False" spack config add "packages:all:providers:mpi:[mpich@4.0.2]" spack config add "packages:all:compiler:[gcc@11.2.0]" diff --git a/spack b/spack index c5a3c25ba..6df7bb3b1 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit c5a3c25ba193fe039015c0693be93b03816257dd +Subproject commit 6df7bb3b1c45f29a1d191a52af24030bde64af8a