From 72ec9f8e5d59d03f234a72dbd77fbc7e2f35f702 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Mon, 4 Nov 2024 07:26:38 -0700
Subject: [PATCH 01/16] Update .gitmodules and submodule pointer for spack for
 code review and testing

---
 .gitmodules | 6 ++++--
 spack       | 2 +-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/.gitmodules b/.gitmodules
index a69f3e4f1d..8db64f1364 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,7 +1,9 @@
 [submodule "spack"]
   path = spack
-  url = https://github.com/jcsda/spack
-  branch = spack-stack-dev
+  #url = https://github.com/jcsda/spack
+  #branch = spack-stack-dev
+  url = https://github.com/climbfuji/spack
+  branch = feature/update_from_spack_dev_20241031
 [submodule "doc/CMakeModules"]
   path = doc/CMakeModules
   url = https://github.com/noaa-emc/cmakemodules
diff --git a/spack b/spack
index 990f0b84b7..2f31a163f8 160000
--- a/spack
+++ b/spack
@@ -1 +1 @@
-Subproject commit 990f0b84b7f6096c0542d04784c291666cdc2b7b
+Subproject commit 2f31a163f862793c6e94ee76d13640366f3c1834

From 15a91da195b9b954767aab2b6e932ea0a9239a28 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Mon, 4 Nov 2024 07:26:59 -0700
Subject: [PATCH 02/16] On blackpearl, only use one build job for spack

---
 configs/sites/tier2/blackpearl/config.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/configs/sites/tier2/blackpearl/config.yaml b/configs/sites/tier2/blackpearl/config.yaml
index 77a2e80adb..0eb7669fbf 100644
--- a/configs/sites/tier2/blackpearl/config.yaml
+++ b/configs/sites/tier2/blackpearl/config.yaml
@@ -1,2 +1,2 @@
 config:
-  build_jobs: 2
+  build_jobs: 1

From 9a2e3048f8e75cf423a5f543cf3e3da8a86a3eab Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Mon, 4 Nov 2024 07:27:26 -0700
Subject: [PATCH 03/16] Bug fix in configs/templates/unified-dev/spack.yaml:
 gsi-env does not depend on ESMF

---
 configs/templates/unified-dev/spack.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/configs/templates/unified-dev/spack.yaml b/configs/templates/unified-dev/spack.yaml
index 370a03e7c2..7447897784 100644
--- a/configs/templates/unified-dev/spack.yaml
+++ b/configs/templates/unified-dev/spack.yaml
@@ -13,7 +13,7 @@ spack:
     - geos-gcm-env          ^esmf@=8.6.1
     - global-workflow-env   ^esmf@=8.6.1
     - gmao-swell-env
-    - gsi-env               ^esmf@=8.6.1
+    - gsi-env
     - jedi-fv3-env
     - jedi-geos-env         ^esmf@=8.6.1
     - jedi-mpas-env

From 321d42f7b83363e6fd23127729b0b4f588b8339f Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Mon, 4 Nov 2024 07:28:13 -0700
Subject: [PATCH 04/16] Update configs/common/packages.yaml: pin gmake to
 either 4.2.1 or 4.4.1, and remove noavx512 variant for openblas

---
 configs/common/packages.yaml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml
index 7248b1b61a..011e5027ca 100644
--- a/configs/common/packages.yaml
+++ b/configs/common/packages.yaml
@@ -102,6 +102,10 @@ packages:
     # Note: Uncommenting this entry will break
     # the container builds.
     #require: '@2.11.0'
+    # To avoid duplicate packages being built
+  gmake:
+    require:
+    - one_of: ['@=4.2.1', '@=4.4.1']
   grib-util:
     require: '@1.4.0'
   gsibec:
@@ -170,7 +174,7 @@ packages:
   odc:
     require: '@1.5.2 ~fortran'
   openblas:
-    require: '@0.3.24 +noavx512'
+    require: '@0.3.24'
   openmpi:
     require: '~internal-hwloc +two_level_namespace'
   openssl:

From 8a651e38e36d5017c74e73db3da5f818039388e5 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Mon, 4 Nov 2024 08:43:58 -0700
Subject: [PATCH 05/16] Pin meson to 1.5.1 or higher to avoid duplicate
 packages being built

---
 configs/common/packages.yaml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml
index 011e5027ca..168ccf3422 100644
--- a/configs/common/packages.yaml
+++ b/configs/common/packages.yaml
@@ -102,7 +102,7 @@ packages:
     # Note: Uncommenting this entry will break
     # the container builds.
     #require: '@2.11.0'
-    # To avoid duplicate packages being built
+  # To avoid duplicate packages being built
   gmake:
     require:
     - one_of: ['@=4.2.1', '@=4.4.1']
@@ -142,6 +142,10 @@ packages:
   mapl:
     require: '@2.46.3 ~shared ~f2py'
     variants: '+pflogger'
+  # To avoid duplicate packages being built
+  meson:
+    require:
+    - '@1.5.1:'
   # If making changes here, also check the Discover site configs and the CI workflows
   met:
     require: '@11.1.1 +python +grib2'

From 46705640588a386731594b64f508f477842f7711 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Mon, 4 Nov 2024 13:39:51 -0700
Subject: [PATCH 06/16] Update submodule pointer for spack

---
 spack | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/spack b/spack
index 2f31a163f8..7c25b422cf 160000
--- a/spack
+++ b/spack
@@ -1 +1 @@
-Subproject commit 2f31a163f862793c6e94ee76d13640366f3c1834
+Subproject commit 7c25b422cfeaa1e9d64ccc5cc447bcce7ec736ef

From e347cc36450eda858246f4355ae57fb1c0a30a62 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Tue, 5 Nov 2024 12:48:47 -0700
Subject: [PATCH 07/16] Update openmpi from 5.0.3 to 5.0.5 in several
 site/container configs

---
 configs/containers/docker-ubuntu-gcc-openmpi.yaml | 2 +-
 configs/sites/tier2/blackpearl/packages_gcc.yaml  | 2 +-
 configs/sites/tier2/bounty/packages_gcc.yaml      | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/configs/containers/docker-ubuntu-gcc-openmpi.yaml b/configs/containers/docker-ubuntu-gcc-openmpi.yaml
index f0e495073e..526d475895 100644
--- a/configs/containers/docker-ubuntu-gcc-openmpi.yaml
+++ b/configs/containers/docker-ubuntu-gcc-openmpi.yaml
@@ -28,7 +28,7 @@ spack:
       require: '%gcc'
       target: [x86_64]
       providers:
-        mpi: [openmpi@5.0.3]
+        mpi: [openmpi@5.0.5]
       compiler: [gcc@12.3.0]
     gcc:
       buildable: false
diff --git a/configs/sites/tier2/blackpearl/packages_gcc.yaml b/configs/sites/tier2/blackpearl/packages_gcc.yaml
index a389fbcfcf..7306013420 100644
--- a/configs/sites/tier2/blackpearl/packages_gcc.yaml
+++ b/configs/sites/tier2/blackpearl/packages_gcc.yaml
@@ -2,4 +2,4 @@ packages:
   all:
     compiler:: [gcc@13.3.0]
     providers:
-      mpi:: [openmpi@5.0.3]
+      mpi:: [openmpi@5.0.5]
diff --git a/configs/sites/tier2/bounty/packages_gcc.yaml b/configs/sites/tier2/bounty/packages_gcc.yaml
index a914083661..0ddd36693c 100644
--- a/configs/sites/tier2/bounty/packages_gcc.yaml
+++ b/configs/sites/tier2/bounty/packages_gcc.yaml
@@ -2,4 +2,4 @@ packages:
   all:
     compiler:: [gcc@12.3.0]
     providers:
-      mpi:: [openmpi@5.0.3]
+      mpi:: [openmpi@5.0.5]

From d57737d0a95ace943b68cdc199545205b860152e Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Wed, 6 Nov 2024 08:53:31 -0700
Subject: [PATCH 08/16] Update GitHub actions workflows: make macOS workflow
 similar to Ubuntu workflows

---
 .github/workflows/macos-ci-aarch64.yaml       | 63 +++++++++++++++----
 .github/workflows/ubuntu-ci-x86_64-gnu.yaml   | 18 ++++--
 .github/workflows/ubuntu-ci-x86_64-intel.yaml | 11 ++--
 .../workflows/ubuntu-ci-x86_64-oneapi.yaml    | 30 ++++-----
 4 files changed, 82 insertions(+), 40 deletions(-)

diff --git a/.github/workflows/macos-ci-aarch64.yaml b/.github/workflows/macos-ci-aarch64.yaml
index e905915f25..0146b0d51d 100644
--- a/.github/workflows/macos-ci-aarch64.yaml
+++ b/.github/workflows/macos-ci-aarch64.yaml
@@ -35,7 +35,7 @@ jobs:
           mkdir -p /Users/ec2-user/spack-stack/build-cache
           mkdir -p /Users/ec2-user/spack-stack/source-cache
 
-      - name: create-env
+      - name: create-buildcache
         run: |
           # Get day of week to decide whether to use build caches or not
           DOW=$(date +%u)
@@ -56,15 +56,13 @@ jobs:
           source ./setup.sh
           # Important!
           export SPACK_PYTHON=/usr/bin/python3
-          export ENVNAME=ue-apple-clang-14.0.3
-          export ENVDIR=$PWD/envs/${ENVNAME}
-
           spack clean -ab
           spack bootstrap now
-
+          #
+          export ENVNAME=ue-apple-clang-14.0.3-buildcache
+          export ENVDIR=$PWD/envs/${ENVNAME}
           spack stack create env --site macos.default --template unified-dev --name ${ENVNAME} --compiler apple-clang
           spack env activate ${ENVDIR}
-          spack add ${{ inputs.specs || '' }}
           export SPACK_SYSTEM_CONFIG_PATH="${ENVDIR}/site"
 
           # Find external packages
@@ -86,8 +84,11 @@ jobs:
 
           export -n SPACK_SYSTEM_CONFIG_PATH
 
+          # For buildcaches
+          spack config add config:install_tree:padded_length:200
+
           # Set compiler and MPI specs
-          spack config add "packages:all:providers:mpi:[openmpi@5.0.3]"
+          spack config add "packages:all:providers:mpi:[openmpi@5.0.5]"
           spack config add "packages:all:compiler:[apple-clang@14.0.3]"
 
           # Add additional variants for MET packages, different from config/common/packages.yaml
@@ -97,8 +98,8 @@ jobs:
           # *DH
 
           # Concretize and check for duplicates
-          spack concretize 2>&1 | tee log.concretize.apple-clang-14.0.3
-          ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.apple-clang-14.0.3 -i fms -i crtm -i esmf -i mapl
+          spack concretize 2>&1 | tee log.concretize.apple-clang-14.0.3-buildcache
+          ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.apple-clang-14.0.3-buildcache -i fms -i crtm -i esmf -i mapl
 
           # Add and update source cache
           spack mirror add local-source file:///Users/ec2-user/spack-stack/source-cache/
@@ -128,6 +129,11 @@ jobs:
           spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.apple-clang-14.0.3.jedi-base-env
           spack buildcache create -a -u /Users/ec2-user/spack-stack/build-cache/ jedi-base-env
 
+          # jedi-ufs-env
+          echo "jedi-ufs-env ..."
+          spack install --fail-fast --source --no-check-signature jedi-ufs-env 2>&1 | tee log.install.apple-clang-14.0.3.jedi-ufs-env
+          spack buildcache create -a -u /Users/ec2-user/spack-stack/build-cache/ jedi-ufs-env
+
           # the rest
           echo "unified-env ..."
           spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.apple-clang-14.0.3.unified-env
@@ -138,12 +144,44 @@ jobs:
             spack mirror rm local-binary
           fi
 
+          # Remove buildcache config settings
+          spack config remove config:install_tree:padded_length
+
+          # Next steps: synchronize source and build cache to a central/combined mirror?
+
+          # Cleanup
+          spack clean -a
+          spack env deactivate
+
+      - name: create-env
+        run: |
+          # Set up spack-stack
+          source ./setup.sh
+          # Important!
+          export SPACK_PYTHON=/usr/bin/python3
+          export BUILDCACHE_ENVNAME=ue-apple-clang-14.0.3-builddcache
+          export BUILDCACHE_ENVDIR=$PWD/envs/${BUILDCACHE_ENVNAME}
+          export ENVNAME=ue-apple-clang-14.0.3
+          export ENVDIR=$PWD/envs/${ENVNAME}
+          rsync -av --exclude='install' --exclude='spack.lock' --exclude='.spack_db' ${BUILDCACHE_ENVDIR}/ ${ENVDIR}/
+          spack env activate ${ENVDIR}
+
+          # Concretize and check for duplicates
+          spack concretize --force 2>&1 | tee log.concretize.apple-clang-14.0.3
+          ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.apple-clang-14.0.3 -i fms -i crtm -i esmf -i mapl
+
+          # Add binary cache back in
+          spack mirror add local-binary file:///Users/ec2-user/spack-stack/build-cache/
+          echo "Packages in combined spack build caches:"
+          spack buildcache list
+
+          # Install from cache
+          spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.apple-clang-14.0.3.unified-env
+
           # Create modules
           spack clean -a
           spack module lmod refresh -y
           spack stack setup-meta-modules
-
-          # Next steps: synchronize source and build cache to a central/combined mirror?
           spack env deactivate
 
           # Test environment chaining
@@ -160,7 +198,6 @@ jobs:
 
       - name: test-env
         run: |
-          # Set up homebrew and lmod support
           eval "$(/opt/homebrew/bin/brew shellenv)"
           source /opt/homebrew/opt/lmod/init/profile
 
@@ -170,7 +207,7 @@ jobs:
 
           module use ${ENVDIR}/install/modulefiles/Core
           module load stack-apple-clang/14.0.3
-          module load stack-openmpi/5.0.3
+          module load stack-openmpi/5.0.5
           module load stack-python/3.11.7
           module available
 
diff --git a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml
index 3b12d1c11f..5452a9ab5d 100644
--- a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml
+++ b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml
@@ -122,6 +122,11 @@ jobs:
             spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.gnu-11.4.0-buildcache.jedi-base-env
             spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env
 
+            # jedi-ufs-env
+            echo "jedi-ufs-env ..."
+            spack install --fail-fast --source --no-check-signature jedi-ufs-env 2>&1 | tee log.install.gnu-11.4.0-buildcache.jedi-ufs-env
+            spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ jedi-ufs-env
+
             # the rest
             echo "${TEMPLATE} ..."
             spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.gnu-11.4.0-buildcache.${TEMPLATE}
@@ -145,6 +150,7 @@ jobs:
 
       - name: create-env
         run: |
+          # Set up spack-stack
           source ./setup.sh
           export BUILDCACHE_ENVNAME=ue-gcc-11.4.0-buildcache
           export BUILDCACHE_ENVDIR=$PWD/envs/${BUILDCACHE_ENVNAME}
@@ -153,20 +159,22 @@ jobs:
           rsync -av --exclude='install' --exclude='spack.lock' --exclude='.spack_db' ${BUILDCACHE_ENVDIR}/ ${ENVDIR}/
           spack env activate ${ENVDIR}
 
-          # Concretize
+          # Concretize and check for duplicates
           spack concretize --force 2>&1 | tee log.concretize.gnu-11.4.0
           ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.gnu-11.4.0 -i fms -i crtm -i esmf -i mapl
 
-          # Add binary cache back in and reindex it
+          # Add binary cache back in
           spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/
           echo "Packages in combined spack build caches:"
           spack buildcache list
 
-          # base-env
-          echo "base-env ..."
+          # Install from cache
           spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.gnu-11.4.0.unified-env
 
+          # Check shared libraries
           ${SPACK_STACK_DIR}/util/ldd_check.py $SPACK_ENV 2>&1 | tee log.ldd_check
+
+          # Create modules
           spack clean -a
           spack module tcl refresh -y
           spack stack setup-meta-modules
@@ -201,7 +209,7 @@ jobs:
 
           module use ${ENVDIR}/install/modulefiles/Core
           module load stack-gcc/11.4.0
-          module load stack-openmpi/5.0.3
+          module load stack-openmpi/5.0.5
           module load stack-python/3.11.7
           module available
 
diff --git a/.github/workflows/ubuntu-ci-x86_64-intel.yaml b/.github/workflows/ubuntu-ci-x86_64-intel.yaml
index eed14ce5b2..513dbb4d2f 100644
--- a/.github/workflows/ubuntu-ci-x86_64-intel.yaml
+++ b/.github/workflows/ubuntu-ci-x86_64-intel.yaml
@@ -183,6 +183,7 @@ jobs:
 
       - name: create-env
         run: |
+          # Set up spack-stack
           source ./setup.sh
           export BUILDCACHE_ENVNAME=ue-intel-2021.10.0-buildcache
           export BUILDCACHE_ENVDIR=$PWD/envs/${BUILDCACHE_ENVNAME}
@@ -191,20 +192,22 @@ jobs:
           rsync -av --exclude='install' --exclude='spack.lock' --exclude='.spack_db' ${BUILDCACHE_ENVDIR}/ ${ENVDIR}/
           spack env activate ${ENVDIR}
 
-          # Concretize
+          # Concretize and check for duplicates
           spack concretize --force 2>&1 | tee log.concretize.intel-2021.10.0
           ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.intel-2021.10.0 -i fms -i crtm -i esmf -i mapl
 
-          # Add binary cache back in and reindex it
+          # Add binary cache back in
           spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/
           echo "Packages in combined spack build caches:"
           spack buildcache list
 
-          # base-env
-          echo "base-env ..."
+          # Install from cache
           spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.intel-2021.10.0.unified-env
 
+          # Check shared libraries
           ${SPACK_STACK_DIR}/util/ldd_check.py $SPACK_ENV 2>&1 | tee log.ldd_check
+
+          # Create modules
           spack clean -a
           spack module tcl refresh -y
           spack stack setup-meta-modules
diff --git a/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml b/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml
index 0b4f7d3375..263fbe0c47 100644
--- a/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml
+++ b/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml
@@ -168,6 +168,11 @@ jobs:
           spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.oneapi-2024.2.0-buildcache.jedi-base-env
           spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env
 
+          # jedi-ufs-env
+          echo "jedi-ufs-env ..."
+          spack install --fail-fast --source --no-check-signature jedi-ufs-env 2>&1 | tee log.install.oneapi-2024.2.0-buildcache.jedi-ufs-env
+          spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ jedi-ufs-env
+
           # the rest
           echo "unified-env ..."
           spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.oneapi-2024.2.0-buildcache.unified-env
@@ -189,6 +194,7 @@ jobs:
 
       - name: create-env
         run: |
+          # Set up spack-stack
           source ./setup.sh
           export BUILDCACHE_ENVNAME=ue-oneapi-2024.2.0-buildcache
           export BUILDCACHE_ENVDIR=$PWD/envs/${BUILDCACHE_ENVNAME}
@@ -197,39 +203,27 @@ jobs:
           rsync -av --exclude='install' --exclude='spack.lock' --exclude='.spack_db' ${BUILDCACHE_ENVDIR}/ ${ENVDIR}/
           spack env activate ${ENVDIR}
 
-          # Concretize
+          # Concretize and check for duplicates
           spack concretize --force 2>&1 | tee log.concretize.oneapi-2024.2.0
           ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.oneapi-2024.2.0 -i fms -i crtm -i esmf -i mapl
 
-          # Add binary cache back in and reindex it
+          # Add binary cache back in
           spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/
           echo "Packages in combined spack build caches:"
           spack buildcache list
 
-          # base-env
-          echo "base-env ..."
+          # Install from cache
           spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.oneapi-2024.2.0.unified-env
 
+          # Check shared libraries
           ${SPACK_STACK_DIR}/util/ldd_check.py $SPACK_ENV 2>&1 | tee log.ldd_check
+
+          # Create modules
           spack clean -a
           spack module tcl refresh -y
           spack stack setup-meta-modules
           spack env deactivate
 
-          ### # Test environment chaining
-          ### echo "Test environment chaining"
-          ### spack stack create env --name chaintest --template empty --site linux.default --upstream ${ENVDIR}/install --compiler intel
-          ### # Retain config from upstream so we don't have to rebuild:
-          ### cp -r ${ENVDIR}/{site,common} $PWD/envs/chaintest/.
-          ### spack env activate ${PWD}/envs/chaintest
-          ### # Pin gmake to avoid duplicate packages (excluded gmake from spack external find above)
-          ### spack config add "packages:gmake:require:'@:4.2'"
-          ### spack add nccmp@1.9.0.1%oneapi
-          ### spack concretize | tee envs/chaintest/log.concretize
-          ### unwanted_duplicates=$(( cat envs/chaintest/log.concretize | grep -E '^ - ' | grep -Fv 'nccmp@1.9.0.1' || true ) | wc -l)
-          ### if [ ${unwanted_duplicates} -gt 0 ]; then echo "Environment chaining test failed"; exit 1; fi
-          ### spack env deactivate
-
       - name: test-env
         run: |
           source /etc/profile.d/modules.sh

From f24ccd5c154c5ef5cfb75f77c3936512ae6a0184 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Wed, 6 Nov 2024 08:53:49 -0700
Subject: [PATCH 09/16] Update PrgEnv in
 configs/sites/tier1/narwhal/compilers.yaml

---
 configs/sites/tier1/narwhal/compilers.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/configs/sites/tier1/narwhal/compilers.yaml b/configs/sites/tier1/narwhal/compilers.yaml
index 838a8a1dd5..ec9b170c06 100644
--- a/configs/sites/tier1/narwhal/compilers.yaml
+++ b/configs/sites/tier1/narwhal/compilers.yaml
@@ -9,7 +9,7 @@ compilers::
       flags: {}
       operating_system: sles15
       modules:
-      - PrgEnv-intel/8.3.3
+      - PrgEnv-intel/8.4.0
       - intel-classic/2023.2.0
       - cray-libsci/23.05.1.4
       - libfabric/1.12.1.2.2.1
@@ -55,7 +55,7 @@ compilers::
       flags: {}
       operating_system: sles15
       modules:
-      - PrgEnv-gnu/8.3.3
+      - PrgEnv-gnu/8.4.0
       - gcc/10.3.0
       - cray-libsci/23.05.1.4
       - libfabric/1.12.1.2.2.1

From cc080b80040860954965e08eb21e81d44c4c76ba Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Thu, 7 Nov 2024 09:40:29 -0700
Subject: [PATCH 10/16] Fix typo in .github/workflows/macos-ci-aarch64.yaml

---
 .github/workflows/macos-ci-aarch64.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/macos-ci-aarch64.yaml b/.github/workflows/macos-ci-aarch64.yaml
index 0146b0d51d..43a8e9d6cd 100644
--- a/.github/workflows/macos-ci-aarch64.yaml
+++ b/.github/workflows/macos-ci-aarch64.yaml
@@ -159,7 +159,7 @@ jobs:
           source ./setup.sh
           # Important!
           export SPACK_PYTHON=/usr/bin/python3
-          export BUILDCACHE_ENVNAME=ue-apple-clang-14.0.3-builddcache
+          export BUILDCACHE_ENVNAME=ue-apple-clang-14.0.3-buildcache
           export BUILDCACHE_ENVDIR=$PWD/envs/${BUILDCACHE_ENVNAME}
           export ENVNAME=ue-apple-clang-14.0.3
           export ENVDIR=$PWD/envs/${ENVNAME}

From 9bd2e34d92fd9dbde96b683194bc9738ead9666a Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Thu, 7 Nov 2024 09:40:53 -0700
Subject: [PATCH 11/16] Set correct variant for intel-oneapi-mpi in
 .github/workflows/ubuntu-ci-x86_64-intel.yaml

---
 .github/workflows/ubuntu-ci-x86_64-intel.yaml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ubuntu-ci-x86_64-intel.yaml b/.github/workflows/ubuntu-ci-x86_64-intel.yaml
index 513dbb4d2f..f67e7e99a9 100644
--- a/.github/workflows/ubuntu-ci-x86_64-intel.yaml
+++ b/.github/workflows/ubuntu-ci-x86_64-intel.yaml
@@ -93,7 +93,7 @@ jobs:
           echo "  intel-oneapi-mpi:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
           echo "    buildable: false" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
           echo "    externals:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
-          echo "    - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
+          echo "    - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0 +classic-names" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
           echo "      prefix: /opt/intel/oneapi" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
 
           # Add external Intel MKL and oneAPI runtime
@@ -162,6 +162,11 @@ jobs:
           spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.intel-2021.10.0-buildcache.jedi-base-env
           spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env
 
+          # jedi-ufs-env
+          echo "jedi-ufs-env ..."
+          spack install --fail-fast --source --no-check-signature jedi-ufs-env 2>&1 | tee log.install.intel-2021.10.0-buildcache.jedi-ufs-env
+          spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ jedi-ufs-env
+
           # the rest
           echo "unified-env ..."
           spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.intel-2021.10.0-buildcache.unified-env

From a2fab24380e122467c6c91aa8bf0acc72c51832d Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Thu, 7 Nov 2024 13:15:40 -0700
Subject: [PATCH 12/16] Remove deprecated '-a' from all 'spack buildcache
 create' in GitHub workflows

---
 .github/workflows/macos-ci-aarch64.yaml        | 8 ++++----
 .github/workflows/ubuntu-ci-x86_64-gnu.yaml    | 8 ++++----
 .github/workflows/ubuntu-ci-x86_64-intel.yaml  | 8 ++++----
 .github/workflows/ubuntu-ci-x86_64-oneapi.yaml | 8 ++++----
 4 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/.github/workflows/macos-ci-aarch64.yaml b/.github/workflows/macos-ci-aarch64.yaml
index 43a8e9d6cd..cb4c6c2197 100644
--- a/.github/workflows/macos-ci-aarch64.yaml
+++ b/.github/workflows/macos-ci-aarch64.yaml
@@ -122,22 +122,22 @@ jobs:
           # base-env
           echo "base-env ..."
           spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.apple-clang-14.0.3.base-env
-          spack buildcache create -a -u /Users/ec2-user/spack-stack/build-cache/ base-env
+          spack buildcache create -u /Users/ec2-user/spack-stack/build-cache/ base-env
 
           # jedi-base-env
           echo "jedi-base-env ..."
           spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.apple-clang-14.0.3.jedi-base-env
-          spack buildcache create -a -u /Users/ec2-user/spack-stack/build-cache/ jedi-base-env
+          spack buildcache create -u /Users/ec2-user/spack-stack/build-cache/ jedi-base-env
 
           # jedi-ufs-env
           echo "jedi-ufs-env ..."
           spack install --fail-fast --source --no-check-signature jedi-ufs-env 2>&1 | tee log.install.apple-clang-14.0.3.jedi-ufs-env
-          spack buildcache create -a -u /Users/ec2-user/spack-stack/build-cache/ jedi-ufs-env
+          spack buildcache create -u /Users/ec2-user/spack-stack/build-cache/ jedi-ufs-env
 
           # the rest
           echo "unified-env ..."
           spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.apple-clang-14.0.3.unified-env
-          spack buildcache create -a -u /Users/ec2-user/spack-stack/build-cache/
+          spack buildcache create -u /Users/ec2-user/spack-stack/build-cache/
 
           # Remove binary cache for next round of concretization
           if [ "$USE_BINARY_CACHE" = true ] ; then
diff --git a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml
index 5452a9ab5d..1a73ad69b4 100644
--- a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml
+++ b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml
@@ -115,22 +115,22 @@ jobs:
             # base-env
             echo "base-env ..."
             spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.gnu-11.4.0-buildcache.base-env
-            spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ base-env
+            spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ base-env
 
             # jedi-base-env
             echo "jedi-base-env ..."
             spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.gnu-11.4.0-buildcache.jedi-base-env
-            spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env
+            spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env
 
             # jedi-ufs-env
             echo "jedi-ufs-env ..."
             spack install --fail-fast --source --no-check-signature jedi-ufs-env 2>&1 | tee log.install.gnu-11.4.0-buildcache.jedi-ufs-env
-            spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ jedi-ufs-env
+            spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-ufs-env
 
             # the rest
             echo "${TEMPLATE} ..."
             spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.gnu-11.4.0-buildcache.${TEMPLATE}
-            spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/
+            spack buildcache create -u /home/ubuntu/spack-stack/build-cache/
 
             # Remove binary cache for next round of concretization
             if [ "$USE_BINARY_CACHE" = true ] ; then
diff --git a/.github/workflows/ubuntu-ci-x86_64-intel.yaml b/.github/workflows/ubuntu-ci-x86_64-intel.yaml
index f67e7e99a9..f391ce9014 100644
--- a/.github/workflows/ubuntu-ci-x86_64-intel.yaml
+++ b/.github/workflows/ubuntu-ci-x86_64-intel.yaml
@@ -155,22 +155,22 @@ jobs:
           # base-env
           echo "base-env ..."
           spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.intel-2021.10.0-buildcache.base-env
-          spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ base-env
+          spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ base-env
 
           # jedi-base-env
           echo "jedi-base-env ..."
           spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.intel-2021.10.0-buildcache.jedi-base-env
-          spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env
+          spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env
 
           # jedi-ufs-env
           echo "jedi-ufs-env ..."
           spack install --fail-fast --source --no-check-signature jedi-ufs-env 2>&1 | tee log.install.intel-2021.10.0-buildcache.jedi-ufs-env
-          spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ jedi-ufs-env
+          spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-ufs-env
 
           # the rest
           echo "unified-env ..."
           spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.intel-2021.10.0-buildcache.unified-env
-          spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/
+          spack buildcache create -u /home/ubuntu/spack-stack/build-cache/
 
           # Remove binary cache for next round of concretization
           if [ "$USE_BINARY_CACHE" = true ] ; then
diff --git a/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml b/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml
index 263fbe0c47..976813438c 100644
--- a/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml
+++ b/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml
@@ -161,22 +161,22 @@ jobs:
           # base-env
           echo "base-env ..."
           spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.oneapi-2024.2.0-buildcache.base-env
-          spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ base-env
+          spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ base-env
 
           # jedi-base-env
           echo "jedi-base-env ..."
           spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.oneapi-2024.2.0-buildcache.jedi-base-env
-          spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env
+          spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env
 
           # jedi-ufs-env
           echo "jedi-ufs-env ..."
           spack install --fail-fast --source --no-check-signature jedi-ufs-env 2>&1 | tee log.install.oneapi-2024.2.0-buildcache.jedi-ufs-env
-          spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ jedi-ufs-env
+          spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-ufs-env
 
           # the rest
           echo "unified-env ..."
           spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.oneapi-2024.2.0-buildcache.unified-env
-          spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/
+          spack buildcache create -u /home/ubuntu/spack-stack/build-cache/
 
           # Remove binary cache for next round of concretization
           if [ "$USE_BINARY_CACHE" = true ] ; then

From 9b86458660304ed8cd01a3231f4d560f93a4d5bd Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Thu, 7 Nov 2024 20:49:42 -0700
Subject: [PATCH 13/16] Trigger CI


From c84ae63e41a884dd328fec342394f62211035c2a Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Fri, 8 Nov 2024 08:01:17 -0700
Subject: [PATCH 14/16] Update submodule pointer for spack

---
 spack | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/spack b/spack
index 7c25b422cf..a3dd052fff 160000
--- a/spack
+++ b/spack
@@ -1 +1 @@
-Subproject commit 7c25b422cfeaa1e9d64ccc5cc447bcce7ec736ef
+Subproject commit a3dd052fffce42379cfe814c5550cae59539d895

From efd7126e6f18acb5d7f775aae1261a6ce536d9d6 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Wed, 13 Nov 2024 15:24:24 -0700
Subject: [PATCH 15/16] Revert .gitmodules and update submodule pointer for
 spack

---
 .gitmodules | 6 ++----
 spack       | 2 +-
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/.gitmodules b/.gitmodules
index 8db64f1364..a69f3e4f1d 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,9 +1,7 @@
 [submodule "spack"]
   path = spack
-  #url = https://github.com/jcsda/spack
-  #branch = spack-stack-dev
-  url = https://github.com/climbfuji/spack
-  branch = feature/update_from_spack_dev_20241031
+  url = https://github.com/jcsda/spack
+  branch = spack-stack-dev
 [submodule "doc/CMakeModules"]
   path = doc/CMakeModules
   url = https://github.com/noaa-emc/cmakemodules
diff --git a/spack b/spack
index a3dd052fff..b6ef265ec4 160000
--- a/spack
+++ b/spack
@@ -1 +1 @@
-Subproject commit a3dd052fffce42379cfe814c5550cae59539d895
+Subproject commit b6ef265ec4582ddca131534a9d314e233919ae69

From 16a3785032048a3c184e26ffcd885e8ef02979ef Mon Sep 17 00:00:00 2001
From: Alex Richert <alexander.richert@noaa.gov>
Date: Wed, 13 Nov 2024 15:58:27 -0800
Subject: [PATCH 16/16] pin gdal@3.8.5 for nco template (cxx issues)

---
 configs/templates/nco/spack.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/configs/templates/nco/spack.yaml b/configs/templates/nco/spack.yaml
index 12787de138..e1263204ed 100644
--- a/configs/templates/nco/spack.yaml
+++ b/configs/templates/nco/spack.yaml
@@ -35,7 +35,7 @@ spack:
   - g2
   - g2c
   - g2tmpl
-  - gdal
+  - gdal@3.8.5
   - geos
   - gfsio
   - gftl-shared