From 2cf6ad870c477b265cea3936e655fd7401454fde Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Mon, 25 Aug 2025 13:05:07 -0600 Subject: [PATCH 1/3] New Derecho environment ncarenv/24.12 for release/1.9.0 --- configs/sites/tier1/derecho/compilers.yaml | 35 ++++++++++--------- configs/sites/tier1/derecho/packages.yaml | 25 ++++++------- configs/sites/tier1/derecho/packages_gcc.yaml | 12 +++---- .../sites/tier1/derecho/packages_oneapi.yaml | 26 ++++++++++---- configs/templates/unified-dev/spack.yaml | 1 + 5 files changed, 56 insertions(+), 43 deletions(-) diff --git a/configs/sites/tier1/derecho/compilers.yaml b/configs/sites/tier1/derecho/compilers.yaml index 5a7891255..47736702b 100644 --- a/configs/sites/tier1/derecho/compilers.yaml +++ b/configs/sites/tier1/derecho/compilers.yaml @@ -2,21 +2,21 @@ compilers:: - compiler: spec: oneapi@2024.2.1 paths: - cc: /glade/u/apps/common/23.08/spack/opt/spack/intel-oneapi-compilers/2024.2.1/compiler/2024.2/bin/icx - cxx: /glade/u/apps/common/23.08/spack/opt/spack/intel-oneapi-compilers/2024.2.1/compiler/2024.2/bin/icpx - f77: /glade/u/apps/common/23.08/spack/opt/spack/intel-oneapi-compilers/2024.2.1/compiler/2024.2/bin/ifort - fc: /glade/u/apps/common/23.08/spack/opt/spack/intel-oneapi-compilers/2024.2.1/compiler/2024.2/bin/ifort + cc: /glade/u/apps/casper/24.12/spack/opt/spack/intel-oneapi-compilers/2024.2.1/gcc/12.4.0/iq3b/compiler/2024.2/bin/icx + cxx: /glade/u/apps/casper/24.12/spack/opt/spack/intel-oneapi-compilers/2024.2.1/gcc/12.4.0/iq3b/compiler/2024.2/bin/icpx + f77: /glade/u/apps/casper/24.12/spack/opt/spack/intel-oneapi-compilers/2024.2.1/gcc/12.4.0/iq3b/compiler/2024.2/bin/ifort + fc: /glade/u/apps/casper/24.12/spack/opt/spack/intel-oneapi-compilers/2024.2.1/gcc/12.4.0/iq3b/compiler/2024.2/bin/ifort flags: {} operating_system: sles15 target: x86_64 modules: - - ncarenv/23.09 - - intel-oneapi/2024.2.1 + - intel/2024.2.1 + - ncarenv/24.12 environment: prepend_path: - PATH: '/opt/cray/pe/gcc/12.2.0/snos/bin' - CPATH: '/opt/cray/pe/gcc/12.2.0/snos/include' - LD_LIBRARY_PATH: '/opt/cray/pe/gcc/12.2.0/snos/lib:/opt/cray/pe/gcc/12.2.0/snos/lib64' + PATH: '/glade/u/apps/casper/24.12/spack/opt/spack/gcc/12.4.0/5b5c/bin' + CPATH: '/glade/u/apps/casper/24.12/spack/opt/spack/gcc/12.4.0/5b5c/include' + LD_LIBRARY_PATH: '/glade/u/apps/casper/24.12/spack/opt/spack/gcc/12.4.0/5b5c/lib:/glade/u/apps/casper/24.12/spack/opt/spack/gcc/12.4.0/5b5c/lib64' set: # https://github.com/JCSDA/spack-stack/issues/957 FI_CXI_RX_MATCH_MODE: 'hybrid' @@ -24,23 +24,24 @@ compilers:: I_MPI_EXTRA_FILESYSTEM: 'ON' extra_rpaths: [] - compiler: - spec: gcc@12.2.0 + spec: gcc@12.4.0 paths: - cc: /opt/cray/pe/gcc/12.2.0/bin/gcc - cxx: /opt/cray/pe/gcc/12.2.0/bin/g++ - f77: /opt/cray/pe/gcc/12.2.0/bin/gfortran - fc: /opt/cray/pe/gcc/12.2.0/bin/gfortran + cc: /glade/u/apps/casper/24.12/spack/opt/spack/gcc/12.4.0/5b5c/bin/gcc + cxx: /glade/u/apps/casper/24.12/spack/opt/spack/gcc/12.4.0/5b5c/bin/g++ + f77: /glade/u/apps/casper/24.12/spack/opt/spack/gcc/12.4.0/5b5c/bin/gfortran + fc: /glade/u/apps/casper/24.12/spack/opt/spack/gcc/12.4.0/5b5c/bin/gfortran flags: {} operating_system: sles15 target: x86_64 modules: - - ncarenv/23.09 - - gcc/12.2.0 + - ncarenv/24.12 + - gcc/12.4.0 environment: prepend_path: # For compiling scotch (which requires Intel MPI) with gcc in the oneapi@2024.2.1 unified environment - LD_LIBRARY_PATH: '/glade/u/apps/common/23.08/spack/opt/spack/intel-oneapi-compilers/2024.2.1/compiler/2024.2/lib' + LD_LIBRARY_PATH: '/glade/u/apps/casper/24.12/spack/opt/spack/intel-oneapi-compilers/2024.2.1/gcc/12.4.0/iq3b/compiler/2024.2/lib' set: # https://github.com/JCSDA/spack-stack/issues/957 FI_CXI_RX_MATCH_MODE: 'hybrid' extra_rpaths: [] + diff --git a/configs/sites/tier1/derecho/packages.yaml b/configs/sites/tier1/derecho/packages.yaml index c1f625d06..3e452397a 100644 --- a/configs/sites/tier1/derecho/packages.yaml +++ b/configs/sites/tier1/derecho/packages.yaml @@ -34,10 +34,6 @@ packages: externals: - spec: coreutils@8.32 prefix: /usr - curl: - externals: - - spec: curl@8.1.2+gssapi+ldap+nghttp2 - prefix: /glade/u/apps/derecho/23.09/spack/opt/spack/curl/8.1.2/gcc/7.5.0/uq6y cvs: externals: - spec: cvs@1.12.13 @@ -56,14 +52,12 @@ packages: prefix: /usr git: externals: - - spec: git@2.41.0+tcltk - prefix: /glade/u/apps/derecho/23.09/spack/opt/spack/git/2.41.0/gcc/7.5.0/jgni - - spec: git@2.35.3+tcltk - prefix: /usr + - spec: git@2.47.0 + prefix: /glade/u/apps/casper/24.12/spack/opt/spack/git/2.47.0/gcc/12.4.0/cn62 git-lfs: externals: - - spec: git-lfs@3.3.0 - prefix: /glade/u/apps/derecho/23.09/spack/opt/spack/git-lfs/3.3.0/gcc/7.5.0/u3kh + - spec: git-lfs@3.5.1 + prefix: /glade/u/apps/casper/24.12/spack/opt/spack/git-lfs/3.5.1/gcc/12.4.0/aivb gmake: externals: - spec: gmake@4.2.1 @@ -85,7 +79,7 @@ packages: - spec: m4@1.4.18 prefix: /usr mysql: - buildable: False + buildable: false externals: - spec: mysql@8.0.33 prefix: /glade/work/epicufsrt/contrib/spack-stack/derecho/mysql-8.0.33 @@ -93,9 +87,9 @@ packages: externals: - spec: openssh@8.4p1 prefix: /usr - # spack concretizer gets confused when omitting %gcc from the external spec (a spack bug?) +# spack concretizer gets confused when omitting %gcc from the external spec (a spack bug?) openssl: - buildable: False + buildable: false externals: - spec: openssl@1.1.1l%gcc prefix: /usr @@ -104,9 +98,12 @@ packages: - spec: pkg-config@0.29.2 prefix: /usr qt: - buildable: False + buildable: false externals: - spec: qt@5.14.2 + # Newer versions don't work, some headers are missing + #prefix: /glade/u/apps/casper/24.12/spack/opt/spack/qt/5.14.2/gcc/12.4.0/6suc + #prefix: /glade/u/apps/casper/24.12/spack/opt/spack/qt/5.14.2/gcc/12.4.0/axtw prefix: /glade/u/apps/derecho/23.09/spack/opt/spack/qt/5.14.2/gcc/7.5.0/eqcj sed: externals: diff --git a/configs/sites/tier1/derecho/packages_gcc.yaml b/configs/sites/tier1/derecho/packages_gcc.yaml index 332c6117b..bcfda60b9 100644 --- a/configs/sites/tier1/derecho/packages_gcc.yaml +++ b/configs/sites/tier1/derecho/packages_gcc.yaml @@ -1,15 +1,15 @@ packages: all: - compiler:: [gcc@12.2.0] + compiler:: [gcc@12.4.0] providers: - mpi:: [cray-mpich@8.1.27] + mpi:: [cray-mpich@8.1.29] mpi: buildable: False cray-mpich: externals: - - spec: cray-mpich@8.1.27%gcc@12.2.0 +wrappers + - spec: cray-mpich@8.1.29%gcc@12.4.0 +wrappers modules: - - craype/2.7.20 - - cray-mpich/8.1.27 + - craype/2.7.31 + - ncarenv/24.12 + - cray-mpich/8.1.29 - libfabric/1.15.2.0 - - cray-pals/1.2.11 diff --git a/configs/sites/tier1/derecho/packages_oneapi.yaml b/configs/sites/tier1/derecho/packages_oneapi.yaml index bfb807e6e..72c4bc589 100644 --- a/configs/sites/tier1/derecho/packages_oneapi.yaml +++ b/configs/sites/tier1/derecho/packages_oneapi.yaml @@ -1,23 +1,37 @@ packages: all: - compiler:: [oneapi@2024.2.1, gcc@12.2.0] + compiler:: [oneapi@2024.2.1, gcc@12.4.0] providers: mpi:: [cray-mpich@8.1.29] mpi: - buildable: False + buildable: true cray-mpich: externals: - spec: cray-mpich@8.1.29%oneapi@2024.2.1 +wrappers modules: - - craype/2.7.20 - - cray-mpich/8.1.29 + - intel/2024.2.1 + - ncarenv/24.12 + - craype/2.7.31 - libfabric/1.15.2.0 - - cray-pals/1.2.11 + environment: + prepend_path: + PATH: '/opt/cray/pe/pals/1.2.12/bin' + PKG_CONFIG_PATH: '/opt/cray/pe/pals/1.2.12/lib/pkgconfig' + MANPATH: '/opt/cray/pe/pals/1.2.12/man' + PE_PALS_PKGCONFIG_LIBS: 'libpals' + PE_PKGCONFIG_PRODUCTS: 'PE_PALS' + set: + # https://github.com/JCSDA/spack-stack/issues/957 + FI_CXI_RX_MATCH_MODE: 'hybrid' + # https://github.com/JCSDA/spack-stack/issues/1012 + I_MPI_EXTRA_FILESYSTEM: 'ON' + I_MPI_OFI_LIBRARY: '/opt/cray/libfabric/1.15.2.0/lib64/libfabric.so.1' + extra_rpaths: [] intel-oneapi-runtime: buildable: False externals: - spec: intel-oneapi-runtime@2024.2.1%oneapi@2024.2.1 - prefix: /glade/u/apps/common/23.08/spack/opt/spack/intel-oneapi-compilers/2024.2.1 + prefix: /glade/u/apps/casper/24.12/spack/opt/spack/intel-oneapi-compilers/2024.2.1/gcc/12.4.0/iq3b/compiler/2024.2 intel-oneapi-mkl: # No intel-oneapi-mkl installed on Derecho, install via spack. # Prefer the version that Intel releases with the above compiler diff --git a/configs/templates/unified-dev/spack.yaml b/configs/templates/unified-dev/spack.yaml index 2badbba65..c7f240f52 100644 --- a/configs/templates/unified-dev/spack.yaml +++ b/configs/templates/unified-dev/spack.yaml @@ -42,6 +42,7 @@ spack: # MADIS for WCOSS2 decoders. - madis@4.5 + - sp@2.5.0 specs: - matrix: From 80213a485e802a9cb7f0bc3550ef2cd8267b9683 Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Wed, 27 Aug 2025 09:14:58 -0600 Subject: [PATCH 2/3] updating Derecho packages_gcc.yaml --- configs/sites/tier1/derecho/packages_gcc.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/configs/sites/tier1/derecho/packages_gcc.yaml b/configs/sites/tier1/derecho/packages_gcc.yaml index bcfda60b9..eb0e0754d 100644 --- a/configs/sites/tier1/derecho/packages_gcc.yaml +++ b/configs/sites/tier1/derecho/packages_gcc.yaml @@ -13,3 +13,18 @@ packages: - ncarenv/24.12 - cray-mpich/8.1.29 - libfabric/1.15.2.0 + environment: + prepend_path: + PATH: '/opt/cray/pe/pals/1.2.12/bin' + PKG_CONFIG_PATH: '/opt/cray/pe/pals/1.2.12/lib/pkgconfig' + MANPATH: '/opt/cray/pe/pals/1.2.12/man' + PE_PALS_PKGCONFIG_LIBS: 'libpals' + PE_PKGCONFIG_PRODUCTS: 'PE_PALS' + set: + # https://github.com/JCSDA/spack-stack/issues/957 + FI_CXI_RX_MATCH_MODE: 'hybrid' + # https://github.com/JCSDA/spack-stack/issues/1012 + I_MPI_EXTRA_FILESYSTEM: 'ON' + I_MPI_OFI_LIBRARY: '/opt/cray/libfabric/1.15.2.0/lib64/libfabric.so.1' + extra_rpaths: [] + From 07e4201e4d246778f3f4281f5b6ebf7537084b6a Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Fri, 29 Aug 2025 07:23:42 -0600 Subject: [PATCH 3/3] Derecho environment with ncarenv/24.12: use external cray-mpich --- configs/sites/tier1/derecho/compilers.yaml | 17 +---------------- configs/sites/tier1/derecho/modules.yaml | 1 - configs/sites/tier1/derecho/packages_gcc.yaml | 18 ++---------------- .../sites/tier1/derecho/packages_oneapi.yaml | 19 +++---------------- 4 files changed, 6 insertions(+), 49 deletions(-) diff --git a/configs/sites/tier1/derecho/compilers.yaml b/configs/sites/tier1/derecho/compilers.yaml index 47736702b..2391018cd 100644 --- a/configs/sites/tier1/derecho/compilers.yaml +++ b/configs/sites/tier1/derecho/compilers.yaml @@ -10,19 +10,13 @@ compilers:: operating_system: sles15 target: x86_64 modules: - - intel/2024.2.1 - ncarenv/24.12 + - intel/2024.2.1 environment: prepend_path: PATH: '/glade/u/apps/casper/24.12/spack/opt/spack/gcc/12.4.0/5b5c/bin' CPATH: '/glade/u/apps/casper/24.12/spack/opt/spack/gcc/12.4.0/5b5c/include' LD_LIBRARY_PATH: '/glade/u/apps/casper/24.12/spack/opt/spack/gcc/12.4.0/5b5c/lib:/glade/u/apps/casper/24.12/spack/opt/spack/gcc/12.4.0/5b5c/lib64' - set: - # https://github.com/JCSDA/spack-stack/issues/957 - FI_CXI_RX_MATCH_MODE: 'hybrid' - # https://github.com/JCSDA/spack-stack/issues/1012 - I_MPI_EXTRA_FILESYSTEM: 'ON' - extra_rpaths: [] - compiler: spec: gcc@12.4.0 paths: @@ -36,12 +30,3 @@ compilers:: modules: - ncarenv/24.12 - gcc/12.4.0 - environment: - prepend_path: - # For compiling scotch (which requires Intel MPI) with gcc in the oneapi@2024.2.1 unified environment - LD_LIBRARY_PATH: '/glade/u/apps/casper/24.12/spack/opt/spack/intel-oneapi-compilers/2024.2.1/gcc/12.4.0/iq3b/compiler/2024.2/lib' - set: - # https://github.com/JCSDA/spack-stack/issues/957 - FI_CXI_RX_MATCH_MODE: 'hybrid' - extra_rpaths: [] - diff --git a/configs/sites/tier1/derecho/modules.yaml b/configs/sites/tier1/derecho/modules.yaml index 960198ae7..65801706f 100644 --- a/configs/sites/tier1/derecho/modules.yaml +++ b/configs/sites/tier1/derecho/modules.yaml @@ -4,5 +4,4 @@ modules: - lmod lmod: include: - - cray-mpich - python diff --git a/configs/sites/tier1/derecho/packages_gcc.yaml b/configs/sites/tier1/derecho/packages_gcc.yaml index eb0e0754d..fbc9a9baa 100644 --- a/configs/sites/tier1/derecho/packages_gcc.yaml +++ b/configs/sites/tier1/derecho/packages_gcc.yaml @@ -11,20 +11,6 @@ packages: modules: - craype/2.7.31 - ncarenv/24.12 - - cray-mpich/8.1.29 + - gcc/12.4.0 - libfabric/1.15.2.0 - environment: - prepend_path: - PATH: '/opt/cray/pe/pals/1.2.12/bin' - PKG_CONFIG_PATH: '/opt/cray/pe/pals/1.2.12/lib/pkgconfig' - MANPATH: '/opt/cray/pe/pals/1.2.12/man' - PE_PALS_PKGCONFIG_LIBS: 'libpals' - PE_PKGCONFIG_PRODUCTS: 'PE_PALS' - set: - # https://github.com/JCSDA/spack-stack/issues/957 - FI_CXI_RX_MATCH_MODE: 'hybrid' - # https://github.com/JCSDA/spack-stack/issues/1012 - I_MPI_EXTRA_FILESYSTEM: 'ON' - I_MPI_OFI_LIBRARY: '/opt/cray/libfabric/1.15.2.0/lib64/libfabric.so.1' - extra_rpaths: [] - + - cray-mpich/8.1.29 diff --git a/configs/sites/tier1/derecho/packages_oneapi.yaml b/configs/sites/tier1/derecho/packages_oneapi.yaml index 72c4bc589..1edce74c5 100644 --- a/configs/sites/tier1/derecho/packages_oneapi.yaml +++ b/configs/sites/tier1/derecho/packages_oneapi.yaml @@ -4,29 +4,16 @@ packages: providers: mpi:: [cray-mpich@8.1.29] mpi: - buildable: true + buildable: false cray-mpich: externals: - spec: cray-mpich@8.1.29%oneapi@2024.2.1 +wrappers modules: - - intel/2024.2.1 - ncarenv/24.12 - craype/2.7.31 + - intel/2024.2.1 - libfabric/1.15.2.0 - environment: - prepend_path: - PATH: '/opt/cray/pe/pals/1.2.12/bin' - PKG_CONFIG_PATH: '/opt/cray/pe/pals/1.2.12/lib/pkgconfig' - MANPATH: '/opt/cray/pe/pals/1.2.12/man' - PE_PALS_PKGCONFIG_LIBS: 'libpals' - PE_PKGCONFIG_PRODUCTS: 'PE_PALS' - set: - # https://github.com/JCSDA/spack-stack/issues/957 - FI_CXI_RX_MATCH_MODE: 'hybrid' - # https://github.com/JCSDA/spack-stack/issues/1012 - I_MPI_EXTRA_FILESYSTEM: 'ON' - I_MPI_OFI_LIBRARY: '/opt/cray/libfabric/1.15.2.0/lib64/libfabric.so.1' - extra_rpaths: [] + - cray-mpich/8.1.29 intel-oneapi-runtime: buildable: False externals: