From 4ce47d20bde4fa01077d3ba0314c3d70e0de57b3 Mon Sep 17 00:00:00 2001 From: apcraig Date: Thu, 24 Jul 2025 13:06:07 -0500 Subject: [PATCH] Update Carpenter port - Update intel and intelimpi due to failures in debug mode associated with a compiler bug. Change -fpe0 to -fpe1. - Update the intel and intelimpi modules to the latest - Add inteloneapi --- .../scripts/machines/Macros.carpenter_intel | 4 +- .../machines/Macros.carpenter_intelimpi | 4 +- .../machines/Macros.carpenter_inteloneapi | 62 +++++++++++++++++++ .../scripts/machines/env.carpenter_intel | 8 +-- .../scripts/machines/env.carpenter_intelimpi | 8 +-- .../machines/env.carpenter_inteloneapi | 57 +++++++++++++++++ 6 files changed, 133 insertions(+), 10 deletions(-) create mode 100644 configuration/scripts/machines/Macros.carpenter_inteloneapi create mode 100644 configuration/scripts/machines/env.carpenter_inteloneapi diff --git a/configuration/scripts/machines/Macros.carpenter_intel b/configuration/scripts/machines/Macros.carpenter_intel index d53f959e4..deb6f1549 100644 --- a/configuration/scripts/machines/Macros.carpenter_intel +++ b/configuration/scripts/machines/Macros.carpenter_intel @@ -13,7 +13,9 @@ FFLAGS := -fp-model precise -convert big_endian -assume byterecl -ftz -trace FFLAGS_NOOPT:= -O0 ifeq ($(ICE_BLDDEBUG), true) - FFLAGS += -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created +# fpe0 produces an abort in hdf5 ~v1.14 which is fixed in 1.14.4.2 + FFLAGS += -O0 -g -check uninit -check bounds -check pointers -fpe1 -check noarg_temp_created +# FFLAGS += -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created # FFLAGS += -O0 -g -check all -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -init=snan,arrays else FFLAGS += -O2 diff --git a/configuration/scripts/machines/Macros.carpenter_intelimpi b/configuration/scripts/machines/Macros.carpenter_intelimpi index 0c1aa5812..b20ed04e7 100644 --- a/configuration/scripts/machines/Macros.carpenter_intelimpi +++ b/configuration/scripts/machines/Macros.carpenter_intelimpi @@ -13,7 +13,9 @@ FFLAGS := -fp-model precise -convert big_endian -assume byterecl -ftz -trace FFLAGS_NOOPT:= -O0 ifeq ($(ICE_BLDDEBUG), true) - FFLAGS += -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created +# fpe0 produces an abort in hdf5 ~v1.14 which is fixed in 1.14.4.2 + FFLAGS += -O0 -g -check uninit -check bounds -check pointers -fpe1 -check noarg_temp_created +# FFLAGS += -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created # FFLAGS += -O0 -g -check all -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -init=snan,arrays else FFLAGS += -O2 diff --git a/configuration/scripts/machines/Macros.carpenter_inteloneapi b/configuration/scripts/machines/Macros.carpenter_inteloneapi new file mode 100644 index 000000000..685e99c36 --- /dev/null +++ b/configuration/scripts/machines/Macros.carpenter_inteloneapi @@ -0,0 +1,62 @@ +#============================================================================== +# Macros file for ERDC carpenter, inteloneapi compiler +#============================================================================== + +CPP := fpp +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} +CFLAGS := -c -O2 -fp-model precise -fcommon + +FIXEDFLAGS := -132 +FREEFLAGS := -FR +FFLAGS := -fp-model precise -convert big_endian -assume byterecl -ftz -traceback +# -mcmodel medium -shared-intel +FFLAGS_NOOPT:= -O0 + +ifeq ($(ICE_BLDDEBUG), true) +# -check uninit is needed on the ld step but it still throws errors in 2023.* and 2024.0.*, likely compiler bug + FFLAGS += -O0 -g -check bounds -check pointers -fpe1 -check noarg_temp_created -link_mpi=dbg +# FFLAGS += -O0 -g -check bounds -check pointers -fpe0 -check noarg_temp_created -link_mpi=dbg +# FFLAGS += -O0 -g -check_uninit -check bounds -check pointers -fpe0 -check noarg_temp_created -link_mpi=dbg +# LDFLAGS += -check uninit +else + FFLAGS += -O2 +endif + +SCC := icx +SFC := ifx +#SCC := cc +#SFC := ftn +MPICC := cc +MPIFC := ftn + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif +LD:= $(FC) + +# defined by module +#NETCDF_PATH := $(NETCDF) +#PNETCDF_PATH := $(PNETCDF) +#PNETCDF_PATH := /glade/apps/opt/pnetcdf/1.3.0/intel/default +#LAPACK_LIBDIR := /glade/apps/opt/lapack/3.4.2/intel/12.1.5/lib + +#PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs + +INCLDIR := $(INCLDIR) +INCLDIR += -I$(NETCDF_PATH)/include + +LIB_NETCDF := $(NETCDF_PATH)/lib +#LIB_PNETCDF := $(PNETCDF_PATH)/lib +#LIB_MPI := $(IMPILIBDIR) +SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -qopenmp + CFLAGS += -qopenmp + FFLAGS += -qopenmp +endif + diff --git a/configuration/scripts/machines/env.carpenter_intel b/configuration/scripts/machines/env.carpenter_intel index c97a7d25a..141a5a82a 100644 --- a/configuration/scripts/machines/env.carpenter_intel +++ b/configuration/scripts/machines/env.carpenter_intel @@ -12,7 +12,7 @@ module unload PrgEnv-cray module unload PrgEnv-gnu module unload PrgEnv-intel module unload PrgEnv-pgi -module load PrgEnv-intel/8.4.0 +module load PrgEnv-intel/8.5.0 module unload intel module load intel/2023.0.0 @@ -20,7 +20,7 @@ module load intel/2023.0.0 module unload cray-mpich module unload mpi module unload openmpi -module load cray-mpich/8.1.26 +module load cray-mpich/8.1.30 #module load mpi/2021.11 #module load openmpi/4.1.6 @@ -30,7 +30,7 @@ module unload cray-netcdf-hdf5parallel module unload cray-parallel-netcdf module unload netcdf module load cray-netcdf/4.9.0.3 -module load cray-hdf5/1.12.2.3 +module load cray-hdf5/1.14.3.1 setenv NETCDF_PATH ${NETCDF_DIR} limit coredumpsize unlimited @@ -44,7 +44,7 @@ endif setenv ICE_MACHINE_MACHNAME carpenter setenv ICE_MACHINE_MACHINFO "Cray EX4000 AMD 9654 Genoa 2.1GHz, Slingshot Interconnect" setenv ICE_MACHINE_ENVNAME intel -setenv ICE_MACHINE_ENVINFO "ifort 2021.8.0 20221119, cray-mpich/8.1.26, netcdf/4.9.0.3" +setenv ICE_MACHINE_ENVINFO "ifort 2021.8.0 20221119, cray-mpich/8.1.30, netcdf/4.9.0.3" setenv ICE_MACHINE_MAKE gmake setenv ICE_MACHINE_WKDIR $WORKDIR/CICE_RUNS setenv ICE_MACHINE_INPUTDATA /p/app/unsupported/RASM/cice_consortium diff --git a/configuration/scripts/machines/env.carpenter_intelimpi b/configuration/scripts/machines/env.carpenter_intelimpi index 25385b09a..ef43363cd 100644 --- a/configuration/scripts/machines/env.carpenter_intelimpi +++ b/configuration/scripts/machines/env.carpenter_intelimpi @@ -12,7 +12,7 @@ module unload PrgEnv-cray module unload PrgEnv-gnu module unload PrgEnv-intel module unload PrgEnv-pgi -module load PrgEnv-intel/8.4.0 +module load PrgEnv-intel/8.5.0 module unload intel module load intel/2023.0.0 @@ -21,7 +21,7 @@ module unload cray-mpich module unload mpi module unload openmpi #module load cray-mpich/8.1.26 -module load mpi/2021.11 +module load mpi/2021.13 #module load openmpi/4.1.6 module unload cray-hdf5 @@ -30,7 +30,7 @@ module unload cray-netcdf-hdf5parallel module unload cray-parallel-netcdf module unload netcdf module load cray-netcdf/4.9.0.3 -module load cray-hdf5/1.12.2.3 +module load cray-hdf5/1.14.3.1 setenv NETCDF_PATH ${NETCDF_DIR} limit coredumpsize unlimited @@ -44,7 +44,7 @@ endif setenv ICE_MACHINE_MACHNAME carpenter setenv ICE_MACHINE_MACHINFO "Cray EX4000 AMD 9654 Genoa 2.1GHz, Slingshot Interconnect" setenv ICE_MACHINE_ENVNAME intelimpi -setenv ICE_MACHINE_ENVINFO "ifort 2021.8.0 20221119, intel mpi 2021.11, netcdf/4.9.0.3" +setenv ICE_MACHINE_ENVINFO "ifort 2021.8.0 20221119, intel mpi 2021.13, netcdf/4.9.0.3" setenv ICE_MACHINE_MAKE gmake setenv ICE_MACHINE_WKDIR $WORKDIR/CICE_RUNS setenv ICE_MACHINE_INPUTDATA /p/app/unsupported/RASM/cice_consortium diff --git a/configuration/scripts/machines/env.carpenter_inteloneapi b/configuration/scripts/machines/env.carpenter_inteloneapi new file mode 100644 index 000000000..2480918c1 --- /dev/null +++ b/configuration/scripts/machines/env.carpenter_inteloneapi @@ -0,0 +1,57 @@ +#!/bin/csh -f +set inp = "undefined" +if ($#argv == 1) then + set inp = $1 +endif + +if ("$inp" != "-nomodules") then + +source ${MODULESHOME}/init/csh + +module unload PrgEnv-cray +module unload PrgEnv-gnu +module unload PrgEnv-intel +module unload PrgEnv-pgi +module load PrgEnv-intel/8.5.0 + +module unload intel +module load intel-oneapi/2024.2 + +module unload cray-mpich +module unload mpi +module unload openmpi +module load cray-mpich/8.1.30 +#module load mpi/2021.11 +#module load openmpi/4.1.6 + +module unload cray-hdf5 +module unload cray-hdf5-parallel +module unload cray-netcdf-hdf5parallel +module unload cray-parallel-netcdf +module unload netcdf +module load cray-netcdf/4.9.0.13 +module load cray-hdf5/1.14.3.1 + +setenv NETCDF_PATH ${NETCDF_DIR} +limit coredumpsize unlimited +limit stacksize unlimited +setenv OMP_STACKSIZE 128M +setenv OMP_WAIT_POLICY PASSIVE +setenv FI_CXI_RX_MATCH_MODE hybrid + +endif + +setenv ICE_MACHINE_MACHNAME carpenter +setenv ICE_MACHINE_MACHINFO "Cray EX4000 AMD 9654 Genoa 2.1GHz, Slingshot Interconnect" +setenv ICE_MACHINE_ENVNAME inteloneapi +setenv ICE_MACHINE_ENVINFO "Intel oneAPI DPC++/C++/icx/ifx 2024.2.0 20240602, cray-mpich/8.1.30, netcdf/4.9.0.13" +setenv ICE_MACHINE_MAKE gmake +setenv ICE_MACHINE_WKDIR $WORKDIR/CICE_RUNS +setenv ICE_MACHINE_INPUTDATA /p/app/unsupported/RASM/cice_consortium +setenv ICE_MACHINE_BASELINE $WORKDIR/CICE_BASELINE +setenv ICE_MACHINE_SUBMIT "qsub " +setenv ICE_MACHINE_ACCT P00000000 +setenv ICE_MACHINE_QUEUE "debug" +setenv ICE_MACHINE_TPNODE 192 # tasks per node +setenv ICE_MACHINE_BLDTHRDS 12 +setenv ICE_MACHINE_QSTAT "qstat "