From d669c28fb4ae962843c52e12f4f45b405684faf5 Mon Sep 17 00:00:00 2001 From: Robert Grumbine Date: Thu, 6 Mar 2025 07:30:24 -0500 Subject: [PATCH 1/3] gaea cray building --- configuration/scripts/cice.batch.csh | 2 +- .../scripts/machines/Macros.gaea_cray | 57 +++++++++++++++++++ configuration/scripts/machines/env.gaea_cray | 44 ++++++++++++++ configuration/scripts/machines/env.gaea_intel | 27 +++++++-- 4 files changed, 123 insertions(+), 7 deletions(-) create mode 100644 configuration/scripts/machines/Macros.gaea_cray create mode 100644 configuration/scripts/machines/env.gaea_cray diff --git a/configuration/scripts/cice.batch.csh b/configuration/scripts/cice.batch.csh index bb2edc17e..2354de9cb 100755 --- a/configuration/scripts/cice.batch.csh +++ b/configuration/scripts/cice.batch.csh @@ -355,7 +355,7 @@ cat >> ${jobfile} << EOFB #SBATCH --partition=batch #SBATCH --qos=${queue} #SBATCH --account=nggps_emc -#SBATCH --clusters=c3 +#SBATCH --clusters=c5 #SBATCH --time=${batchtime} #SBATCH --nodes=${nnodes} #SBATCH --ntasks-per-node=${taskpernodelimit} diff --git a/configuration/scripts/machines/Macros.gaea_cray b/configuration/scripts/machines/Macros.gaea_cray new file mode 100644 index 000000000..851134514 --- /dev/null +++ b/configuration/scripts/machines/Macros.gaea_cray @@ -0,0 +1,57 @@ +#============================================================================== +# Makefile macros for NOAA hera, intel compiler +#============================================================================== + +CPP := fpp +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} +CFLAGS := -c -O2 + +FIXEDFLAGS := -132 +FREEFLAGS := +FFLAGS := -hbyteswapio +FFLAGS_NOOPT:= -O0 +LDLAGS := -hbyteswapio + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -hfp0 -g -Rbcdps -Ktrap=fp +else + FFLAGS += -O2 -hfp0 +endif + +SCC := cc +SFC := ftn +MPICC := cc +MPIFC := ftn + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif +LD:= $(FC) + +NETCDF_PATH := $(NETCDF) + +#PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs + +#PNETCDF_PATH := $(PNETCDF) +#PNETCDF_PATH := /glade/u/apps/ch/opt/pio/2.2/mpt/2.15f/intel/17.0.1/lib + +INC_NETCDF := $(NETCDF_PATH)/include +LIB_NETCDF := $(NETCDF_PATH)/lib + +#LIB_PNETCDF := $(PNETCDF_PATH)/lib +#LIB_MPI := $(IMPILIBDIR) + +INCLDIR := $(INCLDIR) -I$(INC_NETCDF) +#SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff -L$(LIB_PNETCDF) -lpnetcdf -lgptl +SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -fopenmp + CFLAGS += -fopenmp + FFLAGS += -fopenmp +endif + diff --git a/configuration/scripts/machines/env.gaea_cray b/configuration/scripts/machines/env.gaea_cray new file mode 100644 index 000000000..db62615ee --- /dev/null +++ b/configuration/scripts/machines/env.gaea_cray @@ -0,0 +1,44 @@ +#!/bin/csh -f + +set inp = "undefined" +if ($#argv == 1) then + set inp = $1 +endif + +if ("$inp" != "-nomodules") then + +#source /lustre/f2/pdata/esrl/gsd/contrib/lua-5.1.4.9/init/init_lmod.csh +source $MODULESHOME/init/csh +module purge +module load PrgEnv-cray +module load cce/18.0.0 +module load cray-libsci/24.07.0 +module load cray-hdf5/1.14.3.1 +module load cray-netcdf/4.9.0.13 +setenv NETCDF $NETCDF_DIR +module list + +# May be needed for OpenMP memory +#setenv OMP_STACKSIZE 64M + +endif + +# May be needed for OpenMP memory +#setenv OMP_STACKSIZE 64M + +endif + +setenv ICE_MACHINE_MACHNAME gaea +setenv ICE_MACHINE_MACHINFO "Cray XC40 Intel Haswell/Broadwell 2.3GHz, Gemini Interconnect" +setenv ICE_MACHINE_ENVNAME intel +setenv ICE_MACHINE_ENVINFO "ifort 18.0.5 20180823, cray-mpich, cray-netcdf" +setenv ICE_MACHINE_MAKE gmake +setenv ICE_MACHINE_WKDIR $HOME/scratch/CICE_RUNS +setenv ICE_MACHINE_INPUTDATA /ncrc/home1/Robert.Grumbine/rgdev/CICE_INPUTDATA +setenv ICE_MACHINE_BASELINE $HOME/scratch/CICE_BASELINE +setenv ICE_MACHINE_SUBMIT "sbatch" +setenv ICE_MACHINE_TPNODE 40 +setenv ICE_MACHINE_ACCT P0000000 +setenv ICE_MACHINE_QUEUE "normal" +setenv ICE_MACHINE_BLDTHRDS 1 +setenv ICE_MACHINE_QSTAT "squeue --jobs=" diff --git a/configuration/scripts/machines/env.gaea_intel b/configuration/scripts/machines/env.gaea_intel index 3601bfcfb..cd3301ea7 100644 --- a/configuration/scripts/machines/env.gaea_intel +++ b/configuration/scripts/machines/env.gaea_intel @@ -7,18 +7,33 @@ endif if ("$inp" != "-nomodules") then -source /lustre/f2/pdata/esrl/gsd/contrib/lua-5.1.4.9/init/init_lmod.csh -#module list +#source /lustre/f2/pdata/esrl/gsd/contrib/lua-5.1.4.9/init/init_lmod.csh module purge -module load intel -module load cray-mpich -module load cray-netcdf -module load PrgEnv-intel/6.0.5 +module load PrgEnv-cray +#module load cray-mpich +#module load intel-classic +module load cce/18.0.0 +module load cray-libsci/24.07.0 +module load cray-hdf5/1.14.3.1 +module load cray-netcdf/4.9.0.13 +export NETCDF=$NETCDF_DIR +#module load PrgEnv-cray/8.5.0 +#module load cray-mpich +#module load intel/2023.2.0 +#module load cray-hdf5/1.14.3.1 +#module load cray-netcdf/4.9.0.13 +##module load cce/18.0.0 +##module load cray-parallel-netcdf/1.12.3.11 module list # May be needed for OpenMP memory #setenv OMP_STACKSIZE 64M +endif + +# May be needed for OpenMP memory +#setenv OMP_STACKSIZE 64M + endif setenv ICE_MACHINE_MACHNAME gaea From d80896f0a5e2ac682cb84e175f5c3bcd40e672b6 Mon Sep 17 00:00:00 2001 From: Robert Grumbine Date: Mon, 24 Mar 2025 09:42:32 -0400 Subject: [PATCH 2/3] compile and execute quick_suite on gaea --- configuration/scripts/machines/env.gaea_intel | 30 +++++++------------ 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/configuration/scripts/machines/env.gaea_intel b/configuration/scripts/machines/env.gaea_intel index cd3301ea7..69bddb428 100644 --- a/configuration/scripts/machines/env.gaea_intel +++ b/configuration/scripts/machines/env.gaea_intel @@ -8,34 +8,26 @@ endif if ("$inp" != "-nomodules") then #source /lustre/f2/pdata/esrl/gsd/contrib/lua-5.1.4.9/init/init_lmod.csh -module purge -module load PrgEnv-cray -#module load cray-mpich -#module load intel-classic -module load cce/18.0.0 -module load cray-libsci/24.07.0 +source $MODULESHOME/init/csh +#module list +module load PrgEnv-intel +module load intel +#module load intel/2023.2.0 +#module load cce/18.0.0 module load cray-hdf5/1.14.3.1 module load cray-netcdf/4.9.0.13 -export NETCDF=$NETCDF_DIR -#module load PrgEnv-cray/8.5.0 -#module load cray-mpich -#module load intel/2023.2.0 -#module load cray-hdf5/1.14.3.1 -#module load cray-netcdf/4.9.0.13 -##module load cce/18.0.0 -##module load cray-parallel-netcdf/1.12.3.11 +setenv NETCDF $NETCDF_DIR +echo zzz final module list module list +#module avail intel # May be needed for OpenMP memory -#setenv OMP_STACKSIZE 64M +setenv OMP_STACKSIZE 64M endif -# May be needed for OpenMP memory -#setenv OMP_STACKSIZE 64M +env | grep NETCDF -endif - setenv ICE_MACHINE_MACHNAME gaea setenv ICE_MACHINE_MACHINFO "Cray XC40 Intel Haswell/Broadwell 2.3GHz, Gemini Interconnect" setenv ICE_MACHINE_ENVNAME intel From 0e365fb3f7dc79ace5468a651ff1af91f9bd1788 Mon Sep 17 00:00:00 2001 From: Robert Grumbine Date: Wed, 23 Apr 2025 09:19:31 -0400 Subject: [PATCH 3/3] specify c5 vs. c6 gaea --- configuration/scripts/cice.batch.csh | 19 ++++++- .../scripts/machines/Macros.gaeac5_intel | 56 +++++++++++++++++++ .../scripts/machines/env.gaeac5_intel | 44 +++++++++++++++ 3 files changed, 118 insertions(+), 1 deletion(-) create mode 100644 configuration/scripts/machines/Macros.gaeac5_intel create mode 100644 configuration/scripts/machines/env.gaeac5_intel diff --git a/configuration/scripts/cice.batch.csh b/configuration/scripts/cice.batch.csh index 2354de9cb..4e9058b71 100755 --- a/configuration/scripts/cice.batch.csh +++ b/configuration/scripts/cice.batch.csh @@ -349,7 +349,7 @@ cat >> ${jobfile} << EOFB #PBS -l walltime=${batchtime} EOFB -else if (${ICE_MACHINE} =~ gaea*) then +else if (${ICE_MACHINE} =~ gaeac5*) then cat >> ${jobfile} << EOFB #SBATCH -J ${ICE_CASENAME} #SBATCH --partition=batch @@ -366,6 +366,23 @@ cat >> ${jobfile} << EOFB ##SBATCH --mail-user=xxx@noaa.gov EOFB +else if (${ICE_MACHINE} =~ gaeac6*) then +cat >> ${jobfile} << EOFB +#SBATCH -J ${ICE_CASENAME} +#SBATCH --partition=batch +#SBATCH --qos=${queue} +#SBATCH --account=sfs_emc +#SBATCH --clusters=c6 +#SBATCH --time=${batchtime} +#SBATCH --nodes=${nnodes} +#SBATCH --ntasks-per-node=${taskpernodelimit} +#SBATCH --cpus-per-task=${nthrds} +#SBATCH -e slurm%j.err +#SBATCH -o slurm%j.out +##SBATCH --mail-type FAIL +##SBATCH --mail-user=xxx@noaa.gov +EOFB + else if (${ICE_MACHINE} =~ hera*) then cat >> ${jobfile} << EOFB #SBATCH -J ${ICE_CASENAME} diff --git a/configuration/scripts/machines/Macros.gaeac5_intel b/configuration/scripts/machines/Macros.gaeac5_intel new file mode 100644 index 000000000..794070214 --- /dev/null +++ b/configuration/scripts/machines/Macros.gaeac5_intel @@ -0,0 +1,56 @@ +#============================================================================== +# Makefile macros for NOAA gaeac5, intel compiler +#============================================================================== + +CPP := fpp +CPPDEFS := -DFORTRANUNDERSCORE ${ICE_CPPDEFS} +CFLAGS := -c -O2 -fp-model precise -xHost + +FIXEDFLAGS := -132 +FREEFLAGS := -FR +FFLAGS := -fp-model precise -convert big_endian -assume byterecl -ftz -traceback -align array64byte -xHost +FFLAGS_NOOPT:= -O0 + +ifeq ($(ICE_BLDDEBUG), true) + FFLAGS += -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created -link_mpi=dbg +else + FFLAGS += -O2 +endif + +SCC := cc +SFC := ftn +MPICC := cc +MPIFC := ftn + +ifeq ($(ICE_COMMDIR), mpi) + FC := $(MPIFC) + CC := $(MPICC) +else + FC := $(SFC) + CC := $(SCC) +endif +LD:= $(FC) + +NETCDF_PATH := $(NETCDF) + +PIO_CONFIG_OPTS:= --enable-filesystem-hints=gpfs + +#PNETCDF_PATH := $(PNETCDF) +#PNETCDF_PATH := /glade/u/apps/ch/opt/pio/2.2/mpt/2.15f/intel/17.0.1/lib + +INC_NETCDF := $(NETCDF_PATH)/include +LIB_NETCDF := $(NETCDF_PATH)/lib + +#LIB_PNETCDF := $(PNETCDF_PATH)/lib +#LIB_MPI := $(IMPILIBDIR) + +INCLDIR := $(INCLDIR) -I$(INC_NETCDF) +#SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff -L$(LIB_PNETCDF) -lpnetcdf -lgptl +SLIBS := -L$(LIB_NETCDF) -lnetcdf -lnetcdff + +ifeq ($(ICE_THREADED), true) + LDFLAGS += -qopenmp + CFLAGS += -qopenmp + FFLAGS += -qopenmp +endif + diff --git a/configuration/scripts/machines/env.gaeac5_intel b/configuration/scripts/machines/env.gaeac5_intel new file mode 100644 index 000000000..69bddb428 --- /dev/null +++ b/configuration/scripts/machines/env.gaeac5_intel @@ -0,0 +1,44 @@ +#!/bin/csh -f + +set inp = "undefined" +if ($#argv == 1) then + set inp = $1 +endif + +if ("$inp" != "-nomodules") then + +#source /lustre/f2/pdata/esrl/gsd/contrib/lua-5.1.4.9/init/init_lmod.csh +source $MODULESHOME/init/csh +#module list +module load PrgEnv-intel +module load intel +#module load intel/2023.2.0 +#module load cce/18.0.0 +module load cray-hdf5/1.14.3.1 +module load cray-netcdf/4.9.0.13 +setenv NETCDF $NETCDF_DIR +echo zzz final module list +module list +#module avail intel + +# May be needed for OpenMP memory +setenv OMP_STACKSIZE 64M + +endif + +env | grep NETCDF + +setenv ICE_MACHINE_MACHNAME gaea +setenv ICE_MACHINE_MACHINFO "Cray XC40 Intel Haswell/Broadwell 2.3GHz, Gemini Interconnect" +setenv ICE_MACHINE_ENVNAME intel +setenv ICE_MACHINE_ENVINFO "ifort 18.0.5 20180823, cray-mpich, cray-netcdf" +setenv ICE_MACHINE_MAKE gmake +setenv ICE_MACHINE_WKDIR $HOME/scratch/CICE_RUNS +setenv ICE_MACHINE_INPUTDATA /ncrc/home1/Robert.Grumbine/rgdev/CICE_INPUTDATA +setenv ICE_MACHINE_BASELINE $HOME/scratch/CICE_BASELINE +setenv ICE_MACHINE_SUBMIT "sbatch" +setenv ICE_MACHINE_TPNODE 40 +setenv ICE_MACHINE_ACCT P0000000 +setenv ICE_MACHINE_QUEUE "normal" +setenv ICE_MACHINE_BLDTHRDS 1 +setenv ICE_MACHINE_QSTAT "squeue --jobs="