From 330749987034d381fdc037fa852b6836b586b1c1 Mon Sep 17 00:00:00 2001
From: apcraig <anthony.p.craig@gmail.com>
Date: Wed, 17 Mar 2021 17:27:15 -0700
Subject: [PATCH] add machine compy

---
 configuration/scripts/cice.batch.csh          | 16 +++++++
 configuration/scripts/cice.launch.csh         | 12 +++++
 .../scripts/machines/Macros.compy_intel       | 44 +++++++++++++++++++
 .../scripts/machines/env.compy_intel          | 42 ++++++++++++++++++
 4 files changed, 114 insertions(+)
 create mode 100644 configuration/scripts/machines/Macros.compy_intel
 create mode 100755 configuration/scripts/machines/env.compy_intel

diff --git a/configuration/scripts/cice.batch.csh b/configuration/scripts/cice.batch.csh
index 6d1f735a4..60073abfd 100755
--- a/configuration/scripts/cice.batch.csh
+++ b/configuration/scripts/cice.batch.csh
@@ -133,6 +133,22 @@ cat >> ${jobfile} << EOFB
 ###SBATCH --mail-user username@domain.com
 EOFB
 
+else if (${ICE_MACHINE} =~ compy*) then
+if (${runlength} <= 2) set queue = "short"
+cat >> ${jobfile} <<EOFB
+#SBATCH -J ${ICE_CASENAME}
+#SBATCH -A ${acct}
+#SBATCH --qos ${queue}
+#SBATCH --time ${batchtime}
+#SBATCH --nodes ${nnodes}
+#SBATCH --ntasks ${ntasks}
+#SBATCH --cpus-per-task ${nthrds}
+###SBATCH -e filename
+###SBATCH -o filename
+###SBATCH --mail-type FAIL
+###SBATCH --mail-user username@domain.com
+EOFB
+
 else if (${ICE_MACHINE} =~ badger*) then
 cat >> ${jobfile} << EOFB
 #SBATCH -J ${ICE_CASENAME}
diff --git a/configuration/scripts/cice.launch.csh b/configuration/scripts/cice.launch.csh
index a05b3a9d3..7d45a387f 100755
--- a/configuration/scripts/cice.launch.csh
+++ b/configuration/scripts/cice.launch.csh
@@ -81,6 +81,18 @@ srun --cpu-bind=cores ./cice >&! \$ICE_RUNLOG_FILE
 EOFR
 endif
 
+#=======
+else if (${ICE_MACHINE} =~ compy*) then
+if (${ICE_COMMDIR} =~ serial*) then
+cat >> ${jobfile} << EOFR
+./cice >&! \$ICE_RUNLOG_FILE
+EOFR
+else
+cat >> ${jobfile} << EOFR
+srun --mpi=pmi2 --kill-on-bad-exit --cpu-bind=cores ./cice >&! \$ICE_RUNLOG_FILE
+EOFR
+endif
+
 #=======
 else if (${ICE_MACHINE} =~ badger*) then
 if (${ICE_COMMDIR} =~ serial*) then
diff --git a/configuration/scripts/machines/Macros.compy_intel b/configuration/scripts/machines/Macros.compy_intel
new file mode 100644
index 000000000..604337f59
--- /dev/null
+++ b/configuration/scripts/machines/Macros.compy_intel
@@ -0,0 +1,44 @@
+#==============================================================================
+# Makefile macro for PNNL compy, intel compiler
+#==============================================================================
+
+CPP        := fpp
+CPPDEFS    := -DFORTRANUNDERSCORE ${ICE_CPPDEFS}
+CFLAGS     := -c -O2 -fp-model precise   -xHost
+
+FIXEDFLAGS := -132
+FREEFLAGS  := -FR
+FFLAGS     := -fp-model precise -convert big_endian -assume byterecl -ftz -traceback   -xHost
+FFLAGS_NOOPT:= -O0
+
+ifeq ($(ICE_BLDDEBUG), true)
+  FFLAGS     += -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created
+else
+  FFLAGS     += -O2
+endif
+
+SCC := icc
+SFC := ifort
+MPICC := mpiicc
+MPIFC := mpiifort
+ifeq ($(ICE_COMMDIR), mpi)
+  FC := $(MPIFC)
+  CC := $(MPICC)
+else
+  FC := $(SFC)
+  CC := $(SCC)
+endif
+LD := $(FC)
+
+INC_NETCDF := $(NETCDF_PATH)/include
+LIB_NETCDF := $(NETCDF_PATH)/lib
+
+INCLDIR := $(INCLDIR) -I$(INC_NETCDF)
+SLIBS   := -L$(LIB_NETCDF) -lnetcdf -lnetcdff
+
+ifeq ($(ICE_THREADED), true) 
+   LDFLAGS += -qopenmp 
+   CFLAGS += -qopenmp 
+   FFLAGS += -qopenmp 
+endif
+
diff --git a/configuration/scripts/machines/env.compy_intel b/configuration/scripts/machines/env.compy_intel
new file mode 100755
index 000000000..fe3511aa6
--- /dev/null
+++ b/configuration/scripts/machines/env.compy_intel
@@ -0,0 +1,42 @@
+#!/bin/csh -f
+
+set inp = "undefined"
+if ($#argv == 1) then
+  set inp = $1
+endif
+
+if ("$inp" != "-nomodules") then
+
+source /share/apps/modules/init/csh
+
+module purge
+module load intel/19.0.5
+module load intelmpi/2019u4
+module load netcdf/4.6.3 
+module load hdf5/1.10.5
+
+#setenv NETCDF_PATH ${NETCDF_DIR}
+setenv NETCDF_PATH /share/apps/netcdf/4.6.3/intel/19.0.5
+setenv OMP_PROC_BIND true
+setenv OMP_PLACES threads
+setenv I_MPI_ADJUST_ALLREDUCE 1
+limit coredumpsize unlimited
+limit stacksize unlimited
+
+endif
+
+setenv ICE_MACHINE_MACHNAME compy
+setenv ICE_MACHINE_MACHINFO "PNNL Intel Xeon Skylake with 192 GB of DDR4 DRAM"
+setenv ICE_MACHINE_ENVNAME intel
+setenv ICE_MACHINE_ENVINFO "intel/19.0.5 intelmpi/2019u4 netcdf/4.6.3"
+setenv ICE_MACHINE_MAKE gmake
+setenv ICE_MACHINE_WKDIR /compyfs/$USER/CICE_RUNS
+setenv ICE_MACHINE_INPUTDATA /compyfs/inputdata/cice-consortium/
+setenv ICE_MACHINE_BASELINE /compyfs/$USER/CICE_BASELINE
+setenv ICE_MACHINE_SUBMIT "sbatch "
+setenv ICE_MACHINE_ACCT e3sm
+setenv ICE_MACHINE_QUEUE "slurm"
+setenv ICE_MACHINE_TPNODE 40    # tasks per node
+setenv ICE_MACHINE_BLDTHRDS 4
+setenv ICE_MACHINE_QSTAT "squeue --jobs="
+