From 5c1bbc7a303ac9e72044307153c4978cfcd4d61c Mon Sep 17 00:00:00 2001
From: Ben Prather <bprathr2@illinois.edu>
Date: Wed, 6 Sep 2023 12:45:25 -0600
Subject: [PATCH] Add bach Chicoma machinefile, fix compile on Crays

---
 machines/chicoma.sh | 40 ++++++++++++++++++++++++++++++++++++++++
 make.sh             |  2 +-
 2 files changed, 41 insertions(+), 1 deletion(-)
 create mode 100644 machines/chicoma.sh

diff --git a/machines/chicoma.sh b/machines/chicoma.sh
new file mode 100644
index 00000000..41fc59b6
--- /dev/null
+++ b/machines/chicoma.sh
@@ -0,0 +1,40 @@
+# LANL Machines: HPC and IC
+
+# Chicoma
+if [[ "$HOST" == "ch-fe"* ]]; then
+  HOST_ARCH="ZEN2"
+
+  # Cray environments get confused easy
+  # Make things as simple as possible
+  # TODO version with Cray wrappers?
+  module purge
+  export CRAY_CPU_TARGET="x86-64"
+  if [[ "$ARGS" == *"cuda"* ]]; then
+    DEVICE_ARCH="AMPERE80"
+    # System HDF5 can't use compression
+    EXTRA_FLAGS="-DPARTHENON_DISABLE_HDF5_COMPRESSION=ON $EXTRA_FLAGS"
+    # Runtime
+    MPI_NUM_PROCS=4
+    if [[ "$ARGS" == *"gnu"* ]]; then
+      module load PrgEnv-gnu cpe-cuda cuda
+    elif [[ "$ARGS" == *"intel"* ]]; then
+      module load PrgEnv-intel
+    elif [[ "$ARGS" == *"nvc++"* ]]; then
+      module load PrgEnv-nvhpc cray-hdf5-parallel
+      EXTRA_FLAGS="-DCMAKE_CUDA_COMPILER=$HOME/bin/nvc++-wrapper -DCMAKE_CUDA_COMPILER_ID=NVHPC -DCMAKE_CUDA_COMPILER_VERSION=11.6 $EXTRA_FLAGS"
+    else
+      module load PrgEnv-nvhpc cray-hdf5-parallel
+    fi
+  else
+    module load PrgEnv-aocc
+  fi
+  module load cmake
+
+  # Runtime
+  MPI_NUM_PROCS=4
+  MPI_EXE=srun
+  MPI_EXTRA_ARGS="--cpu-bind=mask_cpu:0x0*16,0x1*16,0x2*16,0x3*16 ~/bin/select-gpu"
+  unset OMP_NUM_THREADS
+  unset OMP_PROC_BIND
+  unset OMP_PLACES
+fi
diff --git a/make.sh b/make.sh
index 64b33f50..e9b6329c 100755
--- a/make.sh
+++ b/make.sh
@@ -105,7 +105,7 @@ if [[ -z "$CXX_NATIVE" ]]; then
     CXX_NATIVE=CC
     C_NATIVE=cc
     # In case this isn't Cray, use the more common flag
-    OMP_FLAG="-fopenomp"
+    #OMP_FLAG="-fopenomp"
   # Prefer Intel oneAPI compiler over legacy, both over generic
   elif which icpx >/dev/null 2>&1; then
     CXX_NATIVE=icpx