Compile updates for Frontier, warning fixes

.gitignore

@@ -66,7 +66,7 @@ make_args
 # Executables
 *.host*
 *.cuda*
-*.rocm*
+*.hip*
 *.sycl*
 *.exe
 *.out

CMakeLists.txt

@@ -30,7 +30,6 @@ set(ENABLE_COMPILER_WARNINGS OFF CACHE BOOL "KHARMA Override")
 set(HDF5_USE_STATIC_LIBRARIES ON CACHE BOOL "KHARMA Override")
 
 # Kokkos options
-set(Kokkos_ENABLE_OPENMP ON CACHE BOOL "KHARMA Override")
 set(Kokkos_ENABLE_CUDA_LAMBDA ON CACHE BOOL "KHARMA Override")
 set(Kokkos_ENABLE_CUDA_CONSTEXPR ON CACHE BOOL "KHARMA Override")
 set(Kokkos_ENABLE_HWLOC OFF CACHE BOOL "KHARMA Override") # Possible speed improvement?
@@ -51,8 +50,9 @@ else()
   include_directories(SYSTEM ${MPI_INCLUDE_PATH})
 endif()
 
-# OpenMP is strictly required
-find_package(OpenMP REQUIRED)
+# OpenMP is usually used host-side.  We're letting Parthenon/Kokkos
+# find it though, as sometimes we require disabling it fully
+#find_package(OpenMP REQUIRED)
 
 # Build Parthenon
 add_subdirectory(external/parthenon)

kharma/CMakeLists.txt

@@ -5,6 +5,8 @@ if (Kokkos_ENABLE_CUDA)
     set(EXE_NAME "kharma.cuda")
 elseif(Kokkos_ENABLE_SYCL)
     set(EXE_NAME "kharma.sycl")
+elseif(Kokkos_ENABLE_HIP)
+    set(EXE_NAME "kharma.hip")
 else()
     set(EXE_NAME "kharma.host")
 endif()

kharma/prob/seed_B.hpp

@@ -46,8 +46,11 @@ enum BSeedType{constant, monopole, monopole_cube, sane, mad, mad_quadrupole, r3s
 
 #define SEEDA_ARGS GReal *x, double rho, double rin, double min_A, double A0
 
+// This will also act as the default implementation for unspecified types,
+// which should all be filled as B field by seed_b below.
+// So, we want to set it to something dramatic.
 template<BSeedType T>
-KOKKOS_INLINE_FUNCTION Real seed_a(SEEDA_ARGS) {}
+KOKKOS_INLINE_FUNCTION Real seed_a(SEEDA_ARGS) { return 0./0.;}
 
 // EHT comparison SANE
 template<>

machines/frontier.sh

@@ -0,0 +1,50 @@
+
+# Config for OLCF Frontier
+
+if [[ $HOST == *".frontier.olcf.ornl.gov" ]]
+then
+  HOST_ARCH=ZEN3
+  DEVICE_ARCH=VEGA90A
+
+  MPI_EXE=srun
+  NPROC=64
+
+  if [[ $ARGS == *"hip"* ]]; then
+    # HIP compile for AMD GPUs
+
+    if [[ $ARGS == *"cray"* ]]; then
+      module load PrgEnv-cray
+      module load craype-accel-amd-gfx90a
+      module load amd-mixed
+    else
+      module load PrgEnv-amd
+      module load craype-accel-amd-gfx90a
+    fi
+
+    module load cray-hdf5-parallel
+
+    if [[ $ARGS == *"hipcc"* ]]; then
+      # TODO LINK MPI RIGHT
+      CXX_NATIVE=hipcc
+      C_NATIVE=hipcc
+      export CXXFLAGS="-I$CRAY_HDF5_PARALLEL_PREFIX/include -L$CRAY_HDF5_PARALLEL_PREFIX/lib -l:libhdf5_parallel.a"
+      #export PATH="$CRAY_HDF5_PARALLEL_PREFIX/bin:$PATH"
+    else
+      CXX_NATIVE=CC
+      C_NATIVE=cc
+      export CXXFLAGS="-noopenmp -mllvm -amdgpu-function-calls=false $CXXFLAGS"
+    fi
+
+    # Runtime
+    MPI_NUM_PROCS=8
+    MPI_EXTRA_ARGS="-c1 --gpus-per-node=8 --gpu-bind=closest"
+    export MPICH_GPU_SUPPORT_ENABLED=1
+
+   # Old workaround, for non-GPU MPI only!
+   #export MPICH_SMP_SINGLE_COPY_MODE=NONE
+  else
+    # CPU Compile
+    # TODO -c etc etc
+    MPI_NUM_PROCS=1
+  fi
+fi

make.sh

@@ -161,7 +161,7 @@ if [[ "$ARGS" == *"sycl"* ]]; then
 elif [[ "$ARGS" == *"hip"* ]]; then
   OUTER_LAYOUT="MANUAL1D_LOOP"
   INNER_LAYOUT="TVR_INNER_LOOP"
-  ENABLE_OPENMP="ON"
+  ENABLE_OPENMP="OFF"
   ENABLE_CUDA="OFF"
   ENABLE_SYCL="OFF"
   ENABLE_HIP="ON"

run.sh

@@ -34,6 +34,8 @@ if [ -f $KHARMA_DIR/kharma.cuda ]; then
   EXE_NAME=kharma.cuda
 elif [ -f $KHARMA_DIR/kharma.sycl ]; then
   EXE_NAME=kharma.sycl
+elif [ -f $KHARMA_DIR/kharma.hip ]; then
+  EXE_NAME=kharma.hip
 elif [ -f $KHARMA_DIR/kharma.host ]; then
   EXE_NAME=kharma.host
 else