From 14c98520ab9fbbab272ed73a27a6527e90d51111 Mon Sep 17 00:00:00 2001
From: Ben Prather <bprathr2@illinois.edu>
Date: Fri, 8 Dec 2023 16:05:37 -0600
Subject: [PATCH] Properly support MPI/No MPI option. CMake cleanup.

---
 CMakeLists.txt        |  5 +----
 kharma/CMakeLists.txt | 22 ++++++++++++----------
 kharma/decs.hpp       |  2 +-
 3 files changed, 14 insertions(+), 15 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index c3b61010..5819fe04 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -38,6 +38,7 @@ set(Kokkos_ENABLE_CUDA_CONSTEXPR ON CACHE BOOL "KHARMA Override")
 set(Kokkos_ENABLE_HWLOC OFF CACHE BOOL "KHARMA Override") # Possible speed improvement?
 set(Kokkos_ENABLE_AGGRESSIVE_VECTORIZATION ON CACHE BOOL "KHARMA Override")
 
+# For including the current git revision in the exe
 include(GetGitRevisionDescription)
 get_git_head_revision(GIT_REFSPEC GIT_SHA1)
 git_describe_working_tree(GIT_VERSION --tags)
@@ -53,10 +54,6 @@ else()
   include_directories(SYSTEM ${MPI_INCLUDE_PATH})
 endif()
 
-# OpenMP is usually used host-side.  We're letting Parthenon/Kokkos
-# find it though, as sometimes we require disabling it fully
-#find_package(OpenMP REQUIRED)
-
 # Build Parthenon
 add_subdirectory(external/parthenon)
 include_directories(external/parthenon/src)
diff --git a/kharma/CMakeLists.txt b/kharma/CMakeLists.txt
index aec167cc..614ff4b1 100644
--- a/kharma/CMakeLists.txt
+++ b/kharma/CMakeLists.txt
@@ -92,11 +92,16 @@ else()
 endif()
 
 # OPTIONS
-# These are almost universally performance trade-offs
+# These are almost universally performance trade-offs,
+# or disabling features that need dependencies.
 # TODO is there any way to make compile options less painful in CMake?
 option(FUSE_FLUX_KERNELS "Bundle the usual four flux calculation kernels (floors,R,L,apply) into one" ON)
 option(FUSE_FLOOR_KERNELS "Bundle applying the floors and ceilings into one kernel" ON)
 option(FAST_CARTESIAN "Break operation in curved spacetimes to make Cartesian Minkowski space computations faster" OFF)
+option(KHARMA_DISABLE_IMPLICIT "Disable the implicit solver, which requires bundled kokkos-kernels. Default false" OFF)
+option(KHARMA_DISABLE_CLEANUP "Disable the magnetic field cleanup module, which requires recent Parthenon. Default false" OFF)
+option(KHARMA_TRACE "Compile with tracing: print entry and exit of important functions. Default false" OFF)
+
 if(FUSE_FLUX_KERNELS)
     target_compile_definitions(${EXE_NAME} PUBLIC FUSE_FLUX_KERNELS=1)
 else()
@@ -108,13 +113,11 @@ else()
     target_compile_definitions(${EXE_NAME} PUBLIC FUSE_FLOOR_KERNELS=0)
 endif()
 if(FAST_CARTESIAN)
+    message("Compiling for Cartesian coordinates only. GRMHD will be disabled!")
     target_compile_definitions(${EXE_NAME} PUBLIC FAST_CARTESIAN=1)
 else()
     target_compile_definitions(${EXE_NAME} PUBLIC FAST_CARTESIAN=0)
 endif()
-option(KHARMA_DISABLE_IMPLICIT "Disable the implicit solver, which requires bundled kokkos-kernels. Default false" OFF)
-option(KHARMA_DISABLE_CLEANUP "Disable the magnetic field cleanup module, which requires recent Parthenon. Default false" OFF)
-option(KHARMA_TRACE "Compile with tracing: print entry and exit of important functions" OFF)
 if(KHARMA_DISABLE_IMPLICIT)
     message("Compiling without the implicit solver.  Extended GRMHD will be disabled!")
     target_compile_definitions(${EXE_NAME} PUBLIC DISABLE_IMPLICIT=1)
@@ -134,14 +137,13 @@ if(KHARMA_TRACE)
 else()
     target_compile_definitions(${EXE_NAME} PUBLIC TRACE=0)
 endif()
-option(KHARMA_DISABLE_IMPLICIT "Compile the implicit solver, requiring kokkos-kernels. Default true" OFF)
-option(KHARMA_TRACE "Compile with tracing: print entry and exit of important functions" OFF)
-if(KHARMA_DISABLE_IMPLICIT)
-    message("Compiling without the implicit solver.  Extended GRMHD will be disabled!")
-    target_compile_definitions(${EXE_NAME} PUBLIC ENABLE_IMPLICIT=0)
+if(KHARMA_DISABLE_MPI)
+    message("Compiling without MPI!")
+    target_compile_definitions(${EXE_NAME} PUBLIC ENABLE_MPI=0)
 else()
-    target_compile_definitions(${EXE_NAME} PUBLIC ENABLE_IMPLICIT=1)
+    target_compile_definitions(${EXE_NAME} PUBLIC ENABLE_MPI=1)
 endif()
+
 # FLAGS
 if(CMAKE_BUILD_TYPE)
     if(${CMAKE_BUILD_TYPE} STREQUAL "Debug")
diff --git a/kharma/decs.hpp b/kharma/decs.hpp
index ee86102d..6795d020 100644
--- a/kharma/decs.hpp
+++ b/kharma/decs.hpp
@@ -155,7 +155,7 @@ inline int MPIRank()
 }
 inline int MPIBarrier()
 {
-#if MPI_PARALLEL
+#if ENABLE_MPI
     return MPI_Barrier(MPI_COMM_WORLD);
 #else
     return 0;