Merge branch 'kharma-next' of github:afd-illinois/kharma into feature…

…/amr

machines/darwin.sh

@@ -10,10 +10,12 @@ if [[ $HOSTNAME == "cn"* || $HOSTNAME == "darwin"* ]]; then
   module purge
   module load cmake
 
-  # Always our own HDF5
-  # Run ""./make.sh <usual args> hdf5" to build it
-  PREFIX_PATH="$SOURCE_DIR/external/hdf5"
+  # Help Darwin find the right modules in automated jobs
+  if [[ "$ARGS" == *"cuda"* ]]; then
+    export MODULEPATH="/projects/darwin-nv/modulefiles/rhel8/aarch64:/projects/darwin-nv/modulefiles/rhel8/aarch64"
+  fi
 
+  # Load modules based on first argument...
   if [[ "$ARGS" == *"cuda"* ]]; then
     if [[ "$ARGS" == *"gcc12"* ]]; then
       module load cuda/12.0.0 openmpi gcc/12.1.0
@@ -24,13 +26,19 @@ if [[ $HOSTNAME == "cn"* || $HOSTNAME == "darwin"* ]]; then
       C_NATIVE=gcc
       CXX_NATIVE=g++
     else
-      module load nvhpc/23.3 cuda/11.7.0
+      module load nvhpc/22.1 cuda/12.0.0
       C_NATIVE="nvc"
       CXX_NATIVE="nvc++"
       # New NVHPC doesn't like CUDA_HOME
       export NVHPC_CUDA_HOME="$CUDA_HOME"
       unset CUDA_HOME
     fi
+  elif [[ "$ARGS" == *"hip"* ]]; then
+    module load rocm/5.4.3 #openmpi/5.0.0rc11-gcc_13.1.0
+    source ~/libs/env.sh
+    C_NATIVE=hipcc
+    CXX_NATIVE=hipcc
+    export CXXFLAGS="-fopenmp $CXXFLAGS"
   else
     if [[ "$ARGS" == *"gcc"* ]]; then
       module load openmpi gcc/10.2.0
@@ -44,14 +52,20 @@ if [[ $HOSTNAME == "cn"* || $HOSTNAME == "darwin"* ]]; then
     fi
   fi
 
-  # These are orthogonal to above.
-  # Just don't compile for an nv arch without "cuda"
+  # ...and set architecture according to second.
+  # These are orthogonal to above, so long as the hardware
+  # supports the paradigm
   NPROC=$(($(nproc) / 2))
-  if [[ "$ARGS" == *"arm-nv"* ]]; then
+  if [[ "$ARGS" == *"arm-ampere"* ]]; then
     HOST_ARCH="ARMV81"
     DEVICE_ARCH="AMPERE80"
     MPI_NUM_PROCS=2
     NODE_SLICE=2
+  elif [[ "$ARGS" == *"arm-hopper"* ]]; then
+    HOST_ARCH="ARMV81"
+    DEVICE_ARCH="HOPPER90"
+    MPI_NUM_PROCS=1
+    NODE_SLICE=1
   elif [[ "$ARGS" == *"ampere"* ]]; then
     HOST_ARCH="ZEN3"
     DEVICE_ARCH="AMPERE80"
@@ -72,6 +86,11 @@ if [[ $HOSTNAME == "cn"* || $HOSTNAME == "darwin"* ]]; then
     HOST_ARCH="HSW"
     MPI_NUM_PROCS=1
     NODE_SLICE=1
+  elif [[ "$ARGS" == *"mi250"* ]]; then
+    HOST_ARCH=ZEN3
+    DEVICE_ARCH=VEGA90A
+    MPI_NUM_PROCS=8
+    NODE_SLICE=16
   else
     echo "Must specify an architecture on Darwin!"
     exit

make.sh

@@ -74,6 +74,7 @@ fi
 ### Enivoronment Prep ###
 if [[ "$(which python3 2>/dev/null)" == *"conda"* ]]; then
   echo
+  echo "make.sh note:"
   echo "It looks like you have Anaconda loaded."
   echo "Anaconda loads a serial version of HDF5 which may make this compile impossible."
   echo "If you run into trouble, deactivate your environment with 'conda deactivate'"
@@ -283,7 +284,12 @@ fi
 if [[ "$ARGS" == *"clean"* ]]; then
 
   cd external/parthenon
-  git apply --quiet ../patches/parthenon-*.patch
+  if [[ $(( $(git --version | cut -d '.' -f 2) > 35 )) ]]; then
+    git apply --quiet ../patches/parthenon-*.patch
+  else
+    echo "make.sh note: You may see errors applying patches below. These are normal."
+    git apply ../patches/parthenon-*.patch
+  fi
   cd -
 
   rm -rf build