From 4a31fb32725c84963c80420681ab978c9c55f90c Mon Sep 17 00:00:00 2001
From: Edward Hartnett <Edward.Hartnett@noaa.gov>
Date: Wed, 10 Mar 2021 07:39:19 -0700
Subject: [PATCH 1/5] converted to MPI test

---
 cmake/LibMPI.cmake                      | 124 ++++++++++++++++++++++++
 tests/chres_cube/CMakeLists.txt         |   9 +-
 tests/chres_cube/ftst_program_setup.F90 |  33 ++++---
 3 files changed, 152 insertions(+), 14 deletions(-)
 create mode 100644 cmake/LibMPI.cmake

diff --git a/cmake/LibMPI.cmake b/cmake/LibMPI.cmake
new file mode 100644
index 000000000..18079f6c4
--- /dev/null
+++ b/cmake/LibMPI.cmake
@@ -0,0 +1,124 @@
+# This file contains CMake code related to MPI. (Taken from the
+# ParallelIO project.)
+
+# Jim Edwards
+include (CMakeParseArguments)
+
+# Find Valgrind to perform memory leak check
+if (PIO_VALGRIND_CHECK)
+    find_program (VALGRIND_COMMAND NAMES valgrind)
+    if (VALGRIND_COMMAND)
+        set (VALGRIND_COMMAND_OPTIONS --leak-check=full --show-reachable=yes)
+    else ()
+        message (WARNING "Valgrind not found: memory leak check could not be performed")
+        set (VALGRIND_COMMAND "")
+    endif ()
+endif ()
+
+#
+# - Functions for parallel testing with CTest
+#
+
+#==============================================================================
+# - Get the machine platform-specific
+#
+# Syntax:  platform_name (RETURN_VARIABLE)
+#
+function (platform_name RETURN_VARIABLE)
+
+    # Determine platform name from site name...
+    site_name (SITENAME)
+
+
+    if (SITENAME MATCHES "^laramie" OR
+            SITENAME MATCHES "^cheyenne" OR
+	    SITENAME MATCHES "^chadmin")
+
+	set (${RETURN_VARIABLE} "nwscla" PARENT_SCOPE)
+
+    # ALCF/Argonne Machines
+    elseif (SITENAME MATCHES "^mira" OR
+            SITENAME MATCHES "^cetus" OR
+            SITENAME MATCHES "^vesta" OR
+            SITENAME MATCHES "^cooley")
+
+        set (${RETURN_VARIABLE} "alcf" PARENT_SCOPE)
+
+    # NERSC Machines
+    elseif (SITENAME MATCHES "^edison" OR
+        SITENAME MATCHES "^cori")
+
+        set (${RETURN_VARIABLE} "nersc" PARENT_SCOPE)
+
+    # NCSA Machine (Blue Waters)
+    elseif (SITENAME MATCHES "^h2ologin")
+
+        set (${RETURN_VARIABLE} "ncsa" PARENT_SCOPE)
+
+    # OLCF/Oak Ridge Machines
+    elseif (SITENAME MATCHES "^eos" OR
+            SITENAME MATCHES "^titan")
+
+        set (${RETURN_VARIABLE} "olcf" PARENT_SCOPE)
+
+    else ()
+
+        set (${RETURN_VARIABLE} "unknown" PARENT_SCOPE)
+
+    endif ()
+
+endfunction ()
+
+#==============================================================================
+# - Add a new parallel test
+#
+# Syntax:  add_mpi_test (<TESTNAME>
+#                        EXECUTABLE <command>
+#                        ARGUMENTS <arg1> <arg2> ...
+#                        NUMPROCS <num_procs>
+#                        TIMEOUT <timeout>)
+function (add_mpi_test TESTNAME)
+
+    # Parse the input arguments
+    set (options)
+    set (oneValueArgs NUMPROCS TIMEOUT EXECUTABLE)
+    set (multiValueArgs ARGUMENTS)
+    cmake_parse_arguments (${TESTNAME} "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
+
+    # Store parsed arguments for convenience
+    set (exec_file ${${TESTNAME}_EXECUTABLE})
+    set (exec_args ${${TESTNAME}_ARGUMENTS})
+    set (num_procs ${${TESTNAME}_NUMPROCS})
+    set (timeout ${${TESTNAME}_TIMEOUT})
+
+    # Get the platform name
+    platform_name (PLATFORM)
+
+    get_property(WITH_MPIEXEC GLOBAL PROPERTY WITH_MPIEXEC)
+    if (WITH_MPIEXEC)
+      set(MPIEXEC "${WITH_MPIEXEC}")
+    endif ()
+
+    # Default ("unknown" platform) execution
+    if (PLATFORM STREQUAL "unknown")
+
+        # Run tests directly from the command line
+        set(EXE_CMD ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${num_procs}
+                    ${MPIEXEC_PREFLAGS} ${VALGRIND_COMMAND} ${VALGRIND_COMMAND_OPTIONS} ${exec_file}
+                    ${MPIEXEC_POSTFLAGS} ${exec_args})
+
+    else ()
+
+        # Run tests from the platform-specific executable
+        set (EXE_CMD ${CMAKE_SOURCE_DIR}/cmake/mpiexec.${PLATFORM}
+                     ${num_procs} ${VALGRIND_COMMAND} ${VALGRIND_COMMAND_OPTIONS} ${exec_file} ${exec_args})
+
+    endif ()
+
+    # Add the test to CTest
+    add_test(NAME ${TESTNAME} COMMAND ${EXE_CMD})
+
+    # Adjust the test timeout
+    set_tests_properties(${TESTNAME} PROPERTIES TIMEOUT ${timeout})
+
+endfunction()
diff --git a/tests/chres_cube/CMakeLists.txt b/tests/chres_cube/CMakeLists.txt
index 0d3232d68..893b0e914 100644
--- a/tests/chres_cube/CMakeLists.txt
+++ b/tests/chres_cube/CMakeLists.txt
@@ -3,6 +3,9 @@
 #
 # George Gayno, Lin Gan, Ed Hartnett
 
+# Include cmake to allow parallel I/O tests.
+include (LibMPI)
+
 if(CMAKE_Fortran_COMPILER_ID MATCHES "^(Intel)$")
   set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -r8 -convert big_endian -assume byterecl")
 elseif(CMAKE_Fortran_COMPILER_ID MATCHES "^(GNU)$")
@@ -51,7 +54,6 @@ add_executable(ftst_program_setup "${CMAKE_SOURCE_DIR}/sorc/chgres_cube.fd/input
   "${CMAKE_SOURCE_DIR}/sorc/chgres_cube.fd/search_util.F90"
   "${CMAKE_SOURCE_DIR}/sorc/chgres_cube.fd/utils.F90"
   ftst_program_setup.F90)
-add_test(NAME ftst_program_setup COMMAND ftst_program_setup)
 target_link_libraries(
   ftst_program_setup
   nemsio::nemsio
@@ -69,5 +71,10 @@ if(OpenMP_Fortran_FOUND)
   target_link_libraries(ftst_program_setup OpenMP::OpenMP_Fortran)
 endif()
 
+# Cause test to be run with MPI.
+add_mpi_test(ftst_program_setup
+  EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/ftst_program_setup
+  NUMPROCS 4
+  TIMEOUT 60)
 
 
diff --git a/tests/chres_cube/ftst_program_setup.F90 b/tests/chres_cube/ftst_program_setup.F90
index ddb6064ae..759324a38 100644
--- a/tests/chres_cube/ftst_program_setup.F90
+++ b/tests/chres_cube/ftst_program_setup.F90
@@ -3,15 +3,21 @@
 ! Ed Hartnett 2/16/21
 
 program ftst_program_setup
+  use mpi
   use esmf
   use netcdf
   use program_setup
   implicit none
   integer :: is
+  integer :: my_rank, nprocs
+  integer :: ierr
 
-  print*, "Starting test of program_setup."
+  call mpi_init(ierr)
+  call MPI_Comm_rank(MPI_COMM_WORLD, my_rank, ierr)
+  call MPI_Comm_size(MPI_COMM_WORLD, nprocs, ierr)
 
-  print*, "testing read_setup_namelist with file fort.41..."
+  if (my_rank .eq. 0) print*, "Starting test of program_setup."
+  if (my_rank .eq. 0) print*, "testing read_setup_namelist with file fort.41..."
   call read_setup_namelist()
   if (cycle_mon .ne. 7 .or. cycle_day .ne. 4 .or. cycle_hour .ne. 12) stop 4
   if (.not. convert_atm .or. .not. convert_sfc .or. .not. convert_nst) stop 5
@@ -43,9 +49,9 @@ program ftst_program_setup
      tracers(is) = "NULL"
      tracers_input(is) = "NULL"
   enddo
-  print*, "OK"
+  if (my_rank .eq. 0) print*, "OK"
 
-  print*, "testing read_setup_namelist with config_fv3_tiled..."
+  if (my_rank .eq. 0) print*, "testing read_setup_namelist with config_fv3_tiled..."
   call read_setup_namelist("config_fv3_tiled.nml")
   if (cycle_mon .ne. 10 .or. cycle_day .ne. 3 .or. cycle_hour .ne. 0) stop 34
   if (.not. convert_atm .or. .not. convert_sfc .or. .not. convert_nst) stop 35
@@ -73,14 +79,14 @@ program ftst_program_setup
      tracers(is) = "NULL"
      tracers_input(is) = "NULL"
   enddo
-  print*, "OK"
+  if (my_rank .eq. 0) print*, "OK"
 
   ! Reading this namelist fails for some reason.
   ! print*, "testing read_setup_namelist with config_fv3_tiled_warm_restart..."
   ! call read_setup_namelist("config_fv3_tiled_warm_restart.nml")
   ! print*, "OK"
 
-  print*, "testing read_setup_namelist with config_gaussian_nemsio..."
+  if (my_rank .eq. 0) print*, "testing read_setup_namelist with config_gaussian_nemsio..."
   call read_setup_namelist("config_gaussian_nemsio.nml")
   if (cycle_mon .ne. 7 .or. cycle_day .ne. 4 .or. cycle_hour .ne. 12) stop 74
   if (.not. convert_atm .or. .not. convert_sfc .or. .not. convert_nst) stop 75
@@ -98,7 +104,6 @@ program ftst_program_setup
   if (atm_weight_file .ne. "NULL") stop 88
   if (trim(input_type) .ne. "gaussian_nemsio") stop 89
   if (trim(external_model) .ne. "GFS") stop 90
-  print *,num_tracers
   if (num_tracers .ne. 7) stop 21
   if (tracers(1) .ne. "sphum" .or. tracers(2) .ne. "liq_wat" .or. tracers(3) .ne. "o3mr" .or. &
        tracers(4) .ne. "ice_wat" .or. tracers(5) .ne. "rainwat" .or. tracers(6) .ne. "snowwat" .or. &
@@ -113,9 +118,9 @@ program ftst_program_setup
      tracers(is) = "NULL"
      tracers_input(is) = "NULL"
   enddo
-  print*, "OK"
+  if (my_rank .eq. 0) print*, "OK"
 
-  print*, "testing read_setup_namelist with config_spectral_sigio..."
+  if (my_rank .eq. 0) print*, "testing read_setup_namelist with config_spectral_sigio..."
   call read_setup_namelist("config_spectral_sigio.nml")
   if (cycle_mon .ne. 7 .or. cycle_day .ne. 17 .or. cycle_hour .ne. 0) stop 114
   if (.not. convert_atm .or. .not. convert_sfc .or. convert_nst) stop 115
@@ -143,9 +148,9 @@ program ftst_program_setup
      tracers(is) = "NULL"
      tracers_input(is) = "NULL"
   enddo
-  print*, "OK"
+  if (my_rank .eq. 0) print*, "OK"
 
-  print*, "testing read_setup_namelist with config_gfs_grib2..."
+  if (my_rank .eq. 0) print*, "testing read_setup_namelist with config_gfs_grib2..."
   call read_setup_namelist("config_gfs_grib2.nml")
   if (cycle_mon .ne. 11 .or. cycle_day .ne. 4 .or. cycle_hour .ne. 0) stop 94
   if (.not. convert_atm .or. .not. convert_sfc .or. convert_nst) stop 95
@@ -177,7 +182,9 @@ program ftst_program_setup
      tracers(is) = "NULL"
      tracers_input(is) = "NULL"
   enddo
-  print*, "OK"
+  if (my_rank .eq. 0) print*, "OK"
 
-  print*, "SUCCESS!"
+  if (my_rank .eq. 0) print*, "SUCCESS!"
+
+ call mpi_finalize(ierr)  
 end program ftst_program_setup

From 6e0a0a7d0996ac0285f840a50f1c72e7a5b8b4e6 Mon Sep 17 00:00:00 2001
From: kgerheiser <kgerheiser@icloud.com>
Date: Mon, 5 Apr 2021 10:36:39 -0400
Subject: [PATCH 2/5] MPI test scripts for hera, orion, and jet

---
 cmake/LibMPI.cmake  | 251 ++++++++++++++++++++++++++++++--------------
 cmake/mpiexec.hera  |  12 +++
 cmake/mpiexec.jet   |  12 +++
 cmake/mpiexec.orion |  12 +++
 4 files changed, 209 insertions(+), 78 deletions(-)
 create mode 100644 cmake/mpiexec.hera
 create mode 100644 cmake/mpiexec.jet
 create mode 100644 cmake/mpiexec.orion

diff --git a/cmake/LibMPI.cmake b/cmake/LibMPI.cmake
index 18079f6c4..13733f656 100644
--- a/cmake/LibMPI.cmake
+++ b/cmake/LibMPI.cmake
@@ -6,13 +6,13 @@ include (CMakeParseArguments)
 
 # Find Valgrind to perform memory leak check
 if (PIO_VALGRIND_CHECK)
-    find_program (VALGRIND_COMMAND NAMES valgrind)
-    if (VALGRIND_COMMAND)
-        set (VALGRIND_COMMAND_OPTIONS --leak-check=full --show-reachable=yes)
-    else ()
-        message (WARNING "Valgrind not found: memory leak check could not be performed")
-        set (VALGRIND_COMMAND "")
-    endif ()
+  find_program (VALGRIND_COMMAND NAMES valgrind)
+  if (VALGRIND_COMMAND)
+    set (VALGRIND_COMMAND_OPTIONS --leak-check=full --show-reachable=yes)
+  else ()
+    message (WARNING "Valgrind not found: memory leak check could not be performed")
+    set (VALGRIND_COMMAND "")
+  endif ()
 endif ()
 
 #
@@ -26,46 +26,141 @@ endif ()
 #
 function (platform_name RETURN_VARIABLE)
 
-    # Determine platform name from site name...
-    site_name (SITENAME)
-
-
-    if (SITENAME MATCHES "^laramie" OR
-            SITENAME MATCHES "^cheyenne" OR
-	    SITENAME MATCHES "^chadmin")
-
-	set (${RETURN_VARIABLE} "nwscla" PARENT_SCOPE)
-
-    # ALCF/Argonne Machines
-    elseif (SITENAME MATCHES "^mira" OR
-            SITENAME MATCHES "^cetus" OR
-            SITENAME MATCHES "^vesta" OR
-            SITENAME MATCHES "^cooley")
-
-        set (${RETURN_VARIABLE} "alcf" PARENT_SCOPE)
-
-    # NERSC Machines
-    elseif (SITENAME MATCHES "^edison" OR
-        SITENAME MATCHES "^cori")
-
-        set (${RETURN_VARIABLE} "nersc" PARENT_SCOPE)
-
-    # NCSA Machine (Blue Waters)
-    elseif (SITENAME MATCHES "^h2ologin")
-
-        set (${RETURN_VARIABLE} "ncsa" PARENT_SCOPE)
-
-    # OLCF/Oak Ridge Machines
-    elseif (SITENAME MATCHES "^eos" OR
-            SITENAME MATCHES "^titan")
-
-        set (${RETURN_VARIABLE} "olcf" PARENT_SCOPE)
-
-    else ()
-
-        set (${RETURN_VARIABLE} "unknown" PARENT_SCOPE)
-
-    endif ()
+  # Determine platform name from site name...
+  site_name (SITENAME)
+
+
+  # hera
+  if (SITENAME MATCHES "^hfe01" OR
+      SITENAME MATCHES "^hfe02" OR
+      SITENAME MATCHES "^hfe03" OR
+      SITENAME MATCHES "^hfe04" OR
+      SITENAME MATCHES "^hfe05" OR
+      SITENAME MATCHES "^hfe06" OR
+      SITENAME MATCHES "^hfe07" OR
+      SITENAME MATCHES "^hfe08" OR
+      SITENAME MATCHES "^hfe09" OR
+      SITENAME MATCHES "^hfe10" OR
+      SITENAME MATCHES "^hfe11" OR
+      SITENAME MATCHES "^hfe12")
+
+    set (${RETURN_VARIABLE} "hera" PARENT_SCOPE)
+
+  # wcoss_cray (Luna)
+  elseif (SITENAME MATCHES "^llogin1" OR
+      SITENAME MATCHES "^llogin2" OR
+      SITENAME MATCHES "^llogin3")
+
+    set (${RETURN_VARIABLE} "wcoss_cray" PARENT_SCOPE)
+    
+  # wcoss_cray (Surge)
+  elseif (SITENAME MATCHES "^slogin1" OR
+      SITENAME MATCHES "^slogin2" OR
+      SITENAME MATCHES "^slogin3")
+
+    set (${RETURN_VARIABLE} "wcoss_cray" PARENT_SCOPE)
+    
+  # wcoss_dell_p3 (Venus)
+  elseif (SITENAME MATCHES "^v71a1.ncep.noaa.gov" OR
+      SITENAME MATCHES "^v71a2.ncep.noaa.gov" OR
+      SITENAME MATCHES "^v71a3.ncep.noaa.gov" OR
+      SITENAME MATCHES "^v72a1.ncep.noaa.gov" OR
+      SITENAME MATCHES "^v72a2.ncep.noaa.gov" OR
+      SITENAME MATCHES "^v72a3.ncep.noaa.gov")
+
+    set (${RETURN_VARIABLE} "wcoss_dell_p3" PARENT_SCOPE)
+  
+  # wcoss_dell_p3 (Mars)
+  elseif (SITENAME MATCHES "^m71a1.ncep.noaa.gov" OR
+      SITENAME MATCHES "^m71a2.ncep.noaa.gov" OR
+      SITENAME MATCHES "^m71a3.ncep.noaa.gov" OR
+      SITENAME MATCHES "^m72a1.ncep.noaa.gov" OR
+      SITENAME MATCHES "^m72a2.ncep.noaa.gov" OR
+      SITENAME MATCHES "^m72a3.ncep.noaa.gov")
+
+    set (${RETURN_VARIABLE} "wcoss_dell_p3" PARENT_SCOPE)
+
+  # wcoss2
+  elseif (SITENAME MATCHES "^along01" OR
+      SITENAME MATCHES "^alogin02")
+
+    set (${RETURN_VARIABLE} "wcoss2" PARENT_SCOPE)
+
+  # gaea
+  elseif (SITENAME MATCHES "^gaea9" OR
+      SITENAME MATCHES "^gaea10" OR
+      SITENAME MATCHES "^gaea11" OR
+      SITENAME MATCHES "^gaea12" OR
+      SITENAME MATCHES "^gaea13" OR
+      SITENAME MATCHES "^gaea14" OR
+      SITENAME MATCHES "^gaea15" OR
+      SITENAME MATCHES "^gaea16" OR
+      SITENAME MATCHES "^gaea9.ncrc.gov)" OR
+      SITENAME MATCHES "^gaea10.ncrc.gov)" OR
+      SITENAME MATCHES "^gaea11.ncrc.gov)" OR
+      SITENAME MATCHES "^gaea12.ncrc.gov)" OR
+      SITENAME MATCHES "^gaea13.ncrc.gov)" OR
+      SITENAME MATCHES "^gaea14.ncrc.gov)" OR
+      SITENAME MATCHES "^gaea15.ncrc.gov" OR
+      SITENAME MATCHES "^gaea16.ncrc.gov")
+
+    set (${RETURN_VARIABLE} "gaea" PARENT_SCOPE)
+    
+  # jet 
+  elseif (SITENAME MATCHES "^fe1" OR
+      SITENAME MATCHES "^fe2" OR
+      SITENAME MATCHES "^fe3" OR
+      SITENAME MATCHES "^fe4" OR
+      SITENAME MATCHES "^fe5" OR
+      SITENAME MATCHES "^fe6" OR
+      SITENAME MATCHES "^fe7" OR
+      SITENAME MATCHES "^fe8" OR
+      SITENAME MATCHES "^tfe1" OR
+      SITENAME MATCHES "^tfe2")
+
+    set (${RETURN_VARIABLE} "jet" PARENT_SCOPE)
+
+  elseif (SITENAME MATCHES "^Orion-login-1.HPC.MsState.Edu" OR
+      SITENAME MATCHES "^Orion-login-2.HPC.MsState.Edu" OR
+      SITENAME MATCHES "^Orion-login-3.HPC.MsState.Edu" OR
+      SITENAME MATCHES "^Orion-login-4.HPC.MsState.Edu")
+
+    set (${RETURN_VARIABLE} "orion" PARENT_SCOPE)
+
+  elseif (SITENAME MATCHES "^cheyenne1.cheyenne.ucar.edu" OR
+      SITENAME MATCHES "^cheyenne1.cheyenne.ucar.edu" OR
+      SITENAME MATCHES "^cheyenne2.cheyenne.ucar.edu" OR
+      SITENAME MATCHES "^cheyenne3.cheyenne.ucar.edu" OR
+      SITENAME MATCHES "^cheyenne4.cheyenne.ucar.edu" OR
+      SITENAME MATCHES "^cheyenne5.cheyenne.ucar.edu" OR
+      SITENAME MATCHES "^cheyenne6.cheyenne.ucar.edu" OR
+      SITENAME MATCHES "^cheyenne1.ib0.cheyenne.ucar.edu" OR
+      SITENAME MATCHES "^cheyenne2.ib0.cheyenne.ucar.edu" OR
+      SITENAME MATCHES "^cheyenne3.ib0.cheyenne.ucar.edu" OR
+      SITENAME MATCHES "^cheyenne4.ib0.cheyenne.ucar.edu" OR
+      SITENAME MATCHES "^cheyenne5.ib0.cheyenne.ucar.edu" OR
+      SITENAME MATCHES "^cheyenne6.ib0.cheyenne.ucar.edu" OR
+      SITENAME MATCHES "^chadmin1.ib0.cheyenne.ucar.edu" OR
+      SITENAME MATCHES "^chadmin2.ib0.cheyenne.ucar.edu" OR
+      SITENAME MATCHES "^chadmin3.ib0.cheyenne.ucar.edu" OR
+      SITENAME MATCHES "^chadmin4.ib0.cheyenne.ucar.edu" OR
+      SITENAME MATCHES "^chadmin5.ib0.cheyenne.ucar.edu" OR
+      SITENAME MATCHES "^chadmin6.ib0.cheyenne.ucar.edu")
+
+    set (${RETURN_VARIABLE} "cheyenne" PARENT_SCOPE)
+  elseif (SITENAME MATCHES "^login1.stampede2.tacc.utexas.edu" OR
+      SITENAME MATCHES "^login2.stampede2.tacc.utexas.edu" OR
+      SITENAME MATCHES "^login3.stampede2.tacc.utexas.edu" OR
+      SITENAME MATCHES "^login4.stampede2.tacc.utexas.edu")
+    
+
+    set (${RETURN_VARIABLE} "stampede" PARENT_SCOPE)
+          
+  else ()
+
+    set (${RETURN_VARIABLE} "unknown" PARENT_SCOPE)
+
+  endif ()
 
 endfunction ()
 
@@ -79,46 +174,46 @@ endfunction ()
 #                        TIMEOUT <timeout>)
 function (add_mpi_test TESTNAME)
 
-    # Parse the input arguments
-    set (options)
-    set (oneValueArgs NUMPROCS TIMEOUT EXECUTABLE)
-    set (multiValueArgs ARGUMENTS)
-    cmake_parse_arguments (${TESTNAME} "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
+  # Parse the input arguments
+  set (options)
+  set (oneValueArgs NUMPROCS TIMEOUT EXECUTABLE)
+  set (multiValueArgs ARGUMENTS)
+  cmake_parse_arguments (${TESTNAME} "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
 
-    # Store parsed arguments for convenience
-    set (exec_file ${${TESTNAME}_EXECUTABLE})
-    set (exec_args ${${TESTNAME}_ARGUMENTS})
-    set (num_procs ${${TESTNAME}_NUMPROCS})
-    set (timeout ${${TESTNAME}_TIMEOUT})
+  # Store parsed arguments for convenience
+  set (exec_file ${${TESTNAME}_EXECUTABLE})
+  set (exec_args ${${TESTNAME}_ARGUMENTS})
+  set (num_procs ${${TESTNAME}_NUMPROCS})
+  set (timeout ${${TESTNAME}_TIMEOUT})
 
-    # Get the platform name
-    platform_name (PLATFORM)
+  # Get the platform name
+  platform_name (PLATFORM)
 
-    get_property(WITH_MPIEXEC GLOBAL PROPERTY WITH_MPIEXEC)
-    if (WITH_MPIEXEC)
-      set(MPIEXEC "${WITH_MPIEXEC}")
-    endif ()
+  get_property(WITH_MPIEXEC GLOBAL PROPERTY WITH_MPIEXEC)
+  if (WITH_MPIEXEC)
+    set(MPIEXEC "${WITH_MPIEXEC}")
+  endif ()
 
-    # Default ("unknown" platform) execution
-    if (PLATFORM STREQUAL "unknown")
+  # Default ("unknown" platform) execution
+  if (PLATFORM STREQUAL "unknown")
 
-        # Run tests directly from the command line
-        set(EXE_CMD ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${num_procs}
-                    ${MPIEXEC_PREFLAGS} ${VALGRIND_COMMAND} ${VALGRIND_COMMAND_OPTIONS} ${exec_file}
-                    ${MPIEXEC_POSTFLAGS} ${exec_args})
+    # Run tests directly from the command line
+    set(EXE_CMD ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${num_procs}
+      ${MPIEXEC_PREFLAGS} ${VALGRIND_COMMAND} ${VALGRIND_COMMAND_OPTIONS} ${exec_file}
+      ${MPIEXEC_POSTFLAGS} ${exec_args})
 
-    else ()
+  else ()
 
-        # Run tests from the platform-specific executable
-        set (EXE_CMD ${CMAKE_SOURCE_DIR}/cmake/mpiexec.${PLATFORM}
-                     ${num_procs} ${VALGRIND_COMMAND} ${VALGRIND_COMMAND_OPTIONS} ${exec_file} ${exec_args})
+    # Run tests from the platform-specific executable
+    set (EXE_CMD ${CMAKE_SOURCE_DIR}/cmake/mpiexec.${PLATFORM}
+      ${num_procs} ${VALGRIND_COMMAND} ${VALGRIND_COMMAND_OPTIONS} ${exec_file} ${exec_args})
 
-    endif ()
+  endif ()
 
-    # Add the test to CTest
-    add_test(NAME ${TESTNAME} COMMAND ${EXE_CMD})
+  # Add the test to CTest
+  add_test(NAME ${TESTNAME} COMMAND ${EXE_CMD})
 
-    # Adjust the test timeout
-    set_tests_properties(${TESTNAME} PROPERTIES TIMEOUT ${timeout})
+  # Adjust the test timeout
+  set_tests_properties(${TESTNAME} PROPERTIES TIMEOUT ${timeout})
 
 endfunction()
diff --git a/cmake/mpiexec.hera b/cmake/mpiexec.hera
new file mode 100644
index 000000000..e8774b0e9
--- /dev/null
+++ b/cmake/mpiexec.hera
@@ -0,0 +1,12 @@
+#!/bin/bash
+#
+# Arguments:
+#
+#   $1  - Number of MPI Tasks
+#   $2+ - Executable and its arguments
+#
+
+NP=$1
+shift
+
+srun -n $NP $@
diff --git a/cmake/mpiexec.jet b/cmake/mpiexec.jet
new file mode 100644
index 000000000..e8774b0e9
--- /dev/null
+++ b/cmake/mpiexec.jet
@@ -0,0 +1,12 @@
+#!/bin/bash
+#
+# Arguments:
+#
+#   $1  - Number of MPI Tasks
+#   $2+ - Executable and its arguments
+#
+
+NP=$1
+shift
+
+srun -n $NP $@
diff --git a/cmake/mpiexec.orion b/cmake/mpiexec.orion
new file mode 100644
index 000000000..e8774b0e9
--- /dev/null
+++ b/cmake/mpiexec.orion
@@ -0,0 +1,12 @@
+#!/bin/bash
+#
+# Arguments:
+#
+#   $1  - Number of MPI Tasks
+#   $2+ - Executable and its arguments
+#
+
+NP=$1
+shift
+
+srun -n $NP $@

From f00efda7ab59f220a4780b2d79be692333e8c798 Mon Sep 17 00:00:00 2001
From: kgerheiser <kgerheiser@icloud.com>
Date: Mon, 5 Apr 2021 10:46:52 -0400
Subject: [PATCH 3/5] Fix syntax

---
 cmake/LibMPI.cmake | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/cmake/LibMPI.cmake b/cmake/LibMPI.cmake
index 13733f656..a8539adce 100644
--- a/cmake/LibMPI.cmake
+++ b/cmake/LibMPI.cmake
@@ -95,12 +95,12 @@ function (platform_name RETURN_VARIABLE)
       SITENAME MATCHES "^gaea14" OR
       SITENAME MATCHES "^gaea15" OR
       SITENAME MATCHES "^gaea16" OR
-      SITENAME MATCHES "^gaea9.ncrc.gov)" OR
-      SITENAME MATCHES "^gaea10.ncrc.gov)" OR
-      SITENAME MATCHES "^gaea11.ncrc.gov)" OR
-      SITENAME MATCHES "^gaea12.ncrc.gov)" OR
-      SITENAME MATCHES "^gaea13.ncrc.gov)" OR
-      SITENAME MATCHES "^gaea14.ncrc.gov)" OR
+      SITENAME MATCHES "^gaea9.ncrc.gov" OR
+      SITENAME MATCHES "^gaea10.ncrc.gov" OR
+      SITENAME MATCHES "^gaea11.ncrc.gov" OR
+      SITENAME MATCHES "^gaea12.ncrc.gov" OR
+      SITENAME MATCHES "^gaea13.ncrc.gov" OR
+      SITENAME MATCHES "^gaea14.ncrc.gov" OR
       SITENAME MATCHES "^gaea15.ncrc.gov" OR
       SITENAME MATCHES "^gaea16.ncrc.gov")
 

From 97d7f9740077e54f758e42be28c19e197acc8dd2 Mon Sep 17 00:00:00 2001
From: kgerheiser <kgerheiser@icloud.com>
Date: Mon, 5 Apr 2021 10:59:07 -0400
Subject: [PATCH 4/5] Make scripts executable, pass account to srun

---
 cmake/mpiexec.hera  | 4 +++-
 cmake/mpiexec.jet   | 4 +++-
 cmake/mpiexec.orion | 4 +++-
 3 files changed, 9 insertions(+), 3 deletions(-)
 mode change 100644 => 100755 cmake/mpiexec.hera
 mode change 100644 => 100755 cmake/mpiexec.jet
 mode change 100644 => 100755 cmake/mpiexec.orion

diff --git a/cmake/mpiexec.hera b/cmake/mpiexec.hera
old mode 100644
new mode 100755
index e8774b0e9..d906ad47d
--- a/cmake/mpiexec.hera
+++ b/cmake/mpiexec.hera
@@ -6,7 +6,9 @@
 #   $2+ - Executable and its arguments
 #
 
+ACCOUNT=
+
 NP=$1
 shift
 
-srun -n $NP $@
+srun -A $ACCOUNT -n $NP $@
diff --git a/cmake/mpiexec.jet b/cmake/mpiexec.jet
old mode 100644
new mode 100755
index e8774b0e9..d906ad47d
--- a/cmake/mpiexec.jet
+++ b/cmake/mpiexec.jet
@@ -6,7 +6,9 @@
 #   $2+ - Executable and its arguments
 #
 
+ACCOUNT=
+
 NP=$1
 shift
 
-srun -n $NP $@
+srun -A $ACCOUNT -n $NP $@
diff --git a/cmake/mpiexec.orion b/cmake/mpiexec.orion
old mode 100644
new mode 100755
index e8774b0e9..d906ad47d
--- a/cmake/mpiexec.orion
+++ b/cmake/mpiexec.orion
@@ -6,7 +6,9 @@
 #   $2+ - Executable and its arguments
 #
 
+ACCOUNT=
+
 NP=$1
 shift
 
-srun -n $NP $@
+srun -A $ACCOUNT -n $NP $@

From 0f883f6f52086f953e7ede919de03008644e745d Mon Sep 17 00:00:00 2001
From: kgerheiser <kgerheiser@icloud.com>
Date: Mon, 5 Apr 2021 11:11:33 -0400
Subject: [PATCH 5/5] Add qos

---
 cmake/mpiexec.hera  | 3 ++-
 cmake/mpiexec.jet   | 3 ++-
 cmake/mpiexec.orion | 3 ++-
 3 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/cmake/mpiexec.hera b/cmake/mpiexec.hera
index d906ad47d..332b33e29 100755
--- a/cmake/mpiexec.hera
+++ b/cmake/mpiexec.hera
@@ -7,8 +7,9 @@
 #
 
 ACCOUNT=
+QOS=debug
 
 NP=$1
 shift
 
-srun -A $ACCOUNT -n $NP $@
+srun -A $ACCOUNT -q $QOS -n $NP $@
diff --git a/cmake/mpiexec.jet b/cmake/mpiexec.jet
index d906ad47d..332b33e29 100755
--- a/cmake/mpiexec.jet
+++ b/cmake/mpiexec.jet
@@ -7,8 +7,9 @@
 #
 
 ACCOUNT=
+QOS=debug
 
 NP=$1
 shift
 
-srun -A $ACCOUNT -n $NP $@
+srun -A $ACCOUNT -q $QOS -n $NP $@
diff --git a/cmake/mpiexec.orion b/cmake/mpiexec.orion
index d906ad47d..332b33e29 100755
--- a/cmake/mpiexec.orion
+++ b/cmake/mpiexec.orion
@@ -7,8 +7,9 @@
 #
 
 ACCOUNT=
+QOS=debug
 
 NP=$1
 shift
 
-srun -A $ACCOUNT -n $NP $@
+srun -A $ACCOUNT -q $QOS -n $NP $@