From 9beb292b88638a37a80797775ee9e1cbfa11f07d Mon Sep 17 00:00:00 2001
From: Jian Sun <sunjian@ucar.edu>
Date: Wed, 22 Mar 2023 00:17:31 -0600
Subject: [PATCH 01/24] Add Jim's changes for Gust's modules from
 https://github.com/jedwards4b/ccs_config_cesm/commit/87b9539ee4c280f5262c92db18eae2b07cc53b67
 Use nvhpc/22.11, cray-mpich/8.1.21, and esmf/8.4.1.b02

	modified:   machines/cmake_macros/gust.cmake
	modified:   machines/config_machines.xml
---
 machines/cmake_macros/gust.cmake |  2 +-
 machines/config_machines.xml     | 60 +++++++++++++-------------------
 2 files changed, 26 insertions(+), 36 deletions(-)

diff --git a/machines/cmake_macros/gust.cmake b/machines/cmake_macros/gust.cmake
index 5332d19c9e..0bb6f4f244 100644
--- a/machines/cmake_macros/gust.cmake
+++ b/machines/cmake_macros/gust.cmake
@@ -7,4 +7,4 @@ set(PNETCDF_PATH "$ENV{PNETCDF}")
 # If we want to use cray-libsci instead of mkl uncomment this line as well as the module in config_machines.xml
 #string(REPLACE "-mkl=cluster" "" SLIBS "${SLIBS}")
 string(REPLACE "-mkl=cluster" "-qmkl=cluster" SLIBS "${SLIBS}")
-#string(APPEND CPPDEFS " -DNO_SHR_VMATH ")
\ No newline at end of file
+string(APPEND CPPDEFS " -DHAVE_GETTID")
diff --git a/machines/config_machines.xml b/machines/config_machines.xml
index b9e229bf58..ec188bcf62 100644
--- a/machines/config_machines.xml
+++ b/machines/config_machines.xml
@@ -1857,80 +1857,70 @@ This allows using a different mpirun command to launch unit tests
       </arguments>
     </mpirun>
     <module_system type="module" allow_error="true">
-      <init_path lang="perl">/glade/u/apps/gust/default/spack/opt/spack/lmod/8.7.2/gcc/7.5.0/lmod/lmod/init/perl</init_path>
-      <init_path lang="python">/glade/u/apps/gust/default/spack/opt/spack/lmod/8.7.2/gcc/7.5.0/lmod/lmod/init/env_modules_python.py</init_path>
-      <init_path lang="sh">/glade/u/apps/gust/default/spack/opt/spack/lmod/8.7.2/gcc/7.5.0/lmod/lmod/init/sh</init_path>
-      <init_path lang="csh">/glade/u/apps/gust/default/spack/opt/spack/lmod/8.7.2/gcc/7.5.0/lmod/lmod/init/csh</init_path>
-      <cmd_path lang="perl">/glade/u/apps/gust/default/spack/opt/spack/lmod/8.7.2/gcc/7.5.0/lmod/lmod/libexec/lmod perl</cmd_path>
-      <cmd_path lang="python">/glade/u/apps/gust/default/spack/opt/spack/lmod/8.7.2/gcc/7.5.0/lmod/lmod/libexec/lmod python</cmd_path>
+      <init_path lang="perl">$ENV{LMOD_ROOT}/lmod/init/perl</init_path>
+      <init_path lang="python">$ENV{LMOD_ROOT}/lmod/init/env_modules_python.py</init_path>
+      <init_path lang="sh">$ENV{LMOD_ROOT}/lmod/init/sh</init_path>
+      <init_path lang="csh">$ENV{LMOD_ROOT}/lmod/init/csh</init_path>
+      <cmd_path lang="perl">$ENV{LMOD_ROOT}/lmod/libexec/lmod perl</cmd_path>
+      <cmd_path lang="python">$ENV{LMOD_ROOT}/lmod/libexec/lmod python</cmd_path>
       <cmd_path lang="sh">module</cmd_path>
       <cmd_path lang="csh">module</cmd_path>
       <modules>
         <command name="reset"/>
 	<command name="use">/opt/cray/pe/craype-targets/default/modulefiles</command>
+        <command name="load">cmake</command>
       </modules>
 
       <modules compiler="intel">
-        <command name="load">intel/2021.6.0</command>
+        <command name="load">intel/2021.7.1</command>
       </modules>
       <modules compiler="oneapi">
-        <command name="load">oneapi/2022.1.0</command>
+        <command name="load">oneapi/2023.0.0</command>
       </modules>
       <modules compiler="cray">
-        <command name="load">cce/15.0.0</command>
+        <command name="load">cce/15.0.1</command>
       </modules>
       <modules compiler="gnu">
         <command name="load">gcc/12.1.0</command>
       </modules>
       <modules compiler="nvhpc">
-        <command name="load">nvhpc/22.7</command>
-      </modules>
-      <modules mpilib="mpich" compiler="!cray">
-	<command name="load">cray-mpich/8.1.19</command>
+        <command name="load">nvhpc/22.11</command>
       </modules>
-      <modules mpilib="mpich" compiler="cray">
-	<command name="switch">cray-mpich/8.1.21</command>
+      <modules mpilib="mpich">
+	<command name="load">cray-mpich/8.1.21</command>
       </modules>
       <modules mpilib="mpi-serial">
         <command name="load">mpi-serial/2.3.0</command>
       </modules>
-      
-      <modules>
-        <command name="load">cmake/3.23.2</command>
-        <command name="load">cray-libsci/22.08.1.1</command>
+      <modules compiler="!intel">
+        <command name="load">cray-libsci/23.02.1.1</command>
       </modules>
       <!-- to use cray-libsci instead of mkl comment these out and
 	   edit cmake_macros/gust.cmake -->
       <modules compiler="intel">
-        <command name="load">mkl/2022.1.0</command>
+        <command name="load">mkl/2022.2.1</command>
       </modules>
 
       <modules compiler="oneapi">
-        <command name="load">mkl/2022.1.0</command>
+        <command name="load">mkl/2022.2.1</command>
       </modules>	
       <modules mpilib="mpi-serial">
-        <command name="load">netcdf/4.9.0</command>
+        <command name="load">netcdf/4.9.1</command>
       </modules>
 
       <modules mpilib="!mpi-serial">
 	<command name="rm">hdf5</command>
         <command name="rm">netcdf</command>
-        <command name="load">netcdf-mpi/4.9.0</command>
+        <command name="load">netcdf-mpi/4.9.1</command>
         <command name="load">parallel-netcdf/1.12.3</command>
       </modules>
-      <modules DEBUG="TRUE" compiler="!cray">
-	<command name="load">parallelio/2.5.9-debug</command>
-	<command name="load">esmf/8.4.0b20-debug</command>
-      </modules>
-      <modules DEBUG="FALSE" compiler="!cray">
-	<command name="load">parallelio/2.5.9</command>
-	<command name="load">esmf/8.4.0b20</command>
-      </modules>
-      <modules DEBUG="TRUE" compiler="cray">
-	<command name="load">esmf/8.4.0b21-debug</command>
+      <modules DEBUG="TRUE">
+	<command name="load">parallelio/2.5.10-debug</command>
+	<command name="load">esmf/8.4.1b02-debug</command>
       </modules>
-      <modules DEBUG="FALSE" compiler="cray">
-	<command name="load">esmf/8.4.0b21</command>
+      <modules DEBUG="FALSE">
+	<command name="load">parallelio/2.5.10</command>
+	<command name="load">esmf/8.4.1b02</command>
       </modules>
     </module_system>
 

From 4134af3498e13e9b63db40dbd61f77ff9a2e2506 Mon Sep 17 00:00:00 2001
From: Jian Sun <sunjian@ucar.edu>
Date: Wed, 22 Mar 2023 00:39:23 -0600
Subject: [PATCH 02/24] Add Jim's gpu_improvement branch at
 https://github.com/jedwards4b/ccs_config_cesm/compare/52c06b3..fbc05d6 Update
 settings on Gust and Casper

	modified:   Depends.nvhpc
	deleted:    Depends.nvhpc-gpu
	deleted:    cmake_macros/nvhpc-gpu.cmake
	deleted:    cmake_macros/nvhpc-gpu_casper.cmake
	modified:   cmake_macros/nvhpc.cmake
	modified:   cmake_macros/nvhpc_casper.cmake
	deleted:    cmake_macros/pgi-gpu.cmake
	deleted:    cmake_macros/pgi-gpu_casper.cmake
	modified:   config_batch.xml
	modified:   config_machines.xml
	deleted:    mpi_run_gpu.casper
---
 machines/Depends.nvhpc                       | 16 ++++-
 machines/Depends.nvhpc-gpu                   | 16 -----
 machines/cmake_macros/nvhpc-gpu.cmake        | 46 ------------
 machines/cmake_macros/nvhpc-gpu_casper.cmake | 15 ----
 machines/cmake_macros/nvhpc.cmake            | 18 +++++
 machines/cmake_macros/nvhpc_casper.cmake     |  1 +
 machines/cmake_macros/pgi-gpu.cmake          | 46 ------------
 machines/cmake_macros/pgi-gpu_casper.cmake   | 15 ----
 machines/config_batch.xml                    | 24 ++-----
 machines/config_machines.xml                 | 73 ++++++--------------
 machines/mpi_run_gpu.casper                  |  6 --
 11 files changed, 59 insertions(+), 217 deletions(-)
 delete mode 100644 machines/Depends.nvhpc-gpu
 delete mode 100644 machines/cmake_macros/nvhpc-gpu.cmake
 delete mode 100644 machines/cmake_macros/nvhpc-gpu_casper.cmake
 delete mode 100644 machines/cmake_macros/pgi-gpu.cmake
 delete mode 100644 machines/cmake_macros/pgi-gpu_casper.cmake
 delete mode 100755 machines/mpi_run_gpu.casper

diff --git a/machines/Depends.nvhpc b/machines/Depends.nvhpc
index 812904f7c4..2269cd3190 100644
--- a/machines/Depends.nvhpc
+++ b/machines/Depends.nvhpc
@@ -3,5 +3,19 @@
 REDUCED_OPT_OBJS=\
 esm.o
 
+PUMAS_OBJS=\
+micro_mg1_0.o \
+micro_pumas_v1.o \
+micro_pumas_data.o \
+micro_pumas_utils.o \
+micro_pumas_cam.o \
+wv_sat_methods.o \
+wv_saturation.o \
+macrop_driver.o \
+shr_spfn_mod.o
+
 $(REDUCED_OPT_OBJS): %.o: %.F90
-		     $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS)  -O1 $<
+	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O1 $<
+
+$(PUMAS_OBJS): %.o: %.F90
+	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) $(GPUFLAGS) $<
diff --git a/machines/Depends.nvhpc-gpu b/machines/Depends.nvhpc-gpu
deleted file mode 100644
index 4817ee0e8a..0000000000
--- a/machines/Depends.nvhpc-gpu
+++ /dev/null
@@ -1,16 +0,0 @@
-#
-PUMAS_OBJS=\
-micro_mg1_0.o \
-micro_pumas_v1.o \
-micro_pumas_data.o \
-micro_pumas_utils.o \
-micro_pumas_cam.o \
-wv_sat_methods.o \
-wv_saturation.o \
-macrop_driver.o \
-shr_spfn_mod.o
-
-ifeq ($(DEBUG),FALSE)
-  $(PUMAS_OBJS): %.o: %.F90
-	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -acc -gpu=cc70,lineinfo,nofma -Minfo=accel $<
-endif
diff --git a/machines/cmake_macros/nvhpc-gpu.cmake b/machines/cmake_macros/nvhpc-gpu.cmake
deleted file mode 100644
index 679e81b2f4..0000000000
--- a/machines/cmake_macros/nvhpc-gpu.cmake
+++ /dev/null
@@ -1,46 +0,0 @@
-string(APPEND CFLAGS " -gopt  -time")
-if (compile_threaded)
-  string(APPEND CFLAGS " -mp")
-endif()
-string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DNO_SHR_VMATH -DNO_R16  -DCPRPGI")
-set(CXX_LINKER "CXX")
-set(FC_AUTO_R8 "-r8")
-string(APPEND FFLAGS " -i4 -gopt  -time -Mextend -byteswapio -Mflushz -Kieee")
-if (compile_threaded)
-  string(APPEND FFLAGS " -mp")
-endif()
-if (DEBUG)
-  string(APPEND FFLAGS " -O0 -g -Ktrap=fp -Mbounds -Kieee")
-endif()
-if (COMP_NAME STREQUAL datm)
-  string(APPEND FFLAGS " -Mnovect")
-endif()
-if (COMP_NAME STREQUAL dlnd)
-  string(APPEND FFLAGS " -Mnovect")
-endif()
-if (COMP_NAME STREQUAL drof)
-  string(APPEND FFLAGS " -Mnovect")
-endif()
-if (COMP_NAME STREQUAL dwav)
-  string(APPEND FFLAGS " -Mnovect")
-endif()
-if (COMP_NAME STREQUAL dice)
-  string(APPEND FFLAGS " -Mnovect")
-endif()
-if (COMP_NAME STREQUAL docn)
-  string(APPEND FFLAGS " -Mnovect")
-endif()
-set(FFLAGS_NOOPT "-O0")
-set(FIXEDFLAGS "-Mfixed")
-set(FREEFLAGS "-Mfree")
-set(HAS_F2008_CONTIGUOUS "FALSE")
-set(LDFLAGS "-time -Wl,--allow-multiple-definition")
-if (compile_threaded)
-  string(APPEND LDFLAGS " -mp")
-endif()
-set(MPICC "mpicc")
-set(MPICXX "mpicxx")
-set(MPIFC "mpif90")
-set(SCC "nvc")
-set(SCXX "nvc++")
-set(SFC "nvfortran")
diff --git a/machines/cmake_macros/nvhpc-gpu_casper.cmake b/machines/cmake_macros/nvhpc-gpu_casper.cmake
deleted file mode 100644
index ed2adbc6c9..0000000000
--- a/machines/cmake_macros/nvhpc-gpu_casper.cmake
+++ /dev/null
@@ -1,15 +0,0 @@
-if (NOT DEBUG)
-  string(APPEND CFLAGS " -O -tp=skylake -Mnofma")
-endif()
-if (NOT DEBUG)
-  string(APPEND FFLAGS " -O -tp=skylake -Mnofma")
-endif()
-string(APPEND FFLAGS " -I$(EXEROOT)/ocn/obj/FMS")
-if (NOT DEBUG)
-  string(APPEND LDFLAGS " -O -tp=skylake -Mnofma -acc -gpu=cc70,lineinfo,nofma -Minfo=accel")
-endif()
-string(APPEND SLIBS " -llapack -lblas")
-if (MPILIB STREQUAL mpi-serial)
-  string(APPEND SLIBS " -ldl")
-endif()
-string(APPEND SLIBS " -L${NETCDF_PATH}/lib -lnetcdf -lnetcdff")
diff --git a/machines/cmake_macros/nvhpc.cmake b/machines/cmake_macros/nvhpc.cmake
index 3e1b451624..1e6036d978 100644
--- a/machines/cmake_macros/nvhpc.cmake
+++ b/machines/cmake_macros/nvhpc.cmake
@@ -49,3 +49,21 @@ set(MPIFC "mpif90")
 set(SCC "nvc")
 set(SCXX "nvc++")
 set(SFC "nvfortran")
+if (GPU_TYPE STREQUAL v100 AND GPU_OFFLOAD STREQUAL openacc)
+   string(APPEND GPUFLAGS  " -acc -gpu=cc70,lineinfo,nofma -Minfo=accel ")
+endif()
+if (GPU_TYPE STREQUAL v100 AND GPU_OFFLOAD STREQUAL openmp)
+   string(APPEND GPUFLAGS  " -mp=gpu -gpu=cc70,lineinfo,nofma -Minfo=accel ")
+endif()
+if (GPU_TYPE STREQUAL v100 AND GPU_OFFLOAD STREQUAL combined)
+   string(APPEND GPUFLAGS  " -acc -gpu=cc70,lineinfo,nofma -mp=gpu -Minfo=accel ")
+endif()
+if (GPU_TYPE STREQUAL a100 AND GPU_OFFLOAD STREQUAL openacc)
+   string(APPEND GPUFLAGS  " -acc -gpu=cc80,lineinfo,nofma -Minfo=accel ")
+endif()
+if (GPU_TYPE STREQUAL a100 AND GPU_OFFLOAD STREQUAL openmp)
+   string(APPEND GPUFLAGS  " -mp=gpu -gpu=cc80,lineinfo,nofma -Minfo=accel ")
+endif()
+if (GPU_TYPE STREQUAL a100 AND GPU_OFFLOAD STREQUAL combined)
+   string(APPEND GPUFLAGS  " -acc -gpu=cc80,lineinfo,nofma -mp=gpu -Minfo=accel")
+endif()
diff --git a/machines/cmake_macros/nvhpc_casper.cmake b/machines/cmake_macros/nvhpc_casper.cmake
index f3eb207d1e..08bc6bbecb 100644
--- a/machines/cmake_macros/nvhpc_casper.cmake
+++ b/machines/cmake_macros/nvhpc_casper.cmake
@@ -13,3 +13,4 @@ if (MPILIB STREQUAL mpi-serial)
   string(APPEND SLIBS " -ldl")
 endif()
 string(APPEND SLIBS " -L${NETCDF_PATH}/lib -lnetcdf -lnetcdff")
+message("GPU_TYPE is ${GPU_TYPE} GPU_OFFLOAD is ${GPU_OFFLOAD}")
diff --git a/machines/cmake_macros/pgi-gpu.cmake b/machines/cmake_macros/pgi-gpu.cmake
deleted file mode 100644
index 856574795e..0000000000
--- a/machines/cmake_macros/pgi-gpu.cmake
+++ /dev/null
@@ -1,46 +0,0 @@
-string(APPEND CFLAGS " -gopt  -time")
-if (compile_threaded)
-  string(APPEND CFLAGS " -mp")
-endif()
-string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DNO_SHR_VMATH -DNO_R16  -DCPRPGI")
-set(CXX_LINKER "CXX")
-set(FC_AUTO_R8 "-r8")
-string(APPEND FFLAGS " -i4 -gopt  -time -Mextend -byteswapio -Mflushz -Kieee")
-if (compile_threaded)
-  string(APPEND FFLAGS " -mp")
-endif()
-if (DEBUG)
-  string(APPEND FFLAGS " -O0 -g -Ktrap=fp -Mbounds -Kieee")
-endif()
-if (COMP_NAME STREQUAL datm)
-  string(APPEND FFLAGS " -Mnovect")
-endif()
-if (COMP_NAME STREQUAL dlnd)
-  string(APPEND FFLAGS " -Mnovect")
-endif()
-if (COMP_NAME STREQUAL drof)
-  string(APPEND FFLAGS " -Mnovect")
-endif()
-if (COMP_NAME STREQUAL dwav)
-  string(APPEND FFLAGS " -Mnovect")
-endif()
-if (COMP_NAME STREQUAL dice)
-  string(APPEND FFLAGS " -Mnovect")
-endif()
-if (COMP_NAME STREQUAL docn)
-  string(APPEND FFLAGS " -Mnovect")
-endif()
-set(FFLAGS_NOOPT "-O0")
-set(FIXEDFLAGS "-Mfixed")
-set(FREEFLAGS "-Mfree")
-set(HAS_F2008_CONTIGUOUS "FALSE")
-set(LDFLAGS "-time -Wl,--allow-multiple-definition")
-if (compile_threaded)
-  string(APPEND LDFLAGS " -mp")
-endif()
-set(MPICC "mpicc")
-set(MPICXX "mpicxx")
-set(MPIFC "mpif90")
-set(SCC "pgcc")
-set(SCXX "pgc++")
-set(SFC "pgf95")
diff --git a/machines/cmake_macros/pgi-gpu_casper.cmake b/machines/cmake_macros/pgi-gpu_casper.cmake
deleted file mode 100644
index 600521a1bb..0000000000
--- a/machines/cmake_macros/pgi-gpu_casper.cmake
+++ /dev/null
@@ -1,15 +0,0 @@
-if (NOT DEBUG)
-  string(APPEND CFLAGS " -O -tp=skylake -Mnofma")
-endif()
-if (NOT DEBUG)
-  string(APPEND FFLAGS " -O -tp=skylake -Mnofma")
-endif()
-string(APPEND FFLAGS " -I$(EXEROOT)/ocn/obj/FMS")
-if (NOT DEBUG)
-  string(APPEND LDFLAGS " -O -tp=skylake -Mnofma -acc -ta=tesla:cc70,lineinfo,nofma -Minfo=accel")
-endif()
-string(APPEND SLIBS " -llapack -lblas")
-if (MPILIB STREQUAL mpi-serial)
-  string(APPEND SLIBS " -ldl")
-endif()
-string(APPEND SLIBS " -L${NETCDF_PATH}/lib -lnetcdf -lnetcdff")
diff --git a/machines/config_batch.xml b/machines/config_batch.xml
index 4ea11977e5..9d93418d4f 100644
--- a/machines/config_batch.xml
+++ b/machines/config_batch.xml
@@ -226,31 +226,17 @@
   <!-- casper pbs -->
   <batch_system MACH="casper" type="pbs">
     <batch_submit>qsub</batch_submit>
-
-    <directives queue="casper" compiler="pgi-gpu">
-      <!-- Turn on MPS server manually -->
-      <!-- This is a temporary solution and should be removed once MPS is integrated into PBS on Casper -->
-      <directive default="/bin/bash" > -S /glade/u/apps/dav/opt/nvidia-mps/mps_bash </directive>
-      <directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=700GB:ngpus={{ ngpus_per_node }} </directive>
-      <directive> -l gpu_type=v100 </directive>
-    </directives>
-
-    <directives queue="casper" compiler="nvhpc-gpu">
-      <!-- Turn on MPS server manually -->
-      <!-- This is a temporary solution and should be removed once MPS is integrated into PBS on Casper -->
-      <directive default="/bin/bash" > -S /glade/u/apps/dav/opt/nvidia-mps/mps_bash </directive>
-      <directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=700GB:ngpus={{ ngpus_per_node }} </directive>
-      <directive> -l gpu_type=v100 </directive>
-    </directives>
-
+    <submit_args>
+      <argument> -l gpu_type=$GPU_TYPE </argument>
+    </submit_args>
     <directives queue="casper" compiler="nvhpc">
       <directive default="/bin/bash" > -S {{ shell }} </directive>
       <directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=700GB:ngpus={{ ngpus_per_node }} </directive>
     </directives>
 
-    <directives queue="casper" compiler="pgi">
+    <directives queue="casper" compiler="nvhpc" gpu_enabled="true">
       <directive default="/bin/bash" > -S {{ shell }} </directive>
-      <directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=700GB:ngpus={{ ngpus_per_node }} </directive>
+      <directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=700GB:ngpus={{ ngpus_per_node }}:mps=1 </directive>
     </directives>
 
     <directives queue="casper" compiler="intel">
diff --git a/machines/config_machines.xml b/machines/config_machines.xml
index ec188bcf62..f326c8d15b 100644
--- a/machines/config_machines.xml
+++ b/machines/config_machines.xml
@@ -410,18 +410,19 @@ This allows using a different mpirun command to launch unit tests
     <DESC>NCAR GPU platform, os is Linux, 36 pes/node, batch system is pbs</DESC>
     <NODENAME_REGEX>casper*</NODENAME_REGEX>
     <OS>LINUX</OS>
-    <COMPILERS>pgi,intel,nvhpc,pgi-gpu,nvhpc-gpu</COMPILERS>
+    <COMPILERS>nvhpc,intel</COMPILERS>
     <MPILIBS>openmpi</MPILIBS>
     <CIME_OUTPUT_ROOT>/glade/scratch/$USER</CIME_OUTPUT_ROOT>
     <DIN_LOC_ROOT>$ENV{CESMDATAROOT}/inputdata</DIN_LOC_ROOT>
     <DIN_LOC_ROOT_CLMFORC>/glade/p/cgd/tss/CTSM_datm_forcing_data</DIN_LOC_ROOT_CLMFORC>
     <DOUT_S_ROOT>$CIME_OUTPUT_ROOT/archive/$CASE</DOUT_S_ROOT>
     <BASELINE_ROOT>$ENV{CESMDATAROOT}/cesm_baselines</BASELINE_ROOT>
+    <CCSM_CPRNC>$ENV{CESMDATAROOT}/tools/cime/tools/cprnc/cprnc</CCSM_CPRNC>
     <GMAKE_J>8</GMAKE_J>
     <BATCH_SYSTEM>pbs</BATCH_SYSTEM>
     <SUPPORTED_BY>ASAP/CISL</SUPPORTED_BY>
     <MAX_TASKS_PER_NODE>36</MAX_TASKS_PER_NODE>
-    <MAX_GPUS_PER_NODE>8</MAX_GPUS_PER_NODE>
+    <MAX_GPUS_PER_NODE compiler="nvhpc">8</MAX_GPUS_PER_NODE>
     <MAX_MPITASKS_PER_NODE>36</MAX_MPITASKS_PER_NODE>
     <PROJECT_REQUIRED>TRUE</PROJECT_REQUIRED>
     <mpirun mpilib="default">
@@ -450,56 +451,30 @@ This allows using a different mpirun command to launch unit tests
         <command name="load">ncarenv/1.3</command>
         <command name="load">cmake/3.18.2</command>
       </modules>
-      <modules compiler="pgi">
-        <command name="load">pgi/20.4</command>
-      </modules>
-      <modules compiler="pgi-gpu">
-        <command name="load">pgi/20.4</command>
-      </modules>
       <modules compiler="nvhpc">
         <command name="load">nvhpc/22.2</command>
       </modules>
-      <modules compiler="nvhpc-gpu">
-        <command name="load">nvhpc/22.2</command>
-      </modules>
       <modules compiler="intel">
         <command name="load">intel/19.1.1</command>
         <command name="load">mkl/2020.0.1</command>
       </modules>
-      <modules mpilib="openmpi" compiler="pgi">
-        <command name="load">openmpi/4.1.0</command>
-        <command name="load">netcdf-mpi/4.8.0</command>
-        <command name="load">pnetcdf/1.12.2</command>
+      <modules mpilib="openmpi" compiler="nvhpc" gpu_offload="openacc">
+        <command name="load">cuda/11.6</command>
       </modules>
-      <modules mpilib="mpi-serial" compiler="pgi">
-        <command name="load">netcdf/4.8.0</command>
+      <modules mpilib="openmpi" compiler="nvhpc" gpu_offload="openmp">
+        <command name="load">cuda/11.6</command>
       </modules>
-      <modules mpilib="openmpi" compiler="pgi-gpu">
-        <command name="load">openmpi/4.1.0</command>
-        <command name="load">netcdf-mpi/4.7.4</command>
-        <command name="load">pnetcdf/1.12.2</command>
-        <command name="load">cuda/11.0.3</command>
-      </modules>
-      <modules mpilib="mpi-serial" compiler="pgi-gpu">
-        <command name="load">netcdf/4.7.4</command>
+      <modules mpilib="openmpi" compiler="nvhpc" gpu_offload="combined">
+        <command name="load">cuda/11.6</command>
       </modules>
       <modules mpilib="openmpi" compiler="nvhpc">
-        <command name="load">openmpi/4.1.1</command>
+        <command name="load">openmpi/4.1.4</command>
         <command name="load">netcdf-mpi/4.8.1</command>
-        <command name="load">pnetcdf/1.12.2</command>
+        <command name="load">pnetcdf/1.12.3</command>
       </modules>
       <modules mpilib="mpi-serial" compiler="nvhpc">
         <command name="load">netcdf/4.8.1</command>
       </modules>
-      <modules mpilib="openmpi" compiler="nvhpc-gpu">
-        <command name="load">openmpi/4.1.1</command>
-        <command name="load">netcdf-mpi/4.8.1</command>
-        <command name="load">pnetcdf/1.12.2</command>
-        <command name="load">cuda/11.4.0</command>
-      </modules>
-      <modules mpilib="mpi-serial" compiler="nvhpc-gpu">
-        <command name="load">netcdf/4.8.1</command>
-      </modules>
       <modules mpilib="openmpi" compiler="intel">
         <command name="load">openmpi/4.1.1</command>
         <command name="load">netcdf-mpi/4.8.1</command>
@@ -517,30 +492,22 @@ This allows using a different mpirun command to launch unit tests
         <command name="use">/glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/intel/19.1.1/</command>
         <command name="load">esmf-8.4.0b08_casper-ncdfio-openmpi-O</command>
       </modules>
-      <modules compiler="nvhpc-gpu" mpilib="openmpi" DEBUG="TRUE">
+      <modules compiler="nvhpc" mpilib="openmpi" DEBUG="TRUE">
         <command name="use">/glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/nvhpc/22.2/</command>
-        <command name="load">esmf-8.4.0b08_casper-ncdfio-openmpi-g</command>
+        <command name="load">esmf-8.4.1_casper-ncdfio-openmpi-g</command>
       </modules>
-      <modules compiler="nvhpc-gpu" mpilib="openmpi" DEBUG="FALSE">
+      <modules compiler="nvhpc" mpilib="openmpi" DEBUG="FALSE">
         <command name="use">/glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/nvhpc/22.2/</command>
-        <command name="load">esmf-8.4.0b08_casper-ncdfio-openmpi-O</command>
-      </modules>
-      <modules compiler="pgi" mpilib="openmpi" DEBUG="TRUE">
-        <command name="use">/glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/pgi/20.4/</command>
-        <command name="load">esmf-8.4.0b08_casper-ncdfio-openmpi-g</command>
-      </modules>
-      <modules compiler="pgi" mpilib="openmpi" DEBUG="FALSE">
-        <command name="use">/glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/pgi/20.4/</command>
-        <command name="load">esmf-8.2.0b11_casper-ncdfio-openmpi-O</command>
+        <command name="load">esmf-8.4.1_casper-ncdfio-openmpi-O</command>
       </modules>
       <modules>
         <command name="load">ncarcompilers/0.5.0</command>
       </modules>
-      <modules compiler="!pgi" DEBUG="FALSE" mpilib="openmpi">
-        <command name="load">pio/2.5.7</command>
+      <modules DEBUG="FALSE" mpilib="openmpi">
+        <command name="load">pio/2.5.10</command>
       </modules>
-      <modules compiler="!pgi" DEBUG="TRUE" mpilib="openmpi">
-        <command name="load">pio/2.5.7d</command>
+      <modules DEBUG="TRUE" mpilib="openmpi">
+        <command name="load">pio/2.5.10d</command>
       </modules>
     </module_system>
     <environment_variables>
@@ -577,7 +544,7 @@ This allows using a different mpirun command to launch unit tests
     <DIN_LOC_ROOT_CLMFORC>/glade/p/cgd/tss/CTSM_datm_forcing_data</DIN_LOC_ROOT_CLMFORC>
     <DOUT_S_ROOT>$CIME_OUTPUT_ROOT/archive/$CASE</DOUT_S_ROOT>
     <BASELINE_ROOT>$ENV{CESMDATAROOT}/cesm_baselines</BASELINE_ROOT>
-    <CCSM_CPRNC>$ENV{CESMDATAROOT}/tools/cime/tools/cprnc/cprnc.cheyenne</CCSM_CPRNC>
+    <CCSM_CPRNC>$ENV{CESMDATAROOT}/tools/cime/tools/cprnc/cprnc</CCSM_CPRNC>
     <GMAKE_J>8</GMAKE_J>
     <BATCH_SYSTEM>pbs</BATCH_SYSTEM>
     <SUPPORTED_BY>cseg</SUPPORTED_BY>
diff --git a/machines/mpi_run_gpu.casper b/machines/mpi_run_gpu.casper
deleted file mode 100755
index ade12d50ef..0000000000
--- a/machines/mpi_run_gpu.casper
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/bin/bash
-unset CUDA_VISIBLE_DEVICES
-let dev_id=$OMPI_COMM_WORLD_LOCAL_RANK%{{ ngpus_per_node }}
-export ACC_DEVICE_NUM=$dev_id
-export CUDA_VISIBLE_DEVICES=$dev_id
-exec $*

From 3b7a18255b1cb9c26d7c938d2a70b6f6eb07f5fe Mon Sep 17 00:00:00 2001
From: Jian Sun <sunjian@ucar.edu>
Date: Wed, 22 Mar 2023 22:57:38 -0600
Subject: [PATCH 03/24] Add max_cputasks_per_gpu_node, gpu_type and gpu_offload
 options for Casper and Gust

	modified:   machines/config_machines.xml
---
 machines/config_machines.xml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/machines/config_machines.xml b/machines/config_machines.xml
index f326c8d15b..1dee63e9df 100644
--- a/machines/config_machines.xml
+++ b/machines/config_machines.xml
@@ -424,6 +424,9 @@ This allows using a different mpirun command to launch unit tests
     <MAX_TASKS_PER_NODE>36</MAX_TASKS_PER_NODE>
     <MAX_GPUS_PER_NODE compiler="nvhpc">8</MAX_GPUS_PER_NODE>
     <MAX_MPITASKS_PER_NODE>36</MAX_MPITASKS_PER_NODE>
+    <MAX_CPUTASKS_PER_GPU_NODE>36</MAX_CPUTASKS_PER_GPU_NODE>
+    <GPU_TYPE>none,v100</GPU_TYPE>
+    <GPU_OFFLOAD>none,openacc,openmp,combined</GPU_TYPE>
     <PROJECT_REQUIRED>TRUE</PROJECT_REQUIRED>
     <mpirun mpilib="default">
       <executable>mpirun</executable>
@@ -1815,6 +1818,9 @@ This allows using a different mpirun command to launch unit tests
     <SUPPORTED_BY>cseg</SUPPORTED_BY>
     <MAX_TASKS_PER_NODE>128</MAX_TASKS_PER_NODE>
     <MAX_MPITASKS_PER_NODE>128</MAX_MPITASKS_PER_NODE>
+    <MAX_CPUTASKS_PER_GPU_NODE>64</MAX_CPUTASKS_PER_GPU_NODE>
+    <GPU_TYPE>none,a100</GPU_TYPE>
+    <GPU_OFFLOAD>none,openacc,openmp,combined</GPU_TYPE>
     <PROJECT_REQUIRED>TRUE</PROJECT_REQUIRED>
     <mpirun mpilib="default">
       <executable>mpiexec</executable>

From ee726b5d640950786b84497ba032781c3d009bc9 Mon Sep 17 00:00:00 2001
From: Jian Sun <sunjian@ucar.edu>
Date: Thu, 23 Mar 2023 15:08:39 -0600
Subject: [PATCH 04/24] update batch script template for Gust GPU node

	modified:   config_batch.xml
---
 machines/config_batch.xml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/machines/config_batch.xml b/machines/config_batch.xml
index 9d93418d4f..27c64bf0f1 100644
--- a/machines/config_batch.xml
+++ b/machines/config_batch.xml
@@ -476,10 +476,17 @@
 
   <batch_system MACH="gust" type="pbs" >
     <batch_submit>qsub</batch_submit>
+    <submit_args>
+      <argument> -l gpu_type=$GPU_TYPE </argument>
+    </submit_args>
     <directives>
       <directive default="/bin/bash" > -S {{ shell }}  </directive>
       <directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}</directive>
     </directives>
+    <directives queue="main" gpu_enabled="true">
+      <directive default="/bin/bash" > -S {{ shell }} </directive>
+      <directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=430GB:ngpus={{ ngpus_per_node }}:mps=1 </directive>
+    </directives>
     <queues>
       <queue walltimemax="4:00:00" nodemin="1" nodemax="8" >main</queue>
       <queue walltimemax="1:00:00" nodemin="9" nodemax="16" >bigcpu</queue>

From 1f5eb30b532ea1ac0f418f067f9b02ec80d2174d Mon Sep 17 00:00:00 2001
From: Jian Sun <sunjian@ucar.edu>
Date: Thu, 23 Mar 2023 15:15:42 -0600
Subject: [PATCH 05/24] typo fix for mismatched tags for GPU options

	modified:   machines/config_machines.xml
---
 machines/config_machines.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/machines/config_machines.xml b/machines/config_machines.xml
index 1dee63e9df..389de064fe 100644
--- a/machines/config_machines.xml
+++ b/machines/config_machines.xml
@@ -426,7 +426,7 @@ This allows using a different mpirun command to launch unit tests
     <MAX_MPITASKS_PER_NODE>36</MAX_MPITASKS_PER_NODE>
     <MAX_CPUTASKS_PER_GPU_NODE>36</MAX_CPUTASKS_PER_GPU_NODE>
     <GPU_TYPE>none,v100</GPU_TYPE>
-    <GPU_OFFLOAD>none,openacc,openmp,combined</GPU_TYPE>
+    <GPU_OFFLOAD>none,openacc,openmp,combined</GPU_OFFLOAD>
     <PROJECT_REQUIRED>TRUE</PROJECT_REQUIRED>
     <mpirun mpilib="default">
       <executable>mpirun</executable>
@@ -1820,7 +1820,7 @@ This allows using a different mpirun command to launch unit tests
     <MAX_MPITASKS_PER_NODE>128</MAX_MPITASKS_PER_NODE>
     <MAX_CPUTASKS_PER_GPU_NODE>64</MAX_CPUTASKS_PER_GPU_NODE>
     <GPU_TYPE>none,a100</GPU_TYPE>
-    <GPU_OFFLOAD>none,openacc,openmp,combined</GPU_TYPE>
+    <GPU_OFFLOAD>none,openacc,openmp,combined</GPU_OFFLOAD>
     <PROJECT_REQUIRED>TRUE</PROJECT_REQUIRED>
     <mpirun mpilib="default">
       <executable>mpiexec</executable>

From 8649a95a7daa2bd91fc3ea54e5b3ce5c7be19a67 Mon Sep 17 00:00:00 2001
From: Jian Sun <sunjian@ucar.edu>
Date: Thu, 23 Mar 2023 22:23:02 -0600
Subject: [PATCH 06/24] Add missing "MAX_GPUS_PER_NODE"

	modified:   machines/config_machines.xml
---
 machines/config_machines.xml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/machines/config_machines.xml b/machines/config_machines.xml
index 389de064fe..9c6ac5f495 100644
--- a/machines/config_machines.xml
+++ b/machines/config_machines.xml
@@ -1817,6 +1817,7 @@ This allows using a different mpirun command to launch unit tests
     <BATCH_SYSTEM>pbs</BATCH_SYSTEM>
     <SUPPORTED_BY>cseg</SUPPORTED_BY>
     <MAX_TASKS_PER_NODE>128</MAX_TASKS_PER_NODE>
+    <MAX_GPUS_PER_NODE>4</MAX_GPUS_PER_NODE>
     <MAX_MPITASKS_PER_NODE>128</MAX_MPITASKS_PER_NODE>
     <MAX_CPUTASKS_PER_GPU_NODE>64</MAX_CPUTASKS_PER_GPU_NODE>
     <GPU_TYPE>none,a100</GPU_TYPE>

From e7bfbe189bd268479a3cdc45cc77f22d6ac71abd Mon Sep 17 00:00:00 2001
From: Jian Sun <sunjian@ucar.edu>
Date: Fri, 24 Mar 2023 20:48:30 -0600
Subject: [PATCH 07/24] downgrade netcdf-mpi fpr nvhpc/22.11

	modified:   machines/config_machines.xml
---
 machines/config_machines.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/machines/config_machines.xml b/machines/config_machines.xml
index 9c6ac5f495..017916d5d8 100644
--- a/machines/config_machines.xml
+++ b/machines/config_machines.xml
@@ -1879,13 +1879,13 @@ This allows using a different mpirun command to launch unit tests
         <command name="load">mkl/2022.2.1</command>
       </modules>	
       <modules mpilib="mpi-serial">
-        <command name="load">netcdf/4.9.1</command>
+        <command name="load">netcdf/4.9.0</command>
       </modules>
 
       <modules mpilib="!mpi-serial">
 	<command name="rm">hdf5</command>
         <command name="rm">netcdf</command>
-        <command name="load">netcdf-mpi/4.9.1</command>
+        <command name="load">netcdf-mpi/4.9.0</command>
         <command name="load">parallel-netcdf/1.12.3</command>
       </modules>
       <modules DEBUG="TRUE">

From bf149677d4d1a8e0c994421ee9fc496a32993889 Mon Sep 17 00:00:00 2001
From: Jian Sun <sunjian@ucar.edu>
Date: Fri, 24 Mar 2023 22:59:33 -0600
Subject: [PATCH 08/24] bug fix of batch script template with gpu_enabled
 option

	modified:   machines/config_batch.xml
---
 machines/config_batch.xml | 12 +++---------
 1 file changed, 3 insertions(+), 9 deletions(-)

diff --git a/machines/config_batch.xml b/machines/config_batch.xml
index 27c64bf0f1..943c20863c 100644
--- a/machines/config_batch.xml
+++ b/machines/config_batch.xml
@@ -229,17 +229,11 @@
     <submit_args>
       <argument> -l gpu_type=$GPU_TYPE </argument>
     </submit_args>
-    <directives queue="casper" compiler="nvhpc">
-      <directive default="/bin/bash" > -S {{ shell }} </directive>
-      <directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=700GB:ngpus={{ ngpus_per_node }} </directive>
-    </directives>
-
     <directives queue="casper" compiler="nvhpc" gpu_enabled="true">
       <directive default="/bin/bash" > -S {{ shell }} </directive>
       <directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=700GB:ngpus={{ ngpus_per_node }}:mps=1 </directive>
     </directives>
-
-    <directives queue="casper" compiler="intel">
+    <directives queue="casper" gpu_enabled="false">
       <directive default="/bin/bash" > -S {{ shell }} </directive>
       <directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=700GB:ngpus={{ ngpus_per_node }} </directive>
     </directives>
@@ -479,13 +473,13 @@
     <submit_args>
       <argument> -l gpu_type=$GPU_TYPE </argument>
     </submit_args>
-    <directives>
+    <directives gpu_enabled="false">
       <directive default="/bin/bash" > -S {{ shell }}  </directive>
       <directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}</directive>
     </directives>
     <directives queue="main" gpu_enabled="true">
       <directive default="/bin/bash" > -S {{ shell }} </directive>
-      <directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=430GB:ngpus={{ ngpus_per_node }}:mps=1 </directive>
+      <directive> -l select={{ num_nodes }}:ncpus={{ max_cputasks_per_gpu_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=430GB:ngpus={{ ngpus_per_node }}:mps=1 </directive>
     </directives>
     <queues>
       <queue walltimemax="4:00:00" nodemin="1" nodemax="8" >main</queue>

From 14f605eae88cf1ccf000ef0dad3edf281f26d186 Mon Sep 17 00:00:00 2001
From: Jian Sun <sunjian@ucar.edu>
Date: Sat, 25 Mar 2023 00:01:26 -0600
Subject: [PATCH 09/24] remove -target=zen3 flag on gust load cuda module on
 Gust for a GPU run

	deleted:    cmake_macros/nvhpc_gust.cmake
	modified:   config_machines.xml
---
 machines/cmake_macros/nvhpc_gust.cmake | 3 ---
 machines/config_machines.xml           | 4 ++++
 2 files changed, 4 insertions(+), 3 deletions(-)
 delete mode 100644 machines/cmake_macros/nvhpc_gust.cmake

diff --git a/machines/cmake_macros/nvhpc_gust.cmake b/machines/cmake_macros/nvhpc_gust.cmake
deleted file mode 100644
index 461137315d..0000000000
--- a/machines/cmake_macros/nvhpc_gust.cmake
+++ /dev/null
@@ -1,3 +0,0 @@
-if (NOT DEBUG)
-  string(APPEND FFLAGS " -target=zen3")
-endif()
diff --git a/machines/config_machines.xml b/machines/config_machines.xml
index 017916d5d8..d0959a84e1 100644
--- a/machines/config_machines.xml
+++ b/machines/config_machines.xml
@@ -1875,6 +1875,10 @@ This allows using a different mpirun command to launch unit tests
         <command name="load">mkl/2022.2.1</command>
       </modules>
 
+      <modules gpu_type="!none">
+        <command name="load">cuda/11.7.1</command>
+      </modules>
+
       <modules compiler="oneapi">
         <command name="load">mkl/2022.2.1</command>
       </modules>	

From 59086afcd63f4a773a1cc374ad328e7403aeb947 Mon Sep 17 00:00:00 2001
From: Jian Sun <sunjian@ucar.edu>
Date: Sat, 25 Mar 2023 20:15:45 -0600
Subject: [PATCH 10/24] update to nvhpc/23.1 and related modules

	modified:   machines/config_machines.xml
---
 machines/config_machines.xml | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/machines/config_machines.xml b/machines/config_machines.xml
index d0959a84e1..81e8f2f2f2 100644
--- a/machines/config_machines.xml
+++ b/machines/config_machines.xml
@@ -1858,10 +1858,10 @@ This allows using a different mpirun command to launch unit tests
         <command name="load">gcc/12.1.0</command>
       </modules>
       <modules compiler="nvhpc">
-        <command name="load">nvhpc/22.11</command>
+        <command name="load">nvhpc/23.1</command>
       </modules>
       <modules mpilib="mpich">
-	<command name="load">cray-mpich/8.1.21</command>
+	<command name="load">cray-mpich/8.1.24</command>
       </modules>
       <modules mpilib="mpi-serial">
         <command name="load">mpi-serial/2.3.0</command>
@@ -1874,31 +1874,28 @@ This allows using a different mpirun command to launch unit tests
       <modules compiler="intel">
         <command name="load">mkl/2022.2.1</command>
       </modules>
-
       <modules gpu_type="!none">
         <command name="load">cuda/11.7.1</command>
       </modules>
-
       <modules compiler="oneapi">
         <command name="load">mkl/2022.2.1</command>
       </modules>	
       <modules mpilib="mpi-serial">
         <command name="load">netcdf/4.9.0</command>
       </modules>
-
       <modules mpilib="!mpi-serial">
 	<command name="rm">hdf5</command>
         <command name="rm">netcdf</command>
-        <command name="load">netcdf-mpi/4.9.0</command>
+        <command name="load">netcdf-mpi/4.9.1</command>
         <command name="load">parallel-netcdf/1.12.3</command>
       </modules>
       <modules DEBUG="TRUE">
 	<command name="load">parallelio/2.5.10-debug</command>
-	<command name="load">esmf/8.4.1b02-debug</command>
+	<command name="load">esmf/8.5.0b17-debug</command>
       </modules>
       <modules DEBUG="FALSE">
 	<command name="load">parallelio/2.5.10</command>
-	<command name="load">esmf/8.4.1b02</command>
+	<command name="load">esmf/8.5.0b17</command>
       </modules>
     </module_system>
 

From 918e3a8dd9617c29363969ec4e6c2f2fdea15d2c Mon Sep 17 00:00:00 2001
From: Jian Sun <sunjian@ucar.edu>
Date: Sun, 26 Mar 2023 13:28:06 -0600
Subject: [PATCH 11/24] turn off MPS on Gust

	modified:   config_batch.xml
---
 machines/config_batch.xml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/machines/config_batch.xml b/machines/config_batch.xml
index 943c20863c..2cabfab8eb 100644
--- a/machines/config_batch.xml
+++ b/machines/config_batch.xml
@@ -479,7 +479,9 @@
     </directives>
     <directives queue="main" gpu_enabled="true">
       <directive default="/bin/bash" > -S {{ shell }} </directive>
-      <directive> -l select={{ num_nodes }}:ncpus={{ max_cputasks_per_gpu_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=430GB:ngpus={{ ngpus_per_node }}:mps=1 </directive>
+      <directive> -l select={{ num_nodes }}:ncpus={{ max_cputasks_per_gpu_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=430GB:ngpus={{ ngpus_per_node }} </directive>
+      <!-- Do not use MPS on Gust yet as it conflicts with cray-mpich library -->
+      <!-- <directive> -l select={{ num_nodes }}:ncpus={{ max_cputasks_per_gpu_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=430GB:ngpus={{ ngpus_per_node }}:mps=1 </directive> -->
     </directives>
     <queues>
       <queue walltimemax="4:00:00" nodemin="1" nodemax="8" >main</queue>

From d0093a5611bcca461fdf5a08831241cf49bc2d92 Mon Sep 17 00:00:00 2001
From: Jian Sun <sunjian@ucar.edu>
Date: Tue, 11 Apr 2023 15:39:28 -0600
Subject: [PATCH 12/24] Update module versions for intel compiler

	modified:   machines/config_batch.xml
	modified:   machines/config_machines.xml
---
 machines/config_batch.xml    |  2 +-
 machines/config_machines.xml | 16 ++++++++--------
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/machines/config_batch.xml b/machines/config_batch.xml
index 2cabfab8eb..430b41d572 100644
--- a/machines/config_batch.xml
+++ b/machines/config_batch.xml
@@ -473,7 +473,7 @@
     <submit_args>
       <argument> -l gpu_type=$GPU_TYPE </argument>
     </submit_args>
-    <directives gpu_enabled="false">
+    <directives queue="main" gpu_enabled="false">
       <directive default="/bin/bash" > -S {{ shell }}  </directive>
       <directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}</directive>
     </directives>
diff --git a/machines/config_machines.xml b/machines/config_machines.xml
index 81e8f2f2f2..a5785addf0 100644
--- a/machines/config_machines.xml
+++ b/machines/config_machines.xml
@@ -1846,10 +1846,10 @@ This allows using a different mpirun command to launch unit tests
       </modules>
 
       <modules compiler="intel">
-        <command name="load">intel/2021.7.1</command>
+        <command name="load">intel/2023.0.0</command>
       </modules>
       <modules compiler="oneapi">
-        <command name="load">oneapi/2023.0.0</command>
+        <command name="load">intel-oneapi/2023.0.0</command>
       </modules>
       <modules compiler="cray">
         <command name="load">cce/15.0.1</command>
@@ -1861,7 +1861,7 @@ This allows using a different mpirun command to launch unit tests
         <command name="load">nvhpc/23.1</command>
       </modules>
       <modules mpilib="mpich">
-	<command name="load">cray-mpich/8.1.24</command>
+	<command name="load">cray-mpich/8.1.25</command>
       </modules>
       <modules mpilib="mpi-serial">
         <command name="load">mpi-serial/2.3.0</command>
@@ -1872,16 +1872,16 @@ This allows using a different mpirun command to launch unit tests
       <!-- to use cray-libsci instead of mkl comment these out and
 	   edit cmake_macros/gust.cmake -->
       <modules compiler="intel">
-        <command name="load">mkl/2022.2.1</command>
+        <command name="load">mkl/2023.0.0</command>
       </modules>
       <modules gpu_type="!none">
         <command name="load">cuda/11.7.1</command>
       </modules>
       <modules compiler="oneapi">
-        <command name="load">mkl/2022.2.1</command>
+        <command name="load">mkl/2023.0.0</command>
       </modules>	
       <modules mpilib="mpi-serial">
-        <command name="load">netcdf/4.9.0</command>
+        <command name="load">netcdf/4.9.1</command>
       </modules>
       <modules mpilib="!mpi-serial">
 	<command name="rm">hdf5</command>
@@ -1891,11 +1891,11 @@ This allows using a different mpirun command to launch unit tests
       </modules>
       <modules DEBUG="TRUE">
 	<command name="load">parallelio/2.5.10-debug</command>
-	<command name="load">esmf/8.5.0b17-debug</command>
+	<command name="load">esmf/8.5.0b19-debug</command>
       </modules>
       <modules DEBUG="FALSE">
 	<command name="load">parallelio/2.5.10</command>
-	<command name="load">esmf/8.5.0b17</command>
+	<command name="load">esmf/8.5.0b19</command>
       </modules>
     </module_system>
 

From 7224a3757a86d32e8a24998ca529db6cce721689 Mon Sep 17 00:00:00 2001
From: Jian Sun <sunjian@ucar.edu>
Date: Tue, 18 Apr 2023 13:10:06 -0600
Subject: [PATCH 13/24] load ncarenv and ncarcompilers manually

	modified:   config_machines.xml
---
 machines/config_machines.xml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/machines/config_machines.xml b/machines/config_machines.xml
index 26a3948fab..eda8c6cab1 100644
--- a/machines/config_machines.xml
+++ b/machines/config_machines.xml
@@ -1844,7 +1844,9 @@ This allows using a different mpirun command to launch unit tests
 	<command name="use">/opt/cray/pe/craype-targets/default/modulefiles</command>
         <command name="load">cmake</command>
       </modules>
-
+      <modules>
+        <command name="load">ncarenv/23.03</command>
+      </modules>
       <modules compiler="intel">
         <command name="load">intel/2023.0.0</command>
       </modules>
@@ -1897,6 +1899,9 @@ This allows using a different mpirun command to launch unit tests
 	<command name="load">parallelio/2.5.10</command>
 	<command name="load">esmf/8.5.0b19</command>
       </modules>
+      <modules>
+        <command name="load">ncarcompilers/0.8.0</command>
+      </modules>
     </module_system>
 
     <environment_variables>

From 822f7b71191052eb6f592cafad4157b621b82c5f Mon Sep 17 00:00:00 2001
From: Jian Sun <sunjian@ucar.edu>
Date: Tue, 25 Apr 2023 15:18:33 -0600
Subject: [PATCH 14/24] update the ncarenv and cesmdev module on Gust

	modified:   machines/config_machines.xml
---
 machines/config_machines.xml | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/machines/config_machines.xml b/machines/config_machines.xml
index eda8c6cab1..98dfead3fa 100644
--- a/machines/config_machines.xml
+++ b/machines/config_machines.xml
@@ -1840,12 +1840,10 @@ This allows using a different mpirun command to launch unit tests
       <cmd_path lang="sh">module</cmd_path>
       <cmd_path lang="csh">module</cmd_path>
       <modules>
-        <command name="reset"/>
-	<command name="use">/opt/cray/pe/craype-targets/default/modulefiles</command>
-        <command name="load">cmake</command>
-      </modules>
-      <modules>
-        <command name="load">ncarenv/23.03</command>
+        <command name="purge"/>
+        <command name="load">cesmdev/1.0</command>
+        <command name="load">ncarenv/23.04</command>
+        <command name="load">cmake/3.25.2</command>
       </modules>
       <modules compiler="intel">
         <command name="load">intel/2023.0.0</command>

From 3287d575685dd3359e8b1a7b35d110df11e87240 Mon Sep 17 00:00:00 2001
From: Jian Sun <sunjian@ucar.edu>
Date: Fri, 4 Aug 2023 22:39:05 -0600
Subject: [PATCH 15/24] remove unused files

---
 machines/Depends.nvhpc-gpu | 94 --------------------------------------
 machines/Depends.pgi-gpu   | 16 -------
 2 files changed, 110 deletions(-)
 delete mode 100644 machines/Depends.nvhpc-gpu
 delete mode 100644 machines/Depends.pgi-gpu

diff --git a/machines/Depends.nvhpc-gpu b/machines/Depends.nvhpc-gpu
deleted file mode 100644
index f8eaf17bba..0000000000
--- a/machines/Depends.nvhpc-gpu
+++ /dev/null
@@ -1,94 +0,0 @@
-# 10/13/2022 nvhpc compiler on gust.hpc.ucar.edu produced incorrect mpitask cound when esm.F90 was optimized
-# 04/05/2023 do not apply optimization flags to "SHR_RANDNUM_C_OBJS" for the nvhpc compiler, which will fail the ensemble consistency test
-#
-# Apply a different optimization level consistent with the Intel compiler
-PERFOBJS=\
-prim_advection_mod.o \
-edge_mod.o \
-derivative_mod.o \
-bndry_mod.o \
-prim_advance_mod.o
-
-REDUCED_OPT_OBJS=\
-SatellitePhenologyMod.o \
-esm.o
-
-REDUCED_PRECISION_OBJS=\
-shr_wv_sat_mod.o
-
-SHR_RANDNUM_FORT_OBJS=\
-kissvec_mod.o \
-mersennetwister_mod.o \
-dSFMT_interface.o \
-shr_RandNum_mod.o
-
-SHR_RANDNUM_C_OBJS=\
-dSFMT.o \
-dSFMT_utils.o \
-kissvec.o
-
-# Apply GPU flags to the following code
-PUMAS_OBJS=\
-micro_mg1_0.o \
-micro_pumas_v1.o \
-micro_pumas_data.o \
-micro_pumas_utils.o \
-micro_pumas_cam.o \
-wv_sat_methods.o \
-wv_saturation.o \
-macrop_driver.o \
-shr_spfn_mod.o
-
-RRTMGP_OBJS=\
-mo_rte_util_array.o \
-mo_rte_sw.o \
-mo_rte_lw.o \
-mo_cloud_optics.o \
-mo_gas_optics_rrtmgp.o \
-mo_gas_concentrations.o \
-mo_fluxes_broadband_kernels.o \
-mo_fluxes_byband_kernels.o \
-mo_rrtmgp_util_reorder_kernels.o \
-mo_compute_bc.o\
-mo_fluxes_byband.o\
-mo_fluxes_bygpoint.o\
-mo_heating_rates.o\
-mo_rrtmgp_clr_all_sky.o\
-mo_cloud_sampling.o\
-mo_solar_variability.o\
-mo_gas_optics.o\
-mo_rrtmgp_constants.o\
-mo_rrtmgp_util_reorder.o\
-mo_rrtmgp_util_string.o\
-mo_fluxes.o\
-mo_optical_props.o\
-mo_rte_config.o\
-mo_rte_kind.o\
-mo_source_functions.o\
-mo_optical_props_kernels.o\
-mo_rte_solver_kernels.o\
-mo_optical_props_kernels.o\
-mo_rte_solver_kernels.o\
-rrtmgp_driver.o
-
-RRTMGP_NOOPT=\
-mo_gas_optics_kernels.o
-
-ifeq ($(DEBUG),FALSE)
-  $(PERFOBJS): %.o: %.F90
-	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O3 -Mfprelaxed=div $<
-  $(REDUCED_OPT_OBJS): %.o: %.F90
-	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O1 $<
-  $(REDUCED_PRECISION_OBJS): %.o: %.F90
-	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -fast $<
-  $(SHR_RANDNUM_FORT_OBJS): %.o: %.F90
-	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O3 -fast -Mfprelaxed=div,sqrt $<
-#  $(SHR_RANDNUM_C_OBJS): %.o: %.c
-#	  $(CC) -c $(INCLDIR) $(INCS) $(CFLAGS) -O3 -fast $<
-  $(PUMAS_OBJS): %.o: %.F90
-	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O3 -fastsse -Mnofma -Mflushz -Mfprelaxed=sqrt -acc -gpu=cc70,lineinfo,nofma,flushz,fastmath -Minfo=accel $<
-  $(RRTMGP_OBJS): %.o: %.F90
-	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -acc -gpu=cc70,lineinfo,nofma -Minfo=accel -Mnoopenmp $<
-  $(RRTMGP_NOOPT): %.o: %.F90
-	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -acc -gpu=cc70,lineinfo,nofma -Minfo=accel -O0 $<
-endif
diff --git a/machines/Depends.pgi-gpu b/machines/Depends.pgi-gpu
deleted file mode 100644
index a4bb6df226..0000000000
--- a/machines/Depends.pgi-gpu
+++ /dev/null
@@ -1,16 +0,0 @@
-#
-PUMAS_MG_OBJS=\
-micro_mg1_0.o \
-micro_pumas_v1.o \
-micro_pumas_data.o \
-micro_pumas_utils.o \
-micro_pumas_cam.o \
-wv_sat_methods.o \
-wv_saturation.o \
-macrop_driver.o \
-shr_spfn_mod.o
-
-ifeq ($(DEBUG),FALSE)
-  $(PUMAS_MG_OBJS): %.o: %.F90
-	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -acc -ta=tesla:cc70,lineinfo,nofma -Minfo=accel $<
-endif

From fa7a978858f602f0aba39991a72c45579cc8336e Mon Sep 17 00:00:00 2001
From: Jian Sun <sunjian@ucar.edu>
Date: Fri, 4 Aug 2023 22:54:49 -0600
Subject: [PATCH 16/24] resolve conflicts and add pbs info for derecho

---
 machines/config_batch.xml    | 11 ++++++-
 machines/config_machines.xml | 59 +++---------------------------------
 2 files changed, 15 insertions(+), 55 deletions(-)

diff --git a/machines/config_batch.xml b/machines/config_batch.xml
index 28f321a9c2..f40caac8d0 100644
--- a/machines/config_batch.xml
+++ b/machines/config_batch.xml
@@ -358,10 +358,19 @@
 
   <batch_system MACH="derecho" type="pbs" >
     <batch_submit>qsub</batch_submit>
-    <directives>
+    <submit_args>
+      <argument> -l gpu_type=$GPU_TYPE </argument>
+    </submit_args>
+    <directives gpu_enabled="false">
       <directive default="/bin/bash" > -S {{ shell }}  </directive>
       <directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}</directive>
     </directives>
+    <directives gpu_enabled="true">
+      <directive default="/bin/bash" > -S {{ shell }} </directive>
+      <directive> -l select={{ num_nodes }}:ncpus={{ max_cputasks_per_gpu_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=430GB:ngpus={{ ngpus_per_node }} </directive>
+      <!-- Do not use MPS on Gust yet as it conflicts with cray-mpich library -->
+      <!-- <directive> -l select={{ num_nodes }}:ncpus={{ max_cputasks_per_gpu_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=430GB:ngpus={{ ngpus_per_node }}:mps=1 </directive> -->
+    </directives>
     <queues>
       <queue walltimemax="4:00:00" nodemin="1" nodemax="2488" >main</queue>
     </queues>
diff --git a/machines/config_machines.xml b/machines/config_machines.xml
index e6bfd54f92..58e5b6576f 100644
--- a/machines/config_machines.xml
+++ b/machines/config_machines.xml
@@ -1272,7 +1272,11 @@ This allows using a different mpirun command to launch unit tests
     <BATCH_SYSTEM>pbs</BATCH_SYSTEM>
     <SUPPORTED_BY>cseg</SUPPORTED_BY>
     <MAX_TASKS_PER_NODE>128</MAX_TASKS_PER_NODE>
+    <MAX_GPUS_PER_NODE>4</MAX_GPUS_PER_NODE>
     <MAX_MPITASKS_PER_NODE>128</MAX_MPITASKS_PER_NODE>
+    <MAX_CPUTASKS_PER_GPU_NODE>64</MAX_CPUTASKS_PER_GPU_NODE>
+    <GPU_TYPE>none,a100</GPU_TYPE>
+    <GPU_OFFLOAD>none,openacc,openmp,combined</GPU_OFFLOAD>
     <PROJECT_REQUIRED>TRUE</PROJECT_REQUIRED>
     <mpirun mpilib="default">
       <executable>mpiexec</executable>
@@ -1958,21 +1962,6 @@ This allows using a different mpirun command to launch unit tests
       <cmd_path lang="sh">module</cmd_path>
       <cmd_path lang="csh">module</cmd_path>
       <modules>
-<<<<<<< HEAD
-        <command name="purge"/>
-        <command name="load">cesmdev/1.0</command>
-        <command name="load">ncarenv/23.04</command>
-        <command name="load">cmake/3.25.2</command>
-      </modules>
-      <modules compiler="intel">
-        <command name="load">intel/2023.0.0</command>
-      </modules>
-      <modules compiler="oneapi">
-        <command name="load">intel-oneapi/2023.0.0</command>
-      </modules>
-      <modules compiler="cray">
-        <command name="load">cce/15.0.1</command>
-=======
 	<command name="load">cesmdev/1.0</command>
 	<command name="load">ncarenv/23.04</command>
         <command name="purge"/>
@@ -1993,7 +1982,6 @@ This allows using a different mpirun command to launch unit tests
       <modules compiler="cray">
         <command name="load">cce/15.0.1</command>
         <command name="load">cray-libsci/23.02.1.1</command>
->>>>>>> main
       </modules>
       <modules compiler="gnu">
         <command name="load">gcc/12.2.0</command>
@@ -2002,58 +1990,22 @@ This allows using a different mpirun command to launch unit tests
       <modules compiler="nvhpc">
         <command name="load">nvhpc/23.1</command>
       </modules>
-<<<<<<< HEAD
-=======
       <modules>
 	<command name="load">ncarcompilers/0.8.0</command>
         <command name="load">cmake</command>
       </modules>
->>>>>>> main
       <modules mpilib="mpich">
 	<command name="load">cray-mpich/8.1.25</command>
       </modules>
       <modules mpilib="mpi-serial">
         <command name="load">mpi-serial/2.3.0</command>
       </modules>
-<<<<<<< HEAD
-      <modules compiler="!intel">
-        <command name="load">cray-libsci/23.02.1.1</command>
-      </modules>
-      <!-- to use cray-libsci instead of mkl comment these out and
-	   edit cmake_macros/gust.cmake -->
-      <modules compiler="intel">
-        <command name="load">mkl/2023.0.0</command>
-      </modules>
-      <modules gpu_type="!none">
-        <command name="load">cuda/11.7.1</command>
-      </modules>
-      <modules compiler="oneapi">
-        <command name="load">mkl/2023.0.0</command>
-      </modules>	
-=======
 
->>>>>>> main
       <modules mpilib="mpi-serial">
         <command name="load">netcdf/4.9.1</command>
       </modules>
+
       <modules mpilib="!mpi-serial">
-<<<<<<< HEAD
-	<command name="rm">hdf5</command>
-        <command name="rm">netcdf</command>
-        <command name="load">netcdf-mpi/4.9.1</command>
-        <command name="load">parallel-netcdf/1.12.3</command>
-      </modules>
-      <modules DEBUG="TRUE">
-	<command name="load">parallelio/2.5.10-debug</command>
-	<command name="load">esmf/8.5.0b19-debug</command>
-      </modules>
-      <modules DEBUG="FALSE">
-	<command name="load">parallelio/2.5.10</command>
-	<command name="load">esmf/8.5.0b19</command>
-      </modules>
-      <modules>
-        <command name="load">ncarcompilers/0.8.0</command>
-=======
         <command name="load">netcdf-mpi/4.9.1</command>
         <command name="load">parallel-netcdf/1.12.3</command>
       </modules>
@@ -2066,7 +2018,6 @@ This allows using a different mpirun command to launch unit tests
       <modules DEBUG="FALSE">
 	<command name="load">parallelio/2.5.10</command>
 	<command name="load">esmf/8.5.0b21</command>
->>>>>>> main
       </modules>
     </module_system>
 

From 7215ebb98c39afa0747ca912554d238a2a0fc459 Mon Sep 17 00:00:00 2001
From: Jian Sun <sunjian@ucar.edu>
Date: Fri, 4 Aug 2023 22:58:33 -0600
Subject: [PATCH 17/24] minor update

---
 machines/cmake_macros/nvhpc_derecho.cmake | 1 +
 1 file changed, 1 insertion(+)

diff --git a/machines/cmake_macros/nvhpc_derecho.cmake b/machines/cmake_macros/nvhpc_derecho.cmake
index b7fdbd6998..ab1d0d572e 100644
--- a/machines/cmake_macros/nvhpc_derecho.cmake
+++ b/machines/cmake_macros/nvhpc_derecho.cmake
@@ -4,3 +4,4 @@ endif()
 if (NOT DEBUG)
   string(APPEND FFLAGS " -target=zen3")
 endif()
+message("GPU_TYPE is ${GPU_TYPE} GPU_OFFLOAD is ${GPU_OFFLOAD}")

From 3eb4cdacd69fcb260297ab4cbd379c0e4ce3c009 Mon Sep 17 00:00:00 2001
From: Jian Sun <sunjian@ucar.edu>
Date: Sun, 6 Aug 2023 22:28:16 -0600
Subject: [PATCH 18/24] update GPU settings for Derecho

---
 machines/config_batch.xml    |  4 ++--
 machines/config_machines.xml | 10 +++++++++-
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/machines/config_batch.xml b/machines/config_batch.xml
index f40caac8d0..39d5ca3a3b 100644
--- a/machines/config_batch.xml
+++ b/machines/config_batch.xml
@@ -363,11 +363,11 @@
     </submit_args>
     <directives gpu_enabled="false">
       <directive default="/bin/bash" > -S {{ shell }}  </directive>
-      <directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}</directive>
+      <directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=240GB</directive>
     </directives>
     <directives gpu_enabled="true">
       <directive default="/bin/bash" > -S {{ shell }} </directive>
-      <directive> -l select={{ num_nodes }}:ncpus={{ max_cputasks_per_gpu_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=430GB:ngpus={{ ngpus_per_node }} </directive>
+      <directive> -l select={{ num_nodes }}:ncpus={{ max_cputasks_per_gpu_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=490GB:ngpus={{ ngpus_per_node }} </directive>
       <!-- Do not use MPS on Gust yet as it conflicts with cray-mpich library -->
       <!-- <directive> -l select={{ num_nodes }}:ncpus={{ max_cputasks_per_gpu_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=430GB:ngpus={{ ngpus_per_node }}:mps=1 </directive> -->
     </directives>
diff --git a/machines/config_machines.xml b/machines/config_machines.xml
index 58e5b6576f..fa6f68124d 100644
--- a/machines/config_machines.xml
+++ b/machines/config_machines.xml
@@ -1324,6 +1324,7 @@ This allows using a different mpirun command to launch unit tests
       <modules compiler="nvhpc">
         <command name="load">nvhpc/23.1</command>
       </modules>
+
       <modules>
 	<command name="load">ncarcompilers/1.0.0</command>
         <command name="load">cmake</command>
@@ -1335,6 +1336,10 @@ This allows using a different mpirun command to launch unit tests
         <command name="load">mpi-serial/2.3.0</command>
       </modules>
 
+      <modules mpilib="mpich" compiler="nvhpc" gpu_offload="!none">
+        <command name="load">cuda/11.7.1</command>
+      </modules>
+
       <modules mpilib="mpi-serial">
         <command name="load">netcdf/4.9.2</command>
       </modules>
@@ -1352,7 +1357,7 @@ This allows using a different mpirun command to launch unit tests
       <modules DEBUG="FALSE">
 	<command name="load">parallelio/2.6.0</command>
 	<command name="load">esmf/8.5.0b23</command>
-        </modules>
+      </modules>
     </module_system>
 
     <environment_variables>
@@ -1371,6 +1376,9 @@ This allows using a different mpirun command to launch unit tests
       <env name="ESMF_RUNTIME_PROFILE">ON</env>
       <env name="ESMF_RUNTIME_PROFILE_OUTPUT">SUMMARY</env>
     </environment_variables>
+    <environment_variables compiler="nvhpc" gpu_offload="!none">
+      <env name="RTE_KERNELS">openacc</env>
+    </environment_variables>
   </machine>
 
   <machine MACH="eastwind">

From 9e77b246cb365a0a2344a9c709bb480753cef6b9 Mon Sep 17 00:00:00 2001
From: Jian Sun <sunjian@ucar.edu>
Date: Sun, 6 Aug 2023 23:39:17 -0600
Subject: [PATCH 19/24] remove unused flag

---
 machines/config_batch.xml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/machines/config_batch.xml b/machines/config_batch.xml
index 39d5ca3a3b..0b0508c437 100644
--- a/machines/config_batch.xml
+++ b/machines/config_batch.xml
@@ -358,9 +358,6 @@
 
   <batch_system MACH="derecho" type="pbs" >
     <batch_submit>qsub</batch_submit>
-    <submit_args>
-      <argument> -l gpu_type=$GPU_TYPE </argument>
-    </submit_args>
     <directives gpu_enabled="false">
       <directive default="/bin/bash" > -S {{ shell }}  </directive>
       <directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=240GB</directive>

From 75c3bfb854e7eff1ab0b0f1cb99864b8f04b0a67 Mon Sep 17 00:00:00 2001
From: Jian Sun <sunjian@ucar.edu>
Date: Mon, 7 Aug 2023 11:23:14 -0600
Subject: [PATCH 20/24] remove PBS -V option

---
 machines/config_batch.xml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/machines/config_batch.xml b/machines/config_batch.xml
index 0b0508c437..b5efa5602d 100644
--- a/machines/config_batch.xml
+++ b/machines/config_batch.xml
@@ -156,7 +156,6 @@
       <directive default="n"> -r {{ rerunnable }} </directive>
       <!-- <directive> -j oe {{ job_id }} </directive> -->
       <directive> -j oe </directive>
-      <directive> -V </directive>
     </directives>
   </batch_system>
 

From d1b8e681a8ef14d3eeeeba040b35c3b9f77caa06 Mon Sep 17 00:00:00 2001
From: Jian Sun <sunjian@ucar.edu>
Date: Tue, 8 Aug 2023 09:34:49 -0600
Subject: [PATCH 21/24] Add "get_local_rank" as an attribute

---
 machines/config_machines.xml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/machines/config_machines.xml b/machines/config_machines.xml
index fa6f68124d..76d3538837 100644
--- a/machines/config_machines.xml
+++ b/machines/config_machines.xml
@@ -1277,6 +1277,7 @@ This allows using a different mpirun command to launch unit tests
     <MAX_CPUTASKS_PER_GPU_NODE>64</MAX_CPUTASKS_PER_GPU_NODE>
     <GPU_TYPE>none,a100</GPU_TYPE>
     <GPU_OFFLOAD>none,openacc,openmp,combined</GPU_OFFLOAD>
+    <MPI_GPU_WRAPPER_SCRIPT>get_local_rank</MPI_GPU_WRAPPER_SCRIPT>
     <PROJECT_REQUIRED>TRUE</PROJECT_REQUIRED>
     <mpirun mpilib="default">
       <executable>mpiexec</executable>

From d5da373b99d7709c6bcc4a1292a8d351538d8d1f Mon Sep 17 00:00:00 2001
From: Jian Sun <sjsprecious@gmail.com>
Date: Wed, 9 Aug 2023 09:17:53 -0600
Subject: [PATCH 22/24] Update memory request on Derecho

---
 machines/config_batch.xml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/machines/config_batch.xml b/machines/config_batch.xml
index b5efa5602d..1712283825 100644
--- a/machines/config_batch.xml
+++ b/machines/config_batch.xml
@@ -359,11 +359,11 @@
     <batch_submit>qsub</batch_submit>
     <directives gpu_enabled="false">
       <directive default="/bin/bash" > -S {{ shell }}  </directive>
-      <directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=240GB</directive>
+      <directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=230GB</directive>
     </directives>
     <directives gpu_enabled="true">
       <directive default="/bin/bash" > -S {{ shell }} </directive>
-      <directive> -l select={{ num_nodes }}:ncpus={{ max_cputasks_per_gpu_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=490GB:ngpus={{ ngpus_per_node }} </directive>
+      <directive> -l select={{ num_nodes }}:ncpus={{ max_cputasks_per_gpu_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=480GB:ngpus={{ ngpus_per_node }} </directive>
       <!-- Do not use MPS on Gust yet as it conflicts with cray-mpich library -->
       <!-- <directive> -l select={{ num_nodes }}:ncpus={{ max_cputasks_per_gpu_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=430GB:ngpus={{ ngpus_per_node }}:mps=1 </directive> -->
     </directives>

From 8241e8c0167fab28a836062e6447cf04a7b5277f Mon Sep 17 00:00:00 2001
From: Jian Sun <sjsprecious@gmail.com>
Date: Wed, 9 Aug 2023 10:28:24 -0600
Subject: [PATCH 23/24] Turn on MPS for Derecho

---
 machines/config_batch.xml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/machines/config_batch.xml b/machines/config_batch.xml
index 1712283825..fef0ba1fd6 100644
--- a/machines/config_batch.xml
+++ b/machines/config_batch.xml
@@ -363,9 +363,7 @@
     </directives>
     <directives gpu_enabled="true">
       <directive default="/bin/bash" > -S {{ shell }} </directive>
-      <directive> -l select={{ num_nodes }}:ncpus={{ max_cputasks_per_gpu_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=480GB:ngpus={{ ngpus_per_node }} </directive>
-      <!-- Do not use MPS on Gust yet as it conflicts with cray-mpich library -->
-      <!-- <directive> -l select={{ num_nodes }}:ncpus={{ max_cputasks_per_gpu_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=430GB:ngpus={{ ngpus_per_node }}:mps=1 </directive> -->
+      <directive> -l select={{ num_nodes }}:ncpus={{ max_cputasks_per_gpu_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=480GB:ngpus={{ ngpus_per_node }}:mps=1 </directive>
     </directives>
     <queues>
       <queue walltimemax="4:00:00" nodemin="1" nodemax="2488" >main</queue>

From 560c043bee42c48429fd5b25d1e9f6593dc9649a Mon Sep 17 00:00:00 2001
From: Jian Sun <sjsprecious@gmail.com>
Date: Wed, 9 Aug 2023 12:00:56 -0600
Subject: [PATCH 24/24] remove duplicated lines

---
 machines/Depends.nvhpc | 2 --
 1 file changed, 2 deletions(-)

diff --git a/machines/Depends.nvhpc b/machines/Depends.nvhpc
index 15675846f4..b51e3c901b 100644
--- a/machines/Depends.nvhpc
+++ b/machines/Depends.nvhpc
@@ -67,8 +67,6 @@ mo_rte_kind.o\
 mo_source_functions.o\
 mo_optical_props_kernels.o\
 mo_rte_solver_kernels.o\
-mo_optical_props_kernels.o\
-mo_rte_solver_kernels.o\
 rrtmgp_driver.o\
 mo_gas_optics_kernels.o