diff --git a/machines/Depends.nvhpc b/machines/Depends.nvhpc
index 7b190cc09a..b51e3c901b 100644
--- a/machines/Depends.nvhpc
+++ b/machines/Depends.nvhpc
@@ -67,11 +67,7 @@ mo_rte_kind.o\
mo_source_functions.o\
mo_optical_props_kernels.o\
mo_rte_solver_kernels.o\
-mo_optical_props_kernels.o\
-mo_rte_solver_kernels.o\
-rrtmgp_driver.o
-
-RRTMGP_NOOPT=\
+rrtmgp_driver.o\
mo_gas_optics_kernels.o
ifeq ($(DEBUG),FALSE)
@@ -86,9 +82,7 @@ ifeq ($(DEBUG),FALSE)
# $(SHR_RANDNUM_C_OBJS): %.o: %.c
# $(CC) -c $(INCLDIR) $(INCS) $(CFLAGS) -O3 -fast $<
$(PUMAS_OBJS): %.o: %.F90
- $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O3 -fastsse -Mnofma -Mflushz -Mfprelaxed=sqrt $<
+ $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O3 -fastsse -Mnofma -Mflushz -Mfprelaxed=sqrt $(GPUFLAGS) $<
$(RRTMGP_OBJS): %.o: %.F90
- $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -Mnoopenmp $<
- $(RRTMGP_NOOPT): %.o: %.F90
- $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O0 $<
+ $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) $(GPUFLAGS) $<
endif
diff --git a/machines/Depends.nvhpc-gpu b/machines/Depends.nvhpc-gpu
deleted file mode 100644
index f8eaf17bba..0000000000
--- a/machines/Depends.nvhpc-gpu
+++ /dev/null
@@ -1,94 +0,0 @@
-# 10/13/2022 nvhpc compiler on gust.hpc.ucar.edu produced incorrect mpitask cound when esm.F90 was optimized
-# 04/05/2023 do not apply optimization flags to "SHR_RANDNUM_C_OBJS" for the nvhpc compiler, which will fail the ensemble consistency test
-#
-# Apply a different optimization level consistent with the Intel compiler
-PERFOBJS=\
-prim_advection_mod.o \
-edge_mod.o \
-derivative_mod.o \
-bndry_mod.o \
-prim_advance_mod.o
-
-REDUCED_OPT_OBJS=\
-SatellitePhenologyMod.o \
-esm.o
-
-REDUCED_PRECISION_OBJS=\
-shr_wv_sat_mod.o
-
-SHR_RANDNUM_FORT_OBJS=\
-kissvec_mod.o \
-mersennetwister_mod.o \
-dSFMT_interface.o \
-shr_RandNum_mod.o
-
-SHR_RANDNUM_C_OBJS=\
-dSFMT.o \
-dSFMT_utils.o \
-kissvec.o
-
-# Apply GPU flags to the following code
-PUMAS_OBJS=\
-micro_mg1_0.o \
-micro_pumas_v1.o \
-micro_pumas_data.o \
-micro_pumas_utils.o \
-micro_pumas_cam.o \
-wv_sat_methods.o \
-wv_saturation.o \
-macrop_driver.o \
-shr_spfn_mod.o
-
-RRTMGP_OBJS=\
-mo_rte_util_array.o \
-mo_rte_sw.o \
-mo_rte_lw.o \
-mo_cloud_optics.o \
-mo_gas_optics_rrtmgp.o \
-mo_gas_concentrations.o \
-mo_fluxes_broadband_kernels.o \
-mo_fluxes_byband_kernels.o \
-mo_rrtmgp_util_reorder_kernels.o \
-mo_compute_bc.o\
-mo_fluxes_byband.o\
-mo_fluxes_bygpoint.o\
-mo_heating_rates.o\
-mo_rrtmgp_clr_all_sky.o\
-mo_cloud_sampling.o\
-mo_solar_variability.o\
-mo_gas_optics.o\
-mo_rrtmgp_constants.o\
-mo_rrtmgp_util_reorder.o\
-mo_rrtmgp_util_string.o\
-mo_fluxes.o\
-mo_optical_props.o\
-mo_rte_config.o\
-mo_rte_kind.o\
-mo_source_functions.o\
-mo_optical_props_kernels.o\
-mo_rte_solver_kernels.o\
-mo_optical_props_kernels.o\
-mo_rte_solver_kernels.o\
-rrtmgp_driver.o
-
-RRTMGP_NOOPT=\
-mo_gas_optics_kernels.o
-
-ifeq ($(DEBUG),FALSE)
- $(PERFOBJS): %.o: %.F90
- $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O3 -Mfprelaxed=div $<
- $(REDUCED_OPT_OBJS): %.o: %.F90
- $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O1 $<
- $(REDUCED_PRECISION_OBJS): %.o: %.F90
- $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -fast $<
- $(SHR_RANDNUM_FORT_OBJS): %.o: %.F90
- $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O3 -fast -Mfprelaxed=div,sqrt $<
-# $(SHR_RANDNUM_C_OBJS): %.o: %.c
-# $(CC) -c $(INCLDIR) $(INCS) $(CFLAGS) -O3 -fast $<
- $(PUMAS_OBJS): %.o: %.F90
- $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O3 -fastsse -Mnofma -Mflushz -Mfprelaxed=sqrt -acc -gpu=cc70,lineinfo,nofma,flushz,fastmath -Minfo=accel $<
- $(RRTMGP_OBJS): %.o: %.F90
- $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -acc -gpu=cc70,lineinfo,nofma -Minfo=accel -Mnoopenmp $<
- $(RRTMGP_NOOPT): %.o: %.F90
- $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -acc -gpu=cc70,lineinfo,nofma -Minfo=accel -O0 $<
-endif
diff --git a/machines/Depends.pgi-gpu b/machines/Depends.pgi-gpu
deleted file mode 100644
index a4bb6df226..0000000000
--- a/machines/Depends.pgi-gpu
+++ /dev/null
@@ -1,16 +0,0 @@
-#
-PUMAS_MG_OBJS=\
-micro_mg1_0.o \
-micro_pumas_v1.o \
-micro_pumas_data.o \
-micro_pumas_utils.o \
-micro_pumas_cam.o \
-wv_sat_methods.o \
-wv_saturation.o \
-macrop_driver.o \
-shr_spfn_mod.o
-
-ifeq ($(DEBUG),FALSE)
- $(PUMAS_MG_OBJS): %.o: %.F90
- $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -acc -ta=tesla:cc70,lineinfo,nofma -Minfo=accel $<
-endif
diff --git a/machines/cmake_macros/nvhpc-gpu.cmake b/machines/cmake_macros/nvhpc-gpu.cmake
deleted file mode 100644
index 679e81b2f4..0000000000
--- a/machines/cmake_macros/nvhpc-gpu.cmake
+++ /dev/null
@@ -1,46 +0,0 @@
-string(APPEND CFLAGS " -gopt -time")
-if (compile_threaded)
- string(APPEND CFLAGS " -mp")
-endif()
-string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DNO_SHR_VMATH -DNO_R16 -DCPRPGI")
-set(CXX_LINKER "CXX")
-set(FC_AUTO_R8 "-r8")
-string(APPEND FFLAGS " -i4 -gopt -time -Mextend -byteswapio -Mflushz -Kieee")
-if (compile_threaded)
- string(APPEND FFLAGS " -mp")
-endif()
-if (DEBUG)
- string(APPEND FFLAGS " -O0 -g -Ktrap=fp -Mbounds -Kieee")
-endif()
-if (COMP_NAME STREQUAL datm)
- string(APPEND FFLAGS " -Mnovect")
-endif()
-if (COMP_NAME STREQUAL dlnd)
- string(APPEND FFLAGS " -Mnovect")
-endif()
-if (COMP_NAME STREQUAL drof)
- string(APPEND FFLAGS " -Mnovect")
-endif()
-if (COMP_NAME STREQUAL dwav)
- string(APPEND FFLAGS " -Mnovect")
-endif()
-if (COMP_NAME STREQUAL dice)
- string(APPEND FFLAGS " -Mnovect")
-endif()
-if (COMP_NAME STREQUAL docn)
- string(APPEND FFLAGS " -Mnovect")
-endif()
-set(FFLAGS_NOOPT "-O0")
-set(FIXEDFLAGS "-Mfixed")
-set(FREEFLAGS "-Mfree")
-set(HAS_F2008_CONTIGUOUS "FALSE")
-set(LDFLAGS "-time -Wl,--allow-multiple-definition")
-if (compile_threaded)
- string(APPEND LDFLAGS " -mp")
-endif()
-set(MPICC "mpicc")
-set(MPICXX "mpicxx")
-set(MPIFC "mpif90")
-set(SCC "nvc")
-set(SCXX "nvc++")
-set(SFC "nvfortran")
diff --git a/machines/cmake_macros/nvhpc-gpu_casper.cmake b/machines/cmake_macros/nvhpc-gpu_casper.cmake
deleted file mode 100644
index ed2adbc6c9..0000000000
--- a/machines/cmake_macros/nvhpc-gpu_casper.cmake
+++ /dev/null
@@ -1,15 +0,0 @@
-if (NOT DEBUG)
- string(APPEND CFLAGS " -O -tp=skylake -Mnofma")
-endif()
-if (NOT DEBUG)
- string(APPEND FFLAGS " -O -tp=skylake -Mnofma")
-endif()
-string(APPEND FFLAGS " -I$(EXEROOT)/ocn/obj/FMS")
-if (NOT DEBUG)
- string(APPEND LDFLAGS " -O -tp=skylake -Mnofma -acc -gpu=cc70,lineinfo,nofma -Minfo=accel")
-endif()
-string(APPEND SLIBS " -llapack -lblas")
-if (MPILIB STREQUAL mpi-serial)
- string(APPEND SLIBS " -ldl")
-endif()
-string(APPEND SLIBS " -L${NETCDF_PATH}/lib -lnetcdf -lnetcdff")
diff --git a/machines/cmake_macros/nvhpc.cmake b/machines/cmake_macros/nvhpc.cmake
index 3e1b451624..1e6036d978 100644
--- a/machines/cmake_macros/nvhpc.cmake
+++ b/machines/cmake_macros/nvhpc.cmake
@@ -49,3 +49,21 @@ set(MPIFC "mpif90")
set(SCC "nvc")
set(SCXX "nvc++")
set(SFC "nvfortran")
+if (GPU_TYPE STREQUAL v100 AND GPU_OFFLOAD STREQUAL openacc)
+ string(APPEND GPUFLAGS " -acc -gpu=cc70,lineinfo,nofma -Minfo=accel ")
+endif()
+if (GPU_TYPE STREQUAL v100 AND GPU_OFFLOAD STREQUAL openmp)
+ string(APPEND GPUFLAGS " -mp=gpu -gpu=cc70,lineinfo,nofma -Minfo=accel ")
+endif()
+if (GPU_TYPE STREQUAL v100 AND GPU_OFFLOAD STREQUAL combined)
+ string(APPEND GPUFLAGS " -acc -gpu=cc70,lineinfo,nofma -mp=gpu -Minfo=accel ")
+endif()
+if (GPU_TYPE STREQUAL a100 AND GPU_OFFLOAD STREQUAL openacc)
+ string(APPEND GPUFLAGS " -acc -gpu=cc80,lineinfo,nofma -Minfo=accel ")
+endif()
+if (GPU_TYPE STREQUAL a100 AND GPU_OFFLOAD STREQUAL openmp)
+ string(APPEND GPUFLAGS " -mp=gpu -gpu=cc80,lineinfo,nofma -Minfo=accel ")
+endif()
+if (GPU_TYPE STREQUAL a100 AND GPU_OFFLOAD STREQUAL combined)
+ string(APPEND GPUFLAGS " -acc -gpu=cc80,lineinfo,nofma -mp=gpu -Minfo=accel")
+endif()
diff --git a/machines/cmake_macros/nvhpc_casper.cmake b/machines/cmake_macros/nvhpc_casper.cmake
index f3eb207d1e..08bc6bbecb 100644
--- a/machines/cmake_macros/nvhpc_casper.cmake
+++ b/machines/cmake_macros/nvhpc_casper.cmake
@@ -13,3 +13,4 @@ if (MPILIB STREQUAL mpi-serial)
string(APPEND SLIBS " -ldl")
endif()
string(APPEND SLIBS " -L${NETCDF_PATH}/lib -lnetcdf -lnetcdff")
+message("GPU_TYPE is ${GPU_TYPE} GPU_OFFLOAD is ${GPU_OFFLOAD}")
diff --git a/machines/cmake_macros/nvhpc_derecho.cmake b/machines/cmake_macros/nvhpc_derecho.cmake
index b7fdbd6998..ab1d0d572e 100644
--- a/machines/cmake_macros/nvhpc_derecho.cmake
+++ b/machines/cmake_macros/nvhpc_derecho.cmake
@@ -4,3 +4,4 @@ endif()
if (NOT DEBUG)
string(APPEND FFLAGS " -target=zen3")
endif()
+message("GPU_TYPE is ${GPU_TYPE} GPU_OFFLOAD is ${GPU_OFFLOAD}")
diff --git a/machines/cmake_macros/pgi-gpu.cmake b/machines/cmake_macros/pgi-gpu.cmake
deleted file mode 100644
index 856574795e..0000000000
--- a/machines/cmake_macros/pgi-gpu.cmake
+++ /dev/null
@@ -1,46 +0,0 @@
-string(APPEND CFLAGS " -gopt -time")
-if (compile_threaded)
- string(APPEND CFLAGS " -mp")
-endif()
-string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DNO_SHR_VMATH -DNO_R16 -DCPRPGI")
-set(CXX_LINKER "CXX")
-set(FC_AUTO_R8 "-r8")
-string(APPEND FFLAGS " -i4 -gopt -time -Mextend -byteswapio -Mflushz -Kieee")
-if (compile_threaded)
- string(APPEND FFLAGS " -mp")
-endif()
-if (DEBUG)
- string(APPEND FFLAGS " -O0 -g -Ktrap=fp -Mbounds -Kieee")
-endif()
-if (COMP_NAME STREQUAL datm)
- string(APPEND FFLAGS " -Mnovect")
-endif()
-if (COMP_NAME STREQUAL dlnd)
- string(APPEND FFLAGS " -Mnovect")
-endif()
-if (COMP_NAME STREQUAL drof)
- string(APPEND FFLAGS " -Mnovect")
-endif()
-if (COMP_NAME STREQUAL dwav)
- string(APPEND FFLAGS " -Mnovect")
-endif()
-if (COMP_NAME STREQUAL dice)
- string(APPEND FFLAGS " -Mnovect")
-endif()
-if (COMP_NAME STREQUAL docn)
- string(APPEND FFLAGS " -Mnovect")
-endif()
-set(FFLAGS_NOOPT "-O0")
-set(FIXEDFLAGS "-Mfixed")
-set(FREEFLAGS "-Mfree")
-set(HAS_F2008_CONTIGUOUS "FALSE")
-set(LDFLAGS "-time -Wl,--allow-multiple-definition")
-if (compile_threaded)
- string(APPEND LDFLAGS " -mp")
-endif()
-set(MPICC "mpicc")
-set(MPICXX "mpicxx")
-set(MPIFC "mpif90")
-set(SCC "pgcc")
-set(SCXX "pgc++")
-set(SFC "pgf95")
diff --git a/machines/cmake_macros/pgi-gpu_casper.cmake b/machines/cmake_macros/pgi-gpu_casper.cmake
deleted file mode 100644
index 600521a1bb..0000000000
--- a/machines/cmake_macros/pgi-gpu_casper.cmake
+++ /dev/null
@@ -1,15 +0,0 @@
-if (NOT DEBUG)
- string(APPEND CFLAGS " -O -tp=skylake -Mnofma")
-endif()
-if (NOT DEBUG)
- string(APPEND FFLAGS " -O -tp=skylake -Mnofma")
-endif()
-string(APPEND FFLAGS " -I$(EXEROOT)/ocn/obj/FMS")
-if (NOT DEBUG)
- string(APPEND LDFLAGS " -O -tp=skylake -Mnofma -acc -ta=tesla:cc70,lineinfo,nofma -Minfo=accel")
-endif()
-string(APPEND SLIBS " -llapack -lblas")
-if (MPILIB STREQUAL mpi-serial)
- string(APPEND SLIBS " -ldl")
-endif()
-string(APPEND SLIBS " -L${NETCDF_PATH}/lib -lnetcdf -lnetcdff")
diff --git a/machines/config_batch.xml b/machines/config_batch.xml
index c4b309b57a..fef0ba1fd6 100644
--- a/machines/config_batch.xml
+++ b/machines/config_batch.xml
@@ -156,7 +156,6 @@
-r {{ rerunnable }}
-j oe
- -V
@@ -226,34 +225,14 @@
qsub
-
-
-
-
- -S /glade/u/apps/dav/opt/nvidia-mps/mps_bash
- -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=700GB:ngpus={{ ngpus_per_node }}
- -l gpu_type=v100
-
-
-
-
-
- -S /glade/u/apps/dav/opt/nvidia-mps/mps_bash
- -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=700GB:ngpus={{ ngpus_per_node }}
- -l gpu_type=v100
-
-
-
- -S {{ shell }}
- -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=700GB:ngpus={{ ngpus_per_node }}
-
-
-
+
+ -l gpu_type=$GPU_TYPE
+
+
-S {{ shell }}
- -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=700GB:ngpus={{ ngpus_per_node }}
+ -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=700GB:ngpus={{ ngpus_per_node }}:mps=1
-
-
+
-S {{ shell }}
-l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=700GB:ngpus={{ ngpus_per_node }}
@@ -378,9 +357,13 @@
qsub
-
+
-S {{ shell }}
- -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}
+ -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=230GB
+
+
+ -S {{ shell }}
+ -l select={{ num_nodes }}:ncpus={{ max_cputasks_per_gpu_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=480GB:ngpus={{ ngpus_per_node }}:mps=1
main
@@ -501,10 +484,19 @@
qsub
-
+
+ -l gpu_type=$GPU_TYPE
+
+
-S {{ shell }}
-l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}
+
+ -S {{ shell }}
+ -l select={{ num_nodes }}:ncpus={{ max_cputasks_per_gpu_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=430GB:ngpus={{ ngpus_per_node }}
+
+
+
main
bigcpu
diff --git a/machines/config_machines.xml b/machines/config_machines.xml
index 504f97c682..794b7608f6 100644
--- a/machines/config_machines.xml
+++ b/machines/config_machines.xml
@@ -410,19 +410,23 @@ This allows using a different mpirun command to launch unit tests
NCAR GPU platform, os is Linux, 36 pes/node, batch system is pbs
casper*
LINUX
- pgi,intel,nvhpc,pgi-gpu,nvhpc-gpu
+ nvhpc,intel
openmpi
/glade/scratch/$USER
$ENV{CESMDATAROOT}/inputdata
/glade/p/cgd/tss/CTSM_datm_forcing_data
$CIME_OUTPUT_ROOT/archive/$CASE
$ENV{CESMDATAROOT}/cesm_baselines
+ $ENV{CESMDATAROOT}/tools/cime/tools/cprnc/cprnc
8
pbs
ASAP/CISL
36
- 8
+ 8
36
+ 36
+ none,v100
+ none,openacc,openmp,combined
TRUE
mpirun
@@ -450,38 +454,15 @@ This allows using a different mpirun command to launch unit tests
ncarenv/1.3
cmake/3.18.2
-
- pgi/20.4
-
-
- pgi/20.4
-
nvhpc/22.2
-
- nvhpc/22.2
-
intel/19.1.1
mkl/2020.0.1
-
- openmpi/4.1.0
- netcdf-mpi/4.8.0
- pnetcdf/1.12.2
-
-
- netcdf/4.8.0
-
-
- openmpi/4.1.0
- netcdf-mpi/4.7.4
- pnetcdf/1.12.2
- cuda/11.0.3
-
-
- netcdf/4.7.4
+
+ cuda/11.6
openmpi/4.1.4
@@ -491,15 +472,6 @@ This allows using a different mpirun command to launch unit tests
netcdf/4.8.1
-
- openmpi/4.1.4
- netcdf-mpi/4.8.1
- pnetcdf/1.12.3
- cuda/11.4.0
-
-
- netcdf/4.8.1
-
openmpi/4.1.1
netcdf-mpi/4.8.1
@@ -517,11 +489,11 @@ This allows using a different mpirun command to launch unit tests
/glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/intel/19.1.1/
esmf-8.4.0b08_casper-ncdfio-openmpi-O
-
+
/glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/nvhpc/22.2/
esmf-8.4.1b01-ncdfio-openmpi-g
-
+
/glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/nvhpc/22.2/
esmf-8.4.1b01-ncdfio-openmpi-O
@@ -554,7 +526,7 @@ This allows using a different mpirun command to launch unit tests
ON
SUMMARY
-
+
openacc
@@ -580,7 +552,7 @@ This allows using a different mpirun command to launch unit tests
/glade/p/cgd/tss/CTSM_datm_forcing_data
$CIME_OUTPUT_ROOT/archive/$CASE
$ENV{CESMDATAROOT}/cesm_baselines
- $ENV{CESMDATAROOT}/tools/cime/tools/cprnc/cprnc.cheyenne
+ $ENV{CESMDATAROOT}/tools/cime/tools/cprnc/cprnc
8
pbs
cseg
@@ -1300,7 +1272,12 @@ This allows using a different mpirun command to launch unit tests
pbs
cseg
128
+ 4
128
+ 64
+ none,a100
+ none,openacc,openmp,combined
+ get_local_rank
TRUE
mpiexec
@@ -1346,8 +1323,9 @@ This allows using a different mpirun command to launch unit tests
cray-libsci/23.02.1.1
- nvhpc/23.1
+ nvhpc/23.5
+
ncarcompilers/1.0.0
cmake
@@ -1359,6 +1337,10 @@ This allows using a different mpirun command to launch unit tests
mpi-serial/2.3.0
+
+ cuda/11.7.1
+
+
netcdf/4.9.2
@@ -1370,13 +1352,13 @@ This allows using a different mpirun command to launch unit tests
parallelio/2.6.0-debug
- esmf/8.6.0b03
+ esmf/8.6.0b03-debug
parallelio/2.6.0
esmf/8.6.0b03
-
+
@@ -1395,6 +1377,9 @@ This allows using a different mpirun command to launch unit tests
ON
SUMMARY
+
+ openacc
+
@@ -1963,7 +1948,11 @@ This allows using a different mpirun command to launch unit tests
pbs
cseg
128
+ 4
128
+ 64
+ none,a100
+ none,openacc,openmp,combined
TRUE
mpiexec
diff --git a/machines/mpi_run_gpu.casper b/machines/mpi_run_gpu.casper
deleted file mode 100755
index ade12d50ef..0000000000
--- a/machines/mpi_run_gpu.casper
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/bin/bash
-unset CUDA_VISIBLE_DEVICES
-let dev_id=$OMPI_COMM_WORLD_LOCAL_RANK%{{ ngpus_per_node }}
-export ACC_DEVICE_NUM=$dev_id
-export CUDA_VISIBLE_DEVICES=$dev_id
-exec $*