diff --git a/machines/Depends.nvhpc b/machines/Depends.nvhpc index 7b190cc09a..b51e3c901b 100644 --- a/machines/Depends.nvhpc +++ b/machines/Depends.nvhpc @@ -67,11 +67,7 @@ mo_rte_kind.o\ mo_source_functions.o\ mo_optical_props_kernels.o\ mo_rte_solver_kernels.o\ -mo_optical_props_kernels.o\ -mo_rte_solver_kernels.o\ -rrtmgp_driver.o - -RRTMGP_NOOPT=\ +rrtmgp_driver.o\ mo_gas_optics_kernels.o ifeq ($(DEBUG),FALSE) @@ -86,9 +82,7 @@ ifeq ($(DEBUG),FALSE) # $(SHR_RANDNUM_C_OBJS): %.o: %.c # $(CC) -c $(INCLDIR) $(INCS) $(CFLAGS) -O3 -fast $< $(PUMAS_OBJS): %.o: %.F90 - $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O3 -fastsse -Mnofma -Mflushz -Mfprelaxed=sqrt $< + $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O3 -fastsse -Mnofma -Mflushz -Mfprelaxed=sqrt $(GPUFLAGS) $< $(RRTMGP_OBJS): %.o: %.F90 - $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -Mnoopenmp $< - $(RRTMGP_NOOPT): %.o: %.F90 - $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O0 $< + $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) $(GPUFLAGS) $< endif diff --git a/machines/Depends.nvhpc-gpu b/machines/Depends.nvhpc-gpu deleted file mode 100644 index f8eaf17bba..0000000000 --- a/machines/Depends.nvhpc-gpu +++ /dev/null @@ -1,94 +0,0 @@ -# 10/13/2022 nvhpc compiler on gust.hpc.ucar.edu produced incorrect mpitask cound when esm.F90 was optimized -# 04/05/2023 do not apply optimization flags to "SHR_RANDNUM_C_OBJS" for the nvhpc compiler, which will fail the ensemble consistency test -# -# Apply a different optimization level consistent with the Intel compiler -PERFOBJS=\ -prim_advection_mod.o \ -edge_mod.o \ -derivative_mod.o \ -bndry_mod.o \ -prim_advance_mod.o - -REDUCED_OPT_OBJS=\ -SatellitePhenologyMod.o \ -esm.o - -REDUCED_PRECISION_OBJS=\ -shr_wv_sat_mod.o - -SHR_RANDNUM_FORT_OBJS=\ -kissvec_mod.o \ -mersennetwister_mod.o \ -dSFMT_interface.o \ -shr_RandNum_mod.o - -SHR_RANDNUM_C_OBJS=\ -dSFMT.o \ -dSFMT_utils.o \ -kissvec.o - -# Apply GPU flags to the following code -PUMAS_OBJS=\ -micro_mg1_0.o \ -micro_pumas_v1.o \ -micro_pumas_data.o \ -micro_pumas_utils.o \ -micro_pumas_cam.o \ -wv_sat_methods.o \ -wv_saturation.o \ -macrop_driver.o \ -shr_spfn_mod.o - -RRTMGP_OBJS=\ -mo_rte_util_array.o \ -mo_rte_sw.o \ -mo_rte_lw.o \ -mo_cloud_optics.o \ -mo_gas_optics_rrtmgp.o \ -mo_gas_concentrations.o \ -mo_fluxes_broadband_kernels.o \ -mo_fluxes_byband_kernels.o \ -mo_rrtmgp_util_reorder_kernels.o \ -mo_compute_bc.o\ -mo_fluxes_byband.o\ -mo_fluxes_bygpoint.o\ -mo_heating_rates.o\ -mo_rrtmgp_clr_all_sky.o\ -mo_cloud_sampling.o\ -mo_solar_variability.o\ -mo_gas_optics.o\ -mo_rrtmgp_constants.o\ -mo_rrtmgp_util_reorder.o\ -mo_rrtmgp_util_string.o\ -mo_fluxes.o\ -mo_optical_props.o\ -mo_rte_config.o\ -mo_rte_kind.o\ -mo_source_functions.o\ -mo_optical_props_kernels.o\ -mo_rte_solver_kernels.o\ -mo_optical_props_kernels.o\ -mo_rte_solver_kernels.o\ -rrtmgp_driver.o - -RRTMGP_NOOPT=\ -mo_gas_optics_kernels.o - -ifeq ($(DEBUG),FALSE) - $(PERFOBJS): %.o: %.F90 - $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O3 -Mfprelaxed=div $< - $(REDUCED_OPT_OBJS): %.o: %.F90 - $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O1 $< - $(REDUCED_PRECISION_OBJS): %.o: %.F90 - $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -fast $< - $(SHR_RANDNUM_FORT_OBJS): %.o: %.F90 - $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O3 -fast -Mfprelaxed=div,sqrt $< -# $(SHR_RANDNUM_C_OBJS): %.o: %.c -# $(CC) -c $(INCLDIR) $(INCS) $(CFLAGS) -O3 -fast $< - $(PUMAS_OBJS): %.o: %.F90 - $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O3 -fastsse -Mnofma -Mflushz -Mfprelaxed=sqrt -acc -gpu=cc70,lineinfo,nofma,flushz,fastmath -Minfo=accel $< - $(RRTMGP_OBJS): %.o: %.F90 - $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -acc -gpu=cc70,lineinfo,nofma -Minfo=accel -Mnoopenmp $< - $(RRTMGP_NOOPT): %.o: %.F90 - $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -acc -gpu=cc70,lineinfo,nofma -Minfo=accel -O0 $< -endif diff --git a/machines/Depends.pgi-gpu b/machines/Depends.pgi-gpu deleted file mode 100644 index a4bb6df226..0000000000 --- a/machines/Depends.pgi-gpu +++ /dev/null @@ -1,16 +0,0 @@ -# -PUMAS_MG_OBJS=\ -micro_mg1_0.o \ -micro_pumas_v1.o \ -micro_pumas_data.o \ -micro_pumas_utils.o \ -micro_pumas_cam.o \ -wv_sat_methods.o \ -wv_saturation.o \ -macrop_driver.o \ -shr_spfn_mod.o - -ifeq ($(DEBUG),FALSE) - $(PUMAS_MG_OBJS): %.o: %.F90 - $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -acc -ta=tesla:cc70,lineinfo,nofma -Minfo=accel $< -endif diff --git a/machines/cmake_macros/nvhpc-gpu.cmake b/machines/cmake_macros/nvhpc-gpu.cmake deleted file mode 100644 index 679e81b2f4..0000000000 --- a/machines/cmake_macros/nvhpc-gpu.cmake +++ /dev/null @@ -1,46 +0,0 @@ -string(APPEND CFLAGS " -gopt -time") -if (compile_threaded) - string(APPEND CFLAGS " -mp") -endif() -string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DNO_SHR_VMATH -DNO_R16 -DCPRPGI") -set(CXX_LINKER "CXX") -set(FC_AUTO_R8 "-r8") -string(APPEND FFLAGS " -i4 -gopt -time -Mextend -byteswapio -Mflushz -Kieee") -if (compile_threaded) - string(APPEND FFLAGS " -mp") -endif() -if (DEBUG) - string(APPEND FFLAGS " -O0 -g -Ktrap=fp -Mbounds -Kieee") -endif() -if (COMP_NAME STREQUAL datm) - string(APPEND FFLAGS " -Mnovect") -endif() -if (COMP_NAME STREQUAL dlnd) - string(APPEND FFLAGS " -Mnovect") -endif() -if (COMP_NAME STREQUAL drof) - string(APPEND FFLAGS " -Mnovect") -endif() -if (COMP_NAME STREQUAL dwav) - string(APPEND FFLAGS " -Mnovect") -endif() -if (COMP_NAME STREQUAL dice) - string(APPEND FFLAGS " -Mnovect") -endif() -if (COMP_NAME STREQUAL docn) - string(APPEND FFLAGS " -Mnovect") -endif() -set(FFLAGS_NOOPT "-O0") -set(FIXEDFLAGS "-Mfixed") -set(FREEFLAGS "-Mfree") -set(HAS_F2008_CONTIGUOUS "FALSE") -set(LDFLAGS "-time -Wl,--allow-multiple-definition") -if (compile_threaded) - string(APPEND LDFLAGS " -mp") -endif() -set(MPICC "mpicc") -set(MPICXX "mpicxx") -set(MPIFC "mpif90") -set(SCC "nvc") -set(SCXX "nvc++") -set(SFC "nvfortran") diff --git a/machines/cmake_macros/nvhpc-gpu_casper.cmake b/machines/cmake_macros/nvhpc-gpu_casper.cmake deleted file mode 100644 index ed2adbc6c9..0000000000 --- a/machines/cmake_macros/nvhpc-gpu_casper.cmake +++ /dev/null @@ -1,15 +0,0 @@ -if (NOT DEBUG) - string(APPEND CFLAGS " -O -tp=skylake -Mnofma") -endif() -if (NOT DEBUG) - string(APPEND FFLAGS " -O -tp=skylake -Mnofma") -endif() -string(APPEND FFLAGS " -I$(EXEROOT)/ocn/obj/FMS") -if (NOT DEBUG) - string(APPEND LDFLAGS " -O -tp=skylake -Mnofma -acc -gpu=cc70,lineinfo,nofma -Minfo=accel") -endif() -string(APPEND SLIBS " -llapack -lblas") -if (MPILIB STREQUAL mpi-serial) - string(APPEND SLIBS " -ldl") -endif() -string(APPEND SLIBS " -L${NETCDF_PATH}/lib -lnetcdf -lnetcdff") diff --git a/machines/cmake_macros/nvhpc.cmake b/machines/cmake_macros/nvhpc.cmake index 3e1b451624..1e6036d978 100644 --- a/machines/cmake_macros/nvhpc.cmake +++ b/machines/cmake_macros/nvhpc.cmake @@ -49,3 +49,21 @@ set(MPIFC "mpif90") set(SCC "nvc") set(SCXX "nvc++") set(SFC "nvfortran") +if (GPU_TYPE STREQUAL v100 AND GPU_OFFLOAD STREQUAL openacc) + string(APPEND GPUFLAGS " -acc -gpu=cc70,lineinfo,nofma -Minfo=accel ") +endif() +if (GPU_TYPE STREQUAL v100 AND GPU_OFFLOAD STREQUAL openmp) + string(APPEND GPUFLAGS " -mp=gpu -gpu=cc70,lineinfo,nofma -Minfo=accel ") +endif() +if (GPU_TYPE STREQUAL v100 AND GPU_OFFLOAD STREQUAL combined) + string(APPEND GPUFLAGS " -acc -gpu=cc70,lineinfo,nofma -mp=gpu -Minfo=accel ") +endif() +if (GPU_TYPE STREQUAL a100 AND GPU_OFFLOAD STREQUAL openacc) + string(APPEND GPUFLAGS " -acc -gpu=cc80,lineinfo,nofma -Minfo=accel ") +endif() +if (GPU_TYPE STREQUAL a100 AND GPU_OFFLOAD STREQUAL openmp) + string(APPEND GPUFLAGS " -mp=gpu -gpu=cc80,lineinfo,nofma -Minfo=accel ") +endif() +if (GPU_TYPE STREQUAL a100 AND GPU_OFFLOAD STREQUAL combined) + string(APPEND GPUFLAGS " -acc -gpu=cc80,lineinfo,nofma -mp=gpu -Minfo=accel") +endif() diff --git a/machines/cmake_macros/nvhpc_casper.cmake b/machines/cmake_macros/nvhpc_casper.cmake index f3eb207d1e..08bc6bbecb 100644 --- a/machines/cmake_macros/nvhpc_casper.cmake +++ b/machines/cmake_macros/nvhpc_casper.cmake @@ -13,3 +13,4 @@ if (MPILIB STREQUAL mpi-serial) string(APPEND SLIBS " -ldl") endif() string(APPEND SLIBS " -L${NETCDF_PATH}/lib -lnetcdf -lnetcdff") +message("GPU_TYPE is ${GPU_TYPE} GPU_OFFLOAD is ${GPU_OFFLOAD}") diff --git a/machines/cmake_macros/nvhpc_derecho.cmake b/machines/cmake_macros/nvhpc_derecho.cmake index b7fdbd6998..ab1d0d572e 100644 --- a/machines/cmake_macros/nvhpc_derecho.cmake +++ b/machines/cmake_macros/nvhpc_derecho.cmake @@ -4,3 +4,4 @@ endif() if (NOT DEBUG) string(APPEND FFLAGS " -target=zen3") endif() +message("GPU_TYPE is ${GPU_TYPE} GPU_OFFLOAD is ${GPU_OFFLOAD}") diff --git a/machines/cmake_macros/pgi-gpu.cmake b/machines/cmake_macros/pgi-gpu.cmake deleted file mode 100644 index 856574795e..0000000000 --- a/machines/cmake_macros/pgi-gpu.cmake +++ /dev/null @@ -1,46 +0,0 @@ -string(APPEND CFLAGS " -gopt -time") -if (compile_threaded) - string(APPEND CFLAGS " -mp") -endif() -string(APPEND CPPDEFS " -DFORTRANUNDERSCORE -DNO_SHR_VMATH -DNO_R16 -DCPRPGI") -set(CXX_LINKER "CXX") -set(FC_AUTO_R8 "-r8") -string(APPEND FFLAGS " -i4 -gopt -time -Mextend -byteswapio -Mflushz -Kieee") -if (compile_threaded) - string(APPEND FFLAGS " -mp") -endif() -if (DEBUG) - string(APPEND FFLAGS " -O0 -g -Ktrap=fp -Mbounds -Kieee") -endif() -if (COMP_NAME STREQUAL datm) - string(APPEND FFLAGS " -Mnovect") -endif() -if (COMP_NAME STREQUAL dlnd) - string(APPEND FFLAGS " -Mnovect") -endif() -if (COMP_NAME STREQUAL drof) - string(APPEND FFLAGS " -Mnovect") -endif() -if (COMP_NAME STREQUAL dwav) - string(APPEND FFLAGS " -Mnovect") -endif() -if (COMP_NAME STREQUAL dice) - string(APPEND FFLAGS " -Mnovect") -endif() -if (COMP_NAME STREQUAL docn) - string(APPEND FFLAGS " -Mnovect") -endif() -set(FFLAGS_NOOPT "-O0") -set(FIXEDFLAGS "-Mfixed") -set(FREEFLAGS "-Mfree") -set(HAS_F2008_CONTIGUOUS "FALSE") -set(LDFLAGS "-time -Wl,--allow-multiple-definition") -if (compile_threaded) - string(APPEND LDFLAGS " -mp") -endif() -set(MPICC "mpicc") -set(MPICXX "mpicxx") -set(MPIFC "mpif90") -set(SCC "pgcc") -set(SCXX "pgc++") -set(SFC "pgf95") diff --git a/machines/cmake_macros/pgi-gpu_casper.cmake b/machines/cmake_macros/pgi-gpu_casper.cmake deleted file mode 100644 index 600521a1bb..0000000000 --- a/machines/cmake_macros/pgi-gpu_casper.cmake +++ /dev/null @@ -1,15 +0,0 @@ -if (NOT DEBUG) - string(APPEND CFLAGS " -O -tp=skylake -Mnofma") -endif() -if (NOT DEBUG) - string(APPEND FFLAGS " -O -tp=skylake -Mnofma") -endif() -string(APPEND FFLAGS " -I$(EXEROOT)/ocn/obj/FMS") -if (NOT DEBUG) - string(APPEND LDFLAGS " -O -tp=skylake -Mnofma -acc -ta=tesla:cc70,lineinfo,nofma -Minfo=accel") -endif() -string(APPEND SLIBS " -llapack -lblas") -if (MPILIB STREQUAL mpi-serial) - string(APPEND SLIBS " -ldl") -endif() -string(APPEND SLIBS " -L${NETCDF_PATH}/lib -lnetcdf -lnetcdff") diff --git a/machines/config_batch.xml b/machines/config_batch.xml index c4b309b57a..fef0ba1fd6 100644 --- a/machines/config_batch.xml +++ b/machines/config_batch.xml @@ -156,7 +156,6 @@ -r {{ rerunnable }} -j oe - -V @@ -226,34 +225,14 @@ qsub - - - - - -S /glade/u/apps/dav/opt/nvidia-mps/mps_bash - -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=700GB:ngpus={{ ngpus_per_node }} - -l gpu_type=v100 - - - - - - -S /glade/u/apps/dav/opt/nvidia-mps/mps_bash - -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=700GB:ngpus={{ ngpus_per_node }} - -l gpu_type=v100 - - - - -S {{ shell }} - -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=700GB:ngpus={{ ngpus_per_node }} - - - + + -l gpu_type=$GPU_TYPE + + -S {{ shell }} - -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=700GB:ngpus={{ ngpus_per_node }} + -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=700GB:ngpus={{ ngpus_per_node }}:mps=1 - - + -S {{ shell }} -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=700GB:ngpus={{ ngpus_per_node }} @@ -378,9 +357,13 @@ qsub - + -S {{ shell }} - -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }} + -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=230GB + + + -S {{ shell }} + -l select={{ num_nodes }}:ncpus={{ max_cputasks_per_gpu_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=480GB:ngpus={{ ngpus_per_node }}:mps=1 main @@ -501,10 +484,19 @@ qsub - + + -l gpu_type=$GPU_TYPE + + -S {{ shell }} -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }} + + -S {{ shell }} + -l select={{ num_nodes }}:ncpus={{ max_cputasks_per_gpu_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=430GB:ngpus={{ ngpus_per_node }} + + + main bigcpu diff --git a/machines/config_machines.xml b/machines/config_machines.xml index 504f97c682..794b7608f6 100644 --- a/machines/config_machines.xml +++ b/machines/config_machines.xml @@ -410,19 +410,23 @@ This allows using a different mpirun command to launch unit tests NCAR GPU platform, os is Linux, 36 pes/node, batch system is pbs casper* LINUX - pgi,intel,nvhpc,pgi-gpu,nvhpc-gpu + nvhpc,intel openmpi /glade/scratch/$USER $ENV{CESMDATAROOT}/inputdata /glade/p/cgd/tss/CTSM_datm_forcing_data $CIME_OUTPUT_ROOT/archive/$CASE $ENV{CESMDATAROOT}/cesm_baselines + $ENV{CESMDATAROOT}/tools/cime/tools/cprnc/cprnc 8 pbs ASAP/CISL 36 - 8 + 8 36 + 36 + none,v100 + none,openacc,openmp,combined TRUE mpirun @@ -450,38 +454,15 @@ This allows using a different mpirun command to launch unit tests ncarenv/1.3 cmake/3.18.2 - - pgi/20.4 - - - pgi/20.4 - nvhpc/22.2 - - nvhpc/22.2 - intel/19.1.1 mkl/2020.0.1 - - openmpi/4.1.0 - netcdf-mpi/4.8.0 - pnetcdf/1.12.2 - - - netcdf/4.8.0 - - - openmpi/4.1.0 - netcdf-mpi/4.7.4 - pnetcdf/1.12.2 - cuda/11.0.3 - - - netcdf/4.7.4 + + cuda/11.6 openmpi/4.1.4 @@ -491,15 +472,6 @@ This allows using a different mpirun command to launch unit tests netcdf/4.8.1 - - openmpi/4.1.4 - netcdf-mpi/4.8.1 - pnetcdf/1.12.3 - cuda/11.4.0 - - - netcdf/4.8.1 - openmpi/4.1.1 netcdf-mpi/4.8.1 @@ -517,11 +489,11 @@ This allows using a different mpirun command to launch unit tests /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/intel/19.1.1/ esmf-8.4.0b08_casper-ncdfio-openmpi-O - + /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/nvhpc/22.2/ esmf-8.4.1b01-ncdfio-openmpi-g - + /glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/nvhpc/22.2/ esmf-8.4.1b01-ncdfio-openmpi-O @@ -554,7 +526,7 @@ This allows using a different mpirun command to launch unit tests ON SUMMARY - + openacc @@ -580,7 +552,7 @@ This allows using a different mpirun command to launch unit tests /glade/p/cgd/tss/CTSM_datm_forcing_data $CIME_OUTPUT_ROOT/archive/$CASE $ENV{CESMDATAROOT}/cesm_baselines - $ENV{CESMDATAROOT}/tools/cime/tools/cprnc/cprnc.cheyenne + $ENV{CESMDATAROOT}/tools/cime/tools/cprnc/cprnc 8 pbs cseg @@ -1300,7 +1272,12 @@ This allows using a different mpirun command to launch unit tests pbs cseg 128 + 4 128 + 64 + none,a100 + none,openacc,openmp,combined + get_local_rank TRUE mpiexec @@ -1346,8 +1323,9 @@ This allows using a different mpirun command to launch unit tests cray-libsci/23.02.1.1 - nvhpc/23.1 + nvhpc/23.5 + ncarcompilers/1.0.0 cmake @@ -1359,6 +1337,10 @@ This allows using a different mpirun command to launch unit tests mpi-serial/2.3.0 + + cuda/11.7.1 + + netcdf/4.9.2 @@ -1370,13 +1352,13 @@ This allows using a different mpirun command to launch unit tests parallelio/2.6.0-debug - esmf/8.6.0b03 + esmf/8.6.0b03-debug parallelio/2.6.0 esmf/8.6.0b03 - + @@ -1395,6 +1377,9 @@ This allows using a different mpirun command to launch unit tests ON SUMMARY + + openacc + @@ -1963,7 +1948,11 @@ This allows using a different mpirun command to launch unit tests pbs cseg 128 + 4 128 + 64 + none,a100 + none,openacc,openmp,combined TRUE mpiexec diff --git a/machines/mpi_run_gpu.casper b/machines/mpi_run_gpu.casper deleted file mode 100755 index ade12d50ef..0000000000 --- a/machines/mpi_run_gpu.casper +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -unset CUDA_VISIBLE_DEVICES -let dev_id=$OMPI_COMM_WORLD_LOCAL_RANK%{{ ngpus_per_node }} -export ACC_DEVICE_NUM=$dev_id -export CUDA_VISIBLE_DEVICES=$dev_id -exec $*