Skip to content

Add flexible controls of GPU configuration and remove old GPU configuration #97

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jedwards4b merged 27 commits into from
Aug 9, 2023
Merged

Add flexible controls of GPU configuration and remove old GPU configuration #97

Show file tree
Hide file tree
Changes from 22 commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
9beb292
Add Jim's changes for Gust's modules from https://github.com/jedwards…
sjsprecious Mar 22, 2023
4134af3
Add Jim's gpu_improvement branch at https://github.com/jedwards4b/ccs…
sjsprecious Mar 22, 2023
3b7a182
Add max_cputasks_per_gpu_node, gpu_type and gpu_offload options for C…
sjsprecious Mar 23, 2023
ee726b5
update batch script template for Gust GPU node
sjsprecious Mar 23, 2023
1f5eb30
typo fix for mismatched tags for GPU options
sjsprecious Mar 23, 2023
8649a95
Add missing "MAX_GPUS_PER_NODE"
sjsprecious Mar 24, 2023
e7bfbe1
downgrade netcdf-mpi fpr nvhpc/22.11
sjsprecious Mar 25, 2023
bf14967
bug fix of batch script template with gpu_enabled option
sjsprecious Mar 25, 2023
14f605e
remove -target=zen3 flag on gust
sjsprecious Mar 25, 2023
59086af
update to nvhpc/23.1 and related modules
sjsprecious Mar 26, 2023
918e3a8
turn off MPS on Gust
sjsprecious Mar 26, 2023
d0093a5
Update module versions for intel compiler
sjsprecious Apr 11, 2023
4023d67
Merge branch 'main' into add_gpu_gust
sjsprecious Apr 17, 2023
7224a37
load ncarenv and ncarcompilers manually
sjsprecious Apr 18, 2023
822f7b7
update the ncarenv and cesmdev module on Gust
sjsprecious Apr 25, 2023
71de769
Merge branch 'main' into add_gpu_gust
sjsprecious Aug 5, 2023
3287d57
remove unused files
sjsprecious Aug 5, 2023
fa7a978
resolve conflicts and add pbs info for derecho
sjsprecious Aug 5, 2023
7215ebb
minor update
sjsprecious Aug 5, 2023
3eb4cda
update GPU settings for Derecho
sjsprecious Aug 7, 2023
9e77b24
remove unused flag
sjsprecious Aug 7, 2023
75c3bfb
remove PBS -V option
sjsprecious Aug 7, 2023
d1b8e68
Add "get_local_rank" as an attribute
sjsprecious Aug 8, 2023
d5da373
Update memory request on Derecho
sjsprecious Aug 9, 2023
8241e8c
Turn on MPS for Derecho
sjsprecious Aug 9, 2023
560c043
remove duplicated lines
sjsprecious Aug 9, 2023
7a12059
Merge branch 'main' into add_gpu_gust
sjsprecious Aug 9, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 3 additions & 7 deletions machines/Depends.nvhpc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,9 +69,7 @@ mo_optical_props_kernels.o\
mo_rte_solver_kernels.o\
mo_optical_props_kernels.o\
mo_rte_solver_kernels.o\
rrtmgp_driver.o

RRTMGP_NOOPT=\
rrtmgp_driver.o\
mo_gas_optics_kernels.o

ifeq ($(DEBUG),FALSE)
Expand All @@ -86,9 +84,7 @@ ifeq ($(DEBUG),FALSE)
# $(SHR_RANDNUM_C_OBJS): %.o: %.c
# $(CC) -c $(INCLDIR) $(INCS) $(CFLAGS) -O3 -fast $<
$(PUMAS_OBJS): %.o: %.F90
$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O3 -fastsse -Mnofma -Mflushz -Mfprelaxed=sqrt $<
$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O3 -fastsse -Mnofma -Mflushz -Mfprelaxed=sqrt $(GPUFLAGS) $<
$(RRTMGP_OBJS): %.o: %.F90
$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -Mnoopenmp $<
$(RRTMGP_NOOPT): %.o: %.F90
$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O0 $<
$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) $(GPUFLAGS) $<
endif
94 changes: 0 additions & 94 deletions machines/Depends.nvhpc-gpu
View file
Open in desktop

This file was deleted.

16 changes: 0 additions & 16 deletions machines/Depends.pgi-gpu
View file
Open in desktop

This file was deleted.

46 changes: 0 additions & 46 deletions machines/cmake_macros/nvhpc-gpu.cmake
View file
Open in desktop

This file was deleted.

15 changes: 0 additions & 15 deletions machines/cmake_macros/nvhpc-gpu_casper.cmake
View file
Open in desktop

This file was deleted.

18 changes: 18 additions & 0 deletions machines/cmake_macros/nvhpc.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,3 +49,21 @@ set(MPIFC "mpif90")
set(SCC "nvc")
set(SCXX "nvc++")
set(SFC "nvfortran")
if (GPU_TYPE STREQUAL v100 AND GPU_OFFLOAD STREQUAL openacc)
string(APPEND GPUFLAGS " -acc -gpu=cc70,lineinfo,nofma -Minfo=accel ")
endif()
if (GPU_TYPE STREQUAL v100 AND GPU_OFFLOAD STREQUAL openmp)
string(APPEND GPUFLAGS " -mp=gpu -gpu=cc70,lineinfo,nofma -Minfo=accel ")
endif()
if (GPU_TYPE STREQUAL v100 AND GPU_OFFLOAD STREQUAL combined)
string(APPEND GPUFLAGS " -acc -gpu=cc70,lineinfo,nofma -mp=gpu -Minfo=accel ")
endif()
if (GPU_TYPE STREQUAL a100 AND GPU_OFFLOAD STREQUAL openacc)
string(APPEND GPUFLAGS " -acc -gpu=cc80,lineinfo,nofma -Minfo=accel ")
endif()
if (GPU_TYPE STREQUAL a100 AND GPU_OFFLOAD STREQUAL openmp)
string(APPEND GPUFLAGS " -mp=gpu -gpu=cc80,lineinfo,nofma -Minfo=accel ")
endif()
if (GPU_TYPE STREQUAL a100 AND GPU_OFFLOAD STREQUAL combined)
string(APPEND GPUFLAGS " -acc -gpu=cc80,lineinfo,nofma -mp=gpu -Minfo=accel")
endif()
1 change: 1 addition & 0 deletions machines/cmake_macros/nvhpc_casper.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,3 +13,4 @@ if (MPILIB STREQUAL mpi-serial)
string(APPEND SLIBS " -ldl")
endif()
string(APPEND SLIBS " -L${NETCDF_PATH}/lib -lnetcdf -lnetcdff")
message("GPU_TYPE is ${GPU_TYPE} GPU_OFFLOAD is ${GPU_OFFLOAD}")
1 change: 1 addition & 0 deletions machines/cmake_macros/nvhpc_derecho.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,3 +4,4 @@ endif()
if (NOT DEBUG)
string(APPEND FFLAGS " -target=zen3")
endif()
message("GPU_TYPE is ${GPU_TYPE} GPU_OFFLOAD is ${GPU_OFFLOAD}")
46 changes: 0 additions & 46 deletions machines/cmake_macros/pgi-gpu.cmake
View file
Open in desktop

This file was deleted.

15 changes: 0 additions & 15 deletions machines/cmake_macros/pgi-gpu_casper.cmake
View file
Open in desktop

This file was deleted.

54 changes: 24 additions & 30 deletions machines/config_batch.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,6 @@
<directive default="n"> -r {{ rerunnable }} </directive>
<!-- <directive> -j oe {{ job_id }} </directive> -->
<directive> -j oe </directive>
<directive> -V </directive>
</directives>
</batch_system>

Expand Down Expand Up @@ -226,34 +225,14 @@
<!-- casper pbs -->
<batch_system MACH="casper" type="pbs">
<batch_submit>qsub</batch_submit>

<directives queue="casper" compiler="pgi-gpu">
<!-- Turn on MPS server manually -->
<!-- This is a temporary solution and should be removed once MPS is integrated into PBS on Casper -->
<directive default="/bin/bash" > -S /glade/u/apps/dav/opt/nvidia-mps/mps_bash </directive>
<directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=700GB:ngpus={{ ngpus_per_node }} </directive>
<directive> -l gpu_type=v100 </directive>
</directives>

<directives queue="casper" compiler="nvhpc-gpu">
<!-- Turn on MPS server manually -->
<!-- This is a temporary solution and should be removed once MPS is integrated into PBS on Casper -->
<directive default="/bin/bash" > -S /glade/u/apps/dav/opt/nvidia-mps/mps_bash </directive>
<directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=700GB:ngpus={{ ngpus_per_node }} </directive>
<directive> -l gpu_type=v100 </directive>
</directives>

<directives queue="casper" compiler="nvhpc">
<directive default="/bin/bash" > -S {{ shell }} </directive>
<directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=700GB:ngpus={{ ngpus_per_node }} </directive>
</directives>

<directives queue="casper" compiler="pgi">
<submit_args>
<argument> -l gpu_type=$GPU_TYPE </argument>
</submit_args>
<directives queue="casper" compiler="nvhpc" gpu_enabled="true">
<directive default="/bin/bash" > -S {{ shell }} </directive>
<directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=700GB:ngpus={{ ngpus_per_node }} </directive>
<directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=700GB:ngpus={{ ngpus_per_node }}:mps=1 </directive>
</directives>

<directives queue="casper" compiler="intel">
<directives queue="casper" gpu_enabled="false">
<directive default="/bin/bash" > -S {{ shell }} </directive>
<directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=700GB:ngpus={{ ngpus_per_node }} </directive>
</directives>
Expand Down Expand Up @@ -378,9 +357,15 @@

<batch_system MACH="derecho" type="pbs" >
<batch_submit>qsub</batch_submit>
<directives>
<directives gpu_enabled="false">
<directive default="/bin/bash" > -S {{ shell }} </directive>
<directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}</directive>
<directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=240GB</directive>
</directives>
<directives gpu_enabled="true">
<directive default="/bin/bash" > -S {{ shell }} </directive>
<directive> -l select={{ num_nodes }}:ncpus={{ max_cputasks_per_gpu_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=490GB:ngpus={{ ngpus_per_node }} </directive>
<!-- Do not use MPS on Gust yet as it conflicts with cray-mpich library -->
<!-- <directive> -l select={{ num_nodes }}:ncpus={{ max_cputasks_per_gpu_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=430GB:ngpus={{ ngpus_per_node }}:mps=1 </directive> -->
</directives>
<queues>
<queue walltimemax="4:00:00" nodemin="1" nodemax="2488" >main</queue>
Expand Down Expand Up @@ -501,10 +486,19 @@

<batch_system MACH="gust" type="pbs" >
<batch_submit>qsub</batch_submit>
<directives>
<submit_args>
<argument> -l gpu_type=$GPU_TYPE </argument>
</submit_args>
<directives queue="main" gpu_enabled="false">
<directive default="/bin/bash" > -S {{ shell }} </directive>
<directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}</directive>
</directives>
<directives queue="main" gpu_enabled="true">
<directive default="/bin/bash" > -S {{ shell }} </directive>
<directive> -l select={{ num_nodes }}:ncpus={{ max_cputasks_per_gpu_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=430GB:ngpus={{ ngpus_per_node }} </directive>
<!-- Do not use MPS on Gust yet as it conflicts with cray-mpich library -->
<!-- <directive> -l select={{ num_nodes }}:ncpus={{ max_cputasks_per_gpu_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=430GB:ngpus={{ ngpus_per_node }}:mps=1 </directive> -->
</directives>
<queues>
<queue walltimemax="4:00:00" nodemin="1" nodemax="8" >main</queue>
<queue walltimemax="1:00:00" nodemin="9" nodemax="16" >bigcpu</queue>
Expand Down
Loading