Skip to content
Merged
Show file tree
Hide file tree
Changes from 6 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions projects/rccl/src/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -849,9 +849,11 @@ if (gen_py_result)
endif()

if (GENERATE_SYM_KERNELS)
# Execute the python script to generate required symmetric memory kernels
# Execute the python script to generate required symmetric memory kernels.
# GPU_TARGETS selects the arch-specific algos (the Tma* kernels need gfx1250's
# Tensor Data Mover).
execute_process(
COMMAND ${Python3_EXECUTABLE} ${RCCL_SOURCE_DIR}/src/device/symmetric/generate.py ${GEN_SYM_DIR}
COMMAND ${Python3_EXECUTABLE} ${RCCL_SOURCE_DIR}/src/device/symmetric/generate.py ${GEN_SYM_DIR} "${GPU_TARGETS}"
WORKING_DIRECTORY ${RCCL_SOURCE_DIR}
RESULT_VARIABLE gen_sym_py_result
ERROR_VARIABLE gen_sym_py_error
Expand Down
55 changes: 21 additions & 34 deletions projects/rccl/src/device/symmetric/all_gather.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -27,21 +27,21 @@ static __device__ void bcastDeep(ncclSymkArgsHandler const& handler, int tn, int

Pack* inpPacks = (Pack*)input.localPtr() + intptr_t(w) * UnrollPacks * WARP_SIZE +
(
#if __CUDA_ARCH__ >= 1000
#if NCCL_SYMK_ASYNC_TILE
EnableTma ? 0 :
#endif
lane);

ncclSymPtr<Pack> outPacks = (ncclSymPtr<Pack>)output + intptr_t(w) * UnrollPacks * WARP_SIZE +
(
#if __CUDA_ARCH__ >= 1000
#if NCCL_SYMK_ASYNC_TILE
EnableTma ? 0 :
#endif
lane);

Pack tmp[UnrollPacks];

#if __CUDA_ARCH__ >= 1000
#if NCCL_SYMK_ASYNC_TILE
int lw = threadIdx.x / WARP_SIZE;
extern __shared__ char smemScratch[];
using tmaSmemStruct_t = tmaSmemStruct<Pack, UnrollPacks>;
Expand All @@ -51,28 +51,22 @@ static __device__ void bcastDeep(ncclSymkArgsHandler const& handler, int tn, int
#endif
bool skip = false; // all lanes issue loads/stores

#if __CUDA_ARCH__ >= 1000
#if NCCL_SYMK_ASYNC_TILE
size_t pending = 0;
if NCCL_IF_CONSTEXPR (EnableTma) {
if (lane == 0) {
// lane0 issues async.cp.bulk commands
init(&tmaSmem->bar, 1);
} else {
// other lanes can skip the loop
skip = true;
}
ncclSymkTileBarInit(&tmaSmem->bar, /*arrivers=*/1, lane);
// TMA issues from lane 0, so the rest of the warp has nothing left to do.
// TDM needs the whole warp to build the descriptor, so every lane stays in.
skip = NCCL_SYMK_TILE_TMA && lane != 0;
}
#endif

nIters -= w;
if (0 < nIters) {
#if __CUDA_ARCH__ >= 1000
#if NCCL_SYMK_ASYNC_TILE
if NCCL_IF_CONSTEXPR (EnableTma) {
if (lane == 0) {
cuda::device::memcpy_async_tx(tmaSmem->buff[0], inpPacks, cuda::aligned_size_t<16>(tileSize), tmaSmem->bar);
cuda::barrier<cuda::thread_scope_block>::arrival_token token =
cuda::device::barrier_arrive_tx(tmaSmem->bar, 1, tileSize);
tmaSmem->bar.wait(std::move(token));
}
ncclSymkTileLoad(tmaSmem->buff[0], inpPacks, tileSize, tmaSmem->bar, pending, lane);
ncclSymkTileLoadWait</*Arrivers=*/1>(tmaSmem->bar, pending, lane);
} else
#endif
{
Expand All @@ -97,9 +91,9 @@ static __device__ void bcastDeep(ncclSymkArgsHandler const& handler, int tn, int
NVCC_PRAGMA_UNROLL_AUTO
for (int ur = 0; ur < UnrollPeers - partial; ur++) {
if (partial && dr == nRanks) break;
#if __CUDA_ARCH__ >= 1000
#if NCCL_SYMK_ASYNC_TILE
if NCCL_IF_CONSTEXPR (EnableTma) {
ptx::cp_async_bulk(ptx::space_global, ptx::space_shared, outPacks.lsaPtr(r), tmaSmem->buff[0], tileSize);
ncclSymkTileStore(outPacks.lsaPtr(r), tmaSmem->buff[0], tileSize, lane);
} else
#endif
{
Expand All @@ -110,12 +104,9 @@ static __device__ void bcastDeep(ncclSymkArgsHandler const& handler, int tn, int
}
if (++r == nRanks) r = 0;
}
#if __CUDA_ARCH__ >= 1000
#if NCCL_SYMK_ASYNC_TILE
if NCCL_IF_CONSTEXPR (EnableTma) {
if (lane == 0) {
ptx::cp_async_bulk_commit_group();
ptx::cp_async_bulk_wait_group_read(ptx::n32_t<0>());
}
ncclSymkTileStoreWait(lane);
Comment thread
alex-breslow-amd marked this conversation as resolved.
Resolved
}
#endif
}
Expand All @@ -124,14 +115,10 @@ static __device__ void bcastDeep(ncclSymkArgsHandler const& handler, int tn, int
outPacks += intptr_t(wn) * UnrollPacks * WARP_SIZE;
nIters -= wn;
if (nIters <= 0) break;
#if __CUDA_ARCH__ >= 1000
#if NCCL_SYMK_ASYNC_TILE
if NCCL_IF_CONSTEXPR (EnableTma) {
if (lane == 0) {
cuda::device::memcpy_async_tx(tmaSmem->buff[0], inpPacks, cuda::aligned_size_t<16>(tileSize), tmaSmem->bar);
cuda::barrier<cuda::thread_scope_block>::arrival_token token =
cuda::device::barrier_arrive_tx(tmaSmem->bar, 1, tileSize);
tmaSmem->bar.wait(std::move(token));
}
ncclSymkTileLoad(tmaSmem->buff[0], inpPacks, tileSize, tmaSmem->bar, pending, lane);
ncclSymkTileLoadWait</*Arrivers=*/1>(tmaSmem->bar, pending, lane);
} else
#endif
{
Expand Down Expand Up @@ -185,15 +172,15 @@ static __device__ void bcast(ncclSymkArgsHandler const& handler, int tn, int t,

uint32_t alignment = uint32_t(input.offset - output.offset);
uint32_t nPreBytes =
#if __CUDA_ARCH__ >= 1000
#if NCCL_SYMK_ASYNC_TILE
(EnableTma && alignment % 256 == 0) ? (256 - input.offset) % 256 :
#endif
(16 - input.offset) % 16;

nPreBytes = min((size_t)nPreBytes, nBytes);
uintptr_t cursor = nPreBytes;

#if __CUDA_ARCH__ >= 1000
#if NCCL_SYMK_ASYNC_TILE
if NCCL_IF_CONSTEXPR (EnableTma) {
if (alignment % 256 == 0) {
constexpr int BytePerPack = ncclSymkBytePerPack, UnrollPacks = ncclSymkAlign256BDeepUnrollPacks, UnrollPeers = 2;
Expand Down
93 changes: 32 additions & 61 deletions projects/rccl/src/device/symmetric/all_reduce.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -33,21 +33,21 @@ static __device__ __forceinline__ void allreduceDeep(ncclSymkArgsHandler const&

ncclSymPtr<Pack> inpPacks = (ncclSymPtr<Pack>)input + intptr_t(w) * UnrollPacks * WARP_SIZE +
(
#if __CUDA_ARCH__ >= 1000
#if NCCL_SYMK_ASYNC_TILE
EnableTma ? 0 :
#endif
lane);

ncclSymPtr<Pack> outPacks = (ncclSymPtr<Pack>)output + intptr_t(w) * UnrollPacks * WARP_SIZE +
(
#if __CUDA_ARCH__ >= 1000
#if NCCL_SYMK_ASYNC_TILE
EnableTma ? 0 :
#endif
lane);

Pack acc0[UnrollPacks];

#if __CUDA_ARCH__ >= 1000
#if NCCL_SYMK_ASYNC_TILE
int lw = threadIdx.x / WARP_SIZE;
extern __shared__ char smemScratch[];
using tmaSmemStruct_t = tmaSmemStruct<Pack, UnrollPacks, UnrollPeers>;
Expand All @@ -58,21 +58,15 @@ static __device__ __forceinline__ void allreduceDeep(ncclSymkArgsHandler const&
size_t tmaSize = 0;

if NCCL_IF_CONSTEXPR (EnableTma) {
if (lane == 0) {
init(&tmaSmem->bar, WARP_SIZE);
}
ncclSymkTileBarInit(&tmaSmem->bar, /*arrivers=*/WARP_SIZE, lane);
}
#endif

nIters -= w;
if (0 < nIters) {
#if __CUDA_ARCH__ >= 1000
#if NCCL_SYMK_ASYNC_TILE
if NCCL_IF_CONSTEXPR (EnableTma) {
if (lane == 0) {
cuda::device::memcpy_async_tx(tmaSmem->buff[0], inpPacks.peerPtr(world, rank),
cuda::aligned_size_t<16>(tileSize), tmaSmem->bar);
tmaSize += tileSize;
}
ncclSymkTileLoad(tmaSmem->buff[0], inpPacks.peerPtr(world, rank), tileSize, tmaSmem->bar, tmaSize, lane);
} else
#endif
{
Expand All @@ -92,17 +86,10 @@ static __device__ __forceinline__ void allreduceDeep(ncclSymkArgsHandler const&
if (++r == nRanks) r = 0;
{
Pack tmp1[UnrollPacks];
#if __CUDA_ARCH__ >= 1000
#if NCCL_SYMK_ASYNC_TILE
if NCCL_IF_CONSTEXPR (EnableTma) {
if (lane == 0) {
cuda::device::memcpy_async_tx(tmaSmem->buff[1], inpPacks.peerPtr(world, r),
cuda::aligned_size_t<16>(tileSize), tmaSmem->bar);
tmaSize += tileSize;
}
cuda::barrier<cuda::thread_scope_block>::arrival_token token =
cuda::device::barrier_arrive_tx(tmaSmem->bar, 1, tmaSize);
tmaSmem->bar.wait(std::move(token));
tmaSize = 0;
ncclSymkTileLoad(tmaSmem->buff[1], inpPacks.peerPtr(world, r), tileSize, tmaSmem->bar, tmaSize, lane);
ncclSymkTileLoadWait</*Arrivers=*/WARP_SIZE>(tmaSmem->bar, tmaSize, lane);
} else
#endif
{
Expand All @@ -113,7 +100,7 @@ static __device__ __forceinline__ void allreduceDeep(ncclSymkArgsHandler const&
}
NVCC_PRAGMA_UNROLL_AUTO
for (int u = 0; u < UnrollPacks; u++) {
#if __CUDA_ARCH__ >= 1000
#if NCCL_SYMK_ASYNC_TILE
if NCCL_IF_CONSTEXPR (EnableTma) {
acc0[u] = tmaSmem->buff[0][lane + WARP_SIZE * u];
tmp1[u] = tmaSmem->buff[1][lane + WARP_SIZE * u];
Expand All @@ -133,23 +120,20 @@ static __device__ __forceinline__ void allreduceDeep(ncclSymkArgsHandler const&
if (partial && dr == nRanks) break;

Pack tmp1[UnrollPeers][UnrollPacks];
#if __CUDA_ARCH__ >= 1000
#if NCCL_SYMK_ASYNC_TILE
if NCCL_IF_CONSTEXPR (EnableTma) {
// lane 0 waits for all threads to reduce tmp1 before next batch of TMA loads
// The next batch reloads these same tiles, so hold until every lane has
// finished reducing out of them.
__syncwarp();
}
#endif

NVCC_PRAGMA_UNROLL_AUTO
for (int ur = 0; ur < UnrollPeers - partial; ur++) {
if (partial && ur != 0 && dr + ur == nRanks) break;
#if __CUDA_ARCH__ >= 1000
#if NCCL_SYMK_ASYNC_TILE
if NCCL_IF_CONSTEXPR (EnableTma) {
if (lane == 0) {
cuda::device::memcpy_async_tx(tmaSmem->buff[ur], inpPacks.peerPtr(world, r),
cuda::aligned_size_t<16>(tileSize), tmaSmem->bar);
tmaSize += tileSize;
}
ncclSymkTileLoad(tmaSmem->buff[ur], inpPacks.peerPtr(world, r), tileSize, tmaSmem->bar, tmaSize, lane);
} else
#endif
{
Expand All @@ -160,20 +144,17 @@ static __device__ __forceinline__ void allreduceDeep(ncclSymkArgsHandler const&
}
if (++r == nRanks) r = 0;
}
#if __CUDA_ARCH__ >= 1000
#if NCCL_SYMK_ASYNC_TILE
if NCCL_IF_CONSTEXPR (EnableTma) {
cuda::barrier<cuda::thread_scope_block>::arrival_token token =
cuda::device::barrier_arrive_tx(tmaSmem->bar, 1, tmaSize);
tmaSmem->bar.wait(std::move(token));
tmaSize = 0;
ncclSymkTileLoadWait</*Arrivers=*/WARP_SIZE>(tmaSmem->bar, tmaSize, lane);
}
#endif
NVCC_PRAGMA_UNROLL_AUTO
for (int ur = 0; ur < UnrollPeers - partial; ur++) {
if (partial && ur != 0 && dr + ur == nRanks) break;
NVCC_PRAGMA_UNROLL(UnrollPacks)
for (int u = 0; u < UnrollPacks; u++) {
#if __CUDA_ARCH__ >= 1000
#if NCCL_SYMK_ASYNC_TILE
if NCCL_IF_CONSTEXPR (EnableTma) {
tmp1[ur][u] = tmaSmem->buff[ur][lane + WARP_SIZE * u];
}
Expand All @@ -186,7 +167,7 @@ static __device__ __forceinline__ void allreduceDeep(ncclSymkArgsHandler const&

NVCC_PRAGMA_UNROLL_AUTO
for (int u = 0; u < UnrollPacks; u++) {
#if __CUDA_ARCH__ >= 1000
#if NCCL_SYMK_ASYNC_TILE
if NCCL_IF_CONSTEXPR (EnableTma) {
tmaSmem->buff[0][lane + WARP_SIZE * u] = applyCast<Acc, T>(acc1[u]);
} else
Expand All @@ -196,11 +177,10 @@ static __device__ __forceinline__ void allreduceDeep(ncclSymkArgsHandler const&
}
}

#if __CUDA_ARCH__ >= 1000
#if NCCL_SYMK_ASYNC_TILE
if NCCL_IF_CONSTEXPR (EnableTma) {
// threads flush data to point of consistency for async proxy
ptx::fence_proxy_async(ptx::space_shared);
__syncwarp();
// Publish the lanes' reduced tile to the engine that is about to read it.
ncclSymkTileFenceSmem();
}
#endif

Expand All @@ -213,12 +193,9 @@ static __device__ __forceinline__ void allreduceDeep(ncclSymkArgsHandler const&
NVCC_PRAGMA_UNROLL_AUTO
for (int ur = 0; ur < UnrollPeers - partial; ur++) {
if (partial && dr == nRanks) break;
#if __CUDA_ARCH__ >= 1000
#if NCCL_SYMK_ASYNC_TILE
if NCCL_IF_CONSTEXPR (EnableTma) {
if (lane == 0) {
ptx::cp_async_bulk(ptx::space_global, ptx::space_shared, outPacks.peerPtr(world, r), tmaSmem->buff[0],
tileSize);
}
ncclSymkTileStore(outPacks.peerPtr(world, r), tmaSmem->buff[0], tileSize, lane);
} else
#endif
{
Expand All @@ -231,12 +208,10 @@ static __device__ __forceinline__ void allreduceDeep(ncclSymkArgsHandler const&
}
}
}
#if __CUDA_ARCH__ >= 1000
#if NCCL_SYMK_ASYNC_TILE
if NCCL_IF_CONSTEXPR (EnableTma) {
if (lane == 0) {
ptx::cp_async_bulk_commit_group();
ptx::cp_async_bulk_wait_group_read(ptx::n32_t<0>());
}
// Drain every peer's store before the next iteration refills the tile.
ncclSymkTileStoreWait(lane);
__syncwarp();
}
#endif
Expand All @@ -246,14 +221,10 @@ static __device__ __forceinline__ void allreduceDeep(ncclSymkArgsHandler const&
nIters -= wn;
if (nIters <= 0) break;

#if __CUDA_ARCH__ >= 1000
#if NCCL_SYMK_ASYNC_TILE
// Load data for next iteration.
if NCCL_IF_CONSTEXPR (EnableTma) {
if (lane == 0) {
cuda::device::memcpy_async_tx(tmaSmem->buff[0], inpPacks.peerPtr(world, rank),
cuda::aligned_size_t<16>(tileSize), tmaSmem->bar);
tmaSize += tileSize;
}
ncclSymkTileLoad(tmaSmem->buff[0], inpPacks.peerPtr(world, rank), tileSize, tmaSmem->bar, tmaSize, lane);
} else
#endif
{
Expand Down Expand Up @@ -351,13 +322,13 @@ static __device__ void allreduce(ncclSymkArgsHandler const& handler, int tn, int
if ((input.offset - output.offset) % 16 == 0) {
constexpr int BytePerPack = ncclSymkBytePerPack,
UnrollPacks =
#if __CUDA_ARCH__ >= 1000
EnableTma ? ncclSymkDeepUnrollPacks :
#if NCCL_SYMK_ASYNC_TILE
EnableTma ? ncclSymkDeepUnrollPacks(sizeof(T)) :
#endif
ncclSymkUnrollPacks,
UnrollPeers = 2;

constexpr int BytePerChunk = EnableTma ? ncclSymkDeepBytePerChunk : ncclSymkBytePerChunk;
constexpr int BytePerChunk = EnableTma ? ncclSymkDeepBytePerChunk(sizeof(T)) : ncclSymkBytePerChunk;
Comment thread
alex-breslow-amd marked this conversation as resolved.
Resolved
uint32_t chunks = (nBytes - cursor) / BytePerChunk;
chunks -= imodFast32(chunks, nRanks * nBlocks, nRanks_nBlocks_rcp32);
if (chunks != 0) {
Expand Down
Loading
Loading