Skip to content

perf(rccl): default gfx1250 collectives to a 192-thread block #14564

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
PedramAlizadeh merged 3 commits into from
Nov 3, 2026
Merged

perf(rccl): default gfx1250 collectives to a 192-thread block #14564

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
88459fa
perf(rccl): default gfx1250 collectives to a 192-thread block
PedramAlizadeh Oct 29, 2026
4781d71
fix(rccl): limit the gfx1250 192-thread default to DPX
PedramAlizadeh Oct 30, 2026
a8379fb
fix(rccl): keep PAT on a 256-thread block under the gfx1250 DPX cap
PedramAlizadeh Nov 1, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions projects/rccl/src/include/rccl_common.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,10 @@ typedef enum RcclTunableColls {
#define RCCL_SINGLE_NODE_MAX_NTHREADS 256
#define RCCL_GFX950_MAX_NTHREADS 256 // for Simple and LL64/LL128 gfx950
#define RCCL_DEFAULT_MAX_NTHREADS 256 // for Simple and LL64/LL128 other archs
// gfx1250 DPX launch default. SPX stays at 256 until it is measured. A lower
// NCCL_NTHREADS / NCCL_LL128_NTHREADS is not raised. The 256-thread cap stays
// RCCL_DEFAULT_MAX_NTHREADS so RCCL_THREADS_PER_BLOCK can still select 256.
#define RCCL_GFX1250_DEFAULT_NTHREADS 192
#define RCCL_LL_MAX_NTHREADS 256
#define RCCL_P2P_MAX_NTHREADS 256
#define RCCL_MI3XX_MAX_MULTI_NODE_CHANNELS 64
Expand Down
8 changes: 8 additions & 0 deletions projects/rccl/src/rccl_wrap.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2948,6 +2948,14 @@ void rcclOptThreadBlockSize(struct ncclComm* comm, struct ncclTaskColl* info, si
}
return;
}
// gfx1250 DPX: cap the block at 192. A lower NCCL_NTHREADS / NCCL_LL128_NTHREADS,
// already in nThreads, still wins. SPX is unmeasured and keeps 256.
// RCCL_THREADS_PER_BLOCK is handled above. Channel selection still uses maxThreads.
if (comm->archName && IsArchMatch(comm->archName, "gfx1250") && comm->archThresholds != nullptr &&
Comment thread
PedramAlizadeh marked this conversation as resolved.
Resolved
comm->archThresholds == rcclGetArchThresholds(comm->archName, RCCL_TUNING_PARTITION_DPX)) {
Comment thread
PedramAlizadeh marked this conversation as resolved.
Resolved
nThreads = std::min(nThreads, RCCL_GFX1250_DEFAULT_NTHREADS);
Comment thread
PedramAlizadeh marked this conversation as resolved.
Resolved
return;
}
if (info->algorithm == NCCL_ALGO_TREE) nThreads = maxNthreads[NCCL_PROTO_SIMPLE]; // Tree now uses all threads always.
if (info->algorithm == NCCL_ALGO_PAT) nThreads = maxNthreads[NCCL_PROTO_SIMPLE];
if (comm->nNodes == 1)
Expand Down
101 changes: 101 additions & 0 deletions projects/rccl/test/host/wrap-test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@
// binary -- declaring a second copy here would be a duplicate-symbol error.
#include "fakes/nccl_fakes.h" // extern g_loadParam
#include "fakes/param_redirect.h" // NCCL_PARAM/RCCL_PARAM -> g_loadParam
#include "fakes/rccl_arch_thresholds_fakes.h" // g_archThresholdsTable (DPX vs SPX)

// WRAP_CC_PATH is defined by test/host/CMakeLists.txt as the hipified copy of
// src/rccl_wrap.cc, e.g. ${PROJECT_BINARY_DIR}/hipify/src/rccl_wrap.cc.
Expand Down Expand Up @@ -3278,6 +3279,106 @@ TEST(WrapMicrotestIsolated, OptThreadBlockSize_SingleNodeUsesHalfThreads) {
});
}

// gfx1250 DPX caps a 256-thread request at 192. Single-node would otherwise
// force RCCL_SINGLE_NODE_MAX_NTHREADS, and multi-node LL would force
// RCCL_LL_MAX_NTHREADS. nThreads is seeded with 256, the value the caller
// passes from maxThreads when NCCL_NTHREADS is unset.
TEST(WrapMicrotestIsolated, OptThreadBlockSize_Gfx1250DpxDefaultIs192) {
RUN_ISOLATED_TEST(
"Wrap_OptThreadBlockSize_Gfx1250DpxDefaultIs192",
[]() {
static rcclArchThresholds spx{};
static rcclArchThresholds dpx{};
g_archThresholdsTable[RCCL_TUNING_PARTITION_SPX] = &spx;
g_archThresholdsTable[RCCL_TUNING_PARTITION_DPX] = &dpx;
ncclComm* comm = MakeCommWithArch("gfx1250");
comm->WarpSize = 32;
comm->archThresholds = &dpx;
ncclTaskColl info{};
info.algorithm = NCCL_ALGO_RING;
info.protocol = NCCL_PROTO_SIMPLE;
info.func = ncclFuncBroadcast;
int nThreads = 256;
comm->nNodes = 1;
rcclOptThreadBlockSize(comm, &info, /*nBytes=*/1 << 20, nThreads);
EXPECT_EQ(RCCL_GFX1250_DEFAULT_NTHREADS, nThreads);
comm->nNodes = 2;
info.protocol = NCCL_PROTO_LL;
nThreads = 256;
rcclOptThreadBlockSize(comm, &info, /*nBytes=*/1024, nThreads);
EXPECT_EQ(RCCL_GFX1250_DEFAULT_NTHREADS, nThreads);
DeleteCommWithArch(comm);
});
}

// A lower NCCL_NTHREADS / NCCL_LL128_NTHREADS is already in nThreads. DPX must
// not raise it to 192.
TEST(WrapMicrotestIsolated, OptThreadBlockSize_Gfx1250DpxKeepsLowerNthreads) {
RUN_ISOLATED_TEST(
"Wrap_OptThreadBlockSize_Gfx1250DpxKeepsLowerNthreads",
[]() {
static rcclArchThresholds dpx{};
g_archThresholdsTable[RCCL_TUNING_PARTITION_DPX] = &dpx;
ncclComm* comm = MakeCommWithArch("gfx1250");
comm->WarpSize = 32;
comm->nNodes = 2;
comm->archThresholds = &dpx;
ncclTaskColl info{};
info.algorithm = NCCL_ALGO_RING;
info.protocol = NCCL_PROTO_SIMPLE;
info.func = ncclFuncBroadcast;
int nThreads = 128;
rcclOptThreadBlockSize(comm, &info, /*nBytes=*/1 << 20, nThreads);
EXPECT_EQ(128, nThreads);
DeleteCommWithArch(comm);
});
}

// SPX is unmeasured, so gfx1250 SPX keeps the 256-thread single-node default.
TEST(WrapMicrotestIsolated, OptThreadBlockSize_Gfx1250SpxStays256) {
RUN_ISOLATED_TEST(
"Wrap_OptThreadBlockSize_Gfx1250SpxStays256",
[]() {
static rcclArchThresholds spx{};
static rcclArchThresholds dpx{};
g_archThresholdsTable[RCCL_TUNING_PARTITION_SPX] = &spx;
g_archThresholdsTable[RCCL_TUNING_PARTITION_DPX] = &dpx;
ncclComm* comm = MakeCommWithArch("gfx1250");
comm->WarpSize = 32;
comm->nNodes = 1;
comm->archThresholds = &spx;
ncclTaskColl info{};
info.algorithm = NCCL_ALGO_RING;
info.protocol = NCCL_PROTO_SIMPLE;
info.func = ncclFuncBroadcast;
int nThreads = 256;
rcclOptThreadBlockSize(comm, &info, /*nBytes=*/1 << 20, nThreads);
EXPECT_EQ(RCCL_SINGLE_NODE_MAX_NTHREADS, nThreads);
DeleteCommWithArch(comm);
});
}

// The 192 default must not lower the cap: an explicit 256-thread request on
// gfx1250 still launches 256.
TEST(WrapMicrotestIsolated, OptThreadBlockSize_Gfx1250UserOverrideKeeps256) {
RUN_ISOLATED_TEST(
"Wrap_OptThreadBlockSize_Gfx1250UserOverrideKeeps256",
[]() {
g_loadParam = ForceParam("RCCL_THREADS_PER_BLOCK", int64_t(256));
ncclComm* comm = MakeCommWithArch("gfx1250");
comm->WarpSize = 32;
ncclTaskColl info{};
info.algorithm = NCCL_ALGO_RING;
info.protocol = NCCL_PROTO_SIMPLE;
info.func = ncclFuncBroadcast;
comm->nNodes = 1;
int nThreads = -1;
rcclOptThreadBlockSize(comm, &info, /*nBytes=*/1 << 20, nThreads);
EXPECT_EQ(256, nThreads);
DeleteCommWithArch(comm);
});
}

TEST(WrapMicrotestIsolated, OptThreadBlockSize_ReduceScatterSmallCountUsesLLThreads) {
RUN_ISOLATED_TEST(
"Wrap_OptThreadBlockSize_ReduceScatterSmallCountUsesLLThreads",
Expand Down
Loading