Skip to content

fix(rccl): Redirect P2P SendRecv protocol logs to NCCL_COLL debug subsystem #11010

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rahulvaidya20 merged 1 commit into from
Sep 1, 2026
Merged

fix(rccl): Redirect P2P SendRecv protocol logs to NCCL_COLL debug subsystem #11010

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions projects/rccl/src/enqueue.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1303,10 +1303,10 @@ static ncclResult_t addP2pToPlan(struct ncclComm* comm, struct ncclKernelPlan* p
if (bytes[dir] != -1) protoLatency[dir] &= bytes[dir] <= nChannels[dir] * latencyThreshold;
protocol[dir] = protoLatency[dir] ? (useLL128SendRecv ? NCCL_PROTO_LL128 : NCCL_PROTO_LL) : NCCL_PROTO_SIMPLE;

// Emit the selected protocol so tests (and NCCL_DEBUG=INFO) can confirm the latency protocol
// was actually chosen rather than silently falling back to SIMPLE.
// Emit the selected protocol so tests (and NCCL_DEBUG=INFO with NCCL_DEBUG_SUBSYS=COLL) can confirm
// the latency protocol was actually chosen rather than silently falling back to SIMPLE.
if (bytes[dir] != -1)
INFO(NCCL_INIT, "RCCL P2P SendRecv protocol=%s dir=%s bytes=%ld nChannels=%d", ncclProtoStr[protocol[dir]],
INFO(NCCL_COLL, "RCCL P2P SendRecv protocol=%s dir=%s bytes=%ld nChannels=%d", ncclProtoStr[protocol[dir]],
dir ? "send" : "recv", (long)bytes[dir], nChannels[dir]);

stepSize[dir] = comm->buffSizes[protocol[dir]] / NCCL_STEPS;
Expand Down
25 changes: 13 additions & 12 deletions projects/rccl/test/SendRecvTests.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,10 +47,11 @@ namespace RcclUnitTesting
return a.find("gfx942") != std::string::npos || a.find("gfx950") != std::string::npos;
}

// Scan the NCCL_DEBUG=INFO log files matching globPattern for the per-op protocol line emitted by
// the P2P send/recv enqueue path ("RCCL P2P SendRecv protocol=<proto>"), returning true if any
// line reports the given protocol ("LL128" or "Simple"). Used to assert that LL128 (or SIMPLE) was
// actually selected rather than relying on data correctness alone, which both protocols satisfy.
// Scan the NCCL_DEBUG_SUBSYS=COLL log files matching globPattern for the per-op protocol line
// emitted by the P2P send/recv enqueue path ("RCCL P2P SendRecv protocol=<proto>"), returning
// true if any line reports the given protocol ("LL128" or "Simple"). Used to assert that LL128
// (or SIMPLE) was actually selected rather than relying on data correctness alone, which both
// protocols satisfy.
Comment thread
rahulvaidya20 marked this conversation as resolved.
Resolved
static bool DebugLogsContainProtocol(const std::string& globPattern, const char* protocol)
{
// Trailing space so "LL" does not also match the "LL128" log line (the enqueue log prints
Expand Down Expand Up @@ -282,7 +283,7 @@ namespace RcclUnitTesting
// buffer allocation for correctness.
//
// Protocol selection is asserted (not just data correctness, which SIMPLE also satisfies) by
// scraping the NCCL_DEBUG=INFO protocol line. Protocol is chosen per channel (payload <=
// scraping the NCCL_DEBUG_SUBSYS=COLL protocol line. Protocol is chosen per channel (payload <=
// nChannels * <threshold>), so the caller pins NCCL_MAX_P2P_NCHANNELS=1 (single channel) and pins
// the relevant threshold env var (NCCL_P2P_LL128_THRESHOLD or NCCL_P2P_LL_THRESHOLD) to 16384 so
// the 16 KiB boundary the element counts below straddle is deterministic and independent of the
Expand Down Expand Up @@ -503,7 +504,7 @@ namespace RcclUnitTesting
std::string const debugGlob = "/tmp/rccl_legacy_ll_" + std::to_string(getpid()) + ".*";
RemoveGlobbedFiles(debugGlob);
setenv("NCCL_DEBUG", "INFO", 1);
setenv("NCCL_DEBUG_SUBSYS", "INIT", 1);
setenv("NCCL_DEBUG_SUBSYS", "COLL", 1);
setenv("NCCL_DEBUG_FILE", ("/tmp/rccl_legacy_ll_" + std::to_string(getpid()) + ".%p").c_str(), 1);
{
TestBed testBed;
Expand Down Expand Up @@ -534,7 +535,7 @@ namespace RcclUnitTesting
std::string const debugGlob = "/tmp/rccl_ll128_disabled_" + std::to_string(getpid()) + ".*";
RemoveGlobbedFiles(debugGlob);
setenv("NCCL_DEBUG", "INFO", 1);
setenv("NCCL_DEBUG_SUBSYS", "INIT", 1);
setenv("NCCL_DEBUG_SUBSYS", "COLL", 1);
setenv("NCCL_DEBUG_FILE", ("/tmp/rccl_ll128_disabled_" + std::to_string(getpid()) + ".%p").c_str(), 1);
{
TestBed testBed;
Expand Down Expand Up @@ -571,7 +572,7 @@ namespace RcclUnitTesting
std::string const debugGlob = "/tmp/rccl_ll128_enabled_" + std::to_string(getpid()) + ".*";
RemoveGlobbedFiles(debugGlob);
setenv("NCCL_DEBUG", "INFO", 1);
setenv("NCCL_DEBUG_SUBSYS", "INIT", 1);
setenv("NCCL_DEBUG_SUBSYS", "COLL", 1);
setenv("NCCL_DEBUG_FILE", ("/tmp/rccl_ll128_enabled_" + std::to_string(getpid()) + ".%p").c_str(), 1);
{
TestBed testBed;
Expand Down Expand Up @@ -612,7 +613,7 @@ namespace RcclUnitTesting
std::string const debugGlob = "/tmp/rccl_ll128_disabled_gate_" + std::to_string(getpid()) + ".*";
RemoveGlobbedFiles(debugGlob);
setenv("NCCL_DEBUG", "INFO", 1);
setenv("NCCL_DEBUG_SUBSYS", "INIT", 1);
setenv("NCCL_DEBUG_SUBSYS", "COLL", 1);
setenv("NCCL_DEBUG_FILE", ("/tmp/rccl_ll128_disabled_gate_" + std::to_string(getpid()) + ".%p").c_str(), 1);
{
TestBed testBed;
Expand Down Expand Up @@ -645,7 +646,7 @@ namespace RcclUnitTesting
std::string const debugGlob = "/tmp/rccl_ll128_indep_" + std::to_string(getpid()) + ".*";
RemoveGlobbedFiles(debugGlob);
setenv("NCCL_DEBUG", "INFO", 1);
setenv("NCCL_DEBUG_SUBSYS", "INIT", 1);
setenv("NCCL_DEBUG_SUBSYS", "COLL", 1);
setenv("NCCL_DEBUG_FILE", ("/tmp/rccl_ll128_indep_" + std::to_string(getpid()) + ".%p").c_str(), 1);
{
TestBed testBed;
Expand Down Expand Up @@ -689,7 +690,7 @@ namespace RcclUnitTesting
std::string const debugGlob = "/tmp/rccl_sep_ll128_" + std::to_string(getpid()) + ".*";
RemoveGlobbedFiles(debugGlob);
setenv("NCCL_DEBUG", "INFO", 1);
setenv("NCCL_DEBUG_SUBSYS", "INIT", 1);
setenv("NCCL_DEBUG_SUBSYS", "COLL", 1);
setenv("NCCL_DEBUG_FILE", ("/tmp/rccl_sep_ll128_" + std::to_string(getpid()) + ".%p").c_str(), 1);
{
TestBed testBed;
Expand Down Expand Up @@ -726,7 +727,7 @@ namespace RcclUnitTesting
std::string const debugGlob = "/tmp/rccl_sep_legacy_ll_" + std::to_string(getpid()) + ".*";
RemoveGlobbedFiles(debugGlob);
setenv("NCCL_DEBUG", "INFO", 1);
setenv("NCCL_DEBUG_SUBSYS", "INIT", 1);
setenv("NCCL_DEBUG_SUBSYS", "COLL", 1);
setenv("NCCL_DEBUG_FILE", ("/tmp/rccl_sep_legacy_ll_" + std::to_string(getpid()) + ".%p").c_str(), 1);
{
TestBed testBed;
Expand Down
Loading