Skip to content
Closed
Show file tree
Hide file tree
Changes from 7 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions projects/rccl/src/graph/paths.cc
Original file line number Diff line number Diff line change
Expand Up @@ -807,6 +807,9 @@ int ncclPxnDisable(struct ncclComm* comm) {
// pxnDisable); without one, honour the env default and skip PXN.
if (comm == NULL) return ncclParamPxnDisable();
if (comm->pxnDisable > RCCL_VALUE_INVALID) return comm->pxnDisable;
// commDestroySync() consults this on the teardown path, where a comm whose init failed before the topology was
// built can still reach us with a live user handle. rcclSetPxn() reads comm->topo unconditionally.
if (comm->topo == NULL) return ncclParamPxnDisable();
if (comm->ncclNetVer == 4) {
INFO(NCCL_INIT, "PXN Disabled as plugin is v4");
comm->pxnDisable = 1;
Expand Down
1 change: 1 addition & 0 deletions projects/rccl/src/include/comm.h
Original file line number Diff line number Diff line change
Expand Up @@ -1054,6 +1054,7 @@ struct ncclComm {
bool globalRmaProxySupport;
bool hostRmaSupport;
int childCount;
bool hasExcludedHostRank; // NCCL_SHRINK_ABORT excluded a rank on this host; skip destroy barrier

struct ncclDevrState devrState; // The symmetric runtime state
struct ncclSymkState symkState; // The symmetric kernels state (built on previous)
Expand Down
25 changes: 21 additions & 4 deletions projects/rccl/src/init.cc
Original file line number Diff line number Diff line change
Expand Up @@ -4260,14 +4260,20 @@ static ncclResult_t commDestroySync(struct ncclAsyncJob* job_) {
comm->commHash, comm->rank);
}
}
if (COMPILER_ATOMIC_LOAD(comm->abortFlag, std::memory_order_acquire) == 0) {
if (COMPILER_ATOMIC_LOAD(comm->abortFlag, std::memory_order_acquire) == 0 && ncclPxnDisable(comm) == 0 &&
Comment thread
prasanna-amd marked this conversation as resolved.
Resolved
!comm->hasExcludedHostRank) {
Comment thread
prasanna-amd marked this conversation as resolved.
Resolved
int* hostRanks;
int hostRank = 0;
int nHostRanks = 0;
// Wait for all host-local ranks before stopping the proxy threads, to ensure that PXN connection establishment
// can complete if some ranks were to try destroying the communicator early. As an optimization, filter
// comm->localRanks to the local host only, since on MNNVL systems it can include other hosts, while PXN is
// strictly host-local.
// can complete if some ranks were to try destroying the communicator early. The barrier only exists to protect
// PXN, so skip it when PXN is disabled for this comm; it is a cross-process rendezvous with no cancellation, and
// running it when nothing depends on it is how the single-node teardown deadlock happens. ncclPxnDisable() is
// derived from arch, nRanks and env, so every rank agrees on it -- which matters here, because a barrier that
// some ranks enter and others skip is exactly what hangs. Skip when hasExcludedHostRank is set because an
// NCCL_SHRINK_ABORT excluded a peer on this host, which has aborted and will never arrive. As an optimization,
// filter comm->localRanks to the local host only, since on MNNVL systems it can include other hosts, while PXN
// is strictly host-local.
Comment thread
prasanna-amd marked this conversation as resolved.
Resolved
NCCLCHECKGOTO(ncclCalloc(&hostRanks, comm->localRanks), ret, fail);
for (int i = 0; i < comm->localRanks; i++) {
if (comm->peerInfo[comm->localRankToRank[i]].hostHash == comm->peerInfo[comm->rank].hostHash) {
Expand Down Expand Up @@ -4767,6 +4773,17 @@ static ncclResult_t ncclCommInitChildComm(ncclComm_t comm, ncclComm_t* newcomm,
job->excludeRanksCount = excludeRanksCount;
NCCLCHECKGOTO(ncclCalloc(&job->excludeRanksList, excludeRanksCount), res, fail);
memcpy(job->excludeRanksList, excludeRanksList, excludeRanksCount * sizeof(int));
// Only NCCL_SHRINK_ABORT leaves behind an excluded rank that will never reach the destroy barrier.
// Under NCCL_SHRINK_DEFAULT the excluded rank is healthy and still destroys its parent normally, so it
// does enter the barrier -- skipping it here would hang that rank instead.
if ((flags & NCCL_SHRINK_ABORT) && comm->peerInfo) {
for (int i = 0; i < excludeRanksCount; i++) {
if (comm->peerInfo[excludeRanksList[i]].hostHash == comm->peerInfo[comm->rank].hostHash) {
comm->hasExcludedHostRank = true;
break;
}
Comment thread
prasanna-amd marked this conversation as resolved.
Resolved
}
}
} else {
// each split has to lead to a unique comm, so increment the childCount
job->childCount = ++comm->childCount;
Expand Down
186 changes: 186 additions & 0 deletions projects/rccl/test/TeardownStressTests.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,16 @@
*
* See LICENSE.txt for license information
************************************************************************/
#include <signal.h>
#include <sys/wait.h>
#include <unistd.h>

#include <chrono>
#include <cstdlib>
#include <thread>

#include "TestBed.hpp"
#include "common/ProcessIsolatedTestRunner.hpp"

namespace RcclUnitTesting
{
Expand Down Expand Up @@ -104,4 +113,181 @@ namespace RcclUnitTesting
EXPECT_TRUE(isCorrect);
testBed.Finalize();
}

// AICOMRCCL-2467 / AICOMRCCL-2468: commDestroySync() runs a host-local barrier
// (bootstrapIntraNodeBarrier) before stopping the proxy, so that a rank cannot
// stop its proxy while a host-local peer is still establishing a PXN
// connection. The barrier requires every host-local rank to participate, and
// nothing cancels it when one of them does not. A rank with a non-zero
// abortFlag skips the barrier, so a peer that entered it waits forever in
// bootstrapRecv -- the PyTorch ProcessGroupNCCL watchdog hits exactly this.
//
// These run on a single-node communicator at nRanks == 2, where the barrier
// cannot be protecting anything: RCCL defaults NCCL_PXN_DISABLE=1 and
// rcclSetPxn() only auto-enables PXN at nRanks >= 64 (gfx942) / >= 32
// (gfx950), so there is no PXN relay to keep alive.
namespace
{
// EnvVars reports 0 GPUs in a process re-exec'd by ProcessIsolatedTestRunner, and a
// skipped child is scored as a pass -- so every GPU gate below must run in the parent
// only, or the coverage silently evaporates into a green run.
bool isIsolatedChild()
{
return std::getenv(ProcessIsolatedTestRunner::kReexecMarkerEnvVar) != nullptr;
}

// Never call hipGetDeviceCount() here: HIP state does not survive the fork() below.
int getDetectedGpuCount()
{
static const int detectedGpus = EnvVars().GetNumDetectedGpus();
return detectedGpus;
}

enum class PeerTeardown
{
Abort, // peer calls ncclCommAbort -- non-zero abortFlag, skips the barrier
Destroy // peer calls ncclCommDestroy -- control, both ranks reach the barrier
};

// Two ranks, one process each, one collective, then a deliberately divergent
// teardown. Returns true if the rank taking the normal destroy path returned
// from ncclCommDestroy before the deadline.
//
// Nothing here may touch HIP or RCCL before fork(): initializing the runtime
// in the parent leaves the children with unusable state and they die in
// hipSetDevice.
bool DestroyReturnsWhenPeer(PeerTeardown peer, int deadlineSeconds)
{
constexpr int nRanks = 2;
constexpr size_t numElements = 32 * 1024;

int idpipe[2];
if (pipe(idpipe) != 0) return false;

pid_t kids[nRanks] = {-1, -1};
for (int rank = 0; rank < nRanks; ++rank)
{
pid_t pid = fork();
if (pid < 0)
{
close(idpipe[0]);
close(idpipe[1]);
return false;
}
if (pid == 0)
{
ncclUniqueId id;
if (rank == 0)
{
close(idpipe[0]);
if (ncclGetUniqueId(&id) != ncclSuccess) _exit(70);
if (write(idpipe[1], &id, sizeof(id)) != (ssize_t)sizeof(id)) _exit(71);
close(idpipe[1]);
}
else
{
close(idpipe[1]);
size_t got = 0;
while (got < sizeof(id))
{
ssize_t n = read(idpipe[0], ((char*)&id) + got, sizeof(id) - got);
if (n <= 0) _exit(72);
got += n;
}
close(idpipe[0]);
}

if (hipSetDevice(rank) != hipSuccess) _exit(73);
hipStream_t stream;
if (hipStreamCreate(&stream) != hipSuccess) _exit(74);
float* buf = nullptr;
if (hipMalloc(&buf, numElements * sizeof(float)) != hipSuccess) _exit(75);
if (hipMemset(buf, 0, numElements * sizeof(float)) != hipSuccess) _exit(76);

ncclComm_t comm;
if (ncclCommInitRank(&comm, nRanks, id, rank) != ncclSuccess) _exit(77);
if (ncclAllReduce(buf, buf, numElements, ncclFloat, ncclSum, comm, stream) != ncclSuccess)
_exit(78);
if (hipStreamSynchronize(stream) != hipSuccess) _exit(79);

if (rank == 1 && peer == PeerTeardown::Abort)
ncclCommAbort(comm);
else
ncclCommDestroy(comm); // rank 0 always takes the barrier path

_exit(0);
Comment thread
prasanna-amd marked this conversation as resolved.
Resolved
}
kids[rank] = pid;
}
close(idpipe[0]);
close(idpipe[1]);

// Poll for rank 0 only. It is the one that enters the barrier, so it is
// the one that hangs on regression.
bool returned = false;
auto const deadline =
std::chrono::steady_clock::now() + std::chrono::seconds(deadlineSeconds);
while (std::chrono::steady_clock::now() < deadline)
{
int status = 0;
pid_t done = waitpid(kids[0], &status, WNOHANG);
if (done == kids[0])
{
// WEXITSTATUS is meaningless for a signal-killed child; checking it
// alone would report a crash as a clean pass.
returned = !WIFSIGNALED(status) && WEXITSTATUS(status) == 0;
break;
}
if (done < 0) break;
std::this_thread::sleep_for(std::chrono::milliseconds(50));
}

for (int i = 0; i < nRanks; ++i)
{
if (kids[i] <= 0) continue;
kill(kids[i], SIGKILL);
waitpid(kids[i], nullptr, 0);
}
return returned;
}
}

// Control: both ranks destroy normally, so both reach the barrier and it
// completes. If this fails the harness itself is broken and the companion
// test below proves nothing.
TEST(Teardown, DivergentTeardown_PeerDestroys_Completes)
{
if (!isIsolatedChild() && getDetectedGpuCount() < 2)
GTEST_SKIP() << "Divergent teardown requires at least 2 GPUs (detected "
<< getDetectedGpuCount() << ")";

RUN_ISOLATED_TESTS(
ProcessIsolatedTestRunner::TestConfig(
"DivergentTeardown_PeerDestroys",
[]() {
EXPECT_TRUE(DestroyReturnsWhenPeer(PeerTeardown::Destroy, /*deadlineSeconds*/ 60))
<< "rank 0 did not return from ncclCommDestroy even though its peer "
"also destroyed -- the host-local destroy barrier is broken";
})
.withTimeout(std::chrono::seconds(120)));
}

// Regression: the peer aborts, so it skips the barrier. Rank 0 must still
// return instead of blocking forever in bootstrapRecv.
TEST(Teardown, DivergentTeardown_PeerAborts_DoesNotHang)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This test pins NCCL_PXN_DISABLE=1 and calls ncclCommAbort with no shrink, so it passes via the PXN skip, not via the filtering. The main new logic of this PR has no end-to-end test. Can we add a real shrink(SHRINK_ABORT) + destroy repro with PXN on that actually hits the filter?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The filtering this was asking about is gone as of b7bc5b6, so the specific test you asked for -- shrink(SHRINK_ABORT) + destroy with PXN on, hitting the filter -- no longer has anything to hit.

But I do not want to close this out on that technicality, because the underlying complaint survives the revert: the main new logic of this PR still has no end-to-end test. It is now the timeout rather than the filter, and the gap is the same one you identified. What exists today:

  • DivergentTeardown_PeerAborts_DoesNotHang pins NCCL_PXN_DISABLE=1, so it passes via the PXN skip and never reaches the barrier -- exactly your original point, still true.
  • DivergentTeardown_PeerDestroys_PxnEnabled_Completes pins NCCL_PXN_DISABLE=0 and does run the barrier, but both ranks arrive, so it proves the barrier still works rather than proving the deadline fires.

What is missing is the case that matters: NCCL_PXN_DISABLE=0 plus one rank aborting, asserting that the survivor returns from ncclCommDestroy in about the timeout rather than hanging, and with a non-success rc so the assertion is falsifiable. That is the test that would have failed before this PR and passes after it. I am writing it next, along with host-level coverage of the timeout value actually reaching bootstrapIntraNodeBarrierTimeout and of an expiry not failing teardown.

One caveat I would rather state than have you discover: I cannot run GPU tests from my current checkout -- the compute nodes on this cluster have no path to it -- so the end-to-end case will be verified in PR CI, not locally. I will post the run link rather than just asserting it passes.

Holding this thread open until that is up.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The test is up, in 38d7a5b5e5d: DivergentTeardown_PeerAborts_PxnEnabled_TimesOut.

NCCL_PXN_DISABLE=0 plus one rank calling ncclCommAbort. Rank 0 genuinely enters the barrier, the peer never arrives, and only the deadline can release it. Before this PR that hung forever.

Two details aimed at the vacuous-pass problem you raised, since "the test passes" is worth very little on its own here:

  • RCCL_DESTROY_BARRIER_TIMEOUT_MSEC is pinned to 5000 in the child. At the 120000 default the runner's own timeout fires first and the case reports a hang rather than the deadline working.
  • Destroy must take at least 4000ms. This is the part that keeps it honest: a prompt return would mean the barrier was skipped via the PXN gate, so the deadline was never exercised -- precisely the "passes via the skip, not via the new logic" failure you called out originally. Without this floor the case would still go green if someone later widened the gate.

Expiry reports ncclRemoteError, which is correct rather than a failure, so this case asserts on duration instead of rc. The other two keep the strict rc == ncclSuccess check from pvallem's thread.

Also removed a comment above DivergentTeardown_PeerAborts_DoesNotHang that still claimed the PXN-on abort case "needs a cancellation protocol and is tracked separately". That was true of the old design and would have warned the next person off writing exactly this test.

Caveat I stated earlier and will restate: I cannot run GPU tests from my checkout, so this is verified by syntax check locally and must be proven in CI. I am not claiming it passes until I can post the run. CI has not run on this branch since 2026-10-08 -- the ci:single-arch label gate explains part of that but not all of it, and I am chasing the rest. Holding this thread open until the run link is here.

{
if (!isIsolatedChild() && getDetectedGpuCount() < 2)
GTEST_SKIP() << "Divergent teardown requires at least 2 GPUs (detected "
<< getDetectedGpuCount() << ")";

RUN_ISOLATED_TESTS(
ProcessIsolatedTestRunner::TestConfig(
"DivergentTeardown_PeerAborts",
[]() {
EXPECT_TRUE(DestroyReturnsWhenPeer(PeerTeardown::Abort, /*deadlineSeconds*/ 60))
<< "rank 0 hung in ncclCommDestroy after its host-local peer aborted "
"and skipped the destroy barrier (AICOMRCCL-2467 / AICOMRCCL-2468)";
})
.withTimeout(std::chrono::seconds(120)));
}
Comment thread
prasanna-amd marked this conversation as resolved.
Resolved
}
79 changes: 79 additions & 0 deletions projects/rccl/test/host/init-test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -8053,6 +8053,11 @@ class Grow_ParentComm {
comm_->cudaDev = kGrow_ParentCudaDev;
comm_->config.minCTAs = kGrow_ParentMinCTAs;
comm_->config.maxCTAs = kGrow_ParentMinCTAs;
for (int i = 0; i < kGrow_TargetRanks; i++) {
memset(&peerInfo_[i], 0, sizeof(peerInfo_[i]));
peerInfo_[i].hostHash = static_cast<uint64_t>(i) + 1;
}
comm_->peerInfo = peerInfo_;
}
ncclComm* get() { return comm_.get(); }
ncclComm* operator->() { return comm_.get(); }
Expand All @@ -8061,6 +8066,7 @@ class Grow_ParentComm {
uint32_t abortFlag_ = 0;
uint32_t abortFlagDev_ = 0;
int abortFlagRefCount_ = 1;
ncclPeerInfo peerInfo_[kGrow_TargetRanks] = {};
Comment thread
prasanna-amd marked this conversation as resolved.
Resolved
std::unique_ptr<ncclComm> comm_;
};

Expand Down Expand Up @@ -8922,6 +8928,79 @@ TEST_F(InitMicrotest, InitChildComm_Shrink_SortsCallerListAndCopiesItIntoTheJob)
Rank_ReleaseComm(r.jobComm, /*ownsAbortResources=*/true);
}

TEST_F(InitMicrotest, InitChildComm_ShrinkAbort_ExcludingSameHostRank_SetsHasExcludedHostRank) {
Grow_AllowHostAlloc();
Grow_ParentComm parent;
parent->peerInfo[0].hostHash = 0xAAAA;
parent->peerInfo[1].hostHash = 0xAAAA;
parent->peerInfo[2].hostHash = 0xBBBB;
parent->peerInfo[3].hostHash = 0xBBBB;
parent->rank = 0;
Comment thread
prasanna-amd marked this conversation as resolved.
Resolved
int exclude[1] = {1};
Grow_LaunchSpy spy;
ncclComm_t out = kGrow_NewcommPoison;

ASSERT_EQ(ncclSuccess, Grow_RunShrink(parent.get(), NCCL_SHRINK_ABORT, exclude, 1, &out));

EXPECT_TRUE(parent->hasExcludedHostRank) << "excluded rank 1 is on same host as rank 0";
Rank_ReleaseComm(spy.rec().jobComm, /*ownsAbortResources=*/true);
}

TEST_F(InitMicrotest, InitChildComm_ShrinkDefault_ExcludingSameHostRank_DoesNotSetFlag) {
Grow_AllowHostAlloc();
Grow_ParentComm parent;
parent->peerInfo[0].hostHash = 0xAAAA;
parent->peerInfo[1].hostHash = 0xAAAA;
parent->peerInfo[2].hostHash = 0xBBBB;
parent->peerInfo[3].hostHash = 0xBBBB;
parent->rank = 0;
int exclude[1] = {1};
Grow_LaunchSpy spy;
ncclComm_t out = kGrow_NewcommPoison;

ASSERT_EQ(ncclSuccess, Grow_RunShrink(parent.get(), NCCL_SHRINK_DEFAULT, exclude, 1, &out));

EXPECT_FALSE(parent->hasExcludedHostRank)
<< "a NCCL_SHRINK_DEFAULT excluded rank is healthy and still enters the destroy barrier";
Rank_ReleaseComm(spy.rec().jobComm, /*ownsAbortResources=*/true);
}

TEST_F(InitMicrotest, InitChildComm_ShrinkAbort_ExcludingOnlyRemoteHostRank_DoesNotSetFlag) {
Grow_AllowHostAlloc();
Grow_ParentComm parent;
parent->peerInfo[0].hostHash = 0xAAAA;
parent->peerInfo[1].hostHash = 0xAAAA;
parent->peerInfo[2].hostHash = 0xBBBB;
parent->peerInfo[3].hostHash = 0xBBBB;
parent->rank = 0;
int exclude[1] = {3};
Grow_LaunchSpy spy;
ncclComm_t out = kGrow_NewcommPoison;

ASSERT_EQ(ncclSuccess, Grow_RunShrink(parent.get(), NCCL_SHRINK_ABORT, exclude, 1, &out));

EXPECT_FALSE(parent->hasExcludedHostRank) << "excluded rank 3 is on a different host from rank 0";
Rank_ReleaseComm(spy.rec().jobComm, /*ownsAbortResources=*/true);
}

TEST_F(InitMicrotest, InitChildComm_ShrinkAbort_MultiExcludeHitsLocalOnSecondEntry_SetsFlag) {
Grow_AllowHostAlloc();
Grow_ParentComm parent;
parent->peerInfo[0].hostHash = 0xAAAA;
parent->peerInfo[1].hostHash = 0xBBBB;
parent->peerInfo[2].hostHash = 0xBBBB;
parent->peerInfo[3].hostHash = 0xAAAA;
parent->rank = 0;
int exclude[2] = {1, 3};
Grow_LaunchSpy spy;
ncclComm_t out = kGrow_NewcommPoison;

ASSERT_EQ(ncclSuccess, Grow_RunShrink(parent.get(), NCCL_SHRINK_ABORT, exclude, 2, &out));

EXPECT_TRUE(parent->hasExcludedHostRank) << "rank 3 shares host with rank 0; loop must continue past rank 1";
Rank_ReleaseComm(spy.rec().jobComm, /*ownsAbortResources=*/true);
}

TEST_F(InitMicrotest, InitChildComm_ConfigProvided_ParsesInsteadOfCopyingParent) {
Grow_AllowHostAlloc();
Grow_ParentComm parent;
Expand Down
Loading