Repository navigation
fix(rccl): gate and bound the destroy-path intranode barrier #12446
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 7 commits
77c7d8c
326f5f6
9e9f606
829fd93
3545856
baf39df
e66003d
7118726
b7bc5b6
e29dfad
5f363b8
c0e6c60
38d7a5b
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -3,7 +3,16 @@ | |
| * | ||
| * See LICENSE.txt for license information | ||
| ************************************************************************/ | ||
| #include <signal.h> | ||
| #include <sys/wait.h> | ||
| #include <unistd.h> | ||
|
|
||
| #include <chrono> | ||
| #include <cstdlib> | ||
| #include <thread> | ||
|
|
||
| #include "TestBed.hpp" | ||
| #include "common/ProcessIsolatedTestRunner.hpp" | ||
|
|
||
| namespace RcclUnitTesting | ||
| { | ||
|
|
@@ -104,4 +113,181 @@ namespace RcclUnitTesting | |
| EXPECT_TRUE(isCorrect); | ||
| testBed.Finalize(); | ||
| } | ||
|
|
||
| // AICOMRCCL-2467 / AICOMRCCL-2468: commDestroySync() runs a host-local barrier | ||
| // (bootstrapIntraNodeBarrier) before stopping the proxy, so that a rank cannot | ||
| // stop its proxy while a host-local peer is still establishing a PXN | ||
| // connection. The barrier requires every host-local rank to participate, and | ||
| // nothing cancels it when one of them does not. A rank with a non-zero | ||
| // abortFlag skips the barrier, so a peer that entered it waits forever in | ||
| // bootstrapRecv -- the PyTorch ProcessGroupNCCL watchdog hits exactly this. | ||
| // | ||
| // These run on a single-node communicator at nRanks == 2, where the barrier | ||
| // cannot be protecting anything: RCCL defaults NCCL_PXN_DISABLE=1 and | ||
| // rcclSetPxn() only auto-enables PXN at nRanks >= 64 (gfx942) / >= 32 | ||
| // (gfx950), so there is no PXN relay to keep alive. | ||
| namespace | ||
| { | ||
| // EnvVars reports 0 GPUs in a process re-exec'd by ProcessIsolatedTestRunner, and a | ||
| // skipped child is scored as a pass -- so every GPU gate below must run in the parent | ||
| // only, or the coverage silently evaporates into a green run. | ||
| bool isIsolatedChild() | ||
| { | ||
| return std::getenv(ProcessIsolatedTestRunner::kReexecMarkerEnvVar) != nullptr; | ||
| } | ||
|
|
||
| // Never call hipGetDeviceCount() here: HIP state does not survive the fork() below. | ||
| int getDetectedGpuCount() | ||
| { | ||
| static const int detectedGpus = EnvVars().GetNumDetectedGpus(); | ||
| return detectedGpus; | ||
| } | ||
|
|
||
| enum class PeerTeardown | ||
| { | ||
| Abort, // peer calls ncclCommAbort -- non-zero abortFlag, skips the barrier | ||
| Destroy // peer calls ncclCommDestroy -- control, both ranks reach the barrier | ||
| }; | ||
|
|
||
| // Two ranks, one process each, one collective, then a deliberately divergent | ||
| // teardown. Returns true if the rank taking the normal destroy path returned | ||
| // from ncclCommDestroy before the deadline. | ||
| // | ||
| // Nothing here may touch HIP or RCCL before fork(): initializing the runtime | ||
| // in the parent leaves the children with unusable state and they die in | ||
| // hipSetDevice. | ||
| bool DestroyReturnsWhenPeer(PeerTeardown peer, int deadlineSeconds) | ||
| { | ||
| constexpr int nRanks = 2; | ||
| constexpr size_t numElements = 32 * 1024; | ||
|
|
||
| int idpipe[2]; | ||
| if (pipe(idpipe) != 0) return false; | ||
|
|
||
| pid_t kids[nRanks] = {-1, -1}; | ||
| for (int rank = 0; rank < nRanks; ++rank) | ||
| { | ||
| pid_t pid = fork(); | ||
| if (pid < 0) | ||
| { | ||
| close(idpipe[0]); | ||
| close(idpipe[1]); | ||
| return false; | ||
| } | ||
| if (pid == 0) | ||
| { | ||
| ncclUniqueId id; | ||
| if (rank == 0) | ||
| { | ||
| close(idpipe[0]); | ||
| if (ncclGetUniqueId(&id) != ncclSuccess) _exit(70); | ||
| if (write(idpipe[1], &id, sizeof(id)) != (ssize_t)sizeof(id)) _exit(71); | ||
| close(idpipe[1]); | ||
| } | ||
| else | ||
| { | ||
| close(idpipe[1]); | ||
| size_t got = 0; | ||
| while (got < sizeof(id)) | ||
| { | ||
| ssize_t n = read(idpipe[0], ((char*)&id) + got, sizeof(id) - got); | ||
| if (n <= 0) _exit(72); | ||
| got += n; | ||
| } | ||
| close(idpipe[0]); | ||
| } | ||
|
|
||
| if (hipSetDevice(rank) != hipSuccess) _exit(73); | ||
| hipStream_t stream; | ||
| if (hipStreamCreate(&stream) != hipSuccess) _exit(74); | ||
| float* buf = nullptr; | ||
| if (hipMalloc(&buf, numElements * sizeof(float)) != hipSuccess) _exit(75); | ||
| if (hipMemset(buf, 0, numElements * sizeof(float)) != hipSuccess) _exit(76); | ||
|
|
||
| ncclComm_t comm; | ||
| if (ncclCommInitRank(&comm, nRanks, id, rank) != ncclSuccess) _exit(77); | ||
| if (ncclAllReduce(buf, buf, numElements, ncclFloat, ncclSum, comm, stream) != ncclSuccess) | ||
| _exit(78); | ||
| if (hipStreamSynchronize(stream) != hipSuccess) _exit(79); | ||
|
|
||
| if (rank == 1 && peer == PeerTeardown::Abort) | ||
| ncclCommAbort(comm); | ||
| else | ||
| ncclCommDestroy(comm); // rank 0 always takes the barrier path | ||
|
|
||
| _exit(0); | ||
|
prasanna-amd marked this conversation as resolved.
Resolved
|
||
| } | ||
| kids[rank] = pid; | ||
| } | ||
| close(idpipe[0]); | ||
| close(idpipe[1]); | ||
|
|
||
| // Poll for rank 0 only. It is the one that enters the barrier, so it is | ||
| // the one that hangs on regression. | ||
| bool returned = false; | ||
| auto const deadline = | ||
| std::chrono::steady_clock::now() + std::chrono::seconds(deadlineSeconds); | ||
| while (std::chrono::steady_clock::now() < deadline) | ||
| { | ||
| int status = 0; | ||
| pid_t done = waitpid(kids[0], &status, WNOHANG); | ||
| if (done == kids[0]) | ||
| { | ||
| // WEXITSTATUS is meaningless for a signal-killed child; checking it | ||
| // alone would report a crash as a clean pass. | ||
| returned = !WIFSIGNALED(status) && WEXITSTATUS(status) == 0; | ||
| break; | ||
| } | ||
| if (done < 0) break; | ||
| std::this_thread::sleep_for(std::chrono::milliseconds(50)); | ||
| } | ||
|
|
||
| for (int i = 0; i < nRanks; ++i) | ||
| { | ||
| if (kids[i] <= 0) continue; | ||
| kill(kids[i], SIGKILL); | ||
| waitpid(kids[i], nullptr, 0); | ||
| } | ||
| return returned; | ||
| } | ||
| } | ||
|
|
||
| // Control: both ranks destroy normally, so both reach the barrier and it | ||
| // completes. If this fails the harness itself is broken and the companion | ||
| // test below proves nothing. | ||
| TEST(Teardown, DivergentTeardown_PeerDestroys_Completes) | ||
| { | ||
| if (!isIsolatedChild() && getDetectedGpuCount() < 2) | ||
| GTEST_SKIP() << "Divergent teardown requires at least 2 GPUs (detected " | ||
| << getDetectedGpuCount() << ")"; | ||
|
|
||
| RUN_ISOLATED_TESTS( | ||
| ProcessIsolatedTestRunner::TestConfig( | ||
| "DivergentTeardown_PeerDestroys", | ||
| []() { | ||
| EXPECT_TRUE(DestroyReturnsWhenPeer(PeerTeardown::Destroy, /*deadlineSeconds*/ 60)) | ||
| << "rank 0 did not return from ncclCommDestroy even though its peer " | ||
| "also destroyed -- the host-local destroy barrier is broken"; | ||
| }) | ||
| .withTimeout(std::chrono::seconds(120))); | ||
| } | ||
|
|
||
| // Regression: the peer aborts, so it skips the barrier. Rank 0 must still | ||
| // return instead of blocking forever in bootstrapRecv. | ||
| TEST(Teardown, DivergentTeardown_PeerAborts_DoesNotHang) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This test pins NCCL_PXN_DISABLE=1 and calls ncclCommAbort with no shrink, so it passes via the PXN skip, not via the filtering. The main new logic of this PR has no end-to-end test. Can we add a real shrink(SHRINK_ABORT) + destroy repro with PXN on that actually hits the filter?
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The filtering this was asking about is gone as of b7bc5b6, so the specific test you asked for -- shrink(SHRINK_ABORT) + destroy with PXN on, hitting the filter -- no longer has anything to hit. But I do not want to close this out on that technicality, because the underlying complaint survives the revert: the main new logic of this PR still has no end-to-end test. It is now the timeout rather than the filter, and the gap is the same one you identified. What exists today:
What is missing is the case that matters: One caveat I would rather state than have you discover: I cannot run GPU tests from my current checkout -- the compute nodes on this cluster have no path to it -- so the end-to-end case will be verified in PR CI, not locally. I will post the run link rather than just asserting it passes. Holding this thread open until that is up.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The test is up, in
Two details aimed at the vacuous-pass problem you raised, since "the test passes" is worth very little on its own here:
Expiry reports Also removed a comment above Caveat I stated earlier and will restate: I cannot run GPU tests from my checkout, so this is verified by syntax check locally and must be proven in CI. I am not claiming it passes until I can post the run. CI has not run on this branch since 2026-10-08 -- the |
||
| { | ||
| if (!isIsolatedChild() && getDetectedGpuCount() < 2) | ||
| GTEST_SKIP() << "Divergent teardown requires at least 2 GPUs (detected " | ||
| << getDetectedGpuCount() << ")"; | ||
|
|
||
| RUN_ISOLATED_TESTS( | ||
| ProcessIsolatedTestRunner::TestConfig( | ||
| "DivergentTeardown_PeerAborts", | ||
| []() { | ||
| EXPECT_TRUE(DestroyReturnsWhenPeer(PeerTeardown::Abort, /*deadlineSeconds*/ 60)) | ||
| << "rank 0 hung in ncclCommDestroy after its host-local peer aborted " | ||
| "and skipped the destroy barrier (AICOMRCCL-2467 / AICOMRCCL-2468)"; | ||
| }) | ||
| .withTimeout(std::chrono::seconds(120))); | ||
| } | ||
|
prasanna-amd marked this conversation as resolved.
Resolved
|
||
| } | ||
Uh oh!
There was an error while loading. Please reload this page.