diff --git a/projects/rccl/src/transport/net_ib_cast/reg.cc b/projects/rccl/src/transport/net_ib_cast/reg.cc index 401cfa8999d3..15671b369b38 100644 --- a/projects/rccl/src/transport/net_ib_cast/reg.cc +++ b/projects/rccl/src/transport/net_ib_cast/reg.cc @@ -70,7 +70,15 @@ static ncclResult_t ncclIbRegMrDmaBufInternal2(ncclIbNetCommDevBase* base, void* ncclResult_t IbCastRegMrDmaBufInternal(void* comm, void* data, size_t size, int type, uint64_t offset, int fd, uint64_t mrFlags, void** mhandle) { ncclResult_t ret = ncclSuccess; - assert(size > 0); + // A caller that lost its connection reaches here with a null comm, and reading + // the device list off it segfaults inside the plugin instead of telling the + // caller what was wrong. size == 0 hit the same fate below via the page-count + // math, so it is reported the same way rather than left as a release-build + // assert. + if (comm == NULL || mhandle == NULL || size == 0) { + WARN("NET/IB-CAST: regMr/regMrDmaBuf called with comm=%p mhandle=%p size=%zu", comm, (void*)mhandle, size); + return ncclInvalidArgument; + } struct ncclIbNetCommBase* base = (struct ncclIbNetCommBase*)comm; struct ncclIbMrHandle* mhandleWrapper = (struct ncclIbMrHandle*)malloc(sizeof(struct ncclIbMrHandle)); if (mhandleWrapper == nullptr) { @@ -122,6 +130,10 @@ ncclResult_t IbCastDeregMrInternal(ncclIbNetCommDevBase* base, ibv_mr* mhandle) ncclResult_t IbCastDeregMr(void* comm, void* mhandle) { if (mhandle == NULL) return ncclSuccess; + if (comm == NULL) { + WARN("NET/IB-CAST: deregMr called with a null comm"); + return ncclInvalidArgument; + } struct ncclIbMrHandle* mhandleWrapper = (struct ncclIbMrHandle*)mhandle; struct ncclIbNetCommBase* base = (struct ncclIbNetCommBase*)comm; diff --git a/projects/rccl/test/transport/NetIbMPI/CastTests.cpp b/projects/rccl/test/transport/NetIbMPI/CastTests.cpp index 2d6fda829ecb..215664b5954a 100644 --- a/projects/rccl/test/transport/NetIbMPI/CastTests.cpp +++ b/projects/rccl/test/transport/NetIbMPI/CastTests.cpp @@ -41,7 +41,7 @@ TEST_F(NetIbMPITest, CastEqualWeightsTwoQPsTokenCounts) { void* listenComm = nullptr; void* sendComm = nullptr; void* recvComm = nullptr; - SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm); + ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm)); constexpr size_t kMsgSize = 1024; char sendBuf[kMsgSize], recvBuf[kMsgSize]; @@ -102,7 +102,7 @@ TEST_F(NetIbMPITest, CastWeightsDistributionOneRound) { void* listenComm = nullptr; void* sendComm = nullptr; void* recvComm = nullptr; - SetupCastConnection(0, &listenComm, &sendComm, &recvComm); + ASSERT_NO_FATAL_FAILURE(SetupCastConnection(0, &listenComm, &sendComm, &recvComm)); constexpr int kTotTokens = 100; constexpr size_t kMsgSz = 64; @@ -211,7 +211,7 @@ TEST_F(NetIbMPITest, CastTokenSumInvariantAfterConsumption) { void* listenComm = nullptr; void* sendComm = nullptr; void* recvComm = nullptr; - SetupCastConnection(0, &listenComm, &sendComm, &recvComm); + ASSERT_NO_FATAL_FAILURE(SetupCastConnection(0, &listenComm, &sendComm, &recvComm)); constexpr size_t kMsgSize = 128; constexpr int kNMsgs = 10; @@ -264,7 +264,7 @@ TEST_F(NetIbMPITest, CastSingleQPBypassesWrr) { void* listenComm = nullptr; void* sendComm = nullptr; void* recvComm = nullptr; - SetupCastConnection(0, &listenComm, &sendComm, &recvComm); + ASSERT_NO_FATAL_FAILURE(SetupCastConnection(0, &listenComm, &sendComm, &recvComm)); constexpr size_t kMsgSize = 256; char sendBuf[kMsgSize], recvBuf[kMsgSize]; @@ -317,7 +317,7 @@ TEST_F(NetIbMPITest, CastSchedParmsReflectEnvVars) { void* listenComm = nullptr; void* sendComm = nullptr; void* recvComm = nullptr; - SetupCastConnection(0, &listenComm, &sendComm, &recvComm); + ASSERT_NO_FATAL_FAILURE(SetupCastConnection(0, &listenComm, &sendComm, &recvComm)); constexpr size_t kMsgSize = 64; char sendBuf[kMsgSize], recvBuf[kMsgSize]; @@ -363,7 +363,7 @@ TEST_F(NetIbMPITest, CastCursorWrapsAtNqpsBoundary) { void* listenComm = nullptr; void* sendComm = nullptr; void* recvComm = nullptr; - SetupCastConnection(0, &listenComm, &sendComm, &recvComm); + ASSERT_NO_FATAL_FAILURE(SetupCastConnection(0, &listenComm, &sendComm, &recvComm)); constexpr size_t kMsgSize = 128; char sendBuf[kMsgSize], recvBuf[kMsgSize]; @@ -422,7 +422,7 @@ TEST_F(NetIbMPITest, CastMaxQPCount128) { void* listenComm = nullptr; void* sendComm = nullptr; void* recvComm = nullptr; - SetupCastConnection(0, &listenComm, &sendComm, &recvComm); + ASSERT_NO_FATAL_FAILURE(SetupCastConnection(0, &listenComm, &sendComm, &recvComm)); constexpr size_t kMsgSz = 32; constexpr size_t kBufSz = (NCCL_IB_MAX_QPS + 1) * kMsgSz; @@ -496,7 +496,7 @@ TEST_F(NetIbMPITest, CastFourQPsMonotonicOrder) { void* listenComm = nullptr; void* sendComm = nullptr; void* recvComm = nullptr; - SetupCastConnection(0, &listenComm, &sendComm, &recvComm); + ASSERT_NO_FATAL_FAILURE(SetupCastConnection(0, &listenComm, &sendComm, &recvComm)); constexpr int kNMsgs = 100; constexpr size_t kMsgSz = 32; @@ -654,7 +654,7 @@ TEST_F(NetIbMPITest, CastSplitDataThresholdBoundary) { void* listenComm = nullptr; void* sendComm = nullptr; void* recvComm = nullptr; - SetupCastConnection(0, &listenComm, &sendComm, &recvComm); + ASSERT_NO_FATAL_FAILURE(SetupCastConnection(0, &listenComm, &sendComm, &recvComm)); void* comm = (rank == 0) ? recvComm : sendComm; @@ -749,7 +749,7 @@ TEST_F(NetIbMPITest, CastAlternatingWrrNonWrr) { void* listenComm = nullptr; void* sendComm = nullptr; void* recvComm = nullptr; - SetupCastConnection(0, &listenComm, &sendComm, &recvComm); + ASSERT_NO_FATAL_FAILURE(SetupCastConnection(0, &listenComm, &sendComm, &recvComm)); constexpr int kPhase = 10; constexpr size_t kMsgSz = 64; @@ -879,7 +879,7 @@ TEST_F(NetIbMPITest, CastEnableDisableSplitData) { void* listenComm = nullptr; void* sendComm = nullptr; void* recvComm = nullptr; - SetupCastConnection(0, &listenComm, &sendComm, &recvComm); + ASSERT_NO_FATAL_FAILURE(SetupCastConnection(0, &listenComm, &sendComm, &recvComm)); void* comm = (rank == 0) ? recvComm : sendComm; @@ -987,7 +987,7 @@ TEST_F(NetIbMPITest, CastEnableDisableSched) { void* listenComm = nullptr; void* sendComm = nullptr; void* recvComm = nullptr; - SetupCastConnection(0, &listenComm, &sendComm, &recvComm); + ASSERT_NO_FATAL_FAILURE(SetupCastConnection(0, &listenComm, &sendComm, &recvComm)); // Must be strictly below splitDataMin so dataPerQp < splitDataMin for any nqps, // ensuring messages take the WRR path (not the split path). @@ -1166,7 +1166,7 @@ TEST_F(NetIbMPITest, CastSendRecvMultipleSizes) { void* listenComm = nullptr; void* sendComm = nullptr; void* recvComm = nullptr; - SetupCastConnection(0, &listenComm, &sendComm, &recvComm); + ASSERT_NO_FATAL_FAILURE(SetupCastConnection(0, &listenComm, &sendComm, &recvComm)); constexpr size_t kBufSz = 524288; // 512 KB — fits all test sizes std::vector sendBuf(kBufSz); @@ -1323,7 +1323,7 @@ TEST_F(NetIbMPITest, CastLargeTransfer) { void* listenComm = nullptr; void* sendComm = nullptr; void* recvComm = nullptr; - SetupCastConnection(0, &listenComm, &sendComm, &recvComm); + ASSERT_NO_FATAL_FAILURE(SetupCastConnection(0, &listenComm, &sendComm, &recvComm)); constexpr size_t kMsgSz = kLargeBufferSize; // 16 MB std::vector sendBuf(kMsgSz); @@ -1434,7 +1434,7 @@ TEST_F(NetIbMPITest, CastSendRecvZeroSize) { void* listenComm = nullptr; void* sendComm = nullptr; void* recvComm = nullptr; - SetupCastConnection(0, &listenComm, &sendComm, &recvComm); + ASSERT_NO_FATAL_FAILURE(SetupCastConnection(0, &listenComm, &sendComm, &recvComm)); constexpr size_t kRegSz = 64; char buf[kRegSz]; @@ -1648,8 +1648,50 @@ TEST_F(NetIbMPITest, CastStressMultiRoundTwoConns) { std::vector listenComms(kNConns, nullptr); std::vector sendComms(kNConns, nullptr); std::vector recvComms(kNConns, nullptr); - for (int c = 0; c < kNConns; c++) - SetupCastConnection(/*dev=*/0, &listenComms[c], &sendComms[c], &recvComms[c]); + // Declared here rather than beside their registration loops so the guard below can + // release them: closing a communicator releases the plugin's own memory regions, not + // the ones a test registered, and the IB-CAST registration cache is per device -- so + // a wrapper left behind outlives this test, and its cache reference with it. + std::vector mhandles(kNConns, nullptr); + std::vector rampHandles(kNConns, nullptr); + // Declared before the guard, and sized once their sizes are known, because + // destruction runs in reverse: declared after it, their storage would be freed while + // the guard's registrations still described it, and the guard would then deregister + // memory that no longer exists. + std::vector> sendBufs, recvBufs, rampSend, rampRecv; + + // Releases whatever is still held on any exit from here, the teardown's own failures + // included: otherwise a setup, registration or close failure returns from this body + // and leaves connections open and memory still registered for the rest of the + // process, which is the contamination the helper stopped creating inside itself. + // Registrations go before the communicator they belong to. A slot is nulled only + // after successful cleanup, so the guard can retry a failed teardown. + auto connsScope = makeScopeGuard([&]() { + for (int c = 0; c < kNConns; c++) { + void* comm = (rank == 0) ? recvComms[c] : sendComms[c]; + if (comm && rampHandles[c] + && DeregisterMemory(comm, rampHandles[c]) == ncclSuccess) + rampHandles[c] = nullptr; + if (comm && mhandles[c] && DeregisterMemory(comm, mhandles[c]) == ncclSuccess) + mhandles[c] = nullptr; + if (rampHandles[c] || mhandles[c]) continue; + if (recvComms[c] && CloseRecvComm(recvComms[c]) == ncclSuccess) + recvComms[c] = nullptr; + if (sendComms[c] && CloseSendComm(sendComms[c]) == ncclSuccess) + sendComms[c] = nullptr; + if (!recvComms[c] && !sendComms[c] && listenComms[c] + && CloseListenComm(listenComms[c]) == ncclSuccess) + listenComms[c] = nullptr; + } + }); + + // Wrapped, because the helper ends in a fatal assertion: without this a failed setup + // returns from the helper and not from here, and the loop would work through the + // remaining connections at up to 30 s each instead of reporting what went wrong. + for (int c = 0; c < kNConns; c++) { + ASSERT_NO_FATAL_FAILURE( + SetupCastConnection(/*dev=*/0, &listenComms[c], &sendComms[c], &recvComms[c])); + } // Scale msgs per connection inversely with connection count so total work stays constant. constexpr int kNMsgsTotal = 10000; @@ -1661,8 +1703,8 @@ TEST_F(NetIbMPITest, CastStressMultiRoundTwoConns) { const size_t kBufSz = static_cast(kNMsgs) * kMsgSz; // One send/recv buffer pair per connection. - std::vector> sendBufs(kNConns, std::vector(kBufSz)); - std::vector> recvBufs(kNConns, std::vector(kBufSz)); + sendBufs.assign(kNConns, std::vector(kBufSz)); + recvBufs.assign(kNConns, std::vector(kBufSz)); for (int c = 0; c < kNConns; c++) { for (size_t i = 0; i < kBufSz; i++) { sendBufs[c][i] = static_cast(((i + c) * 3 + 7) & 0xFF); @@ -1670,7 +1712,6 @@ TEST_F(NetIbMPITest, CastStressMultiRoundTwoConns) { memset(recvBufs[c].data(), 0, kBufSz); } - std::vector mhandles(kNConns, nullptr); for (int c = 0; c < kNConns; c++) { void* comm = (rank == 0) ? recvComms[c] : sendComms[c]; char* regBuf = (rank == 0) ? recvBufs[c].data() : sendBufs[c].data(); @@ -1788,15 +1829,14 @@ TEST_F(NetIbMPITest, CastStressMultiRoundTwoConns) { constexpr int kLargeRounds = 20; const size_t kRampBufSz = kRampSizes.back(); // one MR per conn, sized for largest - std::vector> rampSend(kNConns, std::vector(kRampBufSz)); - std::vector> rampRecv(kNConns, std::vector(kRampBufSz)); + rampSend.assign(kNConns, std::vector(kRampBufSz)); + rampRecv.assign(kNConns, std::vector(kRampBufSz)); for (int c = 0; c < kNConns; c++) { for (size_t i = 0; i < kRampBufSz; i++) rampSend[c][i] = static_cast(((i + c) * 7 + 3) & 0xFF); memset(rampRecv[c].data(), 0, kRampBufSz); } - std::vector rampHandles(kNConns, nullptr); for (int c = 0; c < kNConns; c++) { void* comm = (rank == 0) ? recvComms[c] : sendComms[c]; char* regBuf = (rank == 0) ? rampRecv[c].data() : rampSend[c].data(); @@ -1829,17 +1869,44 @@ TEST_F(NetIbMPITest, CastStressMultiRoundTwoConns) { MPI_Barrier(MPI_COMM_WORLD); // ── Teardown ───────────────────────────────────────────────────────────── + // Reported, not asserted, and for the barrier below: these checks are rank-local, so a + // fatal one would return from this rank while its peer waited in that collective -- + // the hang ListenCloseListen was rewritten to remove. Each slot is also nulled as it + // is closed, so the guard can stay armed through the loop and close whatever a failure + // leaves behind. + bool teardownOk = true; for (int c = 0; c < kNConns; c++) { void* comm = (rank == 0) ? recvComms[c] : sendComms[c]; - ASSERT_EQ(DeregisterMemory(comm, rampHandles[c]), ncclSuccess); - ASSERT_EQ(DeregisterMemory(comm, mhandles[c]), ncclSuccess); + const ncclResult_t deregRamp = DeregisterMemory(comm, rampHandles[c]); + if (deregRamp == ncclSuccess) rampHandles[c] = nullptr; + else teardownOk = false; + const ncclResult_t deregMsg = DeregisterMemory(comm, mhandles[c]); + if (deregMsg == ncclSuccess) mhandles[c] = nullptr; + else teardownOk = false; + EXPECT_EQ(deregRamp, ncclSuccess) + << "deregistering the ramp buffer failed on conn " << c; + EXPECT_EQ(deregMsg, ncclSuccess) + << "deregistering the phase-1 buffer failed on conn " << c; + if (rampHandles[c] || mhandles[c]) continue; if (rank == 0) { - ASSERT_EQ(CloseRecvComm(recvComms[c]), ncclSuccess); - ASSERT_EQ(CloseListenComm(listenComms[c]), ncclSuccess); + const ncclResult_t closedRecv = CloseRecvComm(recvComms[c]); + if (closedRecv == ncclSuccess) recvComms[c] = nullptr; + else teardownOk = false; + EXPECT_EQ(closedRecv, ncclSuccess) << "CloseRecvComm failed on conn " << c; + if (!recvComms[c]) { + const ncclResult_t closedListen = CloseListenComm(listenComms[c]); + if (closedListen == ncclSuccess) listenComms[c] = nullptr; + else teardownOk = false; + EXPECT_EQ(closedListen, ncclSuccess) << "CloseListenComm failed on conn " << c; + } } else { - ASSERT_EQ(CloseSendComm(sendComms[c]), ncclSuccess); + const ncclResult_t closedSend = CloseSendComm(sendComms[c]); + if (closedSend == ncclSuccess) sendComms[c] = nullptr; + else teardownOk = false; + EXPECT_EQ(closedSend, ncclSuccess) << "CloseSendComm failed on conn " << c; } } + if (teardownOk) connsScope.dismiss(); MPI_Barrier(MPI_COMM_WORLD); } @@ -1873,7 +1940,7 @@ TEST_F(NetIbMPITest, CastGrhSetOnRoceQp) { void* listenComm = nullptr; void* sendComm = nullptr; void* recvComm = nullptr; - SetupCastConnection(0, &listenComm, &sendComm, &recvComm); + ASSERT_NO_FATAL_FAILURE(SetupCastConnection(0, &listenComm, &sendComm, &recvComm)); constexpr size_t kMsgSize = 1024; char sendBuf[kMsgSize] = {}, recvBuf[kMsgSize] = {}; @@ -1956,7 +2023,7 @@ TEST_F(NetIbMPITest, CastSubnetAwareRoutingSameSubnet) { void* listenComm = nullptr; void* sendComm = nullptr; void* recvComm = nullptr; - SetupCastConnection(0, &listenComm, &sendComm, &recvComm); + ASSERT_NO_FATAL_FAILURE(SetupCastConnection(0, &listenComm, &sendComm, &recvComm)); constexpr size_t kMsgSize = 1024; char sendBuf[kMsgSize] = {}, recvBuf[kMsgSize] = {}; @@ -2019,7 +2086,7 @@ TEST_F(NetIbMPITest, CastMultiplaneConnectionSmoke) { void* listenComm = nullptr; void* sendComm = nullptr; void* recvComm = nullptr; - SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm); + ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm)); // Data integrity: fill with a known pattern, verify after transfer. constexpr size_t kMsgSize = 4096; @@ -2071,7 +2138,7 @@ TEST_F(NetIbMPITest, CastMultiplaneMultiSizeTransfer) { void* listenComm = nullptr; void* sendComm = nullptr; void* recvComm = nullptr; - SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm); + ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm)); constexpr size_t kMaxSize = 4 * 1024 * 1024; // 4 MB std::vector sendBuf(kMaxSize); @@ -2133,7 +2200,7 @@ TEST_F(NetIbMPITest, CastMultiplaneDisabledNoRegression) { void* listenComm = nullptr; void* sendComm = nullptr; void* recvComm = nullptr; - SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm); + ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm)); constexpr size_t kMsgSize = 2048; char sendBuf[kMsgSize] = {}, recvBuf[kMsgSize] = {}; @@ -2154,4 +2221,102 @@ TEST_F(NetIbMPITest, CastMultiplaneDisabledNoRegression) { TeardownConnection(recvComm, listenComm, sendComm, mhandle); } +// ============================================================================= +// Test: CastRegistrationRejectsBadArguments +// +// Regression cover for the production guard this branch adds to net_ib_cast's +// regMr/regMrDmaBuf/deregMr. On develop each of these dereferences the +// communicator or the handle before looking at them, so the suite's reaction to a +// failed connection setup was a SIGSEGV inside librccl rather than a readable +// failure; the guard returns ncclInvalidArgument instead. +// +// The existing negative registration tests pass a null *buffer* with a valid +// communicator, and the new setup wrappers return before registration when setup +// fails, so nothing else in the suite reaches these arms. +// +// A null communicator needs no connection: the guard runs before the arguments reach +// the device. deregMr is the exception -- it answers a null handle with success +// before it looks at the communicator -- so the null-comm arm there has to carry a +// handle, and the documented shortcut is checked on its own. The zero-size arm does +// need a live communicator, since with a null one the null-comm check answers first. +// Both ranks run the same checks and reach the barrier from the same place. +// ============================================================================= +TEST_F(NetIbMPITest, CastRegistrationRejectsBadArguments) { + SKIP_UNLESS_MPI_PREREQS(kExactTwoProcesses, kExactTwoProcesses, + false, kMinGpusPerNode, kNoNodeLimit); + + // No CAST_ENV_CHECK_OR_SKIP here, unlike the 16 tests above: the arms under test + // are argument checks that run before the scheduler, so the WRR env vars are not + // needed and the macro would only turn a pass into a skip. + net_ = &netIbCast; + // Not AssertInitAndGetDevices, and not that helper wrapped in + // ASSERT_NO_FATAL_FAILURE either. Its assertions are rank-local -- a node whose IB + // port is down reports ndev == 0 on that rank alone -- so making the failure fatal + // here, however it is propagated, returns from this body on the rank that failed + // while its peer walks into SetupCastConnection and blocks in the MPI_Allreduce + // there. Nothing downstream resynchronises them: cleanupTestCommunicator returns + // before its barriers when no test communicator was created. Wrapping a rank-local + // assertion trades an unset out-parameter for a hang, which is a worse failure than + // the one it fixes. + // + // So the checks are recorded non-fatally and reduced first, the way both setup + // helpers do it: every rank reaches the same collective and the same verdict. + // Both return codes go into the verdict, not just the device count. IbCastInit + // populates the merged-device list first (IbCastInitDevices) and can still fail + // afterwards -- starting the port-recovery thread, allocating the context -- while + // IbCastDevices goes on reporting the count that was already filled in. Reducing + // ndev alone would let both ranks agree they were ready and walk into setup on a + // plugin whose init had failed. + const ncclResult_t initRet = InitNetIb(); + EXPECT_EQ(initRet, ncclSuccess) << "InitNetIb failed"; + int ndev = 0; + const ncclResult_t devRet = GetDeviceCount(&ndev); + EXPECT_EQ(devRet, ncclSuccess) << "GetDeviceCount failed"; + int ready = (initRet == ncclSuccess && devRet == ncclSuccess && ndev > 0) ? 1 : 0; + MPI_Allreduce(MPI_IN_PLACE, &ready, 1, MPI_INT, MPI_MIN, MPI_COMM_WORLD); + ASSERT_EQ(ready, 1) << "init or device discovery failed on at least one rank"; + + // Fixed, not allocated: a rank-local allocation failure would end this rank + // before the barrier below while its peer waited there. + static char buffer[4096]; + void* mhandle = nullptr; + + // No connection required for these three. + EXPECT_EQ(RegisterMemory(nullptr, buffer, sizeof(buffer), NCCL_PTR_HOST, &mhandle), + ncclInvalidArgument) + << "regMr must reject a null communicator instead of dereferencing it"; + EXPECT_EQ(RegisterDmaBufMemory(nullptr, buffer, sizeof(buffer), NCCL_PTR_HOST, + /*offset=*/0, /*fd=*/-1, &mhandle), + ncclInvalidArgument) + << "regMrDmaBuf must reject a null communicator"; + // Not the mhandle above: it is still null, since the two calls above return before + // assigning it, and a null handle takes deregMr's early success. The guard rejects + // the communicator before the handle is read, so a value that points at nothing + // usable reaches the arm under test safely. + void* const unusedMhandle = reinterpret_cast(0x1); + EXPECT_EQ(DeregisterMemory(nullptr, unusedMhandle), ncclInvalidArgument) + << "deregMr must reject a null communicator"; + EXPECT_EQ(DeregisterMemory(nullptr, nullptr), ncclSuccess) + << "deregMr must keep treating a null handle as a no-op, ahead of the comm check"; + + void* listenComm = nullptr; + void* sendComm = nullptr; + void* recvComm = nullptr; + ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm)); + + const int rank = MPIEnvironment::world_rank; + void* comm = (rank == 0) ? recvComm : sendComm; + ASSERT_NE(comm, nullptr); + + EXPECT_EQ(RegisterMemory(comm, buffer, /*size=*/0, NCCL_PTR_HOST, &mhandle), + ncclInvalidArgument) + << "regMr must reject a zero size"; + EXPECT_EQ(RegisterMemory(comm, buffer, sizeof(buffer), NCCL_PTR_HOST, nullptr), + ncclInvalidArgument) + << "regMr must reject a null out-handle"; + + MPI_Barrier(MPI_COMM_WORLD); + TeardownConnection(recvComm, listenComm, sendComm, nullptr); +} + #endif // MPI_TESTS_ENABLED diff --git a/projects/rccl/test/transport/NetIbMPI/FaultInjectTests.cpp b/projects/rccl/test/transport/NetIbMPI/FaultInjectTests.cpp index 6f579d967ba1..7100455902d6 100644 --- a/projects/rccl/test/transport/NetIbMPI/FaultInjectTests.cpp +++ b/projects/rccl/test/transport/NetIbMPI/FaultInjectTests.cpp @@ -141,7 +141,7 @@ TEST_F(NetIbMPITest, FaultInjCastQpErrorIsFatal) { void* listenComm = nullptr; void* sendComm = nullptr; void* recvComm = nullptr; - SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm); + ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm)); constexpr size_t kMsgSize = 1024; std::vector sendBuf(kMsgSize), recvBuf(kMsgSize); @@ -278,7 +278,7 @@ TEST_F(NetIbMPITest, FaultInjCastSlowQpRebalances) { void* listenComm = nullptr; void* sendComm = nullptr; void* recvComm = nullptr; - SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm); + ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm)); constexpr int kNMsgs = 500; // Keep below splitDataMin so messages take the WRR token path. @@ -483,7 +483,7 @@ TEST_F(NetIbMPITest, FaultInjCastDelayDataIntegrity) { void* listenComm = nullptr; void* sendComm = nullptr; void* recvComm = nullptr; - SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm); + ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm)); constexpr int kNMsgs = 50; constexpr size_t kMsgSz = 8192; @@ -553,7 +553,7 @@ TEST_F(NetIbMPITest, FaultInjCastSingleQpErrorIsFatal) { void* listenComm = nullptr; void* sendComm = nullptr; void* recvComm = nullptr; - SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm); + ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm)); constexpr size_t kMsgSize = 1024; // below splitDataMin → single-QP WRR path std::vector sendBuf(kMsgSize), recvBuf(kMsgSize); @@ -802,7 +802,7 @@ TEST_F(NetIbMPITest, FaultInjCastQpErrorClearRecovers) { void* listenComm1 = nullptr; void* sendComm1 = nullptr; void* recvComm1 = nullptr; - SetupCastConnection(/*dev=*/0, &listenComm1, &sendComm1, &recvComm1); + ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/0, &listenComm1, &sendComm1, &recvComm1)); constexpr size_t kMsgSize = 1024; std::vector sendBuf(kMsgSize), recvBuf(kMsgSize); @@ -897,7 +897,7 @@ TEST_F(NetIbMPITest, FaultInjCastQpErrorClearRecovers) { void* listenComm2 = nullptr; void* sendComm2 = nullptr; void* recvComm2 = nullptr; - SetupCastConnection(/*dev=*/0, &listenComm2, &sendComm2, &recvComm2); + ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/0, &listenComm2, &sendComm2, &recvComm2)); void* comm2 = (rank == 0) ? recvComm2 : sendComm2; char* regBuf2 = (rank == 0) ? recvBuf2.data() : sendBuf2.data(); @@ -954,7 +954,7 @@ TEST_F(NetIbMPITest, FailoverErrorCodeWhitelist) { void* listenComm = nullptr; void* sendComm = nullptr; void* recvComm = nullptr; - SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm); + ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm)); struct WhitelistResult { int hasResiliency; @@ -1086,7 +1086,7 @@ TEST_F(NetIbMPITest, FailoverCqeErrorRecovered) { void* listenComm = nullptr; void* sendComm = nullptr; void* recvComm = nullptr; - SetupCastConnection(/*dev=*/mergedDev, &listenComm, &sendComm, &recvComm); + ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/mergedDev, &listenComm, &sendComm, &recvComm)); constexpr size_t kMsgSize = 8192; std::vector sendBuf(kMsgSize), recvBuf(kMsgSize); @@ -1235,7 +1235,7 @@ TEST_F(NetIbMPITest, FailoverSingleDeviceTopology) { void* listenComm = nullptr; void* sendComm = nullptr; void* recvComm = nullptr; - SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm); + ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm)); constexpr size_t kMsgSize = 4096; std::vector sendBuf(kMsgSize), recvBuf(kMsgSize); @@ -1358,7 +1358,7 @@ TEST_F(NetIbMPITest, FailoverAllDevicesFailed) { void* listenComm = nullptr; void* sendComm = nullptr; void* recvComm = nullptr; - SetupCastConnection(/*dev=*/mergedDev, &listenComm, &sendComm, &recvComm); + ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/mergedDev, &listenComm, &sendComm, &recvComm)); constexpr size_t kMsgSize = 4096; std::vector sendBuf(kMsgSize), recvBuf(kMsgSize); @@ -1520,7 +1520,7 @@ TEST_F(NetIbMPITest, FailoverLargeMessageDataIntegrity) { void* listenComm = nullptr; void* sendComm = nullptr; void* recvComm = nullptr; - SetupCastConnection(/*dev=*/mergedDev, &listenComm, &sendComm, &recvComm); + ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/mergedDev, &listenComm, &sendComm, &recvComm)); // Large message to ensure data spans multiple QPs constexpr size_t kMsgSize = 65536; @@ -1661,7 +1661,7 @@ TEST_F(NetIbMPITest, FailoverDeviceOneFailure) { void* listenComm = nullptr; void* sendComm = nullptr; void* recvComm = nullptr; - SetupCastConnection(/*dev=*/mergedDev, &listenComm, &sendComm, &recvComm); + ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/mergedDev, &listenComm, &sendComm, &recvComm)); constexpr size_t kMsgSize = 8192; std::vector sendBuf(kMsgSize), recvBuf(kMsgSize); @@ -1841,7 +1841,7 @@ TEST_F(NetIbMPITest, FailoverMultiRequestInFlight) { void* listenComm = nullptr; void* sendComm = nullptr; void* recvComm = nullptr; - SetupCastConnection(/*dev=*/mergedDev, &listenComm, &sendComm, &recvComm); + ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/mergedDev, &listenComm, &sendComm, &recvComm)); constexpr int kNumReqs = 4; constexpr size_t kMsgSize = 4096; @@ -1993,7 +1993,7 @@ TEST_F(NetIbMPITest, RecoveryThreadStartedOnlyWithParam) { void* listenComm = nullptr; void* sendComm = nullptr; void* recvComm = nullptr; - SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm); + ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm)); struct RecoveryEnabledResult { int recoveryEnabled; @@ -2202,7 +2202,7 @@ TEST_F(NetIbMPITest, RecoverySuccessRestoresTraffic) { void* listenComm = nullptr; void* sendComm = nullptr; void* recvComm = nullptr; - SetupCastConnection(/*dev=*/mergedDev, &listenComm, &sendComm, &recvComm); + ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/mergedDev, &listenComm, &sendComm, &recvComm)); constexpr size_t kMsgSize = 8192; constexpr int kPostRecoveryMsgs = 20; @@ -2579,7 +2579,7 @@ TEST_F(NetIbMPITest, RecoveryPendingWhileLinkDown) { void* listenComm = nullptr; void* sendComm = nullptr; void* recvComm = nullptr; - SetupCastConnection(/*dev=*/mergedDev, &listenComm, &sendComm, &recvComm); + ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/mergedDev, &listenComm, &sendComm, &recvComm)); constexpr size_t kMsgSize = 8192; std::vector sendBuf(kMsgSize), recvBuf(kMsgSize); @@ -2744,7 +2744,7 @@ TEST_F(NetIbMPITest, RecoveryDeviceOneFailure) { void* listenComm = nullptr; void* sendComm = nullptr; void* recvComm = nullptr; - SetupCastConnection(/*dev=*/mergedDev, &listenComm, &sendComm, &recvComm); + ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/mergedDev, &listenComm, &sendComm, &recvComm)); constexpr size_t kMsgSize = 8192; const size_t kBufSize = kMsgSize * 2; @@ -2991,7 +2991,7 @@ TEST_F(NetIbMPITest, RecoveryUdTimeoutExhaustsAttempts) { void* listenComm = nullptr; void* sendComm = nullptr; void* recvComm = nullptr; - SetupCastConnection(/*dev=*/mergedDev, &listenComm, &sendComm, &recvComm); + ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/mergedDev, &listenComm, &sendComm, &recvComm)); constexpr size_t kMsgSize = 8192; std::vector sendBuf(kMsgSize), recvBuf(kMsgSize); @@ -3120,7 +3120,7 @@ TEST_F(NetIbMPITest, FaultInjCastOpsPostSendErrno) { void* listenComm = nullptr; void* sendComm = nullptr; void* recvComm = nullptr; - SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm); + ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm)); constexpr size_t kMsgSize = 1024; std::vector sendBuf(kMsgSize), recvBuf(kMsgSize); @@ -3251,7 +3251,7 @@ TEST_F(NetIbMPITest, FaultInjCastOpsPollCqFlushNonFatal) { void* listenComm = nullptr; void* sendComm = nullptr; void* recvComm = nullptr; - SetupCastConnection(/*dev=*/mergedDev, &listenComm, &sendComm, &recvComm); + ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/mergedDev, &listenComm, &sendComm, &recvComm)); constexpr size_t kMsgSize = 8192; std::vector sendBuf(kMsgSize), recvBuf(kMsgSize); @@ -3382,7 +3382,7 @@ TEST_F(NetIbMPITest, FaultInjCastOpsPollCqSynthFatal) { void* listenComm = nullptr; void* sendComm = nullptr; void* recvComm = nullptr; - SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm); + ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm)); constexpr size_t kMsgSize = 1024; std::vector sendBuf(kMsgSize), recvBuf(kMsgSize); @@ -3499,7 +3499,7 @@ TEST_F(NetIbMPITest, FaultInjCastOpsPostRecvErrno) { void* listenComm = nullptr; void* sendComm = nullptr; void* recvComm = nullptr; - SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm); + ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm)); constexpr size_t kMsgSize = 1024; std::vector sendBuf(kMsgSize), recvBuf(kMsgSize); @@ -3626,7 +3626,7 @@ TEST_F(NetIbMPITest, FaultInjCastOpsPollCqInjectCountFinite) { void* listenComm = nullptr; void* sendComm = nullptr; void* recvComm = nullptr; - SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm); + ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm)); constexpr size_t kMsgSize = 1024; std::vector sendBuf(kMsgSize), recvBuf(kMsgSize); @@ -3803,7 +3803,7 @@ TEST_F(NetIbMPITest, FaultInjCastOpsApiInvalidArgs) { void* listenComm = nullptr; void* sendComm = nullptr; void* recvComm = nullptr; - SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm); + ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm)); constexpr size_t kMsgSize = 1024; std::vector buf(kMsgSize, 0); @@ -3878,7 +3878,7 @@ TEST_F(NetIbMPITest, FaultInjectionShimsAbsentUnlessRequested) { void* listenComm = nullptr; void* sendComm = nullptr; void* recvComm = nullptr; - SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm); + ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm)); constexpr size_t kMsgSize = 1024; std::vector buf(kMsgSize, 0); diff --git a/projects/rccl/test/transport/NetIbMPI/GdrFlushTests.cpp b/projects/rccl/test/transport/NetIbMPI/GdrFlushTests.cpp index 8dbeaf65b4a8..5a431182c6b5 100644 --- a/projects/rccl/test/transport/NetIbMPI/GdrFlushTests.cpp +++ b/projects/rccl/test/transport/NetIbMPI/GdrFlushTests.cpp @@ -93,7 +93,7 @@ class GdrFlushTest : public NetIbMPITest { ConnectionPair pair; NetConnectionGuard connGuard(net_); - SetupConnectionWithGuard(0, pair, connGuard); + ASSERT_NO_FATAL_FAILURE(SetupConnectionWithGuard(0, pair, connGuard)); const size_t bufferSize = kSmallBufferSize; void* buffer = nullptr; @@ -166,7 +166,8 @@ TEST_F(GdrFlushTest, CuMemDmaBuf_GpuRecvFlush_NoAsyncFatal) { if (!gdrPtrSupport()) GTEST_SKIP() << "no GDR backend (neither peermem nor dma-buf) on this device"; ncclResult_t flush = ncclSuccess; - RunRecvFlushBurst(/*iterations=*/4, /*verifyData=*/true, &flush); + ASSERT_NO_FATAL_FAILURE(RunRecvFlushBurst(/*iterations=*/4, /*verifyData=*/true, + &flush)); if (MPIEnvironment::world_rank == 0) EXPECT_EQ(flush, ncclSuccess) << "write+read flush over dma-buf scratchpad must not fault"; } @@ -182,7 +183,8 @@ TEST_F(GdrFlushTest, Peermem_GpuRecvFlush_NoAsyncFatal) { GTEST_SKIP() << "peermem (NCCL_PTR_CUDA) not available for the reg_mr scratchpad"; ncclResult_t flush = ncclSuccess; - RunRecvFlushBurst(/*iterations=*/4, /*verifyData=*/true, &flush); + ASSERT_NO_FATAL_FAILURE(RunRecvFlushBurst(/*iterations=*/4, /*verifyData=*/true, + &flush)); if (MPIEnvironment::world_rank == 0) EXPECT_EQ(flush, ncclSuccess) << "peermem RO=0 scratchpad flush must succeed"; } @@ -198,7 +200,8 @@ TEST_F(GdrFlushTest, FeatureDisabled_FallbackReadRecvBuffer) { if (!gdrPtrSupport()) GTEST_SKIP() << "no GDR backend (neither peermem nor dma-buf) on this device"; ncclResult_t flush = ncclSuccess; - RunRecvFlushBurst(/*iterations=*/4, /*verifyData=*/true, &flush); + ASSERT_NO_FATAL_FAILURE(RunRecvFlushBurst(/*iterations=*/4, /*verifyData=*/true, + &flush)); if (MPIEnvironment::world_rank == 0) EXPECT_EQ(flush, ncclSuccess) << "fallback flush (read recv buffer) must succeed"; } @@ -212,7 +215,8 @@ TEST_F(GdrFlushTest, RepeatedFlush_NoFaultBurst) { if (!gdrPtrSupport()) GTEST_SKIP() << "no GDR backend (neither peermem nor dma-buf) on this device"; ncclResult_t flush = ncclSuccess; - RunRecvFlushBurst(/*iterations=*/50, /*verifyData=*/false, &flush); + ASSERT_NO_FATAL_FAILURE(RunRecvFlushBurst(/*iterations=*/50, /*verifyData=*/false, + &flush)); if (MPIEnvironment::world_rank == 0) EXPECT_EQ(flush, ncclSuccess) << "no flush in the burst may raise a QP async-fatal"; } diff --git a/projects/rccl/test/transport/NetIbMPI/GeneralTests.cpp b/projects/rccl/test/transport/NetIbMPI/GeneralTests.cpp index 50b043f207d2..a6a7e5497f0f 100644 --- a/projects/rccl/test/transport/NetIbMPI/GeneralTests.cpp +++ b/projects/rccl/test/transport/NetIbMPI/GeneralTests.cpp @@ -89,7 +89,7 @@ TEST_F(NetIbMPITest, ListenAndConnect) { const int rank = MPIEnvironment::world_rank; ConnectionPair pair; NetConnectionGuard connGuard(net_); - SetupConnectionWithGuard(0, pair, connGuard); + ASSERT_NO_FATAL_FAILURE(SetupConnectionWithGuard(0, pair, connGuard)); if (rank == 0) { EXPECT_NE(pair.recvComm, nullptr) << "Recv comm should be established"; @@ -127,7 +127,7 @@ TEST_F(NetIbMPITest, RegisterHostMemory) { const int rank = MPIEnvironment::world_rank; ConnectionPair pair; NetConnectionGuard connGuard(net_); - SetupConnectionWithGuard(0, pair, connGuard); + ASSERT_NO_FATAL_FAILURE(SetupConnectionWithGuard(0, pair, connGuard)); const size_t bufferSize = kSmallBufferSize; void* buffer = malloc(bufferSize); @@ -154,7 +154,7 @@ TEST_F(NetIbMPITest, RegisterGpuMemory) { const int rank = MPIEnvironment::world_rank; ConnectionPair pair; NetConnectionGuard connGuard(net_); - SetupConnectionWithGuard(0, pair, connGuard); + ASSERT_NO_FATAL_FAILURE(SetupConnectionWithGuard(0, pair, connGuard)); const size_t bufferSize = kSmallBufferSize; void* buffer = nullptr; @@ -181,7 +181,7 @@ TEST_F(NetIbMPITest, RegisterMemoryNullPointer) { const int rank = MPIEnvironment::world_rank; ConnectionPair pair; NetConnectionGuard connGuard(net_); - SetupConnectionWithGuard(0, pair, connGuard); + ASSERT_NO_FATAL_FAILURE(SetupConnectionWithGuard(0, pair, connGuard)); void* mhandle = nullptr; void* comm = (rank == 0) ? pair.recvComm : pair.sendComm; @@ -201,7 +201,7 @@ TEST_F(NetIbMPITest, DeregisterNullHandle) { const int rank = MPIEnvironment::world_rank; ConnectionPair pair; NetConnectionGuard connGuard(net_); - SetupConnectionWithGuard(0, pair, connGuard); + ASSERT_NO_FATAL_FAILURE(SetupConnectionWithGuard(0, pair, connGuard)); void* comm = (rank == 0) ? pair.recvComm : pair.sendComm; @@ -229,7 +229,7 @@ TEST_F(NetIbMPITest, SimpleSendRecv) { if (nThreads == 1) { ConnectionPair pair; NetConnectionGuard connGuard(net_); - SetupConnectionWithGuard(0, pair, connGuard); + ASSERT_NO_FATAL_FAILURE(SetupConnectionWithGuard(0, pair, connGuard)); const size_t bufferSize = kSmallBufferSize; const int tag = 42; @@ -384,7 +384,7 @@ TEST_F(NetIbMPITest, SendRecvMultipleSizes) { ConnectionPair pair; NetConnectionGuard connGuard(net_); - SetupConnectionWithGuard(0, pair, connGuard); + ASSERT_NO_FATAL_FAILURE(SetupConnectionWithGuard(0, pair, connGuard)); for (size_t size : testSizes) { const int tag = 100; @@ -441,7 +441,7 @@ TEST_F(NetIbMPITest, SendRecvZeroSize) { const int rank = MPIEnvironment::world_rank; ConnectionPair pair; NetConnectionGuard connGuard(net_); - SetupConnectionWithGuard(0, pair, connGuard); + ASSERT_NO_FATAL_FAILURE(SetupConnectionWithGuard(0, pair, connGuard)); const size_t bufferSize = kSmallBufferSize; const int tag = 50; @@ -605,7 +605,7 @@ TEST_F(NetIbMPITest, MultipleSequentialTransfers) { if (nThreads == 1) { ConnectionPair pair; NetConnectionGuard connGuard(net_); - SetupConnectionWithGuard(0, pair, connGuard); + ASSERT_NO_FATAL_FAILURE(SetupConnectionWithGuard(0, pair, connGuard)); void* sendBuffer = nullptr; void* recvBuffer = nullptr; @@ -791,7 +791,7 @@ TEST_F(NetIbMPITest, LargeTransfer) { ConnectionPair pair; NetConnectionGuard connGuard(net_); - SetupConnectionWithGuard(0, pair, connGuard); + ASSERT_NO_FATAL_FAILURE(SetupConnectionWithGuard(0, pair, connGuard)); const size_t bufferSize = kLargeBufferSize; // 16 MB const int tag = 400; @@ -843,7 +843,7 @@ TEST_F(NetIbMPITest, CloseWithoutWaitingForCompletion) { ConnectionPair pair; NetConnectionGuard connGuard(net_); - SetupConnectionWithGuard(0, pair, connGuard); + ASSERT_NO_FATAL_FAILURE(SetupConnectionWithGuard(0, pair, connGuard)); } TEST_F(NetIbMPITest, ListenCloseListen) { @@ -861,65 +861,111 @@ TEST_F(NetIbMPITest, ListenCloseListen) { } for (int iter = 0; iter < 3; iter++) { - // Iteration 3: abandoned listen-close + // Iteration 3: abandoned listen-close. Self-contained on rank 0 and + // nothing on rank 1 waits on it, so a failure here is recorded rather + // than made fatal: a fatal exit would skip this iteration's handshake + // below and strand rank 1 in its MPI_Recv. if (iter == 2) { if (rank == 0) { void* abandonedListen = nullptr; ncclNetHandle_t abandonedHandle; - ASSERT_EQ(CreateListenComm(mergedDev, &abandonedHandle, &abandonedListen), + EXPECT_EQ(CreateListenComm(mergedDev, &abandonedHandle, &abandonedListen), ncclSuccess) << "Iter 3: abandoned listen failed"; - ASSERT_NE(abandonedListen, nullptr); - ASSERT_EQ(CloseListenComm(abandonedListen), ncclSuccess) + EXPECT_NE(abandonedListen, nullptr); + EXPECT_EQ(CloseListenComm(abandonedListen), ncclSuccess) << "Iter 3: close abandoned listen failed"; } } ConnectionPair pair; - ncclNetHandle_t handle; - - if (rank == 0) { - ASSERT_EQ(CreateListenComm(mergedDev, &handle, &pair.listenComm), ncclSuccess) - << "Listen failed iter " << iter; - ASSERT_NE(pair.listenComm, nullptr); - } + // A failed listen on rank 0 must still reach rank 1: a fatal ASSERT_EQ + // here would return before the handle is ever sent, and rank 1's + // MPI_Recv below has no timeout, so it would block until the suite + // timeout instead of failing with this test. The status word makes the + // failure observable to both ranks instead. + // + // handshake.handle sits at a 4-byte offset (after `status`), but + // ncclIbListen writes a uint64_t magic through the pointer it's given, + // so listen()/connect() must see pair.handle (8-byte aligned, same + // layout SetupConnection relies on) rather than &handshake.handle + // directly; the handshake only carries a byte-copy of it. + struct SetupHandshake { + int status; // 1 when rank 0's listener is ready and the handle is valid + ncclNetHandle_t handle; + } handshake = {}; if (rank == 0) { - MPI_Send(&handle, sizeof(handle), MPI_BYTE, peerRank, 0, MPI_COMM_WORLD); + ncclResult_t local = CreateListenComm(mergedDev, &pair.handle, &pair.listenComm); + handshake.status = (local == ncclSuccess && pair.listenComm != nullptr) ? 1 : 0; + if (handshake.status) memcpy(handshake.handle, pair.handle, sizeof(pair.handle)); + // Sent even on failure: the peer is waiting for this message. + MPI_Send(&handshake, sizeof(handshake), MPI_BYTE, peerRank, 0, MPI_COMM_WORLD); } else { - MPI_Recv(&handle, sizeof(handle), MPI_BYTE, peerRank, 0, + MPI_Recv(&handshake, sizeof(handshake), MPI_BYTE, peerRank, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE); + if (handshake.status) memcpy(pair.handle, handshake.handle, sizeof(pair.handle)); } int connectFailed = 0; if (rank == 0) { - ncclResult_t r = ncclSuccess; - while (!pair.recvComm && r == ncclSuccess) - r = AcceptConnection(pair.listenComm, &pair.recvComm); - if (r != ncclSuccess || !pair.recvComm) connectFailed = 1; + if (!handshake.status) { + connectFailed = 1; + } else { + // Bounded and paced. Accept is asynchronous, so it can keep + // returning ncclSuccess with a null comm -- the state the shared + // setup helper guards against -- and an unbounded spin here would + // hang this rank while the peer waits in the reduction below, which + // is the failure this test exists to have reported instead. + ncclResult_t r = ncclSuccess; + for (int attempt = 0; attempt < kConnectTimeoutMs / kPollIntervalMs + && !pair.recvComm && r == ncclSuccess; attempt++) { + r = AcceptConnection(pair.listenComm, &pair.recvComm); + if (!pair.recvComm && r == ncclSuccess) usleep(kPollIntervalUs); + } + if (r != ncclSuccess || !pair.recvComm) connectFailed = 1; + } } else { - ncclResult_t r = ncclSuccess; - while (!pair.sendComm && r == ncclSuccess) - r = ConnectToRemote(mergedDev, &handle, &pair.sendComm); - if (r != ncclSuccess || !pair.sendComm) connectFailed = 1; + if (!handshake.status) { + connectFailed = 1; // rank 0's listen failed + } else { + // Bounded and paced, for the same reason as the accept side. + ncclResult_t r = ncclSuccess; + for (int attempt = 0; attempt < kConnectTimeoutMs / kPollIntervalMs + && !pair.sendComm && r == ncclSuccess; attempt++) { + r = ConnectToRemote(mergedDev, &pair.handle, &pair.sendComm); + if (!pair.sendComm && r == ncclSuccess) usleep(kPollIntervalUs); + } + if (r != ncclSuccess || !pair.sendComm) connectFailed = 1; + } } MPI_Allreduce(MPI_IN_PLACE, &connectFailed, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD); if (connectFailed) { - if (rank == 0 && pair.listenComm) CloseListenComm(pair.listenComm); - FAIL() << "IB QP connect/accept failed (cross-subnet node pair)"; + // Either side may have completed its own comm before the peer failed, + // and FAIL() ends the test before the loop's normal close path, so + // every local handle is released here. Data comms first, then listen. + if (pair.sendComm) CloseSendComm(pair.sendComm); + if (pair.recvComm) CloseRecvComm(pair.recvComm); + if (pair.listenComm) CloseListenComm(pair.listenComm); + FAIL() << "IB QP listen/connect/accept failed iter " << iter + << " (cross-subnet node pair, or rank 0's listen failed)"; } - // Close: data comms first, then listen + // Close: data comms first, then listen. Reported, not asserted: a fatal + // assertion here leaves this rank's loop while its peer starts the next + // iteration and blocks in the MPI_Recv above -- the hang this test was + // rewritten to remove. ncclIbCloseRecv is a chain of NCCLCHECKs, so it can + // genuinely fail, and both ranks have to keep walking the same iterations. if (rank == 0) { - ASSERT_EQ(CloseRecvComm(pair.recvComm), ncclSuccess) + EXPECT_EQ(CloseRecvComm(pair.recvComm), ncclSuccess) << "CloseRecvComm failed iter " << iter; } else { - ASSERT_EQ(CloseSendComm(pair.sendComm), ncclSuccess) + EXPECT_EQ(CloseSendComm(pair.sendComm), ncclSuccess) << "CloseSendComm failed iter " << iter; } if (rank == 0) { - ASSERT_EQ(CloseListenComm(pair.listenComm), ncclSuccess) + EXPECT_EQ(CloseListenComm(pair.listenComm), ncclSuccess) << "CloseListenComm failed iter " << iter; } } diff --git a/projects/rccl/test/transport/NetIbMPI/GidChangeTests.cpp b/projects/rccl/test/transport/NetIbMPI/GidChangeTests.cpp index 324f8f16c46f..8c90028a21b5 100644 --- a/projects/rccl/test/transport/NetIbMPI/GidChangeTests.cpp +++ b/projects/rccl/test/transport/NetIbMPI/GidChangeTests.cpp @@ -285,7 +285,7 @@ TEST_P(NetIbGidChangeTest, ConnectSnapshotsDeviceCache) { void* listenComm = nullptr; void* sendComm = nullptr; void* recvComm = nullptr; - SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm); + ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm)); std::vector buf(kMsgSize, 0x5A); void* comm = (rank == 0) ? recvComm : sendComm; @@ -324,7 +324,7 @@ TEST_P(NetIbGidChangeTest, PortRecoveryRefreshesStaleCommGid) { void* listenComm = nullptr; void* sendComm = nullptr; void* recvComm = nullptr; - SetupCastConnection(mergedDev, &listenComm, &sendComm, &recvComm); + ASSERT_NO_FATAL_FAILURE(SetupCastConnection(mergedDev, &listenComm, &sendComm, &recvComm)); const size_t bufSize = kMsgSize * (kPostRecoveryMsgs + 1); std::vector sendBuf(bufSize), recvBuf(bufSize, 0); diff --git a/projects/rccl/test/transport/NetIbMPI/NetIbMPITestBase.hpp b/projects/rccl/test/transport/NetIbMPI/NetIbMPITestBase.hpp index 18f283424978..7aee7cc0dc6f 100644 --- a/projects/rccl/test/transport/NetIbMPI/NetIbMPITestBase.hpp +++ b/projects/rccl/test/transport/NetIbMPI/NetIbMPITestBase.hpp @@ -460,59 +460,95 @@ class NetIbMPITest : public MPITestBase { ncclNetHandle_t handle{}; }; + // Both ranks leave this together, or neither does. + // + // Every failure here used to return early, and each early return stranded the + // peer: a listen failure never sent the handle, so the connector sat in + // MPI_Recv until the suite timeout, and a failure after that skipped the + // closing barrier, so the other side sat there instead. A caller cannot fix + // that from outside, however carefully it checks the return value, because by + // then its peer is already blocked. So the handle message carries a status + // word, and the closing barrier became a reduction of it. ncclResult_t SetupConnection(int dev, ConnectionPair& pair, int rank, int peerRank) { // Cap the accept/connect handshake so a dead fabric fails fast instead // of spinning forever (AICOMRCCL-1577). const int maxAttempts = kConnectTimeoutMs / kPollIntervalMs; - if (rank == 0) { - // Rank 0: Listen - RCCL_TEST_CHECK(CreateListenComm(dev, &pair.handle, &pair.listenComm)); + struct SetupHandshake { + int status; // 1 when the listener is ready and the handle is valid + ncclNetHandle_t handle; + } handshake = {}; + ncclResult_t local = ncclSuccess; - // Send handle to peer - MPI_Send(&pair.handle, sizeof(ncclNetHandle_t), MPI_BYTE, peerRank, 0, MPI_COMM_WORLD); + if (rank == 0) { + local = CreateListenComm(dev, &pair.handle, &pair.listenComm); + // Non-null as well as the return code, so this reads the same as the cast + // and ListenCloseListen handshakes. Nothing reaches it today: ncclIbListen + // assigns *listenComm only on its success path. + handshake.status = (local == ncclSuccess && pair.listenComm != nullptr) ? 1 : 0; + if (handshake.status) memcpy(handshake.handle, pair.handle, sizeof(pair.handle)); + // Sent even on failure: the peer is waiting for this message. + MPI_Send(&handshake, sizeof(handshake), MPI_BYTE, peerRank, 0, MPI_COMM_WORLD); - // Accept connection int done = 0; int attempts = 0; - while (!done) { - ncclResult_t result = AcceptConnection(pair.listenComm, &pair.recvComm); - if (result != ncclSuccess) { - return result; - } + while (handshake.status && local == ncclSuccess && !done) { + local = AcceptConnection(pair.listenComm, &pair.recvComm); + if (local != ncclSuccess) break; if (pair.recvComm != nullptr) { done = 1; break; } if (++attempts >= maxAttempts) { - return ncclInternalError; + local = ncclInternalError; + break; } usleep(kPollIntervalUs); } } else { - // Rank 1: Connect - MPI_Recv(&pair.handle, sizeof(ncclNetHandle_t), MPI_BYTE, peerRank, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE); - - // Connect to peer - int done = 0; - int attempts = 0; - while (!done) { - ncclResult_t result = ConnectToRemote(dev, &pair.handle, &pair.sendComm); - if (result != ncclSuccess) { - return result; - } - if (pair.sendComm != nullptr) { - done = 1; - break; - } - if (++attempts >= maxAttempts) { - return ncclInternalError; + MPI_Recv(&handshake, sizeof(handshake), MPI_BYTE, peerRank, 0, MPI_COMM_WORLD, + MPI_STATUS_IGNORE); + if (!handshake.status) { + local = ncclRemoteError; // the peer could not listen + } else { + memcpy(pair.handle, handshake.handle, sizeof(pair.handle)); + int done = 0; + int attempts = 0; + while (!done) { + local = ConnectToRemote(dev, &pair.handle, &pair.sendComm); + if (local != ncclSuccess) break; + if (pair.sendComm != nullptr) { + done = 1; + break; + } + if (++attempts >= maxAttempts) { + local = ncclInternalError; + break; + } + usleep(kPollIntervalUs); } - usleep(kPollIntervalUs); } } - MPI_Barrier(MPI_COMM_WORLD); - return ncclSuccess; + // Replaces the closing barrier, so a one-sided failure ends the setup on + // both ranks instead of leaving one of them in the barrier. + int ok = (local == ncclSuccess) ? 1 : 0; + MPI_Allreduce(MPI_IN_PLACE, &ok, 1, MPI_INT, MPI_MIN, MPI_COMM_WORLD); + if (local == ncclSuccess && ok) return ncclSuccess; + + // Now that the failure is recoverable rather than fatal, whatever this rank did + // create has to be closed here, because nothing else will. + // + // Not because the callers construct their NetConnectionGuard after checking this + // result -- SetupConnectionWithGuard's callers construct it first, and hand it in. + // What holds is that the guard is only ever *populated* on success: that helper + // asserts on this return value and gives the guard its comms only afterwards, so + // on every failure path the guard owns nothing and has nothing to close. A leaked + // listener or QP would outlive the test and contaminate the rest of the process. + // Data comms first, then the listener. + if (pair.sendComm) { CloseSendComm(pair.sendComm); pair.sendComm = nullptr; } + if (pair.recvComm) { CloseRecvComm(pair.recvComm); pair.recvComm = nullptr; } + if (pair.listenComm) { CloseListenComm(pair.listenComm); pair.listenComm = nullptr; } + return (local != ncclSuccess) ? local : ncclRemoteError; } // Retry until the FIFO slot is ready. Re-seed the hint; a NULL isend clears it. @@ -888,37 +924,82 @@ class NetIbMPITest : public MPITestBase { // On return: rank 0 owns listenComm+recvComm, rank 1 owns sendComm. // Caller is responsible for closing all comms. + // Same contract as SetupConnection: the ranks agree before anyone asserts, so + // a one-sided failure cannot leave the peer waiting for a handle or a barrier. + // The assertion at the end is reached by both ranks with the same verdict, + // which is what makes a fatal failure safe here. void SetupCastConnection(int dev, void** listenComm, void** sendComm, void** recvComm) { const int rank = MPIEnvironment::world_rank; const int peer = 1 - rank; - ncclNetHandle_t handle; - memset(&handle, 0, sizeof(handle)); + struct SetupHandshake { + int status; + ncclNetHandle_t handle; + } handshake = {}; + // ncclIbListen writes a uint64_t magic through the handle pointer it's + // given, so listen()/connect() need an 8-byte-aligned buffer rather + // than &handshake.handle (which sits at a 4-byte offset after status). + // This local is aligned; the handshake only ever carries a byte-copy + // of it, same as SetupConnection's pair.handle. + alignas(8) ncclNetHandle_t handle{}; + // Tracks this rank's own outcome, kept separate from the peer's status + // in the handshake so the assertion message below can tell "my own + // accept/connect failed" apart from "the peer's listen failed", instead + // of both ranks reporting the same generic verdict. + bool localOk = true; + const char* localReason = "ok"; if (rank == 0) { - ASSERT_EQ(CreateListenComm(dev, &handle, listenComm), ncclSuccess); - ASSERT_NE(*listenComm, nullptr); - - MPI_Send(&handle, sizeof(handle), MPI_BYTE, peer, 0, MPI_COMM_WORLD); - - for (int i = 0; i < kMaxRetryAttempts && *recvComm == nullptr; i++) { - ASSERT_EQ(AcceptConnection(*listenComm, recvComm), ncclSuccess); - if (*recvComm == nullptr) usleep(kPollIntervalUs); + localOk = (CreateListenComm(dev, &handle, listenComm) == ncclSuccess) + && *listenComm != nullptr; + handshake.status = localOk ? 1 : 0; + if (!localOk) localReason = "listen failed"; + if (localOk) memcpy(handshake.handle, handle, sizeof(handle)); + // Sent even on failure: the peer is waiting for this message. + MPI_Send(&handshake, sizeof(handshake), MPI_BYTE, peer, 0, MPI_COMM_WORLD); + + for (int i = 0; localOk && i < kConnectTimeoutMs / kPollIntervalMs + && *recvComm == nullptr; i++) { + if (AcceptConnection(*listenComm, recvComm) != ncclSuccess) { + localOk = false; + localReason = "accept failed"; + } + if (localOk && *recvComm == nullptr) usleep(kPollIntervalUs); } - ASSERT_NE(*recvComm, nullptr); + if (localOk && *recvComm == nullptr) { localOk = false; localReason = "accept timed out"; } } else { - MPI_Recv(&handle, sizeof(handle), MPI_BYTE, peer, 0, MPI_COMM_WORLD, + MPI_Recv(&handshake, sizeof(handshake), MPI_BYTE, peer, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE); - - for (int i = 0; i < kMaxRetryAttempts && *sendComm == nullptr; i++) { - ncclResult_t r = ConnectToRemote(dev, &handle, sendComm); - ASSERT_EQ(r, ncclSuccess); - if (*sendComm == nullptr) usleep(kPollIntervalUs); + if (handshake.status != 1) { + localOk = false; + localReason = "peer's listen failed"; + } else { + memcpy(handle, handshake.handle, sizeof(handle)); + for (int i = 0; localOk && i < kConnectTimeoutMs / kPollIntervalMs + && *sendComm == nullptr; i++) { + if (ConnectToRemote(dev, &handle, sendComm) != ncclSuccess) { + localOk = false; + localReason = "connect failed"; + } + if (localOk && *sendComm == nullptr) usleep(kPollIntervalUs); + } + if (localOk && *sendComm == nullptr) { localOk = false; localReason = "connect timed out"; } } - ASSERT_NE(*sendComm, nullptr); } - MPI_Barrier(MPI_COMM_WORLD); + int ok = localOk ? 1 : 0; + MPI_Allreduce(MPI_IN_PLACE, &ok, 1, MPI_INT, MPI_MIN, MPI_COMM_WORLD); + if (!ok) { + // The assertion below is fatal and unwinds through the caller's + // ASSERT_NO_FATAL_FAILURE before its teardown runs, so anything this + // rank created must be released here or it stays open for the rest + // of the process. Data comms first, then the listener. + if (*sendComm) { CloseSendComm(*sendComm); *sendComm = nullptr; } + if (*recvComm) { CloseRecvComm(*recvComm); *recvComm = nullptr; } + if (*listenComm) { CloseListenComm(*listenComm); *listenComm = nullptr; } + } + ASSERT_EQ(ok, 1) << "IB-CAST connection setup failed on at least one rank (this rank: " + << localReason << ")"; } // Composite block: Warmup send + read real nqps from sendComm on rank 1. @@ -3350,16 +3431,21 @@ class NetIbMPITest : public MPITestBase { // Deregister mhandle, close comms rank-conditionally, barrier. // rank 0 closes recvComm + listenComm; rank 1 closes sendComm. // Call after any pre-teardown MPI_Barrier the test needs. + // Reported, not asserted, because of the barrier at the end: these checks are all + // rank-local, and a fatal one would return from this helper while the peer waited in + // that collective -- the hang this suite has been removing everywhere else. Every + // caller reaches the barrier now, and a close that failed is still a failed test. void TeardownConnection(void* recvComm, void* listenComm, void* sendComm, void* mhandle) { const int rank = MPIEnvironment::world_rank; void* comm = (rank == 0) ? recvComm : sendComm; - ASSERT_EQ(DeregisterMemory(comm, mhandle), ncclSuccess); + EXPECT_EQ(DeregisterMemory(comm, mhandle), ncclSuccess) + << "deregistering the connection's memory failed on rank " << rank; if (rank == 0) { - ASSERT_EQ(CloseRecvComm(recvComm), ncclSuccess); - ASSERT_EQ(CloseListenComm(listenComm), ncclSuccess); + EXPECT_EQ(CloseRecvComm(recvComm), ncclSuccess) << "CloseRecvComm failed"; + EXPECT_EQ(CloseListenComm(listenComm), ncclSuccess) << "CloseListenComm failed"; } else { - ASSERT_EQ(CloseSendComm(sendComm), ncclSuccess); + EXPECT_EQ(CloseSendComm(sendComm), ncclSuccess) << "CloseSendComm failed"; } MPI_Barrier(MPI_COMM_WORLD); } diff --git a/projects/rccl/test/transport/NetIbMPI/NicFusionTests.cpp b/projects/rccl/test/transport/NetIbMPI/NicFusionTests.cpp index 1e0a1a30cafd..af636f453954 100644 --- a/projects/rccl/test/transport/NetIbMPI/NicFusionTests.cpp +++ b/projects/rccl/test/transport/NetIbMPI/NicFusionTests.cpp @@ -262,7 +262,7 @@ TEST_F(NetIbMPITest, ConnectAndTransfer_VNic) { const int rank = MPIEnvironment::world_rank; ConnectionPair pair; NetConnectionGuard connGuard(net_); - SetupConnectionWithGuard(vdev, pair, connGuard); + ASSERT_NO_FATAL_FAILURE(SetupConnectionWithGuard(vdev, pair, connGuard)); const size_t bufferSize = kSmallBufferSize; const int tag = 500; @@ -532,7 +532,7 @@ TEST_F(NetIbMPITest, CloseWithoutTransfer_VNic) { { ConnectionPair pair; NetConnectionGuard connGuard(net_); - SetupConnectionWithGuard(vdev, pair, connGuard); + ASSERT_NO_FATAL_FAILURE(SetupConnectionWithGuard(vdev, pair, connGuard)); // Guard triggers closeSend/closeRecv/closeListen on a connection that // never had regMr, isend, or irecv called. } @@ -542,7 +542,7 @@ TEST_F(NetIbMPITest, CloseWithoutTransfer_VNic) { // Phase 2: reconnect on the same vdev and do a transfer to verify no corruption. ConnectionPair pair2; NetConnectionGuard connGuard2(net_); - SetupConnectionWithGuard(vdev, pair2, connGuard2); + ASSERT_NO_FATAL_FAILURE(SetupConnectionWithGuard(vdev, pair2, connGuard2)); const size_t bufferSize = kSmallBufferSize; const int tag = 520; @@ -676,7 +676,7 @@ TEST_F(NetIbMPITest, RegDeregCycling_VNic) { ConnectionPair pair; NetConnectionGuard connGuard(net_); - SetupConnectionWithGuard(vdev, pair, connGuard); + ASSERT_NO_FATAL_FAILURE(SetupConnectionWithGuard(vdev, pair, connGuard)); const size_t bufferSize = kSmallBufferSize; void* buffer = malloc(bufferSize); @@ -773,7 +773,7 @@ TEST_F(NetIbMPITest, LargeTransfer_VNic) { ConnectionPair pair; NetConnectionGuard connGuard(net_); - SetupConnectionWithGuard(vdev, pair, connGuard); + ASSERT_NO_FATAL_FAILURE(SetupConnectionWithGuard(vdev, pair, connGuard)); // 64MB buffer — ncclIbMultiSend stripes this across the doubled QPs. const size_t bufferSize = 64 * 1024 * 1024; @@ -860,7 +860,7 @@ TEST_F(NetIbMPITest, MixedSizes_VNic) { ConnectionPair pair; NetConnectionGuard connGuard(net_); - SetupConnectionWithGuard(vdev, pair, connGuard); + ASSERT_NO_FATAL_FAILURE(SetupConnectionWithGuard(vdev, pair, connGuard)); for (size_t idx = 0; idx < testSizes.size(); idx++) { size_t size = testSizes[idx]; @@ -945,7 +945,7 @@ TEST_F(NetIbMPITest, UnalignedSizeTransfer_VNic) { ConnectionPair pair; NetConnectionGuard connGuard(net_); - SetupConnectionWithGuard(vdev, pair, connGuard); + ASSERT_NO_FATAL_FAILURE(SetupConnectionWithGuard(vdev, pair, connGuard)); for (size_t idx = 0; idx < testSizes.size(); idx++) { size_t size = testSizes[idx]; @@ -1274,7 +1274,7 @@ TEST_F(NetIbMPITest, FlushRepeated_VNic) { ConnectionPair pair; NetConnectionGuard connGuard(net_); - SetupConnectionWithGuard(vdev, pair, connGuard); + ASSERT_NO_FATAL_FAILURE(SetupConnectionWithGuard(vdev, pair, connGuard)); const size_t bufferSize = kSmallBufferSize; const int tag = 600; @@ -1388,7 +1388,7 @@ TEST_F(NetIbMPITest, SequentialTransfers_VNic) { // Single connection through the vNIC, reused across all 100 iterations. ConnectionPair pair; NetConnectionGuard connGuard(net_); - SetupConnectionWithGuard(vdev, pair, connGuard); + ASSERT_NO_FATAL_FAILURE(SetupConnectionWithGuard(vdev, pair, connGuard)); const size_t bufferSize = kSmallBufferSize; const int tag = 700; diff --git a/projects/rccl/test/transport/NetIbMPI/OptRecvCompletionTests.cpp b/projects/rccl/test/transport/NetIbMPI/OptRecvCompletionTests.cpp index 0a5a4daca002..0169932f8829 100644 --- a/projects/rccl/test/transport/NetIbMPI/OptRecvCompletionTests.cpp +++ b/projects/rccl/test/transport/NetIbMPI/OptRecvCompletionTests.cpp @@ -18,7 +18,7 @@ TEST_F(NetIbMPITest, OptRecvCompletionFlagEnabled) { ConnectionPair pair; NetConnectionGuard connGuard(net_); - SetupConnectionWithGuard(0, pair, connGuard); + ASSERT_NO_FATAL_FAILURE(SetupConnectionWithGuard(0, pair, connGuard)); const int rank = MPIEnvironment::world_rank; void* comm = (rank == 0) ? pair.recvComm : pair.sendComm; @@ -37,7 +37,7 @@ TEST_F(NetIbMPITest, OptRecvCompletionFlagDisabledByQpSched) { ConnectionPair pair; NetConnectionGuard connGuard(net_); - SetupConnectionWithGuard(0, pair, connGuard); + ASSERT_NO_FATAL_FAILURE(SetupConnectionWithGuard(0, pair, connGuard)); const int rank = MPIEnvironment::world_rank; void* comm = (rank == 0) ? pair.recvComm : pair.sendComm; @@ -59,7 +59,7 @@ TEST_F(NetIbMPITest, OptRecvCompletionDisabledKeepsRecvSize) { ConnectionPair pair; NetConnectionGuard connGuard(net_); - SetupConnectionWithGuard(0, pair, connGuard); + ASSERT_NO_FATAL_FAILURE(SetupConnectionWithGuard(0, pair, connGuard)); const size_t bufferSize = kSmallBufferSize; const int tag = 78; @@ -117,7 +117,7 @@ TEST_F(NetIbMPITest, OptRecvCompletionSkipWritePath) { ConnectionPair pair; NetConnectionGuard connGuard(net_); - SetupConnectionWithGuard(0, pair, connGuard); + ASSERT_NO_FATAL_FAILURE(SetupConnectionWithGuard(0, pair, connGuard)); const size_t bufferSize = kSmallBufferSize; const int tag = 77; @@ -174,7 +174,7 @@ void NetIbMPITest::OptRecvCompletionRunMultiRecv(bool optRecvHint) { const int rank = MPIEnvironment::world_rank; ConnectionPair pair; NetConnectionGuard connGuard(net_); - SetupConnectionWithGuard(0, pair, connGuard); + ASSERT_NO_FATAL_FAILURE(SetupConnectionWithGuard(0, pair, connGuard)); constexpr int kN = 4; size_t sizes[kN] = {64, 1024, 4096, 65536}; diff --git a/projects/rccl/test/transport/NetIbMPI/StressTests.cpp b/projects/rccl/test/transport/NetIbMPI/StressTests.cpp index 6fdd677f7e0e..4a2e892d8e3f 100644 --- a/projects/rccl/test/transport/NetIbMPI/StressTests.cpp +++ b/projects/rccl/test/transport/NetIbMPI/StressTests.cpp @@ -52,7 +52,7 @@ TEST_F(NetIbMPITest, InvalidRecvCount) { ConnectionPair cp; NetConnectionGuard guard(net_); - SetupConnectionWithGuard(/*dev=*/0, cp, guard); + ASSERT_NO_FATAL_FAILURE(SetupConnectionWithGuard(/*dev=*/0, cp, guard)); if (rank == 0) { // n=9 > NCCL_NET_IB_MAX_RECVS (8) — must return ncclInternalError @@ -151,7 +151,7 @@ TEST_F(NetIbMPITest, MrCacheRefCount) { ConnectionPair cp; NetConnectionGuard guard(net_); - SetupConnectionWithGuard(/*dev=*/0, cp, guard); + ASSERT_NO_FATAL_FAILURE(SetupConnectionWithGuard(/*dev=*/0, cp, guard)); void* comm = (rank == 0) ? cp.recvComm : cp.sendComm; @@ -188,7 +188,7 @@ TEST_F(NetIbMPITest, SendSizeClamping) { ConnectionPair cp; NetConnectionGuard guard(net_); - SetupConnectionWithGuard(/*dev=*/0, cp, guard); + ASSERT_NO_FATAL_FAILURE(SetupConnectionWithGuard(/*dev=*/0, cp, guard)); static constexpr size_t kRecvSize = 4096; static constexpr size_t kSendSize = 65536; // larger than kRecvSize — will be clamped @@ -314,7 +314,7 @@ TEST_F(NetIbMPITest, TagZeroReuse) { ConnectionPair cp; NetConnectionGuard guard(net_); - SetupConnectionWithGuard(/*dev=*/0, cp, guard); + ASSERT_NO_FATAL_FAILURE(SetupConnectionWithGuard(/*dev=*/0, cp, guard)); const size_t sz = kSmallBufferSize; auto buf = makeHostBufferAutoGuard(malloc(sz)); @@ -349,7 +349,7 @@ TEST_F(NetIbMPITest, AdaptiveRoutingThresholdBoundary) { ConnectionPair cp; NetConnectionGuard guard(net_); - SetupConnectionWithGuard(0, cp, guard); + ASSERT_NO_FATAL_FAILURE(SetupConnectionWithGuard(0, cp, guard)); const size_t maxSz = 16384; auto buf = makeHostBufferAutoGuard(malloc(maxSz)); @@ -390,7 +390,7 @@ TEST_F(NetIbMPITest, InlineSendBoundary) { ConnectionPair cp; NetConnectionGuard guard(net_); - SetupConnectionWithGuard(0, cp, guard); + ASSERT_NO_FATAL_FAILURE(SetupConnectionWithGuard(0, cp, guard)); const size_t maxSz = 1024 * 1024; // 1 MB auto buf = makeHostBufferAutoGuard(malloc(maxSz)); @@ -434,7 +434,7 @@ TEST_F(NetIbMPITest, MixedSizeBarrage) { ConnectionPair cp; NetConnectionGuard guard(net_); - SetupConnectionWithGuard(0, cp, guard); + ASSERT_NO_FATAL_FAILURE(SetupConnectionWithGuard(0, cp, guard)); // From the shared ladder, not a constant beside it: a size added to // kBarrageSizes would otherwise send DoSendRecv past the end of this buffer. @@ -588,7 +588,7 @@ TEST_F(NetIbMPITest, FifoPressureSenderFast) { ConnectionPair cp; NetConnectionGuard guard(net_); - SetupConnectionWithGuard(0, cp, guard); + ASSERT_NO_FATAL_FAILURE(SetupConnectionWithGuard(0, cp, guard)); const size_t sz = kSmallBufferSize; auto buf = makeHostBufferAutoGuard(malloc(sz)); @@ -725,7 +725,7 @@ TEST_F(NetIbMPITest, RequestSlotExhaustion) { ConnectionPair cp; NetConnectionGuard guard(net_); - SetupConnectionWithGuard(0, cp, guard); + ASSERT_NO_FATAL_FAILURE(SetupConnectionWithGuard(0, cp, guard)); const size_t sz = 256; auto buf = makeHostBufferAutoGuard(malloc(sz * kMaxReqsPerComm)); @@ -880,7 +880,7 @@ TEST_F(NetIbMPITest, MemoryRegistrationStorm) { ConnectionPair cp; NetConnectionGuard guard(net_); - SetupConnectionWithGuard(0, cp, guard); + ASSERT_NO_FATAL_FAILURE(SetupConnectionWithGuard(0, cp, guard)); void* comm = (rank == 0) ? cp.recvComm : cp.sendComm; @@ -1321,7 +1321,7 @@ TEST_F(NetIbMPITest, MultiQpSplitDataStress) { ConnectionPair cp; NetConnectionGuard guard(net_); - SetupConnectionWithGuard(0, cp, guard); + ASSERT_NO_FATAL_FAILURE(SetupConnectionWithGuard(0, cp, guard)); const size_t maxSz = 1024 * 1024; // 1 MB auto buf = makeHostBufferAutoGuard(malloc(maxSz)); @@ -1368,7 +1368,7 @@ TEST_F(NetIbMPITest, MultiQpNoSplitStress) { ConnectionPair cp; NetConnectionGuard guard(net_); - SetupConnectionWithGuard(0, cp, guard); + ASSERT_NO_FATAL_FAILURE(SetupConnectionWithGuard(0, cp, guard)); const size_t maxSz = 1024 * 1024; auto buf = makeHostBufferAutoGuard(malloc(maxSz)); @@ -1872,7 +1872,7 @@ TEST_F(NetIbMPITest, LongRunningEndurance) { ConnectionPair cp; NetConnectionGuard guard(net_); - SetupConnectionWithGuard(0, cp, guard); + ASSERT_NO_FATAL_FAILURE(SetupConnectionWithGuard(0, cp, guard)); const size_t sz = kSmallBufferSize; auto buf = makeHostBufferAutoGuard(malloc(sz)); @@ -2027,7 +2027,7 @@ TEST_F(NetIbMPITest, GpuMemoryTransferStress) { ConnectionPair cp; NetConnectionGuard guard(net_); - SetupConnectionWithGuard(0, cp, guard); + ASSERT_NO_FATAL_FAILURE(SetupConnectionWithGuard(0, cp, guard)); static constexpr int kCycles = 5; const size_t sizes[] = {512 * 1024, 1024 * 1024, 2 * 1024 * 1024, 4 * 1024 * 1024, 1024 * 1024}; @@ -2124,7 +2124,7 @@ TEST_F(NetIbMPITest, RapidRecvPostDrain) { ConnectionPair cp; NetConnectionGuard guard(net_); - SetupConnectionWithGuard(0, cp, guard); + ASSERT_NO_FATAL_FAILURE(SetupConnectionWithGuard(0, cp, guard)); static constexpr int kCycles = 100; const size_t sz = 256; diff --git a/projects/rccl/tools/scripts/test_runner/configs/mi300_mellanox_ib_func.json b/projects/rccl/tools/scripts/test_runner/configs/mi300_mellanox_ib_func.json index 956422a87c1c..4f9e1458291f 100644 --- a/projects/rccl/tools/scripts/test_runner/configs/mi300_mellanox_ib_func.json +++ b/projects/rccl/tools/scripts/test_runner/configs/mi300_mellanox_ib_func.json @@ -613,6 +613,11 @@ "name": "Cast_SendRecvZeroSize", "description": "Zero-byte send: WRR oneQp path taken (dataPerQp=0 < splitDataMin), 1 token consumed, received size==0", "test_filter": "NetIbMPITest.CastSendRecvZeroSize" + }, + { + "name": "Cast_RegistrationRejectsBadArguments", + "description": "regMr, regMrDmaBuf and deregMr reject a null communicator, and regMr rejects a zero size, instead of dereferencing it", + "test_filter": "NetIbMPITest.CastRegistrationRejectsBadArguments" } ] }, diff --git a/projects/rccl/tools/scripts/test_runner/configs/mi300x_mellanox_ib.json b/projects/rccl/tools/scripts/test_runner/configs/mi300x_mellanox_ib.json index 977a13a1dac0..08306ccfb66a 100644 --- a/projects/rccl/tools/scripts/test_runner/configs/mi300x_mellanox_ib.json +++ b/projects/rccl/tools/scripts/test_runner/configs/mi300x_mellanox_ib.json @@ -1856,6 +1856,11 @@ "name": "Cast_SendRecvZeroSize", "description": "Zero-byte send: WRR oneQp path taken (dataPerQp=0 < splitDataMin), 1 token consumed, received size==0", "test_filter": "NetIbMPITest.CastSendRecvZeroSize" + }, + { + "name": "Cast_RegistrationRejectsBadArguments", + "description": "regMr, regMrDmaBuf and deregMr reject a null communicator, and regMr rejects a zero size, instead of dereferencing it", + "test_filter": "NetIbMPITest.CastRegistrationRejectsBadArguments" } ] }, diff --git a/projects/rccl/tools/scripts/test_runner/configs/mi355x_thor2_roce.json b/projects/rccl/tools/scripts/test_runner/configs/mi355x_thor2_roce.json index 9b34b771a7ab..599e06c8c991 100644 --- a/projects/rccl/tools/scripts/test_runner/configs/mi355x_thor2_roce.json +++ b/projects/rccl/tools/scripts/test_runner/configs/mi355x_thor2_roce.json @@ -1937,6 +1937,11 @@ "name": "Cast_SendRecvZeroSize", "description": "Zero-byte send: WRR oneQp path taken (dataPerQp=0 < splitDataMin), 1 token consumed, received size==0", "test_filter": "NetIbMPITest.CastSendRecvZeroSize" + }, + { + "name": "Cast_RegistrationRejectsBadArguments", + "description": "regMr, regMrDmaBuf and deregMr reject a null communicator, and regMr rejects a zero size, instead of dereferencing it", + "test_filter": "NetIbMPITest.CastRegistrationRejectsBadArguments" } ] }, diff --git a/projects/rccl/tools/scripts/test_runner/configs/mi355x_thor2_roce_func.json b/projects/rccl/tools/scripts/test_runner/configs/mi355x_thor2_roce_func.json index 2becaccc974f..6a38216d5cb1 100644 --- a/projects/rccl/tools/scripts/test_runner/configs/mi355x_thor2_roce_func.json +++ b/projects/rccl/tools/scripts/test_runner/configs/mi355x_thor2_roce_func.json @@ -619,6 +619,11 @@ "name": "Cast_SendRecvZeroSize", "description": "Zero-byte send: WRR oneQp path taken (dataPerQp=0 < splitDataMin), 1 token consumed, received size==0", "test_filter": "NetIbMPITest.CastSendRecvZeroSize" + }, + { + "name": "Cast_RegistrationRejectsBadArguments", + "description": "regMr, regMrDmaBuf and deregMr reject a null communicator, and regMr rejects a zero size, instead of dereferencing it", + "test_filter": "NetIbMPITest.CastRegistrationRejectsBadArguments" } ] }, diff --git a/projects/rccl/tools/scripts/test_runner/configs/mi455_ainic_roce.json b/projects/rccl/tools/scripts/test_runner/configs/mi455_ainic_roce.json index 04fb9f4b3024..0c062aefed6c 100644 --- a/projects/rccl/tools/scripts/test_runner/configs/mi455_ainic_roce.json +++ b/projects/rccl/tools/scripts/test_runner/configs/mi455_ainic_roce.json @@ -628,6 +628,11 @@ "name": "Cast_SendRecvZeroSize", "description": "Zero-byte send: WRR oneQp path taken (dataPerQp=0 < splitDataMin), 1 token consumed, received size==0", "test_filter": "NetIbMPITest.CastSendRecvZeroSize" + }, + { + "name": "Cast_RegistrationRejectsBadArguments", + "description": "regMr, regMrDmaBuf and deregMr reject a null communicator, and regMr rejects a zero size, instead of dereferencing it", + "test_filter": "NetIbMPITest.CastRegistrationRejectsBadArguments" } ] }, diff --git a/projects/rccl/tools/scripts/test_runner/configs/mi455_ainic_roce_func.json b/projects/rccl/tools/scripts/test_runner/configs/mi455_ainic_roce_func.json index 68c92285f639..b24d2a77c13d 100644 --- a/projects/rccl/tools/scripts/test_runner/configs/mi455_ainic_roce_func.json +++ b/projects/rccl/tools/scripts/test_runner/configs/mi455_ainic_roce_func.json @@ -613,6 +613,11 @@ "name": "Cast_SendRecvZeroSize", "description": "Zero-byte send: WRR oneQp path taken (dataPerQp=0 < splitDataMin), 1 token consumed, received size==0", "test_filter": "NetIbMPITest.CastSendRecvZeroSize" + }, + { + "name": "Cast_RegistrationRejectsBadArguments", + "description": "regMr, regMrDmaBuf and deregMr reject a null communicator, and regMr rejects a zero size, instead of dereferencing it", + "test_filter": "NetIbMPITest.CastRegistrationRejectsBadArguments" } ] }, diff --git a/projects/rccl/tools/scripts/test_runner/configs/net_ib_transport.json b/projects/rccl/tools/scripts/test_runner/configs/net_ib_transport.json index f1f92fd4fa64..07a34f13d06b 100644 --- a/projects/rccl/tools/scripts/test_runner/configs/net_ib_transport.json +++ b/projects/rccl/tools/scripts/test_runner/configs/net_ib_transport.json @@ -502,6 +502,11 @@ "name": "Cast_SendRecvZeroSize", "description": "Zero-byte send: WRR oneQp path taken (dataPerQp=0 < splitDataMin), 1 token consumed, received size==0", "test_filter": "NetIbMPITest.CastSendRecvZeroSize" + }, + { + "name": "Cast_RegistrationRejectsBadArguments", + "description": "regMr, regMrDmaBuf and deregMr reject a null communicator, and regMr rejects a zero size, instead of dereferencing it", + "test_filter": "NetIbMPITest.CastRegistrationRejectsBadArguments" } ] },