Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 13 additions & 1 deletion projects/rccl/src/transport/net_ib_cast/reg.cc
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,15 @@ static ncclResult_t ncclIbRegMrDmaBufInternal2(ncclIbNetCommDevBase* base, void*
ncclResult_t IbCastRegMrDmaBufInternal(void* comm, void* data, size_t size, int type, uint64_t offset, int fd,
uint64_t mrFlags, void** mhandle) {
ncclResult_t ret = ncclSuccess;
assert(size > 0);
// A caller that lost its connection reaches here with a null comm, and reading
// the device list off it segfaults inside the plugin instead of telling the
// caller what was wrong. size == 0 hit the same fate below via the page-count
// math, so it is reported the same way rather than left as a release-build
// assert.
if (comm == NULL || mhandle == NULL || size == 0) {
WARN("NET/IB-CAST: regMr/regMrDmaBuf called with comm=%p mhandle=%p size=%zu", comm, (void*)mhandle, size);
return ncclInvalidArgument;
Comment thread
i-kosarev marked this conversation as resolved.
Resolved
Comment thread
i-kosarev marked this conversation as resolved.
Resolved
Comment thread
i-kosarev marked this conversation as resolved.
Resolved
}
struct ncclIbNetCommBase* base = (struct ncclIbNetCommBase*)comm;
struct ncclIbMrHandle* mhandleWrapper = (struct ncclIbMrHandle*)malloc(sizeof(struct ncclIbMrHandle));
if (mhandleWrapper == nullptr) {
Expand Down Expand Up @@ -122,6 +130,10 @@ ncclResult_t IbCastDeregMrInternal(ncclIbNetCommDevBase* base, ibv_mr* mhandle)

ncclResult_t IbCastDeregMr(void* comm, void* mhandle) {
if (mhandle == NULL) return ncclSuccess;
if (comm == NULL) {
WARN("NET/IB-CAST: deregMr called with a null comm");
return ncclInvalidArgument;
}

struct ncclIbMrHandle* mhandleWrapper = (struct ncclIbMrHandle*)mhandle;
struct ncclIbNetCommBase* base = (struct ncclIbNetCommBase*)comm;
Expand Down
231 changes: 198 additions & 33 deletions projects/rccl/test/transport/NetIbMPI/CastTests.cpp

Large diffs are not rendered by default.

50 changes: 25 additions & 25 deletions projects/rccl/test/transport/NetIbMPI/FaultInjectTests.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ TEST_F(NetIbMPITest, FaultInjCastQpErrorIsFatal) {
void* listenComm = nullptr;
void* sendComm = nullptr;
void* recvComm = nullptr;
SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm);
ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm));

constexpr size_t kMsgSize = 1024;
std::vector<char> sendBuf(kMsgSize), recvBuf(kMsgSize);
Expand Down Expand Up @@ -278,7 +278,7 @@ TEST_F(NetIbMPITest, FaultInjCastSlowQpRebalances) {
void* listenComm = nullptr;
void* sendComm = nullptr;
void* recvComm = nullptr;
SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm);
ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm));

constexpr int kNMsgs = 500;
// Keep below splitDataMin so messages take the WRR token path.
Expand Down Expand Up @@ -483,7 +483,7 @@ TEST_F(NetIbMPITest, FaultInjCastDelayDataIntegrity) {
void* listenComm = nullptr;
void* sendComm = nullptr;
void* recvComm = nullptr;
SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm);
ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm));

constexpr int kNMsgs = 50;
constexpr size_t kMsgSz = 8192;
Expand Down Expand Up @@ -553,7 +553,7 @@ TEST_F(NetIbMPITest, FaultInjCastSingleQpErrorIsFatal) {
void* listenComm = nullptr;
void* sendComm = nullptr;
void* recvComm = nullptr;
SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm);
ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm));

constexpr size_t kMsgSize = 1024; // below splitDataMin → single-QP WRR path
std::vector<char> sendBuf(kMsgSize), recvBuf(kMsgSize);
Expand Down Expand Up @@ -802,7 +802,7 @@ TEST_F(NetIbMPITest, FaultInjCastQpErrorClearRecovers) {
void* listenComm1 = nullptr;
void* sendComm1 = nullptr;
void* recvComm1 = nullptr;
SetupCastConnection(/*dev=*/0, &listenComm1, &sendComm1, &recvComm1);
ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/0, &listenComm1, &sendComm1, &recvComm1));

constexpr size_t kMsgSize = 1024;
std::vector<char> sendBuf(kMsgSize), recvBuf(kMsgSize);
Expand Down Expand Up @@ -897,7 +897,7 @@ TEST_F(NetIbMPITest, FaultInjCastQpErrorClearRecovers) {
void* listenComm2 = nullptr;
void* sendComm2 = nullptr;
void* recvComm2 = nullptr;
SetupCastConnection(/*dev=*/0, &listenComm2, &sendComm2, &recvComm2);
ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/0, &listenComm2, &sendComm2, &recvComm2));

void* comm2 = (rank == 0) ? recvComm2 : sendComm2;
char* regBuf2 = (rank == 0) ? recvBuf2.data() : sendBuf2.data();
Expand Down Expand Up @@ -954,7 +954,7 @@ TEST_F(NetIbMPITest, FailoverErrorCodeWhitelist) {
void* listenComm = nullptr;
void* sendComm = nullptr;
void* recvComm = nullptr;
SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm);
ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm));

struct WhitelistResult {
int hasResiliency;
Expand Down Expand Up @@ -1086,7 +1086,7 @@ TEST_F(NetIbMPITest, FailoverCqeErrorRecovered) {
void* listenComm = nullptr;
void* sendComm = nullptr;
void* recvComm = nullptr;
SetupCastConnection(/*dev=*/mergedDev, &listenComm, &sendComm, &recvComm);
ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/mergedDev, &listenComm, &sendComm, &recvComm));

constexpr size_t kMsgSize = 8192;
std::vector<char> sendBuf(kMsgSize), recvBuf(kMsgSize);
Expand Down Expand Up @@ -1235,7 +1235,7 @@ TEST_F(NetIbMPITest, FailoverSingleDeviceTopology) {
void* listenComm = nullptr;
void* sendComm = nullptr;
void* recvComm = nullptr;
SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm);
ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm));

constexpr size_t kMsgSize = 4096;
std::vector<char> sendBuf(kMsgSize), recvBuf(kMsgSize);
Expand Down Expand Up @@ -1358,7 +1358,7 @@ TEST_F(NetIbMPITest, FailoverAllDevicesFailed) {
void* listenComm = nullptr;
void* sendComm = nullptr;
void* recvComm = nullptr;
SetupCastConnection(/*dev=*/mergedDev, &listenComm, &sendComm, &recvComm);
ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/mergedDev, &listenComm, &sendComm, &recvComm));

constexpr size_t kMsgSize = 4096;
std::vector<char> sendBuf(kMsgSize), recvBuf(kMsgSize);
Expand Down Expand Up @@ -1520,7 +1520,7 @@ TEST_F(NetIbMPITest, FailoverLargeMessageDataIntegrity) {
void* listenComm = nullptr;
void* sendComm = nullptr;
void* recvComm = nullptr;
SetupCastConnection(/*dev=*/mergedDev, &listenComm, &sendComm, &recvComm);
ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/mergedDev, &listenComm, &sendComm, &recvComm));

// Large message to ensure data spans multiple QPs
constexpr size_t kMsgSize = 65536;
Expand Down Expand Up @@ -1661,7 +1661,7 @@ TEST_F(NetIbMPITest, FailoverDeviceOneFailure) {
void* listenComm = nullptr;
void* sendComm = nullptr;
void* recvComm = nullptr;
SetupCastConnection(/*dev=*/mergedDev, &listenComm, &sendComm, &recvComm);
ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/mergedDev, &listenComm, &sendComm, &recvComm));

constexpr size_t kMsgSize = 8192;
std::vector<char> sendBuf(kMsgSize), recvBuf(kMsgSize);
Expand Down Expand Up @@ -1841,7 +1841,7 @@ TEST_F(NetIbMPITest, FailoverMultiRequestInFlight) {
void* listenComm = nullptr;
void* sendComm = nullptr;
void* recvComm = nullptr;
SetupCastConnection(/*dev=*/mergedDev, &listenComm, &sendComm, &recvComm);
ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/mergedDev, &listenComm, &sendComm, &recvComm));

constexpr int kNumReqs = 4;
constexpr size_t kMsgSize = 4096;
Expand Down Expand Up @@ -1993,7 +1993,7 @@ TEST_F(NetIbMPITest, RecoveryThreadStartedOnlyWithParam) {
void* listenComm = nullptr;
void* sendComm = nullptr;
void* recvComm = nullptr;
SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm);
ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm));

struct RecoveryEnabledResult {
int recoveryEnabled;
Expand Down Expand Up @@ -2202,7 +2202,7 @@ TEST_F(NetIbMPITest, RecoverySuccessRestoresTraffic) {
void* listenComm = nullptr;
void* sendComm = nullptr;
void* recvComm = nullptr;
SetupCastConnection(/*dev=*/mergedDev, &listenComm, &sendComm, &recvComm);
ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/mergedDev, &listenComm, &sendComm, &recvComm));

constexpr size_t kMsgSize = 8192;
constexpr int kPostRecoveryMsgs = 20;
Expand Down Expand Up @@ -2579,7 +2579,7 @@ TEST_F(NetIbMPITest, RecoveryPendingWhileLinkDown) {
void* listenComm = nullptr;
void* sendComm = nullptr;
void* recvComm = nullptr;
SetupCastConnection(/*dev=*/mergedDev, &listenComm, &sendComm, &recvComm);
ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/mergedDev, &listenComm, &sendComm, &recvComm));

constexpr size_t kMsgSize = 8192;
std::vector<char> sendBuf(kMsgSize), recvBuf(kMsgSize);
Expand Down Expand Up @@ -2744,7 +2744,7 @@ TEST_F(NetIbMPITest, RecoveryDeviceOneFailure) {
void* listenComm = nullptr;
void* sendComm = nullptr;
void* recvComm = nullptr;
SetupCastConnection(/*dev=*/mergedDev, &listenComm, &sendComm, &recvComm);
ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/mergedDev, &listenComm, &sendComm, &recvComm));

constexpr size_t kMsgSize = 8192;
const size_t kBufSize = kMsgSize * 2;
Expand Down Expand Up @@ -2991,7 +2991,7 @@ TEST_F(NetIbMPITest, RecoveryUdTimeoutExhaustsAttempts) {
void* listenComm = nullptr;
void* sendComm = nullptr;
void* recvComm = nullptr;
SetupCastConnection(/*dev=*/mergedDev, &listenComm, &sendComm, &recvComm);
ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/mergedDev, &listenComm, &sendComm, &recvComm));

constexpr size_t kMsgSize = 8192;
std::vector<char> sendBuf(kMsgSize), recvBuf(kMsgSize);
Expand Down Expand Up @@ -3120,7 +3120,7 @@ TEST_F(NetIbMPITest, FaultInjCastOpsPostSendErrno) {
void* listenComm = nullptr;
void* sendComm = nullptr;
void* recvComm = nullptr;
SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm);
ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm));

constexpr size_t kMsgSize = 1024;
std::vector<char> sendBuf(kMsgSize), recvBuf(kMsgSize);
Expand Down Expand Up @@ -3251,7 +3251,7 @@ TEST_F(NetIbMPITest, FaultInjCastOpsPollCqFlushNonFatal) {
void* listenComm = nullptr;
void* sendComm = nullptr;
void* recvComm = nullptr;
SetupCastConnection(/*dev=*/mergedDev, &listenComm, &sendComm, &recvComm);
ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/mergedDev, &listenComm, &sendComm, &recvComm));

constexpr size_t kMsgSize = 8192;
std::vector<char> sendBuf(kMsgSize), recvBuf(kMsgSize);
Expand Down Expand Up @@ -3382,7 +3382,7 @@ TEST_F(NetIbMPITest, FaultInjCastOpsPollCqSynthFatal) {
void* listenComm = nullptr;
void* sendComm = nullptr;
void* recvComm = nullptr;
SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm);
ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm));

constexpr size_t kMsgSize = 1024;
std::vector<char> sendBuf(kMsgSize), recvBuf(kMsgSize);
Expand Down Expand Up @@ -3499,7 +3499,7 @@ TEST_F(NetIbMPITest, FaultInjCastOpsPostRecvErrno) {
void* listenComm = nullptr;
void* sendComm = nullptr;
void* recvComm = nullptr;
SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm);
ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm));

constexpr size_t kMsgSize = 1024;
std::vector<char> sendBuf(kMsgSize), recvBuf(kMsgSize);
Expand Down Expand Up @@ -3626,7 +3626,7 @@ TEST_F(NetIbMPITest, FaultInjCastOpsPollCqInjectCountFinite) {
void* listenComm = nullptr;
void* sendComm = nullptr;
void* recvComm = nullptr;
SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm);
ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm));

constexpr size_t kMsgSize = 1024;
std::vector<char> sendBuf(kMsgSize), recvBuf(kMsgSize);
Expand Down Expand Up @@ -3803,7 +3803,7 @@ TEST_F(NetIbMPITest, FaultInjCastOpsApiInvalidArgs) {
void* listenComm = nullptr;
void* sendComm = nullptr;
void* recvComm = nullptr;
SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm);
ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm));

constexpr size_t kMsgSize = 1024;
std::vector<char> buf(kMsgSize, 0);
Expand Down Expand Up @@ -3878,7 +3878,7 @@ TEST_F(NetIbMPITest, FaultInjectionShimsAbsentUnlessRequested) {
void* listenComm = nullptr;
void* sendComm = nullptr;
void* recvComm = nullptr;
SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm);
ASSERT_NO_FATAL_FAILURE(SetupCastConnection(/*dev=*/0, &listenComm, &sendComm, &recvComm));

constexpr size_t kMsgSize = 1024;
std::vector<char> buf(kMsgSize, 0);
Expand Down
14 changes: 9 additions & 5 deletions projects/rccl/test/transport/NetIbMPI/GdrFlushTests.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ class GdrFlushTest : public NetIbMPITest {

ConnectionPair pair;
NetConnectionGuard connGuard(net_);
SetupConnectionWithGuard(0, pair, connGuard);
ASSERT_NO_FATAL_FAILURE(SetupConnectionWithGuard(0, pair, connGuard));
Comment thread
i-kosarev marked this conversation as resolved.
Resolved

const size_t bufferSize = kSmallBufferSize;
void* buffer = nullptr;
Expand Down Expand Up @@ -166,7 +166,8 @@ TEST_F(GdrFlushTest, CuMemDmaBuf_GpuRecvFlush_NoAsyncFatal) {
if (!gdrPtrSupport()) GTEST_SKIP() << "no GDR backend (neither peermem nor dma-buf) on this device";

ncclResult_t flush = ncclSuccess;
RunRecvFlushBurst(/*iterations=*/4, /*verifyData=*/true, &flush);
ASSERT_NO_FATAL_FAILURE(RunRecvFlushBurst(/*iterations=*/4, /*verifyData=*/true,
&flush));
if (MPIEnvironment::world_rank == 0)
EXPECT_EQ(flush, ncclSuccess) << "write+read flush over dma-buf scratchpad must not fault";
}
Expand All @@ -182,7 +183,8 @@ TEST_F(GdrFlushTest, Peermem_GpuRecvFlush_NoAsyncFatal) {
GTEST_SKIP() << "peermem (NCCL_PTR_CUDA) not available for the reg_mr scratchpad";

ncclResult_t flush = ncclSuccess;
RunRecvFlushBurst(/*iterations=*/4, /*verifyData=*/true, &flush);
ASSERT_NO_FATAL_FAILURE(RunRecvFlushBurst(/*iterations=*/4, /*verifyData=*/true,
&flush));
if (MPIEnvironment::world_rank == 0)
EXPECT_EQ(flush, ncclSuccess) << "peermem RO=0 scratchpad flush must succeed";
}
Expand All @@ -198,7 +200,8 @@ TEST_F(GdrFlushTest, FeatureDisabled_FallbackReadRecvBuffer) {
if (!gdrPtrSupport()) GTEST_SKIP() << "no GDR backend (neither peermem nor dma-buf) on this device";

ncclResult_t flush = ncclSuccess;
RunRecvFlushBurst(/*iterations=*/4, /*verifyData=*/true, &flush);
ASSERT_NO_FATAL_FAILURE(RunRecvFlushBurst(/*iterations=*/4, /*verifyData=*/true,
&flush));
if (MPIEnvironment::world_rank == 0)
EXPECT_EQ(flush, ncclSuccess) << "fallback flush (read recv buffer) must succeed";
}
Expand All @@ -212,7 +215,8 @@ TEST_F(GdrFlushTest, RepeatedFlush_NoFaultBurst) {
if (!gdrPtrSupport()) GTEST_SKIP() << "no GDR backend (neither peermem nor dma-buf) on this device";

ncclResult_t flush = ncclSuccess;
RunRecvFlushBurst(/*iterations=*/50, /*verifyData=*/false, &flush);
ASSERT_NO_FATAL_FAILURE(RunRecvFlushBurst(/*iterations=*/50, /*verifyData=*/false,
&flush));
if (MPIEnvironment::world_rank == 0)
EXPECT_EQ(flush, ncclSuccess) << "no flush in the burst may raise a QP async-fatal";
}
Expand Down
Loading
Loading