Skip to content
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 31 additions & 0 deletions projects/rccl/test/host/dev-runtime-test.cc
Original file line number Diff line number Diff line change
Expand Up @@ -2420,6 +2420,37 @@ TEST_F(SymMemoryObtainRollbackTest, RmaRegisterFails_ReturnsSpaceWithoutLinking)
EXPECT_EQ(comm->devrState.memHead, nullptr);
}

// AICOMRCCL-2428: GIN register succeeds, then RMA register fails. That is the
// only way to reach fail_mem's free(mem->ginSegmentInfos) with a live array --
// GinRegisterFails never allocates one that survives to fail_mem, and
// RmaRegisterFails leaves ginEnabled false. deregister is the observable
// witness that symMemoryUnregister saw ginSegmentInfos; free runs on the same
// path before Obtain returns (mem is gone by then so the pointer cannot be
// checked). rmaHostWins[0] stays null, so the RMA deregister arm is skipped.
Comment thread
dlamd1dai marked this conversation as resolved.
Resolved
TEST_F(SymMemoryObtainRollbackTest, GinSucceedsThenRmaFails_UnregistersGinAndUnlinks) {
PushTeam();
comm->devrState.ginEnabled = true;
RmaProxyTerms(comm, true);
ScopedHook gather(g_devrBootstrapAllGather, agreeing);
ScopedHook alloc(g_devrSpaceAlloc, AllocAt(0));
ScopedHook spaceFree(g_devrSpaceFree, [](ncclSpace*, int64_t, int64_t) { return ncclSuccess; });
ScopedHook ginReg(g_devrGinRegister,
[](ncclComm*, void*, size_t, void*[], ncclGinWindow_t[], int, bool, int) {
return ncclSuccess;
});
ScopedHook ginDereg(g_devrGinDeregister, [](ncclComm*, void*[]) { return ncclSuccess; });
ScopedHook rmaReg(g_devrRmaProxyRegister, [](ncclComm*, void*, size_t, void*[]) { return ncclSystemError; });
ScopedHook rmaDereg(g_devrRmaProxyDeregister, [](ncclComm*, void*[]) { return ncclSuccess; });

EXPECT_NE(Obtain(), ncclSuccess);
EXPECT_EQ(ginReg.calls, 1);
EXPECT_EQ(rmaReg.calls, 1);
EXPECT_EQ(ginDereg.calls, 1); // symMemoryUnregister GIN arm
EXPECT_EQ(rmaDereg.calls, 0); // rmaHostWins[0] witness never set
EXPECT_EQ(spaceFree.calls, 1);
EXPECT_EQ(comm->devrState.memHead, nullptr);
}

// Label fail_mem: a failure before the reservation must not free space that was
// never taken.
TEST_F(SymMemoryObtainRollbackTest, EarlyFailure_DoesNotFreeUnreservedSpace) {
Expand Down