From 8388c4aecb5281d840fd61a7e89c5af1c8986fea Mon Sep 17 00:00:00 2001 From: Donglai Dai Date: Tue, 22 Sep 2026 19:32:11 -0400 Subject: [PATCH 1/2] test(rccl): cover GIN-success then RMA-fail obtain rollback (AICOMRCCL-2428) Exercise fail_mem_space_teams after a successful GIN register so symMemoryUnregister's GIN arm and free(ginSegmentInfos) run, then assert memHead is unlinked. Kept out of #11863 so AICOMRCCL-2241 can land first. --- projects/rccl/test/host/dev-runtime-test.cc | 31 +++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/projects/rccl/test/host/dev-runtime-test.cc b/projects/rccl/test/host/dev-runtime-test.cc index a90b5592ea56..0241973e9f4a 100644 --- a/projects/rccl/test/host/dev-runtime-test.cc +++ b/projects/rccl/test/host/dev-runtime-test.cc @@ -2420,6 +2420,37 @@ TEST_F(SymMemoryObtainRollbackTest, RmaRegisterFails_ReturnsSpaceWithoutLinking) EXPECT_EQ(comm->devrState.memHead, nullptr); } +// AICOMRCCL-2428: GIN register succeeds, then RMA register fails. That is the +// only way to reach fail_mem's free(mem->ginSegmentInfos) with a live array -- +// GinRegisterFails never allocates one that survives to fail_mem, and +// RmaRegisterFails leaves ginEnabled false. deregister is the observable +// witness that symMemoryUnregister saw ginSegmentInfos; free runs on the same +// path before Obtain returns (mem is gone by then so the pointer cannot be +// checked). rmaHostWins[0] stays null, so the RMA deregister arm is skipped. +TEST_F(SymMemoryObtainRollbackTest, GinSucceedsThenRmaFails_UnregistersGinAndUnlinks) { + PushTeam(); + comm->devrState.ginEnabled = true; + RmaProxyTerms(comm, true); + ScopedHook gather(g_devrBootstrapAllGather, agreeing); + ScopedHook alloc(g_devrSpaceAlloc, AllocAt(0)); + ScopedHook spaceFree(g_devrSpaceFree, [](ncclSpace*, int64_t, int64_t) { return ncclSuccess; }); + ScopedHook ginReg(g_devrGinRegister, + [](ncclComm*, void*, size_t, void*[], ncclGinWindow_t[], int, bool, int) { + return ncclSuccess; + }); + ScopedHook ginDereg(g_devrGinDeregister, [](ncclComm*, void*[]) { return ncclSuccess; }); + ScopedHook rmaReg(g_devrRmaProxyRegister, [](ncclComm*, void*, size_t, void*[]) { return ncclSystemError; }); + ScopedHook rmaDereg(g_devrRmaProxyDeregister, [](ncclComm*, void*[]) { return ncclSuccess; }); + + EXPECT_NE(Obtain(), ncclSuccess); + EXPECT_EQ(ginReg.calls, 1); + EXPECT_EQ(rmaReg.calls, 1); + EXPECT_EQ(ginDereg.calls, 1); // symMemoryUnregister GIN arm + EXPECT_EQ(rmaDereg.calls, 0); // rmaHostWins[0] witness never set + EXPECT_EQ(spaceFree.calls, 1); + EXPECT_EQ(comm->devrState.memHead, nullptr); +} + // Label fail_mem: a failure before the reservation must not free space that was // never taken. TEST_F(SymMemoryObtainRollbackTest, EarlyFailure_DoesNotFreeUnreservedSpace) { From 1b9de8dcab464e4ffa4695481a7aaa2a7b75aff5 Mon Sep 17 00:00:00 2001 From: Donglai Dai Date: Sat, 3 Oct 2026 02:50:30 -0400 Subject: [PATCH 2/2] test(rccl): apply Argus wording on the GIN-then-RMA rollback comment Name the fail_mem path and ginDereg witness the way the review asked. Co-authored-by: Cursor --- projects/rccl/test/host/dev-runtime-test.cc | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/projects/rccl/test/host/dev-runtime-test.cc b/projects/rccl/test/host/dev-runtime-test.cc index 0241973e9f4a..d8bd98f78d9b 100644 --- a/projects/rccl/test/host/dev-runtime-test.cc +++ b/projects/rccl/test/host/dev-runtime-test.cc @@ -2420,13 +2420,9 @@ TEST_F(SymMemoryObtainRollbackTest, RmaRegisterFails_ReturnsSpaceWithoutLinking) EXPECT_EQ(comm->devrState.memHead, nullptr); } -// AICOMRCCL-2428: GIN register succeeds, then RMA register fails. That is the -// only way to reach fail_mem's free(mem->ginSegmentInfos) with a live array -- -// GinRegisterFails never allocates one that survives to fail_mem, and -// RmaRegisterFails leaves ginEnabled false. deregister is the observable -// witness that symMemoryUnregister saw ginSegmentInfos; free runs on the same -// path before Obtain returns (mem is gone by then so the pointer cannot be -// checked). rmaHostWins[0] stays null, so the RMA deregister arm is skipped. +// Label fail_mem: AICOMRCCL-2428. GIN succeeding then RMA failing is the only way to reach +// free(mem->ginSegmentInfos) with a live array; the sibling cases never build one that survives. +// ginDereg is the witness that symMemoryUnregister saw it, since mem is gone before Obtain returns. TEST_F(SymMemoryObtainRollbackTest, GinSucceedsThenRmaFails_UnregistersGinAndUnlinks) { PushTeam(); comm->devrState.ginEnabled = true;