From f81f40749bb7d65558d2bc2860f0b0a653a36f68 Mon Sep 17 00:00:00 2001 From: Alex Duforest Date: Mon, 12 Oct 2026 14:54:19 -0700 Subject: [PATCH 01/26] test(rccl): add host microtests for nccl_device/utility.h math helpers --- projects/rccl/test/host/CMakeLists.txt | 1 + projects/rccl/test/host/MICROTEST_README.md | 5 + projects/rccl/test/host/nccl-utility-test.cc | 379 +++++++++++++++++++ 3 files changed, 385 insertions(+) create mode 100644 projects/rccl/test/host/nccl-utility-test.cc diff --git a/projects/rccl/test/host/CMakeLists.txt b/projects/rccl/test/host/CMakeLists.txt index 26a486a70cd9..e1d32bb1ec2a 100644 --- a/projects/rccl/test/host/CMakeLists.txt +++ b/projects/rccl/test/host/CMakeLists.txt @@ -97,6 +97,7 @@ function(rccl_define_micro_source_lists) devcomm-test.cc wrap-test.cc dev-runtime-test.cc + nccl-utility-test.cc # src/include/nccl_device/utility.h (header-only, no fakes) # sym-kernels-index-test.cc: added conditionally below, only when GENERATE_SYM_KERNELS is ON. gdr-probe-test.cc # src/misc/gdr_probe.cc runtime GDR registration probe process-isolated-runner-test.cc diff --git a/projects/rccl/test/host/MICROTEST_README.md b/projects/rccl/test/host/MICROTEST_README.md index a1df556582cc..087e9cdfbfbf 100644 --- a/projects/rccl/test/host/MICROTEST_README.md +++ b/projects/rccl/test/host/MICROTEST_README.md @@ -80,6 +80,11 @@ export colliding non-`static` symbols; otherwise a unit needs its own binary: - `dev_runtime.cc` (`DEV_RUNTIME_CC_PATH`, from `dev-runtime-test.cc`); suites `Alloc*`, `Comm*`, `Compute*`, `DeepCopy*`, `Dev*`, `Gin*`, `Nccl*`, `Sym*`, and `Win*`. + - `src/include/nccl_device/utility.h` (header-only, from + `nccl-utility-test.cc`); suite `NcclUtilityMicrotest.*` (also caught by the + `Nccl*` pattern above). No `_CC_PATH` macro needed since it's a header, + not a `.cc` production TU, and no fakes since every targeted function is + `NCCL_HOST_DEVICE_INLINE` pure host/device math with no HIP dependency. - the generated `sym_kernels_host.cc` (`SYM_KERNELS_HOST_CC_PATH`, from `sym-kernels-index-test.cc`); suites `SymKernelIndex*`, `SymAllEmittedCombinations/*`, `SymUnhandledCombinations/*` (covered by the diff --git a/projects/rccl/test/host/nccl-utility-test.cc b/projects/rccl/test/host/nccl-utility-test.cc new file mode 100644 index 000000000000..4c494456503a --- /dev/null +++ b/projects/rccl/test/host/nccl-utility-test.cc @@ -0,0 +1,379 @@ +/************************************************************************* + * Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +// Host-only microtests for the pure host/device math helpers in +// src/include/nccl_device/utility.h: rounding/alignment arithmetic, the +// fast-division reciprocal family, the memory_order conversion helpers, and +// the Optional/Present variadic construction helpers. All targeted +// functions are NCCL_HOST_DEVICE_INLINE with no HIP/device-only dependency, +// so the real header compiles and runs unmodified in a host-only TU -- no +// fakes needed. + +#include + +#include +#include +#include + +#include "nccl_device/utility.h" + +using nccl::utility::acquireOrderOf; +using nccl::utility::add4G; +using nccl::utility::alignDown; +using nccl::utility::alignUp; +using nccl::utility::divUp; +using nccl::utility::idivFast32; +using nccl::utility::idivFast64; +using nccl::utility::idivmodFast32; +using nccl::utility::idivmodFast64; +using nccl::utility::idivRcp32; +using nccl::utility::idivRcp64; +using nccl::utility::imodFast32; +using nccl::utility::imodFast64; +using nccl::utility::imulRcp32; +using nccl::utility::imulRcp64; +using nccl::utility::isPow2; +using nccl::utility::mul32hi; +using nccl::utility::mul64hi; +using nccl::utility::Optional; +using nccl::utility::present; +using nccl::utility::Present; +using nccl::utility::releaseOrderOf; +using nccl::utility::roundDown; +using nccl::utility::roundUp; +using nccl::utility::rollingLessEq; +using nccl::utility::rollingLessThan; +using nccl::utility::toAtomicBuiltinOrder; +using nccl::utility::toCudaOrder; + +namespace { + +// Brute-force reference: idivFast32/64 claim idivFast(x, y, idivRcp(y)) == x/y +// for every valid (x, y); these helpers recompute that division the ordinary +// way so the fast path can't share a bug with its own golden. +uint32_t refDiv32(uint32_t x, uint32_t y) { return x / y; } +uint64_t refDiv64(uint64_t x, uint64_t y) { return x / y; } + +} // namespace + +TEST(NcclUtilityMicrotest, DivUp_RoundsAwayFromZeroOnlyWhenRemainderIsNonzero) { + EXPECT_EQ(divUp(0u, 4u), 0u); + EXPECT_EQ(divUp(1u, 4u), 1u); + EXPECT_EQ(divUp(4u, 4u), 1u); + EXPECT_EQ(divUp(5u, 4u), 2u); + EXPECT_EQ(divUp(7u, 3u), 3u); +} + +TEST(NcclUtilityMicrotest, RoundUpDown_AreInverseDirectionsAroundAMultiple) { + EXPECT_EQ(roundUp(0u, 8u), 0u); + EXPECT_EQ(roundUp(1u, 8u), 8u); + EXPECT_EQ(roundUp(8u, 8u), 8u); + EXPECT_EQ(roundUp(9u, 8u), 16u); + + EXPECT_EQ(roundDown(0u, 8u), 0u); + EXPECT_EQ(roundDown(7u, 8u), 0u); + EXPECT_EQ(roundDown(8u, 8u), 8u); + EXPECT_EQ(roundDown(15u, 8u), 8u); +} + +TEST(NcclUtilityMicrotest, AlignUpDown_ScalarOverload_AssumesPowerOfTwoAlignment) { + EXPECT_EQ(alignUp(0u, 16u), 0u); + EXPECT_EQ(alignUp(1u, 16u), 16u); + EXPECT_EQ(alignUp(16u, 16u), 16u); + EXPECT_EQ(alignUp(17u, 16u), 32u); + + EXPECT_EQ(alignDown(0u, 16u), 0u); + EXPECT_EQ(alignDown(15u, 16u), 0u); + EXPECT_EQ(alignDown(16u, 16u), 16u); + EXPECT_EQ(alignDown(31u, 16u), 16u); +} + +TEST(NcclUtilityMicrotest, AlignUpDown_PointerOverload_RoundsTheAddressNotTheByte) { + alignas(32) char buf[64]; + // Single-byte-type overload: static_assert(sizeof(T) == 1) gates this at + // compile time, so the test itself is part of the contract being checked. + char* base = buf + 3; + char* up = alignUp(base, size_t(16)); + char* down = alignDown(base, size_t(16)); + EXPECT_EQ(reinterpret_cast(up) % 16, 0u); + EXPECT_EQ(reinterpret_cast(down) % 16, 0u); + EXPECT_GE(up, base); + EXPECT_LE(down, base); + EXPECT_LT(reinterpret_cast(up) - reinterpret_cast(down), 32u); +} + +TEST(NcclUtilityMicrotest, IsPow2_TrueOnlyForPowersOfTwoAndZero) { + EXPECT_TRUE(isPow2(0u)); + EXPECT_TRUE(isPow2(1u)); + EXPECT_TRUE(isPow2(2u)); + EXPECT_TRUE(isPow2(1024u)); + EXPECT_FALSE(isPow2(3u)); + EXPECT_FALSE(isPow2(6u)); + EXPECT_FALSE(isPow2(1023u)); +} + +TEST(NcclUtilityMicrotest, RollingLessEqAndLessThan_WrapAroundHalfTheRangeCorrectly) { + // 8-bit rolling counter: 250 is "less than" 4 (wrapped), but not vice versa. + EXPECT_TRUE(rollingLessEq(uint8_t(250), uint8_t(4), 8)); + EXPECT_FALSE(rollingLessEq(uint8_t(4), uint8_t(250), 8)); + EXPECT_TRUE(rollingLessThan(uint8_t(250), uint8_t(4), 8)); + EXPECT_FALSE(rollingLessThan(uint8_t(4), uint8_t(250), 8)); + + // Equal values: LessEq true, LessThan false (strict). + EXPECT_TRUE(rollingLessEq(uint8_t(7), uint8_t(7), 8)); + EXPECT_FALSE(rollingLessThan(uint8_t(7), uint8_t(7), 8)); + + // rollingLessThan is defined as !rollingLessEq(b, a); verify that identity + // directly so a copy-paste swap of a/b in either function is caught. + for (uint8_t a : {uint8_t(0), uint8_t(1), uint8_t(127), uint8_t(128), uint8_t(200)}) { + for (uint8_t b : {uint8_t(0), uint8_t(1), uint8_t(127), uint8_t(128), uint8_t(200)}) { + EXPECT_EQ(rollingLessThan(a, b, 8), !rollingLessEq(b, a, 8)) << "a=" << int(a) << " b=" << int(b); + } + } +} + +TEST(NcclUtilityMicrotest, IdivRcp32_MatchesExactFormulaIncludingPow2Correction) { + // Formula: uint32_t(-1)/x + isPow2(x). The +1 correction for powers of two + // is what makes idivFast32 exact when y is a power of two; drop it and the + // round-trip test below would start failing for y in {1,2,4,...}. + EXPECT_EQ(idivRcp32(1u), uint32_t(-1) / 1u + 1u); + EXPECT_EQ(idivRcp32(2u), uint32_t(-1) / 2u + 1u); + EXPECT_EQ(idivRcp32(3u), uint32_t(-1) / 3u + 0u); + EXPECT_EQ(idivRcp32(100u), uint32_t(-1) / 100u + 0u); +} + +TEST(NcclUtilityMicrotest, IdivRcp64_MatchesExactFormulaIncludingPow2Correction) { + EXPECT_EQ(idivRcp64(1ull), uint64_t(-1) / 1ull + 1ull); + EXPECT_EQ(idivRcp64(2ull), uint64_t(-1) / 2ull + 1ull); + EXPECT_EQ(idivRcp64(3ull), uint64_t(-1) / 3ull + 0ull); + EXPECT_EQ(idivRcp64(1ull << 40), uint64_t(-1) / (1ull << 40) + 1ull); +} + +TEST(NcclUtilityMicrotest, Mul32Hi_ReturnsTheHighHalfOfTheFull64BitProduct) { + EXPECT_EQ(mul32hi(0u, 0xFFFFFFFFu), 0u); + EXPECT_EQ(mul32hi(0xFFFFFFFFu, 0xFFFFFFFFu), uint32_t((uint64_t(0xFFFFFFFFu) * 0xFFFFFFFFu) >> 32)); + EXPECT_EQ(mul32hi(0x10000u, 0x10000u), 1u); // 2^16 * 2^16 == 2^32, high half == 1 +} + +TEST(NcclUtilityMicrotest, Mul64Hi_ReturnsTheHighHalfOfTheFull128BitProduct) { + EXPECT_EQ(mul64hi(0ull, ~0ull), 0ull); + uint64_t big = 0x1'0000'0000ull; // 2^32 + EXPECT_EQ(mul64hi(big, big), 1ull); // 2^32 * 2^32 == 2^64, high half == 1 +} + +TEST(NcclUtilityMicrotest, ImulRcp32_HandlesZeroReciprocalShortCircuitsAndMatchesIdivRcpOfProduct) { + // xrcp == 0 or yrcp == 0 short-circuits to the other operand's reciprocal + // directly -- callers use this to mean "treat this factor as already 1". + EXPECT_EQ(imulRcp32(5u, 0u, 7u, idivRcp32(7u)), idivRcp32(7u)); + EXPECT_EQ(imulRcp32(5u, idivRcp32(5u), 7u, 0u), idivRcp32(5u)); + + // x=3, y=5 is chosen (rather than some arbitrary pair) because it's one of + // the smallest inputs that actually drives the `if (x*y <= rem) rcp += 1;` + // rounding correction true -- an arbitrary pair like (6,7) can land on the + // false side and leave that correction untested. + uint32_t x = 3, y = 5; + uint32_t got = imulRcp32(x, idivRcp32(x), y, idivRcp32(y)); + EXPECT_EQ(got, idivRcp32(x * y)); + + // x=2, y=2 exercises the other side of that same branch (no correction + // needed), so both directions of the rounding check are verified. + EXPECT_EQ(imulRcp32(2u, idivRcp32(2u), 2u, idivRcp32(2u)), idivRcp32(4u)); +} + +TEST(NcclUtilityMicrotest, ImulRcp64_HandlesZeroReciprocalShortCircuitsAndMatchesIdivRcpOfProduct) { + EXPECT_EQ(imulRcp64(5ull, 0ull, 7ull, idivRcp64(7ull)), idivRcp64(7ull)); + EXPECT_EQ(imulRcp64(5ull, idivRcp64(5ull), 7ull, 0ull), idivRcp64(5ull)); + + // Same rationale as the 32-bit test above: (3,5) drives the rounding + // correction branch true, which a value like (6,7) does not for the 64-bit + // reciprocal (the correction's trigger condition depends on word width). + uint64_t x = 3, y = 5; + uint64_t got = imulRcp64(x, idivRcp64(x), y, idivRcp64(y)); + EXPECT_EQ(got, idivRcp64(x * y)); + + // (2,2) exercises the other side of that same branch (no correction needed). + EXPECT_EQ(imulRcp64(2ull, idivRcp64(2ull), 2ull, idivRcp64(2ull)), idivRcp64(4ull)); +} + +TEST(NcclUtilityMicrotest, IdivmodFast32_MatchesOrdinaryDivisionAcrossDivisorsAndYrcpZeroShortcut) { + for (uint32_t y : {1u, 2u, 3u, 7u, 16u, 100u, 4096u}) { + uint32_t yrcp = idivRcp32(y); + for (uint32_t x : {0u, 1u, 5u, 41u, 4095u, 4096u, 1000000u}) { + uint32_t q, r; + idivmodFast32(&q, &r, x, y, yrcp); + EXPECT_EQ(q, x / y) << "x=" << x << " y=" << y; + EXPECT_EQ(r, x % y) << "x=" << x << " y=" << y; + } + } + // yrcp == 0 shortcut: q = x directly (caller's contract for "divisor is 1"). + uint32_t q, r; + idivmodFast32(&q, &r, 42u, 1u, 0u); + EXPECT_EQ(q, 42u); + EXPECT_EQ(r, 0u); +} + +TEST(NcclUtilityMicrotest, IdivmodFast64_MatchesOrdinaryDivisionAcrossDivisorsAndYrcpZeroShortcut) { + for (uint64_t y : {1ull, 2ull, 3ull, 7ull, 16ull, 100ull, 1ull << 20}) { + uint64_t yrcp = idivRcp64(y); + for (uint64_t x : {0ull, 1ull, 5ull, 41ull, (1ull << 20) - 1, 1ull << 20, 1ull << 40}) { + uint64_t q, r; + idivmodFast64(&q, &r, x, y, yrcp); + EXPECT_EQ(q, x / y) << "x=" << x << " y=" << y; + EXPECT_EQ(r, x % y) << "x=" << x << " y=" << y; + } + } + uint64_t q, r; + idivmodFast64(&q, &r, 42ull, 1ull, 0ull); + EXPECT_EQ(q, 42ull); + EXPECT_EQ(r, 0ull); +} + +TEST(NcclUtilityMicrotest, IdivFast32_MatchesRefDivisionAndIdivFast64TruncatesToLow32Bits) { + for (uint32_t y : {1u, 3u, 16u, 4096u}) { + uint32_t yrcp = idivRcp32(y); + for (uint32_t x : {0u, 41u, 4095u, 1000000u}) { + EXPECT_EQ(idivFast32(x, y, yrcp), refDiv32(x, y)); + } + } + // idivFast64 returns a uint32_t truncation of a 64-bit quotient; pick a + // quotient that fits in 32 bits so truncation is a correctness check, not + // a free pass. + uint64_t y = 1000, yrcp64 = idivRcp64(y); + uint64_t x = 123456789ull; + EXPECT_EQ(idivFast64(x, y, yrcp64), uint32_t(refDiv64(x, y))); +} + +TEST(NcclUtilityMicrotest, ImodFast32And64_MatchRefModulo) { + for (uint32_t y : {1u, 3u, 16u, 4096u}) { + uint32_t yrcp = idivRcp32(y); + for (uint32_t x : {0u, 41u, 4095u, 1000000u}) { + EXPECT_EQ(imodFast32(x, y, yrcp), x % y); + } + } + for (uint64_t y : {1ull, 3ull, 16ull, 1ull << 20}) { + uint64_t yrcp = idivRcp64(y); + for (uint64_t x : {0ull, 41ull, (1ull << 20) - 1, 1ull << 40}) { + EXPECT_EQ(imodFast64(x, y, yrcp), x % y); + } + } +} + +TEST(NcclUtilityMicrotest, Optional_DefaultAndAbsentConstructLeavePresentFalse) { + Optional a; + EXPECT_FALSE(a.present); + Optional b{nccl::utility::Absent{}}; + EXPECT_FALSE(b.present); +} + +TEST(NcclUtilityMicrotest, Optional_PresentConstructorForwardsArgsAndSetsPresentTrue) { + // present(1, 2) packs two ints into a Present; Optional's + // Present-taking constructor must forward both through to T's constructor + // in the right order, not just construct a default T. + struct Pair { + int a, b; + NCCL_HOST_DEVICE_INLINE Pair(int a_, int b_) : a(a_), b(b_) {} + }; + Optional opt(present(3, 4)); + ASSERT_TRUE(opt.present); + EXPECT_EQ(opt.thing.a, 3); + EXPECT_EQ(opt.thing.b, 4); +} + +TEST(NcclUtilityMicrotest, Optional_DestructorOnlyRunsWhenPresentIsTrue) { + static int destructorCalls = 0; + struct Counted { + NCCL_HOST_DEVICE_INLINE Counted() {} + NCCL_HOST_DEVICE_INLINE ~Counted() { destructorCalls++; } + }; + + destructorCalls = 0; + { + Optional absent; // present == false: ~Counted() must NOT run. + } + EXPECT_EQ(destructorCalls, 0); + + destructorCalls = 0; + { + Optional present_opt(present()); + } + EXPECT_EQ(destructorCalls, 1); +} + +TEST(NcclUtilityMicrotest, Add4G_AddsDeltaIntoTheHighHalfOfA64BitValueViaUnion) { + // add4G reinterprets a T (must be 8 bytes) as two uint32_t halves and + // adds delta4G into the high half (index 1 is the high word on this + // little-endian build). Verify against plain 64-bit arithmetic. + uint64_t base = 0x0000'0000'1234'5678ull; + EXPECT_EQ(add4G(base, 1), base + (uint64_t(1) << 32)); + EXPECT_EQ(add4G(base, 5), base + (uint64_t(5) << 32)); + // Negative delta: subtracts a 4G chunk from the high half. + uint64_t highBase = 0x0000'0003'0000'0000ull; + EXPECT_EQ(add4G(highBase, -1), highBase - (uint64_t(1) << 32)); +} + +TEST(NcclUtilityMicrotest, AcquireOrderOfStd_DowngradesReleaseAndAcqRelPassesEverythingElseThrough) { + EXPECT_EQ(acquireOrderOf(std::memory_order_release), std::memory_order_relaxed); + EXPECT_EQ(acquireOrderOf(std::memory_order_acq_rel), std::memory_order_acquire); + EXPECT_EQ(acquireOrderOf(std::memory_order_relaxed), std::memory_order_relaxed); + EXPECT_EQ(acquireOrderOf(std::memory_order_acquire), std::memory_order_acquire); + EXPECT_EQ(acquireOrderOf(std::memory_order_seq_cst), std::memory_order_seq_cst); +} + +TEST(NcclUtilityMicrotest, ReleaseOrderOfStd_DowngradesAcquireAndAcqRelPassesEverythingElseThrough) { + EXPECT_EQ(releaseOrderOf(std::memory_order_acquire), std::memory_order_relaxed); + EXPECT_EQ(releaseOrderOf(std::memory_order_acq_rel), std::memory_order_release); + EXPECT_EQ(releaseOrderOf(std::memory_order_relaxed), std::memory_order_relaxed); + EXPECT_EQ(releaseOrderOf(std::memory_order_release), std::memory_order_release); + EXPECT_EQ(releaseOrderOf(std::memory_order_seq_cst), std::memory_order_seq_cst); +} + +TEST(NcclUtilityMicrotest, ToAtomicBuiltinOrder_MapsEveryStdOrderAndFallsBackToSeqCstForConsume) { + EXPECT_EQ(toAtomicBuiltinOrder(std::memory_order_relaxed), __ATOMIC_RELAXED); + EXPECT_EQ(toAtomicBuiltinOrder(std::memory_order_acquire), __ATOMIC_ACQUIRE); + EXPECT_EQ(toAtomicBuiltinOrder(std::memory_order_release), __ATOMIC_RELEASE); + EXPECT_EQ(toAtomicBuiltinOrder(std::memory_order_acq_rel), __ATOMIC_ACQ_REL); + EXPECT_EQ(toAtomicBuiltinOrder(std::memory_order_seq_cst), __ATOMIC_SEQ_CST); + // memory_order_consume has no case label; must fall through to default. + EXPECT_EQ(toAtomicBuiltinOrder(std::memory_order_consume), __ATOMIC_SEQ_CST); +} + +TEST(NcclUtilityMicrotest, AcquireOrderOfCuda_DowngradesReleaseAndAcqRelPassesEverythingElseThrough) { + EXPECT_EQ(acquireOrderOf(cuda::memory_order_release), cuda::memory_order_relaxed); + EXPECT_EQ(acquireOrderOf(cuda::memory_order_acq_rel), cuda::memory_order_acquire); + EXPECT_EQ(acquireOrderOf(cuda::memory_order_relaxed), cuda::memory_order_relaxed); + EXPECT_EQ(acquireOrderOf(cuda::memory_order_acquire), cuda::memory_order_acquire); + EXPECT_EQ(acquireOrderOf(cuda::memory_order_seq_cst), cuda::memory_order_seq_cst); +} + +TEST(NcclUtilityMicrotest, ReleaseOrderOfCuda_DowngradesAcquireAndAcqRelPassesEverythingElseThrough) { + EXPECT_EQ(releaseOrderOf(cuda::memory_order_acquire), cuda::memory_order_relaxed); + EXPECT_EQ(releaseOrderOf(cuda::memory_order_acq_rel), cuda::memory_order_release); + EXPECT_EQ(releaseOrderOf(cuda::memory_order_relaxed), cuda::memory_order_relaxed); + EXPECT_EQ(releaseOrderOf(cuda::memory_order_release), cuda::memory_order_release); + EXPECT_EQ(releaseOrderOf(cuda::memory_order_seq_cst), cuda::memory_order_seq_cst); +} + +TEST(NcclUtilityMicrotest, ToCudaOrder_MapsEveryStdOrderAndFallsBackToSeqCstForConsume) { + EXPECT_EQ(toCudaOrder(std::memory_order_relaxed), cuda::memory_order_relaxed); + EXPECT_EQ(toCudaOrder(std::memory_order_acquire), cuda::memory_order_acquire); + EXPECT_EQ(toCudaOrder(std::memory_order_release), cuda::memory_order_release); + EXPECT_EQ(toCudaOrder(std::memory_order_acq_rel), cuda::memory_order_acq_rel); + EXPECT_EQ(toCudaOrder(std::memory_order_seq_cst), cuda::memory_order_seq_cst); + EXPECT_EQ(toCudaOrder(std::memory_order_consume), cuda::memory_order_seq_cst); +} + +TEST(NcclUtilityMicrotest, Present_GetByIndexReturnsEachArgumentInDeclarationOrder) { + // Pass lvalues so H deduces to int& (present(H&&, ...) with an lvalue arg + // collapses to a reference to the original), keeping p's stored references + // valid for the lifetime of a,b,c below -- a prvalue-literal argument would + // deduce H=int and leave p holding a dangling int&& into a temporary that + // expires at the end of the present(...) call's full expression. + int a = 10, b = 20, c = 30; + auto p = present(a, b, c); + EXPECT_EQ(p.get(nccl::utility::IntSeq<0>{}), 10); + EXPECT_EQ(p.get(nccl::utility::IntSeq<1>{}), 20); + EXPECT_EQ(p.get(nccl::utility::IntSeq<2>{}), 30); +} From 067754716a7e7789013b5a91bd95590337552882 Mon Sep 17 00:00:00 2001 From: Alex Duforest Date: Mon, 12 Oct 2026 22:25:43 -0700 Subject: [PATCH 02/26] test(rccl): address ARGUS findings on utility.h microtests - rollingLessEq: add a partial-width (nBits=32 on a uint64_t) pair; every existing call passed nBits == full width, leaving the mask shift dead. - idivFast64: add a quotient above 2^32 (1<<40 / 1); the existing case's quotient fit in 32 bits, so the truncation it claimed to test was inert. - AlignUp/AlignDown pointer overload: pin exact addresses instead of mod/bound checks that an off-by-one in the production formula also passes. - roundUp/roundDown: add a non-power-of-two modulus pair (3), since modulus-8-only can't tell % from a power-of-two mask. - idivRcp32/64 sweeps: add 0xFFFFFFFFu / ~0ull to catch an over-estimating reciprocal, which the existing +1 under-estimate correction can't. - Remove a tautological loop in the rolling-comparator test: it reduced to the same call on both sides regardless of a swap bug, so it could not fail. - Fix two comments: the IdivRcp32 pow2-correction is a branch-avoidance optimisation, not required for round-trip correctness; and the 'no HIP/device-only dependency' header/README claim doesn't hold for the four memory_order helpers (true only because this build's --offload-host-only path still invokes hipcc). - Two self-referential assertions (mul32hi/mul64hi) replaced with literal expected values. - present_opt -> presentOpt naming nit. --- projects/rccl/test/host/MICROTEST_README.md | 7 +- projects/rccl/test/host/nccl-utility-test.cc | 74 +++++++++++++------- 2 files changed, 52 insertions(+), 29 deletions(-) diff --git a/projects/rccl/test/host/MICROTEST_README.md b/projects/rccl/test/host/MICROTEST_README.md index 087e9cdfbfbf..45c0ea2e12fc 100644 --- a/projects/rccl/test/host/MICROTEST_README.md +++ b/projects/rccl/test/host/MICROTEST_README.md @@ -83,8 +83,11 @@ export colliding non-`static` symbols; otherwise a unit needs its own binary: - `src/include/nccl_device/utility.h` (header-only, from `nccl-utility-test.cc`); suite `NcclUtilityMicrotest.*` (also caught by the `Nccl*` pattern above). No `_CC_PATH` macro needed since it's a header, - not a `.cc` production TU, and no fakes since every targeted function is - `NCCL_HOST_DEVICE_INLINE` pure host/device math with no HIP dependency. + not a `.cc` production TU, and no fakes since every targeted function + compiles cleanly in a host-only TU -- most are `NCCL_HOST_DEVICE_INLINE` + pure math with no HIP dependency; the four `memory_order` conversion + helpers sit inside `#if NCCL_CHECK_CUDACC`, which this build satisfies + because this target's `--offload-host-only` build still invokes hipcc. - the generated `sym_kernels_host.cc` (`SYM_KERNELS_HOST_CC_PATH`, from `sym-kernels-index-test.cc`); suites `SymKernelIndex*`, `SymAllEmittedCombinations/*`, `SymUnhandledCombinations/*` (covered by the diff --git a/projects/rccl/test/host/nccl-utility-test.cc b/projects/rccl/test/host/nccl-utility-test.cc index 4c494456503a..e5b233d17073 100644 --- a/projects/rccl/test/host/nccl-utility-test.cc +++ b/projects/rccl/test/host/nccl-utility-test.cc @@ -7,16 +7,18 @@ // Host-only microtests for the pure host/device math helpers in // src/include/nccl_device/utility.h: rounding/alignment arithmetic, the // fast-division reciprocal family, the memory_order conversion helpers, and -// the Optional/Present variadic construction helpers. All targeted -// functions are NCCL_HOST_DEVICE_INLINE with no HIP/device-only dependency, -// so the real header compiles and runs unmodified in a host-only TU -- no -// fakes needed. +// the Optional/Present variadic construction helpers. Most targeted +// functions are NCCL_HOST_DEVICE_INLINE with no HIP/device-only dependency; +// acquireOrderOf/releaseOrderOf/toAtomicBuiltinOrder/toCudaOrder are the +// exception -- they sit inside `#if NCCL_CHECK_CUDACC`, which this build +// satisfies because both the real device build and this host-only +// `--offload-host-only` build still invoke hipcc. The real header compiles +// and runs unmodified in a host-only TU either way -- no fakes needed. #include #include #include -#include #include "nccl_device/utility.h" @@ -77,6 +79,11 @@ TEST(NcclUtilityMicrotest, RoundUpDown_AreInverseDirectionsAroundAMultiple) { EXPECT_EQ(roundDown(7u, 8u), 0u); EXPECT_EQ(roundDown(8u, 8u), 8u); EXPECT_EQ(roundDown(15u, 8u), 8u); + + // Non-power-of-two modulus: these use % y, not a power-of-two mask. Without + // this pair, modulus 8 alone can't tell `% y` apart from `& (y - 1)`. + EXPECT_EQ(roundUp(5u, 3u), 6u); + EXPECT_EQ(roundDown(7u, 3u), 6u); } TEST(NcclUtilityMicrotest, AlignUpDown_ScalarOverload_AssumesPowerOfTwoAlignment) { @@ -95,14 +102,15 @@ TEST(NcclUtilityMicrotest, AlignUpDown_PointerOverload_RoundsTheAddressNotTheByt alignas(32) char buf[64]; // Single-byte-type overload: static_assert(sizeof(T) == 1) gates this at // compile time, so the test itself is part of the contract being checked. + // Exact expectations, not just "divisible by 16": the mod/bound checks + // below would equally pass an off-by-one mutation of utility.h's + // `(addr + a - 1) & -uintptr_t(a)` to `(addr + a) & -uintptr_t(a)`, since + // buf + 3 rounds to buf + 16 either way. char* base = buf + 3; char* up = alignUp(base, size_t(16)); char* down = alignDown(base, size_t(16)); - EXPECT_EQ(reinterpret_cast(up) % 16, 0u); - EXPECT_EQ(reinterpret_cast(down) % 16, 0u); - EXPECT_GE(up, base); - EXPECT_LE(down, base); - EXPECT_LT(reinterpret_cast(up) - reinterpret_cast(down), 32u); + EXPECT_EQ(up, buf + 16); + EXPECT_EQ(down, buf); } TEST(NcclUtilityMicrotest, IsPow2_TrueOnlyForPowersOfTwoAndZero) { @@ -126,19 +134,22 @@ TEST(NcclUtilityMicrotest, RollingLessEqAndLessThan_WrapAroundHalfTheRangeCorrec EXPECT_TRUE(rollingLessEq(uint8_t(7), uint8_t(7), 8)); EXPECT_FALSE(rollingLessThan(uint8_t(7), uint8_t(7), 8)); - // rollingLessThan is defined as !rollingLessEq(b, a); verify that identity - // directly so a copy-paste swap of a/b in either function is caught. - for (uint8_t a : {uint8_t(0), uint8_t(1), uint8_t(127), uint8_t(128), uint8_t(200)}) { - for (uint8_t b : {uint8_t(0), uint8_t(1), uint8_t(127), uint8_t(128), uint8_t(200)}) { - EXPECT_EQ(rollingLessThan(a, b, 8), !rollingLessEq(b, a, 8)) << "a=" << int(a) << " b=" << int(b); - } - } + // Every call above passes nBits == 8*sizeof(Uint) (a no-op mask), so the + // `m = Uint(-1) >> (8*sizeof(Uint) - nBits)` shift at utility.h:197 is + // never actually exercised -- production calls it with a narrower runtime + // width (gin_barrier__funcs.h:110 passes 32 for a uint64_t counter). Use a + // wider type with an explicit partial-width nBits so the mask is load-bearing: + // true only because the high 32 bits are masked out of the wraparound check. + EXPECT_TRUE(rollingLessEq(uint64_t(0xFFFFFFF0ull), uint64_t(0x10ull), 32)); + EXPECT_FALSE(rollingLessEq(uint64_t(0x10ull), uint64_t(0xFFFFFFF0ull), 32)); } TEST(NcclUtilityMicrotest, IdivRcp32_MatchesExactFormulaIncludingPow2Correction) { - // Formula: uint32_t(-1)/x + isPow2(x). The +1 correction for powers of two - // is what makes idivFast32 exact when y is a power of two; drop it and the - // round-trip test below would start failing for y in {1,2,4,...}. + // Formula: uint32_t(-1)/x + isPow2(x). The +1 term is a branch-avoidance + // optimisation, not a correctness requirement: idivFast32's own + // `if (r >= y) { q += 1; r -= y; }` correction step already recovers the + // right answer even without it. These assertions pin the exact reciprocal + // value the formula produces, not just that the round-trip still works. EXPECT_EQ(idivRcp32(1u), uint32_t(-1) / 1u + 1u); EXPECT_EQ(idivRcp32(2u), uint32_t(-1) / 2u + 1u); EXPECT_EQ(idivRcp32(3u), uint32_t(-1) / 3u + 0u); @@ -154,7 +165,7 @@ TEST(NcclUtilityMicrotest, IdivRcp64_MatchesExactFormulaIncludingPow2Correction) TEST(NcclUtilityMicrotest, Mul32Hi_ReturnsTheHighHalfOfTheFull64BitProduct) { EXPECT_EQ(mul32hi(0u, 0xFFFFFFFFu), 0u); - EXPECT_EQ(mul32hi(0xFFFFFFFFu, 0xFFFFFFFFu), uint32_t((uint64_t(0xFFFFFFFFu) * 0xFFFFFFFFu) >> 32)); + EXPECT_EQ(mul32hi(0xFFFFFFFFu, 0xFFFFFFFFu), 0xFFFFFFFEu); EXPECT_EQ(mul32hi(0x10000u, 0x10000u), 1u); // 2^16 * 2^16 == 2^32, high half == 1 } @@ -162,6 +173,7 @@ TEST(NcclUtilityMicrotest, Mul64Hi_ReturnsTheHighHalfOfTheFull128BitProduct) { EXPECT_EQ(mul64hi(0ull, ~0ull), 0ull); uint64_t big = 0x1'0000'0000ull; // 2^32 EXPECT_EQ(mul64hi(big, big), 1ull); // 2^32 * 2^32 == 2^64, high half == 1 + EXPECT_EQ(mul64hi(~0ull, ~0ull), 0xFFFFFFFFFFFFFFFEull); } TEST(NcclUtilityMicrotest, ImulRcp32_HandlesZeroReciprocalShortCircuitsAndMatchesIdivRcpOfProduct) { @@ -201,7 +213,12 @@ TEST(NcclUtilityMicrotest, ImulRcp64_HandlesZeroReciprocalShortCircuitsAndMatche TEST(NcclUtilityMicrotest, IdivmodFast32_MatchesOrdinaryDivisionAcrossDivisorsAndYrcpZeroShortcut) { for (uint32_t y : {1u, 2u, 3u, 7u, 16u, 100u, 4096u}) { uint32_t yrcp = idivRcp32(y); - for (uint32_t x : {0u, 1u, 5u, 41u, 4095u, 4096u, 1000000u}) { + // 0xFFFFFFFFu exercises the top of the range: idivRcp32's single + // `if (r >= y)` correction step only recovers an *under*-estimating + // reciprocal, so an over-estimate (e.g. an erroneous unconditional +1) + // is only caught near the dividend's upper bound, not by the smaller + // values below. + for (uint32_t x : {0u, 1u, 5u, 41u, 4095u, 4096u, 1000000u, 0xFFFFFFFFu}) { uint32_t q, r; idivmodFast32(&q, &r, x, y, yrcp); EXPECT_EQ(q, x / y) << "x=" << x << " y=" << y; @@ -218,7 +235,8 @@ TEST(NcclUtilityMicrotest, IdivmodFast32_MatchesOrdinaryDivisionAcrossDivisorsAn TEST(NcclUtilityMicrotest, IdivmodFast64_MatchesOrdinaryDivisionAcrossDivisorsAndYrcpZeroShortcut) { for (uint64_t y : {1ull, 2ull, 3ull, 7ull, 16ull, 100ull, 1ull << 20}) { uint64_t yrcp = idivRcp64(y); - for (uint64_t x : {0ull, 1ull, 5ull, 41ull, (1ull << 20) - 1, 1ull << 20, 1ull << 40}) { + // ~0ull is the 64-bit counterpart of the 32-bit sweep's 0xFFFFFFFFu above. + for (uint64_t x : {0ull, 1ull, 5ull, 41ull, (1ull << 20) - 1, 1ull << 20, 1ull << 40, ~0ull}) { uint64_t q, r; idivmodFast64(&q, &r, x, y, yrcp); EXPECT_EQ(q, x / y) << "x=" << x << " y=" << y; @@ -238,12 +256,14 @@ TEST(NcclUtilityMicrotest, IdivFast32_MatchesRefDivisionAndIdivFast64TruncatesTo EXPECT_EQ(idivFast32(x, y, yrcp), refDiv32(x, y)); } } - // idivFast64 returns a uint32_t truncation of a 64-bit quotient; pick a - // quotient that fits in 32 bits so truncation is a correctness check, not - // a free pass. + // idivFast64 narrows a 64-bit quotient to uint32_t. A quotient that fits in + // 32 bits leaves the narrowing inert (the oracle's own uint32_t cast is a + // no-op too), so the second case below -- whose true quotient is far above + // 2^32 -- is the one that actually exercises the truncation. uint64_t y = 1000, yrcp64 = idivRcp64(y); uint64_t x = 123456789ull; EXPECT_EQ(idivFast64(x, y, yrcp64), uint32_t(refDiv64(x, y))); + EXPECT_EQ(idivFast64(1ull << 40, 1ull, idivRcp64(1ull)), 0u); } TEST(NcclUtilityMicrotest, ImodFast32And64_MatchRefModulo) { @@ -297,7 +317,7 @@ TEST(NcclUtilityMicrotest, Optional_DestructorOnlyRunsWhenPresentIsTrue) { destructorCalls = 0; { - Optional present_opt(present()); + Optional presentOpt(present()); } EXPECT_EQ(destructorCalls, 1); } From 47a12553f3b0487d753d89fcb75aa2e1e52dade1 Mon Sep 17 00:00:00 2001 From: Alex Duforest Date: Tue, 13 Oct 2026 12:05:46 -0700 Subject: [PATCH 03/26] test(rccl): pin alignUp's - 1, rollingLessThan's nBits forward, trim wrapper sweeps Co-Authored-By: Claude --- projects/rccl/test/host/nccl-utility-test.cc | 50 +++++++++----------- 1 file changed, 22 insertions(+), 28 deletions(-) diff --git a/projects/rccl/test/host/nccl-utility-test.cc b/projects/rccl/test/host/nccl-utility-test.cc index e5b233d17073..10a83346b076 100644 --- a/projects/rccl/test/host/nccl-utility-test.cc +++ b/projects/rccl/test/host/nccl-utility-test.cc @@ -53,10 +53,8 @@ using nccl::utility::toCudaOrder; namespace { -// Brute-force reference: idivFast32/64 claim idivFast(x, y, idivRcp(y)) == x/y -// for every valid (x, y); these helpers recompute that division the ordinary -// way so the fast path can't share a bug with its own golden. -uint32_t refDiv32(uint32_t x, uint32_t y) { return x / y; } +// Reference for idivFast64's truncation check: recomputes the division the +// ordinary way so the fast path can't share a bug with its own golden. uint64_t refDiv64(uint64_t x, uint64_t y) { return x / y; } } // namespace @@ -102,15 +100,17 @@ TEST(NcclUtilityMicrotest, AlignUpDown_PointerOverload_RoundsTheAddressNotTheByt alignas(32) char buf[64]; // Single-byte-type overload: static_assert(sizeof(T) == 1) gates this at // compile time, so the test itself is part of the contract being checked. - // Exact expectations, not just "divisible by 16": the mod/bound checks - // below would equally pass an off-by-one mutation of utility.h's - // `(addr + a - 1) & -uintptr_t(a)` to `(addr + a) & -uintptr_t(a)`, since - // buf + 3 rounds to buf + 16 either way. + // buf + 3 rounds to buf + 16 under both `(addr + a - 1) & -uintptr_t(a)` and + // an off-by-one `(addr + a) & -uintptr_t(a)`, so the unaligned case alone + // cannot pin the `- 1`. An already-aligned pointer can: correct returns buf, + // the mutant returns buf + 16. alignDown has no `- 1` term, so only alignUp + // needs the extra case. char* base = buf + 3; char* up = alignUp(base, size_t(16)); char* down = alignDown(base, size_t(16)); EXPECT_EQ(up, buf + 16); EXPECT_EQ(down, buf); + EXPECT_EQ(alignUp(buf, size_t(16)), buf); } TEST(NcclUtilityMicrotest, IsPow2_TrueOnlyForPowersOfTwoAndZero) { @@ -142,6 +142,11 @@ TEST(NcclUtilityMicrotest, RollingLessEqAndLessThan_WrapAroundHalfTheRangeCorrec // true only because the high 32 bits are masked out of the wraparound check. EXPECT_TRUE(rollingLessEq(uint64_t(0xFFFFFFF0ull), uint64_t(0x10ull), 32)); EXPECT_FALSE(rollingLessEq(uint64_t(0x10ull), uint64_t(0xFFFFFFF0ull), 32)); + + // rollingLessThan must forward nBits rather than fall back to its own + // default: dropping the forward at utility.h:202 flips both of these. + EXPECT_TRUE(rollingLessThan(uint64_t(0xFFFFFFF0ull), uint64_t(0x10ull), 32)); + EXPECT_FALSE(rollingLessThan(uint64_t(0x10ull), uint64_t(0xFFFFFFF0ull), 32)); } TEST(NcclUtilityMicrotest, IdivRcp32_MatchesExactFormulaIncludingPow2Correction) { @@ -249,13 +254,10 @@ TEST(NcclUtilityMicrotest, IdivmodFast64_MatchesOrdinaryDivisionAcrossDivisorsAn EXPECT_EQ(r, 0ull); } -TEST(NcclUtilityMicrotest, IdivFast32_MatchesRefDivisionAndIdivFast64TruncatesToLow32Bits) { - for (uint32_t y : {1u, 3u, 16u, 4096u}) { - uint32_t yrcp = idivRcp32(y); - for (uint32_t x : {0u, 41u, 4095u, 1000000u}) { - EXPECT_EQ(idivFast32(x, y, yrcp), refDiv32(x, y)); - } - } +TEST(NcclUtilityMicrotest, IdivFast32_ReturnsTheQuotientAndIdivFast64TruncatesToLow32Bits) { + // idivFast32 is a one-line wrapper over idivmodFast32, which the sweep above + // already checks; one case pins that it returns q (13), not r (2). + EXPECT_EQ(idivFast32(41u, 3u, idivRcp32(3u)), 13u); // idivFast64 narrows a 64-bit quotient to uint32_t. A quotient that fits in // 32 bits leaves the narrowing inert (the oracle's own uint32_t cast is a // no-op too), so the second case below -- whose true quotient is far above @@ -266,19 +268,11 @@ TEST(NcclUtilityMicrotest, IdivFast32_MatchesRefDivisionAndIdivFast64TruncatesTo EXPECT_EQ(idivFast64(1ull << 40, 1ull, idivRcp64(1ull)), 0u); } -TEST(NcclUtilityMicrotest, ImodFast32And64_MatchRefModulo) { - for (uint32_t y : {1u, 3u, 16u, 4096u}) { - uint32_t yrcp = idivRcp32(y); - for (uint32_t x : {0u, 41u, 4095u, 1000000u}) { - EXPECT_EQ(imodFast32(x, y, yrcp), x % y); - } - } - for (uint64_t y : {1ull, 3ull, 16ull, 1ull << 20}) { - uint64_t yrcp = idivRcp64(y); - for (uint64_t x : {0ull, 41ull, (1ull << 20) - 1, 1ull << 40}) { - EXPECT_EQ(imodFast64(x, y, yrcp), x % y); - } - } +TEST(NcclUtilityMicrotest, ImodFast32And64_ReturnTheRemainder) { + // imodFast32/64 are one-line wrappers over idivmodFast32/64, which the sweeps + // above already check; one case each pins that they return r (2), not q (13). + EXPECT_EQ(imodFast32(41u, 3u, idivRcp32(3u)), 2u); + EXPECT_EQ(imodFast64(41ull, 3ull, idivRcp64(3ull)), 2ull); } TEST(NcclUtilityMicrotest, Optional_DefaultAndAbsentConstructLeavePresentFalse) { From d1eb14cf6b99b8689fa05795f159356cbdcab412 Mon Sep 17 00:00:00 2001 From: Alex Duforest Date: Tue, 13 Oct 2026 13:53:40 -0700 Subject: [PATCH 04/26] test(rccl): correct the imulRcp32 counterexample comment, note the bitops.h using-declaration trap Co-Authored-By: Claude --- projects/rccl/test/host/nccl-utility-test.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/projects/rccl/test/host/nccl-utility-test.cc b/projects/rccl/test/host/nccl-utility-test.cc index 10a83346b076..36107fbde779 100644 --- a/projects/rccl/test/host/nccl-utility-test.cc +++ b/projects/rccl/test/host/nccl-utility-test.cc @@ -22,6 +22,9 @@ #include "nccl_device/utility.h" +// bitops.h declares same-signature ::divUp, ::isPow2, ::add4G, ::idivRcp32 and more, so these +// global-scope using-declarations become ill-formed if it ever reaches this TU; qualify the calls +// instead if a new include pulls it in (test/BitOpsTests.cpp records the same trap for add4G). using nccl::utility::acquireOrderOf; using nccl::utility::add4G; using nccl::utility::alignDown; @@ -189,8 +192,8 @@ TEST(NcclUtilityMicrotest, ImulRcp32_HandlesZeroReciprocalShortCircuitsAndMatche // x=3, y=5 is chosen (rather than some arbitrary pair) because it's one of // the smallest inputs that actually drives the `if (x*y <= rem) rcp += 1;` - // rounding correction true -- an arbitrary pair like (6,7) can land on the - // false side and leave that correction untested. + // rounding correction true: rcp comes back as 0x11111110, rem == 16, and + // 15 <= 16. Pairs like the (2,2) case below land on the false side instead. uint32_t x = 3, y = 5; uint32_t got = imulRcp32(x, idivRcp32(x), y, idivRcp32(y)); EXPECT_EQ(got, idivRcp32(x * y)); From 87d73103f23f9cfe1cbb3832b575af40608abf32 Mon Sep 17 00:00:00 2001 From: Alex Duforest Date: Tue, 13 Oct 2026 19:56:47 -0700 Subject: [PATCH 05/26] test(rccl): pin alignDown's negation, rollingLessEq's half-range boundary, and imulRcp's rem negation Co-Authored-By: Claude --- projects/rccl/test/host/nccl-utility-test.cc | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/projects/rccl/test/host/nccl-utility-test.cc b/projects/rccl/test/host/nccl-utility-test.cc index 36107fbde779..d3302371351d 100644 --- a/projects/rccl/test/host/nccl-utility-test.cc +++ b/projects/rccl/test/host/nccl-utility-test.cc @@ -97,6 +97,9 @@ TEST(NcclUtilityMicrotest, AlignUpDown_ScalarOverload_AssumesPowerOfTwoAlignment EXPECT_EQ(alignDown(15u, 16u), 0u); EXPECT_EQ(alignDown(16u, 16u), 16u); EXPECT_EQ(alignDown(31u, 16u), 16u); + // Above 2*a: below it, `x & -Z(a)` (utility.h:166) cannot be told from `x & Z(a)`. + EXPECT_EQ(alignDown(48u, 16u), 48u); + EXPECT_EQ(alignDown(33u, 16u), 32u); } TEST(NcclUtilityMicrotest, AlignUpDown_PointerOverload_RoundsTheAddressNotTheByte) { @@ -137,6 +140,11 @@ TEST(NcclUtilityMicrotest, RollingLessEqAndLessThan_WrapAroundHalfTheRangeCorrec EXPECT_TRUE(rollingLessEq(uint8_t(7), uint8_t(7), 8)); EXPECT_FALSE(rollingLessThan(uint8_t(7), uint8_t(7), 8)); + // The half-range boundary itself: m is 0xFF here, so m >> 1 is 127. A + // distance of exactly 127 is still "less or equal", 128 is not. + EXPECT_TRUE(rollingLessEq(uint8_t(0), uint8_t(127), 8)); + EXPECT_FALSE(rollingLessEq(uint8_t(0), uint8_t(128), 8)); + // Every call above passes nBits == 8*sizeof(Uint) (a no-op mask), so the // `m = Uint(-1) >> (8*sizeof(Uint) - nBits)` shift at utility.h:197 is // never actually exercised -- production calls it with a narrower runtime @@ -201,6 +209,11 @@ TEST(NcclUtilityMicrotest, ImulRcp32_HandlesZeroReciprocalShortCircuitsAndMatche // x=2, y=2 exercises the other side of that same branch (no correction // needed), so both directions of the rounding check are verified. EXPECT_EQ(imulRcp32(2u, idivRcp32(2u), 2u, idivRcp32(2u)), idivRcp32(4u)); + + // x=3, y=3 pins the `0u -` in rem: mul32hi already returns the exact + // 0x1C71C71C, so rem is 4 and 9 <= 4 wants no correction. Without the + // negation rem is 0xFFFFFFFC and the correction fires, returning one too many. + EXPECT_EQ(imulRcp32(3u, idivRcp32(3u), 3u, idivRcp32(3u)), idivRcp32(9u)); } TEST(NcclUtilityMicrotest, ImulRcp64_HandlesZeroReciprocalShortCircuitsAndMatchesIdivRcpOfProduct) { @@ -216,6 +229,10 @@ TEST(NcclUtilityMicrotest, ImulRcp64_HandlesZeroReciprocalShortCircuitsAndMatche // (2,2) exercises the other side of that same branch (no correction needed). EXPECT_EQ(imulRcp64(2ull, idivRcp64(2ull), 2ull, idivRcp64(2ull)), idivRcp64(4ull)); + + // (3,3) pins the `0 -` in rem the same way: mul64hi returns the exact + // 0x1C71C71C71C71C71, so rem is 7 and 9 <= 7 wants no correction. + EXPECT_EQ(imulRcp64(3ull, idivRcp64(3ull), 3ull, idivRcp64(3ull)), idivRcp64(9ull)); } TEST(NcclUtilityMicrotest, IdivmodFast32_MatchesOrdinaryDivisionAcrossDivisorsAndYrcpZeroShortcut) { From f8e4b96e3fdd9066f363523486fa4132af249d82 Mon Sep 17 00:00:00 2001 From: Alex Duforest Date: Mon, 12 Oct 2026 14:35:49 -0700 Subject: [PATCH 06/26] test(rccl): add host microtests for nccl_device device-resource requirement builders --- projects/rccl/test/host/CMakeLists.txt | 24 + projects/rccl/test/host/MICROTEST_README.md | 15 + .../host/nccl-device-requirements-test.cc | 418 ++++++++++++++++++ projects/rccl/test/host/run_host_tests.sh | 1 + projects/rccl/test/test_categories_micro.yaml | 2 + 5 files changed, 460 insertions(+) create mode 100644 projects/rccl/test/host/nccl-device-requirements-test.cc diff --git a/projects/rccl/test/host/CMakeLists.txt b/projects/rccl/test/host/CMakeLists.txt index 1344ae8b3bf2..078b4ea97ae4 100644 --- a/projects/rccl/test/host/CMakeLists.txt +++ b/projects/rccl/test/host/CMakeLists.txt @@ -293,6 +293,16 @@ function(rccl_define_micro_source_lists) ../common/main_altrsmi.cpp PARENT_SCOPE) + # rccl-UnitTestsMicroNcclDevice -- src/nccl_device/{cft_barrier,gin_barrier,gin_scratch,ll_a2a, + # lsa_barrier}.cc. Its own binary: fakes/dev_runtime_micro_fakes.cc (needed by dev-runtime-test.cc + # in rccl-UnitTestsMicro) fakes ncclLsaBarrierCreateRequirement/ncclGinBarrierCreateRequirement/ + # ncclCftBarrierCreateRequirement, which this target compiles as the real thing under test. + set(TEST_MICRO_NCCL_DEVICE_SOURCE_FILES + nccl-device-requirements-test.cc + ../common/ProcessIsolatedTestRunner.cpp + ../common/main_altrsmi.cpp + PARENT_SCOPE) + # rccl-UnitTestsMicroSymKernels -- REAL src/sym_kernels.cc + generated host TU. set(TEST_MICRO_SYM_KERNELS_SOURCE_FILES sym-kernels-test.cc @@ -1055,6 +1065,20 @@ rccl_add_micro_binary(rccl-UnitTestsMicroClientSupport CLIENT_SUPPORT_CC_PATH="${RCCL_HOST_HIPIFY_DIR}/src/ras/client_support.cc" CATEGORIES_YAML "${CMAKE_CURRENT_SOURCE_DIR}/../test_categories_micro.yaml") +# --------------------------------------------------------------------------- +# rccl-UnitTestsMicroNcclDevice +# --------------------------------------------------------------------------- +rccl_add_micro_binary(rccl-UnitTestsMicroNcclDevice + SOURCES ${TEST_MICRO_NCCL_DEVICE_SOURCE_FILES} + EXTRA_DEFS + ${_MICRO_MODE_DEFS} + CFT_BARRIER_CC_PATH="${RCCL_HOST_HIPIFY_DIR}/src/nccl_device/cft_barrier.cc" + GIN_BARRIER_CC_PATH="${RCCL_HOST_HIPIFY_DIR}/src/nccl_device/gin_barrier.cc" + GIN_SCRATCH_CC_PATH="${RCCL_HOST_HIPIFY_DIR}/src/nccl_device/gin_scratch.cc" + LL_A2A_CC_PATH="${RCCL_HOST_HIPIFY_DIR}/src/nccl_device/ll_a2a.cc" + LSA_BARRIER_CC_PATH="${RCCL_HOST_HIPIFY_DIR}/src/nccl_device/lsa_barrier.cc" + CATEGORIES_YAML "${CMAKE_CURRENT_SOURCE_DIR}/../test_categories_micro.yaml") + # --------------------------------------------------------------------------- # rccl-UnitTestsMicroSymKernels # --------------------------------------------------------------------------- diff --git a/projects/rccl/test/host/MICROTEST_README.md b/projects/rccl/test/host/MICROTEST_README.md index 1ff07e6588f5..525e22166349 100644 --- a/projects/rccl/test/host/MICROTEST_README.md +++ b/projects/rccl/test/host/MICROTEST_README.md @@ -248,6 +248,21 @@ export colliding non-`static` symbols; otherwise a unit needs its own binary: guard-less positional pair `libc_seam.h` / `libc_seam_undef.h` must bracket the production source include, after system headers and before test helpers. CMake pins the copied `ncclFuncStr` table to `src/init.cc`. +- **`rccl-UnitTestsMicroNcclDevice`**: the six device-resource requirement + builders in `src/nccl_device/{cft_barrier,gin_barrier,gin_scratch,ll_a2a, + lsa_barrier}.cc` (via `CFT_BARRIER_CC_PATH`, `GIN_BARRIER_CC_PATH`, + `GIN_SCRATCH_CC_PATH`, `LL_A2A_CC_PATH`, `LSA_BARRIER_CC_PATH`, from + `nccl-device-requirements-test.cc`); suite `NcclDeviceRequirementsMicrotest.*`. + Its own binary: `fakes/dev_runtime_micro_fakes.cc` (needed by + `dev-runtime-test.cc` in `rccl-UnitTestsMicro`) fakes + `ncclLsaBarrierCreateRequirement`/`ncclGinBarrierCreateRequirement`/ + `ncclCftBarrierCreateRequirement`, which this target compiles as the real + thing under test. These functions are pure host computation (no HIP/device + calls), so five of the six port directly from the existing GPU-binary + coverage in `test/DeviceResourceRequirementTests.cpp` (AICOMRCCL-2446, + including its `ProcessIsolatedTestRunner`-based divide-by-zero crash pin); + the sixth, `ncclCftBarrierCreateRequirement`, had no coverage anywhere and is + new here. - **`rccl-UnitTestsMicroSymKernels`** — the REAL `src/sym_kernels.cc` (via `SYM_KERNELS_CC_PATH`, from `sym-kernels-test.cc`), compiled together with the GENERATED `sym_kernels_host.cc` it calls into; suites `SymKernelMicrotest.*`, diff --git a/projects/rccl/test/host/nccl-device-requirements-test.cc b/projects/rccl/test/host/nccl-device-requirements-test.cc new file mode 100644 index 000000000000..6250d61d31de --- /dev/null +++ b/projects/rccl/test/host/nccl-device-requirements-test.cc @@ -0,0 +1,418 @@ +/************************************************************************* + * Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +// Host-only microtests for the six device-resource requirement builders in +// src/nccl_device/{cft_barrier,gin_barrier,gin_scratch,ll_a2a,lsa_barrier}.cc. +// They compute buffer sizes and GIN signal counts that device kernels index +// into; an off-by-one here under-allocates scratch and surfaces as device +// corruption. Five of the six (everything but ncclCftBarrierCreateRequirement) +// already have GPU-binary coverage in test/DeviceResourceRequirementTests.cpp +// (AICOMRCCL-2446); this file ports those same goldens host-only rather than +// duplicating the reasoning, since the functions under test are pure host +// computation with no HIP/device calls. + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "../common/ProcessIsolatedTestRunner.hpp" + +// ncclDevResourceRequirements / ncclTeam live behind core_tmp.h, transitively included below. +#include "nccl_device/impl/ll_a2a__types.h" +#include "nccl_device/impl/lsa_barrier__types.h" +#include "nccl_device/impl/gin_barrier__types.h" +#include "nccl_device/impl/cft_barrier__types.h" +// gin_scratch__types.h expects the unqualified alignUp/imodFast32 that "core.h" gives production TUs. +#include "bitops.h" +// Outbox/inbox declarations still live at their pre-public-split location, not under nccl_device/impl/. +#include "device/symmetric/gin_scratch.h" + +#include CFT_BARRIER_CC_PATH +#include GIN_BARRIER_CC_PATH +#include GIN_SCRATCH_CC_PATH +#include LL_A2A_CC_PATH +#include LSA_BARRIER_CC_PATH + +namespace { + +// Re-implemented via division, not the production bit-mask trick, so a bug there can't hide here. +constexpr size_t alignUpLocal(size_t x, size_t a) { + return ((x + a - 1) / a) * a; +} + +// Non-zero poison so a missing memset() would leave visible garbage instead of a false-zero read. +ncclDevResourceRequirements_t poisonedRequirement() { + ncclDevResourceRequirements_t req; + std::memset(&req, 0xAA, sizeof(req)); + return req; +} + +// Same poison for handle out-params: an unpoisoned handle can pass by accident on a zeroed stack. +template +T poisonedHandle() { + T handle; + std::memset(&handle, 0xAA, sizeof(handle)); + return handle; +} + +// GIN-signal fields untouched by the buffer-only builders (LLA2A, outbox, LSA/CFT barrier). +void ExpectNoGinSignalFields(const ncclDevResourceRequirements_t& req) { + EXPECT_EQ(req.next, nullptr); + EXPECT_EQ(req.ginSignalCount, 0); + EXPECT_EQ(req.ginCounterCount, 0); + EXPECT_EQ(req.outGinSignalStart, nullptr); + EXPECT_EQ(req.outGinCounterStart, nullptr); +} + +// GIN-counter fields untouched by the signal-setting builders; ginSignalCount/outGinSignalStart +// are set and checked separately by each test. +void ExpectNoGinCounterFields(const ncclDevResourceRequirements_t& req) { + EXPECT_EQ(req.next, nullptr); + EXPECT_EQ(req.ginCounterCount, 0); + EXPECT_EQ(req.outGinCounterStart, nullptr); +} + +// The isolated child re-execs, resetting PR_SET_DUMPABLE, so it must be set again here rather than +// once in the runner. ci-precheckin.json builds Debug with coverage on, so a core dump is sizable. +void DisableCoreDumpsForExpectedCrash() { +#if defined(__linux__) + if (prctl(PR_SET_DUMPABLE, 0) != 0) { + std::perror("prctl(PR_SET_DUMPABLE) failed"); + _exit(1); + } +#endif + const struct rlimit noCore = {0, 0}; + if (setrlimit(RLIMIT_CORE, &noCore) != 0) { + std::perror("setrlimit(RLIMIT_CORE) failed"); + _exit(1); + } +} + +} // namespace + +// --------------------------------------------------------------------------- +// ncclLLA2ACalcSlots / ncclLLA2ACreateRequirement (src/nccl_device/ll_a2a.cc) +// --------------------------------------------------------------------------- + +// divUp rounding: maxEltSize of 1, 7, 8 all round up to one 8-byte slot; 9 and 16 need two. +TEST(NcclDeviceRequirementsMicrotest, LLA2ACalcSlots_DivUpRounding) { + struct Case { + const char* name; + int maxEltSize; + int expectedSlots; + }; + const std::vector cases = { + {"eltSize_1", 1, 1}, {"eltSize_7", 7, 1}, {"eltSize_8", 8, 1}, {"eltSize_9", 9, 2}, {"eltSize_16", 16, 2}, + }; + for (const Case& c : cases) { + SCOPED_TRACE(c.name); + EXPECT_EQ(ncclLLA2ACalcSlots(/*maxElts=*/1, c.maxEltSize), c.expectedSlots); + } +} + +// Scales linearly with maxElts against a known-correct literal, not ncclLLA2ACalcSlots's own +// output: that would hold for any maxElts * g(maxEltSize) shape, including a wrong g. +TEST(NcclDeviceRequirementsMicrotest, LLA2ACalcSlots_LinearInMaxElts) { + EXPECT_EQ(ncclLLA2ACalcSlots(/*maxElts=*/5, /*maxEltSize=*/8), 5); +} + +// maxElts = 0 always yields 0 slots, independent of maxEltSize. +TEST(NcclDeviceRequirementsMicrotest, LLA2ACalcSlots_ZeroMaxEltsIsZero) { + EXPECT_EQ(ncclLLA2ACalcSlots(/*maxElts=*/0, /*maxEltSize=*/8), 0); +} + +// bufferSize == nBlocks * (1 + 2*nSlots) * 16; nSlots == 0 is the header-only case. Values stay +// sane on purpose: the plain-int formula can overflow before promoting to size_t, and pinning UB is out of scope. +TEST(NcclDeviceRequirementsMicrotest, LLA2ACreateRequirement_BufferSizeFormula) { + struct Case { + const char* name; + int nBlocks; + int nSlots; + }; + const std::vector cases = { + {"header_only", 1, 0}, + {"small", 2, 3}, + {"larger", 4, 5}, + }; + for (const Case& c : cases) { + SCOPED_TRACE(c.name); + ncclLLA2AHandle_t handle = poisonedHandle(); + ncclDevResourceRequirements_t req = poisonedRequirement(); + ASSERT_EQ(ncclLLA2ACreateRequirement(c.nBlocks, c.nSlots, &handle, &req), ncclSuccess); + EXPECT_EQ(req.bufferSize, static_cast(c.nBlocks) * (1 + 2 * c.nSlots) * 16); + EXPECT_EQ(req.bufferAlign, 16u); + EXPECT_EQ(req.outBufferHandle, &handle.bufHandle); + EXPECT_EQ(handle.nSlots, static_cast(c.nSlots)); + // Fields this function never sets must still read back zero: proves the memset prologue ran. + ExpectNoGinSignalFields(req); + } +} + +// Realistic-usage row (mirrors src/sym_kernels.cc), not a contract. Expected bufferSize is a +// literal rather than a formula built from nSlots, since nSlots is the function under test's own +// output and would hold regardless of its value. ncclLLA2ACalcSlots(4, 16) == 8. +TEST(NcclDeviceRequirementsMicrotest, LLA2ACreateRequirement_RoundTripWithCalcSlots) { + const int nSlots = ncclLLA2ACalcSlots(/*maxElts=*/4, /*maxEltSize=*/16); + const int nBlocks = 3; + ncclLLA2AHandle_t handle = poisonedHandle(); + ncclDevResourceRequirements_t req = poisonedRequirement(); + ASSERT_EQ(ncclLLA2ACreateRequirement(nBlocks, nSlots, &handle, &req), ncclSuccess); + EXPECT_EQ(req.bufferSize, size_t{816}); // nBlocks=3, nSlots=8: 3 * (1 + 2*8) * 16 + EXPECT_EQ(handle.nSlots, static_cast(nSlots)); +} + +// --------------------------------------------------------------------------- +// ncclGinOutboxCreateRequirement / ncclGinInboxA2ACreateRequirement (src/nccl_device/gin_scratch.cc) +// --------------------------------------------------------------------------- + +// The 128-byte floor: size_log2 below 7 clamps to 7, and outHandle reads back the clamped value. +TEST(NcclDeviceRequirementsMicrotest, GinOutboxCreateRequirement_FloorClampsSizeLog2Below7) { + const std::vector belowFloor = {0, 5, 6, 7}; + for (int sizeLog2 : belowFloor) { + ncclGinOutboxHandle handle = poisonedHandle(); + ncclDevResourceRequirements_t req = poisonedRequirement(); + ASSERT_EQ(ncclGinOutboxCreateRequirement(/*nBlocks=*/1, sizeLog2, &handle, &req), ncclSuccess) + << "case: size_log2=" << sizeLog2; + EXPECT_EQ(handle.size_log2, 7u) << "case: size_log2=" << sizeLog2; + } +} + +// Above the floor, size_log2 passes through unclamped. +TEST(NcclDeviceRequirementsMicrotest, GinOutboxCreateRequirement_SizeLog2AboveFloorIsUnclamped) { + ncclGinOutboxHandle handle = poisonedHandle(); + ncclDevResourceRequirements_t req = poisonedRequirement(); + ASSERT_EQ(ncclGinOutboxCreateRequirement(/*nBlocks=*/1, /*size_log2=*/8, &handle, &req), ncclSuccess); + EXPECT_EQ(handle.size_log2, 8u); +} + +// bufferSize == nBlocks * (sizeof(state) + RequestBytes + alignUp(1< cases = { + {"clamped_floor", 1, 0, 8448}, // 1 * (128 + 8192 + 128) + {"at_floor", 3, 7, 25344}, // 3 * (128 + 8192 + 128) + {"above_floor", 2, 8, 17152}, // 2 * (128 + 8192 + 256) + }; + for (const Case& c : cases) { + SCOPED_TRACE(c.name); + ncclGinOutboxHandle handle = poisonedHandle(); + ncclDevResourceRequirements_t req = poisonedRequirement(); + ASSERT_EQ(ncclGinOutboxCreateRequirement(c.nBlocks, c.sizeLog2, &handle, &req), ncclSuccess); + const int clampedLog2 = std::max(c.sizeLog2, 7); + const size_t expected = + static_cast(c.nBlocks) * (sizeof(ncclGinOutboxState) + ncclGinOutboxState::RequestBytes + + alignUpLocal(size_t(1) << clampedLog2, alignof(ncclGinOutboxState))); + EXPECT_EQ(req.bufferSize, expected); + EXPECT_EQ(req.bufferSize, c.goldenBufferSize); + EXPECT_EQ(req.bufferAlign, 128u); + EXPECT_EQ(req.outBufferHandle, &handle.bufHandle); + // The outbox has no GIN signals; only the inbox A2A sibling below sets ginSignalCount. + ExpectNoGinSignalFields(req); + } +} + +// Unlike the outbox, the inbox A2A applies no floor to size_log2: pinned as observed, not endorsed. +TEST(NcclDeviceRequirementsMicrotest, GinInboxA2ACreateRequirement_NoFloorOnSizeLog2) { + ncclTeam_t peers{/*nRanks=*/2, /*rank=*/0, /*stride=*/1}; + ncclGinInboxA2AHandle handle = poisonedHandle(); + ncclDevResourceRequirements_t req = poisonedRequirement(); + ASSERT_EQ(ncclGinInboxA2ACreateRequirement(peers, /*nBlocks=*/1, /*size_log2=*/0, &handle, &req), ncclSuccess); + EXPECT_EQ(handle.size_log2, 0u) << "asymmetric with the outbox's 128-byte floor; no floor here today"; + // The only case anywhere in this file where alignUpLocal's rounding actually fires: every other + // size_log2 used elsewhere is already >= 7, so 1< cases = { + {"smallest_safe_team", 2, 1, 7, 256, 2052}, // 1*(128+128); 1*4*(1+512) + {"larger_team", 5, 2, 8, 768, 4128}, // 2*(128+256); 2*4*(4+512) + }; + for (const Case& c : cases) { + SCOPED_TRACE(c.name); + ncclTeam_t peers{c.nRanks, /*rank=*/0, /*stride=*/1}; + ncclGinInboxA2AHandle handle = poisonedHandle(); + ncclDevResourceRequirements_t req = poisonedRequirement(); + ASSERT_EQ(ncclGinInboxA2ACreateRequirement(peers, c.nBlocks, c.sizeLog2, &handle, &req), ncclSuccess); + const int nPeers = c.nRanks - 1; + const size_t expectedBufferSize = + static_cast(c.nBlocks) * + (sizeof(ncclGinInboxA2AState) + alignUpLocal(size_t(1) << c.sizeLog2, alignof(ncclGinInboxA2AState))); + EXPECT_EQ(req.bufferSize, expectedBufferSize); + EXPECT_EQ(req.bufferSize, c.goldenBufferSize); + EXPECT_EQ(req.bufferAlign, 128u); + EXPECT_EQ(req.outBufferHandle, &handle.bufHandle); + EXPECT_EQ(req.ginSignalCount, c.nBlocks * 4 * (nPeers + (1 << ncclGinScratchMaxBufs_log2))); + EXPECT_EQ(req.ginSignalCount, c.goldenGinSignalCount); + EXPECT_EQ(req.outGinSignalStart, &handle.signals); + ExpectNoGinCounterFields(req); + // handle.nPeers_rcp32 feeds imodFast32 in device code (gin_scratch__types.h / __funcs.h); check + // the reciprocal contract, not just that some value got stored, so a wrong-operand builder fails. + for (uint32_t x : {0u, 1u, 3u, 7u, 100u}) { + EXPECT_EQ(imodFast32(x, static_cast(nPeers), handle.nPeers_rcp32), x % static_cast(nPeers)) + << "x=" << x; + } + } +} + +// A 1-rank team makes nPeers = 0, and idivRcp32(0) (bitops.h) divides by zero at runtime: a real +// defect, unreachable from any in-tree caller (src/sym_kernels.cc guards nRanks >= 2 via +// computeLsaSize's GCD construction). A crash-containment pin, not an assertion on the exact +// signal: ProcessIsolatedTestRunner's public API clears per-test results before executeAllTests() +// returns, so nothing here can read back WTERMSIG. A gtest death test would pin SIGFPE precisely, +// but this file compiles for both host and the gfx1151 device pass, which never defines __linux__, +// so GTEST_HAS_DEATH_TEST is false there and EXPECT_EXIT does not exist (confirmed by a scratch +// build). Verified empirically instead: the isolated child is killed by signal 8 today. +// +// x86_64-only: idivRcp32 is a runtime 64-bit divide, and only x86 raises #DE on divide-by-zero. +// AArch64 UDIV is defined to return 0 without trapping, so the same call would return ncclSuccess +// there, and this pin would misreport the still-present defect as fixed. Same guard idiom as +// src/proxy.cc:887. No aarch64 target exists in tools/scripts/test_runner/configs/ today. +TEST(NcclDeviceRequirementsMicrotest, GinInboxA2ACreateRequirement_SingleRankTeam_CrashesOnDivideByZero) { +#if defined(__x86_64__) + // Not RUN_ISOLATED_TEST (it EXPECT_TRUE's the run); no lambda assertion either, so a future fix + // that makes the call return normally flips executeAllTests() true and fails the EXPECT_FALSE below. + RcclUnitTesting::ProcessIsolatedTestRunner::registerTest("GinInboxA2A_SingleRankTeam_DivideByZero", [] { + DisableCoreDumpsForExpectedCrash(); + ncclTeam_t peers{/*nRanks=*/1, /*rank=*/0, /*stride=*/1}; + ncclGinInboxA2AHandle handle = poisonedHandle(); + ncclDevResourceRequirements_t req = poisonedRequirement(); + // Never expected to return: SIGFPE fires inside idivRcp32 before this line completes. + ncclGinInboxA2ACreateRequirement(peers, /*nBlocks=*/1, /*size_log2=*/7, &handle, &req); + }); + EXPECT_FALSE(RcclUnitTesting::ProcessIsolatedTestRunner::executeAllTests()) + << "expected the isolated child to be killed by SIGFPE; if it now passes, the defect was fixed"; +#else + GTEST_SKIP() << "divide-by-zero trap is x86_64-specific; AArch64 UDIV returns 0 without trapping"; +#endif +} + +// --------------------------------------------------------------------------- +// ncclLsaBarrierCreateRequirement (src/nccl_device/lsa_barrier.cc) +// --------------------------------------------------------------------------- + +// bufferSize == (3*nBarriers + nBarriers*team.nRanks) * sizeof(uint32_t); nBarriers == 0 gives 0. +// Same out-of-scope-overflow reasoning as the LLA2A formula above applies to this plain-int formula. +// goldenBufferSize hand-computes the same value independently of the formula below, same reasoning. +TEST(NcclDeviceRequirementsMicrotest, LsaBarrierCreateRequirement_BufferSizeFormula) { + struct Case { + const char* name; + int nBarriers; + int nRanks; + size_t goldenBufferSize; + }; + const std::vector cases = { + {"zero_barriers", 0, 4, 0}, // (0 + 0) * 4 + {"small", 2, 4, 56}, // (6 + 8) * 4 + {"larger", 5, 3, 120}, // (15 + 15) * 4 + }; + for (const Case& c : cases) { + SCOPED_TRACE(c.name); + ncclTeam_t team{c.nRanks, /*rank=*/0, /*stride=*/1}; + ncclLsaBarrierHandle_t handle = poisonedHandle(); + ncclDevResourceRequirements_t req = poisonedRequirement(); + ASSERT_EQ(ncclLsaBarrierCreateRequirement(team, c.nBarriers, &handle, &req), ncclSuccess); + const size_t expected = static_cast(3 * c.nBarriers + c.nBarriers * c.nRanks) * sizeof(uint32_t); + EXPECT_EQ(req.bufferSize, expected); + EXPECT_EQ(req.bufferSize, c.goldenBufferSize); + EXPECT_EQ(req.bufferAlign, alignof(uint32_t)); + EXPECT_EQ(req.outBufferHandle, &handle.bufHandle); + EXPECT_EQ(handle.nBarriers, c.nBarriers); + // The buffer-only sibling of the GIN/CFT barriers below: no GIN signals here. + ExpectNoGinSignalFields(req); + } +} + +// --------------------------------------------------------------------------- +// ncclGinBarrierCreateRequirement (src/nccl_device/gin_barrier.cc) +// --------------------------------------------------------------------------- + +// comm is unused, so nullptr is the contract that makes this testable without a GPU. Mirrors the +// LSA/CFT barriers: this one sets ginSignalCount but no buffer, those set a buffer but no signals. +TEST(NcclDeviceRequirementsMicrotest, GinBarrierCreateRequirement_NullCommAndSignalCountFormula) { + struct Case { + const char* name; + int nBarriers; + int nRanks; + }; + const std::vector cases = { + {"zero_barriers", 0, 5}, + {"small", 3, 4}, + }; + for (const Case& c : cases) { + SCOPED_TRACE(c.name); + ncclTeam_t team{c.nRanks, /*rank=*/0, /*stride=*/1}; + ncclGinBarrierHandle_t handle = poisonedHandle(); + ncclDevResourceRequirements_t req = poisonedRequirement(); + ASSERT_EQ(ncclGinBarrierCreateRequirement(/*comm=*/nullptr, team, c.nBarriers, &handle, &req), ncclSuccess); + EXPECT_EQ(req.ginSignalCount, c.nBarriers * c.nRanks); + EXPECT_EQ(req.outGinSignalStart, &handle.signal0); + EXPECT_EQ(req.bufferSize, 0u); + EXPECT_EQ(req.bufferAlign, 0u); + EXPECT_EQ(req.outBufferHandle, nullptr); + ExpectNoGinCounterFields(req); + } +} + +// --------------------------------------------------------------------------- +// ncclCftBarrierCreateRequirement (src/nccl_device/cft_barrier.cc) -- not covered anywhere else +// --------------------------------------------------------------------------- + +// bufferSize == (3*nBarriers + nBarriers*team.nRanks) * NCCL_CFT_BARRIER_GRAN. Same shape as the +// LSA barrier above (both are buffer-only, no GIN signals), but a distinct granularity constant +// (NCCL_CFT_BARRIER_GRAN == NCCL_CFT_BARRIER_ALIGN == 16, vs. the LSA barrier's sizeof(uint32_t) == 4): +// swapping the two constants would still compile and this is the only place that would catch it. +TEST(NcclDeviceRequirementsMicrotest, CftBarrierCreateRequirement_BufferSizeFormula) { + struct Case { + const char* name; + int nBarriers; + int nRanks; + size_t goldenBufferSize; + }; + const std::vector cases = { + {"zero_barriers", 0, 4, 0}, // (0 + 0) * 16 + {"small", 2, 4, 224}, // (6 + 8) * 16 + {"larger", 5, 3, 480}, // (15 + 15) * 16 + }; + for (const Case& c : cases) { + SCOPED_TRACE(c.name); + ncclTeam_t team{c.nRanks, /*rank=*/0, /*stride=*/1}; + ncclCftBarrierHandle_t handle = poisonedHandle(); + ncclDevResourceRequirements_t req = poisonedRequirement(); + ASSERT_EQ(ncclCftBarrierCreateRequirement(team, c.nBarriers, &handle, &req), ncclSuccess); + const size_t expected = static_cast(3 * c.nBarriers + c.nBarriers * c.nRanks) * NCCL_CFT_BARRIER_GRAN; + EXPECT_EQ(req.bufferSize, expected); + EXPECT_EQ(req.bufferSize, c.goldenBufferSize); + EXPECT_EQ(req.bufferAlign, static_cast(NCCL_CFT_BARRIER_ALIGN)); + EXPECT_EQ(req.outBufferHandle, &handle.bufHandle); + EXPECT_EQ(handle.nBarriers, c.nBarriers); + ExpectNoGinSignalFields(req); + } +} diff --git a/projects/rccl/test/host/run_host_tests.sh b/projects/rccl/test/host/run_host_tests.sh index ec702f4f62b1..494f7187da90 100755 --- a/projects/rccl/test/host/run_host_tests.sh +++ b/projects/rccl/test/host/run_host_tests.sh @@ -167,6 +167,7 @@ do_host_tests() { "rccl-UnitTestsMicroTaskPrep:$SCRIPT_DIR/host_tests_micro_taskprep.xml" "rccl-UnitTestsMicroCe:$SCRIPT_DIR/host_tests_micro_ce.xml" "rccl-UnitTestsMicroClientSupport:$SCRIPT_DIR/host_tests_micro_client_support.xml" + "rccl-UnitTestsMicroNcclDevice:$SCRIPT_DIR/host_tests_micro_nccl_device.xml" "rccl-UnitTestsMicroGinHost:$SCRIPT_DIR/host_tests_micro_gin_host.xml" "rccl-UnitTestsMicroDiagnostics:$SCRIPT_DIR/host_tests_micro_diagnostics.xml" ) diff --git a/projects/rccl/test/test_categories_micro.yaml b/projects/rccl/test/test_categories_micro.yaml index a940ce5f4f7c..f8b4c67b0c3c 100644 --- a/projects/rccl/test/test_categories_micro.yaml +++ b/projects/rccl/test/test_categories_micro.yaml @@ -114,6 +114,8 @@ test_categories: # ce_coll.cc (rccl-UnitTestsMicroCe) - "CeCollMicrotest.*" - "LaunchPaths/CeAlltoAllvSyncMicrotest.*" + # nccl_device/{cft_barrier,gin_barrier,gin_scratch,ll_a2a,lsa_barrier}.cc (rccl-UnitTestsMicroNcclDevice) + - "NcclDeviceRequirementsMicrotest.*" # dev_runtime.cc - "Alloc*" - "Comm*" From 54ccb4dd82d01f076d02a9cd9c5c08f2d3024b27 Mon Sep 17 00:00:00 2001 From: Alex Duforest Date: Mon, 12 Oct 2026 22:31:46 -0700 Subject: [PATCH 07/26] test(rccl): address ARGUS findings on device-requirement microtests - Replace the ProcessIsolatedTestRunner-based divide-by-zero pin with a real EXPECT_EXIT(..., KilledBySignal(SIGFPE), "") death test: this binary is always --offload-host-only, so GTEST_HAS_DEATH_TEST is true here and the old comment's reason for avoiding one didn't hold. EXPECT_FALSE(executeAllTests()) passed on any child failure, not only SIGFPE; the new test pins the exact signal and can no longer pass on a fork failure, timeout, or unrelated crash. - Build the crashing team's nRanks from a volatile int so the divide can't be constant-folded away by an optimizing build. - Drop the now-unused ProcessIsolatedTestRunner include and CMake source entry (nothing else in this binary used it). - LLA2ACalcSlots_LinearInMaxElts: add a (2,4) row distinguishing maxElts*divUp(maxEltSize,8) from divUp(maxElts*maxEltSize,8), which (5,8) alone could not. - Note AICOMRCCL-2912 (filed) for retiring the duplicate GPU-binary copy in test/DeviceResourceRequirementTests.cpp once this suite is trusted in CI. - Nits: rename the floor-clamp test/vector for the at-or-below-7 case, fix include order. --- projects/rccl/test/host/CMakeLists.txt | 1 - .../host/nccl-device-requirements-test.cc | 63 +++++++++++-------- 2 files changed, 38 insertions(+), 26 deletions(-) diff --git a/projects/rccl/test/host/CMakeLists.txt b/projects/rccl/test/host/CMakeLists.txt index 078b4ea97ae4..cef6d9270e02 100644 --- a/projects/rccl/test/host/CMakeLists.txt +++ b/projects/rccl/test/host/CMakeLists.txt @@ -299,7 +299,6 @@ function(rccl_define_micro_source_lists) # ncclCftBarrierCreateRequirement, which this target compiles as the real thing under test. set(TEST_MICRO_NCCL_DEVICE_SOURCE_FILES nccl-device-requirements-test.cc - ../common/ProcessIsolatedTestRunner.cpp ../common/main_altrsmi.cpp PARENT_SCOPE) diff --git a/projects/rccl/test/host/nccl-device-requirements-test.cc b/projects/rccl/test/host/nccl-device-requirements-test.cc index 6250d61d31de..742aaaa7c2a5 100644 --- a/projects/rccl/test/host/nccl-device-requirements-test.cc +++ b/projects/rccl/test/host/nccl-device-requirements-test.cc @@ -12,20 +12,22 @@ // already have GPU-binary coverage in test/DeviceResourceRequirementTests.cpp // (AICOMRCCL-2446); this file ports those same goldens host-only rather than // duplicating the reasoning, since the functions under test are pure host -// computation with no HIP/device calls. +// computation with no HIP/device calls. Both copies are built and run today +// (test/CMakeLists.txt, ci-precheckin.json) and have already drifted at a few +// lines; retiring the GPU-binary copy once this suite has run clean in CI for +// a while is tracked separately in AICOMRCCL-2912, not done here. #include #include #include #include +#include #include #include #include #include -#include "../common/ProcessIsolatedTestRunner.hpp" - // ncclDevResourceRequirements / ncclTeam live behind core_tmp.h, transitively included below. #include "nccl_device/impl/ll_a2a__types.h" #include "nccl_device/impl/lsa_barrier__types.h" @@ -123,6 +125,10 @@ TEST(NcclDeviceRequirementsMicrotest, LLA2ACalcSlots_DivUpRounding) { // output: that would hold for any maxElts * g(maxEltSize) shape, including a wrong g. TEST(NcclDeviceRequirementsMicrotest, LLA2ACalcSlots_LinearInMaxElts) { EXPECT_EQ(ncclLLA2ACalcSlots(/*maxElts=*/5, /*maxEltSize=*/8), 5); + // maxElts=2, maxEltSize=4: correct is maxElts * divUp(maxEltSize, 8) == 2 * 1 == 2; hoisting the + // divUp outside the multiply (divUp(maxElts * maxEltSize, 8) == divUp(8, 8)) would give 1 instead. + // The row above can't catch that: 5 * divUp(8,8) == divUp(5*8,8) == 5 either way. + EXPECT_EQ(ncclLLA2ACalcSlots(/*maxElts=*/2, /*maxEltSize=*/4), 2); } // maxElts = 0 always yields 0 slots, independent of maxEltSize. @@ -174,10 +180,10 @@ TEST(NcclDeviceRequirementsMicrotest, LLA2ACreateRequirement_RoundTripWithCalcSl // ncclGinOutboxCreateRequirement / ncclGinInboxA2ACreateRequirement (src/nccl_device/gin_scratch.cc) // --------------------------------------------------------------------------- -// The 128-byte floor: size_log2 below 7 clamps to 7, and outHandle reads back the clamped value. -TEST(NcclDeviceRequirementsMicrotest, GinOutboxCreateRequirement_FloorClampsSizeLog2Below7) { - const std::vector belowFloor = {0, 5, 6, 7}; - for (int sizeLog2 : belowFloor) { +// The 128-byte floor: size_log2 at or below 7 clamps to 7, and outHandle reads back the clamped value. +TEST(NcclDeviceRequirementsMicrotest, GinOutboxCreateRequirement_FloorClampsSizeLog2AtOrBelow7) { + const std::vector atOrBelowFloor = {0, 5, 6, 7}; + for (int sizeLog2 : atOrBelowFloor) { ncclGinOutboxHandle handle = poisonedHandle(); ncclDevResourceRequirements_t req = poisonedRequirement(); ASSERT_EQ(ncclGinOutboxCreateRequirement(/*nBlocks=*/1, sizeLog2, &handle, &req), ncclSuccess) @@ -284,12 +290,15 @@ TEST(NcclDeviceRequirementsMicrotest, GinInboxA2ACreateRequirement_BufferSizeAnd // A 1-rank team makes nPeers = 0, and idivRcp32(0) (bitops.h) divides by zero at runtime: a real // defect, unreachable from any in-tree caller (src/sym_kernels.cc guards nRanks >= 2 via -// computeLsaSize's GCD construction). A crash-containment pin, not an assertion on the exact -// signal: ProcessIsolatedTestRunner's public API clears per-test results before executeAllTests() -// returns, so nothing here can read back WTERMSIG. A gtest death test would pin SIGFPE precisely, -// but this file compiles for both host and the gfx1151 device pass, which never defines __linux__, -// so GTEST_HAS_DEATH_TEST is false there and EXPECT_EXIT does not exist (confirmed by a scratch -// build). Verified empirically instead: the isolated child is killed by signal 8 today. +// computeLsaSize's GCD construction). This binary (rccl-UnitTestsMicroNcclDevice) is always built +// with --offload-host-only (cmake/RcclTargetHelpers.cmake:78), so __linux__ is defined and +// GTEST_HAS_DEATH_TEST is true here; pin the exact signal with EXPECT_EXIT, same idiom as +// init-test.cc's P2pSchedule_ZeroGroupSizeParam_DiesOnDivideByZero. +// +// peers/handle/req are built from a volatile int so the divide stays real at -O2 too: with plain +// automatic storage, inlining collapses ncclGinInboxA2ACreateRequirement's `idivRcp32(nPeers)` call +// into a compile-time-zero divisor, which is undefined behaviour an optimising build is free to +// fold away entirely rather than trap. // // x86_64-only: idivRcp32 is a runtime 64-bit divide, and only x86 raises #DE on divide-by-zero. // AArch64 UDIV is defined to return 0 without trapping, so the same call would return ncclSuccess @@ -297,18 +306,22 @@ TEST(NcclDeviceRequirementsMicrotest, GinInboxA2ACreateRequirement_BufferSizeAnd // src/proxy.cc:887. No aarch64 target exists in tools/scripts/test_runner/configs/ today. TEST(NcclDeviceRequirementsMicrotest, GinInboxA2ACreateRequirement_SingleRankTeam_CrashesOnDivideByZero) { #if defined(__x86_64__) - // Not RUN_ISOLATED_TEST (it EXPECT_TRUE's the run); no lambda assertion either, so a future fix - // that makes the call return normally flips executeAllTests() true and fails the EXPECT_FALSE below. - RcclUnitTesting::ProcessIsolatedTestRunner::registerTest("GinInboxA2A_SingleRankTeam_DivideByZero", [] { - DisableCoreDumpsForExpectedCrash(); - ncclTeam_t peers{/*nRanks=*/1, /*rank=*/0, /*stride=*/1}; - ncclGinInboxA2AHandle handle = poisonedHandle(); - ncclDevResourceRequirements_t req = poisonedRequirement(); - // Never expected to return: SIGFPE fires inside idivRcp32 before this line completes. - ncclGinInboxA2ACreateRequirement(peers, /*nBlocks=*/1, /*size_log2=*/7, &handle, &req); - }); - EXPECT_FALSE(RcclUnitTesting::ProcessIsolatedTestRunner::executeAllTests()) - << "expected the isolated child to be killed by SIGFPE; if it now passes, the defect was fixed"; + volatile int nRanksVolatile = 1; + // Field-by-field assignment, not a braced aggregate init: a braced-init-list's top-level commas + // are not protected by the macro's parens and split EXPECT_EXIT's own argument list instead. + EXPECT_EXIT( + { + DisableCoreDumpsForExpectedCrash(); + ncclTeam_t peers; + peers.nRanks = nRanksVolatile; + peers.rank = 0; + peers.stride = 1; + ncclGinInboxA2AHandle handle = poisonedHandle(); + ncclDevResourceRequirements_t req = poisonedRequirement(); + // Never expected to return: SIGFPE fires inside idivRcp32 before this line completes. + ncclGinInboxA2ACreateRequirement(peers, /*nBlocks=*/1, /*size_log2=*/7, &handle, &req); + }, + ::testing::KilledBySignal(SIGFPE), ""); #else GTEST_SKIP() << "divide-by-zero trap is x86_64-specific; AArch64 UDIV returns 0 without trapping"; #endif From 68b9e901a8e68d66b0d09bdd8bd8f28e517139fd Mon Sep 17 00:00:00 2001 From: Alex Duforest Date: Tue, 13 Oct 2026 11:07:17 -0700 Subject: [PATCH 08/26] test(rccl): fix DisableCoreDumpsForExpectedCrash rationale for gtest's fork-style death test Co-Authored-By: Claude --- projects/rccl/test/host/nccl-device-requirements-test.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/rccl/test/host/nccl-device-requirements-test.cc b/projects/rccl/test/host/nccl-device-requirements-test.cc index 742aaaa7c2a5..e045011a11a1 100644 --- a/projects/rccl/test/host/nccl-device-requirements-test.cc +++ b/projects/rccl/test/host/nccl-device-requirements-test.cc @@ -83,8 +83,8 @@ void ExpectNoGinCounterFields(const ncclDevResourceRequirements_t& req) { EXPECT_EQ(req.outGinCounterStart, nullptr); } -// The isolated child re-execs, resetting PR_SET_DUMPABLE, so it must be set again here rather than -// once in the runner. ci-precheckin.json builds Debug with coverage on, so a core dump is sizable. +// gtest's default death-test style forks without re-exec, so clear this inside the death statement: +// only the child is affected. ci-precheckin.json builds Debug with coverage on, so a dump is sizable. void DisableCoreDumpsForExpectedCrash() { #if defined(__linux__) if (prctl(PR_SET_DUMPABLE, 0) != 0) { From f6d697c15da6053fd21b58bb160b78f09c4871eb Mon Sep 17 00:00:00 2001 From: Alex Duforest Date: Tue, 13 Oct 2026 20:01:29 -0700 Subject: [PATCH 09/26] test(rccl): non-identity team rank/stride, an inexact-reciprocal peer row, literal alignment goldens Co-Authored-By: Claude --- .../host/nccl-device-requirements-test.cc | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/projects/rccl/test/host/nccl-device-requirements-test.cc b/projects/rccl/test/host/nccl-device-requirements-test.cc index e045011a11a1..1aba34afeace 100644 --- a/projects/rccl/test/host/nccl-device-requirements-test.cc +++ b/projects/rccl/test/host/nccl-device-requirements-test.cc @@ -12,9 +12,9 @@ // already have GPU-binary coverage in test/DeviceResourceRequirementTests.cpp // (AICOMRCCL-2446); this file ports those same goldens host-only rather than // duplicating the reasoning, since the functions under test are pure host -// computation with no HIP/device calls. Both copies are built and run today -// (test/CMakeLists.txt, ci-precheckin.json) and have already drifted at a few -// lines; retiring the GPU-binary copy once this suite has run clean in CI for +// computation with no HIP/device calls. Both copies are built from test/CMakeLists.txt; the GPU +// copy runs from ci-precheckin.json and this one from test/host/run_host_tests.sh. They have +// already drifted at a few lines; retiring the GPU-binary copy once this suite has run clean in CI for // a while is tracked separately in AICOMRCCL-2912, not done here. #include @@ -260,10 +260,13 @@ TEST(NcclDeviceRequirementsMicrotest, GinInboxA2ACreateRequirement_BufferSizeAnd const std::vector cases = { {"smallest_safe_team", 2, 1, 7, 256, 2052}, // 1*(128+128); 1*4*(1+512) {"larger_team", 5, 2, 8, 768, 4128}, // 2*(128+256); 2*4*(4+512) + // nPeers=3 is the only row whose reciprocal is inexact, so the r >= y correction actually runs. + {"non_pow2_peers", 4, 1, 7, 256, 2060}, // 1*(128+128); 1*4*(3+512) }; for (const Case& c : cases) { SCOPED_TRACE(c.name); - ncclTeam_t peers{c.nRanks, /*rank=*/0, /*stride=*/1}; + // Non-identity rank and stride, so a stray `+ team.rank` or `* team.stride` would show. + ncclTeam_t peers{c.nRanks, /*rank=*/c.nRanks - 1, /*stride=*/4}; ncclGinInboxA2AHandle handle = poisonedHandle(); ncclDevResourceRequirements_t req = poisonedRequirement(); ASSERT_EQ(ncclGinInboxA2ACreateRequirement(peers, c.nBlocks, c.sizeLog2, &handle, &req), ncclSuccess); @@ -348,14 +351,14 @@ TEST(NcclDeviceRequirementsMicrotest, LsaBarrierCreateRequirement_BufferSizeForm }; for (const Case& c : cases) { SCOPED_TRACE(c.name); - ncclTeam_t team{c.nRanks, /*rank=*/0, /*stride=*/1}; + ncclTeam_t team{c.nRanks, /*rank=*/c.nRanks - 1, /*stride=*/4}; ncclLsaBarrierHandle_t handle = poisonedHandle(); ncclDevResourceRequirements_t req = poisonedRequirement(); ASSERT_EQ(ncclLsaBarrierCreateRequirement(team, c.nBarriers, &handle, &req), ncclSuccess); const size_t expected = static_cast(3 * c.nBarriers + c.nBarriers * c.nRanks) * sizeof(uint32_t); EXPECT_EQ(req.bufferSize, expected); EXPECT_EQ(req.bufferSize, c.goldenBufferSize); - EXPECT_EQ(req.bufferAlign, alignof(uint32_t)); + EXPECT_EQ(req.bufferAlign, size_t{4}); // alignof(uint32_t) today; a literal, so a change fails here EXPECT_EQ(req.outBufferHandle, &handle.bufHandle); EXPECT_EQ(handle.nBarriers, c.nBarriers); // The buffer-only sibling of the GIN/CFT barriers below: no GIN signals here. @@ -381,7 +384,7 @@ TEST(NcclDeviceRequirementsMicrotest, GinBarrierCreateRequirement_NullCommAndSig }; for (const Case& c : cases) { SCOPED_TRACE(c.name); - ncclTeam_t team{c.nRanks, /*rank=*/0, /*stride=*/1}; + ncclTeam_t team{c.nRanks, /*rank=*/c.nRanks - 1, /*stride=*/4}; ncclGinBarrierHandle_t handle = poisonedHandle(); ncclDevResourceRequirements_t req = poisonedRequirement(); ASSERT_EQ(ncclGinBarrierCreateRequirement(/*comm=*/nullptr, team, c.nBarriers, &handle, &req), ncclSuccess); @@ -416,14 +419,14 @@ TEST(NcclDeviceRequirementsMicrotest, CftBarrierCreateRequirement_BufferSizeForm }; for (const Case& c : cases) { SCOPED_TRACE(c.name); - ncclTeam_t team{c.nRanks, /*rank=*/0, /*stride=*/1}; + ncclTeam_t team{c.nRanks, /*rank=*/c.nRanks - 1, /*stride=*/4}; ncclCftBarrierHandle_t handle = poisonedHandle(); ncclDevResourceRequirements_t req = poisonedRequirement(); ASSERT_EQ(ncclCftBarrierCreateRequirement(team, c.nBarriers, &handle, &req), ncclSuccess); const size_t expected = static_cast(3 * c.nBarriers + c.nBarriers * c.nRanks) * NCCL_CFT_BARRIER_GRAN; EXPECT_EQ(req.bufferSize, expected); EXPECT_EQ(req.bufferSize, c.goldenBufferSize); - EXPECT_EQ(req.bufferAlign, static_cast(NCCL_CFT_BARRIER_ALIGN)); + EXPECT_EQ(req.bufferAlign, size_t{16}); // NCCL_CFT_BARRIER_ALIGN today; a literal, so a change fails here EXPECT_EQ(req.outBufferHandle, &handle.bufHandle); EXPECT_EQ(handle.nBarriers, c.nBarriers); ExpectNoGinSignalFields(req); From 171f6809e54eb5b37e44b3563365de1031f9d6e3 Mon Sep 17 00:00:00 2001 From: Alex Duforest Date: Mon, 12 Oct 2026 15:02:33 -0700 Subject: [PATCH 10/26] test(rccl): add host microtests for nccl_device/core.cc team helpers --- projects/rccl/test/host/CMakeLists.txt | 39 ++++ projects/rccl/test/host/MICROTEST_README.md | 10 ++ projects/rccl/test/host/nccl-core-test.cc | 188 ++++++++++++++++++++ projects/rccl/test/host/run_host_tests.sh | 1 + 4 files changed, 238 insertions(+) create mode 100644 projects/rccl/test/host/nccl-core-test.cc diff --git a/projects/rccl/test/host/CMakeLists.txt b/projects/rccl/test/host/CMakeLists.txt index 1344ae8b3bf2..f2c9b0d9a9be 100644 --- a/projects/rccl/test/host/CMakeLists.txt +++ b/projects/rccl/test/host/CMakeLists.txt @@ -293,6 +293,33 @@ function(rccl_define_micro_source_lists) ../common/main_altrsmi.cpp PARENT_SCOPE) + # rccl-UnitTestsMicroNcclDeviceCore -- src/nccl_device/core.cc. Its own binary: + # dev_runtime_micro_fakes.cc (ncclTeamWorld/ncclTeamRail/ncclTeamCft/ncclTeamCftMultimem), + # devcomm_fakes.cc (ncclTeamLsa), and gin-host-test.cc (ncclTeamRankToWorld) between them + # fake 6 of this file's 7 functions for rccl-UnitTestsMicro's sake; this target compiles + # core.cc as the real thing under test, so none of those three can share a binary with it. + set(TEST_MICRO_NCCL_DEVICE_CORE_SOURCE_FILES + nccl-core-test.cc + fakes/bootstrap_stubs.cc + fakes/topo_stubs.cc + fakes/transport_stubs.cc + fakes/nccl_stubs.cc + fakes/nccl_fakes.cc + fakes/hip_fakes.cc + fakes/collectives_fakes.cc + fakes/comm_fakes.cc + fakes/dev_runtime_fakes.cc + fakes/env_fakes.cc + fakes/os_fakes.cc + fakes/recorder_fakes.cc + fakes/strongstream_stubs.cc + fakes/tuning_fakes.cc + ${RCCL_HOST_HIPIFY_DIR}/src/misc/argcheck.cc + ${RCCL_HOST_HIPIFY_DIR}/src/misc/archinfo.cc + ${RCCL_HOST_HIPIFY_DIR}/src/misc/utils.cc + ../common/main_altrsmi.cpp + PARENT_SCOPE) + # rccl-UnitTestsMicroSymKernels -- REAL src/sym_kernels.cc + generated host TU. set(TEST_MICRO_SYM_KERNELS_SOURCE_FILES sym-kernels-test.cc @@ -1055,6 +1082,18 @@ rccl_add_micro_binary(rccl-UnitTestsMicroClientSupport CLIENT_SUPPORT_CC_PATH="${RCCL_HOST_HIPIFY_DIR}/src/ras/client_support.cc" CATEGORIES_YAML "${CMAKE_CURRENT_SOURCE_DIR}/../test_categories_micro.yaml") +# --------------------------------------------------------------------------- +# rccl-UnitTestsMicroNcclDeviceCore +# --------------------------------------------------------------------------- +rccl_add_micro_binary(rccl-UnitTestsMicroNcclDeviceCore + SOURCES ${TEST_MICRO_NCCL_DEVICE_CORE_SOURCE_FILES} + EXTRA_INCLUDES + ${RCCL_HOST_HIPIFY_DIR}/src/include/nccl_device # core.cc includes "nccl_device/impl/core__funcs.h" unqualified + EXTRA_DEFS + ${_MICRO_MODE_DEFS} + CORE_CC_PATH="${RCCL_HOST_HIPIFY_DIR}/src/nccl_device/core.cc" + CATEGORIES_YAML "${CMAKE_CURRENT_SOURCE_DIR}/../test_categories_micro.yaml") + # --------------------------------------------------------------------------- # rccl-UnitTestsMicroSymKernels # --------------------------------------------------------------------------- diff --git a/projects/rccl/test/host/MICROTEST_README.md b/projects/rccl/test/host/MICROTEST_README.md index 1ff07e6588f5..2b7a1d532ed5 100644 --- a/projects/rccl/test/host/MICROTEST_README.md +++ b/projects/rccl/test/host/MICROTEST_README.md @@ -248,6 +248,16 @@ export colliding non-`static` symbols; otherwise a unit needs its own binary: guard-less positional pair `libc_seam.h` / `libc_seam_undef.h` must bracket the production source include, after system headers and before test helpers. CMake pins the copied `ncclFuncStr` table to `src/init.cc`. +- **`rccl-UnitTestsMicroNcclDeviceCore`**: `src/nccl_device/core.cc` (via + `CORE_CC_PATH`, from `nccl-core-test.cc`); suite `NcclCoreMicrotest.*`. Its + own binary because `rccl-UnitTestsMicro`'s `fakes/dev_runtime_micro_fakes.cc` + (`ncclTeamWorld`/`ncclTeamRail`/`ncclTeamCft`/`ncclTeamCftMultimem`), + `fakes/devcomm_fakes.cc` (`ncclTeamLsa`), and `gin-host-test.cc` + (`ncclTeamRankToWorld`) between them fake 6 of this file's 7 functions. + Links the lightweight `fakes/dev_runtime_fakes.cc` (hookable + `ncclDevrInitOnce`, default `ncclSuccess`) rather than the real + `dev_runtime.cc` -- core.cc only calls `ncclDevrInitOnce` itself, so the full + production file and its much larger fake surface aren't needed. - **`rccl-UnitTestsMicroSymKernels`** — the REAL `src/sym_kernels.cc` (via `SYM_KERNELS_CC_PATH`, from `sym-kernels-test.cc`), compiled together with the GENERATED `sym_kernels_host.cc` it calls into; suites `SymKernelMicrotest.*`, diff --git a/projects/rccl/test/host/nccl-core-test.cc b/projects/rccl/test/host/nccl-core-test.cc new file mode 100644 index 000000000000..bf57a259eda7 --- /dev/null +++ b/projects/rccl/test/host/nccl-core-test.cc @@ -0,0 +1,188 @@ +/************************************************************************* + * Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +// Host-only microtests for src/nccl_device/core.cc: the ncclTeam* family that +// maps a communicator onto its world/LSA/CFT/rail views, plus the +// world/LSA rank-translation helpers. Every function but ncclTeamRankToWorld +// gates through ncclDevrInitOnce first, so the fake's hook lets us drive both +// the success path (devrState fields read back verbatim) and the failure +// path (all five return a poisoned/zeroed answer rather than touching +// uninitialised devrState). + +#include + +#include + +#include "fakes/dev_runtime_fakes.h" + +#include CORE_CC_PATH + +namespace { + +class NcclCoreMicrotest : public ::testing::Test { +protected: + std::unique_ptr commStorage; + ncclComm* comm = nullptr; + + void SetUp() override { + commStorage = std::make_unique(); // value-initialised: POD members zeroed + comm = commStorage.get(); + } + + void TearDown() override { ResetDevRuntimeFakes(); } +}; + +} // namespace + +TEST_F(NcclCoreMicrotest, TeamWorld_ReturnsCommNRanksAndRankWithStrideOne) { + comm->nRanks = 4; + comm->rank = 2; + ncclTeam_t team = ncclTeamWorld(comm); + EXPECT_EQ(team.nRanks, 4); + EXPECT_EQ(team.rank, 2); + EXPECT_EQ(team.stride, 1); +} + +TEST_F(NcclCoreMicrotest, TeamLsa_SuccessPath_ReadsLsaSizeAndSelfFromDevrState) { + comm->devrState.lsaSize = 8; + comm->devrState.lsaSelf = 3; + ncclTeam_t team = ncclTeamLsa(comm); + EXPECT_EQ(team.nRanks, 8); + EXPECT_EQ(team.rank, 3); + EXPECT_EQ(team.stride, 1); +} + +TEST_F(NcclCoreMicrotest, TeamLsa_InitOnceFails_ReturnsZeroedTeamWithoutReadingDevrState) { + // Poison lsaSize/lsaSelf so that if the function ignored the failure and + // read them anyway, the zero-check below would fail loudly. + comm->devrState.lsaSize = 99; + comm->devrState.lsaSelf = 99; + g_devrInitOnce = [](ncclComm*) { return ncclSystemError; }; + ncclTeam_t team = ncclTeamLsa(comm); + EXPECT_EQ(team.nRanks, 0); + EXPECT_EQ(team.rank, 0); + EXPECT_EQ(team.stride, 0); +} + +TEST_F(NcclCoreMicrotest, TeamCftMultimem_SuccessPath_ReadsCftMcSizeAndSelfFromDevrState) { + comm->devrState.cftMcSize = 16; + comm->devrState.cftMcSelf = 5; + ncclTeam_t team = ncclTeamCftMultimem(comm); + EXPECT_EQ(team.nRanks, 16); + EXPECT_EQ(team.rank, 5); + EXPECT_EQ(team.stride, 1); +} + +TEST_F(NcclCoreMicrotest, TeamCftMultimem_InitOnceFails_ReturnsZeroedTeam) { + comm->devrState.cftMcSize = 99; + comm->devrState.cftMcSelf = 99; + g_devrInitOnce = [](ncclComm*) { return ncclSystemError; }; + ncclTeam_t team = ncclTeamCftMultimem(comm); + EXPECT_EQ(team.nRanks, 0); + EXPECT_EQ(team.rank, 0); + EXPECT_EQ(team.stride, 0); +} + +TEST_F(NcclCoreMicrotest, TeamRail_SuccessPath_DividesCommNRanksAndRankByLsaSize) { + comm->nRanks = 16; + comm->rank = 10; + comm->devrState.lsaSize = 4; + ncclTeam_t team = ncclTeamRail(comm); + EXPECT_EQ(team.nRanks, 4); // 16 / 4 + EXPECT_EQ(team.rank, 2); // 10 / 4 + EXPECT_EQ(team.stride, 4); // == lsaSize +} + +TEST_F(NcclCoreMicrotest, TeamRail_InitOnceFails_ReturnsZeroedTeamWithoutDividingByLsaSize) { + // If the failure branch were missing, this would divide by zero instead. + comm->nRanks = 16; + comm->rank = 10; + comm->devrState.lsaSize = 0; + g_devrInitOnce = [](ncclComm*) { return ncclSystemError; }; + ncclTeam_t team = ncclTeamRail(comm); + EXPECT_EQ(team.nRanks, 0); + EXPECT_EQ(team.rank, 0); + EXPECT_EQ(team.stride, 0); +} + +TEST_F(NcclCoreMicrotest, TeamCft_FlatMode_ReturnsFlatTeamFromCftSizeAndSelf) { + comm->devrState.cftSize = 12; + comm->devrState.cftSelf = 7; + ncclTeam_t team = ncclTeamCft(comm, NCCL_CFT_TEAM_FLAT); + EXPECT_EQ(team.nRanks, 12); + EXPECT_EQ(team.rank, 7); + EXPECT_EQ(team.stride, 1); +} + +TEST_F(NcclCoreMicrotest, TeamCft_HierMultimemMode_OuterFactorsTheFlatTeamByCftMcSize) { + // ncclTeamOuterFactor(flat, innerSize): nRanks = flat.nRanks/innerSize, + // rank = flat.rank/innerSize, stride = flat.stride*innerSize (== 1*innerSize + // here since the flat team's stride is always 1). + comm->devrState.cftSize = 12; + comm->devrState.cftSelf = 7; + comm->devrState.cftMcSize = 3; + ncclTeam_t team = ncclTeamCft(comm, NCCL_CFT_TEAM_HIER_MULTIMEM); + EXPECT_EQ(team.nRanks, 4); // 12 / 3 + EXPECT_EQ(team.rank, 2); // 7 / 3 + EXPECT_EQ(team.stride, 3); // 1 * 3 +} + +TEST_F(NcclCoreMicrotest, TeamCft_HierLsaMode_OuterFactorsTheFlatTeamByLsaSize) { + comm->devrState.cftSize = 20; + comm->devrState.cftSelf = 9; + comm->devrState.lsaSize = 5; + ncclTeam_t team = ncclTeamCft(comm, NCCL_CFT_TEAM_HIER_LSA); + EXPECT_EQ(team.nRanks, 4); // 20 / 5 + EXPECT_EQ(team.rank, 1); // 9 / 5 + EXPECT_EQ(team.stride, 5); // 1 * 5 +} + +TEST_F(NcclCoreMicrotest, TeamCft_UnrecognisedMode_ReturnsZeroedTeam) { + comm->devrState.cftSize = 12; + comm->devrState.cftSelf = 7; + ncclTeam_t team = ncclTeamCft(comm, static_cast(99)); + EXPECT_EQ(team.nRanks, 0); + EXPECT_EQ(team.rank, 0); + EXPECT_EQ(team.stride, 0); +} + +TEST_F(NcclCoreMicrotest, TeamCft_InitOnceFails_ReturnsZeroedTeamRegardlessOfMode) { + comm->devrState.cftSize = 99; + comm->devrState.cftSelf = 99; + g_devrInitOnce = [](ncclComm*) { return ncclSystemError; }; + for (ncclCftTeamMode_t mode : {NCCL_CFT_TEAM_FLAT, NCCL_CFT_TEAM_HIER_MULTIMEM, NCCL_CFT_TEAM_HIER_LSA}) { + ncclTeam_t team = ncclTeamCft(comm, mode); + EXPECT_EQ(team.nRanks, 0); + EXPECT_EQ(team.rank, 0); + EXPECT_EQ(team.stride, 0); + } +} + +TEST_F(NcclCoreMicrotest, TeamRankToWorld_MapsATeamLocalRankBackToItsWorldRankViaStride) { + // Does not gate on ncclDevrInitOnce at all -- pure arithmetic over comm->rank + // and the caller-supplied team, so no fake wiring is needed for this one. + comm->rank = 10; + ncclTeam_t team{/*nRanks=*/4, /*rank=*/2, /*stride=*/3}; + // comm->rank + (rank - team.rank) * team.stride + EXPECT_EQ(ncclTeamRankToWorld(comm, team, team.rank), 10); // self: offset 0 + EXPECT_EQ(ncclTeamRankToWorld(comm, team, team.rank + 1), 13); + EXPECT_EQ(ncclTeamRankToWorld(comm, team, team.rank - 1), 7); +} + +TEST_F(NcclCoreMicrotest, TeamRankToLsa_SuccessPath_MapsATeamLocalRankToAnLsaRankViaLsaSelf) { + comm->devrState.lsaSelf = 5; + ncclTeam_t team{/*nRanks=*/4, /*rank=*/2, /*stride=*/3}; + // devrState.lsaSelf + (rank - team.rank) * team.stride + EXPECT_EQ(ncclTeamRankToLsa(comm, team, team.rank), 5); + EXPECT_EQ(ncclTeamRankToLsa(comm, team, team.rank + 1), 8); +} + +TEST_F(NcclCoreMicrotest, TeamRankToLsa_InitOnceFails_ReturnsNegativeOneWithoutReadingLsaSelf) { + comm->devrState.lsaSelf = 99; + g_devrInitOnce = [](ncclComm*) { return ncclSystemError; }; + ncclTeam_t team{/*nRanks=*/4, /*rank=*/2, /*stride=*/3}; + EXPECT_EQ(ncclTeamRankToLsa(comm, team, team.rank), -1); +} diff --git a/projects/rccl/test/host/run_host_tests.sh b/projects/rccl/test/host/run_host_tests.sh index ec702f4f62b1..b0400485ebd8 100755 --- a/projects/rccl/test/host/run_host_tests.sh +++ b/projects/rccl/test/host/run_host_tests.sh @@ -167,6 +167,7 @@ do_host_tests() { "rccl-UnitTestsMicroTaskPrep:$SCRIPT_DIR/host_tests_micro_taskprep.xml" "rccl-UnitTestsMicroCe:$SCRIPT_DIR/host_tests_micro_ce.xml" "rccl-UnitTestsMicroClientSupport:$SCRIPT_DIR/host_tests_micro_client_support.xml" + "rccl-UnitTestsMicroNcclDeviceCore:$SCRIPT_DIR/host_tests_micro_nccl_device_core.xml" "rccl-UnitTestsMicroGinHost:$SCRIPT_DIR/host_tests_micro_gin_host.xml" "rccl-UnitTestsMicroDiagnostics:$SCRIPT_DIR/host_tests_micro_diagnostics.xml" ) From 12d94130bac9ba64ca45313362940923cd94d452 Mon Sep 17 00:00:00 2001 From: Alex Duforest Date: Mon, 12 Oct 2026 22:18:49 -0700 Subject: [PATCH 11/26] test(rccl): address ARGUS findings on core.cc microtests - TeamRail_SuccessPath: avoid lsaSize^2 == nRanks so dividing by lsaSize can't be mistaken for reading it back directly. - HierMultimem/HierLsa mode tests: give the other mode's field a distinct non-zero value, so a swapped arm divides by a real (wrong) number instead of silently dividing by zero. - TeamRankToLsa: add a negative-offset case. - Fix header comment: 5 of 7 functions gate on ncclDevrInitOnce, not 6. - Drop the nccl_device EXTRA_INCLUDES entry: verified empirically (binary still builds and all tests pass without it) that core.cc does not need it; the stated reason didn't match core.cc's own include spelling. --- projects/rccl/test/host/CMakeLists.txt | 2 -- projects/rccl/test/host/nccl-core-test.cc | 27 +++++++++++++++++------ 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/projects/rccl/test/host/CMakeLists.txt b/projects/rccl/test/host/CMakeLists.txt index f2c9b0d9a9be..9c7717f995ba 100644 --- a/projects/rccl/test/host/CMakeLists.txt +++ b/projects/rccl/test/host/CMakeLists.txt @@ -1087,8 +1087,6 @@ rccl_add_micro_binary(rccl-UnitTestsMicroClientSupport # --------------------------------------------------------------------------- rccl_add_micro_binary(rccl-UnitTestsMicroNcclDeviceCore SOURCES ${TEST_MICRO_NCCL_DEVICE_CORE_SOURCE_FILES} - EXTRA_INCLUDES - ${RCCL_HOST_HIPIFY_DIR}/src/include/nccl_device # core.cc includes "nccl_device/impl/core__funcs.h" unqualified EXTRA_DEFS ${_MICRO_MODE_DEFS} CORE_CC_PATH="${RCCL_HOST_HIPIFY_DIR}/src/nccl_device/core.cc" diff --git a/projects/rccl/test/host/nccl-core-test.cc b/projects/rccl/test/host/nccl-core-test.cc index bf57a259eda7..854c84dc24f9 100644 --- a/projects/rccl/test/host/nccl-core-test.cc +++ b/projects/rccl/test/host/nccl-core-test.cc @@ -6,11 +6,11 @@ // Host-only microtests for src/nccl_device/core.cc: the ncclTeam* family that // maps a communicator onto its world/LSA/CFT/rail views, plus the -// world/LSA rank-translation helpers. Every function but ncclTeamRankToWorld -// gates through ncclDevrInitOnce first, so the fake's hook lets us drive both -// the success path (devrState fields read back verbatim) and the failure -// path (all five return a poisoned/zeroed answer rather than touching -// uninitialised devrState). +// world/LSA rank-translation helpers. Five of the seven gate through +// ncclDevrInitOnce first (all but ncclTeamWorld and ncclTeamRankToWorld), so +// the fake's hook lets us drive both the success path (devrState fields read +// back verbatim) and the failure path (all five return a poisoned/zeroed +// answer rather than touching uninitialised devrState). #include @@ -87,11 +87,14 @@ TEST_F(NcclCoreMicrotest, TeamCftMultimem_InitOnceFails_ReturnsZeroedTeam) { } TEST_F(NcclCoreMicrotest, TeamRail_SuccessPath_DividesCommNRanksAndRankByLsaSize) { - comm->nRanks = 16; + // nRanks=24, lsaSize=4 deliberately avoids lsaSize^2 == nRanks: with the + // earlier 16/4 the expected team.nRanks (4) was indistinguishable from a + // mutation that replaced the division with a bare `lsaSize` readback. + comm->nRanks = 24; comm->rank = 10; comm->devrState.lsaSize = 4; ncclTeam_t team = ncclTeamRail(comm); - EXPECT_EQ(team.nRanks, 4); // 16 / 4 + EXPECT_EQ(team.nRanks, 6); // 24 / 4 EXPECT_EQ(team.rank, 2); // 10 / 4 EXPECT_EQ(team.stride, 4); // == lsaSize } @@ -121,9 +124,15 @@ TEST_F(NcclCoreMicrotest, TeamCft_HierMultimemMode_OuterFactorsTheFlatTeamByCftM // ncclTeamOuterFactor(flat, innerSize): nRanks = flat.nRanks/innerSize, // rank = flat.rank/innerSize, stride = flat.stride*innerSize (== 1*innerSize // here since the flat team's stride is always 1). + // + // lsaSize is set to a distinct non-zero value (the HIER_LSA test's own + // divisor) so that if the two mode arms in production were swapped, this + // test would divide by a real, wrong number and get a mis-answer to diff + // against -- not silently divide by zero and crash identically either way. comm->devrState.cftSize = 12; comm->devrState.cftSelf = 7; comm->devrState.cftMcSize = 3; + comm->devrState.lsaSize = 2; ncclTeam_t team = ncclTeamCft(comm, NCCL_CFT_TEAM_HIER_MULTIMEM); EXPECT_EQ(team.nRanks, 4); // 12 / 3 EXPECT_EQ(team.rank, 2); // 7 / 3 @@ -131,9 +140,12 @@ TEST_F(NcclCoreMicrotest, TeamCft_HierMultimemMode_OuterFactorsTheFlatTeamByCftM } TEST_F(NcclCoreMicrotest, TeamCft_HierLsaMode_OuterFactorsTheFlatTeamByLsaSize) { + // cftMcSize is likewise set to a distinct non-zero value for the same + // swapped-arm reason as the HIER_MULTIMEM test above. comm->devrState.cftSize = 20; comm->devrState.cftSelf = 9; comm->devrState.lsaSize = 5; + comm->devrState.cftMcSize = 3; ncclTeam_t team = ncclTeamCft(comm, NCCL_CFT_TEAM_HIER_LSA); EXPECT_EQ(team.nRanks, 4); // 20 / 5 EXPECT_EQ(team.rank, 1); // 9 / 5 @@ -178,6 +190,7 @@ TEST_F(NcclCoreMicrotest, TeamRankToLsa_SuccessPath_MapsATeamLocalRankToAnLsaRan // devrState.lsaSelf + (rank - team.rank) * team.stride EXPECT_EQ(ncclTeamRankToLsa(comm, team, team.rank), 5); EXPECT_EQ(ncclTeamRankToLsa(comm, team, team.rank + 1), 8); + EXPECT_EQ(ncclTeamRankToLsa(comm, team, team.rank - 1), 2); // 5 + (-1)*3 } TEST_F(NcclCoreMicrotest, TeamRankToLsa_InitOnceFails_ReturnsNegativeOneWithoutReadingLsaSelf) { From efc642106b00aecceae120e90c53952a7cb470bb Mon Sep 17 00:00:00 2001 From: Alex Duforest Date: Tue, 13 Oct 2026 11:00:26 -0700 Subject: [PATCH 12/26] test(rccl): pick a TeamRail rank where / and % differ rank=10 with lsaSize=4 gives 2 for both 10/4 and 10%4, so core.cc's rank division was unpinned. rank=9 keeps the expected 2 and makes % give 1. Co-Authored-By: Claude --- projects/rccl/test/host/nccl-core-test.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/projects/rccl/test/host/nccl-core-test.cc b/projects/rccl/test/host/nccl-core-test.cc index 854c84dc24f9..269cbf31a793 100644 --- a/projects/rccl/test/host/nccl-core-test.cc +++ b/projects/rccl/test/host/nccl-core-test.cc @@ -87,15 +87,15 @@ TEST_F(NcclCoreMicrotest, TeamCftMultimem_InitOnceFails_ReturnsZeroedTeam) { } TEST_F(NcclCoreMicrotest, TeamRail_SuccessPath_DividesCommNRanksAndRankByLsaSize) { - // nRanks=24, lsaSize=4 deliberately avoids lsaSize^2 == nRanks: with the - // earlier 16/4 the expected team.nRanks (4) was indistinguishable from a - // mutation that replaced the division with a bare `lsaSize` readback. + // Constants chosen so each field separates `/` from its look-alikes: nRanks 24/4=6 is + // neither lsaSize (4) nor 24%4=0, and rank 9/4=2 differs from 9%4=1 (ncclTeamInnerFactor + // uses `%` on the same field, so that mix-up is the realistic one). comm->nRanks = 24; - comm->rank = 10; + comm->rank = 9; comm->devrState.lsaSize = 4; ncclTeam_t team = ncclTeamRail(comm); EXPECT_EQ(team.nRanks, 6); // 24 / 4 - EXPECT_EQ(team.rank, 2); // 10 / 4 + EXPECT_EQ(team.rank, 2); // 9 / 4 EXPECT_EQ(team.stride, 4); // == lsaSize } From 854da5023deef703b87d5469f6f2a74875bcf84b Mon Sep 17 00:00:00 2001 From: Alex Duforest Date: Tue, 13 Oct 2026 12:42:10 -0700 Subject: [PATCH 13/26] test(rccl): pin ncclTeamWorld/RankToWorld staying ungated, fix fake-ownership wording, explicit yaml pattern Co-Authored-By: Claude --- projects/rccl/test/host/CMakeLists.txt | 9 +++++---- projects/rccl/test/host/MICROTEST_README.md | 7 ++++--- projects/rccl/test/host/nccl-core-test.cc | 14 ++++++++++++++ projects/rccl/test/test_categories_micro.yaml | 2 ++ 4 files changed, 25 insertions(+), 7 deletions(-) diff --git a/projects/rccl/test/host/CMakeLists.txt b/projects/rccl/test/host/CMakeLists.txt index 9c7717f995ba..e3d99bc76fd1 100644 --- a/projects/rccl/test/host/CMakeLists.txt +++ b/projects/rccl/test/host/CMakeLists.txt @@ -294,10 +294,11 @@ function(rccl_define_micro_source_lists) PARENT_SCOPE) # rccl-UnitTestsMicroNcclDeviceCore -- src/nccl_device/core.cc. Its own binary: - # dev_runtime_micro_fakes.cc (ncclTeamWorld/ncclTeamRail/ncclTeamCft/ncclTeamCftMultimem), - # devcomm_fakes.cc (ncclTeamLsa), and gin-host-test.cc (ncclTeamRankToWorld) between them - # fake 6 of this file's 7 functions for rccl-UnitTestsMicro's sake; this target compiles - # core.cc as the real thing under test, so none of those three can share a binary with it. + # dev_runtime_micro_fakes.cc (ncclTeamWorld/ncclTeamRail/ncclTeamCft/ncclTeamCftMultimem) + # and devcomm_fakes.cc (ncclTeamLsa) fake 5 of this file's 7 for rccl-UnitTestsMicro, + # and gin-host-test.cc fakes ncclTeamWorld/ncclTeamRail/ncclTeamRankToWorld for + # rccl-UnitTestsMicroGinHost; this target compiles core.cc as the real thing under + # test, so none of those three can share a binary with it. set(TEST_MICRO_NCCL_DEVICE_CORE_SOURCE_FILES nccl-core-test.cc fakes/bootstrap_stubs.cc diff --git a/projects/rccl/test/host/MICROTEST_README.md b/projects/rccl/test/host/MICROTEST_README.md index 2b7a1d532ed5..5e7f20eeb897 100644 --- a/projects/rccl/test/host/MICROTEST_README.md +++ b/projects/rccl/test/host/MICROTEST_README.md @@ -251,9 +251,10 @@ export colliding non-`static` symbols; otherwise a unit needs its own binary: - **`rccl-UnitTestsMicroNcclDeviceCore`**: `src/nccl_device/core.cc` (via `CORE_CC_PATH`, from `nccl-core-test.cc`); suite `NcclCoreMicrotest.*`. Its own binary because `rccl-UnitTestsMicro`'s `fakes/dev_runtime_micro_fakes.cc` - (`ncclTeamWorld`/`ncclTeamRail`/`ncclTeamCft`/`ncclTeamCftMultimem`), - `fakes/devcomm_fakes.cc` (`ncclTeamLsa`), and `gin-host-test.cc` - (`ncclTeamRankToWorld`) between them fake 6 of this file's 7 functions. + (`ncclTeamWorld`/`ncclTeamRail`/`ncclTeamCft`/`ncclTeamCftMultimem`) and + `fakes/devcomm_fakes.cc` (`ncclTeamLsa`) fake 5 of this file's 7 functions, and + `gin-host-test.cc` fakes `ncclTeamWorld`/`ncclTeamRail`/`ncclTeamRankToWorld` + for `rccl-UnitTestsMicroGinHost`. Links the lightweight `fakes/dev_runtime_fakes.cc` (hookable `ncclDevrInitOnce`, default `ncclSuccess`) rather than the real `dev_runtime.cc` -- core.cc only calls `ncclDevrInitOnce` itself, so the full diff --git a/projects/rccl/test/host/nccl-core-test.cc b/projects/rccl/test/host/nccl-core-test.cc index 269cbf31a793..e706a239195a 100644 --- a/projects/rccl/test/host/nccl-core-test.cc +++ b/projects/rccl/test/host/nccl-core-test.cc @@ -46,6 +46,20 @@ TEST_F(NcclCoreMicrotest, TeamWorld_ReturnsCommNRanksAndRankWithStrideOne) { EXPECT_EQ(team.stride, 1); } +TEST_F(NcclCoreMicrotest, TeamWorldAndRankToWorld_IgnoreAFailingInitOnce) { + // The only two of the seven that don't gate on ncclDevrInitOnce: with it + // failing, both must still return real values rather than a zeroed/-1 answer. + g_devrInitOnce = [](ncclComm*) { return ncclSystemError; }; + comm->nRanks = 4; + comm->rank = 2; + ncclTeam_t world = ncclTeamWorld(comm); + EXPECT_EQ(world.nRanks, 4); + EXPECT_EQ(world.rank, 2); + EXPECT_EQ(world.stride, 1); + ncclTeam_t team{/*nRanks=*/4, /*rank=*/2, /*stride=*/3}; + EXPECT_EQ(ncclTeamRankToWorld(comm, team, team.rank + 1), 5); // 2 + (3 - 2) * 3 +} + TEST_F(NcclCoreMicrotest, TeamLsa_SuccessPath_ReadsLsaSizeAndSelfFromDevrState) { comm->devrState.lsaSize = 8; comm->devrState.lsaSelf = 3; diff --git a/projects/rccl/test/test_categories_micro.yaml b/projects/rccl/test/test_categories_micro.yaml index a940ce5f4f7c..dc6b303c9584 100644 --- a/projects/rccl/test/test_categories_micro.yaml +++ b/projects/rccl/test/test_categories_micro.yaml @@ -114,6 +114,8 @@ test_categories: # ce_coll.cc (rccl-UnitTestsMicroCe) - "CeCollMicrotest.*" - "LaunchPaths/CeAlltoAllvSyncMicrotest.*" + # nccl_device/core.cc (rccl-UnitTestsMicroNcclDeviceCore) + - "NcclCoreMicrotest.*" # dev_runtime.cc - "Alloc*" - "Comm*" From f648bbde97ddbefebc4f917f6463119ac7a24ce8 Mon Sep 17 00:00:00 2001 From: Alex Duforest Date: Mon, 12 Oct 2026 15:18:18 -0700 Subject: [PATCH 14/26] test(rccl): add host microtests for src/cft_dev_runtime.cc --- projects/rccl/test/host/CMakeLists.txt | 46 ++ projects/rccl/test/host/MICROTEST_README.md | 25 + .../test/host/fakes/cft_dev_runtime_fakes.cc | 68 +++ .../test/host/fakes/cft_dev_runtime_fakes.h | 51 ++ .../test/host/nccl-cft-dev-runtime-test.cc | 512 ++++++++++++++++++ projects/rccl/test/host/run_host_tests.sh | 1 + 6 files changed, 703 insertions(+) create mode 100644 projects/rccl/test/host/fakes/cft_dev_runtime_fakes.cc create mode 100644 projects/rccl/test/host/fakes/cft_dev_runtime_fakes.h create mode 100644 projects/rccl/test/host/nccl-cft-dev-runtime-test.cc diff --git a/projects/rccl/test/host/CMakeLists.txt b/projects/rccl/test/host/CMakeLists.txt index 1344ae8b3bf2..6933fed44290 100644 --- a/projects/rccl/test/host/CMakeLists.txt +++ b/projects/rccl/test/host/CMakeLists.txt @@ -293,6 +293,39 @@ function(rccl_define_micro_source_lists) ../common/main_altrsmi.cpp PARENT_SCOPE) + # rccl-UnitTestsMicroNcclDeviceCft -- src/nccl_device/core.cc + src/cft_dev_runtime.cc. + # Its own binary: rccl-UnitTestsMicro's fakes/dev_runtime_micro_fakes.cc, fakes/devcomm_fakes.cc, + # and gin-host-test.cc between them fake 6 of core.cc's 7 functions. core.cc is compiled for + # real here (not faked) because cft_dev_runtime.cc's ncclGet*DeviceLeInfo functions call + # ncclTeamCft/ncclTeamCftMultimem, and faking those would just re-derive core.cc's own + # already-tested logic. Links the lightweight fakes/dev_runtime_fakes.cc (hookable + # ncclDevrInitOnce) for core.cc's sake, and the new fakes/cft_dev_runtime_fakes.cc for + # cft_dev_runtime.cc's own dev_runtime.cc callees (computeLsaSize, symTeamObtain, + # findCommAndHostWindowFromDeviceWindow, ncclDevrWinRegEnabled) -- not the real + # dev_runtime.cc, which neither production file needs beyond those four symbols. + set(TEST_MICRO_NCCL_DEVICE_CFT_SOURCE_FILES + nccl-cft-dev-runtime-test.cc + fakes/cft_dev_runtime_fakes.cc + fakes/bootstrap_stubs.cc + fakes/topo_stubs.cc + fakes/transport_stubs.cc + fakes/nccl_stubs.cc + fakes/nccl_fakes.cc + fakes/hip_fakes.cc + fakes/collectives_fakes.cc + fakes/comm_fakes.cc + fakes/dev_runtime_fakes.cc + fakes/env_fakes.cc + fakes/os_fakes.cc + fakes/recorder_fakes.cc + fakes/strongstream_stubs.cc + fakes/tuning_fakes.cc + ${RCCL_HOST_HIPIFY_DIR}/src/misc/argcheck.cc + ${RCCL_HOST_HIPIFY_DIR}/src/misc/archinfo.cc + ${RCCL_HOST_HIPIFY_DIR}/src/misc/utils.cc + ../common/main_altrsmi.cpp + PARENT_SCOPE) + # rccl-UnitTestsMicroSymKernels -- REAL src/sym_kernels.cc + generated host TU. set(TEST_MICRO_SYM_KERNELS_SOURCE_FILES sym-kernels-test.cc @@ -1055,6 +1088,19 @@ rccl_add_micro_binary(rccl-UnitTestsMicroClientSupport CLIENT_SUPPORT_CC_PATH="${RCCL_HOST_HIPIFY_DIR}/src/ras/client_support.cc" CATEGORIES_YAML "${CMAKE_CURRENT_SOURCE_DIR}/../test_categories_micro.yaml") +# --------------------------------------------------------------------------- +# rccl-UnitTestsMicroNcclDeviceCft +# --------------------------------------------------------------------------- +rccl_add_micro_binary(rccl-UnitTestsMicroNcclDeviceCft + SOURCES ${TEST_MICRO_NCCL_DEVICE_CFT_SOURCE_FILES} + EXTRA_INCLUDES + ${RCCL_HOST_HIPIFY_DIR}/src/include/nccl_device # core.cc includes "nccl_device/impl/core__funcs.h" unqualified + EXTRA_DEFS + ${_MICRO_MODE_DEFS} + CORE_CC_PATH="${RCCL_HOST_HIPIFY_DIR}/src/nccl_device/core.cc" + CFT_DEV_RUNTIME_CC_PATH="${RCCL_HOST_HIPIFY_DIR}/src/cft_dev_runtime.cc" + CATEGORIES_YAML "${CMAKE_CURRENT_SOURCE_DIR}/../test_categories_micro.yaml") + # --------------------------------------------------------------------------- # rccl-UnitTestsMicroSymKernels # --------------------------------------------------------------------------- diff --git a/projects/rccl/test/host/MICROTEST_README.md b/projects/rccl/test/host/MICROTEST_README.md index 1ff07e6588f5..e7ccc0e6bccf 100644 --- a/projects/rccl/test/host/MICROTEST_README.md +++ b/projects/rccl/test/host/MICROTEST_README.md @@ -248,6 +248,31 @@ export colliding non-`static` symbols; otherwise a unit needs its own binary: guard-less positional pair `libc_seam.h` / `libc_seam_undef.h` must bracket the production source include, after system headers and before test helpers. CMake pins the copied `ncclFuncStr` table to `src/init.cc`. +- **`rccl-UnitTestsMicroNcclDeviceCft`**: `src/nccl_device/core.cc` (via + `CORE_CC_PATH`) + `src/cft_dev_runtime.cc` (via `CFT_DEV_RUNTIME_CC_PATH`, + from `nccl-cft-dev-runtime-test.cc`); suites `NcclCftDevRuntimeMicrotest.*`, + `NcclGetMultimemDeviceLeInfoTest.*`, `NcclGetCftDeviceLeInfoTest.*`, + `NcclGetPeerDeviceLeInfoTest.*` (all caught by the `Nccl*` pattern). Its own + binary for the same reason as `rccl-UnitTestsMicroNcclDeviceCore`: 6 of + core.cc's 7 functions are faked elsewhere in `rccl-UnitTestsMicro`. core.cc + is compiled for real here too (not faked) because + `ncclGetMultimemDeviceLeInfo`/`ncclGetCftDeviceLeInfo`/`ncclGetPeerDeviceLeInfo` + call `ncclTeamCft`/`ncclTeamCftMultimem`. Links `fakes/dev_runtime_fakes.cc` + for core.cc's `ncclDevrInitOnce`, and the new `fakes/cft_dev_runtime_fakes.cc` + for `cft_dev_runtime.cc`'s own `dev_runtime.cc` callees (`computeLsaSize`, + `symTeamObtain`, `findCommAndHostWindowFromDeviceWindow`, + `ncclDevrWinRegEnabled`) -- not the real `dev_runtime.cc`, which neither + production file needs beyond those four symbols. This build's `CUDA_VERSION` + is below 13030, so the CUDA logical-endpoint driver calls throughout + `cft_dev_runtime.cc` are compiled out (no ROCm equivalent exists); the tests + cover what remains: the zero-and-succeed body of `ncclGpuCftSupport`, the + host-side branch logic of `computeCftSize`/`computeCftMcSize`, the + condition-evaluation (now-empty) bodies of `symBindTeamLe`/`symUnbindTeamLe`, + the trivial-failure stand-ins for `symTeamObtainUcLe`/`symTeamObtainMcLe`, + and the three unconditional `ncclGet*DeviceLeInfo` query functions. Lines are + 100% covered; regions/branches have two residual, well-understood gaps + documented in AICOMRCCL-2825 PR4's description (`ncclParamCftEnable`, and + `NCCLCHECK`'s own macro-internal branches). - **`rccl-UnitTestsMicroSymKernels`** — the REAL `src/sym_kernels.cc` (via `SYM_KERNELS_CC_PATH`, from `sym-kernels-test.cc`), compiled together with the GENERATED `sym_kernels_host.cc` it calls into; suites `SymKernelMicrotest.*`, diff --git a/projects/rccl/test/host/fakes/cft_dev_runtime_fakes.cc b/projects/rccl/test/host/fakes/cft_dev_runtime_fakes.cc new file mode 100644 index 000000000000..62ed86c48bb1 --- /dev/null +++ b/projects/rccl/test/host/fakes/cft_dev_runtime_fakes.cc @@ -0,0 +1,68 @@ +/************************************************************************* + * Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +// See cft_dev_runtime_fakes.h. + +#include "cft_dev_runtime_fakes.h" + +#include "dev_runtime_internal.h" +#include "fail_loud.h" +#include "signature-drift.h" + +ASSERT_HOOK_MATCHES_PROD(g_computeLsaSize, computeLsaSize); +ASSERT_HOOK_MATCHES_PROD(g_devrWinRegEnabled, ncclDevrWinRegEnabled); +ASSERT_HOOK_MATCHES_PROD(g_findCommAndHostWindowFromDeviceWindow, findCommAndHostWindowFromDeviceWindow); +ASSERT_HOOK_MATCHES_PROD(g_symTeamObtain, symTeamObtain); +#undef ASSERT_HOOK_MATCHES_PROD + +static int DefaultComputeLsaSize(struct ncclComm*) { return 1; } +std::function g_computeLsaSize = DefaultComputeLsaSize; +int computeLsaSize(struct ncclComm* comm) { return g_computeLsaSize(comm); } + +static bool DefaultDevrWinRegEnabled(int, ncclDevrRegisterCapability) { return true; } +std::function g_devrWinRegEnabled = DefaultDevrWinRegEnabled; +bool ncclDevrWinRegEnabled(int winFlags, ncclDevrRegisterCapability capability) { + return g_devrWinRegEnabled(winFlags, capability); +} + +ncclComm_t g_foundComm = nullptr; +struct ncclDevrWindow* g_foundWinHost = nullptr; +static ncclResult_t FailLoudFindCommAndHostWindow(ncclWindow_t, ncclComm_t*, struct ncclDevrWindow**) { + FailLoudUnfaked("cft_dev_runtime_fakes", "findCommAndHostWindowFromDeviceWindow"); +} +std::function + g_findCommAndHostWindowFromDeviceWindow = FailLoudFindCommAndHostWindow; +ncclResult_t findCommAndHostWindowFromDeviceWindow(ncclWindow_t devWindow, ncclComm_t* foundComm, + struct ncclDevrWindow** hostWindow) { + return g_findCommAndHostWindowFromDeviceWindow(devWindow, foundComm, hostWindow); +} +static ncclResult_t ResolveFoundCommAndWindowImpl(ncclWindow_t, ncclComm_t* foundComm, + struct ncclDevrWindow** hostWindow) { + *foundComm = g_foundComm; + *hostWindow = g_foundWinHost; + return ncclSuccess; +} +void UseFoundCommAndWindow() { g_findCommAndHostWindowFromDeviceWindow = ResolveFoundCommAndWindowImpl; } + +static ncclResult_t FailLoudSymTeamObtain(struct ncclComm*, struct ncclTeam, bool, bool, bool, bool, + struct ncclDevrTeam**, bool*) { + FailLoudUnfaked("cft_dev_runtime_fakes", "symTeamObtain"); +} +std::function + g_symTeamObtain = FailLoudSymTeamObtain; +ncclResult_t symTeamObtain(struct ncclComm* comm, struct ncclTeam team, bool multimem, bool counted, bool wantsLeUc, + bool wantsLeMc, struct ncclDevrTeam** outTeam, bool* needBarrier) { + return g_symTeamObtain(comm, team, multimem, counted, wantsLeUc, wantsLeMc, outTeam, needBarrier); +} + +void ResetCftDevRuntimeFakes() { + g_computeLsaSize = DefaultComputeLsaSize; + g_devrWinRegEnabled = DefaultDevrWinRegEnabled; + g_findCommAndHostWindowFromDeviceWindow = FailLoudFindCommAndHostWindow; + g_foundComm = nullptr; + g_foundWinHost = nullptr; + g_symTeamObtain = FailLoudSymTeamObtain; +} diff --git a/projects/rccl/test/host/fakes/cft_dev_runtime_fakes.h b/projects/rccl/test/host/fakes/cft_dev_runtime_fakes.h new file mode 100644 index 000000000000..c06a71c66d75 --- /dev/null +++ b/projects/rccl/test/host/fakes/cft_dev_runtime_fakes.h @@ -0,0 +1,51 @@ +/************************************************************************* + * Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +// Seams for src/cft_dev_runtime.cc's callees that live in src/dev_runtime.cc +// (computeLsaSize, symTeamObtain, findCommAndHostWindowFromDeviceWindow) and +// src/dev_runtime.cc's ncclDevrWinRegEnabled. Targets linking this file +// compile the real cft_dev_runtime.cc but not the much larger dev_runtime.cc, +// so these four symbols need a seam of their own rather than the full file. + +#ifndef RCCL_TEST_HOST_CFT_DEV_RUNTIME_FAKES_H_ +#define RCCL_TEST_HOST_CFT_DEV_RUNTIME_FAKES_H_ + +#include + +#include "nccl.h" +// ncclDevrRegisterCapability is a plain (non-fixed-underlying-type) enum, which C++ +// does not allow forward-declaring -- pull in the real declarations instead. +#include "dev_runtime.h" + +struct ncclComm; +struct ncclTeam; + +// computeLsaSize: generous default (1) -- matches a comm with no LSA peers. +extern std::function g_computeLsaSize; + +// ncclDevrWinRegEnabled: default true -- matches a window registered for every capability. +extern std::function g_devrWinRegEnabled; + +// findCommAndHostWindowFromDeviceWindow: fail-loud default; a test must set +// g_foundComm/g_foundWinHost and install ResolveFoundCommAndWindow (or its own +// hook) before calling a function that depends on it. +extern std::function + g_findCommAndHostWindowFromDeviceWindow; +extern ncclComm_t g_foundComm; +extern struct ncclDevrWindow* g_foundWinHost; +// Installs the default hook that returns g_foundComm/g_foundWinHost directly. +void UseFoundCommAndWindow(); + +// symTeamObtain: fail-loud default; a test must install its own hook (e.g. one +// that heap-allocates a struct ncclDevrTeam with a populated worldRankList) +// before calling ncclGetMultimemDeviceLeInfo. +extern std::function + g_symTeamObtain; + +void ResetCftDevRuntimeFakes(); + +#endif // RCCL_TEST_HOST_CFT_DEV_RUNTIME_FAKES_H_ diff --git a/projects/rccl/test/host/nccl-cft-dev-runtime-test.cc b/projects/rccl/test/host/nccl-cft-dev-runtime-test.cc new file mode 100644 index 000000000000..e6d45c16af5e --- /dev/null +++ b/projects/rccl/test/host/nccl-cft-dev-runtime-test.cc @@ -0,0 +1,512 @@ +/************************************************************************* + * Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +// Host-only microtests for src/cft_dev_runtime.cc: the CFT (confidential/ +// fault-tolerant?) logical-endpoint support-probe, size-computation, and +// bind/unbind helpers, plus the three public LE-info query functions. This +// build's CUDA_VERSION is below 13030, so the real CUDA logical-endpoint +// driver calls are compiled out everywhere in this file (no ROCm +// equivalent exists); what remains compiled is exercised here: the +// zero-everything-and-succeed body of ncclGpuCftSupport, the host-side +// branch logic of computeCftSize/computeCftMcSize, the condition-evaluation +// (but now-empty) bodies of symBindTeamLe/symUnbindTeamLe, the +// trivial-failure stand-ins for symTeamObtainUcLe/symTeamObtainMcLe, and the +// three unconditional ncclGet*DeviceLeInfo query functions. +// +// core.cc is compiled for real alongside cft_dev_runtime.cc (not faked): +// ncclGetMultimemDeviceLeInfo/ncclGetCftDeviceLeInfo/ncclGetPeerDeviceLeInfo +// call ncclTeamCftMultimem/ncclTeamCft, and faking those would just be +// re-deriving core.cc's own already-tested logic (AICOMRCCL-2825 PR3) rather +// than reusing it. + +#include + +#include +#include + +#include "fakes/cft_dev_runtime_fakes.h" +#include "fakes/dev_runtime_fakes.h" + +#include CORE_CC_PATH +#include CFT_DEV_RUNTIME_CC_PATH + +namespace { + +class NcclCftDevRuntimeMicrotest : public ::testing::Test { +protected: + std::unique_ptr commStorage; + ncclComm* comm = nullptr; + + void SetUp() override { + commStorage = std::make_unique(); // value-initialised: POD members zeroed + comm = commStorage.get(); + // bigSize != 0 makes the real ncclDevrInitOnce (called by ncclTeamCft/ + // ncclTeamCftMultimem via core.cc) a no-op that leaves devrState as set below. + comm->devrState.bigSize = 1; + } + + void TearDown() override { + ResetDevRuntimeFakes(); + ResetCftDevRuntimeFakes(); + } +}; + +} // namespace + +// --------------------------------------------------------------------------- +// ncclGpuCftSupport: CUDA_VERSION < 13030 on this build, so the entire +// capability-probe body is compiled out; only the zero-and-succeed prologue +// remains. +// --------------------------------------------------------------------------- + +TEST_F(NcclCftDevRuntimeMicrotest, GpuCftSupport_ZeroesAllThreeOutputsAndSucceeds) { + int gpuCftSupport = 99; + bool gpuCftMulticastSupport = true, gpuCftCountedSupport = true; + EXPECT_EQ(ncclGpuCftSupport(comm, &gpuCftSupport, &gpuCftMulticastSupport, &gpuCftCountedSupport), ncclSuccess); + EXPECT_EQ(gpuCftSupport, 0); + EXPECT_FALSE(gpuCftMulticastSupport); + EXPECT_FALSE(gpuCftCountedSupport); +} + +// --------------------------------------------------------------------------- +// computeCftSize / computeCftMcSize +// --------------------------------------------------------------------------- + +TEST_F(NcclCftDevRuntimeMicrotest, ComputeCftSize_AlreadySized_ReturnsCftSizeWithoutCallingComputeLsaSize) { + comm->devrState.bigSize = 7; + comm->devrState.cftSize = 42; + bool called = false; + g_computeLsaSize = [&](ncclComm*) { called = true; return 1; }; + EXPECT_EQ(computeCftSize(comm), 42); + EXPECT_FALSE(called); +} + +TEST_F(NcclCftDevRuntimeMicrotest, ComputeCftSize_NotSizedNoGpuCftSupport_ReturnsOneWithoutCallingComputeLsaSize) { + comm->devrState.bigSize = 0; + comm->gpuCftSupport = 0; + bool called = false; + g_computeLsaSize = [&](ncclComm*) { called = true; return 99; }; + EXPECT_EQ(computeCftSize(comm), 1); + EXPECT_FALSE(called); +} + +TEST_F(NcclCftDevRuntimeMicrotest, ComputeCftSize_NotSizedWithGpuCftSupport_DelegatesToComputeLsaSize) { + comm->devrState.bigSize = 0; + comm->gpuCftSupport = 13030; + g_computeLsaSize = [](ncclComm*) { return 8; }; + EXPECT_EQ(computeCftSize(comm), 8); +} + +TEST_F(NcclCftDevRuntimeMicrotest, ComputeCftMcSize_AlreadySized_ReturnsCftMcSizeWithoutCallingComputeLsaSize) { + comm->devrState.bigSize = 7; + comm->devrState.cftMcSize = 24; + bool called = false; + g_computeLsaSize = [&](ncclComm*) { called = true; return 1; }; + EXPECT_EQ(computeCftMcSize(comm), 24); + EXPECT_FALSE(called); +} + +TEST_F(NcclCftDevRuntimeMicrotest, ComputeCftMcSize_NotSizedNoGpuCftSupport_ReturnsOneWithoutCallingComputeLsaSize) { + comm->devrState.bigSize = 0; + comm->gpuCftSupport = 0; + bool called = false; + g_computeLsaSize = [&](ncclComm*) { called = true; return 99; }; + EXPECT_EQ(computeCftMcSize(comm), 1); + EXPECT_FALSE(called); +} + +TEST_F(NcclCftDevRuntimeMicrotest, ComputeCftMcSize_NotSizedWithGpuCftSupport_DelegatesToComputeLsaSize) { + comm->devrState.bigSize = 0; + comm->gpuCftSupport = 13030; + g_computeLsaSize = [](ncclComm*) { return 6; }; + EXPECT_EQ(computeCftMcSize(comm), 6); +} + +// --------------------------------------------------------------------------- +// symBindTeamLe / symUnbindTeamLe: on this build the CUDA bind/unbind calls +// inside the condition are compiled out, so the only externally observable +// contract left is "always succeeds" and "always consults +// ncclDevrWinRegEnabled with this memory's winFlags and ncclDevrRegisterCft". +// --------------------------------------------------------------------------- + +TEST_F(NcclCftDevRuntimeMicrotest, SymBindTeamLe_AlwaysSucceedsAndConsultsWinRegEnabledWithCftCapability) { + auto mem = std::make_unique(); + mem->winFlags = 0x5a; + int sawWinFlags = -1; + ncclDevrRegisterCapability sawCapability = ncclDevrRegisterAll; + g_devrWinRegEnabled = [&](int winFlags, ncclDevrRegisterCapability capability) { + sawWinFlags = winFlags; + sawCapability = capability; + return true; + }; + EXPECT_EQ(symBindTeamLe(comm, mem.get(), /*le=*/7), ncclSuccess); + EXPECT_EQ(sawWinFlags, 0x5a); + EXPECT_EQ(sawCapability, ncclDevrRegisterCft); +} + +TEST_F(NcclCftDevRuntimeMicrotest, SymBindTeamLe_WinRegDisabled_StillSucceeds) { + auto mem = std::make_unique(); + g_devrWinRegEnabled = [](int, ncclDevrRegisterCapability) { return false; }; + EXPECT_EQ(symBindTeamLe(comm, mem.get(), /*le=*/7), ncclSuccess); +} + +TEST_F(NcclCftDevRuntimeMicrotest, SymBindTeamLe_InvalidLe_StillSucceeds) { + auto mem = std::make_unique(); + g_devrWinRegEnabled = [](int, ncclDevrRegisterCapability) { return true; }; + EXPECT_EQ(symBindTeamLe(comm, mem.get(), /*le=*/NCCL_LE_ID_INVALID), ncclSuccess); +} + +TEST_F(NcclCftDevRuntimeMicrotest, SymBindTeamLe_GlobalHasSysmemSegment_StillSucceeds) { + auto mem = std::make_unique(); + mem->globalHasSysmemSegment = true; + g_devrWinRegEnabled = [](int, ncclDevrRegisterCapability) { return true; }; + EXPECT_EQ(symBindTeamLe(comm, mem.get(), /*le=*/7), ncclSuccess); +} + +TEST_F(NcclCftDevRuntimeMicrotest, SymUnbindTeamLe_AlwaysSucceedsAndConsultsWinRegEnabledWithCftCapability) { + auto mem = std::make_unique(); + mem->winFlags = 0x3c; + int sawWinFlags = -1; + g_devrWinRegEnabled = [&](int winFlags, ncclDevrRegisterCapability) { + sawWinFlags = winFlags; + return true; + }; + EXPECT_EQ(symUnbindTeamLe(comm, mem.get(), /*le=*/3), ncclSuccess); + EXPECT_EQ(sawWinFlags, 0x3c); +} + +TEST_F(NcclCftDevRuntimeMicrotest, SymUnbindTeamLe_WinRegDisabled_StillSucceeds) { + auto mem = std::make_unique(); + g_devrWinRegEnabled = [](int, ncclDevrRegisterCapability) { return false; }; + EXPECT_EQ(symUnbindTeamLe(comm, mem.get(), /*le=*/3), ncclSuccess); +} + +TEST_F(NcclCftDevRuntimeMicrotest, SymUnbindTeamLe_InvalidLe_StillSucceeds) { + auto mem = std::make_unique(); + g_devrWinRegEnabled = [](int, ncclDevrRegisterCapability) { return true; }; + EXPECT_EQ(symUnbindTeamLe(comm, mem.get(), /*le=*/NCCL_LE_ID_INVALID), ncclSuccess); +} + +TEST_F(NcclCftDevRuntimeMicrotest, SymUnbindTeamLe_GlobalHasSysmemSegment_StillSucceeds) { + auto mem = std::make_unique(); + mem->globalHasSysmemSegment = true; + g_devrWinRegEnabled = [](int, ncclDevrRegisterCapability) { return true; }; + EXPECT_EQ(symUnbindTeamLe(comm, mem.get(), /*le=*/3), ncclSuccess); +} + +// --------------------------------------------------------------------------- +// symTeamObtainUcLe / symTeamObtainMcLe: this build compiles the +// CUDA_VERSION < 13030 stand-ins, which ignore every argument and fail. +// --------------------------------------------------------------------------- + +TEST_F(NcclCftDevRuntimeMicrotest, SymTeamObtainUcLe_UnsupportedOnThisPlatform_ReturnsInvalidUsage) { + bool needBarrier = false; + EXPECT_EQ(symTeamObtainUcLe(comm, /*t=*/nullptr, /*devr=*/nullptr, &needBarrier, /*counted=*/false), + ncclInvalidUsage); +} + +TEST_F(NcclCftDevRuntimeMicrotest, SymTeamObtainMcLe_UnsupportedOnThisPlatform_ReturnsInvalidUsage) { + bool needBarrier = false; + EXPECT_EQ(symTeamObtainMcLe(comm, /*t=*/nullptr, /*devr=*/nullptr, &needBarrier, /*counted=*/true), + ncclInvalidUsage); +} + +// --------------------------------------------------------------------------- +// ncclGetMultimemDeviceLeInfo +// --------------------------------------------------------------------------- + +class NcclGetMultimemDeviceLeInfoTest : public NcclCftDevRuntimeMicrotest { +protected: + ncclDevrWindow winHost{}; + int dummyWindowTarget = 0; + ncclWindow_t window = reinterpret_cast(&dummyWindowTarget); + + void SetUp() override { + NcclCftDevRuntimeMicrotest::SetUp(); + comm->gpuCftSupport = 13030; + winHost.bigOffset = 0x1000; + g_foundComm = comm; + g_foundWinHost = &winHost; + UseFoundCommAndWindow(); + } +}; + +TEST_F(NcclGetMultimemDeviceLeInfoTest, NullArgs_RejectedByPtrCheckBeforeAnyLookup) { + ncclCftLeId leId; + size_t leOffset; + EXPECT_NE(ncclGetMultimemDeviceLeInfo(nullptr, 0, &leId, &leOffset), ncclSuccess); + EXPECT_NE(ncclGetMultimemDeviceLeInfo(window, 0, nullptr, &leOffset), ncclSuccess); + EXPECT_NE(ncclGetMultimemDeviceLeInfo(window, 0, &leId, nullptr), ncclSuccess); +} + +TEST_F(NcclGetMultimemDeviceLeInfoTest, WindowNotCftRegistered_ReturnsInvalidUsage) { + g_devrWinRegEnabled = [](int, ncclDevrRegisterCapability) { return false; }; + ncclCftLeId leId; + size_t leOffset; + EXPECT_EQ(ncclGetMultimemDeviceLeInfo(window, 0, &leId, &leOffset), ncclInvalidUsage); +} + +TEST_F(NcclGetMultimemDeviceLeInfoTest, CommHasNoGpuCftSupport_ReturnsInvalidArgument) { + comm->gpuCftSupport = 0; + ncclCftLeId leId; + size_t leOffset; + EXPECT_EQ(ncclGetMultimemDeviceLeInfo(window, 0, &leId, &leOffset), ncclInvalidArgument); +} + +TEST_F(NcclGetMultimemDeviceLeInfoTest, FindCommAndHostWindowFails_PropagatesErrorViaNcclcheck) { + g_findCommAndHostWindowFromDeviceWindow = [](ncclWindow_t, ncclComm_t*, ncclDevrWindow**) { + return ncclSystemError; + }; + ncclCftLeId leId; + size_t leOffset; + EXPECT_EQ(ncclGetMultimemDeviceLeInfo(window, 0, &leId, &leOffset), ncclSystemError); +} + +TEST_F(NcclGetMultimemDeviceLeInfoTest, SymTeamObtainFails_PropagatesErrorViaNcclcheck) { + g_symTeamObtain = [](ncclComm*, ncclTeam, bool, bool, bool, bool, ncclDevrTeam**, bool*) { + return ncclInvalidUsage; + }; + ncclCftLeId leId; + size_t leOffset; + EXPECT_EQ(ncclGetMultimemDeviceLeInfo(window, 0, &leId, &leOffset), ncclInvalidUsage); +} + +// NOTE: the NCCLCHECK(bootstrapIntraNodeBarrier(...)) propagation branch (needBarrier +// path) is not independently exercised here: fakes/bootstrap_stubs.cc's +// bootstrapIntraNodeBarrier is a parameterless-result `{ return ncclSuccess; }` stub +// shared by every other suite that links it, with no failure hook. Giving it one is out +// of scope for this file's tests -- see the PR description's coverage note. + +TEST_F(NcclGetMultimemDeviceLeInfoTest, SuccessPath_ReadsMcLeIdByCountedBitAndOffsetsByBigOffsetPlusOffset) { + comm->devrState.cftMcSize = 4; + comm->devrState.cftMcSelf = 1; + winHost.winFlags = NCCL_WIN_CFT_COUNTED; + + // ncclDevrTeam has a flexible array member (worldRankList[]), which clang + // refuses to let a lambda capture directly (even by reference) -- capture a + // pointer to it instead. + ncclDevrTeam capturedTeamStorage{}; + capturedTeamStorage.mcLeId[0] = 111; + capturedTeamStorage.mcLeId[1] = 222; // counted == true selects this one + ncclDevrTeam* capturedTeamPtr = &capturedTeamStorage; + bool sawNeedBarrierRequested = false; + g_symTeamObtain = [&, capturedTeamPtr](ncclComm*, ncclTeam team, bool multimem, bool counted, bool wantsLeUc, + bool wantsLeMc, ncclDevrTeam** outTeam, bool* needBarrier) { + EXPECT_FALSE(multimem); + EXPECT_TRUE(counted); // from winHost.winFlags & NCCL_WIN_CFT_COUNTED + EXPECT_FALSE(wantsLeUc); + EXPECT_TRUE(wantsLeMc); + EXPECT_EQ(team.nRanks, 4); + EXPECT_EQ(team.rank, 1); + *outTeam = capturedTeamPtr; + *needBarrier = true; + sawNeedBarrierRequested = true; + return ncclSuccess; + }; + + ncclCftLeId leId = 0; + size_t leOffset = 0; + ASSERT_EQ(ncclGetMultimemDeviceLeInfo(window, /*offset=*/0x20, &leId, &leOffset), ncclSuccess); + EXPECT_TRUE(sawNeedBarrierRequested); + EXPECT_EQ(leId, 222u); + EXPECT_EQ(leOffset, winHost.bigOffset + 0x20); +} + +TEST_F(NcclGetMultimemDeviceLeInfoTest, UncountedWindow_ReadsMcLeIdZeroSlot) { + comm->devrState.cftMcSize = 4; + comm->devrState.cftMcSelf = 0; + winHost.winFlags = 0; // not counted + + ncclDevrTeam capturedTeamStorage{}; + capturedTeamStorage.mcLeId[0] = 55; + capturedTeamStorage.mcLeId[1] = 66; + ncclDevrTeam* capturedTeamPtr = &capturedTeamStorage; + g_symTeamObtain = [capturedTeamPtr](ncclComm*, ncclTeam, bool, bool counted, bool, bool, ncclDevrTeam** outTeam, + bool* needBarrier) { + EXPECT_FALSE(counted); + *outTeam = capturedTeamPtr; + *needBarrier = false; // no barrier requested this time + return ncclSuccess; + }; + + ncclCftLeId leId = 0; + size_t leOffset = 0; + ASSERT_EQ(ncclGetMultimemDeviceLeInfo(window, /*offset=*/0, &leId, &leOffset), ncclSuccess); + EXPECT_EQ(leId, 55u); +} + +// --------------------------------------------------------------------------- +// ncclGetCftDeviceLeInfo +// --------------------------------------------------------------------------- + +class NcclGetCftDeviceLeInfoTest : public NcclCftDevRuntimeMicrotest { +protected: + ncclDevrWindow winHost{}; + int dummyWindowTarget = 0; + ncclWindow_t window = reinterpret_cast(&dummyWindowTarget); + + void SetUp() override { + NcclCftDevRuntimeMicrotest::SetUp(); + comm->gpuCftSupport = 13030; + winHost.bigOffset = 0x2000; + g_foundComm = comm; + g_foundWinHost = &winHost; + UseFoundCommAndWindow(); + } +}; + +TEST_F(NcclGetCftDeviceLeInfoTest, NullArgs_RejectedByPtrCheck) { + ncclTeam_t cftTeam{4, 0, 1}; + ncclCftLeId leId; + size_t leOffset; + EXPECT_NE(ncclGetCftDeviceLeInfo(nullptr, 0, 0, cftTeam, &leId, &leOffset), ncclSuccess); +} + +TEST_F(NcclGetCftDeviceLeInfoTest, WindowNotCftRegistered_ReturnsInvalidUsage) { + g_devrWinRegEnabled = [](int, ncclDevrRegisterCapability) { return false; }; + ncclTeam_t cftTeam{4, 0, 1}; + ncclCftLeId leId; + size_t leOffset; + EXPECT_EQ(ncclGetCftDeviceLeInfo(window, 0, 0, cftTeam, &leId, &leOffset), ncclInvalidUsage); +} + +TEST_F(NcclGetCftDeviceLeInfoTest, CommHasNoGpuCftSupport_ReturnsInvalidArgument) { + comm->gpuCftSupport = 0; + ncclTeam_t cftTeam{4, 0, 1}; + ncclCftLeId leId; + size_t leOffset; + EXPECT_EQ(ncclGetCftDeviceLeInfo(window, 0, 0, cftTeam, &leId, &leOffset), ncclInvalidArgument); +} + +TEST_F(NcclGetCftDeviceLeInfoTest, FindCommAndHostWindowFails_PropagatesErrorViaNcclcheck) { + g_findCommAndHostWindowFromDeviceWindow = [](ncclWindow_t, ncclComm_t*, ncclDevrWindow**) { + return ncclSystemError; + }; + ncclTeam_t cftTeam{4, 0, 1}; + ncclCftLeId leId; + size_t leOffset; + EXPECT_EQ(ncclGetCftDeviceLeInfo(window, 0, 0, cftTeam, &leId, &leOffset), ncclSystemError); +} + +TEST_F(NcclGetCftDeviceLeInfoTest, BaseLeIdNotYetCreated_ReturnsInvalidUsage) { + comm->devrState.le[0].baseId = NCCL_LE_ID_INVALID; + ncclTeam_t cftTeam{4, 0, 1}; + ncclCftLeId leId; + size_t leOffset; + EXPECT_EQ(ncclGetCftDeviceLeInfo(window, 0, 0, cftTeam, &leId, &leOffset), ncclInvalidUsage); +} + +TEST_F(NcclGetCftDeviceLeInfoTest, SuccessPath_AddsFlatTeamRankAndPeerOffsetToBaseLeId) { + // ncclTeamCft(comm, FLAT) reads straight from devrState.cftSize/cftSelf (bigSize != 0 + // from the fixture keeps ncclDevrInitOnce a no-op). + comm->devrState.cftSize = 8; + comm->devrState.cftSelf = 3; + comm->devrState.le[0].baseId = 1000; + winHost.winFlags = 0; // uncounted -> le[0] + + ncclTeam_t cftTeam{4, /*rank=*/1, /*stride=*/2}; + ncclCftLeId leId = 0; + size_t leOffset = 0; + // *leId = baseLeId + flatTeam.rank + (peerCft - cftTeam.rank) * cftTeam.stride + // = 1000 + 3 + (5 - 1) * 2 = 1011 + ASSERT_EQ(ncclGetCftDeviceLeInfo(window, /*offset=*/0x10, /*peerCft=*/5, cftTeam, &leId, &leOffset), ncclSuccess); + EXPECT_EQ(leId, 1011u); + EXPECT_EQ(leOffset, winHost.bigOffset + 0x10); +} + +TEST_F(NcclGetCftDeviceLeInfoTest, CountedWindow_UsesCountedBaseLeIdSlot) { + comm->devrState.cftSize = 4; + comm->devrState.cftSelf = 0; + comm->devrState.le[0].baseId = NCCL_LE_ID_INVALID; // uncounted slot deliberately unset + comm->devrState.le[1].baseId = 500; // counted slot is what must be read + winHost.winFlags = NCCL_WIN_CFT_COUNTED; + + ncclTeam_t cftTeam{4, 0, 1}; + ncclCftLeId leId = 0; + size_t leOffset = 0; + ASSERT_EQ(ncclGetCftDeviceLeInfo(window, 0, /*peerCft=*/0, cftTeam, &leId, &leOffset), ncclSuccess); + EXPECT_EQ(leId, 500u); // 500 + flatTeam.rank(0) + (0 - 0) * 1 +} + +// --------------------------------------------------------------------------- +// ncclGetPeerDeviceLeInfo +// --------------------------------------------------------------------------- + +class NcclGetPeerDeviceLeInfoTest : public NcclCftDevRuntimeMicrotest { +protected: + ncclDevrWindow winHost{}; + int dummyWindowTarget = 0; + ncclWindow_t window = reinterpret_cast(&dummyWindowTarget); + + void SetUp() override { + NcclCftDevRuntimeMicrotest::SetUp(); + comm->gpuCftSupport = 13030; + winHost.bigOffset = 0x3000; + g_foundComm = comm; + g_foundWinHost = &winHost; + UseFoundCommAndWindow(); + } +}; + +TEST_F(NcclGetPeerDeviceLeInfoTest, WindowNotCftRegistered_ReturnsInvalidUsage) { + g_devrWinRegEnabled = [](int, ncclDevrRegisterCapability) { return false; }; + ncclCftLeId leId; + size_t leOffset; + EXPECT_EQ(ncclGetPeerDeviceLeInfo(window, 0, 0, &leId, &leOffset), ncclInvalidUsage); +} + +TEST_F(NcclGetPeerDeviceLeInfoTest, CommHasNoGpuCftSupport_ReturnsInvalidArgument) { + comm->gpuCftSupport = 0; + ncclCftLeId leId; + size_t leOffset; + EXPECT_EQ(ncclGetPeerDeviceLeInfo(window, 0, 0, &leId, &leOffset), ncclInvalidArgument); +} + +TEST_F(NcclGetPeerDeviceLeInfoTest, FindCommAndHostWindowFails_PropagatesErrorViaNcclcheck) { + g_findCommAndHostWindowFromDeviceWindow = [](ncclWindow_t, ncclComm_t*, ncclDevrWindow**) { + return ncclSystemError; + }; + ncclCftLeId leId; + size_t leOffset; + EXPECT_EQ(ncclGetPeerDeviceLeInfo(window, 0, 0, &leId, &leOffset), ncclSystemError); +} + +TEST_F(NcclGetPeerDeviceLeInfoTest, PeerOutsideFlatTeamBoundaries_ReturnsInvalidArgument) { + // flatStart = comm->rank - flatTeam.rank; valid range is [flatStart, flatStart + nRanks). + comm->rank = 10; + comm->devrState.cftSize = 4; + comm->devrState.cftSelf = 2; // flatTeam.rank == 2 -> flatStart == 8, valid peers in [8, 12) + comm->devrState.le[0].baseId = 1; // must be valid so we reach the boundary check + ncclCftLeId leId; + size_t leOffset; + EXPECT_EQ(ncclGetPeerDeviceLeInfo(window, 0, /*peerWorld=*/20, &leId, &leOffset), ncclInvalidArgument); + EXPECT_EQ(ncclGetPeerDeviceLeInfo(window, 0, /*peerWorld=*/7, &leId, &leOffset), ncclInvalidArgument); +} + +TEST_F(NcclGetPeerDeviceLeInfoTest, BaseLeIdNotYetCreated_ReturnsInvalidUsage) { + comm->rank = 10; + comm->devrState.cftSize = 4; + comm->devrState.cftSelf = 2; + comm->devrState.le[0].baseId = NCCL_LE_ID_INVALID; + ncclCftLeId leId; + size_t leOffset; + EXPECT_EQ(ncclGetPeerDeviceLeInfo(window, 0, /*peerWorld=*/10, &leId, &leOffset), ncclInvalidUsage); +} + +TEST_F(NcclGetPeerDeviceLeInfoTest, SuccessPath_AddsFlatTeamRankAndWorldOffsetToBaseLeId) { + comm->rank = 10; + comm->devrState.cftSize = 4; + comm->devrState.cftSelf = 2; // flatStart = 8, valid peers in [8, 12) + comm->devrState.le[0].baseId = 1000; + winHost.winFlags = 0; + + ncclCftLeId leId = 0; + size_t leOffset = 0; + // *leId = baseLeId + flatTeam.rank + (peerWorld - comm->rank) = 1000 + 2 + (11 - 10) = 1003 + ASSERT_EQ(ncclGetPeerDeviceLeInfo(window, /*offset=*/0x30, /*peerWorld=*/11, &leId, &leOffset), ncclSuccess); + EXPECT_EQ(leId, 1003u); + EXPECT_EQ(leOffset, winHost.bigOffset + 0x30); +} diff --git a/projects/rccl/test/host/run_host_tests.sh b/projects/rccl/test/host/run_host_tests.sh index ec702f4f62b1..e03682484e35 100755 --- a/projects/rccl/test/host/run_host_tests.sh +++ b/projects/rccl/test/host/run_host_tests.sh @@ -167,6 +167,7 @@ do_host_tests() { "rccl-UnitTestsMicroTaskPrep:$SCRIPT_DIR/host_tests_micro_taskprep.xml" "rccl-UnitTestsMicroCe:$SCRIPT_DIR/host_tests_micro_ce.xml" "rccl-UnitTestsMicroClientSupport:$SCRIPT_DIR/host_tests_micro_client_support.xml" + "rccl-UnitTestsMicroNcclDeviceCft:$SCRIPT_DIR/host_tests_micro_nccl_device_cft.xml" "rccl-UnitTestsMicroGinHost:$SCRIPT_DIR/host_tests_micro_gin_host.xml" "rccl-UnitTestsMicroDiagnostics:$SCRIPT_DIR/host_tests_micro_diagnostics.xml" ) From b1f8208499cc854f1b131952e9bd81f1771b7378 Mon Sep 17 00:00:00 2001 From: Alex Duforest Date: Mon, 12 Oct 2026 22:42:52 -0700 Subject: [PATCH 15/26] test(rccl): address ARGUS findings on cft_dev_runtime.cc microtests - Refactor the three ncclGet*DeviceLeInfo fixture classes to share a common base (NcclGetDeviceLeInfoTestBase) with a one-line BigOffset() override each, removing ~15 duplicated lines while keeping all four gtest suite names and the Nccl* CTest pattern intact. - Add a MakeDevrTeam() helper that heap-allocates ncclDevrTeam sized for its flexible-array worldRankList (same idiom as dev_runtime.cc's own symTeamObtain), replacing an automatic ncclDevrTeam whose zero-element worldRankList the production code indexed out of bounds at team.nRanks. - ncclGetMultimemDeviceLeInfo's success test: OR NCCL_WIN_COLL_SYMMETRIC into the counted window's winFlags, so is tested as a real bit-mask, not just equality (every other winFlags in this file was exactly NCCL_WIN_CFT_COUNTED or exactly 0). - Add a dedicated NullArgs test to NcclGetPeerDeviceLeInfoTest (previously missing entirely), and extend NcclGetCftDeviceLeInfoTest's existing one to cover leId/leOffset, not just window; all three now pin the exact ncclInvalidArgument code instead of just EXPECT_NE(..., ncclSuccess). - Add a CountedWindow_UsesCountedBaseLeIdSlot case to NcclGetPeerDeviceLeInfoTest, mirroring the Cft sibling's existing one. - PeerOutsideFlatTeamBoundaries: drop the dead/backwards-commented le[0].baseId=1 setup (the boundary check runs before baseLeId is ever read) and pin both exact edges (peerWorld=12 rejecting, peerWorld=8 succeeding), which the previous 20/7 probes left both uncaught. - SymUnbindTeamLe_AlwaysSucceeds...: also assert the capability argument (mirrors the already-correct SymBindTeamLe twin). - Fix two comments: the fixture's bigSize=1 is load-bearing only for computeCftSize/computeCftMcSize, not ncclTeamCft/ncclTeamCftMultimem (ncclDevrInitOnce here is an always-succeeds fake that never reads devrState); and cft_dev_runtime_fakes.h's wrong function name. - Drop the nccl_device EXTRA_INCLUDES entry: verified empirically that this binary builds and all tests pass without it, same as PR3's core.cc finding. - Fix MICROTEST_README.md's dangling reference to rccl-UnitTestsMicroNcclDeviceCore, which doesn't exist on this branch. Re-measured coverage on cft_dev_runtime.cc after these fixes: regions 72.06%->82.35%, branches 57.89%->71.05%, lines still 100%. Remaining gaps are the same two documented categories from the PR description (ncclParamCftEnable, NCCLCHECK's own macro-internal branches). --- projects/rccl/test/host/CMakeLists.txt | 2 - projects/rccl/test/host/MICROTEST_README.md | 7 +- .../test/host/fakes/cft_dev_runtime_fakes.h | 2 +- .../test/host/nccl-cft-dev-runtime-test.cc | 176 ++++++++++++------ 4 files changed, 119 insertions(+), 68 deletions(-) diff --git a/projects/rccl/test/host/CMakeLists.txt b/projects/rccl/test/host/CMakeLists.txt index 6933fed44290..3dcbabec9ca4 100644 --- a/projects/rccl/test/host/CMakeLists.txt +++ b/projects/rccl/test/host/CMakeLists.txt @@ -1093,8 +1093,6 @@ rccl_add_micro_binary(rccl-UnitTestsMicroClientSupport # --------------------------------------------------------------------------- rccl_add_micro_binary(rccl-UnitTestsMicroNcclDeviceCft SOURCES ${TEST_MICRO_NCCL_DEVICE_CFT_SOURCE_FILES} - EXTRA_INCLUDES - ${RCCL_HOST_HIPIFY_DIR}/src/include/nccl_device # core.cc includes "nccl_device/impl/core__funcs.h" unqualified EXTRA_DEFS ${_MICRO_MODE_DEFS} CORE_CC_PATH="${RCCL_HOST_HIPIFY_DIR}/src/nccl_device/core.cc" diff --git a/projects/rccl/test/host/MICROTEST_README.md b/projects/rccl/test/host/MICROTEST_README.md index e7ccc0e6bccf..b5f31cbfa944 100644 --- a/projects/rccl/test/host/MICROTEST_README.md +++ b/projects/rccl/test/host/MICROTEST_README.md @@ -253,9 +253,10 @@ export colliding non-`static` symbols; otherwise a unit needs its own binary: from `nccl-cft-dev-runtime-test.cc`); suites `NcclCftDevRuntimeMicrotest.*`, `NcclGetMultimemDeviceLeInfoTest.*`, `NcclGetCftDeviceLeInfoTest.*`, `NcclGetPeerDeviceLeInfoTest.*` (all caught by the `Nccl*` pattern). Its own - binary for the same reason as `rccl-UnitTestsMicroNcclDeviceCore`: 6 of - core.cc's 7 functions are faked elsewhere in `rccl-UnitTestsMicro`. core.cc - is compiled for real here too (not faked) because + binary because `rccl-UnitTestsMicro`'s `fakes/dev_runtime_micro_fakes.cc`, + `fakes/devcomm_fakes.cc`, and `gin-host-test.cc` between them fake 6 of + core.cc's 7 functions. core.cc is compiled for real here too (not faked) + because `ncclGetMultimemDeviceLeInfo`/`ncclGetCftDeviceLeInfo`/`ncclGetPeerDeviceLeInfo` call `ncclTeamCft`/`ncclTeamCftMultimem`. Links `fakes/dev_runtime_fakes.cc` for core.cc's `ncclDevrInitOnce`, and the new `fakes/cft_dev_runtime_fakes.cc` diff --git a/projects/rccl/test/host/fakes/cft_dev_runtime_fakes.h b/projects/rccl/test/host/fakes/cft_dev_runtime_fakes.h index c06a71c66d75..1104cc4776aa 100644 --- a/projects/rccl/test/host/fakes/cft_dev_runtime_fakes.h +++ b/projects/rccl/test/host/fakes/cft_dev_runtime_fakes.h @@ -30,7 +30,7 @@ extern std::function g_computeLsaSize; extern std::function g_devrWinRegEnabled; // findCommAndHostWindowFromDeviceWindow: fail-loud default; a test must set -// g_foundComm/g_foundWinHost and install ResolveFoundCommAndWindow (or its own +// g_foundComm/g_foundWinHost and call UseFoundCommAndWindow() (or install its own // hook) before calling a function that depends on it. extern std::function g_findCommAndHostWindowFromDeviceWindow; diff --git a/projects/rccl/test/host/nccl-cft-dev-runtime-test.cc b/projects/rccl/test/host/nccl-cft-dev-runtime-test.cc index e6d45c16af5e..d4f95d4b1944 100644 --- a/projects/rccl/test/host/nccl-cft-dev-runtime-test.cc +++ b/projects/rccl/test/host/nccl-cft-dev-runtime-test.cc @@ -24,7 +24,7 @@ #include -#include +#include #include #include "fakes/cft_dev_runtime_fakes.h" @@ -35,6 +35,25 @@ namespace { +// Heap-allocates a ncclDevrTeam sized and populated for its flexible-array +// worldRankList, same idiom dev_runtime.cc's own symTeamObtain uses. An +// automatic (stack) ncclDevrTeam's worldRankList has zero elements; a fake +// that hands one to code indexing it at team.nRanks-1 is out of bounds. +struct DevrTeamDeleter { + void operator()(ncclDevrTeam* t) const { std::free(t); } +}; +using DevrTeamPtr = std::unique_ptr; + +DevrTeamPtr MakeDevrTeam(ncclTeam_t team, ncclCftLeId mcLeId0, ncclCftLeId mcLeId1) { + auto* t = static_cast(std::malloc(sizeof(ncclDevrTeam) + team.nRanks * sizeof(int))); + t->team = team; + t->ucLeId[0] = t->ucLeId[1] = NCCL_LE_ID_INVALID; + t->mcLeId[0] = mcLeId0; + t->mcLeId[1] = mcLeId1; + for (int i = 0; i < team.nRanks; i++) t->worldRankList[i] = i; + return DevrTeamPtr(t); +} + class NcclCftDevRuntimeMicrotest : public ::testing::Test { protected: std::unique_ptr commStorage; @@ -43,8 +62,11 @@ class NcclCftDevRuntimeMicrotest : public ::testing::Test { void SetUp() override { commStorage = std::make_unique(); // value-initialised: POD members zeroed comm = commStorage.get(); - // bigSize != 0 makes the real ncclDevrInitOnce (called by ncclTeamCft/ - // ncclTeamCftMultimem via core.cc) a no-op that leaves devrState as set below. + // Load-bearing only for computeCftSize/computeCftMcSize, which check + // devrState.bigSize themselves: ncclDevrInitOnce in this binary is the + // always-succeeds fake seam at fakes/dev_runtime_fakes.cc, which never + // reads devrState, so bigSize is inert on the ncclTeamCft/ + // ncclTeamCftMultimem path. comm->devrState.bigSize = 1; } @@ -170,12 +192,15 @@ TEST_F(NcclCftDevRuntimeMicrotest, SymUnbindTeamLe_AlwaysSucceedsAndConsultsWinR auto mem = std::make_unique(); mem->winFlags = 0x3c; int sawWinFlags = -1; - g_devrWinRegEnabled = [&](int winFlags, ncclDevrRegisterCapability) { + ncclDevrRegisterCapability sawCapability = ncclDevrRegisterAll; + g_devrWinRegEnabled = [&](int winFlags, ncclDevrRegisterCapability capability) { sawWinFlags = winFlags; + sawCapability = capability; return true; }; EXPECT_EQ(symUnbindTeamLe(comm, mem.get(), /*le=*/3), ncclSuccess); EXPECT_EQ(sawWinFlags, 0x3c); + EXPECT_EQ(sawCapability, ncclDevrRegisterCft); } TEST_F(NcclCftDevRuntimeMicrotest, SymUnbindTeamLe_WinRegDisabled_StillSucceeds) { @@ -214,32 +239,44 @@ TEST_F(NcclCftDevRuntimeMicrotest, SymTeamObtainMcLe_UnsupportedOnThisPlatform_R ncclInvalidUsage); } -// --------------------------------------------------------------------------- -// ncclGetMultimemDeviceLeInfo -// --------------------------------------------------------------------------- - -class NcclGetMultimemDeviceLeInfoTest : public NcclCftDevRuntimeMicrotest { +// Shared window/comm-resolution setup for all three ncclGet*DeviceLeInfo +// query functions; BigOffset() keeps each derived suite's winHost.bigOffset +// distinct, same reasoning as the handle.* field checks elsewhere in this +// file: a test mixing up which window it is reading would get a visibly +// wrong offset back instead of an accidental match. +class NcclGetDeviceLeInfoTestBase : public NcclCftDevRuntimeMicrotest { protected: ncclDevrWindow winHost{}; int dummyWindowTarget = 0; ncclWindow_t window = reinterpret_cast(&dummyWindowTarget); + virtual size_t BigOffset() const = 0; + void SetUp() override { NcclCftDevRuntimeMicrotest::SetUp(); comm->gpuCftSupport = 13030; - winHost.bigOffset = 0x1000; + winHost.bigOffset = BigOffset(); g_foundComm = comm; g_foundWinHost = &winHost; UseFoundCommAndWindow(); } }; +// --------------------------------------------------------------------------- +// ncclGetMultimemDeviceLeInfo +// --------------------------------------------------------------------------- + +class NcclGetMultimemDeviceLeInfoTest : public NcclGetDeviceLeInfoTestBase { +protected: + size_t BigOffset() const override { return 0x1000; } +}; + TEST_F(NcclGetMultimemDeviceLeInfoTest, NullArgs_RejectedByPtrCheckBeforeAnyLookup) { ncclCftLeId leId; size_t leOffset; - EXPECT_NE(ncclGetMultimemDeviceLeInfo(nullptr, 0, &leId, &leOffset), ncclSuccess); - EXPECT_NE(ncclGetMultimemDeviceLeInfo(window, 0, nullptr, &leOffset), ncclSuccess); - EXPECT_NE(ncclGetMultimemDeviceLeInfo(window, 0, &leId, nullptr), ncclSuccess); + EXPECT_EQ(ncclGetMultimemDeviceLeInfo(nullptr, 0, &leId, &leOffset), ncclInvalidArgument); + EXPECT_EQ(ncclGetMultimemDeviceLeInfo(window, 0, nullptr, &leOffset), ncclInvalidArgument); + EXPECT_EQ(ncclGetMultimemDeviceLeInfo(window, 0, &leId, nullptr), ncclInvalidArgument); } TEST_F(NcclGetMultimemDeviceLeInfoTest, WindowNotCftRegistered_ReturnsInvalidUsage) { @@ -283,25 +320,25 @@ TEST_F(NcclGetMultimemDeviceLeInfoTest, SymTeamObtainFails_PropagatesErrorViaNcc TEST_F(NcclGetMultimemDeviceLeInfoTest, SuccessPath_ReadsMcLeIdByCountedBitAndOffsetsByBigOffsetPlusOffset) { comm->devrState.cftMcSize = 4; comm->devrState.cftMcSelf = 1; - winHost.winFlags = NCCL_WIN_CFT_COUNTED; - - // ncclDevrTeam has a flexible array member (worldRankList[]), which clang - // refuses to let a lambda capture directly (even by reference) -- capture a - // pointer to it instead. - ncclDevrTeam capturedTeamStorage{}; - capturedTeamStorage.mcLeId[0] = 111; - capturedTeamStorage.mcLeId[1] = 222; // counted == true selects this one - ncclDevrTeam* capturedTeamPtr = &capturedTeamStorage; + // NCCL_WIN_COLL_SYMMETRIC is ORed in alongside NCCL_WIN_CFT_COUNTED so this + // exercises `winFlags & NCCL_WIN_CFT_COUNTED` as a real bit test: every + // other winFlags value elsewhere in this file is exactly + // NCCL_WIN_CFT_COUNTED or exactly 0, which `winFlags == NCCL_WIN_CFT_COUNTED` + // would also satisfy. + winHost.winFlags = NCCL_WIN_CFT_COUNTED | NCCL_WIN_COLL_SYMMETRIC; + + DevrTeamPtr team = MakeDevrTeam(ncclTeam_t{4, 1, 1}, /*mcLeId0=*/111, /*mcLeId1=*/222); // counted==true selects mcLeId1 + ncclDevrTeam* teamPtr = team.get(); bool sawNeedBarrierRequested = false; - g_symTeamObtain = [&, capturedTeamPtr](ncclComm*, ncclTeam team, bool multimem, bool counted, bool wantsLeUc, - bool wantsLeMc, ncclDevrTeam** outTeam, bool* needBarrier) { + g_symTeamObtain = [&, teamPtr](ncclComm*, ncclTeam team, bool multimem, bool counted, bool wantsLeUc, + bool wantsLeMc, ncclDevrTeam** outTeam, bool* needBarrier) { EXPECT_FALSE(multimem); EXPECT_TRUE(counted); // from winHost.winFlags & NCCL_WIN_CFT_COUNTED EXPECT_FALSE(wantsLeUc); EXPECT_TRUE(wantsLeMc); EXPECT_EQ(team.nRanks, 4); EXPECT_EQ(team.rank, 1); - *outTeam = capturedTeamPtr; + *outTeam = teamPtr; *needBarrier = true; sawNeedBarrierRequested = true; return ncclSuccess; @@ -320,14 +357,12 @@ TEST_F(NcclGetMultimemDeviceLeInfoTest, UncountedWindow_ReadsMcLeIdZeroSlot) { comm->devrState.cftMcSelf = 0; winHost.winFlags = 0; // not counted - ncclDevrTeam capturedTeamStorage{}; - capturedTeamStorage.mcLeId[0] = 55; - capturedTeamStorage.mcLeId[1] = 66; - ncclDevrTeam* capturedTeamPtr = &capturedTeamStorage; - g_symTeamObtain = [capturedTeamPtr](ncclComm*, ncclTeam, bool, bool counted, bool, bool, ncclDevrTeam** outTeam, - bool* needBarrier) { + DevrTeamPtr team = MakeDevrTeam(ncclTeam_t{4, 0, 1}, /*mcLeId0=*/55, /*mcLeId1=*/66); + ncclDevrTeam* teamPtr = team.get(); + g_symTeamObtain = [teamPtr](ncclComm*, ncclTeam, bool, bool counted, bool, bool, ncclDevrTeam** outTeam, + bool* needBarrier) { EXPECT_FALSE(counted); - *outTeam = capturedTeamPtr; + *outTeam = teamPtr; *needBarrier = false; // no barrier requested this time return ncclSuccess; }; @@ -342,27 +377,18 @@ TEST_F(NcclGetMultimemDeviceLeInfoTest, UncountedWindow_ReadsMcLeIdZeroSlot) { // ncclGetCftDeviceLeInfo // --------------------------------------------------------------------------- -class NcclGetCftDeviceLeInfoTest : public NcclCftDevRuntimeMicrotest { +class NcclGetCftDeviceLeInfoTest : public NcclGetDeviceLeInfoTestBase { protected: - ncclDevrWindow winHost{}; - int dummyWindowTarget = 0; - ncclWindow_t window = reinterpret_cast(&dummyWindowTarget); - - void SetUp() override { - NcclCftDevRuntimeMicrotest::SetUp(); - comm->gpuCftSupport = 13030; - winHost.bigOffset = 0x2000; - g_foundComm = comm; - g_foundWinHost = &winHost; - UseFoundCommAndWindow(); - } + size_t BigOffset() const override { return 0x2000; } }; TEST_F(NcclGetCftDeviceLeInfoTest, NullArgs_RejectedByPtrCheck) { ncclTeam_t cftTeam{4, 0, 1}; ncclCftLeId leId; size_t leOffset; - EXPECT_NE(ncclGetCftDeviceLeInfo(nullptr, 0, 0, cftTeam, &leId, &leOffset), ncclSuccess); + EXPECT_EQ(ncclGetCftDeviceLeInfo(nullptr, 0, 0, cftTeam, &leId, &leOffset), ncclInvalidArgument); + EXPECT_EQ(ncclGetCftDeviceLeInfo(window, 0, 0, cftTeam, nullptr, &leOffset), ncclInvalidArgument); + EXPECT_EQ(ncclGetCftDeviceLeInfo(window, 0, 0, cftTeam, &leId, nullptr), ncclInvalidArgument); } TEST_F(NcclGetCftDeviceLeInfoTest, WindowNotCftRegistered_ReturnsInvalidUsage) { @@ -400,8 +426,9 @@ TEST_F(NcclGetCftDeviceLeInfoTest, BaseLeIdNotYetCreated_ReturnsInvalidUsage) { } TEST_F(NcclGetCftDeviceLeInfoTest, SuccessPath_AddsFlatTeamRankAndPeerOffsetToBaseLeId) { - // ncclTeamCft(comm, FLAT) reads straight from devrState.cftSize/cftSelf (bigSize != 0 - // from the fixture keeps ncclDevrInitOnce a no-op). + // ncclTeamCft(comm, FLAT) reads straight from devrState.cftSize/cftSelf; it never + // consults devrState.bigSize itself (that's load-bearing only for computeCftSize/ + // computeCftMcSize -- see NcclCftDevRuntimeMicrotest::SetUp's comment). comm->devrState.cftSize = 8; comm->devrState.cftSelf = 3; comm->devrState.le[0].baseId = 1000; @@ -435,22 +462,19 @@ TEST_F(NcclGetCftDeviceLeInfoTest, CountedWindow_UsesCountedBaseLeIdSlot) { // ncclGetPeerDeviceLeInfo // --------------------------------------------------------------------------- -class NcclGetPeerDeviceLeInfoTest : public NcclCftDevRuntimeMicrotest { +class NcclGetPeerDeviceLeInfoTest : public NcclGetDeviceLeInfoTestBase { protected: - ncclDevrWindow winHost{}; - int dummyWindowTarget = 0; - ncclWindow_t window = reinterpret_cast(&dummyWindowTarget); - - void SetUp() override { - NcclCftDevRuntimeMicrotest::SetUp(); - comm->gpuCftSupport = 13030; - winHost.bigOffset = 0x3000; - g_foundComm = comm; - g_foundWinHost = &winHost; - UseFoundCommAndWindow(); - } + size_t BigOffset() const override { return 0x3000; } }; +TEST_F(NcclGetPeerDeviceLeInfoTest, NullArgs_RejectedByPtrCheck) { + ncclCftLeId leId; + size_t leOffset; + EXPECT_EQ(ncclGetPeerDeviceLeInfo(nullptr, 0, 0, &leId, &leOffset), ncclInvalidArgument); + EXPECT_EQ(ncclGetPeerDeviceLeInfo(window, 0, 0, nullptr, &leOffset), ncclInvalidArgument); + EXPECT_EQ(ncclGetPeerDeviceLeInfo(window, 0, 0, &leId, nullptr), ncclInvalidArgument); +} + TEST_F(NcclGetPeerDeviceLeInfoTest, WindowNotCftRegistered_ReturnsInvalidUsage) { g_devrWinRegEnabled = [](int, ncclDevrRegisterCapability) { return false; }; ncclCftLeId leId; @@ -476,14 +500,42 @@ TEST_F(NcclGetPeerDeviceLeInfoTest, FindCommAndHostWindowFails_PropagatesErrorVi TEST_F(NcclGetPeerDeviceLeInfoTest, PeerOutsideFlatTeamBoundaries_ReturnsInvalidArgument) { // flatStart = comm->rank - flatTeam.rank; valid range is [flatStart, flatStart + nRanks). + // The boundary check runs before baseLeId is ever read, so no valid baseId is needed for + // the two out-of-range probes below. 20 and 7 alone don't pin either exact edge though: a + // relaxed `>=` -> `>` survives because 12 (the first rejected value above the range) is + // never probed, and a relaxed `<` -> `<=` survives because 8 (the first accepted value) is + // never probed either. 12 closes the first; 8 (further down, where it can succeed) closes + // the second. comm->rank = 10; comm->devrState.cftSize = 4; comm->devrState.cftSelf = 2; // flatTeam.rank == 2 -> flatStart == 8, valid peers in [8, 12) - comm->devrState.le[0].baseId = 1; // must be valid so we reach the boundary check ncclCftLeId leId; size_t leOffset; EXPECT_EQ(ncclGetPeerDeviceLeInfo(window, 0, /*peerWorld=*/20, &leId, &leOffset), ncclInvalidArgument); EXPECT_EQ(ncclGetPeerDeviceLeInfo(window, 0, /*peerWorld=*/7, &leId, &leOffset), ncclInvalidArgument); + EXPECT_EQ(ncclGetPeerDeviceLeInfo(window, 0, /*peerWorld=*/12, &leId, &leOffset), ncclInvalidArgument); + + // peerWorld=8 is the lower inclusive edge: it must succeed, not reject. This path proceeds + // past the boundary check into the baseLeId read, so (unlike the three probes above) it + // needs a valid baseId. + comm->devrState.le[0].baseId = 1; + ASSERT_EQ(ncclGetPeerDeviceLeInfo(window, 0, /*peerWorld=*/8, &leId, &leOffset), ncclSuccess); + EXPECT_EQ(leId, 1u); // baseLeId(1) + flatTeam.rank(2) + (peerWorld(8) - comm->rank(10)) == 1 +} + +TEST_F(NcclGetPeerDeviceLeInfoTest, CountedWindow_UsesCountedBaseLeIdSlot) { + comm->rank = 10; + comm->devrState.cftSize = 4; + comm->devrState.cftSelf = 2; // flatStart = 8, valid peers in [8, 12) + comm->devrState.le[0].baseId = NCCL_LE_ID_INVALID; // uncounted slot deliberately unset + comm->devrState.le[1].baseId = 2000; // counted slot is what must be read + winHost.winFlags = NCCL_WIN_CFT_COUNTED; + + ncclCftLeId leId = 0; + size_t leOffset = 0; + // *leId = baseLeId + flatTeam.rank + (peerWorld - comm->rank) = 2000 + 2 + (11 - 10) = 2003 + ASSERT_EQ(ncclGetPeerDeviceLeInfo(window, 0, /*peerWorld=*/11, &leId, &leOffset), ncclSuccess); + EXPECT_EQ(leId, 2003u); } TEST_F(NcclGetPeerDeviceLeInfoTest, BaseLeIdNotYetCreated_ReturnsInvalidUsage) { From 50a8f37196ddc28638ae8d044cc5a5fa5fe3f4e0 Mon Sep 17 00:00:00 2001 From: Alex Duforest Date: Tue, 13 Oct 2026 12:13:47 -0700 Subject: [PATCH 16/26] test(rccl): pin CFT registration args in all LE-info suites, symmetric-only and 13029 probes, dedupe prologue tests Co-Authored-By: Claude --- .../test/host/nccl-cft-dev-runtime-test.cc | 180 +++++++++++------- 1 file changed, 108 insertions(+), 72 deletions(-) diff --git a/projects/rccl/test/host/nccl-cft-dev-runtime-test.cc b/projects/rccl/test/host/nccl-cft-dev-runtime-test.cc index d4f95d4b1944..a41ae9a014d0 100644 --- a/projects/rccl/test/host/nccl-cft-dev-runtime-test.cc +++ b/projects/rccl/test/host/nccl-cft-dev-runtime-test.cc @@ -4,8 +4,8 @@ * See LICENSE.txt for license information ************************************************************************/ -// Host-only microtests for src/cft_dev_runtime.cc: the CFT (confidential/ -// fault-tolerant?) logical-endpoint support-probe, size-computation, and +// Host-only microtests for src/cft_dev_runtime.cc: the CFT (Compute Fabric +// Transport) logical-endpoint support-probe, size-computation, and // bind/unbind helpers, plus the three public LE-info query functions. This // build's CUDA_VERSION is below 13030, so the real CUDA logical-endpoint // driver calls are compiled out everywhere in this file (no ROCm @@ -122,6 +122,18 @@ TEST_F(NcclCftDevRuntimeMicrotest, ComputeCftSize_NotSizedWithGpuCftSupport_Dele EXPECT_EQ(computeCftSize(comm), 8); } +TEST_F(NcclCftDevRuntimeMicrotest, ComputeCftSizeAndMcSize_GpuCftSupportJustBelow13030_ReturnsOne) { + // Every other probe uses 0 or exactly 13030, far from the `>= 13030` edge at + // cft_dev_runtime.cc:54 and 61; 13029 is the first value it must reject. + comm->devrState.bigSize = 0; + comm->gpuCftSupport = 13029; + bool called = false; + g_computeLsaSize = [&](ncclComm*) { called = true; return 99; }; + EXPECT_EQ(computeCftSize(comm), 1); + EXPECT_EQ(computeCftMcSize(comm), 1); + EXPECT_FALSE(called); +} + TEST_F(NcclCftDevRuntimeMicrotest, ComputeCftMcSize_AlreadySized_ReturnsCftMcSizeWithoutCallingComputeLsaSize) { comm->devrState.bigSize = 7; comm->devrState.cftMcSize = 24; @@ -249,8 +261,15 @@ class NcclGetDeviceLeInfoTestBase : public NcclCftDevRuntimeMicrotest { ncclDevrWindow winHost{}; int dummyWindowTarget = 0; ncclWindow_t window = reinterpret_cast(&dummyWindowTarget); + // What the ncclDevrWinRegEnabled seam was last asked, and what it answers. + int sawWinFlags = -1; + ncclDevrRegisterCapability sawCapability = ncclDevrRegisterAll; + bool winRegEnabled = true; virtual size_t BigOffset() const = 0; + // Calls this suite's query function with its own extra arguments held fixed, + // so the shared prologue checks below run against all three functions. + virtual ncclResult_t Invoke(ncclWindow_t w, ncclCftLeId* leId, size_t* leOffset) = 0; void SetUp() override { NcclCftDevRuntimeMicrotest::SetUp(); @@ -259,6 +278,49 @@ class NcclGetDeviceLeInfoTestBase : public NcclCftDevRuntimeMicrotest { g_foundComm = comm; g_foundWinHost = &winHost; UseFoundCommAndWindow(); + g_devrWinRegEnabled = [this](int winFlags, ncclDevrRegisterCapability capability) { + sawWinFlags = winFlags; + sawCapability = capability; + return winRegEnabled; + }; + } + + // The prologue below is identical across the three query functions + // (cft_dev_runtime.cc:295-309 and its two twins). + void ExpectNullArgsRejectedByPtrCheck() { + ncclCftLeId leId; + size_t leOffset; + EXPECT_EQ(Invoke(nullptr, &leId, &leOffset), ncclInvalidArgument); + EXPECT_EQ(Invoke(window, nullptr, &leOffset), ncclInvalidArgument); + EXPECT_EQ(Invoke(window, &leId, nullptr), ncclInvalidArgument); + } + + void ExpectWindowNotCftRegisteredReturnsInvalidUsage() { + // Also pins what the registration check asks: this window's own flags, + // against the CFT capability rather than e.g. ncclDevrRegisterAll. + winHost.winFlags = 0x5a; + winRegEnabled = false; + ncclCftLeId leId; + size_t leOffset; + EXPECT_EQ(Invoke(window, &leId, &leOffset), ncclInvalidUsage); + EXPECT_EQ(sawWinFlags, 0x5a); + EXPECT_EQ(sawCapability, ncclDevrRegisterCft); + } + + void ExpectNoGpuCftSupportReturnsInvalidArgument() { + comm->gpuCftSupport = 0; + ncclCftLeId leId; + size_t leOffset; + EXPECT_EQ(Invoke(window, &leId, &leOffset), ncclInvalidArgument); + } + + void ExpectFindCommAndHostWindowFailurePropagates() { + g_findCommAndHostWindowFromDeviceWindow = [](ncclWindow_t, ncclComm_t*, ncclDevrWindow**) { + return ncclSystemError; + }; + ncclCftLeId leId; + size_t leOffset; + EXPECT_EQ(Invoke(window, &leId, &leOffset), ncclSystemError); } }; @@ -269,37 +331,22 @@ class NcclGetDeviceLeInfoTestBase : public NcclCftDevRuntimeMicrotest { class NcclGetMultimemDeviceLeInfoTest : public NcclGetDeviceLeInfoTestBase { protected: size_t BigOffset() const override { return 0x1000; } + ncclResult_t Invoke(ncclWindow_t w, ncclCftLeId* leId, size_t* leOffset) override { + return ncclGetMultimemDeviceLeInfo(w, 0, leId, leOffset); + } }; TEST_F(NcclGetMultimemDeviceLeInfoTest, NullArgs_RejectedByPtrCheckBeforeAnyLookup) { - ncclCftLeId leId; - size_t leOffset; - EXPECT_EQ(ncclGetMultimemDeviceLeInfo(nullptr, 0, &leId, &leOffset), ncclInvalidArgument); - EXPECT_EQ(ncclGetMultimemDeviceLeInfo(window, 0, nullptr, &leOffset), ncclInvalidArgument); - EXPECT_EQ(ncclGetMultimemDeviceLeInfo(window, 0, &leId, nullptr), ncclInvalidArgument); + ExpectNullArgsRejectedByPtrCheck(); } - TEST_F(NcclGetMultimemDeviceLeInfoTest, WindowNotCftRegistered_ReturnsInvalidUsage) { - g_devrWinRegEnabled = [](int, ncclDevrRegisterCapability) { return false; }; - ncclCftLeId leId; - size_t leOffset; - EXPECT_EQ(ncclGetMultimemDeviceLeInfo(window, 0, &leId, &leOffset), ncclInvalidUsage); + ExpectWindowNotCftRegisteredReturnsInvalidUsage(); } - TEST_F(NcclGetMultimemDeviceLeInfoTest, CommHasNoGpuCftSupport_ReturnsInvalidArgument) { - comm->gpuCftSupport = 0; - ncclCftLeId leId; - size_t leOffset; - EXPECT_EQ(ncclGetMultimemDeviceLeInfo(window, 0, &leId, &leOffset), ncclInvalidArgument); + ExpectNoGpuCftSupportReturnsInvalidArgument(); } - TEST_F(NcclGetMultimemDeviceLeInfoTest, FindCommAndHostWindowFails_PropagatesErrorViaNcclcheck) { - g_findCommAndHostWindowFromDeviceWindow = [](ncclWindow_t, ncclComm_t*, ncclDevrWindow**) { - return ncclSystemError; - }; - ncclCftLeId leId; - size_t leOffset; - EXPECT_EQ(ncclGetMultimemDeviceLeInfo(window, 0, &leId, &leOffset), ncclSystemError); + ExpectFindCommAndHostWindowFailurePropagates(); } TEST_F(NcclGetMultimemDeviceLeInfoTest, SymTeamObtainFails_PropagatesErrorViaNcclcheck) { @@ -373,6 +420,29 @@ TEST_F(NcclGetMultimemDeviceLeInfoTest, UncountedWindow_ReadsMcLeIdZeroSlot) { EXPECT_EQ(leId, 55u); } +TEST_F(NcclGetMultimemDeviceLeInfoTest, SymmetricOnlyWindow_IsUncountedAndReadsMcLeIdZeroSlot) { + // NCCL_WIN_COLL_SYMMETRIC without NCCL_WIN_CFT_COUNTED: a counted-bit mask + // that tested the companion bit instead would read the counted slot here. + comm->devrState.cftMcSize = 4; + comm->devrState.cftMcSelf = 0; + winHost.winFlags = NCCL_WIN_COLL_SYMMETRIC; + + DevrTeamPtr team = MakeDevrTeam(ncclTeam_t{4, 0, 1}, /*mcLeId0=*/77, /*mcLeId1=*/88); + ncclDevrTeam* teamPtr = team.get(); + g_symTeamObtain = [teamPtr](ncclComm*, ncclTeam, bool, bool counted, bool, bool, ncclDevrTeam** outTeam, + bool* needBarrier) { + EXPECT_FALSE(counted); + *outTeam = teamPtr; + *needBarrier = false; + return ncclSuccess; + }; + + ncclCftLeId leId = 0; + size_t leOffset = 0; + ASSERT_EQ(ncclGetMultimemDeviceLeInfo(window, /*offset=*/0, &leId, &leOffset), ncclSuccess); + EXPECT_EQ(leId, 77u); +} + // --------------------------------------------------------------------------- // ncclGetCftDeviceLeInfo // --------------------------------------------------------------------------- @@ -380,41 +450,22 @@ TEST_F(NcclGetMultimemDeviceLeInfoTest, UncountedWindow_ReadsMcLeIdZeroSlot) { class NcclGetCftDeviceLeInfoTest : public NcclGetDeviceLeInfoTestBase { protected: size_t BigOffset() const override { return 0x2000; } + ncclResult_t Invoke(ncclWindow_t w, ncclCftLeId* leId, size_t* leOffset) override { + return ncclGetCftDeviceLeInfo(w, 0, /*peerCft=*/0, ncclTeam_t{4, 0, 1}, leId, leOffset); + } }; TEST_F(NcclGetCftDeviceLeInfoTest, NullArgs_RejectedByPtrCheck) { - ncclTeam_t cftTeam{4, 0, 1}; - ncclCftLeId leId; - size_t leOffset; - EXPECT_EQ(ncclGetCftDeviceLeInfo(nullptr, 0, 0, cftTeam, &leId, &leOffset), ncclInvalidArgument); - EXPECT_EQ(ncclGetCftDeviceLeInfo(window, 0, 0, cftTeam, nullptr, &leOffset), ncclInvalidArgument); - EXPECT_EQ(ncclGetCftDeviceLeInfo(window, 0, 0, cftTeam, &leId, nullptr), ncclInvalidArgument); + ExpectNullArgsRejectedByPtrCheck(); } - TEST_F(NcclGetCftDeviceLeInfoTest, WindowNotCftRegistered_ReturnsInvalidUsage) { - g_devrWinRegEnabled = [](int, ncclDevrRegisterCapability) { return false; }; - ncclTeam_t cftTeam{4, 0, 1}; - ncclCftLeId leId; - size_t leOffset; - EXPECT_EQ(ncclGetCftDeviceLeInfo(window, 0, 0, cftTeam, &leId, &leOffset), ncclInvalidUsage); + ExpectWindowNotCftRegisteredReturnsInvalidUsage(); } - TEST_F(NcclGetCftDeviceLeInfoTest, CommHasNoGpuCftSupport_ReturnsInvalidArgument) { - comm->gpuCftSupport = 0; - ncclTeam_t cftTeam{4, 0, 1}; - ncclCftLeId leId; - size_t leOffset; - EXPECT_EQ(ncclGetCftDeviceLeInfo(window, 0, 0, cftTeam, &leId, &leOffset), ncclInvalidArgument); + ExpectNoGpuCftSupportReturnsInvalidArgument(); } - TEST_F(NcclGetCftDeviceLeInfoTest, FindCommAndHostWindowFails_PropagatesErrorViaNcclcheck) { - g_findCommAndHostWindowFromDeviceWindow = [](ncclWindow_t, ncclComm_t*, ncclDevrWindow**) { - return ncclSystemError; - }; - ncclTeam_t cftTeam{4, 0, 1}; - ncclCftLeId leId; - size_t leOffset; - EXPECT_EQ(ncclGetCftDeviceLeInfo(window, 0, 0, cftTeam, &leId, &leOffset), ncclSystemError); + ExpectFindCommAndHostWindowFailurePropagates(); } TEST_F(NcclGetCftDeviceLeInfoTest, BaseLeIdNotYetCreated_ReturnsInvalidUsage) { @@ -465,37 +516,22 @@ TEST_F(NcclGetCftDeviceLeInfoTest, CountedWindow_UsesCountedBaseLeIdSlot) { class NcclGetPeerDeviceLeInfoTest : public NcclGetDeviceLeInfoTestBase { protected: size_t BigOffset() const override { return 0x3000; } + ncclResult_t Invoke(ncclWindow_t w, ncclCftLeId* leId, size_t* leOffset) override { + return ncclGetPeerDeviceLeInfo(w, 0, /*peerWorld=*/0, leId, leOffset); + } }; TEST_F(NcclGetPeerDeviceLeInfoTest, NullArgs_RejectedByPtrCheck) { - ncclCftLeId leId; - size_t leOffset; - EXPECT_EQ(ncclGetPeerDeviceLeInfo(nullptr, 0, 0, &leId, &leOffset), ncclInvalidArgument); - EXPECT_EQ(ncclGetPeerDeviceLeInfo(window, 0, 0, nullptr, &leOffset), ncclInvalidArgument); - EXPECT_EQ(ncclGetPeerDeviceLeInfo(window, 0, 0, &leId, nullptr), ncclInvalidArgument); + ExpectNullArgsRejectedByPtrCheck(); } - TEST_F(NcclGetPeerDeviceLeInfoTest, WindowNotCftRegistered_ReturnsInvalidUsage) { - g_devrWinRegEnabled = [](int, ncclDevrRegisterCapability) { return false; }; - ncclCftLeId leId; - size_t leOffset; - EXPECT_EQ(ncclGetPeerDeviceLeInfo(window, 0, 0, &leId, &leOffset), ncclInvalidUsage); + ExpectWindowNotCftRegisteredReturnsInvalidUsage(); } - TEST_F(NcclGetPeerDeviceLeInfoTest, CommHasNoGpuCftSupport_ReturnsInvalidArgument) { - comm->gpuCftSupport = 0; - ncclCftLeId leId; - size_t leOffset; - EXPECT_EQ(ncclGetPeerDeviceLeInfo(window, 0, 0, &leId, &leOffset), ncclInvalidArgument); + ExpectNoGpuCftSupportReturnsInvalidArgument(); } - TEST_F(NcclGetPeerDeviceLeInfoTest, FindCommAndHostWindowFails_PropagatesErrorViaNcclcheck) { - g_findCommAndHostWindowFromDeviceWindow = [](ncclWindow_t, ncclComm_t*, ncclDevrWindow**) { - return ncclSystemError; - }; - ncclCftLeId leId; - size_t leOffset; - EXPECT_EQ(ncclGetPeerDeviceLeInfo(window, 0, 0, &leId, &leOffset), ncclSystemError); + ExpectFindCommAndHostWindowFailurePropagates(); } TEST_F(NcclGetPeerDeviceLeInfoTest, PeerOutsideFlatTeamBoundaries_ReturnsInvalidArgument) { From 60262dd4e910f02c6c58b31bf125ea434d8990d9 Mon Sep 17 00:00:00 2001 From: Alex Duforest Date: Tue, 13 Oct 2026 13:53:22 -0700 Subject: [PATCH 17/26] test(rccl): seed a non-degenerate CFT team so peer prologue checks are reached, drop a vacuous flag Co-Authored-By: Claude --- projects/rccl/test/host/nccl-cft-dev-runtime-test.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/projects/rccl/test/host/nccl-cft-dev-runtime-test.cc b/projects/rccl/test/host/nccl-cft-dev-runtime-test.cc index a41ae9a014d0..4623f897afff 100644 --- a/projects/rccl/test/host/nccl-cft-dev-runtime-test.cc +++ b/projects/rccl/test/host/nccl-cft-dev-runtime-test.cc @@ -274,6 +274,9 @@ class NcclGetDeviceLeInfoTestBase : public NcclCftDevRuntimeMicrotest { void SetUp() override { NcclCftDevRuntimeMicrotest::SetUp(); comm->gpuCftSupport = 13030; + // Non-degenerate flat CFT team: at cftSize 0 the peer query's range check rejects every + // peer first, so the shared prologue checks below would never reach the check they name. + comm->devrState.cftSize = 4; winHost.bigOffset = BigOffset(); g_foundComm = comm; g_foundWinHost = &winHost; @@ -362,7 +365,7 @@ TEST_F(NcclGetMultimemDeviceLeInfoTest, SymTeamObtainFails_PropagatesErrorViaNcc // path) is not independently exercised here: fakes/bootstrap_stubs.cc's // bootstrapIntraNodeBarrier is a parameterless-result `{ return ncclSuccess; }` stub // shared by every other suite that links it, with no failure hook. Giving it one is out -// of scope for this file's tests -- see the PR description's coverage note. +// of scope for this file's tests. TEST_F(NcclGetMultimemDeviceLeInfoTest, SuccessPath_ReadsMcLeIdByCountedBitAndOffsetsByBigOffsetPlusOffset) { comm->devrState.cftMcSize = 4; @@ -376,9 +379,8 @@ TEST_F(NcclGetMultimemDeviceLeInfoTest, SuccessPath_ReadsMcLeIdByCountedBitAndOf DevrTeamPtr team = MakeDevrTeam(ncclTeam_t{4, 1, 1}, /*mcLeId0=*/111, /*mcLeId1=*/222); // counted==true selects mcLeId1 ncclDevrTeam* teamPtr = team.get(); - bool sawNeedBarrierRequested = false; - g_symTeamObtain = [&, teamPtr](ncclComm*, ncclTeam team, bool multimem, bool counted, bool wantsLeUc, - bool wantsLeMc, ncclDevrTeam** outTeam, bool* needBarrier) { + g_symTeamObtain = [teamPtr](ncclComm*, ncclTeam team, bool multimem, bool counted, bool wantsLeUc, + bool wantsLeMc, ncclDevrTeam** outTeam, bool* needBarrier) { EXPECT_FALSE(multimem); EXPECT_TRUE(counted); // from winHost.winFlags & NCCL_WIN_CFT_COUNTED EXPECT_FALSE(wantsLeUc); @@ -387,14 +389,12 @@ TEST_F(NcclGetMultimemDeviceLeInfoTest, SuccessPath_ReadsMcLeIdByCountedBitAndOf EXPECT_EQ(team.rank, 1); *outTeam = teamPtr; *needBarrier = true; - sawNeedBarrierRequested = true; return ncclSuccess; }; ncclCftLeId leId = 0; size_t leOffset = 0; ASSERT_EQ(ncclGetMultimemDeviceLeInfo(window, /*offset=*/0x20, &leId, &leOffset), ncclSuccess); - EXPECT_TRUE(sawNeedBarrierRequested); EXPECT_EQ(leId, 222u); EXPECT_EQ(leOffset, winHost.bigOffset + 0x20); } From 8dedee9eb253fac5df6f12fe5bdcba290329ab2f Mon Sep 17 00:00:00 2001 From: Alex Duforest Date: Wed, 14 Oct 2026 13:38:32 -0700 Subject: [PATCH 18/26] docs(rccl): record cft_dev_runtime_fakes.cc in the MICROTEST_README fakes table Co-Authored-By: Claude --- projects/rccl/test/host/MICROTEST_README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/projects/rccl/test/host/MICROTEST_README.md b/projects/rccl/test/host/MICROTEST_README.md index b5f31cbfa944..e2b6cf07dfbc 100644 --- a/projects/rccl/test/host/MICROTEST_README.md +++ b/projects/rccl/test/host/MICROTEST_README.md @@ -460,6 +460,10 @@ had become before this map existed. dependency of another unit (`rccl-UnitTestsMicroEnqueue` doesn't compile it, so it fakes the file's own entry points). The two never define the same symbol. +`src/dev_runtime.cc` has two rows for a narrower reason: `rccl-UnitTestsMicroNcclDeviceCft` +needs only the four `dev_runtime.cc` symbols `cft_dev_runtime.cc` calls, kept in their own +file so that target does not drag in the rest of `fakes/dev_runtime_fakes.cc`'s seams. +The two never define the same symbol either. | Production TU | Fakes file | |---|---| @@ -468,6 +472,7 @@ symbol. | `src/ce_coll.cc` (targets that do not compile the real file) | `fakes/ce_fakes.cc` | | `src/collectives.cc` | `fakes/collectives_fakes.cc` | | `src/dev_runtime.cc` (targets that do not compile the real file) | `fakes/dev_runtime_fakes.cc` | +| `src/dev_runtime.cc`'s callees from `src/cft_dev_runtime.cc` (`computeLsaSize`, `symTeamObtain`, `findCommAndHostWindowFromDeviceWindow`, `ncclDevrWinRegEnabled`; `rccl-UnitTestsMicroNcclDeviceCft`, which compiles the real `cft_dev_runtime.cc`) | `fakes/cft_dev_runtime_fakes.cc` | | `src/diagnostics.cc` (`ncclDiagChildRun*` external-tool runners) | `fakes/diagnostics_fakes.cc` | | `src/diagnostics/device/p2p.cu` (`ncclDiagP2p*` kernel launchers) | `fakes/diagnostics_p2p_device_fakes.cc` | | `src/graph/*.cc` (topo, paths, search, connect, rome consensus) | `fakes/topo_stubs.cc` | From d53bedee4495b660e9e81bf25326bee314dbc118 Mon Sep 17 00:00:00 2001 From: Alex Duforest Date: Wed, 14 Oct 2026 15:39:52 -0700 Subject: [PATCH 19/26] test(rccl): trim microtest docs and comments to match the other host tests Shorten the MICROTEST_README entry, CMake comments and test-file comments to the length the other host microtests use. Comment and doc changes only. Co-Authored-By: Claude --- projects/rccl/test/host/CMakeLists.txt | 6 +- projects/rccl/test/host/MICROTEST_README.md | 20 +--- .../host/nccl-device-requirements-test.cc | 98 +++++-------------- 3 files changed, 30 insertions(+), 94 deletions(-) diff --git a/projects/rccl/test/host/CMakeLists.txt b/projects/rccl/test/host/CMakeLists.txt index cef6d9270e02..beae08289865 100644 --- a/projects/rccl/test/host/CMakeLists.txt +++ b/projects/rccl/test/host/CMakeLists.txt @@ -293,10 +293,8 @@ function(rccl_define_micro_source_lists) ../common/main_altrsmi.cpp PARENT_SCOPE) - # rccl-UnitTestsMicroNcclDevice -- src/nccl_device/{cft_barrier,gin_barrier,gin_scratch,ll_a2a, - # lsa_barrier}.cc. Its own binary: fakes/dev_runtime_micro_fakes.cc (needed by dev-runtime-test.cc - # in rccl-UnitTestsMicro) fakes ncclLsaBarrierCreateRequirement/ncclGinBarrierCreateRequirement/ - # ncclCftBarrierCreateRequirement, which this target compiles as the real thing under test. + # rccl-UnitTestsMicroNcclDevice -- src/nccl_device/ requirement builders. Own binary: + # fakes/dev_runtime_micro_fakes.cc fakes three of them in rccl-UnitTestsMicro. set(TEST_MICRO_NCCL_DEVICE_SOURCE_FILES nccl-device-requirements-test.cc ../common/main_altrsmi.cpp diff --git a/projects/rccl/test/host/MICROTEST_README.md b/projects/rccl/test/host/MICROTEST_README.md index 525e22166349..155f789aa876 100644 --- a/projects/rccl/test/host/MICROTEST_README.md +++ b/projects/rccl/test/host/MICROTEST_README.md @@ -248,21 +248,11 @@ export colliding non-`static` symbols; otherwise a unit needs its own binary: guard-less positional pair `libc_seam.h` / `libc_seam_undef.h` must bracket the production source include, after system headers and before test helpers. CMake pins the copied `ncclFuncStr` table to `src/init.cc`. -- **`rccl-UnitTestsMicroNcclDevice`**: the six device-resource requirement - builders in `src/nccl_device/{cft_barrier,gin_barrier,gin_scratch,ll_a2a, - lsa_barrier}.cc` (via `CFT_BARRIER_CC_PATH`, `GIN_BARRIER_CC_PATH`, - `GIN_SCRATCH_CC_PATH`, `LL_A2A_CC_PATH`, `LSA_BARRIER_CC_PATH`, from - `nccl-device-requirements-test.cc`); suite `NcclDeviceRequirementsMicrotest.*`. - Its own binary: `fakes/dev_runtime_micro_fakes.cc` (needed by - `dev-runtime-test.cc` in `rccl-UnitTestsMicro`) fakes - `ncclLsaBarrierCreateRequirement`/`ncclGinBarrierCreateRequirement`/ - `ncclCftBarrierCreateRequirement`, which this target compiles as the real - thing under test. These functions are pure host computation (no HIP/device - calls), so five of the six port directly from the existing GPU-binary - coverage in `test/DeviceResourceRequirementTests.cpp` (AICOMRCCL-2446, - including its `ProcessIsolatedTestRunner`-based divide-by-zero crash pin); - the sixth, `ncclCftBarrierCreateRequirement`, had no coverage anywhere and is - new here. +- **`rccl-UnitTestsMicroNcclDevice`**: the requirement builders in + `src/nccl_device/{cft_barrier,gin_barrier,gin_scratch,ll_a2a,lsa_barrier}.cc` + (via `_CC_PATH`, from `nccl-device-requirements-test.cc`); suite + `NcclDeviceRequirementsMicrotest.*`. Its own binary because + `fakes/dev_runtime_micro_fakes.cc` fakes three of these builders. - **`rccl-UnitTestsMicroSymKernels`** — the REAL `src/sym_kernels.cc` (via `SYM_KERNELS_CC_PATH`, from `sym-kernels-test.cc`), compiled together with the GENERATED `sym_kernels_host.cc` it calls into; suites `SymKernelMicrotest.*`, diff --git a/projects/rccl/test/host/nccl-device-requirements-test.cc b/projects/rccl/test/host/nccl-device-requirements-test.cc index 1aba34afeace..be59fb4328c4 100644 --- a/projects/rccl/test/host/nccl-device-requirements-test.cc +++ b/projects/rccl/test/host/nccl-device-requirements-test.cc @@ -4,18 +4,9 @@ * See LICENSE.txt for license information ************************************************************************/ -// Host-only microtests for the six device-resource requirement builders in -// src/nccl_device/{cft_barrier,gin_barrier,gin_scratch,ll_a2a,lsa_barrier}.cc. -// They compute buffer sizes and GIN signal counts that device kernels index -// into; an off-by-one here under-allocates scratch and surfaces as device -// corruption. Five of the six (everything but ncclCftBarrierCreateRequirement) -// already have GPU-binary coverage in test/DeviceResourceRequirementTests.cpp -// (AICOMRCCL-2446); this file ports those same goldens host-only rather than -// duplicating the reasoning, since the functions under test are pure host -// computation with no HIP/device calls. Both copies are built from test/CMakeLists.txt; the GPU -// copy runs from ci-precheckin.json and this one from test/host/run_host_tests.sh. They have -// already drifted at a few lines; retiring the GPU-binary copy once this suite has run clean in CI for -// a while is tracked separately in AICOMRCCL-2912, not done here. +// Host-only microtests for the device-resource requirement builders in +// src/nccl_device/{cft_barrier,gin_barrier,gin_scratch,ll_a2a,lsa_barrier}.cc. Goldens are +// ported from test/DeviceResourceRequirementTests.cpp. #include @@ -75,16 +66,14 @@ void ExpectNoGinSignalFields(const ncclDevResourceRequirements_t& req) { EXPECT_EQ(req.outGinCounterStart, nullptr); } -// GIN-counter fields untouched by the signal-setting builders; ginSignalCount/outGinSignalStart -// are set and checked separately by each test. +// GIN-counter fields untouched by the signal-setting builders. void ExpectNoGinCounterFields(const ncclDevResourceRequirements_t& req) { EXPECT_EQ(req.next, nullptr); EXPECT_EQ(req.ginCounterCount, 0); EXPECT_EQ(req.outGinCounterStart, nullptr); } -// gtest's default death-test style forks without re-exec, so clear this inside the death statement: -// only the child is affected. ci-precheckin.json builds Debug with coverage on, so a dump is sizable. +// Only the forked child is affected: skip its core dump. void DisableCoreDumpsForExpectedCrash() { #if defined(__linux__) if (prctl(PR_SET_DUMPABLE, 0) != 0) { @@ -101,9 +90,7 @@ void DisableCoreDumpsForExpectedCrash() { } // namespace -// --------------------------------------------------------------------------- // ncclLLA2ACalcSlots / ncclLLA2ACreateRequirement (src/nccl_device/ll_a2a.cc) -// --------------------------------------------------------------------------- // divUp rounding: maxEltSize of 1, 7, 8 all round up to one 8-byte slot; 9 and 16 need two. TEST(NcclDeviceRequirementsMicrotest, LLA2ACalcSlots_DivUpRounding) { @@ -121,13 +108,10 @@ TEST(NcclDeviceRequirementsMicrotest, LLA2ACalcSlots_DivUpRounding) { } } -// Scales linearly with maxElts against a known-correct literal, not ncclLLA2ACalcSlots's own -// output: that would hold for any maxElts * g(maxEltSize) shape, including a wrong g. +// Against literals, not ncclLLA2ACalcSlots's own output. TEST(NcclDeviceRequirementsMicrotest, LLA2ACalcSlots_LinearInMaxElts) { EXPECT_EQ(ncclLLA2ACalcSlots(/*maxElts=*/5, /*maxEltSize=*/8), 5); - // maxElts=2, maxEltSize=4: correct is maxElts * divUp(maxEltSize, 8) == 2 * 1 == 2; hoisting the - // divUp outside the multiply (divUp(maxElts * maxEltSize, 8) == divUp(8, 8)) would give 1 instead. - // The row above can't catch that: 5 * divUp(8,8) == divUp(5*8,8) == 5 either way. + // 2 * divUp(4, 8) == 2; hoisting the divUp, divUp(2 * 4, 8), would give 1. EXPECT_EQ(ncclLLA2ACalcSlots(/*maxElts=*/2, /*maxEltSize=*/4), 2); } @@ -136,8 +120,7 @@ TEST(NcclDeviceRequirementsMicrotest, LLA2ACalcSlots_ZeroMaxEltsIsZero) { EXPECT_EQ(ncclLLA2ACalcSlots(/*maxElts=*/0, /*maxEltSize=*/8), 0); } -// bufferSize == nBlocks * (1 + 2*nSlots) * 16; nSlots == 0 is the header-only case. Values stay -// sane on purpose: the plain-int formula can overflow before promoting to size_t, and pinning UB is out of scope. +// bufferSize == nBlocks * (1 + 2*nSlots) * 16; nSlots == 0 is the header-only case. TEST(NcclDeviceRequirementsMicrotest, LLA2ACreateRequirement_BufferSizeFormula) { struct Case { const char* name; @@ -158,14 +141,12 @@ TEST(NcclDeviceRequirementsMicrotest, LLA2ACreateRequirement_BufferSizeFormula) EXPECT_EQ(req.bufferAlign, 16u); EXPECT_EQ(req.outBufferHandle, &handle.bufHandle); EXPECT_EQ(handle.nSlots, static_cast(c.nSlots)); - // Fields this function never sets must still read back zero: proves the memset prologue ran. + // Unset fields read back zero: the memset ran. ExpectNoGinSignalFields(req); } } -// Realistic-usage row (mirrors src/sym_kernels.cc), not a contract. Expected bufferSize is a -// literal rather than a formula built from nSlots, since nSlots is the function under test's own -// output and would hold regardless of its value. ncclLLA2ACalcSlots(4, 16) == 8. +// Realistic usage (mirrors src/sym_kernels.cc); ncclLLA2ACalcSlots(4, 16) == 8. TEST(NcclDeviceRequirementsMicrotest, LLA2ACreateRequirement_RoundTripWithCalcSlots) { const int nSlots = ncclLLA2ACalcSlots(/*maxElts=*/4, /*maxEltSize=*/16); const int nBlocks = 3; @@ -176,9 +157,7 @@ TEST(NcclDeviceRequirementsMicrotest, LLA2ACreateRequirement_RoundTripWithCalcSl EXPECT_EQ(handle.nSlots, static_cast(nSlots)); } -// --------------------------------------------------------------------------- // ncclGinOutboxCreateRequirement / ncclGinInboxA2ACreateRequirement (src/nccl_device/gin_scratch.cc) -// --------------------------------------------------------------------------- // The 128-byte floor: size_log2 at or below 7 clamps to 7, and outHandle reads back the clamped value. TEST(NcclDeviceRequirementsMicrotest, GinOutboxCreateRequirement_FloorClampsSizeLog2AtOrBelow7) { @@ -201,8 +180,6 @@ TEST(NcclDeviceRequirementsMicrotest, GinOutboxCreateRequirement_SizeLog2AboveFl } // bufferSize == nBlocks * (sizeof(state) + RequestBytes + alignUp(1<(); ncclDevResourceRequirements_t req = poisonedRequirement(); ASSERT_EQ(ncclGinInboxA2ACreateRequirement(peers, /*nBlocks=*/1, /*size_log2=*/0, &handle, &req), ncclSuccess); EXPECT_EQ(handle.size_log2, 0u) << "asymmetric with the outbox's 128-byte floor; no floor here today"; - // The only case anywhere in this file where alignUpLocal's rounding actually fires: every other - // size_log2 used elsewhere is already >= 7, so 1<(nPeers), handle.nPeers_rcp32), x % static_cast(nPeers)) << "x=" << x; @@ -291,27 +264,13 @@ TEST(NcclDeviceRequirementsMicrotest, GinInboxA2ACreateRequirement_BufferSizeAnd } } -// A 1-rank team makes nPeers = 0, and idivRcp32(0) (bitops.h) divides by zero at runtime: a real -// defect, unreachable from any in-tree caller (src/sym_kernels.cc guards nRanks >= 2 via -// computeLsaSize's GCD construction). This binary (rccl-UnitTestsMicroNcclDevice) is always built -// with --offload-host-only (cmake/RcclTargetHelpers.cmake:78), so __linux__ is defined and -// GTEST_HAS_DEATH_TEST is true here; pin the exact signal with EXPECT_EXIT, same idiom as -// init-test.cc's P2pSchedule_ZeroGroupSizeParam_DiesOnDivideByZero. -// -// peers/handle/req are built from a volatile int so the divide stays real at -O2 too: with plain -// automatic storage, inlining collapses ncclGinInboxA2ACreateRequirement's `idivRcp32(nPeers)` call -// into a compile-time-zero divisor, which is undefined behaviour an optimising build is free to -// fold away entirely rather than trap. -// -// x86_64-only: idivRcp32 is a runtime 64-bit divide, and only x86 raises #DE on divide-by-zero. -// AArch64 UDIV is defined to return 0 without trapping, so the same call would return ncclSuccess -// there, and this pin would misreport the still-present defect as fixed. Same guard idiom as -// src/proxy.cc:887. No aarch64 target exists in tools/scripts/test_runner/configs/ today. +// A 1-rank team makes nPeers = 0 and idivRcp32(0) divides by zero: a real defect, unreachable from +// in-tree callers (sym_kernels.cc needs nRanks >= 2). The volatile keeps the divide from being +// folded away at -O2. x86_64 only: AArch64 UDIV returns 0 instead of trapping. TEST(NcclDeviceRequirementsMicrotest, GinInboxA2ACreateRequirement_SingleRankTeam_CrashesOnDivideByZero) { #if defined(__x86_64__) volatile int nRanksVolatile = 1; - // Field-by-field assignment, not a braced aggregate init: a braced-init-list's top-level commas - // are not protected by the macro's parens and split EXPECT_EXIT's own argument list instead. + // Assigned field by field: a braced init's commas would split EXPECT_EXIT's arguments. EXPECT_EXIT( { DisableCoreDumpsForExpectedCrash(); @@ -321,7 +280,7 @@ TEST(NcclDeviceRequirementsMicrotest, GinInboxA2ACreateRequirement_SingleRankTea peers.stride = 1; ncclGinInboxA2AHandle handle = poisonedHandle(); ncclDevResourceRequirements_t req = poisonedRequirement(); - // Never expected to return: SIGFPE fires inside idivRcp32 before this line completes. + // Not reached: SIGFPE fires inside idivRcp32. ncclGinInboxA2ACreateRequirement(peers, /*nBlocks=*/1, /*size_log2=*/7, &handle, &req); }, ::testing::KilledBySignal(SIGFPE), ""); @@ -330,13 +289,9 @@ TEST(NcclDeviceRequirementsMicrotest, GinInboxA2ACreateRequirement_SingleRankTea #endif } -// --------------------------------------------------------------------------- // ncclLsaBarrierCreateRequirement (src/nccl_device/lsa_barrier.cc) -// --------------------------------------------------------------------------- // bufferSize == (3*nBarriers + nBarriers*team.nRanks) * sizeof(uint32_t); nBarriers == 0 gives 0. -// Same out-of-scope-overflow reasoning as the LLA2A formula above applies to this plain-int formula. -// goldenBufferSize hand-computes the same value independently of the formula below, same reasoning. TEST(NcclDeviceRequirementsMicrotest, LsaBarrierCreateRequirement_BufferSizeFormula) { struct Case { const char* name; @@ -366,12 +321,9 @@ TEST(NcclDeviceRequirementsMicrotest, LsaBarrierCreateRequirement_BufferSizeForm } } -// --------------------------------------------------------------------------- // ncclGinBarrierCreateRequirement (src/nccl_device/gin_barrier.cc) -// --------------------------------------------------------------------------- -// comm is unused, so nullptr is the contract that makes this testable without a GPU. Mirrors the -// LSA/CFT barriers: this one sets ginSignalCount but no buffer, those set a buffer but no signals. +// comm is unused, so nullptr works. Sets ginSignalCount but no buffer, unlike the LSA/CFT barriers. TEST(NcclDeviceRequirementsMicrotest, GinBarrierCreateRequirement_NullCommAndSignalCountFormula) { struct Case { const char* name; @@ -397,14 +349,10 @@ TEST(NcclDeviceRequirementsMicrotest, GinBarrierCreateRequirement_NullCommAndSig } } -// --------------------------------------------------------------------------- -// ncclCftBarrierCreateRequirement (src/nccl_device/cft_barrier.cc) -- not covered anywhere else -// --------------------------------------------------------------------------- +// ncclCftBarrierCreateRequirement (src/nccl_device/cft_barrier.cc) -// bufferSize == (3*nBarriers + nBarriers*team.nRanks) * NCCL_CFT_BARRIER_GRAN. Same shape as the -// LSA barrier above (both are buffer-only, no GIN signals), but a distinct granularity constant -// (NCCL_CFT_BARRIER_GRAN == NCCL_CFT_BARRIER_ALIGN == 16, vs. the LSA barrier's sizeof(uint32_t) == 4): -// swapping the two constants would still compile and this is the only place that would catch it. +// bufferSize == (3*nBarriers + nBarriers*team.nRanks) * NCCL_CFT_BARRIER_GRAN (16, vs. the LSA +// barrier's 4), so swapping the two constants fails here. TEST(NcclDeviceRequirementsMicrotest, CftBarrierCreateRequirement_BufferSizeFormula) { struct Case { const char* name; From f41e1d5ee67a35ce6efbb965ea333ec47f63e466 Mon Sep 17 00:00:00 2001 From: Alex Duforest Date: Wed, 14 Oct 2026 15:40:03 -0700 Subject: [PATCH 20/26] test(rccl): trim microtest docs and comments to match the other host tests Shorten the MICROTEST_README entry, CMake comments and test-file comments to the length the other host microtests use. Comment and doc changes only. Co-Authored-By: Claude --- projects/rccl/test/host/MICROTEST_README.md | 10 +- projects/rccl/test/host/nccl-utility-test.cc | 108 +++++-------------- 2 files changed, 27 insertions(+), 91 deletions(-) diff --git a/projects/rccl/test/host/MICROTEST_README.md b/projects/rccl/test/host/MICROTEST_README.md index 45c0ea2e12fc..88215181402e 100644 --- a/projects/rccl/test/host/MICROTEST_README.md +++ b/projects/rccl/test/host/MICROTEST_README.md @@ -80,14 +80,8 @@ export colliding non-`static` symbols; otherwise a unit needs its own binary: - `dev_runtime.cc` (`DEV_RUNTIME_CC_PATH`, from `dev-runtime-test.cc`); suites `Alloc*`, `Comm*`, `Compute*`, `DeepCopy*`, `Dev*`, `Gin*`, `Nccl*`, `Sym*`, and `Win*`. - - `src/include/nccl_device/utility.h` (header-only, from - `nccl-utility-test.cc`); suite `NcclUtilityMicrotest.*` (also caught by the - `Nccl*` pattern above). No `_CC_PATH` macro needed since it's a header, - not a `.cc` production TU, and no fakes since every targeted function - compiles cleanly in a host-only TU -- most are `NCCL_HOST_DEVICE_INLINE` - pure math with no HIP dependency; the four `memory_order` conversion - helpers sit inside `#if NCCL_CHECK_CUDACC`, which this build satisfies - because this target's `--offload-host-only` build still invokes hipcc. + - `include/nccl_device/utility.h` (header-only, from `nccl-utility-test.cc`); + suite `NcclUtilityMicrotest.*`. - the generated `sym_kernels_host.cc` (`SYM_KERNELS_HOST_CC_PATH`, from `sym-kernels-index-test.cc`); suites `SymKernelIndex*`, `SymAllEmittedCombinations/*`, `SymUnhandledCombinations/*` (covered by the diff --git a/projects/rccl/test/host/nccl-utility-test.cc b/projects/rccl/test/host/nccl-utility-test.cc index d3302371351d..716c707db603 100644 --- a/projects/rccl/test/host/nccl-utility-test.cc +++ b/projects/rccl/test/host/nccl-utility-test.cc @@ -4,16 +4,7 @@ * See LICENSE.txt for license information ************************************************************************/ -// Host-only microtests for the pure host/device math helpers in -// src/include/nccl_device/utility.h: rounding/alignment arithmetic, the -// fast-division reciprocal family, the memory_order conversion helpers, and -// the Optional/Present variadic construction helpers. Most targeted -// functions are NCCL_HOST_DEVICE_INLINE with no HIP/device-only dependency; -// acquireOrderOf/releaseOrderOf/toAtomicBuiltinOrder/toCudaOrder are the -// exception -- they sit inside `#if NCCL_CHECK_CUDACC`, which this build -// satisfies because both the real device build and this host-only -// `--offload-host-only` build still invoke hipcc. The real header compiles -// and runs unmodified in a host-only TU either way -- no fakes needed. +// Host-only microtests for the host/device math helpers in src/include/nccl_device/utility.h. #include @@ -22,9 +13,8 @@ #include "nccl_device/utility.h" -// bitops.h declares same-signature ::divUp, ::isPow2, ::add4G, ::idivRcp32 and more, so these -// global-scope using-declarations become ill-formed if it ever reaches this TU; qualify the calls -// instead if a new include pulls it in (test/BitOpsTests.cpp records the same trap for add4G). +// bitops.h declares the same ::divUp, ::isPow2, ... so these using-declarations break if it is +// ever included here; qualify the calls instead. using nccl::utility::acquireOrderOf; using nccl::utility::add4G; using nccl::utility::alignDown; @@ -56,8 +46,7 @@ using nccl::utility::toCudaOrder; namespace { -// Reference for idivFast64's truncation check: recomputes the division the -// ordinary way so the fast path can't share a bug with its own golden. +// Plain division, so the fast path can't share a bug with its golden. uint64_t refDiv64(uint64_t x, uint64_t y) { return x / y; } } // namespace @@ -81,8 +70,7 @@ TEST(NcclUtilityMicrotest, RoundUpDown_AreInverseDirectionsAroundAMultiple) { EXPECT_EQ(roundDown(8u, 8u), 8u); EXPECT_EQ(roundDown(15u, 8u), 8u); - // Non-power-of-two modulus: these use % y, not a power-of-two mask. Without - // this pair, modulus 8 alone can't tell `% y` apart from `& (y - 1)`. + // Non-power-of-two modulus, so `% y` can't pass as `& (y - 1)`. EXPECT_EQ(roundUp(5u, 3u), 6u); EXPECT_EQ(roundDown(7u, 3u), 6u); } @@ -97,20 +85,14 @@ TEST(NcclUtilityMicrotest, AlignUpDown_ScalarOverload_AssumesPowerOfTwoAlignment EXPECT_EQ(alignDown(15u, 16u), 0u); EXPECT_EQ(alignDown(16u, 16u), 16u); EXPECT_EQ(alignDown(31u, 16u), 16u); - // Above 2*a: below it, `x & -Z(a)` (utility.h:166) cannot be told from `x & Z(a)`. + // Above 2*a, where `x & -Z(a)` differs from `x & Z(a)`. EXPECT_EQ(alignDown(48u, 16u), 48u); EXPECT_EQ(alignDown(33u, 16u), 32u); } TEST(NcclUtilityMicrotest, AlignUpDown_PointerOverload_RoundsTheAddressNotTheByte) { alignas(32) char buf[64]; - // Single-byte-type overload: static_assert(sizeof(T) == 1) gates this at - // compile time, so the test itself is part of the contract being checked. - // buf + 3 rounds to buf + 16 under both `(addr + a - 1) & -uintptr_t(a)` and - // an off-by-one `(addr + a) & -uintptr_t(a)`, so the unaligned case alone - // cannot pin the `- 1`. An already-aligned pointer can: correct returns buf, - // the mutant returns buf + 16. alignDown has no `- 1` term, so only alignUp - // needs the extra case. + // An already-aligned pointer pins alignUp's `- 1`: buf + 3 rounds to buf + 16 either way. char* base = buf + 3; char* up = alignUp(base, size_t(16)); char* down = alignDown(base, size_t(16)); @@ -140,32 +122,22 @@ TEST(NcclUtilityMicrotest, RollingLessEqAndLessThan_WrapAroundHalfTheRangeCorrec EXPECT_TRUE(rollingLessEq(uint8_t(7), uint8_t(7), 8)); EXPECT_FALSE(rollingLessThan(uint8_t(7), uint8_t(7), 8)); - // The half-range boundary itself: m is 0xFF here, so m >> 1 is 127. A - // distance of exactly 127 is still "less or equal", 128 is not. + // The half-range boundary: 127 is still "less or equal", 128 is not. EXPECT_TRUE(rollingLessEq(uint8_t(0), uint8_t(127), 8)); EXPECT_FALSE(rollingLessEq(uint8_t(0), uint8_t(128), 8)); - // Every call above passes nBits == 8*sizeof(Uint) (a no-op mask), so the - // `m = Uint(-1) >> (8*sizeof(Uint) - nBits)` shift at utility.h:197 is - // never actually exercised -- production calls it with a narrower runtime - // width (gin_barrier__funcs.h:110 passes 32 for a uint64_t counter). Use a - // wider type with an explicit partial-width nBits so the mask is load-bearing: - // true only because the high 32 bits are masked out of the wraparound check. + // A partial nBits on a wider type, as production uses it (gin_barrier__funcs.h passes 32 for a + // uint64_t counter): true only because the high 32 bits are masked out. EXPECT_TRUE(rollingLessEq(uint64_t(0xFFFFFFF0ull), uint64_t(0x10ull), 32)); EXPECT_FALSE(rollingLessEq(uint64_t(0x10ull), uint64_t(0xFFFFFFF0ull), 32)); - // rollingLessThan must forward nBits rather than fall back to its own - // default: dropping the forward at utility.h:202 flips both of these. + // rollingLessThan must forward nBits: dropping it flips both. EXPECT_TRUE(rollingLessThan(uint64_t(0xFFFFFFF0ull), uint64_t(0x10ull), 32)); EXPECT_FALSE(rollingLessThan(uint64_t(0x10ull), uint64_t(0xFFFFFFF0ull), 32)); } TEST(NcclUtilityMicrotest, IdivRcp32_MatchesExactFormulaIncludingPow2Correction) { - // Formula: uint32_t(-1)/x + isPow2(x). The +1 term is a branch-avoidance - // optimisation, not a correctness requirement: idivFast32's own - // `if (r >= y) { q += 1; r -= y; }` correction step already recovers the - // right answer even without it. These assertions pin the exact reciprocal - // value the formula produces, not just that the round-trip still works. + // uint32_t(-1)/x + isPow2(x), pinned exactly: idivFast32's correction would hide a wrong +1. EXPECT_EQ(idivRcp32(1u), uint32_t(-1) / 1u + 1u); EXPECT_EQ(idivRcp32(2u), uint32_t(-1) / 2u + 1u); EXPECT_EQ(idivRcp32(3u), uint32_t(-1) / 3u + 0u); @@ -193,26 +165,19 @@ TEST(NcclUtilityMicrotest, Mul64Hi_ReturnsTheHighHalfOfTheFull128BitProduct) { } TEST(NcclUtilityMicrotest, ImulRcp32_HandlesZeroReciprocalShortCircuitsAndMatchesIdivRcpOfProduct) { - // xrcp == 0 or yrcp == 0 short-circuits to the other operand's reciprocal - // directly -- callers use this to mean "treat this factor as already 1". + // A zero reciprocal means "this factor is 1", so the other one is returned. EXPECT_EQ(imulRcp32(5u, 0u, 7u, idivRcp32(7u)), idivRcp32(7u)); EXPECT_EQ(imulRcp32(5u, idivRcp32(5u), 7u, 0u), idivRcp32(5u)); - // x=3, y=5 is chosen (rather than some arbitrary pair) because it's one of - // the smallest inputs that actually drives the `if (x*y <= rem) rcp += 1;` - // rounding correction true: rcp comes back as 0x11111110, rem == 16, and - // 15 <= 16. Pairs like the (2,2) case below land on the false side instead. + // (3,5) takes the `x*y <= rem` rounding correction. uint32_t x = 3, y = 5; uint32_t got = imulRcp32(x, idivRcp32(x), y, idivRcp32(y)); EXPECT_EQ(got, idivRcp32(x * y)); - // x=2, y=2 exercises the other side of that same branch (no correction - // needed), so both directions of the rounding check are verified. + // (2,2) doesn't. EXPECT_EQ(imulRcp32(2u, idivRcp32(2u), 2u, idivRcp32(2u)), idivRcp32(4u)); - // x=3, y=3 pins the `0u -` in rem: mul32hi already returns the exact - // 0x1C71C71C, so rem is 4 and 9 <= 4 wants no correction. Without the - // negation rem is 0xFFFFFFFC and the correction fires, returning one too many. + // (3,3) pins the `0u -` in rem: without it the correction fires and returns one too many. EXPECT_EQ(imulRcp32(3u, idivRcp32(3u), 3u, idivRcp32(3u)), idivRcp32(9u)); } @@ -220,29 +185,20 @@ TEST(NcclUtilityMicrotest, ImulRcp64_HandlesZeroReciprocalShortCircuitsAndMatche EXPECT_EQ(imulRcp64(5ull, 0ull, 7ull, idivRcp64(7ull)), idivRcp64(7ull)); EXPECT_EQ(imulRcp64(5ull, idivRcp64(5ull), 7ull, 0ull), idivRcp64(5ull)); - // Same rationale as the 32-bit test above: (3,5) drives the rounding - // correction branch true, which a value like (6,7) does not for the 64-bit - // reciprocal (the correction's trigger condition depends on word width). + // Same three cases as the 32-bit version. uint64_t x = 3, y = 5; uint64_t got = imulRcp64(x, idivRcp64(x), y, idivRcp64(y)); EXPECT_EQ(got, idivRcp64(x * y)); - // (2,2) exercises the other side of that same branch (no correction needed). EXPECT_EQ(imulRcp64(2ull, idivRcp64(2ull), 2ull, idivRcp64(2ull)), idivRcp64(4ull)); - // (3,3) pins the `0 -` in rem the same way: mul64hi returns the exact - // 0x1C71C71C71C71C71, so rem is 7 and 9 <= 7 wants no correction. EXPECT_EQ(imulRcp64(3ull, idivRcp64(3ull), 3ull, idivRcp64(3ull)), idivRcp64(9ull)); } TEST(NcclUtilityMicrotest, IdivmodFast32_MatchesOrdinaryDivisionAcrossDivisorsAndYrcpZeroShortcut) { for (uint32_t y : {1u, 2u, 3u, 7u, 16u, 100u, 4096u}) { uint32_t yrcp = idivRcp32(y); - // 0xFFFFFFFFu exercises the top of the range: idivRcp32's single - // `if (r >= y)` correction step only recovers an *under*-estimating - // reciprocal, so an over-estimate (e.g. an erroneous unconditional +1) - // is only caught near the dividend's upper bound, not by the smaller - // values below. + // The top of the range catches an over-estimated reciprocal; smaller values don't. for (uint32_t x : {0u, 1u, 5u, 41u, 4095u, 4096u, 1000000u, 0xFFFFFFFFu}) { uint32_t q, r; idivmodFast32(&q, &r, x, y, yrcp); @@ -250,7 +206,7 @@ TEST(NcclUtilityMicrotest, IdivmodFast32_MatchesOrdinaryDivisionAcrossDivisorsAn EXPECT_EQ(r, x % y) << "x=" << x << " y=" << y; } } - // yrcp == 0 shortcut: q = x directly (caller's contract for "divisor is 1"). + // yrcp == 0: q = x ("divisor is 1"). uint32_t q, r; idivmodFast32(&q, &r, 42u, 1u, 0u); EXPECT_EQ(q, 42u); @@ -260,7 +216,6 @@ TEST(NcclUtilityMicrotest, IdivmodFast32_MatchesOrdinaryDivisionAcrossDivisorsAn TEST(NcclUtilityMicrotest, IdivmodFast64_MatchesOrdinaryDivisionAcrossDivisorsAndYrcpZeroShortcut) { for (uint64_t y : {1ull, 2ull, 3ull, 7ull, 16ull, 100ull, 1ull << 20}) { uint64_t yrcp = idivRcp64(y); - // ~0ull is the 64-bit counterpart of the 32-bit sweep's 0xFFFFFFFFu above. for (uint64_t x : {0ull, 1ull, 5ull, 41ull, (1ull << 20) - 1, 1ull << 20, 1ull << 40, ~0ull}) { uint64_t q, r; idivmodFast64(&q, &r, x, y, yrcp); @@ -275,13 +230,9 @@ TEST(NcclUtilityMicrotest, IdivmodFast64_MatchesOrdinaryDivisionAcrossDivisorsAn } TEST(NcclUtilityMicrotest, IdivFast32_ReturnsTheQuotientAndIdivFast64TruncatesToLow32Bits) { - // idivFast32 is a one-line wrapper over idivmodFast32, which the sweep above - // already checks; one case pins that it returns q (13), not r (2). + // Wrapper over idivmodFast32: returns q (13), not r (2). EXPECT_EQ(idivFast32(41u, 3u, idivRcp32(3u)), 13u); - // idivFast64 narrows a 64-bit quotient to uint32_t. A quotient that fits in - // 32 bits leaves the narrowing inert (the oracle's own uint32_t cast is a - // no-op too), so the second case below -- whose true quotient is far above - // 2^32 -- is the one that actually exercises the truncation. + // The second case's quotient is far above 2^32, so the narrowing to uint32_t matters. uint64_t y = 1000, yrcp64 = idivRcp64(y); uint64_t x = 123456789ull; EXPECT_EQ(idivFast64(x, y, yrcp64), uint32_t(refDiv64(x, y))); @@ -289,8 +240,7 @@ TEST(NcclUtilityMicrotest, IdivFast32_ReturnsTheQuotientAndIdivFast64TruncatesTo } TEST(NcclUtilityMicrotest, ImodFast32And64_ReturnTheRemainder) { - // imodFast32/64 are one-line wrappers over idivmodFast32/64, which the sweeps - // above already check; one case each pins that they return r (2), not q (13). + // Wrappers over idivmodFast32/64: return r (2), not q (13). EXPECT_EQ(imodFast32(41u, 3u, idivRcp32(3u)), 2u); EXPECT_EQ(imodFast64(41ull, 3ull, idivRcp64(3ull)), 2ull); } @@ -303,9 +253,7 @@ TEST(NcclUtilityMicrotest, Optional_DefaultAndAbsentConstructLeavePresentFalse) } TEST(NcclUtilityMicrotest, Optional_PresentConstructorForwardsArgsAndSetsPresentTrue) { - // present(1, 2) packs two ints into a Present; Optional's - // Present-taking constructor must forward both through to T's constructor - // in the right order, not just construct a default T. + // Both values must reach T's constructor, in order. struct Pair { int a, b; NCCL_HOST_DEVICE_INLINE Pair(int a_, int b_) : a(a_), b(b_) {} @@ -337,9 +285,7 @@ TEST(NcclUtilityMicrotest, Optional_DestructorOnlyRunsWhenPresentIsTrue) { } TEST(NcclUtilityMicrotest, Add4G_AddsDeltaIntoTheHighHalfOfA64BitValueViaUnion) { - // add4G reinterprets a T (must be 8 bytes) as two uint32_t halves and - // adds delta4G into the high half (index 1 is the high word on this - // little-endian build). Verify against plain 64-bit arithmetic. + // add4G adds delta4G into the high 32 bits; check against plain 64-bit arithmetic. uint64_t base = 0x0000'0000'1234'5678ull; EXPECT_EQ(add4G(base, 1), base + (uint64_t(1) << 32)); EXPECT_EQ(add4G(base, 5), base + (uint64_t(5) << 32)); @@ -400,11 +346,7 @@ TEST(NcclUtilityMicrotest, ToCudaOrder_MapsEveryStdOrderAndFallsBackToSeqCstForC } TEST(NcclUtilityMicrotest, Present_GetByIndexReturnsEachArgumentInDeclarationOrder) { - // Pass lvalues so H deduces to int& (present(H&&, ...) with an lvalue arg - // collapses to a reference to the original), keeping p's stored references - // valid for the lifetime of a,b,c below -- a prvalue-literal argument would - // deduce H=int and leave p holding a dangling int&& into a temporary that - // expires at the end of the present(...) call's full expression. + // Lvalues, so p holds references to a, b, c rather than to dangling temporaries. int a = 10, b = 20, c = 30; auto p = present(a, b, c); EXPECT_EQ(p.get(nccl::utility::IntSeq<0>{}), 10); From 9d54be41db0f6eecf4200879969827d91fb41d1f Mon Sep 17 00:00:00 2001 From: Alex Duforest Date: Wed, 14 Oct 2026 15:40:13 -0700 Subject: [PATCH 21/26] test(rccl): trim microtest docs and comments to match the other host tests Shorten the MICROTEST_README entry, CMake comments and test-file comments to the length the other host microtests use. Comment and doc changes only. Co-Authored-By: Claude --- projects/rccl/test/host/CMakeLists.txt | 8 ++--- projects/rccl/test/host/MICROTEST_README.md | 11 ++----- projects/rccl/test/host/nccl-core-test.cc | 34 +++++---------------- 3 files changed, 12 insertions(+), 41 deletions(-) diff --git a/projects/rccl/test/host/CMakeLists.txt b/projects/rccl/test/host/CMakeLists.txt index e3d99bc76fd1..5f0d2f8e3dfc 100644 --- a/projects/rccl/test/host/CMakeLists.txt +++ b/projects/rccl/test/host/CMakeLists.txt @@ -293,12 +293,8 @@ function(rccl_define_micro_source_lists) ../common/main_altrsmi.cpp PARENT_SCOPE) - # rccl-UnitTestsMicroNcclDeviceCore -- src/nccl_device/core.cc. Its own binary: - # dev_runtime_micro_fakes.cc (ncclTeamWorld/ncclTeamRail/ncclTeamCft/ncclTeamCftMultimem) - # and devcomm_fakes.cc (ncclTeamLsa) fake 5 of this file's 7 for rccl-UnitTestsMicro, - # and gin-host-test.cc fakes ncclTeamWorld/ncclTeamRail/ncclTeamRankToWorld for - # rccl-UnitTestsMicroGinHost; this target compiles core.cc as the real thing under - # test, so none of those three can share a binary with it. + # rccl-UnitTestsMicroNcclDeviceCore -- src/nccl_device/core.cc. Own binary: + # dev_runtime_micro_fakes.cc and devcomm_fakes.cc fake five of its seven functions. set(TEST_MICRO_NCCL_DEVICE_CORE_SOURCE_FILES nccl-core-test.cc fakes/bootstrap_stubs.cc diff --git a/projects/rccl/test/host/MICROTEST_README.md b/projects/rccl/test/host/MICROTEST_README.md index 5e7f20eeb897..f3082fa04738 100644 --- a/projects/rccl/test/host/MICROTEST_README.md +++ b/projects/rccl/test/host/MICROTEST_README.md @@ -250,15 +250,8 @@ export colliding non-`static` symbols; otherwise a unit needs its own binary: helpers. CMake pins the copied `ncclFuncStr` table to `src/init.cc`. - **`rccl-UnitTestsMicroNcclDeviceCore`**: `src/nccl_device/core.cc` (via `CORE_CC_PATH`, from `nccl-core-test.cc`); suite `NcclCoreMicrotest.*`. Its - own binary because `rccl-UnitTestsMicro`'s `fakes/dev_runtime_micro_fakes.cc` - (`ncclTeamWorld`/`ncclTeamRail`/`ncclTeamCft`/`ncclTeamCftMultimem`) and - `fakes/devcomm_fakes.cc` (`ncclTeamLsa`) fake 5 of this file's 7 functions, and - `gin-host-test.cc` fakes `ncclTeamWorld`/`ncclTeamRail`/`ncclTeamRankToWorld` - for `rccl-UnitTestsMicroGinHost`. - Links the lightweight `fakes/dev_runtime_fakes.cc` (hookable - `ncclDevrInitOnce`, default `ncclSuccess`) rather than the real - `dev_runtime.cc` -- core.cc only calls `ncclDevrInitOnce` itself, so the full - production file and its much larger fake surface aren't needed. + own binary because `fakes/dev_runtime_micro_fakes.cc` and + `fakes/devcomm_fakes.cc` fake five of its seven functions. - **`rccl-UnitTestsMicroSymKernels`** — the REAL `src/sym_kernels.cc` (via `SYM_KERNELS_CC_PATH`, from `sym-kernels-test.cc`), compiled together with the GENERATED `sym_kernels_host.cc` it calls into; suites `SymKernelMicrotest.*`, diff --git a/projects/rccl/test/host/nccl-core-test.cc b/projects/rccl/test/host/nccl-core-test.cc index e706a239195a..fb8024228a90 100644 --- a/projects/rccl/test/host/nccl-core-test.cc +++ b/projects/rccl/test/host/nccl-core-test.cc @@ -4,13 +4,8 @@ * See LICENSE.txt for license information ************************************************************************/ -// Host-only microtests for src/nccl_device/core.cc: the ncclTeam* family that -// maps a communicator onto its world/LSA/CFT/rail views, plus the -// world/LSA rank-translation helpers. Five of the seven gate through -// ncclDevrInitOnce first (all but ncclTeamWorld and ncclTeamRankToWorld), so -// the fake's hook lets us drive both the success path (devrState fields read -// back verbatim) and the failure path (all five return a poisoned/zeroed -// answer rather than touching uninitialised devrState). +// Host-only microtests for src/nccl_device/core.cc's ncclTeam* and rank-translation helpers. +// ncclDevrInitOnce is faked so both its success and failure paths run. #include @@ -47,8 +42,7 @@ TEST_F(NcclCoreMicrotest, TeamWorld_ReturnsCommNRanksAndRankWithStrideOne) { } TEST_F(NcclCoreMicrotest, TeamWorldAndRankToWorld_IgnoreAFailingInitOnce) { - // The only two of the seven that don't gate on ncclDevrInitOnce: with it - // failing, both must still return real values rather than a zeroed/-1 answer. + // The two that don't gate on ncclDevrInitOnce still return real values when it fails. g_devrInitOnce = [](ncclComm*) { return ncclSystemError; }; comm->nRanks = 4; comm->rank = 2; @@ -70,8 +64,7 @@ TEST_F(NcclCoreMicrotest, TeamLsa_SuccessPath_ReadsLsaSizeAndSelfFromDevrState) } TEST_F(NcclCoreMicrotest, TeamLsa_InitOnceFails_ReturnsZeroedTeamWithoutReadingDevrState) { - // Poison lsaSize/lsaSelf so that if the function ignored the failure and - // read them anyway, the zero-check below would fail loudly. + // Poisoned, so reading them despite the failure shows up below. comm->devrState.lsaSize = 99; comm->devrState.lsaSelf = 99; g_devrInitOnce = [](ncclComm*) { return ncclSystemError; }; @@ -101,9 +94,7 @@ TEST_F(NcclCoreMicrotest, TeamCftMultimem_InitOnceFails_ReturnsZeroedTeam) { } TEST_F(NcclCoreMicrotest, TeamRail_SuccessPath_DividesCommNRanksAndRankByLsaSize) { - // Constants chosen so each field separates `/` from its look-alikes: nRanks 24/4=6 is - // neither lsaSize (4) nor 24%4=0, and rank 9/4=2 differs from 9%4=1 (ncclTeamInnerFactor - // uses `%` on the same field, so that mix-up is the realistic one). + // 24/4 = 6 and 9/4 = 2 differ from the `%` results ncclTeamInnerFactor would give. comm->nRanks = 24; comm->rank = 9; comm->devrState.lsaSize = 4; @@ -135,14 +126,7 @@ TEST_F(NcclCoreMicrotest, TeamCft_FlatMode_ReturnsFlatTeamFromCftSizeAndSelf) { } TEST_F(NcclCoreMicrotest, TeamCft_HierMultimemMode_OuterFactorsTheFlatTeamByCftMcSize) { - // ncclTeamOuterFactor(flat, innerSize): nRanks = flat.nRanks/innerSize, - // rank = flat.rank/innerSize, stride = flat.stride*innerSize (== 1*innerSize - // here since the flat team's stride is always 1). - // - // lsaSize is set to a distinct non-zero value (the HIER_LSA test's own - // divisor) so that if the two mode arms in production were swapped, this - // test would divide by a real, wrong number and get a mis-answer to diff - // against -- not silently divide by zero and crash identically either way. + // lsaSize is a distinct non-zero value, so swapped mode arms give a wrong answer, not a crash. comm->devrState.cftSize = 12; comm->devrState.cftSelf = 7; comm->devrState.cftMcSize = 3; @@ -154,8 +138,7 @@ TEST_F(NcclCoreMicrotest, TeamCft_HierMultimemMode_OuterFactorsTheFlatTeamByCftM } TEST_F(NcclCoreMicrotest, TeamCft_HierLsaMode_OuterFactorsTheFlatTeamByLsaSize) { - // cftMcSize is likewise set to a distinct non-zero value for the same - // swapped-arm reason as the HIER_MULTIMEM test above. + // cftMcSize is distinct for the same reason. comm->devrState.cftSize = 20; comm->devrState.cftSelf = 9; comm->devrState.lsaSize = 5; @@ -188,8 +171,7 @@ TEST_F(NcclCoreMicrotest, TeamCft_InitOnceFails_ReturnsZeroedTeamRegardlessOfMod } TEST_F(NcclCoreMicrotest, TeamRankToWorld_MapsATeamLocalRankBackToItsWorldRankViaStride) { - // Does not gate on ncclDevrInitOnce at all -- pure arithmetic over comm->rank - // and the caller-supplied team, so no fake wiring is needed for this one. + // Pure arithmetic: doesn't gate on ncclDevrInitOnce. comm->rank = 10; ncclTeam_t team{/*nRanks=*/4, /*rank=*/2, /*stride=*/3}; // comm->rank + (rank - team.rank) * team.stride From f8d922e77dfc4a861624718cdacc9c78c8910bad Mon Sep 17 00:00:00 2001 From: Alex Duforest Date: Wed, 14 Oct 2026 15:40:21 -0700 Subject: [PATCH 22/26] test(rccl): trim microtest docs and comments to match the other host tests Shorten the MICROTEST_README entry, CMake comments and test-file comments to the length the other host microtests use. Comment and doc changes only. Co-Authored-By: Claude --- projects/rccl/test/host/CMakeLists.txt | 12 +- projects/rccl/test/host/MICROTEST_README.md | 36 +----- .../test/host/nccl-cft-dev-runtime-test.cc | 108 +++--------------- 3 files changed, 26 insertions(+), 130 deletions(-) diff --git a/projects/rccl/test/host/CMakeLists.txt b/projects/rccl/test/host/CMakeLists.txt index 3dcbabec9ca4..7a0001fe8bbd 100644 --- a/projects/rccl/test/host/CMakeLists.txt +++ b/projects/rccl/test/host/CMakeLists.txt @@ -293,16 +293,8 @@ function(rccl_define_micro_source_lists) ../common/main_altrsmi.cpp PARENT_SCOPE) - # rccl-UnitTestsMicroNcclDeviceCft -- src/nccl_device/core.cc + src/cft_dev_runtime.cc. - # Its own binary: rccl-UnitTestsMicro's fakes/dev_runtime_micro_fakes.cc, fakes/devcomm_fakes.cc, - # and gin-host-test.cc between them fake 6 of core.cc's 7 functions. core.cc is compiled for - # real here (not faked) because cft_dev_runtime.cc's ncclGet*DeviceLeInfo functions call - # ncclTeamCft/ncclTeamCftMultimem, and faking those would just re-derive core.cc's own - # already-tested logic. Links the lightweight fakes/dev_runtime_fakes.cc (hookable - # ncclDevrInitOnce) for core.cc's sake, and the new fakes/cft_dev_runtime_fakes.cc for - # cft_dev_runtime.cc's own dev_runtime.cc callees (computeLsaSize, symTeamObtain, - # findCommAndHostWindowFromDeviceWindow, ncclDevrWinRegEnabled) -- not the real - # dev_runtime.cc, which neither production file needs beyond those four symbols. + # rccl-UnitTestsMicroNcclDeviceCft -- src/cft_dev_runtime.cc + src/nccl_device/core.cc. Own + # binary: rccl-UnitTestsMicro fakes most of core.cc, which the LE-info queries call. set(TEST_MICRO_NCCL_DEVICE_CFT_SOURCE_FILES nccl-cft-dev-runtime-test.cc fakes/cft_dev_runtime_fakes.cc diff --git a/projects/rccl/test/host/MICROTEST_README.md b/projects/rccl/test/host/MICROTEST_README.md index e2b6cf07dfbc..f689e31800cf 100644 --- a/projects/rccl/test/host/MICROTEST_README.md +++ b/projects/rccl/test/host/MICROTEST_README.md @@ -248,32 +248,10 @@ export colliding non-`static` symbols; otherwise a unit needs its own binary: guard-less positional pair `libc_seam.h` / `libc_seam_undef.h` must bracket the production source include, after system headers and before test helpers. CMake pins the copied `ncclFuncStr` table to `src/init.cc`. -- **`rccl-UnitTestsMicroNcclDeviceCft`**: `src/nccl_device/core.cc` (via - `CORE_CC_PATH`) + `src/cft_dev_runtime.cc` (via `CFT_DEV_RUNTIME_CC_PATH`, - from `nccl-cft-dev-runtime-test.cc`); suites `NcclCftDevRuntimeMicrotest.*`, - `NcclGetMultimemDeviceLeInfoTest.*`, `NcclGetCftDeviceLeInfoTest.*`, - `NcclGetPeerDeviceLeInfoTest.*` (all caught by the `Nccl*` pattern). Its own - binary because `rccl-UnitTestsMicro`'s `fakes/dev_runtime_micro_fakes.cc`, - `fakes/devcomm_fakes.cc`, and `gin-host-test.cc` between them fake 6 of - core.cc's 7 functions. core.cc is compiled for real here too (not faked) - because - `ncclGetMultimemDeviceLeInfo`/`ncclGetCftDeviceLeInfo`/`ncclGetPeerDeviceLeInfo` - call `ncclTeamCft`/`ncclTeamCftMultimem`. Links `fakes/dev_runtime_fakes.cc` - for core.cc's `ncclDevrInitOnce`, and the new `fakes/cft_dev_runtime_fakes.cc` - for `cft_dev_runtime.cc`'s own `dev_runtime.cc` callees (`computeLsaSize`, - `symTeamObtain`, `findCommAndHostWindowFromDeviceWindow`, - `ncclDevrWinRegEnabled`) -- not the real `dev_runtime.cc`, which neither - production file needs beyond those four symbols. This build's `CUDA_VERSION` - is below 13030, so the CUDA logical-endpoint driver calls throughout - `cft_dev_runtime.cc` are compiled out (no ROCm equivalent exists); the tests - cover what remains: the zero-and-succeed body of `ncclGpuCftSupport`, the - host-side branch logic of `computeCftSize`/`computeCftMcSize`, the - condition-evaluation (now-empty) bodies of `symBindTeamLe`/`symUnbindTeamLe`, - the trivial-failure stand-ins for `symTeamObtainUcLe`/`symTeamObtainMcLe`, - and the three unconditional `ncclGet*DeviceLeInfo` query functions. Lines are - 100% covered; regions/branches have two residual, well-understood gaps - documented in AICOMRCCL-2825 PR4's description (`ncclParamCftEnable`, and - `NCCLCHECK`'s own macro-internal branches). +- **`rccl-UnitTestsMicroNcclDeviceCft`**: `src/cft_dev_runtime.cc` and + `src/nccl_device/core.cc` (via `CFT_DEV_RUNTIME_CC_PATH` / `CORE_CC_PATH`, + from `nccl-cft-dev-runtime-test.cc`); suites `Nccl*`. Its own binary because + `rccl-UnitTestsMicro` fakes most of `core.cc`, which the LE-info queries call. - **`rccl-UnitTestsMicroSymKernels`** — the REAL `src/sym_kernels.cc` (via `SYM_KERNELS_CC_PATH`, from `sym-kernels-test.cc`), compiled together with the GENERATED `sym_kernels_host.cc` it calls into; suites `SymKernelMicrotest.*`, @@ -460,10 +438,6 @@ had become before this map existed. dependency of another unit (`rccl-UnitTestsMicroEnqueue` doesn't compile it, so it fakes the file's own entry points). The two never define the same symbol. -`src/dev_runtime.cc` has two rows for a narrower reason: `rccl-UnitTestsMicroNcclDeviceCft` -needs only the four `dev_runtime.cc` symbols `cft_dev_runtime.cc` calls, kept in their own -file so that target does not drag in the rest of `fakes/dev_runtime_fakes.cc`'s seams. -The two never define the same symbol either. | Production TU | Fakes file | |---|---| @@ -472,7 +446,7 @@ The two never define the same symbol either. | `src/ce_coll.cc` (targets that do not compile the real file) | `fakes/ce_fakes.cc` | | `src/collectives.cc` | `fakes/collectives_fakes.cc` | | `src/dev_runtime.cc` (targets that do not compile the real file) | `fakes/dev_runtime_fakes.cc` | -| `src/dev_runtime.cc`'s callees from `src/cft_dev_runtime.cc` (`computeLsaSize`, `symTeamObtain`, `findCommAndHostWindowFromDeviceWindow`, `ncclDevrWinRegEnabled`; `rccl-UnitTestsMicroNcclDeviceCft`, which compiles the real `cft_dev_runtime.cc`) | `fakes/cft_dev_runtime_fakes.cc` | +| `src/dev_runtime.cc` callees of `src/cft_dev_runtime.cc` (`rccl-UnitTestsMicroNcclDeviceCft`) | `fakes/cft_dev_runtime_fakes.cc` | | `src/diagnostics.cc` (`ncclDiagChildRun*` external-tool runners) | `fakes/diagnostics_fakes.cc` | | `src/diagnostics/device/p2p.cu` (`ncclDiagP2p*` kernel launchers) | `fakes/diagnostics_p2p_device_fakes.cc` | | `src/graph/*.cc` (topo, paths, search, connect, rome consensus) | `fakes/topo_stubs.cc` | diff --git a/projects/rccl/test/host/nccl-cft-dev-runtime-test.cc b/projects/rccl/test/host/nccl-cft-dev-runtime-test.cc index 4623f897afff..ae04eca38c2a 100644 --- a/projects/rccl/test/host/nccl-cft-dev-runtime-test.cc +++ b/projects/rccl/test/host/nccl-cft-dev-runtime-test.cc @@ -4,23 +4,8 @@ * See LICENSE.txt for license information ************************************************************************/ -// Host-only microtests for src/cft_dev_runtime.cc: the CFT (Compute Fabric -// Transport) logical-endpoint support-probe, size-computation, and -// bind/unbind helpers, plus the three public LE-info query functions. This -// build's CUDA_VERSION is below 13030, so the real CUDA logical-endpoint -// driver calls are compiled out everywhere in this file (no ROCm -// equivalent exists); what remains compiled is exercised here: the -// zero-everything-and-succeed body of ncclGpuCftSupport, the host-side -// branch logic of computeCftSize/computeCftMcSize, the condition-evaluation -// (but now-empty) bodies of symBindTeamLe/symUnbindTeamLe, the -// trivial-failure stand-ins for symTeamObtainUcLe/symTeamObtainMcLe, and the -// three unconditional ncclGet*DeviceLeInfo query functions. -// -// core.cc is compiled for real alongside cft_dev_runtime.cc (not faked): -// ncclGetMultimemDeviceLeInfo/ncclGetCftDeviceLeInfo/ncclGetPeerDeviceLeInfo -// call ncclTeamCftMultimem/ncclTeamCft, and faking those would just be -// re-deriving core.cc's own already-tested logic (AICOMRCCL-2825 PR3) rather -// than reusing it. +// Host-only microtests for src/cft_dev_runtime.cc, with src/nccl_device/core.cc compiled for real. +// CUDA_VERSION is below 13030 here, so the CUDA logical-endpoint calls are compiled out. #include @@ -35,10 +20,7 @@ namespace { -// Heap-allocates a ncclDevrTeam sized and populated for its flexible-array -// worldRankList, same idiom dev_runtime.cc's own symTeamObtain uses. An -// automatic (stack) ncclDevrTeam's worldRankList has zero elements; a fake -// that hands one to code indexing it at team.nRanks-1 is out of bounds. +// Heap-allocated so worldRankList has team.nRanks elements, like symTeamObtain's own teams. struct DevrTeamDeleter { void operator()(ncclDevrTeam* t) const { std::free(t); } }; @@ -62,11 +44,7 @@ class NcclCftDevRuntimeMicrotest : public ::testing::Test { void SetUp() override { commStorage = std::make_unique(); // value-initialised: POD members zeroed comm = commStorage.get(); - // Load-bearing only for computeCftSize/computeCftMcSize, which check - // devrState.bigSize themselves: ncclDevrInitOnce in this binary is the - // always-succeeds fake seam at fakes/dev_runtime_fakes.cc, which never - // reads devrState, so bigSize is inert on the ncclTeamCft/ - // ncclTeamCftMultimem path. + // Only computeCftSize/computeCftMcSize read bigSize; the ncclDevrInitOnce fake ignores it. comm->devrState.bigSize = 1; } @@ -78,11 +56,7 @@ class NcclCftDevRuntimeMicrotest : public ::testing::Test { } // namespace -// --------------------------------------------------------------------------- -// ncclGpuCftSupport: CUDA_VERSION < 13030 on this build, so the entire -// capability-probe body is compiled out; only the zero-and-succeed prologue -// remains. -// --------------------------------------------------------------------------- +// ncclGpuCftSupport: only the zero-and-succeed prologue is compiled on this build. TEST_F(NcclCftDevRuntimeMicrotest, GpuCftSupport_ZeroesAllThreeOutputsAndSucceeds) { int gpuCftSupport = 99; @@ -93,9 +67,7 @@ TEST_F(NcclCftDevRuntimeMicrotest, GpuCftSupport_ZeroesAllThreeOutputsAndSucceed EXPECT_FALSE(gpuCftCountedSupport); } -// --------------------------------------------------------------------------- // computeCftSize / computeCftMcSize -// --------------------------------------------------------------------------- TEST_F(NcclCftDevRuntimeMicrotest, ComputeCftSize_AlreadySized_ReturnsCftSizeWithoutCallingComputeLsaSize) { comm->devrState.bigSize = 7; @@ -123,8 +95,7 @@ TEST_F(NcclCftDevRuntimeMicrotest, ComputeCftSize_NotSizedWithGpuCftSupport_Dele } TEST_F(NcclCftDevRuntimeMicrotest, ComputeCftSizeAndMcSize_GpuCftSupportJustBelow13030_ReturnsOne) { - // Every other probe uses 0 or exactly 13030, far from the `>= 13030` edge at - // cft_dev_runtime.cc:54 and 61; 13029 is the first value it must reject. + // 13029 is the first value the `>= 13030` check must reject. comm->devrState.bigSize = 0; comm->gpuCftSupport = 13029; bool called = false; @@ -159,12 +130,8 @@ TEST_F(NcclCftDevRuntimeMicrotest, ComputeCftMcSize_NotSizedWithGpuCftSupport_De EXPECT_EQ(computeCftMcSize(comm), 6); } -// --------------------------------------------------------------------------- -// symBindTeamLe / symUnbindTeamLe: on this build the CUDA bind/unbind calls -// inside the condition are compiled out, so the only externally observable -// contract left is "always succeeds" and "always consults -// ncclDevrWinRegEnabled with this memory's winFlags and ncclDevrRegisterCft". -// --------------------------------------------------------------------------- +// symBindTeamLe / symUnbindTeamLe: the CUDA calls are compiled out, so they always succeed and +// only the ncclDevrWinRegEnabled query is observable. TEST_F(NcclCftDevRuntimeMicrotest, SymBindTeamLe_AlwaysSucceedsAndConsultsWinRegEnabledWithCftCapability) { auto mem = std::make_unique(); @@ -234,10 +201,7 @@ TEST_F(NcclCftDevRuntimeMicrotest, SymUnbindTeamLe_GlobalHasSysmemSegment_StillS EXPECT_EQ(symUnbindTeamLe(comm, mem.get(), /*le=*/3), ncclSuccess); } -// --------------------------------------------------------------------------- -// symTeamObtainUcLe / symTeamObtainMcLe: this build compiles the -// CUDA_VERSION < 13030 stand-ins, which ignore every argument and fail. -// --------------------------------------------------------------------------- +// symTeamObtainUcLe / symTeamObtainMcLe: this build's stand-ins always fail. TEST_F(NcclCftDevRuntimeMicrotest, SymTeamObtainUcLe_UnsupportedOnThisPlatform_ReturnsInvalidUsage) { bool needBarrier = false; @@ -251,11 +215,7 @@ TEST_F(NcclCftDevRuntimeMicrotest, SymTeamObtainMcLe_UnsupportedOnThisPlatform_R ncclInvalidUsage); } -// Shared window/comm-resolution setup for all three ncclGet*DeviceLeInfo -// query functions; BigOffset() keeps each derived suite's winHost.bigOffset -// distinct, same reasoning as the handle.* field checks elsewhere in this -// file: a test mixing up which window it is reading would get a visibly -// wrong offset back instead of an accidental match. +// Shared setup for the three ncclGet*DeviceLeInfo queries; BigOffset() is distinct per suite. class NcclGetDeviceLeInfoTestBase : public NcclCftDevRuntimeMicrotest { protected: ncclDevrWindow winHost{}; @@ -267,15 +227,13 @@ class NcclGetDeviceLeInfoTestBase : public NcclCftDevRuntimeMicrotest { bool winRegEnabled = true; virtual size_t BigOffset() const = 0; - // Calls this suite's query function with its own extra arguments held fixed, - // so the shared prologue checks below run against all three functions. + // Calls this suite's query with its other arguments fixed. virtual ncclResult_t Invoke(ncclWindow_t w, ncclCftLeId* leId, size_t* leOffset) = 0; void SetUp() override { NcclCftDevRuntimeMicrotest::SetUp(); comm->gpuCftSupport = 13030; - // Non-degenerate flat CFT team: at cftSize 0 the peer query's range check rejects every - // peer first, so the shared prologue checks below would never reach the check they name. + // At cftSize 0 the peer query rejects every peer before reaching the checks below. comm->devrState.cftSize = 4; winHost.bigOffset = BigOffset(); g_foundComm = comm; @@ -288,8 +246,6 @@ class NcclGetDeviceLeInfoTestBase : public NcclCftDevRuntimeMicrotest { }; } - // The prologue below is identical across the three query functions - // (cft_dev_runtime.cc:295-309 and its two twins). void ExpectNullArgsRejectedByPtrCheck() { ncclCftLeId leId; size_t leOffset; @@ -299,8 +255,7 @@ class NcclGetDeviceLeInfoTestBase : public NcclCftDevRuntimeMicrotest { } void ExpectWindowNotCftRegisteredReturnsInvalidUsage() { - // Also pins what the registration check asks: this window's own flags, - // against the CFT capability rather than e.g. ncclDevrRegisterAll. + // Asked with this window's flags, against the CFT capability. winHost.winFlags = 0x5a; winRegEnabled = false; ncclCftLeId leId; @@ -327,9 +282,7 @@ class NcclGetDeviceLeInfoTestBase : public NcclCftDevRuntimeMicrotest { } }; -// --------------------------------------------------------------------------- // ncclGetMultimemDeviceLeInfo -// --------------------------------------------------------------------------- class NcclGetMultimemDeviceLeInfoTest : public NcclGetDeviceLeInfoTestBase { protected: @@ -361,20 +314,13 @@ TEST_F(NcclGetMultimemDeviceLeInfoTest, SymTeamObtainFails_PropagatesErrorViaNcc EXPECT_EQ(ncclGetMultimemDeviceLeInfo(window, 0, &leId, &leOffset), ncclInvalidUsage); } -// NOTE: the NCCLCHECK(bootstrapIntraNodeBarrier(...)) propagation branch (needBarrier -// path) is not independently exercised here: fakes/bootstrap_stubs.cc's -// bootstrapIntraNodeBarrier is a parameterless-result `{ return ncclSuccess; }` stub -// shared by every other suite that links it, with no failure hook. Giving it one is out -// of scope for this file's tests. +// The bootstrapIntraNodeBarrier failure branch isn't covered: fakes/bootstrap_stubs.cc has no +// failure hook. TEST_F(NcclGetMultimemDeviceLeInfoTest, SuccessPath_ReadsMcLeIdByCountedBitAndOffsetsByBigOffsetPlusOffset) { comm->devrState.cftMcSize = 4; comm->devrState.cftMcSelf = 1; - // NCCL_WIN_COLL_SYMMETRIC is ORed in alongside NCCL_WIN_CFT_COUNTED so this - // exercises `winFlags & NCCL_WIN_CFT_COUNTED` as a real bit test: every - // other winFlags value elsewhere in this file is exactly - // NCCL_WIN_CFT_COUNTED or exactly 0, which `winFlags == NCCL_WIN_CFT_COUNTED` - // would also satisfy. + // SYMMETRIC alongside CFT_COUNTED, so `== NCCL_WIN_CFT_COUNTED` would fail where `&` passes. winHost.winFlags = NCCL_WIN_CFT_COUNTED | NCCL_WIN_COLL_SYMMETRIC; DevrTeamPtr team = MakeDevrTeam(ncclTeam_t{4, 1, 1}, /*mcLeId0=*/111, /*mcLeId1=*/222); // counted==true selects mcLeId1 @@ -421,8 +367,7 @@ TEST_F(NcclGetMultimemDeviceLeInfoTest, UncountedWindow_ReadsMcLeIdZeroSlot) { } TEST_F(NcclGetMultimemDeviceLeInfoTest, SymmetricOnlyWindow_IsUncountedAndReadsMcLeIdZeroSlot) { - // NCCL_WIN_COLL_SYMMETRIC without NCCL_WIN_CFT_COUNTED: a counted-bit mask - // that tested the companion bit instead would read the counted slot here. + // SYMMETRIC without CFT_COUNTED: testing the wrong bit would read the counted slot. comm->devrState.cftMcSize = 4; comm->devrState.cftMcSelf = 0; winHost.winFlags = NCCL_WIN_COLL_SYMMETRIC; @@ -443,9 +388,7 @@ TEST_F(NcclGetMultimemDeviceLeInfoTest, SymmetricOnlyWindow_IsUncountedAndReadsM EXPECT_EQ(leId, 77u); } -// --------------------------------------------------------------------------- // ncclGetCftDeviceLeInfo -// --------------------------------------------------------------------------- class NcclGetCftDeviceLeInfoTest : public NcclGetDeviceLeInfoTestBase { protected: @@ -477,9 +420,6 @@ TEST_F(NcclGetCftDeviceLeInfoTest, BaseLeIdNotYetCreated_ReturnsInvalidUsage) { } TEST_F(NcclGetCftDeviceLeInfoTest, SuccessPath_AddsFlatTeamRankAndPeerOffsetToBaseLeId) { - // ncclTeamCft(comm, FLAT) reads straight from devrState.cftSize/cftSelf; it never - // consults devrState.bigSize itself (that's load-bearing only for computeCftSize/ - // computeCftMcSize -- see NcclCftDevRuntimeMicrotest::SetUp's comment). comm->devrState.cftSize = 8; comm->devrState.cftSelf = 3; comm->devrState.le[0].baseId = 1000; @@ -509,9 +449,7 @@ TEST_F(NcclGetCftDeviceLeInfoTest, CountedWindow_UsesCountedBaseLeIdSlot) { EXPECT_EQ(leId, 500u); // 500 + flatTeam.rank(0) + (0 - 0) * 1 } -// --------------------------------------------------------------------------- // ncclGetPeerDeviceLeInfo -// --------------------------------------------------------------------------- class NcclGetPeerDeviceLeInfoTest : public NcclGetDeviceLeInfoTestBase { protected: @@ -535,13 +473,7 @@ TEST_F(NcclGetPeerDeviceLeInfoTest, FindCommAndHostWindowFails_PropagatesErrorVi } TEST_F(NcclGetPeerDeviceLeInfoTest, PeerOutsideFlatTeamBoundaries_ReturnsInvalidArgument) { - // flatStart = comm->rank - flatTeam.rank; valid range is [flatStart, flatStart + nRanks). - // The boundary check runs before baseLeId is ever read, so no valid baseId is needed for - // the two out-of-range probes below. 20 and 7 alone don't pin either exact edge though: a - // relaxed `>=` -> `>` survives because 12 (the first rejected value above the range) is - // never probed, and a relaxed `<` -> `<=` survives because 8 (the first accepted value) is - // never probed either. 12 closes the first; 8 (further down, where it can succeed) closes - // the second. + // Valid range is [flatStart, flatStart + nRanks) = [8, 12); 12 and 8 pin the two edges. comm->rank = 10; comm->devrState.cftSize = 4; comm->devrState.cftSelf = 2; // flatTeam.rank == 2 -> flatStart == 8, valid peers in [8, 12) @@ -551,9 +483,7 @@ TEST_F(NcclGetPeerDeviceLeInfoTest, PeerOutsideFlatTeamBoundaries_ReturnsInvalid EXPECT_EQ(ncclGetPeerDeviceLeInfo(window, 0, /*peerWorld=*/7, &leId, &leOffset), ncclInvalidArgument); EXPECT_EQ(ncclGetPeerDeviceLeInfo(window, 0, /*peerWorld=*/12, &leId, &leOffset), ncclInvalidArgument); - // peerWorld=8 is the lower inclusive edge: it must succeed, not reject. This path proceeds - // past the boundary check into the baseLeId read, so (unlike the three probes above) it - // needs a valid baseId. + // 8 is the lower inclusive edge, so it must succeed. comm->devrState.le[0].baseId = 1; ASSERT_EQ(ncclGetPeerDeviceLeInfo(window, 0, /*peerWorld=*/8, &leId, &leOffset), ncclSuccess); EXPECT_EQ(leId, 1u); // baseLeId(1) + flatTeam.rank(2) + (peerWorld(8) - comm->rank(10)) == 1 From 40f672320d53b2953a1c748393d000001accbcce Mon Sep 17 00:00:00 2001 From: Alex Duforest Date: Wed, 14 Oct 2026 16:58:38 -0700 Subject: [PATCH 23/26] test(rccl): follow the microtest naming and registration rules Give the binary its own test_categories_micro_.yaml and register it in ci-precheckin.json and test_executor.py's llvm-cov list, as MICROTEST_README's naming section requires, so the CI test runner and coverage pick it up. Co-Authored-By: Claude --- projects/rccl/test/host/CMakeLists.txt | 2 +- projects/rccl/test/host/MICROTEST_README.md | 5 +- projects/rccl/test/test_categories_micro.yaml | 2 - .../test_categories_micro_nccl_device.yaml | 46 +++++++++++++++++++ .../test_runner/configs/ci-precheckin.json | 17 +++++++ .../scripts/test_runner/lib/test_executor.py | 3 +- 6 files changed, 70 insertions(+), 5 deletions(-) create mode 100644 projects/rccl/test/test_categories_micro_nccl_device.yaml diff --git a/projects/rccl/test/host/CMakeLists.txt b/projects/rccl/test/host/CMakeLists.txt index beae08289865..f84455f7253b 100644 --- a/projects/rccl/test/host/CMakeLists.txt +++ b/projects/rccl/test/host/CMakeLists.txt @@ -1074,7 +1074,7 @@ rccl_add_micro_binary(rccl-UnitTestsMicroNcclDevice GIN_SCRATCH_CC_PATH="${RCCL_HOST_HIPIFY_DIR}/src/nccl_device/gin_scratch.cc" LL_A2A_CC_PATH="${RCCL_HOST_HIPIFY_DIR}/src/nccl_device/ll_a2a.cc" LSA_BARRIER_CC_PATH="${RCCL_HOST_HIPIFY_DIR}/src/nccl_device/lsa_barrier.cc" - CATEGORIES_YAML "${CMAKE_CURRENT_SOURCE_DIR}/../test_categories_micro.yaml") + CATEGORIES_YAML "${CMAKE_CURRENT_SOURCE_DIR}/../test_categories_micro_nccl_device.yaml") # --------------------------------------------------------------------------- # rccl-UnitTestsMicroSymKernels diff --git a/projects/rccl/test/host/MICROTEST_README.md b/projects/rccl/test/host/MICROTEST_README.md index 155f789aa876..e65923c574f4 100644 --- a/projects/rccl/test/host/MICROTEST_README.md +++ b/projects/rccl/test/host/MICROTEST_README.md @@ -252,7 +252,8 @@ export colliding non-`static` symbols; otherwise a unit needs its own binary: `src/nccl_device/{cft_barrier,gin_barrier,gin_scratch,ll_a2a,lsa_barrier}.cc` (via `_CC_PATH`, from `nccl-device-requirements-test.cc`); suite `NcclDeviceRequirementsMicrotest.*`. Its own binary because - `fakes/dev_runtime_micro_fakes.cc` fakes three of these builders. + `fakes/dev_runtime_micro_fakes.cc` fakes three of these builders. See + `test_categories_micro_nccl_device.yaml`. - **`rccl-UnitTestsMicroSymKernels`** — the REAL `src/sym_kernels.cc` (via `SYM_KERNELS_CC_PATH`, from `sym-kernels-test.cc`), compiled together with the GENERATED `sym_kernels_host.cc` it calls into; suites `SymKernelMicrotest.*`, @@ -834,6 +835,7 @@ can be configured **directly** to build every host binary — `rccl-HostUnitTest `rccl-UnitTestsMicro`, `rccl-UnitTestsMicroWarpSpeed`, `rccl-UnitTestsMicroInit[-uncached|-faultinj]`, `rccl-UnitTestsMicroEnqueue[-devlinker]`, `rccl-UnitTestsMicroGinHost`, `rccl-UnitTestsMicroCe`, `rccl-UnitTestsMicroDiagnostics`, +`rccl-UnitTestsMicroNcclDevice`, `rccl-UnitTestsMicroSymKernels` and `rccl-UnitTestsMicroTaskPrep` — **without configuring/building all of librccl**. It compiles just the tests + fakes + the hipified unit-under-test sources. @@ -874,6 +876,7 @@ cmake --build build -j"$(nproc)" ./build/rccl-UnitTestsMicroTaskPrep # src/enqueue/task_prep/ + task_sched/ tests ./build/rccl-UnitTestsMicroDiagnostics # src/diagnostics/{p2p,ib_write_bw}.cc tests ./build/rccl-UnitTestsMicroGinHost # src/gin/gin_host.cc GIN_PROXY_NTHREADS +./build/rccl-UnitTestsMicroNcclDevice # nccl_device requirement builders tests ./build/rccl-UnitTestsMicroDiagnostics # src/diagnostics/p2p.cc tests ./build/rccl-UnitTestsMicroCe # src/ce_coll.cc tests ./build/rccl-HostUnitTests diff --git a/projects/rccl/test/test_categories_micro.yaml b/projects/rccl/test/test_categories_micro.yaml index f8b4c67b0c3c..a940ce5f4f7c 100644 --- a/projects/rccl/test/test_categories_micro.yaml +++ b/projects/rccl/test/test_categories_micro.yaml @@ -114,8 +114,6 @@ test_categories: # ce_coll.cc (rccl-UnitTestsMicroCe) - "CeCollMicrotest.*" - "LaunchPaths/CeAlltoAllvSyncMicrotest.*" - # nccl_device/{cft_barrier,gin_barrier,gin_scratch,ll_a2a,lsa_barrier}.cc (rccl-UnitTestsMicroNcclDevice) - - "NcclDeviceRequirementsMicrotest.*" # dev_runtime.cc - "Alloc*" - "Comm*" diff --git a/projects/rccl/test/test_categories_micro_nccl_device.yaml b/projects/rccl/test/test_categories_micro_nccl_device.yaml new file mode 100644 index 000000000000..2cb9ecc4980a --- /dev/null +++ b/projects/rccl/test/test_categories_micro_nccl_device.yaml @@ -0,0 +1,46 @@ +# rccl-UnitTestsMicroNcclDevice — shared/ctest schema. +# Host-only micro-tests for src/nccl_device/ requirement builders (no GPU, no librccl). +# RCCL does not support Windows: exclude_windows matches all tests so generated suites are empty on Windows. + +test_categories: + quick: µ_nccl_device_quick + description: "Smoke — host-only device-resource requirement builder micro-tests, sub-second." + test_patterns: + - "NcclDevice*" + exclude_windows: + - "*" + labels: + - "rccl" + - "quick" + - "smoke" + - "precheck" + - "microtest" + standard: + <<: *micro_nccl_device_quick + labels: + - "rccl" + - "standard" + - "regression" + - "microtest" + comprehensive: + <<: *micro_nccl_device_quick + labels: + - "rccl" + - "comprehensive" + - "comprehensive_regression" + - "microtest" + full: + <<: *micro_nccl_device_quick + labels: + - "rccl" + - "full" + - "full_regression" + - "microtest" + +execution_settings: + timeout_multiplier: 1 + category_timeouts: + quick: 600 + standard: 600 + comprehensive: 600 + full: 600 diff --git a/projects/rccl/tools/scripts/test_runner/configs/ci-precheckin.json b/projects/rccl/tools/scripts/test_runner/configs/ci-precheckin.json index 07da6b28ba73..d219f834a401 100644 --- a/projects/rccl/tools/scripts/test_runner/configs/ci-precheckin.json +++ b/projects/rccl/tools/scripts/test_runner/configs/ci-precheckin.json @@ -257,6 +257,17 @@ "tests": [ {"name": "GinHostMicrotests", "description": "Host-only gin/gin_host.cc microtests (no GPU, no HIP/RCCL runtime); covers GIN_PROXY_NTHREADS (NVIDIA/nccl#2279 / AICOMRCCL-2017) and proxy-thread CPU affinity (AICOMRCCL-1859)", "test_filter": "*"} ] + }, + "unit_tests_micro_nccl_device": { + "is_gtest": true, + "binary": "host/rccl-UnitTestsMicroNcclDevice", + "num_ranks": 1, + "num_nodes": 1, + "num_gpus": 1, + "timeout": 60, + "tests": [ + {"name": "NcclDeviceMicrotests", "description": "Host-only nccl_device requirement builders microtests (no GPU, no HIP/RCCL runtime)", "test_filter": "*"} + ] } }, "test_suites": [ @@ -319,6 +330,12 @@ "description": "rccl-UnitTestsMicroGinHost (host-only gin/gin_host.cc microtests, no GPU/HIP/RCCL runtime)", "config": "unit_tests_micro_gin_host", "enabled": true + }, + { + "name": "RCCL Microtests - nccl_device requirement builders", + "description": "rccl-UnitTestsMicroNcclDevice (host-only nccl_device requirement builders microtests, no GPU/HIP/RCCL runtime)", + "config": "unit_tests_micro_nccl_device", + "enabled": true } ] } diff --git a/projects/rccl/tools/scripts/test_runner/lib/test_executor.py b/projects/rccl/tools/scripts/test_runner/lib/test_executor.py index 70add7053ab2..f9fadf264536 100644 --- a/projects/rccl/tools/scripts/test_runner/lib/test_executor.py +++ b/projects/rccl/tools/scripts/test_runner/lib/test_executor.py @@ -3292,7 +3292,8 @@ def generate_coverage_report(self): "rccl-UnitTestsMicroInit-faultinj", "rccl-UnitTestsMicroEnqueue", "rccl-UnitTestsMicroSymKernels", - "rccl-UnitTestsMicroGinHost"]: + "rccl-UnitTestsMicroGinHost", + "rccl-UnitTestsMicroNcclDevice"]: binary_path = os.path.join(host_test_dir, binary) if os.path.isfile(binary_path): object_files.extend(["--object", binary_path]) From b8776e63d810095393509bbd4b7b11d8be1ab0c1 Mon Sep 17 00:00:00 2001 From: Alex Duforest Date: Wed, 14 Oct 2026 16:58:48 -0700 Subject: [PATCH 24/26] test(rccl): follow the microtest naming and registration rules Give the binary its own test_categories_micro_.yaml and register it in ci-precheckin.json and test_executor.py's llvm-cov list, as MICROTEST_README's naming section requires, so the CI test runner and coverage pick it up. Rename to core-test.cc / rccl-UnitTestsMicroCore / CoreMicrotest per the rules. Co-Authored-By: Claude --- projects/rccl/test/host/CMakeLists.txt | 14 +++--- projects/rccl/test/host/MICROTEST_README.md | 9 ++-- .../host/{nccl-core-test.cc => core-test.cc} | 34 +++++++------- projects/rccl/test/host/run_host_tests.sh | 2 +- projects/rccl/test/test_categories_micro.yaml | 2 - .../rccl/test/test_categories_micro_core.yaml | 46 +++++++++++++++++++ .../test_runner/configs/ci-precheckin.json | 17 +++++++ .../scripts/test_runner/lib/test_executor.py | 3 +- 8 files changed, 96 insertions(+), 31 deletions(-) rename projects/rccl/test/host/{nccl-core-test.cc => core-test.cc} (80%) create mode 100644 projects/rccl/test/test_categories_micro_core.yaml diff --git a/projects/rccl/test/host/CMakeLists.txt b/projects/rccl/test/host/CMakeLists.txt index 5f0d2f8e3dfc..3204c0f6fdd9 100644 --- a/projects/rccl/test/host/CMakeLists.txt +++ b/projects/rccl/test/host/CMakeLists.txt @@ -293,10 +293,10 @@ function(rccl_define_micro_source_lists) ../common/main_altrsmi.cpp PARENT_SCOPE) - # rccl-UnitTestsMicroNcclDeviceCore -- src/nccl_device/core.cc. Own binary: + # rccl-UnitTestsMicroCore -- src/nccl_device/core.cc. Own binary: # dev_runtime_micro_fakes.cc and devcomm_fakes.cc fake five of its seven functions. - set(TEST_MICRO_NCCL_DEVICE_CORE_SOURCE_FILES - nccl-core-test.cc + set(TEST_MICRO_CORE_SOURCE_FILES + core-test.cc fakes/bootstrap_stubs.cc fakes/topo_stubs.cc fakes/transport_stubs.cc @@ -1080,14 +1080,14 @@ rccl_add_micro_binary(rccl-UnitTestsMicroClientSupport CATEGORIES_YAML "${CMAKE_CURRENT_SOURCE_DIR}/../test_categories_micro.yaml") # --------------------------------------------------------------------------- -# rccl-UnitTestsMicroNcclDeviceCore +# rccl-UnitTestsMicroCore # --------------------------------------------------------------------------- -rccl_add_micro_binary(rccl-UnitTestsMicroNcclDeviceCore - SOURCES ${TEST_MICRO_NCCL_DEVICE_CORE_SOURCE_FILES} +rccl_add_micro_binary(rccl-UnitTestsMicroCore + SOURCES ${TEST_MICRO_CORE_SOURCE_FILES} EXTRA_DEFS ${_MICRO_MODE_DEFS} CORE_CC_PATH="${RCCL_HOST_HIPIFY_DIR}/src/nccl_device/core.cc" - CATEGORIES_YAML "${CMAKE_CURRENT_SOURCE_DIR}/../test_categories_micro.yaml") + CATEGORIES_YAML "${CMAKE_CURRENT_SOURCE_DIR}/../test_categories_micro_core.yaml") # --------------------------------------------------------------------------- # rccl-UnitTestsMicroSymKernels diff --git a/projects/rccl/test/host/MICROTEST_README.md b/projects/rccl/test/host/MICROTEST_README.md index f3082fa04738..13a413ea4794 100644 --- a/projects/rccl/test/host/MICROTEST_README.md +++ b/projects/rccl/test/host/MICROTEST_README.md @@ -248,10 +248,11 @@ export colliding non-`static` symbols; otherwise a unit needs its own binary: guard-less positional pair `libc_seam.h` / `libc_seam_undef.h` must bracket the production source include, after system headers and before test helpers. CMake pins the copied `ncclFuncStr` table to `src/init.cc`. -- **`rccl-UnitTestsMicroNcclDeviceCore`**: `src/nccl_device/core.cc` (via - `CORE_CC_PATH`, from `nccl-core-test.cc`); suite `NcclCoreMicrotest.*`. Its +- **`rccl-UnitTestsMicroCore`**: `src/nccl_device/core.cc` (via + `CORE_CC_PATH`, from `core-test.cc`); suite `CoreMicrotest.*`. Its own binary because `fakes/dev_runtime_micro_fakes.cc` and - `fakes/devcomm_fakes.cc` fake five of its seven functions. + `fakes/devcomm_fakes.cc` fake five of its seven functions. See + `test_categories_micro_core.yaml`. - **`rccl-UnitTestsMicroSymKernels`** — the REAL `src/sym_kernels.cc` (via `SYM_KERNELS_CC_PATH`, from `sym-kernels-test.cc`), compiled together with the GENERATED `sym_kernels_host.cc` it calls into; suites `SymKernelMicrotest.*`, @@ -833,6 +834,7 @@ can be configured **directly** to build every host binary — `rccl-HostUnitTest `rccl-UnitTestsMicro`, `rccl-UnitTestsMicroWarpSpeed`, `rccl-UnitTestsMicroInit[-uncached|-faultinj]`, `rccl-UnitTestsMicroEnqueue[-devlinker]`, `rccl-UnitTestsMicroGinHost`, `rccl-UnitTestsMicroCe`, `rccl-UnitTestsMicroDiagnostics`, +`rccl-UnitTestsMicroCore`, `rccl-UnitTestsMicroSymKernels` and `rccl-UnitTestsMicroTaskPrep` — **without configuring/building all of librccl**. It compiles just the tests + fakes + the hipified unit-under-test sources. @@ -873,6 +875,7 @@ cmake --build build -j"$(nproc)" ./build/rccl-UnitTestsMicroTaskPrep # src/enqueue/task_prep/ + task_sched/ tests ./build/rccl-UnitTestsMicroDiagnostics # src/diagnostics/{p2p,ib_write_bw}.cc tests ./build/rccl-UnitTestsMicroGinHost # src/gin/gin_host.cc GIN_PROXY_NTHREADS +./build/rccl-UnitTestsMicroCore # nccl_device/core.cc tests ./build/rccl-UnitTestsMicroDiagnostics # src/diagnostics/p2p.cc tests ./build/rccl-UnitTestsMicroCe # src/ce_coll.cc tests ./build/rccl-HostUnitTests diff --git a/projects/rccl/test/host/nccl-core-test.cc b/projects/rccl/test/host/core-test.cc similarity index 80% rename from projects/rccl/test/host/nccl-core-test.cc rename to projects/rccl/test/host/core-test.cc index fb8024228a90..c70ef6ce3714 100644 --- a/projects/rccl/test/host/nccl-core-test.cc +++ b/projects/rccl/test/host/core-test.cc @@ -17,7 +17,7 @@ namespace { -class NcclCoreMicrotest : public ::testing::Test { +class CoreMicrotest : public ::testing::Test { protected: std::unique_ptr commStorage; ncclComm* comm = nullptr; @@ -32,7 +32,7 @@ class NcclCoreMicrotest : public ::testing::Test { } // namespace -TEST_F(NcclCoreMicrotest, TeamWorld_ReturnsCommNRanksAndRankWithStrideOne) { +TEST_F(CoreMicrotest, TeamWorld_ReturnsCommNRanksAndRankWithStrideOne) { comm->nRanks = 4; comm->rank = 2; ncclTeam_t team = ncclTeamWorld(comm); @@ -41,7 +41,7 @@ TEST_F(NcclCoreMicrotest, TeamWorld_ReturnsCommNRanksAndRankWithStrideOne) { EXPECT_EQ(team.stride, 1); } -TEST_F(NcclCoreMicrotest, TeamWorldAndRankToWorld_IgnoreAFailingInitOnce) { +TEST_F(CoreMicrotest, TeamWorldAndRankToWorld_IgnoreAFailingInitOnce) { // The two that don't gate on ncclDevrInitOnce still return real values when it fails. g_devrInitOnce = [](ncclComm*) { return ncclSystemError; }; comm->nRanks = 4; @@ -54,7 +54,7 @@ TEST_F(NcclCoreMicrotest, TeamWorldAndRankToWorld_IgnoreAFailingInitOnce) { EXPECT_EQ(ncclTeamRankToWorld(comm, team, team.rank + 1), 5); // 2 + (3 - 2) * 3 } -TEST_F(NcclCoreMicrotest, TeamLsa_SuccessPath_ReadsLsaSizeAndSelfFromDevrState) { +TEST_F(CoreMicrotest, TeamLsa_SuccessPath_ReadsLsaSizeAndSelfFromDevrState) { comm->devrState.lsaSize = 8; comm->devrState.lsaSelf = 3; ncclTeam_t team = ncclTeamLsa(comm); @@ -63,7 +63,7 @@ TEST_F(NcclCoreMicrotest, TeamLsa_SuccessPath_ReadsLsaSizeAndSelfFromDevrState) EXPECT_EQ(team.stride, 1); } -TEST_F(NcclCoreMicrotest, TeamLsa_InitOnceFails_ReturnsZeroedTeamWithoutReadingDevrState) { +TEST_F(CoreMicrotest, TeamLsa_InitOnceFails_ReturnsZeroedTeamWithoutReadingDevrState) { // Poisoned, so reading them despite the failure shows up below. comm->devrState.lsaSize = 99; comm->devrState.lsaSelf = 99; @@ -74,7 +74,7 @@ TEST_F(NcclCoreMicrotest, TeamLsa_InitOnceFails_ReturnsZeroedTeamWithoutReadingD EXPECT_EQ(team.stride, 0); } -TEST_F(NcclCoreMicrotest, TeamCftMultimem_SuccessPath_ReadsCftMcSizeAndSelfFromDevrState) { +TEST_F(CoreMicrotest, TeamCftMultimem_SuccessPath_ReadsCftMcSizeAndSelfFromDevrState) { comm->devrState.cftMcSize = 16; comm->devrState.cftMcSelf = 5; ncclTeam_t team = ncclTeamCftMultimem(comm); @@ -83,7 +83,7 @@ TEST_F(NcclCoreMicrotest, TeamCftMultimem_SuccessPath_ReadsCftMcSizeAndSelfFromD EXPECT_EQ(team.stride, 1); } -TEST_F(NcclCoreMicrotest, TeamCftMultimem_InitOnceFails_ReturnsZeroedTeam) { +TEST_F(CoreMicrotest, TeamCftMultimem_InitOnceFails_ReturnsZeroedTeam) { comm->devrState.cftMcSize = 99; comm->devrState.cftMcSelf = 99; g_devrInitOnce = [](ncclComm*) { return ncclSystemError; }; @@ -93,7 +93,7 @@ TEST_F(NcclCoreMicrotest, TeamCftMultimem_InitOnceFails_ReturnsZeroedTeam) { EXPECT_EQ(team.stride, 0); } -TEST_F(NcclCoreMicrotest, TeamRail_SuccessPath_DividesCommNRanksAndRankByLsaSize) { +TEST_F(CoreMicrotest, TeamRail_SuccessPath_DividesCommNRanksAndRankByLsaSize) { // 24/4 = 6 and 9/4 = 2 differ from the `%` results ncclTeamInnerFactor would give. comm->nRanks = 24; comm->rank = 9; @@ -104,7 +104,7 @@ TEST_F(NcclCoreMicrotest, TeamRail_SuccessPath_DividesCommNRanksAndRankByLsaSize EXPECT_EQ(team.stride, 4); // == lsaSize } -TEST_F(NcclCoreMicrotest, TeamRail_InitOnceFails_ReturnsZeroedTeamWithoutDividingByLsaSize) { +TEST_F(CoreMicrotest, TeamRail_InitOnceFails_ReturnsZeroedTeamWithoutDividingByLsaSize) { // If the failure branch were missing, this would divide by zero instead. comm->nRanks = 16; comm->rank = 10; @@ -116,7 +116,7 @@ TEST_F(NcclCoreMicrotest, TeamRail_InitOnceFails_ReturnsZeroedTeamWithoutDividin EXPECT_EQ(team.stride, 0); } -TEST_F(NcclCoreMicrotest, TeamCft_FlatMode_ReturnsFlatTeamFromCftSizeAndSelf) { +TEST_F(CoreMicrotest, TeamCft_FlatMode_ReturnsFlatTeamFromCftSizeAndSelf) { comm->devrState.cftSize = 12; comm->devrState.cftSelf = 7; ncclTeam_t team = ncclTeamCft(comm, NCCL_CFT_TEAM_FLAT); @@ -125,7 +125,7 @@ TEST_F(NcclCoreMicrotest, TeamCft_FlatMode_ReturnsFlatTeamFromCftSizeAndSelf) { EXPECT_EQ(team.stride, 1); } -TEST_F(NcclCoreMicrotest, TeamCft_HierMultimemMode_OuterFactorsTheFlatTeamByCftMcSize) { +TEST_F(CoreMicrotest, TeamCft_HierMultimemMode_OuterFactorsTheFlatTeamByCftMcSize) { // lsaSize is a distinct non-zero value, so swapped mode arms give a wrong answer, not a crash. comm->devrState.cftSize = 12; comm->devrState.cftSelf = 7; @@ -137,7 +137,7 @@ TEST_F(NcclCoreMicrotest, TeamCft_HierMultimemMode_OuterFactorsTheFlatTeamByCftM EXPECT_EQ(team.stride, 3); // 1 * 3 } -TEST_F(NcclCoreMicrotest, TeamCft_HierLsaMode_OuterFactorsTheFlatTeamByLsaSize) { +TEST_F(CoreMicrotest, TeamCft_HierLsaMode_OuterFactorsTheFlatTeamByLsaSize) { // cftMcSize is distinct for the same reason. comm->devrState.cftSize = 20; comm->devrState.cftSelf = 9; @@ -149,7 +149,7 @@ TEST_F(NcclCoreMicrotest, TeamCft_HierLsaMode_OuterFactorsTheFlatTeamByLsaSize) EXPECT_EQ(team.stride, 5); // 1 * 5 } -TEST_F(NcclCoreMicrotest, TeamCft_UnrecognisedMode_ReturnsZeroedTeam) { +TEST_F(CoreMicrotest, TeamCft_UnrecognisedMode_ReturnsZeroedTeam) { comm->devrState.cftSize = 12; comm->devrState.cftSelf = 7; ncclTeam_t team = ncclTeamCft(comm, static_cast(99)); @@ -158,7 +158,7 @@ TEST_F(NcclCoreMicrotest, TeamCft_UnrecognisedMode_ReturnsZeroedTeam) { EXPECT_EQ(team.stride, 0); } -TEST_F(NcclCoreMicrotest, TeamCft_InitOnceFails_ReturnsZeroedTeamRegardlessOfMode) { +TEST_F(CoreMicrotest, TeamCft_InitOnceFails_ReturnsZeroedTeamRegardlessOfMode) { comm->devrState.cftSize = 99; comm->devrState.cftSelf = 99; g_devrInitOnce = [](ncclComm*) { return ncclSystemError; }; @@ -170,7 +170,7 @@ TEST_F(NcclCoreMicrotest, TeamCft_InitOnceFails_ReturnsZeroedTeamRegardlessOfMod } } -TEST_F(NcclCoreMicrotest, TeamRankToWorld_MapsATeamLocalRankBackToItsWorldRankViaStride) { +TEST_F(CoreMicrotest, TeamRankToWorld_MapsATeamLocalRankBackToItsWorldRankViaStride) { // Pure arithmetic: doesn't gate on ncclDevrInitOnce. comm->rank = 10; ncclTeam_t team{/*nRanks=*/4, /*rank=*/2, /*stride=*/3}; @@ -180,7 +180,7 @@ TEST_F(NcclCoreMicrotest, TeamRankToWorld_MapsATeamLocalRankBackToItsWorldRankVi EXPECT_EQ(ncclTeamRankToWorld(comm, team, team.rank - 1), 7); } -TEST_F(NcclCoreMicrotest, TeamRankToLsa_SuccessPath_MapsATeamLocalRankToAnLsaRankViaLsaSelf) { +TEST_F(CoreMicrotest, TeamRankToLsa_SuccessPath_MapsATeamLocalRankToAnLsaRankViaLsaSelf) { comm->devrState.lsaSelf = 5; ncclTeam_t team{/*nRanks=*/4, /*rank=*/2, /*stride=*/3}; // devrState.lsaSelf + (rank - team.rank) * team.stride @@ -189,7 +189,7 @@ TEST_F(NcclCoreMicrotest, TeamRankToLsa_SuccessPath_MapsATeamLocalRankToAnLsaRan EXPECT_EQ(ncclTeamRankToLsa(comm, team, team.rank - 1), 2); // 5 + (-1)*3 } -TEST_F(NcclCoreMicrotest, TeamRankToLsa_InitOnceFails_ReturnsNegativeOneWithoutReadingLsaSelf) { +TEST_F(CoreMicrotest, TeamRankToLsa_InitOnceFails_ReturnsNegativeOneWithoutReadingLsaSelf) { comm->devrState.lsaSelf = 99; g_devrInitOnce = [](ncclComm*) { return ncclSystemError; }; ncclTeam_t team{/*nRanks=*/4, /*rank=*/2, /*stride=*/3}; diff --git a/projects/rccl/test/host/run_host_tests.sh b/projects/rccl/test/host/run_host_tests.sh index b0400485ebd8..22ce7c9da1c5 100755 --- a/projects/rccl/test/host/run_host_tests.sh +++ b/projects/rccl/test/host/run_host_tests.sh @@ -167,7 +167,7 @@ do_host_tests() { "rccl-UnitTestsMicroTaskPrep:$SCRIPT_DIR/host_tests_micro_taskprep.xml" "rccl-UnitTestsMicroCe:$SCRIPT_DIR/host_tests_micro_ce.xml" "rccl-UnitTestsMicroClientSupport:$SCRIPT_DIR/host_tests_micro_client_support.xml" - "rccl-UnitTestsMicroNcclDeviceCore:$SCRIPT_DIR/host_tests_micro_nccl_device_core.xml" + "rccl-UnitTestsMicroCore:$SCRIPT_DIR/host_tests_micro_core.xml" "rccl-UnitTestsMicroGinHost:$SCRIPT_DIR/host_tests_micro_gin_host.xml" "rccl-UnitTestsMicroDiagnostics:$SCRIPT_DIR/host_tests_micro_diagnostics.xml" ) diff --git a/projects/rccl/test/test_categories_micro.yaml b/projects/rccl/test/test_categories_micro.yaml index dc6b303c9584..a940ce5f4f7c 100644 --- a/projects/rccl/test/test_categories_micro.yaml +++ b/projects/rccl/test/test_categories_micro.yaml @@ -114,8 +114,6 @@ test_categories: # ce_coll.cc (rccl-UnitTestsMicroCe) - "CeCollMicrotest.*" - "LaunchPaths/CeAlltoAllvSyncMicrotest.*" - # nccl_device/core.cc (rccl-UnitTestsMicroNcclDeviceCore) - - "NcclCoreMicrotest.*" # dev_runtime.cc - "Alloc*" - "Comm*" diff --git a/projects/rccl/test/test_categories_micro_core.yaml b/projects/rccl/test/test_categories_micro_core.yaml new file mode 100644 index 000000000000..97d74200b11d --- /dev/null +++ b/projects/rccl/test/test_categories_micro_core.yaml @@ -0,0 +1,46 @@ +# rccl-UnitTestsMicroCore — shared/ctest schema. +# Host-only micro-tests for src/nccl_device/core.cc (no GPU, no librccl). +# RCCL does not support Windows: exclude_windows matches all tests so generated suites are empty on Windows. + +test_categories: + quick: µ_core_quick + description: "Smoke — host-only nccl_device/core.cc team-helper micro-tests, sub-second." + test_patterns: + - "Core*" + exclude_windows: + - "*" + labels: + - "rccl" + - "quick" + - "smoke" + - "precheck" + - "microtest" + standard: + <<: *micro_core_quick + labels: + - "rccl" + - "standard" + - "regression" + - "microtest" + comprehensive: + <<: *micro_core_quick + labels: + - "rccl" + - "comprehensive" + - "comprehensive_regression" + - "microtest" + full: + <<: *micro_core_quick + labels: + - "rccl" + - "full" + - "full_regression" + - "microtest" + +execution_settings: + timeout_multiplier: 1 + category_timeouts: + quick: 600 + standard: 600 + comprehensive: 600 + full: 600 diff --git a/projects/rccl/tools/scripts/test_runner/configs/ci-precheckin.json b/projects/rccl/tools/scripts/test_runner/configs/ci-precheckin.json index 07da6b28ba73..cd2f19721d43 100644 --- a/projects/rccl/tools/scripts/test_runner/configs/ci-precheckin.json +++ b/projects/rccl/tools/scripts/test_runner/configs/ci-precheckin.json @@ -257,6 +257,17 @@ "tests": [ {"name": "GinHostMicrotests", "description": "Host-only gin/gin_host.cc microtests (no GPU, no HIP/RCCL runtime); covers GIN_PROXY_NTHREADS (NVIDIA/nccl#2279 / AICOMRCCL-2017) and proxy-thread CPU affinity (AICOMRCCL-1859)", "test_filter": "*"} ] + }, + "unit_tests_micro_core": { + "is_gtest": true, + "binary": "host/rccl-UnitTestsMicroCore", + "num_ranks": 1, + "num_nodes": 1, + "num_gpus": 1, + "timeout": 60, + "tests": [ + {"name": "CoreMicrotests", "description": "Host-only nccl_device/core.cc microtests (no GPU, no HIP/RCCL runtime)", "test_filter": "*"} + ] } }, "test_suites": [ @@ -319,6 +330,12 @@ "description": "rccl-UnitTestsMicroGinHost (host-only gin/gin_host.cc microtests, no GPU/HIP/RCCL runtime)", "config": "unit_tests_micro_gin_host", "enabled": true + }, + { + "name": "RCCL Microtests - nccl_device/core.cc", + "description": "rccl-UnitTestsMicroCore (host-only nccl_device/core.cc microtests, no GPU/HIP/RCCL runtime)", + "config": "unit_tests_micro_core", + "enabled": true } ] } diff --git a/projects/rccl/tools/scripts/test_runner/lib/test_executor.py b/projects/rccl/tools/scripts/test_runner/lib/test_executor.py index 70add7053ab2..c689c2d0ccab 100644 --- a/projects/rccl/tools/scripts/test_runner/lib/test_executor.py +++ b/projects/rccl/tools/scripts/test_runner/lib/test_executor.py @@ -3292,7 +3292,8 @@ def generate_coverage_report(self): "rccl-UnitTestsMicroInit-faultinj", "rccl-UnitTestsMicroEnqueue", "rccl-UnitTestsMicroSymKernels", - "rccl-UnitTestsMicroGinHost"]: + "rccl-UnitTestsMicroGinHost", + "rccl-UnitTestsMicroCore"]: binary_path = os.path.join(host_test_dir, binary) if os.path.isfile(binary_path): object_files.extend(["--object", binary_path]) From 4e28aebc8737cb0fe183ca33f1cb88af7af6d12a Mon Sep 17 00:00:00 2001 From: Alex Duforest Date: Wed, 14 Oct 2026 16:58:56 -0700 Subject: [PATCH 25/26] test(rccl): follow the microtest naming and registration rules Give the binary its own test_categories_micro_.yaml and register it in ci-precheckin.json and test_executor.py's llvm-cov list, as MICROTEST_README's naming section requires, so the CI test runner and coverage pick it up. Rename to cft-dev-runtime-test.cc / rccl-UnitTestsMicroCftDevRuntime / CftDevRuntime*Microtest per the rules. Co-Authored-By: Claude --- projects/rccl/test/host/CMakeLists.txt | 14 +-- projects/rccl/test/host/MICROTEST_README.md | 11 ++- ...untime-test.cc => cft-dev-runtime-test.cc} | 94 +++++++++---------- projects/rccl/test/host/run_host_tests.sh | 2 +- ...test_categories_micro_cft_dev_runtime.yaml | 46 +++++++++ .../test_runner/configs/ci-precheckin.json | 17 ++++ .../scripts/test_runner/lib/test_executor.py | 3 +- 7 files changed, 127 insertions(+), 60 deletions(-) rename projects/rccl/test/host/{nccl-cft-dev-runtime-test.cc => cft-dev-runtime-test.cc} (80%) create mode 100644 projects/rccl/test/test_categories_micro_cft_dev_runtime.yaml diff --git a/projects/rccl/test/host/CMakeLists.txt b/projects/rccl/test/host/CMakeLists.txt index 7a0001fe8bbd..cd2b2dd4773e 100644 --- a/projects/rccl/test/host/CMakeLists.txt +++ b/projects/rccl/test/host/CMakeLists.txt @@ -293,10 +293,10 @@ function(rccl_define_micro_source_lists) ../common/main_altrsmi.cpp PARENT_SCOPE) - # rccl-UnitTestsMicroNcclDeviceCft -- src/cft_dev_runtime.cc + src/nccl_device/core.cc. Own + # rccl-UnitTestsMicroCftDevRuntime -- src/cft_dev_runtime.cc + src/nccl_device/core.cc. Own # binary: rccl-UnitTestsMicro fakes most of core.cc, which the LE-info queries call. - set(TEST_MICRO_NCCL_DEVICE_CFT_SOURCE_FILES - nccl-cft-dev-runtime-test.cc + set(TEST_MICRO_CFT_DEV_RUNTIME_SOURCE_FILES + cft-dev-runtime-test.cc fakes/cft_dev_runtime_fakes.cc fakes/bootstrap_stubs.cc fakes/topo_stubs.cc @@ -1081,15 +1081,15 @@ rccl_add_micro_binary(rccl-UnitTestsMicroClientSupport CATEGORIES_YAML "${CMAKE_CURRENT_SOURCE_DIR}/../test_categories_micro.yaml") # --------------------------------------------------------------------------- -# rccl-UnitTestsMicroNcclDeviceCft +# rccl-UnitTestsMicroCftDevRuntime # --------------------------------------------------------------------------- -rccl_add_micro_binary(rccl-UnitTestsMicroNcclDeviceCft - SOURCES ${TEST_MICRO_NCCL_DEVICE_CFT_SOURCE_FILES} +rccl_add_micro_binary(rccl-UnitTestsMicroCftDevRuntime + SOURCES ${TEST_MICRO_CFT_DEV_RUNTIME_SOURCE_FILES} EXTRA_DEFS ${_MICRO_MODE_DEFS} CORE_CC_PATH="${RCCL_HOST_HIPIFY_DIR}/src/nccl_device/core.cc" CFT_DEV_RUNTIME_CC_PATH="${RCCL_HOST_HIPIFY_DIR}/src/cft_dev_runtime.cc" - CATEGORIES_YAML "${CMAKE_CURRENT_SOURCE_DIR}/../test_categories_micro.yaml") + CATEGORIES_YAML "${CMAKE_CURRENT_SOURCE_DIR}/../test_categories_micro_cft_dev_runtime.yaml") # --------------------------------------------------------------------------- # rccl-UnitTestsMicroSymKernels diff --git a/projects/rccl/test/host/MICROTEST_README.md b/projects/rccl/test/host/MICROTEST_README.md index f689e31800cf..74451b835313 100644 --- a/projects/rccl/test/host/MICROTEST_README.md +++ b/projects/rccl/test/host/MICROTEST_README.md @@ -248,10 +248,11 @@ export colliding non-`static` symbols; otherwise a unit needs its own binary: guard-less positional pair `libc_seam.h` / `libc_seam_undef.h` must bracket the production source include, after system headers and before test helpers. CMake pins the copied `ncclFuncStr` table to `src/init.cc`. -- **`rccl-UnitTestsMicroNcclDeviceCft`**: `src/cft_dev_runtime.cc` and +- **`rccl-UnitTestsMicroCftDevRuntime`**: `src/cft_dev_runtime.cc` and `src/nccl_device/core.cc` (via `CFT_DEV_RUNTIME_CC_PATH` / `CORE_CC_PATH`, - from `nccl-cft-dev-runtime-test.cc`); suites `Nccl*`. Its own binary because - `rccl-UnitTestsMicro` fakes most of `core.cc`, which the LE-info queries call. + from `cft-dev-runtime-test.cc`); suites `CftDevRuntime*`. Its own binary because + `rccl-UnitTestsMicro` fakes most of `core.cc`, which the LE-info queries call. See + `test_categories_micro_cft_dev_runtime.yaml`. - **`rccl-UnitTestsMicroSymKernels`** — the REAL `src/sym_kernels.cc` (via `SYM_KERNELS_CC_PATH`, from `sym-kernels-test.cc`), compiled together with the GENERATED `sym_kernels_host.cc` it calls into; suites `SymKernelMicrotest.*`, @@ -446,7 +447,7 @@ symbol. | `src/ce_coll.cc` (targets that do not compile the real file) | `fakes/ce_fakes.cc` | | `src/collectives.cc` | `fakes/collectives_fakes.cc` | | `src/dev_runtime.cc` (targets that do not compile the real file) | `fakes/dev_runtime_fakes.cc` | -| `src/dev_runtime.cc` callees of `src/cft_dev_runtime.cc` (`rccl-UnitTestsMicroNcclDeviceCft`) | `fakes/cft_dev_runtime_fakes.cc` | +| `src/dev_runtime.cc` callees of `src/cft_dev_runtime.cc` (`rccl-UnitTestsMicroCftDevRuntime`) | `fakes/cft_dev_runtime_fakes.cc` | | `src/diagnostics.cc` (`ncclDiagChildRun*` external-tool runners) | `fakes/diagnostics_fakes.cc` | | `src/diagnostics/device/p2p.cu` (`ncclDiagP2p*` kernel launchers) | `fakes/diagnostics_p2p_device_fakes.cc` | | `src/graph/*.cc` (topo, paths, search, connect, rome consensus) | `fakes/topo_stubs.cc` | @@ -834,6 +835,7 @@ can be configured **directly** to build every host binary — `rccl-HostUnitTest `rccl-UnitTestsMicro`, `rccl-UnitTestsMicroWarpSpeed`, `rccl-UnitTestsMicroInit[-uncached|-faultinj]`, `rccl-UnitTestsMicroEnqueue[-devlinker]`, `rccl-UnitTestsMicroGinHost`, `rccl-UnitTestsMicroCe`, `rccl-UnitTestsMicroDiagnostics`, +`rccl-UnitTestsMicroCftDevRuntime`, `rccl-UnitTestsMicroSymKernels` and `rccl-UnitTestsMicroTaskPrep` — **without configuring/building all of librccl**. It compiles just the tests + fakes + the hipified unit-under-test sources. @@ -874,6 +876,7 @@ cmake --build build -j"$(nproc)" ./build/rccl-UnitTestsMicroTaskPrep # src/enqueue/task_prep/ + task_sched/ tests ./build/rccl-UnitTestsMicroDiagnostics # src/diagnostics/{p2p,ib_write_bw}.cc tests ./build/rccl-UnitTestsMicroGinHost # src/gin/gin_host.cc GIN_PROXY_NTHREADS +./build/rccl-UnitTestsMicroCftDevRuntime # cft_dev_runtime.cc tests ./build/rccl-UnitTestsMicroDiagnostics # src/diagnostics/p2p.cc tests ./build/rccl-UnitTestsMicroCe # src/ce_coll.cc tests ./build/rccl-HostUnitTests diff --git a/projects/rccl/test/host/nccl-cft-dev-runtime-test.cc b/projects/rccl/test/host/cft-dev-runtime-test.cc similarity index 80% rename from projects/rccl/test/host/nccl-cft-dev-runtime-test.cc rename to projects/rccl/test/host/cft-dev-runtime-test.cc index ae04eca38c2a..a61b1d81a236 100644 --- a/projects/rccl/test/host/nccl-cft-dev-runtime-test.cc +++ b/projects/rccl/test/host/cft-dev-runtime-test.cc @@ -36,7 +36,7 @@ DevrTeamPtr MakeDevrTeam(ncclTeam_t team, ncclCftLeId mcLeId0, ncclCftLeId mcLeI return DevrTeamPtr(t); } -class NcclCftDevRuntimeMicrotest : public ::testing::Test { +class CftDevRuntimeMicrotest : public ::testing::Test { protected: std::unique_ptr commStorage; ncclComm* comm = nullptr; @@ -58,7 +58,7 @@ class NcclCftDevRuntimeMicrotest : public ::testing::Test { // ncclGpuCftSupport: only the zero-and-succeed prologue is compiled on this build. -TEST_F(NcclCftDevRuntimeMicrotest, GpuCftSupport_ZeroesAllThreeOutputsAndSucceeds) { +TEST_F(CftDevRuntimeMicrotest, GpuCftSupport_ZeroesAllThreeOutputsAndSucceeds) { int gpuCftSupport = 99; bool gpuCftMulticastSupport = true, gpuCftCountedSupport = true; EXPECT_EQ(ncclGpuCftSupport(comm, &gpuCftSupport, &gpuCftMulticastSupport, &gpuCftCountedSupport), ncclSuccess); @@ -69,7 +69,7 @@ TEST_F(NcclCftDevRuntimeMicrotest, GpuCftSupport_ZeroesAllThreeOutputsAndSucceed // computeCftSize / computeCftMcSize -TEST_F(NcclCftDevRuntimeMicrotest, ComputeCftSize_AlreadySized_ReturnsCftSizeWithoutCallingComputeLsaSize) { +TEST_F(CftDevRuntimeMicrotest, ComputeCftSize_AlreadySized_ReturnsCftSizeWithoutCallingComputeLsaSize) { comm->devrState.bigSize = 7; comm->devrState.cftSize = 42; bool called = false; @@ -78,7 +78,7 @@ TEST_F(NcclCftDevRuntimeMicrotest, ComputeCftSize_AlreadySized_ReturnsCftSizeWit EXPECT_FALSE(called); } -TEST_F(NcclCftDevRuntimeMicrotest, ComputeCftSize_NotSizedNoGpuCftSupport_ReturnsOneWithoutCallingComputeLsaSize) { +TEST_F(CftDevRuntimeMicrotest, ComputeCftSize_NotSizedNoGpuCftSupport_ReturnsOneWithoutCallingComputeLsaSize) { comm->devrState.bigSize = 0; comm->gpuCftSupport = 0; bool called = false; @@ -87,14 +87,14 @@ TEST_F(NcclCftDevRuntimeMicrotest, ComputeCftSize_NotSizedNoGpuCftSupport_Return EXPECT_FALSE(called); } -TEST_F(NcclCftDevRuntimeMicrotest, ComputeCftSize_NotSizedWithGpuCftSupport_DelegatesToComputeLsaSize) { +TEST_F(CftDevRuntimeMicrotest, ComputeCftSize_NotSizedWithGpuCftSupport_DelegatesToComputeLsaSize) { comm->devrState.bigSize = 0; comm->gpuCftSupport = 13030; g_computeLsaSize = [](ncclComm*) { return 8; }; EXPECT_EQ(computeCftSize(comm), 8); } -TEST_F(NcclCftDevRuntimeMicrotest, ComputeCftSizeAndMcSize_GpuCftSupportJustBelow13030_ReturnsOne) { +TEST_F(CftDevRuntimeMicrotest, ComputeCftSizeAndMcSize_GpuCftSupportJustBelow13030_ReturnsOne) { // 13029 is the first value the `>= 13030` check must reject. comm->devrState.bigSize = 0; comm->gpuCftSupport = 13029; @@ -105,7 +105,7 @@ TEST_F(NcclCftDevRuntimeMicrotest, ComputeCftSizeAndMcSize_GpuCftSupportJustBelo EXPECT_FALSE(called); } -TEST_F(NcclCftDevRuntimeMicrotest, ComputeCftMcSize_AlreadySized_ReturnsCftMcSizeWithoutCallingComputeLsaSize) { +TEST_F(CftDevRuntimeMicrotest, ComputeCftMcSize_AlreadySized_ReturnsCftMcSizeWithoutCallingComputeLsaSize) { comm->devrState.bigSize = 7; comm->devrState.cftMcSize = 24; bool called = false; @@ -114,7 +114,7 @@ TEST_F(NcclCftDevRuntimeMicrotest, ComputeCftMcSize_AlreadySized_ReturnsCftMcSiz EXPECT_FALSE(called); } -TEST_F(NcclCftDevRuntimeMicrotest, ComputeCftMcSize_NotSizedNoGpuCftSupport_ReturnsOneWithoutCallingComputeLsaSize) { +TEST_F(CftDevRuntimeMicrotest, ComputeCftMcSize_NotSizedNoGpuCftSupport_ReturnsOneWithoutCallingComputeLsaSize) { comm->devrState.bigSize = 0; comm->gpuCftSupport = 0; bool called = false; @@ -123,7 +123,7 @@ TEST_F(NcclCftDevRuntimeMicrotest, ComputeCftMcSize_NotSizedNoGpuCftSupport_Retu EXPECT_FALSE(called); } -TEST_F(NcclCftDevRuntimeMicrotest, ComputeCftMcSize_NotSizedWithGpuCftSupport_DelegatesToComputeLsaSize) { +TEST_F(CftDevRuntimeMicrotest, ComputeCftMcSize_NotSizedWithGpuCftSupport_DelegatesToComputeLsaSize) { comm->devrState.bigSize = 0; comm->gpuCftSupport = 13030; g_computeLsaSize = [](ncclComm*) { return 6; }; @@ -133,7 +133,7 @@ TEST_F(NcclCftDevRuntimeMicrotest, ComputeCftMcSize_NotSizedWithGpuCftSupport_De // symBindTeamLe / symUnbindTeamLe: the CUDA calls are compiled out, so they always succeed and // only the ncclDevrWinRegEnabled query is observable. -TEST_F(NcclCftDevRuntimeMicrotest, SymBindTeamLe_AlwaysSucceedsAndConsultsWinRegEnabledWithCftCapability) { +TEST_F(CftDevRuntimeMicrotest, SymBindTeamLe_AlwaysSucceedsAndConsultsWinRegEnabledWithCftCapability) { auto mem = std::make_unique(); mem->winFlags = 0x5a; int sawWinFlags = -1; @@ -148,26 +148,26 @@ TEST_F(NcclCftDevRuntimeMicrotest, SymBindTeamLe_AlwaysSucceedsAndConsultsWinReg EXPECT_EQ(sawCapability, ncclDevrRegisterCft); } -TEST_F(NcclCftDevRuntimeMicrotest, SymBindTeamLe_WinRegDisabled_StillSucceeds) { +TEST_F(CftDevRuntimeMicrotest, SymBindTeamLe_WinRegDisabled_StillSucceeds) { auto mem = std::make_unique(); g_devrWinRegEnabled = [](int, ncclDevrRegisterCapability) { return false; }; EXPECT_EQ(symBindTeamLe(comm, mem.get(), /*le=*/7), ncclSuccess); } -TEST_F(NcclCftDevRuntimeMicrotest, SymBindTeamLe_InvalidLe_StillSucceeds) { +TEST_F(CftDevRuntimeMicrotest, SymBindTeamLe_InvalidLe_StillSucceeds) { auto mem = std::make_unique(); g_devrWinRegEnabled = [](int, ncclDevrRegisterCapability) { return true; }; EXPECT_EQ(symBindTeamLe(comm, mem.get(), /*le=*/NCCL_LE_ID_INVALID), ncclSuccess); } -TEST_F(NcclCftDevRuntimeMicrotest, SymBindTeamLe_GlobalHasSysmemSegment_StillSucceeds) { +TEST_F(CftDevRuntimeMicrotest, SymBindTeamLe_GlobalHasSysmemSegment_StillSucceeds) { auto mem = std::make_unique(); mem->globalHasSysmemSegment = true; g_devrWinRegEnabled = [](int, ncclDevrRegisterCapability) { return true; }; EXPECT_EQ(symBindTeamLe(comm, mem.get(), /*le=*/7), ncclSuccess); } -TEST_F(NcclCftDevRuntimeMicrotest, SymUnbindTeamLe_AlwaysSucceedsAndConsultsWinRegEnabledWithCftCapability) { +TEST_F(CftDevRuntimeMicrotest, SymUnbindTeamLe_AlwaysSucceedsAndConsultsWinRegEnabledWithCftCapability) { auto mem = std::make_unique(); mem->winFlags = 0x3c; int sawWinFlags = -1; @@ -182,19 +182,19 @@ TEST_F(NcclCftDevRuntimeMicrotest, SymUnbindTeamLe_AlwaysSucceedsAndConsultsWinR EXPECT_EQ(sawCapability, ncclDevrRegisterCft); } -TEST_F(NcclCftDevRuntimeMicrotest, SymUnbindTeamLe_WinRegDisabled_StillSucceeds) { +TEST_F(CftDevRuntimeMicrotest, SymUnbindTeamLe_WinRegDisabled_StillSucceeds) { auto mem = std::make_unique(); g_devrWinRegEnabled = [](int, ncclDevrRegisterCapability) { return false; }; EXPECT_EQ(symUnbindTeamLe(comm, mem.get(), /*le=*/3), ncclSuccess); } -TEST_F(NcclCftDevRuntimeMicrotest, SymUnbindTeamLe_InvalidLe_StillSucceeds) { +TEST_F(CftDevRuntimeMicrotest, SymUnbindTeamLe_InvalidLe_StillSucceeds) { auto mem = std::make_unique(); g_devrWinRegEnabled = [](int, ncclDevrRegisterCapability) { return true; }; EXPECT_EQ(symUnbindTeamLe(comm, mem.get(), /*le=*/NCCL_LE_ID_INVALID), ncclSuccess); } -TEST_F(NcclCftDevRuntimeMicrotest, SymUnbindTeamLe_GlobalHasSysmemSegment_StillSucceeds) { +TEST_F(CftDevRuntimeMicrotest, SymUnbindTeamLe_GlobalHasSysmemSegment_StillSucceeds) { auto mem = std::make_unique(); mem->globalHasSysmemSegment = true; g_devrWinRegEnabled = [](int, ncclDevrRegisterCapability) { return true; }; @@ -203,20 +203,20 @@ TEST_F(NcclCftDevRuntimeMicrotest, SymUnbindTeamLe_GlobalHasSysmemSegment_StillS // symTeamObtainUcLe / symTeamObtainMcLe: this build's stand-ins always fail. -TEST_F(NcclCftDevRuntimeMicrotest, SymTeamObtainUcLe_UnsupportedOnThisPlatform_ReturnsInvalidUsage) { +TEST_F(CftDevRuntimeMicrotest, SymTeamObtainUcLe_UnsupportedOnThisPlatform_ReturnsInvalidUsage) { bool needBarrier = false; EXPECT_EQ(symTeamObtainUcLe(comm, /*t=*/nullptr, /*devr=*/nullptr, &needBarrier, /*counted=*/false), ncclInvalidUsage); } -TEST_F(NcclCftDevRuntimeMicrotest, SymTeamObtainMcLe_UnsupportedOnThisPlatform_ReturnsInvalidUsage) { +TEST_F(CftDevRuntimeMicrotest, SymTeamObtainMcLe_UnsupportedOnThisPlatform_ReturnsInvalidUsage) { bool needBarrier = false; EXPECT_EQ(symTeamObtainMcLe(comm, /*t=*/nullptr, /*devr=*/nullptr, &needBarrier, /*counted=*/true), ncclInvalidUsage); } // Shared setup for the three ncclGet*DeviceLeInfo queries; BigOffset() is distinct per suite. -class NcclGetDeviceLeInfoTestBase : public NcclCftDevRuntimeMicrotest { +class CftDevRuntimeLeInfoMicrotestBase : public CftDevRuntimeMicrotest { protected: ncclDevrWindow winHost{}; int dummyWindowTarget = 0; @@ -231,7 +231,7 @@ class NcclGetDeviceLeInfoTestBase : public NcclCftDevRuntimeMicrotest { virtual ncclResult_t Invoke(ncclWindow_t w, ncclCftLeId* leId, size_t* leOffset) = 0; void SetUp() override { - NcclCftDevRuntimeMicrotest::SetUp(); + CftDevRuntimeMicrotest::SetUp(); comm->gpuCftSupport = 13030; // At cftSize 0 the peer query rejects every peer before reaching the checks below. comm->devrState.cftSize = 4; @@ -284,7 +284,7 @@ class NcclGetDeviceLeInfoTestBase : public NcclCftDevRuntimeMicrotest { // ncclGetMultimemDeviceLeInfo -class NcclGetMultimemDeviceLeInfoTest : public NcclGetDeviceLeInfoTestBase { +class CftDevRuntimeMultimemLeInfoMicrotest : public CftDevRuntimeLeInfoMicrotestBase { protected: size_t BigOffset() const override { return 0x1000; } ncclResult_t Invoke(ncclWindow_t w, ncclCftLeId* leId, size_t* leOffset) override { @@ -292,20 +292,20 @@ class NcclGetMultimemDeviceLeInfoTest : public NcclGetDeviceLeInfoTestBase { } }; -TEST_F(NcclGetMultimemDeviceLeInfoTest, NullArgs_RejectedByPtrCheckBeforeAnyLookup) { +TEST_F(CftDevRuntimeMultimemLeInfoMicrotest, NullArgs_RejectedByPtrCheckBeforeAnyLookup) { ExpectNullArgsRejectedByPtrCheck(); } -TEST_F(NcclGetMultimemDeviceLeInfoTest, WindowNotCftRegistered_ReturnsInvalidUsage) { +TEST_F(CftDevRuntimeMultimemLeInfoMicrotest, WindowNotCftRegistered_ReturnsInvalidUsage) { ExpectWindowNotCftRegisteredReturnsInvalidUsage(); } -TEST_F(NcclGetMultimemDeviceLeInfoTest, CommHasNoGpuCftSupport_ReturnsInvalidArgument) { +TEST_F(CftDevRuntimeMultimemLeInfoMicrotest, CommHasNoGpuCftSupport_ReturnsInvalidArgument) { ExpectNoGpuCftSupportReturnsInvalidArgument(); } -TEST_F(NcclGetMultimemDeviceLeInfoTest, FindCommAndHostWindowFails_PropagatesErrorViaNcclcheck) { +TEST_F(CftDevRuntimeMultimemLeInfoMicrotest, FindCommAndHostWindowFails_PropagatesErrorViaNcclcheck) { ExpectFindCommAndHostWindowFailurePropagates(); } -TEST_F(NcclGetMultimemDeviceLeInfoTest, SymTeamObtainFails_PropagatesErrorViaNcclcheck) { +TEST_F(CftDevRuntimeMultimemLeInfoMicrotest, SymTeamObtainFails_PropagatesErrorViaNcclcheck) { g_symTeamObtain = [](ncclComm*, ncclTeam, bool, bool, bool, bool, ncclDevrTeam**, bool*) { return ncclInvalidUsage; }; @@ -317,7 +317,7 @@ TEST_F(NcclGetMultimemDeviceLeInfoTest, SymTeamObtainFails_PropagatesErrorViaNcc // The bootstrapIntraNodeBarrier failure branch isn't covered: fakes/bootstrap_stubs.cc has no // failure hook. -TEST_F(NcclGetMultimemDeviceLeInfoTest, SuccessPath_ReadsMcLeIdByCountedBitAndOffsetsByBigOffsetPlusOffset) { +TEST_F(CftDevRuntimeMultimemLeInfoMicrotest, SuccessPath_ReadsMcLeIdByCountedBitAndOffsetsByBigOffsetPlusOffset) { comm->devrState.cftMcSize = 4; comm->devrState.cftMcSelf = 1; // SYMMETRIC alongside CFT_COUNTED, so `== NCCL_WIN_CFT_COUNTED` would fail where `&` passes. @@ -345,7 +345,7 @@ TEST_F(NcclGetMultimemDeviceLeInfoTest, SuccessPath_ReadsMcLeIdByCountedBitAndOf EXPECT_EQ(leOffset, winHost.bigOffset + 0x20); } -TEST_F(NcclGetMultimemDeviceLeInfoTest, UncountedWindow_ReadsMcLeIdZeroSlot) { +TEST_F(CftDevRuntimeMultimemLeInfoMicrotest, UncountedWindow_ReadsMcLeIdZeroSlot) { comm->devrState.cftMcSize = 4; comm->devrState.cftMcSelf = 0; winHost.winFlags = 0; // not counted @@ -366,7 +366,7 @@ TEST_F(NcclGetMultimemDeviceLeInfoTest, UncountedWindow_ReadsMcLeIdZeroSlot) { EXPECT_EQ(leId, 55u); } -TEST_F(NcclGetMultimemDeviceLeInfoTest, SymmetricOnlyWindow_IsUncountedAndReadsMcLeIdZeroSlot) { +TEST_F(CftDevRuntimeMultimemLeInfoMicrotest, SymmetricOnlyWindow_IsUncountedAndReadsMcLeIdZeroSlot) { // SYMMETRIC without CFT_COUNTED: testing the wrong bit would read the counted slot. comm->devrState.cftMcSize = 4; comm->devrState.cftMcSelf = 0; @@ -390,7 +390,7 @@ TEST_F(NcclGetMultimemDeviceLeInfoTest, SymmetricOnlyWindow_IsUncountedAndReadsM // ncclGetCftDeviceLeInfo -class NcclGetCftDeviceLeInfoTest : public NcclGetDeviceLeInfoTestBase { +class CftDevRuntimeCftLeInfoMicrotest : public CftDevRuntimeLeInfoMicrotestBase { protected: size_t BigOffset() const override { return 0x2000; } ncclResult_t Invoke(ncclWindow_t w, ncclCftLeId* leId, size_t* leOffset) override { @@ -398,20 +398,20 @@ class NcclGetCftDeviceLeInfoTest : public NcclGetDeviceLeInfoTestBase { } }; -TEST_F(NcclGetCftDeviceLeInfoTest, NullArgs_RejectedByPtrCheck) { +TEST_F(CftDevRuntimeCftLeInfoMicrotest, NullArgs_RejectedByPtrCheck) { ExpectNullArgsRejectedByPtrCheck(); } -TEST_F(NcclGetCftDeviceLeInfoTest, WindowNotCftRegistered_ReturnsInvalidUsage) { +TEST_F(CftDevRuntimeCftLeInfoMicrotest, WindowNotCftRegistered_ReturnsInvalidUsage) { ExpectWindowNotCftRegisteredReturnsInvalidUsage(); } -TEST_F(NcclGetCftDeviceLeInfoTest, CommHasNoGpuCftSupport_ReturnsInvalidArgument) { +TEST_F(CftDevRuntimeCftLeInfoMicrotest, CommHasNoGpuCftSupport_ReturnsInvalidArgument) { ExpectNoGpuCftSupportReturnsInvalidArgument(); } -TEST_F(NcclGetCftDeviceLeInfoTest, FindCommAndHostWindowFails_PropagatesErrorViaNcclcheck) { +TEST_F(CftDevRuntimeCftLeInfoMicrotest, FindCommAndHostWindowFails_PropagatesErrorViaNcclcheck) { ExpectFindCommAndHostWindowFailurePropagates(); } -TEST_F(NcclGetCftDeviceLeInfoTest, BaseLeIdNotYetCreated_ReturnsInvalidUsage) { +TEST_F(CftDevRuntimeCftLeInfoMicrotest, BaseLeIdNotYetCreated_ReturnsInvalidUsage) { comm->devrState.le[0].baseId = NCCL_LE_ID_INVALID; ncclTeam_t cftTeam{4, 0, 1}; ncclCftLeId leId; @@ -419,7 +419,7 @@ TEST_F(NcclGetCftDeviceLeInfoTest, BaseLeIdNotYetCreated_ReturnsInvalidUsage) { EXPECT_EQ(ncclGetCftDeviceLeInfo(window, 0, 0, cftTeam, &leId, &leOffset), ncclInvalidUsage); } -TEST_F(NcclGetCftDeviceLeInfoTest, SuccessPath_AddsFlatTeamRankAndPeerOffsetToBaseLeId) { +TEST_F(CftDevRuntimeCftLeInfoMicrotest, SuccessPath_AddsFlatTeamRankAndPeerOffsetToBaseLeId) { comm->devrState.cftSize = 8; comm->devrState.cftSelf = 3; comm->devrState.le[0].baseId = 1000; @@ -435,7 +435,7 @@ TEST_F(NcclGetCftDeviceLeInfoTest, SuccessPath_AddsFlatTeamRankAndPeerOffsetToBa EXPECT_EQ(leOffset, winHost.bigOffset + 0x10); } -TEST_F(NcclGetCftDeviceLeInfoTest, CountedWindow_UsesCountedBaseLeIdSlot) { +TEST_F(CftDevRuntimeCftLeInfoMicrotest, CountedWindow_UsesCountedBaseLeIdSlot) { comm->devrState.cftSize = 4; comm->devrState.cftSelf = 0; comm->devrState.le[0].baseId = NCCL_LE_ID_INVALID; // uncounted slot deliberately unset @@ -451,7 +451,7 @@ TEST_F(NcclGetCftDeviceLeInfoTest, CountedWindow_UsesCountedBaseLeIdSlot) { // ncclGetPeerDeviceLeInfo -class NcclGetPeerDeviceLeInfoTest : public NcclGetDeviceLeInfoTestBase { +class CftDevRuntimePeerLeInfoMicrotest : public CftDevRuntimeLeInfoMicrotestBase { protected: size_t BigOffset() const override { return 0x3000; } ncclResult_t Invoke(ncclWindow_t w, ncclCftLeId* leId, size_t* leOffset) override { @@ -459,20 +459,20 @@ class NcclGetPeerDeviceLeInfoTest : public NcclGetDeviceLeInfoTestBase { } }; -TEST_F(NcclGetPeerDeviceLeInfoTest, NullArgs_RejectedByPtrCheck) { +TEST_F(CftDevRuntimePeerLeInfoMicrotest, NullArgs_RejectedByPtrCheck) { ExpectNullArgsRejectedByPtrCheck(); } -TEST_F(NcclGetPeerDeviceLeInfoTest, WindowNotCftRegistered_ReturnsInvalidUsage) { +TEST_F(CftDevRuntimePeerLeInfoMicrotest, WindowNotCftRegistered_ReturnsInvalidUsage) { ExpectWindowNotCftRegisteredReturnsInvalidUsage(); } -TEST_F(NcclGetPeerDeviceLeInfoTest, CommHasNoGpuCftSupport_ReturnsInvalidArgument) { +TEST_F(CftDevRuntimePeerLeInfoMicrotest, CommHasNoGpuCftSupport_ReturnsInvalidArgument) { ExpectNoGpuCftSupportReturnsInvalidArgument(); } -TEST_F(NcclGetPeerDeviceLeInfoTest, FindCommAndHostWindowFails_PropagatesErrorViaNcclcheck) { +TEST_F(CftDevRuntimePeerLeInfoMicrotest, FindCommAndHostWindowFails_PropagatesErrorViaNcclcheck) { ExpectFindCommAndHostWindowFailurePropagates(); } -TEST_F(NcclGetPeerDeviceLeInfoTest, PeerOutsideFlatTeamBoundaries_ReturnsInvalidArgument) { +TEST_F(CftDevRuntimePeerLeInfoMicrotest, PeerOutsideFlatTeamBoundaries_ReturnsInvalidArgument) { // Valid range is [flatStart, flatStart + nRanks) = [8, 12); 12 and 8 pin the two edges. comm->rank = 10; comm->devrState.cftSize = 4; @@ -489,7 +489,7 @@ TEST_F(NcclGetPeerDeviceLeInfoTest, PeerOutsideFlatTeamBoundaries_ReturnsInvalid EXPECT_EQ(leId, 1u); // baseLeId(1) + flatTeam.rank(2) + (peerWorld(8) - comm->rank(10)) == 1 } -TEST_F(NcclGetPeerDeviceLeInfoTest, CountedWindow_UsesCountedBaseLeIdSlot) { +TEST_F(CftDevRuntimePeerLeInfoMicrotest, CountedWindow_UsesCountedBaseLeIdSlot) { comm->rank = 10; comm->devrState.cftSize = 4; comm->devrState.cftSelf = 2; // flatStart = 8, valid peers in [8, 12) @@ -504,7 +504,7 @@ TEST_F(NcclGetPeerDeviceLeInfoTest, CountedWindow_UsesCountedBaseLeIdSlot) { EXPECT_EQ(leId, 2003u); } -TEST_F(NcclGetPeerDeviceLeInfoTest, BaseLeIdNotYetCreated_ReturnsInvalidUsage) { +TEST_F(CftDevRuntimePeerLeInfoMicrotest, BaseLeIdNotYetCreated_ReturnsInvalidUsage) { comm->rank = 10; comm->devrState.cftSize = 4; comm->devrState.cftSelf = 2; @@ -514,7 +514,7 @@ TEST_F(NcclGetPeerDeviceLeInfoTest, BaseLeIdNotYetCreated_ReturnsInvalidUsage) { EXPECT_EQ(ncclGetPeerDeviceLeInfo(window, 0, /*peerWorld=*/10, &leId, &leOffset), ncclInvalidUsage); } -TEST_F(NcclGetPeerDeviceLeInfoTest, SuccessPath_AddsFlatTeamRankAndWorldOffsetToBaseLeId) { +TEST_F(CftDevRuntimePeerLeInfoMicrotest, SuccessPath_AddsFlatTeamRankAndWorldOffsetToBaseLeId) { comm->rank = 10; comm->devrState.cftSize = 4; comm->devrState.cftSelf = 2; // flatStart = 8, valid peers in [8, 12) diff --git a/projects/rccl/test/host/run_host_tests.sh b/projects/rccl/test/host/run_host_tests.sh index e03682484e35..e78a78cad1ba 100755 --- a/projects/rccl/test/host/run_host_tests.sh +++ b/projects/rccl/test/host/run_host_tests.sh @@ -167,7 +167,7 @@ do_host_tests() { "rccl-UnitTestsMicroTaskPrep:$SCRIPT_DIR/host_tests_micro_taskprep.xml" "rccl-UnitTestsMicroCe:$SCRIPT_DIR/host_tests_micro_ce.xml" "rccl-UnitTestsMicroClientSupport:$SCRIPT_DIR/host_tests_micro_client_support.xml" - "rccl-UnitTestsMicroNcclDeviceCft:$SCRIPT_DIR/host_tests_micro_nccl_device_cft.xml" + "rccl-UnitTestsMicroCftDevRuntime:$SCRIPT_DIR/host_tests_micro_cft_dev_runtime.xml" "rccl-UnitTestsMicroGinHost:$SCRIPT_DIR/host_tests_micro_gin_host.xml" "rccl-UnitTestsMicroDiagnostics:$SCRIPT_DIR/host_tests_micro_diagnostics.xml" ) diff --git a/projects/rccl/test/test_categories_micro_cft_dev_runtime.yaml b/projects/rccl/test/test_categories_micro_cft_dev_runtime.yaml new file mode 100644 index 000000000000..9e6dd846d7f4 --- /dev/null +++ b/projects/rccl/test/test_categories_micro_cft_dev_runtime.yaml @@ -0,0 +1,46 @@ +# rccl-UnitTestsMicroCftDevRuntime — shared/ctest schema. +# Host-only micro-tests for src/cft_dev_runtime.cc (no GPU, no librccl). +# RCCL does not support Windows: exclude_windows matches all tests so generated suites are empty on Windows. + +test_categories: + quick: µ_cft_dev_runtime_quick + description: "Smoke — host-only cft_dev_runtime.cc micro-tests, sub-second." + test_patterns: + - "CftDevRuntime*" + exclude_windows: + - "*" + labels: + - "rccl" + - "quick" + - "smoke" + - "precheck" + - "microtest" + standard: + <<: *micro_cft_dev_runtime_quick + labels: + - "rccl" + - "standard" + - "regression" + - "microtest" + comprehensive: + <<: *micro_cft_dev_runtime_quick + labels: + - "rccl" + - "comprehensive" + - "comprehensive_regression" + - "microtest" + full: + <<: *micro_cft_dev_runtime_quick + labels: + - "rccl" + - "full" + - "full_regression" + - "microtest" + +execution_settings: + timeout_multiplier: 1 + category_timeouts: + quick: 600 + standard: 600 + comprehensive: 600 + full: 600 diff --git a/projects/rccl/tools/scripts/test_runner/configs/ci-precheckin.json b/projects/rccl/tools/scripts/test_runner/configs/ci-precheckin.json index 07da6b28ba73..578f207a21d6 100644 --- a/projects/rccl/tools/scripts/test_runner/configs/ci-precheckin.json +++ b/projects/rccl/tools/scripts/test_runner/configs/ci-precheckin.json @@ -257,6 +257,17 @@ "tests": [ {"name": "GinHostMicrotests", "description": "Host-only gin/gin_host.cc microtests (no GPU, no HIP/RCCL runtime); covers GIN_PROXY_NTHREADS (NVIDIA/nccl#2279 / AICOMRCCL-2017) and proxy-thread CPU affinity (AICOMRCCL-1859)", "test_filter": "*"} ] + }, + "unit_tests_micro_cft_dev_runtime": { + "is_gtest": true, + "binary": "host/rccl-UnitTestsMicroCftDevRuntime", + "num_ranks": 1, + "num_nodes": 1, + "num_gpus": 1, + "timeout": 60, + "tests": [ + {"name": "CftDevRuntimeMicrotests", "description": "Host-only cft_dev_runtime.cc microtests (no GPU, no HIP/RCCL runtime)", "test_filter": "*"} + ] } }, "test_suites": [ @@ -319,6 +330,12 @@ "description": "rccl-UnitTestsMicroGinHost (host-only gin/gin_host.cc microtests, no GPU/HIP/RCCL runtime)", "config": "unit_tests_micro_gin_host", "enabled": true + }, + { + "name": "RCCL Microtests - cft_dev_runtime.cc", + "description": "rccl-UnitTestsMicroCftDevRuntime (host-only cft_dev_runtime.cc microtests, no GPU/HIP/RCCL runtime)", + "config": "unit_tests_micro_cft_dev_runtime", + "enabled": true } ] } diff --git a/projects/rccl/tools/scripts/test_runner/lib/test_executor.py b/projects/rccl/tools/scripts/test_runner/lib/test_executor.py index 70add7053ab2..fea2cd2ca07f 100644 --- a/projects/rccl/tools/scripts/test_runner/lib/test_executor.py +++ b/projects/rccl/tools/scripts/test_runner/lib/test_executor.py @@ -3292,7 +3292,8 @@ def generate_coverage_report(self): "rccl-UnitTestsMicroInit-faultinj", "rccl-UnitTestsMicroEnqueue", "rccl-UnitTestsMicroSymKernels", - "rccl-UnitTestsMicroGinHost"]: + "rccl-UnitTestsMicroGinHost", + "rccl-UnitTestsMicroCftDevRuntime"]: binary_path = os.path.join(host_test_dir, binary) if os.path.isfile(binary_path): object_files.extend(["--object", binary_path]) From cf0a98af4a4281699edf8e60b517aaeeefec641c Mon Sep 17 00:00:00 2001 From: Alex Duforest Date: Fri, 16 Oct 2026 14:26:18 -0700 Subject: [PATCH 26/26] test(rccl): move the cft_dev_runtime hooks into dev_runtime_fakes The four dev_runtime.cc hooks now live in fakes/dev_runtime_fakes.cc and fail loud by default. Drop the six symBindTeamLe/symUnbindTeamLe tests that cannot fail on this build, and name the divide-by-zero pin KnownBug_. Co-Authored-By: Claude --- projects/rccl/test/host/CMakeLists.txt | 1 - projects/rccl/test/host/MICROTEST_README.md | 1 - .../rccl/test/host/cft-dev-runtime-test.cc | 40 ----------- .../test/host/fakes/cft_dev_runtime_fakes.cc | 68 ------------------- .../test/host/fakes/cft_dev_runtime_fakes.h | 51 -------------- .../rccl/test/host/fakes/dev_runtime_fakes.cc | 52 ++++++++++++++ .../rccl/test/host/fakes/dev_runtime_fakes.h | 17 +++++ .../host/nccl-device-requirements-test.cc | 10 ++- 8 files changed, 73 insertions(+), 167 deletions(-) delete mode 100644 projects/rccl/test/host/fakes/cft_dev_runtime_fakes.cc delete mode 100644 projects/rccl/test/host/fakes/cft_dev_runtime_fakes.h diff --git a/projects/rccl/test/host/CMakeLists.txt b/projects/rccl/test/host/CMakeLists.txt index e52947bef7ed..a08f8e702618 100644 --- a/projects/rccl/test/host/CMakeLists.txt +++ b/projects/rccl/test/host/CMakeLists.txt @@ -354,7 +354,6 @@ function(rccl_define_micro_source_lists) # binary: rccl-UnitTestsMicro fakes most of core.cc, which the LE-info queries call. set(TEST_MICRO_CFT_DEV_RUNTIME_SOURCE_FILES cft-dev-runtime-test.cc - fakes/cft_dev_runtime_fakes.cc fakes/bootstrap_stubs.cc fakes/topo_stubs.cc fakes/transport_stubs.cc diff --git a/projects/rccl/test/host/MICROTEST_README.md b/projects/rccl/test/host/MICROTEST_README.md index 0b9c9c936656..3196de9caba1 100644 --- a/projects/rccl/test/host/MICROTEST_README.md +++ b/projects/rccl/test/host/MICROTEST_README.md @@ -487,7 +487,6 @@ symbol. | `src/ce_coll.cc` (targets that do not compile the real file) | `fakes/ce_fakes.cc` | | `src/collectives.cc` | `fakes/collectives_fakes.cc` | | `src/dev_runtime.cc` (targets that do not compile the real file) | `fakes/dev_runtime_fakes.cc` | -| `src/dev_runtime.cc` callees of `src/cft_dev_runtime.cc` (`rccl-UnitTestsMicroCftDevRuntime`) | `fakes/cft_dev_runtime_fakes.cc` | | `src/diagnostics.cc` (`ncclDiagChildRun*` external-tool runners) | `fakes/diagnostics_fakes.cc` | | `src/diagnostics/device/p2p.cu` (`ncclDiagP2p*` kernel launchers) | `fakes/diagnostics_p2p_device_fakes.cc` | | `src/graph/*.cc` (topo, paths, search, connect, rome consensus) | `fakes/topo_stubs.cc` | diff --git a/projects/rccl/test/host/cft-dev-runtime-test.cc b/projects/rccl/test/host/cft-dev-runtime-test.cc index a61b1d81a236..0b3cd354a6bc 100644 --- a/projects/rccl/test/host/cft-dev-runtime-test.cc +++ b/projects/rccl/test/host/cft-dev-runtime-test.cc @@ -12,7 +12,6 @@ #include #include -#include "fakes/cft_dev_runtime_fakes.h" #include "fakes/dev_runtime_fakes.h" #include CORE_CC_PATH @@ -50,7 +49,6 @@ class CftDevRuntimeMicrotest : public ::testing::Test { void TearDown() override { ResetDevRuntimeFakes(); - ResetCftDevRuntimeFakes(); } }; @@ -148,25 +146,6 @@ TEST_F(CftDevRuntimeMicrotest, SymBindTeamLe_AlwaysSucceedsAndConsultsWinRegEnab EXPECT_EQ(sawCapability, ncclDevrRegisterCft); } -TEST_F(CftDevRuntimeMicrotest, SymBindTeamLe_WinRegDisabled_StillSucceeds) { - auto mem = std::make_unique(); - g_devrWinRegEnabled = [](int, ncclDevrRegisterCapability) { return false; }; - EXPECT_EQ(symBindTeamLe(comm, mem.get(), /*le=*/7), ncclSuccess); -} - -TEST_F(CftDevRuntimeMicrotest, SymBindTeamLe_InvalidLe_StillSucceeds) { - auto mem = std::make_unique(); - g_devrWinRegEnabled = [](int, ncclDevrRegisterCapability) { return true; }; - EXPECT_EQ(symBindTeamLe(comm, mem.get(), /*le=*/NCCL_LE_ID_INVALID), ncclSuccess); -} - -TEST_F(CftDevRuntimeMicrotest, SymBindTeamLe_GlobalHasSysmemSegment_StillSucceeds) { - auto mem = std::make_unique(); - mem->globalHasSysmemSegment = true; - g_devrWinRegEnabled = [](int, ncclDevrRegisterCapability) { return true; }; - EXPECT_EQ(symBindTeamLe(comm, mem.get(), /*le=*/7), ncclSuccess); -} - TEST_F(CftDevRuntimeMicrotest, SymUnbindTeamLe_AlwaysSucceedsAndConsultsWinRegEnabledWithCftCapability) { auto mem = std::make_unique(); mem->winFlags = 0x3c; @@ -182,25 +161,6 @@ TEST_F(CftDevRuntimeMicrotest, SymUnbindTeamLe_AlwaysSucceedsAndConsultsWinRegEn EXPECT_EQ(sawCapability, ncclDevrRegisterCft); } -TEST_F(CftDevRuntimeMicrotest, SymUnbindTeamLe_WinRegDisabled_StillSucceeds) { - auto mem = std::make_unique(); - g_devrWinRegEnabled = [](int, ncclDevrRegisterCapability) { return false; }; - EXPECT_EQ(symUnbindTeamLe(comm, mem.get(), /*le=*/3), ncclSuccess); -} - -TEST_F(CftDevRuntimeMicrotest, SymUnbindTeamLe_InvalidLe_StillSucceeds) { - auto mem = std::make_unique(); - g_devrWinRegEnabled = [](int, ncclDevrRegisterCapability) { return true; }; - EXPECT_EQ(symUnbindTeamLe(comm, mem.get(), /*le=*/NCCL_LE_ID_INVALID), ncclSuccess); -} - -TEST_F(CftDevRuntimeMicrotest, SymUnbindTeamLe_GlobalHasSysmemSegment_StillSucceeds) { - auto mem = std::make_unique(); - mem->globalHasSysmemSegment = true; - g_devrWinRegEnabled = [](int, ncclDevrRegisterCapability) { return true; }; - EXPECT_EQ(symUnbindTeamLe(comm, mem.get(), /*le=*/3), ncclSuccess); -} - // symTeamObtainUcLe / symTeamObtainMcLe: this build's stand-ins always fail. TEST_F(CftDevRuntimeMicrotest, SymTeamObtainUcLe_UnsupportedOnThisPlatform_ReturnsInvalidUsage) { diff --git a/projects/rccl/test/host/fakes/cft_dev_runtime_fakes.cc b/projects/rccl/test/host/fakes/cft_dev_runtime_fakes.cc deleted file mode 100644 index 62ed86c48bb1..000000000000 --- a/projects/rccl/test/host/fakes/cft_dev_runtime_fakes.cc +++ /dev/null @@ -1,68 +0,0 @@ -/************************************************************************* - * Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. - * - * See LICENSE.txt for license information - ************************************************************************/ - -// See cft_dev_runtime_fakes.h. - -#include "cft_dev_runtime_fakes.h" - -#include "dev_runtime_internal.h" -#include "fail_loud.h" -#include "signature-drift.h" - -ASSERT_HOOK_MATCHES_PROD(g_computeLsaSize, computeLsaSize); -ASSERT_HOOK_MATCHES_PROD(g_devrWinRegEnabled, ncclDevrWinRegEnabled); -ASSERT_HOOK_MATCHES_PROD(g_findCommAndHostWindowFromDeviceWindow, findCommAndHostWindowFromDeviceWindow); -ASSERT_HOOK_MATCHES_PROD(g_symTeamObtain, symTeamObtain); -#undef ASSERT_HOOK_MATCHES_PROD - -static int DefaultComputeLsaSize(struct ncclComm*) { return 1; } -std::function g_computeLsaSize = DefaultComputeLsaSize; -int computeLsaSize(struct ncclComm* comm) { return g_computeLsaSize(comm); } - -static bool DefaultDevrWinRegEnabled(int, ncclDevrRegisterCapability) { return true; } -std::function g_devrWinRegEnabled = DefaultDevrWinRegEnabled; -bool ncclDevrWinRegEnabled(int winFlags, ncclDevrRegisterCapability capability) { - return g_devrWinRegEnabled(winFlags, capability); -} - -ncclComm_t g_foundComm = nullptr; -struct ncclDevrWindow* g_foundWinHost = nullptr; -static ncclResult_t FailLoudFindCommAndHostWindow(ncclWindow_t, ncclComm_t*, struct ncclDevrWindow**) { - FailLoudUnfaked("cft_dev_runtime_fakes", "findCommAndHostWindowFromDeviceWindow"); -} -std::function - g_findCommAndHostWindowFromDeviceWindow = FailLoudFindCommAndHostWindow; -ncclResult_t findCommAndHostWindowFromDeviceWindow(ncclWindow_t devWindow, ncclComm_t* foundComm, - struct ncclDevrWindow** hostWindow) { - return g_findCommAndHostWindowFromDeviceWindow(devWindow, foundComm, hostWindow); -} -static ncclResult_t ResolveFoundCommAndWindowImpl(ncclWindow_t, ncclComm_t* foundComm, - struct ncclDevrWindow** hostWindow) { - *foundComm = g_foundComm; - *hostWindow = g_foundWinHost; - return ncclSuccess; -} -void UseFoundCommAndWindow() { g_findCommAndHostWindowFromDeviceWindow = ResolveFoundCommAndWindowImpl; } - -static ncclResult_t FailLoudSymTeamObtain(struct ncclComm*, struct ncclTeam, bool, bool, bool, bool, - struct ncclDevrTeam**, bool*) { - FailLoudUnfaked("cft_dev_runtime_fakes", "symTeamObtain"); -} -std::function - g_symTeamObtain = FailLoudSymTeamObtain; -ncclResult_t symTeamObtain(struct ncclComm* comm, struct ncclTeam team, bool multimem, bool counted, bool wantsLeUc, - bool wantsLeMc, struct ncclDevrTeam** outTeam, bool* needBarrier) { - return g_symTeamObtain(comm, team, multimem, counted, wantsLeUc, wantsLeMc, outTeam, needBarrier); -} - -void ResetCftDevRuntimeFakes() { - g_computeLsaSize = DefaultComputeLsaSize; - g_devrWinRegEnabled = DefaultDevrWinRegEnabled; - g_findCommAndHostWindowFromDeviceWindow = FailLoudFindCommAndHostWindow; - g_foundComm = nullptr; - g_foundWinHost = nullptr; - g_symTeamObtain = FailLoudSymTeamObtain; -} diff --git a/projects/rccl/test/host/fakes/cft_dev_runtime_fakes.h b/projects/rccl/test/host/fakes/cft_dev_runtime_fakes.h deleted file mode 100644 index 1104cc4776aa..000000000000 --- a/projects/rccl/test/host/fakes/cft_dev_runtime_fakes.h +++ /dev/null @@ -1,51 +0,0 @@ -/************************************************************************* - * Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. - * - * See LICENSE.txt for license information - ************************************************************************/ - -// Seams for src/cft_dev_runtime.cc's callees that live in src/dev_runtime.cc -// (computeLsaSize, symTeamObtain, findCommAndHostWindowFromDeviceWindow) and -// src/dev_runtime.cc's ncclDevrWinRegEnabled. Targets linking this file -// compile the real cft_dev_runtime.cc but not the much larger dev_runtime.cc, -// so these four symbols need a seam of their own rather than the full file. - -#ifndef RCCL_TEST_HOST_CFT_DEV_RUNTIME_FAKES_H_ -#define RCCL_TEST_HOST_CFT_DEV_RUNTIME_FAKES_H_ - -#include - -#include "nccl.h" -// ncclDevrRegisterCapability is a plain (non-fixed-underlying-type) enum, which C++ -// does not allow forward-declaring -- pull in the real declarations instead. -#include "dev_runtime.h" - -struct ncclComm; -struct ncclTeam; - -// computeLsaSize: generous default (1) -- matches a comm with no LSA peers. -extern std::function g_computeLsaSize; - -// ncclDevrWinRegEnabled: default true -- matches a window registered for every capability. -extern std::function g_devrWinRegEnabled; - -// findCommAndHostWindowFromDeviceWindow: fail-loud default; a test must set -// g_foundComm/g_foundWinHost and call UseFoundCommAndWindow() (or install its own -// hook) before calling a function that depends on it. -extern std::function - g_findCommAndHostWindowFromDeviceWindow; -extern ncclComm_t g_foundComm; -extern struct ncclDevrWindow* g_foundWinHost; -// Installs the default hook that returns g_foundComm/g_foundWinHost directly. -void UseFoundCommAndWindow(); - -// symTeamObtain: fail-loud default; a test must install its own hook (e.g. one -// that heap-allocates a struct ncclDevrTeam with a populated worldRankList) -// before calling ncclGetMultimemDeviceLeInfo. -extern std::function - g_symTeamObtain; - -void ResetCftDevRuntimeFakes(); - -#endif // RCCL_TEST_HOST_CFT_DEV_RUNTIME_FAKES_H_ diff --git a/projects/rccl/test/host/fakes/dev_runtime_fakes.cc b/projects/rccl/test/host/fakes/dev_runtime_fakes.cc index a7149af3a2e6..34fa12c23b70 100644 --- a/projects/rccl/test/host/fakes/dev_runtime_fakes.cc +++ b/projects/rccl/test/host/fakes/dev_runtime_fakes.cc @@ -10,6 +10,7 @@ #include "dev_runtime_fakes.h" #include "dev_runtime.h" +#include "dev_runtime_internal.h" #include "fail_loud.h" #include "signature-drift.h" @@ -20,6 +21,10 @@ ASSERT_HOOK_MATCHES_PROD(g_devrInitOnce, ncclDevrInitOnce); ASSERT_HOOK_MATCHES_PROD(g_devrGetLsaRankPtr, ncclDevrGetLsaRankPtr); ASSERT_HOOK_MATCHES_PROD(g_devrWindowRegisterInGroup, ncclDevrWindowRegisterInGroup); ASSERT_HOOK_MATCHES_PROD(g_devrNcclCommWindowDeregister, ncclCommWindowDeregister); +ASSERT_HOOK_MATCHES_PROD(g_computeLsaSize, computeLsaSize); +ASSERT_HOOK_MATCHES_PROD(g_devrWinRegEnabled, ncclDevrWinRegEnabled); +ASSERT_HOOK_MATCHES_PROD(g_symTeamObtain, symTeamObtain); +ASSERT_HOOK_MATCHES_PROD(g_findCommAndHostWindowFromDeviceWindow, findCommAndHostWindowFromDeviceWindow); #undef ASSERT_HOOK_MATCHES_PROD static ncclResult_t DefaultDevrFindWindow(struct ncclComm*, void const*, struct ncclDevrWindow** window) { @@ -80,6 +85,47 @@ ncclResult_t ncclCommWindowDeregister(ncclComm_t comm, ncclWindow_t win) { return g_devrNcclCommWindowDeregister(comm, win); } +static int DefaultComputeLsaSize(struct ncclComm*) { FailLoudUnfaked("dev_runtime_fakes", "computeLsaSize"); } +std::function g_computeLsaSize = DefaultComputeLsaSize; +int computeLsaSize(struct ncclComm* comm) { return g_computeLsaSize(comm); } + +static bool DefaultDevrWinRegEnabled(int, ncclDevrRegisterCapability) { + FailLoudUnfaked("dev_runtime_fakes", "ncclDevrWinRegEnabled"); +} +std::function g_devrWinRegEnabled = DefaultDevrWinRegEnabled; +bool ncclDevrWinRegEnabled(int winFlags, ncclDevrRegisterCapability capability) { + return g_devrWinRegEnabled(winFlags, capability); +} + +static ncclResult_t DefaultSymTeamObtain(struct ncclComm*, struct ncclTeam, bool, bool, bool, bool, + struct ncclDevrTeam**, bool*) { + FailLoudUnfaked("dev_runtime_fakes", "symTeamObtain"); +} +std::function + g_symTeamObtain = DefaultSymTeamObtain; +ncclResult_t symTeamObtain(struct ncclComm* comm, struct ncclTeam team, bool multimem, bool counted, bool wantsLeUc, + bool wantsLeMc, struct ncclDevrTeam** outTeam, bool* needBarrier) { + return g_symTeamObtain(comm, team, multimem, counted, wantsLeUc, wantsLeMc, outTeam, needBarrier); +} + +static ncclResult_t DefaultFindCommAndHostWindow(ncclWindow_t, ncclComm_t*, struct ncclDevrWindow**) { + FailLoudUnfaked("dev_runtime_fakes", "findCommAndHostWindowFromDeviceWindow"); +} +std::function + g_findCommAndHostWindowFromDeviceWindow = DefaultFindCommAndHostWindow; +ncclResult_t findCommAndHostWindowFromDeviceWindow(ncclWindow_t devWindow, ncclComm_t* foundComm, + struct ncclDevrWindow** hostWindow) { + return g_findCommAndHostWindowFromDeviceWindow(devWindow, foundComm, hostWindow); +} +ncclComm_t g_foundComm = nullptr; +struct ncclDevrWindow* g_foundWinHost = nullptr; +static ncclResult_t ReturnFoundCommAndWindow(ncclWindow_t, ncclComm_t* foundComm, struct ncclDevrWindow** hostWindow) { + *foundComm = g_foundComm; + *hostWindow = g_foundWinHost; + return ncclSuccess; +} +void UseFoundCommAndWindow() { g_findCommAndHostWindowFromDeviceWindow = ReturnFoundCommAndWindow; } + // Floors for the LSA addressing ce_coll.cc's copy paths reach. ncclResult_t ncclDevrWorldToLsaRank(struct ncclComm*, int, int*) { FailLoudUnfaked("dev_runtime_fakes", "ncclDevrWorldToLsaRank"); @@ -98,4 +144,10 @@ void ResetDevRuntimeFakes() { g_devrGetLsaRankPtr = DefaultDevrGetLsaRankPtr; g_devrWindowRegisterInGroup = DefaultDevrWindowRegisterInGroup; g_devrNcclCommWindowDeregister = DefaultCommWindowDeregister; + g_computeLsaSize = DefaultComputeLsaSize; + g_devrWinRegEnabled = DefaultDevrWinRegEnabled; + g_symTeamObtain = DefaultSymTeamObtain; + g_findCommAndHostWindowFromDeviceWindow = DefaultFindCommAndHostWindow; + g_foundComm = nullptr; + g_foundWinHost = nullptr; } diff --git a/projects/rccl/test/host/fakes/dev_runtime_fakes.h b/projects/rccl/test/host/fakes/dev_runtime_fakes.h index f4851ef81e4c..33322e43b841 100644 --- a/projects/rccl/test/host/fakes/dev_runtime_fakes.h +++ b/projects/rccl/test/host/fakes/dev_runtime_fakes.h @@ -15,6 +15,8 @@ #include #include "nccl.h" +// ncclDevrRegisterCapability is a plain enum, which C++ cannot forward-declare. +#include "dev_runtime.h" struct ncclComm; struct ncclDevrWindow; @@ -38,6 +40,21 @@ extern std::function g_devrNcclCommWindowDeregister; +// The dev_runtime.cc functions src/cft_dev_runtime.cc calls. All fail loud by default. +struct ncclTeam; +struct ncclDevrTeam; +extern std::function g_computeLsaSize; +extern std::function g_devrWinRegEnabled; +extern std::function + g_symTeamObtain; +extern std::function + g_findCommAndHostWindowFromDeviceWindow; +// UseFoundCommAndWindow() installs a g_findCommAndHostWindowFromDeviceWindow that returns these. +extern ncclComm_t g_foundComm; +extern struct ncclDevrWindow* g_foundWinHost; +void UseFoundCommAndWindow(); + void ResetDevRuntimeFakes(); #endif // RCCL_TEST_HOST_DEV_RUNTIME_FAKES_H_ diff --git a/projects/rccl/test/host/nccl-device-requirements-test.cc b/projects/rccl/test/host/nccl-device-requirements-test.cc index be59fb4328c4..095b251afac3 100644 --- a/projects/rccl/test/host/nccl-device-requirements-test.cc +++ b/projects/rccl/test/host/nccl-device-requirements-test.cc @@ -264,12 +264,10 @@ TEST(NcclDeviceRequirementsMicrotest, GinInboxA2ACreateRequirement_BufferSizeAnd } } -// A 1-rank team makes nPeers = 0 and idivRcp32(0) divides by zero: a real defect, unreachable from -// in-tree callers (sym_kernels.cc needs nRanks >= 2). The volatile keeps the divide from being -// folded away at -O2. x86_64 only: AArch64 UDIV returns 0 instead of trapping. -TEST(NcclDeviceRequirementsMicrotest, GinInboxA2ACreateRequirement_SingleRankTeam_CrashesOnDivideByZero) { -#if defined(__x86_64__) - volatile int nRanksVolatile = 1; +// A 1-rank team makes nPeers = 0, so idivRcp32(0) divides by zero; a fix flips this pin. +TEST(NcclDeviceRequirementsMicrotest, GinInboxA2ACreateRequirement_KnownBug_SingleRankTeamDividesByZero) { +#if defined(__x86_64__) // AArch64 UDIV returns 0 instead of trapping. + volatile int nRanksVolatile = 1; // keeps -O2 from folding the divide away // Assigned field by field: a braced init's commas would split EXPECT_EXIT's arguments. EXPECT_EXIT( {