From 991a70a0964e67776db5c9c6179c9c7ad8956398 Mon Sep 17 00:00:00 2001 From: Alex Duforest Date: Mon, 28 Sep 2026 14:02:51 -0700 Subject: [PATCH 1/4] test(rccl): add host microtests for diagnostics_env.cc --- projects/rccl/test/host/CMakeLists.txt | 2 + projects/rccl/test/host/MICROTEST_README.md | 4 + .../test/host/ras-diagnostics-env-test.cc | 660 ++++++++++++++++++ projects/rccl/test/test_categories_micro.yaml | 2 + 4 files changed, 668 insertions(+) create mode 100644 projects/rccl/test/host/ras-diagnostics-env-test.cc diff --git a/projects/rccl/test/host/CMakeLists.txt b/projects/rccl/test/host/CMakeLists.txt index ae39e2c86619..6a397cbe45e7 100644 --- a/projects/rccl/test/host/CMakeLists.txt +++ b/projects/rccl/test/host/CMakeLists.txt @@ -65,6 +65,7 @@ function(rccl_define_micro_source_lists) rma-ce-test.cc ras-test.cc ras-diagnostics-common-test.cc # src/ras/diagnostics_checks_common.cc + ras-diagnostics-env-test.cc # src/ras/diagnostics_env.cc tuning-general-test.cc ras-client-test.cc group-test.cc @@ -732,6 +733,7 @@ rccl_add_micro_binary(rccl-UnitTestsMicro P2P_CC_PATH="${RCCL_HOST_HIPIFY_DIR}/src/transport/p2p_tmp.cc" RAS_CC_PATH="${RCCL_HOST_HIPIFY_DIR}/src/ras/ras.cc" RAS_DIAGNOSTICS_COMMON_CC_PATH="${RCCL_HOST_HIPIFY_DIR}/src/ras/diagnostics_checks_common.cc" + RAS_DIAGNOSTICS_ENV_CC_PATH="${RCCL_HOST_HIPIFY_DIR}/src/ras/diagnostics_env.cc" TUNING_GENERAL_CC_PATH="${RCCL_HOST_HIPIFY_DIR}/src/tuning/tuning_general.cc" RAS_CLIENT_CC_PATH="${RCCL_HOST_HIPIFY_DIR}/src/ras/client.cc" RMA_PROXY_LAUNCH_CC_PATH="${RCCL_HOST_HIPIFY_DIR}/src/rma/rma_proxy_launch.cc" diff --git a/projects/rccl/test/host/MICROTEST_README.md b/projects/rccl/test/host/MICROTEST_README.md index 8970dea599ad..5c4915328bc1 100644 --- a/projects/rccl/test/host/MICROTEST_README.md +++ b/projects/rccl/test/host/MICROTEST_README.md @@ -90,6 +90,10 @@ export colliding non-`static` symbols; otherwise a unit needs its own binary: `RasDiagnosticsCommonMicrotest.*`. Covers communicator snapshots and filtering, aligned local-record collection, allocation and callback failures, rank ordering and formatting, and reporter output. + - `ras/diagnostics_env.cc` (`RAS_DIAGNOSTICS_ENV_CC_PATH`, from + `ras-diagnostics-env-test.cc`); suite `RasDiagnosticsEnvMicrotest.*`. + Covers NCCL environment collection, filtering, truncation, aggregation, + mismatch reporting, and reporter errors. - `ras/client.cc` (`RAS_CLIENT_CC_PATH`, from `ras-client-test.cc`); suite `RasClientMicrotest.*`. With `NCCL_RAS_CLIENT` defined, `ras_internal.h` reduces to four macros, so this diff --git a/projects/rccl/test/host/ras-diagnostics-env-test.cc b/projects/rccl/test/host/ras-diagnostics-env-test.cc new file mode 100644 index 000000000000..6e6ee5944b33 --- /dev/null +++ b/projects/rccl/test/host/ras-diagnostics-env-test.cc @@ -0,0 +1,660 @@ +/************************************************************************* + * Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +// Host-only microtests for `src/ras/diagnostics_env.cc`. + +#include +#include +#include +#include +#include + +#include + +#include "comm.h" +#include "ras/diagnostics_checks_common.h" + +namespace { + +int g_allocationCalls; +int g_failAllocationCall; +std::vector g_allocationCounts; + +template +ncclResult_t DiagnosticsEnvTestCalloc(T** ptr, size_t count) { + ++g_allocationCalls; + g_allocationCounts.push_back(count); + if (g_allocationCalls == g_failAllocationCall) return ncclSystemError; + return ncclCallocDebug(ptr, count, __FILE__, __LINE__, __func__, false); +} + +template +ncclResult_t DiagnosticsEnvTestCalloc(ncclUniquePtr& ptr, size_t count) { + ++g_allocationCalls; + g_allocationCounts.push_back(count); + if (g_allocationCalls == g_failAllocationCall) return ncclSystemError; + return ncclCallocDebug(ptr, count, __FILE__, __LINE__, __func__, false); +} + +} // namespace + +#undef ncclCalloc +#define ncclCalloc(...) DiagnosticsEnvTestCalloc(__VA_ARGS__) + +#include RAS_DIAGNOSTICS_ENV_CC_PATH + +#undef ncclCalloc + +namespace { + +int g_emitCalls; +int g_emitFailAt; +std::vector g_emittedLines; + +ncclResult_t RecordingEmit(void* /*target*/, const char* line) { + ++g_emitCalls; + if (g_emitCalls == g_emitFailAt) return ncclSystemError; + g_emittedLines.emplace_back(line); + return ncclSuccess; +} + +rasDiagnosticsReporter MakeRecordingReporter() { + rasDiagnosticsReporter reporter{}; + reporter.emit = RecordingEmit; + return reporter; +} + +bool AnyLineContains(const std::vector& lines, const std::string& needle) { + for (auto& l : lines) + if (l.find(needle) != std::string::npos) return true; + return false; +} + +struct FakeComm { + std::unique_ptr comm{new ncclComm{}}; + std::vector peerInfos; + + FakeComm(uint64_t commHash, int rank, int nRanks) { + comm->commHash = commHash; + comm->rank = rank; + comm->nRanks = nRanks; + comm->peerInfoValid = true; + comm->cudaDev = 0; + comm->nvmlDev = 0; + comm->busId = 0; + comm->localRank = 0; + comm->localRanks = 1; + peerInfos.assign(1, ncclPeerInfo{}); + peerInfos[0].hostHash = commHash + 1; + peerInfos[0].pidHash = commHash + 2; + comm->peerInfo = peerInfos.data(); + } + ncclComm* get() { return comm.get(); } +}; + +void InstallNcclComms(std::vector comms) { + free(ncclComms); + nNcclComms = static_cast(comms.size()); + ncclComms = static_cast(calloc(comms.size() ? comms.size() : 1, sizeof(*ncclComms))); + for (size_t i = 0; i < comms.size(); i++) ncclComms[i] = comms[i]; +} + +// --------- Fake `environ` plumbing --------- + +char** g_realEnviron; +std::vector g_envStorage; +std::vector g_envPointers; + +void SetFakeEnviron(const std::vector& entries) { + g_envStorage = entries; + g_envPointers.clear(); + for (auto& s : g_envStorage) g_envPointers.push_back(const_cast(s.c_str())); + g_envPointers.push_back(nullptr); + environ = g_envPointers.data(); +} + +void ResetWholeFileSeams() { + g_allocationCalls = 0; + g_failAllocationCall = 0; + g_allocationCounts.clear(); + g_emitCalls = 0; + g_emitFailAt = 0; + g_emittedLines.clear(); + free(ncclComms); + ncclComms = nullptr; + nNcclComms = 0; + SetFakeEnviron({}); +} + +class RasDiagnosticsEnvMicrotest : public ::testing::Test { + protected: + void SetUp() override { + g_realEnviron = environ; + ResetWholeFileSeams(); + } + void TearDown() override { + ResetWholeFileSeams(); + environ = g_realEnviron; + } +}; + +} // namespace + +// =========================================================================== +// rasDiagnosticsNcclEnvCollectLocal (via the real rasDiagnosticsCollectLocalRecords) +// =========================================================================== + +TEST_F(RasDiagnosticsEnvMicrotest, CollectLocal_NoMatchingCommsReturnsEmpty) { + struct rasDiagnosticsContext ctx{}; + struct rasDiagnosticsLocalData data{}; + ASSERT_EQ(ncclSuccess, rasDiagnosticsNcclEnvCollectLocal(&ctx, &data)); + EXPECT_EQ(0, data.nRecords); +} + +TEST_F(RasDiagnosticsEnvMicrotest, CollectLocal_FiltersNonNcclPrefixedVars) { + FakeComm fc(0x1000, 0, 1); + InstallNcclComms({fc.get()}); + SetFakeEnviron({"PATH=/usr/bin", "NCCL_DEBUG=INFO", "HOME=/root"}); + + struct rasDiagnosticsContext ctx{}; + struct rasDiagnosticsLocalData data{}; + ASSERT_EQ(ncclSuccess, rasDiagnosticsNcclEnvCollectLocal(&ctx, &data)); + ASSERT_EQ(1, data.nRecords); + const auto* envData = reinterpret_cast( + data.records + sizeof(struct rasDiagnosticsRankHeader)); + EXPECT_EQ(0, envData->truncated); + std::string blob(envData->data, envData->bytesUsed); + EXPECT_NE(std::string::npos, blob.find("NCCL_DEBUG=INFO")); + EXPECT_EQ(std::string::npos, blob.find("PATH=")); + free(data.records); +} + +TEST_F(RasDiagnosticsEnvMicrotest, CollectLocal_NoNcclVarsProducesEmptyPayload) { + FakeComm fc(0x1000, 0, 1); + InstallNcclComms({fc.get()}); + SetFakeEnviron({"PATH=/usr/bin"}); + + struct rasDiagnosticsContext ctx{}; + struct rasDiagnosticsLocalData data{}; + ASSERT_EQ(ncclSuccess, rasDiagnosticsNcclEnvCollectLocal(&ctx, &data)); + ASSERT_EQ(1, data.nRecords); + const auto* envData = reinterpret_cast( + data.records + sizeof(struct rasDiagnosticsRankHeader)); + EXPECT_EQ(0, envData->bytesUsed); + free(data.records); +} + +TEST_F(RasDiagnosticsEnvMicrotest, CollectLocal_ExceedingByteBudgetSetsTruncated) { + FakeComm fc(0x1000, 0, 1); + InstallNcclComms({fc.get()}); + std::vector entries; + // Each ~1040 bytes; ~17 of them exceed the 16384-byte budget. + for (int i = 0; i < 17; i++) entries.push_back("NCCL_VAR" + std::to_string(i) + "=" + std::string(1024, 'a')); + SetFakeEnviron(entries); + + struct rasDiagnosticsContext ctx{}; + struct rasDiagnosticsLocalData data{}; + ASSERT_EQ(ncclSuccess, rasDiagnosticsNcclEnvCollectLocal(&ctx, &data)); + ASSERT_EQ(1, data.nRecords); + const auto* envData = reinterpret_cast( + data.records + sizeof(struct rasDiagnosticsRankHeader)); + EXPECT_EQ(1, envData->truncated); + free(data.records); +} + +TEST_F(RasDiagnosticsEnvMicrotest, CollectLocal_ExactFitAtTheByteBudgetIsNotTruncated) { + FakeComm fc(0x1000, 0, 1); + InstallNcclComms({fc.get()}); + // "NCCL_X=" (7) + value: total strlen == RAS_DIAG_ENV_BYTES - 1, so len (strlen+1, + // including the NUL) exactly equals the remaining budget -- fits exactly, at the + // boundary between "fits" and "doesn't fit". + std::string value(RAS_DIAG_ENV_BYTES - 1 - 7, 'a'); + SetFakeEnviron({"NCCL_X=" + value}); + + struct rasDiagnosticsContext ctx{}; + struct rasDiagnosticsLocalData data{}; + ASSERT_EQ(ncclSuccess, rasDiagnosticsNcclEnvCollectLocal(&ctx, &data)); + ASSERT_EQ(1, data.nRecords); + const auto* envData = reinterpret_cast( + data.records + sizeof(struct rasDiagnosticsRankHeader)); + EXPECT_EQ(0, envData->truncated); + EXPECT_EQ(RAS_DIAG_ENV_BYTES, envData->bytesUsed); + free(data.records); +} + +// =========================================================================== +// rasDiagnosticsNcclEnvSummarize: argument validation +// =========================================================================== + +TEST_F(RasDiagnosticsEnvMicrotest, Summarize_NullReporterReturnsInternalError) { + EXPECT_EQ(ncclInternalError, rasDiagnosticsNcclEnvSummarize(nullptr, nullptr, nullptr, 0)); +} + +TEST_F(RasDiagnosticsEnvMicrotest, Summarize_ReporterWithNullEmitReturnsInternalError) { + struct rasDiagnosticsReporter reporter{}; + EXPECT_EQ(ncclInternalError, rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, nullptr, 0)); +} + +TEST_F(RasDiagnosticsEnvMicrotest, Summarize_ZeroDataIsNoOp) { + auto reporter = MakeRecordingReporter(); + EXPECT_EQ(ncclSuccess, rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, nullptr, 0)); + EXPECT_EQ(0, g_emitCalls); +} + +TEST_F(RasDiagnosticsEnvMicrotest, Summarize_NullDataWithNonzeroSizeReturnsInternalError) { + auto reporter = MakeRecordingReporter(); + EXPECT_EQ(ncclInternalError, rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, nullptr, 8)); +} + +TEST_F(RasDiagnosticsEnvMicrotest, Summarize_SizeNotMultipleOfStrideReturnsInternalError) { + auto reporter = MakeRecordingReporter(); + char buf[4] = {}; + EXPECT_EQ(ncclInternalError, rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, buf, 1)); +} + +// =========================================================================== +// rasDiagnosticsNcclEnvSummarize: end-to-end via CollectLocal-produced records +// =========================================================================== + +namespace { + +// Runs CollectLocal for the currently installed ncclComms/environ, then feeds +// the result straight into Summarize -- exercising the exact production +// pairing (one rank's real gathered record, not a hand-built stand-in). +ncclResult_t CollectThenSummarize(struct rasDiagnosticsReporter* reporter) { + struct rasDiagnosticsContext ctx{}; + struct rasDiagnosticsLocalData data{}; + ncclResult_t ret = rasDiagnosticsNcclEnvCollectLocal(&ctx, &data); + if (ret != ncclSuccess) return ret; + ret = rasDiagnosticsNcclEnvSummarize(&ctx, reporter, data.records, data.recordsBytes); + free(data.records); + return ret; +} + +} // namespace + +TEST_F(RasDiagnosticsEnvMicrotest, Summarize_SingleRankConsistentReport) { + FakeComm fc(0x2000, 0, 1); + InstallNcclComms({fc.get()}); + SetFakeEnviron({"NCCL_DEBUG=INFO"}); + auto reporter = MakeRecordingReporter(); + + ASSERT_EQ(ncclSuccess, CollectThenSummarize(&reporter)); + ASSERT_EQ(1u, g_emittedLines.size()); + EXPECT_EQ("[OK] NCCL environment: NCCL_* env vars consistent across 1 ranks in comm 0x2000", g_emittedLines[0]); + EXPECT_FALSE(AnyLineContains(g_emittedLines, "mismatch")); +} + +TEST_F(RasDiagnosticsEnvMicrotest, Summarize_MultipleRanksWithEqualValuesAreConsistent) { + const size_t stride = rasDiagnosticsLocalRecordStride(sizeof(struct rasDiagnosticsNcclEnvData)); + std::vector buf(stride * 2, 0); + + for (int rank = 0; rank < 2; rank++) { + auto* rankHeader = reinterpret_cast(buf.data() + rank * stride); + rankHeader->commId.commHash = 0x2800; + rankHeader->commId.hostHash = 0x2801; + rankHeader->commId.pidHash = 0x2802; + rankHeader->commRank = rank; + rankHeader->commNRanks = 2; + auto* envData = reinterpret_cast( + buf.data() + rank * stride + sizeof(struct rasDiagnosticsRankHeader)); + constexpr char entry[] = "NCCL_DEBUG=INFO"; + memcpy(envData->data, entry, sizeof(entry)); + envData->bytesUsed = sizeof(entry); + } + + auto reporter = MakeRecordingReporter(); + ASSERT_EQ(ncclSuccess, rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, buf.data(), static_cast(buf.size()))); + ASSERT_EQ(1u, g_emittedLines.size()); + EXPECT_EQ("[OK] NCCL environment: NCCL_* env vars consistent across 2 ranks in comm 0x2800", g_emittedLines[0]); + EXPECT_FALSE(AnyLineContains(g_emittedLines, "mismatch")); +} + +TEST_F(RasDiagnosticsEnvMicrotest, Summarize_IncompleteGroupReportsIncomplete) { + // commNRanks says 2, but only rank 0 is present in ncclComms -- an incomplete gather. + FakeComm fc(0x3000, 0, 2); + InstallNcclComms({fc.get()}); + SetFakeEnviron({}); + auto reporter = MakeRecordingReporter(); + + ASSERT_EQ(ncclSuccess, CollectThenSummarize(&reporter)); + ASSERT_EQ(1u, g_emittedLines.size()); + EXPECT_EQ("[INFO] NCCL environment: diagnostics incomplete, gathered 1/2 ranks in comm 0x3000/0x3001/0x3002 " + "(RAS overlay may not be ready)", + g_emittedLines[0]); + EXPECT_FALSE(AnyLineContains(g_emittedLines, "consistent")); +} + +TEST_F(RasDiagnosticsEnvMicrotest, Summarize_TwoRanksMismatchedValueReportsMismatch) { + // Two distinct ncclComm entries, same commHash/hostHash/pidHash pairing style but + // different rank -- built by hand below since FakeComm's per-object environ can't + // differ per rank (only one process-wide `environ`), so we hand-build two records + // directly instead of going through CollectLocal twice. + const size_t stride = rasDiagnosticsLocalRecordStride(sizeof(struct rasDiagnosticsNcclEnvData)); + std::vector buf(stride * 2, 0); + + auto fillRecord = [&](int rank, const char* value) { + auto* rank_hdr = reinterpret_cast(buf.data() + rank * stride); + rank_hdr->commId.commHash = 0x4000; + rank_hdr->commId.hostHash = 0x4001; + rank_hdr->commId.pidHash = 0x4002; + rank_hdr->commRank = rank; + rank_hdr->commNRanks = 2; + auto* envData = reinterpret_cast(buf.data() + rank * stride + + sizeof(struct rasDiagnosticsRankHeader)); + std::string entry = std::string("NCCL_DEBUG=") + value; + memcpy(envData->data, entry.c_str(), entry.size() + 1); + envData->bytesUsed = static_cast(entry.size() + 1); + envData->truncated = 0; + }; + fillRecord(0, "INFO"); + fillRecord(1, "WARN"); + + auto reporter = MakeRecordingReporter(); + ASSERT_EQ(ncclSuccess, rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, buf.data(), static_cast(buf.size()))); + ASSERT_EQ(4u, g_emittedLines.size()); + EXPECT_EQ("[INFO] NCCL environment: mismatch across 2 ranks in comm 0x4000 for NCCL_DEBUG", g_emittedLines[0]); + EXPECT_EQ("[INFO] NCCL environment: NCCL_DEBUG=INFO on rank(s) {0}", g_emittedLines[1]); + EXPECT_EQ("[INFO] NCCL environment: NCCL_DEBUG=WARN on rank(s) {1}", g_emittedLines[2]); + EXPECT_EQ("[INFO] NCCL environment: 1 NCCL_* env var(s) differ across ranks in comm 0x4000", g_emittedLines[3]); + EXPECT_FALSE(AnyLineContains(g_emittedLines, "consistent")); +} + +TEST_F(RasDiagnosticsEnvMicrotest, Summarize_FourRanksGroupsRepeatedValuesOnce) { + const size_t stride = rasDiagnosticsLocalRecordStride(sizeof(struct rasDiagnosticsNcclEnvData)); + std::vector buf(stride * 4, 0); + const int ranks[] = {3, 1, 2, 0}; + const char* values[] = {"WARN", "WARN", "INFO", "INFO"}; + + for (int i = 0; i < 4; i++) { + auto* rankHeader = reinterpret_cast(buf.data() + i * stride); + rankHeader->commId.commHash = 0x4100; + rankHeader->commRank = ranks[i]; + rankHeader->commNRanks = 4; + auto* envData = reinterpret_cast( + buf.data() + i * stride + sizeof(struct rasDiagnosticsRankHeader)); + std::string entry = std::string("NCCL_DEBUG=") + values[i]; + memcpy(envData->data, entry.c_str(), entry.size() + 1); + envData->bytesUsed = static_cast(entry.size() + 1); + } + + auto reporter = MakeRecordingReporter(); + ASSERT_EQ(ncclSuccess, rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, buf.data(), static_cast(buf.size()))); + ASSERT_EQ(4u, g_emittedLines.size()); + EXPECT_EQ("[INFO] NCCL environment: mismatch across 4 ranks in comm 0x4100 for NCCL_DEBUG", g_emittedLines[0]); + EXPECT_EQ("[INFO] NCCL environment: NCCL_DEBUG=INFO on rank(s) {0,2}", g_emittedLines[1]); + EXPECT_EQ("[INFO] NCCL environment: NCCL_DEBUG=WARN on rank(s) {1,3}", g_emittedLines[2]); + EXPECT_EQ("[INFO] NCCL environment: 1 NCCL_* env var(s) differ across ranks in comm 0x4100", g_emittedLines[3]); + EXPECT_FALSE(AnyLineContains(g_emittedLines, "consistent")); +} + +TEST_F(RasDiagnosticsEnvMicrotest, Summarize_TwoDistinctKeysAreSortedAndDeduplicated) { + const size_t stride = rasDiagnosticsLocalRecordStride(sizeof(struct rasDiagnosticsNcclEnvData)); + std::vector buf(stride * 2, 0); + + auto fillRecord = [&](int rank, const char* first, const char* second) { + auto* rankHeader = reinterpret_cast(buf.data() + rank * stride); + rankHeader->commId.commHash = 0x4150; + rankHeader->commRank = rank; + rankHeader->commNRanks = 2; + auto* envData = reinterpret_cast( + buf.data() + rank * stride + sizeof(struct rasDiagnosticsRankHeader)); + const size_t firstLen = strlen(first) + 1; + const size_t secondLen = strlen(second) + 1; + memcpy(envData->data, first, firstLen); + memcpy(envData->data + firstLen, second, secondLen); + envData->bytesUsed = static_cast(firstLen + secondLen); + }; + fillRecord(0, "NCCL_DEBUG_SUBSYS=INIT", "NCCL_DEBUG=INFO"); + fillRecord(1, "NCCL_DEBUG_SUBSYS=COLL", "NCCL_DEBUG=WARN"); + + auto reporter = MakeRecordingReporter(); + ASSERT_EQ(ncclSuccess, rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, buf.data(), static_cast(buf.size()))); + ASSERT_EQ(7u, g_emittedLines.size()); + EXPECT_EQ("[INFO] NCCL environment: mismatch across 2 ranks in comm 0x4150 for NCCL_DEBUG", g_emittedLines[0]); + EXPECT_EQ("[INFO] NCCL environment: NCCL_DEBUG=INFO on rank(s) {0}", g_emittedLines[1]); + EXPECT_EQ("[INFO] NCCL environment: NCCL_DEBUG=WARN on rank(s) {1}", g_emittedLines[2]); + EXPECT_EQ("[INFO] NCCL environment: mismatch across 2 ranks in comm 0x4150 for NCCL_DEBUG_SUBSYS", + g_emittedLines[3]); + EXPECT_EQ("[INFO] NCCL environment: NCCL_DEBUG_SUBSYS=INIT on rank(s) {0}", g_emittedLines[4]); + EXPECT_EQ("[INFO] NCCL environment: NCCL_DEBUG_SUBSYS=COLL on rank(s) {1}", g_emittedLines[5]); + EXPECT_EQ("[INFO] NCCL environment: 2 NCCL_* env var(s) differ across ranks in comm 0x4150", g_emittedLines[6]); + EXPECT_FALSE(AnyLineContains(g_emittedLines, "consistent")); + ASSERT_GE(g_allocationCounts.size(), 2u); + EXPECT_EQ(buf.size(), g_allocationCounts[0]); + EXPECT_EQ(4u, g_allocationCounts[1]); +} + +TEST_F(RasDiagnosticsEnvMicrotest, Summarize_AllocationFailuresPropagate) { + const size_t stride = rasDiagnosticsLocalRecordStride(sizeof(struct rasDiagnosticsNcclEnvData)); + std::vector buf(stride * 2, 0); + for (int rank = 0; rank < 2; rank++) { + auto* rankHeader = reinterpret_cast(buf.data() + rank * stride); + rankHeader->commId.commHash = 0x4175; + rankHeader->commRank = rank; + rankHeader->commNRanks = 2; + auto* envData = reinterpret_cast( + buf.data() + rank * stride + sizeof(struct rasDiagnosticsRankHeader)); + const char* entry = rank == 0 ? "NCCL_DEBUG=INFO" : "NCCL_DEBUG=WARN"; + memcpy(envData->data, entry, strlen(entry) + 1); + envData->bytesUsed = static_cast(strlen(entry) + 1); + } + + auto reporter = MakeRecordingReporter(); + for (int failAt = 1; failAt <= 5; failAt++) { + g_allocationCalls = 0; + g_allocationCounts.clear(); + g_failAllocationCall = failAt; + EXPECT_EQ(ncclSystemError, + rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, buf.data(), static_cast(buf.size()))) + << failAt; + } +} + +TEST_F(RasDiagnosticsEnvMicrotest, CollectLocal_EntryArrayAllocationFailurePropagates) { + FakeComm fc(0x4180, 0, 1); + InstallNcclComms({fc.get()}); + SetFakeEnviron({"NCCL_DEBUG=INFO"}); + struct rasDiagnosticsContext ctx{}; + struct rasDiagnosticsLocalData data{}; + g_allocationCalls = 0; + g_failAllocationCall = 1; + EXPECT_EQ(ncclSystemError, rasDiagnosticsNcclEnvCollectLocal(&ctx, &data)); + EXPECT_EQ(nullptr, data.records); +} + +TEST_F(RasDiagnosticsEnvMicrotest, Summarize_ReporterFailuresPropagate) { + const size_t stride = rasDiagnosticsLocalRecordStride(sizeof(struct rasDiagnosticsNcclEnvData)); + std::vector mismatch(stride * 2, 0); + for (int rank = 0; rank < 2; rank++) { + auto* rankHeader = reinterpret_cast(mismatch.data() + rank * stride); + rankHeader->commId.commHash = 0x4200; + rankHeader->commRank = rank; + rankHeader->commNRanks = 2; + auto* envData = reinterpret_cast( + mismatch.data() + rank * stride + sizeof(struct rasDiagnosticsRankHeader)); + std::string entry = std::string("NCCL_DEBUG=") + (rank == 0 ? "INFO" : "WARN"); + memcpy(envData->data, entry.c_str(), entry.size() + 1); + envData->bytesUsed = static_cast(entry.size() + 1); + } + + auto reporter = MakeRecordingReporter(); + for (int failAt = 1; failAt <= 4; failAt++) { + g_emitCalls = 0; + g_emitFailAt = failAt; + EXPECT_EQ(ncclSystemError, + rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, mismatch.data(), static_cast(mismatch.size()))); + } + + std::vector single(stride, 0); + auto* rankHeader = reinterpret_cast(single.data()); + rankHeader->commId.commHash = 0x4300; + rankHeader->commRank = 0; + rankHeader->commNRanks = 1; + auto* envData = reinterpret_cast( + single.data() + sizeof(struct rasDiagnosticsRankHeader)); + + g_emitCalls = 0; + g_emitFailAt = 1; + EXPECT_EQ(ncclSystemError, + rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, single.data(), static_cast(single.size()))); + + envData->truncated = 1; + g_emitCalls = 0; + EXPECT_EQ(ncclSystemError, + rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, single.data(), static_cast(single.size()))); + + rankHeader->commNRanks = 2; + envData->truncated = 0; + g_emitCalls = 0; + EXPECT_EQ(ncclSystemError, + rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, single.data(), static_cast(single.size()))); +} + +TEST_F(RasDiagnosticsEnvMicrotest, Summarize_ValueSetOnOneRankUnsetOnAnotherIsAMismatch) { + const size_t stride = rasDiagnosticsLocalRecordStride(sizeof(struct rasDiagnosticsNcclEnvData)); + std::vector buf(stride * 2, 0); + + auto* r0 = reinterpret_cast(buf.data()); + r0->commId.commHash = 0x5000; + r0->commRank = 0; + r0->commNRanks = 2; + auto* env0 = + reinterpret_cast(buf.data() + sizeof(struct rasDiagnosticsRankHeader)); + const char* entry0 = "NCCL_FOO=bar"; + memcpy(env0->data, entry0, strlen(entry0) + 1); + env0->bytesUsed = static_cast(strlen(entry0) + 1); + + auto* r1 = reinterpret_cast(buf.data() + stride); + r1->commId.commHash = 0x5000; + r1->commRank = 1; + r1->commNRanks = 2; + auto* env1 = reinterpret_cast(buf.data() + stride + + sizeof(struct rasDiagnosticsRankHeader)); + env1->bytesUsed = 0; // NCCL_FOO not set on rank 1. + + auto reporter = MakeRecordingReporter(); + ASSERT_EQ(ncclSuccess, rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, buf.data(), static_cast(buf.size()))); + EXPECT_TRUE(AnyLineContains(g_emittedLines, "NCCL_FOO=(unset) on rank(s) {1}")); + EXPECT_TRUE(AnyLineContains(g_emittedLines, "NCCL_FOO=bar on rank(s) {0}")); +} + +TEST_F(RasDiagnosticsEnvMicrotest, Summarize_TruncatedRankReportsTruncationWarning) { + const size_t stride = rasDiagnosticsLocalRecordStride(sizeof(struct rasDiagnosticsNcclEnvData)); + std::vector buf(stride, 0); + auto* rank_hdr = reinterpret_cast(buf.data()); + rank_hdr->commId.commHash = 0x6000; + rank_hdr->commRank = 0; + rank_hdr->commNRanks = 1; + auto* envData = + reinterpret_cast(buf.data() + sizeof(struct rasDiagnosticsRankHeader)); + const char* entry = "NCCL_DEBUG=INFO"; + memcpy(envData->data, entry, strlen(entry) + 1); + envData->bytesUsed = static_cast(strlen(entry) + 1); + envData->truncated = 1; + + auto reporter = MakeRecordingReporter(); + ASSERT_EQ(ncclSuccess, rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, buf.data(), static_cast(buf.size()))); + EXPECT_TRUE(AnyLineContains(g_emittedLines, "1 rank(s) had >16384 bytes")); +} + +TEST_F(RasDiagnosticsEnvMicrotest, Summarize_MalformedEnvPayloadReturnsInternalError) { + const size_t stride = rasDiagnosticsLocalRecordStride(sizeof(struct rasDiagnosticsNcclEnvData)); + std::vector buf(stride, 0); + auto* rank_hdr = reinterpret_cast(buf.data()); + rank_hdr->commId.commHash = 0x7000; + rank_hdr->commRank = 0; + rank_hdr->commNRanks = 1; + auto* envData = + reinterpret_cast(buf.data() + sizeof(struct rasDiagnosticsRankHeader)); + envData->bytesUsed = 5; + envData->data[4] = 'x'; // Not NUL-terminated at the claimed end -- malformed. + + auto reporter = MakeRecordingReporter(); + EXPECT_EQ(ncclInternalError, + rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, buf.data(), static_cast(buf.size()))); +} + +TEST_F(RasDiagnosticsEnvMicrotest, Summarize_TwoDistinctCommsAreReportedSeparately) { + const size_t stride = rasDiagnosticsLocalRecordStride(sizeof(struct rasDiagnosticsNcclEnvData)); + std::vector buf(stride * 3, 0); + + auto fillSolo = [&](int idx, uint64_t commHash, uint64_t hostHash, uint64_t pidHash) { + auto* rank_hdr = reinterpret_cast(buf.data() + idx * stride); + rank_hdr->commId.commHash = commHash; + rank_hdr->commId.hostHash = hostHash; + rank_hdr->commId.pidHash = pidHash; + rank_hdr->commRank = 0; + rank_hdr->commNRanks = 1; + auto* envData = reinterpret_cast(buf.data() + idx * stride + + sizeof(struct rasDiagnosticsRankHeader)); + envData->bytesUsed = 0; + }; + fillSolo(0, 0x8000, 0x8100, 0x8200); + fillSolo(1, 0x8000, 0x8300, 0x8400); + fillSolo(2, 0x9000, 0x9100, 0x9200); + + auto reporter = MakeRecordingReporter(); + ASSERT_EQ(ncclSuccess, rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, buf.data(), static_cast(buf.size()))); + int consistentCount = 0; + for (auto& l : g_emittedLines) { + if (l.find("consistent across 1 ranks") != std::string::npos) consistentCount++; + } + EXPECT_EQ(3, consistentCount); +} + +TEST_F(RasDiagnosticsEnvMicrotest, Summarize_LongKeyAndValueAreTruncatedWithEllipsis) { + const size_t stride = rasDiagnosticsLocalRecordStride(sizeof(struct rasDiagnosticsNcclEnvData)); + std::vector buf(stride * 2, 0); + + std::string longKey = "NCCL_" + std::string(300, 'K'); + std::string longVal(300, 'V'); + std::string entry0 = longKey + "=" + longVal; + std::string entry1 = longKey + "=" + std::string(300, 'W'); // Different value -> mismatch. + + auto fillRecord = [&](int rank, const std::string& entry) { + auto* rank_hdr = reinterpret_cast(buf.data() + rank * stride); + rank_hdr->commId.commHash = 0xA000; + rank_hdr->commRank = rank; + rank_hdr->commNRanks = 2; + auto* envData = reinterpret_cast(buf.data() + rank * stride + + sizeof(struct rasDiagnosticsRankHeader)); + memcpy(envData->data, entry.c_str(), entry.size() + 1); + envData->bytesUsed = static_cast(entry.size() + 1); + }; + fillRecord(0, entry0); + fillRecord(1, entry1); + + auto reporter = MakeRecordingReporter(); + ASSERT_EQ(ncclSuccess, rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, buf.data(), static_cast(buf.size()))); + EXPECT_TRUE(AnyLineContains(g_emittedLines, "KKK...")); + EXPECT_TRUE(AnyLineContains(g_emittedLines, "VVV...")); +} + +TEST_F(RasDiagnosticsEnvMicrotest, Summarize_ControlCharactersAreSanitizedToQuestionMarks) { + const size_t stride = rasDiagnosticsLocalRecordStride(sizeof(struct rasDiagnosticsNcclEnvData)); + std::vector buf(stride * 2, 0); + + std::string entry0 = std::string("NCCL_X=a\x01" "b"); + std::string entry1 = "NCCL_X=different"; + + auto fillRecord = [&](int rank, const std::string& entry) { + auto* rank_hdr = reinterpret_cast(buf.data() + rank * stride); + rank_hdr->commId.commHash = 0xB000; + rank_hdr->commRank = rank; + rank_hdr->commNRanks = 2; + auto* envData = reinterpret_cast(buf.data() + rank * stride + + sizeof(struct rasDiagnosticsRankHeader)); + memcpy(envData->data, entry.c_str(), entry.size() + 1); + envData->bytesUsed = static_cast(entry.size() + 1); + }; + fillRecord(0, entry0); + fillRecord(1, entry1); + + auto reporter = MakeRecordingReporter(); + ASSERT_EQ(ncclSuccess, rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, buf.data(), static_cast(buf.size()))); + EXPECT_TRUE(AnyLineContains(g_emittedLines, "NCCL_X=a?b")); +} diff --git a/projects/rccl/test/test_categories_micro.yaml b/projects/rccl/test/test_categories_micro.yaml index 90854c79a88d..185179fee941 100644 --- a/projects/rccl/test/test_categories_micro.yaml +++ b/projects/rccl/test/test_categories_micro.yaml @@ -36,6 +36,8 @@ test_categories: - "RasMicrotest.*" # ras/diagnostics_checks_common.cc - "RasDiagnosticsCommonMicrotest.*" + # ras/diagnostics_env.cc + - "RasDiagnosticsEnvMicrotest.*" # ras/client.cc - "RasClientMicrotest.*" # rma/rma_proxy_launch.cc From f67f93457ec111a81c845edc7f4c2233bc81b94b Mon Sep 17 00:00:00 2001 From: Alex Duforest Date: Wed, 30 Sep 2026 10:30:25 -0700 Subject: [PATCH 2/4] test(rccl): address diagnostics environment microtest review findings --- .../test/host/ras-diagnostics-env-test.cc | 273 +++++++----------- 1 file changed, 100 insertions(+), 173 deletions(-) diff --git a/projects/rccl/test/host/ras-diagnostics-env-test.cc b/projects/rccl/test/host/ras-diagnostics-env-test.cc index 6e6ee5944b33..3946456b7633 100644 --- a/projects/rccl/test/host/ras-diagnostics-env-test.cc +++ b/projects/rccl/test/host/ras-diagnostics-env-test.cc @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -73,6 +74,32 @@ bool AnyLineContains(const std::vector& lines, const std::string& n return false; } +size_t EnvRecordStride() { return rasDiagnosticsLocalRecordStride(sizeof(struct rasDiagnosticsNcclEnvData)); } + +struct rasDiagnosticsNcclEnvData* FillEnvRecord(std::vector& records, size_t index, + struct rasCommId commId, int rank, int nRanks, + std::initializer_list entries = {}, + bool truncated = false) { + char* record = records.data() + index * EnvRecordStride(); + auto* rankHeader = reinterpret_cast(record); + rankHeader->commId = commId; + rankHeader->commRank = rank; + rankHeader->commNRanks = nRanks; + + auto* envData = reinterpret_cast(record + sizeof(*rankHeader)); + size_t bytesUsed = 0; + for (const std::string& entry : entries) { + const size_t entryBytes = entry.size() + 1; + EXPECT_LE(bytesUsed + entryBytes, sizeof(envData->data)); + if (bytesUsed + entryBytes > sizeof(envData->data)) break; + memcpy(envData->data + bytesUsed, entry.c_str(), entryBytes); + bytesUsed += entryBytes; + } + envData->bytesUsed = static_cast(bytesUsed); + envData->truncated = truncated; + return envData; +} + struct FakeComm { std::unique_ptr comm{new ncclComm{}}; std::vector peerInfos; @@ -99,6 +126,7 @@ void InstallNcclComms(std::vector comms) { free(ncclComms); nNcclComms = static_cast(comms.size()); ncclComms = static_cast(calloc(comms.size() ? comms.size() : 1, sizeof(*ncclComms))); + ASSERT_NE(nullptr, ncclComms); for (size_t i = 0; i < comms.size(); i++) ncclComms[i] = comms[i]; } @@ -144,7 +172,7 @@ class RasDiagnosticsEnvMicrotest : public ::testing::Test { } // namespace // =========================================================================== -// rasDiagnosticsNcclEnvCollectLocal (via the real rasDiagnosticsCollectLocalRecords) +// rasDiagnosticsNcclEnvCollectLocal (through the shared microtest copy of rasDiagnosticsCollectLocalRecords) // =========================================================================== TEST_F(RasDiagnosticsEnvMicrotest, CollectLocal_NoMatchingCommsReturnsEmpty) { @@ -157,7 +185,7 @@ TEST_F(RasDiagnosticsEnvMicrotest, CollectLocal_NoMatchingCommsReturnsEmpty) { TEST_F(RasDiagnosticsEnvMicrotest, CollectLocal_FiltersNonNcclPrefixedVars) { FakeComm fc(0x1000, 0, 1); InstallNcclComms({fc.get()}); - SetFakeEnviron({"PATH=/usr/bin", "NCCL_DEBUG=INFO", "HOME=/root"}); + SetFakeEnviron({"PATH=/usr/bin", "NCCL_DEBUG=INFO", "NCCLFOO=1", "HOME=/root"}); struct rasDiagnosticsContext ctx{}; struct rasDiagnosticsLocalData data{}; @@ -168,6 +196,7 @@ TEST_F(RasDiagnosticsEnvMicrotest, CollectLocal_FiltersNonNcclPrefixedVars) { EXPECT_EQ(0, envData->truncated); std::string blob(envData->data, envData->bytesUsed); EXPECT_NE(std::string::npos, blob.find("NCCL_DEBUG=INFO")); + EXPECT_EQ(std::string::npos, blob.find("NCCLFOO")); EXPECT_EQ(std::string::npos, blob.find("PATH=")); free(data.records); } @@ -202,6 +231,8 @@ TEST_F(RasDiagnosticsEnvMicrotest, CollectLocal_ExceedingByteBudgetSetsTruncated const auto* envData = reinterpret_cast( data.records + sizeof(struct rasDiagnosticsRankHeader)); EXPECT_EQ(1, envData->truncated); + EXPECT_EQ(15532, envData->bytesUsed); + EXPECT_EQ(0u, std::string(envData->data, envData->bytesUsed).find("NCCL_VAR0=")); free(data.records); } @@ -246,13 +277,16 @@ TEST_F(RasDiagnosticsEnvMicrotest, Summarize_ZeroDataIsNoOp) { TEST_F(RasDiagnosticsEnvMicrotest, Summarize_NullDataWithNonzeroSizeReturnsInternalError) { auto reporter = MakeRecordingReporter(); - EXPECT_EQ(ncclInternalError, rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, nullptr, 8)); + const int stride = static_cast(EnvRecordStride()); + EXPECT_EQ(ncclInternalError, rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, nullptr, stride)); } TEST_F(RasDiagnosticsEnvMicrotest, Summarize_SizeNotMultipleOfStrideReturnsInternalError) { auto reporter = MakeRecordingReporter(); char buf[4] = {}; EXPECT_EQ(ncclInternalError, rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, buf, 1)); + EXPECT_EQ(ncclInternalError, + rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, buf, -static_cast(EnvRecordStride()))); } // =========================================================================== @@ -279,31 +313,22 @@ ncclResult_t CollectThenSummarize(struct rasDiagnosticsReporter* reporter) { TEST_F(RasDiagnosticsEnvMicrotest, Summarize_SingleRankConsistentReport) { FakeComm fc(0x2000, 0, 1); InstallNcclComms({fc.get()}); - SetFakeEnviron({"NCCL_DEBUG=INFO"}); + SetFakeEnviron({"NCCL_A_NO_EQUALS", "NCCL_DEBUG=INFO"}); auto reporter = MakeRecordingReporter(); ASSERT_EQ(ncclSuccess, CollectThenSummarize(&reporter)); ASSERT_EQ(1u, g_emittedLines.size()); EXPECT_EQ("[OK] NCCL environment: NCCL_* env vars consistent across 1 ranks in comm 0x2000", g_emittedLines[0]); + EXPECT_FALSE(AnyLineContains(g_emittedLines, "NCCL_A_NO_EQUALS")); EXPECT_FALSE(AnyLineContains(g_emittedLines, "mismatch")); } TEST_F(RasDiagnosticsEnvMicrotest, Summarize_MultipleRanksWithEqualValuesAreConsistent) { - const size_t stride = rasDiagnosticsLocalRecordStride(sizeof(struct rasDiagnosticsNcclEnvData)); + const size_t stride = EnvRecordStride(); std::vector buf(stride * 2, 0); for (int rank = 0; rank < 2; rank++) { - auto* rankHeader = reinterpret_cast(buf.data() + rank * stride); - rankHeader->commId.commHash = 0x2800; - rankHeader->commId.hostHash = 0x2801; - rankHeader->commId.pidHash = 0x2802; - rankHeader->commRank = rank; - rankHeader->commNRanks = 2; - auto* envData = reinterpret_cast( - buf.data() + rank * stride + sizeof(struct rasDiagnosticsRankHeader)); - constexpr char entry[] = "NCCL_DEBUG=INFO"; - memcpy(envData->data, entry, sizeof(entry)); - envData->bytesUsed = sizeof(entry); + FillEnvRecord(buf, rank, {0x2800, 0x2801, 0x2802}, rank, 2, {"NCCL_DEBUG=INFO"}); } auto reporter = MakeRecordingReporter(); @@ -333,25 +358,11 @@ TEST_F(RasDiagnosticsEnvMicrotest, Summarize_TwoRanksMismatchedValueReportsMisma // different rank -- built by hand below since FakeComm's per-object environ can't // differ per rank (only one process-wide `environ`), so we hand-build two records // directly instead of going through CollectLocal twice. - const size_t stride = rasDiagnosticsLocalRecordStride(sizeof(struct rasDiagnosticsNcclEnvData)); + const size_t stride = EnvRecordStride(); std::vector buf(stride * 2, 0); - auto fillRecord = [&](int rank, const char* value) { - auto* rank_hdr = reinterpret_cast(buf.data() + rank * stride); - rank_hdr->commId.commHash = 0x4000; - rank_hdr->commId.hostHash = 0x4001; - rank_hdr->commId.pidHash = 0x4002; - rank_hdr->commRank = rank; - rank_hdr->commNRanks = 2; - auto* envData = reinterpret_cast(buf.data() + rank * stride + - sizeof(struct rasDiagnosticsRankHeader)); - std::string entry = std::string("NCCL_DEBUG=") + value; - memcpy(envData->data, entry.c_str(), entry.size() + 1); - envData->bytesUsed = static_cast(entry.size() + 1); - envData->truncated = 0; - }; - fillRecord(0, "INFO"); - fillRecord(1, "WARN"); + FillEnvRecord(buf, 0, {0x4000, 0x4001, 0x4002}, 0, 2, {"NCCL_DEBUG=INFO"}); + FillEnvRecord(buf, 1, {0x4000, 0x4001, 0x4002}, 1, 2, {"NCCL_DEBUG=WARN"}); auto reporter = MakeRecordingReporter(); ASSERT_EQ(ncclSuccess, rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, buf.data(), static_cast(buf.size()))); @@ -364,21 +375,13 @@ TEST_F(RasDiagnosticsEnvMicrotest, Summarize_TwoRanksMismatchedValueReportsMisma } TEST_F(RasDiagnosticsEnvMicrotest, Summarize_FourRanksGroupsRepeatedValuesOnce) { - const size_t stride = rasDiagnosticsLocalRecordStride(sizeof(struct rasDiagnosticsNcclEnvData)); + const size_t stride = EnvRecordStride(); std::vector buf(stride * 4, 0); const int ranks[] = {3, 1, 2, 0}; const char* values[] = {"WARN", "WARN", "INFO", "INFO"}; for (int i = 0; i < 4; i++) { - auto* rankHeader = reinterpret_cast(buf.data() + i * stride); - rankHeader->commId.commHash = 0x4100; - rankHeader->commRank = ranks[i]; - rankHeader->commNRanks = 4; - auto* envData = reinterpret_cast( - buf.data() + i * stride + sizeof(struct rasDiagnosticsRankHeader)); - std::string entry = std::string("NCCL_DEBUG=") + values[i]; - memcpy(envData->data, entry.c_str(), entry.size() + 1); - envData->bytesUsed = static_cast(entry.size() + 1); + FillEnvRecord(buf, i, {0x4100, 0, 0}, ranks[i], 4, {std::string("NCCL_DEBUG=") + values[i]}); } auto reporter = MakeRecordingReporter(); @@ -392,24 +395,11 @@ TEST_F(RasDiagnosticsEnvMicrotest, Summarize_FourRanksGroupsRepeatedValuesOnce) } TEST_F(RasDiagnosticsEnvMicrotest, Summarize_TwoDistinctKeysAreSortedAndDeduplicated) { - const size_t stride = rasDiagnosticsLocalRecordStride(sizeof(struct rasDiagnosticsNcclEnvData)); + const size_t stride = EnvRecordStride(); std::vector buf(stride * 2, 0); - auto fillRecord = [&](int rank, const char* first, const char* second) { - auto* rankHeader = reinterpret_cast(buf.data() + rank * stride); - rankHeader->commId.commHash = 0x4150; - rankHeader->commRank = rank; - rankHeader->commNRanks = 2; - auto* envData = reinterpret_cast( - buf.data() + rank * stride + sizeof(struct rasDiagnosticsRankHeader)); - const size_t firstLen = strlen(first) + 1; - const size_t secondLen = strlen(second) + 1; - memcpy(envData->data, first, firstLen); - memcpy(envData->data + firstLen, second, secondLen); - envData->bytesUsed = static_cast(firstLen + secondLen); - }; - fillRecord(0, "NCCL_DEBUG_SUBSYS=INIT", "NCCL_DEBUG=INFO"); - fillRecord(1, "NCCL_DEBUG_SUBSYS=COLL", "NCCL_DEBUG=WARN"); + FillEnvRecord(buf, 0, {0x4150, 0, 0}, 0, 2, {"NCCL_DEBUG_SUBSYS=INIT", "NCCL_DEBUG=INFO"}); + FillEnvRecord(buf, 1, {0x4150, 0, 0}, 1, 2, {"NCCL_DEBUG_SUBSYS=COLL", "NCCL_DEBUG=WARN"}); auto reporter = MakeRecordingReporter(); ASSERT_EQ(ncclSuccess, rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, buf.data(), static_cast(buf.size()))); @@ -429,22 +419,14 @@ TEST_F(RasDiagnosticsEnvMicrotest, Summarize_TwoDistinctKeysAreSortedAndDeduplic } TEST_F(RasDiagnosticsEnvMicrotest, Summarize_AllocationFailuresPropagate) { - const size_t stride = rasDiagnosticsLocalRecordStride(sizeof(struct rasDiagnosticsNcclEnvData)); + const size_t stride = EnvRecordStride(); std::vector buf(stride * 2, 0); - for (int rank = 0; rank < 2; rank++) { - auto* rankHeader = reinterpret_cast(buf.data() + rank * stride); - rankHeader->commId.commHash = 0x4175; - rankHeader->commRank = rank; - rankHeader->commNRanks = 2; - auto* envData = reinterpret_cast( - buf.data() + rank * stride + sizeof(struct rasDiagnosticsRankHeader)); - const char* entry = rank == 0 ? "NCCL_DEBUG=INFO" : "NCCL_DEBUG=WARN"; - memcpy(envData->data, entry, strlen(entry) + 1); - envData->bytesUsed = static_cast(strlen(entry) + 1); - } + FillEnvRecord(buf, 0, {0x4175, 0, 0}, 0, 2, {"NCCL_DEBUG=INFO"}); + FillEnvRecord(buf, 1, {0x4175, 0, 0}, 1, 2, {"NCCL_DEBUG=WARN"}); auto reporter = MakeRecordingReporter(); for (int failAt = 1; failAt <= 5; failAt++) { + SCOPED_TRACE(failAt); g_allocationCalls = 0; g_allocationCounts.clear(); g_failAllocationCall = failAt; @@ -462,79 +444,58 @@ TEST_F(RasDiagnosticsEnvMicrotest, CollectLocal_EntryArrayAllocationFailurePropa struct rasDiagnosticsLocalData data{}; g_allocationCalls = 0; g_failAllocationCall = 1; + // The diagnostics-env seam controls the callback's entry array; shared collector allocations + // are covered by RasDiagnosticsCommonMicrotest. EXPECT_EQ(ncclSystemError, rasDiagnosticsNcclEnvCollectLocal(&ctx, &data)); EXPECT_EQ(nullptr, data.records); } TEST_F(RasDiagnosticsEnvMicrotest, Summarize_ReporterFailuresPropagate) { - const size_t stride = rasDiagnosticsLocalRecordStride(sizeof(struct rasDiagnosticsNcclEnvData)); + const size_t stride = EnvRecordStride(); std::vector mismatch(stride * 2, 0); - for (int rank = 0; rank < 2; rank++) { - auto* rankHeader = reinterpret_cast(mismatch.data() + rank * stride); - rankHeader->commId.commHash = 0x4200; - rankHeader->commRank = rank; - rankHeader->commNRanks = 2; - auto* envData = reinterpret_cast( - mismatch.data() + rank * stride + sizeof(struct rasDiagnosticsRankHeader)); - std::string entry = std::string("NCCL_DEBUG=") + (rank == 0 ? "INFO" : "WARN"); - memcpy(envData->data, entry.c_str(), entry.size() + 1); - envData->bytesUsed = static_cast(entry.size() + 1); - } + FillEnvRecord(mismatch, 0, {0x4200, 0, 0}, 0, 2, {"NCCL_DEBUG=INFO"}); + FillEnvRecord(mismatch, 1, {0x4200, 0, 0}, 1, 2, {"NCCL_DEBUG=WARN"}); auto reporter = MakeRecordingReporter(); for (int failAt = 1; failAt <= 4; failAt++) { + SCOPED_TRACE(failAt); g_emitCalls = 0; + g_emittedLines.clear(); g_emitFailAt = failAt; EXPECT_EQ(ncclSystemError, rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, mismatch.data(), static_cast(mismatch.size()))); + EXPECT_EQ(failAt, g_emitCalls); } std::vector single(stride, 0); - auto* rankHeader = reinterpret_cast(single.data()); - rankHeader->commId.commHash = 0x4300; - rankHeader->commRank = 0; - rankHeader->commNRanks = 1; - auto* envData = reinterpret_cast( - single.data() + sizeof(struct rasDiagnosticsRankHeader)); + auto* envData = FillEnvRecord(single, 0, {0x4300, 0, 0}, 0, 1); g_emitCalls = 0; g_emitFailAt = 1; EXPECT_EQ(ncclSystemError, rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, single.data(), static_cast(single.size()))); + EXPECT_EQ(1, g_emitCalls); envData->truncated = 1; g_emitCalls = 0; EXPECT_EQ(ncclSystemError, rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, single.data(), static_cast(single.size()))); + EXPECT_EQ(1, g_emitCalls); - rankHeader->commNRanks = 2; + reinterpret_cast(single.data())->commNRanks = 2; envData->truncated = 0; g_emitCalls = 0; EXPECT_EQ(ncclSystemError, rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, single.data(), static_cast(single.size()))); + EXPECT_EQ(1, g_emitCalls); } TEST_F(RasDiagnosticsEnvMicrotest, Summarize_ValueSetOnOneRankUnsetOnAnotherIsAMismatch) { - const size_t stride = rasDiagnosticsLocalRecordStride(sizeof(struct rasDiagnosticsNcclEnvData)); + const size_t stride = EnvRecordStride(); std::vector buf(stride * 2, 0); - auto* r0 = reinterpret_cast(buf.data()); - r0->commId.commHash = 0x5000; - r0->commRank = 0; - r0->commNRanks = 2; - auto* env0 = - reinterpret_cast(buf.data() + sizeof(struct rasDiagnosticsRankHeader)); - const char* entry0 = "NCCL_FOO=bar"; - memcpy(env0->data, entry0, strlen(entry0) + 1); - env0->bytesUsed = static_cast(strlen(entry0) + 1); - - auto* r1 = reinterpret_cast(buf.data() + stride); - r1->commId.commHash = 0x5000; - r1->commRank = 1; - r1->commNRanks = 2; - auto* env1 = reinterpret_cast(buf.data() + stride + - sizeof(struct rasDiagnosticsRankHeader)); - env1->bytesUsed = 0; // NCCL_FOO not set on rank 1. + FillEnvRecord(buf, 0, {0x5000, 0, 0}, 0, 2, {"NCCL_FOO=bar"}); + FillEnvRecord(buf, 1, {0x5000, 0, 0}, 1, 2); auto reporter = MakeRecordingReporter(); ASSERT_EQ(ncclSuccess, rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, buf.data(), static_cast(buf.size()))); @@ -543,59 +504,39 @@ TEST_F(RasDiagnosticsEnvMicrotest, Summarize_ValueSetOnOneRankUnsetOnAnotherIsAM } TEST_F(RasDiagnosticsEnvMicrotest, Summarize_TruncatedRankReportsTruncationWarning) { - const size_t stride = rasDiagnosticsLocalRecordStride(sizeof(struct rasDiagnosticsNcclEnvData)); + const size_t stride = EnvRecordStride(); std::vector buf(stride, 0); - auto* rank_hdr = reinterpret_cast(buf.data()); - rank_hdr->commId.commHash = 0x6000; - rank_hdr->commRank = 0; - rank_hdr->commNRanks = 1; - auto* envData = - reinterpret_cast(buf.data() + sizeof(struct rasDiagnosticsRankHeader)); - const char* entry = "NCCL_DEBUG=INFO"; - memcpy(envData->data, entry, strlen(entry) + 1); - envData->bytesUsed = static_cast(strlen(entry) + 1); - envData->truncated = 1; + FillEnvRecord(buf, 0, {0x6000, 0, 0}, 0, 1, {"NCCL_DEBUG=INFO"}, true); auto reporter = MakeRecordingReporter(); ASSERT_EQ(ncclSuccess, rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, buf.data(), static_cast(buf.size()))); EXPECT_TRUE(AnyLineContains(g_emittedLines, "1 rank(s) had >16384 bytes")); + EXPECT_FALSE(AnyLineContains(g_emittedLines, "consistent across")); } -TEST_F(RasDiagnosticsEnvMicrotest, Summarize_MalformedEnvPayloadReturnsInternalError) { - const size_t stride = rasDiagnosticsLocalRecordStride(sizeof(struct rasDiagnosticsNcclEnvData)); +TEST_F(RasDiagnosticsEnvMicrotest, Summarize_MalformedEnvPayloadsReturnInternalError) { + const size_t stride = EnvRecordStride(); std::vector buf(stride, 0); - auto* rank_hdr = reinterpret_cast(buf.data()); - rank_hdr->commId.commHash = 0x7000; - rank_hdr->commRank = 0; - rank_hdr->commNRanks = 1; - auto* envData = - reinterpret_cast(buf.data() + sizeof(struct rasDiagnosticsRankHeader)); + auto* envData = FillEnvRecord(buf, 0, {0x7000, 0, 0}, 0, 1); envData->bytesUsed = 5; envData->data[4] = 'x'; // Not NUL-terminated at the claimed end -- malformed. auto reporter = MakeRecordingReporter(); EXPECT_EQ(ncclInternalError, rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, buf.data(), static_cast(buf.size()))); + + envData->bytesUsed = RAS_DIAG_ENV_BYTES + 1; + EXPECT_EQ(ncclInternalError, + rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, buf.data(), static_cast(buf.size()))); } TEST_F(RasDiagnosticsEnvMicrotest, Summarize_TwoDistinctCommsAreReportedSeparately) { - const size_t stride = rasDiagnosticsLocalRecordStride(sizeof(struct rasDiagnosticsNcclEnvData)); + const size_t stride = EnvRecordStride(); std::vector buf(stride * 3, 0); - auto fillSolo = [&](int idx, uint64_t commHash, uint64_t hostHash, uint64_t pidHash) { - auto* rank_hdr = reinterpret_cast(buf.data() + idx * stride); - rank_hdr->commId.commHash = commHash; - rank_hdr->commId.hostHash = hostHash; - rank_hdr->commId.pidHash = pidHash; - rank_hdr->commRank = 0; - rank_hdr->commNRanks = 1; - auto* envData = reinterpret_cast(buf.data() + idx * stride + - sizeof(struct rasDiagnosticsRankHeader)); - envData->bytesUsed = 0; - }; - fillSolo(0, 0x8000, 0x8100, 0x8200); - fillSolo(1, 0x8000, 0x8300, 0x8400); - fillSolo(2, 0x9000, 0x9100, 0x9200); + FillEnvRecord(buf, 0, {0x8000, 0x8100, 0x8200}, 0, 1); + FillEnvRecord(buf, 1, {0x8000, 0x8300, 0x8400}, 0, 1); + FillEnvRecord(buf, 2, {0x9000, 0x9100, 0x9200}, 0, 1); auto reporter = MakeRecordingReporter(); ASSERT_EQ(ncclSuccess, rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, buf.data(), static_cast(buf.size()))); @@ -607,7 +548,7 @@ TEST_F(RasDiagnosticsEnvMicrotest, Summarize_TwoDistinctCommsAreReportedSeparate } TEST_F(RasDiagnosticsEnvMicrotest, Summarize_LongKeyAndValueAreTruncatedWithEllipsis) { - const size_t stride = rasDiagnosticsLocalRecordStride(sizeof(struct rasDiagnosticsNcclEnvData)); + const size_t stride = EnvRecordStride(); std::vector buf(stride * 2, 0); std::string longKey = "NCCL_" + std::string(300, 'K'); @@ -615,46 +556,32 @@ TEST_F(RasDiagnosticsEnvMicrotest, Summarize_LongKeyAndValueAreTruncatedWithElli std::string entry0 = longKey + "=" + longVal; std::string entry1 = longKey + "=" + std::string(300, 'W'); // Different value -> mismatch. - auto fillRecord = [&](int rank, const std::string& entry) { - auto* rank_hdr = reinterpret_cast(buf.data() + rank * stride); - rank_hdr->commId.commHash = 0xA000; - rank_hdr->commRank = rank; - rank_hdr->commNRanks = 2; - auto* envData = reinterpret_cast(buf.data() + rank * stride + - sizeof(struct rasDiagnosticsRankHeader)); - memcpy(envData->data, entry.c_str(), entry.size() + 1); - envData->bytesUsed = static_cast(entry.size() + 1); - }; - fillRecord(0, entry0); - fillRecord(1, entry1); + FillEnvRecord(buf, 0, {0xA000, 0, 0}, 0, 2, {entry0}); + FillEnvRecord(buf, 1, {0xA000, 0, 0}, 1, 2, {entry1}); auto reporter = MakeRecordingReporter(); ASSERT_EQ(ncclSuccess, rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, buf.data(), static_cast(buf.size()))); - EXPECT_TRUE(AnyLineContains(g_emittedLines, "KKK...")); - EXPECT_TRUE(AnyLineContains(g_emittedLines, "VVV...")); + const std::string keyDisplay = "NCCL_" + std::string(247, 'K') + "..."; + const std::string valueDisplay = std::string(252, 'V') + "..."; + ASSERT_EQ(4u, g_emittedLines.size()); + EXPECT_EQ("[INFO] NCCL environment: mismatch across 2 ranks in comm 0xa000 for " + keyDisplay, g_emittedLines[0]); + EXPECT_EQ("[INFO] NCCL environment: " + keyDisplay + "=" + valueDisplay + " on rank(s) {0}", g_emittedLines[1]); } TEST_F(RasDiagnosticsEnvMicrotest, Summarize_ControlCharactersAreSanitizedToQuestionMarks) { - const size_t stride = rasDiagnosticsLocalRecordStride(sizeof(struct rasDiagnosticsNcclEnvData)); + const size_t stride = EnvRecordStride(); std::vector buf(stride * 2, 0); - std::string entry0 = std::string("NCCL_X=a\x01" "b"); - std::string entry1 = "NCCL_X=different"; - - auto fillRecord = [&](int rank, const std::string& entry) { - auto* rank_hdr = reinterpret_cast(buf.data() + rank * stride); - rank_hdr->commId.commHash = 0xB000; - rank_hdr->commRank = rank; - rank_hdr->commNRanks = 2; - auto* envData = reinterpret_cast(buf.data() + rank * stride + - sizeof(struct rasDiagnosticsRankHeader)); - memcpy(envData->data, entry.c_str(), entry.size() + 1); - envData->bytesUsed = static_cast(entry.size() + 1); - }; - fillRecord(0, entry0); - fillRecord(1, entry1); + FillEnvRecord(buf, 0, {0xB000, 0, 0}, 0, 2, + {std::string("NCCL_A\x01" "KEY=left"), std::string("NCCL_B\x7f" "KEY=a\x7f" "b"), + std::string("NCCL_X=a\x01" "b")}); + FillEnvRecord(buf, 1, {0xB000, 0, 0}, 1, 2, + {std::string("NCCL_A\x01" "KEY=right"), std::string("NCCL_B\x7f" "KEY=different"), + "NCCL_X=different"}); auto reporter = MakeRecordingReporter(); ASSERT_EQ(ncclSuccess, rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, buf.data(), static_cast(buf.size()))); + EXPECT_TRUE(AnyLineContains(g_emittedLines, "NCCL_A?KEY")); + EXPECT_TRUE(AnyLineContains(g_emittedLines, "NCCL_B?KEY=a?b")); EXPECT_TRUE(AnyLineContains(g_emittedLines, "NCCL_X=a?b")); } From 68767d53d43161346b0ea2aa7b0e46efa3f21d45 Mon Sep 17 00:00:00 2001 From: Alex Duforest Date: Thu, 1 Oct 2026 23:24:21 -0700 Subject: [PATCH 3/4] test(rccl): cover environment offsets across communicators --- .../test/host/ras-diagnostics-env-test.cc | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/projects/rccl/test/host/ras-diagnostics-env-test.cc b/projects/rccl/test/host/ras-diagnostics-env-test.cc index 3946456b7633..b904a272b787 100644 --- a/projects/rccl/test/host/ras-diagnostics-env-test.cc +++ b/projects/rccl/test/host/ras-diagnostics-env-test.cc @@ -532,19 +532,26 @@ TEST_F(RasDiagnosticsEnvMicrotest, Summarize_MalformedEnvPayloadsReturnInternalE TEST_F(RasDiagnosticsEnvMicrotest, Summarize_TwoDistinctCommsAreReportedSeparately) { const size_t stride = EnvRecordStride(); - std::vector buf(stride * 3, 0); + std::vector buf(stride * 4, 0); - FillEnvRecord(buf, 0, {0x8000, 0x8100, 0x8200}, 0, 1); - FillEnvRecord(buf, 1, {0x8000, 0x8300, 0x8400}, 0, 1); + FillEnvRecord(buf, 0, {0x8000, 0x8100, 0x8200}, 0, 1, {"NCCL_FIRST=ONLY"}); + FillEnvRecord(buf, 1, {0x8000, 0x8300, 0x8400}, 1, 2, {"NCCL_DEBUG=WARN"}); FillEnvRecord(buf, 2, {0x9000, 0x9100, 0x9200}, 0, 1); + FillEnvRecord(buf, 3, {0x8000, 0x8300, 0x8400}, 0, 2, {"NCCL_DEBUG=INFO"}); auto reporter = MakeRecordingReporter(); ASSERT_EQ(ncclSuccess, rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, buf.data(), static_cast(buf.size()))); - int consistentCount = 0; - for (auto& l : g_emittedLines) { - if (l.find("consistent across 1 ranks") != std::string::npos) consistentCount++; - } - EXPECT_EQ(3, consistentCount); + const std::vector expectedLines = { + "[OK] NCCL environment: NCCL_* env vars consistent across 1 ranks in comm 0x8000", + "[INFO] NCCL environment: mismatch across 2 ranks in comm 0x8000 for NCCL_DEBUG", + "[INFO] NCCL environment: NCCL_DEBUG=INFO on rank(s) {0}", + "[INFO] NCCL environment: NCCL_DEBUG=WARN on rank(s) {1}", + "[INFO] NCCL environment: 1 NCCL_* env var(s) differ across ranks in comm 0x8000", + "[OK] NCCL environment: NCCL_* env vars consistent across 1 ranks in comm 0x9000", + }; + EXPECT_EQ(expectedLines, g_emittedLines); + const std::vector expectedAllocationCounts = {buf.size(), 1, 1, 2, 2, 2, 2}; + EXPECT_EQ(expectedAllocationCounts, g_allocationCounts); } TEST_F(RasDiagnosticsEnvMicrotest, Summarize_LongKeyAndValueAreTruncatedWithEllipsis) { From 3ac5eae440e12cea95789a2fc2d12d2d2a462705 Mon Sep 17 00:00:00 2001 From: Alex Duforest Date: Fri, 2 Oct 2026 16:45:36 -0700 Subject: [PATCH 4/4] test(rccl): address RAS diagnostics review feedback --- projects/rccl/test/host/MICROTEST_README.md | 23 ++ projects/rccl/test/host/fakes/env_fakes.cc | 68 ++++- projects/rccl/test/host/fakes/env_fakes.h | 4 + .../host/fakes/ras_diagnostics_test_support.h | 55 ++++ .../host/fakes/ras_registry_test_support.h | 34 +++ .../test/host/ras-diagnostics-common-test.cc | 66 +---- .../test/host/ras-diagnostics-env-test.cc | 247 +++++++++++------- projects/rccl/test/host/ras-test.cc | 27 +- 8 files changed, 351 insertions(+), 173 deletions(-) create mode 100644 projects/rccl/test/host/fakes/ras_diagnostics_test_support.h create mode 100644 projects/rccl/test/host/fakes/ras_registry_test_support.h diff --git a/projects/rccl/test/host/MICROTEST_README.md b/projects/rccl/test/host/MICROTEST_README.md index 3e766d972675..fc64c8ef6752 100644 --- a/projects/rccl/test/host/MICROTEST_README.md +++ b/projects/rccl/test/host/MICROTEST_README.md @@ -739,3 +739,26 @@ cmake --build build -j"$(nproc)" Disable coverage instrumentation for the standalone host-only test binaries with `-DHOST_TEST_COVERAGE=OFF`. + +### Shared RAS diagnostic fixtures + +`fakes/ras_registry_test_support.h` owns test installation/reset of the +`ras.cc` communicator registry. Both operations hold `ncclCommsMutex` and +clear `ncclCommsSorted`; installation validates allocation before replacing +the old registry and preserves explicit vacant slots. The RAS and diagnostics +suites use the same helper. `fakes/ras_diagnostics_test_support.h` shares the +owned communicator and recording reporter used by the diagnostic suites. + +`SetMicroEnviron` in `fakes/env_fakes.{h,cc}` replaces the scripted environment +for enumeration tests, preserving input order and malformed entries while +making valid entries visible through `micro_getenv` and `getenv`. While that +override is active, `SetMicroEnv` and `SetMicroEnvAbsent` also update `environ`. +`ClearMicroEnv`/`ResetEnvFakes` restore the original process environment and +clear the lookup map. Map-only tests retain the existing fallback behavior +unless they opt into the enumeration override. + +`DISABLED_Summarize_HashCollisionsReportFullCommIdentity` tracks AICOMRCCL-2743. +The environment reporters currently omit host and process hashes, so distinct +communicators can appear identical. The disabled regression specifies the full +identity for consistent, mismatch, and mismatch-summary reports. Enable it +with the separate production fix; enabled tests still pin correct grouping. diff --git a/projects/rccl/test/host/fakes/env_fakes.cc b/projects/rccl/test/host/fakes/env_fakes.cc index c4569ec55d53..cc6211602d0e 100644 --- a/projects/rccl/test/host/fakes/env_fakes.cc +++ b/projects/rccl/test/host/fakes/env_fakes.cc @@ -14,10 +14,34 @@ #include #include #include +#include +#include + +#include #include "nccl.h" namespace { +struct MicroEnvironState { + bool active = false; + char** original = nullptr; + std::vector entries; + std::vector pointers; +}; + +MicroEnvironState& microEnvironState() { + static MicroEnvironState state; + return state; +} + +void PublishMicroEnviron() { + auto& state = microEnvironState(); + state.pointers.clear(); + for (auto& entry : state.entries) state.pointers.push_back(entry.data()); + state.pointers.push_back(nullptr); + environ = state.pointers.data(); +} + // A nullopt entry means "absent". Unmapped names read as unset via micro_getenv, real via the getenv interposer. std::unordered_map>& microEnvMap() { static std::unordered_map> m; @@ -54,11 +78,53 @@ void SetMicroEnv(const char* name, const char* value) { if (name == nullptr) return; if (value == nullptr) microEnvMap()[name] = std::nullopt; else microEnvMap()[name] = value; + auto& state = microEnvironState(); + if (state.active) { + const std::string prefix = std::string(name) + "="; + std::vector entries; + bool replaced = false; + for (const auto& entry : state.entries) { + if (entry.compare(0, prefix.size(), prefix) != 0) { + entries.push_back(entry); + } else if (!replaced && value != nullptr) { + entries.push_back(prefix + value); + replaced = true; + } + } + if (!replaced && value != nullptr) entries.push_back(prefix + value); + state.entries = std::move(entries); + PublishMicroEnviron(); + } } void SetMicroEnvAbsent(const char* name) { SetMicroEnv(name, nullptr); } -void ClearMicroEnv() { microEnvMap().clear(); } +void SetMicroEnviron(const std::vector& entries) { + auto& state = microEnvironState(); + if (!state.active) state.original = environ; + state.active = true; + state.entries = entries; + auto& values = microEnvMap(); + values.clear(); + for (const auto& entry : entries) { + const auto separator = entry.find('='); + if (separator != std::string::npos) + values.emplace(entry.substr(0, separator), entry.substr(separator + 1)); + } + PublishMicroEnviron(); +} + +void ClearMicroEnv() { + auto& state = microEnvironState(); + if (state.active) { + environ = state.original; + state.active = false; + state.original = nullptr; + state.entries.clear(); + state.pointers.clear(); + } + microEnvMap().clear(); +} void ResetEnvFakes() { ClearMicroEnv(); } diff --git a/projects/rccl/test/host/fakes/env_fakes.h b/projects/rccl/test/host/fakes/env_fakes.h index 373c44cb0ee6..b05c5a8869a8 100644 --- a/projects/rccl/test/host/fakes/env_fakes.h +++ b/projects/rccl/test/host/fakes/env_fakes.h @@ -21,6 +21,9 @@ #ifndef RCCL_TEST_HOST_ENV_FAKES_H_ #define RCCL_TEST_HOST_ENV_FAKES_H_ +#include +#include + // Strict: a name the fixture has not scripted reads as unset, so no test can // depend on the ambient environment. ncclGetEnv routes here. const char* micro_getenv(const char* name); @@ -29,6 +32,7 @@ const char* micro_getenv(const char* name); // fall through to the real getenv. void SetMicroEnv(const char* name, const char* value); void SetMicroEnvAbsent(const char* name); +void SetMicroEnviron(const std::vector& entries); void ClearMicroEnv(); // Named like every other per-TU reset so a Reset*Fakes() chain reads as one list diff --git a/projects/rccl/test/host/fakes/ras_diagnostics_test_support.h b/projects/rccl/test/host/fakes/ras_diagnostics_test_support.h new file mode 100644 index 000000000000..11e12b14d5c6 --- /dev/null +++ b/projects/rccl/test/host/fakes/ras_diagnostics_test_support.h @@ -0,0 +1,55 @@ +#ifndef RCCL_TEST_HOST_RAS_DIAGNOSTICS_TEST_SUPPORT_H_ +#define RCCL_TEST_HOST_RAS_DIAGNOSTICS_TEST_SUPPORT_H_ + +#include +#include +#include + +#include "comm.h" +#include "ras/diagnostics.h" +#include "fakes/ras_registry_test_support.h" + +namespace ras_test { + +struct OwnedComm { + OwnedComm(uint64_t commHash, uint64_t hostHash, uint64_t pidHash, int rank, bool peerInfoValid = true) + : comm(std::make_unique()), peers(std::make_unique(1)) { + comm->commHash = commHash; + comm->peerInfo = peers.get(); + comm->peerInfoValid = peerInfoValid; + comm->rank = rank; + comm->nRanks = 8; + comm->cudaDev = rank + 10; + comm->nvmlDev = rank + 20; + comm->busId = 0x1000 + rank; + comm->localRank = rank % 4; + comm->localRanks = 4; + peers[0].hostHash = hostHash; + peers[0].pidHash = pidHash; + } + + std::unique_ptr comm; + std::unique_ptr peers; +}; + +struct ReporterState { + std::vector lines; + ncclResult_t emitResult = ncclSuccess; + int calls = 0; + int failCall = 0; +}; + +inline ncclResult_t CaptureReport(void* target, const char* line) { + auto* state = static_cast(target); + state->lines.emplace_back(line); + ++state->calls; + return state->failCall == 0 || state->calls == state->failCall ? state->emitResult : ncclSuccess; +} + +inline rasDiagnosticsReporter MakeReporter(ReporterState* state) { + return rasDiagnosticsReporter{CaptureReport, nullptr, state}; +} + +} + +#endif diff --git a/projects/rccl/test/host/fakes/ras_registry_test_support.h b/projects/rccl/test/host/fakes/ras_registry_test_support.h new file mode 100644 index 000000000000..fc0c5250d391 --- /dev/null +++ b/projects/rccl/test/host/fakes/ras_registry_test_support.h @@ -0,0 +1,34 @@ +#ifndef RCCL_TEST_HOST_RAS_REGISTRY_TEST_SUPPORT_H_ +#define RCCL_TEST_HOST_RAS_REGISTRY_TEST_SUPPORT_H_ + +#include +#include +#include + +#include +#include "ras/ras_internal.h" + +namespace ras_test { + +inline void InstallNcclComms(const std::vector& comms) { + std::lock_guard lock(ncclCommsMutex); + auto** replacement = static_cast(std::calloc(comms.empty() ? 1 : comms.size(), sizeof(ncclComm*))); + ASSERT_NE(nullptr, replacement); + std::free(ncclComms); + ncclComms = replacement; + nNcclComms = static_cast(comms.size()); + ncclCommsSorted = false; + for (size_t index = 0; index < comms.size(); ++index) ncclComms[index] = comms[index]; +} + +inline void ResetNcclComms() { + std::lock_guard lock(ncclCommsMutex); + std::free(ncclComms); + ncclComms = nullptr; + nNcclComms = 0; + ncclCommsSorted = false; +} + +} + +#endif diff --git a/projects/rccl/test/host/ras-diagnostics-common-test.cc b/projects/rccl/test/host/ras-diagnostics-common-test.cc index 1912814d1d12..ff9f02b51827 100644 --- a/projects/rccl/test/host/ras-diagnostics-common-test.cc +++ b/projects/rccl/test/host/ras-diagnostics-common-test.cc @@ -22,6 +22,7 @@ #include "alloc.h" #include "checks.h" #include "comm.h" +#include "fakes/ras_diagnostics_test_support.h" #include "compiler.h" #include "ras/diagnostics_checks_common.h" #include "ras/ras_internal.h" @@ -58,26 +59,10 @@ struct TestPayload { uint32_t marker; }; -struct OwnedComm { - OwnedComm(uint64_t commHash, uint64_t hostHash, uint64_t pidHash, int rank, bool peerInfoValid = true) - : comm(std::make_unique()), peers(std::make_unique(1)) { - comm->commHash = commHash; - comm->peerInfo = peers.get(); - comm->peerInfoValid = peerInfoValid; - comm->rank = rank; - comm->nRanks = 8; - comm->cudaDev = rank + 10; - comm->nvmlDev = rank + 20; - comm->busId = 0x1000 + rank; - comm->localRank = rank % 4; - comm->localRanks = 4; - peers[0].hostHash = hostHash; - peers[0].pidHash = pidHash; - } - - std::unique_ptr comm; - std::unique_ptr peers; -}; +using ras_test::OwnedComm; +using ras_test::ReporterState; +using ras_test::CaptureReport; +using ras_test::InstallNcclComms; std::vector g_fillSnapshots; int g_fillCalls = 0; @@ -102,23 +87,8 @@ ncclResult_t FillTestPayload(const rasDiagnosticsCommSnapshot* snapshot, void* c return ncclSuccess; } -struct ReporterState { - std::vector lines; - ncclResult_t emitResult = ncclSuccess; -}; - -ncclResult_t CaptureReport(void* target, const char* line) { - auto* state = static_cast(target); - state->lines.emplace_back(line); - return state->emitResult; -} - void ResetDiagnosticsCommonState() { - std::lock_guard lock(ncclCommsMutex); - std::free(ncclComms); - ncclComms = nullptr; - nNcclComms = 0; - ncclCommsSorted = false; + ras_test::ResetNcclComms(); g_allocationCalls = 0; g_failAllocationCall = 0; g_allocationCounts.clear(); @@ -128,16 +98,6 @@ void ResetDiagnosticsCommonState() { g_fillObservedUnlockedMutex = true; } -void InstallComms(std::initializer_list comms) { - std::lock_guard lock(ncclCommsMutex); - std::free(ncclComms); - ncclComms = static_cast(std::calloc(comms.size(), sizeof(*ncclComms))); - ASSERT_NE(nullptr, ncclComms); - nNcclComms = static_cast(comms.size()); - int index = 0; - for (ncclComm* comm : comms) ncclComms[index++] = comm; -} - rasDiagnosticsContext UnfilteredContext() { rasDiagnosticsContext ctx{}; ctx.hasCommFilter = false; @@ -307,7 +267,7 @@ TEST_F(RasDiagnosticsCommonMicrotest, CollectRejectsStrideRoundedPastIntMax) { TEST_F(RasDiagnosticsCommonMicrotest, CollectRejectsCombinedRecordBytesPastIntMax) { OwnedComm first(1, 2, 3, 0); OwnedComm second(4, 5, 6, 1); - InstallComms({first.comm.get(), second.comm.get()}); + InstallNcclComms({first.comm.get(), second.comm.get()}); const rasDiagnosticsContext ctx = UnfilteredContext(); rasDiagnosticsLocalData data{}; const size_t payloadSize = static_cast(INT_MAX) / 2; @@ -323,7 +283,7 @@ TEST_F(RasDiagnosticsCommonMicrotest, CollectRejectsCombinedRecordBytesPastIntMa TEST_F(RasDiagnosticsCommonMicrotest, CollectAcceptsRecordBytesExactlyAtIntMaxLimit) { OwnedComm first(1, 2, 3, 0); OwnedComm second(4, 5, 6, 1); - InstallComms({first.comm.get(), second.comm.get()}); + InstallNcclComms({first.comm.get(), second.comm.get()}); const rasDiagnosticsContext ctx = UnfilteredContext(); rasDiagnosticsLocalData data{}; const size_t payloadSize = 1000000000; @@ -341,7 +301,7 @@ TEST_F(RasDiagnosticsCommonMicrotest, CollectAcceptsRecordBytesExactlyAtIntMaxLi TEST_F(RasDiagnosticsCommonMicrotest, CollectReturnsEmptyForNullInvalidAndFilteredComms) { OwnedComm invalid(10, 20, 30, 0, false); OwnedComm mismatch(11, 20, 30, 1); - InstallComms({nullptr, invalid.comm.get(), mismatch.comm.get()}); + InstallNcclComms({nullptr, invalid.comm.get(), mismatch.comm.get()}); const rasDiagnosticsContext ctx = FilteredContext(10, 20, 30); rasDiagnosticsLocalData data{reinterpret_cast(1), 2, 3, 4}; @@ -363,7 +323,7 @@ TEST_F(RasDiagnosticsCommonMicrotest, CollectFiltersCommsAndBuildsAlignedRecords first.comm->nRanks = 0x55667788; second.comm->rank = 0x22334455; second.comm->nRanks = 0x66778899; - InstallComms({first.comm.get(), nullptr, invalid.comm.get(), mismatch.comm.get(), second.comm.get(), nullptr}); + InstallNcclComms({first.comm.get(), nullptr, invalid.comm.get(), mismatch.comm.get(), second.comm.get(), nullptr}); const rasDiagnosticsContext ctx = FilteredContext(10, 20, 30); rasDiagnosticsLocalData data{}; @@ -407,7 +367,7 @@ TEST_F(RasDiagnosticsCommonMicrotest, CollectFiltersCommsAndBuildsAlignedRecords TEST_F(RasDiagnosticsCommonMicrotest, CollectPropagatesSnapshotAllocationFailure) { OwnedComm owned(1, 2, 3, 0); - InstallComms({owned.comm.get()}); + InstallNcclComms({owned.comm.get()}); const rasDiagnosticsContext ctx = UnfilteredContext(); rasDiagnosticsLocalData data{}; g_failAllocationCall = 1; @@ -421,7 +381,7 @@ TEST_F(RasDiagnosticsCommonMicrotest, CollectPropagatesSnapshotAllocationFailure TEST_F(RasDiagnosticsCommonMicrotest, CollectPropagatesRecordAllocationFailure) { OwnedComm owned(1, 2, 3, 0); - InstallComms({owned.comm.get()}); + InstallNcclComms({owned.comm.get()}); const rasDiagnosticsContext ctx = UnfilteredContext(); rasDiagnosticsLocalData data{}; g_failAllocationCall = 2; @@ -435,7 +395,7 @@ TEST_F(RasDiagnosticsCommonMicrotest, CollectPropagatesRecordAllocationFailure) TEST_F(RasDiagnosticsCommonMicrotest, CollectPropagatesFillFailureWithoutPublishingPartialData) { OwnedComm owned(1, 2, 3, 0); - InstallComms({owned.comm.get()}); + InstallNcclComms({owned.comm.get()}); const rasDiagnosticsContext ctx = UnfilteredContext(); rasDiagnosticsLocalData data{}; g_failFillCall = 1; diff --git a/projects/rccl/test/host/ras-diagnostics-env-test.cc b/projects/rccl/test/host/ras-diagnostics-env-test.cc index b904a272b787..8cbc301a0dcf 100644 --- a/projects/rccl/test/host/ras-diagnostics-env-test.cc +++ b/projects/rccl/test/host/ras-diagnostics-env-test.cc @@ -12,38 +12,30 @@ #include #include #include +#include #include #include "comm.h" +#include "fakes/env_fakes.h" +#include "fakes/ras_registry_test_support.h" #include "ras/diagnostics_checks_common.h" namespace { int g_allocationCalls; int g_failAllocationCall; -std::vector g_allocationCounts; - -template -ncclResult_t DiagnosticsEnvTestCalloc(T** ptr, size_t count) { +template +ncclResult_t DiagnosticsEnvTestCalloc(const char* file, int line, const char* function, Args&&... args) { ++g_allocationCalls; - g_allocationCounts.push_back(count); if (g_allocationCalls == g_failAllocationCall) return ncclSystemError; - return ncclCallocDebug(ptr, count, __FILE__, __LINE__, __func__, false); -} - -template -ncclResult_t DiagnosticsEnvTestCalloc(ncclUniquePtr& ptr, size_t count) { - ++g_allocationCalls; - g_allocationCounts.push_back(count); - if (g_allocationCalls == g_failAllocationCall) return ncclSystemError; - return ncclCallocDebug(ptr, count, __FILE__, __LINE__, __func__, false); + return ncclCallocDebug(std::forward(args)..., file, line, function, false); } } // namespace #undef ncclCalloc -#define ncclCalloc(...) DiagnosticsEnvTestCalloc(__VA_ARGS__) +#define ncclCalloc(...) DiagnosticsEnvTestCalloc(__FILE__, __LINE__, __func__, __VA_ARGS__) #include RAS_DIAGNOSTICS_ENV_CC_PATH @@ -122,50 +114,26 @@ struct FakeComm { ncclComm* get() { return comm.get(); } }; -void InstallNcclComms(std::vector comms) { - free(ncclComms); - nNcclComms = static_cast(comms.size()); - ncclComms = static_cast(calloc(comms.size() ? comms.size() : 1, sizeof(*ncclComms))); - ASSERT_NE(nullptr, ncclComms); - for (size_t i = 0; i < comms.size(); i++) ncclComms[i] = comms[i]; -} - -// --------- Fake `environ` plumbing --------- - -char** g_realEnviron; -std::vector g_envStorage; -std::vector g_envPointers; - -void SetFakeEnviron(const std::vector& entries) { - g_envStorage = entries; - g_envPointers.clear(); - for (auto& s : g_envStorage) g_envPointers.push_back(const_cast(s.c_str())); - g_envPointers.push_back(nullptr); - environ = g_envPointers.data(); -} +using ras_test::InstallNcclComms; void ResetWholeFileSeams() { g_allocationCalls = 0; g_failAllocationCall = 0; - g_allocationCounts.clear(); g_emitCalls = 0; g_emitFailAt = 0; g_emittedLines.clear(); - free(ncclComms); - ncclComms = nullptr; - nNcclComms = 0; - SetFakeEnviron({}); + ras_test::ResetNcclComms(); + ResetEnvFakes(); } class RasDiagnosticsEnvMicrotest : public ::testing::Test { protected: void SetUp() override { - g_realEnviron = environ; ResetWholeFileSeams(); + SetMicroEnviron({}); } void TearDown() override { ResetWholeFileSeams(); - environ = g_realEnviron; } }; @@ -175,21 +143,72 @@ class RasDiagnosticsEnvMicrotest : public ::testing::Test { // rasDiagnosticsNcclEnvCollectLocal (through the shared microtest copy of rasDiagnosticsCollectLocalRecords) // =========================================================================== +TEST_F(RasDiagnosticsEnvMicrotest, EnvironmentViews_EnumerationAndLookupStayInSync) { + SetMicroEnviron({"NCCL_DEBUG=INFO", "PATH=/scripted"}); + EXPECT_STREQ("INFO", micro_getenv("NCCL_DEBUG")); + EXPECT_STREQ("INFO", std::getenv("NCCL_DEBUG")); + ASSERT_NE(nullptr, environ[0]); + EXPECT_STREQ("NCCL_DEBUG=INFO", environ[0]); + + SetMicroEnv("NCCL_DEBUG", "WARN"); + EXPECT_STREQ("WARN", micro_getenv("NCCL_DEBUG")); + EXPECT_STREQ("WARN", std::getenv("NCCL_DEBUG")); + EXPECT_STREQ("NCCL_DEBUG=WARN", environ[0]); + SetMicroEnv("NCCL_NEW", "1"); + EXPECT_STREQ("NCCL_NEW=1", environ[2]); + EXPECT_STREQ("1", micro_getenv("NCCL_NEW")); + + SetMicroEnvAbsent("NCCL_DEBUG"); + EXPECT_EQ(nullptr, micro_getenv("NCCL_DEBUG")); + EXPECT_EQ(nullptr, std::getenv("NCCL_DEBUG")); + EXPECT_STREQ("PATH=/scripted", environ[0]); + EXPECT_STREQ("NCCL_NEW=1", environ[1]); + EXPECT_EQ(nullptr, environ[2]); +} + +TEST_F(RasDiagnosticsEnvMicrotest, EnvironmentViews_ResetRestoresOriginalEnviron) { + ResetEnvFakes(); + char** original = environ; + SetMicroEnviron({"NCCL_DEBUG=INFO"}); + SetMicroEnviron({"NCCL_DEBUG=WARN"}); + ResetEnvFakes(); + EXPECT_EQ(original, environ); + EXPECT_EQ(nullptr, micro_getenv("NCCL_DEBUG")); +} + +TEST_F(RasDiagnosticsEnvMicrotest, EnvironmentViews_DuplicateUpdatesPreserveOtherEntries) { + SetMicroEnviron({"NCCL_NO_EQUALS", "NCCL_DEBUG=INFO", "NCCL_DEBUG=WARN", "NCCL_OTHER=a=b"}); + EXPECT_STREQ("INFO", micro_getenv("NCCL_DEBUG")); + EXPECT_STREQ("INFO", std::getenv("NCCL_DEBUG")); + EXPECT_STREQ("a=b", micro_getenv("NCCL_OTHER")); + SetMicroEnv("NCCL_DEBUG", "TRACE"); + EXPECT_STREQ("NCCL_NO_EQUALS", environ[0]); + EXPECT_STREQ("NCCL_DEBUG=TRACE", environ[1]); + EXPECT_STREQ("NCCL_OTHER=a=b", environ[2]); + EXPECT_EQ(nullptr, environ[3]); + EXPECT_STREQ("TRACE", micro_getenv("NCCL_DEBUG")); + EXPECT_STREQ("TRACE", std::getenv("NCCL_DEBUG")); +} + TEST_F(RasDiagnosticsEnvMicrotest, CollectLocal_NoMatchingCommsReturnsEmpty) { struct rasDiagnosticsContext ctx{}; struct rasDiagnosticsLocalData data{}; - ASSERT_EQ(ncclSuccess, rasDiagnosticsNcclEnvCollectLocal(&ctx, &data)); + const auto result = rasDiagnosticsNcclEnvCollectLocal(&ctx, &data); + ncclUniquePtr recordsOwner(data.records); + ASSERT_EQ(ncclSuccess, result); EXPECT_EQ(0, data.nRecords); } TEST_F(RasDiagnosticsEnvMicrotest, CollectLocal_FiltersNonNcclPrefixedVars) { FakeComm fc(0x1000, 0, 1); InstallNcclComms({fc.get()}); - SetFakeEnviron({"PATH=/usr/bin", "NCCL_DEBUG=INFO", "NCCLFOO=1", "HOME=/root"}); + SetMicroEnviron({"PATH=/usr/bin", "NCCL_DEBUG=INFO", "NCCLFOO=1", "HOME=/root"}); struct rasDiagnosticsContext ctx{}; struct rasDiagnosticsLocalData data{}; - ASSERT_EQ(ncclSuccess, rasDiagnosticsNcclEnvCollectLocal(&ctx, &data)); + const auto result = rasDiagnosticsNcclEnvCollectLocal(&ctx, &data); + ncclUniquePtr recordsOwner(data.records); + ASSERT_EQ(ncclSuccess, result); ASSERT_EQ(1, data.nRecords); const auto* envData = reinterpret_cast( data.records + sizeof(struct rasDiagnosticsRankHeader)); @@ -198,22 +217,22 @@ TEST_F(RasDiagnosticsEnvMicrotest, CollectLocal_FiltersNonNcclPrefixedVars) { EXPECT_NE(std::string::npos, blob.find("NCCL_DEBUG=INFO")); EXPECT_EQ(std::string::npos, blob.find("NCCLFOO")); EXPECT_EQ(std::string::npos, blob.find("PATH=")); - free(data.records); } TEST_F(RasDiagnosticsEnvMicrotest, CollectLocal_NoNcclVarsProducesEmptyPayload) { FakeComm fc(0x1000, 0, 1); InstallNcclComms({fc.get()}); - SetFakeEnviron({"PATH=/usr/bin"}); + SetMicroEnviron({"PATH=/usr/bin"}); struct rasDiagnosticsContext ctx{}; struct rasDiagnosticsLocalData data{}; - ASSERT_EQ(ncclSuccess, rasDiagnosticsNcclEnvCollectLocal(&ctx, &data)); + const auto result = rasDiagnosticsNcclEnvCollectLocal(&ctx, &data); + ncclUniquePtr recordsOwner(data.records); + ASSERT_EQ(ncclSuccess, result); ASSERT_EQ(1, data.nRecords); const auto* envData = reinterpret_cast( data.records + sizeof(struct rasDiagnosticsRankHeader)); EXPECT_EQ(0, envData->bytesUsed); - free(data.records); } TEST_F(RasDiagnosticsEnvMicrotest, CollectLocal_ExceedingByteBudgetSetsTruncated) { @@ -222,18 +241,19 @@ TEST_F(RasDiagnosticsEnvMicrotest, CollectLocal_ExceedingByteBudgetSetsTruncated std::vector entries; // Each ~1040 bytes; ~17 of them exceed the 16384-byte budget. for (int i = 0; i < 17; i++) entries.push_back("NCCL_VAR" + std::to_string(i) + "=" + std::string(1024, 'a')); - SetFakeEnviron(entries); + SetMicroEnviron(entries); struct rasDiagnosticsContext ctx{}; struct rasDiagnosticsLocalData data{}; - ASSERT_EQ(ncclSuccess, rasDiagnosticsNcclEnvCollectLocal(&ctx, &data)); + const auto result = rasDiagnosticsNcclEnvCollectLocal(&ctx, &data); + ncclUniquePtr recordsOwner(data.records); + ASSERT_EQ(ncclSuccess, result); ASSERT_EQ(1, data.nRecords); const auto* envData = reinterpret_cast( data.records + sizeof(struct rasDiagnosticsRankHeader)); EXPECT_EQ(1, envData->truncated); EXPECT_EQ(15532, envData->bytesUsed); EXPECT_EQ(0u, std::string(envData->data, envData->bytesUsed).find("NCCL_VAR0=")); - free(data.records); } TEST_F(RasDiagnosticsEnvMicrotest, CollectLocal_ExactFitAtTheByteBudgetIsNotTruncated) { @@ -243,17 +263,18 @@ TEST_F(RasDiagnosticsEnvMicrotest, CollectLocal_ExactFitAtTheByteBudgetIsNotTrun // including the NUL) exactly equals the remaining budget -- fits exactly, at the // boundary between "fits" and "doesn't fit". std::string value(RAS_DIAG_ENV_BYTES - 1 - 7, 'a'); - SetFakeEnviron({"NCCL_X=" + value}); + SetMicroEnviron({"NCCL_X=" + value}); struct rasDiagnosticsContext ctx{}; struct rasDiagnosticsLocalData data{}; - ASSERT_EQ(ncclSuccess, rasDiagnosticsNcclEnvCollectLocal(&ctx, &data)); + const auto result = rasDiagnosticsNcclEnvCollectLocal(&ctx, &data); + ncclUniquePtr recordsOwner(data.records); + ASSERT_EQ(ncclSuccess, result); ASSERT_EQ(1, data.nRecords); const auto* envData = reinterpret_cast( data.records + sizeof(struct rasDiagnosticsRankHeader)); EXPECT_EQ(0, envData->truncated); EXPECT_EQ(RAS_DIAG_ENV_BYTES, envData->bytesUsed); - free(data.records); } // =========================================================================== @@ -285,6 +306,11 @@ TEST_F(RasDiagnosticsEnvMicrotest, Summarize_SizeNotMultipleOfStrideReturnsInter auto reporter = MakeRecordingReporter(); char buf[4] = {}; EXPECT_EQ(ncclInternalError, rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, buf, 1)); +} + +TEST_F(RasDiagnosticsEnvMicrotest, Summarize_NegativeSizeReturnsInternalError) { + auto reporter = MakeRecordingReporter(); + char buf[4] = {}; EXPECT_EQ(ncclInternalError, rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, buf, -static_cast(EnvRecordStride()))); } @@ -302,9 +328,9 @@ ncclResult_t CollectThenSummarize(struct rasDiagnosticsReporter* reporter) { struct rasDiagnosticsContext ctx{}; struct rasDiagnosticsLocalData data{}; ncclResult_t ret = rasDiagnosticsNcclEnvCollectLocal(&ctx, &data); + ncclUniquePtr recordsOwner(data.records); if (ret != ncclSuccess) return ret; ret = rasDiagnosticsNcclEnvSummarize(&ctx, reporter, data.records, data.recordsBytes); - free(data.records); return ret; } @@ -313,7 +339,7 @@ ncclResult_t CollectThenSummarize(struct rasDiagnosticsReporter* reporter) { TEST_F(RasDiagnosticsEnvMicrotest, Summarize_SingleRankConsistentReport) { FakeComm fc(0x2000, 0, 1); InstallNcclComms({fc.get()}); - SetFakeEnviron({"NCCL_A_NO_EQUALS", "NCCL_DEBUG=INFO"}); + SetMicroEnviron({"NCCL_A_NO_EQUALS", "NCCL_DEBUG=INFO"}); auto reporter = MakeRecordingReporter(); ASSERT_EQ(ncclSuccess, CollectThenSummarize(&reporter)); @@ -342,7 +368,7 @@ TEST_F(RasDiagnosticsEnvMicrotest, Summarize_IncompleteGroupReportsIncomplete) { // commNRanks says 2, but only rank 0 is present in ncclComms -- an incomplete gather. FakeComm fc(0x3000, 0, 2); InstallNcclComms({fc.get()}); - SetFakeEnviron({}); + SetMicroEnviron({}); auto reporter = MakeRecordingReporter(); ASSERT_EQ(ncclSuccess, CollectThenSummarize(&reporter)); @@ -413,9 +439,6 @@ TEST_F(RasDiagnosticsEnvMicrotest, Summarize_TwoDistinctKeysAreSortedAndDeduplic EXPECT_EQ("[INFO] NCCL environment: NCCL_DEBUG_SUBSYS=COLL on rank(s) {1}", g_emittedLines[5]); EXPECT_EQ("[INFO] NCCL environment: 2 NCCL_* env var(s) differ across ranks in comm 0x4150", g_emittedLines[6]); EXPECT_FALSE(AnyLineContains(g_emittedLines, "consistent")); - ASSERT_GE(g_allocationCounts.size(), 2u); - EXPECT_EQ(buf.size(), g_allocationCounts[0]); - EXPECT_EQ(4u, g_allocationCounts[1]); } TEST_F(RasDiagnosticsEnvMicrotest, Summarize_AllocationFailuresPropagate) { @@ -425,10 +448,12 @@ TEST_F(RasDiagnosticsEnvMicrotest, Summarize_AllocationFailuresPropagate) { FillEnvRecord(buf, 1, {0x4175, 0, 0}, 1, 2, {"NCCL_DEBUG=WARN"}); auto reporter = MakeRecordingReporter(); - for (int failAt = 1; failAt <= 5; failAt++) { + ASSERT_EQ(ncclSuccess, rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, buf.data(), static_cast(buf.size()))); + const int allocationCount = g_allocationCalls; + ASSERT_GT(allocationCount, 0); + for (int failAt = 1; failAt <= allocationCount; failAt++) { SCOPED_TRACE(failAt); g_allocationCalls = 0; - g_allocationCounts.clear(); g_failAllocationCall = failAt; EXPECT_EQ(ncclSystemError, rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, buf.data(), static_cast(buf.size()))) @@ -439,7 +464,7 @@ TEST_F(RasDiagnosticsEnvMicrotest, Summarize_AllocationFailuresPropagate) { TEST_F(RasDiagnosticsEnvMicrotest, CollectLocal_EntryArrayAllocationFailurePropagates) { FakeComm fc(0x4180, 0, 1); InstallNcclComms({fc.get()}); - SetFakeEnviron({"NCCL_DEBUG=INFO"}); + SetMicroEnviron({"NCCL_DEBUG=INFO"}); struct rasDiagnosticsContext ctx{}; struct rasDiagnosticsLocalData data{}; g_allocationCalls = 0; @@ -450,7 +475,7 @@ TEST_F(RasDiagnosticsEnvMicrotest, CollectLocal_EntryArrayAllocationFailurePropa EXPECT_EQ(nullptr, data.records); } -TEST_F(RasDiagnosticsEnvMicrotest, Summarize_ReporterFailuresPropagate) { +TEST_F(RasDiagnosticsEnvMicrotest, Summarize_MismatchReporterFailuresPropagate) { const size_t stride = EnvRecordStride(); std::vector mismatch(stride * 2, 0); FillEnvRecord(mismatch, 0, {0x4200, 0, 0}, 0, 2, {"NCCL_DEBUG=INFO"}); @@ -467,27 +492,6 @@ TEST_F(RasDiagnosticsEnvMicrotest, Summarize_ReporterFailuresPropagate) { EXPECT_EQ(failAt, g_emitCalls); } - std::vector single(stride, 0); - auto* envData = FillEnvRecord(single, 0, {0x4300, 0, 0}, 0, 1); - - g_emitCalls = 0; - g_emitFailAt = 1; - EXPECT_EQ(ncclSystemError, - rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, single.data(), static_cast(single.size()))); - EXPECT_EQ(1, g_emitCalls); - - envData->truncated = 1; - g_emitCalls = 0; - EXPECT_EQ(ncclSystemError, - rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, single.data(), static_cast(single.size()))); - EXPECT_EQ(1, g_emitCalls); - - reinterpret_cast(single.data())->commNRanks = 2; - envData->truncated = 0; - g_emitCalls = 0; - EXPECT_EQ(ncclSystemError, - rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, single.data(), static_cast(single.size()))); - EXPECT_EQ(1, g_emitCalls); } TEST_F(RasDiagnosticsEnvMicrotest, Summarize_ValueSetOnOneRankUnsetOnAnotherIsAMismatch) { @@ -510,7 +514,7 @@ TEST_F(RasDiagnosticsEnvMicrotest, Summarize_TruncatedRankReportsTruncationWarni auto reporter = MakeRecordingReporter(); ASSERT_EQ(ncclSuccess, rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, buf.data(), static_cast(buf.size()))); - EXPECT_TRUE(AnyLineContains(g_emittedLines, "1 rank(s) had >16384 bytes")); + EXPECT_TRUE(AnyLineContains(g_emittedLines, "1 rank(s) had >" + std::to_string(RAS_DIAG_ENV_BYTES) + " bytes")); EXPECT_FALSE(AnyLineContains(g_emittedLines, "consistent across")); } @@ -541,17 +545,32 @@ TEST_F(RasDiagnosticsEnvMicrotest, Summarize_TwoDistinctCommsAreReportedSeparate auto reporter = MakeRecordingReporter(); ASSERT_EQ(ncclSuccess, rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, buf.data(), static_cast(buf.size()))); - const std::vector expectedLines = { - "[OK] NCCL environment: NCCL_* env vars consistent across 1 ranks in comm 0x8000", - "[INFO] NCCL environment: mismatch across 2 ranks in comm 0x8000 for NCCL_DEBUG", - "[INFO] NCCL environment: NCCL_DEBUG=INFO on rank(s) {0}", - "[INFO] NCCL environment: NCCL_DEBUG=WARN on rank(s) {1}", - "[INFO] NCCL environment: 1 NCCL_* env var(s) differ across ranks in comm 0x8000", - "[OK] NCCL environment: NCCL_* env vars consistent across 1 ranks in comm 0x9000", - }; - EXPECT_EQ(expectedLines, g_emittedLines); - const std::vector expectedAllocationCounts = {buf.size(), 1, 1, 2, 2, 2, 2}; - EXPECT_EQ(expectedAllocationCounts, g_allocationCounts); + ASSERT_EQ(6u, g_emittedLines.size()); + EXPECT_EQ(0u, g_emittedLines[0].find("[OK] NCCL environment: NCCL_* env vars consistent across 1 ranks in comm 0x8000")); + EXPECT_EQ(0u, g_emittedLines[1].find("[INFO] NCCL environment: mismatch across 2 ranks in comm 0x8000")); + EXPECT_NE(std::string::npos, g_emittedLines[1].find(" for NCCL_DEBUG")); + EXPECT_EQ("[INFO] NCCL environment: NCCL_DEBUG=INFO on rank(s) {0}", g_emittedLines[2]); + EXPECT_EQ("[INFO] NCCL environment: NCCL_DEBUG=WARN on rank(s) {1}", g_emittedLines[3]); + EXPECT_EQ(0u, g_emittedLines[4].find("[INFO] NCCL environment: 1 NCCL_* env var(s) differ across ranks in comm 0x8000")); + EXPECT_EQ(0u, g_emittedLines[5].find("[OK] NCCL environment: NCCL_* env vars consistent across 1 ranks in comm 0x9000")); +} + +TEST_F(RasDiagnosticsEnvMicrotest, DISABLED_Summarize_HashCollisionsReportFullCommIdentity) { + SCOPED_TRACE("AICOMRCCL-2743: reporters omit host and process hashes from the communicator identity"); + std::vector records(EnvRecordStride() * 3, 0); + FillEnvRecord(records, 0, {0x8000, 0x8100, 0x8200}, 0, 1, {"NCCL_FIRST=ONLY"}); + FillEnvRecord(records, 1, {0x8000, 0x8300, 0x8400}, 1, 2, {"NCCL_DEBUG=WARN"}); + FillEnvRecord(records, 2, {0x8000, 0x8300, 0x8400}, 0, 2, {"NCCL_DEBUG=INFO"}); + auto reporter = MakeRecordingReporter(); + ASSERT_EQ(ncclSuccess, + rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, records.data(), static_cast(records.size()))); + ASSERT_EQ(5u, g_emittedLines.size()); + EXPECT_EQ("[OK] NCCL environment: NCCL_* env vars consistent across 1 ranks in comm 0x8000/0x8100/0x8200", + g_emittedLines[0]); + EXPECT_EQ("[INFO] NCCL environment: mismatch across 2 ranks in comm 0x8000/0x8300/0x8400 for NCCL_DEBUG", + g_emittedLines[1]); + EXPECT_EQ("[INFO] NCCL environment: 1 NCCL_* env var(s) differ across ranks in comm 0x8000/0x8300/0x8400", + g_emittedLines[4]); } TEST_F(RasDiagnosticsEnvMicrotest, Summarize_LongKeyAndValueAreTruncatedWithEllipsis) { @@ -592,3 +611,33 @@ TEST_F(RasDiagnosticsEnvMicrotest, Summarize_ControlCharactersAreSanitizedToQues EXPECT_TRUE(AnyLineContains(g_emittedLines, "NCCL_B?KEY=a?b")); EXPECT_TRUE(AnyLineContains(g_emittedLines, "NCCL_X=a?b")); } + +TEST_F(RasDiagnosticsEnvMicrotest, Summarize_ConsistentReporterFailurePropagates) { + std::vector records(EnvRecordStride(), 0); + FillEnvRecord(records, 0, {0x4300, 0, 0}, 0, 1, {}, false); + auto reporter = MakeRecordingReporter(); + g_emitFailAt = 1; + EXPECT_EQ(ncclSystemError, + rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, records.data(), static_cast(records.size()))); + EXPECT_EQ(1, g_emitCalls); +} + +TEST_F(RasDiagnosticsEnvMicrotest, Summarize_TruncatedReporterFailurePropagates) { + std::vector records(EnvRecordStride(), 0); + FillEnvRecord(records, 0, {0x4300, 0, 0}, 0, 1, {}, true); + auto reporter = MakeRecordingReporter(); + g_emitFailAt = 1; + EXPECT_EQ(ncclSystemError, + rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, records.data(), static_cast(records.size()))); + EXPECT_EQ(1, g_emitCalls); +} + +TEST_F(RasDiagnosticsEnvMicrotest, Summarize_IncompleteReporterFailurePropagates) { + std::vector records(EnvRecordStride(), 0); + FillEnvRecord(records, 0, {0x4300, 0, 0}, 0, 2, {}, false); + auto reporter = MakeRecordingReporter(); + g_emitFailAt = 1; + EXPECT_EQ(ncclSystemError, + rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, records.data(), static_cast(records.size()))); + EXPECT_EQ(1, g_emitCalls); +} diff --git a/projects/rccl/test/host/ras-test.cc b/projects/rccl/test/host/ras-test.cc index eb2e025bd8af..369b38b7885f 100644 --- a/projects/rccl/test/host/ras-test.cc +++ b/projects/rccl/test/host/ras-test.cc @@ -23,6 +23,7 @@ #include #include "comm.h" +#include "fakes/ras_registry_test_support.h" #include "fakes/nccl_fakes.h" #include "fakes/param_redirect.h" #include "fakes/signature-drift.h" @@ -146,10 +147,7 @@ class RasMicrotest : public ::testing::Test { rasInitialized = false; rasInitRefCount = 0; rasNotificationPipe[0] = rasNotificationPipe[1] = NCCL_SOCKET_PAIR_INVALID; - std::free(ncclComms); - ncclComms = nullptr; - nNcclComms = 0; - ncclCommsSorted = false; + ras_test::ResetNcclComms(); std::free(rasPfds); rasPfds = nullptr; nRasPfds = 0; @@ -161,9 +159,7 @@ class RasMicrotest : public ::testing::Test { std::free(rasPfds); rasPfds = nullptr; nRasPfds = 0; - std::free(ncclComms); - ncclComms = nullptr; - nNcclComms = 0; + ras_test::ResetNcclComms(); rasInitialized = false; rasInitRefCount = 0; g_socketProgress = DefaultSocketProgress; @@ -197,11 +193,9 @@ TEST_F(RasMicrotest, CommInitAlreadyInitializedAcceptsNullRank) { TEST_F(RasMicrotest, CommInitReusesVacantCommSlot) { rasInitialized = true; - nNcclComms = 2; - ncclComms = static_cast(std::calloc(2, sizeof(*ncclComms))); auto incumbent = std::make_unique(); auto newcomer = std::make_unique(); - ncclComms[0] = incumbent.get(); + ras_test::InstallNcclComms({incumbent.get(), nullptr}); rasRankInit rank{}; ASSERT_EQ(ncclSuccess, ncclRasCommInit(newcomer.get(), &rank)); EXPECT_EQ(newcomer.get(), ncclComms[1]); @@ -210,13 +204,10 @@ TEST_F(RasMicrotest, CommInitReusesVacantCommSlot) { TEST_F(RasMicrotest, CommInitGrowthPreservesRegisteredComms) { rasInitialized = true; - nNcclComms = 2; - ncclComms = static_cast(std::calloc(2, sizeof(*ncclComms))); auto first = std::make_unique(); auto second = std::make_unique(); auto newcomer = std::make_unique(); - ncclComms[0] = first.get(); - ncclComms[1] = second.get(); + ras_test::InstallNcclComms({first.get(), second.get()}); rasRankInit rank{}; ASSERT_EQ(ncclSuccess, ncclRasCommInit(newcomer.get(), &rank)); EXPECT_EQ(first.get(), ncclComms[0]); @@ -283,12 +274,9 @@ TEST_F(RasMicrotest, CommFiniUninitializedIsNoOp) { TEST_F(RasMicrotest, CommFiniRemovesMatchingCommAndDropsReference) { rasInitialized = true; rasInitRefCount = 1; - nNcclComms = 2; - ncclComms = static_cast(std::calloc(2, sizeof(*ncclComms))); auto first = std::make_unique(); auto second = std::make_unique(); - ncclComms[0] = first.get(); - ncclComms[1] = second.get(); + ras_test::InstallNcclComms({first.get(), second.get()}); ncclCommsSorted = true; EXPECT_EQ(ncclSuccess, ncclRasCommFini(second.get())); EXPECT_EQ(first.get(), ncclComms[0]); @@ -386,8 +374,7 @@ TEST_F(RasMicrotest, LocalHandleRunsDiagnosticsAndIgnoresHandlerFailure) { TEST_F(RasMicrotest, ThreadCleanupResetsAllGlobalState) { rasInitialized = true; rasInitRefCount = 3; - nNcclComms = 1; - ncclComms = static_cast(std::calloc(1, sizeof(*ncclComms))); + ras_test::InstallNcclComms({nullptr}); InitPollFds(1); rasThreadCleanup(); for (int calls : g_cleanupCalls) EXPECT_EQ(1, calls);