diff --git a/projects/rccl/test/host/CMakeLists.txt b/projects/rccl/test/host/CMakeLists.txt index a8070aca2be2..ff575edd7f12 100644 --- a/projects/rccl/test/host/CMakeLists.txt +++ b/projects/rccl/test/host/CMakeLists.txt @@ -79,6 +79,7 @@ function(rccl_define_micro_source_lists) ras-diagnostics-common-test.cc # src/ras/diagnostics_checks_common.cc ras-collectives-test.cc # src/ras/collectives.cc ras-diagnostics-test.cc # src/ras/diagnostics.cc + ras-diagnostics-env-test.cc # src/ras/diagnostics_env.cc ras-param-test.cc # src/ras/ras_param.cc tuning-test.cc # src/tuning/tuning.cc tuning-general-test.cc @@ -768,6 +769,7 @@ rccl_add_micro_binary(rccl-UnitTestsMicro # so add_file_unique() renames this one. Re-check after any source-list reorder. COLLECTIVES_CC_PATH="${RCCL_HOST_HIPIFY_DIR}/src/ras/collectives_tmp.cc" RAS_DIAGNOSTICS_CC_PATH="${RCCL_HOST_HIPIFY_DIR}/src/ras/diagnostics_tmp.cc" + RAS_DIAGNOSTICS_ENV_CC_PATH="${RCCL_HOST_HIPIFY_DIR}/src/ras/diagnostics_env.cc" RAS_PARAM_CC_PATH="${RCCL_HOST_HIPIFY_DIR}/src/ras/ras_param.cc" # graph/tuning.cc keeps the basename, so add_file_unique() renames this one. TUNING_CC_PATH="${RCCL_HOST_HIPIFY_DIR}/src/tuning/tuning_tmp.cc" diff --git a/projects/rccl/test/host/MICROTEST_README.md b/projects/rccl/test/host/MICROTEST_README.md index 136f6ca76693..a8126efe54ac 100644 --- a/projects/rccl/test/host/MICROTEST_README.md +++ b/projects/rccl/test/host/MICROTEST_README.md @@ -111,6 +111,10 @@ export colliding non-`static` symbols; otherwise a unit needs its own binary: before validation. Its raw-byte fixture reproduces the enum UBSan failure; enable it after the production fix. The defined `RAS_DIAG_CHECK_COUNT` sentinel remains covered by enabled dispatch and peer-payload tests. + - `ras/diagnostics_env.cc` (`RAS_DIAGNOSTICS_ENV_CC_PATH`, from + `ras-diagnostics-env-test.cc`); suite `RasDiagnosticsEnvMicrotest.*`. + Covers NCCL environment collection, filtering, truncation, aggregation, + mismatch reporting, and reporter errors. - `ras/client.cc` (`RAS_CLIENT_CC_PATH`, from `ras-client-test.cc`); suite `RasClientMicrotest.*`. With `NCCL_RAS_CLIENT` defined, `ras_internal.h` reduces to four macros, so this @@ -771,3 +775,26 @@ cmake --build build -j"$(nproc)" Disable coverage instrumentation for the standalone host-only test binaries with `-DHOST_TEST_COVERAGE=OFF`. + +### Shared RAS diagnostic fixtures + +`fakes/ras_registry_test_support.h` owns test installation/reset of the +`ras.cc` communicator registry. Both operations hold `ncclCommsMutex` and +clear `ncclCommsSorted`; installation validates allocation before replacing +the old registry and preserves explicit vacant slots. The RAS and diagnostics +suites use the same helper. `fakes/ras_diagnostics_test_support.h` shares the +owned communicator and recording reporter used by the diagnostic suites. + +`SetMicroEnviron` in `fakes/env_fakes.{h,cc}` replaces the scripted environment +for enumeration tests, preserving input order and malformed entries while +making valid entries visible through `micro_getenv` and `getenv`. While that +override is active, `SetMicroEnv` and `SetMicroEnvAbsent` also update `environ`. +`ClearMicroEnv`/`ResetEnvFakes` restore the original process environment and +clear the lookup map. Map-only tests retain the existing fallback behavior +unless they opt into the enumeration override. + +`DISABLED_Summarize_HashCollisionsReportFullCommIdentity` tracks AICOMRCCL-2743. +The environment reporters currently omit host and process hashes, so distinct +communicators can appear identical. The disabled regression specifies the full +identity for consistent, mismatch, and mismatch-summary reports. Enable it +with the separate production fix; enabled tests still pin correct grouping. diff --git a/projects/rccl/test/host/fakes/env_fakes.cc b/projects/rccl/test/host/fakes/env_fakes.cc index c4569ec55d53..cc6211602d0e 100644 --- a/projects/rccl/test/host/fakes/env_fakes.cc +++ b/projects/rccl/test/host/fakes/env_fakes.cc @@ -14,10 +14,34 @@ #include #include #include +#include +#include + +#include #include "nccl.h" namespace { +struct MicroEnvironState { + bool active = false; + char** original = nullptr; + std::vector entries; + std::vector pointers; +}; + +MicroEnvironState& microEnvironState() { + static MicroEnvironState state; + return state; +} + +void PublishMicroEnviron() { + auto& state = microEnvironState(); + state.pointers.clear(); + for (auto& entry : state.entries) state.pointers.push_back(entry.data()); + state.pointers.push_back(nullptr); + environ = state.pointers.data(); +} + // A nullopt entry means "absent". Unmapped names read as unset via micro_getenv, real via the getenv interposer. std::unordered_map>& microEnvMap() { static std::unordered_map> m; @@ -54,11 +78,53 @@ void SetMicroEnv(const char* name, const char* value) { if (name == nullptr) return; if (value == nullptr) microEnvMap()[name] = std::nullopt; else microEnvMap()[name] = value; + auto& state = microEnvironState(); + if (state.active) { + const std::string prefix = std::string(name) + "="; + std::vector entries; + bool replaced = false; + for (const auto& entry : state.entries) { + if (entry.compare(0, prefix.size(), prefix) != 0) { + entries.push_back(entry); + } else if (!replaced && value != nullptr) { + entries.push_back(prefix + value); + replaced = true; + } + } + if (!replaced && value != nullptr) entries.push_back(prefix + value); + state.entries = std::move(entries); + PublishMicroEnviron(); + } } void SetMicroEnvAbsent(const char* name) { SetMicroEnv(name, nullptr); } -void ClearMicroEnv() { microEnvMap().clear(); } +void SetMicroEnviron(const std::vector& entries) { + auto& state = microEnvironState(); + if (!state.active) state.original = environ; + state.active = true; + state.entries = entries; + auto& values = microEnvMap(); + values.clear(); + for (const auto& entry : entries) { + const auto separator = entry.find('='); + if (separator != std::string::npos) + values.emplace(entry.substr(0, separator), entry.substr(separator + 1)); + } + PublishMicroEnviron(); +} + +void ClearMicroEnv() { + auto& state = microEnvironState(); + if (state.active) { + environ = state.original; + state.active = false; + state.original = nullptr; + state.entries.clear(); + state.pointers.clear(); + } + microEnvMap().clear(); +} void ResetEnvFakes() { ClearMicroEnv(); } diff --git a/projects/rccl/test/host/fakes/env_fakes.h b/projects/rccl/test/host/fakes/env_fakes.h index 373c44cb0ee6..b05c5a8869a8 100644 --- a/projects/rccl/test/host/fakes/env_fakes.h +++ b/projects/rccl/test/host/fakes/env_fakes.h @@ -21,6 +21,9 @@ #ifndef RCCL_TEST_HOST_ENV_FAKES_H_ #define RCCL_TEST_HOST_ENV_FAKES_H_ +#include +#include + // Strict: a name the fixture has not scripted reads as unset, so no test can // depend on the ambient environment. ncclGetEnv routes here. const char* micro_getenv(const char* name); @@ -29,6 +32,7 @@ const char* micro_getenv(const char* name); // fall through to the real getenv. void SetMicroEnv(const char* name, const char* value); void SetMicroEnvAbsent(const char* name); +void SetMicroEnviron(const std::vector& entries); void ClearMicroEnv(); // Named like every other per-TU reset so a Reset*Fakes() chain reads as one list diff --git a/projects/rccl/test/host/fakes/ras_diagnostics_test_support.h b/projects/rccl/test/host/fakes/ras_diagnostics_test_support.h new file mode 100644 index 000000000000..11e12b14d5c6 --- /dev/null +++ b/projects/rccl/test/host/fakes/ras_diagnostics_test_support.h @@ -0,0 +1,55 @@ +#ifndef RCCL_TEST_HOST_RAS_DIAGNOSTICS_TEST_SUPPORT_H_ +#define RCCL_TEST_HOST_RAS_DIAGNOSTICS_TEST_SUPPORT_H_ + +#include +#include +#include + +#include "comm.h" +#include "ras/diagnostics.h" +#include "fakes/ras_registry_test_support.h" + +namespace ras_test { + +struct OwnedComm { + OwnedComm(uint64_t commHash, uint64_t hostHash, uint64_t pidHash, int rank, bool peerInfoValid = true) + : comm(std::make_unique()), peers(std::make_unique(1)) { + comm->commHash = commHash; + comm->peerInfo = peers.get(); + comm->peerInfoValid = peerInfoValid; + comm->rank = rank; + comm->nRanks = 8; + comm->cudaDev = rank + 10; + comm->nvmlDev = rank + 20; + comm->busId = 0x1000 + rank; + comm->localRank = rank % 4; + comm->localRanks = 4; + peers[0].hostHash = hostHash; + peers[0].pidHash = pidHash; + } + + std::unique_ptr comm; + std::unique_ptr peers; +}; + +struct ReporterState { + std::vector lines; + ncclResult_t emitResult = ncclSuccess; + int calls = 0; + int failCall = 0; +}; + +inline ncclResult_t CaptureReport(void* target, const char* line) { + auto* state = static_cast(target); + state->lines.emplace_back(line); + ++state->calls; + return state->failCall == 0 || state->calls == state->failCall ? state->emitResult : ncclSuccess; +} + +inline rasDiagnosticsReporter MakeReporter(ReporterState* state) { + return rasDiagnosticsReporter{CaptureReport, nullptr, state}; +} + +} + +#endif diff --git a/projects/rccl/test/host/fakes/ras_registry_test_support.h b/projects/rccl/test/host/fakes/ras_registry_test_support.h new file mode 100644 index 000000000000..fc0c5250d391 --- /dev/null +++ b/projects/rccl/test/host/fakes/ras_registry_test_support.h @@ -0,0 +1,34 @@ +#ifndef RCCL_TEST_HOST_RAS_REGISTRY_TEST_SUPPORT_H_ +#define RCCL_TEST_HOST_RAS_REGISTRY_TEST_SUPPORT_H_ + +#include +#include +#include + +#include +#include "ras/ras_internal.h" + +namespace ras_test { + +inline void InstallNcclComms(const std::vector& comms) { + std::lock_guard lock(ncclCommsMutex); + auto** replacement = static_cast(std::calloc(comms.empty() ? 1 : comms.size(), sizeof(ncclComm*))); + ASSERT_NE(nullptr, replacement); + std::free(ncclComms); + ncclComms = replacement; + nNcclComms = static_cast(comms.size()); + ncclCommsSorted = false; + for (size_t index = 0; index < comms.size(); ++index) ncclComms[index] = comms[index]; +} + +inline void ResetNcclComms() { + std::lock_guard lock(ncclCommsMutex); + std::free(ncclComms); + ncclComms = nullptr; + nNcclComms = 0; + ncclCommsSorted = false; +} + +} + +#endif diff --git a/projects/rccl/test/host/ras-diagnostics-common-test.cc b/projects/rccl/test/host/ras-diagnostics-common-test.cc index 7bd26f194f67..e330a1e55883 100644 --- a/projects/rccl/test/host/ras-diagnostics-common-test.cc +++ b/projects/rccl/test/host/ras-diagnostics-common-test.cc @@ -22,6 +22,7 @@ #include "alloc.h" #include "checks.h" #include "comm.h" +#include "fakes/ras_diagnostics_test_support.h" #include "compiler.h" #include "ras/diagnostics_checks_common.h" #include "ras/ras_internal.h" @@ -58,26 +59,10 @@ struct TestPayload { uint32_t marker; }; -struct OwnedComm { - OwnedComm(uint64_t commHash, uint64_t hostHash, uint64_t pidHash, int rank, bool peerInfoValid = true) - : comm(std::make_unique()), peers(std::make_unique(1)) { - comm->commHash = commHash; - comm->peerInfo = peers.get(); - comm->peerInfoValid = peerInfoValid; - comm->rank = rank; - comm->nRanks = 8; - comm->cudaDev = rank + 10; - comm->nvmlDev = rank + 20; - comm->busId = 0x1000 + rank; - comm->localRank = rank % 4; - comm->localRanks = 4; - peers[0].hostHash = hostHash; - peers[0].pidHash = pidHash; - } - - std::unique_ptr comm; - std::unique_ptr peers; -}; +using ras_test::OwnedComm; +using ras_test::ReporterState; +using ras_test::CaptureReport; +using ras_test::InstallNcclComms; std::vector g_fillSnapshots; int g_fillCalls = 0; @@ -102,23 +87,8 @@ ncclResult_t FillTestPayload(const rasDiagnosticsCommSnapshot* snapshot, void* c return ncclSuccess; } -struct ReporterState { - std::vector lines; - ncclResult_t emitResult = ncclSuccess; -}; - -ncclResult_t CaptureReport(void* target, const char* line) { - auto* state = static_cast(target); - state->lines.emplace_back(line); - return state->emitResult; -} - void ResetDiagnosticsCommonState() { - std::lock_guard lock(ncclCommsMutex); - std::free(ncclComms); - ncclComms = nullptr; - nNcclComms = 0; - ncclCommsSorted = false; + ras_test::ResetNcclComms(); g_allocationCalls = 0; g_failAllocationCall = 0; g_allocationCounts.clear(); @@ -128,16 +98,6 @@ void ResetDiagnosticsCommonState() { g_fillObservedUnlockedMutex = true; } -void InstallComms(std::initializer_list comms) { - std::lock_guard lock(ncclCommsMutex); - std::free(ncclComms); - ncclComms = static_cast(std::calloc(comms.size(), sizeof(*ncclComms))); - ASSERT_NE(nullptr, ncclComms); - nNcclComms = static_cast(comms.size()); - int index = 0; - for (ncclComm* comm : comms) ncclComms[index++] = comm; -} - rasDiagnosticsContext UnfilteredContext() { rasDiagnosticsContext ctx{}; ctx.hasCommFilter = false; @@ -307,7 +267,7 @@ TEST_F(RasDiagnosticsCommonMicrotest, CollectRejectsStrideRoundedPastIntMax) { TEST_F(RasDiagnosticsCommonMicrotest, CollectRejectsCombinedRecordBytesPastIntMax) { OwnedComm first(1, 2, 3, 0); OwnedComm second(4, 5, 6, 1); - InstallComms({first.comm.get(), second.comm.get()}); + InstallNcclComms({first.comm.get(), second.comm.get()}); const rasDiagnosticsContext ctx = UnfilteredContext(); rasDiagnosticsLocalData data{}; const size_t payloadSize = static_cast(INT_MAX) / 2; @@ -323,7 +283,7 @@ TEST_F(RasDiagnosticsCommonMicrotest, CollectRejectsCombinedRecordBytesPastIntMa TEST_F(RasDiagnosticsCommonMicrotest, CollectAcceptsRecordBytesExactlyAtIntMaxLimit) { OwnedComm first(1, 2, 3, 0); OwnedComm second(4, 5, 6, 1); - InstallComms({first.comm.get(), second.comm.get()}); + InstallNcclComms({first.comm.get(), second.comm.get()}); const rasDiagnosticsContext ctx = UnfilteredContext(); rasDiagnosticsLocalData data{}; const size_t payloadSize = 1000000000; @@ -341,7 +301,7 @@ TEST_F(RasDiagnosticsCommonMicrotest, CollectAcceptsRecordBytesExactlyAtIntMaxLi TEST_F(RasDiagnosticsCommonMicrotest, CollectReturnsEmptyForNullInvalidAndFilteredComms) { OwnedComm invalid(10, 20, 30, 0, false); OwnedComm mismatch(11, 20, 30, 1); - InstallComms({nullptr, invalid.comm.get(), mismatch.comm.get()}); + InstallNcclComms({nullptr, invalid.comm.get(), mismatch.comm.get()}); const rasDiagnosticsContext ctx = FilteredContext(10, 20, 30); rasDiagnosticsLocalData data{reinterpret_cast(1), 2, 3, 4}; @@ -363,7 +323,7 @@ TEST_F(RasDiagnosticsCommonMicrotest, CollectFiltersCommsAndBuildsAlignedRecords first.comm->nRanks = 0x55667788; second.comm->rank = 0x22334455; second.comm->nRanks = 0x66778899; - InstallComms({first.comm.get(), nullptr, invalid.comm.get(), mismatch.comm.get(), second.comm.get(), nullptr}); + InstallNcclComms({first.comm.get(), nullptr, invalid.comm.get(), mismatch.comm.get(), second.comm.get(), nullptr}); const rasDiagnosticsContext ctx = FilteredContext(10, 20, 30); rasDiagnosticsLocalData data{}; @@ -407,7 +367,7 @@ TEST_F(RasDiagnosticsCommonMicrotest, CollectFiltersCommsAndBuildsAlignedRecords TEST_F(RasDiagnosticsCommonMicrotest, CollectPropagatesSnapshotAllocationFailure) { OwnedComm owned(1, 2, 3, 0); - InstallComms({owned.comm.get()}); + InstallNcclComms({owned.comm.get()}); const rasDiagnosticsContext ctx = UnfilteredContext(); rasDiagnosticsLocalData data{}; g_failAllocationCall = 1; @@ -421,7 +381,7 @@ TEST_F(RasDiagnosticsCommonMicrotest, CollectPropagatesSnapshotAllocationFailure TEST_F(RasDiagnosticsCommonMicrotest, CollectPropagatesRecordAllocationFailure) { OwnedComm owned(1, 2, 3, 0); - InstallComms({owned.comm.get()}); + InstallNcclComms({owned.comm.get()}); const rasDiagnosticsContext ctx = UnfilteredContext(); rasDiagnosticsLocalData data{}; g_failAllocationCall = 2; @@ -435,7 +395,7 @@ TEST_F(RasDiagnosticsCommonMicrotest, CollectPropagatesRecordAllocationFailure) TEST_F(RasDiagnosticsCommonMicrotest, CollectPropagatesFillFailureWithoutPublishingPartialData) { OwnedComm owned(1, 2, 3, 0); - InstallComms({owned.comm.get()}); + InstallNcclComms({owned.comm.get()}); const rasDiagnosticsContext ctx = UnfilteredContext(); rasDiagnosticsLocalData data{}; g_failFillCall = 1; diff --git a/projects/rccl/test/host/ras-diagnostics-env-test.cc b/projects/rccl/test/host/ras-diagnostics-env-test.cc new file mode 100644 index 000000000000..524ada75a339 --- /dev/null +++ b/projects/rccl/test/host/ras-diagnostics-env-test.cc @@ -0,0 +1,642 @@ +/************************************************************************* + * Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +// Host-only microtests for `src/ras/diagnostics_env.cc`. + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "comm.h" +#include "fakes/env_fakes.h" +#include "fakes/ras_registry_test_support.h" +#include "ras/diagnostics_checks_common.h" + +namespace { + +int g_allocationCalls; +int g_failAllocationCall; +template +ncclResult_t DiagnosticsEnvTestCalloc(const char* file, int line, const char* function, Args&&... args) { + ++g_allocationCalls; + if (g_allocationCalls == g_failAllocationCall) return ncclSystemError; + return ncclCallocDebug(std::forward(args)..., file, line, function, false); +} + +} // namespace + +#undef ncclCalloc +#define ncclCalloc(...) DiagnosticsEnvTestCalloc(__FILE__, __LINE__, __func__, __VA_ARGS__) + +#include RAS_DIAGNOSTICS_ENV_CC_PATH + +#undef ncclCalloc + +namespace { + +int g_emitCalls; +int g_emitFailAt; +std::vector g_emittedLines; + +ncclResult_t RecordingEmit(void* /*target*/, const char* line) { + ++g_emitCalls; + if (g_emitCalls == g_emitFailAt) return ncclSystemError; + g_emittedLines.emplace_back(line); + return ncclSuccess; +} + +rasDiagnosticsReporter MakeRecordingReporter() { + rasDiagnosticsReporter reporter{}; + reporter.emit = RecordingEmit; + return reporter; +} + +bool AnyLineContains(const std::vector& lines, const std::string& needle) { + for (auto& l : lines) + if (l.find(needle) != std::string::npos) return true; + return false; +} + +size_t EnvRecordStride() { return rasDiagnosticsLocalRecordStride(sizeof(struct rasDiagnosticsNcclEnvData)); } + +struct rasDiagnosticsNcclEnvData* FillEnvRecord(std::vector& records, size_t index, + struct rasCommId commId, int rank, int nRanks, + std::initializer_list entries = {}, + bool truncated = false) { + char* record = records.data() + index * EnvRecordStride(); + auto* rankHeader = reinterpret_cast(record); + rankHeader->commId = commId; + rankHeader->commRank = rank; + rankHeader->commNRanks = nRanks; + + auto* envData = reinterpret_cast(record + sizeof(*rankHeader)); + size_t bytesUsed = 0; + for (const std::string& entry : entries) { + const size_t entryBytes = entry.size() + 1; + EXPECT_LE(bytesUsed + entryBytes, sizeof(envData->data)); + if (bytesUsed + entryBytes > sizeof(envData->data)) break; + memcpy(envData->data + bytesUsed, entry.c_str(), entryBytes); + bytesUsed += entryBytes; + } + envData->bytesUsed = static_cast(bytesUsed); + envData->truncated = truncated; + return envData; +} + +struct FakeComm { + std::unique_ptr comm{new ncclComm{}}; + std::vector peerInfos; + + FakeComm(uint64_t commHash, int rank, int nRanks) { + comm->commHash = commHash; + comm->rank = rank; + comm->nRanks = nRanks; + comm->peerInfoValid = true; + comm->cudaDev = 0; + comm->nvmlDev = 0; + comm->busId = 0; + comm->localRank = 0; + comm->localRanks = 1; + peerInfos.assign(1, ncclPeerInfo{}); + peerInfos[0].hostHash = commHash + 1; + peerInfos[0].pidHash = commHash + 2; + comm->peerInfo = peerInfos.data(); + } + ncclComm* get() { return comm.get(); } +}; + +using ras_test::InstallNcclComms; + +void ResetWholeFileSeams() { + g_allocationCalls = 0; + g_failAllocationCall = 0; + g_emitCalls = 0; + g_emitFailAt = 0; + g_emittedLines.clear(); + ras_test::ResetNcclComms(); + ResetEnvFakes(); +} + +class RasDiagnosticsEnvMicrotest : public ::testing::Test { + protected: + void SetUp() override { + ResetWholeFileSeams(); + SetMicroEnviron({}); + } + void TearDown() override { + ResetWholeFileSeams(); + } +}; + +} // namespace + +// =========================================================================== +// rasDiagnosticsNcclEnvCollectLocal (through the shared microtest copy of rasDiagnosticsCollectLocalRecords) +// =========================================================================== + +TEST_F(RasDiagnosticsEnvMicrotest, EnvironmentViews_EnumerationAndLookupStayInSync) { + SetMicroEnviron({"NCCL_DEBUG=INFO", "PATH=/scripted"}); + EXPECT_STREQ("INFO", micro_getenv("NCCL_DEBUG")); + EXPECT_STREQ("INFO", std::getenv("NCCL_DEBUG")); + ASSERT_NE(nullptr, environ[0]); + EXPECT_STREQ("NCCL_DEBUG=INFO", environ[0]); + + SetMicroEnv("NCCL_DEBUG", "WARN"); + EXPECT_STREQ("WARN", micro_getenv("NCCL_DEBUG")); + EXPECT_STREQ("WARN", std::getenv("NCCL_DEBUG")); + EXPECT_STREQ("NCCL_DEBUG=WARN", environ[0]); + SetMicroEnv("NCCL_NEW", "1"); + EXPECT_STREQ("NCCL_NEW=1", environ[2]); + EXPECT_STREQ("1", micro_getenv("NCCL_NEW")); + + SetMicroEnvAbsent("NCCL_DEBUG"); + EXPECT_EQ(nullptr, micro_getenv("NCCL_DEBUG")); + EXPECT_EQ(nullptr, std::getenv("NCCL_DEBUG")); + EXPECT_STREQ("PATH=/scripted", environ[0]); + EXPECT_STREQ("NCCL_NEW=1", environ[1]); + EXPECT_EQ(nullptr, environ[2]); +} + +TEST_F(RasDiagnosticsEnvMicrotest, EnvironmentViews_ResetRestoresOriginalEnviron) { + ResetEnvFakes(); + char** original = environ; + SetMicroEnviron({"NCCL_DEBUG=INFO"}); + SetMicroEnviron({"NCCL_DEBUG=WARN"}); + ResetEnvFakes(); + EXPECT_EQ(original, environ); + EXPECT_EQ(nullptr, micro_getenv("NCCL_DEBUG")); +} + +TEST_F(RasDiagnosticsEnvMicrotest, EnvironmentViews_DuplicateUpdatesPreserveOtherEntries) { + SetMicroEnviron({"NCCL_NO_EQUALS", "NCCL_DEBUG=INFO", "NCCL_DEBUG=WARN", "NCCL_OTHER=a=b"}); + EXPECT_STREQ("INFO", micro_getenv("NCCL_DEBUG")); + EXPECT_STREQ("INFO", std::getenv("NCCL_DEBUG")); + EXPECT_STREQ("a=b", micro_getenv("NCCL_OTHER")); + SetMicroEnv("NCCL_DEBUG", "TRACE"); + EXPECT_STREQ("NCCL_NO_EQUALS", environ[0]); + EXPECT_STREQ("NCCL_DEBUG=TRACE", environ[1]); + EXPECT_STREQ("NCCL_OTHER=a=b", environ[2]); + EXPECT_EQ(nullptr, environ[3]); + EXPECT_STREQ("TRACE", micro_getenv("NCCL_DEBUG")); + EXPECT_STREQ("TRACE", std::getenv("NCCL_DEBUG")); +} + +TEST_F(RasDiagnosticsEnvMicrotest, CollectLocal_NoMatchingCommsReturnsEmpty) { + struct rasDiagnosticsContext ctx{}; + struct rasDiagnosticsLocalData data{}; + const auto result = rasDiagnosticsNcclEnvCollectLocal(&ctx, &data); + ncclUniquePtr recordsOwner(data.records); + ASSERT_EQ(ncclSuccess, result); + EXPECT_EQ(0, data.nRecords); +} + +TEST_F(RasDiagnosticsEnvMicrotest, CollectLocal_FiltersNonNcclPrefixedVars) { + FakeComm fc(0x1000, 0, 1); + InstallNcclComms({fc.get()}); + SetMicroEnviron({"PATH=/usr/bin", "NCCL_DEBUG=INFO", "NCCLFOO=1", "HOME=/root"}); + + struct rasDiagnosticsContext ctx{}; + struct rasDiagnosticsLocalData data{}; + const auto result = rasDiagnosticsNcclEnvCollectLocal(&ctx, &data); + ncclUniquePtr recordsOwner(data.records); + ASSERT_EQ(ncclSuccess, result); + ASSERT_EQ(1, data.nRecords); + const auto* envData = reinterpret_cast( + data.records + sizeof(struct rasDiagnosticsRankHeader)); + EXPECT_EQ(0, envData->truncated); + std::string blob(envData->data, envData->bytesUsed); + EXPECT_NE(std::string::npos, blob.find("NCCL_DEBUG=INFO")); + EXPECT_EQ(std::string::npos, blob.find("NCCLFOO")); + EXPECT_EQ(std::string::npos, blob.find("PATH=")); +} + +TEST_F(RasDiagnosticsEnvMicrotest, CollectLocal_NoNcclVarsProducesEmptyPayload) { + FakeComm fc(0x1000, 0, 1); + InstallNcclComms({fc.get()}); + SetMicroEnviron({"PATH=/usr/bin"}); + + struct rasDiagnosticsContext ctx{}; + struct rasDiagnosticsLocalData data{}; + const auto result = rasDiagnosticsNcclEnvCollectLocal(&ctx, &data); + ncclUniquePtr recordsOwner(data.records); + ASSERT_EQ(ncclSuccess, result); + ASSERT_EQ(1, data.nRecords); + const auto* envData = reinterpret_cast( + data.records + sizeof(struct rasDiagnosticsRankHeader)); + EXPECT_EQ(0, envData->bytesUsed); +} + +TEST_F(RasDiagnosticsEnvMicrotest, CollectLocal_ExceedingByteBudgetSetsTruncated) { + FakeComm fc(0x1000, 0, 1); + InstallNcclComms({fc.get()}); + std::vector entries; + // Each ~1040 bytes; ~17 of them exceed the 16384-byte budget. + for (int i = 0; i < 17; i++) entries.push_back("NCCL_VAR" + std::to_string(i) + "=" + std::string(1024, 'a')); + SetMicroEnviron(entries); + + struct rasDiagnosticsContext ctx{}; + struct rasDiagnosticsLocalData data{}; + const auto result = rasDiagnosticsNcclEnvCollectLocal(&ctx, &data); + ncclUniquePtr recordsOwner(data.records); + ASSERT_EQ(ncclSuccess, result); + ASSERT_EQ(1, data.nRecords); + const auto* envData = reinterpret_cast( + data.records + sizeof(struct rasDiagnosticsRankHeader)); + EXPECT_EQ(1, envData->truncated); + EXPECT_EQ(15532, envData->bytesUsed); + EXPECT_EQ(0u, std::string(envData->data, envData->bytesUsed).find("NCCL_VAR0=")); +} + +TEST_F(RasDiagnosticsEnvMicrotest, CollectLocal_ExactFitAtTheByteBudgetIsNotTruncated) { + FakeComm fc(0x1000, 0, 1); + InstallNcclComms({fc.get()}); + // "NCCL_X=" (7) + value: total strlen == RAS_DIAG_ENV_BYTES - 1, so len (strlen+1, + // including the NUL) exactly equals the remaining budget -- fits exactly, at the + // boundary between "fits" and "doesn't fit". + std::string value(RAS_DIAG_ENV_BYTES - 1 - 7, 'a'); + SetMicroEnviron({"NCCL_X=" + value}); + + struct rasDiagnosticsContext ctx{}; + struct rasDiagnosticsLocalData data{}; + const auto result = rasDiagnosticsNcclEnvCollectLocal(&ctx, &data); + ncclUniquePtr recordsOwner(data.records); + ASSERT_EQ(ncclSuccess, result); + ASSERT_EQ(1, data.nRecords); + const auto* envData = reinterpret_cast( + data.records + sizeof(struct rasDiagnosticsRankHeader)); + EXPECT_EQ(0, envData->truncated); + EXPECT_EQ(RAS_DIAG_ENV_BYTES, envData->bytesUsed); +} + +// =========================================================================== +// rasDiagnosticsNcclEnvSummarize: argument validation +// =========================================================================== + +TEST_F(RasDiagnosticsEnvMicrotest, Summarize_NullReporterReturnsInternalError) { + EXPECT_EQ(ncclInternalError, rasDiagnosticsNcclEnvSummarize(nullptr, nullptr, nullptr, 0)); +} + +TEST_F(RasDiagnosticsEnvMicrotest, Summarize_ReporterWithNullEmitReturnsInternalError) { + struct rasDiagnosticsReporter reporter{}; + EXPECT_EQ(ncclInternalError, rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, nullptr, 0)); +} + +TEST_F(RasDiagnosticsEnvMicrotest, Summarize_ZeroDataIsNoOp) { + auto reporter = MakeRecordingReporter(); + EXPECT_EQ(ncclSuccess, rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, nullptr, 0)); + EXPECT_EQ(0, g_emitCalls); +} + +TEST_F(RasDiagnosticsEnvMicrotest, Summarize_NullDataWithNonzeroSizeReturnsInternalError) { + auto reporter = MakeRecordingReporter(); + const int stride = static_cast(EnvRecordStride()); + EXPECT_EQ(ncclInternalError, rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, nullptr, stride)); +} + +TEST_F(RasDiagnosticsEnvMicrotest, Summarize_SizeNotMultipleOfStrideReturnsInternalError) { + auto reporter = MakeRecordingReporter(); + char buf[4] = {}; + EXPECT_EQ(ncclInternalError, rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, buf, 1)); +} + +TEST_F(RasDiagnosticsEnvMicrotest, Summarize_NegativeSizeReturnsInternalError) { + auto reporter = MakeRecordingReporter(); + char buf[4] = {}; + EXPECT_EQ(ncclInternalError, + rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, buf, -static_cast(EnvRecordStride()))); +} + +// =========================================================================== +// rasDiagnosticsNcclEnvSummarize: end-to-end via CollectLocal-produced records +// =========================================================================== + +namespace { + +// Runs CollectLocal for the currently installed ncclComms/environ, then feeds +// the result straight into Summarize -- exercising the exact production +// pairing (one rank's real gathered record, not a hand-built stand-in). +ncclResult_t CollectThenSummarize(struct rasDiagnosticsReporter* reporter) { + struct rasDiagnosticsContext ctx{}; + struct rasDiagnosticsLocalData data{}; + ncclResult_t ret = rasDiagnosticsNcclEnvCollectLocal(&ctx, &data); + ncclUniquePtr recordsOwner(data.records); + if (ret != ncclSuccess) return ret; + ret = rasDiagnosticsNcclEnvSummarize(&ctx, reporter, data.records, data.recordsBytes); + return ret; +} + +} // namespace + +TEST_F(RasDiagnosticsEnvMicrotest, Summarize_SingleRankConsistentReport) { + FakeComm fc(0x2000, 0, 1); + InstallNcclComms({fc.get()}); + SetMicroEnviron({"NCCL_A_NO_EQUALS", "NCCL_DEBUG=INFO"}); + auto reporter = MakeRecordingReporter(); + + ASSERT_EQ(ncclSuccess, CollectThenSummarize(&reporter)); + ASSERT_EQ(1u, g_emittedLines.size()); + EXPECT_EQ("[OK] NCCL environment: NCCL_* env vars consistent across 1 ranks in comm 0x2000", g_emittedLines[0]); + EXPECT_FALSE(AnyLineContains(g_emittedLines, "NCCL_A_NO_EQUALS")); + EXPECT_FALSE(AnyLineContains(g_emittedLines, "mismatch")); +} + +TEST_F(RasDiagnosticsEnvMicrotest, Summarize_MultipleRanksWithEqualValuesAreConsistent) { + const size_t stride = EnvRecordStride(); + std::vector buf(stride * 2, 0); + + for (int rank = 0; rank < 2; rank++) { + FillEnvRecord(buf, rank, {0x2800, 0x2801, 0x2802}, rank, 2, {"NCCL_DEBUG=INFO"}); + } + + auto reporter = MakeRecordingReporter(); + ASSERT_EQ(ncclSuccess, rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, buf.data(), static_cast(buf.size()))); + ASSERT_EQ(1u, g_emittedLines.size()); + EXPECT_EQ("[OK] NCCL environment: NCCL_* env vars consistent across 2 ranks in comm 0x2800", g_emittedLines[0]); + EXPECT_FALSE(AnyLineContains(g_emittedLines, "mismatch")); +} + +TEST_F(RasDiagnosticsEnvMicrotest, Summarize_IncompleteGroupReportsIncomplete) { + // commNRanks says 2, but only rank 0 is present in ncclComms -- an incomplete gather. + FakeComm fc(0x3000, 0, 2); + InstallNcclComms({fc.get()}); + SetMicroEnviron({}); + auto reporter = MakeRecordingReporter(); + + ASSERT_EQ(ncclSuccess, CollectThenSummarize(&reporter)); + ASSERT_EQ(1u, g_emittedLines.size()); + EXPECT_EQ("[INFO] NCCL environment: diagnostics incomplete, gathered 1/2 ranks in comm 0x3000/0x3001/0x3002", + g_emittedLines[0]); + EXPECT_FALSE(AnyLineContains(g_emittedLines, "consistent")); +} + +TEST_F(RasDiagnosticsEnvMicrotest, Summarize_TwoRanksMismatchedValueReportsMismatch) { + // Two distinct ncclComm entries, same commHash/hostHash/pidHash pairing style but + // different rank -- built by hand below since FakeComm's per-object environ can't + // differ per rank (only one process-wide `environ`), so we hand-build two records + // directly instead of going through CollectLocal twice. + const size_t stride = EnvRecordStride(); + std::vector buf(stride * 2, 0); + + FillEnvRecord(buf, 0, {0x4000, 0x4001, 0x4002}, 0, 2, {"NCCL_DEBUG=INFO"}); + FillEnvRecord(buf, 1, {0x4000, 0x4001, 0x4002}, 1, 2, {"NCCL_DEBUG=WARN"}); + + auto reporter = MakeRecordingReporter(); + ASSERT_EQ(ncclSuccess, rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, buf.data(), static_cast(buf.size()))); + ASSERT_EQ(4u, g_emittedLines.size()); + EXPECT_EQ("[INFO] NCCL environment: mismatch across 2 ranks in comm 0x4000 for NCCL_DEBUG", g_emittedLines[0]); + EXPECT_EQ("[INFO] NCCL environment: NCCL_DEBUG=INFO on rank(s) {0}", g_emittedLines[1]); + EXPECT_EQ("[INFO] NCCL environment: NCCL_DEBUG=WARN on rank(s) {1}", g_emittedLines[2]); + EXPECT_EQ("[INFO] NCCL environment: 1 NCCL_* env var(s) differ across ranks in comm 0x4000", g_emittedLines[3]); + EXPECT_FALSE(AnyLineContains(g_emittedLines, "consistent")); +} + +TEST_F(RasDiagnosticsEnvMicrotest, Summarize_FourRanksGroupsRepeatedValuesOnce) { + const size_t stride = EnvRecordStride(); + std::vector buf(stride * 4, 0); + const int ranks[] = {3, 1, 2, 0}; + const char* values[] = {"WARN", "WARN", "INFO", "INFO"}; + + for (int i = 0; i < 4; i++) { + FillEnvRecord(buf, i, {0x4100, 0, 0}, ranks[i], 4, {std::string("NCCL_DEBUG=") + values[i]}); + } + + auto reporter = MakeRecordingReporter(); + ASSERT_EQ(ncclSuccess, rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, buf.data(), static_cast(buf.size()))); + ASSERT_EQ(4u, g_emittedLines.size()); + EXPECT_EQ("[INFO] NCCL environment: mismatch across 4 ranks in comm 0x4100 for NCCL_DEBUG", g_emittedLines[0]); + EXPECT_EQ("[INFO] NCCL environment: NCCL_DEBUG=INFO on rank(s) {0,2}", g_emittedLines[1]); + EXPECT_EQ("[INFO] NCCL environment: NCCL_DEBUG=WARN on rank(s) {1,3}", g_emittedLines[2]); + EXPECT_EQ("[INFO] NCCL environment: 1 NCCL_* env var(s) differ across ranks in comm 0x4100", g_emittedLines[3]); + EXPECT_FALSE(AnyLineContains(g_emittedLines, "consistent")); +} + +TEST_F(RasDiagnosticsEnvMicrotest, Summarize_TwoDistinctKeysAreSortedAndDeduplicated) { + const size_t stride = EnvRecordStride(); + std::vector buf(stride * 2, 0); + + FillEnvRecord(buf, 0, {0x4150, 0, 0}, 0, 2, {"NCCL_DEBUG_SUBSYS=INIT", "NCCL_DEBUG=INFO"}); + FillEnvRecord(buf, 1, {0x4150, 0, 0}, 1, 2, {"NCCL_DEBUG_SUBSYS=COLL", "NCCL_DEBUG=WARN"}); + + auto reporter = MakeRecordingReporter(); + ASSERT_EQ(ncclSuccess, rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, buf.data(), static_cast(buf.size()))); + ASSERT_EQ(7u, g_emittedLines.size()); + EXPECT_EQ("[INFO] NCCL environment: mismatch across 2 ranks in comm 0x4150 for NCCL_DEBUG", g_emittedLines[0]); + EXPECT_EQ("[INFO] NCCL environment: NCCL_DEBUG=INFO on rank(s) {0}", g_emittedLines[1]); + EXPECT_EQ("[INFO] NCCL environment: NCCL_DEBUG=WARN on rank(s) {1}", g_emittedLines[2]); + EXPECT_EQ("[INFO] NCCL environment: mismatch across 2 ranks in comm 0x4150 for NCCL_DEBUG_SUBSYS", + g_emittedLines[3]); + EXPECT_EQ("[INFO] NCCL environment: NCCL_DEBUG_SUBSYS=INIT on rank(s) {0}", g_emittedLines[4]); + EXPECT_EQ("[INFO] NCCL environment: NCCL_DEBUG_SUBSYS=COLL on rank(s) {1}", g_emittedLines[5]); + EXPECT_EQ("[INFO] NCCL environment: 2 NCCL_* env var(s) differ across ranks in comm 0x4150", g_emittedLines[6]); + EXPECT_FALSE(AnyLineContains(g_emittedLines, "consistent")); +} + +TEST_F(RasDiagnosticsEnvMicrotest, Summarize_AllocationFailuresPropagate) { + const size_t stride = EnvRecordStride(); + std::vector buf(stride * 2, 0); + FillEnvRecord(buf, 0, {0x4175, 0, 0}, 0, 2, {"NCCL_DEBUG=INFO"}); + FillEnvRecord(buf, 1, {0x4175, 0, 0}, 1, 2, {"NCCL_DEBUG=WARN"}); + + auto reporter = MakeRecordingReporter(); + ASSERT_EQ(ncclSuccess, rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, buf.data(), static_cast(buf.size()))); + const int allocationCount = g_allocationCalls; + ASSERT_GT(allocationCount, 0); + for (int failAt = 1; failAt <= allocationCount; failAt++) { + SCOPED_TRACE(failAt); + g_allocationCalls = 0; + g_failAllocationCall = failAt; + EXPECT_EQ(ncclSystemError, + rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, buf.data(), static_cast(buf.size()))) + << failAt; + } +} + +TEST_F(RasDiagnosticsEnvMicrotest, CollectLocal_EntryArrayAllocationFailurePropagates) { + FakeComm fc(0x4180, 0, 1); + InstallNcclComms({fc.get()}); + SetMicroEnviron({"NCCL_DEBUG=INFO"}); + struct rasDiagnosticsContext ctx{}; + struct rasDiagnosticsLocalData data{}; + g_allocationCalls = 0; + g_failAllocationCall = 1; + // The diagnostics-env seam controls the callback's entry array; shared collector allocations + // are covered by RasDiagnosticsCommonMicrotest. + EXPECT_EQ(ncclSystemError, rasDiagnosticsNcclEnvCollectLocal(&ctx, &data)); + EXPECT_EQ(nullptr, data.records); +} + +TEST_F(RasDiagnosticsEnvMicrotest, Summarize_MismatchReporterFailuresPropagate) { + const size_t stride = EnvRecordStride(); + std::vector mismatch(stride * 2, 0); + FillEnvRecord(mismatch, 0, {0x4200, 0, 0}, 0, 2, {"NCCL_DEBUG=INFO"}); + FillEnvRecord(mismatch, 1, {0x4200, 0, 0}, 1, 2, {"NCCL_DEBUG=WARN"}); + + auto reporter = MakeRecordingReporter(); + for (int failAt = 1; failAt <= 4; failAt++) { + SCOPED_TRACE(failAt); + g_emitCalls = 0; + g_emittedLines.clear(); + g_emitFailAt = failAt; + EXPECT_EQ(ncclSystemError, + rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, mismatch.data(), static_cast(mismatch.size()))); + EXPECT_EQ(failAt, g_emitCalls); + } + +} + +TEST_F(RasDiagnosticsEnvMicrotest, Summarize_ValueSetOnOneRankUnsetOnAnotherIsAMismatch) { + const size_t stride = EnvRecordStride(); + std::vector buf(stride * 2, 0); + + FillEnvRecord(buf, 0, {0x5000, 0, 0}, 0, 2, {"NCCL_FOO=bar"}); + FillEnvRecord(buf, 1, {0x5000, 0, 0}, 1, 2); + + auto reporter = MakeRecordingReporter(); + ASSERT_EQ(ncclSuccess, rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, buf.data(), static_cast(buf.size()))); + EXPECT_TRUE(AnyLineContains(g_emittedLines, "NCCL_FOO=(unset) on rank(s) {1}")); + EXPECT_TRUE(AnyLineContains(g_emittedLines, "NCCL_FOO=bar on rank(s) {0}")); +} + +TEST_F(RasDiagnosticsEnvMicrotest, Summarize_TruncatedRankReportsTruncationWarning) { + const size_t stride = EnvRecordStride(); + std::vector buf(stride, 0); + FillEnvRecord(buf, 0, {0x6000, 0, 0}, 0, 1, {"NCCL_DEBUG=INFO"}, true); + + auto reporter = MakeRecordingReporter(); + ASSERT_EQ(ncclSuccess, rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, buf.data(), static_cast(buf.size()))); + EXPECT_TRUE(AnyLineContains(g_emittedLines, "1 rank(s) had >" + std::to_string(RAS_DIAG_ENV_BYTES) + " bytes")); + EXPECT_FALSE(AnyLineContains(g_emittedLines, "consistent across")); +} + +TEST_F(RasDiagnosticsEnvMicrotest, Summarize_MalformedEnvPayloadsReturnInternalError) { + const size_t stride = EnvRecordStride(); + std::vector buf(stride, 0); + auto* envData = FillEnvRecord(buf, 0, {0x7000, 0, 0}, 0, 1); + envData->bytesUsed = 5; + envData->data[4] = 'x'; // Not NUL-terminated at the claimed end -- malformed. + + auto reporter = MakeRecordingReporter(); + EXPECT_EQ(ncclInternalError, + rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, buf.data(), static_cast(buf.size()))); + + envData->bytesUsed = RAS_DIAG_ENV_BYTES + 1; + EXPECT_EQ(ncclInternalError, + rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, buf.data(), static_cast(buf.size()))); +} + +TEST_F(RasDiagnosticsEnvMicrotest, Summarize_TwoDistinctCommsAreReportedSeparately) { + const size_t stride = EnvRecordStride(); + std::vector buf(stride * 4, 0); + + FillEnvRecord(buf, 0, {0x8000, 0x8100, 0x8200}, 0, 1, {"NCCL_FIRST=ONLY"}); + FillEnvRecord(buf, 1, {0x8000, 0x8300, 0x8400}, 1, 2, {"NCCL_DEBUG=WARN"}); + FillEnvRecord(buf, 2, {0x9000, 0x9100, 0x9200}, 0, 1); + FillEnvRecord(buf, 3, {0x8000, 0x8300, 0x8400}, 0, 2, {"NCCL_DEBUG=INFO"}); + + auto reporter = MakeRecordingReporter(); + ASSERT_EQ(ncclSuccess, rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, buf.data(), static_cast(buf.size()))); + ASSERT_EQ(6u, g_emittedLines.size()); + EXPECT_EQ(0u, g_emittedLines[0].find("[OK] NCCL environment: NCCL_* env vars consistent across 1 ranks in comm 0x8000")); + EXPECT_EQ(0u, g_emittedLines[1].find("[INFO] NCCL environment: mismatch across 2 ranks in comm 0x8000")); + EXPECT_NE(std::string::npos, g_emittedLines[1].find(" for NCCL_DEBUG")); + EXPECT_EQ("[INFO] NCCL environment: NCCL_DEBUG=INFO on rank(s) {0}", g_emittedLines[2]); + EXPECT_EQ("[INFO] NCCL environment: NCCL_DEBUG=WARN on rank(s) {1}", g_emittedLines[3]); + EXPECT_EQ(0u, g_emittedLines[4].find("[INFO] NCCL environment: 1 NCCL_* env var(s) differ across ranks in comm 0x8000")); + EXPECT_EQ(0u, g_emittedLines[5].find("[OK] NCCL environment: NCCL_* env vars consistent across 1 ranks in comm 0x9000")); +} + +TEST_F(RasDiagnosticsEnvMicrotest, DISABLED_Summarize_HashCollisionsReportFullCommIdentity) { + SCOPED_TRACE("AICOMRCCL-2743: reporters omit host and process hashes from the communicator identity"); + std::vector records(EnvRecordStride() * 3, 0); + FillEnvRecord(records, 0, {0x8000, 0x8100, 0x8200}, 0, 1, {"NCCL_FIRST=ONLY"}); + FillEnvRecord(records, 1, {0x8000, 0x8300, 0x8400}, 1, 2, {"NCCL_DEBUG=WARN"}); + FillEnvRecord(records, 2, {0x8000, 0x8300, 0x8400}, 0, 2, {"NCCL_DEBUG=INFO"}); + auto reporter = MakeRecordingReporter(); + ASSERT_EQ(ncclSuccess, + rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, records.data(), static_cast(records.size()))); + ASSERT_EQ(5u, g_emittedLines.size()); + EXPECT_EQ("[OK] NCCL environment: NCCL_* env vars consistent across 1 ranks in comm 0x8000/0x8100/0x8200", + g_emittedLines[0]); + EXPECT_EQ("[INFO] NCCL environment: mismatch across 2 ranks in comm 0x8000/0x8300/0x8400 for NCCL_DEBUG", + g_emittedLines[1]); + EXPECT_EQ("[INFO] NCCL environment: 1 NCCL_* env var(s) differ across ranks in comm 0x8000/0x8300/0x8400", + g_emittedLines[4]); +} + +TEST_F(RasDiagnosticsEnvMicrotest, Summarize_LongKeyAndValueAreTruncatedWithEllipsis) { + const size_t stride = EnvRecordStride(); + std::vector buf(stride * 2, 0); + + std::string longKey = "NCCL_" + std::string(300, 'K'); + std::string longVal(300, 'V'); + std::string entry0 = longKey + "=" + longVal; + std::string entry1 = longKey + "=" + std::string(300, 'W'); // Different value -> mismatch. + + FillEnvRecord(buf, 0, {0xA000, 0, 0}, 0, 2, {entry0}); + FillEnvRecord(buf, 1, {0xA000, 0, 0}, 1, 2, {entry1}); + + auto reporter = MakeRecordingReporter(); + ASSERT_EQ(ncclSuccess, rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, buf.data(), static_cast(buf.size()))); + const std::string keyDisplay = "NCCL_" + std::string(247, 'K') + "..."; + const std::string valueDisplay = std::string(252, 'V') + "..."; + ASSERT_EQ(4u, g_emittedLines.size()); + EXPECT_EQ("[INFO] NCCL environment: mismatch across 2 ranks in comm 0xa000 for " + keyDisplay, g_emittedLines[0]); + EXPECT_EQ("[INFO] NCCL environment: " + keyDisplay + "=" + valueDisplay + " on rank(s) {0}", g_emittedLines[1]); +} + +TEST_F(RasDiagnosticsEnvMicrotest, Summarize_ControlCharactersAreSanitizedToQuestionMarks) { + const size_t stride = EnvRecordStride(); + std::vector buf(stride * 2, 0); + + FillEnvRecord(buf, 0, {0xB000, 0, 0}, 0, 2, + {std::string("NCCL_A\x01" "KEY=left"), std::string("NCCL_B\x7f" "KEY=a\x7f" "b"), + std::string("NCCL_X=a\x01" "b")}); + FillEnvRecord(buf, 1, {0xB000, 0, 0}, 1, 2, + {std::string("NCCL_A\x01" "KEY=right"), std::string("NCCL_B\x7f" "KEY=different"), + "NCCL_X=different"}); + + auto reporter = MakeRecordingReporter(); + ASSERT_EQ(ncclSuccess, rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, buf.data(), static_cast(buf.size()))); + EXPECT_TRUE(AnyLineContains(g_emittedLines, "NCCL_A?KEY")); + EXPECT_TRUE(AnyLineContains(g_emittedLines, "NCCL_B?KEY=a?b")); + EXPECT_TRUE(AnyLineContains(g_emittedLines, "NCCL_X=a?b")); +} + +TEST_F(RasDiagnosticsEnvMicrotest, Summarize_ConsistentReporterFailurePropagates) { + std::vector records(EnvRecordStride(), 0); + FillEnvRecord(records, 0, {0x4300, 0, 0}, 0, 1, {}, false); + auto reporter = MakeRecordingReporter(); + g_emitFailAt = 1; + EXPECT_EQ(ncclSystemError, + rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, records.data(), static_cast(records.size()))); + EXPECT_EQ(1, g_emitCalls); +} + +TEST_F(RasDiagnosticsEnvMicrotest, Summarize_TruncatedReporterFailurePropagates) { + std::vector records(EnvRecordStride(), 0); + FillEnvRecord(records, 0, {0x4300, 0, 0}, 0, 1, {}, true); + auto reporter = MakeRecordingReporter(); + g_emitFailAt = 1; + EXPECT_EQ(ncclSystemError, + rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, records.data(), static_cast(records.size()))); + EXPECT_EQ(1, g_emitCalls); +} + +TEST_F(RasDiagnosticsEnvMicrotest, Summarize_IncompleteReporterFailurePropagates) { + std::vector records(EnvRecordStride(), 0); + FillEnvRecord(records, 0, {0x4300, 0, 0}, 0, 2, {}, false); + auto reporter = MakeRecordingReporter(); + g_emitFailAt = 1; + EXPECT_EQ(ncclSystemError, + rasDiagnosticsNcclEnvSummarize(nullptr, &reporter, records.data(), static_cast(records.size()))); + EXPECT_EQ(1, g_emitCalls); +} diff --git a/projects/rccl/test/host/ras-test.cc b/projects/rccl/test/host/ras-test.cc index 8b9ae0358ec4..092ef7ce3dc8 100644 --- a/projects/rccl/test/host/ras-test.cc +++ b/projects/rccl/test/host/ras-test.cc @@ -23,6 +23,7 @@ #include #include "comm.h" +#include "fakes/ras_registry_test_support.h" #include "fakes/nccl_fakes.h" #include "fakes/param_redirect.h" #include "fakes/signature-drift.h" @@ -149,10 +150,7 @@ class RasMicrotest : public ::testing::Test { rasInitialized = false; rasInitRefCount = 0; rasNotificationPipe[0] = rasNotificationPipe[1] = NCCL_SOCKET_PAIR_INVALID; - std::free(ncclComms); - ncclComms = nullptr; - nNcclComms = 0; - ncclCommsSorted = false; + ras_test::ResetNcclComms(); std::free(rasPfds); rasPfds = nullptr; nRasPfds = 0; @@ -164,9 +162,7 @@ class RasMicrotest : public ::testing::Test { std::free(rasPfds); rasPfds = nullptr; nRasPfds = 0; - std::free(ncclComms); - ncclComms = nullptr; - nNcclComms = 0; + ras_test::ResetNcclComms(); rasInitialized = false; rasInitRefCount = 0; g_socketProgress = DefaultSocketProgress; @@ -200,11 +196,9 @@ TEST_F(RasMicrotest, CommInitAlreadyInitializedAcceptsNullRank) { TEST_F(RasMicrotest, CommInitReusesVacantCommSlot) { rasInitialized = true; - nNcclComms = 2; - ncclComms = static_cast(std::calloc(2, sizeof(*ncclComms))); auto incumbent = std::make_unique(); auto newcomer = std::make_unique(); - ncclComms[0] = incumbent.get(); + ras_test::InstallNcclComms({incumbent.get(), nullptr}); rasRankInit rank{}; ASSERT_EQ(ncclSuccess, ncclRasCommInit(newcomer.get(), &rank)); EXPECT_EQ(newcomer.get(), ncclComms[1]); @@ -213,13 +207,10 @@ TEST_F(RasMicrotest, CommInitReusesVacantCommSlot) { TEST_F(RasMicrotest, CommInitGrowthPreservesRegisteredComms) { rasInitialized = true; - nNcclComms = 2; - ncclComms = static_cast(std::calloc(2, sizeof(*ncclComms))); auto first = std::make_unique(); auto second = std::make_unique(); auto newcomer = std::make_unique(); - ncclComms[0] = first.get(); - ncclComms[1] = second.get(); + ras_test::InstallNcclComms({first.get(), second.get()}); rasRankInit rank{}; ASSERT_EQ(ncclSuccess, ncclRasCommInit(newcomer.get(), &rank)); EXPECT_EQ(first.get(), ncclComms[0]); @@ -286,12 +277,9 @@ TEST_F(RasMicrotest, CommFiniUninitializedIsNoOp) { TEST_F(RasMicrotest, CommFiniRemovesMatchingCommAndDropsReference) { rasInitialized = true; rasInitRefCount = 1; - nNcclComms = 2; - ncclComms = static_cast(std::calloc(2, sizeof(*ncclComms))); auto first = std::make_unique(); auto second = std::make_unique(); - ncclComms[0] = first.get(); - ncclComms[1] = second.get(); + ras_test::InstallNcclComms({first.get(), second.get()}); ncclCommsSorted = true; EXPECT_EQ(ncclSuccess, ncclRasCommFini(second.get())); EXPECT_EQ(first.get(), ncclComms[0]); @@ -389,8 +377,7 @@ TEST_F(RasMicrotest, LocalHandleRunsDiagnosticsAndIgnoresHandlerFailure) { TEST_F(RasMicrotest, ThreadCleanupResetsAllGlobalState) { rasInitialized = true; rasInitRefCount = 3; - nNcclComms = 1; - ncclComms = static_cast(std::calloc(1, sizeof(*ncclComms))); + ras_test::InstallNcclComms({nullptr}); InitPollFds(1); rasThreadCleanup(); for (int calls : g_cleanupCalls) EXPECT_EQ(1, calls); diff --git a/projects/rccl/test/test_categories_micro.yaml b/projects/rccl/test/test_categories_micro.yaml index 83093dc7a9e0..1f74dd1e134f 100644 --- a/projects/rccl/test/test_categories_micro.yaml +++ b/projects/rccl/test/test_categories_micro.yaml @@ -41,6 +41,8 @@ test_categories: - "RasCollectivesMicrotest.*" # ras/diagnostics.cc - "RasDiagnosticsMicrotest.*" + # ras/diagnostics_env.cc + - "RasDiagnosticsEnvMicrotest.*" # ras/client.cc - "RasClientMicrotest.*" # ras/ras_param.cc