test(rccl): add host microtests for src/ras/diagnostics_gpu.cc

projects/rccl/test/host/CMakeLists.txt

@@ -74,6 +74,7 @@ function(rccl_define_micro_source_lists)
     ras-diagnostics-common-test.cc       # src/ras/diagnostics_checks_common.cc
     ras-collectives-test.cc              # src/ras/collectives.cc
     ras-diagnostics-test.cc              # src/ras/diagnostics.cc
+    ras-diagnostics-gpu-test.cc              # src/ras/diagnostics_gpu.cc
     tuning-test.cc                       # src/tuning/tuning.cc
     tuning-general-test.cc
     ras-client-test.cc
@@ -750,6 +751,7 @@ rccl_add_micro_binary(rccl-UnitTestsMicro
     # so add_file_unique() renames this one. Re-check after any source-list reorder.
     COLLECTIVES_CC_PATH="${RCCL_HOST_HIPIFY_DIR}/src/ras/collectives_tmp.cc"
     RAS_DIAGNOSTICS_CC_PATH="${RCCL_HOST_HIPIFY_DIR}/src/ras/diagnostics_tmp.cc"
+    RAS_DIAGNOSTICS_GPU_CC_PATH="${RCCL_HOST_HIPIFY_DIR}/src/ras/diagnostics_gpu.cc"
     # graph/tuning.cc keeps the basename, so add_file_unique() renames this one.
     TUNING_CC_PATH="${RCCL_HOST_HIPIFY_DIR}/src/tuning/tuning_tmp.cc"
     TUNING_GENERAL_CC_PATH="${RCCL_HOST_HIPIFY_DIR}/src/tuning/tuning_general.cc"

projects/rccl/test/host/MICROTEST_README.md

@@ -111,6 +111,11 @@ export colliding non-`static` symbols; otherwise a unit needs its own binary:
     before validation. Its raw-byte fixture reproduces the enum UBSan failure;
     enable it after the production fix. The defined `RAS_DIAG_CHECK_COUNT`
     sentinel remains covered by enabled dispatch and peer-payload tests.
+  - `ras/diagnostics_gpu.cc` (`RAS_DIAGNOSTICS_GPU_CC_PATH`, from
+    `ras-diagnostics-gpu-test.cc`); suite `RasDiagnosticsGpuMicrotest.*`. Covers
+    GPU inventory, CUDA and NVIDIA graphics driver versions, ECC, and NVLink
+    collection and summaries, including failed NVML queries and bounded
+    driver-version strings.
   - `ras/client.cc` (`RAS_CLIENT_CC_PATH`, from `ras-client-test.cc`); suite
     `RasClientMicrotest.*`. With
     `NCCL_RAS_CLIENT` defined, `ras_internal.h` reduces to four macros, so this
@@ -755,3 +760,19 @@ cmake --build build -j"$(nproc)"
 
 Disable coverage instrumentation for the standalone host-only test binaries
 with `-DHOST_TEST_COVERAGE=OFF`.
+
+### Shared RAS diagnostic fixtures
+
+`fakes/ras_registry_test_support.h` owns test installation/reset of the
+`ras.cc` communicator registry. Both operations hold `ncclCommsMutex` and
+clear `ncclCommsSorted`; installation validates allocation before replacing
+the old registry and preserves explicit vacant slots. The RAS and diagnostics
+suites use the same helper. `fakes/ras_diagnostics_test_support.h` shares the
+owned communicator and recording reporter used by the diagnostic suites.
+
+The GPU diagnostic suite exercises payload filling through the public collectors
+and owns returned records with RAII. Each summarizer has an over-capacity rank
+case; together they exercise all seven bounded rank arrays and check truncated
+rank sets with total counts. ECC maxima are checked with the largest values on
+an earlier rank, and incomplete-report fixtures include hexadecimal hashes
+larger than nine.

projects/rccl/test/host/fakes/ras_diagnostics_test_support.h

@@ -0,0 +1,55 @@
+#ifndef RCCL_TEST_HOST_RAS_DIAGNOSTICS_TEST_SUPPORT_H_
+#define RCCL_TEST_HOST_RAS_DIAGNOSTICS_TEST_SUPPORT_H_
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "comm.h"
+#include "ras/diagnostics.h"
+#include "fakes/ras_registry_test_support.h"
+
+namespace ras_test {
+
+struct OwnedComm {
+  OwnedComm(uint64_t commHash, uint64_t hostHash, uint64_t pidHash, int rank, bool peerInfoValid = true)
+      : comm(std::make_unique<ncclComm>()), peers(std::make_unique<ncclPeerInfo[]>(1)) {
+    comm->commHash = commHash;
+    comm->peerInfo = peers.get();
+    comm->peerInfoValid = peerInfoValid;
+    comm->rank = rank;
+    comm->nRanks = 8;
+    comm->cudaDev = rank + 10;
+    comm->nvmlDev = rank + 20;
+    comm->busId = 0x1000 + rank;
+    comm->localRank = rank % 4;
+    comm->localRanks = 4;
+    peers[0].hostHash = hostHash;
+    peers[0].pidHash = pidHash;
+  }
+
+  std::unique_ptr<ncclComm> comm;
+  std::unique_ptr<ncclPeerInfo[]> peers;
+};
+
+struct ReporterState {
+  std::vector<std::string> lines;
+  ncclResult_t emitResult = ncclSuccess;
+  int calls = 0;
+  int failCall = 0;
+};
+
+inline ncclResult_t CaptureReport(void* target, const char* line) {
+  auto* state = static_cast<ReporterState*>(target);
+  state->lines.emplace_back(line);
+  ++state->calls;
+  return state->failCall == 0 || state->calls == state->failCall ? state->emitResult : ncclSuccess;
+}
+
+inline rasDiagnosticsReporter MakeReporter(ReporterState* state) {
+  return rasDiagnosticsReporter{CaptureReport, nullptr, state};
+}
+
+}
+
+#endif

projects/rccl/test/host/fakes/ras_registry_test_support.h

@@ -0,0 +1,34 @@
+#ifndef RCCL_TEST_HOST_RAS_REGISTRY_TEST_SUPPORT_H_
+#define RCCL_TEST_HOST_RAS_REGISTRY_TEST_SUPPORT_H_
+
+#include <cstdlib>
+#include <mutex>
+#include <vector>
+
+#include <gtest/gtest.h>
+#include "ras/ras_internal.h"
+
+namespace ras_test {
+
+inline void InstallNcclComms(const std::vector<ncclComm*>& comms) {
+  std::lock_guard<std::mutex> lock(ncclCommsMutex);
+  auto** replacement = static_cast<ncclComm**>(std::calloc(comms.empty() ? 1 : comms.size(), sizeof(ncclComm*)));
+  ASSERT_NE(nullptr, replacement);
+  std::free(ncclComms);
+  ncclComms = replacement;
+  nNcclComms = static_cast<int>(comms.size());
+  ncclCommsSorted = false;
+  for (size_t index = 0; index < comms.size(); ++index) ncclComms[index] = comms[index];
+}
+
+inline void ResetNcclComms() {
+  std::lock_guard<std::mutex> lock(ncclCommsMutex);
+  std::free(ncclComms);
+  ncclComms = nullptr;
+  nNcclComms = 0;
+  ncclCommsSorted = false;
+}
+
+}
+
+#endif

projects/rccl/test/host/ras-diagnostics-common-test.cc

@@ -22,6 +22,7 @@
 #include "alloc.h"
 #include "checks.h"
 #include "comm.h"
+#include "fakes/ras_diagnostics_test_support.h"
 #include "compiler.h"
 #include "ras/diagnostics_checks_common.h"
 #include "ras/ras_internal.h"
@@ -58,26 +59,10 @@ struct TestPayload {
   uint32_t marker;
 };
 
-struct OwnedComm {
-  OwnedComm(uint64_t commHash, uint64_t hostHash, uint64_t pidHash, int rank, bool peerInfoValid = true)
-      : comm(std::make_unique<ncclComm>()), peers(std::make_unique<ncclPeerInfo[]>(1)) {
-    comm->commHash = commHash;
-    comm->peerInfo = peers.get();
-    comm->peerInfoValid = peerInfoValid;
-    comm->rank = rank;
-    comm->nRanks = 8;
-    comm->cudaDev = rank + 10;
-    comm->nvmlDev = rank + 20;
-    comm->busId = 0x1000 + rank;
-    comm->localRank = rank % 4;
-    comm->localRanks = 4;
-    peers[0].hostHash = hostHash;
-    peers[0].pidHash = pidHash;
-  }
-
-  std::unique_ptr<ncclComm> comm;
-  std::unique_ptr<ncclPeerInfo[]> peers;
-};
+using ras_test::OwnedComm;
+using ras_test::ReporterState;
+using ras_test::CaptureReport;
+using ras_test::InstallNcclComms;
 
 std::vector<rasDiagnosticsCommSnapshot> g_fillSnapshots;
 int g_fillCalls = 0;
@@ -102,23 +87,8 @@ ncclResult_t FillTestPayload(const rasDiagnosticsCommSnapshot* snapshot, void* c
   return ncclSuccess;
 }
 
-struct ReporterState {
-  std::vector<std::string> lines;
-  ncclResult_t emitResult = ncclSuccess;
-};
-
-ncclResult_t CaptureReport(void* target, const char* line) {
-  auto* state = static_cast<ReporterState*>(target);
-  state->lines.emplace_back(line);
-  return state->emitResult;
-}
-
 void ResetDiagnosticsCommonState() {
-  std::lock_guard<std::mutex> lock(ncclCommsMutex);
-  std::free(ncclComms);
-  ncclComms = nullptr;
-  nNcclComms = 0;
-  ncclCommsSorted = false;
+  ras_test::ResetNcclComms();
   g_allocationCalls = 0;
   g_failAllocationCall = 0;
   g_allocationCounts.clear();
@@ -128,16 +98,6 @@ void ResetDiagnosticsCommonState() {
   g_fillObservedUnlockedMutex = true;
 }
 
-void InstallComms(std::initializer_list<ncclComm*> comms) {
-  std::lock_guard<std::mutex> lock(ncclCommsMutex);
-  std::free(ncclComms);
-  ncclComms = static_cast<ncclComm**>(std::calloc(comms.size(), sizeof(*ncclComms)));
-  ASSERT_NE(nullptr, ncclComms);
-  nNcclComms = static_cast<int>(comms.size());
-  int index = 0;
-  for (ncclComm* comm : comms) ncclComms[index++] = comm;
-}
-
 rasDiagnosticsContext UnfilteredContext() {
   rasDiagnosticsContext ctx{};
   ctx.hasCommFilter = false;
@@ -307,7 +267,7 @@ TEST_F(RasDiagnosticsCommonMicrotest, CollectRejectsStrideRoundedPastIntMax) {
 TEST_F(RasDiagnosticsCommonMicrotest, CollectRejectsCombinedRecordBytesPastIntMax) {
   OwnedComm first(1, 2, 3, 0);
   OwnedComm second(4, 5, 6, 1);
-  InstallComms({first.comm.get(), second.comm.get()});
+  InstallNcclComms({first.comm.get(), second.comm.get()});
   const rasDiagnosticsContext ctx = UnfilteredContext();
   rasDiagnosticsLocalData data{};
   const size_t payloadSize = static_cast<size_t>(INT_MAX) / 2;
@@ -323,7 +283,7 @@ TEST_F(RasDiagnosticsCommonMicrotest, CollectRejectsCombinedRecordBytesPastIntMa
 TEST_F(RasDiagnosticsCommonMicrotest, CollectAcceptsRecordBytesExactlyAtIntMaxLimit) {
   OwnedComm first(1, 2, 3, 0);
   OwnedComm second(4, 5, 6, 1);
-  InstallComms({first.comm.get(), second.comm.get()});
+  InstallNcclComms({first.comm.get(), second.comm.get()});
   const rasDiagnosticsContext ctx = UnfilteredContext();
   rasDiagnosticsLocalData data{};
   const size_t payloadSize = 1000000000;
@@ -341,7 +301,7 @@ TEST_F(RasDiagnosticsCommonMicrotest, CollectAcceptsRecordBytesExactlyAtIntMaxLi
 TEST_F(RasDiagnosticsCommonMicrotest, CollectReturnsEmptyForNullInvalidAndFilteredComms) {
   OwnedComm invalid(10, 20, 30, 0, false);
   OwnedComm mismatch(11, 20, 30, 1);
-  InstallComms({nullptr, invalid.comm.get(), mismatch.comm.get()});
+  InstallNcclComms({nullptr, invalid.comm.get(), mismatch.comm.get()});
   const rasDiagnosticsContext ctx = FilteredContext(10, 20, 30);
   rasDiagnosticsLocalData data{reinterpret_cast<char*>(1), 2, 3, 4};
 
@@ -363,7 +323,7 @@ TEST_F(RasDiagnosticsCommonMicrotest, CollectFiltersCommsAndBuildsAlignedRecords
   first.comm->nRanks = 0x55667788;
   second.comm->rank = 0x22334455;
   second.comm->nRanks = 0x66778899;
-  InstallComms({first.comm.get(), nullptr, invalid.comm.get(), mismatch.comm.get(), second.comm.get(), nullptr});
+  InstallNcclComms({first.comm.get(), nullptr, invalid.comm.get(), mismatch.comm.get(), second.comm.get(), nullptr});
   const rasDiagnosticsContext ctx = FilteredContext(10, 20, 30);
   rasDiagnosticsLocalData data{};
 
@@ -407,7 +367,7 @@ TEST_F(RasDiagnosticsCommonMicrotest, CollectFiltersCommsAndBuildsAlignedRecords
 
 TEST_F(RasDiagnosticsCommonMicrotest, CollectPropagatesSnapshotAllocationFailure) {
   OwnedComm owned(1, 2, 3, 0);
-  InstallComms({owned.comm.get()});
+  InstallNcclComms({owned.comm.get()});
   const rasDiagnosticsContext ctx = UnfilteredContext();
   rasDiagnosticsLocalData data{};
   g_failAllocationCall = 1;
@@ -421,7 +381,7 @@ TEST_F(RasDiagnosticsCommonMicrotest, CollectPropagatesSnapshotAllocationFailure
 
 TEST_F(RasDiagnosticsCommonMicrotest, CollectPropagatesRecordAllocationFailure) {
   OwnedComm owned(1, 2, 3, 0);
-  InstallComms({owned.comm.get()});
+  InstallNcclComms({owned.comm.get()});
   const rasDiagnosticsContext ctx = UnfilteredContext();
   rasDiagnosticsLocalData data{};
   g_failAllocationCall = 2;
@@ -435,7 +395,7 @@ TEST_F(RasDiagnosticsCommonMicrotest, CollectPropagatesRecordAllocationFailure)
 
 TEST_F(RasDiagnosticsCommonMicrotest, CollectPropagatesFillFailureWithoutPublishingPartialData) {
   OwnedComm owned(1, 2, 3, 0);
-  InstallComms({owned.comm.get()});
+  InstallNcclComms({owned.comm.get()});
   const rasDiagnosticsContext ctx = UnfilteredContext();
   rasDiagnosticsLocalData data{};
   g_failFillCall = 1;