test(rccl): add binary-mode Recorder RedOp record tests

projects/rccl/test/_RecorderTests.cpp

@@ -13,6 +13,7 @@
 
 #include <cstdio>
 #include <fstream>
+#include <memory>
 #include <thread>
 
 #define HIPCALL(cmd)                                                                          \
@@ -134,6 +135,67 @@ namespace RcclUnitTesting
     );
   }
 
+  /**
+   * \brief Verify binary mode writes PreMulSum scalar/datatype/residence as sendbuff/datatype/root; Destroy drops them
+   * ******************************************************************************************/
+  TEST(Recorder, RedOpRecordBinary)
+  {
+    RUN_ISOLATED_TEST_WITH_ENV(
+      "RedOpRecordBinary",
+      []()
+      {
+        int ranks[] = {2, 3, 5};
+        auto comm = std::make_unique<ncclComm>();
+        comm->nRanks = 4;
+        comm->localRank = 1;
+        comm->localRankToRank = ranks;
+        float scalar = 2.0f;
+        double ignored = 4.0;
+        // Non-default datatype and residence, so a dropped or inverted PreMulSum guard cannot match the defaults.
+        ASSERT_EQ(ncclSuccess, rccl::Recorder::instance().record(rccl::rrRedOpCreatePreMulSum, ncclMax, comm.get(),
+                                                                 ncclFloat32, ncclScalarHostImmediate, &scalar));
+        // Destroy gets non-default scalar arguments too, so a guard that always copies them is caught.
+        ASSERT_EQ(ncclSuccess, rccl::Recorder::instance().record(rccl::rrRedOpDestroy, ncclMax, comm.get(),
+                                                                 ncclFloat64, ncclScalarHostImmediate, &ignored));
+
+        char host[256];
+        gethostname(host, sizeof(host));
+        std::string filename = "/tmp/test_redop." + std::to_string(getpid()) + "." + std::string(host) + ".bin";
+        std::ifstream fp(filename, std::ios::binary);
+        ASSERT_TRUE(fp.is_open()) << "Recorder did not create expected file: " << filename;
+        // One spare slot so a stray extra record shows up in bytesRead.
+        rccl::rcclApiCall calls[3];
+        fp.read(reinterpret_cast<char*>(calls), sizeof(calls));
+        std::streamsize bytesRead = fp.gcount();
+        fp.close();
+        std::remove(filename.c_str());
+        // Binary mode writes exactly one raw struct per call and no header.
+        ASSERT_EQ(bytesRead, static_cast<std::streamsize>(2 * sizeof(rccl::rcclApiCall)));
+
+        const rccl::rcclApiCall& create = calls[0];
+        EXPECT_EQ(create.type, rccl::rrRedOpCreatePreMulSum);
+        EXPECT_EQ(create.op, ncclMax);
+        EXPECT_EQ(create.comm, comm.get());
+        // nRanks and globalRank are taken from the comm and only reach disk in binary mode.
+        EXPECT_EQ(create.nRanks, 4);
+        EXPECT_EQ(create.globalRank, 3);
+        EXPECT_EQ(create.sendbuff, static_cast<const void*>(&scalar)) << "sendbuff must come from scalar";
+        EXPECT_EQ(create.datatype, ncclFloat32) << "datatype must come from the datatype argument";
+        EXPECT_EQ(create.root, static_cast<int>(ncclScalarHostImmediate)) << "root must come from residence";
+
+        const rccl::rcclApiCall& destroy = calls[1];
+        EXPECT_EQ(destroy.type, rccl::rrRedOpDestroy);
+        EXPECT_EQ(destroy.op, ncclMax);
+        EXPECT_EQ(destroy.comm, comm.get());
+        // record() builds from the aggregate ncclInfo{.op, .comm}, so fields Destroy must not set are zero.
+        EXPECT_EQ(destroy.sendbuff, nullptr) << "Destroy must not record the scalar";
+        EXPECT_EQ(destroy.datatype, ncclInt8) << "Destroy must not record the datatype";
+        EXPECT_EQ(destroy.root, 0) << "Destroy must not record the residence";
+      },
+      {{"RCCL_REPLAY_FILE", "/tmp/test_redop.bin"}}
+    );
+  }
+
   /**
    * \brief Verify RCCL Recorder's integrity in multithread context by comparing Recorder
    * instance across different threads.

projects/rccl/tools/scripts/test_runner/configs/ci-precheckin.json

@@ -86,6 +86,7 @@
         {"name": "Recorder.ParseJson", "description": "Recorder parse JSON tests", "test_filter": "Recorder.ParseJson", "timeout": 60},
         {"name": "Recorder.VerifyMultithread", "description": "Recorder multithread verification tests", "test_filter": "Recorder.VerifyMultithread", "timeout": 60},
         {"name": "Recorder.RedOpRecordJson", "description": "Recorder RedOp record JSON output", "test_filter": "Recorder.RedOpRecordJson", "timeout": 60},
+        {"name": "Recorder.RedOpRecordBinary", "description": "Recorder RedOp record binary output", "test_filter": "Recorder.RedOpRecordBinary", "timeout": 60},
         {"name": "ProxyTraceTestFixture.nonEmptySingleton", "description": "ProxyTrace non-empty singleton test", "test_filter": "ProxyTraceTestFixture.nonEmptySingleton", "timeout": 60},
         {"name": "ProxyTraceTestFixture.addTraceOp", "description": "ProxyTrace add trace operation test", "test_filter": "ProxyTraceTestFixture.addTraceOp", "timeout": 60},
         {"name": "ProxyTraceTestFixture.getMapSizeMB", "description": "ProxyTrace get map size test", "test_filter": "ProxyTraceTestFixture.getMapSizeMB", "timeout": 60},

projects/rccl/tools/scripts/test_runner/configs/mi300_mellanox_ib_func.json

@@ -1833,6 +1833,11 @@
           "description": "Recorder RedOp record JSON output",
           "test_filter": "Recorder.RedOpRecordJson"
         },
+        {
+          "name": "Recorder.RedOpRecordBinary",
+          "description": "Recorder RedOp record binary output",
+          "test_filter": "Recorder.RedOpRecordBinary"
+        },
         {
           "name": "ProxyTrace.nonEmptySingleton",
           "description": "Proxy trace non-empty singleton",

projects/rccl/tools/scripts/test_runner/configs/mi300x_mellanox_ib.json

@@ -1196,6 +1196,7 @@
         {"name": "Recorder.ParseJson", "description": "Recorder parse JSON trace", "test_filter": "Recorder.ParseJson"},
         {"name": "Recorder.VerifyMultithread", "description": "Recorder multithreaded verification", "test_filter": "Recorder.VerifyMultithread"},
         {"name": "Recorder.RedOpRecordJson", "description": "Recorder RedOp record JSON output", "test_filter": "Recorder.RedOpRecordJson"},
+        {"name": "Recorder.RedOpRecordBinary", "description": "Recorder RedOp record binary output", "test_filter": "Recorder.RedOpRecordBinary"},
         {"name": "IbvSymbolsEnv.DefaultLoadsLibrary", "description": "NCCL_IBVERBS_LIB unset loads default libibverbs", "test_filter": "IbvSymbolsEnv.DefaultLoadsLibrary"},
         {"name": "IbvSymbolsEnv.EnvOverrideValidSoname", "description": "NCCL_IBVERBS_LIB override to a valid soname is honored", "test_filter": "IbvSymbolsEnv.EnvOverrideValidSoname"},
         {"name": "IbvSymbolsEnv.BogusPathFallsBackToDefault", "description": "NCCL_IBVERBS_LIB bogus path falls back to default libibverbs", "test_filter": "IbvSymbolsEnv.BogusPathFallsBackToDefault"},

projects/rccl/tools/scripts/test_runner/configs/mi355x_thor2_roce.json

@@ -1427,6 +1427,11 @@
           "description": "Recorder RedOp record JSON output",
           "test_filter": "Recorder.RedOpRecordJson"
         },
+        {
+          "name": "Recorder.RedOpRecordBinary",
+          "description": "Recorder RedOp record binary output",
+          "test_filter": "Recorder.RedOpRecordBinary"
+        },
         {
           "name": "ProxyTrace.nonEmptySingleton",
           "description": "Proxy trace non-empty singleton",

projects/rccl/tools/scripts/test_runner/configs/mi355x_thor2_roce_func.json

@@ -1883,6 +1883,11 @@
           "description": "Recorder RedOp record JSON output",
           "test_filter": "Recorder.RedOpRecordJson"
         },
+        {
+          "name": "Recorder.RedOpRecordBinary",
+          "description": "Recorder RedOp record binary output",
+          "test_filter": "Recorder.RedOpRecordBinary"
+        },
         {
           "name": "ProxyTrace.nonEmptySingleton",
           "description": "Proxy trace non-empty singleton",

projects/rccl/tools/scripts/test_runner/configs/mi455_ainic_roce.json

@@ -1970,6 +1970,11 @@
           "description": "Recorder RedOp record JSON output",
           "test_filter": "Recorder.RedOpRecordJson"
         },
+        {
+          "name": "Recorder.RedOpRecordBinary",
+          "description": "Recorder RedOp record binary output",
+          "test_filter": "Recorder.RedOpRecordBinary"
+        },
         {
           "name": "ProxyTrace.nonEmptySingleton",
           "description": "Proxy trace non-empty singleton",

projects/rccl/tools/scripts/test_runner/configs/mi455_ainic_roce_func.json

@@ -2024,6 +2024,11 @@
           "description": "Recorder RedOp record JSON output",
           "test_filter": "Recorder.RedOpRecordJson"
         },
+        {
+          "name": "Recorder.RedOpRecordBinary",
+          "description": "Recorder RedOp record binary output",
+          "test_filter": "Recorder.RedOpRecordBinary"
+        },
         {
           "name": "ProxyTrace.nonEmptySingleton",
           "description": "Proxy trace non-empty singleton",