Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions projects/rccl/test/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -347,6 +347,7 @@ if(BUILD_TESTS)
IpcsocketTests.cpp
TopoEnvPolicyTests.cpp
NetSocketTests.cpp
PluginCompatV11_test.cpp
ProxyTests.cpp
RcclWrapTests.cpp
TransportTests.cpp
Expand All @@ -367,6 +368,12 @@ if(BUILD_TESTS)
# so they can build the path.
target_compile_definitions(rccl-UnitTestsFixturesDebug PRIVATE
RCCL_TEST_SOURCE_DIR="${CMAKE_CURRENT_SOURCE_DIR}")

# PluginCompatV11_test.cpp resolves its in-process mock plugin symbols via
# dlsym(dlopen(NULL), ...), which only sees the executable's dynamic symbols.
target_link_options(rccl-UnitTestsFixturesDebug PRIVATE
-Wl,--export-dynamic-symbol=ncclNetPlugin_v11
-Wl,--export-dynamic-symbol=ncclCollNetPlugin_v11)
endif()

# rccl-UnitTestsAltRsmi: Compiles alt_rsmi.cc directly (not via librccl.so) with
Expand Down
272 changes: 272 additions & 0 deletions projects/rccl/test/PluginCompatV11_test.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,272 @@
/*************************************************************************
* Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved.
*
* See LICENSE.txt for license information
************************************************************************/

// Regression test for AICOMRCCL-1532: the v11 network-plugin compat layer must
// initialize lazily. getNcclNet_v11() must populate only ncclNet.name/.init;
// the remaining entry points must stay null until ncclNet.init() runs (the old
// bug wired them up eagerly, exposing an uninitialized plugin).
//
// Uses a purely in-process mock: it exports the ncclNetPlugin_v11 /
// ncclCollNetPlugin_v11 symbols the compat layer resolves via dlsym, hands it a
// dlopen(NULL) handle, and checks that only ->init is set before init() and the
// rest become non-null after. No GPU or real plugin required.

#include <gtest/gtest.h>

#include <dlfcn.h>

#include "nccl.h"
#include "nccl_net.h"

// Compat-layer entry points from librccl.so (net_v11.cc); global-namespace C++
// functions, so this prototype matches the (Debug) library's mangled symbol.
ncclNet_t* getNcclNet_v11(void* lib);
ncclCollNet_t* getNcclCollNet_v11(void* lib);

namespace RcclUnitTesting {
namespace {

// Mock v11 net plugin: each entry point is a distinct non-null stub so we can
// tell "wired up" from "still null"; init() just returns success.
ncclResult_t mockNetInit(void** ctx, uint64_t, ncclNetCommConfig_v11_t*,
ncclDebugLogger_t, ncclProfilerCallback_t) {
if (ctx) { *ctx = nullptr; }
return ncclSuccess;
}
ncclResult_t mockNetDevices(int* ndev) { if (ndev) { *ndev = 1; } return ncclSuccess; }
ncclResult_t mockNetGetProperties(int, ncclNetProperties_v11_t*) { return ncclSuccess; }
ncclResult_t mockNetListen(void*, int, void*, void**) { return ncclSuccess; }
ncclResult_t mockNetConnect(void*, int, void*, void**, ncclNetDeviceHandle_v11_t**) { return ncclSuccess; }
ncclResult_t mockNetAccept(void*, void**, ncclNetDeviceHandle_v11_t**) { return ncclSuccess; }
ncclResult_t mockNetRegMr(void*, void*, size_t, int, void**) { return ncclSuccess; }
ncclResult_t mockNetRegMrDmaBuf(void*, void*, size_t, int, uint64_t, int, void**) { return ncclSuccess; }
ncclResult_t mockNetDeregMr(void*, void*) { return ncclSuccess; }
ncclResult_t mockNetIsend(void*, void*, size_t, int, void*, void*, void**) { return ncclSuccess; }
ncclResult_t mockNetIrecv(void*, int, void**, size_t*, int*, void**, void**, void**) { return ncclSuccess; }
ncclResult_t mockNetIflush(void*, int, void**, int*, void**, void**) { return ncclSuccess; }
ncclResult_t mockNetTest(void*, int*, int*) { return ncclSuccess; }
ncclResult_t mockNetCloseSend(void*) { return ncclSuccess; }
ncclResult_t mockNetCloseRecv(void*) { return ncclSuccess; }
ncclResult_t mockNetCloseListen(void*) { return ncclSuccess; }
ncclResult_t mockNetGetDeviceMr(void*, void*, void**) { return ncclSuccess; }
ncclResult_t mockNetIrecvConsumed(void*, int, void*) { return ncclSuccess; }
ncclResult_t mockNetMakeVDevice(int*, ncclNetVDeviceProps_v11_t*) { return ncclSuccess; }
ncclResult_t mockNetFinalize(void*) { return ncclSuccess; }
ncclResult_t mockNetSetNetAttr(void*, ncclNetAttr_v11_t*) { return ncclSuccess; }

// ---------------------------------------------------------------------------
// Mock v11 collnet plugin (same lazy-init contract as the net plugin).
// ---------------------------------------------------------------------------
ncclResult_t mockCollInit(void** ctx, uint64_t, ncclDebugLogger_t) {
if (ctx) { *ctx = nullptr; }
return ncclSuccess;
}
ncclResult_t mockCollDevices(int* ndev) { if (ndev) { *ndev = 1; } return ncclSuccess; }
ncclResult_t mockCollGetProperties(int, ncclNetProperties_v11_t*) { return ncclSuccess; }
ncclResult_t mockCollListen(void*, int, void*, void**) { return ncclSuccess; }
ncclResult_t mockCollConnect(void**, int, int, void*, void**) { return ncclSuccess; }
ncclResult_t mockCollReduceSupport(ncclDataType_t, ncclRedOp_t, int*) { return ncclSuccess; }
ncclResult_t mockCollRegMr(void*, void*, size_t, int, void**) { return ncclSuccess; }
ncclResult_t mockCollRegMrDmaBuf(void*, void*, size_t, int, uint64_t, int, void**) { return ncclSuccess; }
ncclResult_t mockCollDeregMr(void*, void*) { return ncclSuccess; }
ncclResult_t mockCollIallreduce(void*, void*, void*, size_t, ncclDataType_t, ncclRedOp_t, void*, void*, void**) { return ncclSuccess; }
ncclResult_t mockCollIallgather(void*, void*, int, ncclNetSGE_v11_t*, size_t, size_t, size_t, void*, void**) { return ncclSuccess; }
ncclResult_t mockCollIreducescatter(void*, int, ncclNetSGE_v11_t*, void*, size_t, size_t, size_t, ncclDataType_t, ncclRedOp_t, void*, void**) { return ncclSuccess; }
ncclResult_t mockCollIflush(void*, void*, int, void*, void**) { return ncclSuccess; }
ncclResult_t mockCollTest(void*, int*, int*) { return ncclSuccess; }
ncclResult_t mockCollCloseColl(void*) { return ncclSuccess; }
ncclResult_t mockCollCloseListen(void*) { return ncclSuccess; }
ncclResult_t mockCollMakeVDevice(int*, ncclNetVDeviceProps_v11_t*) { return ncclSuccess; }
ncclResult_t mockCollFinalize(void*) { return ncclSuccess; }

} // namespace
} // namespace RcclUnitTesting

// These symbols are what getNcclNet_v11()/getNcclCollNet_v11() look up via
// dlsym(). The CMake target exports them (--export-dynamic-symbol) so they land
// in the test executable's dynamic symbol table; they must be named exactly, so
// we place them at global scope with C linkage and default visibility.
extern "C" {

__attribute__((visibility("default")))
ncclNet_v11_t ncclNetPlugin_v11 = {
"mock_v11_net", // name
RcclUnitTesting::mockNetInit, // init
RcclUnitTesting::mockNetDevices, // devices
RcclUnitTesting::mockNetGetProperties, // getProperties
RcclUnitTesting::mockNetListen, // listen
RcclUnitTesting::mockNetConnect, // connect
RcclUnitTesting::mockNetAccept, // accept
RcclUnitTesting::mockNetRegMr, // regMr
RcclUnitTesting::mockNetRegMrDmaBuf, // regMrDmaBuf
RcclUnitTesting::mockNetDeregMr, // deregMr
RcclUnitTesting::mockNetIsend, // isend
RcclUnitTesting::mockNetIrecv, // irecv
RcclUnitTesting::mockNetIflush, // iflush
RcclUnitTesting::mockNetTest, // test
RcclUnitTesting::mockNetCloseSend, // closeSend
RcclUnitTesting::mockNetCloseRecv, // closeRecv
RcclUnitTesting::mockNetCloseListen, // closeListen
RcclUnitTesting::mockNetGetDeviceMr, // getDeviceMr
RcclUnitTesting::mockNetIrecvConsumed, // irecvConsumed
RcclUnitTesting::mockNetMakeVDevice, // makeVDevice
RcclUnitTesting::mockNetFinalize, // finalize
RcclUnitTesting::mockNetSetNetAttr, // setNetAttr
};

__attribute__((visibility("default")))
ncclCollNet_v11_t ncclCollNetPlugin_v11 = {
"mock_v11_collnet", // name
RcclUnitTesting::mockCollInit, // init
RcclUnitTesting::mockCollDevices, // devices
RcclUnitTesting::mockCollGetProperties, // getProperties
RcclUnitTesting::mockCollListen, // listen
RcclUnitTesting::mockCollConnect, // connect
RcclUnitTesting::mockCollReduceSupport, // reduceSupport
RcclUnitTesting::mockCollRegMr, // regMr
RcclUnitTesting::mockCollRegMrDmaBuf, // regMrDmaBuf
RcclUnitTesting::mockCollDeregMr, // deregMr
RcclUnitTesting::mockCollIallreduce, // iallreduce
RcclUnitTesting::mockCollIallgather, // iallgather
RcclUnitTesting::mockCollIreducescatter, // ireducescatter
RcclUnitTesting::mockCollIflush, // iflush
RcclUnitTesting::mockCollTest, // test
RcclUnitTesting::mockCollCloseColl, // closeColl
RcclUnitTesting::mockCollCloseListen, // closeListen
RcclUnitTesting::mockCollMakeVDevice, // makeVDevice
RcclUnitTesting::mockCollFinalize, // finalize
};

} // extern "C"

namespace RcclUnitTesting {

// getNcclNet_v11() must hand back a real vtable with ONLY ->init populated;
// every other entry point stays null until init() runs, and becomes non-null
// afterwards. This is the core AICOMRCCL-1532 contract - the pre-fix code
// populated the whole vtable eagerly and would fail the "before init()" block.
TEST(PluginCompatV11, NetLazyInitialization) {
void* self = dlopen(nullptr, RTLD_NOW | RTLD_GLOBAL);
ASSERT_NE(self, nullptr) << "dlopen(NULL) failed: " << dlerror();

ncclNet_t* net = getNcclNet_v11(self);
ASSERT_NE(net, nullptr)
<< "getNcclNet_v11 returned null; the in-process ncclNetPlugin_v11 symbol "
"was not resolved (is the test linked with -rdynamic?)";

// init() is the single entry point the compat layer is allowed to populate up
// front, because it is the callback that performs the lazy initialization.
EXPECT_NE(net->init, nullptr) << "compat layer must set ncclNet.init";

// Everything else must still be null: the plugin has not been init()'d, so no
// per-device entry point may be exposed yet.
EXPECT_EQ(net->devices, nullptr) << "devices set before init()";
EXPECT_EQ(net->getProperties, nullptr) << "getProperties set before init()";
EXPECT_EQ(net->listen, nullptr) << "listen set before init()";
EXPECT_EQ(net->connect, nullptr) << "connect set before init()";
EXPECT_EQ(net->accept, nullptr) << "accept set before init()";
EXPECT_EQ(net->regMr, nullptr) << "regMr set before init()";
EXPECT_EQ(net->regMrDmaBuf, nullptr) << "regMrDmaBuf set before init()";
EXPECT_EQ(net->deregMr, nullptr) << "deregMr set before init()";
EXPECT_EQ(net->isend, nullptr) << "isend set before init()";
EXPECT_EQ(net->irecv, nullptr) << "irecv set before init()";
EXPECT_EQ(net->iflush, nullptr) << "iflush set before init()";
EXPECT_EQ(net->test, nullptr) << "test set before init()";
EXPECT_EQ(net->closeSend, nullptr) << "closeSend set before init()";
EXPECT_EQ(net->closeRecv, nullptr) << "closeRecv set before init()";
EXPECT_EQ(net->closeListen, nullptr) << "closeListen set before init()";
EXPECT_EQ(net->getDeviceMr, nullptr) << "getDeviceMr set before init()";
EXPECT_EQ(net->irecvConsumed, nullptr) << "irecvConsumed set before init()";
EXPECT_EQ(net->makeVDevice, nullptr) << "makeVDevice set before init()";
EXPECT_EQ(net->finalize, nullptr) << "finalize set before init()";
EXPECT_EQ(net->setNetAttr, nullptr) << "setNetAttr set before init()";

// Run the lazy init. The compat layer now wires up the remaining entry points
// from the (mock) v11 plugin struct.
void* ctx = nullptr;
ASSERT_EQ(net->init(&ctx, /*commId=*/0, /*config=*/nullptr,
/*logFunction=*/nullptr, /*profFunction=*/nullptr),
ncclSuccess);

EXPECT_NE(net->devices, nullptr) << "devices null after init()";
EXPECT_NE(net->getProperties, nullptr) << "getProperties null after init()";
EXPECT_NE(net->listen, nullptr) << "listen null after init()";
EXPECT_NE(net->connect, nullptr) << "connect null after init()";
EXPECT_NE(net->accept, nullptr) << "accept null after init()";
EXPECT_NE(net->regMr, nullptr) << "regMr null after init()";
EXPECT_NE(net->regMrDmaBuf, nullptr) << "regMrDmaBuf null after init()";
EXPECT_NE(net->deregMr, nullptr) << "deregMr null after init()";
EXPECT_NE(net->isend, nullptr) << "isend null after init()";
EXPECT_NE(net->irecv, nullptr) << "irecv null after init()";
EXPECT_NE(net->iflush, nullptr) << "iflush null after init()";
EXPECT_NE(net->test, nullptr) << "test null after init()";
EXPECT_NE(net->closeSend, nullptr) << "closeSend null after init()";
EXPECT_NE(net->closeRecv, nullptr) << "closeRecv null after init()";
EXPECT_NE(net->closeListen, nullptr) << "closeListen null after init()";
EXPECT_NE(net->getDeviceMr, nullptr) << "getDeviceMr null after init()";
EXPECT_NE(net->irecvConsumed, nullptr) << "irecvConsumed null after init()";
EXPECT_NE(net->makeVDevice, nullptr) << "makeVDevice null after init()";
EXPECT_NE(net->finalize, nullptr) << "finalize null after init()";
EXPECT_NE(net->setNetAttr, nullptr) << "setNetAttr null after init()";

dlclose(self);
}

// Same lazy-init contract for the collnet compat layer.
TEST(PluginCompatV11, CollNetLazyInitialization) {
void* self = dlopen(nullptr, RTLD_NOW | RTLD_GLOBAL);
ASSERT_NE(self, nullptr) << "dlopen(NULL) failed: " << dlerror();

ncclCollNet_t* coll = getNcclCollNet_v11(self);
ASSERT_NE(coll, nullptr)
<< "getNcclCollNet_v11 returned null; the in-process ncclCollNetPlugin_v11 "
"symbol was not resolved (is the test linked with -rdynamic?)";

EXPECT_NE(coll->init, nullptr) << "compat layer must set ncclCollNet.init";

EXPECT_EQ(coll->devices, nullptr) << "devices set before init()";
EXPECT_EQ(coll->getProperties, nullptr) << "getProperties set before init()";
EXPECT_EQ(coll->listen, nullptr) << "listen set before init()";
EXPECT_EQ(coll->connect, nullptr) << "connect set before init()";
EXPECT_EQ(coll->reduceSupport, nullptr) << "reduceSupport set before init()";
EXPECT_EQ(coll->regMr, nullptr) << "regMr set before init()";
EXPECT_EQ(coll->regMrDmaBuf, nullptr) << "regMrDmaBuf set before init()";
EXPECT_EQ(coll->deregMr, nullptr) << "deregMr set before init()";
EXPECT_EQ(coll->iallreduce, nullptr) << "iallreduce set before init()";
EXPECT_EQ(coll->iallgather, nullptr) << "iallgather set before init()";
EXPECT_EQ(coll->ireducescatter, nullptr) << "ireducescatter set before init()";
EXPECT_EQ(coll->iflush, nullptr) << "iflush set before init()";
EXPECT_EQ(coll->test, nullptr) << "test set before init()";
EXPECT_EQ(coll->closeColl, nullptr) << "closeColl set before init()";
EXPECT_EQ(coll->closeListen, nullptr) << "closeListen set before init()";
EXPECT_EQ(coll->makeVDevice, nullptr) << "makeVDevice set before init()";
EXPECT_EQ(coll->finalize, nullptr) << "finalize set before init()";

void* ctx = nullptr;
ASSERT_EQ(coll->init(&ctx, /*commId=*/0, /*logFunction=*/nullptr), ncclSuccess);

EXPECT_NE(coll->devices, nullptr) << "devices null after init()";
EXPECT_NE(coll->getProperties, nullptr) << "getProperties null after init()";
EXPECT_NE(coll->listen, nullptr) << "listen null after init()";
EXPECT_NE(coll->connect, nullptr) << "connect null after init()";
EXPECT_NE(coll->reduceSupport, nullptr) << "reduceSupport null after init()";
EXPECT_NE(coll->regMr, nullptr) << "regMr null after init()";
EXPECT_NE(coll->regMrDmaBuf, nullptr) << "regMrDmaBuf null after init()";
EXPECT_NE(coll->deregMr, nullptr) << "deregMr null after init()";
EXPECT_NE(coll->iallreduce, nullptr) << "iallreduce null after init()";
EXPECT_NE(coll->iallgather, nullptr) << "iallgather null after init()";
EXPECT_NE(coll->ireducescatter, nullptr) << "ireducescatter null after init()";
EXPECT_NE(coll->iflush, nullptr) << "iflush null after init()";
EXPECT_NE(coll->test, nullptr) << "test null after init()";
EXPECT_NE(coll->closeColl, nullptr) << "closeColl null after init()";
EXPECT_NE(coll->closeListen, nullptr) << "closeListen null after init()";
EXPECT_NE(coll->makeVDevice, nullptr) << "makeVDevice null after init()";
EXPECT_NE(coll->finalize, nullptr) << "finalize null after init()";

dlclose(self);
}

} // namespace RcclUnitTesting
Original file line number Diff line number Diff line change
Expand Up @@ -783,6 +783,21 @@
{"name": "AsymmetricVisibility.CommInitRankAllReduce", "description": "ROCM-27034: RCCL comm init + AllReduce under asymmetric HIP_VISIBLE_DEVICES", "test_filter": "AsymmetricVisibility.CommInitRankAllReduce"}
]
},
"plugin_compat_v11": {
"is_gtest": true,
"binary": "rccl-UnitTestsFixturesDebug",
"num_ranks": 1,
"num_nodes": 1,
"num_gpus": 1,
"timeout": 60,
"rerun_env_variables": {
"NCCL_DEBUG": "INFO"
},
"tests": [
{"name": "PluginCompatV11.NetLazyInitialization", "description": "AICOMRCCL-1532: v11 net plugin compat layer initializes lazily (only ncclNet.init set until init() runs)", "test_filter": "PluginCompatV11.NetLazyInitialization"},
{"name": "PluginCompatV11.CollNetLazyInitialization", "description": "AICOMRCCL-1532: v11 collnet plugin compat layer initializes lazily", "test_filter": "PluginCompatV11.CollNetLazyInitialization"}
]
},
"debug_tests": {
"is_gtest": true,
"binary": "rccl-UnitTests",
Expand Down Expand Up @@ -3257,6 +3272,12 @@
"config": "unit_tests_fixtures_debug",
"enabled": true
},
{
"name": "Plugin Compat v11 - Lazy Init",
"description": "AICOMRCCL-1532: v11 net/collnet plugin compat layer lazy initialization (Debug only)",
"config": "plugin_compat_v11",
"enabled": true
},
{
"name": "Asymmetric Visibility",
"description": "Comm init + AllReduce under asymmetric HIP_VISIBLE_DEVICES",
Expand Down
Loading