From 0cf7084a1f1862391bd48b580afe86e5e2bf952e Mon Sep 17 00:00:00 2001 From: Alex Duforest Date: Tue, 20 Oct 2026 11:05:07 -0700 Subject: [PATCH 01/15] test(rccl): add net_ib_cast scheduler, recovery and GDR probe fakes Seams for scheduler.cc, p2p_resiliency_recovery.cc, gdr_peermem.cc and gdr_probe.cc, and transport_stubs gains net.cc's NET_OPTIONAL_RECV_COMPLETION param, for net_ib_cast microtest binaries that do not compile those TUs. Co-Authored-By: Claude --- .../rccl/test/host/fakes/gdr_peermem_fakes.cc | 24 +++++++++ .../rccl/test/host/fakes/gdr_peermem_fakes.h | 19 +++++++ .../rccl/test/host/fakes/gdr_probe_fakes.cc | 26 ++++++++++ .../rccl/test/host/fakes/gdr_probe_fakes.h | 22 ++++++++ ...t_ib_cast_p2p_resiliency_recovery_fakes.cc | 50 +++++++++++++++++++ ...et_ib_cast_p2p_resiliency_recovery_fakes.h | 26 ++++++++++ .../host/fakes/net_ib_cast_scheduler_fakes.cc | 38 ++++++++++++++ .../host/fakes/net_ib_cast_scheduler_fakes.h | 26 ++++++++++ .../rccl/test/host/fakes/transport_stubs.cc | 3 ++ 9 files changed, 234 insertions(+) create mode 100644 projects/rccl/test/host/fakes/gdr_peermem_fakes.cc create mode 100644 projects/rccl/test/host/fakes/gdr_peermem_fakes.h create mode 100644 projects/rccl/test/host/fakes/gdr_probe_fakes.cc create mode 100644 projects/rccl/test/host/fakes/gdr_probe_fakes.h create mode 100644 projects/rccl/test/host/fakes/net_ib_cast_p2p_resiliency_recovery_fakes.cc create mode 100644 projects/rccl/test/host/fakes/net_ib_cast_p2p_resiliency_recovery_fakes.h create mode 100644 projects/rccl/test/host/fakes/net_ib_cast_scheduler_fakes.cc create mode 100644 projects/rccl/test/host/fakes/net_ib_cast_scheduler_fakes.h diff --git a/projects/rccl/test/host/fakes/gdr_peermem_fakes.cc b/projects/rccl/test/host/fakes/gdr_peermem_fakes.cc new file mode 100644 index 000000000000..3b9a98fbfdc8 --- /dev/null +++ b/projects/rccl/test/host/fakes/gdr_peermem_fakes.cc @@ -0,0 +1,24 @@ +/************************************************************************* + * Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +// Implementation of the src/misc/gdr_peermem.cc fakes. See gdr_peermem_fakes.h. + +#include "gdr_peermem_fakes.h" + +#include "fail_loud.h" +#include "gdr_peermem.h" +#include "signature-drift.h" + +static int DefaultNcclIbScanDefaultPeerMemClients() { + FailLoudUnfaked("gdr_peermem_fakes", "ncclIbScanDefaultPeerMemClients"); +} +std::function g_ncclIbScanDefaultPeerMemClients = DefaultNcclIbScanDefaultPeerMemClients; + +ASSERT_HOOK_MATCHES_PROD(g_ncclIbScanDefaultPeerMemClients, ncclIbScanDefaultPeerMemClients); + +int ncclIbScanDefaultPeerMemClients(void) { return g_ncclIbScanDefaultPeerMemClients(); } + +void ResetGdrPeermemFakes() { g_ncclIbScanDefaultPeerMemClients = DefaultNcclIbScanDefaultPeerMemClients; } diff --git a/projects/rccl/test/host/fakes/gdr_peermem_fakes.h b/projects/rccl/test/host/fakes/gdr_peermem_fakes.h new file mode 100644 index 000000000000..68ab527b4e08 --- /dev/null +++ b/projects/rccl/test/host/fakes/gdr_peermem_fakes.h @@ -0,0 +1,19 @@ +/************************************************************************* + * Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +// Seam for ncclIbScanDefaultPeerMemClients (src/misc/gdr_peermem.cc), whose real +// answer depends on the host's /sys. Fails loud by default. + +#ifndef RCCL_TEST_HOST_GDR_PEERMEM_FAKES_H_ +#define RCCL_TEST_HOST_GDR_PEERMEM_FAKES_H_ + +#include + +extern std::function g_ncclIbScanDefaultPeerMemClients; + +void ResetGdrPeermemFakes(); + +#endif // RCCL_TEST_HOST_GDR_PEERMEM_FAKES_H_ diff --git a/projects/rccl/test/host/fakes/gdr_probe_fakes.cc b/projects/rccl/test/host/fakes/gdr_probe_fakes.cc new file mode 100644 index 000000000000..7a7616345970 --- /dev/null +++ b/projects/rccl/test/host/fakes/gdr_probe_fakes.cc @@ -0,0 +1,26 @@ +/************************************************************************* + * Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +// Implementation of the src/misc/gdr_probe.cc fakes. See gdr_probe_fakes.h. + +#include "gdr_probe_fakes.h" + +#include "fail_loud.h" +#include "gdr_peermem.h" +#include "signature-drift.h" + +static int DefaultNcclIbProbeGdrSupport(struct ibv_context*, int) { + FailLoudUnfaked("gdr_probe_fakes", "ncclIbProbeGdrSupport"); +} +std::function g_ncclIbProbeGdrSupport = DefaultNcclIbProbeGdrSupport; + +ASSERT_HOOK_MATCHES_PROD(g_ncclIbProbeGdrSupport, ncclIbProbeGdrSupport); + +int ncclIbProbeGdrSupport(struct ibv_context* context, int relaxedOrderingEnabled) { + return g_ncclIbProbeGdrSupport(context, relaxedOrderingEnabled); +} + +void ResetGdrProbeFakes() { g_ncclIbProbeGdrSupport = DefaultNcclIbProbeGdrSupport; } diff --git a/projects/rccl/test/host/fakes/gdr_probe_fakes.h b/projects/rccl/test/host/fakes/gdr_probe_fakes.h new file mode 100644 index 000000000000..f6d547c1defc --- /dev/null +++ b/projects/rccl/test/host/fakes/gdr_probe_fakes.h @@ -0,0 +1,22 @@ +/************************************************************************* + * Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +// Seam for ncclIbProbeGdrSupport (src/misc/gdr_probe.cc), for binaries that do not +// compile gdr_probe.cc (rccl-UnitTestsMicro does, so it cannot link this). Fails +// loud by default. + +#ifndef RCCL_TEST_HOST_GDR_PROBE_FAKES_H_ +#define RCCL_TEST_HOST_GDR_PROBE_FAKES_H_ + +#include + +struct ibv_context; + +extern std::function g_ncclIbProbeGdrSupport; + +void ResetGdrProbeFakes(); + +#endif // RCCL_TEST_HOST_GDR_PROBE_FAKES_H_ diff --git a/projects/rccl/test/host/fakes/net_ib_cast_p2p_resiliency_recovery_fakes.cc b/projects/rccl/test/host/fakes/net_ib_cast_p2p_resiliency_recovery_fakes.cc new file mode 100644 index 000000000000..d0e82a8f8da1 --- /dev/null +++ b/projects/rccl/test/host/fakes/net_ib_cast_p2p_resiliency_recovery_fakes.cc @@ -0,0 +1,50 @@ +/************************************************************************* + * Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +// Implementation of the src/transport/net_ib_cast/p2p_resiliency_recovery.cc +// fakes. See net_ib_cast_p2p_resiliency_recovery_fakes.h. + +#include "net_ib_cast_p2p_resiliency_recovery_fakes.h" + +#include "fail_loud.h" +#include "nccl_fakes.h" +#include "param_redirect.h" +#include "p2p_resiliency_recovery_cast.h" +#include "signature-drift.h" + +// Same line as p2p_resiliency_recovery.cc; test/host/CMakeLists.txt pins both copies. +NCCL_PARAM(IbCastResiliencyPortRecovery, "IB_RESILIENCY_PORT_RECOVERY", 0); + +static ncclResult_t DefaultIbCastPortRecoveryThreadStart() { + FailLoudUnfaked("net_ib_cast_p2p_resiliency_recovery_fakes", "IbCastPortRecoveryThreadStart"); +} +std::function g_ibCastPortRecoveryThreadStart = DefaultIbCastPortRecoveryThreadStart; + +static ncclResult_t DefaultIbCastPortRecoveryThreadStop() { + FailLoudUnfaked("net_ib_cast_p2p_resiliency_recovery_fakes", "IbCastPortRecoveryThreadStop"); +} +std::function g_ibCastPortRecoveryThreadStop = DefaultIbCastPortRecoveryThreadStop; + +static ncclResult_t DefaultIbCastPortRecoveryQpUdToRts(struct ncclIbQp*, int) { + FailLoudUnfaked("net_ib_cast_p2p_resiliency_recovery_fakes", "IbCastPortRecoveryQpUdToRts"); +} +std::function g_ibCastPortRecoveryQpUdToRts = DefaultIbCastPortRecoveryQpUdToRts; + +ASSERT_HOOK_MATCHES_PROD(g_ibCastPortRecoveryThreadStart, IbCastPortRecoveryThreadStart); +ASSERT_HOOK_MATCHES_PROD(g_ibCastPortRecoveryThreadStop, IbCastPortRecoveryThreadStop); +ASSERT_HOOK_MATCHES_PROD(g_ibCastPortRecoveryQpUdToRts, IbCastPortRecoveryQpUdToRts); + +ncclResult_t IbCastPortRecoveryThreadStart() { return g_ibCastPortRecoveryThreadStart(); } +ncclResult_t IbCastPortRecoveryThreadStop() { return g_ibCastPortRecoveryThreadStop(); } +ncclResult_t IbCastPortRecoveryQpUdToRts(struct ncclIbQp* qp, int portNum) { + return g_ibCastPortRecoveryQpUdToRts(qp, portNum); +} + +void ResetNetIbCastP2pResiliencyRecoveryFakes() { + g_ibCastPortRecoveryThreadStart = DefaultIbCastPortRecoveryThreadStart; + g_ibCastPortRecoveryThreadStop = DefaultIbCastPortRecoveryThreadStop; + g_ibCastPortRecoveryQpUdToRts = DefaultIbCastPortRecoveryQpUdToRts; +} diff --git a/projects/rccl/test/host/fakes/net_ib_cast_p2p_resiliency_recovery_fakes.h b/projects/rccl/test/host/fakes/net_ib_cast_p2p_resiliency_recovery_fakes.h new file mode 100644 index 000000000000..625747d0fef5 --- /dev/null +++ b/projects/rccl/test/host/fakes/net_ib_cast_p2p_resiliency_recovery_fakes.h @@ -0,0 +1,26 @@ +/************************************************************************* + * Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +// Seams for the src/transport/net_ib_cast/p2p_resiliency_recovery.cc symbols, for +// binaries that do not compile that TU, plus its IB_RESILIENCY_PORT_RECOVERY param +// through g_loadParam. Every hook fails loud by default. + +#ifndef RCCL_TEST_HOST_NET_IB_CAST_P2P_RESILIENCY_RECOVERY_FAKES_H_ +#define RCCL_TEST_HOST_NET_IB_CAST_P2P_RESILIENCY_RECOVERY_FAKES_H_ + +#include + +#include "nccl.h" + +struct ncclIbQp; + +extern std::function g_ibCastPortRecoveryThreadStart; +extern std::function g_ibCastPortRecoveryThreadStop; +extern std::function g_ibCastPortRecoveryQpUdToRts; + +void ResetNetIbCastP2pResiliencyRecoveryFakes(); + +#endif // RCCL_TEST_HOST_NET_IB_CAST_P2P_RESILIENCY_RECOVERY_FAKES_H_ diff --git a/projects/rccl/test/host/fakes/net_ib_cast_scheduler_fakes.cc b/projects/rccl/test/host/fakes/net_ib_cast_scheduler_fakes.cc new file mode 100644 index 000000000000..900d05e65ac3 --- /dev/null +++ b/projects/rccl/test/host/fakes/net_ib_cast_scheduler_fakes.cc @@ -0,0 +1,38 @@ +/************************************************************************* + * Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +// Implementation of the src/transport/net_ib_cast/scheduler.cc fakes. See +// net_ib_cast_scheduler_fakes.h. + +#include "net_ib_cast_scheduler_fakes.h" + +#include + +#include "common_cast.h" +#include "fail_loud.h" +#include "signature-drift.h" + +struct ncclIbQpSchedParms castGlobalQpSchedParms; + +static bool DefaultRcclUseIbCastQpSched() { FailLoudUnfaked("net_ib_cast_scheduler_fakes", "rcclUseIbCastQpSched"); } +std::function g_rcclUseIbCastQpSched = DefaultRcclUseIbCastQpSched; + +static ncclResult_t DefaultIbCastQpSchedInitParms(struct ncclIbQpSchedParms*) { + FailLoudUnfaked("net_ib_cast_scheduler_fakes", "IbCastQpSchedInitParms"); +} +std::function g_ibCastQpSchedInitParms = DefaultIbCastQpSchedInitParms; + +ASSERT_HOOK_MATCHES_PROD(g_rcclUseIbCastQpSched, rcclUseIbCastQpSched); +ASSERT_HOOK_MATCHES_PROD(g_ibCastQpSchedInitParms, IbCastQpSchedInitParms); + +bool rcclUseIbCastQpSched() { return g_rcclUseIbCastQpSched(); } +ncclResult_t IbCastQpSchedInitParms(struct ncclIbQpSchedParms* parms) { return g_ibCastQpSchedInitParms(parms); } + +void ResetNetIbCastSchedulerFakes() { + memset(&castGlobalQpSchedParms, 0, sizeof(castGlobalQpSchedParms)); + g_rcclUseIbCastQpSched = DefaultRcclUseIbCastQpSched; + g_ibCastQpSchedInitParms = DefaultIbCastQpSchedInitParms; +} diff --git a/projects/rccl/test/host/fakes/net_ib_cast_scheduler_fakes.h b/projects/rccl/test/host/fakes/net_ib_cast_scheduler_fakes.h new file mode 100644 index 000000000000..073dbf87ad36 --- /dev/null +++ b/projects/rccl/test/host/fakes/net_ib_cast_scheduler_fakes.h @@ -0,0 +1,26 @@ +/************************************************************************* + * Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +// Seams for the src/transport/net_ib_cast/scheduler.cc symbols, for binaries that +// do not compile scheduler.cc. castGlobalQpSchedParms starts zeroed (scheduler +// off), as in production before IbCastQpSchedInitParms runs; both hooks fail loud +// by default. + +#ifndef RCCL_TEST_HOST_NET_IB_CAST_SCHEDULER_FAKES_H_ +#define RCCL_TEST_HOST_NET_IB_CAST_SCHEDULER_FAKES_H_ + +#include + +#include "nccl.h" + +struct ncclIbQpSchedParms; + +extern std::function g_rcclUseIbCastQpSched; +extern std::function g_ibCastQpSchedInitParms; + +void ResetNetIbCastSchedulerFakes(); + +#endif // RCCL_TEST_HOST_NET_IB_CAST_SCHEDULER_FAKES_H_ diff --git a/projects/rccl/test/host/fakes/transport_stubs.cc b/projects/rccl/test/host/fakes/transport_stubs.cc index 02ab61f7d938..511a35f74fb0 100644 --- a/projects/rccl/test/host/fakes/transport_stubs.cc +++ b/projects/rccl/test/host/fakes/transport_stubs.cc @@ -37,6 +37,9 @@ struct ncclTopoGraph; // NCCL_PARAM does not cover it. int64_t ncclParamNvlsEnable() { return g_loadParam("NVLS_ENABLE", 2); } +// src/transport/net.cc:212, read by net_ib_cast/common.cc. +int64_t ncclParamNetOptionalRecvCompletion() { return g_loadParam("NET_OPTIONAL_RECV_COMPLETION", 1); } + // src/plugin/net.cc: initTransportsRank's GDR probe. int g_gdrSupportValue = 0; int g_gdrSupportCalls = 0; From f921ea43d41fcd1cd7ee77533efcd2785ad24b99 Mon Sep 17 00:00:00 2001 From: Alex Duforest Date: Tue, 20 Oct 2026 11:15:45 -0700 Subject: [PATCH 02/15] test(rccl): place the NET_OPTIONAL_RECV_COMPLETION stub next to rcclUseAinic Co-Authored-By: Claude --- projects/rccl/test/host/fakes/transport_stubs.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/projects/rccl/test/host/fakes/transport_stubs.cc b/projects/rccl/test/host/fakes/transport_stubs.cc index 511a35f74fb0..eedafab11448 100644 --- a/projects/rccl/test/host/fakes/transport_stubs.cc +++ b/projects/rccl/test/host/fakes/transport_stubs.cc @@ -37,9 +37,6 @@ struct ncclTopoGraph; // NCCL_PARAM does not cover it. int64_t ncclParamNvlsEnable() { return g_loadParam("NVLS_ENABLE", 2); } -// src/transport/net.cc:212, read by net_ib_cast/common.cc. -int64_t ncclParamNetOptionalRecvCompletion() { return g_loadParam("NET_OPTIONAL_RECV_COMPLETION", 1); } - // src/plugin/net.cc: initTransportsRank's GDR probe. int g_gdrSupportValue = 0; int g_gdrSupportCalls = 0; @@ -58,6 +55,9 @@ static bool DefaultUseAinic() { return g_rcclUseAinicValue; } std::function g_useAinic = DefaultUseAinic; bool rcclUseAinic() { return g_useAinic(); } +// src/transport/net.cc:212, read by net_ib_cast/common.cc. +int64_t ncclParamNetOptionalRecvCompletion() { return g_loadParam("NET_OPTIONAL_RECV_COMPLETION", 1); } + static int DefaultPxnDisable(struct ncclComm*) { return 0; } std::function g_pxnDisable = DefaultPxnDisable; #ifndef RCCL_TRANSPORT_STUBS_OMIT_ncclPxnDisable From 196c48ff79531b996ead15cc247ebbc63b75f49e Mon Sep 17 00:00:00 2001 From: Alex Duforest Date: Tue, 20 Oct 2026 12:04:47 -0700 Subject: [PATCH 03/15] test(rccl): add the shared verbs, socket and net_ib_cast fakes Byte-identical copies of the fakes the other net_ib_cast microtest PRs own (verbs, mlx5/ionic direct verbs, sockets, and the connect, p2p, p2p_resiliency and qp_sharing seams), and os_fakes' ncclOsStrSep as #13601 adds it. Co-Authored-By: Claude --- .../rccl/test/host/fakes/ibvwrap_fakes.cc | 237 ++++++++++++++++++ projects/rccl/test/host/fakes/ibvwrap_fakes.h | 67 ++++- .../rccl/test/host/fakes/ionicdvwrap_fakes.cc | 51 ++++ .../rccl/test/host/fakes/ionicdvwrap_fakes.h | 33 +++ .../rccl/test/host/fakes/mlx5dvwrap_fakes.cc | 88 +++++++ .../rccl/test/host/fakes/mlx5dvwrap_fakes.h | 49 ++++ .../host/fakes/net_ib_cast_connect_fakes.cc | 185 ++++++++++++++ .../host/fakes/net_ib_cast_connect_fakes.h | 53 ++++ .../test/host/fakes/net_ib_cast_p2p_fakes.cc | 119 +++++++++ .../test/host/fakes/net_ib_cast_p2p_fakes.h | 42 ++++ .../fakes/net_ib_cast_p2p_resiliency_fakes.cc | 220 ++++++++++++++++ .../fakes/net_ib_cast_p2p_resiliency_fakes.h | 60 +++++ .../fakes/net_ib_cast_qp_sharing_fakes.cc | 28 +++ .../host/fakes/net_ib_cast_qp_sharing_fakes.h | 18 ++ projects/rccl/test/host/fakes/os_fakes.cc | 2 + projects/rccl/test/host/fakes/socket_fakes.cc | 113 +++++++++ projects/rccl/test/host/fakes/socket_fakes.h | 46 ++++ 17 files changed, 1400 insertions(+), 11 deletions(-) create mode 100644 projects/rccl/test/host/fakes/ionicdvwrap_fakes.cc create mode 100644 projects/rccl/test/host/fakes/ionicdvwrap_fakes.h create mode 100644 projects/rccl/test/host/fakes/mlx5dvwrap_fakes.cc create mode 100644 projects/rccl/test/host/fakes/mlx5dvwrap_fakes.h create mode 100644 projects/rccl/test/host/fakes/net_ib_cast_connect_fakes.cc create mode 100644 projects/rccl/test/host/fakes/net_ib_cast_connect_fakes.h create mode 100644 projects/rccl/test/host/fakes/net_ib_cast_p2p_fakes.cc create mode 100644 projects/rccl/test/host/fakes/net_ib_cast_p2p_fakes.h create mode 100644 projects/rccl/test/host/fakes/net_ib_cast_p2p_resiliency_fakes.cc create mode 100644 projects/rccl/test/host/fakes/net_ib_cast_p2p_resiliency_fakes.h create mode 100644 projects/rccl/test/host/fakes/net_ib_cast_qp_sharing_fakes.cc create mode 100644 projects/rccl/test/host/fakes/net_ib_cast_qp_sharing_fakes.h create mode 100644 projects/rccl/test/host/fakes/socket_fakes.cc create mode 100644 projects/rccl/test/host/fakes/socket_fakes.h diff --git a/projects/rccl/test/host/fakes/ibvwrap_fakes.cc b/projects/rccl/test/host/fakes/ibvwrap_fakes.cc index dbaf99fa96a6..a87e64b0de44 100644 --- a/projects/rccl/test/host/fakes/ibvwrap_fakes.cc +++ b/projects/rccl/test/host/fakes/ibvwrap_fakes.cc @@ -30,11 +30,151 @@ std::function g_ib static ncclResult_t DefaultIbvDeregMr(struct ibv_mr*) { return ncclSuccess; } std::function g_ibvDeregMr = DefaultIbvDeregMr; +static ncclResult_t DefaultIbvRegMr(struct ibv_mr** ret, struct ibv_pd*, void*, size_t, int) { + if (ret) *ret = nullptr; + return ncclSystemError; +} +std::function g_ibvRegMr = DefaultIbvRegMr; + +static ncclResult_t DefaultIbvRegDmabufMr(struct ibv_mr** ret, struct ibv_pd*, uint64_t, size_t, uint64_t, int, int) { + if (ret) *ret = nullptr; + return ncclSystemError; +} +std::function g_ibvRegDmabufMr = + DefaultIbvRegDmabufMr; + +static ncclResult_t DefaultIbvCreateCq(struct ibv_cq** ret, struct ibv_context*, int, void*, struct ibv_comp_channel*, + int) { + if (ret) *ret = nullptr; + return ncclSystemError; +} +std::function + g_ibvCreateCq = DefaultIbvCreateCq; + +static ncclResult_t DefaultIbvDestroyCq(struct ibv_cq*) { return ncclSuccess; } +std::function g_ibvDestroyCq = DefaultIbvDestroyCq; + +static ncclResult_t DefaultIbvCreateQp(struct ibv_qp** ret, struct ibv_pd*, struct ibv_qp_init_attr*) { + if (ret) *ret = nullptr; + return ncclSystemError; +} +std::function g_ibvCreateQp = + DefaultIbvCreateQp; + +static ncclResult_t DefaultIbvModifyQp(struct ibv_qp*, struct ibv_qp_attr*, int) { return ncclSystemError; } +std::function g_ibvModifyQp = DefaultIbvModifyQp; + +static ncclResult_t DefaultIbvDestroyQp(struct ibv_qp*) { return ncclSuccess; } +std::function g_ibvDestroyQp = DefaultIbvDestroyQp; + +static ncclResult_t DefaultIbvQueryGid(struct ibv_context*, uint8_t, int, union ibv_gid*) { return ncclSystemError; } +std::function g_ibvQueryGid = DefaultIbvQueryGid; + +static ncclResult_t DefaultIbvQueryEce(struct ibv_qp*, struct ibv_ece*, int*) { return ncclSystemError; } +std::function g_ibvQueryEce = DefaultIbvQueryEce; + +static ncclResult_t DefaultIbvSetEce(struct ibv_qp*, struct ibv_ece*, int*) { return ncclSystemError; } +std::function g_ibvSetEce = DefaultIbvSetEce; + +static const char* DefaultIbvGetDeviceName(struct ibv_device*) { return nullptr; } +std::function g_ibvGetDeviceName = DefaultIbvGetDeviceName; + +static ncclResult_t DefaultIbvSymbols() { return ncclSystemError; } +std::function g_ibvSymbols = DefaultIbvSymbols; + +static ncclResult_t DefaultIbvForkInit() { return ncclSystemError; } +std::function g_ibvForkInit = DefaultIbvForkInit; + +static ncclResult_t DefaultIbvGetDeviceList(struct ibv_device*** ret, int* num_devices) { + if (ret) *ret = nullptr; + if (num_devices) *num_devices = 0; + return ncclSystemError; +} +std::function g_ibvGetDeviceList = DefaultIbvGetDeviceList; + +static ncclResult_t DefaultIbvFreeDeviceList(struct ibv_device**) { return ncclSuccess; } +std::function g_ibvFreeDeviceList = DefaultIbvFreeDeviceList; + +static ncclResult_t DefaultIbvOpenDevice(struct ibv_context** ret, struct ibv_device*) { + if (ret) *ret = nullptr; + return ncclSystemError; +} +std::function g_ibvOpenDevice = DefaultIbvOpenDevice; + +static ncclResult_t DefaultIbvCloseDevice(struct ibv_context*) { return ncclSuccess; } +std::function g_ibvCloseDevice = DefaultIbvCloseDevice; + +static ncclResult_t DefaultIbvGetAsyncEvent(struct ibv_context*, struct ibv_async_event*) { return ncclSystemError; } +std::function g_ibvGetAsyncEvent = DefaultIbvGetAsyncEvent; + +static ncclResult_t DefaultIbvAckAsyncEvent(struct ibv_async_event*) { return ncclSuccess; } +std::function g_ibvAckAsyncEvent = DefaultIbvAckAsyncEvent; + +static ncclResult_t DefaultIbvEventTypeStr(char** ret, enum ibv_event_type) { + if (ret) *ret = nullptr; + return ncclSystemError; +} +std::function g_ibvEventTypeStr = DefaultIbvEventTypeStr; + +static ncclResult_t DefaultIbvQueryDevice(struct ibv_context*, struct ibv_device_attr*) { return ncclSystemError; } +std::function g_ibvQueryDevice = DefaultIbvQueryDevice; + +static ncclResult_t DefaultIbvQueryPort(struct ibv_context*, uint8_t, struct ibv_port_attr*) { return ncclSystemError; } +std::function g_ibvQueryPort = DefaultIbvQueryPort; + +static ncclResult_t DefaultIbvQueryPortSpeed(struct ibv_context*, uint8_t, uint64_t*) { return ncclSystemError; } +std::function g_ibvQueryPortSpeed = DefaultIbvQueryPortSpeed; + +static ncclResult_t DefaultIbvQueryQp(struct ibv_qp*, struct ibv_qp_attr*, int, struct ibv_qp_init_attr*) { + return ncclSystemError; +} +std::function + g_ibvQueryQp = DefaultIbvQueryQp; + +static ncclResult_t DefaultIbvRegMrIova2(struct ibv_mr** ret, struct ibv_pd*, void*, size_t, uint64_t, int) { + if (ret) *ret = nullptr; + return ncclSystemError; +} +std::function + g_ibvRegMrIova2 = DefaultIbvRegMrIova2; + +static struct ibv_mr* DefaultIbvDirectRegDmabufMr(struct ibv_pd*, uint64_t, size_t, uint64_t, int, int) { + return nullptr; +} +std::function + g_ibvDirectRegDmabufMr = DefaultIbvDirectRegDmabufMr; + ASSERT_HOOK_MATCHES_PROD(g_ibvAllocPd, wrap_ibv_alloc_pd); ASSERT_HOOK_MATCHES_PROD(g_ibvDeallocPd, wrap_ibv_dealloc_pd); ASSERT_HOOK_MATCHES_PROD(g_ibvDirectRegMr, wrap_direct_ibv_reg_mr); ASSERT_HOOK_MATCHES_PROD(g_ibvDirectRegMrIova2, wrap_direct_ibv_reg_mr_iova2); ASSERT_HOOK_MATCHES_PROD(g_ibvDeregMr, wrap_ibv_dereg_mr); +ASSERT_HOOK_MATCHES_PROD(g_ibvRegMr, wrap_ibv_reg_mr); +ASSERT_HOOK_MATCHES_PROD(g_ibvRegDmabufMr, wrap_ibv_reg_dmabuf_mr); +ASSERT_HOOK_MATCHES_PROD(g_ibvCreateCq, wrap_ibv_create_cq); +ASSERT_HOOK_MATCHES_PROD(g_ibvDestroyCq, wrap_ibv_destroy_cq); +ASSERT_HOOK_MATCHES_PROD(g_ibvCreateQp, wrap_ibv_create_qp); +ASSERT_HOOK_MATCHES_PROD(g_ibvModifyQp, wrap_ibv_modify_qp); +ASSERT_HOOK_MATCHES_PROD(g_ibvDestroyQp, wrap_ibv_destroy_qp); +ASSERT_HOOK_MATCHES_PROD(g_ibvQueryGid, wrap_ibv_query_gid); +ASSERT_HOOK_MATCHES_PROD(g_ibvQueryEce, wrap_ibv_query_ece); +ASSERT_HOOK_MATCHES_PROD(g_ibvSetEce, wrap_ibv_set_ece); +ASSERT_HOOK_MATCHES_PROD(g_ibvGetDeviceName, wrap_ibv_get_device_name); +ASSERT_HOOK_MATCHES_PROD(g_ibvSymbols, wrap_ibv_symbols); +ASSERT_HOOK_MATCHES_PROD(g_ibvForkInit, wrap_ibv_fork_init); +ASSERT_HOOK_MATCHES_PROD(g_ibvGetDeviceList, wrap_ibv_get_device_list); +ASSERT_HOOK_MATCHES_PROD(g_ibvFreeDeviceList, wrap_ibv_free_device_list); +ASSERT_HOOK_MATCHES_PROD(g_ibvOpenDevice, wrap_ibv_open_device); +ASSERT_HOOK_MATCHES_PROD(g_ibvCloseDevice, wrap_ibv_close_device); +ASSERT_HOOK_MATCHES_PROD(g_ibvGetAsyncEvent, wrap_ibv_get_async_event); +ASSERT_HOOK_MATCHES_PROD(g_ibvAckAsyncEvent, wrap_ibv_ack_async_event); +ASSERT_HOOK_MATCHES_PROD(g_ibvEventTypeStr, wrap_ibv_event_type_str); +ASSERT_HOOK_MATCHES_PROD(g_ibvQueryDevice, wrap_ibv_query_device); +ASSERT_HOOK_MATCHES_PROD(g_ibvQueryPort, wrap_ibv_query_port); +ASSERT_HOOK_MATCHES_PROD(g_ibvQueryPortSpeed, wrap_ibv_query_port_speed); +ASSERT_HOOK_MATCHES_PROD(g_ibvQueryQp, wrap_ibv_query_qp); +ASSERT_HOOK_MATCHES_PROD(g_ibvRegMrIova2, wrap_ibv_reg_mr_iova2); +ASSERT_HOOK_MATCHES_PROD(g_ibvDirectRegDmabufMr, wrap_direct_ibv_reg_dmabuf_mr); ncclResult_t wrap_ibv_alloc_pd(struct ibv_pd** ret, struct ibv_context* context) { return g_ibvAllocPd(ret, context); @@ -48,6 +188,77 @@ struct ibv_mr* wrap_direct_ibv_reg_mr_iova2(struct ibv_pd* pd, void* addr, size_ return g_ibvDirectRegMrIova2(pd, addr, length, iova, access); } ncclResult_t wrap_ibv_dereg_mr(struct ibv_mr* mr) { return g_ibvDeregMr(mr); } +ncclResult_t wrap_ibv_reg_mr(struct ibv_mr** ret, struct ibv_pd* pd, void* addr, size_t length, int access) { + return g_ibvRegMr(ret, pd, addr, length, access); +} +ncclResult_t wrap_ibv_reg_dmabuf_mr(struct ibv_mr** ret, struct ibv_pd* pd, uint64_t offset, size_t length, + uint64_t iova, int fd, int access) { + return g_ibvRegDmabufMr(ret, pd, offset, length, iova, fd, access); +} +ncclResult_t wrap_ibv_create_cq(struct ibv_cq** ret, struct ibv_context* context, int cqe, void* cq_context, + struct ibv_comp_channel* channel, int comp_vector) { + return g_ibvCreateCq(ret, context, cqe, cq_context, channel, comp_vector); +} +ncclResult_t wrap_ibv_destroy_cq(struct ibv_cq* cq) { return g_ibvDestroyCq(cq); } +ncclResult_t wrap_ibv_create_qp(struct ibv_qp** ret, struct ibv_pd* pd, struct ibv_qp_init_attr* qp_init_attr) { + return g_ibvCreateQp(ret, pd, qp_init_attr); +} +ncclResult_t wrap_ibv_modify_qp(struct ibv_qp* qp, struct ibv_qp_attr* attr, int attr_mask) { + return g_ibvModifyQp(qp, attr, attr_mask); +} +ncclResult_t wrap_ibv_destroy_qp(struct ibv_qp* qp) { return g_ibvDestroyQp(qp); } +ncclResult_t wrap_ibv_query_gid(struct ibv_context* context, uint8_t port_num, int index, union ibv_gid* gid) { + return g_ibvQueryGid(context, port_num, index, gid); +} +ncclResult_t wrap_ibv_query_ece(struct ibv_qp* qp, struct ibv_ece* ece, int* supported) { + return g_ibvQueryEce(qp, ece, supported); +} +ncclResult_t wrap_ibv_set_ece(struct ibv_qp* qp, struct ibv_ece* ece, int* supported) { + return g_ibvSetEce(qp, ece, supported); +} +const char* wrap_ibv_get_device_name(struct ibv_device* device) { return g_ibvGetDeviceName(device); } +ncclResult_t wrap_ibv_symbols(void) { return g_ibvSymbols(); } +ncclResult_t wrap_ibv_fork_init(void) { return g_ibvForkInit(); } +ncclResult_t wrap_ibv_get_device_list(struct ibv_device*** ret, int* num_devices) { + return g_ibvGetDeviceList(ret, num_devices); +} +ncclResult_t wrap_ibv_free_device_list(struct ibv_device** list) { return g_ibvFreeDeviceList(list); } +ncclResult_t wrap_ibv_open_device(struct ibv_context** ret, struct ibv_device* device) { + return g_ibvOpenDevice(ret, device); +} +ncclResult_t wrap_ibv_close_device(struct ibv_context* context) { return g_ibvCloseDevice(context); } +ncclResult_t wrap_ibv_get_async_event(struct ibv_context* context, struct ibv_async_event* event) { + return g_ibvGetAsyncEvent(context, event); +} +ncclResult_t wrap_ibv_ack_async_event(struct ibv_async_event* event) { return g_ibvAckAsyncEvent(event); } +ncclResult_t wrap_ibv_event_type_str(char** ret, enum ibv_event_type event) { return g_ibvEventTypeStr(ret, event); } +ncclResult_t wrap_ibv_query_device(struct ibv_context* context, struct ibv_device_attr* device_attr) { + return g_ibvQueryDevice(context, device_attr); +} +ncclResult_t wrap_ibv_query_port(struct ibv_context* context, uint8_t port_num, struct ibv_port_attr* port_attr) { + return g_ibvQueryPort(context, port_num, port_attr); +} +ncclResult_t wrap_ibv_query_port_speed(struct ibv_context* context, uint8_t port_num, uint64_t* speed) { + return g_ibvQueryPortSpeed(context, port_num, speed); +} +ncclResult_t wrap_ibv_query_qp(struct ibv_qp* qp, struct ibv_qp_attr* attr, int attr_mask, + struct ibv_qp_init_attr* init_attr) { + return g_ibvQueryQp(qp, attr, attr_mask, init_attr); +} +ncclResult_t wrap_ibv_reg_mr_iova2(struct ibv_mr** ret, struct ibv_pd* pd, void* addr, size_t length, uint64_t iova, + int access) { + return g_ibvRegMrIova2(ret, pd, addr, length, iova, access); +} +struct ibv_mr* wrap_direct_ibv_reg_dmabuf_mr(struct ibv_pd* pd, uint64_t offset, size_t length, uint64_t iova, int fd, + int access) { + return g_ibvDirectRegDmabufMr(pd, offset, length, iova, fd, access); +} + +// Plain string helpers: pure functions of their argument, nothing to control. +const char* ibvWcStatusStr(enum ibv_wc_status status) { + return status == IBV_WC_SUCCESS ? "IBV_WC_SUCCESS" : "IBV_WC_ERROR(fake)"; +} +const char* ibvWcOpcodeStr(enum ibv_wc_opcode) { return "IBV_WC_OPCODE(fake)"; } void ResetIbvwrapFakes() { g_ibvAllocPd = DefaultIbvAllocPd; @@ -55,4 +266,30 @@ void ResetIbvwrapFakes() { g_ibvDirectRegMr = DefaultIbvDirectRegMr; g_ibvDirectRegMrIova2 = DefaultIbvDirectRegMrIova2; g_ibvDeregMr = DefaultIbvDeregMr; + g_ibvRegMr = DefaultIbvRegMr; + g_ibvRegDmabufMr = DefaultIbvRegDmabufMr; + g_ibvCreateCq = DefaultIbvCreateCq; + g_ibvDestroyCq = DefaultIbvDestroyCq; + g_ibvCreateQp = DefaultIbvCreateQp; + g_ibvModifyQp = DefaultIbvModifyQp; + g_ibvDestroyQp = DefaultIbvDestroyQp; + g_ibvQueryGid = DefaultIbvQueryGid; + g_ibvQueryEce = DefaultIbvQueryEce; + g_ibvSetEce = DefaultIbvSetEce; + g_ibvGetDeviceName = DefaultIbvGetDeviceName; + g_ibvSymbols = DefaultIbvSymbols; + g_ibvForkInit = DefaultIbvForkInit; + g_ibvGetDeviceList = DefaultIbvGetDeviceList; + g_ibvFreeDeviceList = DefaultIbvFreeDeviceList; + g_ibvOpenDevice = DefaultIbvOpenDevice; + g_ibvCloseDevice = DefaultIbvCloseDevice; + g_ibvGetAsyncEvent = DefaultIbvGetAsyncEvent; + g_ibvAckAsyncEvent = DefaultIbvAckAsyncEvent; + g_ibvEventTypeStr = DefaultIbvEventTypeStr; + g_ibvQueryDevice = DefaultIbvQueryDevice; + g_ibvQueryPort = DefaultIbvQueryPort; + g_ibvQueryPortSpeed = DefaultIbvQueryPortSpeed; + g_ibvQueryQp = DefaultIbvQueryQp; + g_ibvRegMrIova2 = DefaultIbvRegMrIova2; + g_ibvDirectRegDmabufMr = DefaultIbvDirectRegDmabufMr; } diff --git a/projects/rccl/test/host/fakes/ibvwrap_fakes.h b/projects/rccl/test/host/fakes/ibvwrap_fakes.h index bef15ccfc970..49655f683275 100644 --- a/projects/rccl/test/host/fakes/ibvwrap_fakes.h +++ b/projects/rccl/test/host/fakes/ibvwrap_fakes.h @@ -5,13 +5,13 @@ ************************************************************************/ // Controllable seams for the IB verbs wrappers defined by src/misc/ibvwrap.cc. -// Only the protection-domain and memory-registration entry points are faked; -// add more here as further units come under test. +// Only the entry points the units under test reach are faked; add more here as +// further units come under test. // -// Defaults are fail-loud for acquisitions (alloc_pd fails with *ret = NULL, the -// reg_mr entry points return NULL, matching the real wrappers on failure) and -// succeed for releases (dealloc_pd / dereg_mr), so an un-hooked call path can -// never report a resource it did not actually get. +// Defaults are fail-loud for acquisitions and queries (they fail with *ret = +// NULL, the direct reg_mr entry points return NULL, matching the real wrappers +// on failure) and succeed for releases (dealloc_pd / dereg_mr / destroy_*), so an +// un-hooked call path can never report a resource it did not actually get. #ifndef RCCL_TEST_HOST_IBVWRAP_FAKES_H_ #define RCCL_TEST_HOST_IBVWRAP_FAKES_H_ @@ -20,11 +20,7 @@ #include #include -#include "nccl.h" - -struct ibv_context; -struct ibv_pd; -struct ibv_mr; +#include "ibvwrap.h" extern std::function g_ibvAllocPd; extern std::function g_ibvDeallocPd; @@ -34,6 +30,55 @@ extern std::function g_ibvDirectRegMrIova2; extern std::function g_ibvDeregMr; +extern std::function + g_ibvRegMr; +extern std::function + g_ibvRegDmabufMr; +extern std::function + g_ibvCreateCq; +extern std::function g_ibvDestroyCq; +extern std::function + g_ibvCreateQp; +extern std::function + g_ibvModifyQp; +extern std::function g_ibvDestroyQp; +extern std::function + g_ibvQueryGid; +extern std::function g_ibvQueryEce; +extern std::function g_ibvSetEce; +extern std::function g_ibvGetDeviceName; + +extern std::function g_ibvSymbols; +extern std::function g_ibvForkInit; +extern std::function g_ibvGetDeviceList; +extern std::function g_ibvFreeDeviceList; +extern std::function g_ibvOpenDevice; +extern std::function g_ibvCloseDevice; +extern std::function + g_ibvGetAsyncEvent; +extern std::function g_ibvAckAsyncEvent; +extern std::function g_ibvEventTypeStr; +extern std::function + g_ibvQueryDevice; +extern std::function + g_ibvQueryPort; +extern std::function + g_ibvQueryPortSpeed; +extern std::function + g_ibvQueryQp; +extern std::function + g_ibvRegMrIova2; +extern std::function + g_ibvDirectRegDmabufMr; void ResetIbvwrapFakes(); diff --git a/projects/rccl/test/host/fakes/ionicdvwrap_fakes.cc b/projects/rccl/test/host/fakes/ionicdvwrap_fakes.cc new file mode 100644 index 000000000000..e35f0a364e15 --- /dev/null +++ b/projects/rccl/test/host/fakes/ionicdvwrap_fakes.cc @@ -0,0 +1,51 @@ +/************************************************************************* + * Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +// Implementation of the src/misc/ionicdvwrap.cc fakes. See ionicdvwrap_fakes.h. + +#include "ionicdvwrap_fakes.h" + +#include "ionic/ionicdvwrap.h" +#include "signature-drift.h" + +static ncclResult_t DefaultIonicdvSymbols() { return ncclSystemError; } +std::function g_ionicdvSymbols = DefaultIonicdvSymbols; + +static ncclResult_t DefaultIonicdvQpSetGda(struct ibv_qp*, bool, bool) { return ncclSystemError; } +std::function g_ionicdvQpSetGda = DefaultIonicdvQpSetGda; + +static ncclResult_t DefaultIonicdvPdSetUdmaMask(struct ibv_pd*, uint8_t) { return ncclSystemError; } +std::function g_ionicdvPdSetUdmaMask = DefaultIonicdvPdSetUdmaMask; + +static ncclResult_t DefaultIonicdvQpSetPuecPlaneRoute(struct ibv_qp*, uint8_t, struct ionic_dv_puec_route*) { + return ncclSystemError; +} +std::function g_ionicdvQpSetPuecPlaneRoute = + DefaultIonicdvQpSetPuecPlaneRoute; + +ASSERT_HOOK_MATCHES_PROD(g_ionicdvSymbols, wrap_ionicdv_symbols); +ASSERT_HOOK_MATCHES_PROD(g_ionicdvQpSetGda, wrap_ionicdv_qp_set_gda); +ASSERT_HOOK_MATCHES_PROD(g_ionicdvPdSetUdmaMask, wrap_ionicdv_pd_set_udma_mask); +ASSERT_HOOK_MATCHES_PROD(g_ionicdvQpSetPuecPlaneRoute, wrap_ionicdv_qp_set_puec_plane_route); + +ncclResult_t wrap_ionicdv_symbols(void) { return g_ionicdvSymbols(); } +ncclResult_t wrap_ionicdv_qp_set_gda(struct ibv_qp* ibqp, bool enable_send, bool enable_recv) { + return g_ionicdvQpSetGda(ibqp, enable_send, enable_recv); +} +ncclResult_t wrap_ionicdv_pd_set_udma_mask(struct ibv_pd* ibpd, uint8_t udma_mask) { + return g_ionicdvPdSetUdmaMask(ibpd, udma_mask); +} +ncclResult_t wrap_ionicdv_qp_set_puec_plane_route(struct ibv_qp* qp, uint8_t plane_idx, + struct ionic_dv_puec_route* route) { + return g_ionicdvQpSetPuecPlaneRoute(qp, plane_idx, route); +} + +void ResetIonicdvwrapFakes() { + g_ionicdvSymbols = DefaultIonicdvSymbols; + g_ionicdvQpSetGda = DefaultIonicdvQpSetGda; + g_ionicdvPdSetUdmaMask = DefaultIonicdvPdSetUdmaMask; + g_ionicdvQpSetPuecPlaneRoute = DefaultIonicdvQpSetPuecPlaneRoute; +} diff --git a/projects/rccl/test/host/fakes/ionicdvwrap_fakes.h b/projects/rccl/test/host/fakes/ionicdvwrap_fakes.h new file mode 100644 index 000000000000..1b9e3679c2ca --- /dev/null +++ b/projects/rccl/test/host/fakes/ionicdvwrap_fakes.h @@ -0,0 +1,33 @@ +/************************************************************************* + * Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +// Controllable seams for the ionic direct-verbs wrappers defined by +// src/misc/ionicdvwrap.cc. Every default fails with ncclSystemError, the real +// wrappers' answer when libionic is missing. + +#ifndef RCCL_TEST_HOST_IONICDVWRAP_FAKES_H_ +#define RCCL_TEST_HOST_IONICDVWRAP_FAKES_H_ + +#include +#include + +#include "nccl.h" + +struct ibv_pd; +struct ibv_qp; +struct ionic_dv_puec_route; + +extern std::function g_ionicdvSymbols; +extern std::function + g_ionicdvQpSetGda; +extern std::function g_ionicdvPdSetUdmaMask; +extern std::function + g_ionicdvQpSetPuecPlaneRoute; + +void ResetIonicdvwrapFakes(); + +#endif // RCCL_TEST_HOST_IONICDVWRAP_FAKES_H_ diff --git a/projects/rccl/test/host/fakes/mlx5dvwrap_fakes.cc b/projects/rccl/test/host/fakes/mlx5dvwrap_fakes.cc new file mode 100644 index 000000000000..b081ead12652 --- /dev/null +++ b/projects/rccl/test/host/fakes/mlx5dvwrap_fakes.cc @@ -0,0 +1,88 @@ +/************************************************************************* + * Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +// Implementation of the src/misc/mlx5dvwrap.cc fakes. See mlx5dvwrap_fakes.h. + +#include "mlx5dvwrap_fakes.h" + +#include "mlx5/mlx5dvwrap.h" +#include "signature-drift.h" + +static ncclResult_t DefaultMlx5dvSymbols() { return ncclSystemError; } +std::function g_mlx5dvSymbols = DefaultMlx5dvSymbols; + +static bool DefaultMlx5dvIsSupported(struct ibv_device*) { return false; } +std::function g_mlx5dvIsSupported = DefaultMlx5dvIsSupported; + +static ncclResult_t DefaultMlx5dvGetDataDirectSysfsPath(struct ibv_context*, char*, size_t) { + return ncclSystemError; +} +std::function g_mlx5dvGetDataDirectSysfsPath = + DefaultMlx5dvGetDataDirectSysfsPath; + +static ncclResult_t DefaultMlx5dvRegDmabufMr(struct ibv_mr** ret, struct ibv_pd*, uint64_t, size_t, uint64_t, int, int, + int) { + if (ret) *ret = nullptr; + return ncclSystemError; +} +std::function + g_mlx5dvRegDmabufMr = DefaultMlx5dvRegDmabufMr; + +static struct ibv_mr* DefaultMlx5dvDirectRegDmabufMr(struct ibv_pd*, uint64_t, size_t, uint64_t, int, int, int) { + return nullptr; +} +std::function g_mlx5dvDirectRegDmabufMr = + DefaultMlx5dvDirectRegDmabufMr; + +static ncclResult_t DefaultMlx5dvQueryDevice(struct ibv_context*, struct mlx5dv_context*) { return ncclSystemError; } +std::function g_mlx5dvQueryDevice = + DefaultMlx5dvQueryDevice; + +static struct ibv_qp* DefaultMlx5dvCreateQp(struct ibv_context*, struct ibv_qp_init_attr_ex*, + struct mlx5dv_qp_init_attr*) { + return nullptr; +} +std::function + g_mlx5dvCreateQp = DefaultMlx5dvCreateQp; + +ASSERT_HOOK_MATCHES_PROD(g_mlx5dvSymbols, wrap_mlx5dv_symbols); +ASSERT_HOOK_MATCHES_PROD(g_mlx5dvIsSupported, wrap_mlx5dv_is_supported); +ASSERT_HOOK_MATCHES_PROD(g_mlx5dvGetDataDirectSysfsPath, wrap_mlx5dv_get_data_direct_sysfs_path); +ASSERT_HOOK_MATCHES_PROD(g_mlx5dvRegDmabufMr, wrap_mlx5dv_reg_dmabuf_mr); +ASSERT_HOOK_MATCHES_PROD(g_mlx5dvDirectRegDmabufMr, wrap_direct_mlx5dv_reg_dmabuf_mr); +ASSERT_HOOK_MATCHES_PROD(g_mlx5dvQueryDevice, wrap_mlx5dv_query_device); +ASSERT_HOOK_MATCHES_PROD(g_mlx5dvCreateQp, wrap_mlx5dv_create_qp); + +ncclResult_t wrap_mlx5dv_symbols(void) { return g_mlx5dvSymbols(); } +bool wrap_mlx5dv_is_supported(struct ibv_device* device) { return g_mlx5dvIsSupported(device); } +ncclResult_t wrap_mlx5dv_get_data_direct_sysfs_path(struct ibv_context* context, char* buf, size_t buf_len) { + return g_mlx5dvGetDataDirectSysfsPath(context, buf, buf_len); +} +ncclResult_t wrap_mlx5dv_reg_dmabuf_mr(struct ibv_mr** ret, struct ibv_pd* pd, uint64_t offset, size_t length, + uint64_t iova, int fd, int access, int mlx5_access) { + return g_mlx5dvRegDmabufMr(ret, pd, offset, length, iova, fd, access, mlx5_access); +} +struct ibv_mr* wrap_direct_mlx5dv_reg_dmabuf_mr(struct ibv_pd* pd, uint64_t offset, size_t length, uint64_t iova, + int fd, int access, int mlx5_access) { + return g_mlx5dvDirectRegDmabufMr(pd, offset, length, iova, fd, access, mlx5_access); +} +ncclResult_t wrap_mlx5dv_query_device(struct ibv_context* ctx_in, struct mlx5dv_context* attrs_out) { + return g_mlx5dvQueryDevice(ctx_in, attrs_out); +} +struct ibv_qp* wrap_mlx5dv_create_qp(struct ibv_context* context, struct ibv_qp_init_attr_ex* qp_attr, + struct mlx5dv_qp_init_attr* mlx5_qp_attr) { + return g_mlx5dvCreateQp(context, qp_attr, mlx5_qp_attr); +} + +void ResetMlx5dvwrapFakes() { + g_mlx5dvSymbols = DefaultMlx5dvSymbols; + g_mlx5dvIsSupported = DefaultMlx5dvIsSupported; + g_mlx5dvGetDataDirectSysfsPath = DefaultMlx5dvGetDataDirectSysfsPath; + g_mlx5dvRegDmabufMr = DefaultMlx5dvRegDmabufMr; + g_mlx5dvDirectRegDmabufMr = DefaultMlx5dvDirectRegDmabufMr; + g_mlx5dvQueryDevice = DefaultMlx5dvQueryDevice; + g_mlx5dvCreateQp = DefaultMlx5dvCreateQp; +} diff --git a/projects/rccl/test/host/fakes/mlx5dvwrap_fakes.h b/projects/rccl/test/host/fakes/mlx5dvwrap_fakes.h new file mode 100644 index 000000000000..4ae1df90d215 --- /dev/null +++ b/projects/rccl/test/host/fakes/mlx5dvwrap_fakes.h @@ -0,0 +1,49 @@ +/************************************************************************* + * Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +// Controllable seams for the mlx5 direct-verbs wrappers defined by +// src/misc/mlx5dvwrap.cc. Every default fails: acquisitions return NULL or an +// error and wrap_mlx5dv_is_supported reports false, so an un-hooked path never +// sees an mlx5 device. + +#ifndef RCCL_TEST_HOST_MLX5DVWRAP_FAKES_H_ +#define RCCL_TEST_HOST_MLX5DVWRAP_FAKES_H_ + +#include +#include +#include + +#include "nccl.h" + +struct ibv_context; +struct ibv_device; +struct ibv_pd; +struct ibv_mr; +struct ibv_qp; +struct ibv_qp_init_attr_ex; +struct mlx5dv_context; +struct mlx5dv_qp_init_attr; + +extern std::function g_mlx5dvSymbols; +extern std::function g_mlx5dvIsSupported; +extern std::function + g_mlx5dvGetDataDirectSysfsPath; +extern std::function + g_mlx5dvRegDmabufMr; +extern std::function + g_mlx5dvDirectRegDmabufMr; +extern std::function + g_mlx5dvQueryDevice; +extern std::function + g_mlx5dvCreateQp; + +void ResetMlx5dvwrapFakes(); + +#endif // RCCL_TEST_HOST_MLX5DVWRAP_FAKES_H_ diff --git a/projects/rccl/test/host/fakes/net_ib_cast_connect_fakes.cc b/projects/rccl/test/host/fakes/net_ib_cast_connect_fakes.cc new file mode 100644 index 000000000000..0d2d2f589778 --- /dev/null +++ b/projects/rccl/test/host/fakes/net_ib_cast_connect_fakes.cc @@ -0,0 +1,185 @@ +/************************************************************************* + * Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +// Implementation of the src/transport/net_ib_cast/connect.cc fakes. See net_ib_cast_connect_fakes.h. + +#include "net_ib_cast_connect_fakes.h" + +#include "connect_cast.h" +#include "fail_loud.h" +#include "fakes/nccl_fakes.h" +#include "param_redirect.h" +#include "signature-drift.h" + +// Same lines as connect.cc; test/host/CMakeLists.txt pins both copies. +NCCL_PARAM(IbCastTimeout, "IB_TIMEOUT", 20); +NCCL_PARAM(IbCastRetryCnt, "IB_RETRY_CNT", 7); +NCCL_PARAM(IbCastPkey, "IB_PKEY", 0); +NCCL_PARAM(IbCastQpsPerConn, "IB_QPS_PER_CONNECTION", 2); +NCCL_PARAM(IbCastUseInline, "IB_USE_INLINE", 0); +NCCL_PARAM(IbCastGdrFlushDisable, "GDR_FLUSH_DISABLE", 1); +NCCL_PARAM(IbCastSl, "IB_SL", -1); +NCCL_PARAM(IbCastTc, "IB_TC", -1); +RCCL_PARAM(IbCastUDMAPolicy, "IB_UDMA_POLICY", 0); + +static ncclResult_t DefaultIbCastQpCreate(struct ncclIbQp*, struct ncclIbQpCreateAttr*) { + FailLoudUnfaked("net_ib_cast_connect_fakes", "IbCastQpCreate"); +} +static void DefaultIbCastBuildDataQpCreateAttr(struct ncclIbNetCommBase*, int, struct ncclIbQpCreateAttr*) { + FailLoudUnfaked("net_ib_cast_connect_fakes", "IbCastBuildDataQpCreateAttr"); +} +static ncclResult_t DefaultIbCastQpInit(struct ncclIbQp*) { + FailLoudUnfaked("net_ib_cast_connect_fakes", "IbCastQpInit"); +} +static ncclResult_t DefaultIbCastQpRtr(struct ncclIbQp*) { FailLoudUnfaked("net_ib_cast_connect_fakes", "IbCastQpRtr"); } +static ncclResult_t DefaultIbCastQpRts(struct ncclIbQp*) { FailLoudUnfaked("net_ib_cast_connect_fakes", "IbCastQpRts"); } +static ncclResult_t DefaultIbCastQpReset(struct ncclIbQp*) { + FailLoudUnfaked("net_ib_cast_connect_fakes", "IbCastQpReset"); +} +static ncclResult_t DefaultIbCastQpError(struct ncclIbQp*) { + FailLoudUnfaked("net_ib_cast_connect_fakes", "IbCastQpError"); +} +static ncclResult_t DefaultIbCastPostReceiveWorkRequestsOnQp(struct ncclIbRecvComm*, struct ncclIbQp*) { + FailLoudUnfaked("net_ib_cast_connect_fakes", "IbCastPostReceiveWorkRequestsOnQp"); +} +static bool DefaultIbCastByOrderRequested() { return false; } +static ncclResult_t DefaultIbCastListen(void*, int, void*, void**) { + FailLoudUnfaked("net_ib_cast_connect_fakes", "IbCastListen"); +} +static ncclResult_t DefaultIbCastConnect(void*, int, void*, void**, ncclNetDeviceHandle_t**) { + FailLoudUnfaked("net_ib_cast_connect_fakes", "IbCastConnect"); +} +static ncclResult_t DefaultIbCastConnectImpl(void*, int, void*, void**, ncclNetDeviceHandle_t**, int) { + FailLoudUnfaked("net_ib_cast_connect_fakes", "IbCastConnectImpl"); +} +static ncclResult_t DefaultIbCastAccept(void*, void**, ncclNetDeviceHandle_t**) { + FailLoudUnfaked("net_ib_cast_connect_fakes", "IbCastAccept"); +} +static ncclResult_t DefaultIbCastCloseSend(void*) { FailLoudUnfaked("net_ib_cast_connect_fakes", "IbCastCloseSend"); } +static ncclResult_t DefaultIbCastCloseRecv(void*) { FailLoudUnfaked("net_ib_cast_connect_fakes", "IbCastCloseRecv"); } +static ncclResult_t DefaultIbCastCloseListen(void*) { + FailLoudUnfaked("net_ib_cast_connect_fakes", "IbCastCloseListen"); +} +static ncclResult_t DefaultIbCastGetGidIndex(struct ibv_context*, uint8_t, struct ibv_port_attr*, int*) { + FailLoudUnfaked("net_ib_cast_connect_fakes", "IbCastGetGidIndex"); +} +static ncclResult_t DefaultIbCastGidInfoQuery(struct ibv_context*, uint8_t, struct ibv_port_attr*, + struct ncclIbGidInfo*) { + FailLoudUnfaked("net_ib_cast_connect_fakes", "IbCastGidInfoQuery"); +} +static void DefaultIbCastSetTrafficClass(void*, int) { + FailLoudUnfaked("net_ib_cast_connect_fakes", "IbCastSetTrafficClass"); +} + +std::function g_ibCastQpCreate = DefaultIbCastQpCreate; +std::function g_ibCastBuildDataQpCreateAttr = + DefaultIbCastBuildDataQpCreateAttr; +std::function g_ibCastQpInit = DefaultIbCastQpInit; +std::function g_ibCastQpRtr = DefaultIbCastQpRtr; +std::function g_ibCastQpRts = DefaultIbCastQpRts; +std::function g_ibCastQpReset = DefaultIbCastQpReset; +std::function g_ibCastQpError = DefaultIbCastQpError; +std::function g_ibCastPostReceiveWorkRequestsOnQp = + DefaultIbCastPostReceiveWorkRequestsOnQp; +std::function g_ibCastByOrderRequested = DefaultIbCastByOrderRequested; +std::function g_ibCastListen = DefaultIbCastListen; +std::function g_ibCastConnect = DefaultIbCastConnect; +std::function g_ibCastConnectImpl = + DefaultIbCastConnectImpl; +std::function g_ibCastAccept = DefaultIbCastAccept; +std::function g_ibCastCloseSend = DefaultIbCastCloseSend; +std::function g_ibCastCloseRecv = DefaultIbCastCloseRecv; +std::function g_ibCastCloseListen = DefaultIbCastCloseListen; +std::function g_ibCastGetGidIndex = + DefaultIbCastGetGidIndex; +std::function + g_ibCastGidInfoQuery = DefaultIbCastGidInfoQuery; +std::function g_ibCastSetTrafficClass = DefaultIbCastSetTrafficClass; +int g_ibCastReportMatchingSchemeCalls = 0; + +ASSERT_HOOK_MATCHES_PROD(g_ibCastQpCreate, IbCastQpCreate); +ASSERT_HOOK_MATCHES_PROD(g_ibCastBuildDataQpCreateAttr, IbCastBuildDataQpCreateAttr); +ASSERT_HOOK_MATCHES_PROD(g_ibCastQpInit, IbCastQpInit); +ASSERT_HOOK_MATCHES_PROD(g_ibCastQpRtr, IbCastQpRtr); +ASSERT_HOOK_MATCHES_PROD(g_ibCastQpRts, IbCastQpRts); +ASSERT_HOOK_MATCHES_PROD(g_ibCastQpReset, IbCastQpReset); +ASSERT_HOOK_MATCHES_PROD(g_ibCastQpError, IbCastQpError); +ASSERT_HOOK_MATCHES_PROD(g_ibCastPostReceiveWorkRequestsOnQp, IbCastPostReceiveWorkRequestsOnQp); +ASSERT_HOOK_MATCHES_PROD(g_ibCastByOrderRequested, IbCastByOrderRequested); +ASSERT_HOOK_MATCHES_PROD(g_ibCastListen, IbCastListen); +ASSERT_HOOK_MATCHES_PROD(g_ibCastConnect, IbCastConnect); +ASSERT_HOOK_MATCHES_PROD(g_ibCastConnectImpl, IbCastConnectImpl); +ASSERT_HOOK_MATCHES_PROD(g_ibCastAccept, IbCastAccept); +ASSERT_HOOK_MATCHES_PROD(g_ibCastCloseSend, IbCastCloseSend); +ASSERT_HOOK_MATCHES_PROD(g_ibCastCloseRecv, IbCastCloseRecv); +ASSERT_HOOK_MATCHES_PROD(g_ibCastCloseListen, IbCastCloseListen); +ASSERT_HOOK_MATCHES_PROD(g_ibCastGetGidIndex, IbCastGetGidIndex); +ASSERT_HOOK_MATCHES_PROD(g_ibCastGidInfoQuery, IbCastGidInfoQuery); +ASSERT_HOOK_MATCHES_PROD(g_ibCastSetTrafficClass, IbCastSetTrafficClass); + +ncclResult_t IbCastQpCreate(struct ncclIbQp* qp, struct ncclIbQpCreateAttr* attr) { return g_ibCastQpCreate(qp, attr); } +void IbCastBuildDataQpCreateAttr(struct ncclIbNetCommBase* base, int devIndex, struct ncclIbQpCreateAttr* out) { + g_ibCastBuildDataQpCreateAttr(base, devIndex, out); +} +ncclResult_t IbCastQpInit(struct ncclIbQp* qp) { return g_ibCastQpInit(qp); } +ncclResult_t IbCastQpRtr(struct ncclIbQp* qp) { return g_ibCastQpRtr(qp); } +ncclResult_t IbCastQpRts(struct ncclIbQp* qp) { return g_ibCastQpRts(qp); } +ncclResult_t IbCastQpReset(struct ncclIbQp* qp) { return g_ibCastQpReset(qp); } +ncclResult_t IbCastQpError(struct ncclIbQp* qp) { return g_ibCastQpError(qp); } +ncclResult_t IbCastPostReceiveWorkRequestsOnQp(struct ncclIbRecvComm* recvComm, struct ncclIbQp* dataQp) { + return g_ibCastPostReceiveWorkRequestsOnQp(recvComm, dataQp); +} +bool IbCastByOrderRequested() { return g_ibCastByOrderRequested(); } +ncclResult_t IbCastListen(void* ctx, int dev, void* opaqueHandle, void** listenComm) { + return g_ibCastListen(ctx, dev, opaqueHandle, listenComm); +} +ncclResult_t IbCastConnect(void* ctx, int dev, void* opaqueHandle, void** sendComm, + ncclNetDeviceHandle_t** sendDevComm) { + return g_ibCastConnect(ctx, dev, opaqueHandle, sendComm, sendDevComm); +} +ncclResult_t IbCastConnectImpl(void* ctx, int dev, void* opaqueHandle, void** sendComm, + ncclNetDeviceHandle_t** sendDevComm, int envTrafficClass) { + return g_ibCastConnectImpl(ctx, dev, opaqueHandle, sendComm, sendDevComm, envTrafficClass); +} +ncclResult_t IbCastAccept(void* listenComm, void** recvComm, ncclNetDeviceHandle_t** recvDevComm) { + return g_ibCastAccept(listenComm, recvComm, recvDevComm); +} +ncclResult_t IbCastCloseSend(void* sendComm) { return g_ibCastCloseSend(sendComm); } +ncclResult_t IbCastCloseRecv(void* recvComm) { return g_ibCastCloseRecv(recvComm); } +ncclResult_t IbCastCloseListen(void* listenComm) { return g_ibCastCloseListen(listenComm); } +ncclResult_t IbCastGetGidIndex(struct ibv_context* context, uint8_t portNum, struct ibv_port_attr* portAttr, + int* gidIndex) { + return g_ibCastGetGidIndex(context, portNum, portAttr, gidIndex); +} +ncclResult_t IbCastGidInfoQuery(struct ibv_context* context, uint8_t portNum, struct ibv_port_attr* portAttr, + struct ncclIbGidInfo* gidInfo) { + return g_ibCastGidInfoQuery(context, portNum, portAttr, gidInfo); +} +void IbCastSetTrafficClass(void* ctx, int trafficClass) { g_ibCastSetTrafficClass(ctx, trafficClass); } +void IbCastReportMatchingScheme() { g_ibCastReportMatchingSchemeCalls++; } + +void ResetNetIbCastConnectFakes() { + g_ibCastQpCreate = DefaultIbCastQpCreate; + g_ibCastBuildDataQpCreateAttr = DefaultIbCastBuildDataQpCreateAttr; + g_ibCastQpInit = DefaultIbCastQpInit; + g_ibCastQpRtr = DefaultIbCastQpRtr; + g_ibCastQpRts = DefaultIbCastQpRts; + g_ibCastQpReset = DefaultIbCastQpReset; + g_ibCastQpError = DefaultIbCastQpError; + g_ibCastPostReceiveWorkRequestsOnQp = DefaultIbCastPostReceiveWorkRequestsOnQp; + g_ibCastByOrderRequested = DefaultIbCastByOrderRequested; + g_ibCastListen = DefaultIbCastListen; + g_ibCastConnect = DefaultIbCastConnect; + g_ibCastConnectImpl = DefaultIbCastConnectImpl; + g_ibCastAccept = DefaultIbCastAccept; + g_ibCastCloseSend = DefaultIbCastCloseSend; + g_ibCastCloseRecv = DefaultIbCastCloseRecv; + g_ibCastCloseListen = DefaultIbCastCloseListen; + g_ibCastGetGidIndex = DefaultIbCastGetGidIndex; + g_ibCastGidInfoQuery = DefaultIbCastGidInfoQuery; + g_ibCastSetTrafficClass = DefaultIbCastSetTrafficClass; + g_ibCastReportMatchingSchemeCalls = 0; +} diff --git a/projects/rccl/test/host/fakes/net_ib_cast_connect_fakes.h b/projects/rccl/test/host/fakes/net_ib_cast_connect_fakes.h new file mode 100644 index 000000000000..dae5c28cecfe --- /dev/null +++ b/projects/rccl/test/host/fakes/net_ib_cast_connect_fakes.h @@ -0,0 +1,53 @@ +/************************************************************************* + * Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +// Seams for the src/transport/net_ib_cast/connect.cc symbols, for binaries that +// do not compile connect.cc. Every hook fails loud by default except +// IbCastByOrderRequested, which answers false (BY_ID, the default scheme). +// IbCastReportMatchingScheme only counts its calls. + +#ifndef RCCL_TEST_HOST_NET_IB_CAST_CONNECT_FAKES_H_ +#define RCCL_TEST_HOST_NET_IB_CAST_CONNECT_FAKES_H_ + +#include +#include + +#include "nccl.h" +#include "nccl_device/net_device.h" + +struct ibv_context; +struct ibv_port_attr; +struct ncclIbGidInfo; +struct ncclIbQp; +struct ncclIbQpCreateAttr; +struct ncclIbNetCommBase; +struct ncclIbRecvComm; + +extern std::function g_ibCastQpCreate; +extern std::function g_ibCastBuildDataQpCreateAttr; +extern std::function g_ibCastQpInit; +extern std::function g_ibCastQpRtr; +extern std::function g_ibCastQpRts; +extern std::function g_ibCastQpReset; +extern std::function g_ibCastQpError; +extern std::function g_ibCastPostReceiveWorkRequestsOnQp; +extern std::function g_ibCastByOrderRequested; +extern std::function g_ibCastListen; +extern std::function g_ibCastConnect; +extern std::function g_ibCastConnectImpl; +extern std::function g_ibCastAccept; +extern std::function g_ibCastCloseSend; +extern std::function g_ibCastCloseRecv; +extern std::function g_ibCastCloseListen; +extern std::function g_ibCastGetGidIndex; +extern std::function + g_ibCastGidInfoQuery; +extern std::function g_ibCastSetTrafficClass; +extern int g_ibCastReportMatchingSchemeCalls; + +void ResetNetIbCastConnectFakes(); + +#endif // RCCL_TEST_HOST_NET_IB_CAST_CONNECT_FAKES_H_ diff --git a/projects/rccl/test/host/fakes/net_ib_cast_p2p_fakes.cc b/projects/rccl/test/host/fakes/net_ib_cast_p2p_fakes.cc new file mode 100644 index 000000000000..7b9ef57ddf63 --- /dev/null +++ b/projects/rccl/test/host/fakes/net_ib_cast_p2p_fakes.cc @@ -0,0 +1,119 @@ +/************************************************************************* + * Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +// Implementation of the src/transport/net_ib_cast/p2p.cc fakes. See +// net_ib_cast_p2p_fakes.h. + +#include "net_ib_cast_p2p_fakes.h" + +#include "fail_loud.h" +#include "nccl_fakes.h" +#include "signature-drift.h" + +const char* IbCastReqTypeStr[] = {"Unused", "Send", "Recv", "Flush", "IPut"}; +int64_t IbCastArThreshold = 8192; + +int64_t ncclParamIbCastArThreshold() { return g_loadParam("IB_AR_THRESHOLD", -2); } +int64_t ncclParamIbCastReceiverSideMatchingScheme() { return g_loadParam("IB_RECEIVER_SIDE_MATCHING_SCHEME", -2); } +int64_t rcclParamIbCastGdrFlushGpuMemNoRelaxedOrdering() { + return g_loadParam("RCCL_GDR_FLUSH_GPU_MEM_NO_RELAXED_ORDERING", 1); +} + +static ncclResult_t DefaultIbCastMultiSend(struct ncclIbSendComm*, int, int, int, bool, bool) { + FailLoudUnfaked("net_ib_cast_p2p_fakes", "IbCastMultiSend"); +} +std::function g_ibCastMultiSend = + DefaultIbCastMultiSend; + +static ncclResult_t DefaultIbCastPostFifo(struct ncclIbRecvComm*, struct ncclIbRequest*, int, int) { + FailLoudUnfaked("net_ib_cast_p2p_fakes", "IbCastPostFifo"); +} +std::function g_ibCastPostFifo = + DefaultIbCastPostFifo; + +static void DefaultIbCastAddEvent(struct ncclIbRequest*, int) { + FailLoudUnfaked("net_ib_cast_p2p_fakes", "IbCastAddEvent"); +} +std::function g_ibCastAddEvent = DefaultIbCastAddEvent; + +static ncclResult_t DefaultIbCastGetRequest(struct ncclIbNetCommBase*, struct ncclIbRequest**) { + FailLoudUnfaked("net_ib_cast_p2p_fakes", "IbCastGetRequest"); +} +std::function + g_ibCastGetRequest = DefaultIbCastGetRequest; + +static ncclResult_t DefaultIbCastFreeRequest(struct ncclIbRequest*) { + FailLoudUnfaked("net_ib_cast_p2p_fakes", "IbCastFreeRequest"); +} +std::function g_ibCastFreeRequest = DefaultIbCastFreeRequest; + +static ncclResult_t DefaultIbCastIsend(void*, void*, size_t, int, void*, void*, void**) { + FailLoudUnfaked("net_ib_cast_p2p_fakes", "IbCastIsend"); +} +std::function g_ibCastIsend = DefaultIbCastIsend; + +static ncclResult_t DefaultIbCastIrecv(void*, int, void**, size_t*, int*, void**, void**, void**) { + FailLoudUnfaked("net_ib_cast_p2p_fakes", "IbCastIrecv"); +} +std::function + g_ibCastIrecv = DefaultIbCastIrecv; + +static ncclResult_t DefaultIbCastIflush(void*, int, void**, int*, void**, void**) { + FailLoudUnfaked("net_ib_cast_p2p_fakes", "IbCastIflush"); +} +std::function g_ibCastIflush = DefaultIbCastIflush; + +static ncclResult_t DefaultIbCastTest(void*, int*, int*) { + FailLoudUnfaked("net_ib_cast_p2p_fakes", "IbCastTest"); +} +std::function g_ibCastTest = DefaultIbCastTest; + +ASSERT_HOOK_MATCHES_PROD(g_ibCastMultiSend, IbCastMultiSend); +ASSERT_HOOK_MATCHES_PROD(g_ibCastPostFifo, IbCastPostFifo); +ASSERT_HOOK_MATCHES_PROD(g_ibCastAddEvent, IbCastAddEvent); +ASSERT_HOOK_MATCHES_PROD(g_ibCastGetRequest, IbCastGetRequest); +ASSERT_HOOK_MATCHES_PROD(g_ibCastFreeRequest, IbCastFreeRequest); +ASSERT_HOOK_MATCHES_PROD(g_ibCastIsend, IbCastIsend); +ASSERT_HOOK_MATCHES_PROD(g_ibCastIrecv, IbCastIrecv); +ASSERT_HOOK_MATCHES_PROD(g_ibCastIflush, IbCastIflush); +ASSERT_HOOK_MATCHES_PROD(g_ibCastTest, IbCastTest); + +ncclResult_t IbCastMultiSend(struct ncclIbSendComm* comm, int slot, int nqps, int startQpIndex, bool wrrSched, + bool useWriteOp) { + return g_ibCastMultiSend(comm, slot, nqps, startQpIndex, wrrSched, useWriteOp); +} +ncclResult_t IbCastPostFifo(struct ncclIbRecvComm* comm, struct ncclIbRequest* req, int slot, int n) { + return g_ibCastPostFifo(comm, req, slot, n); +} +void IbCastAddEvent(struct ncclIbRequest* req, int devIndex) { g_ibCastAddEvent(req, devIndex); } +ncclResult_t IbCastGetRequest(struct ncclIbNetCommBase* base, struct ncclIbRequest** req) { + return g_ibCastGetRequest(base, req); +} +ncclResult_t IbCastFreeRequest(struct ncclIbRequest* r) { return g_ibCastFreeRequest(r); } +ncclResult_t IbCastIsend(void* sendComm, void* data, size_t size, int tag, void* mhandle, void* phandle, + void** request) { + return g_ibCastIsend(sendComm, data, size, tag, mhandle, phandle, request); +} +ncclResult_t IbCastIrecv(void* recvComm, int n, void** data, size_t* sizes, int* tags, void** mhandles, void** phandles, + void** request) { + return g_ibCastIrecv(recvComm, n, data, sizes, tags, mhandles, phandles, request); +} +ncclResult_t IbCastIflush(void* recvComm, int n, void** data, int* sizes, void** mhandles, void** request) { + return g_ibCastIflush(recvComm, n, data, sizes, mhandles, request); +} +ncclResult_t IbCastTest(void* request, int* done, int* sizes) { return g_ibCastTest(request, done, sizes); } + +void ResetNetIbCastP2pFakes() { + g_ibCastMultiSend = DefaultIbCastMultiSend; + g_ibCastPostFifo = DefaultIbCastPostFifo; + g_ibCastAddEvent = DefaultIbCastAddEvent; + g_ibCastGetRequest = DefaultIbCastGetRequest; + g_ibCastFreeRequest = DefaultIbCastFreeRequest; + g_ibCastIsend = DefaultIbCastIsend; + g_ibCastIrecv = DefaultIbCastIrecv; + g_ibCastIflush = DefaultIbCastIflush; + g_ibCastTest = DefaultIbCastTest; +} diff --git a/projects/rccl/test/host/fakes/net_ib_cast_p2p_fakes.h b/projects/rccl/test/host/fakes/net_ib_cast_p2p_fakes.h new file mode 100644 index 000000000000..09f997cf0e13 --- /dev/null +++ b/projects/rccl/test/host/fakes/net_ib_cast_p2p_fakes.h @@ -0,0 +1,42 @@ +/************************************************************************* + * Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +// Fakes for the symbols defined by src/transport/net_ib_cast/p2p.cc: its +// params through g_loadParam, IbCastReqTypeStr, IbCastArThreshold, and +// fail-loud hooks for its functions. + +#ifndef RCCL_TEST_HOST_NET_IB_CAST_P2P_FAKES_H_ +#define RCCL_TEST_HOST_NET_IB_CAST_P2P_FAKES_H_ + +#include + +#include "p2p_cast.h" + +extern std::function + g_ibCastMultiSend; +extern std::function + g_ibCastPostFifo; +extern std::function g_ibCastAddEvent; + +extern std::function + g_ibCastGetRequest; +extern std::function g_ibCastFreeRequest; +extern std::function + g_ibCastIsend; +extern std::function + g_ibCastIrecv; +extern std::function + g_ibCastIflush; +extern std::function g_ibCastTest; + +void ResetNetIbCastP2pFakes(); + +#endif // RCCL_TEST_HOST_NET_IB_CAST_P2P_FAKES_H_ diff --git a/projects/rccl/test/host/fakes/net_ib_cast_p2p_resiliency_fakes.cc b/projects/rccl/test/host/fakes/net_ib_cast_p2p_resiliency_fakes.cc new file mode 100644 index 000000000000..c3725c05345a --- /dev/null +++ b/projects/rccl/test/host/fakes/net_ib_cast_p2p_resiliency_fakes.cc @@ -0,0 +1,220 @@ +/************************************************************************* + * Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +// Implementation of the src/transport/net_ib_cast/p2p_resiliency.cc fakes. See +// net_ib_cast_p2p_resiliency_fakes.h. + +#include "net_ib_cast_p2p_resiliency_fakes.h" + +#include "fail_loud.h" +#include "nccl_fakes.h" +#include "signature-drift.h" + +int64_t ncclParamIbCastResiliencyPortFailover() { return g_loadParam("IB_RESILIENCY_PORT_FAILOVER", 0); } +int64_t ncclParamIbCastResiliencyPortFailoverMaxAttempts() { + return g_loadParam("IB_RESILIENCY_PORT_FAILOVER_MAX_ATTEMPTS", 1); +} +int64_t ncclParamIbCastResiliencyPortFailoverProbeDelay() { + return g_loadParam("IB_RESILIENCY_PORT_FAILOVER_PROBE_DELAY", 10); +} + +static ncclResult_t DefaultIbCastResiliencyRequestIsComplete(struct ncclIbRequest*, bool*) { + FailLoudUnfaked("net_ib_cast_p2p_resiliency_fakes", "IbCastResiliencyRequestIsComplete"); +} +std::function + g_ibCastResiliencyRequestIsComplete = DefaultIbCastResiliencyRequestIsComplete; + +static ncclResult_t DefaultIbCastResiliencyHandleCompletionError(struct ncclIbResiliency*, struct ibv_wc*, int) { + FailLoudUnfaked("net_ib_cast_p2p_resiliency_fakes", "IbCastResiliencyHandleCompletionError"); +} +std::function + g_ibCastResiliencyHandleCompletionError = DefaultIbCastResiliencyHandleCompletionError; + +static ncclResult_t DefaultIbCastResiliencyProgress(struct ncclIbResiliency*) { + FailLoudUnfaked("net_ib_cast_p2p_resiliency_fakes", "IbCastResiliencyProgress"); +} +std::function g_ibCastResiliencyProgress = DefaultIbCastResiliencyProgress; + +static ncclResult_t DefaultIbCastResiliencyInit(struct ncclIbNetCommBase*, struct ncclIbResiliency**) { + FailLoudUnfaked("net_ib_cast_p2p_resiliency_fakes", "IbCastResiliencyInit"); +} +std::function + g_ibCastResiliencyInit = DefaultIbCastResiliencyInit; + +static ncclResult_t DefaultIbCastResiliencyDestroy(struct ncclIbResiliency**) { + FailLoudUnfaked("net_ib_cast_p2p_resiliency_fakes", "IbCastResiliencyDestroy"); +} +std::function g_ibCastResiliencyDestroy = DefaultIbCastResiliencyDestroy; + +static ncclResult_t DefaultIbCastResiliencyDevInit(struct ncclIbResiliency*, uint, ncclIbDev*) { + FailLoudUnfaked("net_ib_cast_p2p_resiliency_fakes", "IbCastResiliencyDevInit"); +} +std::function + g_ibCastResiliencyDevInit = DefaultIbCastResiliencyDevInit; + +static ncclResult_t DefaultIbCastResiliencyDevDestroy(struct ncclIbResiliency*, uint) { + FailLoudUnfaked("net_ib_cast_p2p_resiliency_fakes", "IbCastResiliencyDevDestroy"); +} +std::function + g_ibCastResiliencyDevDestroy = DefaultIbCastResiliencyDevDestroy; + +static ncclResult_t DefaultIbCastResiliencyDataCqSizeGet(struct ncclIbResiliency*, uint, int*) { + FailLoudUnfaked("net_ib_cast_p2p_resiliency_fakes", "IbCastResiliencyDataCqSizeGet"); +} +std::function + g_ibCastResiliencyDataCqSizeGet = DefaultIbCastResiliencyDataCqSizeGet; + +static ncclResult_t DefaultIbCastResiliencyDataRqSizeGet(struct ncclIbResiliency*, uint, uint32_t*) { + FailLoudUnfaked("net_ib_cast_p2p_resiliency_fakes", "IbCastResiliencyDataRqSizeGet"); +} +std::function + g_ibCastResiliencyDataRqSizeGet = DefaultIbCastResiliencyDataRqSizeGet; + +static ncclResult_t DefaultIbCastResiliencyDeviceNumSet(struct ncclIbResiliency*, int, int) { + FailLoudUnfaked("net_ib_cast_p2p_resiliency_fakes", "IbCastResiliencyDeviceNumSet"); +} +std::function + g_ibCastResiliencyDeviceNumSet = DefaultIbCastResiliencyDeviceNumSet; + +static ncclResult_t DefaultIbCastResiliencySenderCreateQps(struct ncclIbResiliency*, struct ncclIbResiliencyInfo*) { + FailLoudUnfaked("net_ib_cast_p2p_resiliency_fakes", "IbCastResiliencySenderCreateQps"); +} +std::function + g_ibCastResiliencySenderCreateQps = DefaultIbCastResiliencySenderCreateQps; + +static ncclResult_t DefaultIbCastResiliencySenderQpsToRts(struct ncclIbResiliency*, struct ncclIbConnectionMetadata*) { + FailLoudUnfaked("net_ib_cast_p2p_resiliency_fakes", "IbCastResiliencySenderQpsToRts"); +} +std::function + g_ibCastResiliencySenderQpsToRts = DefaultIbCastResiliencySenderQpsToRts; + +static ncclResult_t DefaultIbCastResiliencyReceiverQpsCreateToRts(struct ncclIbResiliency*, + struct ncclIbConnectionMetadata*, + struct ncclIbResiliencyInfo*) { + FailLoudUnfaked("net_ib_cast_p2p_resiliency_fakes", "IbCastResiliencyReceiverQpsCreateToRts"); +} +std::function + g_ibCastResiliencyReceiverQpsCreateToRts = DefaultIbCastResiliencyReceiverQpsCreateToRts; + +static ncclResult_t DefaultIbCastResiliencyQpsReconfigure(struct ncclIbResiliency*, int, bool*) { + FailLoudUnfaked("net_ib_cast_p2p_resiliency_fakes", "IbCastResiliencyQpsReconfigure"); +} +std::function + g_ibCastResiliencyQpsReconfigure = DefaultIbCastResiliencyQpsReconfigure; + +static ncclResult_t DefaultIbCastResiliencyQpReconfigure(struct ncclIbResiliency*, struct ncclIbQp*, + struct ncclIbNetCommDevBase*, int, bool*) { + FailLoudUnfaked("net_ib_cast_p2p_resiliency_fakes", "IbCastResiliencyQpReconfigure"); +} +std::function + g_ibCastResiliencyQpReconfigure = DefaultIbCastResiliencyQpReconfigure; + +static ncclResult_t DefaultIbCastResiliencyClose(struct ncclIbResiliency*) { + FailLoudUnfaked("net_ib_cast_p2p_resiliency_fakes", "IbCastResiliencyClose"); +} +std::function g_ibCastResiliencyClose = DefaultIbCastResiliencyClose; + +static ncclResult_t DefaultIbCastResiliencyRemoteCompletionRecordsSet(struct ncclIbResiliency*, uint32_t, uint64_t, + uint) { + FailLoudUnfaked("net_ib_cast_p2p_resiliency_fakes", "IbCastResiliencyRemoteCompletionRecordsSet"); +} +std::function + g_ibCastResiliencyRemoteCompletionRecordsSet = DefaultIbCastResiliencyRemoteCompletionRecordsSet; + +ASSERT_HOOK_MATCHES_PROD(g_ibCastResiliencyRequestIsComplete, IbCastResiliencyRequestIsComplete); +ASSERT_HOOK_MATCHES_PROD(g_ibCastResiliencyHandleCompletionError, IbCastResiliencyHandleCompletionError); +ASSERT_HOOK_MATCHES_PROD(g_ibCastResiliencyProgress, IbCastResiliencyProgress); +ASSERT_HOOK_MATCHES_PROD(g_ibCastResiliencyInit, IbCastResiliencyInit); +ASSERT_HOOK_MATCHES_PROD(g_ibCastResiliencyDestroy, IbCastResiliencyDestroy); +ASSERT_HOOK_MATCHES_PROD(g_ibCastResiliencyDevInit, IbCastResiliencyDevInit); +ASSERT_HOOK_MATCHES_PROD(g_ibCastResiliencyDevDestroy, IbCastResiliencyDevDestroy); +ASSERT_HOOK_MATCHES_PROD(g_ibCastResiliencyDataCqSizeGet, IbCastResiliencyDataCqSizeGet); +ASSERT_HOOK_MATCHES_PROD(g_ibCastResiliencyDataRqSizeGet, IbCastResiliencyDataRqSizeGet); +ASSERT_HOOK_MATCHES_PROD(g_ibCastResiliencyDeviceNumSet, IbCastResiliencyDeviceNumSet); +ASSERT_HOOK_MATCHES_PROD(g_ibCastResiliencySenderCreateQps, IbCastResiliencySenderCreateQps); +ASSERT_HOOK_MATCHES_PROD(g_ibCastResiliencySenderQpsToRts, IbCastResiliencySenderQpsToRts); +ASSERT_HOOK_MATCHES_PROD(g_ibCastResiliencyReceiverQpsCreateToRts, IbCastResiliencyReceiverQpsCreateToRts); +ASSERT_HOOK_MATCHES_PROD(g_ibCastResiliencyQpsReconfigure, IbCastResiliencyQpsReconfigure); +ASSERT_HOOK_MATCHES_PROD(g_ibCastResiliencyQpReconfigure, IbCastResiliencyQpReconfigure); +ASSERT_HOOK_MATCHES_PROD(g_ibCastResiliencyClose, IbCastResiliencyClose); +ASSERT_HOOK_MATCHES_PROD(g_ibCastResiliencyRemoteCompletionRecordsSet, IbCastResiliencyRemoteCompletionRecordsSet); + +ncclResult_t IbCastResiliencyRequestIsComplete(struct ncclIbRequest* request, bool* isComplete) { + return g_ibCastResiliencyRequestIsComplete(request, isComplete); +} +ncclResult_t IbCastResiliencyHandleCompletionError(struct ncclIbResiliency* resCtx, struct ibv_wc* wc, int devIndex) { + return g_ibCastResiliencyHandleCompletionError(resCtx, wc, devIndex); +} +ncclResult_t IbCastResiliencyProgress(struct ncclIbResiliency* resCtx) { + return g_ibCastResiliencyProgress(resCtx); +} +ncclResult_t IbCastResiliencyInit(struct ncclIbNetCommBase* baseComm, struct ncclIbResiliency** resCtx) { + return g_ibCastResiliencyInit(baseComm, resCtx); +} +ncclResult_t IbCastResiliencyDestroy(struct ncclIbResiliency** resCtx) { + return g_ibCastResiliencyDestroy(resCtx); +} +ncclResult_t IbCastResiliencyDevInit(struct ncclIbResiliency* resCtx, uint devIndex, ncclIbDev* ibDev) { + return g_ibCastResiliencyDevInit(resCtx, devIndex, ibDev); +} +ncclResult_t IbCastResiliencyDevDestroy(struct ncclIbResiliency* resCtx, uint devIndex) { + return g_ibCastResiliencyDevDestroy(resCtx, devIndex); +} +ncclResult_t IbCastResiliencyDataCqSizeGet(struct ncclIbResiliency* resCtx, uint devIndex, int* cqSize) { + return g_ibCastResiliencyDataCqSizeGet(resCtx, devIndex, cqSize); +} +ncclResult_t IbCastResiliencyDataRqSizeGet(struct ncclIbResiliency* resCtx, uint devIndex, uint32_t* rqSize) { + return g_ibCastResiliencyDataRqSizeGet(resCtx, devIndex, rqSize); +} +ncclResult_t IbCastResiliencyDeviceNumSet(struct ncclIbResiliency* resCtx, int nLocalDevs, int nRemDevs) { + return g_ibCastResiliencyDeviceNumSet(resCtx, nLocalDevs, nRemDevs); +} +ncclResult_t IbCastResiliencySenderCreateQps(struct ncclIbResiliency* resCtx, + struct ncclIbResiliencyInfo* localResiliencyInfo) { + return g_ibCastResiliencySenderCreateQps(resCtx, localResiliencyInfo); +} +ncclResult_t IbCastResiliencySenderQpsToRts(struct ncclIbResiliency* resCtx, struct ncclIbConnectionMetadata* remInfo) { + return g_ibCastResiliencySenderQpsToRts(resCtx, remInfo); +} +ncclResult_t IbCastResiliencyReceiverQpsCreateToRts(struct ncclIbResiliency* resCtx, + struct ncclIbConnectionMetadata* remInfo, + struct ncclIbResiliencyInfo* localResiliencyInfo) { + return g_ibCastResiliencyReceiverQpsCreateToRts(resCtx, remInfo, localResiliencyInfo); +} +ncclResult_t IbCastResiliencyQpsReconfigure(struct ncclIbResiliency* resCtx, int devIndex, bool* success) { + return g_ibCastResiliencyQpsReconfigure(resCtx, devIndex, success); +} +ncclResult_t IbCastResiliencyQpReconfigure(struct ncclIbResiliency* resCtx, struct ncclIbQp* qp, + struct ncclIbNetCommDevBase* devBase, int devIndex, bool* success) { + return g_ibCastResiliencyQpReconfigure(resCtx, qp, devBase, devIndex, success); +} +ncclResult_t IbCastResiliencyClose(struct ncclIbResiliency* resCtx) { + return g_ibCastResiliencyClose(resCtx); +} +ncclResult_t IbCastResiliencyRemoteCompletionRecordsSet(struct ncclIbResiliency* resCtx, uint32_t cmplsRecordsRkey, + uint64_t cmplsRecordsAddr, uint devIndex) { + return g_ibCastResiliencyRemoteCompletionRecordsSet(resCtx, cmplsRecordsRkey, cmplsRecordsAddr, devIndex); +} + +void ResetNetIbCastP2pResiliencyFakes() { + g_ibCastResiliencyRequestIsComplete = DefaultIbCastResiliencyRequestIsComplete; + g_ibCastResiliencyHandleCompletionError = DefaultIbCastResiliencyHandleCompletionError; + g_ibCastResiliencyProgress = DefaultIbCastResiliencyProgress; + g_ibCastResiliencyInit = DefaultIbCastResiliencyInit; + g_ibCastResiliencyDestroy = DefaultIbCastResiliencyDestroy; + g_ibCastResiliencyDevInit = DefaultIbCastResiliencyDevInit; + g_ibCastResiliencyDevDestroy = DefaultIbCastResiliencyDevDestroy; + g_ibCastResiliencyDataCqSizeGet = DefaultIbCastResiliencyDataCqSizeGet; + g_ibCastResiliencyDataRqSizeGet = DefaultIbCastResiliencyDataRqSizeGet; + g_ibCastResiliencyDeviceNumSet = DefaultIbCastResiliencyDeviceNumSet; + g_ibCastResiliencySenderCreateQps = DefaultIbCastResiliencySenderCreateQps; + g_ibCastResiliencySenderQpsToRts = DefaultIbCastResiliencySenderQpsToRts; + g_ibCastResiliencyReceiverQpsCreateToRts = DefaultIbCastResiliencyReceiverQpsCreateToRts; + g_ibCastResiliencyQpsReconfigure = DefaultIbCastResiliencyQpsReconfigure; + g_ibCastResiliencyQpReconfigure = DefaultIbCastResiliencyQpReconfigure; + g_ibCastResiliencyClose = DefaultIbCastResiliencyClose; + g_ibCastResiliencyRemoteCompletionRecordsSet = DefaultIbCastResiliencyRemoteCompletionRecordsSet; +} diff --git a/projects/rccl/test/host/fakes/net_ib_cast_p2p_resiliency_fakes.h b/projects/rccl/test/host/fakes/net_ib_cast_p2p_resiliency_fakes.h new file mode 100644 index 000000000000..3b829af22969 --- /dev/null +++ b/projects/rccl/test/host/fakes/net_ib_cast_p2p_resiliency_fakes.h @@ -0,0 +1,60 @@ +/************************************************************************* + * Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +// Controllable seams for src/transport/net_ib_cast/p2p_resiliency.cc, plus its +// params through g_loadParam. Every hook fails loud by default: production +// only reaches these with a non-NULL resiliency context, which a test must +// set up on purpose. + +#ifndef RCCL_TEST_HOST_NET_IB_CAST_P2P_RESILIENCY_FAKES_H_ +#define RCCL_TEST_HOST_NET_IB_CAST_P2P_RESILIENCY_FAKES_H_ + +#include + +#include "p2p_resiliency_cast.h" + +extern std::function + g_ibCastResiliencyRequestIsComplete; +extern std::function + g_ibCastResiliencyHandleCompletionError; +extern std::function + g_ibCastResiliencyProgress; +extern std::function + g_ibCastResiliencyInit; +extern std::function + g_ibCastResiliencyDestroy; +extern std::function + g_ibCastResiliencyDevInit; +extern std::function + g_ibCastResiliencyDevDestroy; +extern std::function + g_ibCastResiliencyDataCqSizeGet; +extern std::function + g_ibCastResiliencyDataRqSizeGet; +extern std::function + g_ibCastResiliencyDeviceNumSet; +extern std::function + g_ibCastResiliencySenderCreateQps; +extern std::function + g_ibCastResiliencySenderQpsToRts; +extern std::function + g_ibCastResiliencyReceiverQpsCreateToRts; +extern std::function + g_ibCastResiliencyQpsReconfigure; +extern std::function + g_ibCastResiliencyQpReconfigure; +extern std::function + g_ibCastResiliencyClose; +extern std::function + g_ibCastResiliencyRemoteCompletionRecordsSet; + +void ResetNetIbCastP2pResiliencyFakes(); + +#endif // RCCL_TEST_HOST_NET_IB_CAST_P2P_RESILIENCY_FAKES_H_ diff --git a/projects/rccl/test/host/fakes/net_ib_cast_qp_sharing_fakes.cc b/projects/rccl/test/host/fakes/net_ib_cast_qp_sharing_fakes.cc new file mode 100644 index 000000000000..f1ed75a37aae --- /dev/null +++ b/projects/rccl/test/host/fakes/net_ib_cast_qp_sharing_fakes.cc @@ -0,0 +1,28 @@ +/************************************************************************* + * Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +// Implementation of the src/transport/net_ib_cast/qp_sharing.cc fakes. See net_ib_cast_qp_sharing_fakes.h. + +#include "net_ib_cast_qp_sharing_fakes.h" + +#include "fakes/nccl_fakes.h" +#include "param_redirect.h" +#include "qp_sharing.h" + +// Same lines as qp_sharing.cc; test/host/CMakeLists.txt pins both copies. +RCCL_PARAM(IbCastQpSharingEnable, "IB_QP_SHARING_ENABLE", 0); +RCCL_PARAM(IbCastCommNGroups, "IB_COMM_NGROUPS", 4); +RCCL_PARAM(IbCastQpSharingValidatePool, "IB_QP_SHARING_VALIDATE_POOL", 0); + +bool IbCastQpSharingGlobalEnable = false; + +int g_ibCastValidateSharedQpPoolCalls = 0; +void IbCastValidateSharedQpPool(void) { g_ibCastValidateSharedQpPoolCalls++; } + +void ResetNetIbCastQpSharingFakes() { + IbCastQpSharingGlobalEnable = false; + g_ibCastValidateSharedQpPoolCalls = 0; +} diff --git a/projects/rccl/test/host/fakes/net_ib_cast_qp_sharing_fakes.h b/projects/rccl/test/host/fakes/net_ib_cast_qp_sharing_fakes.h new file mode 100644 index 000000000000..220d6cac8428 --- /dev/null +++ b/projects/rccl/test/host/fakes/net_ib_cast_qp_sharing_fakes.h @@ -0,0 +1,18 @@ +/************************************************************************* + * Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +// State owned by src/transport/net_ib_cast/qp_sharing.cc, for binaries that do not compile it. +// IbCastQpSharingGlobalEnable itself is declared by qp_sharing.h and starts false, as in production. +// IbCastValidateSharedQpPool only counts its calls. + +#ifndef RCCL_TEST_HOST_NET_IB_CAST_QP_SHARING_FAKES_H_ +#define RCCL_TEST_HOST_NET_IB_CAST_QP_SHARING_FAKES_H_ + +extern int g_ibCastValidateSharedQpPoolCalls; + +void ResetNetIbCastQpSharingFakes(); + +#endif // RCCL_TEST_HOST_NET_IB_CAST_QP_SHARING_FAKES_H_ diff --git a/projects/rccl/test/host/fakes/os_fakes.cc b/projects/rccl/test/host/fakes/os_fakes.cc index 206f35fb8b50..eb16af38a432 100644 --- a/projects/rccl/test/host/fakes/os_fakes.cc +++ b/projects/rccl/test/host/fakes/os_fakes.cc @@ -26,6 +26,8 @@ void ncclOsAlignedFree(void* ptr) { free(ptr); } ncclResult_t ncclOsInitialize() { return ncclSuccess; } uint64_t ncclOsGetPid() { return 4321; } +ncclResult_t ncclOsSetFilesLimit() { return ncclSuccess; } +char* ncclOsStrSep(char** stringp, const char* delim) { return strsep(stringp, delim); } size_t ncclOsGetPageSize() { return 4096; } size_t ncclOsGetCommMempoolMaxSize() { return 0; } // src/os/linux.cc: 0 = no cap diff --git a/projects/rccl/test/host/fakes/socket_fakes.cc b/projects/rccl/test/host/fakes/socket_fakes.cc new file mode 100644 index 000000000000..589392d3fa3f --- /dev/null +++ b/projects/rccl/test/host/fakes/socket_fakes.cc @@ -0,0 +1,113 @@ +/************************************************************************* + * Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +// Implementation of the src/misc/socket.cc fakes. See socket_fakes.h. + +#include "socket_fakes.h" + +#include + +#include "signature-drift.h" + +static ncclResult_t DefaultNcclSocketInit(struct ncclSocket*, const union ncclSocketAddress*, uint64_t, + enum ncclSocketType, volatile uint32_t*, int, int) { + return ncclSystemError; +} +std::function + g_ncclSocketInit = DefaultNcclSocketInit; + +static ncclResult_t DefaultNcclSocketListen(struct ncclSocket*) { return ncclSystemError; } +std::function g_ncclSocketListen = DefaultNcclSocketListen; + +static ncclResult_t DefaultNcclSocketGetAddr(struct ncclSocket*, union ncclSocketAddress*) { return ncclSystemError; } +std::function g_ncclSocketGetAddr = + DefaultNcclSocketGetAddr; + +static ncclResult_t DefaultNcclSocketConnect(struct ncclSocket*) { return ncclSystemError; } +std::function g_ncclSocketConnect = DefaultNcclSocketConnect; + +static ncclResult_t DefaultNcclSocketReady(struct ncclSocket*, int*) { return ncclSystemError; } +std::function g_ncclSocketReady = DefaultNcclSocketReady; + +static ncclResult_t DefaultNcclSocketAccept(struct ncclSocket*, struct ncclSocket*, bool) { return ncclSystemError; } +std::function g_ncclSocketAccept = + DefaultNcclSocketAccept; + +static ncclResult_t DefaultNcclSocketProgress(int, struct ncclSocket*, void*, int, int*, bool*) { + return ncclSystemError; +} +std::function g_ncclSocketProgress = + DefaultNcclSocketProgress; + +static ncclResult_t DefaultNcclSocketClose(struct ncclSocket*, bool) { return ncclSuccess; } +std::function g_ncclSocketClose = DefaultNcclSocketClose; + +static const char* DefaultNcclSocketToString(const union ncclSocketAddress*, char* buf, size_t bufLen, const int) { + if (buf && bufLen > 0) buf[0] = '\0'; + return buf; +} +std::function g_ncclSocketToString = + DefaultNcclSocketToString; + +static ncclResult_t DefaultNcclFindInterfaces(char*, union ncclSocketAddress*, int, int, int* nIfs) { + if (nIfs) *nIfs = 0; + return ncclSuccess; +} +std::function g_ncclFindInterfaces = + DefaultNcclFindInterfaces; + +ASSERT_HOOK_MATCHES_PROD(g_ncclSocketInit, ncclSocketInit); +ASSERT_HOOK_MATCHES_PROD(g_ncclSocketListen, ncclSocketListen); +ASSERT_HOOK_MATCHES_PROD(g_ncclSocketGetAddr, ncclSocketGetAddr); +ASSERT_HOOK_MATCHES_PROD(g_ncclSocketConnect, ncclSocketConnect); +ASSERT_HOOK_MATCHES_PROD(g_ncclSocketReady, ncclSocketReady); +ASSERT_HOOK_MATCHES_PROD(g_ncclSocketAccept, ncclSocketAccept); +ASSERT_HOOK_MATCHES_PROD(g_ncclSocketProgress, ncclSocketProgress); +ASSERT_HOOK_MATCHES_PROD(g_ncclSocketClose, ncclSocketClose); +ASSERT_HOOK_MATCHES_PROD(g_ncclSocketToString, ncclSocketToString); +ASSERT_HOOK_MATCHES_PROD(g_ncclFindInterfaces, ncclFindInterfaces); + +uint64_t ncclSocketDefaultMagic(void) { return NCCL_SOCKET_MAGIC; } + +ncclResult_t ncclSocketInit(struct ncclSocket* sock, const union ncclSocketAddress* addr, uint64_t magic, + enum ncclSocketType type, volatile uint32_t* abortFlag, int asyncFlag, int customRetry) { + return g_ncclSocketInit(sock, addr, magic, type, abortFlag, asyncFlag, customRetry); +} +ncclResult_t ncclSocketListen(struct ncclSocket* sock) { return g_ncclSocketListen(sock); } +ncclResult_t ncclSocketGetAddr(struct ncclSocket* sock, union ncclSocketAddress* addr) { + return g_ncclSocketGetAddr(sock, addr); +} +ncclResult_t ncclSocketConnect(struct ncclSocket* sock) { return g_ncclSocketConnect(sock); } +ncclResult_t ncclSocketReady(struct ncclSocket* sock, int* running) { return g_ncclSocketReady(sock, running); } +ncclResult_t ncclSocketAccept(struct ncclSocket* sock, struct ncclSocket* ulistenSock, bool retry) { + return g_ncclSocketAccept(sock, ulistenSock, retry); +} +ncclResult_t ncclSocketProgress(int op, struct ncclSocket* sock, void* ptr, int size, int* offset, bool* closed) { + return g_ncclSocketProgress(op, sock, ptr, size, offset, closed); +} +ncclResult_t ncclSocketClose(struct ncclSocket* sock, bool wait) { return g_ncclSocketClose(sock, wait); } +const char* ncclSocketToString(const union ncclSocketAddress* addr, char* buf, size_t bufLen, + const int numericHostForm) { + return g_ncclSocketToString(addr, buf, bufLen, numericHostForm); +} +ncclResult_t ncclFindInterfaces(char* ifNames, union ncclSocketAddress* ifAddrs, int ifNameMaxSize, int maxIfs, + int* nIfs) { + return g_ncclFindInterfaces(ifNames, ifAddrs, ifNameMaxSize, maxIfs, nIfs); +} + +void ResetSocketFakes() { + g_ncclSocketInit = DefaultNcclSocketInit; + g_ncclSocketListen = DefaultNcclSocketListen; + g_ncclSocketGetAddr = DefaultNcclSocketGetAddr; + g_ncclSocketConnect = DefaultNcclSocketConnect; + g_ncclSocketReady = DefaultNcclSocketReady; + g_ncclSocketAccept = DefaultNcclSocketAccept; + g_ncclSocketProgress = DefaultNcclSocketProgress; + g_ncclSocketClose = DefaultNcclSocketClose; + g_ncclSocketToString = DefaultNcclSocketToString; + g_ncclFindInterfaces = DefaultNcclFindInterfaces; +} diff --git a/projects/rccl/test/host/fakes/socket_fakes.h b/projects/rccl/test/host/fakes/socket_fakes.h new file mode 100644 index 000000000000..44de52864bc4 --- /dev/null +++ b/projects/rccl/test/host/fakes/socket_fakes.h @@ -0,0 +1,46 @@ +/************************************************************************* + * Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +// Controllable seams for the socket layer defined by src/misc/socket.cc, for +// units that reach it but are not about it. Defaults: every acquisition, +// transfer and query fails with ncclSystemError; ncclSocketClose succeeds; +// ncclFindInterfaces finds no interface; ncclSocketToString writes an empty +// string; ncclSocketDefaultMagic is NCCL_SOCKET_MAGIC, its unset-env value. + +#ifndef RCCL_TEST_HOST_SOCKET_FAKES_H_ +#define RCCL_TEST_HOST_SOCKET_FAKES_H_ + +#include +#include +#include + +#include "socket.h" + +extern std::function + g_ncclSocketInit; +extern std::function g_ncclSocketListen; +extern std::function + g_ncclSocketGetAddr; +extern std::function g_ncclSocketConnect; +extern std::function g_ncclSocketReady; +extern std::function + g_ncclSocketAccept; +extern std::function + g_ncclSocketProgress; +extern std::function g_ncclSocketClose; +extern std::function + g_ncclSocketToString; +extern std::function + g_ncclFindInterfaces; + +void ResetSocketFakes(); + +#endif // RCCL_TEST_HOST_SOCKET_FAKES_H_ From 56646bf77e0e25fdb1cd44e362686b01f6366e2a Mon Sep 17 00:00:00 2001 From: Alex Duforest Date: Tue, 20 Oct 2026 12:16:00 -0700 Subject: [PATCH 04/15] test(rccl): add host microtests for net_ib_cast init, capability and GIN rccl-UnitTestsMicroNetIbCastInit covers src/transport/net_ib_cast/{init,common, capability,reg,gdr,multiplane,gin,net_ib_cast_gid_inspect}.cc over fake verbs contexts, with the real xml.cc, utils.cc and net_telemetry.cc. Production bugs are pinned as KnownBug_ tests (AICOMRCCL-3088..3096). Co-Authored-By: Claude --- projects/rccl/test/host/CMakeLists.txt | 77 + projects/rccl/test/host/MICROTEST_README.md | 12 +- .../test/host/net-ib-cast-capability-test.cc | 403 +++++ .../rccl/test/host/net-ib-cast-common-test.cc | 438 ++++++ .../rccl/test/host/net-ib-cast-gdr-test.cc | 275 ++++ .../test/host/net-ib-cast-gid-inspect-test.cc | 236 +++ .../rccl/test/host/net-ib-cast-gin-test.cc | 1151 ++++++++++++++ .../rccl/test/host/net-ib-cast-init-test.cc | 1360 +++++++++++++++++ .../test/host/net-ib-cast-multiplane-test.cc | 296 ++++ .../rccl/test/host/net-ib-cast-reg-test.cc | 333 ++++ .../rccl/test/host/net-ib-cast-test-common.h | 255 ++++ projects/rccl/test/host/run_host_tests.sh | 1 + ...est_categories_micro_net_ib_cast_init.yaml | 47 + .../test_runner/configs/ci-precheckin.json | 17 + .../scripts/test_runner/lib/test_executor.py | 3 +- 15 files changed, 4901 insertions(+), 3 deletions(-) create mode 100644 projects/rccl/test/host/net-ib-cast-capability-test.cc create mode 100644 projects/rccl/test/host/net-ib-cast-common-test.cc create mode 100644 projects/rccl/test/host/net-ib-cast-gdr-test.cc create mode 100644 projects/rccl/test/host/net-ib-cast-gid-inspect-test.cc create mode 100644 projects/rccl/test/host/net-ib-cast-gin-test.cc create mode 100644 projects/rccl/test/host/net-ib-cast-init-test.cc create mode 100644 projects/rccl/test/host/net-ib-cast-multiplane-test.cc create mode 100644 projects/rccl/test/host/net-ib-cast-reg-test.cc create mode 100644 projects/rccl/test/host/net-ib-cast-test-common.h create mode 100644 projects/rccl/test/test_categories_micro_net_ib_cast_init.yaml diff --git a/projects/rccl/test/host/CMakeLists.txt b/projects/rccl/test/host/CMakeLists.txt index 795b3c2fc0a7..ddfd7854b1e7 100644 --- a/projects/rccl/test/host/CMakeLists.txt +++ b/projects/rccl/test/host/CMakeLists.txt @@ -105,6 +105,17 @@ foreach(_path_cc "${_rccl_src_root}/graph/topo.cc" "${CMAKE_CURRENT_LIST_DIR}/fa rccl_assert_source_line("${_path_cc}" "const char* topoPathTypeStr[] = {\"LOC\", \"XGMI\", \"NVB\", \"C2C\", \"PIX\", \ \"PXB\", \"P2C\", \"PXN\", \"PHB\", \"SYS\", \"NET\", \"DIS\"};" "fakes/topo_stubs.cc: topoPathTypeStr") endforeach() +foreach(_path_cc "${_rccl_src_root}/transport/net_ib_cast/p2p_resiliency_recovery.cc" + "${CMAKE_CURRENT_LIST_DIR}/fakes/net_ib_cast_p2p_resiliency_recovery_fakes.cc") + rccl_assert_source_line("${_path_cc}" "NCCL_PARAM(IbCastResiliencyPortRecovery, \"IB_RESILIENCY_PORT_RECOVERY\", 0);" + "fakes/net_ib_cast_p2p_resiliency_recovery_fakes.cc: ncclParamIbCastResiliencyPortRecovery()") +endforeach() +rccl_assert_source_line("${_rccl_src_root}/transport/net.cc" + "NCCL_PARAM(NetOptionalRecvCompletion, \"NET_OPTIONAL_RECV_COMPLETION\", 1);" + "fakes/transport_stubs.cc: ncclParamNetOptionalRecvCompletion()") +rccl_assert_source_line("${CMAKE_CURRENT_LIST_DIR}/fakes/transport_stubs.cc" + "int64_t ncclParamNetOptionalRecvCompletion() { return g_loadParam(\"NET_OPTIONAL_RECV_COMPLETION\", 1); }" + "the NetOptionalRecvCompletion fake default") unset(_rccl_src_root) # Define the per-target micro-test source lists shared by BOTH build paths. @@ -675,6 +686,43 @@ function(rccl_define_micro_source_lists) fakes/utils_fakes.cc # getBusId (alloc.h side stream) ../common/main_altrsmi.cpp PARENT_SCOPE) + + # rccl-UnitTestsMicroNetIbCastInit -- src/transport/net_ib_cast/{init,common,capability,reg,gdr,multiplane,gin, + # net_ib_cast_gid_inspect}.cc. Own binary: it fakes gdr_probe.cc, which rccl-UnitTestsMicro compiles. + set(TEST_MICRO_NET_IB_CAST_INIT_SOURCE_FILES + net-ib-cast-init-test.cc + net-ib-cast-capability-test.cc + net-ib-cast-common-test.cc + net-ib-cast-gdr-test.cc + net-ib-cast-gid-inspect-test.cc + net-ib-cast-gin-test.cc + net-ib-cast-multiplane-test.cc + net-ib-cast-reg-test.cc + fakes/env_fakes.cc + fakes/gdr_peermem_fakes.cc + fakes/gdr_probe_fakes.cc + fakes/hip_fakes.cc + fakes/ibvwrap_fakes.cc + fakes/ionicdvwrap_fakes.cc + fakes/libc_fakes.cc + fakes/mlx5dvwrap_fakes.cc + fakes/nccl_fakes.cc + fakes/net_ib_cast_connect_fakes.cc + fakes/net_ib_cast_p2p_fakes.cc + fakes/net_ib_cast_p2p_resiliency_fakes.cc + fakes/net_ib_cast_p2p_resiliency_recovery_fakes.cc + fakes/net_ib_cast_qp_sharing_fakes.cc + fakes/net_ib_cast_scheduler_fakes.cc + fakes/os_fakes.cc + fakes/rocmwrap_fakes.cc # rocmLibraryInit (gdr.cc DMA-BUF probe) + fakes/socket_fakes.cc + fakes/transport_stubs.cc # rcclUseAinic, NET_OPTIONAL_RECV_COMPLETION + ${RCCL_HOST_HIPIFY_DIR}/src/graph/xml.cc # real parser for the multiplane map + ${RCCL_HOST_HIPIFY_DIR}/src/misc/utils.cc # real NCCL_IB_HCA parsing, pciPathToInt64 + ${RCCL_HOST_HIPIFY_DIR}/src/transport/net_telemetry.cc # real, off unless RCCL_TELEMETRY_ENABLE=1 + ../common/ProcessIsolatedTestRunner.cpp + ../common/main_altrsmi.cpp + PARENT_SCOPE) endfunction() # Create one host-only micro-test binary and apply all the boilerplate that @@ -1702,6 +1750,35 @@ rccl_add_micro_binary(rccl-UnitTestsMicroDda FABRIC_GPU_BARRIER_CC_PATH="${RCCL_HOST_HIPIFY_DIR}/src/algorithms/dda/fabric/fabric_gpu_barrier.cu.cpp" CATEGORIES_YAML "${CMAKE_CURRENT_SOURCE_DIR}/../test_categories_micro_dda.yaml") +# --------------------------------------------------------------------------- +# rccl-UnitTestsMicroNetIbCastInit +# --------------------------------------------------------------------------- +set_source_files_properties( + ${RCCL_HOST_HIPIFY_DIR}/src/graph/xml.cc + ${RCCL_HOST_HIPIFY_DIR}/src/misc/utils.cc + ${RCCL_HOST_HIPIFY_DIR}/src/transport/net_telemetry.cc + PROPERTIES GENERATED TRUE) +rccl_add_micro_binary(rccl-UnitTestsMicroNetIbCastInit + SOURCES ${TEST_MICRO_NET_IB_CAST_INIT_SOURCE_FILES} + EXTRA_INCLUDES + ${RCCL_HOST_HIPIFY_DIR}/src/transport/net_ib_cast # common_cast.h and the other cast headers + ${RCCL_HOST_HIPIFY_DIR}/src/transport # net_ib_limits.h, net_ib_gid_inspect.h + EXTRA_DEFS + ${_MICRO_MODE_DEFS} + ${_MICRO_AMDSMI_DEF} + ENABLE_FAULT_INJECTION # net_ib_cast_gid_inspect.cc and the GID-change hook exist only with it + # init, common, reg, gdr and gin share basenames with net_ib/ (and init.cc with src/), which + # src/CMakeLists.txt lists first, so add_file_unique() renames these copies. + NET_IB_CAST_INIT_CC_PATH="${RCCL_HOST_HIPIFY_DIR}/src/transport/net_ib_cast/init_tmp.cc" + NET_IB_CAST_COMMON_CC_PATH="${RCCL_HOST_HIPIFY_DIR}/src/transport/net_ib_cast/common_tmp.cc" + NET_IB_CAST_REG_CC_PATH="${RCCL_HOST_HIPIFY_DIR}/src/transport/net_ib_cast/reg_tmp.cc" + NET_IB_CAST_GDR_CC_PATH="${RCCL_HOST_HIPIFY_DIR}/src/transport/net_ib_cast/gdr_tmp.cc" + NET_IB_CAST_GIN_CC_PATH="${RCCL_HOST_HIPIFY_DIR}/src/transport/net_ib_cast/gin_tmp.cc" + NET_IB_CAST_CAPABILITY_CC_PATH="${RCCL_HOST_HIPIFY_DIR}/src/transport/net_ib_cast/capability.cc" + NET_IB_CAST_MULTIPLANE_CC_PATH="${RCCL_HOST_HIPIFY_DIR}/src/transport/net_ib_cast/multiplane.cc" + NET_IB_CAST_GID_INSPECT_CC_PATH="${RCCL_HOST_HIPIFY_DIR}/src/transport/net_ib_cast/net_ib_cast_gid_inspect.cc" + CATEGORIES_YAML "${CMAKE_CURRENT_SOURCE_DIR}/../test_categories_micro_net_ib_cast_init.yaml") + # ===================================================================== # Standalone-only tail: coverage + no CTest registration. # ===================================================================== diff --git a/projects/rccl/test/host/MICROTEST_README.md b/projects/rccl/test/host/MICROTEST_README.md index 159a22ac3138..8fd2398291ff 100644 --- a/projects/rccl/test/host/MICROTEST_README.md +++ b/projects/rccl/test/host/MICROTEST_README.md @@ -454,6 +454,13 @@ export colliding non-`static` symbols; otherwise a unit needs its own binary: - `fabric/fabric_gpu_barrier.cu` (`FABRIC_GPU_BARRIER_CC_PATH`, from `fabric-gpu-barrier-test.cc`); suites `FabricGpuBarrier*`. +- **`rccl-UnitTestsMicroNetIbCastInit`** — `transport/net_ib_cast/init.cc` + (`NET_IB_CAST_INIT_CC_PATH`, from `net-ib-cast-init-test.cc`), plus `common.cc`, + `capability.cc`, `reg.cc`, `gdr.cc`, `multiplane.cc`, `gin.cc` and + `net_ib_cast_gid_inspect.cc` from `net-ib-cast--test.cc`; suites `NetIbCast*`. + Its own binary: it fakes `misc/gdr_probe.cc`, which `rccl-UnitTestsMicro` compiles. + See `test_categories_micro_net_ib_cast_init.yaml`. + Everything below (seams, fakes, coverage) applies to all of them; the concrete examples use `p2p.cc`. @@ -964,8 +971,8 @@ can be configured **directly** to build every host binary — `rccl-HostUnitTest `rccl-UnitTestsMicroInit[-uncached|-faultinj]`, `rccl-UnitTestsMicroEnqueue[-devlinker]`, `rccl-UnitTestsMicroGinHost`, `rccl-UnitTestsMicroGinHostFinalize`, `rccl-UnitTestsMicroDdaIpc`, `rccl-UnitTestsMicroCe`, `rccl-UnitTestsMicroDiagnostics`, `rccl-UnitTestsMicroGraph`, `rccl-UnitTestsMicroDiagnostics`, `rccl-UnitTestsMicroDda`, -`rccl-UnitTestsMicroSymKernels`, `rccl-UnitTestsMicroTaskPrep` and -`rccl-UnitTestsMicroRmaProxy` — **without configuring/building all of +`rccl-UnitTestsMicroSymKernels`, `rccl-UnitTestsMicroTaskPrep`, +`rccl-UnitTestsMicroRmaProxy` and `rccl-UnitTestsMicroNetIbCastInit` — **without configuring/building all of librccl**. It compiles just the tests + fakes + the hipified unit-under-test sources. @@ -1030,6 +1037,7 @@ cmake --build build -j"$(nproc)" ./build/rccl-UnitTestsMicroRoctxNvml-roctx # same, ROCTX_ENABLE arm ./build/rccl-UnitTestsMicroRocmwrap # src/misc/rocmwrap.cc cuMem + DMA-BUF init ./build/rccl-UnitTestsMicroAmdsmiWrap # src/misc/amdsmi_wrap.cc +./build/rccl-UnitTestsMicroNetIbCastInit # src/transport/net_ib_cast/ init, capability and GIN ``` Disable coverage instrumentation for the standalone host-only test binaries diff --git a/projects/rccl/test/host/net-ib-cast-capability-test.cc b/projects/rccl/test/host/net-ib-cast-capability-test.cc new file mode 100644 index 000000000000..4d39d51f6c5a --- /dev/null +++ b/projects/rccl/test/host/net-ib-cast-capability-test.cc @@ -0,0 +1,403 @@ +/************************************************************************* + * Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +// Host-only microtests for src/transport/net_ib_cast/capability.cc (NET_IB_CAST_CAPABILITY_CC_PATH): +// the UD and RDMA READ probes the resiliency features depend on, over fake verbs. + +#include + +#include +#include + +#include "../common/LogCapture.hpp" +#include "net-ib-cast-test-common.h" + +#include NET_IB_CAST_CAPABILITY_CC_PATH + +namespace { + +using RcclUnitTesting::CaptureLog; +using RcclUnitTesting::LogHas; +using RcclUnitTesting::ScopedDebugLogging; + +// The step a probe fails at; None lets it run to the end. +enum class Fail { None, AllocPd, CreateCq, RegMr, CreateQp, NoQp, GidIndex, QueryGid, Init, Rtr, Rts, PostSend }; + +class NetIbCastCapabilityMicrotest : public ::testing::Test { + protected: + void SetUp() override { + InitFakeContext(&ctx_, &device_); + dev_ = &IbCastDevs[0]; + dev_->context = &ctx_; + dev_->portNum = 2; + snprintf(dev_->devName, sizeof(dev_->devName), "ionic_0"); + dev_->portAttr.active_mtu = IBV_MTU_4096; + dev_->portAttr.link_layer = IBV_LINK_LAYER_ETHERNET; + dev_->portAttr.lid = 9; + dev_->portAttr.flags = 3; + dev_->udSupported = -1; + dev_->rdmaReadSupported = -1; + IbCastNDevs = 1; + qp_.qp_num = 77; + qp_.context = &ctx_; + cq_.context = &ctx_; + } + void TearDown() override { + NetIbCastResetGlobals(); + ResetIbvwrapFakes(); + ResetNetIbCastConnectFakes(); + ResetNetIbCastP2pResiliencyRecoveryFakes(); + } + + bool Stops(Fail at) const { return fail_ == at; } + + ParamScript params_; + Fail fail_ = Fail::None; + struct ibv_device device_ {}; + struct ibv_context ctx_; + struct ncclIbDev* dev_ = nullptr; + struct ibv_pd pd_ {}; + struct ibv_cq cq_ {}; + struct ibv_mr mr_ {}; + struct ibv_qp qp_ {}; + struct ncclIbQpCreateAttr created_ {}; + struct ncclIbQp configured_ {}; // the probe QP as last configured + int destroyedQps_ = 0, destroyedCqs_ = 0, deallocs_ = 0, deregs_ = 0; + + HookOf allocPd_{g_ibvAllocPd, [this](struct ibv_pd** ret, struct ibv_context* ctx) { + EXPECT_EQ(ctx, &ctx_); + *ret = Stops(Fail::AllocPd) ? nullptr : &pd_; + return Stops(Fail::AllocPd) ? ncclSystemError : ncclSuccess; + }}; + HookOf createCq_{ + g_ibvCreateCq, [this](struct ibv_cq** ret, struct ibv_context*, int, void*, struct ibv_comp_channel*, int) { + *ret = Stops(Fail::CreateCq) ? nullptr : &cq_; + return Stops(Fail::CreateCq) ? ncclSystemError : ncclSuccess; + }}; + HookOf regMr_{g_ibvRegMr, [this](struct ibv_mr** ret, struct ibv_pd*, void* addr, size_t, + int access) { + EXPECT_EQ(access, IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ); + mr_.addr = addr; + mr_.lkey = 0x11; + mr_.rkey = 0x22; + *ret = Stops(Fail::RegMr) ? nullptr : &mr_; + return Stops(Fail::RegMr) ? ncclSystemError : ncclSuccess; + }}; + HookOf qpCreate_{g_ibCastQpCreate, [this](struct ncclIbQp* qp, + struct ncclIbQpCreateAttr* attr) { + created_ = *attr; + qp->qp = Stops(Fail::NoQp) || Stops(Fail::CreateQp) ? nullptr : &qp_; + return Stops(Fail::CreateQp) ? ncclSystemError : ncclSuccess; + }}; + HookOf gidIndex_{ + g_ibCastGetGidIndex, [this](struct ibv_context*, uint8_t port, struct ibv_port_attr* attr, int* index) { + EXPECT_EQ(port, 2); + EXPECT_EQ(attr, &dev_->portAttr); + *index = 3; + return Stops(Fail::GidIndex) ? ncclSystemError : ncclSuccess; + }}; + HookOf queryGid_{g_ibvQueryGid, [this](struct ibv_context*, uint8_t, int index, + union ibv_gid* gid) { + EXPECT_EQ(index, 3); + memset(gid->raw, 0xee, sizeof(gid->raw)); + return Stops(Fail::QueryGid) ? ncclSystemError : ncclSuccess; + }}; + HookOf qpInit_{g_ibCastQpInit, [this](struct ncclIbQp* qp) { + configured_ = *qp; + return Stops(Fail::Init) ? ncclSystemError : ncclSuccess; + }}; + HookOf qpRtr_{g_ibCastQpRtr, [this](struct ncclIbQp* qp) { + configured_ = *qp; + return Stops(Fail::Rtr) ? ncclSystemError : ncclSuccess; + }}; + HookOf qpRts_{g_ibCastQpRts, [this](struct ncclIbQp* qp) { + configured_ = *qp; + return Stops(Fail::Rts) ? ncclSystemError : ncclSuccess; + }}; + HookOf destroyQp_{g_ibvDestroyQp, [this](struct ibv_qp* qp) { + EXPECT_EQ(qp, &qp_); + destroyedQps_++; + return ncclSuccess; + }}; + HookOf destroyCq_{g_ibvDestroyCq, [this](struct ibv_cq*) { + destroyedCqs_++; + return ncclSuccess; + }}; + HookOf dealloc_{g_ibvDeallocPd, [this](struct ibv_pd*) { + deallocs_++; + return ncclSuccess; + }}; + HookOf dereg_{g_ibvDeregMr, [this](struct ibv_mr*) { + deregs_++; + return ncclSuccess; + }}; +}; + +TEST_F(NetIbCastCapabilityMicrotest, ProbeDevices_NoResiliencyFeature_ProbesNothing) { + ASSERT_EQ(IbCastCapProbeDevices(), ncclSuccess); + EXPECT_EQ(allocPd_.calls, 0); + EXPECT_EQ(dev_->udSupported, -1); + EXPECT_EQ(dev_->rdmaReadSupported, -1); +} + + +class NetIbCastCapabilityUdMicrotest : public NetIbCastCapabilityMicrotest { + protected: + void SetUp() override { + NetIbCastCapabilityMicrotest::SetUp(); + params_.Set("IB_RESILIENCY_PORT_RECOVERY", 1); + } + HookOf udToRts_{g_ibCastPortRecoveryQpUdToRts, + [this](struct ncclIbQp* qp, int port) { + EXPECT_EQ(qp->qp, &qp_); + EXPECT_EQ(port, 2); + return udToRtsResult_; + }}; + ncclResult_t udToRtsResult_ = ncclSuccess; +}; + +TEST_F(NetIbCastCapabilityUdMicrotest, ProbeUd_QpReachesRts_IsSupportedAndReleasesEverything) { + ScopedDebugLogging info(NCCL_LOG_INFO, NCCL_ALL); + std::string log = CaptureLog([&] { ASSERT_EQ(IbCastCapProbeDevices(), ncclSuccess); }); + EXPECT_EQ(dev_->udSupported, 1); + EXPECT_TRUE(IbCastCapUdSupported(dev_)); + EXPECT_EQ(created_.type, IBV_QPT_UD); + EXPECT_EQ(created_.cq, &cq_); + EXPECT_EQ(created_.pd, &pd_); + EXPECT_EQ(created_.maxSendWorkRequest, (uint32_t)NCCL_IB_RESILIENCY_PORT_RECOVERY_ALIVE_MSG_BATCH_SIZE_MAX); + EXPECT_EQ(created_.maxRecvWorkRequest, (uint32_t)NCCL_IB_RESILIENCY_PORT_RECOVERY_ALIVE_MSG_BATCH_SIZE_MAX); + EXPECT_EQ(created_.qpContext, &dev_->stats); + EXPECT_EQ(created_.ctsQpSlot, (int8_t)NCCL_CTS_QP_SLOT_INVALID); + EXPECT_FALSE(created_.isQpSharingEnabled); + EXPECT_EQ(createCq_.calls, 1); + EXPECT_EQ(destroyedQps_, 1); + EXPECT_EQ(destroyedCqs_, 1); + EXPECT_EQ(deallocs_, 1); + EXPECT_TRUE(LogHas(log, "device ionic_0 capability probe: UD=yes")) << log; + // Probed once: a second pass leaves the answer alone. + ASSERT_EQ(IbCastCapProbeDevices(), ncclSuccess); + EXPECT_EQ(allocPd_.calls, 1); +} + +TEST_F(NetIbCastCapabilityUdMicrotest, ProbeUd_CreateCqUsesThePortRecoveryCqSize) { + ScopedHook cq(g_ibvCreateCq, [this](struct ibv_cq** ret, struct ibv_context*, int cqe, void*, + struct ibv_comp_channel*, int) { + EXPECT_EQ(cqe, NCCL_IB_RESILIENCY_PORT_RECOVERY_CQ_SIZE); + *ret = &cq_; + return ncclSuccess; + }); + ASSERT_EQ(IbCastCapProbeDevices(), ncclSuccess); + EXPECT_EQ(cq.calls, 1); +} + +class NetIbCastCapabilityUdFailMicrotest : public NetIbCastCapabilityUdMicrotest, + public ::testing::WithParamInterface {}; + +TEST_P(NetIbCastCapabilityUdFailMicrotest, ProbeUd_AnyStepFails_IsUnsupportedAndWarns) { + fail_ = GetParam(); + if (fail_ == Fail::Rts) udToRtsResult_ = ncclSystemError; + std::string log = CaptureLog([&] { ASSERT_EQ(IbCastCapProbeDevices(), ncclSuccess); }); + EXPECT_EQ(dev_->udSupported, 0); + EXPECT_FALSE(IbCastCapUdSupported(dev_)); + EXPECT_TRUE(LogHas(log, "device ionic_0 cannot create a UD queue pair; port recovery disabled on it")) << log; + // Whatever was acquired is released. + EXPECT_EQ(deallocs_, fail_ == Fail::AllocPd ? 0 : 1); + EXPECT_EQ(destroyedCqs_, fail_ == Fail::AllocPd || fail_ == Fail::CreateCq ? 0 : 1); + EXPECT_EQ(destroyedQps_, fail_ == Fail::Rts ? 1 : 0); +} + +INSTANTIATE_TEST_SUITE_P(NetIbCastCapability, NetIbCastCapabilityUdFailMicrotest, + ::testing::Values(Fail::AllocPd, Fail::CreateCq, Fail::CreateQp, Fail::NoQp, Fail::Rts)); + + +class NetIbCastCapabilityReadMicrotest : public NetIbCastCapabilityMicrotest { + protected: + void SetUp() override { + NetIbCastCapabilityMicrotest::SetUp(); + params_.Set("IB_RESILIENCY_PORT_FAILOVER", 1); + } + // The loopback READ: copies the source word into the destination, as the NIC would. + HookOf postSend_{g_fakePostSend, [this](struct ibv_qp* qp, struct ibv_send_wr* wr, + struct ibv_send_wr**) { + EXPECT_EQ(qp, &qp_); + posted_ = *wr; + postedSge_ = *wr->sg_list; + if (Stops(Fail::PostSend)) return EIO; + if (copy_) + memcpy((void*)wr->sg_list->addr, (void*)wr->wr.rdma.remote_addr, + wr->sg_list->length); + return 0; + }}; + HookOf pollCq_{g_fakePollCq, [this](struct ibv_cq* cq, int n, struct ibv_wc* wc) { + EXPECT_EQ(cq, &cq_); + EXPECT_EQ(n, 1); + if (pollResult_ <= 0) return pollResult_; + memset(wc, 0, sizeof(*wc)); + wc->status = status_; + wc->vendor_err = 0x55; + return 1; + }}; + bool copy_ = true; + int pollResult_ = 1; + enum ibv_wc_status status_ = IBV_WC_SUCCESS; + struct ibv_send_wr posted_ {}; + struct ibv_sge postedSge_ {}; +}; + +TEST_F(NetIbCastCapabilityReadMicrotest, ProbeRead_LoopbackReadCompletes_IsSupported) { + params_.Set("IB_PKEY", 4); + params_.Set("IB_TC", 96); + params_.Set("IB_SL", 2); + params_.Set("IB_TIMEOUT", 14); + params_.Set("IB_RETRY_CNT", 5); + ScopedDebugLogging info(NCCL_LOG_INFO, NCCL_ALL); + std::string log = CaptureLog([&] { ASSERT_EQ(IbCastCapProbeDevices(), ncclSuccess); }); + EXPECT_EQ(dev_->rdmaReadSupported, 1); + EXPECT_TRUE(IbCastCapRdmaReadSupported(dev_)); + EXPECT_TRUE(LogHas(log, "device ionic_0 capability probe: RDMA_READ=yes")) << log; + // An RC QP of depth 1 that connects to itself. + EXPECT_EQ(created_.type, IBV_QPT_RC); + EXPECT_EQ(created_.maxSendWorkRequest, 1u); + EXPECT_EQ(created_.maxRecvWorkRequest, 1u); + EXPECT_EQ(configured_.initAttr.state, IBV_QPS_INIT); + EXPECT_EQ(configured_.initAttr.pkeyIndex, 4); + EXPECT_EQ(configured_.initAttr.portNum, 2); + EXPECT_EQ(configured_.initAttr.qpAccessFlags, IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ); + EXPECT_EQ(configured_.rtrAttr.mtu, IBV_MTU_4096); + EXPECT_EQ(configured_.rtrAttr.linkLayer, IBV_LINK_LAYER_ETHERNET); + EXPECT_EQ(configured_.rtrAttr.tc, 96); + EXPECT_EQ(configured_.rtrAttr.sl, 2); + EXPECT_EQ(configured_.rtrAttr.remoteQpNum, 77u); + EXPECT_EQ(configured_.rtrAttr.remoteLid, 9u); + EXPECT_EQ(configured_.rtrAttr.localIbPort, 2); + EXPECT_EQ(configured_.rtrAttr.localPortFlags, 3); + EXPECT_EQ(configured_.rtrAttr.localGidIndex, 3); + EXPECT_EQ(configured_.rtrAttr.remoteGid.raw[0], 0xee); + EXPECT_EQ(memcmp(&configured_.rtrAttr.localGid, &configured_.rtrAttr.remoteGid, sizeof(union ibv_gid)), 0); + EXPECT_EQ(configured_.rtsAttr.timeout, 14); + EXPECT_EQ(configured_.rtsAttr.retryCnt, 5); + // One signaled READ of the source word into the destination, both in the probe's MR. + EXPECT_EQ(posted_.opcode, IBV_WR_RDMA_READ); + EXPECT_EQ(posted_.send_flags, (unsigned)IBV_SEND_SIGNALED); + EXPECT_EQ(posted_.num_sge, 1); + EXPECT_EQ(posted_.wr.rdma.rkey, 0x22u); + EXPECT_EQ(postedSge_.lkey, 0x11u); + EXPECT_EQ(postedSge_.length, sizeof(uint64_t)); + EXPECT_EQ(postedSge_.addr, posted_.wr.rdma.remote_addr + sizeof(uint64_t)); + EXPECT_EQ(posted_.wr.rdma.remote_addr, (uint64_t)(uintptr_t)mr_.addr); + EXPECT_EQ(destroyedQps_, 1); + EXPECT_EQ(deregs_, 1); + EXPECT_EQ(destroyedCqs_, 1); + EXPECT_EQ(deallocs_, 1); +} + +TEST_F(NetIbCastCapabilityReadMicrotest, ProbeRead_TcAndSlUnset_UseZero) { + ASSERT_EQ(IbCastCapProbeDevices(), ncclSuccess); + EXPECT_EQ(configured_.rtrAttr.tc, 0); + EXPECT_EQ(configured_.rtrAttr.sl, 0); +} + +TEST_F(NetIbCastCapabilityReadMicrotest, ProbeRead_CompletionError_IsUnsupportedAndReportsTheStatus) { + status_ = IBV_WC_REM_ACCESS_ERR; + ScopedDebugLogging info(NCCL_LOG_INFO, NCCL_ALL); + std::string log = CaptureLog([&] { ASSERT_EQ(IbCastCapProbeDevices(), ncclSuccess); }); + EXPECT_EQ(dev_->rdmaReadSupported, 0); + EXPECT_TRUE(LogHas(log, "RDMA READ probe completed with status")) << log; + EXPECT_TRUE(LogHas(log, "vendor_err 85")) << log; + EXPECT_TRUE(LogHas(log, "cannot complete an RDMA READ; port failover probing will fail on it")) << log; +} + +TEST_F(NetIbCastCapabilityReadMicrotest, ProbeRead_SuccessWithoutTheData_IsUnsupported) { + copy_ = false; + ASSERT_EQ(IbCastCapProbeDevices(), ncclSuccess); + EXPECT_EQ(dev_->rdmaReadSupported, 0); +} + +TEST_F(NetIbCastCapabilityReadMicrotest, ProbeRead_PollFails_IsUnsupported) { + pollResult_ = -1; + ASSERT_EQ(IbCastCapProbeDevices(), ncclSuccess); + EXPECT_EQ(dev_->rdmaReadSupported, 0); + EXPECT_EQ(pollCq_.calls, 1); +} + +TEST_F(NetIbCastCapabilityReadMicrotest, ProbeRead_NeverCompletes_GivesUpAfterTheTimeout) { + pollResult_ = 0; + const uint64_t start = clockNano(); + ASSERT_EQ(IbCastCapProbeDevices(), ncclSuccess); + EXPECT_GE(clockNano() - start, NCCL_IB_CAP_RDMA_READ_PROBE_TIMEOUT_NS); + EXPECT_EQ(dev_->rdmaReadSupported, 0); + EXPECT_GT(pollCq_.calls, 1); + EXPECT_EQ(destroyedQps_, 1); // the QP goes before the buffer it may still read into +} + +class NetIbCastCapabilityReadFailMicrotest : public NetIbCastCapabilityReadMicrotest, + public ::testing::WithParamInterface {}; + +TEST_P(NetIbCastCapabilityReadFailMicrotest, ProbeRead_AnySetupStepFails_IsUnsupportedAndReleases) { + fail_ = GetParam(); + ASSERT_EQ(IbCastCapProbeDevices(), ncclSuccess); + EXPECT_EQ(dev_->rdmaReadSupported, 0); + EXPECT_EQ(pollCq_.calls, 0); + EXPECT_EQ(deallocs_, fail_ == Fail::AllocPd ? 0 : 1); + EXPECT_EQ(destroyedCqs_, fail_ == Fail::AllocPd || fail_ == Fail::CreateCq ? 0 : 1); + EXPECT_EQ(deregs_, fail_ == Fail::AllocPd || fail_ == Fail::CreateCq || fail_ == Fail::RegMr ? 0 : 1); + const bool qpMade = fail_ != Fail::AllocPd && fail_ != Fail::CreateCq && fail_ != Fail::RegMr && + fail_ != Fail::CreateQp && fail_ != Fail::NoQp; + EXPECT_EQ(destroyedQps_, qpMade ? 1 : 0); +} + +INSTANTIATE_TEST_SUITE_P(NetIbCastCapability, NetIbCastCapabilityReadFailMicrotest, + ::testing::Values(Fail::AllocPd, Fail::CreateCq, Fail::RegMr, Fail::CreateQp, Fail::NoQp, + Fail::GidIndex, Fail::QueryGid, Fail::Init, Fail::Rtr, Fail::Rts, + Fail::PostSend)); + +TEST_F(NetIbCastCapabilityReadMicrotest, ProbeDevices_AlreadyProbed_IsNotProbedAgain) { + dev_->rdmaReadSupported = 0; + dev_->udSupported = 1; + params_.Set("IB_RESILIENCY_PORT_RECOVERY", 1); + ASSERT_EQ(IbCastCapProbeDevices(), ncclSuccess); + EXPECT_EQ(allocPd_.calls, 0); +} + + +TEST_F(NetIbCastCapabilityMicrotest, GetDeviceCaps_AMergedDeviceHasOnlyWhatEveryMemberHas) { + IbCastNDevs = 2; + IbCastNMergedDevs = 2; + IbCastDevs[0].udSupported = 1; + IbCastDevs[0].rdmaReadSupported = 1; + IbCastDevs[1].udSupported = 1; + IbCastDevs[1].rdmaReadSupported = 0; + IbCastMergedDevs[0].vProps.ndevs = 1; + IbCastMergedDevs[0].vProps.devs[0] = 0; + IbCastMergedDevs[1].vProps.ndevs = 2; + IbCastMergedDevs[1].vProps.devs[0] = 0; + IbCastMergedDevs[1].vProps.devs[1] = 1; + struct ncclIbCastDeviceCaps caps; + ASSERT_EQ(ncclIbCastGetDeviceCaps(0, &caps), ncclSuccess); + EXPECT_TRUE(caps.udSupported); + EXPECT_TRUE(caps.rdmaReadSupported); + ASSERT_EQ(ncclIbCastGetDeviceCaps(1, &caps), ncclSuccess); + EXPECT_TRUE(caps.udSupported); + EXPECT_FALSE(caps.rdmaReadSupported); + IbCastDevs[0].udSupported = -1; // not probed counts as missing + ASSERT_EQ(ncclIbCastGetDeviceCaps(0, &caps), ncclSuccess); + EXPECT_FALSE(caps.udSupported); +} + +TEST_F(NetIbCastCapabilityMicrotest, GetDeviceCaps_BadArguments_AreRejected) { + IbCastNMergedDevs = 1; + IbCastMergedDevs[0].vProps.ndevs = 1; + IbCastMergedDevs[0].vProps.devs[0] = 5; // past IbCastNDevs + struct ncclIbCastDeviceCaps caps; + EXPECT_EQ(ncclIbCastGetDeviceCaps(0, nullptr), ncclInvalidArgument); + EXPECT_EQ(ncclIbCastGetDeviceCaps(-1, &caps), ncclInvalidArgument); + EXPECT_EQ(ncclIbCastGetDeviceCaps(1, &caps), ncclInvalidArgument); + EXPECT_EQ(ncclIbCastGetDeviceCaps(0, &caps), ncclInvalidArgument); + IbCastMergedDevs[0].vProps.devs[0] = -1; + EXPECT_EQ(ncclIbCastGetDeviceCaps(0, &caps), ncclInvalidArgument); +} + +} // namespace diff --git a/projects/rccl/test/host/net-ib-cast-common-test.cc b/projects/rccl/test/host/net-ib-cast-common-test.cc new file mode 100644 index 000000000000..57ebe7481202 --- /dev/null +++ b/projects/rccl/test/host/net-ib-cast-common-test.cc @@ -0,0 +1,438 @@ +/************************************************************************* + * Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +// Host-only microtests for src/transport/net_ib_cast/common.cc (NET_IB_CAST_COMMON_CC_PATH) and the +// inline helpers of common_cast.h: comm setup, fatal-error accounting and the async-event loop. + +#include + +#include +#include +#include +#include + +#include "../common/LogCapture.hpp" +#include "fakes/net_ib_cast_p2p_resiliency_fakes.h" +#include "fakes/transport_stubs.h" +#include "net-ib-cast-test-common.h" +#include "fakes/param_redirect.h" + +#include NET_IB_CAST_COMMON_CC_PATH + +namespace { + +using RcclUnitTesting::CaptureLog; +using RcclUnitTesting::LogHas; +using RcclUnitTesting::ScopedDebugLogging; + +class NetIbCastCommonMicrotest : public ::testing::Test { + protected: + void TearDown() override { + NetIbCastResetGlobals(); + ResetIbvwrapFakes(); + ResetNetIbCastConnectFakes(); + ResetNetIbCastP2pResiliencyFakes(); + ResetNetIbCastSchedulerFakes(); + } + + ParamScript params_; + // Resiliency is off unless a test turns it on: the comm gets no resiliency context. + HookOf resiliencyInit_{ + g_ibCastResiliencyInit, [this](struct ncclIbNetCommBase*, struct ncclIbResiliency** res) { + *res = resiliency_; + return ncclSuccess; + }}; + struct ncclIbResiliency* resiliency_ = nullptr; +}; + +TEST_F(NetIbCastCommonMicrotest, StatsCheckFatalCount_AnyFatalErrorStopsTheCaller) { + struct ncclIbStats stats; + IbCastStatsInit(&stats); + EXPECT_EQ(IbCastStatsCheckFatalCount(&stats, "here"), ncclSuccess); + IbCastStatsFatalError(&stats); + std::string log = CaptureLog([&] { EXPECT_EQ(IbCastStatsCheckFatalCount(&stats, "here"), ncclSystemError); }); + EXPECT_TRUE(LogHas(log, "fatal error (detected in here)")) << log; + params_.Set("IB_RETURN_ASYNC_EVENTS", 0); + EXPECT_EQ(IbCastStatsCheckFatalCount(&stats, "here"), ncclSuccess); +} + +TEST_F(NetIbCastCommonMicrotest, FatalErrorHelpers_ChargeTheStatsTheObjectCarries) { + struct ncclIbStats qpStats, cqStats; + IbCastStatsInit(&qpStats); + IbCastStatsInit(&cqStats); + struct ibv_qp qp {}; + qp.qp_context = &qpStats; + struct ibv_cq cq {}; + cq.cq_context = &cqStats; + IbCastQpFatalError(&qp); + IbCastQpFatalError(&qp); + IbCastCqFatalError(&cq); + IbCastDevFatalError(&IbCastDevs[0]); + EXPECT_EQ(qpStats.fatalErrorCount, 2); + EXPECT_EQ(cqStats.fatalErrorCount, 1); + EXPECT_EQ(IbCastDevs[0].stats.fatalErrorCount, 1); +} + +TEST_F(NetIbCastCommonMicrotest, GetNetCommDevBase_FindsThePerDeviceBaseOfSendAndRecvComms) { + ZeroedPtr send; + ZeroedPtr recv; + send->base.isSend = true; + recv->base.isSend = false; + EXPECT_EQ(IbCastGetNetCommDevBase(&send->base, 1), &send->devs[1].base); + EXPECT_EQ(IbCastGetNetCommDevBase(&recv->base, 1), &recv->devs[1].base); +} + +TEST_F(NetIbCastCommonMicrotest, SendCommInit_ResetsEveryQpAndTheCommState) { + ZeroedPtr send; + memset(send->base.qps, 0x5a, sizeof(send->base.qps)); + params_.Set("IB_SPLIT_DATA_ON_QPS", 1); + ASSERT_EQ(IbCastSendCommInit(send.get()), ncclSuccess); + EXPECT_TRUE(send->base.isSend); + EXPECT_EQ(send->base.nqps, -1); + EXPECT_EQ(send->base.nDataQps, -1); + EXPECT_EQ(send->base.splitDataOnQps, 1); + EXPECT_FALSE(send->base.optRecvCompletion); + EXPECT_EQ(send->base.ready, 0); + EXPECT_EQ(send->base.resiliency, nullptr); + for (int i = 0; i < NCCL_IB_MAX_QPS; i++) { + const struct ncclIbQp& qp = send->base.qps[i]; + EXPECT_EQ(qp.devIndex, -1) << i; + EXPECT_EQ(qp.remDevIdx, -1) << i; + EXPECT_EQ(qp.eceSupported, 0) << i; + EXPECT_EQ(qp.initAttr.portNum, 0) << i; + EXPECT_EQ(qp.rtrAttr.remoteQpNum, 0u) << i; + EXPECT_EQ(qp.rtsAttr.timeout, 0) << i; + EXPECT_EQ(send->base.activeQps[i], &send->base.qps[i]) << i; + } +} + +TEST_F(NetIbCastCommonMicrotest, BaseCommInit_ResiliencyInitFails_ReturnsTheError) { + ZeroedPtr send; + ScopedHook fails(g_ibCastResiliencyInit, + [](struct ncclIbNetCommBase*, struct ncclIbResiliency**) { return ncclSystemError; }); + EXPECT_EQ(IbCastSendCommInit(send.get()), ncclSystemError); + ZeroedPtr recv; + EXPECT_EQ(IbCastRecvCommInit(recv.get()), ncclSystemError); +} + +TEST_F(NetIbCastCommonMicrotest, RecvCommInit_PostsOnDemandByDefault) { + ZeroedPtr recv; + ASSERT_EQ(IbCastRecvCommInit(recv.get()), ncclSuccess); + EXPECT_FALSE(recv->base.isSend); + EXPECT_FALSE(recv->prepostReceiveWorkRequests); + EXPECT_EQ(recv->ibRecvWorkRequest.wr_id, NCCL_IB_RECV_WR_ID_DUMMY); + EXPECT_EQ(recv->ibRecvWorkRequest.num_sge, 0); + params_.Set("IB_PREPOST_RECEIVE_WORK_REQUESTS", 1); + ASSERT_EQ(IbCastRecvCommInit(recv.get()), ncclSuccess); + EXPECT_TRUE(recv->prepostReceiveWorkRequests); +} + +TEST_F(NetIbCastCommonMicrotest, RecvCommInit_ResiliencyOrOooRq_ForcePrePosting) { + ZeroedPtr recv; + params_.Set("IB_PREPOST_RECEIVE_WORK_REQUESTS", 0); + ScopedDebugLogging info(NCCL_LOG_INFO, NCCL_ALL); + auto res = std::make_unique(); + resiliency_ = res.get(); + std::string log = CaptureLog([&] { ASSERT_EQ(IbCastRecvCommInit(recv.get()), ncclSuccess); }); + EXPECT_TRUE(recv->prepostReceiveWorkRequests); + EXPECT_TRUE(LogHas(log, "Overriding pre-posting to true (1).")) << log; + resiliency_ = nullptr; + params_.Set("IB_OOO_RQ", 1); + log = CaptureLog([&] { ASSERT_EQ(IbCastRecvCommInit(recv.get()), ncclSuccess); }); + EXPECT_TRUE(recv->prepostReceiveWorkRequests); + EXPECT_TRUE(LogHas(log, "OOO RQ is enabled, Overriding pre-posting")) << log; + // Already asked for: no override to announce. + params_.Set("IB_PREPOST_RECEIVE_WORK_REQUESTS", 1); + log = CaptureLog([&] { ASSERT_EQ(IbCastRecvCommInit(recv.get()), ncclSuccess); }); + EXPECT_FALSE(LogHas(log, "Overriding")) << log; +} + +struct OptRecv { + const char* name; + int64_t control; + bool ctsOffload; + int64_t netHint; + bool resiliency; + bool scheduler; + bool expected; +}; + +class NetIbCastCommonOptRecvMicrotest : public NetIbCastCommonMicrotest, + public ::testing::WithParamInterface {}; + +TEST_P(NetIbCastCommonOptRecvMicrotest, InitOptRecvCompletion_OnlyWhenAskedAndNothingNeedsCompletions) { + const OptRecv c = GetParam(); + params_.Set("RCCL_IB_OPTIONAL_RECV_COMPLETION", c.control); + params_.Set("NET_OPTIONAL_RECV_COMPLETION", c.netHint); + castGlobalQpSchedParms.enable = c.scheduler; + ncclIbResiliency res; + ZeroedPtr send; + send->base.resiliency = c.resiliency ? &res : nullptr; + send->base.optRecvCompletion = !c.expected; + IbCastInitOptRecvCompletion(&send->base, c.ctsOffload); + EXPECT_EQ(send->base.optRecvCompletion, c.expected); +} + +INSTANTIATE_TEST_SUITE_P(NetIbCastCommon, NetIbCastCommonOptRecvMicrotest, + ::testing::Values(OptRecv{"Default", 0, false, 1, false, false, false}, + OptRecv{"Control", 1, false, 1, false, false, true}, + OptRecv{"CtsOffload", 0, true, 1, false, false, true}, + OptRecv{"NoNetHint", 1, true, 0, false, false, false}, + OptRecv{"Resiliency", 1, false, 1, true, false, false}, + OptRecv{"Scheduler", 1, true, 1, false, true, false}), + [](const ::testing::TestParamInfo& info) { return info.param.name; }); + +TEST_F(NetIbCastCommonMicrotest, CommBaseQpHelpers_IndexStripeAndLookUpQps) { + ZeroedPtr send; + ASSERT_EQ(IbCastSendCommInit(send.get()), ncclSuccess); + struct ncclIbNetCommBase* base = &send->base; + base->vProps.ndevs = 2; + base->nqps = 4; + struct ibv_qp qps[4] = {}; + for (int i = 0; i < 4; i++) { + qps[i].qp_num = 100 + i; + base->qps[i].qp = &qps[i]; + } + struct ncclIbQp* qp = nullptr; + // QP i lives on device i % ndevs, as the i / ndevs-th QP of that device. + ASSERT_EQ(IbCastCommBaseGetQpByIndex(base, 1, 1, &qp), ncclSuccess); + EXPECT_EQ(qp, &base->qps[3]); + int qpIndex = -1; + ASSERT_EQ(IbCastCommBaseGetQpForRequest(base, 6, 1, &qp, &qpIndex), ncclSuccess); + EXPECT_EQ(qpIndex, 3); + EXPECT_EQ(qp, &base->qps[3]); + ASSERT_EQ(IbCastCommBaseGetQpByQpNum(base, 1, 103, &qp, &qpIndex), ncclSuccess); + EXPECT_EQ(qpIndex, 3); + ASSERT_EQ(IbCastCommBaseGetQpByQpNum(base, 0, 100, &qp, nullptr), ncclSuccess); + EXPECT_EQ(qp, &base->qps[0]); + EXPECT_EQ(IbCastCommBaseGetQpByQpNum(base, 0, 103, &qp, &qpIndex), ncclInternalError); // on device 1 + EXPECT_EQ(qp, nullptr); +} + +TEST_F(NetIbCastCommonMicrotest, NqpsPerRequest_IsAllQpsOnlyWhenSplittingOrWeighting) { + ZeroedPtr send; + send->base.nqps = 4; + EXPECT_EQ(IbCastCommBaseGetNqpsPerRequest(&send->base), 1); + send->base.schedParms.splitData = true; + EXPECT_EQ(IbCastCommBaseGetNqpsPerRequest(&send->base), 4); + send->base.schedParms.splitData = false; + send->base.schedParms.doWrr = true; + EXPECT_EQ(IbCastCommBaseGetNqpsPerRequest(&send->base), 4); +} + +TEST_F(NetIbCastCommonMicrotest, PostRecvWorkRequest_GoesThroughTheQpsContext) { + struct ibv_device device {}; + struct ibv_context ctx; + InitFakeContext(&ctx, &device); + struct ibv_qp qp {}; + qp.context = &ctx; + struct ibv_recv_wr wr {}; + int rc = 0; + ScopedHook post(g_fakePostRecv, [&](struct ibv_qp* q, struct ibv_recv_wr* w, struct ibv_recv_wr**) { + EXPECT_EQ(q, &qp); + EXPECT_EQ(w, &wr); + return rc; + }); + EXPECT_EQ(IbCastPostRecvWorkRequest(&qp, &wr), ncclSuccess); + rc = ENOMEM; + EXPECT_EQ(IbCastPostRecvWorkRequest(&qp, &wr), ncclSystemError); +} + +TEST_F(NetIbCastCommonMicrotest, GidInfoSnapshot_CopiesTheDevicesCurrentGid) { + IbCastDevs[0].gidInfo.localGidIndex = 5; + IbCastDevs[0].gidInfo.link_layer = IBV_LINK_LAYER_ETHERNET; + struct ncclIbNetCommDevBase base {}; + IbCastGidInfoSnapshot(&base, &IbCastDevs[0]); + EXPECT_EQ(base.gidInfo.localGidIndex, 5); + EXPECT_EQ(base.gidInfo.link_layer, IBV_LINK_LAYER_ETHERNET); +} + +TEST_F(NetIbCastCommonMicrotest, NetIbCast_PublishesTheCastEntryPoints) { + EXPECT_STREQ(netIbCast.name, "IB-CAST"); + EXPECT_EQ(netIbCast.init, IbCastInit); + EXPECT_EQ(netIbCast.regMr, IbCastRegMr); + EXPECT_EQ(netIbCast.isend, IbCastIsend); + EXPECT_EQ(netIbCast.getDeviceMr, nullptr); + EXPECT_EQ(netIbCast.irecvConsumed, nullptr); + EXPECT_EQ(netIbCast.makeVDevice, IbCastMakeVDevice); + EXPECT_EQ(netIbCast.finalize, IbCastFinalize); + EXPECT_EQ(netIbCast.setNetAttr, IbCastSetNetAttr); +} + + +class NetIbCastCommonEventMicrotest : public NetIbCastCommonMicrotest { + protected: + void SetUp() override { + InitFakeContext(&ctx_, &device_); + dev_ = &IbCastDevs[0]; + dev_->context = &ctx_; + dev_->portNum = 1; + snprintf(dev_->devName, sizeof(dev_->devName), "ionic_0"); + dev_->gidInfo.link_layer = IBV_LINK_LAYER_ETHERNET; + dev_->gidInfo.localGidIndex = 1; + IbCastStatsInit(&dev_->stats); + IbCastNDevs = 1; + } + // Hands the loop `events` in order, then fails the read so the loop ends. + void Events(std::deque types) { + events_ = std::move(types); + eventRead_.emplace(g_ibvGetAsyncEvent, [this](struct ibv_context* ctx, struct ibv_async_event* event) { + EXPECT_EQ(ctx, &ctx_); + if (events_.empty()) return ncclSystemError; + memset(event, 0, sizeof(*event)); + event->event_type = events_.front(); + events_.pop_front(); + event->element.cq = &cq_; + if (event->event_type >= IBV_EVENT_QP_FATAL && event->event_type <= IBV_EVENT_QP_ACCESS_ERR) + event->element.qp = &qp_; + return ncclSuccess; + }); + } + struct ibv_device device_ {}; + struct ibv_context ctx_; + struct ncclIbDev* dev_ = nullptr; + struct ibv_cq cq_ {}; + struct ibv_qp qp_ {}; + std::deque events_; + std::optional> eventRead_; + HookOf typeStr_{g_ibvEventTypeStr, [](char** ret, enum ibv_event_type) { + static char name[] = "fake event"; + *ret = name; + return ncclSuccess; + }}; + HookOf ack_{g_ibvAckAsyncEvent, [](struct ibv_async_event*) { return ncclSuccess; }}; +}; + +TEST_F(NetIbCastCommonEventMicrotest, GidChange_NotEthernet_IsIgnored) { + dev_->gidInfo.link_layer = IBV_LINK_LAYER_INFINIBAND; + ScopedHook query(g_ibvQueryPort, [](struct ibv_context*, uint8_t, struct ibv_port_attr*) { return ncclSuccess; }); + EXPECT_EQ(ncclIbCastGidChangeEvent(0), ncclSuccess); + EXPECT_EQ(query.calls, 0); +} + +TEST_F(NetIbCastCommonEventMicrotest, GidChange_RefreshesTheGidAndPortAttributes) { + ScopedHook query(g_ibvQueryPort, [](struct ibv_context*, uint8_t port, struct ibv_port_attr* attr) { + EXPECT_EQ(port, 1); + memset(attr, 0, sizeof(*attr)); + attr->lid = 42; + return ncclSuccess; + }); + ScopedHook gid(g_ibCastGidInfoQuery, + [](struct ibv_context*, uint8_t, struct ibv_port_attr* attr, struct ncclIbGidInfo* info) { + EXPECT_EQ(attr->lid, 42); + info->localGidIndex = 2; + memset(info->localGid.raw, 0, sizeof(info->localGid.raw)); + info->localGid.raw[15] = 9; + return ncclSuccess; + }); + ScopedDebugLogging info(NCCL_LOG_INFO, NCCL_ALL); + std::string log = CaptureLog([&] { EXPECT_EQ(ncclIbCastGidChangeEvent(0), ncclSuccess); }); + EXPECT_EQ(dev_->gidInfo.localGidIndex, 2); + EXPECT_EQ(dev_->portAttr.lid, 42); + EXPECT_TRUE(LogHas(log, "GID refreshed: idx 1 -> 2, gid :: -> ::9")) << log; +} + +TEST_F(NetIbCastCommonEventMicrotest, GidChange_QueryFailures_LeaveTheDeviceUnchanged) { + dev_->portAttr.lid = 7; + ncclResult_t portResult = ncclSystemError; + ScopedHook query(g_ibvQueryPort, [&](struct ibv_context*, uint8_t, struct ibv_port_attr* attr) { + attr->lid = 99; + return portResult; + }); + ScopedHook gid(g_ibCastGidInfoQuery, [](struct ibv_context*, uint8_t, struct ibv_port_attr*, + struct ncclIbGidInfo*) { return ncclInternalError; }); + std::string log = CaptureLog([&] { EXPECT_EQ(ncclIbCastGidChangeEvent(0), ncclSystemError); }); + EXPECT_TRUE(LogHas(log, "query_port failed during GID change")) << log; + portResult = ncclSuccess; + log = CaptureLog([&] { EXPECT_EQ(ncclIbCastGidChangeEvent(0), ncclInternalError); }); + EXPECT_TRUE(LogHas(log, "GID info query failed")) << log; + EXPECT_EQ(dev_->portAttr.lid, 7); +} + +TEST_F(NetIbCastCommonEventMicrotest, GidChangeEvent_DeviceOutOfRange_IsRejected) { + EXPECT_EQ(ncclIbCastGidChangeEvent(-1), ncclInvalidArgument); + EXPECT_EQ(ncclIbCastGidChangeEvent(1), ncclInvalidArgument); +} + +TEST_F(NetIbCastCommonEventMicrotest, AsyncThread_FatalEvents_ChargeTheDeviceCqAndQp) { + struct ncclIbStats cqStats, qpStats; + IbCastStatsInit(&cqStats); + IbCastStatsInit(&qpStats); + cq_.cq_context = &cqStats; + qp_.qp_context = &qpStats; + Events({IBV_EVENT_DEVICE_FATAL, IBV_EVENT_CQ_ERR, IBV_EVENT_QP_FATAL, IBV_EVENT_QP_REQ_ERR, IBV_EVENT_QP_ACCESS_ERR}); + ScopedHook query(g_ibvQueryQp, [](struct ibv_qp*, struct ibv_qp_attr* attr, int mask, struct ibv_qp_init_attr*) { + EXPECT_EQ(mask, IBV_QP_STATE | IBV_QP_SQ_PSN | IBV_QP_RQ_PSN); + attr->qp_state = IBV_QPS_ERR; + return ncclSuccess; + }); + std::string log = CaptureLog([&] { EXPECT_EQ(IbCastAsyncThreadMain(dev_), nullptr); }); + EXPECT_EQ(dev_->stats.fatalErrorCount, 1); + EXPECT_EQ(cqStats.fatalErrorCount, 1); + EXPECT_EQ(qpStats.fatalErrorCount, 3); + EXPECT_EQ(query.calls, 3); + EXPECT_EQ(ack_.calls, 5); + EXPECT_TRUE(LogHas(log, "ionic_0:1 async fatal event: fake event")) << log; + EXPECT_TRUE(LogHas(log, "async fatal event on QP")) << log; +} + +TEST_F(NetIbCastCommonEventMicrotest, AsyncThread_QpEventWithCqs_ReportsTheirHandles) { + struct ncclIbStats qpStats; + IbCastStatsInit(&qpStats); + qp_.qp_context = &qpStats; + struct ibv_cq sendCq {}, recvCq {}; + sendCq.handle = 11; + recvCq.handle = 12; + qp_.send_cq = &sendCq; + qp_.recv_cq = &recvCq; + Events({IBV_EVENT_QP_FATAL}); + ScopedHook query(g_ibvQueryQp, [](struct ibv_qp*, struct ibv_qp_attr*, int, struct ibv_qp_init_attr*) { + return ncclSystemError; + }); + std::string log = CaptureLog([&] { IbCastAsyncThreadMain(dev_); }); + EXPECT_TRUE(LogHas(log, "send_cq=11 recv_cq=12")) << log; + EXPECT_EQ(qpStats.fatalErrorCount, 1); +} + +TEST_F(NetIbCastCommonEventMicrotest, AsyncThread_NonFatalEvents_OnlyWarn) { + Events({IBV_EVENT_SRQ_ERR, IBV_EVENT_PATH_MIG_ERR, IBV_EVENT_PORT_ERR, IBV_EVENT_PATH_MIG, IBV_EVENT_PORT_ACTIVE, + IBV_EVENT_SQ_DRAINED, IBV_EVENT_LID_CHANGE, IBV_EVENT_PKEY_CHANGE, IBV_EVENT_SM_CHANGE, + IBV_EVENT_QP_LAST_WQE_REACHED, IBV_EVENT_CLIENT_REREGISTER, IBV_EVENT_SRQ_LIMIT_REACHED, + IBV_EVENT_COMM_EST, (enum ibv_event_type)999}); + std::string log = CaptureLog([&] { IbCastAsyncThreadMain(dev_); }); + EXPECT_EQ(dev_->stats.fatalErrorCount, 0); + EXPECT_EQ(ack_.calls, 14); + EXPECT_TRUE(LogHas(log, "async fatal event on SRQ, unused for now")) << log; + const std::string nonFatal = "Got non-fatal async event: fake event(" + std::to_string(IBV_EVENT_PORT_ERR) + ")"; + EXPECT_TRUE(LogHas(log, nonFatal.c_str())) << log; + EXPECT_TRUE(LogHas(log, "unknown event type (999)")) << log; +} + +TEST_F(NetIbCastCommonEventMicrotest, AsyncThread_GidChangeHandlerFails_MarksTheDeviceFatal) { + Events({IBV_EVENT_GID_CHANGE, IBV_EVENT_GID_CHANGE}); + int portQueries = 0; + ScopedHook query(g_ibvQueryPort, [&](struct ibv_context*, uint8_t, struct ibv_port_attr*) { + return ++portQueries == 1 ? ncclSuccess : ncclSystemError; + }); + ScopedHook gid(g_ibCastGidInfoQuery, [](struct ibv_context*, uint8_t, struct ibv_port_attr*, + struct ncclIbGidInfo*) { return ncclSuccess; }); + std::string log = CaptureLog([&] { IbCastAsyncThreadMain(dev_); }); + EXPECT_EQ(dev_->stats.fatalErrorCount, 1); + EXPECT_TRUE(LogHas(log, "marking device with fatal error after GID-change event handler failed")) << log; +} + +TEST_F(NetIbCastCommonEventMicrotest, AsyncThread_EventNameOrAckFails_EndsTheLoop) { + Events({IBV_EVENT_COMM_EST, IBV_EVENT_COMM_EST}); + ScopedHook ack(g_ibvAckAsyncEvent, [](struct ibv_async_event*) { return ncclSystemError; }); + IbCastAsyncThreadMain(dev_); + EXPECT_EQ(events_.size(), 1u); + ScopedHook name(g_ibvEventTypeStr, [](char**, enum ibv_event_type) { return ncclSystemError; }); + IbCastAsyncThreadMain(dev_); + EXPECT_EQ(events_.size(), 0u); + EXPECT_EQ(ack.calls, 1); +} + +} // namespace diff --git a/projects/rccl/test/host/net-ib-cast-gdr-test.cc b/projects/rccl/test/host/net-ib-cast-gdr-test.cc new file mode 100644 index 000000000000..12ab9ed564d3 --- /dev/null +++ b/projects/rccl/test/host/net-ib-cast-gdr-test.cc @@ -0,0 +1,275 @@ +/************************************************************************* + * Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +// Host-only microtests for src/transport/net_ib_cast/gdr.cc (NET_IB_CAST_GDR_CC_PATH): GPUDirect +// RDMA, peermem and DMA-BUF detection. Every answer is latched by a std::call_once, so each case +// runs in its own process. + +#include + +#include +#include + +#include +#include +#include + +#include "../common/LogCapture.hpp" +#include "../common/ProcessIsolatedTestRunner.hpp" +#include "fakes/gdr_peermem_fakes.h" +#include "fakes/gdr_probe_fakes.h" +#include "fakes/ibvwrap_fakes.h" +#include "fakes/libc_fakes.h" +#include "gdr_peermem.h" +#include "net-ib-cast-test-common.h" +#include "fakes/param_redirect.h" + +// os_fakes.cc answers every sysfs read with one fixed string; the Hyper-V override needs per-file +// answers, so gdr.cc's reads are renamed onto this TU's seam. +static std::function g_gdrSysfsRead = + [](const char*, const char*, char*, int) { return ncclSystemError; }; +static ncclResult_t GdrTestSysfsRead(const char* path, const char* file, char* value, int maxLen) { + return g_gdrSysfsRead(path, file, value, maxLen); +} + +#define ncclOsTopoGetStrFromSys GdrTestSysfsRead +#include "fakes/libc_seam.h" +#include NET_IB_CAST_GDR_CC_PATH +#include "fakes/libc_seam_undef.h" +#undef ncclOsTopoGetStrFromSys + +namespace { + +using RcclUnitTesting::CaptureLog; +using RcclUnitTesting::LogHas; + +// Answers bios_version and numa_balancing; an absent answer is a failed read. +void SysfsAnswers(const char* bios, const char* numaBalancing) { + g_gdrSysfsRead = [=](const char*, const char* file, char* value, int maxLen) { + const char* answer = strcmp(file, "bios_version") == 0 ? bios : strcmp(file, "numa_balancing") == 0 ? numaBalancing + : nullptr; + if (answer == nullptr) return ncclSystemError; + snprintf(value, maxLen, "%s", answer); + return ncclSuccess; + }; +} + +constexpr const char* kHyperV = "Hyper-V UEFI Release v4.1"; + +class NetIbCastGdrMicrotestIsolated : public ::testing::Test {}; + +TEST_F(NetIbCastGdrMicrotestIsolated, GdrSupport_ForceEnable_SucceedsWithoutScanningForPeermem) { + RUN_ISOLATED_TEST("NetIbCastGdr_ForceEnable", []() { + ParamScript params; + params.Set("RCCL_FORCE_ENABLE_GDRDMA", 1); + ScopedHook scan(g_ncclIbScanDefaultPeerMemClients, [] { return 0; }); + EXPECT_EQ(IbCastGdrSupport(), ncclSuccess); + EXPECT_EQ(scan.calls, 0); + }); +} + +TEST_F(NetIbCastGdrMicrotestIsolated, GdrSupport_PeermemClientFound_SucceedsWithoutProbing) { + RUN_ISOLATED_TEST("NetIbCastGdr_PeermemFound", []() { + SysfsAnswers("Generic BIOS", "1"); + ScopedHook scan(g_ncclIbScanDefaultPeerMemClients, [] { return 1; }); + ScopedHook probe(g_ncclIbProbeGdrSupport, [](struct ibv_context*, int) { return 0; }); + EXPECT_EQ(IbCastGdrSupport(), ncclSuccess); + EXPECT_EQ(IbCastGdrSupport(), ncclSuccess); + EXPECT_EQ(scan.calls, 1); + EXPECT_EQ(probe.calls, 0); + }); +} + +TEST_F(NetIbCastGdrMicrotestIsolated, GdrSupport_NoPeermem_ProbesDeviceZeroOnceAndCachesSuccess) { + RUN_ISOLATED_TEST("NetIbCastGdr_ProbeSucceeds", []() { + struct ibv_context ctx {}; + IbCastDevs[0].context = &ctx; + IbCastRelaxedOrderingEnabled = 1; + ScopedHook scan(g_ncclIbScanDefaultPeerMemClients, [] { return 0; }); + ScopedHook probe(g_ncclIbProbeGdrSupport, [&](struct ibv_context* c, int ro) { + EXPECT_EQ(c, &ctx); + EXPECT_EQ(ro, 1); + return 1; + }); + EXPECT_EQ(IbCastGdrSupport(), ncclSuccess); + EXPECT_EQ(IbCastGdrSupport(), ncclSuccess); + EXPECT_EQ(probe.calls, 1); + }); +} + +TEST_F(NetIbCastGdrMicrotestIsolated, GdrSupport_ProbeSaysNo_FailsAndDoesNotProbeAgain) { + RUN_ISOLATED_TEST("NetIbCastGdr_ProbeFails", []() { + ScopedHook scan(g_ncclIbScanDefaultPeerMemClients, [] { return 0; }); + ScopedHook probe(g_ncclIbProbeGdrSupport, [](struct ibv_context*, int) { return 0; }); + EXPECT_EQ(IbCastGdrSupport(), ncclSystemError); + EXPECT_EQ(IbCastGdrSupport(), ncclSystemError); + EXPECT_EQ(probe.calls, 1); + }); +} + +// gdr_peermem.h says a probe that could not run (-1) must not be cached, but call_once latches it. +// Pins AICOMRCCL-3089; a fix flips this pin. +TEST_F(NetIbCastGdrMicrotestIsolated, GdrSupport_KnownBug_ProbeThatCouldNotRunIsNeverRetried) { + RUN_ISOLATED_TEST("NetIbCastGdr_ProbeCouldNotRun", []() { + int answer = -1; + ScopedHook scan(g_ncclIbScanDefaultPeerMemClients, [] { return 0; }); + ScopedHook probe(g_ncclIbProbeGdrSupport, [&](struct ibv_context*, int) { return answer; }); + EXPECT_EQ(IbCastGdrSupport(), ncclSystemError); + answer = 1; // the device could now be probed + EXPECT_EQ(IbCastGdrSupport(), ncclSystemError); + EXPECT_EQ(probe.calls, 1); + }); +} + +TEST_F(NetIbCastGdrMicrotestIsolated, GdrSupport_HyperVWithNumaBalancingAndNoRelaxedOrdering_IsBlocklisted) { + RUN_ISOLATED_TEST("NetIbCastGdr_HyperVBlocklisted", []() { + ParamScript params; + params.Set("IB_PCI_RELAXED_ORDERING", 0); + SysfsAnswers(kHyperV, "1"); + ScopedHook scan(g_ncclIbScanDefaultPeerMemClients, [] { return 1; }); + ScopedHook probe(g_ncclIbProbeGdrSupport, [](struct ibv_context*, int) { return 1; }); + EXPECT_EQ(IbCastGdrSupport(), ncclSystemError); + EXPECT_EQ(probe.calls, 0); // a deliberate override is not probed past + }); +} + +struct HyperVCase { + const char* name; + const char* numaBalancing; // nullptr: the read fails + int64_t relaxedOrdering; +}; + +class NetIbCastGdrHyperVMicrotestIsolated : public ::testing::TestWithParam {}; + +TEST_P(NetIbCastGdrHyperVMicrotestIsolated, GdrSupport_HyperVWithoutTheOtherConditions_KeepsPeermem) { + const HyperVCase c = GetParam(); + RUN_ISOLATED_TEST((std::string("NetIbCastGdr_HyperV_") + c.name).c_str(), [c]() { + ParamScript params; + params.Set("IB_PCI_RELAXED_ORDERING", c.relaxedOrdering); + SysfsAnswers(kHyperV, c.numaBalancing); + ScopedHook scan(g_ncclIbScanDefaultPeerMemClients, [] { return 1; }); + EXPECT_EQ(IbCastGdrSupport(), ncclSuccess); + }); +} + +INSTANTIATE_TEST_SUITE_P(NetIbCastGdr, NetIbCastGdrHyperVMicrotestIsolated, + ::testing::Values(HyperVCase{"NumaBalancingOff", "0", 0}, + HyperVCase{"NumaBalancingUnreadable", nullptr, 0}, + HyperVCase{"RelaxedOrderingOn", "1", 1}, + HyperVCase{"RelaxedOrderingDefault", "1", 2}), + [](const ::testing::TestParamInfo& info) { return info.param.name; }); + +TEST_F(NetIbCastGdrMicrotestIsolated, GdrSupport_BiosUnreadable_SkipsTheHyperVCheck) { + RUN_ISOLATED_TEST("NetIbCastGdr_BiosUnreadable", []() { + ParamScript params; + params.Set("IB_PCI_RELAXED_ORDERING", 0); + SysfsAnswers(nullptr, "1"); + ScopedHook scan(g_ncclIbScanDefaultPeerMemClients, [] { return 1; }); + EXPECT_EQ(IbCastGdrSupport(), ncclSuccess); + }); +} + +TEST_F(NetIbCastGdrMicrotestIsolated, PeerMemSupport_NvidiaPeermemLoaded_IsCheckedOnce) { + RUN_ISOLATED_TEST("NetIbCastGdr_PeerMemLoaded", []() { + std::string probed; + ScopedHook access(g_access, [&](const char* path, int mode) { + probed = path; + EXPECT_EQ(mode, F_OK); + return 0; + }); + EXPECT_EQ(IbCastPeerMemSupport(), ncclSuccess); + EXPECT_EQ(IbCastPeerMemSupport(), ncclSuccess); + EXPECT_EQ(access.calls, 1); + EXPECT_EQ(probed, "/sys/module/nvidia_peermem/version"); + }); +} + +TEST_F(NetIbCastGdrMicrotestIsolated, PeerMemSupport_NoNvidiaPeermem_Fails) { + RUN_ISOLATED_TEST("NetIbCastGdr_PeerMemAbsent", []() { + ScopedHook access(g_access, [](const char*, int) { return -1; }); + EXPECT_EQ(IbCastPeerMemSupport(), ncclSystemError); + }); +} + +// Physical device 1 backs merged device 0, so the probe has to look through the merge table. +void TwoDevices(struct ibv_context* ctx0, struct ibv_context* ctx1) { + IbCastNDevs = 2; + IbCastNMergedDevs = 2; + IbCastDevs[0].context = ctx0; + IbCastDevs[1].context = ctx1; + IbCastMergedDevs[0].vProps.ndevs = 1; + IbCastMergedDevs[0].vProps.devs[0] = 1; + IbCastMergedDevs[1].vProps.ndevs = 1; + IbCastMergedDevs[1].vProps.devs[0] = 0; +} + +TEST_F(NetIbCastGdrMicrotestIsolated, DmaBufSupport_KernelRejectsTheDummyFd_IsSupportedAndCachedPerDevice) { + RUN_ISOLATED_TEST("NetIbCastGdr_DmaBufSupported", []() { + struct ibv_context ctx0 {}, ctx1 {}; + struct ibv_pd pd {}; + TwoDevices(&ctx0, &ctx1); + ScopedHook alloc(g_ibvAllocPd, [&](struct ibv_pd** ret, struct ibv_context* ctx) { + EXPECT_EQ(ctx, &ctx1); + *ret = &pd; + return ncclSuccess; + }); + ScopedHook reg(g_ibvDirectRegDmabufMr, [](struct ibv_pd*, uint64_t, size_t, uint64_t, int fd, int) { + EXPECT_EQ(fd, -1); + errno = EBADF; // a kernel that knows DMA-BUF rejects the bogus fd itself + return (struct ibv_mr*)nullptr; + }); + ScopedHook dealloc(g_ibvDeallocPd, [&](struct ibv_pd* p) { + EXPECT_EQ(p, &pd); + return ncclSuccess; + }); + EXPECT_EQ(IbCastDmaBufSupport(0), ncclSuccess); + EXPECT_EQ(IbCastDevs[1].dmaBufSupported, 1); + EXPECT_EQ(IbCastDmaBufSupport(0), ncclSuccess); + EXPECT_EQ(alloc.calls, 1); + EXPECT_EQ(dealloc.calls, 1); + }); +} + +struct DmaBufFailure { + const char* name; + bool allocFails; + int regErrno; + bool deallocFails; +}; + +class NetIbCastGdrDmaBufMicrotestIsolated : public ::testing::TestWithParam {}; + +TEST_P(NetIbCastGdrDmaBufMicrotestIsolated, DmaBufSupport_IsUnsupported) { + const DmaBufFailure c = GetParam(); + RUN_ISOLATED_TEST((std::string("NetIbCastGdr_DmaBuf_") + c.name).c_str(), [c]() { + struct ibv_context ctx0 {}, ctx1 {}; + struct ibv_pd pd {}; + TwoDevices(&ctx0, &ctx1); + ScopedHook alloc(g_ibvAllocPd, [&](struct ibv_pd** ret, struct ibv_context*) { + *ret = c.allocFails ? nullptr : &pd; + return c.allocFails ? ncclSystemError : ncclSuccess; + }); + ScopedHook reg(g_ibvDirectRegDmabufMr, [&](struct ibv_pd*, uint64_t, size_t, uint64_t, int, int) { + errno = c.regErrno; + return (struct ibv_mr*)nullptr; + }); + ScopedHook dealloc(g_ibvDeallocPd, + [&](struct ibv_pd*) { return c.deallocFails ? ncclSystemError : ncclSuccess; }); + EXPECT_EQ(IbCastDmaBufSupport(1), ncclSystemError); + EXPECT_EQ(IbCastDevs[0].dmaBufSupported, -1); + EXPECT_EQ(IbCastDmaBufSupport(1), ncclSystemError); + EXPECT_EQ(alloc.calls, 1); + }); +} + +INSTANTIATE_TEST_SUITE_P(NetIbCastGdr, NetIbCastGdrDmaBufMicrotestIsolated, + ::testing::Values(DmaBufFailure{"PdAllocFails", true, EBADF, false}, + DmaBufFailure{"KernelLacksDmaBuf", false, EOPNOTSUPP, false}, + DmaBufFailure{"ProtocolUnsupported", false, EPROTONOSUPPORT, false}, + DmaBufFailure{"PdDeallocFails", false, EBADF, true}), + [](const ::testing::TestParamInfo& info) { return info.param.name; }); + +} // namespace diff --git a/projects/rccl/test/host/net-ib-cast-gid-inspect-test.cc b/projects/rccl/test/host/net-ib-cast-gid-inspect-test.cc new file mode 100644 index 000000000000..384451a8e38b --- /dev/null +++ b/projects/rccl/test/host/net-ib-cast-gid-inspect-test.cc @@ -0,0 +1,236 @@ +/************************************************************************* + * Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +// Host-only microtests for src/transport/net_ib_cast/net_ib_cast_gid_inspect.cc +// (NET_IB_CAST_GID_INSPECT_CC_PATH), the fault-injection build's GID inspection hooks. + +#include + +#include +#include + +#include "fakes/ibvwrap_fakes.h" +#include "net-ib-cast-test-common.h" + +#include NET_IB_CAST_GID_INSPECT_CC_PATH + +namespace { + +class NetIbCastGidInspectMicrotest : public ::testing::Test { + protected: + void SetUp() override { + IbCastNDevs = 2; + send_->base.isSend = true; + send_->base.vProps.ndevs = 2; + send_->devs[1].base.ibDevN = 1; + } + void TearDown() override { + NetIbCastResetGlobals(); + ResetIbvwrapFakes(); + } + + static ncclIbGidState State(int linkLayer, int gidIndex, uint8_t fill) { + ncclIbGidState s{}; + s.linkLayer = linkLayer; + s.gidIndex = gidIndex; + memset(s.gid, fill, sizeof(s.gid)); + return s; + } + + ZeroedPtr send_; +}; + +TEST_F(NetIbCastGidInspectMicrotest, Dev_SetThenGet_RoundTripsAndReportsNoComm) { + ncclIbGidState in = State(IBV_LINK_LAYER_ETHERNET, 3, 0x5a); + ASSERT_EQ(ncclIbCastGidSetDev(1, &in), ncclSuccess); + EXPECT_EQ(IbCastDevs[1].gidInfo.localGidIndex, 3); + EXPECT_EQ(IbCastDevs[1].gidInfo.link_layer, IBV_LINK_LAYER_ETHERNET); + ncclIbGidState out = State(0, 0, 0); + out.ibDev = 7; + ASSERT_EQ(ncclIbCastGidGetDev(1, &out), ncclSuccess); + EXPECT_EQ(out.gidIndex, 3); + EXPECT_EQ(out.linkLayer, IBV_LINK_LAYER_ETHERNET); + EXPECT_EQ(memcmp(out.gid, in.gid, sizeof(out.gid)), 0); + EXPECT_EQ(out.ibDev, -1); +} + +TEST_F(NetIbCastGidInspectMicrotest, Dev_OutOfRangeOrNull_IsRejected) { + ncclIbGidState s{}; + EXPECT_EQ(ncclIbCastGidGetDev(-1, &s), ncclInvalidArgument); + EXPECT_EQ(ncclIbCastGidGetDev(2, &s), ncclInvalidArgument); + EXPECT_EQ(ncclIbCastGidGetDev(0, nullptr), ncclInvalidArgument); + EXPECT_EQ(ncclIbCastGidSetDev(-1, &s), ncclInvalidArgument); + EXPECT_EQ(ncclIbCastGidSetDev(2, &s), ncclInvalidArgument); + EXPECT_EQ(ncclIbCastGidSetDev(0, nullptr), ncclInvalidArgument); +} + +TEST_F(NetIbCastGidInspectMicrotest, Comm_SetThenGet_UsesThatDevicesBaseAndReportsItsIbDev) { + ncclIbGidState in = State(IBV_LINK_LAYER_INFINIBAND, 9, 0x11); + ASSERT_EQ(ncclIbCastGidSetComm(send_.get(), 1, &in), ncclSuccess); + EXPECT_EQ(send_->devs[1].base.gidInfo.localGidIndex, 9); + EXPECT_EQ(send_->devs[0].base.gidInfo.localGidIndex, 0); + ncclIbGidState out{}; + ASSERT_EQ(ncclIbCastGidGetComm(send_.get(), 1, &out), ncclSuccess); + EXPECT_EQ(out.gidIndex, 9); + EXPECT_EQ(out.ibDev, 1); + EXPECT_EQ(memcmp(out.gid, in.gid, sizeof(out.gid)), 0); +} + +TEST_F(NetIbCastGidInspectMicrotest, Comm_RecvCommResolvesItsOwnDeviceLayout) { + ZeroedPtr recv; + recv->base.vProps.ndevs = 1; + recv->devs[0].base.ibDevN = 4; + recv->devs[0].base.gidInfo.localGidIndex = 6; + ncclIbGidState out{}; + ASSERT_EQ(ncclIbCastGidGetComm(recv.get(), 0, &out), ncclSuccess); + EXPECT_EQ(out.gidIndex, 6); + EXPECT_EQ(out.ibDev, 4); +} + +TEST_F(NetIbCastGidInspectMicrotest, Comm_BadCommIndexOrNull_IsRejected) { + ncclIbGidState s{}; + EXPECT_EQ(ncclIbCastGidGetComm(nullptr, 0, &s), ncclInvalidArgument); + EXPECT_EQ(ncclIbCastGidGetComm(send_.get(), -1, &s), ncclInvalidArgument); + EXPECT_EQ(ncclIbCastGidGetComm(send_.get(), 2, &s), ncclInvalidArgument); + EXPECT_EQ(ncclIbCastGidGetComm(send_.get(), 0, nullptr), ncclInvalidArgument); + EXPECT_EQ(ncclIbCastGidSetComm(nullptr, 0, &s), ncclInvalidArgument); + EXPECT_EQ(ncclIbCastGidSetComm(send_.get(), 2, &s), ncclInvalidArgument); + EXPECT_EQ(ncclIbCastGidSetComm(send_.get(), 0, nullptr), ncclInvalidArgument); +} + +TEST_F(NetIbCastGidInspectMicrotest, QpState_ReadsBackTheSgidOfEachLiveQp) { + ibv_qp qps[2] = {}; + send_->base.nqps = 3; + send_->base.qps[0] = {}; + send_->base.qps[0].qp = &qps[0]; + send_->base.qps[0].devIndex = 0; + send_->base.qps[0].rtrAttr.localGidIndex = 2; + send_->base.qps[1].qp = nullptr; // not created: reported, but not queried + send_->base.qps[1].devIndex = 1; + send_->base.qps[1].rtrAttr.localGidIndex = 5; + send_->base.qps[2].qp = &qps[1]; + send_->base.qps[2].devIndex = 1; + ScopedHook query(g_ibvQueryQp, [&](struct ibv_qp* qp, struct ibv_qp_attr* attr, int mask, struct ibv_qp_init_attr*) { + EXPECT_EQ(mask, IBV_QP_AV); + if (qp == &qps[1]) return ncclSystemError; + attr->ah_attr.grh.sgid_index = 4; + return ncclSuccess; + }); + ncclIbGidQpState out; + memset(&out, 0xff, sizeof(out)); + ASSERT_EQ(ncclIbCastGidGetQpState(send_.get(), &out), ncclSuccess); + EXPECT_EQ(query.calls, 2); + EXPECT_EQ(out.nqps, 3); + EXPECT_EQ(out.devIndex[1], 1); + EXPECT_EQ(out.rtrGidIndex[0], 2); + EXPECT_EQ(out.rtrGidIndex[1], 5); + EXPECT_TRUE(out.queryOk[0]); + EXPECT_EQ(out.sgidIndex[0], 4); + EXPECT_FALSE(out.queryOk[1]); + EXPECT_FALSE(out.queryOk[2]); + EXPECT_EQ(out.sgidIndex[3], 0); // cleared past nqps +} + +TEST_F(NetIbCastGidInspectMicrotest, QpState_NqpsIsClampedToTheQpArray) { + ncclIbGidQpState out; + send_->base.nqps = -1; + ASSERT_EQ(ncclIbCastGidGetQpState(send_.get(), &out), ncclSuccess); + EXPECT_EQ(out.nqps, 0); + send_->base.nqps = NCCL_IB_MAX_QPS + 5; + ASSERT_EQ(ncclIbCastGidGetQpState(send_.get(), &out), ncclSuccess); + EXPECT_EQ(out.nqps, NCCL_IB_MAX_QPS); + EXPECT_EQ(ncclIbCastGidGetQpState(nullptr, &out), ncclInvalidArgument); + EXPECT_EQ(ncclIbCastGidGetQpState(send_.get(), nullptr), ncclInvalidArgument); +} + +class NetIbCastGidInspectResiliencyMicrotest : public NetIbCastGidInspectMicrotest { + protected: + void SetUp() override { + NetIbCastGidInspectMicrotest::SetUp(); + res_ = std::make_unique(); + res_->ndevs = 2; + send_->base.resiliency = res_.get(); + } + std::unique_ptr res_; +}; + +TEST_F(NetIbCastGidInspectResiliencyMicrotest, DevState_ReportsTheResiliencyStateOfThatDevice) { + res_->devs[1].state.store(ncclIbResiliencyDevStateRecoveryInProgress); + int state = -1; + ASSERT_EQ(ncclIbCastGidGetDevState(send_.get(), 1, &state), ncclSuccess); + EXPECT_EQ(state, (int)ncclIbResiliencyDevStateRecoveryInProgress); + EXPECT_EQ(ncclIbCastGidGetDevState(send_.get(), 2, &state), ncclInvalidArgument); + EXPECT_EQ(ncclIbCastGidGetDevState(send_.get(), -1, &state), ncclInvalidArgument); + EXPECT_EQ(ncclIbCastGidGetDevState(send_.get(), 0, nullptr), ncclInvalidArgument); + EXPECT_EQ(ncclIbCastGidGetDevState(nullptr, 0, &state), ncclInvalidArgument); + send_->base.resiliency = nullptr; + EXPECT_EQ(ncclIbCastGidGetDevState(send_.get(), 0, &state), ncclInvalidArgument); +} + +TEST_F(NetIbCastGidInspectResiliencyMicrotest, RecoveryGidIndex_NeedsAGlobalRecoveryAh) { + int gidIndex = -1; + EXPECT_EQ(ncclIbCastGidGetRecoveryGidIndex(send_.get(), 1, &gidIndex), ncclInvalidArgument); // no AH yet + ibv_ah ah{}; + res_->portRecoveryAh[1] = &ah; + EXPECT_EQ(ncclIbCastGidGetRecoveryGidIndex(send_.get(), 1, &gidIndex), ncclInvalidArgument); // not global + res_->portRecoveryAhAttr[1].is_global = 1; + res_->portRecoveryAhAttr[1].grh.sgid_index = 7; + ASSERT_EQ(ncclIbCastGidGetRecoveryGidIndex(send_.get(), 1, &gidIndex), ncclSuccess); + EXPECT_EQ(gidIndex, 7); + EXPECT_EQ(ncclIbCastGidGetRecoveryGidIndex(send_.get(), NCCL_IB_MAX_DEVS_PER_NIC, &gidIndex), ncclInvalidArgument); + EXPECT_EQ(ncclIbCastGidGetRecoveryGidIndex(send_.get(), -1, &gidIndex), ncclInvalidArgument); + EXPECT_EQ(ncclIbCastGidGetRecoveryGidIndex(send_.get(), 1, nullptr), ncclInvalidArgument); + send_->base.resiliency = nullptr; + EXPECT_EQ(ncclIbCastGidGetRecoveryGidIndex(send_.get(), 1, &gidIndex), ncclInvalidArgument); +} + +TEST_F(NetIbCastGidInspectResiliencyMicrotest, ProbingGidIndex_TakesTheFirstLiveProbeQpOnThatDevice) { + ibv_qp qps[2] = {}; + res_->nProbingQps = 3; + res_->probingQps[0].qp = &qps[0]; + res_->probingQps[0].devIndex = 0; + res_->probingQps[1].qp = nullptr; // not created yet + res_->probingQps[1].devIndex = 1; + res_->probingQps[1].rtrAttr.localGidIndex = 9; + res_->probingQps[2].qp = &qps[1]; + res_->probingQps[2].devIndex = 1; + res_->probingQps[2].rtrAttr.localGidIndex = 3; + int gidIndex = -1; + ASSERT_EQ(ncclIbCastGidGetProbingGidIndex(send_.get(), 1, &gidIndex), ncclSuccess); + EXPECT_EQ(gidIndex, 3); + EXPECT_EQ(ncclIbCastGidGetProbingGidIndex(send_.get(), 2, &gidIndex), ncclInvalidArgument); + EXPECT_EQ(ncclIbCastGidGetProbingGidIndex(send_.get(), 0, nullptr), ncclInvalidArgument); + send_->base.resiliency = nullptr; + EXPECT_EQ(ncclIbCastGidGetProbingGidIndex(send_.get(), 1, &gidIndex), ncclInvalidArgument); +} + +TEST_F(NetIbCastGidInspectMicrotest, DriveQpToError_ModifiesOnlyTheStateToError) { + ibv_qp qp{}; + send_->base.nqps = 2; + send_->base.qps[1].qp = &qp; + ScopedHook modify(g_ibvModifyQp, [&](struct ibv_qp* q, struct ibv_qp_attr* attr, int mask) { + EXPECT_EQ(q, &qp); + EXPECT_EQ(attr->qp_state, IBV_QPS_ERR); + EXPECT_EQ(mask, IBV_QP_STATE); + return ncclSuccess; + }); + ASSERT_EQ(ncclIbCastGidDriveQpToError(send_.get(), 1), ncclSuccess); + EXPECT_EQ(modify.calls, 1); + EXPECT_EQ(ncclIbCastGidDriveQpToError(send_.get(), 0), ncclInvalidArgument); // no QP there + EXPECT_EQ(ncclIbCastGidDriveQpToError(send_.get(), 2), ncclInvalidArgument); + EXPECT_EQ(ncclIbCastGidDriveQpToError(send_.get(), -1), ncclInvalidArgument); + EXPECT_EQ(ncclIbCastGidDriveQpToError(nullptr, 0), ncclInvalidArgument); +} + +TEST_F(NetIbCastGidInspectMicrotest, DriveQpToError_ModifyFails_ReturnsTheError) { + ibv_qp qp{}; + send_->base.nqps = 1; + send_->base.qps[0].qp = &qp; + ScopedHook modify(g_ibvModifyQp, [](struct ibv_qp*, struct ibv_qp_attr*, int) { return ncclSystemError; }); + EXPECT_EQ(ncclIbCastGidDriveQpToError(send_.get(), 0), ncclSystemError); +} + +} // namespace diff --git a/projects/rccl/test/host/net-ib-cast-gin-test.cc b/projects/rccl/test/host/net-ib-cast-gin-test.cc new file mode 100644 index 000000000000..96ddfe60679a --- /dev/null +++ b/projects/rccl/test/host/net-ib-cast-gin-test.cc @@ -0,0 +1,1151 @@ +/************************************************************************* + * Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +// Host-only microtests for src/transport/net_ib_cast/gin.cc (NET_IB_CAST_GIN_CC_PATH): the GIN/RMA +// proxy backend, its bootstrap collectives and the verbs work requests it posts. The GDAKI arm is +// compiled out (RCCL_NET_IB_CAST_ENABLE_GDAKI is unset), as in the shipping build. + +#include + +#include +#include +#include +#include +#include + +#include "../common/LogCapture.hpp" +#include "../common/ProcessIsolatedTestRunner.hpp" +#include "fakes/gdr_peermem_fakes.h" +#include "fakes/gdr_probe_fakes.h" +#include "fakes/net_ib_cast_p2p_fakes.h" +#include "fakes/socket_fakes.h" +#include "net-ib-cast-test-common.h" +#include "fakes/param_redirect.h" + +#include NET_IB_CAST_GIN_CC_PATH + +namespace { + +using RcclUnitTesting::CaptureLog; +using RcclUnitTesting::LogHas; + +// One rank's side of the proxy: a send and a recv comm on device 0, a CQ, a QP and a flush QP, +// all on a fake context; requests come from the comms' own pools the way p2p.cc hands them out. +class NetIbCastGinMicrotest : public ::testing::Test { + protected: + void SetUp() override { + InitFakeContext(&ctx_, &device_); + snprintf(device_.name, sizeof(device_.name), "ionic_0"); + pd_.context = &ctx_; + cq_.context = &ctx_; + qp_.context = &ctx_; + flushQp_.context = &ctx_; + IbCastNDevs = 1; + IbCastNMergedDevs = 1; + IbCastDevs[0].context = &ctx_; + IbCastDevs[0].pd = &pd_; + IbCastMergedDevs[0].vProps.ndevs = 1; + for (struct ncclIbNetCommBase* base : {&send_->base, &recv_->base}) { + base->vProps.ndevs = 1; + base->nqps = 1; + } + send_->base.isSend = true; + send_->devs[0].base.pd = &pd_; + send_->devs[0].base.cq = &cq_; + send_->devs[0].putSignalScratchpadMr = &scratchMr_; + scratchMr_.lkey = 0x5c; + send_->base.qps[0].qp = &qp_; + recv_->devs[0].base.pd = &pd_; + recv_->devs[0].base.cq = &cq_; + recv_->devs[0].gpuFlush.qp.qp = &flushQp_; + cComm_.ctx = &netConfig_; + cComm_.rank = 0; + cComm_.nranks = 2; + cComm_.sendComm = send_.get(); + cComm_.recvComm = recv_.get(); + cComm_.allGather = IbCastGinIbAllGather; + netConfig_.trafficClass = NCCL_NET_TRAFFIC_CLASS_UNDEF; + } + void TearDown() override { + NetIbCastResetGlobals(); + ResetIbvwrapFakes(); + ResetNetIbCastConnectFakes(); + ResetNetIbCastP2pFakes(); + ResetSocketFakes(); + ResetGdrPeermemFakes(); + ResetGdrProbeFakes(); + } + + // An MR per registration, so lkey/rkey tell the registrations apart. + struct ibv_mr* NewMr() { + mrs_.emplace_back(new ibv_mr{}); + mrs_.back()->lkey = 0x100 + (uint32_t)mrs_.size(); + mrs_.back()->rkey = 0x200 + (uint32_t)mrs_.size(); + return mrs_.back().get(); + } + + ParamScript params_; + struct ibv_device device_ {}; + struct ibv_context ctx_; + struct ibv_pd pd_ {}; + struct ibv_cq cq_ {}; + struct ibv_qp qp_ {}, flushQp_ {}; + struct ibv_mr scratchMr_ {}; + ZeroedPtr send_; + ZeroedPtr recv_; + struct CastIbGinCollComm cComm_ {}; + ncclNetCommConfig_t netConfig_ {}; + std::vector> mrs_; + + // GDR is latched once per process; every test in this binary that reaches it sees peermem. + ScopedHook peermem_{g_ncclIbScanDefaultPeerMemClients, [] { return 1; }}; + HookOf regMr_{g_ibvRegMr, [this](struct ibv_mr** ret, struct ibv_pd*, void*, size_t, int) { + *ret = NewMr(); + return ncclSuccess; + }}; + HookOf getRequest_{ + g_ibCastGetRequest, [](struct ncclIbNetCommBase* base, struct ncclIbRequest** req) { + for (struct ncclIbRequest& r : base->reqs) { + if (r.type != NCCL_NET_IB_REQ_UNUSED) continue; + r.base = base; + memset(r.devBases, 0, sizeof(r.devBases)); + memset(r.events, 0, sizeof(r.events)); + *req = &r; + return ncclSuccess; + } + return ncclInternalError; + }}; + HookOf freeRequest_{g_ibCastFreeRequest, [](struct ncclIbRequest* r) { + r->type = NCCL_NET_IB_REQ_UNUSED; + return ncclSuccess; + }}; + HookOf addEvent_{g_ibCastAddEvent, [](struct ncclIbRequest* req, int devIndex) { + req->events[devIndex]++; + req->devBases[devIndex] = IbCastGetNetCommDevBase(req->base, devIndex); + }}; +}; + + +class NetIbCastGinInitMicrotest : public NetIbCastGinMicrotest { + protected: + void SetUp() override {} // discovery builds the device tables + void TearDown() override { + world_.reset(); + NetIbCastGinMicrotest::TearDown(); + } + std::unique_ptr world_ = std::make_unique(); +}; + +TEST_F(NetIbCastGinInitMicrotest, RmaProxyInit_WithADevice_HandsBackAConfigWithNoTrafficClass) { + world_->AddHca("ionic_0"); + void* ctx = nullptr; + ASSERT_EQ(IbCastRmaIbProxy.init(&ctx, 0, nullptr), ncclSuccess); + ASSERT_NE(ctx, nullptr); + EXPECT_EQ(static_cast(ctx)->trafficClass, NCCL_NET_TRAFFIC_CLASS_UNDEF); + int ndev = -1; + ASSERT_EQ(IbCastRmaIbProxy.devices(&ndev), ncclSuccess); + EXPECT_EQ(ndev, 1); + ASSERT_EQ(IbCastRmaIbProxy.finalize(ctx), ncclSuccess); +} + +TEST_F(NetIbCastGinInitMicrotest, RmaProxyInit_NoDevice_Fails) { + void* ctx = nullptr; + EXPECT_EQ(IbCastRmaIbProxyInit(&ctx, 0, nullptr), ncclInternalError); + EXPECT_EQ(ctx, nullptr); + EXPECT_EQ(IbCastGinIbFinalize(nullptr), ncclSuccess); // balances the reference discovery took +} + +TEST_F(NetIbCastGinInitMicrotest, RmaProxyInit_DeviceDiscoveryFails_Fails) { + params_.Set("IB_DISABLE", 1); + void* ctx = nullptr; + EXPECT_EQ(IbCastRmaIbProxyInit(&ctx, 0, nullptr), ncclInternalError); +} + +TEST_F(NetIbCastGinMicrotest, GinProperties_AdvertiseStrongAndVaSignals) { + ncclGinProperties_v14_t props{}; + ASSERT_EQ(IbCastGinIbGetGinProperties(&props), ncclSuccess); + EXPECT_TRUE(props.supportsStrongSignals); + EXPECT_TRUE(props.supportsVASignals); +} + +TEST_F(NetIbCastGinMicrotest, RmaProperties_DoNotClaimSignalsFlushPuts) { + ncclRmaProperties_t props; + props.flushesAllPutsOnAnySignal = true; + ASSERT_EQ(IbCastRmaIbProxy.getRmaProperties(&cComm_, &props), ncclSuccess); + EXPECT_FALSE(props.flushesAllPutsOnAnySignal); +} + +TEST_F(NetIbCastGinMicrotest, RmaProxyGetProperties_IsTheNetDeviceAsAGinProxy) { + snprintf(IbCastDevs[0].devName, sizeof(IbCastDevs[0].devName), "ionic_0"); + snprintf(IbCastMergedDevs[0].devName, sizeof(IbCastMergedDevs[0].devName), "ionic_0"); + ncclNetProperties_t props; + ASSERT_EQ(IbCastRmaIbProxy.getProperties(0, &props), ncclSuccess); + EXPECT_STREQ(props.name, "ionic_0"); + EXPECT_EQ(props.netDeviceType, NCCL_NET_DEVICE_GIN_PROXY); + EXPECT_EQ(IbCastRmaIbProxyGetProperties(1, &props), ncclInvalidUsage); +} + +TEST_F(NetIbCastGinMicrotest, RmaProxy_PublishesTheProxyEntryPointsWithoutProgress) { + EXPECT_STREQ(IbCastRmaIbProxy.name, "RMA_IB_PROXY"); + EXPECT_EQ(IbCastRmaIbProxy.listen, IbCastListen); + EXPECT_EQ(IbCastRmaIbProxy.closeColl, IbCastGinIbCloseColl); + EXPECT_EQ(IbCastRmaIbProxy.closeListen, IbCastCloseListen); + EXPECT_EQ(IbCastRmaIbProxy.iput, IbCastRmaIbProxyIPut); + EXPECT_EQ(IbCastRmaIbProxy.test, IbCastRmaIbProxyTest); + EXPECT_EQ(IbCastRmaIbProxy.rmaProgress, nullptr); + EXPECT_EQ(IbCastRmaIbProxy.queryLastError, nullptr); +} + +class NetIbCastGinMicrotestIsolated : public ::testing::Test {}; + +TEST_F(NetIbCastGinMicrotestIsolated, GdrSupport_NoPeermemButDmaBuf_IsEnough) { + RUN_ISOLATED_TEST("NetIbCastGin_GdrViaDmaBuf", []() { + struct ibv_pd pd {}; + ScopedHook peermem(g_ncclIbScanDefaultPeerMemClients, [] { return 0; }); + ScopedHook probe(g_ncclIbProbeGdrSupport, [](struct ibv_context*, int) { return 0; }); + ScopedHook alloc(g_ibvAllocPd, [&](struct ibv_pd** ret, struct ibv_context*) { + *ret = &pd; + return ncclSuccess; + }); + ScopedHook reg(g_ibvDirectRegDmabufMr, [](struct ibv_pd*, uint64_t, size_t, uint64_t, int, int) { + errno = EBADF; + return (struct ibv_mr*)nullptr; + }); + IbCastNMergedDevs = 1; + bool gdr = false; + ASSERT_EQ(IbCastGinIbGdrSupport(&gdr, false), ncclSuccess); + EXPECT_TRUE(gdr); + EXPECT_EQ(IbCastGinIbGdrGpuSupport(false), ncclSuccess); + }); +} + +TEST_F(NetIbCastGinMicrotestIsolated, GdrSupport_NeitherPeermemNorDmaBuf_RefusesGin) { + RUN_ISOLATED_TEST("NetIbCastGin_NoGdr", []() { + ScopedHook peermem(g_ncclIbScanDefaultPeerMemClients, [] { return 0; }); + ScopedHook probe(g_ncclIbProbeGdrSupport, [](struct ibv_context*, int) { return 0; }); + IbCastNMergedDevs = 1; + bool gdr = true; + ASSERT_EQ(IbCastGinIbGdrSupport(&gdr, false), ncclSuccess); + EXPECT_FALSE(gdr); + std::string log = CaptureLog([&] { EXPECT_EQ(IbCastGinIbGdrGpuSupport(false), ncclInvalidUsage); }); + EXPECT_TRUE(LogHas(log, "Peermem is not supported, and DMA-BUF is not available")) << log; + void* collComm = nullptr; + EXPECT_EQ(IbCastRmaIbProxyConnect(nullptr, nullptr, 2, 0, nullptr, &collComm), ncclInvalidUsage); + }); +} + +TEST_F(NetIbCastGinMicrotestIsolated, InitType_NoGdr_FailsAfterDiscovery) { + RUN_ISOLATED_TEST("NetIbCastGin_InitNoGdr", []() { + FakeIbWorld world; + world.AddHca("ionic_0"); + ScopedHook peermem(g_ncclIbScanDefaultPeerMemClients, [] { return 0; }); + ScopedHook probe(g_ncclIbProbeGdrSupport, [](struct ibv_context*, int) { return 0; }); + void* ctx = nullptr; + EXPECT_EQ(IbCastGinIbInitType(&ctx, 0, nullptr, NCCL_GIN_TYPE_PROXY), ncclInternalError); + EXPECT_EQ(ctx, nullptr); + }); +} + + +// A ring that completes every send and receive at once and fills each received chunk with its +// source rank, so the chunk order shows the ring direction. +class NetIbCastGinRingMicrotest : public NetIbCastGinMicrotest { + protected: + struct Xfer { + void* comm; + size_t offset; + size_t size; + int tag; + void* mhandle; + }; + void SetUp() override { + NetIbCastGinMicrotest::SetUp(); + cComm_.rank = 1; + cComm_.nranks = 3; + send_->devs[0].base.ibDevN = 1; // its own MR cache, so both registrations reach verbs + } + std::vector sends_, recvs_; + char* base_ = nullptr; // the buffer the collective registered + int pendingTests_ = 0; // polls each request needs before it completes + + HookOf isend_{g_ibCastIsend, [this](void* comm, void* data, size_t size, int tag, + void* mhandle, void*, void** request) { + sends_.push_back({comm, (size_t)((char*)data - base_), size, tag, mhandle}); + *request = &sends_; + return ncclSuccess; + }}; + HookOf irecv_{g_ibCastIrecv, [this](void* comm, int n, void** data, size_t* sizes, + int* tags, void** mhandles, void**, void** request) { + EXPECT_EQ(n, 1); + const size_t offset = (char*)data[0] - base_; + recvs_.push_back({comm, offset, sizes[0], tags[0], mhandles[0]}); + memset(data[0], 'A' + (int)(offset / sizes[0]), sizes[0]); + *request = &recvs_; + return ncclSuccess; + }}; + HookOf test_{g_ibCastTest, [this](void*, int* done, int* sizes) { + EXPECT_EQ(sizes, nullptr); + *done = pendingTests_ > 0 ? (pendingTests_--, 0) : 1; + return ncclSuccess; + }}; +}; + +TEST_F(NetIbCastGinRingMicrotest, AllGather_PassesChunksAroundTheRingAndDeregisters) { + char out[3 * 4]; + base_ = out; + const char src[4] = {'s', 'r', 'c', '1'}; + int deregs = 0; + ScopedHook dereg(g_ibvDeregMr, [&](struct ibv_mr*) { + deregs++; + return ncclSuccess; + }); + ASSERT_EQ(IbCastGinIbAllGather(&cComm_, (void*)src, out, 4), ncclSuccess); + EXPECT_EQ(memcmp(out + 4, src, 4), 0); + EXPECT_EQ(std::string(out, 4), "AAAA"); + EXPECT_EQ(std::string(out + 8, 4), "CCCC"); + // Rank 1 sends its own chunk, then forwards what came from rank 0; it hears from 0, then 2. + ASSERT_EQ(sends_.size(), 2u); + EXPECT_EQ(sends_[0].offset, 4u); + EXPECT_EQ(sends_[1].offset, 0u); + ASSERT_EQ(recvs_.size(), 2u); + EXPECT_EQ(recvs_[0].offset, 0u); + EXPECT_EQ(recvs_[1].offset, 8u); + EXPECT_EQ(sends_[0].comm, send_.get()); + EXPECT_EQ(recvs_[0].comm, recv_.get()); + EXPECT_EQ(sends_[0].tag, IBCAST_GIN_IB_ALLGATHER_TAG); + EXPECT_EQ(recvs_[0].tag, IBCAST_GIN_IB_ALLGATHER_TAG); + EXPECT_NE(sends_[0].mhandle, recvs_[0].mhandle); // one registration per comm + EXPECT_EQ(regMr_.calls, 2); + EXPECT_EQ(deregs, 2); +} + +TEST_F(NetIbCastGinRingMicrotest, AllGather_RetriesPostsAndPollsUntilBothSidesComplete) { + char out[3 * 4]; + base_ = out; + int sendTries = 0; + ScopedHook isend(g_ibCastIsend, [&](void*, void*, size_t, int, void*, void*, void** request) { + *request = ++sendTries % 2 ? nullptr : &sends_; // every other post is refused for now + return ncclSuccess; + }); + pendingTests_ = 3; + ASSERT_EQ(IbCastGinIbAllGather(&cComm_, out, out, 4), ncclSuccess); + EXPECT_EQ(sendTries, 4); + EXPECT_EQ(recvs_.size(), 2u); +} + +TEST_F(NetIbCastGinRingMicrotest, AllGather_AnyFailure_ReleasesWhatWasRegistered) { + char out[3 * 4]; + base_ = out; + int deregs = 0; + ScopedHook dereg(g_ibvDeregMr, [&](struct ibv_mr*) { + deregs++; + return ncclSuccess; + }); + { + int regs = 0; + ScopedHook reg(g_ibvRegMr, [&](struct ibv_mr** ret, struct ibv_pd*, void*, size_t, int) { + *ret = NewMr(); + return ++regs == 2 ? ncclSystemError : ncclSuccess; // the send-side registration fails + }); + EXPECT_EQ(IbCastGinIbAllGather(&cComm_, out, out, 4), ncclSystemError); + EXPECT_EQ(deregs, 1); + } + deregs = 0; + { + ScopedHook isend(g_ibCastIsend, [](void*, void*, size_t, int, void*, void*, void**) { return ncclSystemError; }); + EXPECT_EQ(IbCastGinIbAllGather(&cComm_, out, out, 4), ncclSystemError); + EXPECT_EQ(deregs, 2); + } + deregs = 0; + { + ScopedHook irecv(g_ibCastIrecv, [](void*, int, void**, size_t*, int*, void**, void**, void**) { + return ncclSystemError; + }); + EXPECT_EQ(IbCastGinIbAllGather(&cComm_, out, out, 4), ncclSystemError); + EXPECT_EQ(deregs, 2); + } + { + int tests = 0; + ScopedHook test(g_ibCastTest, [&](void*, int* done, int*) { + *done = 0; + return ++tests == 2 ? ncclSystemError : ncclSuccess; // the send-side poll fails + }); + EXPECT_EQ(IbCastGinIbAllGather(&cComm_, out, out, 4), ncclSystemError); + } + { + ScopedHook test(g_ibCastTest, [](void*, int*, int*) { return ncclSystemError; }); + EXPECT_EQ(IbCastGinIbAllGather(&cComm_, out, out, 4), ncclSystemError); + } + { + ScopedHook reg(g_ibvRegMr, [](struct ibv_mr**, struct ibv_pd*, void*, size_t, int) { return ncclSystemError; }); + EXPECT_EQ(IbCastGinIbAllGather(&cComm_, out, out, 4), ncclSystemError); + } +} + +// allGather stand-in for AllToAll: rank r's row j is the chunk r sends to rank j, "rj". +ncclResult_t ScriptedAllGather(struct CastIbGinCollComm* cComm, void* src, void* recv, size_t len) { + for (int r = 0; r < cComm->nranks; r++) { + for (int j = 0; j < cComm->nranks; j++) { + char* chunk = (char*)recv + r * len + j * (len / cComm->nranks); + chunk[0] = (char)('0' + r); + chunk[1] = (char)('0' + j); + } + } + return ncclSuccess; +} +ncclResult_t FailingAllGather(struct CastIbGinCollComm*, void*, void*, size_t) { return ncclRemoteError; } + +TEST_F(NetIbCastGinMicrotest, AllToAll_KeepsTheChunkEachRankAddressedToThisRank) { + cComm_.rank = 1; + cComm_.nranks = 3; + cComm_.allGather = ScriptedAllGather; + char src[3 * 2] = {}, out[3 * 2] = {}; + ASSERT_EQ(IbCastGinIbAllToAll(&cComm_, src, out, 2), ncclSuccess); + EXPECT_EQ(std::string(out, 6), "011121"); + cComm_.allGather = FailingAllGather; + EXPECT_EQ(IbCastGinIbAllToAll(&cComm_, src, out, 2), ncclRemoteError); +} + +TEST_F(NetIbCastGinRingMicrotest, P2PBarrier_IsAnAllGatherOfOneIntPerRank) { + ASSERT_EQ(IbCastGinIbP2PBarrier(&cComm_), ncclSuccess); + ASSERT_EQ(sends_.size(), 2u); + EXPECT_EQ(sends_[0].size, sizeof(int)); + ScopedHook isend(g_ibCastIsend, [](void*, void*, size_t, int, void*, void*, void**) { return ncclSystemError; }); + EXPECT_EQ(IbCastGinIbP2PBarrier(&cComm_), ncclSystemError); +} + + +class NetIbCastGinConnectMicrotest : public NetIbCastGinMicrotest { + protected: + struct ncclIbListenComm lComm_ {}; + std::vector connectTcs_; + int connectRefusals_ = 0; // connects that hand back no comm yet + int acceptRefusals_ = 0; + std::vector connectHandles_; + HookOf connect_{ + g_ibCastConnectImpl, [this](void*, int dev, void* handle, void** sendComm, ncclNetDeviceHandle_t**, int tc) { + EXPECT_EQ(dev, lComm_.dev); + connectTcs_.push_back(tc); + connectHandles_.push_back(handle); + *sendComm = connectRefusals_ > 0 ? (connectRefusals_--, nullptr) : (void*)send_.get(); + return ncclSuccess; + }}; + HookOf accept_{g_ibCastAccept, [this](void* listenComm, void** recvComm, + ncclNetDeviceHandle_t**) { + EXPECT_EQ(listenComm, &lComm_); + *recvComm = acceptRefusals_ > 0 ? (acceptRefusals_--, nullptr) + : (void*)recv_.get(); + return ncclSuccess; + }}; +}; + +TEST_F(NetIbCastGinConnectMicrotest, Connect_RetriesUntilBothSidesAreUpAndFillsTheCollComm) { + connectRefusals_ = 1; + acceptRefusals_ = 2; + int h0, h1; + void* handles[2] = {&h0, &h1}; + void* collComm = nullptr; + ASSERT_EQ(IbCastRmaIbProxy.connect(&netConfig_, handles, 2, 1, &lComm_, &collComm), ncclSuccess); + auto* c = static_cast(collComm); + ASSERT_NE(c, nullptr); + EXPECT_EQ(c->rank, 1); + EXPECT_EQ(c->nranks, 2); + EXPECT_EQ(c->ctx, &netConfig_); + EXPECT_EQ(c->sendComm, send_.get()); + EXPECT_EQ(c->recvComm, recv_.get()); + EXPECT_EQ(c->allGather, IbCastGinIbAllGather); + EXPECT_EQ(c->allToAll, IbCastGinIbAllToAll); + EXPECT_EQ(c->getGidIndex, IbCastGetGidIndex); + EXPECT_EQ(c->ib.context, &ctx_); + EXPECT_EQ(c->ib.pd, &pd_); + EXPECT_EQ(connect_.calls, 2); + EXPECT_EQ(accept_.calls, 3); + EXPECT_EQ(connectTcs_[0], -1); + EXPECT_EQ(connectHandles_[0], &h0); // rank 1 of 2 connects to rank 0, the next one around + ScopedHook closeRecv(g_ibCastCloseRecv, [&](void* comm) { + EXPECT_EQ(comm, recv_.get()); + return ncclSuccess; + }); + ScopedHook closeSend(g_ibCastCloseSend, [&](void* comm) { + EXPECT_EQ(comm, send_.get()); + return ncclSuccess; + }); + ASSERT_EQ(IbCastGinIbCloseColl(collComm), ncclSuccess); + EXPECT_EQ(closeRecv.calls, 1); + EXPECT_EQ(closeSend.calls, 1); +} + +TEST_F(NetIbCastGinConnectMicrotest, Connect_GinTrafficClassWinsOverTheIbOne) { + void* handles[2] = {nullptr, nullptr}; + void* collComm = nullptr; + params_.Set("IB_TC", 40); + ASSERT_EQ(IbCastGinIbConnect(&netConfig_, handles, 2, 0, &lComm_, &collComm), ncclSuccess); + free(collComm); + params_.Set("GIN_IB_TC", 104); + ASSERT_EQ(IbCastGinIbConnect(&netConfig_, handles, 2, 0, &lComm_, &collComm), ncclSuccess); + free(collComm); + EXPECT_EQ(connectTcs_, (std::vector{40, 104})); +} + +TEST_F(NetIbCastGinConnectMicrotest, Connect_ConnectOrAcceptFails_ReturnsTheError) { + void* handles[2] = {nullptr, nullptr}; + void* collComm = reinterpret_cast(0x1); + { + ScopedHook fails(g_ibCastConnectImpl, [](void*, int, void*, void**, ncclNetDeviceHandle_t**, int) { + return ncclRemoteError; + }); + EXPECT_EQ(IbCastGinIbConnect(&netConfig_, handles, 2, 0, &lComm_, &collComm), ncclRemoteError); + EXPECT_EQ(collComm, nullptr); + } +} + +// The coll comm and the send comm already connected are left behind when accept fails. +// Pins AICOMRCCL-3094; a fix flips this pin. +TEST_F(NetIbCastGinConnectMicrotest, Connect_KnownBug_AcceptFailureLeavesTheSendCommOpen) { + void* handles[2] = {nullptr, nullptr}; + void* collComm = nullptr; + ScopedHook fails(g_ibCastAccept, [](void*, void**, ncclNetDeviceHandle_t**) { return ncclSystemError; }); + ScopedHook closeSend(g_ibCastCloseSend, [](void*) { return ncclSuccess; }); + EXPECT_EQ(IbCastGinIbConnect(&netConfig_, handles, 2, 0, &lComm_, &collComm), ncclSystemError); + EXPECT_EQ(connect_.calls, 1); + EXPECT_EQ(closeSend.calls, 0); +} + +TEST_F(NetIbCastGinMicrotest, CloseColl_NullIsANoOpAndCloseErrorsArePassedUp) { + EXPECT_EQ(IbCastGinIbCloseColl(nullptr), ncclSuccess); + auto* c = static_cast(calloc(1, sizeof(struct CastIbGinCollComm))); + c->recvComm = recv_.get(); + c->sendComm = send_.get(); + { + ScopedHook closeRecv(g_ibCastCloseRecv, [](void*) { return ncclSystemError; }); + EXPECT_EQ(IbCastGinIbCloseColl(c), ncclSystemError); + } + c->recvComm = nullptr; + ScopedHook closeSend(g_ibCastCloseSend, [](void*) { return ncclSystemError; }); + EXPECT_EQ(IbCastGinIbCloseColl(c), ncclSystemError); + free(c); + // With neither comm there is nothing to close. + c = static_cast(calloc(1, sizeof(struct CastIbGinCollComm))); + EXPECT_EQ(IbCastGinIbCloseColl(c), ncclSuccess); + EXPECT_EQ(closeSend.calls, 1); +} + +TEST_F(NetIbCastGinMicrotest, RmaProxyCloseColl_FreesTheCollComm) { + void* c = calloc(1, sizeof(struct CastIbGinCollComm)); + EXPECT_EQ(IbCastRmaIbProxyCloseColl(c), ncclSuccess); +} + +// The context: every rank's handle is exchanged, then one send and one recv comm per peer per +// context, with a barrier after each peer. +class NetIbCastGinContextMicrotest : public NetIbCastGinRingMicrotest { + protected: + void SetUp() override { + NetIbCastGinRingMicrotest::SetUp(); + cComm_.rank = 1; + cComm_.nranks = 4; + cComm_.dev = 0; + for (int i = 0; i < 16; i++) { + peerSends_.emplace_back(); + peerRecvs_.emplace_back(); + } + } + void TearDown() override { + for (struct IbCastRmaIbProxyCtx* c : contexts_) { + for (int i = 0; i < c[0].nContexts; i++) { + free(c[i].fullSendComm); + free(c[i].fullRecvComm); + } + free(c); + } + NetIbCastGinRingMicrotest::TearDown(); + } + ncclResult_t Create(int nContexts, int rankStride, void** ctx) { + ncclRmaConfig_t config{}; + config.nContexts = nContexts; + config.trafficClass = 72; + config.rankStride = rankStride; + ncclResult_t res = IbCastRmaIbProxy.createContext(&cComm_, &config, ctx); + if (res == ncclSuccess) contexts_.push_back(static_cast(*ctx)); + return res; + } + + std::vector contexts_; + std::vector connectPeers_, acceptPeers_; + std::deque> peerSends_; + std::deque> peerRecvs_; + struct ncclIbListenComm lComm_ {}; + int listenBefore_ = 0; + std::vector gatheredHandles_; + + HookOf tc_{g_ibCastSetTrafficClass, [this](void* ctx, int tc) { + EXPECT_EQ(ctx, &netConfig_); + EXPECT_EQ(tc, 72); + }}; + HookOf listen_{g_ibCastListen, [this](void* ctx, int dev, void* handle, void** listenComm) { + EXPECT_EQ(ctx, &netConfig_); + memset(handle, 0, NCCL_NET_HANDLE_MAXSIZE); + ((struct ncclIbHandle*)handle)->magic = 0xfeed; + *listenComm = &lComm_; + return ncclSuccess; + }}; + HookOf closeListen_{g_ibCastCloseListen, [this](void* listenComm) { + EXPECT_EQ(listenComm, &lComm_); + return ncclSuccess; + }}; + // The handle exchange: every peer's handle is a copy of ours with its rank in the magic. + static ncclResult_t HandleAllGather(struct CastIbGinCollComm* cComm, void* src, void* recv, size_t len) { + EXPECT_EQ(len, (size_t)NCCL_NET_HANDLE_MAXSIZE); + for (int r = 0; r < cComm->nranks; r++) { + memcpy((char*)recv + r * len, src, len); + ((struct ncclIbHandle*)((char*)recv + r * len))->magic = 0x100 + r; + } + return ncclSuccess; + } + HookOf connect_{ + g_ibCastConnectImpl, [this](void* ctx, int, void* handle, void** sendComm, ncclNetDeviceHandle_t**, int tc) { + const struct ncclIbHandle* h = (const struct ncclIbHandle*)handle; + EXPECT_TRUE(h->isRMA); + connectPeers_.push_back((int)(h->magic - 0x100)); + *sendComm = peerSends_[connectPeers_.size() - 1].get(); + return ncclSuccess; + }}; + HookOf accept_{g_ibCastAccept, [this](void*, void** recvComm, ncclNetDeviceHandle_t**) { + acceptPeers_.push_back(0); + *recvComm = peerRecvs_[acceptPeers_.size() - 1].get(); + return ncclSuccess; + }}; +}; + +TEST_F(NetIbCastGinContextMicrotest, CreateContext_ConnectsToEveryPeerInEachContext) { + // The barrier after each peer is a ring all-gather over the bootstrap comms; the handle + // exchange goes through cComm->allGather, replaced here. + cComm_.allGather = HandleAllGather; + char ring[4 * sizeof(int)]; + base_ = ring; + void* ctx = nullptr; + ASSERT_EQ(Create(2, 1, &ctx), ncclSuccess); + auto* gc = static_cast(ctx); + EXPECT_EQ(gc[0].nContexts, 2); + EXPECT_EQ(gc[0].nranks, 4); + EXPECT_EQ(gc[1].rank, 1); + // Rank 1 connects to 1, 2, 3, 0 and accepts from 1, 0, 3, 2, in each context. + EXPECT_EQ(connectPeers_, (std::vector{1, 2, 3, 0, 1, 2, 3, 0})); + for (int c = 0; c < 2; c++) { + for (int peer = 0; peer < 4; peer++) { + EXPECT_NE(gc[c].fullSendComm[peer], nullptr) << c << " " << peer; + EXPECT_NE(gc[c].fullRecvComm[peer], nullptr) << c << " " << peer; + } + } + EXPECT_EQ(gc[0].fullSendComm[2], peerSends_[1].get()); + EXPECT_EQ(gc[0].fullRecvComm[0], peerRecvs_[1].get()); // accepted second, after itself + EXPECT_EQ(closeListen_.calls, 1); + EXPECT_EQ(sends_.size(), 8u * 3); // 8 barriers of nranks - 1 steps +} + +TEST_F(NetIbCastGinContextMicrotest, CreateContext_RankStride_ConnectsOnlyToThatStride) { + cComm_.allGather = HandleAllGather; + char ring[4 * sizeof(int)]; + base_ = ring; + void* ctx = nullptr; + ASSERT_EQ(Create(1, 2, &ctx), ncclSuccess); + EXPECT_EQ(connectPeers_, (std::vector{1, 3})); + auto* gc = static_cast(ctx); + EXPECT_EQ(gc[0].fullSendComm[0], nullptr); + EXPECT_EQ(gc[0].fullRecvComm[2], nullptr); +} + +TEST_F(NetIbCastGinContextMicrotest, CreateContext_NoRankStride_MeansEveryRank) { + cComm_.allGather = HandleAllGather; + char ring[4 * sizeof(int)]; + base_ = ring; + void* ctx = nullptr; + ASSERT_EQ(Create(1, 0, &ctx), ncclSuccess); + EXPECT_EQ(connectPeers_, (std::vector{1, 2, 3, 0})); +} + +TEST_F(NetIbCastGinContextMicrotest, CreateContext_StrideThatDoesNotDivideTheRanks_IsRejected) { + void* ctx = reinterpret_cast(0x1); + std::string log = CaptureLog([&] { EXPECT_EQ(Create(1, 3, &ctx), ncclInternalError); }); + EXPECT_TRUE(LogHas(log, "invalid rank stride 3, nranks (4) must be a multiple of it")) << log; + EXPECT_EQ(listen_.calls, 0); +} + +TEST_F(NetIbCastGinContextMicrotest, CreateContext_AnyStepFails_ReturnsNoContextAndClosesTheListener) { + cComm_.allGather = HandleAllGather; + char ring[4 * sizeof(int)]; + base_ = ring; + void* ctx = reinterpret_cast(0x1); + { + ScopedHook listen(g_ibCastListen, [](void*, int, void*, void**) { return ncclSystemError; }); + EXPECT_EQ(Create(1, 1, &ctx), ncclSystemError); + EXPECT_EQ(ctx, nullptr); + EXPECT_EQ(closeListen_.calls, 0); + } + cComm_.allGather = FailingAllGather; + EXPECT_EQ(Create(1, 1, &ctx), ncclRemoteError); + EXPECT_EQ(closeListen_.calls, 1); + cComm_.allGather = HandleAllGather; + { + ScopedHook fails(g_ibCastConnectImpl, [](void*, int, void*, void**, ncclNetDeviceHandle_t**, int) { + return ncclSystemError; + }); + EXPECT_EQ(Create(1, 1, &ctx), ncclSystemError); + } + { + ScopedHook fails(g_ibCastAccept, [](void*, void**, ncclNetDeviceHandle_t**) { return ncclSystemError; }); + EXPECT_EQ(Create(1, 1, &ctx), ncclSystemError); + } + { + ScopedHook fails(g_ibCastIsend, [](void*, void*, size_t, int, void*, void*, void**) { return ncclSystemError; }); + EXPECT_EQ(Create(1, 1, &ctx), ncclSystemError); // the barrier + } + EXPECT_EQ(ctx, nullptr); +} + +// Only the context array is freed on failure; the comms already connected stay open. +// Pins AICOMRCCL-3094; a fix flips this pin. +TEST_F(NetIbCastGinContextMicrotest, CreateContext_KnownBug_FailureLeavesConnectedCommsOpen) { + cComm_.allGather = HandleAllGather; + char ring[4 * sizeof(int)]; + base_ = ring; + int connects = 0; + ScopedHook secondFails(g_ibCastConnectImpl, [&](void*, int, void*, void** sendComm, ncclNetDeviceHandle_t**, int) { + if (++connects == 2) return ncclSystemError; + *sendComm = peerSends_[0].get(); + return ncclSuccess; + }); + ScopedHook closeSend(g_ibCastCloseSend, [](void*) { return ncclSuccess; }); + ScopedHook closeRecv(g_ibCastCloseRecv, [](void*) { return ncclSuccess; }); + void* ctx = nullptr; + EXPECT_EQ(Create(1, 1, &ctx), ncclSystemError); + EXPECT_EQ(accept_.calls, 1); + EXPECT_EQ(closeSend.calls, 0); + EXPECT_EQ(closeRecv.calls, 0); +} + +TEST_F(NetIbCastGinContextMicrotest, DestroyContext_ClosesEveryPeerCommOfEveryContext) { + cComm_.allGather = HandleAllGather; + char ring[4 * sizeof(int)]; + base_ = ring; + void* ctx = nullptr; + ASSERT_EQ(Create(2, 1, &ctx), ncclSuccess); + ScopedHook closeRecv(g_ibCastCloseRecv, [](void*) { return ncclSuccess; }); + ScopedHook closeSend(g_ibCastCloseSend, [](void*) { return ncclSuccess; }); + ASSERT_EQ(IbCastRmaIbProxy.destroyContext(ctx), ncclSuccess); + EXPECT_EQ(closeRecv.calls, 8); + EXPECT_EQ(closeSend.calls, 8); + auto* gc = static_cast(ctx); + EXPECT_EQ(gc[1].fullSendComm, nullptr); + EXPECT_EQ(gc[1].fullRecvComm, nullptr); +} + +TEST_F(NetIbCastGinContextMicrotest, DestroyContext_CloseFails_ReturnsTheError) { + cComm_.allGather = HandleAllGather; + char ring[4 * sizeof(int)]; + base_ = ring; + void* ctx = nullptr; + ASSERT_EQ(Create(1, 1, &ctx), ncclSuccess); + { + ScopedHook closeRecv(g_ibCastCloseRecv, [](void*) { return ncclSystemError; }); + EXPECT_EQ(IbCastRmaIbProxyDestroyContext(ctx), ncclSystemError); + } + ScopedHook closeRecv(g_ibCastCloseRecv, [](void*) { return ncclSuccess; }); + ScopedHook closeSend(g_ibCastCloseSend, [](void*) { return ncclSystemError; }); + EXPECT_EQ(IbCastRmaIbProxyDestroyContext(ctx), ncclSystemError); +} + +// The registration exchange: every rank's base VA and rkey, rank r's being 0x1000 * (r + 1) and r. +ncclResult_t MrAllGather(struct CastIbGinCollComm* cComm, void* src, void* recv, size_t len) { + for (int r = 0; r < cComm->nranks; r++) { + if (r == cComm->rank) { + memcpy((char*)recv + r * len, src, len); + } else if (len == sizeof(uintptr_t)) { + ((uintptr_t*)recv)[r] = 0x1000 * (r + 1); + } else { + ((uint32_t*)recv)[r] = (uint32_t)r; + } + } + return ncclSuccess; +} + +TEST_F(NetIbCastGinMicrotest, RegMrSym_RegistersOnTheRecvCommAndGathersEveryRanksVaAndRkey) { + cComm_.allGather = MrAllGather; + char data[64]; + void* mhandle = nullptr; + ASSERT_EQ(IbCastRmaIbProxy.regMrSym(&cComm_, data, sizeof(data), NCCL_PTR_HOST, 0, &mhandle), ncclSuccess); + auto* h = static_cast(mhandle); + EXPECT_EQ(h->mrHandle->mrs[0], mrs_[0].get()); + EXPECT_EQ(IbCastDevs[0].mrCache.population, 1); // the recv comm's device + EXPECT_EQ(h->base_vas[0], (uintptr_t)data); + EXPECT_EQ(h->base_vas[1], 0x2000u); + EXPECT_EQ(h->rkeys[0], mrs_[0]->rkey); + EXPECT_EQ(h->rkeys[1], 1u); + ASSERT_EQ(IbCastRmaIbProxy.deregMrSym(&cComm_, mhandle), ncclSuccess); + EXPECT_EQ(IbCastDevs[0].mrCache.population, 0); +} + +TEST_F(NetIbCastGinMicrotest, RegMrSymDmaBuf_PassesTheFdThrough) { + cComm_.allGather = MrAllGather; + char data[64]; + int fd = -2; + ScopedHook dmabuf(g_ibvRegDmabufMr, [&](struct ibv_mr** ret, struct ibv_pd*, uint64_t, size_t, uint64_t, int f, + int) { + fd = f; + *ret = NewMr(); + return ncclSuccess; + }); + void* mhandle = nullptr; + ASSERT_EQ(IbCastRmaIbProxy.regMrSymDmaBuf(&cComm_, data, sizeof(data), NCCL_PTR_CUDA, 0, 9, 0, &mhandle), + ncclSuccess); + EXPECT_EQ(fd, 9); + ASSERT_EQ(IbCastRmaIbProxyDeregMrSym(&cComm_, mhandle), ncclSuccess); +} + +TEST_F(NetIbCastGinMicrotest, RegMrSym_RegistrationOrExchangeFails_ReturnsTheError) { + char data[64]; + void* mhandle = nullptr; + { + ScopedHook reg(g_ibvRegMr, [](struct ibv_mr**, struct ibv_pd*, void*, size_t, int) { return ncclSystemError; }); + EXPECT_EQ(IbCastRmaIbProxyRegMrSym(&cComm_, data, sizeof(data), NCCL_PTR_HOST, 0, &mhandle), ncclSystemError); + } +} + +// A failed exchange returns without deregistering, so the MR stays in the device's cache. +// Pins AICOMRCCL-3094; a fix flips this pin. +TEST_F(NetIbCastGinMicrotest, RegMrSym_KnownBug_FailedExchangeLeavesTheMrRegistered) { + char data[64]; + void* mhandle = nullptr; + cComm_.allGather = FailingAllGather; + EXPECT_EQ(IbCastRmaIbProxyRegMrSym(&cComm_, data, sizeof(data), NCCL_PTR_HOST, 0, &mhandle), ncclRemoteError); + EXPECT_EQ(mhandle, nullptr); + EXPECT_EQ(IbCastDevs[0].mrCache.population, 1); +} + +TEST_F(NetIbCastGinMicrotest, DeregMrSym_DeregFails_ReturnsTheError) { + cComm_.allGather = MrAllGather; + char data[64]; + void* mhandle = nullptr; + ASSERT_EQ(IbCastRmaIbProxyRegMrSym(&cComm_, data, sizeof(data), NCCL_PTR_HOST, 0, &mhandle), ncclSuccess); + ScopedHook dereg(g_ibvDeregMr, [](struct ibv_mr*) { return ncclSystemError; }); + EXPECT_EQ(IbCastRmaIbProxyDeregMrSym(&cComm_, mhandle), ncclSystemError); +} + + +class NetIbCastGinDataMicrotest : public NetIbCastGinMicrotest { + protected: + void SetUp() override { + NetIbCastGinMicrotest::SetUp(); + proxyCtx_[0].rank = 0; + proxyCtx_[0].nranks = 2; + proxyCtx_[0].nContexts = 1; + proxyCtx_[0].fullSendComm = sendComms_; + proxyCtx_[0].fullRecvComm = recvComms_; + sendComms_[1] = send_.get(); + recvComms_[1] = recv_.get(); + src_ = Handle(0x10, 0xa0, 0xb0); + dst_ = Handle(0x20, 0xa1, 0xb1); + sig_ = Handle(0x30, 0xa2, 0xb2); + recv_->devs[0].gpuFlush.sge.lkey = 0xf1; + } + // A symmetric registration: rank r's buffer at base + r * 0x10000, with rkey rkey0 + r. + struct IbCastRmaProxyMrHandle Handle(uintptr_t base, uint32_t lkey, uint32_t rkey0) { + mrStore_.emplace_back(new ibv_mr{}); + mrStore_.back()->lkey = lkey; + handleStore_.emplace_back(new ncclIbMrHandle{}); + handleStore_.back()->mrs[0] = mrStore_.back().get(); + vas_.emplace_back(new uintptr_t[2]{base, base + 0x10000}); + rkeys_.emplace_back(new uint32_t[2]{rkey0, rkey0 + 1}); + return {handleStore_.back().get(), vas_.back().get(), rkeys_.back().get()}; + } + struct ibv_send_wr Posted(size_t i) const { return posted_.at(i); } + + struct IbCastRmaIbProxyCtx proxyCtx_[1] {}; + void* sendComms_[2] = {}; + void* recvComms_[2] = {}; + struct IbCastRmaProxyMrHandle src_, dst_, sig_; + std::vector> mrStore_; + std::vector> handleStore_; + std::vector> vas_; + std::vector> rkeys_; + std::vector posted_; + std::vector postedSges_; + int postResult_ = 0; + HookOf postSend_{g_fakePostSend, [this](struct ibv_qp* qp, struct ibv_send_wr* wr, + struct ibv_send_wr**) { + postedQp_ = qp; + for (; wr; wr = wr->next) { + posted_.push_back(*wr); + postedSges_.push_back(wr->sg_list ? *wr->sg_list : ibv_sge{}); + } + return postResult_; + }}; + struct ibv_qp* postedQp_ = nullptr; +}; + +TEST_F(NetIbCastGinDataMicrotest, IPut_PostsOneSignaledWriteFromOurBufferToThePeers) { + void* request = nullptr; + ASSERT_EQ(IbCastRmaIbProxy.iput(proxyCtx_, 0, 8, &src_, 64, 16, &dst_, 1, 0, &request), ncclSuccess); + ASSERT_EQ(posted_.size(), 1u); + EXPECT_EQ(postedQp_, &qp_); + const struct ibv_send_wr wr = Posted(0); + EXPECT_EQ(wr.opcode, IBV_WR_RDMA_WRITE); + EXPECT_EQ(wr.send_flags, (unsigned)IBV_SEND_SIGNALED); + EXPECT_EQ(wr.wr.rdma.remote_addr, 0x20u + 0x10000 + 16); + EXPECT_EQ(wr.wr.rdma.rkey, 0xb1u + 1); + EXPECT_EQ(postedSges_[0].addr, 0x10u + 8); + EXPECT_EQ(postedSges_[0].length, 64u); + EXPECT_EQ(postedSges_[0].lkey, 0xa0u); + auto* req = static_cast(request); + EXPECT_EQ(wr.wr_id, (uint64_t)(req - send_->base.reqs)); + EXPECT_EQ(req->type, NCCL_NET_IB_REQ_GIN_IPUT); + EXPECT_EQ(req->iput.rank, 1); + EXPECT_EQ(req->ginProxyCtx, &proxyCtx_[0]); + EXPECT_EQ(req->sock, &send_->base.sock); + EXPECT_EQ(req->events[0], 1); + EXPECT_EQ(req->devBases[0], &send_->devs[0].base); +} + +TEST_F(NetIbCastGinDataMicrotest, IGet_PostsOneSignaledReadFromThePeersBufferIntoOurs) { + void* request = nullptr; + ASSERT_EQ(IbCastRmaIbProxy.iget(proxyCtx_, 0, 4, &dst_, 32, 12, &src_, 1, 0, &request), ncclSuccess); + ASSERT_EQ(posted_.size(), 1u); + const struct ibv_send_wr wr = Posted(0); + EXPECT_EQ(wr.opcode, IBV_WR_RDMA_READ); + EXPECT_EQ(wr.send_flags, (unsigned)IBV_SEND_SIGNALED); + EXPECT_EQ(wr.wr.rdma.remote_addr, 0x20u + 0x10000 + 4); + EXPECT_EQ(wr.wr.rdma.rkey, 0xb2u); + EXPECT_EQ(postedSges_[0].addr, 0x10u + 12); + EXPECT_EQ(postedSges_[0].lkey, 0xa0u); + EXPECT_EQ(static_cast(request)->type, NCCL_NET_IB_REQ_GIN_IGET); + EXPECT_EQ(static_cast(request)->iget.rank, 1); +} + +TEST_F(NetIbCastGinDataMicrotest, IPutSignal_WithData_ChainsAnUnsignaledWriteBeforeTheAtomicAdd) { + void* request = nullptr; + ASSERT_EQ(IbCastRmaIbProxy.iputSignal(proxyCtx_, 0, 0, &src_, 128, 0, &dst_, 1, 8, &sig_, 5, NCCL_NET_SIGNAL_OP_ADD, + true, 0, &request), + ncclSuccess); + ASSERT_EQ(posted_.size(), 2u); + EXPECT_EQ(Posted(0).opcode, IBV_WR_RDMA_WRITE); + EXPECT_EQ(Posted(0).send_flags, 0u); + EXPECT_EQ(postedSges_[0].length, 128u); + const struct ibv_send_wr sig = Posted(1); + EXPECT_EQ(sig.opcode, IBV_WR_ATOMIC_FETCH_AND_ADD); + EXPECT_EQ(sig.send_flags, (unsigned)IBV_SEND_SIGNALED); + EXPECT_EQ(sig.wr.atomic.remote_addr, 0x30u + 0x10000 + 8); + EXPECT_EQ(sig.wr.atomic.compare_add, 5u); + EXPECT_EQ(sig.wr.atomic.rkey, 0xb3u); + EXPECT_EQ(postedSges_[1].addr, (uint64_t)(uintptr_t)&send_->putSignalScratchpad); + EXPECT_EQ(postedSges_[1].lkey, 0x5cu); + EXPECT_EQ(Posted(0).wr_id, Posted(1).wr_id); + EXPECT_EQ(static_cast(request)->events[0], 1); +} + +TEST_F(NetIbCastGinDataMicrotest, IPutSignal_SizeZeroOrNoDestination_PostsOnlyTheSignal) { + void* request = nullptr; + ASSERT_EQ(IbCastRmaIbProxyIPutSignal(proxyCtx_, 0, 0, &src_, 0, 0, &dst_, 1, 0, &sig_, 5, NCCL_NET_SIGNAL_OP_INC, false, + 0, &request), + ncclSuccess); + ASSERT_EQ(posted_.size(), 1u); + EXPECT_EQ(Posted(0).opcode, IBV_WR_ATOMIC_FETCH_AND_ADD); + EXPECT_EQ(Posted(0).wr.atomic.compare_add, 1u); // INC ignores the value +} + +// With a size but no destination the PUT slot is left empty yet still posted, and the signal it +// should have chained to is never sent. Pins AICOMRCCL-3096; a fix flips this pin. +TEST_F(NetIbCastGinDataMicrotest, IPutSignal_KnownBug_DataWithoutADestinationPostsAnEmptyWrNotTheSignal) { + void* request = nullptr; + ASSERT_EQ(IbCastRmaIbProxyIPutSignal(proxyCtx_, 0, 0, &src_, 64, 0, nullptr, 1, 0, &sig_, 5, NCCL_NET_SIGNAL_OP_ADD, + false, 0, &request), + ncclSuccess); + ASSERT_EQ(posted_.size(), 1u); + EXPECT_EQ(Posted(0).num_sge, 0); + EXPECT_EQ(Posted(0).sg_list, nullptr); + EXPECT_NE(Posted(0).opcode, IBV_WR_ATOMIC_FETCH_AND_ADD); +} + +TEST_F(NetIbCastGinDataMicrotest, IPutSignal_UnsupportedOp_IsRejectedBeforeAnyPost) { + void* request = nullptr; + std::string log = CaptureLog([&] { + EXPECT_EQ(IbCastRmaIbProxyIPutSignal(proxyCtx_, 0, 0, &src_, 8, 0, &dst_, 1, 0, &sig_, 1, 0x7, false, 0, &request), + ncclInvalidArgument); + }); + EXPECT_TRUE(LogHas(log, "Unsupported signalOp 7")) << log; + EXPECT_TRUE(posted_.empty()); +} + +TEST_F(NetIbCastGinDataMicrotest, IFlush_ReadsOurOwnRegistrationThroughTheLoopbackFlushQp) { + void* request = nullptr; + ASSERT_EQ(IbCastRmaIbProxy.iflush(proxyCtx_, 0, &dst_, 1, &request), ncclSuccess); + ASSERT_EQ(posted_.size(), 1u); + EXPECT_EQ(postedQp_, &flushQp_); + const struct ibv_send_wr wr = Posted(0); + EXPECT_EQ(wr.opcode, IBV_WR_RDMA_READ); + EXPECT_EQ(wr.wr.rdma.remote_addr, 0x20u); // rank 0's VA, not the peer's + EXPECT_EQ(wr.wr.rdma.rkey, 0xb1u); + EXPECT_EQ(postedSges_[0].lkey, 0xf1u); + auto* req = static_cast(request); + EXPECT_EQ(req->type, NCCL_NET_IB_REQ_FLUSH); + EXPECT_EQ(req->sock, &recv_->base.sock); + EXPECT_EQ(req->events[0], 1); +} + +TEST_F(NetIbCastGinDataMicrotest, DataPath_NoRequestOrPostFails_ReturnsTheError) { + void* request = nullptr; + postResult_ = EIO; + EXPECT_EQ(IbCastRmaIbProxyIPut(proxyCtx_, 0, 0, &src_, 8, 0, &dst_, 1, 0, &request), ncclSystemError); + EXPECT_EQ(IbCastRmaIbProxyIGet(proxyCtx_, 0, 0, &dst_, 8, 0, &src_, 1, 0, &request), ncclSystemError); + EXPECT_EQ(IbCastRmaIbProxyIPutSignal(proxyCtx_, 0, 0, &src_, 8, 0, &dst_, 1, 0, &sig_, 1, NCCL_NET_SIGNAL_OP_INC, false, + 0, &request), + ncclSystemError); + EXPECT_EQ(IbCastRmaIbProxyIFlush(proxyCtx_, 0, &dst_, 1, &request), ncclSystemError); + ScopedHook none(g_ibCastGetRequest, [](struct ncclIbNetCommBase*, struct ncclIbRequest**) { + return ncclInternalError; + }); + EXPECT_EQ(IbCastRmaIbProxyIPut(proxyCtx_, 0, 0, &src_, 8, 0, &dst_, 1, 0, &request), ncclInternalError); + EXPECT_EQ(IbCastRmaIbProxyIGet(proxyCtx_, 0, 0, &dst_, 8, 0, &src_, 1, 0, &request), ncclInternalError); + EXPECT_EQ(IbCastRmaIbProxyIPutSignal(proxyCtx_, 0, 0, &src_, 8, 0, &dst_, 1, 0, &sig_, 1, NCCL_NET_SIGNAL_OP_INC, false, + 0, &request), + ncclInternalError); + EXPECT_EQ(IbCastRmaIbProxyIFlush(proxyCtx_, 0, &dst_, 1, &request), ncclInternalError); +} + +class NetIbCastGinTestMicrotest : public NetIbCastGinDataMicrotest { + protected: + // Completions handed out by the next poll, as (request, status) pairs. + void Completions(std::vector> wcs) { + wcs_ = std::move(wcs); + } + std::vector> wcs_; + int pollResult_ = 0; + HookOf pollCq_{g_fakePollCq, [this](struct ibv_cq* cq, int n, struct ibv_wc* wc) { + EXPECT_EQ(cq, &cq_); + EXPECT_EQ(n, 4); + if (pollResult_ < 0) return pollResult_; + int i = 0; + for (auto& [req, status] : wcs_) { + memset(&wc[i], 0, sizeof(wc[i])); + wc[i].wr_id = req - req->base->reqs; + wc[i].status = status; + wc[i].opcode = IBV_WC_RDMA_WRITE; + wc[i].vendor_err = 0x81; + i++; + } + wcs_.clear(); + return i; + }}; +}; + +TEST_F(NetIbCastGinTestMicrotest, Test_CompletesARequestOnceItsLastEventArrives) { + void* a = nullptr; + void* b = nullptr; + ASSERT_EQ(IbCastRmaIbProxyIPut(proxyCtx_, 0, 0, &src_, 8, 0, &dst_, 1, 0, &a), ncclSuccess); + ASSERT_EQ(IbCastRmaIbProxyIPut(proxyCtx_, 0, 0, &src_, 8, 0, &dst_, 1, 0, &b), ncclSuccess); + auto* ra = static_cast(a); + auto* rb = static_cast(b); + int done = -1; + ASSERT_EQ(IbCastRmaIbProxy.test(&cComm_, a, &done), ncclSuccess); // nothing yet + EXPECT_EQ(done, 0); + // b's completion arrives while polling for a: it is charged to b, a stays pending. + Completions({{rb, IBV_WC_SUCCESS}}); + ASSERT_EQ(IbCastRmaIbProxyTest(&cComm_, a, &done), ncclSuccess); + EXPECT_EQ(done, 0); + EXPECT_EQ(rb->events[0], 0); + Completions({{ra, IBV_WC_SUCCESS}}); + ASSERT_EQ(IbCastRmaIbProxyTest(&cComm_, a, &done), ncclSuccess); + EXPECT_EQ(done, 1); + EXPECT_EQ(ra->type, NCCL_NET_IB_REQ_UNUSED); + // b already has every event: done without polling. + const int polls = pollCq_.calls; + ASSERT_EQ(IbCastRmaIbProxyTest(&cComm_, b, &done), ncclSuccess); + EXPECT_EQ(done, 1); + EXPECT_EQ(pollCq_.calls, polls); + EXPECT_EQ(rb->type, NCCL_NET_IB_REQ_UNUSED); +} + +TEST_F(NetIbCastGinTestMicrotest, Test_FlushRequest_PollsTheRecvCommsCq) { + struct ibv_cq recvCq {}; + recvCq.context = &ctx_; + recv_->devs[0].base.cq = &recvCq; + void* f = nullptr; + ASSERT_EQ(IbCastRmaIbProxyIFlush(proxyCtx_, 0, &dst_, 1, &f), ncclSuccess); + int done = -1; + ScopedHook poll(g_fakePollCq, [&](struct ibv_cq* cq, int, struct ibv_wc*) { + EXPECT_EQ(cq, &recvCq); + return 0; + }); + ASSERT_EQ(IbCastRmaIbProxyTest(&cComm_, f, &done), ncclSuccess); + EXPECT_EQ(poll.calls, 1); +} + +TEST_F(NetIbCastGinTestMicrotest, Test_PollFails_ReturnsTheError) { + void* a = nullptr; + ASSERT_EQ(IbCastRmaIbProxyIPut(proxyCtx_, 0, 0, &src_, 8, 0, &dst_, 1, 0, &a), ncclSuccess); + pollResult_ = -1; + int done = -1; + EXPECT_EQ(IbCastRmaIbProxyTest(&cComm_, a, &done), ncclSystemError); +} + +TEST_F(NetIbCastGinTestMicrotest, Test_ErrorCompletion_IsARemoteErrorNamingTheHcaAndGids) { + send_->devs[0].base.gidInfo.link_layer = IBV_LINK_LAYER_ETHERNET; + send_->devs[0].base.gidInfo.localGid.raw[15] = 1; + send_->base.remDevs[0].remoteGid.raw[15] = 2; + struct ibv_context hcaCtx {}; + struct ibv_device hca {}; + snprintf(hca.name, sizeof(hca.name), "ionic_7"); + hcaCtx.device = &hca; + struct ibv_pd hcaPd {}; + hcaPd.context = &hcaCtx; + send_->devs[0].base.pd = &hcaPd; + void* a = nullptr; + ASSERT_EQ(IbCastRmaIbProxyIPut(proxyCtx_, 0, 0, &src_, 8, 0, &dst_, 1, 0, &a), ncclSuccess); + Completions({{static_cast(a), IBV_WC_REM_ACCESS_ERR}}); + int done = -1; + std::string log = CaptureLog([&] { EXPECT_EQ(IbCastRmaIbProxyTest(&cComm_, a, &done), ncclRemoteError); }); + EXPECT_TRUE(LogHas(log, ("status=" + std::to_string(IBV_WC_REM_ACCESS_ERR)).c_str())) << log; + EXPECT_TRUE(LogHas(log, "vendor err 129 (IPut) localGid ::1 remoteGids::2 hca ionic_7")) << log; + EXPECT_EQ(done, 0); +} + +// The error report indexes devBases[] with the completion's place in the poll batch, so an error +// in the second slot reads a device this one-device comm does not have. +// Pins AICOMRCCL-3095; a fix flips this pin. +TEST_F(NetIbCastGinTestMicrotest, Test_KnownBug_ErrorInTheSecondCompletionSlotCrashes) { + void* a = nullptr; + void* b = nullptr; + ASSERT_EQ(IbCastRmaIbProxyIPut(proxyCtx_, 0, 0, &src_, 8, 0, &dst_, 1, 0, &a), ncclSuccess); + ASSERT_EQ(IbCastRmaIbProxyIPut(proxyCtx_, 0, 0, &src_, 8, 0, &dst_, 1, 0, &b), ncclSuccess); + Completions({{static_cast(b), IBV_WC_SUCCESS}, + {static_cast(b), IBV_WC_REM_ACCESS_ERR}}); + const std::string style = ::testing::GTEST_FLAG(death_test_style); + ::testing::GTEST_FLAG(death_test_style) = "threadsafe"; + EXPECT_EXIT( + { + int done = 0; + IbCastRmaIbProxyTest(&cComm_, a, &done); + exit(0); + }, + ::testing::KilledBySignal(SIGSEGV), ""); + ::testing::GTEST_FLAG(death_test_style) = style; +} + +TEST_F(NetIbCastGinTestMicrotest, Test_ErrorCompletionOnInfiniband_LeavesTheGidsOut) { + struct ibv_context hcaCtx {}; + struct ibv_device hca {}; + hcaCtx.device = &hca; + struct ibv_pd hcaPd {}; + hcaPd.context = &hcaCtx; + send_->devs[0].base.pd = &hcaPd; + send_->devs[0].base.gidInfo.link_layer = IBV_LINK_LAYER_INFINIBAND; + void* a = nullptr; + ASSERT_EQ(IbCastRmaIbProxyIPut(proxyCtx_, 0, 0, &src_, 8, 0, &dst_, 1, 0, &a), ncclSuccess); + Completions({{static_cast(a), IBV_WC_RETRY_EXC_ERR}}); + int done = -1; + std::string log = CaptureLog([&] { EXPECT_EQ(IbCastRmaIbProxyTest(&cComm_, a, &done), ncclRemoteError); }); + EXPECT_FALSE(LogHas(log, "localGid")) << log; +} + +} // namespace diff --git a/projects/rccl/test/host/net-ib-cast-init-test.cc b/projects/rccl/test/host/net-ib-cast-init-test.cc new file mode 100644 index 000000000000..77077d0362d9 --- /dev/null +++ b/projects/rccl/test/host/net-ib-cast-init-test.cc @@ -0,0 +1,1360 @@ +/************************************************************************* + * Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +// Host-only microtests for src/transport/net_ib_cast/init.cc (NET_IB_CAST_INIT_CC_PATH): HCA +// discovery, virtual-device merging, device properties and the AINIC defaults, over fake verbs. + +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "../common/LogCapture.hpp" +#include "../common/ProcessIsolatedTestRunner.hpp" +#include "fakes/env_fakes.h" +#include "fakes/gdr_peermem_fakes.h" +#include "fakes/gdr_probe_fakes.h" +#include "fakes/net_ib_cast_p2p_fakes.h" +#include "fakes/net_ib_cast_qp_sharing_fakes.h" +#include "fakes/transport_stubs.h" +#include "net-ib-cast-test-common.h" +#include "fakes/param_redirect.h" + +// No test host has /sys/class/infiniband//device, so init.cc's realpath() of it is renamed +// onto this seam. +static std::function g_initRealpath = [](const char*, char*) -> char* { return nullptr; }; +static char* InitTestRealpath(const char* path, char* resolved) { return g_initRealpath(path, resolved); } + +#define realpath InitTestRealpath +#include NET_IB_CAST_INIT_CC_PATH +#undef realpath + +std::atomic g_netIbCastAsyncThreadsStarted{0}; +static char g_lastThreadName[64]; + +// init.cc names each async-event thread right after starting it. +void ncclSetThreadName(pthread_t, const char* fmt, ...) { + va_list ap; + va_start(ap, fmt); + vsnprintf(g_lastThreadName, sizeof(g_lastThreadName), fmt, ap); + va_end(ap); + g_netIbCastAsyncThreadsStarted++; +} + +void NetIbCastResetGlobals() { + netRefCount = 0; + for (struct ncclIbDev& dev : IbCastDevs) { + free(dev.pciPath); + free(dev.mrCache.slots); + dev.~ncclIbDev(); + new (&dev) ncclIbDev(); + } + memset(IbCastMergedDevs, 0, sizeof(IbCastMergedDevs)); + IbCastNDevs = -1; + IbCastNMergedDevs = -1; + memset(IbCastIfName, 0, sizeof(IbCastIfName)); + memset(&IbCastIfAddr, 0, sizeof(IbCastIfAddr)); + IbCastRelaxedOrderingEnabled = 0; + IbCastProfilerFunction = nullptr; + IbCastAinicRoce = IbCastMultiplaneEnable = IbCastOffloadEnabled = IbCastUseInline = IbCastAinicCtsInlineData = false; + IbCastGdrFlushDisable = 0; + IbCastArThreshold = 8192; + IbCastQpSharingGlobalEnable = false; + memset(&castGlobalQpSchedParms, 0, sizeof(castGlobalQpSchedParms)); +} + +namespace { + +using RcclUnitTesting::CaptureLog; +using RcclUnitTesting::LogHas; +using RcclUnitTesting::ScopedDebugLogging; + +class NetIbCastInitMicrotest : public ::testing::Test { + protected: + void TearDown() override { + world_.reset(); + NetIbCastResetGlobals(); + ResetEnvFakes(); + ResetIbvwrapFakes(); + ResetMlx5dvwrapFakes(); + ResetIonicdvwrapFakes(); + ResetSocketFakes(); + ResetNetIbCastConnectFakes(); + ResetNetIbCastP2pFakes(); + ResetNetIbCastQpSharingFakes(); + ResetNetIbCastSchedulerFakes(); + ResetNetIbCastP2pResiliencyRecoveryFakes(); + ResetGdrPeermemFakes(); + ResetGdrProbeFakes(); + ResetTransportStubs(); + } + + FakeIbWorld& World() { + if (!world_) world_ = std::make_unique(); + return *world_; + } + // For a second discovery in the same test: the first one's async threads must be gone first. + void ResetForAnotherInit() { + World().Quiesce(); + NetIbCastResetGlobals(); + } + + // A physical device as IbCastInitDevices leaves it, for the tests that start past discovery. + static struct ncclIbDev& Dev(int d, const char* name, uint8_t link = IBV_LINK_LAYER_ETHERNET) { + if (IbCastNDevs <= d) IbCastNDevs = d + 1; + struct ncclIbDev& dev = IbCastDevs[d]; + snprintf(dev.devName, sizeof(dev.devName), "%s", name); + dev.link = link; + dev.speed = 100000; + dev.portNum = 1; + dev.railId = NCCL_NET_ID_UNDEF; + dev.planeId = NCCL_NET_ID_UNDEF; + return dev; + } + + static ncclNetVDeviceProps_t Props(std::initializer_list devs) { + ncclNetVDeviceProps_t props{}; + for (int d : devs) props.devs[props.ndevs++] = d; + return props; + } + + // realpath() answers for /sys/class/infiniband//device. + void PciPaths(std::map paths) { + realpath_.emplace(g_initRealpath, [paths](const char* path, char*) -> char* { + for (auto& [hca, pci] : paths) + if (std::string(path) == "/sys/class/infiniband/" + hca + "/device") return strdup(pci.c_str()); + return nullptr; + }); + } + + ParamScript params_; + std::unique_ptr world_; + std::optional> realpath_; +}; + + +TEST_F(NetIbCastInitMicrotest, Speed_EachRateBitMapsToItsLaneSpeed) { + const int expected[] = {2500, 5000, 10000, 10000, 14000, 25000, 50000, 100000, 200000}; + for (int bit = 0; bit < (int)(sizeof(expected) / sizeof(expected[0])); bit++) { + EXPECT_EQ(ncclIbCastTestSpeed(1 << bit), expected[bit]) << bit; + } + // The lowest set bit wins; no bit at all falls through to the last entry. + EXPECT_EQ(ncclIbCastTestSpeed((1 << 1) | (1 << 5)), 5000); + EXPECT_EQ(ncclIbCastTestSpeed(0), 200000); +} + +TEST_F(NetIbCastInitMicrotest, Width_EachWidthBitMapsToItsLaneCount) { + EXPECT_EQ(IbCastWidth(1), 1); + EXPECT_EQ(IbCastWidth(2), 4); + EXPECT_EQ(IbCastWidth(4), 8); + EXPECT_EQ(IbCastWidth(8), 12); + EXPECT_EQ(IbCastWidth(16), 2); + EXPECT_EQ(IbCastWidth(0), 2); +} + +TEST_F(NetIbCastInitMicrotest, MatchVfPath_MergeVfsIgnoresTheFunctionAndPortDigits) { + char a[] = "/sys/devices/pci0000:00/0000:01:00.0"; + char b[] = "/sys/devices/pci0000:00/0000:01:10.1"; + char c[] = "/sys/devices/pci0000:00/0000:01:00.1"; + char d[] = "/sys/devices/pci0000:00/0000:02:00.0"; + EXPECT_TRUE(IbCastMatchVfPath(a, b)); + EXPECT_FALSE(IbCastMatchVfPath(a, d)); + params_.Set("IB_MERGE_VFS", 0); + EXPECT_FALSE(IbCastMatchVfPath(a, b)); + EXPECT_TRUE(IbCastMatchVfPath(a, c)); +} + +TEST_F(NetIbCastInitMicrotest, CompareDevs_OrdersByPciAddressWithUnknownPathsLast) { + struct ncclIbDev& lo = Dev(0, "lo"); + struct ncclIbDev& hi = Dev(1, "hi"); + struct ncclIbDev& none = Dev(2, "none"); + snprintf(lo.fullPciPath, PATH_MAX, "/sys/devices/pci0000:00/0000:01:00.0"); + snprintf(hi.fullPciPath, PATH_MAX, "/sys/devices/pci0000:80/0000:81:00.0"); + none.fullPciPath[0] = '\0'; + EXPECT_LT(IbCastCompareDevs(&lo, &hi), 0); + EXPECT_GT(IbCastCompareDevs(&hi, &lo), 0); + EXPECT_EQ(IbCastCompareDevs(&lo, &lo), 0); + EXPECT_GT(IbCastCompareDevs(&none, &lo), 0); + EXPECT_LT(IbCastCompareDevs(&lo, &none), 0); +} + +TEST_F(NetIbCastInitMicrotest, GetPciPath_FoldsPortAndVfDigitsAndKeepsTheFullPath) { + PciPaths({{"mlx5_0", "/sys/devices/pci0000:00/0000:01:02.3"}}); + char hca[] = "mlx5_0"; + char full[PATH_MAX]; + char* path = nullptr; + ASSERT_EQ(IbCastGetPciPath(hca, &path, full), ncclSuccess); + EXPECT_STREQ(full, "/sys/devices/pci0000:00/0000:01:02.3"); + EXPECT_STREQ(path, "/sys/devices/pci0000:00/0000:01:00.0"); + free(path); + params_.Set("IB_MERGE_VFS", 0); + ASSERT_EQ(IbCastGetPciPath(hca, &path, full), ncclSuccess); + EXPECT_STREQ(path, "/sys/devices/pci0000:00/0000:01:02.0"); + free(path); + ASSERT_EQ(IbCastGetPciPath(hca, nullptr, full), ncclSuccess); // path not wanted: freed here +} + +TEST_F(NetIbCastInitMicrotest, GetPciPath_NoSysfsEntry_LeavesAnEmptyFullPath) { + char hca[] = "mlx5_9"; + char full[PATH_MAX] = "stale"; + char* path = reinterpret_cast(0x1); + std::string log = CaptureLog([&] { ASSERT_EQ(IbCastGetPciPath(hca, &path, full), ncclSuccess); }); + EXPECT_EQ(path, nullptr); + EXPECT_STREQ(full, ""); + EXPECT_TRUE(LogHas(log, "Could not find real path of mlx5_9")) << log; +} + +TEST_F(NetIbCastInitMicrotest, GetRealPort_CountsEarlierDevicesOnTheSamePhysicalNic) { + char p0[] = "/sys/devices/pci0000:00/0000:01:00.0"; + char p1[] = "/sys/devices/pci0000:00/0000:02:00.0"; + Dev(0, "a").pciPath = p0; + Dev(1, "b").pciPath = p1; + Dev(2, "c").pciPath = p0; + int realPort = -1; + ASSERT_EQ(IbCastGetRealPort(p0, &realPort, 3), ncclSuccess); + EXPECT_EQ(realPort, 2); + ASSERT_EQ(IbCastGetRealPort(p0, &realPort, 0), ncclSuccess); + EXPECT_EQ(realPort, 0); + ASSERT_EQ(IbCastGetRealPort(nullptr, &realPort, 3), ncclSuccess); + EXPECT_EQ(realPort, 0); + for (int d = 0; d < 3; d++) IbCastDevs[d].pciPath = nullptr; +} + +TEST_F(NetIbCastInitMicrotest, PciRootFromPath_TakesTheFirstDomainAndBus) { + char root[8]; + ASSERT_EQ(IbCastGetPciRootFromPath("/sys/devices/pci0000:80/0000:80:03.1/0000:81:00.0", root, sizeof(root)), + ncclSuccess); + EXPECT_STREQ(root, "0000:80"); + // "pcie" and a short "pci1:2" are skipped on the way to the real root. + ASSERT_EQ(IbCastGetPciRootFromPath("/sys/pcie/pci1:2/pci000a:1f/x", root, sizeof(root)), ncclSuccess); + EXPECT_STREQ(root, "000a:1f"); + EXPECT_EQ(IbCastGetPciRootFromPath("/sys/devices/platform/x", root, sizeof(root)), ncclInvalidUsage); + EXPECT_EQ(IbCastGetPciRootFromPath(nullptr, root, sizeof(root)), ncclInvalidUsage); + EXPECT_EQ(IbCastGetPciRootFromPath("/pci0000:80", nullptr, sizeof(root)), ncclInvalidUsage); + EXPECT_EQ(IbCastGetPciRootFromPath("/pci0000:80", root, 7), ncclInvalidUsage); +} + +class NetIbCastInitNumaMicrotest : public NetIbCastInitMicrotest { + protected: + void SetUp() override { + char tmpl[] = "/tmp/net-ib-cast-numa-XXXXXX"; + ASSERT_NE(mkdtemp(tmpl), nullptr); + dir_ = tmpl; + } + void TearDown() override { + std::filesystem::remove_all(dir_); + NetIbCastInitMicrotest::TearDown(); + } + int NumaOf(const char* contents) { + std::ofstream(dir_ + "/numa_node") << contents; + return IbCastGetNumaNodeFromPath(dir_.c_str()); + } + std::string dir_; +}; + +TEST_F(NetIbCastInitNumaMicrotest, NumaNode_ReadsTheSysfsAttribute) { + EXPECT_EQ(NumaOf("1\n"), 1); + EXPECT_EQ(NumaOf("-1\n"), -1); +} + +TEST_F(NetIbCastInitNumaMicrotest, NumaNode_UnreadableOrGarbage_IsUnknown) { + EXPECT_EQ(IbCastGetNumaNodeFromPath(nullptr), -1); + EXPECT_EQ(IbCastGetNumaNodeFromPath(dir_.c_str()), -1); // no numa_node file + EXPECT_EQ(NumaOf(""), -1); + EXPECT_EQ(NumaOf("node"), -1); + EXPECT_EQ(NumaOf("-99999999999999999999"), -1); // ERANGE + std::string longPath(PATH_MAX, 'x'); + EXPECT_EQ(IbCastGetNumaNodeFromPath(longPath.c_str()), -1); +} + +TEST_F(NetIbCastInitMicrotest, PlaneIndex_AssignsIndexesInFirstSeenOrder) { + int16_t count = 1, planes[NCCL_IB_PLANE_MAX_INDEX] = {NCCL_NET_ID_UNDEF}, idx = -1; + ASSERT_EQ(ncclIbCastTestGetPlaneIndex(NCCL_NET_ID_UNDEF, &count, planes, &idx), ncclSuccess); + EXPECT_EQ(idx, 0); + ASSERT_EQ(ncclIbCastTestGetPlaneIndex(5, &count, planes, &idx), ncclSuccess); + EXPECT_EQ(idx, 1); + ASSERT_EQ(ncclIbCastTestGetPlaneIndex(9, &count, planes, &idx), ncclSuccess); + EXPECT_EQ(idx, 2); + ASSERT_EQ(ncclIbCastTestGetPlaneIndex(5, &count, planes, &idx), ncclSuccess); + EXPECT_EQ(idx, 1); + EXPECT_EQ(count, 3); + EXPECT_EQ(ncclIbCastTestGetPlaneIndex(5, nullptr, planes, &idx), ncclInvalidArgument); + EXPECT_EQ(ncclIbCastTestGetPlaneIndex(5, &count, nullptr, &idx), ncclInvalidArgument); + EXPECT_EQ(ncclIbCastTestGetPlaneIndex(5, &count, planes, nullptr), ncclInvalidArgument); +} + +TEST_F(NetIbCastInitMicrotest, PlaneIndex_TableFullOrVirtualBitSet_IsRejected) { + int16_t count = 1, planes[NCCL_IB_PLANE_MAX_INDEX] = {NCCL_NET_ID_UNDEF}, idx = -1; + std::string log = CaptureLog([&] { + EXPECT_EQ(ncclIbCastTestGetPlaneIndex(NCCL_IB_PLANE_VIRT_BIT | 1, &count, planes, &idx), ncclInvalidUsage); + }); + EXPECT_TRUE(LogHas(log, "cannot use a plane ID")) << log; + for (int p = 1; count < NCCL_IB_PLANE_MAX_INDEX - 1; p++) { + ASSERT_EQ(ncclIbCastTestGetPlaneIndex(p, &count, planes, &idx), ncclSuccess); + } + log = CaptureLog([&] { EXPECT_EQ(ncclIbCastTestGetPlaneIndex(100, &count, planes, &idx), ncclInvalidUsage); }); + EXPECT_TRUE(LogHas(log, "cannot use more than 14 plane IDs")) << log; + ASSERT_EQ(ncclIbCastTestGetPlaneIndex(1, &count, planes, &idx), ncclSuccess); // known ones still resolve +} + +TEST_F(NetIbCastInitMicrotest, RelaxedOrdering_NeedsTheModeAndTheIova2Api) { + ncclResult_t iova2 = ncclSystemError; // present: the NULL probe fails on its arguments + ScopedHook probe(g_ibvRegMrIova2, [&](struct ibv_mr** ret, struct ibv_pd* pd, void*, size_t, uint64_t, int) { + EXPECT_EQ(ret, nullptr); + EXPECT_EQ(pd, nullptr); + return iova2; + }); + EXPECT_EQ(IbCastRelaxedOrderingCapable(), 1); // default mode 2 + params_.Set("IB_PCI_RELAXED_ORDERING", 1); + EXPECT_EQ(IbCastRelaxedOrderingCapable(), 1); + iova2 = ncclInternalError; // IBVERBS_1.8 missing + EXPECT_EQ(IbCastRelaxedOrderingCapable(), 0); + params_.Set("IB_PCI_RELAXED_ORDERING", 0); + const int calls = probe.calls; + EXPECT_EQ(IbCastRelaxedOrderingCapable(), 0); + EXPECT_EQ(probe.calls, calls); +} + +struct DmaBufProbe { + const char* name; + bool allocFails; + int ibvErrno; + int mlx5Errno; + bool deallocFails; + bool capable; +}; + +class NetIbCastInitDmaBufMicrotest : public NetIbCastInitMicrotest, + public ::testing::WithParamInterface {}; + +TEST_P(NetIbCastInitDmaBufMicrotest, Mlx5DmaBufCapable_NeedsBothDummyRegistrationsToFailOnlyOnTheFd) { + const DmaBufProbe c = GetParam(); + struct ibv_context ctx {}; + struct ibv_pd pd {}; + ScopedHook alloc(g_ibvAllocPd, [&](struct ibv_pd** ret, struct ibv_context*) { + *ret = c.allocFails ? nullptr : &pd; + return c.allocFails ? ncclSystemError : ncclSuccess; + }); + ScopedHook ibvReg(g_ibvDirectRegDmabufMr, [&](struct ibv_pd*, uint64_t, size_t, uint64_t, int fd, int) { + EXPECT_EQ(fd, -1); + errno = c.ibvErrno; + return (struct ibv_mr*)nullptr; + }); + ScopedHook mlx5Reg(g_mlx5dvDirectRegDmabufMr, [&](struct ibv_pd*, uint64_t, size_t, uint64_t, int fd, int, int) { + EXPECT_EQ(fd, -1); + errno = c.mlx5Errno; + return (struct ibv_mr*)nullptr; + }); + ScopedHook dealloc(g_ibvDeallocPd, [&](struct ibv_pd*) { return c.deallocFails ? ncclSystemError : ncclSuccess; }); + EXPECT_EQ(ncclMlx5dvDmaBufCapable(&ctx), c.capable); +} + +INSTANTIATE_TEST_SUITE_P( + NetIbCastInit, NetIbCastInitDmaBufMicrotest, + ::testing::Values(DmaBufProbe{"Capable", false, EBADF, EBADF, false, true}, + DmaBufProbe{"PdAllocFails", true, EBADF, EBADF, false, false}, + DmaBufProbe{"KernelLacksDmaBuf", false, EOPNOTSUPP, EBADF, false, false}, + DmaBufProbe{"KernelProtocolUnsupported", false, EPROTONOSUPPORT, EBADF, false, false}, + DmaBufProbe{"Mlx5LacksDmaBuf", false, EBADF, EPROTONOSUPPORT, false, false}, + DmaBufProbe{"Mlx5OpUnsupported", false, EBADF, EOPNOTSUPP, false, false}, + DmaBufProbe{"PdDeallocFails", false, EBADF, EBADF, true, false}), + [](const ::testing::TestParamInfo& info) { return info.param.name; }); + +TEST_F(NetIbCastInitMicrotest, OooRqSize_ReportsTheDevicesOooReceiveDepthOnlyWhenEnabled) { + struct ibv_context ctx {}; + uint32_t size = 99; + int maxRc = 64; + uint64_t mask = MLX5DV_CONTEXT_MASK_OOO_RECV_WRS; + ScopedHook query(g_mlx5dvQueryDevice, [&](struct ibv_context*, struct mlx5dv_context* dv) { + EXPECT_EQ(dv->comp_mask, (uint64_t)MLX5DV_CONTEXT_MASK_OOO_RECV_WRS); + dv->comp_mask = mask; + dv->ooo_recv_wrs_caps.max_rc = maxRc; + return ncclSuccess; + }); + ASSERT_EQ(IbCastQueryOooRqSize(&ctx, "mlx5_0", &size), ncclSuccess); + EXPECT_EQ(size, 0u); // NCCL_IB_OOO_RQ off: not asked + EXPECT_EQ(query.calls, 0); + params_.Set("IB_OOO_RQ", 1); + ASSERT_EQ(IbCastQueryOooRqSize(&ctx, "mlx5_0", &size), ncclSuccess); + EXPECT_EQ(size, 64u); + maxRc = 0; + ASSERT_EQ(IbCastQueryOooRqSize(&ctx, "mlx5_0", &size), ncclSuccess); + EXPECT_EQ(size, 0u); + maxRc = 64; + mask = 0; // the provider does not know the capability + ASSERT_EQ(IbCastQueryOooRqSize(&ctx, "mlx5_0", &size), ncclSuccess); + EXPECT_EQ(size, 0u); + EXPECT_EQ(IbCastQueryOooRqSize(&ctx, "mlx5_0", nullptr), ncclInvalidArgument); +} + +TEST_F(NetIbCastInitMicrotest, OooRqSize_QueryFails_IsAnInternalError) { + struct ibv_context ctx {}; + uint32_t size = 0; + params_.Set("IB_OOO_RQ", 1); + ScopedHook query(g_mlx5dvQueryDevice, [](struct ibv_context*, struct mlx5dv_context*) { return ncclSystemError; }); + EXPECT_EQ(IbCastQueryOooRqSize(&ctx, "mlx5_0", &size), ncclInternalError); +} + + +class NetIbCastInitVDeviceMicrotest : public NetIbCastInitMicrotest { + protected: + void SetUp() override { + IbCastNMergedDevs = 0; + Dev(0, "mlx5_0"); + Dev(1, "mlx5_1"); + Dev(2, "mlx5_2"); + } + ncclResult_t Make(std::initializer_list devs, int* d) { + ncclNetVDeviceProps_t props = Props(devs); + return IbCastMakeVDevice(d, &props); + } +}; + +TEST_F(NetIbCastInitVDeviceMicrotest, MakeVDevice_OneDevice_CopiesItsNameSpeedRailAndPlane) { + IbCastDevs[1].railId = 4; + IbCastDevs[1].planeId = 2; + int d = -1; + ASSERT_EQ(Make({1}, &d), ncclSuccess); + EXPECT_EQ(d, 0); + EXPECT_EQ(IbCastNMergedDevs, 1); + const ncclIbMergedDev& m = IbCastMergedDevs[0]; + EXPECT_STREQ(m.devName, "mlx5_1"); + EXPECT_EQ(m.speed, 100000); + EXPECT_EQ(m.railId, 4); + EXPECT_EQ(m.planeId, 2); + EXPECT_EQ(m.vProps.ndevs, 1); + EXPECT_EQ(m.vProps.devs[0], 1); +} + +TEST_F(NetIbCastInitVDeviceMicrotest, MakeVDevice_SeveralDevices_FusesNamesSpeedsAndPlanes) { + IbCastDevs[0].railId = IbCastDevs[2].railId = 3; + IbCastDevs[0].planeIdx = 1; + IbCastDevs[2].planeIdx = 2; + IbCastOffloadEnabled = true; + int d = -1; + ScopedDebugLogging info(NCCL_LOG_INFO, NCCL_ALL); + std::string log = CaptureLog([&] { ASSERT_EQ(Make({0, 2, 0}, &d), ncclSuccess); }); + const ncclIbMergedDev& m = IbCastMergedDevs[d]; + EXPECT_STREQ(m.devName, "mlx5_0+mlx5_2"); // the repeated device is merged once + EXPECT_EQ(m.vProps.ndevs, 2); + EXPECT_EQ(m.speed, 200000); + EXPECT_EQ(m.railId, 3); + EXPECT_EQ(m.planeId, NCCL_IB_PLANE_VIRT_BIT | (1 << 1) | (1 << 2)); + EXPECT_FALSE(IbCastOffloadEnabled); // CTS offload does not support fused NICs + EXPECT_TRUE(LogHas(log, "disabling CTS Offload")) << log; +} + +TEST_F(NetIbCastInitVDeviceMicrotest, MakeVDevice_DifferentRails_LeavesTheRailUndefined) { + IbCastDevs[0].railId = 1; + IbCastDevs[1].railId = 2; + int d = -1; + ASSERT_EQ(Make({0, 1}, &d), ncclSuccess); + EXPECT_EQ(IbCastMergedDevs[d].railId, NCCL_NET_ID_UNDEF); +} + +TEST_F(NetIbCastInitVDeviceMicrotest, MakeVDevice_MergeNicsOff_RefusesToFuse) { + params_.Set("IB_MERGE_NICS", 0); + int d = -1; + std::string log = CaptureLog([&] { EXPECT_EQ(Make({0, 1}, &d), ncclInvalidUsage); }); + EXPECT_TRUE(LogHas(log, "Skipping makeVDevice, Please set NCCL_IB_MERGE_NICS=1")) << log; + EXPECT_EQ(IbCastNMergedDevs, 0); + ASSERT_EQ(Make({0}, &d), ncclSuccess); // a single device is not a fusion +} + +TEST_F(NetIbCastInitVDeviceMicrotest, MakeVDevice_AinicWithMergeNicsNotOne_RefusesToFuse) { + g_rcclUseAinicValue = true; + params_.Set("IB_MERGE_NICS", 2); + int d = -1; + std::string log = CaptureLog([&] { EXPECT_EQ(Make({0, 1}, &d), ncclInvalidUsage); }); + EXPECT_TRUE(LogHas(log, "(disabled by default on AINIC)")) << log; +} + +// AINIC is meant to fuse NICs only on an explicit NCCL_IB_MERGE_NICS=1, but 1 is the default. +// Pins AICOMRCCL-3088; a fix flips this pin. +TEST_F(NetIbCastInitVDeviceMicrotest, MakeVDevice_KnownBug_AinicFusesNicsByDefault) { + g_rcclUseAinicValue = true; + int d = -1; + EXPECT_EQ(Make({0, 1}, &d), ncclSuccess); + EXPECT_EQ(IbCastMergedDevs[d].vProps.ndevs, 2); +} + +TEST_F(NetIbCastInitVDeviceMicrotest, MakeVDevice_BadDeviceCounts_AreRejected) { + int d = -1; + ncclNetVDeviceProps_t props{}; + std::string log = CaptureLog([&] { EXPECT_EQ(IbCastMakeVDevice(&d, &props), ncclInvalidUsage); }); + EXPECT_TRUE(LogHas(log, "with 0 devices")) << log; + props.ndevs = -1; + EXPECT_EQ(IbCastMakeVDevice(&d, &props), ncclInvalidUsage); + props.ndevs = NCCL_IB_MAX_DEVS_PER_NIC + 1; + log = CaptureLog([&] { EXPECT_EQ(IbCastMakeVDevice(&d, &props), ncclInvalidUsage); }); + EXPECT_TRUE(LogHas(log, "Can't make virtual NIC with 9 devices, max 8")) << log; +} + +TEST_F(NetIbCastInitVDeviceMicrotest, MakeVDevice_UnknownPhysicalDevice_IsRejected) { + int d = -1; + std::string log = CaptureLog([&] { EXPECT_EQ(Make({0, 3}, &d), ncclInvalidUsage); }); + EXPECT_TRUE(LogHas(log, "Cannot use physical device 3, max 3")) << log; + EXPECT_EQ(Make({-1}, &d), ncclInvalidUsage); + EXPECT_EQ(IbCastNMergedDevs, 0); +} + +TEST_F(NetIbCastInitVDeviceMicrotest, MakeVDevice_TableFull_IsRejected) { + IbCastNMergedDevs = MAX_IB_VDEVS; + int d = -1; + std::string log = CaptureLog([&] { EXPECT_EQ(Make({0}, &d), ncclInvalidUsage); }); + EXPECT_TRUE(LogHas(log, "Cannot allocate any more virtual devices (256)")) << log; +} + +TEST_F(NetIbCastInitVDeviceMicrotest, MakeVDevice_MixedLinkLayers_IsRejected) { + IbCastDevs[1].link = IBV_LINK_LAYER_INFINIBAND; + int d = -1; + std::string log = CaptureLog([&] { EXPECT_EQ(Make({0, 1}, &d), ncclInvalidUsage); }); + EXPECT_TRUE(LogHas(log, "[0]mlx5_0:1/RoCE and [1]mlx5_1:1/IB")) << log; + EXPECT_EQ(IbCastNMergedDevs, 0); +} + +TEST_F(NetIbCastInitVDeviceMicrotest, MakeVDevice_AcrossNumaNodesOrRootComplexes_WarnsButFuses) { + char tmpl[] = "/tmp/net-ib-cast-vdev-XXXXXX"; + ASSERT_NE(mkdtemp(tmpl), nullptr); + const std::string base = tmpl; + const std::string p0 = base + "/pci0000:00/a", p1 = base + "/pci0000:00/b", p2 = base + "/pci0000:80/c"; + for (const std::string& p : {p0, p1, p2}) std::filesystem::create_directories(p); + std::ofstream(p0 + "/numa_node") << "0\n"; + std::ofstream(p1 + "/numa_node") << "1\n"; + std::ofstream(p2 + "/numa_node") << "0\n"; + IbCastDevs[0].pciPath = strdup(p0.c_str()); + IbCastDevs[1].pciPath = strdup(p1.c_str()); + IbCastDevs[2].pciPath = strdup(p2.c_str()); + int d = -1; + std::string log = CaptureLog([&] { ASSERT_EQ(Make({0, 1}, &d), ncclSuccess); }); + EXPECT_TRUE(LogHas(log, "Merging NICs across NUMA nodes (mlx5_0 numa=0, mlx5_1 numa=1)")) << log; + log = CaptureLog([&] { ASSERT_EQ(Make({0, 2}, &d), ncclSuccess); }); + EXPECT_TRUE(LogHas(log, "PCIe Root Complexes (mlx5_0 root=0000:00, mlx5_2 root=0000:80)")) << log; + EXPECT_FALSE(LogHas(log, "NUMA")) << log; + std::filesystem::remove_all(base); +} + +TEST_F(NetIbCastInitVDeviceMicrotest, SetNetAttr_AcceptsAnything) { + EXPECT_EQ(IbCastSetNetAttr(nullptr, nullptr), ncclSuccess); +} + + +class NetIbCastInitDevicesMicrotest : public NetIbCastInitMicrotest { + protected: + ncclResult_t Init() { return IbCastInitDevices(nullptr, nullptr); } +}; + +TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_OneRoceHca_AddsOnePhysicalAndOneVirtualDevice) { + World().AddHca("ionic_0"); + PciPaths({{"ionic_0", "/sys/devices/pci0000:00/0000:01:00.0"}}); + ScopedDebugLogging info(NCCL_LOG_INFO, NCCL_ALL); + std::string log = CaptureLog([&] { ASSERT_EQ(Init(), ncclSuccess); }); + ASSERT_EQ(IbCastNDevs, 1); + ASSERT_EQ(IbCastNMergedDevs, 1); + const struct ncclIbDev& dev = IbCastDevs[0]; + EXPECT_STREQ(dev.devName, "ionic_0"); + EXPECT_EQ(dev.portNum, 1); + EXPECT_EQ(dev.link, IBV_LINK_LAYER_ETHERNET); + EXPECT_EQ(dev.speed, 25000 * 4); + EXPECT_FALSE(dev.speedFromQuery); + EXPECT_EQ(dev.guid, World().hcas[0].attr.sys_image_guid); + EXPECT_EQ(dev.maxQp, 1024); + EXPECT_EQ(dev.maxCqe, 4096); + EXPECT_EQ(dev.ar, 0); // adaptive routing is an IB default only + EXPECT_EQ(dev.udSupported, -1); + EXPECT_EQ(dev.rdmaReadSupported, -1); + EXPECT_EQ(dev.gidInfo.localGidIndex, 3); + EXPECT_EQ(dev.context, &World().hcas[0].context); + EXPECT_STREQ(dev.pciPath, "/sys/devices/pci0000:00/0000:01:00.0"); + EXPECT_EQ(dev.railId, -1); + EXPECT_EQ(dev.planeId, -1); + EXPECT_STREQ(IbCastMergedDevs[0].devName, "ionic_0"); + EXPECT_STREQ(IbCastIfName, "eth0"); + EXPECT_EQ(IbCastRelaxedOrderingEnabled, 0); + EXPECT_STREQ(g_lastThreadName, "NCCL IbAsync 0"); + EXPECT_TRUE(LogHas(log, "NET/IB : Using [0]ionic_0:1/RoCE")) << log; + EXPECT_EQ(g_ibCastReportMatchingSchemeCalls, 1); +} + +TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_SecondCall_OnlyTakesAReference) { + World().AddHca("ionic_0"); + ASSERT_EQ(Init(), ncclSuccess); + const int lists = World().getDeviceList.calls; + ASSERT_EQ(Init(), ncclSuccess); + EXPECT_EQ(World().getDeviceList.calls, lists); + EXPECT_EQ(IbCastNDevs, 1); + // Both references are released before the pool check runs. + params_.Set("RCCL_IB_QP_SHARING_VALIDATE_POOL", 1); + ASSERT_EQ(IbCastFinalizeDevices(), ncclSuccess); + EXPECT_EQ(g_ibCastValidateSharedQpPoolCalls, 0); + ASSERT_EQ(IbCastFinalizeDevices(), ncclSuccess); + EXPECT_EQ(g_ibCastValidateSharedQpPoolCalls, 1); +} + +TEST_F(NetIbCastInitDevicesMicrotest, FinalizeDevices_LastReferenceWithoutValidation_SkipsThePoolCheck) { + World().AddHca("ionic_0"); + ASSERT_EQ(Init(), ncclSuccess); + ASSERT_EQ(IbCastFinalizeDevices(), ncclSuccess); + EXPECT_EQ(g_ibCastValidateSharedQpPoolCalls, 0); +} + +TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_IbDisabled_FailsBeforeLoadingVerbs) { + params_.Set("IB_DISABLE", 1); + EXPECT_EQ(Init(), ncclInternalError); + EXPECT_EQ(World().ibvSymbols.calls, 0); +} + +// The reference is taken before the checks that can fail and is never dropped, so a retry +// returns at once as if the devices were up. Pins AICOMRCCL-3091; a fix flips this pin. +TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_KnownBug_FailedInitMakesTheNextOneASilentNoOp) { + World().AddHca("ionic_0"); + params_.Set("IB_DISABLE", 1); + ASSERT_EQ(Init(), ncclInternalError); + params_.Set("IB_DISABLE", 0); + EXPECT_EQ(Init(), ncclSuccess); + EXPECT_EQ(World().getDeviceList.calls, 0); + EXPECT_EQ(IbCastNDevs, -1); + int ndev = 0; + ASSERT_EQ(IbCastDevices(&ndev), ncclSuccess); + EXPECT_EQ(ndev, -1); +} + +TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_NoVerbsLibrary_Fails) { + ScopedHook symbols(g_ibvSymbols, [] { return ncclSystemError; }); + EXPECT_EQ(Init(), ncclInternalError); + EXPECT_EQ(IbCastNDevs, -1); +} + +TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_NoMlx5Library_CarriesOn) { + World().AddHca("ionic_0"); + ScopedHook symbols(g_mlx5dvSymbols, [] { return ncclSystemError; }); + ScopedDebugLogging info(NCCL_LOG_INFO, NCCL_ALL); + std::string log = CaptureLog([&] { EXPECT_EQ(Init(), ncclSuccess); }); + EXPECT_TRUE(LogHas(log, "Failed to open mlx5dv symbols")) << log; + EXPECT_EQ(IbCastNDevs, 1); +} + +TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_NoIonicLibrary_DisablesTheTransport) { + World().AddHca("ionic_0"); + ScopedHook symbols(g_ionicdvSymbols, [] { return ncclSystemError; }); + std::string log = CaptureLog([&] { EXPECT_EQ(Init(), ncclInternalError); }); + EXPECT_TRUE(LogHas(log, "libionic is missing")) << log; + EXPECT_EQ(World().getDeviceList.calls, 0); +} + +TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_NoIpInterface_Fails) { + World().AddHca("ionic_0"); + ScopedHook none(g_ncclFindInterfaces, [](char*, union ncclSocketAddress*, int, int, int* n) { + *n = 0; + return ncclSuccess; + }); + std::string log = CaptureLog([&] { EXPECT_EQ(Init(), ncclInternalError); }); + EXPECT_TRUE(LogHas(log, "No IP interface found")) << log; + EXPECT_EQ(World().freeDeviceList.calls, 0); // never listed +} + +TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_InterfaceLookupErrors_ArePassedUp) { + World(); + ScopedHook fails(g_ncclFindInterfaces, [](char*, union ncclSocketAddress*, int, int, int*) { return ncclSystemError; }); + EXPECT_EQ(Init(), ncclSystemError); +} + +TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_DeviceListFails_Fails) { + World(); + ScopedHook list(g_ibvGetDeviceList, [](struct ibv_device***, int*) { return ncclSystemError; }); + EXPECT_EQ(Init(), ncclInternalError); + EXPECT_EQ(World().freeDeviceList.calls, 0); +} + +TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_NoHca_FindsNothingButSucceeds) { + World(); + ScopedDebugLogging info(NCCL_LOG_INFO, NCCL_ALL); + std::string log = CaptureLog([&] { EXPECT_EQ(Init(), ncclSuccess); }); + EXPECT_EQ(IbCastNDevs, 0); + EXPECT_EQ(IbCastNMergedDevs, 0); + EXPECT_TRUE(LogHas(log, "No device found")) << log; + EXPECT_EQ(g_ibCastReportMatchingSchemeCalls, 0); + EXPECT_EQ(World().freeDeviceList.calls, 1); +} + +TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_UnopenableHca_IsSkipped) { + World().AddHca("ionic_0").openFails = true; + World().AddHca("ionic_1"); + std::string log = CaptureLog([&] { ASSERT_EQ(Init(), ncclSuccess); }); + EXPECT_TRUE(LogHas(log, "Unable to open device ionic_0")) << log; + ASSERT_EQ(IbCastNDevs, 1); + EXPECT_STREQ(IbCastDevs[0].devName, "ionic_1"); +} + +TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_OpenSucceedsWithoutAContext_IsSkipped) { + World().AddHca("ionic_0"); + ScopedHook open(g_ibvOpenDevice, [](struct ibv_context** ret, struct ibv_device*) { + *ret = nullptr; + return ncclSuccess; + }); + std::string log = CaptureLog([&] { ASSERT_EQ(Init(), ncclSuccess); }); + EXPECT_TRUE(LogHas(log, "Unable to open device ionic_0")) << log; + EXPECT_EQ(World().queryDevice.calls, 0); + EXPECT_EQ(IbCastNDevs, 0); +} + +TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_UnqueryableHca_IsClosedAndSkipped) { + FakeHca& bad = World().AddHca("ionic_0"); + bad.queryDeviceFails = true; + World().AddHca("ionic_1"); + std::string log = CaptureLog([&] { ASSERT_EQ(Init(), ncclSuccess); }); + EXPECT_TRUE(LogHas(log, "Unable to query device ionic_0")) << log; + EXPECT_EQ(bad.closes, 1); + EXPECT_EQ(IbCastNDevs, 1); +} + +TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_CloseAfterFailedQueryFails_FreesTheListAndFails) { + World().AddHca("ionic_0").queryDeviceFails = true; + ScopedHook close(g_ibvCloseDevice, [](struct ibv_context*) { return ncclSystemError; }); + EXPECT_EQ(Init(), ncclInternalError); + EXPECT_EQ(World().freeDeviceList.calls, 1); +} + +TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_PortsThatAreDownUnqueryableOrNotIbOrRoce_AreSkipped) { + FakeHca& hca = World().AddHca("ionic_0"); + hca.ports[0].state = IBV_PORT_DOWN; + hca.ports.push_back(FakeIbWorld::ActivePort(IBV_LINK_LAYER_UNSPECIFIED)); + hca.ports.push_back(FakeIbWorld::ActivePort(IBV_LINK_LAYER_INFINIBAND)); + hca.attr.phys_port_cnt = 4; // port 4 answers no query + ScopedHook query(g_ibvQueryDevice, [&](struct ibv_context*, struct ibv_device_attr* attr) { + *attr = hca.attr; + return ncclSuccess; + }); + std::string log = CaptureLog([&] { ASSERT_EQ(Init(), ncclSuccess); }); + EXPECT_TRUE(LogHas(log, "Unable to query port_num 4")) << log; + ASSERT_EQ(IbCastNDevs, 1); + EXPECT_EQ(IbCastDevs[0].portNum, 3); + EXPECT_EQ(IbCastDevs[0].link, IBV_LINK_LAYER_INFINIBAND); + EXPECT_EQ(IbCastDevs[0].ar, 1); // IB defaults to adaptive routing + EXPECT_EQ(hca.closes, 0); +} + +TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_HcaWithNoUsablePort_IsClosed) { + FakeHca& hca = World().AddHca("ionic_0"); + hca.ports[0].state = IBV_PORT_DOWN; + ASSERT_EQ(Init(), ncclSuccess); + EXPECT_EQ(hca.closes, 1); + EXPECT_EQ(IbCastNDevs, 0); +} + +TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_ClosingAnUnusedHcaFails_Fails) { + World().AddHca("ionic_0").ports[0].state = IBV_PORT_DOWN; + ScopedHook close(g_ibvCloseDevice, [](struct ibv_context*) { return ncclSystemError; }); + EXPECT_EQ(Init(), ncclInternalError); + EXPECT_EQ(World().freeDeviceList.calls, 1); +} + +TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_FreeingTheListFails_Fails) { + World().AddHca("ionic_0"); + ScopedHook freeList(g_ibvFreeDeviceList, [](struct ibv_device**) { return ncclSystemError; }); + std::string log = CaptureLog([&] { EXPECT_EQ(Init(), ncclInternalError); }); + EXPECT_EQ(freeList.calls, 2); // the failure path tries once more + EXPECT_TRUE(LogHas(log, "Unable to free device list")) << log; +} + +TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_PortSpeedQuery_WinsOverThePortAttributes) { + World().AddHca("ionic_0").portSpeed = 4000; // 400 Gb/s + World().AddHca("ionic_1").ports[0].active_speed_ex = 0x100; // XDR, beyond active_speed + ASSERT_EQ(Init(), ncclSuccess); + ASSERT_EQ(IbCastNDevs, 2); + EXPECT_EQ(IbCastDevs[0].speed, 400000); + int fromQuery = -1; + ASSERT_EQ(ncclIbCastTestSpeedSource(0, &fromQuery), ncclSuccess); + EXPECT_EQ(fromQuery, 1); + EXPECT_EQ(IbCastDevs[1].speed, 200000 * 4); + ASSERT_EQ(ncclIbCastTestSpeedSource(1, &fromQuery), ncclSuccess); + EXPECT_EQ(fromQuery, 0); + EXPECT_EQ(ncclIbCastTestGetNDevs(), 2); + EXPECT_EQ(ncclIbCastTestSpeedSource(2, &fromQuery), ncclInvalidArgument); + EXPECT_EQ(ncclIbCastTestSpeedSource(-1, &fromQuery), ncclInvalidArgument); + EXPECT_EQ(ncclIbCastTestSpeedSource(0, nullptr), ncclInvalidArgument); +} + +TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_IbHcaList_SelectsByPrefixExclusionAndExactName) { + for (const char* name : {"ionic_0", "ionic_1", "ionic_10"}) World().AddHca(name); + SetMicroEnv("NCCL_IB_HCA", "ionic_1"); + ASSERT_EQ(Init(), ncclSuccess); + EXPECT_EQ(IbCastNDevs, 2); // a prefix: ionic_1 and ionic_10 + ResetForAnotherInit(); + SetMicroEnv("NCCL_IB_HCA", "=ionic_1"); + ASSERT_EQ(Init(), ncclSuccess); + ASSERT_EQ(IbCastNDevs, 1); + EXPECT_STREQ(IbCastDevs[0].devName, "ionic_1"); + ResetForAnotherInit(); + SetMicroEnv("NCCL_IB_HCA", "^ionic_1"); + ASSERT_EQ(Init(), ncclSuccess); + ASSERT_EQ(IbCastNDevs, 1); + EXPECT_STREQ(IbCastDevs[0].devName, "ionic_0"); +} + +TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_IbHcaRailAndPlane_AreRecordedAndLogged) { + World().AddHca("ionic_0"); + World().AddHca("ionic_1"); + World().AddHca("ionic_2"); + SetMicroEnv("NCCL_IB_HCA", "ionic_0:1:7,ionic_1::2:5,ionic_2:::6"); + ScopedDebugLogging info(NCCL_LOG_INFO, NCCL_ALL); + std::string log = CaptureLog([&] { ASSERT_EQ(Init(), ncclSuccess); }); + ASSERT_EQ(IbCastNDevs, 3); + EXPECT_EQ(IbCastDevs[0].railId, 7); + EXPECT_EQ(IbCastDevs[1].railId, 2); + EXPECT_EQ(IbCastDevs[1].planeId, 5); + EXPECT_EQ(IbCastDevs[2].planeId, 6); + EXPECT_NE(IbCastDevs[1].planeIdx, IbCastDevs[2].planeIdx); + EXPECT_TRUE(LogHas(log, "[0]ionic_0:1:7/RoCE [1]ionic_1:1:2:5/RoCE [2]ionic_2:1::6/RoCE")) << log; +} + +TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_TooManyPlanes_FailsAfterListing) { + std::string hca; + for (int i = 0; i < NCCL_IB_PLANE_MAX_INDEX; i++) { + const std::string name = "ionic_" + std::string(1, (char)('a' + i)); + World().AddHca(name.c_str()); + hca += (i ? "," : "") + name + ":::" + std::to_string(i + 1); + } + SetMicroEnv("NCCL_IB_HCA", hca.c_str()); + std::string log = CaptureLog([&] { EXPECT_EQ(Init(), ncclInvalidUsage); }); + EXPECT_TRUE(LogHas(log, "cannot use more than")) << log; +} + +TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_SortsByPciPathUnlessDisabled) { + World().AddHca("ionic_0"); + World().AddHca("ionic_1"); + PciPaths({{"ionic_0", "/sys/devices/pci0000:80/0000:81:00.0"}, {"ionic_1", "/sys/devices/pci0000:00/0000:01:00.0"}}); + ASSERT_EQ(Init(), ncclSuccess); + EXPECT_STREQ(IbCastDevs[0].devName, "ionic_1"); + EXPECT_STREQ(IbCastMergedDevs[0].devName, "ionic_1"); + ResetForAnotherInit(); + params_.Set("IB_DEVICE_PCI_ORDER", 0); + ASSERT_EQ(Init(), ncclSuccess); + EXPECT_STREQ(IbCastDevs[0].devName, "ionic_0"); +} + +TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_TwoPortsOfOneNic_GetDistinctRealPorts) { + FakeHca& hca = World().AddHca("ionic_0"); + hca.ports.push_back(FakeIbWorld::ActivePort(IBV_LINK_LAYER_ETHERNET)); + PciPaths({{"ionic_0", "/sys/devices/pci0000:00/0000:01:00.1"}}); + ASSERT_EQ(Init(), ncclSuccess); + ASSERT_EQ(IbCastNDevs, 2); + EXPECT_EQ(IbCastDevs[0].realPort, 0); + EXPECT_EQ(IbCastDevs[1].realPort, 1); + EXPECT_EQ(IbCastNMergedDevs, 2); +} + +TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_AdaptiveRoutingParam_OverridesTheLinkDefault) { + World().AddHca("ionic_0"); + params_.Set("IB_ADAPTIVE_ROUTING", 1); + ASSERT_EQ(Init(), ncclSuccess); + EXPECT_EQ(IbCastDevs[0].ar, 1); +} + +TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_GidQueryFails_Fails) { + World().AddHca("ionic_0"); + ScopedHook gid(g_ibCastGidInfoQuery, [](struct ibv_context*, uint8_t, struct ibv_port_attr*, struct ncclIbGidInfo*) { + return ncclSystemError; + }); + EXPECT_EQ(Init(), ncclSystemError); + EXPECT_EQ(World().freeDeviceList.calls, 1); +} + +TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_RelaxedOrderingAvailable_IsEnabled) { + World(); + ScopedHook iova2(g_ibvRegMrIova2, [](struct ibv_mr**, struct ibv_pd*, void*, size_t, uint64_t, int) { + return ncclSystemError; + }); + ASSERT_EQ(Init(), ncclSuccess); + EXPECT_EQ(IbCastRelaxedOrderingEnabled, 1); +} + +TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_ArThreshold_IsTakenUnlessOooRqIsOn) { + World(); + params_.Set("IB_AR_THRESHOLD", 4096); + ASSERT_EQ(Init(), ncclSuccess); + EXPECT_EQ(IbCastArThreshold, 4096); + ResetForAnotherInit(); + params_.Set("IB_OOO_RQ", 1); + ScopedDebugLogging info(NCCL_LOG_INFO, NCCL_ALL); + std::string log = CaptureLog([&] { ASSERT_EQ(Init(), ncclSuccess); }); + EXPECT_EQ(IbCastArThreshold, 8192); + EXPECT_TRUE(LogHas(log, "AR threshold will be ignored")) << log; +} + +TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_Mlx5WithOooRq_RecordsTheOooDepth) { + World().AddHca("mlx5_0").mlx5 = true; + params_.Set("IB_OOO_RQ", 1); + params_.Set("IB_DATA_DIRECT", 0); + ScopedHook query(g_mlx5dvQueryDevice, [](struct ibv_context*, struct mlx5dv_context* dv) { + dv->ooo_recv_wrs_caps.max_rc = 32; + return ncclSuccess; + }); + ASSERT_EQ(Init(), ncclSuccess); + EXPECT_EQ(IbCastDevs[0].ibProvider, IB_PROVIDER_MLX5); + EXPECT_EQ(IbCastDevs[0].oooRqSize, 32u); +} + +TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_Mlx5OooRqQueryFails_Fails) { + World().AddHca("mlx5_0").mlx5 = true; + params_.Set("IB_OOO_RQ", 1); + ScopedHook query(g_mlx5dvQueryDevice, [](struct ibv_context*, struct mlx5dv_context*) { return ncclSystemError; }); + EXPECT_EQ(Init(), ncclInternalError); +} + +class NetIbCastInitDataDirectMicrotest : public NetIbCastInitDevicesMicrotest { + protected: + void SetUp() override { + World().AddHca("mlx5_0").mlx5 = true; + PciPaths({{"mlx5_0", "/sys/devices/pci0000:00/0000:01:00.0"}}); + } + // The DMA-BUF probe passes; the sysfs path lookup answers `result`. + void DataDirect(ncclResult_t result) { + allocPd_.emplace(g_ibvAllocPd, [this](struct ibv_pd** ret, struct ibv_context*) { + *ret = &pd_; + return ncclSuccess; + }); + regDmabuf_.emplace(g_ibvDirectRegDmabufMr, [](struct ibv_pd*, uint64_t, size_t, uint64_t, int, int) { + errno = EBADF; + return (struct ibv_mr*)nullptr; + }); + mlx5RegDmabuf_.emplace(g_mlx5dvDirectRegDmabufMr, [](struct ibv_pd*, uint64_t, size_t, uint64_t, int, int, int) { + errno = EBADF; + return (struct ibv_mr*)nullptr; + }); + sysfs_.emplace(g_mlx5dvGetDataDirectSysfsPath, [result](struct ibv_context*, char* buf, size_t len) { + if (result == ncclSuccess) snprintf(buf, len, "/devices/dd0"); + return result; + }); + } + struct ibv_pd pd_ {}; + std::optional> allocPd_; + std::optional> regDmabuf_; + std::optional> mlx5RegDmabuf_; + std::optional> sysfs_; +}; + +TEST_F(NetIbCastInitDataDirectMicrotest, InitDevices_DataDirectNic_ExposesOnlyTheDmaDevice) { + DataDirect(ncclSuccess); + ScopedDebugLogging info(NCCL_LOG_INFO, NCCL_ALL); + std::string log = CaptureLog([&] { ASSERT_EQ(Init(), ncclSuccess); }); + ASSERT_EQ(IbCastNDevs, 1); + EXPECT_STREQ(IbCastDevs[0].devName, "mlx5_0_dma"); + EXPECT_STREQ(IbCastDevs[0].pciPath, "/sys/devices/dd0"); + EXPECT_EQ(IbCastDevs[0].capsProvider.mlx5.dataDirect, 1); + EXPECT_STREQ(IbCastDevs[0].fullPciPath, "/sys/devices/pci0000:00/0000:01:00.0"); + EXPECT_TRUE(LogHas(log, "Data Direct DMA Interface is detected for device mlx5_0")) << log; +} + +TEST_F(NetIbCastInitDataDirectMicrotest, InitDevices_DataDirectTwo_ExposesBothDevices) { + DataDirect(ncclSuccess); + params_.Set("IB_DATA_DIRECT", 2); + ASSERT_EQ(Init(), ncclSuccess); + ASSERT_EQ(IbCastNDevs, 2); + EXPECT_EQ(IbCastDevs[0].capsProvider.mlx5.dataDirect + IbCastDevs[1].capsProvider.mlx5.dataDirect, 1); +} + +TEST_F(NetIbCastInitDataDirectMicrotest, InitDevices_NoDataDirectPath_KeepsThePlainDevice) { + DataDirect(ncclInvalidArgument); + ASSERT_EQ(Init(), ncclSuccess); + ASSERT_EQ(IbCastNDevs, 1); + EXPECT_STREQ(IbCastDevs[0].devName, "mlx5_0"); + EXPECT_EQ(IbCastDevs[0].capsProvider.mlx5.dataDirect, 0); +} + +// The error returns past the fail: label, so the device list and the open context are never released. +// Pins AICOMRCCL-3090; a fix flips this pin. +TEST_F(NetIbCastInitDataDirectMicrotest, InitDevices_KnownBug_DataDirectLookupErrorLeaksTheDeviceList) { + DataDirect(ncclSystemError); + std::string log = CaptureLog([&] { EXPECT_EQ(Init(), ncclSystemError); }); + EXPECT_TRUE(LogHas(log, "Error in mlx5dv_get_data_direct_sysfs_path with device mlx5_0")) << log; + EXPECT_EQ(World().freeDeviceList.calls, 0); + EXPECT_EQ(World().hcas[0].closes, 0); +} + +TEST_F(NetIbCastInitDataDirectMicrotest, InitDevices_DataDirectOff_SkipsTheProbe) { + DataDirect(ncclSuccess); + params_.Set("IB_DATA_DIRECT", 0); + ASSERT_EQ(Init(), ncclSuccess); + EXPECT_EQ(sysfs_->calls, 0); + EXPECT_STREQ(IbCastDevs[0].devName, "mlx5_0"); +} + + +class NetIbCastInitAinicMicrotest : public NetIbCastInitDevicesMicrotest { + protected: + void SetUp() override { + g_rcclUseAinicValue = true; + World().AddHca("ionic_0"); + } +}; + +TEST_F(NetIbCastInitAinicMicrotest, InitDevices_Ainic_EnablesCtsOffloadAndInlineByDefault) { + ScopedDebugLogging info(NCCL_LOG_INFO, NCCL_ALL); + std::string log = CaptureLog([&] { ASSERT_EQ(Init(), ncclSuccess); }); + EXPECT_TRUE(IbCastAinicRoce); + EXPECT_TRUE(IbCastOffloadEnabled); + EXPECT_TRUE(IbCastUseInline); + EXPECT_TRUE(IbCastAinicCtsInlineData); + EXPECT_EQ(IbCastGdrFlushDisable, 1); + EXPECT_FALSE(IbCastMultiplaneEnable); + EXPECT_TRUE(LogHas(log, "enabling IB Use Inline Data")) << log; + EXPECT_TRUE(LogHas(log, "CTS Inline Data: Enabled; CTS Offload: Enabled; IB Use Inline: Enabled; GDR Flush: " + "Disabled")) << log; + EXPECT_TRUE(LogHas(log, "AINIC uDMA policy: 0")) << log; +} + +TEST_F(NetIbCastInitAinicMicrotest, InitDevices_AinicOffloadOff_KeepsInlineAsConfigured) { + params_.Set("RCCL_CTS_OFFLOAD_ENABLED", 0); + params_.Set("GDR_FLUSH_DISABLE", 0); + ASSERT_EQ(Init(), ncclSuccess); + EXPECT_FALSE(IbCastOffloadEnabled); + EXPECT_FALSE(IbCastUseInline); + EXPECT_FALSE(IbCastAinicCtsInlineData); + EXPECT_EQ(IbCastGdrFlushDisable, 0); +} + +TEST_F(NetIbCastInitAinicMicrotest, InitDevices_AinicInlineWithoutOffload_KeepsCtsInlineData) { + params_.Set("RCCL_CTS_OFFLOAD_ENABLED", 0); + params_.Set("IB_USE_INLINE", 1); + ASSERT_EQ(Init(), ncclSuccess); + EXPECT_FALSE(IbCastOffloadEnabled); + EXPECT_TRUE(IbCastAinicCtsInlineData); +} + +TEST_F(NetIbCastInitAinicMicrotest, InitDevices_AinicWithQpScheduler_DropsOffloadAndInline) { + ScopedHook sched(g_rcclUseIbCastQpSched, [] { return true; }); + params_.Set("IB_USE_INLINE", 1); + ASSERT_EQ(Init(), ncclSuccess); + EXPECT_FALSE(IbCastOffloadEnabled); + EXPECT_FALSE(IbCastUseInline); + EXPECT_FALSE(IbCastAinicCtsInlineData); +} + +TEST_F(NetIbCastInitAinicMicrotest, InitDevices_AinicWithQpSharing_DropsOffloadOnly) { + params_.Set("RCCL_IB_QP_SHARING_ENABLE", 1); + ASSERT_EQ(Init(), ncclSuccess); + EXPECT_TRUE(IbCastQpSharingGlobalEnable); + EXPECT_FALSE(IbCastOffloadEnabled); + EXPECT_TRUE(IbCastUseInline); +} + +TEST_F(NetIbCastInitAinicMicrotest, InitDevices_AinicWithMapFile_EnablesMultiplane) { + SetMicroEnv("RCCL_MULTIPLANE_MAP_FILE", "/etc/multiplane.xml"); + ASSERT_EQ(Init(), ncclSuccess); + EXPECT_TRUE(IbCastMultiplaneEnable); +} + +TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_MapFileWithoutAinicOrEmpty_LeavesMultiplaneOff) { + World().AddHca("mlx5_0"); + SetMicroEnv("RCCL_MULTIPLANE_MAP_FILE", "/etc/multiplane.xml"); + ASSERT_EQ(Init(), ncclSuccess); + EXPECT_FALSE(IbCastMultiplaneEnable); + ResetForAnotherInit(); + g_rcclUseAinicValue = true; + SetMicroEnv("RCCL_MULTIPLANE_MAP_FILE", ""); + ASSERT_EQ(Init(), ncclSuccess); + EXPECT_FALSE(IbCastMultiplaneEnable); +} + +TEST_F(NetIbCastInitAinicMicrotest, InitDevices_UdmaPolicyOutOfRange_FallsBackToChannelAffinity) { + for (int64_t policy : {(int64_t)ncclIbCastUDMAPolicy::First - 1, (int64_t)ncclIbCastUDMAPolicy::Last + 1}) { + ResetForAnotherInit(); + params_.Set("RCCL_IB_UDMA_POLICY", policy); + ScopedDebugLogging info(NCCL_LOG_INFO, NCCL_ALL); + std::string log = CaptureLog([&] { ASSERT_EQ(Init(), ncclSuccess); }); + EXPECT_TRUE(LogHas(log, ("RCCL_IB_UDMA_POLICY=" + std::to_string(policy) + " is invalid").c_str())) << log; + EXPECT_TRUE(LogHas(log, "AINIC uDMA policy: 0")) << log; + } + ResetForAnotherInit(); + params_.Set("RCCL_IB_UDMA_POLICY", (int64_t)ncclIbCastUDMAPolicy::Last); + ScopedDebugLogging info(NCCL_LOG_INFO, NCCL_ALL); + std::string log = CaptureLog([&] { ASSERT_EQ(Init(), ncclSuccess); }); + EXPECT_TRUE(LogHas(log, "AINIC uDMA policy: 3")) << log; +} + +TEST_F(NetIbCastInitAinicMicrotest, InitDevices_AinicWithResiliency_DropsOffloadAndInline) { + params_.Set("IB_RESILIENCY_PORT_RECOVERY", 1); + ASSERT_EQ(Init(), ncclSuccess); + EXPECT_FALSE(IbCastOffloadEnabled); + EXPECT_FALSE(IbCastUseInline); + EXPECT_FALSE(IbCastAinicCtsInlineData); +} + +TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_NotAinic_LeavesOffloadOffAndReadsInline) { + World().AddHca("mlx5_0"); + params_.Set("IB_USE_INLINE", 1); + ASSERT_EQ(Init(), ncclSuccess); + EXPECT_FALSE(IbCastAinicRoce); + EXPECT_FALSE(IbCastOffloadEnabled); + EXPECT_TRUE(IbCastUseInline); + EXPECT_FALSE(IbCastAinicCtsInlineData); +} + +TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_SchedulerParamsFail_ReturnsTheError) { + World(); + ScopedHook sched(g_ibCastQpSchedInitParms, [](struct ncclIbQpSchedParms*) { return ncclInvalidUsage; }); + EXPECT_EQ(Init(), ncclInvalidUsage); +} + +TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_SchedulerWithResiliency_IsTurnedOff) { + World(); + ScopedHook sched(g_ibCastQpSchedInitParms, [](struct ncclIbQpSchedParms* p) { + p->enable = true; + return ncclSuccess; + }); + ASSERT_EQ(Init(), ncclSuccess); + EXPECT_TRUE(castGlobalQpSchedParms.enable); + ResetForAnotherInit(); + params_.Set("IB_RESILIENCY_PORT_FAILOVER", 1); + ASSERT_EQ(Init(), ncclSuccess); + EXPECT_FALSE(castGlobalQpSchedParms.enable); + ResetForAnotherInit(); + params_.Set("IB_RESILIENCY_PORT_FAILOVER", 0); + params_.Set("IB_RESILIENCY_PORT_RECOVERY", 1); + ASSERT_EQ(Init(), ncclSuccess); + EXPECT_FALSE(castGlobalQpSchedParms.enable); +} + +TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_QpSharing_TurnsTheSchedulerOff) { + World(); + params_.Set("RCCL_IB_QP_SHARING_ENABLE", 1); + ScopedHook sched(g_ibCastQpSchedInitParms, [](struct ncclIbQpSchedParms* p) { + p->enable = true; + return ncclSuccess; + }); + ASSERT_EQ(Init(), ncclSuccess); + EXPECT_TRUE(IbCastQpSharingGlobalEnable); + EXPECT_FALSE(castGlobalQpSchedParms.enable); +} + +TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_QpSharingWithNoGroups_TurnsSharingOff) { + World(); + params_.Set("RCCL_IB_QP_SHARING_ENABLE", 1); + params_.Set("RCCL_IB_COMM_NGROUPS", 0); + std::string log = CaptureLog([&] { ASSERT_EQ(Init(), ncclSuccess); }); + EXPECT_FALSE(IbCastQpSharingGlobalEnable); + EXPECT_TRUE(LogHas(log, "RCCL_IB_COMM_NGROUPS=0 is invalid")) << log; +} + +TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_ByOrderWithFailoverOrOooRq_WarnsAboutWhatItDisables) { + World().AddHca("ionic_0"); + ScopedHook byOrder(g_ibCastByOrderRequested, [] { return true; }); + params_.Set("IB_RESILIENCY_PORT_FAILOVER", 1); + params_.Set("IB_OOO_RQ", 1); + std::string log = CaptureLog([&] { ASSERT_EQ(Init(), ncclSuccess); }); + EXPECT_TRUE(LogHas(log, "disabling resiliency (port failover and port recovery)")) << log; + EXPECT_TRUE(LogHas(log, "disabling out-of-order RQ")) << log; + EXPECT_EQ(g_ibCastReportMatchingSchemeCalls, 1); +} + +TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_ByOrderAlone_OnlyReportsTheScheme) { + World().AddHca("ionic_0"); + ScopedHook byOrder(g_ibCastByOrderRequested, [] { return true; }); + std::string log = CaptureLog([&] { ASSERT_EQ(Init(), ncclSuccess); }); + EXPECT_FALSE(LogHas(log, "BY_ORDER")) << log; + EXPECT_EQ(g_ibCastReportMatchingSchemeCalls, 1); +} + +class NetIbCastInitMicrotestIsolated : public NetIbCastInitMicrotest {}; + +TEST_F(NetIbCastInitMicrotestIsolated, InitDevices_IbHca_IsAnnouncedOncePerProcess) { + RUN_ISOLATED_TEST("NetIbCastInit_IbHcaAnnouncedOnce", []() { + FakeIbWorld world; + SetMicroEnv("NCCL_IB_HCA", "ionic_0"); + ScopedDebugLogging info(NCCL_LOG_INFO, NCCL_ALL); + std::string log = CaptureLog([&] { ASSERT_EQ(IbCastInitDevices(nullptr, nullptr), ncclSuccess); }); + EXPECT_TRUE(LogHas(log, "NCCL_IB_HCA set to ionic_0")) << log; + world.Quiesce(); + NetIbCastResetGlobals(); + log = CaptureLog([&] { ASSERT_EQ(IbCastInitDevices(nullptr, nullptr), ncclSuccess); }); + EXPECT_FALSE(LogHas(log, "NCCL_IB_HCA set to")) << log; + }); +} + +TEST_F(NetIbCastInitMicrotestIsolated, InitDevices_TelemetryCannotStart_CarriesOnWithoutIt) { + RUN_ISOLATED_TEST_WITH_ENV( + "NetIbCastInit_TelemetryCannotStart", + []() { + FakeIbWorld world; + ScopedDebugLogging info(NCCL_LOG_INFO, NCCL_ALL); + std::string log = CaptureLog([&] { EXPECT_EQ(IbCastInitDevices(nullptr, nullptr), ncclSuccess); }); + EXPECT_TRUE(LogHas(log, "telemetry was requested but could not start")) << log; + }, + {{"RCCL_TELEMETRY_ENABLE", "1"}, {"RCCL_TELEMETRY_OUTPUT_DIR", std::string(5000, 'd')}}); +} + +TEST_F(NetIbCastInitMicrotestIsolated, InitDevices_TelemetryOn_RegistersEachDeviceUnderItsSortedIndex) { + RUN_ISOLATED_TEST_WITH_ENV( + "NetIbCastInit_TelemetryOn", + []() { + FakeIbWorld world; + world.AddHca("ionic_0"); + ScopedDebugLogging info(NCCL_LOG_INFO, NCCL_ALL); + std::string log = CaptureLog([&] { EXPECT_EQ(IbCastInitDevices(nullptr, nullptr), ncclSuccess); }); + EXPECT_TRUE(rcclTelemetryOn()); + EXPECT_EQ(rcclTelemetryDevs[0].device_id, 0); + EXPECT_FALSE(LogHas(log, "telemetry did not register")) << log; + }, + {{"RCCL_TELEMETRY_ENABLE", "1"}, {"RCCL_TELEMETRY_OUTPUT_DIR", "/tmp"}}); +} + + +TEST_F(NetIbCastInitDevicesMicrotest, Init_Success_HandsBackAConfigWithTheTrafficClass) { + World().AddHca("ionic_0"); + ncclNetCommConfig_t config{}; + config.trafficClass = 106; + void* ctx = nullptr; + ASSERT_EQ(IbCastInit(&ctx, 0, &config, nullptr, nullptr), ncclSuccess); + ASSERT_NE(ctx, nullptr); + EXPECT_EQ(static_cast(ctx)->trafficClass, 106); + EXPECT_EQ(World().recoveryStart.calls, 1); + int ndev = -1; + ASSERT_EQ(IbCastDevices(&ndev), ncclSuccess); + EXPECT_EQ(ndev, 1); + ASSERT_EQ(IbCastFinalize(ctx), ncclSuccess); + EXPECT_EQ(World().recoveryStop.calls, 1); +} + +TEST_F(NetIbCastInitDevicesMicrotest, Init_DeviceInitFails_ReturnsWithoutStartingRecovery) { + World(); + params_.Set("IB_DISABLE", 1); + void* ctx = nullptr; + ncclNetCommConfig_t config{}; + EXPECT_EQ(IbCastInit(&ctx, 0, &config, nullptr, nullptr), ncclInternalError); + EXPECT_EQ(World().recoveryStart.calls, 0); + EXPECT_EQ(ctx, nullptr); +} + +TEST_F(NetIbCastInitDevicesMicrotest, Init_RecoveryThreadFails_ReleasesTheDevicesReference) { + World().AddHca("ionic_0"); + ScopedHook start(g_ibCastPortRecoveryThreadStart, [] { return ncclSystemError; }); + void* ctx = nullptr; + ncclNetCommConfig_t config{}; + EXPECT_EQ(IbCastInit(&ctx, 0, &config, nullptr, nullptr), ncclSystemError); + EXPECT_EQ(netRefCount, 0); + EXPECT_EQ(ctx, nullptr); +} + +TEST_F(NetIbCastInitDevicesMicrotest, Finalize_RecoveryStopFails_KeepsTheReference) { + World().AddHca("ionic_0"); + ncclNetCommConfig_t config{}; + void* ctx = nullptr; + ASSERT_EQ(IbCastInit(&ctx, 0, &config, nullptr, nullptr), ncclSuccess); + ScopedHook stop(g_ibCastPortRecoveryThreadStop, [] { return ncclSystemError; }); + EXPECT_EQ(IbCastFinalize(ctx), ncclSystemError); + EXPECT_EQ(netRefCount, 1); +} + +class NetIbCastInitPropertiesMicrotest : public NetIbCastInitMicrotest { + protected: + void SetUp() override { + IbCastNMergedDevs = 0; + struct ncclIbDev& dev = Dev(0, "mlx5_0"); + dev.guid = 0xabc; + dev.maxQp = 77; + dev.portNum = 1; + dev.realPort = 1; + dev.railId = 2; + dev.planeId = 3; + int d; + ncclNetVDeviceProps_t props = Props({0}); + ASSERT_EQ(IbCastMakeVDevice(&d, &props), ncclSuccess); + } + // GDR is latched once per process; every test in this binary that reaches it sees peermem. + ScopedHook peermem_{g_ncclIbScanDefaultPeerMemClients, [] { return 1; }}; +}; + +TEST_F(NetIbCastInitPropertiesMicrotest, GetProperties_ReportsTheDeviceAndItsLimits) { + ncclNetProperties_t props; + memset(&props, 0, sizeof(props)); + ASSERT_EQ(IbCastGetProperties(0, &props), ncclSuccess); + EXPECT_STREQ(props.name, "mlx5_0"); + EXPECT_EQ(props.speed, 100000); + EXPECT_EQ(props.guid, 0xabcu); + EXPECT_EQ(props.ptrSupport & (NCCL_PTR_HOST | NCCL_PTR_CUDA), NCCL_PTR_HOST | NCCL_PTR_CUDA); + EXPECT_EQ(props.regIsGlobal, 1); + EXPECT_EQ(props.forceFlush, 0); + EXPECT_EQ(props.port, 2); // ibv port 1 plus real port 1 + EXPECT_EQ(props.maxComms, 77); + EXPECT_EQ(props.maxRecvs, NCCL_NET_IB_MAX_RECVS); + EXPECT_EQ(props.netDeviceType, NCCL_NET_DEVICE_HOST); + EXPECT_EQ(props.netDeviceVersion, NCCL_NET_DEVICE_INVALID_VERSION); + EXPECT_EQ(props.maxP2pBytes, NCCL_MAX_NET_SIZE_BYTES); + EXPECT_EQ(props.maxCollBytes, (size_t)MAX_COLLNET_SIZE); + EXPECT_EQ(props.maxMultiRequestSize, 1); + EXPECT_EQ(props.railId, 2); + EXPECT_EQ(props.planeId, 3); + EXPECT_EQ(props.vProps.ndevs, 1); +} + +TEST_F(NetIbCastInitPropertiesMicrotest, GetProperties_DataDirectAndCtsOffload_ChangeFlushAndRecvs) { + IbCastDevs[0].capsProvider.mlx5.dataDirect = 1; + IbCastOffloadEnabled = true; + params_.Set("RCCL_IB_P2P_DISABLE_CTS", 0); + ncclNetProperties_t props; + ASSERT_EQ(IbCastGetProperties(0, &props), ncclSuccess); + EXPECT_EQ(props.forceFlush, 1); + EXPECT_EQ(props.maxRecvs, 1); + params_.Set("RCCL_IB_P2P_DISABLE_CTS", 1); // offload without CTS: the full count again + ASSERT_EQ(IbCastGetProperties(0, &props), ncclSuccess); + EXPECT_EQ(props.maxRecvs, NCCL_NET_IB_MAX_RECVS); +} + +TEST_F(NetIbCastInitPropertiesMicrotest, GetProperties_MergedDevice_ReportsTheFusedNameSpeedAndIds) { + Dev(1, "mlx5_1"); + int d; + ncclNetVDeviceProps_t props = Props({0, 1}); + ASSERT_EQ(IbCastMakeVDevice(&d, &props), ncclSuccess); + ncclNetProperties_t out; + ASSERT_EQ(IbCastGetProperties(d, &out), ncclSuccess); + EXPECT_STREQ(out.name, "mlx5_0+mlx5_1"); + EXPECT_EQ(out.speed, 200000); + EXPECT_EQ(out.railId, NCCL_NET_ID_UNDEF); + EXPECT_EQ(out.vProps.ndevs, 2); +} + +TEST_F(NetIbCastInitPropertiesMicrotest, GetProperties_UnknownVirtualDevice_IsRejected) { + ncclNetProperties_t props; + std::string log = CaptureLog([&] { EXPECT_EQ(IbCastGetProperties(1, &props), ncclInvalidUsage); }); + EXPECT_TRUE(LogHas(log, "Requested properties for vNic 1, only 1 vNics have been created")) << log; +} + +class NetIbCastInitPropertiesMicrotestIsolated : public NetIbCastInitMicrotest {}; + +TEST_F(NetIbCastInitPropertiesMicrotestIsolated, GetPhysProperties_GdrAndDmaBuf_AddTheirPointerTypes) { + RUN_ISOLATED_TEST("NetIbCastInit_PhysPropertiesDmaBuf", []() { + struct ibv_context ctx {}; + struct ibv_pd pd {}; + IbCastNDevs = 1; + IbCastNMergedDevs = 1; + IbCastDevs[0].context = &ctx; + ScopedHook peermem(g_ncclIbScanDefaultPeerMemClients, [] { return 0; }); + ScopedHook probe(g_ncclIbProbeGdrSupport, [](struct ibv_context*, int) { return 0; }); + ScopedHook alloc(g_ibvAllocPd, [&](struct ibv_pd** ret, struct ibv_context*) { + *ret = &pd; + return ncclSuccess; + }); + ScopedHook reg(g_ibvDirectRegDmabufMr, [](struct ibv_pd*, uint64_t, size_t, uint64_t, int, int) { + errno = EBADF; + return (struct ibv_mr*)nullptr; + }); + ncclNetProperties_t props; + ASSERT_EQ(IbCastGetPhysProperties(0, &props), ncclSuccess); + EXPECT_EQ(props.ptrSupport, NCCL_PTR_HOST | NCCL_PTR_DMABUF); + }); +} + +} // namespace diff --git a/projects/rccl/test/host/net-ib-cast-multiplane-test.cc b/projects/rccl/test/host/net-ib-cast-multiplane-test.cc new file mode 100644 index 000000000000..4a13773c853d --- /dev/null +++ b/projects/rccl/test/host/net-ib-cast-multiplane-test.cc @@ -0,0 +1,296 @@ +/************************************************************************* + * Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +// Host-only microtests for src/transport/net_ib_cast/multiplane.cc +// (NET_IB_CAST_MULTIPLANE_CC_PATH): the VIP-to-PIP map file, parsed by the real xml.cc. + +#include + +#include +#include + +#include +#include +#include +#include + +#include "../common/LogCapture.hpp" +#include "fakes/env_fakes.h" +#include "net-ib-cast-test-common.h" + +#include NET_IB_CAST_MULTIPLANE_CC_PATH + +namespace { + +using RcclUnitTesting::CaptureLog; +using RcclUnitTesting::LogHas; +using RcclUnitTesting::ScopedDebugLogging; + +class NetIbCastMultiplaneMicrotest : public ::testing::Test { + protected: + void SetUp() override { + char tmpl[] = "/tmp/net-ib-cast-multiplane-XXXXXX"; + ASSERT_NE(mkdtemp(tmpl), nullptr); + dir_ = tmpl; + } + void TearDown() override { + ncclIbCastTestMultiplaneReset(); + ResetEnvFakes(); + std::filesystem::remove_all(dir_); + } + + // Writes the map file and points RCCL_MULTIPLANE_MAP_FILE at it. + void UseMap(const std::string& xml) { + const std::string path = dir_ + "/map.xml"; + std::ofstream(path) << xml; + SetMicroEnv("RCCL_MULTIPLANE_MAP_FILE", path.c_str()); + } + + static union ibv_gid Gid(const char* ip) { + union ibv_gid gid; + EXPECT_EQ(ibCastIpToGid(ip, &gid), ncclSuccess) << ip; + return gid; + } + + static bool SameGid(const union ibv_gid& gid, const char* ip) { + union ibv_gid expected = Gid(ip); + return memcmp(&gid, &expected, sizeof(gid)) == 0; + } + + int Resolve(const char* vip, union ibv_gid* pips) { + union ibv_gid vipGid = Gid(vip); + int nPips = -1; + EXPECT_EQ(IbCastMultiplaneGetPipGids(&vipGid, pips, &nPips), ncclSuccess); + return nPips; + } + + std::string dir_; +}; + +constexpr const char* kTwoInterfaces = + "\n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + "\n"; + +TEST_F(NetIbCastMultiplaneMicrotest, IpToGid_Ipv6_CopiesTheAddress) { + struct in6_addr expected; + ASSERT_EQ(inet_pton(AF_INET6, "fe80::1:2", &expected), 1); + uint8_t gid[16]; + ASSERT_EQ(ncclIbCastTestIpToGid("fe80::1:2", gid), ncclSuccess); + EXPECT_EQ(memcmp(gid, &expected, sizeof(gid)), 0); +} + +TEST_F(NetIbCastMultiplaneMicrotest, IpToGid_Ipv4_StoresTheV4MappedForm) { + uint8_t gid[16]; + ASSERT_EQ(ncclIbCastTestIpToGid("10.1.2.3", gid), ncclSuccess); + const uint8_t expected[16] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff, 10, 1, 2, 3}; + EXPECT_EQ(memcmp(gid, expected, sizeof(gid)), 0); +} + +TEST_F(NetIbCastMultiplaneMicrotest, IpToGid_NotAnAddress_IsRejected) { + uint8_t gid[16]; + memset(gid, 0xab, sizeof(gid)); + std::string log = CaptureLog([&] { EXPECT_EQ(ncclIbCastTestIpToGid("10.1.2", gid), ncclInvalidArgument); }); + EXPECT_TRUE(LogHas(log, "invalid IP address '10.1.2'")) << log; + EXPECT_EQ(gid[0], 0xab); // left alone on failure +} + +TEST_F(NetIbCastMultiplaneMicrotest, GidToString_PrintsEightLowercaseGroups) { + uint8_t gid[16]; + for (int i = 0; i < 16; i++) gid[i] = (uint8_t)(0xA0 + i); + char buf[64]; + ncclIbCastTestGidToString(gid, buf, sizeof(buf)); + EXPECT_STREQ(buf, "a0a1:a2a3:a4a5:a6a7:a8a9:aaab:acad:aeaf"); +} + +TEST_F(NetIbCastMultiplaneMicrotest, Load_NoMapFile_SucceedsAndResolvesNothing) { + ASSERT_EQ(IbCastMultiplaneLoad(), ncclSuccess); + union ibv_gid pips[MULTIPLANE_MAX_PIPS]; + EXPECT_EQ(Resolve("fd00::1", pips), 0); +} + +TEST_F(NetIbCastMultiplaneMicrotest, Load_EmptyMapFileName_IsTreatedAsUnset) { + SetMicroEnv("RCCL_MULTIPLANE_MAP_FILE", ""); + EXPECT_EQ(IbCastMultiplaneLoad(), ncclSuccess); + union ibv_gid pips[MULTIPLANE_MAX_PIPS]; + EXPECT_EQ(Resolve("fd00::1", pips), 0); +} + +TEST_F(NetIbCastMultiplaneMicrotest, Load_MissingFile_FailsAndKeepsFailingWithoutRereading) { + const std::string path = dir_ + "/absent.xml"; + SetMicroEnv("RCCL_MULTIPLANE_MAP_FILE", path.c_str()); + std::string log = CaptureLog([&] { EXPECT_EQ(IbCastMultiplaneLoad(), ncclSystemError); }); + EXPECT_TRUE(LogHas(log, "could not open map file")) << log; + // The load runs once: creating the file afterwards changes nothing. + std::ofstream(path) << kTwoInterfaces; + EXPECT_EQ(IbCastMultiplaneLoad(), ncclSystemError); +} + +TEST_F(NetIbCastMultiplaneMicrotest, Load_ValidMap_ResolvesEachVipToItsPipsInOrder) { + UseMap(kTwoInterfaces); + ScopedDebugLogging info(NCCL_LOG_INFO, NCCL_ALL); + std::string log = CaptureLog([&] { ASSERT_EQ(IbCastMultiplaneLoad(), ncclSuccess); }); + EXPECT_TRUE(LogHas(log, "loaded 2 GID mappings")) << log; + EXPECT_FALSE(LogHas(log, "duplicate")) << log; + EXPECT_TRUE(LogHas(log, "ip=10.0.0.1 interface=eth1")) << log; + + union ibv_gid pips[MULTIPLANE_MAX_PIPS]; + ASSERT_EQ(Resolve("fd00::1", pips), 2); + EXPECT_TRUE(SameGid(pips[0], "10.0.0.1")); + EXPECT_TRUE(SameGid(pips[1], "10.0.1.1")); + ASSERT_EQ(Resolve("192.168.7.9", pips), 1); + EXPECT_TRUE(SameGid(pips[0], "fd01::5")); +} + +TEST_F(NetIbCastMultiplaneMicrotest, Load_VipKeyIsCanonicalized_LongFormAndUppercaseMatch) { + UseMap("" + ""); + ASSERT_EQ(IbCastMultiplaneLoad(), ncclSuccess); + union ibv_gid pips[MULTIPLANE_MAX_PIPS]; + EXPECT_EQ(Resolve("fd00::1", pips), 1); +} + +TEST_F(NetIbCastMultiplaneMicrotest, GetPipGids_UnknownVip_ResolvesNothing) { + UseMap(kTwoInterfaces); + ASSERT_EQ(IbCastMultiplaneLoad(), ncclSuccess); + union ibv_gid pips[MULTIPLANE_MAX_PIPS]; + EXPECT_EQ(Resolve("fd00::2", pips), 0); +} + +TEST_F(NetIbCastMultiplaneMicrotest, GetPipGids_MorePipsThanTheMax_ReturnsOnlyTheFirstMax) { + std::string xml = ""; + for (int i = 0; i <= MULTIPLANE_MAX_PIPS; i++) xml += ""; + xml += ""; + UseMap(xml); + ASSERT_EQ(IbCastMultiplaneLoad(), ncclSuccess); + union ibv_gid pips[MULTIPLANE_MAX_PIPS + 1]; + memset(pips, 0, sizeof(pips)); + ASSERT_EQ(Resolve("fd00::1", pips), MULTIPLANE_MAX_PIPS); + const std::string last = "10.0.0." + std::to_string(MULTIPLANE_MAX_PIPS); + EXPECT_TRUE(SameGid(pips[MULTIPLANE_MAX_PIPS - 1], last.c_str())); + union ibv_gid zero; + memset(&zero, 0, sizeof(zero)); + EXPECT_EQ(memcmp(&pips[MULTIPLANE_MAX_PIPS], &zero, sizeof(zero)), 0); +} + +TEST_F(NetIbCastMultiplaneMicrotest, Load_PipWithoutInterface_LeavesTheInterfaceEmpty) { + UseMap(""); + ScopedDebugLogging info(NCCL_LOG_INFO, NCCL_ALL); + std::string log = CaptureLog([&] { ASSERT_EQ(IbCastMultiplaneLoad(), ncclSuccess); }); + EXPECT_TRUE(LogHas(log, "ip=10.0.0.1 interface=\n")) << log; +} + +TEST_F(NetIbCastMultiplaneMicrotest, Load_DuplicateVip_LastEntryWins) { + UseMap("" + "" + "" + ""); + std::string log = CaptureLog([&] { ASSERT_EQ(IbCastMultiplaneLoad(), ncclSuccess); }); + EXPECT_TRUE(LogHas(log, "duplicate GID")) << log; + union ibv_gid pips[MULTIPLANE_MAX_PIPS]; + ASSERT_EQ(Resolve("fd00::1", pips), 2); + EXPECT_TRUE(SameGid(pips[0], "10.0.0.2")); +} + +struct BadMap { + const char* name; + const char* xml; + const char* warning; +}; + +class NetIbCastMultiplaneBadMapMicrotest : public NetIbCastMultiplaneMicrotest, + public ::testing::WithParamInterface {}; + +TEST_P(NetIbCastMultiplaneBadMapMicrotest, Load_IsRejectedAndNothingResolves) { + UseMap(GetParam().xml); + std::string log = CaptureLog([&] { EXPECT_NE(IbCastMultiplaneLoad(), ncclSuccess); }); + EXPECT_TRUE(LogHas(log, GetParam().warning)) << log; + union ibv_gid pips[MULTIPLANE_MAX_PIPS]; + EXPECT_EQ(Resolve("fd00::1", pips), 0); +} + +INSTANTIATE_TEST_SUITE_P( + NetIbCastMultiplane, NetIbCastMultiplaneBadMapMicrotest, + ::testing::Values( + BadMap{"NoRoot", "", "no root element"}, + BadMap{"InterfaceWithoutGid", + "", + " missing 'gid'"}, + BadMap{"PipWithoutIp", + "" + "", + " missing 'ip'"}, + BadMap{"BadPipIp", + "" + "", + "invalid IP address '10.0.0.300'"}, + BadMap{"BadVip", + "" + "", + "invalid IP address 'fd00::g'"}, + BadMap{"Unterminated", "", "failed to parse map file"}), + [](const ::testing::TestParamInfo& info) { return std::string(info.param.name); }); + +// The parser only hands back the elements it has handlers for, so the "skip anything else" +// branches of the tree walk are driven with a hand-built tree. +class NetIbCastMultiplaneTreeMicrotest : public NetIbCastMultiplaneMicrotest { + protected: + struct ncclXmlNode* Add(const char* name, struct ncclXmlNode* parent) { + struct ncclXmlNode* node = &xml_->nodes[xml_->maxIndex++]; + strcpy(node->name, name); + if (parent) parent->subs[parent->nSubs++] = node; + return node; + } + void SetUp() override { + NetIbCastMultiplaneMicrotest::SetUp(); + ASSERT_EQ(xmlAlloc(&xml_, 16), ncclSuccess); + } + void TearDown() override { + free(xml_); + NetIbCastMultiplaneMicrotest::TearDown(); + } + struct ncclXml* xml_ = nullptr; +}; + +TEST_F(NetIbCastMultiplaneTreeMicrotest, PopulateMap_SkipsElementsThatAreNotHostInterfaceOrPip) { + // Each stray element holds what would fail the load if it were walked into. + struct ncclXmlNode* root = Add("multiplane", nullptr); + struct ncclXmlNode* comment = Add("comment", root); + Add("interface", comment); // no gid + struct ncclXmlNode* host = Add("host", root); + struct ncclXmlNode* note = Add("note", host); + Add("pip", note); // no ip + struct ncclXmlNode* iface = Add("interface", host); + ASSERT_EQ(xmlSetAttr(iface, "gid", "fd00::1"), ncclSuccess); + struct ncclXmlNode* label = Add("label", iface); + ASSERT_EQ(xmlSetAttr(label, "ip", "not-an-ip"), ncclSuccess); + struct ncclXmlNode* pip = Add("pip", iface); + ASSERT_EQ(xmlSetAttr(pip, "ip", "10.0.0.1"), ncclSuccess); + + ASSERT_EQ(ibCastMultiplanePopulateMap(xml_), ncclSuccess); + ASSERT_EQ(gidToPipMap.size(), 1u); + EXPECT_EQ(gidToPipMap.begin()->second.size(), 1u); +} + +TEST_F(NetIbCastMultiplaneMicrotest, Enabled_ReportsTheInitTimeFlagAndResetClearsIt) { + bool enabled = false; + IbCastMultiplaneEnable = true; + ASSERT_EQ(IbCastMultiplaneEnabled(&enabled), ncclSuccess); + EXPECT_TRUE(enabled); + ncclIbCastTestMultiplaneReset(); + ASSERT_EQ(IbCastMultiplaneEnabled(&enabled), ncclSuccess); + EXPECT_FALSE(enabled); +} + +} // namespace diff --git a/projects/rccl/test/host/net-ib-cast-reg-test.cc b/projects/rccl/test/host/net-ib-cast-reg-test.cc new file mode 100644 index 000000000000..49d6fe7f21f8 --- /dev/null +++ b/projects/rccl/test/host/net-ib-cast-reg-test.cc @@ -0,0 +1,333 @@ +/************************************************************************* + * Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +// Host-only microtests for src/transport/net_ib_cast/reg.cc (NET_IB_CAST_REG_CC_PATH): memory +// registration through the per-device MR cache, over the shared verbs fakes. + +#include + +#include + +#include + +#include "../common/LogCapture.hpp" +#include "fakes/ibvwrap_fakes.h" +#include "fakes/libc_fakes.h" +#include "fakes/mlx5dvwrap_fakes.h" +#include "net-ib-cast-test-common.h" + +#include "fakes/libc_seam.h" +#include NET_IB_CAST_REG_CC_PATH +#include "fakes/libc_seam_undef.h" + +namespace { + +using RcclUnitTesting::CaptureLog; +using RcclUnitTesting::LogHas; + +constexpr unsigned int kBaseAccess = + IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ | IBV_ACCESS_REMOTE_ATOMIC; + +struct RegCall { + void* addr; + size_t length; + uint64_t iova; + uint64_t offset; + int fd; + int access; + int mlx5Access; +}; + +class NetIbCastRegMicrotest : public ::testing::Test { + protected: + void SetUp() override { + pageSize_ = sysconf(_SC_PAGESIZE); + buf_ = static_cast(aligned_alloc(pageSize_, 64 * pageSize_)); + send_->base.isSend = true; + send_->base.vProps.ndevs = 1; + send_->devs[0].base.ibDevN = 0; + send_->devs[0].base.pd = &pd_; + } + void TearDown() override { + NetIbCastResetGlobals(); + ResetIbvwrapFakes(); + ResetMlx5dvwrapFakes(); + ResetLibcFakes(); + free(buf_); + } + + // Hands out a distinct MR per registration and records what was asked for. + struct ibv_mr* NextMr(void* addr, size_t length, uint64_t iova, uint64_t offset, int fd, int access, int mlx5) { + calls_.push_back({addr, length, iova, offset, fd, access, mlx5}); + mrs_.emplace_back(new ibv_mr{}); + mrs_.back()->lkey = 0x100 + (uint32_t)mrs_.size(); + mrs_.back()->rkey = 0x200 + (uint32_t)mrs_.size(); + return mrs_.back().get(); + } + + struct ncclIbMrCache& Cache(int dev = 0) { return IbCastDevs[dev].mrCache; } + ibv_mr* MrOf(void* mhandle, int dev = 0) { return static_cast(mhandle)->mrs[dev]; } + + size_t pageSize_ = 0; + char* buf_ = nullptr; + struct ibv_pd pd_ {}; + ZeroedPtr send_; + std::vector calls_; + std::vector> mrs_; + ScopedHook regMr_{ + g_ibvRegMr, [this](struct ibv_mr** ret, struct ibv_pd*, void* addr, size_t length, int access) { + *ret = NextMr(addr, length, 0, 0, -1, access, 0); + return ncclSuccess; + }}; +}; + +TEST_F(NetIbCastRegMicrotest, RegMr_RejectsNullCommNullHandleAndZeroSize) { + void* mhandle = nullptr; + std::string log = CaptureLog([&] { + EXPECT_EQ(IbCastRegMr(nullptr, buf_, 8, NCCL_PTR_HOST, &mhandle), ncclInvalidArgument); + EXPECT_EQ(IbCastRegMr(send_.get(), buf_, 8, NCCL_PTR_HOST, nullptr), ncclInvalidArgument); + EXPECT_EQ(IbCastRegMr(send_.get(), buf_, 0, NCCL_PTR_HOST, &mhandle), ncclInvalidArgument); + }); + EXPECT_TRUE(LogHas(log, "regMr/regMrDmaBuf called with comm=")) << log; + EXPECT_EQ(mhandle, nullptr); +} + +TEST_F(NetIbCastRegMicrotest, RegMr_HandleAllocationFails_ReturnsSystemError) { + ScopedHook mallocFails(g_malloc, [](size_t) -> void* { return nullptr; }); + void* mhandle = nullptr; + EXPECT_EQ(IbCastRegMr(send_.get(), buf_, 8, NCCL_PTR_HOST, &mhandle), ncclSystemError); + EXPECT_EQ(mhandle, nullptr); +} + +TEST_F(NetIbCastRegMicrotest, RegMr_Host_RegistersWholePagesWithTheBaseAccessFlags) { + void* mhandle = nullptr; + // Starts mid-page and spills into the next one. + char* data = buf_ + pageSize_ + 16; + ASSERT_EQ(IbCastRegMr(send_.get(), data, pageSize_, NCCL_PTR_HOST, &mhandle), ncclSuccess); + ASSERT_EQ(calls_.size(), 1u); + EXPECT_EQ(calls_[0].addr, buf_ + pageSize_); + EXPECT_EQ(calls_[0].length, 2 * pageSize_); + EXPECT_EQ(calls_[0].access, (int)kBaseAccess); + EXPECT_EQ(MrOf(mhandle), mrs_[0].get()); + ASSERT_EQ(Cache().population, 1); + EXPECT_EQ(Cache().slots[0].addr, (uintptr_t)(buf_ + pageSize_)); + EXPECT_EQ(Cache().slots[0].pages, 2u); + EXPECT_EQ(Cache().slots[0].refs, 1); + EXPECT_EQ(IbCastDeregMr(send_.get(), mhandle), ncclSuccess); +} + +TEST_F(NetIbCastRegMicrotest, RegMr_RelaxedOrdering_UsesIova2WithTheRelaxedFlag) { + IbCastRelaxedOrderingEnabled = 1; + ScopedHook iova2(g_ibvRegMrIova2, + [this](struct ibv_mr** ret, struct ibv_pd*, void* addr, size_t length, uint64_t iova, int access) { + *ret = NextMr(addr, length, iova, 0, -1, access, 0); + return ncclSuccess; + }); + void* mhandle = nullptr; + ASSERT_EQ(IbCastRegMr(send_.get(), buf_, 8, NCCL_PTR_HOST, &mhandle), ncclSuccess); + ASSERT_EQ(calls_.size(), 1u); + EXPECT_EQ(calls_[0].access, (int)(kBaseAccess | IBV_ACCESS_RELAXED_ORDERING)); + EXPECT_EQ(calls_[0].iova, (uint64_t)(uintptr_t)buf_); + EXPECT_EQ(IbCastDeregMr(send_.get(), mhandle), ncclSuccess); +} + +TEST_F(NetIbCastRegMicrotest, RegMr_ForceStrongOrdering_OverridesRelaxedOrdering) { + IbCastRelaxedOrderingEnabled = 1; + void* mhandle = nullptr; + ASSERT_EQ(IbCastRegMrDmaBufInternal(send_.get(), buf_, 8, NCCL_PTR_HOST, 0, -1, NCCL_NET_MR_FLAG_FORCE_SO, &mhandle), + ncclSuccess); + ASSERT_EQ(calls_.size(), 1u); + EXPECT_EQ(calls_[0].access, (int)kBaseAccess); + EXPECT_EQ(IbCastDeregMr(send_.get(), mhandle), ncclSuccess); +} + +TEST_F(NetIbCastRegMicrotest, RegMrDmaBuf_PassesTheFdOffsetAndIova) { + ScopedHook dmabuf(g_ibvRegDmabufMr, [this](struct ibv_mr** ret, struct ibv_pd*, uint64_t offset, size_t length, + uint64_t iova, int fd, int access) { + *ret = NextMr(nullptr, length, iova, offset, fd, access, 0); + return ncclSuccess; + }); + void* mhandle = nullptr; + ASSERT_EQ(IbCastRegMrDmaBuf(send_.get(), buf_, 8, NCCL_PTR_CUDA, 4096, 17, &mhandle), ncclSuccess); + ASSERT_EQ(calls_.size(), 1u); + EXPECT_EQ(calls_[0].fd, 17); + EXPECT_EQ(calls_[0].offset, 4096u); + EXPECT_EQ(calls_[0].iova, (uint64_t)(uintptr_t)buf_); + EXPECT_EQ(calls_[0].length, pageSize_); + EXPECT_EQ(calls_[0].access, (int)kBaseAccess); + EXPECT_EQ(IbCastDeregMr(send_.get(), mhandle), ncclSuccess); +} + +TEST_F(NetIbCastRegMicrotest, RegMrDmaBuf_DataDirectDevice_GoesThroughMlx5WithTheDataDirectFlag) { + IbCastDevs[0].capsProvider.mlx5.dataDirect = 1; + ScopedHook dmabuf(g_mlx5dvRegDmabufMr, [this](struct ibv_mr** ret, struct ibv_pd*, uint64_t offset, size_t length, + uint64_t iova, int fd, int access, int mlx5Access) { + *ret = NextMr(nullptr, length, iova, offset, fd, access, mlx5Access); + return ncclSuccess; + }); + void* mhandle = nullptr; + ASSERT_EQ(IbCastRegMrDmaBuf(send_.get(), buf_, 8, NCCL_PTR_CUDA, 0, 5, &mhandle), ncclSuccess); + ASSERT_EQ(calls_.size(), 1u); + EXPECT_EQ(calls_[0].mlx5Access, MLX5DV_REG_DMABUF_ACCESS_DATA_DIRECT); + EXPECT_EQ(calls_[0].fd, 5); + EXPECT_EQ(IbCastDeregMr(send_.get(), mhandle), ncclSuccess); +} + +TEST_F(NetIbCastRegMicrotest, RegMr_RangeInsideACachedOne_ReusesItsMr) { + void *outer = nullptr, *inner = nullptr; + ASSERT_EQ(IbCastRegMr(send_.get(), buf_, 4 * pageSize_, NCCL_PTR_HOST, &outer), ncclSuccess); + ASSERT_EQ(IbCastRegMr(send_.get(), buf_ + pageSize_, pageSize_, NCCL_PTR_HOST, &inner), ncclSuccess); + EXPECT_EQ(calls_.size(), 1u); + EXPECT_EQ(MrOf(inner), MrOf(outer)); + EXPECT_EQ(Cache().slots[0].refs, 2); + // The MR survives until its last user lets go. + int derefs = 0; + ScopedHook dereg(g_ibvDeregMr, [&](struct ibv_mr*) { + ++derefs; + return ncclSuccess; + }); + ASSERT_EQ(IbCastDeregMr(send_.get(), inner), ncclSuccess); + EXPECT_EQ(derefs, 0); + EXPECT_EQ(Cache().population, 1); + ASSERT_EQ(IbCastDeregMr(send_.get(), outer), ncclSuccess); + EXPECT_EQ(derefs, 1); + EXPECT_EQ(Cache().population, 0); + EXPECT_EQ(Cache().slots, nullptr); + EXPECT_EQ(Cache().capacity, 0); +} + +TEST_F(NetIbCastRegMicrotest, RegMr_RangeRunningPastACachedOne_RegistersAgain) { + void *first = nullptr, *second = nullptr; + ASSERT_EQ(IbCastRegMr(send_.get(), buf_, pageSize_, NCCL_PTR_HOST, &first), ncclSuccess); + ASSERT_EQ(IbCastRegMr(send_.get(), buf_, 2 * pageSize_, NCCL_PTR_HOST, &second), ncclSuccess); + EXPECT_EQ(calls_.size(), 2u); + EXPECT_NE(MrOf(first), MrOf(second)); + EXPECT_EQ(Cache().population, 2); + EXPECT_EQ(IbCastDeregMr(send_.get(), first), ncclSuccess); + EXPECT_EQ(IbCastDeregMr(send_.get(), second), ncclSuccess); +} + +TEST_F(NetIbCastRegMicrotest, RegMr_LowerAddress_IsInsertedInFrontToKeepTheCacheSorted) { + void *high = nullptr, *low = nullptr, *again = nullptr; + ASSERT_EQ(IbCastRegMr(send_.get(), buf_ + 8 * pageSize_, pageSize_, NCCL_PTR_HOST, &high), ncclSuccess); + ASSERT_EQ(IbCastRegMr(send_.get(), buf_, pageSize_, NCCL_PTR_HOST, &low), ncclSuccess); + ASSERT_EQ(Cache().population, 2); + EXPECT_EQ(Cache().slots[0].addr, (uintptr_t)buf_); + EXPECT_EQ(Cache().slots[1].addr, (uintptr_t)(buf_ + 8 * pageSize_)); + ASSERT_EQ(IbCastRegMr(send_.get(), buf_ + 8 * pageSize_, 8, NCCL_PTR_HOST, &again), ncclSuccess); + EXPECT_EQ(MrOf(again), MrOf(high)); + EXPECT_EQ(calls_.size(), 2u); + for (void* h : {high, low, again}) EXPECT_EQ(IbCastDeregMr(send_.get(), h), ncclSuccess); +} + +TEST_F(NetIbCastRegMicrotest, RegMr_CacheFull_GrowsFrom32ToDouble) { + std::vector handles(33); + for (int i = 0; i < 33; i++) { + ASSERT_EQ(IbCastRegMr(send_.get(), buf_ + i * pageSize_, 8, NCCL_PTR_HOST, &handles[i]), ncclSuccess); + EXPECT_EQ(Cache().capacity, i < 32 ? 32 : 64) << i; + } + for (void* h : handles) EXPECT_EQ(IbCastDeregMr(send_.get(), h), ncclSuccess); +} + +TEST_F(NetIbCastRegMicrotest, RegMr_RecvComm_UsesTheRecvDevicesBase) { + ZeroedPtr recv; + recv->base.isSend = false; + recv->base.vProps.ndevs = 1; + recv->devs[0].base.ibDevN = 1; + recv->devs[0].base.pd = &pd_; + void* mhandle = nullptr; + ASSERT_EQ(IbCastRegMr(recv.get(), buf_, 8, NCCL_PTR_HOST, &mhandle), ncclSuccess); + EXPECT_EQ(Cache(1).population, 1); + EXPECT_EQ(Cache(0).population, 0); + EXPECT_EQ(IbCastDeregMr(recv.get(), mhandle), ncclSuccess); +} + +TEST_F(NetIbCastRegMicrotest, RegMr_TwoDevices_RegistersOnEachDevice) { + send_->base.vProps.ndevs = 2; + send_->devs[1].base.ibDevN = 3; + send_->devs[1].base.pd = &pd_; + void* mhandle = nullptr; + ASSERT_EQ(IbCastRegMr(send_.get(), buf_, 8, NCCL_PTR_HOST, &mhandle), ncclSuccess); + EXPECT_EQ(calls_.size(), 2u); + EXPECT_EQ(MrOf(mhandle, 1), mrs_[1].get()); + EXPECT_EQ(Cache(3).population, 1); + EXPECT_EQ(IbCastDeregMr(send_.get(), mhandle), ncclSuccess); + EXPECT_EQ(Cache(3).population, 0); +} + +TEST_F(NetIbCastRegMicrotest, RegMr_RegistrationFails_FreesTheHandleAndReturnsTheError) { + ScopedHook regFails(g_ibvRegMr, [](struct ibv_mr**, struct ibv_pd*, void*, size_t, int) { return ncclSystemError; }); + void* wrapper = nullptr; + ScopedHook alloc(g_malloc, [&](size_t size) { return wrapper = malloc(size); }); + void* mhandle = nullptr; + EXPECT_EQ(IbCastRegMr(send_.get(), buf_, 8, NCCL_PTR_HOST, &mhandle), ncclSystemError); + EXPECT_EQ(mhandle, nullptr); + ASSERT_NE(wrapper, nullptr); + EXPECT_EQ(g_freedPointers.back(), wrapper); + EXPECT_EQ(Cache().population, 0); +} + +// Deregistering moves the last slot into the hole, so the cache is no longer sorted and the +// lookup's early exit misses a cached range. Pins AICOMRCCL-3092; a fix flips this pin. +TEST_F(NetIbCastRegMicrotest, DeregMr_KnownBug_CacheLosesItsOrderAndRegistersTwice) { + void *low = nullptr, *mid = nullptr, *high = nullptr, *again = nullptr; + ASSERT_EQ(IbCastRegMr(send_.get(), buf_, 8, NCCL_PTR_HOST, &low), ncclSuccess); + ASSERT_EQ(IbCastRegMr(send_.get(), buf_ + 2 * pageSize_, 8, NCCL_PTR_HOST, &mid), ncclSuccess); + ASSERT_EQ(IbCastRegMr(send_.get(), buf_ + 4 * pageSize_, 8, NCCL_PTR_HOST, &high), ncclSuccess); + ASSERT_EQ(IbCastDeregMr(send_.get(), low), ncclSuccess); + ASSERT_EQ(IbCastRegMr(send_.get(), buf_ + 2 * pageSize_, 8, NCCL_PTR_HOST, &again), ncclSuccess); + EXPECT_EQ(calls_.size(), 4u); + EXPECT_NE(MrOf(again), MrOf(mid)); + for (void* h : {mid, high, again}) EXPECT_EQ(IbCastDeregMr(send_.get(), h), ncclSuccess); +} + +// A failure on a later device returns without releasing the MRs the earlier devices took. +// Pins AICOMRCCL-3093; a fix flips this pin. +TEST_F(NetIbCastRegMicrotest, RegMr_KnownBug_FailureOnASecondDeviceKeepsTheFirstDevicesMr) { + send_->base.vProps.ndevs = 2; + send_->devs[1].base.ibDevN = 3; + send_->devs[1].base.pd = &pd_; + int regs = 0; + ScopedHook secondFails(g_ibvRegMr, [&](struct ibv_mr** ret, struct ibv_pd*, void* addr, size_t length, int access) { + if (++regs == 2) return ncclSystemError; + *ret = NextMr(addr, length, 0, 0, -1, access, 0); + return ncclSuccess; + }); + void* mhandle = nullptr; + EXPECT_EQ(IbCastRegMr(send_.get(), buf_, 8, NCCL_PTR_HOST, &mhandle), ncclSystemError); + EXPECT_EQ(Cache(0).population, 1); + EXPECT_EQ(Cache(0).slots[0].refs, 1); +} + +TEST_F(NetIbCastRegMicrotest, DeregMr_NullHandle_IsANoOpAndNullCommIsRejected) { + EXPECT_EQ(IbCastDeregMr(send_.get(), nullptr), ncclSuccess); + ncclIbMrHandle handle{}; + std::string log = CaptureLog([&] { EXPECT_EQ(IbCastDeregMr(nullptr, &handle), ncclInvalidArgument); }); + EXPECT_TRUE(LogHas(log, "deregMr called with a null comm")) << log; +} + +TEST_F(NetIbCastRegMicrotest, DeregMr_UnknownMr_IsAnInternalError) { + void* mhandle = nullptr; + ASSERT_EQ(IbCastRegMr(send_.get(), buf_, 8, NCCL_PTR_HOST, &mhandle), ncclSuccess); + ibv_mr stranger{}; + ncclIbMrHandle* bogus = static_cast(malloc(sizeof(ncclIbMrHandle))); + bogus->mrs[0] = &stranger; + std::string log = CaptureLog([&] { EXPECT_EQ(IbCastDeregMr(send_.get(), bogus), ncclInternalError); }); + EXPECT_TRUE(LogHas(log, "inside cache of 1 entries")) << log; + free(bogus); + EXPECT_EQ(IbCastDeregMr(send_.get(), mhandle), ncclSuccess); +} + +TEST_F(NetIbCastRegMicrotest, DeregMr_VerbsDeregFails_ReturnsTheError) { + void* mhandle = nullptr; + ASSERT_EQ(IbCastRegMr(send_.get(), buf_, 8, NCCL_PTR_HOST, &mhandle), ncclSuccess); + ScopedHook dereg(g_ibvDeregMr, [](struct ibv_mr*) { return ncclSystemError; }); + EXPECT_EQ(IbCastDeregMr(send_.get(), mhandle), ncclSystemError); + EXPECT_EQ(Cache().population, 0); + free(mhandle); // the failed deregistration returns before releasing the handle +} + +} // namespace diff --git a/projects/rccl/test/host/net-ib-cast-test-common.h b/projects/rccl/test/host/net-ib-cast-test-common.h new file mode 100644 index 000000000000..c501affd1394 --- /dev/null +++ b/projects/rccl/test/host/net-ib-cast-test-common.h @@ -0,0 +1,255 @@ +/************************************************************************* + * Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +// Shared by the rccl-UnitTestsMicroNetIbCastInit test TUs: param and comm helpers, fake verbs +// objects whose context ops call back into the test, and the reset of the transport's globals. +#pragma once + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ScopedHook.h" +#include "common_cast.h" +#include "fakes/ibvwrap_fakes.h" +#include "fakes/ionicdvwrap_fakes.h" +#include "fakes/mlx5dvwrap_fakes.h" +#include "fakes/nccl_fakes.h" +#include "fakes/net_ib_cast_connect_fakes.h" +#include "fakes/net_ib_cast_p2p_resiliency_recovery_fakes.h" +#include "fakes/net_ib_cast_scheduler_fakes.h" +#include "fakes/signature-drift.h" +#include "fakes/socket_fakes.h" + +// Puts every net_ib_cast global back to its load-time value. Defined in +// net-ib-cast-init-test.cc, the TU that can reach init.cc's file-scope state. +void NetIbCastResetGlobals(); +// IbCastInitDevices starts one async-event thread per device it adds; this counts them. +extern std::atomic g_netIbCastAsyncThreadsStarted; + +namespace { + +// Params reach the units through param_redirect.h's g_loadParam; one not Set() keeps its default. +class ParamScript { + public: + void Set(const char* name, int64_t value) { values_[name] = value; } + + private: + std::map values_; + ScopedHook hook_{g_loadParam, [this](const char* name, int64_t deftVal) { + auto it = values_.find(name); + return it == values_.end() ? deftVal : it->second; + }}; +}; + +// Zeroed heap storage for the large comm structs, as the transport's own calloc gives them. +template +struct ZeroedPtr { + ZeroedPtr() : p(static_cast(calloc(1, sizeof(T)))) {} + ~ZeroedPtr() { free(p); } + ZeroedPtr(const ZeroedPtr&) = delete; + ZeroedPtr& operator=(const ZeroedPtr&) = delete; + T* operator->() const { return p; } + T* get() const { return p; } + T* p; +}; + +// wrap_ibv_post_send/poll_cq/post_recv are inline and call through ibv_context::ops, so a +// fake context routes them here. +std::function g_fakePostSend = + [](struct ibv_qp*, struct ibv_send_wr*, struct ibv_send_wr**) { return EIO; }; +std::function g_fakePollCq = [](struct ibv_cq*, int, struct ibv_wc*) { + return -1; +}; +std::function g_fakePostRecv = + [](struct ibv_qp*, struct ibv_recv_wr*, struct ibv_recv_wr**) { return EIO; }; + +inline void InitFakeContext(struct ibv_context* ctx, struct ibv_device* dev) { + memset(ctx, 0, sizeof(*ctx)); + ctx->device = dev; + ctx->ops.post_send = [](struct ibv_qp* qp, struct ibv_send_wr* wr, struct ibv_send_wr** bad) { + return g_fakePostSend(qp, wr, bad); + }; + ctx->ops.poll_cq = [](struct ibv_cq* cq, int n, struct ibv_wc* wc) { return g_fakePollCq(cq, n, wc); }; + ctx->ops.post_recv = [](struct ibv_qp* qp, struct ibv_recv_wr* wr, struct ibv_recv_wr** bad) { + return g_fakePostRecv(qp, wr, bad); + }; +} + +template +using HookOf = ScopedHook>; + +// One fake HCA: the verbs objects IbCastInitDevices opens and queries. Port n is ports[n - 1]. +struct FakeHca { + struct ibv_device device; + struct ibv_context context; + struct ibv_device_attr attr; + std::vector ports; + uint64_t portSpeed = 0; // ibv_query_port_speed, in 100 Mb/s; 0 falls back to the port attributes + bool openFails = false; + bool queryDeviceFails = false; + bool mlx5 = false; + int closes = 0; +}; + +// A host with HCAs, one IP interface and every library present, as IbCastInitDevices sees it. +// The async-event threads it starts end on their first event read; the destructor waits for them, +// since they read hooks and IbCastDevs that the test is about to reset. +class FakeIbWorld { + public: + FakeIbWorld() : asyncThreadsBefore_(g_netIbCastAsyncThreadsStarted.load()) {} + ~FakeIbWorld() { Quiesce(); } + + // Waits until every async-event thread started so far has read its event and left. + void Quiesce() { + const int started = g_netIbCastAsyncThreadsStarted.load() - asyncThreadsBefore_; + auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(10); + while (asyncExits_.load() < started && std::chrono::steady_clock::now() < deadline) std::this_thread::yield(); + EXPECT_EQ(asyncExits_.load(), started) << "async-event threads still running"; + } + + // An HCA with one active RoCE port at 100 Gb/s (EDR, 4x). + FakeHca& AddHca(const char* name) { + hcas.emplace_back(); + FakeHca& hca = hcas.back(); + memset(&hca.device, 0, sizeof(hca.device)); + snprintf(hca.device.name, sizeof(hca.device.name), "%s", name); + snprintf(hca.device.dev_name, sizeof(hca.device.dev_name), "uverbs%zu", hcas.size() - 1); + InitFakeContext(&hca.context, &hca.device); + memset(&hca.attr, 0, sizeof(hca.attr)); + hca.attr.sys_image_guid = 0x1000 + hcas.size(); + hca.attr.max_qp = 1024; + hca.attr.max_cqe = 4096; + hca.ports.push_back(ActivePort(IBV_LINK_LAYER_ETHERNET)); + list.push_back(&hca.device); + return hca; + } + static struct ibv_port_attr ActivePort(uint8_t linkLayer) { + struct ibv_port_attr port; + memset(&port, 0, sizeof(port)); + port.state = IBV_PORT_ACTIVE; + port.link_layer = linkLayer; + port.active_speed = 32; // EDR + port.active_width = 2; // 4x + port.active_mtu = IBV_MTU_4096; + port.lid = 7; + return port; + } + FakeHca* Find(struct ibv_context* ctx) { + for (FakeHca& hca : hcas) + if (&hca.context == ctx) return &hca; + return nullptr; + } + + std::deque hcas; + std::vector list; + + private: + int asyncThreadsBefore_; + std::atomic asyncExits_{0}; + std::vector listed_; + + public: + HookOf ibvSymbols{g_ibvSymbols, [] { return ncclSuccess; }}; + HookOf mlx5Symbols{g_mlx5dvSymbols, [] { return ncclSuccess; }}; + HookOf ionicSymbols{g_ionicdvSymbols, [] { return ncclSuccess; }}; + HookOf forkInit{g_ibvForkInit, [] { return ncclSuccess; }}; + HookOf findInterfaces{ + g_ncclFindInterfaces, [](char* names, union ncclSocketAddress* addrs, int nameSize, int, int* nIfs) { + snprintf(names, nameSize, "eth0"); + memset(addrs, 0, sizeof(*addrs)); + addrs->sin.sin_family = AF_INET; + addrs->sin.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + *nIfs = 1; + return ncclSuccess; + }}; + // Like ibv_get_device_list: a NULL-terminated array, non-NULL even when there is no HCA. + HookOf getDeviceList{g_ibvGetDeviceList, [this](struct ibv_device*** ret, int* n) { + listed_ = list; + listed_.push_back(nullptr); + *ret = listed_.data(); + *n = (int)list.size(); + return ncclSuccess; + }}; + HookOf freeDeviceList{g_ibvFreeDeviceList, + [](struct ibv_device**) { return ncclSuccess; }}; + HookOf openDevice{g_ibvOpenDevice, [this](struct ibv_context** ret, struct ibv_device* d) { + for (FakeHca& hca : hcas) { + if (&hca.device != d) continue; + *ret = hca.openFails ? nullptr : &hca.context; + return hca.openFails ? ncclSystemError : ncclSuccess; + } + return ncclSystemError; + }}; + HookOf closeDevice{g_ibvCloseDevice, [this](struct ibv_context* ctx) { + if (FakeHca* hca = Find(ctx)) hca->closes++; + return ncclSuccess; + }}; + HookOf mlx5IsSupported{g_mlx5dvIsSupported, [this](struct ibv_device* d) { + for (FakeHca& hca : hcas) + if (&hca.device == d) return hca.mlx5; + return false; + }}; + HookOf queryDevice{g_ibvQueryDevice, [this](struct ibv_context* ctx, + struct ibv_device_attr* attr) { + FakeHca* hca = Find(ctx); + if (hca == nullptr || hca->queryDeviceFails) return ncclSystemError; + *attr = hca->attr; + attr->phys_port_cnt = (uint8_t)hca->ports.size(); + return ncclSuccess; + }}; + HookOf queryPort{g_ibvQueryPort, [this](struct ibv_context* ctx, uint8_t portNum, + struct ibv_port_attr* attr) { + FakeHca* hca = Find(ctx); + if (hca == nullptr || portNum < 1 || portNum > hca->ports.size()) + return ncclSystemError; + *attr = hca->ports[portNum - 1]; + return ncclSuccess; + }}; + HookOf queryPortSpeed{ + g_ibvQueryPortSpeed, [this](struct ibv_context* ctx, uint8_t, uint64_t* speed) { + FakeHca* hca = Find(ctx); + *speed = hca ? hca->portSpeed : 0; + return ncclSuccess; + }}; + HookOf gidInfoQuery{ + g_ibCastGidInfoQuery, [](struct ibv_context*, uint8_t, struct ibv_port_attr* port, struct ncclIbGidInfo* info) { + info->link_layer = port->link_layer; + info->localGidIndex = 3; + return ncclSuccess; + }}; + // Relaxed ordering needs IBVERBS_1.8; this library lacks it unless a test says otherwise. + HookOf regMrIova2{ + g_ibvRegMrIova2, [](struct ibv_mr**, struct ibv_pd*, void*, size_t, uint64_t, int) { return ncclInternalError; }}; + HookOf asyncEvent{g_ibvGetAsyncEvent, [this](struct ibv_context*, struct ibv_async_event*) { + asyncExits_++; + return ncclSystemError; + }}; + HookOf schedInitParms{g_ibCastQpSchedInitParms, [](struct ncclIbQpSchedParms* p) { + memset(p, 0, sizeof(*p)); + return ncclSuccess; + }}; + HookOf useQpSched{g_rcclUseIbCastQpSched, [] { return false; }}; + HookOf recoveryStart{g_ibCastPortRecoveryThreadStart, + [] { return ncclSuccess; }}; + HookOf recoveryStop{g_ibCastPortRecoveryThreadStop, + [] { return ncclSuccess; }}; +}; + +} // namespace diff --git a/projects/rccl/test/host/run_host_tests.sh b/projects/rccl/test/host/run_host_tests.sh index dca4d0a9663a..ed020be1fe49 100755 --- a/projects/rccl/test/host/run_host_tests.sh +++ b/projects/rccl/test/host/run_host_tests.sh @@ -192,6 +192,7 @@ do_host_tests() { "rccl-UnitTestsMicroGraph:$SCRIPT_DIR/host_tests_micro_graph.xml" "rccl-UnitTestsMicroRmaProxy:$SCRIPT_DIR/host_tests_micro_rma_proxy.xml" "rccl-UnitTestsMicroDda:$SCRIPT_DIR/host_tests_micro_dda.xml" + "rccl-UnitTestsMicroNetIbCastInit:$SCRIPT_DIR/host_tests_micro_net_ib_cast_init.xml" ) # Binaries that only exist for some CMake option settings (rccl-UnitTestsMicroSymKernels needs # GENERATE_SYM_KERNELS, off via install.sh --disable-sym-kernels); missing is a skip, not an error. diff --git a/projects/rccl/test/test_categories_micro_net_ib_cast_init.yaml b/projects/rccl/test/test_categories_micro_net_ib_cast_init.yaml new file mode 100644 index 000000000000..8861d664f76d --- /dev/null +++ b/projects/rccl/test/test_categories_micro_net_ib_cast_init.yaml @@ -0,0 +1,47 @@ +# rccl-UnitTestsMicroNetIbCastInit — shared/ctest schema. +# Host-only micro-tests for src/transport/net_ib_cast/{init,common,capability,reg,gdr,multiplane,gin, +# net_ib_cast_gid_inspect}.cc (no GPU, no librccl). +# RCCL does not support Windows: exclude_windows matches all tests so generated suites are empty on Windows. + +test_categories: + quick: µ_net_ib_cast_init_quick + description: "Smoke — host-only net_ib_cast init, capability and GIN micro-tests, sub-second." + test_patterns: + - "NetIbCast*" + exclude_windows: + - "*" + labels: + - "rccl" + - "quick" + - "smoke" + - "precheck" + - "microtest" + standard: + <<: *micro_net_ib_cast_init_quick + labels: + - "rccl" + - "standard" + - "regression" + - "microtest" + comprehensive: + <<: *micro_net_ib_cast_init_quick + labels: + - "rccl" + - "comprehensive" + - "comprehensive_regression" + - "microtest" + full: + <<: *micro_net_ib_cast_init_quick + labels: + - "rccl" + - "full" + - "full_regression" + - "microtest" + +execution_settings: + timeout_multiplier: 1 + category_timeouts: + quick: 600 + standard: 600 + comprehensive: 600 + full: 600 diff --git a/projects/rccl/tools/scripts/test_runner/configs/ci-precheckin.json b/projects/rccl/tools/scripts/test_runner/configs/ci-precheckin.json index 82d6ba29645b..e33389d0f282 100644 --- a/projects/rccl/tools/scripts/test_runner/configs/ci-precheckin.json +++ b/projects/rccl/tools/scripts/test_runner/configs/ci-precheckin.json @@ -473,6 +473,17 @@ "tests": [ {"name": "DdaMicrotests", "description": "Host-only src/algorithms/dda/ microtests (no GPU, no HIP/RCCL runtime)", "test_filter": "*"} ] + }, + "unit_tests_micro_net_ib_cast_init": { + "is_gtest": true, + "binary": "host/rccl-UnitTestsMicroNetIbCastInit", + "num_ranks": 1, + "num_nodes": 1, + "num_gpus": 1, + "timeout": 60, + "tests": [ + {"name": "NetIbCastInitMicrotests", "description": "Host-only net_ib_cast init, capability and GIN microtests (no GPU, no HIP/RCCL runtime)", "test_filter": "*"} + ] } }, "test_suites": [ @@ -649,6 +660,12 @@ "description": "rccl-UnitTestsMicroDda (host-only src/algorithms/dda/ microtests, no GPU/HIP/RCCL runtime)", "config": "unit_tests_micro_dda", "enabled": true + }, + { + "name": "RCCL Microtests - net_ib_cast init, capability and GIN", + "description": "rccl-UnitTestsMicroNetIbCastInit (host-only net_ib_cast init, capability and GIN microtests, no GPU/HIP/RCCL runtime)", + "config": "unit_tests_micro_net_ib_cast_init", + "enabled": true } ] } diff --git a/projects/rccl/tools/scripts/test_runner/lib/test_executor.py b/projects/rccl/tools/scripts/test_runner/lib/test_executor.py index 65427fdd1ee1..86288274fd62 100644 --- a/projects/rccl/tools/scripts/test_runner/lib/test_executor.py +++ b/projects/rccl/tools/scripts/test_runner/lib/test_executor.py @@ -3326,7 +3326,8 @@ def generate_coverage_report(self): "rccl-UnitTestsMicroSymKernels", "rccl-UnitTestsMicroGinHost", "rccl-UnitTestsMicroRomeModels", - "rccl-UnitTestsMicroDda"]: + "rccl-UnitTestsMicroDda", + "rccl-UnitTestsMicroNetIbCastInit"]: binary_path = os.path.join(host_test_dir, binary) if os.path.isfile(binary_path): object_files.extend(["--object", binary_path]) From 4d3155b7cebe530407025fa5b8c71d649b7037b5 Mon Sep 17 00:00:00 2001 From: Alex Duforest Date: Tue, 20 Oct 2026 13:28:12 -0700 Subject: [PATCH 05/15] test(rccl): address review of the net_ib_cast init microtests Pin the copied connect, qp_sharing, p2p, p2p_resiliency and socket fake defaults to production, check the stored PIP interface, let the slot-index crash pin accept ASan's exit, own the GetRealPort paths, and brace the single-statement bodies. Co-Authored-By: Claude --- projects/rccl/test/host/CMakeLists.txt | 52 +++++++++++++++++++ .../test/host/net-ib-cast-capability-test.cc | 3 +- .../rccl/test/host/net-ib-cast-common-test.cc | 3 +- .../rccl/test/host/net-ib-cast-gin-test.cc | 5 +- .../rccl/test/host/net-ib-cast-init-test.cc | 11 ++-- .../test/host/net-ib-cast-multiplane-test.cc | 2 + .../rccl/test/host/net-ib-cast-test-common.h | 9 ++-- 7 files changed, 73 insertions(+), 12 deletions(-) diff --git a/projects/rccl/test/host/CMakeLists.txt b/projects/rccl/test/host/CMakeLists.txt index ddfd7854b1e7..280cbc372748 100644 --- a/projects/rccl/test/host/CMakeLists.txt +++ b/projects/rccl/test/host/CMakeLists.txt @@ -49,6 +49,58 @@ rccl_assert_source_line("${_rccl_src_root}/transport/net_ib/connect.cc" rccl_assert_source_line("${CMAKE_CURRENT_LIST_DIR}/fakes/transport_net_ib_fakes.cc" "int64_t ncclParamIbQpsPerConn() { return g_loadParam(\"IB_QPS_PER_CONNECTION\", 1); }" "the IbQpsPerConn fake default") +# The net_ib_cast connect / qp_sharing fakes repeat these params; both copies must agree. +foreach(_param + "NCCL_PARAM(IbCastTimeout, \"IB_TIMEOUT\", 20)" + "NCCL_PARAM(IbCastRetryCnt, \"IB_RETRY_CNT\", 7)" + "NCCL_PARAM(IbCastPkey, \"IB_PKEY\", 0)" + "NCCL_PARAM(IbCastQpsPerConn, \"IB_QPS_PER_CONNECTION\", 2)" + "NCCL_PARAM(IbCastUseInline, \"IB_USE_INLINE\", 0)" + "NCCL_PARAM(IbCastGdrFlushDisable, \"GDR_FLUSH_DISABLE\", 1)" + "NCCL_PARAM(IbCastSl, \"IB_SL\", -1)" + "NCCL_PARAM(IbCastTc, \"IB_TC\", -1)" + "RCCL_PARAM(IbCastUDMAPolicy, \"IB_UDMA_POLICY\", 0)") + rccl_assert_source_line("${_rccl_src_root}/transport/net_ib_cast/connect.cc" + "${_param}" "fakes/net_ib_cast_connect_fakes.cc") + rccl_assert_source_line("${CMAKE_CURRENT_LIST_DIR}/fakes/net_ib_cast_connect_fakes.cc" + "${_param}" "the connect.cc default") +endforeach() +foreach(_param + "RCCL_PARAM(IbCastQpSharingEnable, \"IB_QP_SHARING_ENABLE\", 0)" + "RCCL_PARAM(IbCastCommNGroups, \"IB_COMM_NGROUPS\", 4)" + "RCCL_PARAM(IbCastQpSharingValidatePool, \"IB_QP_SHARING_VALIDATE_POOL\", 0)") + rccl_assert_source_line("${_rccl_src_root}/transport/net_ib_cast/qp_sharing.cc" + "${_param}" "fakes/net_ib_cast_qp_sharing_fakes.cc") + rccl_assert_source_line("${CMAKE_CURRENT_LIST_DIR}/fakes/net_ib_cast_qp_sharing_fakes.cc" + "${_param}" "the qp_sharing.cc default") +endforeach() +# The net_ib_cast and socket fakes copy these lines from production; pin both ends ("" stands for a semicolon). +set(_rccl_cast_pins + "transport/net_ib_cast/p2p.cc" "NCCL_PARAM(IbCastArThreshold, \"IB_AR_THRESHOLD\", -2)" + "net_ib_cast_p2p_fakes.cc" "return g_loadParam(\"IB_AR_THRESHOLD\", -2) }" + "transport/net_ib_cast/p2p.cc" "int64_t IbCastArThreshold = 8192" + "net_ib_cast_p2p_fakes.cc" "int64_t IbCastArThreshold = 8192" + "transport/net_ib_cast/p2p.cc" "NCCL_PARAM(IbCastReceiverSideMatchingScheme, \"IB_RECEIVER_SIDE_MATCHING_SCHEME\", -2)" + "net_ib_cast_p2p_fakes.cc" "return g_loadParam(\"IB_RECEIVER_SIDE_MATCHING_SCHEME\", -2) }" + "transport/net_ib_cast/p2p.cc" "RCCL_PARAM(IbCastGdrFlushGpuMemNoRelaxedOrdering, \"GDR_FLUSH_GPU_MEM_NO_RELAXED_ORDERING\", 1)" + "net_ib_cast_p2p_fakes.cc" "return g_loadParam(\"RCCL_GDR_FLUSH_GPU_MEM_NO_RELAXED_ORDERING\", 1)" + "transport/net_ib_cast/p2p.cc" "const char* IbCastReqTypeStr[] = {\"Unused\", \"Send\", \"Recv\", \"Flush\", \"IPut\"}" + "net_ib_cast_p2p_fakes.cc" "const char* IbCastReqTypeStr[] = {\"Unused\", \"Send\", \"Recv\", \"Flush\", \"IPut\"}" + "transport/net_ib_cast/p2p_resiliency.cc" "NCCL_PARAM(IbCastResiliencyPortFailover, \"IB_RESILIENCY_PORT_FAILOVER\", 0)" + "net_ib_cast_p2p_resiliency_fakes.cc" "return g_loadParam(\"IB_RESILIENCY_PORT_FAILOVER\", 0) }" + "transport/net_ib_cast/p2p_resiliency.cc" "NCCL_PARAM(IbCastResiliencyPortFailoverMaxAttempts, \"IB_RESILIENCY_PORT_FAILOVER_MAX_ATTEMPTS\", 1)" + "net_ib_cast_p2p_resiliency_fakes.cc" "return g_loadParam(\"IB_RESILIENCY_PORT_FAILOVER_MAX_ATTEMPTS\", 1)" + "transport/net_ib_cast/p2p_resiliency.cc" "NCCL_PARAM(IbCastResiliencyPortFailoverProbeDelay, \"IB_RESILIENCY_PORT_FAILOVER_PROBE_DELAY\", 10)" + "net_ib_cast_p2p_resiliency_fakes.cc" "return g_loadParam(\"IB_RESILIENCY_PORT_FAILOVER_PROBE_DELAY\", 10)" + "misc/socket.cc" "static uint64_t cached = NCCL_SOCKET_MAGIC" + "socket_fakes.cc" "uint64_t ncclSocketDefaultMagic(void) { return NCCL_SOCKET_MAGIC }") +while(_rccl_cast_pins) + list(POP_FRONT _rccl_cast_pins _src _prod _fake _copy) + string(REPLACE "" ";" _prod "${_prod}") + string(REPLACE "" ";" _copy "${_copy}") + rccl_assert_source_line("${_rccl_src_root}/${_src}" "${_prod}" "fakes/${_fake}") + rccl_assert_source_line("${CMAKE_CURRENT_LIST_DIR}/fakes/${_fake}" "${_copy}" "the ${_src} copy") +endwhile() rccl_assert_source_line("${CMAKE_CURRENT_LIST_DIR}/fakes/tuning_fakes.cc" "int64_t g_paramMinNchannels = -2;" "the MinNchannels fake default") rccl_assert_source_line("${CMAKE_CURRENT_LIST_DIR}/fakes/tuning_fakes.cc" diff --git a/projects/rccl/test/host/net-ib-cast-capability-test.cc b/projects/rccl/test/host/net-ib-cast-capability-test.cc index 4d39d51f6c5a..efb8152b32df 100644 --- a/projects/rccl/test/host/net-ib-cast-capability-test.cc +++ b/projects/rccl/test/host/net-ib-cast-capability-test.cc @@ -226,9 +226,10 @@ class NetIbCastCapabilityReadMicrotest : public NetIbCastCapabilityMicrotest { posted_ = *wr; postedSge_ = *wr->sg_list; if (Stops(Fail::PostSend)) return EIO; - if (copy_) + if (copy_) { memcpy((void*)wr->sg_list->addr, (void*)wr->wr.rdma.remote_addr, wr->sg_list->length); + } return 0; }}; HookOf pollCq_{g_fakePollCq, [this](struct ibv_cq* cq, int n, struct ibv_wc* wc) { diff --git a/projects/rccl/test/host/net-ib-cast-common-test.cc b/projects/rccl/test/host/net-ib-cast-common-test.cc index 57ebe7481202..a4a70c175fc4 100644 --- a/projects/rccl/test/host/net-ib-cast-common-test.cc +++ b/projects/rccl/test/host/net-ib-cast-common-test.cc @@ -286,8 +286,9 @@ class NetIbCastCommonEventMicrotest : public NetIbCastCommonMicrotest { event->event_type = events_.front(); events_.pop_front(); event->element.cq = &cq_; - if (event->event_type >= IBV_EVENT_QP_FATAL && event->event_type <= IBV_EVENT_QP_ACCESS_ERR) + if (event->event_type >= IBV_EVENT_QP_FATAL && event->event_type <= IBV_EVENT_QP_ACCESS_ERR) { event->element.qp = &qp_; + } return ncclSuccess; }); } diff --git a/projects/rccl/test/host/net-ib-cast-gin-test.cc b/projects/rccl/test/host/net-ib-cast-gin-test.cc index 96ddfe60679a..4c143638f627 100644 --- a/projects/rccl/test/host/net-ib-cast-gin-test.cc +++ b/projects/rccl/test/host/net-ib-cast-gin-test.cc @@ -10,6 +10,8 @@ #include +#include + #include #include #include @@ -1128,7 +1130,8 @@ TEST_F(NetIbCastGinTestMicrotest, Test_KnownBug_ErrorInTheSecondCompletionSlotCr IbCastRmaIbProxyTest(&cComm_, a, &done); exit(0); }, - ::testing::KilledBySignal(SIGSEGV), ""); + // A plain build dies on the signal; under ASan the runtime reports the SEGV and exits non-zero. + [](int status) { return WIFSIGNALED(status) ? WTERMSIG(status) == SIGSEGV : WEXITSTATUS(status) != 0; }, ""); ::testing::GTEST_FLAG(death_test_style) = style; } diff --git a/projects/rccl/test/host/net-ib-cast-init-test.cc b/projects/rccl/test/host/net-ib-cast-init-test.cc index 77077d0362d9..db2d83252a51 100644 --- a/projects/rccl/test/host/net-ib-cast-init-test.cc +++ b/projects/rccl/test/host/net-ib-cast-init-test.cc @@ -133,8 +133,9 @@ class NetIbCastInitMicrotest : public ::testing::Test { // realpath() answers for /sys/class/infiniband//device. void PciPaths(std::map paths) { realpath_.emplace(g_initRealpath, [paths](const char* path, char*) -> char* { - for (auto& [hca, pci] : paths) + for (auto& [hca, pci] : paths) { if (std::string(path) == "/sys/class/infiniband/" + hca + "/device") return strdup(pci.c_str()); + } return nullptr; }); } @@ -218,10 +219,9 @@ TEST_F(NetIbCastInitMicrotest, GetPciPath_NoSysfsEntry_LeavesAnEmptyFullPath) { TEST_F(NetIbCastInitMicrotest, GetRealPort_CountsEarlierDevicesOnTheSamePhysicalNic) { char p0[] = "/sys/devices/pci0000:00/0000:01:00.0"; - char p1[] = "/sys/devices/pci0000:00/0000:02:00.0"; - Dev(0, "a").pciPath = p0; - Dev(1, "b").pciPath = p1; - Dev(2, "c").pciPath = p0; + Dev(0, "a").pciPath = strdup(p0); + Dev(1, "b").pciPath = strdup("/sys/devices/pci0000:00/0000:02:00.0"); + Dev(2, "c").pciPath = strdup(p0); int realPort = -1; ASSERT_EQ(IbCastGetRealPort(p0, &realPort, 3), ncclSuccess); EXPECT_EQ(realPort, 2); @@ -229,7 +229,6 @@ TEST_F(NetIbCastInitMicrotest, GetRealPort_CountsEarlierDevicesOnTheSamePhysical EXPECT_EQ(realPort, 0); ASSERT_EQ(IbCastGetRealPort(nullptr, &realPort, 3), ncclSuccess); EXPECT_EQ(realPort, 0); - for (int d = 0; d < 3; d++) IbCastDevs[d].pciPath = nullptr; } TEST_F(NetIbCastInitMicrotest, PciRootFromPath_TakesTheFirstDomainAndBus) { diff --git a/projects/rccl/test/host/net-ib-cast-multiplane-test.cc b/projects/rccl/test/host/net-ib-cast-multiplane-test.cc index 4a13773c853d..ba264f9bceb4 100644 --- a/projects/rccl/test/host/net-ib-cast-multiplane-test.cc +++ b/projects/rccl/test/host/net-ib-cast-multiplane-test.cc @@ -189,6 +189,8 @@ TEST_F(NetIbCastMultiplaneMicrotest, Load_PipWithoutInterface_LeavesTheInterface ScopedDebugLogging info(NCCL_LOG_INFO, NCCL_ALL); std::string log = CaptureLog([&] { ASSERT_EQ(IbCastMultiplaneLoad(), ncclSuccess); }); EXPECT_TRUE(LogHas(log, "ip=10.0.0.1 interface=\n")) << log; + ASSERT_EQ(gidToPipMap.size(), 1u); + EXPECT_STREQ(gidToPipMap.begin()->second.at(0).interface, ""); } TEST_F(NetIbCastMultiplaneMicrotest, Load_DuplicateVip_LastEntryWins) { diff --git a/projects/rccl/test/host/net-ib-cast-test-common.h b/projects/rccl/test/host/net-ib-cast-test-common.h index c501affd1394..b840e8252c1d 100644 --- a/projects/rccl/test/host/net-ib-cast-test-common.h +++ b/projects/rccl/test/host/net-ib-cast-test-common.h @@ -152,8 +152,9 @@ class FakeIbWorld { return port; } FakeHca* Find(struct ibv_context* ctx) { - for (FakeHca& hca : hcas) + for (FakeHca& hca : hcas) { if (&hca.context == ctx) return &hca; + } return nullptr; } @@ -202,8 +203,9 @@ class FakeIbWorld { return ncclSuccess; }}; HookOf mlx5IsSupported{g_mlx5dvIsSupported, [this](struct ibv_device* d) { - for (FakeHca& hca : hcas) + for (FakeHca& hca : hcas) { if (&hca.device == d) return hca.mlx5; + } return false; }}; HookOf queryDevice{g_ibvQueryDevice, [this](struct ibv_context* ctx, @@ -217,8 +219,9 @@ class FakeIbWorld { HookOf queryPort{g_ibvQueryPort, [this](struct ibv_context* ctx, uint8_t portNum, struct ibv_port_attr* attr) { FakeHca* hca = Find(ctx); - if (hca == nullptr || portNum < 1 || portNum > hca->ports.size()) + if (hca == nullptr || portNum < 1 || portNum > hca->ports.size()) { return ncclSystemError; + } *attr = hca->ports[portNum - 1]; return ncclSuccess; }}; From f66e1e4e6ae77bbb5756ea72ed48b546e14c7271 Mon Sep 17 00:00:00 2001 From: Alex Duforest Date: Tue, 20 Oct 2026 19:44:58 -0700 Subject: [PATCH 06/15] test(rccl): host-only suites declare num_gpus 0 Co-Authored-By: Claude --- .../rccl/tools/scripts/test_runner/configs/ci-precheckin.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/rccl/tools/scripts/test_runner/configs/ci-precheckin.json b/projects/rccl/tools/scripts/test_runner/configs/ci-precheckin.json index 9187dbe563f6..85f12411849d 100644 --- a/projects/rccl/tools/scripts/test_runner/configs/ci-precheckin.json +++ b/projects/rccl/tools/scripts/test_runner/configs/ci-precheckin.json @@ -490,7 +490,7 @@ "binary": "host/rccl-UnitTestsMicroNetIbCastInit", "num_ranks": 1, "num_nodes": 1, - "num_gpus": 1, + "num_gpus": 0, "timeout": 60, "tests": [ {"name": "NetIbCastInitMicrotests", "description": "Host-only net_ib_cast init, capability and GIN microtests (no GPU, no HIP/RCCL runtime)", "test_filter": "*"} From 5b15c49e5ace90a1e64f38d9d5036bef4b101dda Mon Sep 17 00:00:00 2001 From: Alex Duforest Date: Tue, 20 Oct 2026 22:12:38 -0700 Subject: [PATCH 07/15] test(rccl): skip leaking gin error-path tests under ASan; make the IB_DISABLE test bite Co-Authored-By: Claude --- .../rccl/test/host/net-ib-cast-gin-test.cc | 23 +++++++++++++++++++ .../rccl/test/host/net-ib-cast-init-test.cc | 4 +++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/projects/rccl/test/host/net-ib-cast-gin-test.cc b/projects/rccl/test/host/net-ib-cast-gin-test.cc index 4c143638f627..2d3f55fc57f9 100644 --- a/projects/rccl/test/host/net-ib-cast-gin-test.cc +++ b/projects/rccl/test/host/net-ib-cast-gin-test.cc @@ -29,8 +29,23 @@ #include NET_IB_CAST_GIN_CC_PATH +#define GIN_TEST_ASAN 0 +#if defined(__SANITIZE_ADDRESS__) +#undef GIN_TEST_ASAN +#define GIN_TEST_ASAN 1 +#elif defined(__has_feature) +#if __has_feature(address_sanitizer) +#undef GIN_TEST_ASAN +#define GIN_TEST_ASAN 1 +#endif +#endif + namespace { +// gin.cc's error paths leak what they allocated; tests that take one skip under ASan, whose leak +// check would fail the binary. +constexpr bool kAsan = GIN_TEST_ASAN; + using RcclUnitTesting::CaptureLog; using RcclUnitTesting::LogHas; @@ -415,6 +430,7 @@ TEST_F(NetIbCastGinRingMicrotest, P2PBarrier_IsAnAllGatherOfOneIntPerRank) { ASSERT_EQ(IbCastGinIbP2PBarrier(&cComm_), ncclSuccess); ASSERT_EQ(sends_.size(), 2u); EXPECT_EQ(sends_[0].size, sizeof(int)); + if (kAsan) GTEST_SKIP() << "the failed barrier leaks its buffer"; ScopedHook isend(g_ibCastIsend, [](void*, void*, size_t, int, void*, void*, void**) { return ncclSystemError; }); EXPECT_EQ(IbCastGinIbP2PBarrier(&cComm_), ncclSystemError); } @@ -493,6 +509,7 @@ TEST_F(NetIbCastGinConnectMicrotest, Connect_GinTrafficClassWinsOverTheIbOne) { } TEST_F(NetIbCastGinConnectMicrotest, Connect_ConnectOrAcceptFails_ReturnsTheError) { + if (kAsan) GTEST_SKIP() << "the failed connect leaks the coll comm"; void* handles[2] = {nullptr, nullptr}; void* collComm = reinterpret_cast(0x1); { @@ -507,6 +524,7 @@ TEST_F(NetIbCastGinConnectMicrotest, Connect_ConnectOrAcceptFails_ReturnsTheErro // The coll comm and the send comm already connected are left behind when accept fails. // Pins AICOMRCCL-3094; a fix flips this pin. TEST_F(NetIbCastGinConnectMicrotest, Connect_KnownBug_AcceptFailureLeavesTheSendCommOpen) { + if (kAsan) GTEST_SKIP() << "the failed connect leaks the coll comm"; void* handles[2] = {nullptr, nullptr}; void* collComm = nullptr; ScopedHook fails(g_ibCastAccept, [](void*, void**, ncclNetDeviceHandle_t**) { return ncclSystemError; }); @@ -689,6 +707,7 @@ TEST_F(NetIbCastGinContextMicrotest, CreateContext_AnyStepFails_ReturnsNoContext cComm_.allGather = FailingAllGather; EXPECT_EQ(Create(1, 1, &ctx), ncclRemoteError); EXPECT_EQ(closeListen_.calls, 1); + if (kAsan) GTEST_SKIP() << "a failure past the peer-comm arrays leaks them"; cComm_.allGather = HandleAllGather; { ScopedHook fails(g_ibCastConnectImpl, [](void*, int, void*, void**, ncclNetDeviceHandle_t**, int) { @@ -710,6 +729,7 @@ TEST_F(NetIbCastGinContextMicrotest, CreateContext_AnyStepFails_ReturnsNoContext // Only the context array is freed on failure; the comms already connected stay open. // Pins AICOMRCCL-3094; a fix flips this pin. TEST_F(NetIbCastGinContextMicrotest, CreateContext_KnownBug_FailureLeavesConnectedCommsOpen) { + if (kAsan) GTEST_SKIP() << "a failure past the peer-comm arrays leaks them"; cComm_.allGather = HandleAllGather; char ring[4 * sizeof(int)]; base_ = ring; @@ -807,6 +827,7 @@ TEST_F(NetIbCastGinMicrotest, RegMrSymDmaBuf_PassesTheFdThrough) { } TEST_F(NetIbCastGinMicrotest, RegMrSym_RegistrationOrExchangeFails_ReturnsTheError) { + if (kAsan) GTEST_SKIP() << "the failed registration leaks the MR handle"; char data[64]; void* mhandle = nullptr; { @@ -818,6 +839,7 @@ TEST_F(NetIbCastGinMicrotest, RegMrSym_RegistrationOrExchangeFails_ReturnsTheErr // A failed exchange returns without deregistering, so the MR stays in the device's cache. // Pins AICOMRCCL-3094; a fix flips this pin. TEST_F(NetIbCastGinMicrotest, RegMrSym_KnownBug_FailedExchangeLeavesTheMrRegistered) { + if (kAsan) GTEST_SKIP() << "the failed exchange leaks the MR handle"; char data[64]; void* mhandle = nullptr; cComm_.allGather = FailingAllGather; @@ -827,6 +849,7 @@ TEST_F(NetIbCastGinMicrotest, RegMrSym_KnownBug_FailedExchangeLeavesTheMrRegiste } TEST_F(NetIbCastGinMicrotest, DeregMrSym_DeregFails_ReturnsTheError) { + if (kAsan) GTEST_SKIP() << "the failed deregistration leaks the MR handle"; cComm_.allGather = MrAllGather; char data[64]; void* mhandle = nullptr; diff --git a/projects/rccl/test/host/net-ib-cast-init-test.cc b/projects/rccl/test/host/net-ib-cast-init-test.cc index db2d83252a51..0ff9b2a7c1e8 100644 --- a/projects/rccl/test/host/net-ib-cast-init-test.cc +++ b/projects/rccl/test/host/net-ib-cast-init-test.cc @@ -610,9 +610,11 @@ TEST_F(NetIbCastInitDevicesMicrotest, FinalizeDevices_LastReferenceWithoutValida } TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_IbDisabled_FailsBeforeLoadingVerbs) { + FakeIbWorld& world = World(); + world.AddHca("ionic_0"); params_.Set("IB_DISABLE", 1); EXPECT_EQ(Init(), ncclInternalError); - EXPECT_EQ(World().ibvSymbols.calls, 0); + EXPECT_EQ(world.ibvSymbols.calls, 0); } // The reference is taken before the checks that can fail and is never dropped, so a retry From a7e2c2e5748243295c5b6803bf86d44f2b3c4050 Mon Sep 17 00:00:00 2001 From: Alex Duforest Date: Tue, 20 Oct 2026 23:06:33 -0700 Subject: [PATCH 08/15] test(rccl): make the no-verbs init test bite; give merged devices real PCI roots Co-Authored-By: Claude --- .../rccl/test/host/net-ib-cast-init-test.cc | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/projects/rccl/test/host/net-ib-cast-init-test.cc b/projects/rccl/test/host/net-ib-cast-init-test.cc index 0ff9b2a7c1e8..838dc21224d8 100644 --- a/projects/rccl/test/host/net-ib-cast-init-test.cc +++ b/projects/rccl/test/host/net-ib-cast-init-test.cc @@ -123,6 +123,13 @@ class NetIbCastInitMicrotest : public ::testing::Test { dev.planeId = NCCL_NET_ID_UNDEF; return dev; } + static void SetPciPath(int d, const std::string& path) { + free(IbCastDevs[d].pciPath); + IbCastDevs[d].pciPath = strdup(path.c_str()); + } + // Root complex 0000:00 with no numa_node behind it. Merging devices compares their roots, which + // are left uninitialized for a NULL pciPath. + static void OnRoot0(int d) { SetPciPath(d, "/nonexistent/pci0000:00/0000:0" + std::to_string(d + 1) + ":00.0"); } static ncclNetVDeviceProps_t Props(std::initializer_list devs) { ncclNetVDeviceProps_t props{}; @@ -414,6 +421,7 @@ class NetIbCastInitVDeviceMicrotest : public NetIbCastInitMicrotest { Dev(0, "mlx5_0"); Dev(1, "mlx5_1"); Dev(2, "mlx5_2"); + for (int d = 0; d < 3; d++) OnRoot0(d); } ncclResult_t Make(std::initializer_list devs, int* d) { ncclNetVDeviceProps_t props = Props(devs); @@ -533,9 +541,9 @@ TEST_F(NetIbCastInitVDeviceMicrotest, MakeVDevice_AcrossNumaNodesOrRootComplexes std::ofstream(p0 + "/numa_node") << "0\n"; std::ofstream(p1 + "/numa_node") << "1\n"; std::ofstream(p2 + "/numa_node") << "0\n"; - IbCastDevs[0].pciPath = strdup(p0.c_str()); - IbCastDevs[1].pciPath = strdup(p1.c_str()); - IbCastDevs[2].pciPath = strdup(p2.c_str()); + SetPciPath(0, p0); + SetPciPath(1, p1); + SetPciPath(2, p2); int d = -1; std::string log = CaptureLog([&] { ASSERT_EQ(Make({0, 1}, &d), ncclSuccess); }); EXPECT_TRUE(LogHas(log, "Merging NICs across NUMA nodes (mlx5_0 numa=0, mlx5_1 numa=1)")) << log; @@ -633,8 +641,11 @@ TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_KnownBug_FailedInitMakesTheNex } TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_NoVerbsLibrary_Fails) { + FakeIbWorld& world = World(); + world.AddHca("ionic_0"); ScopedHook symbols(g_ibvSymbols, [] { return ncclSystemError; }); EXPECT_EQ(Init(), ncclInternalError); + EXPECT_EQ(world.ionicSymbols.calls, 0); EXPECT_EQ(IbCastNDevs, -1); } @@ -1316,6 +1327,8 @@ TEST_F(NetIbCastInitPropertiesMicrotest, GetProperties_DataDirectAndCtsOffload_C TEST_F(NetIbCastInitPropertiesMicrotest, GetProperties_MergedDevice_ReportsTheFusedNameSpeedAndIds) { Dev(1, "mlx5_1"); + OnRoot0(0); + OnRoot0(1); int d; ncclNetVDeviceProps_t props = Props({0, 1}); ASSERT_EQ(IbCastMakeVDevice(&d, &props), ncclSuccess); From 022046c5316d7b3b7be6aad23e71705c639aaac1 Mon Sep 17 00:00:00 2001 From: Alex Duforest Date: Wed, 21 Oct 2026 18:55:43 -0700 Subject: [PATCH 09/15] test(rccl): net_ib_cast shared hooks, kAsan header and fewer log checks per review - os_fakes: ncclOsTopoGetStrFromSys goes through a g_ncclOsTopoGetStrFromSys hook (same default), so gdr-test installs it with ScopedHook instead of renaming it. - nccl_fakes: g_ncclSetThreadName hook for the pthread_t overload; FakeIbWorld records the names and counts async threads through it. - AsanCheck.h exports kAsan; gin-test, TopoScene.h and topo-test use it. - README fake-map rows for the new fakes files. - Drop log-text checks where a return code, field or counter already pins the branch (81 -> 21). Co-Authored-By: Claude --- projects/rccl/test/host/AsanCheck.h | 24 +++ projects/rccl/test/host/MICROTEST_README.md | 8 +- projects/rccl/test/host/TopoScene.h | 13 +- projects/rccl/test/host/fakes/nccl_fakes.cc | 14 ++ projects/rccl/test/host/fakes/nccl_fakes.h | 3 + projects/rccl/test/host/fakes/os_fakes.cc | 19 ++- projects/rccl/test/host/fakes/os_fakes.h | 1 + .../test/host/net-ib-cast-capability-test.cc | 15 +- .../rccl/test/host/net-ib-cast-common-test.cc | 39 ++--- .../rccl/test/host/net-ib-cast-gdr-test.cc | 25 +--- .../rccl/test/host/net-ib-cast-gin-test.cc | 33 +---- .../rccl/test/host/net-ib-cast-init-test.cc | 138 +++++------------- .../test/host/net-ib-cast-multiplane-test.cc | 38 +++-- .../rccl/test/host/net-ib-cast-reg-test.cc | 19 +-- .../rccl/test/host/net-ib-cast-test-common.h | 10 +- projects/rccl/test/host/topo-test.cc | 2 +- 16 files changed, 160 insertions(+), 241 deletions(-) create mode 100644 projects/rccl/test/host/AsanCheck.h diff --git a/projects/rccl/test/host/AsanCheck.h b/projects/rccl/test/host/AsanCheck.h new file mode 100644 index 000000000000..0c8628716d43 --- /dev/null +++ b/projects/rccl/test/host/AsanCheck.h @@ -0,0 +1,24 @@ +/************************************************************************* + * Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +// kAsan is true in an AddressSanitizer build; tests that pin a production leak or overflow skip under it. +#ifndef RCCL_TEST_HOST_ASANCHECK_H_ +#define RCCL_TEST_HOST_ASANCHECK_H_ + +#if defined(__SANITIZE_ADDRESS__) +#define RCCL_TEST_ASAN 1 +#elif defined(__has_feature) +#if __has_feature(address_sanitizer) +#define RCCL_TEST_ASAN 1 +#endif +#endif +#ifndef RCCL_TEST_ASAN +#define RCCL_TEST_ASAN 0 +#endif + +constexpr bool kAsan = RCCL_TEST_ASAN; + +#endif // RCCL_TEST_HOST_ASANCHECK_H_ diff --git a/projects/rccl/test/host/MICROTEST_README.md b/projects/rccl/test/host/MICROTEST_README.md index 8bfa32dbe3f1..8d9741e25277 100644 --- a/projects/rccl/test/host/MICROTEST_README.md +++ b/projects/rccl/test/host/MICROTEST_README.md @@ -614,11 +614,16 @@ symbol. | `src/mem_manager.cc` | `fakes/mem_manager_fakes.cc`; `ncclMemTrack*` / `ncclMemUntrack*` in `fakes/nccl_fakes.cc` | | `src/misc/amdsmi_wrap.cc` | `fakes/amdsmi_fakes.cc` | | `src/misc/api_trace.cc` (`NCCL_API` dispatch) | `fakes/api_trace_fakes.cc` | -| `src/misc/ibvwrap.cc` (PD / MR registration wrappers) | `fakes/ibvwrap_fakes.cc` | +| `src/misc/gdr_peermem.cc` | `fakes/gdr_peermem_fakes.cc` | +| `src/misc/gdr_probe.cc` (targets that do not compile the real file) | `fakes/gdr_probe_fakes.cc` | +| `src/misc/ibvwrap.cc` (verbs wrappers) | `fakes/ibvwrap_fakes.cc` | +| `src/misc/ionicdvwrap.cc` | `fakes/ionicdvwrap_fakes.cc` | | ibverbs provider dv libraries (`libionic.so`, `libmlx5.so`) | `fakes/dv_provider_fake.cc` | | `src/misc/kernel_config.cc` | `fakes/kernel_config_fakes.cc` | +| `src/misc/mlx5dvwrap.cc` | `fakes/mlx5dvwrap_fakes.cc` | | `src/misc/param.cc` + `getenv` interposition | `fakes/env_fakes.cc` | | `src/misc/rocmwrap.cc` | `fakes/rocmwrap_fakes.cc` | +| `src/misc/socket.cc` | `fakes/socket_fakes.cc` | | `src/misc/strongstream.cc` | `fakes/strongstream_stubs.cc` | | `src/misc/utils.cc` | `fakes/utils_fakes.cc` | | `src/os/linux.cc` | `fakes/os_fakes.cc` | @@ -635,6 +640,7 @@ symbol. | `src/scheduler/*.cc`'s own public entry points (targets that don't compile the real files, e.g. `rccl-UnitTestsMicroEnqueue`) and the deep launch paths | `fakes/sched_stubs.cc` | | `src/sym_kernels.cc` | `fakes/sym_kernels_fakes.cc` | | `src/transport/net_ib/connect.cc` params (`ncclParamIbQpsPerConn`) | `fakes/transport_net_ib_fakes.cc` | +| `src/transport/net_ib_cast/{connect,p2p,p2p_resiliency,p2p_resiliency_recovery,qp_sharing,scheduler}.cc` (targets that do not compile the real file) | `fakes/net_ib_cast__fakes.cc` | | generated `sym_kernels_host.cc` (targets that do not compile it) | `fakes/sym_kernels_index_fakes.cc` | | rocSHMEM host API | `fakes/rocshmem_fakes.cc` | | `src/transport/p2p.cc` shareable-buffer entry points (`rccl-UnitTestsMicroDiagnostics`) | `fakes/transport_p2p_fakes.cc` | diff --git a/projects/rccl/test/host/TopoScene.h b/projects/rccl/test/host/TopoScene.h index dd8cc406f415..4f8c9cf1387a 100644 --- a/projects/rccl/test/host/TopoScene.h +++ b/projects/rccl/test/host/TopoScene.h @@ -19,6 +19,7 @@ #include #include +#include "AsanCheck.h" #include "fakes/graph_fakes.h" #include "comm.h" @@ -29,21 +30,9 @@ // Defined in topo-test.cc, which owns topo.cc's NCCL_NETDEVS_POLICY statics. void TopoSetNetDevsPolicy(enum netDevsPolicy policy, int num); -#define TOPO_SCENE_ASAN 0 -#if defined(__SANITIZE_ADDRESS__) -#undef TOPO_SCENE_ASAN -#define TOPO_SCENE_ASAN 1 -#elif defined(__has_feature) -#if __has_feature(address_sanitizer) -#undef TOPO_SCENE_ASAN -#define TOPO_SCENE_ASAN 1 -#endif -#endif - namespace topo_scene { constexpr int kXmlNodes = 512; -constexpr bool kAsan = TOPO_SCENE_ASAN; // tests that pin a production leak or overflow skip under ASan struct FreeDeleter { void operator()(void* p) const { free(p); } diff --git a/projects/rccl/test/host/fakes/nccl_fakes.cc b/projects/rccl/test/host/fakes/nccl_fakes.cc index bc70cdc61917..93e716401c3a 100644 --- a/projects/rccl/test/host/fakes/nccl_fakes.cc +++ b/projects/rccl/test/host/fakes/nccl_fakes.cc @@ -89,6 +89,19 @@ void ncclDebugLog(ncclDebugLogLevel /*level*/, std::fputc('\n', stderr); } +// src/debug.cc's pthread_t overload; the hook gets the name as production would set it. Default: no-op, +// like production at its NCCL_SET_THREAD_NAME=0 default. Weak so a suite with its own copy still links. +static void DefaultSetThreadName(pthread_t, const char*) {} +std::function g_ncclSetThreadName = DefaultSetThreadName; +__attribute__((weak)) void ncclSetThreadName(pthread_t thread, const char* fmt, ...) { + char name[NCCL_THREAD_NAMELEN]; + std::va_list ap; + va_start(ap, fmt); + std::vsnprintf(name, sizeof(name), fmt, ap); + va_end(ap); + g_ncclSetThreadName(thread, name); +} + // Every param sits at its default. The NCCL_PARAM bodies that p2p-test.cc redirects through g_loadParam bypass this. int64_t ncclLoadParam(char const*, int64_t deftVal, int64_t, int64_t* cache, int8_t* noCache) { if (cache) { @@ -611,4 +624,5 @@ void ResetNcclFakes() g_shmAllocateShareableBuffer = DefaultShmAllocateShareableBuffer; g_commGraphRegister = DefaultCommGraphRegister; g_commGraphDeregister = DefaultCommGraphDeregister; + g_ncclSetThreadName = DefaultSetThreadName; } diff --git a/projects/rccl/test/host/fakes/nccl_fakes.h b/projects/rccl/test/host/fakes/nccl_fakes.h index 6eae3240bb5a..5f7d1d26c3bf 100644 --- a/projects/rccl/test/host/fakes/nccl_fakes.h +++ b/projects/rccl/test/host/fakes/nccl_fakes.h @@ -165,6 +165,9 @@ extern std::function g_shmAllocateShareableBuffer; +// ncclSetThreadName(pthread_t, fmt, ...) hands the formatted name here. +extern std::function g_ncclSetThreadName; + // Restore every NCCL controllable seam in this header to its default. // Called by ResetP2pFakes(); exposed for tests that only touch NCCL hooks. // Hands back an fd the caller must close. Defaults to ncclSystemError (the diff --git a/projects/rccl/test/host/fakes/os_fakes.cc b/projects/rccl/test/host/fakes/os_fakes.cc index eb16af38a432..28b4cc0dffc3 100644 --- a/projects/rccl/test/host/fakes/os_fakes.cc +++ b/projects/rccl/test/host/fakes/os_fakes.cc @@ -66,14 +66,16 @@ ncclResult_t ncclOsSetAffinity(const ncclAffinity& affinity) { // Also not "1" and not the Hyper-V BIOS string, so numa_balancing / bios_version stay on their benign arms. ncclResult_t g_ncclOsTopoGetStrFromSysResult = ncclSuccess; int g_ncclOsTopoGetStrFromSysCalls = 0; -ncclResult_t ncclOsTopoGetStrFromSys(const char* path, const char* fileName, char* strValue, int maxLen) -{ - ++g_ncclOsTopoGetStrFromSysCalls; - if (g_ncclOsTopoGetStrFromSysResult != ncclSuccess) return g_ncclOsTopoGetStrFromSysResult; - if (strValue && maxLen > 0) { - std::snprintf(strValue, maxLen, "Linux version 6.8.0-microtest"); - } - return ncclSuccess; +static ncclResult_t DefaultOsTopoGetStrFromSys(const char*, const char*, char* value, int maxLen) { + if (g_ncclOsTopoGetStrFromSysResult != ncclSuccess) return g_ncclOsTopoGetStrFromSysResult; + if (value && maxLen > 0) std::snprintf(value, maxLen, "Linux version 6.8.0-microtest"); + return ncclSuccess; +} +std::function g_ncclOsTopoGetStrFromSys = + DefaultOsTopoGetStrFromSys; +ncclResult_t ncclOsTopoGetStrFromSys(const char* path, const char* fileName, char* strValue, int maxLen) { + ++g_ncclOsTopoGetStrFromSysCalls; + return g_ncclOsTopoGetStrFromSys(path, fileName, strValue, maxLen); } // fillInfo MLOPart PCI-function fallback (init.cc ~1093). The default models what sysfs actually @@ -143,6 +145,7 @@ void ResetOsFakes() { g_ncclOsSetAffinityMasks.clear(); g_ncclOsTopoGetStrFromSysResult = ncclSuccess; g_ncclOsTopoGetStrFromSysCalls = 0; + g_ncclOsTopoGetStrFromSys = DefaultOsTopoGetStrFromSys; g_pciDeviceClass = kDefaultPciDeviceClass; g_pciDeviceClassCalls = 0; g_lastPciDeviceClassBusId.clear(); diff --git a/projects/rccl/test/host/fakes/os_fakes.h b/projects/rccl/test/host/fakes/os_fakes.h index 05fc2e9871a6..56b88fd7b1a4 100644 --- a/projects/rccl/test/host/fakes/os_fakes.h +++ b/projects/rccl/test/host/fakes/os_fakes.h @@ -35,6 +35,7 @@ extern std::function g_ncclOsGetAffinity; // ncclInit() NCCLCHECKs this before its own call_once, so it is the one per-call way to fail ncclInit(). extern ncclResult_t g_ncclOsTopoGetStrFromSysResult; extern int g_ncclOsTopoGetStrFromSysCalls; +extern std::function g_ncclOsTopoGetStrFromSys; // fillInfo's MLOPart PCI-function fallback (init.cc:1092) probes sysfs for the class of its own BDF. // The default is an accelerator class, i.e. what sysfs reports for a real GPU's own BDF; a test that diff --git a/projects/rccl/test/host/net-ib-cast-capability-test.cc b/projects/rccl/test/host/net-ib-cast-capability-test.cc index efb8152b32df..41bca5306dc9 100644 --- a/projects/rccl/test/host/net-ib-cast-capability-test.cc +++ b/projects/rccl/test/host/net-ib-cast-capability-test.cc @@ -160,8 +160,7 @@ class NetIbCastCapabilityUdMicrotest : public NetIbCastCapabilityMicrotest { }; TEST_F(NetIbCastCapabilityUdMicrotest, ProbeUd_QpReachesRts_IsSupportedAndReleasesEverything) { - ScopedDebugLogging info(NCCL_LOG_INFO, NCCL_ALL); - std::string log = CaptureLog([&] { ASSERT_EQ(IbCastCapProbeDevices(), ncclSuccess); }); + ASSERT_EQ(IbCastCapProbeDevices(), ncclSuccess); EXPECT_EQ(dev_->udSupported, 1); EXPECT_TRUE(IbCastCapUdSupported(dev_)); EXPECT_EQ(created_.type, IBV_QPT_UD); @@ -176,7 +175,6 @@ TEST_F(NetIbCastCapabilityUdMicrotest, ProbeUd_QpReachesRts_IsSupportedAndReleas EXPECT_EQ(destroyedQps_, 1); EXPECT_EQ(destroyedCqs_, 1); EXPECT_EQ(deallocs_, 1); - EXPECT_TRUE(LogHas(log, "device ionic_0 capability probe: UD=yes")) << log; // Probed once: a second pass leaves the answer alone. ASSERT_EQ(IbCastCapProbeDevices(), ncclSuccess); EXPECT_EQ(allocPd_.calls, 1); @@ -196,13 +194,12 @@ TEST_F(NetIbCastCapabilityUdMicrotest, ProbeUd_CreateCqUsesThePortRecoveryCqSize class NetIbCastCapabilityUdFailMicrotest : public NetIbCastCapabilityUdMicrotest, public ::testing::WithParamInterface {}; -TEST_P(NetIbCastCapabilityUdFailMicrotest, ProbeUd_AnyStepFails_IsUnsupportedAndWarns) { +TEST_P(NetIbCastCapabilityUdFailMicrotest, ProbeUd_AnyStepFails_IsUnsupported) { fail_ = GetParam(); if (fail_ == Fail::Rts) udToRtsResult_ = ncclSystemError; - std::string log = CaptureLog([&] { ASSERT_EQ(IbCastCapProbeDevices(), ncclSuccess); }); + ASSERT_EQ(IbCastCapProbeDevices(), ncclSuccess); EXPECT_EQ(dev_->udSupported, 0); EXPECT_FALSE(IbCastCapUdSupported(dev_)); - EXPECT_TRUE(LogHas(log, "device ionic_0 cannot create a UD queue pair; port recovery disabled on it")) << log; // Whatever was acquired is released. EXPECT_EQ(deallocs_, fail_ == Fail::AllocPd ? 0 : 1); EXPECT_EQ(destroyedCqs_, fail_ == Fail::AllocPd || fail_ == Fail::CreateCq ? 0 : 1); @@ -254,11 +251,9 @@ TEST_F(NetIbCastCapabilityReadMicrotest, ProbeRead_LoopbackReadCompletes_IsSuppo params_.Set("IB_SL", 2); params_.Set("IB_TIMEOUT", 14); params_.Set("IB_RETRY_CNT", 5); - ScopedDebugLogging info(NCCL_LOG_INFO, NCCL_ALL); - std::string log = CaptureLog([&] { ASSERT_EQ(IbCastCapProbeDevices(), ncclSuccess); }); + ASSERT_EQ(IbCastCapProbeDevices(), ncclSuccess); EXPECT_EQ(dev_->rdmaReadSupported, 1); EXPECT_TRUE(IbCastCapRdmaReadSupported(dev_)); - EXPECT_TRUE(LogHas(log, "device ionic_0 capability probe: RDMA_READ=yes")) << log; // An RC QP of depth 1 that connects to itself. EXPECT_EQ(created_.type, IBV_QPT_RC); EXPECT_EQ(created_.maxSendWorkRequest, 1u); @@ -306,9 +301,7 @@ TEST_F(NetIbCastCapabilityReadMicrotest, ProbeRead_CompletionError_IsUnsupported ScopedDebugLogging info(NCCL_LOG_INFO, NCCL_ALL); std::string log = CaptureLog([&] { ASSERT_EQ(IbCastCapProbeDevices(), ncclSuccess); }); EXPECT_EQ(dev_->rdmaReadSupported, 0); - EXPECT_TRUE(LogHas(log, "RDMA READ probe completed with status")) << log; EXPECT_TRUE(LogHas(log, "vendor_err 85")) << log; - EXPECT_TRUE(LogHas(log, "cannot complete an RDMA READ; port failover probing will fail on it")) << log; } TEST_F(NetIbCastCapabilityReadMicrotest, ProbeRead_SuccessWithoutTheData_IsUnsupported) { diff --git a/projects/rccl/test/host/net-ib-cast-common-test.cc b/projects/rccl/test/host/net-ib-cast-common-test.cc index a4a70c175fc4..4ba46c664d8f 100644 --- a/projects/rccl/test/host/net-ib-cast-common-test.cc +++ b/projects/rccl/test/host/net-ib-cast-common-test.cc @@ -26,7 +26,6 @@ namespace { using RcclUnitTesting::CaptureLog; using RcclUnitTesting::LogHas; -using RcclUnitTesting::ScopedDebugLogging; class NetIbCastCommonMicrotest : public ::testing::Test { protected: @@ -53,8 +52,7 @@ TEST_F(NetIbCastCommonMicrotest, StatsCheckFatalCount_AnyFatalErrorStopsTheCalle IbCastStatsInit(&stats); EXPECT_EQ(IbCastStatsCheckFatalCount(&stats, "here"), ncclSuccess); IbCastStatsFatalError(&stats); - std::string log = CaptureLog([&] { EXPECT_EQ(IbCastStatsCheckFatalCount(&stats, "here"), ncclSystemError); }); - EXPECT_TRUE(LogHas(log, "fatal error (detected in here)")) << log; + EXPECT_EQ(IbCastStatsCheckFatalCount(&stats, "here"), ncclSystemError); params_.Set("IB_RETURN_ASYNC_EVENTS", 0); EXPECT_EQ(IbCastStatsCheckFatalCount(&stats, "here"), ncclSuccess); } @@ -133,21 +131,14 @@ TEST_F(NetIbCastCommonMicrotest, RecvCommInit_PostsOnDemandByDefault) { TEST_F(NetIbCastCommonMicrotest, RecvCommInit_ResiliencyOrOooRq_ForcePrePosting) { ZeroedPtr recv; params_.Set("IB_PREPOST_RECEIVE_WORK_REQUESTS", 0); - ScopedDebugLogging info(NCCL_LOG_INFO, NCCL_ALL); auto res = std::make_unique(); resiliency_ = res.get(); - std::string log = CaptureLog([&] { ASSERT_EQ(IbCastRecvCommInit(recv.get()), ncclSuccess); }); + ASSERT_EQ(IbCastRecvCommInit(recv.get()), ncclSuccess); EXPECT_TRUE(recv->prepostReceiveWorkRequests); - EXPECT_TRUE(LogHas(log, "Overriding pre-posting to true (1).")) << log; resiliency_ = nullptr; params_.Set("IB_OOO_RQ", 1); - log = CaptureLog([&] { ASSERT_EQ(IbCastRecvCommInit(recv.get()), ncclSuccess); }); + ASSERT_EQ(IbCastRecvCommInit(recv.get()), ncclSuccess); EXPECT_TRUE(recv->prepostReceiveWorkRequests); - EXPECT_TRUE(LogHas(log, "OOO RQ is enabled, Overriding pre-posting")) << log; - // Already asked for: no override to announce. - params_.Set("IB_PREPOST_RECEIVE_WORK_REQUESTS", 1); - log = CaptureLog([&] { ASSERT_EQ(IbCastRecvCommInit(recv.get()), ncclSuccess); }); - EXPECT_FALSE(LogHas(log, "Overriding")) << log; } struct OptRecv { @@ -329,11 +320,10 @@ TEST_F(NetIbCastCommonEventMicrotest, GidChange_RefreshesTheGidAndPortAttributes info->localGid.raw[15] = 9; return ncclSuccess; }); - ScopedDebugLogging info(NCCL_LOG_INFO, NCCL_ALL); - std::string log = CaptureLog([&] { EXPECT_EQ(ncclIbCastGidChangeEvent(0), ncclSuccess); }); + EXPECT_EQ(ncclIbCastGidChangeEvent(0), ncclSuccess); EXPECT_EQ(dev_->gidInfo.localGidIndex, 2); + EXPECT_EQ(dev_->gidInfo.localGid.raw[15], 9); EXPECT_EQ(dev_->portAttr.lid, 42); - EXPECT_TRUE(LogHas(log, "GID refreshed: idx 1 -> 2, gid :: -> ::9")) << log; } TEST_F(NetIbCastCommonEventMicrotest, GidChange_QueryFailures_LeaveTheDeviceUnchanged) { @@ -345,11 +335,9 @@ TEST_F(NetIbCastCommonEventMicrotest, GidChange_QueryFailures_LeaveTheDeviceUnch }); ScopedHook gid(g_ibCastGidInfoQuery, [](struct ibv_context*, uint8_t, struct ibv_port_attr*, struct ncclIbGidInfo*) { return ncclInternalError; }); - std::string log = CaptureLog([&] { EXPECT_EQ(ncclIbCastGidChangeEvent(0), ncclSystemError); }); - EXPECT_TRUE(LogHas(log, "query_port failed during GID change")) << log; + EXPECT_EQ(ncclIbCastGidChangeEvent(0), ncclSystemError); portResult = ncclSuccess; - log = CaptureLog([&] { EXPECT_EQ(ncclIbCastGidChangeEvent(0), ncclInternalError); }); - EXPECT_TRUE(LogHas(log, "GID info query failed")) << log; + EXPECT_EQ(ncclIbCastGidChangeEvent(0), ncclInternalError); EXPECT_EQ(dev_->portAttr.lid, 7); } @@ -370,14 +358,12 @@ TEST_F(NetIbCastCommonEventMicrotest, AsyncThread_FatalEvents_ChargeTheDeviceCqA attr->qp_state = IBV_QPS_ERR; return ncclSuccess; }); - std::string log = CaptureLog([&] { EXPECT_EQ(IbCastAsyncThreadMain(dev_), nullptr); }); + EXPECT_EQ(IbCastAsyncThreadMain(dev_), nullptr); EXPECT_EQ(dev_->stats.fatalErrorCount, 1); EXPECT_EQ(cqStats.fatalErrorCount, 1); EXPECT_EQ(qpStats.fatalErrorCount, 3); EXPECT_EQ(query.calls, 3); EXPECT_EQ(ack_.calls, 5); - EXPECT_TRUE(LogHas(log, "ionic_0:1 async fatal event: fake event")) << log; - EXPECT_TRUE(LogHas(log, "async fatal event on QP")) << log; } TEST_F(NetIbCastCommonEventMicrotest, AsyncThread_QpEventWithCqs_ReportsTheirHandles) { @@ -406,9 +392,9 @@ TEST_F(NetIbCastCommonEventMicrotest, AsyncThread_NonFatalEvents_OnlyWarn) { std::string log = CaptureLog([&] { IbCastAsyncThreadMain(dev_); }); EXPECT_EQ(dev_->stats.fatalErrorCount, 0); EXPECT_EQ(ack_.calls, 14); - EXPECT_TRUE(LogHas(log, "async fatal event on SRQ, unused for now")) << log; - const std::string nonFatal = "Got non-fatal async event: fake event(" + std::to_string(IBV_EVENT_PORT_ERR) + ")"; - EXPECT_TRUE(LogHas(log, nonFatal.c_str())) << log; + // These arms only warn. + EXPECT_TRUE(LogHas(log, "on SRQ")) << log; + EXPECT_TRUE(LogHas(log, "non-fatal async event")) << log; EXPECT_TRUE(LogHas(log, "unknown event type (999)")) << log; } @@ -420,9 +406,8 @@ TEST_F(NetIbCastCommonEventMicrotest, AsyncThread_GidChangeHandlerFails_MarksThe }); ScopedHook gid(g_ibCastGidInfoQuery, [](struct ibv_context*, uint8_t, struct ibv_port_attr*, struct ncclIbGidInfo*) { return ncclSuccess; }); - std::string log = CaptureLog([&] { IbCastAsyncThreadMain(dev_); }); + IbCastAsyncThreadMain(dev_); EXPECT_EQ(dev_->stats.fatalErrorCount, 1); - EXPECT_TRUE(LogHas(log, "marking device with fatal error after GID-change event handler failed")) << log; } TEST_F(NetIbCastCommonEventMicrotest, AsyncThread_EventNameOrAckFails_EndsTheLoop) { diff --git a/projects/rccl/test/host/net-ib-cast-gdr-test.cc b/projects/rccl/test/host/net-ib-cast-gdr-test.cc index 12ab9ed564d3..8004d7d88449 100644 --- a/projects/rccl/test/host/net-ib-cast-gdr-test.cc +++ b/projects/rccl/test/host/net-ib-cast-gdr-test.cc @@ -23,32 +23,23 @@ #include "fakes/gdr_probe_fakes.h" #include "fakes/ibvwrap_fakes.h" #include "fakes/libc_fakes.h" +#include "fakes/os_fakes.h" #include "gdr_peermem.h" #include "net-ib-cast-test-common.h" #include "fakes/param_redirect.h" -// os_fakes.cc answers every sysfs read with one fixed string; the Hyper-V override needs per-file -// answers, so gdr.cc's reads are renamed onto this TU's seam. -static std::function g_gdrSysfsRead = - [](const char*, const char*, char*, int) { return ncclSystemError; }; -static ncclResult_t GdrTestSysfsRead(const char* path, const char* file, char* value, int maxLen) { - return g_gdrSysfsRead(path, file, value, maxLen); -} - -#define ncclOsTopoGetStrFromSys GdrTestSysfsRead #include "fakes/libc_seam.h" #include NET_IB_CAST_GDR_CC_PATH #include "fakes/libc_seam_undef.h" -#undef ncclOsTopoGetStrFromSys namespace { using RcclUnitTesting::CaptureLog; using RcclUnitTesting::LogHas; -// Answers bios_version and numa_balancing; an absent answer is a failed read. -void SysfsAnswers(const char* bios, const char* numaBalancing) { - g_gdrSysfsRead = [=](const char*, const char* file, char* value, int maxLen) { +// A g_ncclOsTopoGetStrFromSys that answers bios_version and numa_balancing; an absent answer is a failed read. +auto SysfsAnswers(const char* bios, const char* numaBalancing) { + return [=](const char*, const char* file, char* value, int maxLen) { const char* answer = strcmp(file, "bios_version") == 0 ? bios : strcmp(file, "numa_balancing") == 0 ? numaBalancing : nullptr; if (answer == nullptr) return ncclSystemError; @@ -73,7 +64,7 @@ TEST_F(NetIbCastGdrMicrotestIsolated, GdrSupport_ForceEnable_SucceedsWithoutScan TEST_F(NetIbCastGdrMicrotestIsolated, GdrSupport_PeermemClientFound_SucceedsWithoutProbing) { RUN_ISOLATED_TEST("NetIbCastGdr_PeermemFound", []() { - SysfsAnswers("Generic BIOS", "1"); + ScopedHook sysfs(g_ncclOsTopoGetStrFromSys, SysfsAnswers("Generic BIOS", "1")); ScopedHook scan(g_ncclIbScanDefaultPeerMemClients, [] { return 1; }); ScopedHook probe(g_ncclIbProbeGdrSupport, [](struct ibv_context*, int) { return 0; }); EXPECT_EQ(IbCastGdrSupport(), ncclSuccess); @@ -128,7 +119,7 @@ TEST_F(NetIbCastGdrMicrotestIsolated, GdrSupport_HyperVWithNumaBalancingAndNoRel RUN_ISOLATED_TEST("NetIbCastGdr_HyperVBlocklisted", []() { ParamScript params; params.Set("IB_PCI_RELAXED_ORDERING", 0); - SysfsAnswers(kHyperV, "1"); + ScopedHook sysfs(g_ncclOsTopoGetStrFromSys, SysfsAnswers(kHyperV, "1")); ScopedHook scan(g_ncclIbScanDefaultPeerMemClients, [] { return 1; }); ScopedHook probe(g_ncclIbProbeGdrSupport, [](struct ibv_context*, int) { return 1; }); EXPECT_EQ(IbCastGdrSupport(), ncclSystemError); @@ -149,7 +140,7 @@ TEST_P(NetIbCastGdrHyperVMicrotestIsolated, GdrSupport_HyperVWithoutTheOtherCond RUN_ISOLATED_TEST((std::string("NetIbCastGdr_HyperV_") + c.name).c_str(), [c]() { ParamScript params; params.Set("IB_PCI_RELAXED_ORDERING", c.relaxedOrdering); - SysfsAnswers(kHyperV, c.numaBalancing); + ScopedHook sysfs(g_ncclOsTopoGetStrFromSys, SysfsAnswers(kHyperV, c.numaBalancing)); ScopedHook scan(g_ncclIbScanDefaultPeerMemClients, [] { return 1; }); EXPECT_EQ(IbCastGdrSupport(), ncclSuccess); }); @@ -166,7 +157,7 @@ TEST_F(NetIbCastGdrMicrotestIsolated, GdrSupport_BiosUnreadable_SkipsTheHyperVCh RUN_ISOLATED_TEST("NetIbCastGdr_BiosUnreadable", []() { ParamScript params; params.Set("IB_PCI_RELAXED_ORDERING", 0); - SysfsAnswers(nullptr, "1"); + ScopedHook sysfs(g_ncclOsTopoGetStrFromSys, SysfsAnswers(nullptr, "1")); ScopedHook scan(g_ncclIbScanDefaultPeerMemClients, [] { return 1; }); EXPECT_EQ(IbCastGdrSupport(), ncclSuccess); }); diff --git a/projects/rccl/test/host/net-ib-cast-gin-test.cc b/projects/rccl/test/host/net-ib-cast-gin-test.cc index 2d3f55fc57f9..4254b44db8f9 100644 --- a/projects/rccl/test/host/net-ib-cast-gin-test.cc +++ b/projects/rccl/test/host/net-ib-cast-gin-test.cc @@ -20,6 +20,7 @@ #include "../common/LogCapture.hpp" #include "../common/ProcessIsolatedTestRunner.hpp" +#include "AsanCheck.h" #include "fakes/gdr_peermem_fakes.h" #include "fakes/gdr_probe_fakes.h" #include "fakes/net_ib_cast_p2p_fakes.h" @@ -29,23 +30,8 @@ #include NET_IB_CAST_GIN_CC_PATH -#define GIN_TEST_ASAN 0 -#if defined(__SANITIZE_ADDRESS__) -#undef GIN_TEST_ASAN -#define GIN_TEST_ASAN 1 -#elif defined(__has_feature) -#if __has_feature(address_sanitizer) -#undef GIN_TEST_ASAN -#define GIN_TEST_ASAN 1 -#endif -#endif - namespace { -// gin.cc's error paths leak what they allocated; tests that take one skip under ASan, whose leak -// check would fail the binary. -constexpr bool kAsan = GIN_TEST_ASAN; - using RcclUnitTesting::CaptureLog; using RcclUnitTesting::LogHas; @@ -247,8 +233,7 @@ TEST_F(NetIbCastGinMicrotestIsolated, GdrSupport_NeitherPeermemNorDmaBuf_Refuses bool gdr = true; ASSERT_EQ(IbCastGinIbGdrSupport(&gdr, false), ncclSuccess); EXPECT_FALSE(gdr); - std::string log = CaptureLog([&] { EXPECT_EQ(IbCastGinIbGdrGpuSupport(false), ncclInvalidUsage); }); - EXPECT_TRUE(LogHas(log, "Peermem is not supported, and DMA-BUF is not available")) << log; + EXPECT_EQ(IbCastGinIbGdrGpuSupport(false), ncclInvalidUsage); void* collComm = nullptr; EXPECT_EQ(IbCastRmaIbProxyConnect(nullptr, nullptr, 2, 0, nullptr, &collComm), ncclInvalidUsage); }); @@ -688,8 +673,7 @@ TEST_F(NetIbCastGinContextMicrotest, CreateContext_NoRankStride_MeansEveryRank) TEST_F(NetIbCastGinContextMicrotest, CreateContext_StrideThatDoesNotDivideTheRanks_IsRejected) { void* ctx = reinterpret_cast(0x1); - std::string log = CaptureLog([&] { EXPECT_EQ(Create(1, 3, &ctx), ncclInternalError); }); - EXPECT_TRUE(LogHas(log, "invalid rank stride 3, nranks (4) must be a multiple of it")) << log; + EXPECT_EQ(Create(1, 3, &ctx), ncclInternalError); EXPECT_EQ(listen_.calls, 0); } @@ -994,11 +978,8 @@ TEST_F(NetIbCastGinDataMicrotest, IPutSignal_KnownBug_DataWithoutADestinationPos TEST_F(NetIbCastGinDataMicrotest, IPutSignal_UnsupportedOp_IsRejectedBeforeAnyPost) { void* request = nullptr; - std::string log = CaptureLog([&] { - EXPECT_EQ(IbCastRmaIbProxyIPutSignal(proxyCtx_, 0, 0, &src_, 8, 0, &dst_, 1, 0, &sig_, 1, 0x7, false, 0, &request), - ncclInvalidArgument); - }); - EXPECT_TRUE(LogHas(log, "Unsupported signalOp 7")) << log; + EXPECT_EQ(IbCastRmaIbProxyIPutSignal(proxyCtx_, 0, 0, &src_, 8, 0, &dst_, 1, 0, &sig_, 1, 0x7, false, 0, &request), + ncclInvalidArgument); EXPECT_TRUE(posted_.empty()); } @@ -1130,8 +1111,8 @@ TEST_F(NetIbCastGinTestMicrotest, Test_ErrorCompletion_IsARemoteErrorNamingTheHc Completions({{static_cast(a), IBV_WC_REM_ACCESS_ERR}}); int done = -1; std::string log = CaptureLog([&] { EXPECT_EQ(IbCastRmaIbProxyTest(&cComm_, a, &done), ncclRemoteError); }); - EXPECT_TRUE(LogHas(log, ("status=" + std::to_string(IBV_WC_REM_ACCESS_ERR)).c_str())) << log; - EXPECT_TRUE(LogHas(log, "vendor err 129 (IPut) localGid ::1 remoteGids::2 hca ionic_7")) << log; + EXPECT_TRUE(LogHas(log, "localGid ::1 remoteGids::2")) << log; + EXPECT_TRUE(LogHas(log, "hca ionic_7")) << log; EXPECT_EQ(done, 0); } diff --git a/projects/rccl/test/host/net-ib-cast-init-test.cc b/projects/rccl/test/host/net-ib-cast-init-test.cc index 838dc21224d8..4851bb2c2932 100644 --- a/projects/rccl/test/host/net-ib-cast-init-test.cc +++ b/projects/rccl/test/host/net-ib-cast-init-test.cc @@ -10,9 +10,7 @@ #include #include -#include #include -#include #include #include @@ -41,18 +39,6 @@ static char* InitTestRealpath(const char* path, char* resolved) { return g_initR #include NET_IB_CAST_INIT_CC_PATH #undef realpath -std::atomic g_netIbCastAsyncThreadsStarted{0}; -static char g_lastThreadName[64]; - -// init.cc names each async-event thread right after starting it. -void ncclSetThreadName(pthread_t, const char* fmt, ...) { - va_list ap; - va_start(ap, fmt); - vsnprintf(g_lastThreadName, sizeof(g_lastThreadName), fmt, ap); - va_end(ap); - g_netIbCastAsyncThreadsStarted++; -} - void NetIbCastResetGlobals() { netRefCount = 0; for (struct ncclIbDev& dev : IbCastDevs) { @@ -218,10 +204,9 @@ TEST_F(NetIbCastInitMicrotest, GetPciPath_NoSysfsEntry_LeavesAnEmptyFullPath) { char hca[] = "mlx5_9"; char full[PATH_MAX] = "stale"; char* path = reinterpret_cast(0x1); - std::string log = CaptureLog([&] { ASSERT_EQ(IbCastGetPciPath(hca, &path, full), ncclSuccess); }); + ASSERT_EQ(IbCastGetPciPath(hca, &path, full), ncclSuccess); EXPECT_EQ(path, nullptr); EXPECT_STREQ(full, ""); - EXPECT_TRUE(LogHas(log, "Could not find real path of mlx5_9")) << log; } TEST_F(NetIbCastInitMicrotest, GetRealPort_CountsEarlierDevicesOnTheSamePhysicalNic) { @@ -303,15 +288,11 @@ TEST_F(NetIbCastInitMicrotest, PlaneIndex_AssignsIndexesInFirstSeenOrder) { TEST_F(NetIbCastInitMicrotest, PlaneIndex_TableFullOrVirtualBitSet_IsRejected) { int16_t count = 1, planes[NCCL_IB_PLANE_MAX_INDEX] = {NCCL_NET_ID_UNDEF}, idx = -1; - std::string log = CaptureLog([&] { - EXPECT_EQ(ncclIbCastTestGetPlaneIndex(NCCL_IB_PLANE_VIRT_BIT | 1, &count, planes, &idx), ncclInvalidUsage); - }); - EXPECT_TRUE(LogHas(log, "cannot use a plane ID")) << log; + EXPECT_EQ(ncclIbCastTestGetPlaneIndex(NCCL_IB_PLANE_VIRT_BIT | 1, &count, planes, &idx), ncclInvalidUsage); for (int p = 1; count < NCCL_IB_PLANE_MAX_INDEX - 1; p++) { ASSERT_EQ(ncclIbCastTestGetPlaneIndex(p, &count, planes, &idx), ncclSuccess); } - log = CaptureLog([&] { EXPECT_EQ(ncclIbCastTestGetPlaneIndex(100, &count, planes, &idx), ncclInvalidUsage); }); - EXPECT_TRUE(LogHas(log, "cannot use more than 14 plane IDs")) << log; + EXPECT_EQ(ncclIbCastTestGetPlaneIndex(100, &count, planes, &idx), ncclInvalidUsage); ASSERT_EQ(ncclIbCastTestGetPlaneIndex(1, &count, planes, &idx), ncclSuccess); // known ones still resolve } @@ -451,8 +432,7 @@ TEST_F(NetIbCastInitVDeviceMicrotest, MakeVDevice_SeveralDevices_FusesNamesSpeed IbCastDevs[2].planeIdx = 2; IbCastOffloadEnabled = true; int d = -1; - ScopedDebugLogging info(NCCL_LOG_INFO, NCCL_ALL); - std::string log = CaptureLog([&] { ASSERT_EQ(Make({0, 2, 0}, &d), ncclSuccess); }); + ASSERT_EQ(Make({0, 2, 0}, &d), ncclSuccess); const ncclIbMergedDev& m = IbCastMergedDevs[d]; EXPECT_STREQ(m.devName, "mlx5_0+mlx5_2"); // the repeated device is merged once EXPECT_EQ(m.vProps.ndevs, 2); @@ -460,7 +440,6 @@ TEST_F(NetIbCastInitVDeviceMicrotest, MakeVDevice_SeveralDevices_FusesNamesSpeed EXPECT_EQ(m.railId, 3); EXPECT_EQ(m.planeId, NCCL_IB_PLANE_VIRT_BIT | (1 << 1) | (1 << 2)); EXPECT_FALSE(IbCastOffloadEnabled); // CTS offload does not support fused NICs - EXPECT_TRUE(LogHas(log, "disabling CTS Offload")) << log; } TEST_F(NetIbCastInitVDeviceMicrotest, MakeVDevice_DifferentRails_LeavesTheRailUndefined) { @@ -474,8 +453,7 @@ TEST_F(NetIbCastInitVDeviceMicrotest, MakeVDevice_DifferentRails_LeavesTheRailUn TEST_F(NetIbCastInitVDeviceMicrotest, MakeVDevice_MergeNicsOff_RefusesToFuse) { params_.Set("IB_MERGE_NICS", 0); int d = -1; - std::string log = CaptureLog([&] { EXPECT_EQ(Make({0, 1}, &d), ncclInvalidUsage); }); - EXPECT_TRUE(LogHas(log, "Skipping makeVDevice, Please set NCCL_IB_MERGE_NICS=1")) << log; + EXPECT_EQ(Make({0, 1}, &d), ncclInvalidUsage); EXPECT_EQ(IbCastNMergedDevs, 0); ASSERT_EQ(Make({0}, &d), ncclSuccess); // a single device is not a fusion } @@ -484,8 +462,7 @@ TEST_F(NetIbCastInitVDeviceMicrotest, MakeVDevice_AinicWithMergeNicsNotOne_Refus g_rcclUseAinicValue = true; params_.Set("IB_MERGE_NICS", 2); int d = -1; - std::string log = CaptureLog([&] { EXPECT_EQ(Make({0, 1}, &d), ncclInvalidUsage); }); - EXPECT_TRUE(LogHas(log, "(disabled by default on AINIC)")) << log; + EXPECT_EQ(Make({0, 1}, &d), ncclInvalidUsage); } // AINIC is meant to fuse NICs only on an explicit NCCL_IB_MERGE_NICS=1, but 1 is the default. @@ -500,19 +477,16 @@ TEST_F(NetIbCastInitVDeviceMicrotest, MakeVDevice_KnownBug_AinicFusesNicsByDefau TEST_F(NetIbCastInitVDeviceMicrotest, MakeVDevice_BadDeviceCounts_AreRejected) { int d = -1; ncclNetVDeviceProps_t props{}; - std::string log = CaptureLog([&] { EXPECT_EQ(IbCastMakeVDevice(&d, &props), ncclInvalidUsage); }); - EXPECT_TRUE(LogHas(log, "with 0 devices")) << log; + EXPECT_EQ(IbCastMakeVDevice(&d, &props), ncclInvalidUsage); props.ndevs = -1; EXPECT_EQ(IbCastMakeVDevice(&d, &props), ncclInvalidUsage); props.ndevs = NCCL_IB_MAX_DEVS_PER_NIC + 1; - log = CaptureLog([&] { EXPECT_EQ(IbCastMakeVDevice(&d, &props), ncclInvalidUsage); }); - EXPECT_TRUE(LogHas(log, "Can't make virtual NIC with 9 devices, max 8")) << log; + EXPECT_EQ(IbCastMakeVDevice(&d, &props), ncclInvalidUsage); } TEST_F(NetIbCastInitVDeviceMicrotest, MakeVDevice_UnknownPhysicalDevice_IsRejected) { int d = -1; - std::string log = CaptureLog([&] { EXPECT_EQ(Make({0, 3}, &d), ncclInvalidUsage); }); - EXPECT_TRUE(LogHas(log, "Cannot use physical device 3, max 3")) << log; + EXPECT_EQ(Make({0, 3}, &d), ncclInvalidUsage); EXPECT_EQ(Make({-1}, &d), ncclInvalidUsage); EXPECT_EQ(IbCastNMergedDevs, 0); } @@ -520,15 +494,13 @@ TEST_F(NetIbCastInitVDeviceMicrotest, MakeVDevice_UnknownPhysicalDevice_IsReject TEST_F(NetIbCastInitVDeviceMicrotest, MakeVDevice_TableFull_IsRejected) { IbCastNMergedDevs = MAX_IB_VDEVS; int d = -1; - std::string log = CaptureLog([&] { EXPECT_EQ(Make({0}, &d), ncclInvalidUsage); }); - EXPECT_TRUE(LogHas(log, "Cannot allocate any more virtual devices (256)")) << log; + EXPECT_EQ(Make({0}, &d), ncclInvalidUsage); } TEST_F(NetIbCastInitVDeviceMicrotest, MakeVDevice_MixedLinkLayers_IsRejected) { IbCastDevs[1].link = IBV_LINK_LAYER_INFINIBAND; int d = -1; - std::string log = CaptureLog([&] { EXPECT_EQ(Make({0, 1}, &d), ncclInvalidUsage); }); - EXPECT_TRUE(LogHas(log, "[0]mlx5_0:1/RoCE and [1]mlx5_1:1/IB")) << log; + EXPECT_EQ(Make({0, 1}, &d), ncclInvalidUsage); EXPECT_EQ(IbCastNMergedDevs, 0); } @@ -546,9 +518,9 @@ TEST_F(NetIbCastInitVDeviceMicrotest, MakeVDevice_AcrossNumaNodesOrRootComplexes SetPciPath(2, p2); int d = -1; std::string log = CaptureLog([&] { ASSERT_EQ(Make({0, 1}, &d), ncclSuccess); }); - EXPECT_TRUE(LogHas(log, "Merging NICs across NUMA nodes (mlx5_0 numa=0, mlx5_1 numa=1)")) << log; + EXPECT_TRUE(LogHas(log, "across NUMA nodes")) << log; log = CaptureLog([&] { ASSERT_EQ(Make({0, 2}, &d), ncclSuccess); }); - EXPECT_TRUE(LogHas(log, "PCIe Root Complexes (mlx5_0 root=0000:00, mlx5_2 root=0000:80)")) << log; + EXPECT_TRUE(LogHas(log, "PCIe Root Complexes")) << log; EXPECT_FALSE(LogHas(log, "NUMA")) << log; std::filesystem::remove_all(base); } @@ -566,8 +538,7 @@ class NetIbCastInitDevicesMicrotest : public NetIbCastInitMicrotest { TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_OneRoceHca_AddsOnePhysicalAndOneVirtualDevice) { World().AddHca("ionic_0"); PciPaths({{"ionic_0", "/sys/devices/pci0000:00/0000:01:00.0"}}); - ScopedDebugLogging info(NCCL_LOG_INFO, NCCL_ALL); - std::string log = CaptureLog([&] { ASSERT_EQ(Init(), ncclSuccess); }); + ASSERT_EQ(Init(), ncclSuccess); ASSERT_EQ(IbCastNDevs, 1); ASSERT_EQ(IbCastNMergedDevs, 1); const struct ncclIbDev& dev = IbCastDevs[0]; @@ -590,8 +561,7 @@ TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_OneRoceHca_AddsOnePhysicalAndO EXPECT_STREQ(IbCastMergedDevs[0].devName, "ionic_0"); EXPECT_STREQ(IbCastIfName, "eth0"); EXPECT_EQ(IbCastRelaxedOrderingEnabled, 0); - EXPECT_STREQ(g_lastThreadName, "NCCL IbAsync 0"); - EXPECT_TRUE(LogHas(log, "NET/IB : Using [0]ionic_0:1/RoCE")) << log; + EXPECT_EQ(World().threadNames, std::vector{"NCCL IbAsync 0"}); EXPECT_EQ(g_ibCastReportMatchingSchemeCalls, 1); } @@ -652,17 +622,14 @@ TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_NoVerbsLibrary_Fails) { TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_NoMlx5Library_CarriesOn) { World().AddHca("ionic_0"); ScopedHook symbols(g_mlx5dvSymbols, [] { return ncclSystemError; }); - ScopedDebugLogging info(NCCL_LOG_INFO, NCCL_ALL); - std::string log = CaptureLog([&] { EXPECT_EQ(Init(), ncclSuccess); }); - EXPECT_TRUE(LogHas(log, "Failed to open mlx5dv symbols")) << log; + EXPECT_EQ(Init(), ncclSuccess); EXPECT_EQ(IbCastNDevs, 1); } TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_NoIonicLibrary_DisablesTheTransport) { World().AddHca("ionic_0"); ScopedHook symbols(g_ionicdvSymbols, [] { return ncclSystemError; }); - std::string log = CaptureLog([&] { EXPECT_EQ(Init(), ncclInternalError); }); - EXPECT_TRUE(LogHas(log, "libionic is missing")) << log; + EXPECT_EQ(Init(), ncclInternalError); EXPECT_EQ(World().getDeviceList.calls, 0); } @@ -672,8 +639,7 @@ TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_NoIpInterface_Fails) { *n = 0; return ncclSuccess; }); - std::string log = CaptureLog([&] { EXPECT_EQ(Init(), ncclInternalError); }); - EXPECT_TRUE(LogHas(log, "No IP interface found")) << log; + EXPECT_EQ(Init(), ncclInternalError); EXPECT_EQ(World().freeDeviceList.calls, 0); // never listed } @@ -692,11 +658,9 @@ TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_DeviceListFails_Fails) { TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_NoHca_FindsNothingButSucceeds) { World(); - ScopedDebugLogging info(NCCL_LOG_INFO, NCCL_ALL); - std::string log = CaptureLog([&] { EXPECT_EQ(Init(), ncclSuccess); }); + EXPECT_EQ(Init(), ncclSuccess); EXPECT_EQ(IbCastNDevs, 0); EXPECT_EQ(IbCastNMergedDevs, 0); - EXPECT_TRUE(LogHas(log, "No device found")) << log; EXPECT_EQ(g_ibCastReportMatchingSchemeCalls, 0); EXPECT_EQ(World().freeDeviceList.calls, 1); } @@ -704,8 +668,7 @@ TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_NoHca_FindsNothingButSucceeds) TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_UnopenableHca_IsSkipped) { World().AddHca("ionic_0").openFails = true; World().AddHca("ionic_1"); - std::string log = CaptureLog([&] { ASSERT_EQ(Init(), ncclSuccess); }); - EXPECT_TRUE(LogHas(log, "Unable to open device ionic_0")) << log; + ASSERT_EQ(Init(), ncclSuccess); ASSERT_EQ(IbCastNDevs, 1); EXPECT_STREQ(IbCastDevs[0].devName, "ionic_1"); } @@ -716,8 +679,7 @@ TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_OpenSucceedsWithoutAContext_Is *ret = nullptr; return ncclSuccess; }); - std::string log = CaptureLog([&] { ASSERT_EQ(Init(), ncclSuccess); }); - EXPECT_TRUE(LogHas(log, "Unable to open device ionic_0")) << log; + ASSERT_EQ(Init(), ncclSuccess); EXPECT_EQ(World().queryDevice.calls, 0); EXPECT_EQ(IbCastNDevs, 0); } @@ -726,8 +688,7 @@ TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_UnqueryableHca_IsClosedAndSkip FakeHca& bad = World().AddHca("ionic_0"); bad.queryDeviceFails = true; World().AddHca("ionic_1"); - std::string log = CaptureLog([&] { ASSERT_EQ(Init(), ncclSuccess); }); - EXPECT_TRUE(LogHas(log, "Unable to query device ionic_0")) << log; + ASSERT_EQ(Init(), ncclSuccess); EXPECT_EQ(bad.closes, 1); EXPECT_EQ(IbCastNDevs, 1); } @@ -749,8 +710,7 @@ TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_PortsThatAreDownUnqueryableOrN *attr = hca.attr; return ncclSuccess; }); - std::string log = CaptureLog([&] { ASSERT_EQ(Init(), ncclSuccess); }); - EXPECT_TRUE(LogHas(log, "Unable to query port_num 4")) << log; + ASSERT_EQ(Init(), ncclSuccess); ASSERT_EQ(IbCastNDevs, 1); EXPECT_EQ(IbCastDevs[0].portNum, 3); EXPECT_EQ(IbCastDevs[0].link, IBV_LINK_LAYER_INFINIBAND); @@ -776,9 +736,8 @@ TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_ClosingAnUnusedHcaFails_Fails) TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_FreeingTheListFails_Fails) { World().AddHca("ionic_0"); ScopedHook freeList(g_ibvFreeDeviceList, [](struct ibv_device**) { return ncclSystemError; }); - std::string log = CaptureLog([&] { EXPECT_EQ(Init(), ncclInternalError); }); + EXPECT_EQ(Init(), ncclInternalError); EXPECT_EQ(freeList.calls, 2); // the failure path tries once more - EXPECT_TRUE(LogHas(log, "Unable to free device list")) << log; } TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_PortSpeedQuery_WinsOverThePortAttributes) { @@ -821,15 +780,13 @@ TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_IbHcaRailAndPlane_AreRecordedA World().AddHca("ionic_1"); World().AddHca("ionic_2"); SetMicroEnv("NCCL_IB_HCA", "ionic_0:1:7,ionic_1::2:5,ionic_2:::6"); - ScopedDebugLogging info(NCCL_LOG_INFO, NCCL_ALL); - std::string log = CaptureLog([&] { ASSERT_EQ(Init(), ncclSuccess); }); + ASSERT_EQ(Init(), ncclSuccess); ASSERT_EQ(IbCastNDevs, 3); EXPECT_EQ(IbCastDevs[0].railId, 7); EXPECT_EQ(IbCastDevs[1].railId, 2); EXPECT_EQ(IbCastDevs[1].planeId, 5); EXPECT_EQ(IbCastDevs[2].planeId, 6); EXPECT_NE(IbCastDevs[1].planeIdx, IbCastDevs[2].planeIdx); - EXPECT_TRUE(LogHas(log, "[0]ionic_0:1:7/RoCE [1]ionic_1:1:2:5/RoCE [2]ionic_2:1::6/RoCE")) << log; } TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_TooManyPlanes_FailsAfterListing) { @@ -840,8 +797,7 @@ TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_TooManyPlanes_FailsAfterListin hca += (i ? "," : "") + name + ":::" + std::to_string(i + 1); } SetMicroEnv("NCCL_IB_HCA", hca.c_str()); - std::string log = CaptureLog([&] { EXPECT_EQ(Init(), ncclInvalidUsage); }); - EXPECT_TRUE(LogHas(log, "cannot use more than")) << log; + EXPECT_EQ(Init(), ncclInvalidUsage); } TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_SortsByPciPathUnlessDisabled) { @@ -900,10 +856,8 @@ TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_ArThreshold_IsTakenUnlessOooRq EXPECT_EQ(IbCastArThreshold, 4096); ResetForAnotherInit(); params_.Set("IB_OOO_RQ", 1); - ScopedDebugLogging info(NCCL_LOG_INFO, NCCL_ALL); - std::string log = CaptureLog([&] { ASSERT_EQ(Init(), ncclSuccess); }); + ASSERT_EQ(Init(), ncclSuccess); EXPECT_EQ(IbCastArThreshold, 8192); - EXPECT_TRUE(LogHas(log, "AR threshold will be ignored")) << log; } TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_Mlx5WithOooRq_RecordsTheOooDepth) { @@ -960,14 +914,12 @@ class NetIbCastInitDataDirectMicrotest : public NetIbCastInitDevicesMicrotest { TEST_F(NetIbCastInitDataDirectMicrotest, InitDevices_DataDirectNic_ExposesOnlyTheDmaDevice) { DataDirect(ncclSuccess); - ScopedDebugLogging info(NCCL_LOG_INFO, NCCL_ALL); - std::string log = CaptureLog([&] { ASSERT_EQ(Init(), ncclSuccess); }); + ASSERT_EQ(Init(), ncclSuccess); ASSERT_EQ(IbCastNDevs, 1); EXPECT_STREQ(IbCastDevs[0].devName, "mlx5_0_dma"); EXPECT_STREQ(IbCastDevs[0].pciPath, "/sys/devices/dd0"); EXPECT_EQ(IbCastDevs[0].capsProvider.mlx5.dataDirect, 1); EXPECT_STREQ(IbCastDevs[0].fullPciPath, "/sys/devices/pci0000:00/0000:01:00.0"); - EXPECT_TRUE(LogHas(log, "Data Direct DMA Interface is detected for device mlx5_0")) << log; } TEST_F(NetIbCastInitDataDirectMicrotest, InitDevices_DataDirectTwo_ExposesBothDevices) { @@ -990,8 +942,7 @@ TEST_F(NetIbCastInitDataDirectMicrotest, InitDevices_NoDataDirectPath_KeepsThePl // Pins AICOMRCCL-3090; a fix flips this pin. TEST_F(NetIbCastInitDataDirectMicrotest, InitDevices_KnownBug_DataDirectLookupErrorLeaksTheDeviceList) { DataDirect(ncclSystemError); - std::string log = CaptureLog([&] { EXPECT_EQ(Init(), ncclSystemError); }); - EXPECT_TRUE(LogHas(log, "Error in mlx5dv_get_data_direct_sysfs_path with device mlx5_0")) << log; + EXPECT_EQ(Init(), ncclSystemError); EXPECT_EQ(World().freeDeviceList.calls, 0); EXPECT_EQ(World().hcas[0].closes, 0); } @@ -1014,18 +965,13 @@ class NetIbCastInitAinicMicrotest : public NetIbCastInitDevicesMicrotest { }; TEST_F(NetIbCastInitAinicMicrotest, InitDevices_Ainic_EnablesCtsOffloadAndInlineByDefault) { - ScopedDebugLogging info(NCCL_LOG_INFO, NCCL_ALL); - std::string log = CaptureLog([&] { ASSERT_EQ(Init(), ncclSuccess); }); + ASSERT_EQ(Init(), ncclSuccess); EXPECT_TRUE(IbCastAinicRoce); EXPECT_TRUE(IbCastOffloadEnabled); EXPECT_TRUE(IbCastUseInline); EXPECT_TRUE(IbCastAinicCtsInlineData); EXPECT_EQ(IbCastGdrFlushDisable, 1); EXPECT_FALSE(IbCastMultiplaneEnable); - EXPECT_TRUE(LogHas(log, "enabling IB Use Inline Data")) << log; - EXPECT_TRUE(LogHas(log, "CTS Inline Data: Enabled; CTS Offload: Enabled; IB Use Inline: Enabled; GDR Flush: " - "Disabled")) << log; - EXPECT_TRUE(LogHas(log, "AINIC uDMA policy: 0")) << log; } TEST_F(NetIbCastInitAinicMicrotest, InitDevices_AinicOffloadOff_KeepsInlineAsConfigured) { @@ -1081,20 +1027,21 @@ TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_MapFileWithoutAinicOrEmpty_Lea EXPECT_FALSE(IbCastMultiplaneEnable); } +// Init keeps no policy state: it only validates the param and reports what comms will use. TEST_F(NetIbCastInitAinicMicrotest, InitDevices_UdmaPolicyOutOfRange_FallsBackToChannelAffinity) { for (int64_t policy : {(int64_t)ncclIbCastUDMAPolicy::First - 1, (int64_t)ncclIbCastUDMAPolicy::Last + 1}) { ResetForAnotherInit(); params_.Set("RCCL_IB_UDMA_POLICY", policy); ScopedDebugLogging info(NCCL_LOG_INFO, NCCL_ALL); std::string log = CaptureLog([&] { ASSERT_EQ(Init(), ncclSuccess); }); - EXPECT_TRUE(LogHas(log, ("RCCL_IB_UDMA_POLICY=" + std::to_string(policy) + " is invalid").c_str())) << log; - EXPECT_TRUE(LogHas(log, "AINIC uDMA policy: 0")) << log; + EXPECT_TRUE(LogHas(log, "is invalid")) << log; + EXPECT_TRUE(LogHas(log, "uDMA policy: 0")) << log; } ResetForAnotherInit(); params_.Set("RCCL_IB_UDMA_POLICY", (int64_t)ncclIbCastUDMAPolicy::Last); ScopedDebugLogging info(NCCL_LOG_INFO, NCCL_ALL); std::string log = CaptureLog([&] { ASSERT_EQ(Init(), ncclSuccess); }); - EXPECT_TRUE(LogHas(log, "AINIC uDMA policy: 3")) << log; + EXPECT_TRUE(LogHas(log, "uDMA policy: 3")) << log; } TEST_F(NetIbCastInitAinicMicrotest, InitDevices_AinicWithResiliency_DropsOffloadAndInline) { @@ -1156,9 +1103,8 @@ TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_QpSharingWithNoGroups_TurnsSha World(); params_.Set("RCCL_IB_QP_SHARING_ENABLE", 1); params_.Set("RCCL_IB_COMM_NGROUPS", 0); - std::string log = CaptureLog([&] { ASSERT_EQ(Init(), ncclSuccess); }); + ASSERT_EQ(Init(), ncclSuccess); EXPECT_FALSE(IbCastQpSharingGlobalEnable); - EXPECT_TRUE(LogHas(log, "RCCL_IB_COMM_NGROUPS=0 is invalid")) << log; } TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_ByOrderWithFailoverOrOooRq_WarnsAboutWhatItDisables) { @@ -1167,7 +1113,7 @@ TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_ByOrderWithFailoverOrOooRq_War params_.Set("IB_RESILIENCY_PORT_FAILOVER", 1); params_.Set("IB_OOO_RQ", 1); std::string log = CaptureLog([&] { ASSERT_EQ(Init(), ncclSuccess); }); - EXPECT_TRUE(LogHas(log, "disabling resiliency (port failover and port recovery)")) << log; + EXPECT_TRUE(LogHas(log, "disabling resiliency")) << log; EXPECT_TRUE(LogHas(log, "disabling out-of-order RQ")) << log; EXPECT_EQ(g_ibCastReportMatchingSchemeCalls, 1); } @@ -1201,9 +1147,8 @@ TEST_F(NetIbCastInitMicrotestIsolated, InitDevices_TelemetryCannotStart_CarriesO "NetIbCastInit_TelemetryCannotStart", []() { FakeIbWorld world; - ScopedDebugLogging info(NCCL_LOG_INFO, NCCL_ALL); - std::string log = CaptureLog([&] { EXPECT_EQ(IbCastInitDevices(nullptr, nullptr), ncclSuccess); }); - EXPECT_TRUE(LogHas(log, "telemetry was requested but could not start")) << log; + EXPECT_EQ(IbCastInitDevices(nullptr, nullptr), ncclSuccess); + EXPECT_FALSE(rcclTelemetryOn()); }, {{"RCCL_TELEMETRY_ENABLE", "1"}, {"RCCL_TELEMETRY_OUTPUT_DIR", std::string(5000, 'd')}}); } @@ -1214,11 +1159,9 @@ TEST_F(NetIbCastInitMicrotestIsolated, InitDevices_TelemetryOn_RegistersEachDevi []() { FakeIbWorld world; world.AddHca("ionic_0"); - ScopedDebugLogging info(NCCL_LOG_INFO, NCCL_ALL); - std::string log = CaptureLog([&] { EXPECT_EQ(IbCastInitDevices(nullptr, nullptr), ncclSuccess); }); + EXPECT_EQ(IbCastInitDevices(nullptr, nullptr), ncclSuccess); EXPECT_TRUE(rcclTelemetryOn()); EXPECT_EQ(rcclTelemetryDevs[0].device_id, 0); - EXPECT_FALSE(LogHas(log, "telemetry did not register")) << log; }, {{"RCCL_TELEMETRY_ENABLE", "1"}, {"RCCL_TELEMETRY_OUTPUT_DIR", "/tmp"}}); } @@ -1342,8 +1285,7 @@ TEST_F(NetIbCastInitPropertiesMicrotest, GetProperties_MergedDevice_ReportsTheFu TEST_F(NetIbCastInitPropertiesMicrotest, GetProperties_UnknownVirtualDevice_IsRejected) { ncclNetProperties_t props; - std::string log = CaptureLog([&] { EXPECT_EQ(IbCastGetProperties(1, &props), ncclInvalidUsage); }); - EXPECT_TRUE(LogHas(log, "Requested properties for vNic 1, only 1 vNics have been created")) << log; + EXPECT_EQ(IbCastGetProperties(1, &props), ncclInvalidUsage); } class NetIbCastInitPropertiesMicrotestIsolated : public NetIbCastInitMicrotest {}; diff --git a/projects/rccl/test/host/net-ib-cast-multiplane-test.cc b/projects/rccl/test/host/net-ib-cast-multiplane-test.cc index ba264f9bceb4..6b8f72c13a58 100644 --- a/projects/rccl/test/host/net-ib-cast-multiplane-test.cc +++ b/projects/rccl/test/host/net-ib-cast-multiplane-test.cc @@ -27,7 +27,6 @@ namespace { using RcclUnitTesting::CaptureLog; using RcclUnitTesting::LogHas; -using RcclUnitTesting::ScopedDebugLogging; class NetIbCastMultiplaneMicrotest : public ::testing::Test { protected: @@ -101,8 +100,7 @@ TEST_F(NetIbCastMultiplaneMicrotest, IpToGid_Ipv4_StoresTheV4MappedForm) { TEST_F(NetIbCastMultiplaneMicrotest, IpToGid_NotAnAddress_IsRejected) { uint8_t gid[16]; memset(gid, 0xab, sizeof(gid)); - std::string log = CaptureLog([&] { EXPECT_EQ(ncclIbCastTestIpToGid("10.1.2", gid), ncclInvalidArgument); }); - EXPECT_TRUE(LogHas(log, "invalid IP address '10.1.2'")) << log; + EXPECT_EQ(ncclIbCastTestIpToGid("10.1.2", gid), ncclInvalidArgument); EXPECT_EQ(gid[0], 0xab); // left alone on failure } @@ -130,8 +128,7 @@ TEST_F(NetIbCastMultiplaneMicrotest, Load_EmptyMapFileName_IsTreatedAsUnset) { TEST_F(NetIbCastMultiplaneMicrotest, Load_MissingFile_FailsAndKeepsFailingWithoutRereading) { const std::string path = dir_ + "/absent.xml"; SetMicroEnv("RCCL_MULTIPLANE_MAP_FILE", path.c_str()); - std::string log = CaptureLog([&] { EXPECT_EQ(IbCastMultiplaneLoad(), ncclSystemError); }); - EXPECT_TRUE(LogHas(log, "could not open map file")) << log; + EXPECT_EQ(IbCastMultiplaneLoad(), ncclSystemError); // The load runs once: creating the file afterwards changes nothing. std::ofstream(path) << kTwoInterfaces; EXPECT_EQ(IbCastMultiplaneLoad(), ncclSystemError); @@ -139,11 +136,12 @@ TEST_F(NetIbCastMultiplaneMicrotest, Load_MissingFile_FailsAndKeepsFailingWithou TEST_F(NetIbCastMultiplaneMicrotest, Load_ValidMap_ResolvesEachVipToItsPipsInOrder) { UseMap(kTwoInterfaces); - ScopedDebugLogging info(NCCL_LOG_INFO, NCCL_ALL); - std::string log = CaptureLog([&] { ASSERT_EQ(IbCastMultiplaneLoad(), ncclSuccess); }); - EXPECT_TRUE(LogHas(log, "loaded 2 GID mappings")) << log; - EXPECT_FALSE(LogHas(log, "duplicate")) << log; - EXPECT_TRUE(LogHas(log, "ip=10.0.0.1 interface=eth1")) << log; + ASSERT_EQ(IbCastMultiplaneLoad(), ncclSuccess); + ASSERT_EQ(gidToPipMap.size(), 2u); + const std::vector& vip = gidToPipMap.at("fd00:0000:0000:0000:0000:0000:0000:0001"); + ASSERT_EQ(vip.size(), 2u); + EXPECT_STREQ(vip[0].interface, "eth1"); + EXPECT_STREQ(vip[1].interface, "eth2"); union ibv_gid pips[MULTIPLANE_MAX_PIPS]; ASSERT_EQ(Resolve("fd00::1", pips), 2); @@ -186,9 +184,7 @@ TEST_F(NetIbCastMultiplaneMicrotest, GetPipGids_MorePipsThanTheMax_ReturnsOnlyTh TEST_F(NetIbCastMultiplaneMicrotest, Load_PipWithoutInterface_LeavesTheInterfaceEmpty) { UseMap(""); - ScopedDebugLogging info(NCCL_LOG_INFO, NCCL_ALL); - std::string log = CaptureLog([&] { ASSERT_EQ(IbCastMultiplaneLoad(), ncclSuccess); }); - EXPECT_TRUE(LogHas(log, "ip=10.0.0.1 interface=\n")) << log; + ASSERT_EQ(IbCastMultiplaneLoad(), ncclSuccess); ASSERT_EQ(gidToPipMap.size(), 1u); EXPECT_STREQ(gidToPipMap.begin()->second.at(0).interface, ""); } @@ -199,7 +195,7 @@ TEST_F(NetIbCastMultiplaneMicrotest, Load_DuplicateVip_LastEntryWins) { "" ""); std::string log = CaptureLog([&] { ASSERT_EQ(IbCastMultiplaneLoad(), ncclSuccess); }); - EXPECT_TRUE(LogHas(log, "duplicate GID")) << log; + EXPECT_TRUE(LogHas(log, "duplicate GID")) << log; // a duplicate only warns union ibv_gid pips[MULTIPLANE_MAX_PIPS]; ASSERT_EQ(Resolve("fd00::1", pips), 2); EXPECT_TRUE(SameGid(pips[0], "10.0.0.2")); @@ -208,7 +204,7 @@ TEST_F(NetIbCastMultiplaneMicrotest, Load_DuplicateVip_LastEntryWins) { struct BadMap { const char* name; const char* xml; - const char* warning; + const char* warning; // the checks share a return code, so this names the one that fired }; class NetIbCastMultiplaneBadMapMicrotest : public NetIbCastMultiplaneMicrotest, @@ -225,23 +221,23 @@ TEST_P(NetIbCastMultiplaneBadMapMicrotest, Load_IsRejectedAndNothingResolves) { INSTANTIATE_TEST_SUITE_P( NetIbCastMultiplane, NetIbCastMultiplaneBadMapMicrotest, ::testing::Values( - BadMap{"NoRoot", "", "no root element"}, + BadMap{"NoRoot", "", "no root"}, BadMap{"InterfaceWithoutGid", "", - " missing 'gid'"}, + "missing 'gid'"}, BadMap{"PipWithoutIp", "" "", - " missing 'ip'"}, + "missing 'ip'"}, BadMap{"BadPipIp", "" "", - "invalid IP address '10.0.0.300'"}, + "'10.0.0.300'"}, BadMap{"BadVip", "" "", - "invalid IP address 'fd00::g'"}, - BadMap{"Unterminated", "", "failed to parse map file"}), + "'fd00::g'"}, + BadMap{"Unterminated", "", "failed to parse"}), [](const ::testing::TestParamInfo& info) { return std::string(info.param.name); }); // The parser only hands back the elements it has handlers for, so the "skip anything else" diff --git a/projects/rccl/test/host/net-ib-cast-reg-test.cc b/projects/rccl/test/host/net-ib-cast-reg-test.cc index 49d6fe7f21f8..b3e4a934d976 100644 --- a/projects/rccl/test/host/net-ib-cast-reg-test.cc +++ b/projects/rccl/test/host/net-ib-cast-reg-test.cc @@ -13,7 +13,6 @@ #include -#include "../common/LogCapture.hpp" #include "fakes/ibvwrap_fakes.h" #include "fakes/libc_fakes.h" #include "fakes/mlx5dvwrap_fakes.h" @@ -25,9 +24,6 @@ namespace { -using RcclUnitTesting::CaptureLog; -using RcclUnitTesting::LogHas; - constexpr unsigned int kBaseAccess = IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ | IBV_ACCESS_REMOTE_ATOMIC; @@ -86,12 +82,9 @@ class NetIbCastRegMicrotest : public ::testing::Test { TEST_F(NetIbCastRegMicrotest, RegMr_RejectsNullCommNullHandleAndZeroSize) { void* mhandle = nullptr; - std::string log = CaptureLog([&] { - EXPECT_EQ(IbCastRegMr(nullptr, buf_, 8, NCCL_PTR_HOST, &mhandle), ncclInvalidArgument); - EXPECT_EQ(IbCastRegMr(send_.get(), buf_, 8, NCCL_PTR_HOST, nullptr), ncclInvalidArgument); - EXPECT_EQ(IbCastRegMr(send_.get(), buf_, 0, NCCL_PTR_HOST, &mhandle), ncclInvalidArgument); - }); - EXPECT_TRUE(LogHas(log, "regMr/regMrDmaBuf called with comm=")) << log; + EXPECT_EQ(IbCastRegMr(nullptr, buf_, 8, NCCL_PTR_HOST, &mhandle), ncclInvalidArgument); + EXPECT_EQ(IbCastRegMr(send_.get(), buf_, 8, NCCL_PTR_HOST, nullptr), ncclInvalidArgument); + EXPECT_EQ(IbCastRegMr(send_.get(), buf_, 0, NCCL_PTR_HOST, &mhandle), ncclInvalidArgument); EXPECT_EQ(mhandle, nullptr); } @@ -305,8 +298,7 @@ TEST_F(NetIbCastRegMicrotest, RegMr_KnownBug_FailureOnASecondDeviceKeepsTheFirst TEST_F(NetIbCastRegMicrotest, DeregMr_NullHandle_IsANoOpAndNullCommIsRejected) { EXPECT_EQ(IbCastDeregMr(send_.get(), nullptr), ncclSuccess); ncclIbMrHandle handle{}; - std::string log = CaptureLog([&] { EXPECT_EQ(IbCastDeregMr(nullptr, &handle), ncclInvalidArgument); }); - EXPECT_TRUE(LogHas(log, "deregMr called with a null comm")) << log; + EXPECT_EQ(IbCastDeregMr(nullptr, &handle), ncclInvalidArgument); } TEST_F(NetIbCastRegMicrotest, DeregMr_UnknownMr_IsAnInternalError) { @@ -315,8 +307,7 @@ TEST_F(NetIbCastRegMicrotest, DeregMr_UnknownMr_IsAnInternalError) { ibv_mr stranger{}; ncclIbMrHandle* bogus = static_cast(malloc(sizeof(ncclIbMrHandle))); bogus->mrs[0] = &stranger; - std::string log = CaptureLog([&] { EXPECT_EQ(IbCastDeregMr(send_.get(), bogus), ncclInternalError); }); - EXPECT_TRUE(LogHas(log, "inside cache of 1 entries")) << log; + EXPECT_EQ(IbCastDeregMr(send_.get(), bogus), ncclInternalError); free(bogus); EXPECT_EQ(IbCastDeregMr(send_.get(), mhandle), ncclSuccess); } diff --git a/projects/rccl/test/host/net-ib-cast-test-common.h b/projects/rccl/test/host/net-ib-cast-test-common.h index b840e8252c1d..79a0732233fd 100644 --- a/projects/rccl/test/host/net-ib-cast-test-common.h +++ b/projects/rccl/test/host/net-ib-cast-test-common.h @@ -40,8 +40,6 @@ // Puts every net_ib_cast global back to its load-time value. Defined in // net-ib-cast-init-test.cc, the TU that can reach init.cc's file-scope state. void NetIbCastResetGlobals(); -// IbCastInitDevices starts one async-event thread per device it adds; this counts them. -extern std::atomic g_netIbCastAsyncThreadsStarted; namespace { @@ -113,12 +111,11 @@ struct FakeHca { // since they read hooks and IbCastDevs that the test is about to reset. class FakeIbWorld { public: - FakeIbWorld() : asyncThreadsBefore_(g_netIbCastAsyncThreadsStarted.load()) {} ~FakeIbWorld() { Quiesce(); } // Waits until every async-event thread started so far has read its event and left. void Quiesce() { - const int started = g_netIbCastAsyncThreadsStarted.load() - asyncThreadsBefore_; + const int started = setThreadName.calls.load(); auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(10); while (asyncExits_.load() < started && std::chrono::steady_clock::now() < deadline) std::this_thread::yield(); EXPECT_EQ(asyncExits_.load(), started) << "async-event threads still running"; @@ -160,13 +157,16 @@ class FakeIbWorld { std::deque hcas; std::vector list; + // IbCastInitDevices names each async-event thread it starts, so the hook also counts them. + std::vector threadNames; private: - int asyncThreadsBefore_; std::atomic asyncExits_{0}; std::vector listed_; public: + HookOf setThreadName{ + g_ncclSetThreadName, [this](pthread_t, const char* name) { threadNames.push_back(name); }}; HookOf ibvSymbols{g_ibvSymbols, [] { return ncclSuccess; }}; HookOf mlx5Symbols{g_mlx5dvSymbols, [] { return ncclSuccess; }}; HookOf ionicSymbols{g_ionicdvSymbols, [] { return ncclSuccess; }}; diff --git a/projects/rccl/test/host/topo-test.cc b/projects/rccl/test/host/topo-test.cc index 7dee6c01ef2d..bc366a2160b0 100644 --- a/projects/rccl/test/host/topo-test.cc +++ b/projects/rccl/test/host/topo-test.cc @@ -39,7 +39,7 @@ void TopoSetNetDevsPolicy(enum netDevsPolicy policy, int num) { namespace { // ASan turns a SIGSEGV into a report plus exit(1), so KilledBySignal never matches under it. -#if TOPO_SCENE_ASAN +#if RCCL_TEST_ASAN #define DEATH_BY_SEGV ::testing::ExitedWithCode(1) #else #define DEATH_BY_SEGV ::testing::KilledBySignal(SIGSEGV) From 5124b17c5854be464de5d16d78b7e417d3b7781a Mon Sep 17 00:00:00 2001 From: Alex Duforest Date: Wed, 21 Oct 2026 22:44:29 -0700 Subject: [PATCH 10/15] test(rccl): net_ib_cast aligned comms, in-range event, no IbCastDevs[-1] read (pre-merge hardening) - ZeroedPtr allocates with the comm structs' alignas() alignment instead of calloc's 16 bytes, which cleared the UBSan misaligned-access reports in rccl-UnitTestsMicroNetIbCastInit. - The unknown-event case uses IBV_EVENT_DEVICE_SPEED_CHANGE, a real event the loop has no case for, instead of casting 999 into the enum. - MakeVDevice's negative-index case puts -1 second, since production reads IbCastDevs[devs[0]] before its range check. Co-Authored-By: Claude --- projects/rccl/test/host/net-ib-cast-common-test.cc | 5 +++-- projects/rccl/test/host/net-ib-cast-init-test.cc | 5 ++++- projects/rccl/test/host/net-ib-cast-test-common.h | 6 ++++-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/projects/rccl/test/host/net-ib-cast-common-test.cc b/projects/rccl/test/host/net-ib-cast-common-test.cc index 4ba46c664d8f..d3a2b8db0cad 100644 --- a/projects/rccl/test/host/net-ib-cast-common-test.cc +++ b/projects/rccl/test/host/net-ib-cast-common-test.cc @@ -384,18 +384,19 @@ TEST_F(NetIbCastCommonEventMicrotest, AsyncThread_QpEventWithCqs_ReportsTheirHan EXPECT_EQ(qpStats.fatalErrorCount, 1); } +// DEVICE_SPEED_CHANGE is a real event the loop has no case for. TEST_F(NetIbCastCommonEventMicrotest, AsyncThread_NonFatalEvents_OnlyWarn) { Events({IBV_EVENT_SRQ_ERR, IBV_EVENT_PATH_MIG_ERR, IBV_EVENT_PORT_ERR, IBV_EVENT_PATH_MIG, IBV_EVENT_PORT_ACTIVE, IBV_EVENT_SQ_DRAINED, IBV_EVENT_LID_CHANGE, IBV_EVENT_PKEY_CHANGE, IBV_EVENT_SM_CHANGE, IBV_EVENT_QP_LAST_WQE_REACHED, IBV_EVENT_CLIENT_REREGISTER, IBV_EVENT_SRQ_LIMIT_REACHED, - IBV_EVENT_COMM_EST, (enum ibv_event_type)999}); + IBV_EVENT_COMM_EST, IBV_EVENT_DEVICE_SPEED_CHANGE}); std::string log = CaptureLog([&] { IbCastAsyncThreadMain(dev_); }); EXPECT_EQ(dev_->stats.fatalErrorCount, 0); EXPECT_EQ(ack_.calls, 14); // These arms only warn. EXPECT_TRUE(LogHas(log, "on SRQ")) << log; EXPECT_TRUE(LogHas(log, "non-fatal async event")) << log; - EXPECT_TRUE(LogHas(log, "unknown event type (999)")) << log; + EXPECT_TRUE(LogHas(log, "unknown event type")) << log; } TEST_F(NetIbCastCommonEventMicrotest, AsyncThread_GidChangeHandlerFails_MarksTheDeviceFatal) { diff --git a/projects/rccl/test/host/net-ib-cast-init-test.cc b/projects/rccl/test/host/net-ib-cast-init-test.cc index 4851bb2c2932..011406b635ba 100644 --- a/projects/rccl/test/host/net-ib-cast-init-test.cc +++ b/projects/rccl/test/host/net-ib-cast-init-test.cc @@ -487,7 +487,10 @@ TEST_F(NetIbCastInitVDeviceMicrotest, MakeVDevice_BadDeviceCounts_AreRejected) { TEST_F(NetIbCastInitVDeviceMicrotest, MakeVDevice_UnknownPhysicalDevice_IsRejected) { int d = -1; EXPECT_EQ(Make({0, 3}, &d), ncclInvalidUsage); - EXPECT_EQ(Make({-1}, &d), ncclInvalidUsage); + // Not first: makeVDevice reads IbCastDevs[devs[0]] before it checks the range. The later checks + // return the same code, so the warning is what shows this one fired. + std::string log = CaptureLog([&] { EXPECT_EQ(Make({0, -1}, &d), ncclInvalidUsage); }); + EXPECT_TRUE(LogHas(log, "physical device -1")) << log; EXPECT_EQ(IbCastNMergedDevs, 0); } diff --git a/projects/rccl/test/host/net-ib-cast-test-common.h b/projects/rccl/test/host/net-ib-cast-test-common.h index 79a0732233fd..924d72b34887 100644 --- a/projects/rccl/test/host/net-ib-cast-test-common.h +++ b/projects/rccl/test/host/net-ib-cast-test-common.h @@ -56,10 +56,12 @@ class ParamScript { }}; }; -// Zeroed heap storage for the large comm structs, as the transport's own calloc gives them. +// Zeroed heap storage for the large comm structs, aligned as their alignas() asks (calloc only gives 16). template struct ZeroedPtr { - ZeroedPtr() : p(static_cast(calloc(1, sizeof(T)))) {} + ZeroedPtr() : p(static_cast(aligned_alloc(alignof(T), (sizeof(T) + alignof(T) - 1) / alignof(T) * alignof(T)))) { + memset(p, 0, sizeof(T)); + } ~ZeroedPtr() { free(p); } ZeroedPtr(const ZeroedPtr&) = delete; ZeroedPtr& operator=(const ZeroedPtr&) = delete; From c0bb04bb3f45f759c23bc938b8b8ed42761a6156 Mon Sep 17 00:00:00 2001 From: Alex Duforest Date: Thu, 22 Oct 2026 09:51:14 -0700 Subject: [PATCH 11/15] test(rccl): fake the ionic uDMA queries and cover the runtime uDMA count (pre-merge hardening) #12749 made init.cc call wrap_ionicdv_ctx_get_udma_count, so the Init binary no longer linked. ionicdvwrap_fakes gains hooks for it and for wrap_ionicdv_qp_get_udma_idx, defaulting to what the real wrappers return without libionic. Tests cover the normalized per-device count and the mlx5-only data-direct checks in GetProperties and DMA-BUF registration. Co-Authored-By: Claude --- .../rccl/test/host/fakes/ionicdvwrap_fakes.cc | 12 ++++++++ .../rccl/test/host/fakes/ionicdvwrap_fakes.h | 7 +++-- .../rccl/test/host/net-ib-cast-init-test.cc | 30 +++++++++++++++++++ .../rccl/test/host/net-ib-cast-reg-test.cc | 20 +++++++++++++ 4 files changed, 67 insertions(+), 2 deletions(-) diff --git a/projects/rccl/test/host/fakes/ionicdvwrap_fakes.cc b/projects/rccl/test/host/fakes/ionicdvwrap_fakes.cc index e35f0a364e15..67b376f424eb 100644 --- a/projects/rccl/test/host/fakes/ionicdvwrap_fakes.cc +++ b/projects/rccl/test/host/fakes/ionicdvwrap_fakes.cc @@ -26,10 +26,18 @@ static ncclResult_t DefaultIonicdvQpSetPuecPlaneRoute(struct ibv_qp*, uint8_t, s std::function g_ionicdvQpSetPuecPlaneRoute = DefaultIonicdvQpSetPuecPlaneRoute; +static uint8_t DefaultIonicdvCtxGetUdmaCount(struct ibv_context*) { return 0; } +std::function g_ionicdvCtxGetUdmaCount = DefaultIonicdvCtxGetUdmaCount; + +static uint8_t DefaultIonicdvQpGetUdmaIdx(struct ibv_qp*) { return 0xFF; } +std::function g_ionicdvQpGetUdmaIdx = DefaultIonicdvQpGetUdmaIdx; + ASSERT_HOOK_MATCHES_PROD(g_ionicdvSymbols, wrap_ionicdv_symbols); ASSERT_HOOK_MATCHES_PROD(g_ionicdvQpSetGda, wrap_ionicdv_qp_set_gda); ASSERT_HOOK_MATCHES_PROD(g_ionicdvPdSetUdmaMask, wrap_ionicdv_pd_set_udma_mask); ASSERT_HOOK_MATCHES_PROD(g_ionicdvQpSetPuecPlaneRoute, wrap_ionicdv_qp_set_puec_plane_route); +ASSERT_HOOK_MATCHES_PROD(g_ionicdvCtxGetUdmaCount, wrap_ionicdv_ctx_get_udma_count); +ASSERT_HOOK_MATCHES_PROD(g_ionicdvQpGetUdmaIdx, wrap_ionicdv_qp_get_udma_idx); ncclResult_t wrap_ionicdv_symbols(void) { return g_ionicdvSymbols(); } ncclResult_t wrap_ionicdv_qp_set_gda(struct ibv_qp* ibqp, bool enable_send, bool enable_recv) { @@ -42,10 +50,14 @@ ncclResult_t wrap_ionicdv_qp_set_puec_plane_route(struct ibv_qp* qp, uint8_t pla struct ionic_dv_puec_route* route) { return g_ionicdvQpSetPuecPlaneRoute(qp, plane_idx, route); } +uint8_t wrap_ionicdv_ctx_get_udma_count(struct ibv_context* ibctx) { return g_ionicdvCtxGetUdmaCount(ibctx); } +uint8_t wrap_ionicdv_qp_get_udma_idx(struct ibv_qp* ibqp) { return g_ionicdvQpGetUdmaIdx(ibqp); } void ResetIonicdvwrapFakes() { g_ionicdvSymbols = DefaultIonicdvSymbols; g_ionicdvQpSetGda = DefaultIonicdvQpSetGda; g_ionicdvPdSetUdmaMask = DefaultIonicdvPdSetUdmaMask; g_ionicdvQpSetPuecPlaneRoute = DefaultIonicdvQpSetPuecPlaneRoute; + g_ionicdvCtxGetUdmaCount = DefaultIonicdvCtxGetUdmaCount; + g_ionicdvQpGetUdmaIdx = DefaultIonicdvQpGetUdmaIdx; } diff --git a/projects/rccl/test/host/fakes/ionicdvwrap_fakes.h b/projects/rccl/test/host/fakes/ionicdvwrap_fakes.h index 1b9e3679c2ca..cfc1657d76f1 100644 --- a/projects/rccl/test/host/fakes/ionicdvwrap_fakes.h +++ b/projects/rccl/test/host/fakes/ionicdvwrap_fakes.h @@ -5,8 +5,8 @@ ************************************************************************/ // Controllable seams for the ionic direct-verbs wrappers defined by -// src/misc/ionicdvwrap.cc. Every default fails with ncclSystemError, the real -// wrappers' answer when libionic is missing. +// src/misc/ionicdvwrap.cc. Every default gives the real wrapper's answer when +// libionic is missing: ncclSystemError, a uDMA count of 0 or a uDMA index of 0xFF. #ifndef RCCL_TEST_HOST_IONICDVWRAP_FAKES_H_ #define RCCL_TEST_HOST_IONICDVWRAP_FAKES_H_ @@ -16,6 +16,7 @@ #include "nccl.h" +struct ibv_context; struct ibv_pd; struct ibv_qp; struct ionic_dv_puec_route; @@ -27,6 +28,8 @@ extern std::function g_ionicdvQpSetPuecPlaneRoute; +extern std::function g_ionicdvCtxGetUdmaCount; +extern std::function g_ionicdvQpGetUdmaIdx; void ResetIonicdvwrapFakes(); diff --git a/projects/rccl/test/host/net-ib-cast-init-test.cc b/projects/rccl/test/host/net-ib-cast-init-test.cc index 011406b635ba..e18b0bc29fbc 100644 --- a/projects/rccl/test/host/net-ib-cast-init-test.cc +++ b/projects/rccl/test/host/net-ib-cast-init-test.cc @@ -977,6 +977,21 @@ TEST_F(NetIbCastInitAinicMicrotest, InitDevices_Ainic_EnablesCtsOffloadAndInline EXPECT_FALSE(IbCastMultiplaneEnable); } +TEST_F(NetIbCastInitAinicMicrotest, InitDevices_Ainic_StoresEachDevicesNormalizedUdmaCount) { + World().AddHca("ionic_1"); + World().AddHca("ionic_2"); + // 0 is a driver without the query, which keeps the two legacy engines; an 8-bit mask caps it at 8. + const std::map raw = {{"ionic_0", 4}, {"ionic_1", 0}, {"ionic_2", 9}}; + ScopedHook count(g_ionicdvCtxGetUdmaCount, [&](ibv_context* ctx) -> uint8_t { + FakeHca* hca = World().Find(ctx); + return hca ? raw.at(hca->device.name) : 0; + }); + ASSERT_EQ(Init(), ncclSuccess); + std::map got; + for (int d = 0; d < IbCastNDevs; d++) got[IbCastDevs[d].devName] = IbCastDevs[d].capsProvider.ionic.udmaCount; + EXPECT_EQ(got, (std::map{{"ionic_0", 4}, {"ionic_1", 2}, {"ionic_2", 8}})); +} + TEST_F(NetIbCastInitAinicMicrotest, InitDevices_AinicOffloadOff_KeepsInlineAsConfigured) { params_.Set("RCCL_CTS_OFFLOAD_ENABLED", 0); params_.Set("GDR_FLUSH_DISABLE", 0); @@ -1058,7 +1073,9 @@ TEST_F(NetIbCastInitAinicMicrotest, InitDevices_AinicWithResiliency_DropsOffload TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_NotAinic_LeavesOffloadOffAndReadsInline) { World().AddHca("mlx5_0"); params_.Set("IB_USE_INLINE", 1); + ScopedHook count(g_ionicdvCtxGetUdmaCount, [](ibv_context*) -> uint8_t { return 4; }); ASSERT_EQ(Init(), ncclSuccess); + EXPECT_EQ(count.calls, 0); EXPECT_FALSE(IbCastAinicRoce); EXPECT_FALSE(IbCastOffloadEnabled); EXPECT_TRUE(IbCastUseInline); @@ -1259,6 +1276,7 @@ TEST_F(NetIbCastInitPropertiesMicrotest, GetProperties_ReportsTheDeviceAndItsLim } TEST_F(NetIbCastInitPropertiesMicrotest, GetProperties_DataDirectAndCtsOffload_ChangeFlushAndRecvs) { + IbCastDevs[0].ibProvider = IB_PROVIDER_MLX5; IbCastDevs[0].capsProvider.mlx5.dataDirect = 1; IbCastOffloadEnabled = true; params_.Set("RCCL_IB_P2P_DISABLE_CTS", 0); @@ -1271,6 +1289,18 @@ TEST_F(NetIbCastInitPropertiesMicrotest, GetProperties_DataDirectAndCtsOffload_C EXPECT_EQ(props.maxRecvs, NCCL_NET_IB_MAX_RECVS); } +// An ionic device's uDMA count shares capsProvider with mlx5.dataDirect. +TEST_F(NetIbCastInitPropertiesMicrotest, GetProperties_OnlyAnMlx5DataDirectDeviceForcesFlush) { + IbCastDevs[0].capsProvider.ionic.udmaCount = 2; + ncclNetProperties_t props; + ASSERT_EQ(IbCastGetProperties(0, &props), ncclSuccess); + EXPECT_EQ(props.forceFlush, 0); + IbCastDevs[0].ibProvider = IB_PROVIDER_MLX5; + IbCastDevs[0].capsProvider.mlx5.dataDirect = 0; + ASSERT_EQ(IbCastGetProperties(0, &props), ncclSuccess); + EXPECT_EQ(props.forceFlush, 0); +} + TEST_F(NetIbCastInitPropertiesMicrotest, GetProperties_MergedDevice_ReportsTheFusedNameSpeedAndIds) { Dev(1, "mlx5_1"); OnRoot0(0); diff --git a/projects/rccl/test/host/net-ib-cast-reg-test.cc b/projects/rccl/test/host/net-ib-cast-reg-test.cc index b3e4a934d976..9094c7f049b3 100644 --- a/projects/rccl/test/host/net-ib-cast-reg-test.cc +++ b/projects/rccl/test/host/net-ib-cast-reg-test.cc @@ -155,6 +155,7 @@ TEST_F(NetIbCastRegMicrotest, RegMrDmaBuf_PassesTheFdOffsetAndIova) { } TEST_F(NetIbCastRegMicrotest, RegMrDmaBuf_DataDirectDevice_GoesThroughMlx5WithTheDataDirectFlag) { + IbCastDevs[0].ibProvider = IB_PROVIDER_MLX5; IbCastDevs[0].capsProvider.mlx5.dataDirect = 1; ScopedHook dmabuf(g_mlx5dvRegDmabufMr, [this](struct ibv_mr** ret, struct ibv_pd*, uint64_t offset, size_t length, uint64_t iova, int fd, int access, int mlx5Access) { @@ -169,6 +170,25 @@ TEST_F(NetIbCastRegMicrotest, RegMrDmaBuf_DataDirectDevice_GoesThroughMlx5WithTh EXPECT_EQ(IbCastDeregMr(send_.get(), mhandle), ncclSuccess); } +// An ionic device's uDMA count shares capsProvider with mlx5.dataDirect. +TEST_F(NetIbCastRegMicrotest, RegMrDmaBuf_OnlyAnMlx5DataDirectDeviceGoesThroughMlx5) { + IbCastDevs[0].capsProvider.ionic.udmaCount = 2; + ScopedHook dmabuf(g_ibvRegDmabufMr, [this](struct ibv_mr** ret, struct ibv_pd*, uint64_t offset, size_t length, + uint64_t iova, int fd, int access) { + *ret = NextMr(nullptr, length, iova, offset, fd, access, 0); + return ncclSuccess; + }); + void* mhandle = nullptr; + ASSERT_EQ(IbCastRegMrDmaBuf(send_.get(), buf_, 8, NCCL_PTR_CUDA, 0, 5, &mhandle), ncclSuccess); + EXPECT_EQ(dmabuf.calls, 1); + EXPECT_EQ(IbCastDeregMr(send_.get(), mhandle), ncclSuccess); + IbCastDevs[0].ibProvider = IB_PROVIDER_MLX5; + IbCastDevs[0].capsProvider.mlx5.dataDirect = 0; + ASSERT_EQ(IbCastRegMrDmaBuf(send_.get(), buf_, 8, NCCL_PTR_CUDA, 0, 5, &mhandle), ncclSuccess); + EXPECT_EQ(dmabuf.calls, 2); + EXPECT_EQ(IbCastDeregMr(send_.get(), mhandle), ncclSuccess); +} + TEST_F(NetIbCastRegMicrotest, RegMr_RangeInsideACachedOne_ReusesItsMr) { void *outer = nullptr, *inner = nullptr; ASSERT_EQ(IbCastRegMr(send_.get(), buf_, 4 * pageSize_, NCCL_PTR_HOST, &outer), ncclSuccess); From dbe4f99fd6b3606d9f3666fb632fee10cf4abe8f Mon Sep 17 00:00:00 2001 From: Alex Duforest Date: Thu, 22 Oct 2026 14:18:32 -0700 Subject: [PATCH 12/15] test(rccl): address ARGUS findings on net_ib_cast init tests Co-Authored-By: Claude --- .../rccl/test/host/net-ib-cast-common-test.cc | 10 ++++- .../rccl/test/host/net-ib-cast-init-test.cc | 37 +++++++++++++++++-- 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/projects/rccl/test/host/net-ib-cast-common-test.cc b/projects/rccl/test/host/net-ib-cast-common-test.cc index d3a2b8db0cad..8a27456074ff 100644 --- a/projects/rccl/test/host/net-ib-cast-common-test.cc +++ b/projects/rccl/test/host/net-ib-cast-common-test.cc @@ -86,6 +86,12 @@ TEST_F(NetIbCastCommonMicrotest, GetNetCommDevBase_FindsThePerDeviceBaseOfSendAn TEST_F(NetIbCastCommonMicrotest, SendCommInit_ResetsEveryQpAndTheCommState) { ZeroedPtr send; memset(send->base.qps, 0x5a, sizeof(send->base.qps)); + memset(send->base.activeQps, 0x5a, sizeof(send->base.activeQps)); + send->base.nqps = send->base.nDataQps = 7; + send->base.optRecvCompletion = true; + send->base.ready = 1; + ncclIbResiliency res; + resiliency_ = &res; params_.Set("IB_SPLIT_DATA_ON_QPS", 1); ASSERT_EQ(IbCastSendCommInit(send.get()), ncclSuccess); EXPECT_TRUE(send->base.isSend); @@ -94,12 +100,13 @@ TEST_F(NetIbCastCommonMicrotest, SendCommInit_ResetsEveryQpAndTheCommState) { EXPECT_EQ(send->base.splitDataOnQps, 1); EXPECT_FALSE(send->base.optRecvCompletion); EXPECT_EQ(send->base.ready, 0); - EXPECT_EQ(send->base.resiliency, nullptr); + EXPECT_EQ(send->base.resiliency, &res); for (int i = 0; i < NCCL_IB_MAX_QPS; i++) { const struct ncclIbQp& qp = send->base.qps[i]; EXPECT_EQ(qp.devIndex, -1) << i; EXPECT_EQ(qp.remDevIdx, -1) << i; EXPECT_EQ(qp.eceSupported, 0) << i; + EXPECT_EQ(qp.ece.vendor_id | qp.ece.options | qp.ece.comp_mask, 0u) << i; EXPECT_EQ(qp.initAttr.portNum, 0) << i; EXPECT_EQ(qp.rtrAttr.remoteQpNum, 0u) << i; EXPECT_EQ(qp.rtsAttr.timeout, 0) << i; @@ -118,6 +125,7 @@ TEST_F(NetIbCastCommonMicrotest, BaseCommInit_ResiliencyInitFails_ReturnsTheErro TEST_F(NetIbCastCommonMicrotest, RecvCommInit_PostsOnDemandByDefault) { ZeroedPtr recv; + recv->base.isSend = true; ASSERT_EQ(IbCastRecvCommInit(recv.get()), ncclSuccess); EXPECT_FALSE(recv->base.isSend); EXPECT_FALSE(recv->prepostReceiveWorkRequests); diff --git a/projects/rccl/test/host/net-ib-cast-init-test.cc b/projects/rccl/test/host/net-ib-cast-init-test.cc index e18b0bc29fbc..2df670146c2d 100644 --- a/projects/rccl/test/host/net-ib-cast-init-test.cc +++ b/projects/rccl/test/host/net-ib-cast-init-test.cc @@ -792,7 +792,9 @@ TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_IbHcaRailAndPlane_AreRecordedA EXPECT_NE(IbCastDevs[1].planeIdx, IbCastDevs[2].planeIdx); } -TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_TooManyPlanes_FailsAfterListing) { +// The list is freed after enumeration but the pointer is kept, so a plane-index failure frees it +// again on the fail: path. Pins AICOMRCCL-TBD; a fix flips this pin. +TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_KnownBug_TooManyPlanesFreesTheDeviceListTwice) { std::string hca; for (int i = 0; i < NCCL_IB_PLANE_MAX_INDEX; i++) { const std::string name = "ionic_" + std::string(1, (char)('a' + i)); @@ -800,7 +802,14 @@ TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_TooManyPlanes_FailsAfterListin hca += (i ? "," : "") + name + ":::" + std::to_string(i + 1); } SetMicroEnv("NCCL_IB_HCA", hca.c_str()); + std::vector freed; + ScopedHook freeList(g_ibvFreeDeviceList, [&freed](struct ibv_device** list) { + freed.push_back(list); + return ncclSuccess; + }); EXPECT_EQ(Init(), ncclInvalidUsage); + ASSERT_EQ(freed.size(), 2u); + EXPECT_EQ(freed[0], freed[1]); } TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_SortsByPciPathUnlessDisabled) { @@ -1167,8 +1176,13 @@ TEST_F(NetIbCastInitMicrotestIsolated, InitDevices_TelemetryCannotStart_CarriesO "NetIbCastInit_TelemetryCannotStart", []() { FakeIbWorld world; - EXPECT_EQ(IbCastInitDevices(nullptr, nullptr), ncclSuccess); + world.AddHca("ionic_0"); + ScopedDebugLogging info(NCCL_LOG_INFO, NCCL_ALL); + std::string log = CaptureLog([&] { EXPECT_EQ(IbCastInitDevices(nullptr, nullptr), ncclSuccess); }); EXPECT_FALSE(rcclTelemetryOn()); + EXPECT_EQ(IbCastNDevs, 1); + EXPECT_TRUE(LogHas(log, "telemetry was requested but could not start")) << log; + EXPECT_FALSE(LogHas(log, "telemetry did not register")) << log; }, {{"RCCL_TELEMETRY_ENABLE", "1"}, {"RCCL_TELEMETRY_OUTPUT_DIR", std::string(5000, 'd')}}); } @@ -1179,9 +1193,26 @@ TEST_F(NetIbCastInitMicrotestIsolated, InitDevices_TelemetryOn_RegistersEachDevi []() { FakeIbWorld world; world.AddHca("ionic_0"); + world.AddHca("ionic_1"); + // ionic_1 sorts first, so a device registered under its listing index lands in the wrong slot. + HookOf realpath{g_initRealpath, [](const char* path, char*) -> char* { + const std::string p(path); + if (p == "/sys/class/infiniband/ionic_0/device") + return strdup("/sys/devices/pci0000:80/0000:81:00.0"); + if (p == "/sys/class/infiniband/ionic_1/device") + return strdup("/sys/devices/pci0000:00/0000:01:00.0"); + return nullptr; + }}; EXPECT_EQ(IbCastInitDevices(nullptr, nullptr), ncclSuccess); EXPECT_TRUE(rcclTelemetryOn()); - EXPECT_EQ(rcclTelemetryDevs[0].device_id, 0); + ASSERT_EQ(IbCastNDevs, 2); + EXPECT_STREQ(IbCastDevs[0].devName, "ionic_1"); + EXPECT_EQ(rcclTelemetryNumDevs, 2); + for (int d = 0; d < 2; d++) { + EXPECT_EQ(rcclTelemetryDevs[d].device_id, d); + EXPECT_STREQ(rcclTelemetryDevs[d].roce_device, IbCastDevs[d].devName) << d; + EXPECT_STREQ(rcclTelemetryDevs[d].transport, "IB-CAST") << d; + } }, {{"RCCL_TELEMETRY_ENABLE", "1"}, {"RCCL_TELEMETRY_OUTPUT_DIR", "/tmp"}}); } From 305951ac2c7bd768907d84c9ac9e59b4e25121ae Mon Sep 17 00:00:00 2001 From: Alex Duforest Date: Thu, 22 Oct 2026 14:48:44 -0700 Subject: [PATCH 13/15] test(rccl): name the double-free pin's ticket Co-Authored-By: Claude --- projects/rccl/test/host/net-ib-cast-init-test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/rccl/test/host/net-ib-cast-init-test.cc b/projects/rccl/test/host/net-ib-cast-init-test.cc index 2df670146c2d..38446f46acfa 100644 --- a/projects/rccl/test/host/net-ib-cast-init-test.cc +++ b/projects/rccl/test/host/net-ib-cast-init-test.cc @@ -793,7 +793,7 @@ TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_IbHcaRailAndPlane_AreRecordedA } // The list is freed after enumeration but the pointer is kept, so a plane-index failure frees it -// again on the fail: path. Pins AICOMRCCL-TBD; a fix flips this pin. +// again on the fail: path. Pins AICOMRCCL-3143; a fix flips this pin. TEST_F(NetIbCastInitDevicesMicrotest, InitDevices_KnownBug_TooManyPlanesFreesTheDeviceListTwice) { std::string hca; for (int i = 0; i < NCCL_IB_PLANE_MAX_INDEX; i++) { From bba9fe1d4de4c65f6e7ae1ed9a1dca39f4be882d Mon Sep 17 00:00:00 2001 From: Alex Duforest Date: Fri, 23 Oct 2026 18:15:20 -0700 Subject: [PATCH 14/15] test(rccl): set up device 3 so only the range check rejects it per review Co-Authored-By: Claude --- projects/rccl/test/host/net-ib-cast-init-test.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/projects/rccl/test/host/net-ib-cast-init-test.cc b/projects/rccl/test/host/net-ib-cast-init-test.cc index 38446f46acfa..e3a7b6fcbee0 100644 --- a/projects/rccl/test/host/net-ib-cast-init-test.cc +++ b/projects/rccl/test/host/net-ib-cast-init-test.cc @@ -485,6 +485,10 @@ TEST_F(NetIbCastInitVDeviceMicrotest, MakeVDevice_BadDeviceCounts_AreRejected) { } TEST_F(NetIbCastInitVDeviceMicrotest, MakeVDevice_UnknownPhysicalDevice_IsRejected) { + // Device 3 would merge fine, so only the range check can reject it. + Dev(3, "mlx5_3"); + OnRoot0(3); + IbCastNDevs = 3; int d = -1; EXPECT_EQ(Make({0, 3}, &d), ncclInvalidUsage); // Not first: makeVDevice reads IbCastDevs[devs[0]] before it checks the range. The later checks From 62483adfb7f527165043eae198481039d1d33366 Mon Sep 17 00:00:00 2001 From: Alex Duforest Date: Fri, 23 Oct 2026 18:36:46 -0700 Subject: [PATCH 15/15] test(rccl): leave ci-precheckin.json to its code owners Host CI runs these binaries through run_host_tests.sh, so the json entries are not needed. Co-Authored-By: Claude --- .../test_runner/configs/ci-precheckin.json | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/projects/rccl/tools/scripts/test_runner/configs/ci-precheckin.json b/projects/rccl/tools/scripts/test_runner/configs/ci-precheckin.json index 85f12411849d..a256d289843e 100644 --- a/projects/rccl/tools/scripts/test_runner/configs/ci-precheckin.json +++ b/projects/rccl/tools/scripts/test_runner/configs/ci-precheckin.json @@ -484,17 +484,6 @@ "tests": [ {"name": "DdaMicrotests", "description": "Host-only src/algorithms/dda/ microtests (no GPU, no HIP/RCCL runtime)", "test_filter": "*"} ] - }, - "unit_tests_micro_net_ib_cast_init": { - "is_gtest": true, - "binary": "host/rccl-UnitTestsMicroNetIbCastInit", - "num_ranks": 1, - "num_nodes": 1, - "num_gpus": 0, - "timeout": 60, - "tests": [ - {"name": "NetIbCastInitMicrotests", "description": "Host-only net_ib_cast init, capability and GIN microtests (no GPU, no HIP/RCCL runtime)", "test_filter": "*"} - ] } }, "test_suites": [ @@ -677,12 +666,6 @@ "description": "rccl-UnitTestsMicroDda (host-only src/algorithms/dda/ microtests, no GPU/HIP/RCCL runtime)", "config": "unit_tests_micro_dda", "enabled": true - }, - { - "name": "RCCL Microtests - net_ib_cast init, capability and GIN", - "description": "rccl-UnitTestsMicroNetIbCastInit (host-only net_ib_cast init, capability and GIN microtests, no GPU/HIP/RCCL runtime)", - "config": "unit_tests_micro_net_ib_cast_init", - "enabled": true } ] }