Skip to content

NetIbMPI tests multithread mode for IB-CAST and NIC fusion #10540

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
i-kosarev merged 4 commits into from
Sep 29, 2026
Merged

NetIbMPI tests multithread mode for IB-CAST and NIC fusion #10540

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
96d1f2d
net-ib: add worker-safe IB-CAST scheduler inspection helpers
i-kosarev Sep 16, 2026
73af111
net-ib: let a threaded size sweep register per step, not only once
i-kosarev Sep 16, 2026
94d6059
net-ib: run the IB-CAST scheduler and fused-device tests at several w…
i-kosarev Sep 16, 2026
2d8b53a
rccl(ainic): register the threaded IB-CAST and fused-device suites
i-kosarev Sep 16, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
376 changes: 375 additions & 1 deletion projects/rccl/test/transport/NetIbMPI/CastTests.cpp
View file
Open in desktop

Large diffs are not rendered by default.

403 changes: 393 additions & 10 deletions projects/rccl/test/transport/NetIbMPI/NetIbMPITestBase.hpp
View file
Open in desktop

Large diffs are not rendered by default.

282 changes: 272 additions & 10 deletions projects/rccl/test/transport/NetIbMPI/NicFusionTests.cpp
View file
Open in desktop

Large diffs are not rendered by default.

28 changes: 28 additions & 0 deletions projects/rccl/tools/scripts/test_runner/configs/mi300_mellanox_ib_func.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,28 @@
}
]
},
"cast_multithread": {
"extends": "cast_base",
"timeout": 600,
"command_args": "--net_ib_nthreads=4",
"env_variables": {
"NCCL_IB_QPS_PER_CONNECTION": "2",
"NCCL_IB_SPLIT_DATA_ON_QPS": "1",
"RCCL_IB_QP_SCHED_UPDATE_INTERVAL": "10000000"
},
"tests": [
{
"name": "Cast_MultiThread_StressMultiRound",
"description": "Concurrent IB-CAST connections, each auditing its own WRR token ledger",
"test_filter": "NetIbMPITest.CastStressMultiRoundTwoConns"
},
{
"name": "Cast_MultiThread_SendRecvMultipleSizes",
"description": "Concurrent schedulers sweep both sides of the WRR/split boundary",
"test_filter": "NetIbMPITest.CastSendRecvMultipleSizes"
}
]
},
"net_ib_gdr_flush": {
"extends": "net_ib_base",
"timeout": 120,
Expand Down Expand Up @@ -2794,6 +2816,12 @@
"config": "net_ib_multithread_independent",
"enabled": true
},
{
"name": "NET IB - CAST Multithread (4)",
"description": "4 worker threads/rank driving independent IB-CAST connections and WRR schedulers",
"config": "cast_multithread",
"enabled": true
},
{
"name": "GDR Flush - Whitebox ncclIbIflush (Single-Node, 2-Rank)",
"description": "Whitebox coverage of the NET/IB GDR flush: dma-buf and peermem RO=0 scratchpad fence, feature-disabled fallback, and repeated-flush burst",
Expand Down
28 changes: 28 additions & 0 deletions projects/rccl/tools/scripts/test_runner/configs/mi300x_mellanox_ib.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -478,6 +478,28 @@
}
]
},
"cast_multithread": {
"extends": "cast_base",
"timeout": 600,
"command_args": "--net_ib_nthreads=4",
"env_variables": {
"NCCL_IB_QPS_PER_CONNECTION": "2",
"NCCL_IB_SPLIT_DATA_ON_QPS": "1",
"RCCL_IB_QP_SCHED_UPDATE_INTERVAL": "10000000"
},
"tests": [
{
"name": "Cast_MultiThread_StressMultiRound",
"description": "Concurrent IB-CAST connections, each auditing its own WRR token ledger",
"test_filter": "NetIbMPITest.CastStressMultiRoundTwoConns"
},
{
"name": "Cast_MultiThread_SendRecvMultipleSizes",
"description": "Concurrent schedulers sweep both sides of the WRR/split boundary",
"test_filter": "NetIbMPITest.CastSendRecvMultipleSizes"
}
]
},
"net_ib_initialization": {
"extends": "net_ib_base",
"timeout": 60,
Expand Down Expand Up @@ -4204,6 +4226,12 @@
"config": "net_ib_multithread_independent",
"enabled": true
},
{
"name": "NET IB - CAST Multithread (4)",
"description": "4 worker threads/rank driving independent IB-CAST connections and WRR schedulers",
"config": "cast_multithread",
"enabled": true
},
{
"name": "GDR Flush - Whitebox ncclIbIflush (Single-Node, 2-Rank)",
"description": "Whitebox coverage of the NET/IB GDR flush: dma-buf and peermem RO=0 scratchpad fence, feature-disabled fallback, and repeated-flush burst",
Expand Down
17 changes: 17 additions & 0 deletions projects/rccl/tools/scripts/test_runner/configs/mi355x_ainic_tw_roce.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,22 @@
}
]
},
"a_cast_multithread": {
"extends": "cast_base",
"timeout": 600,
"command_args": "--net_ib_nthreads=4",
"description": "A18: four worker threads per rank on independent IB-CAST connections, so several WRR schedulers advance at once.",
"env_variables": {
"RCCL_IB_QP_SCHED_UPDATE_INTERVAL": "10000000"
},
"tests": [
{
"name": "A18_Cast_MultiThread_StressMultiRound",
"description": "Concurrent CAST connections, each auditing its own WRR token ledger.",
"test_filter": "NetIbMPITest.CastStressMultiRoundTwoConns"
}
]
},

"gin_proxy": {
"extends": "gtest_base",
Expand Down Expand Up @@ -489,6 +505,7 @@
{ "config": "gin_ainic_backend", "enabled": true },
{ "config": "bootstrap_socket_poll", "enabled": true, "smoke": true },
{ "config": "a_netib_multithread", "enabled": true, "smoke": true },
{ "config": "a_cast_multithread", "enabled": true, "smoke": true },

{ "config": "ib_baseline", "enabled": true, "smoke": true },
{ "config": "cast_alltoall_full", "enabled": true, "smoke": true },
Expand Down
28 changes: 28 additions & 0 deletions projects/rccl/tools/scripts/test_runner/configs/mi355x_thor2_roce.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -246,6 +246,28 @@
}
]
},
"cast_multithread": {
"extends": "cast_base",
"timeout": 600,
"command_args": "--net_ib_nthreads=4",
"env_variables": {
"NCCL_IB_QPS_PER_CONNECTION": "2",
"NCCL_IB_SPLIT_DATA_ON_QPS": "1",
"RCCL_IB_QP_SCHED_UPDATE_INTERVAL": "10000000"
},
"tests": [
{
"name": "Cast_MultiThread_StressMultiRound",
"description": "Concurrent IB-CAST connections, each auditing its own WRR token ledger",
"test_filter": "NetIbMPITest.CastStressMultiRoundTwoConns"
},
{
"name": "Cast_MultiThread_SendRecvMultipleSizes",
"description": "Concurrent schedulers sweep both sides of the WRR/split boundary",
"test_filter": "NetIbMPITest.CastSendRecvMultipleSizes"
}
]
},
"net_ib_gdr_flush": {
"extends": "net_ib_base",
"timeout": 120,
Expand Down Expand Up @@ -4490,6 +4512,12 @@
"config": "net_ib_multithread_independent",
"enabled": true
},
{
"name": "NET IB - CAST Multithread (4)",
"description": "4 worker threads/rank driving independent IB-CAST connections and WRR schedulers",
"config": "cast_multithread",
"enabled": true
},
{
"name": "GDR Flush - Whitebox ncclIbIflush (Single-Node, 2-Rank)",
"description": "Whitebox coverage of the NET/IB GDR flush: dma-buf and peermem RO=0 scratchpad fence, feature-disabled fallback, and repeated-flush burst",
Expand Down
28 changes: 28 additions & 0 deletions projects/rccl/tools/scripts/test_runner/configs/mi355x_thor2_roce_func.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -233,6 +233,28 @@
}
]
},
"cast_multithread": {
"extends": "cast_base",
"timeout": 600,
"command_args": "--net_ib_nthreads=4",
"env_variables": {
"NCCL_IB_QPS_PER_CONNECTION": "2",
"NCCL_IB_SPLIT_DATA_ON_QPS": "1",
"RCCL_IB_QP_SCHED_UPDATE_INTERVAL": "10000000"
},
"tests": [
{
"name": "Cast_MultiThread_StressMultiRound",
"description": "Concurrent IB-CAST connections, each auditing its own WRR token ledger",
"test_filter": "NetIbMPITest.CastStressMultiRoundTwoConns"
},
{
"name": "Cast_MultiThread_SendRecvMultipleSizes",
"description": "Concurrent schedulers sweep both sides of the WRR/split boundary",
"test_filter": "NetIbMPITest.CastSendRecvMultipleSizes"
}
]
},
"net_ib_gdr_flush": {
"extends": "net_ib_base",
"timeout": 120,
Expand Down Expand Up @@ -2836,6 +2858,12 @@
"config": "net_ib_multithread_independent",
"enabled": true
},
{
"name": "NET IB - CAST Multithread (4)",
"description": "4 worker threads/rank driving independent IB-CAST connections and WRR schedulers",
"config": "cast_multithread",
"enabled": true
},
{
"name": "GDR Flush - Whitebox ncclIbIflush (Single-Node, 2-Rank)",
"description": "Whitebox coverage of the NET/IB GDR flush: dma-buf and peermem RO=0 scratchpad fence, feature-disabled fallback, and repeated-flush burst",
Expand Down
28 changes: 28 additions & 0 deletions projects/rccl/tools/scripts/test_runner/configs/mi455_ainic_roce.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -242,6 +242,28 @@
}
]
},
"cast_multithread": {
"extends": "cast_base",
"timeout": 600,
"command_args": "--net_ib_nthreads=4",
"env_variables": {
"NCCL_IB_QPS_PER_CONNECTION": "2",
"NCCL_IB_SPLIT_DATA_ON_QPS": "1",
"RCCL_IB_QP_SCHED_UPDATE_INTERVAL": "10000000"
},
"tests": [
{
"name": "Cast_MultiThread_StressMultiRound",
"description": "Concurrent IB-CAST connections, each auditing its own WRR token ledger",
"test_filter": "NetIbMPITest.CastStressMultiRoundTwoConns"
},
{
"name": "Cast_MultiThread_SendRecvMultipleSizes",
"description": "Concurrent schedulers sweep both sides of the WRR/split boundary",
"test_filter": "NetIbMPITest.CastSendRecvMultipleSizes"
}
]
},
"net_ib_gdr_flush": {
"extends": "net_ib_base",
"timeout": 120,
Expand Down Expand Up @@ -3648,6 +3670,12 @@
"config": "net_ib_multithread_independent",
"enabled": true
},
{
"name": "NET IB - CAST Multithread (4)",
"description": "4 worker threads/rank driving independent IB-CAST connections and WRR schedulers",
"config": "cast_multithread",
"enabled": true
},
{
"name": "GDR Flush - Whitebox ncclIbIflush (Single-Node, 2-Rank)",
"description": "Whitebox coverage of the NET/IB GDR flush: dma-buf and peermem RO=0 scratchpad fence, feature-disabled fallback, and repeated-flush burst",
Expand Down
28 changes: 28 additions & 0 deletions projects/rccl/tools/scripts/test_runner/configs/mi455_ainic_roce_func.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,28 @@
}
]
},
"cast_multithread": {
"extends": "cast_base",
"timeout": 600,
"command_args": "--net_ib_nthreads=4",
"env_variables": {
"NCCL_IB_QPS_PER_CONNECTION": "2",
"NCCL_IB_SPLIT_DATA_ON_QPS": "1",
"RCCL_IB_QP_SCHED_UPDATE_INTERVAL": "10000000"
},
"tests": [
{
"name": "Cast_MultiThread_StressMultiRound",
"description": "Concurrent IB-CAST connections, each auditing its own WRR token ledger",
"test_filter": "NetIbMPITest.CastStressMultiRoundTwoConns"
},
{
"name": "Cast_MultiThread_SendRecvMultipleSizes",
"description": "Concurrent schedulers sweep both sides of the WRR/split boundary",
"test_filter": "NetIbMPITest.CastSendRecvMultipleSizes"
}
]
},
"net_ib_gdr_flush": {
"extends": "net_ib_base",
"timeout": 120,
Expand Down Expand Up @@ -2789,6 +2811,12 @@
"config": "net_ib_multithread_independent",
"enabled": true
},
{
"name": "NET IB - CAST Multithread (4)",
"description": "4 worker threads/rank driving independent IB-CAST connections and WRR schedulers",
"config": "cast_multithread",
"enabled": true
},
{
"name": "GDR Flush - Whitebox ncclIbIflush (Single-Node, 2-Rank)",
"description": "Whitebox coverage of the NET/IB GDR flush: dma-buf and peermem RO=0 scratchpad fence, feature-disabled fallback, and repeated-flush burst",
Expand Down
96 changes: 96 additions & 0 deletions projects/rccl/tools/scripts/test_runner/configs/net_ib_transport.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1068,6 +1068,36 @@
"name": "NetIB_MultiThread_GpuMemoryTransfer",
"description": "Concurrent GPU-buffer registrations drive the peer-memory path into the shared MR cache",
"test_filter": "NetIbMPITest.GpuMemoryTransferStress"
},
{
"name": "NetIB_MultiThread_VNicRegDeregCycling",
"description": "Concurrent register/deregister cycling on a fused device, fanning out across both members' protection domains",
"test_filter": "NetIbMPITest.RegDeregCycling_VNic"
},
{
"name": "NetIB_MultiThread_VNicSequentialTransfers",
"description": "Concurrent long-lived registrations on one fused device",
"test_filter": "NetIbMPITest.SequentialTransfers_VNic"
},
{
"name": "NetIB_MultiThread_VNicMixedSizes",
"description": "Concurrent uneven QP splits across a fused device",
"test_filter": "NetIbMPITest.MixedSizes_VNic"
},
{
"name": "NetIB_MultiThread_VNicLargeTransfer",
"description": "Concurrent striped transfers over a fused device, 64 MB divided by the worker count so the registrations stay within the device limits",
"test_filter": "NetIbMPITest.LargeTransfer_VNic"
},
{
"name": "NetIB_MultiThread_VNicUnalignedSizes",
"description": "Concurrent transfers at the 128-byte striping boundary of a fused device",
"test_filter": "NetIbMPITest.UnalignedSizeTransfer_VNic"
},
{
"name": "NetIB_MultiThread_VNicFlushRepeated",
"description": "Concurrent GPU receives and flushes on one fused device",
"test_filter": "NetIbMPITest.FlushRepeated_VNic"
}
]
},
Expand Down Expand Up @@ -1119,6 +1149,54 @@
"extends": "net_ib_multithread_multiqp_nosplit",
"command_args": "--net_ib_nthreads=4"
},
"cast_multithread": {
"extends": "cast_base",
"timeout": 300,
"env_variables": {
"NCCL_IB_QPS_PER_CONNECTION": "2",
"NCCL_IB_SPLIT_DATA_ON_QPS": "1",
"RCCL_IB_QP_SCHED_UPDATE_INTERVAL": "10000000"
},
"tests": [
{
"name": "Cast_MultiThread_StressMultiRound",
"description": "Every worker drives its own CAST connection, so the WRR schedulers advance simultaneously instead of one after another",
"test_filter": "NetIbMPITest.CastStressMultiRoundTwoConns"
},
{
"name": "Cast_MultiThread_SendRecvMultipleSizes",
"description": "Concurrent schedulers sweep both sides of the WRR/split boundary, each auditing its own token ledger",
"test_filter": "NetIbMPITest.CastSendRecvMultipleSizes"
},
{
"name": "Cast_MultiThread_LargeTransfer",
"description": "Concurrent 16 MB split-path transfers, each confirming its scheduler consumed no tokens",
"test_filter": "NetIbMPITest.CastLargeTransfer"
},
{
"name": "Cast_MultiThread_SendRecvZeroSize",
"description": "Zero-byte sends still take the WRR path on every worker's scheduler",
"test_filter": "NetIbMPITest.CastSendRecvZeroSize"
},
{
"name": "Cast_MultiThread_SplitDataThresholdBoundary",
"description": "Concurrent schedulers decide the split boundary at the same time, each auditing its own tokens",
"test_filter": "NetIbMPITest.CastSplitDataThresholdBoundary"
}
]
},
"cast_multithread_2": {
"extends": "cast_multithread",
"command_args": "--net_ib_nthreads=2"
},
"cast_multithread_4": {
"extends": "cast_multithread",
"command_args": "--net_ib_nthreads=4"
},
"cast_multithread_16": {
"extends": "cast_multithread",
"command_args": "--net_ib_nthreads=16"
},
"cast_wrr_split_long_interval": {
"extends": "cast_base",
"timeout": 120,
Expand Down Expand Up @@ -1459,6 +1537,24 @@
"config": "net_ib_multithread_multiqp_nosplit_4",
"enabled": true
},
{
"name": "NET IB - CAST Multithread (2)",
"description": "2 worker threads/rank driving independent IB-CAST connections and WRR schedulers",
"config": "cast_multithread_2",
"enabled": true
},
{
"name": "NET IB - CAST Multithread (4)",
"description": "4 worker threads/rank driving independent IB-CAST connections and WRR schedulers",
"config": "cast_multithread_4",
"enabled": true
},
{
"name": "NET IB - CAST Multithread (16)",
"description": "16 worker threads/rank driving independent IB-CAST connections and WRR schedulers",
"config": "cast_multithread_16",
"enabled": true
},
{
"name": "NET IB - CAST WRR+Split, long update interval",
"description": "Scheduler toggle tests that need RCCL_IB_QP_SCHED_UPDATE_INTERVAL >= 10 s; they skip in every other config",
Expand Down
Loading