diff --git a/projects/rccl-tests/src/alltoall.cu b/projects/rccl-tests/src/alltoall.cu index 2d3cf5a892ca..6019f9b4e275 100644 --- a/projects/rccl-tests/src/alltoall.cu +++ b/projects/rccl-tests/src/alltoall.cu @@ -579,15 +579,30 @@ testResult_t AlltoAllDeviceTime(struct threadArgs* args, ncclDataType_t type, nc } #endif +testResult_t AlltoAllGetAlgoProtoChannels(ncclComm_t comm, size_t count, ncclDataType_t type, int* algo, int* proto, int* nchannels) { + if (rcclTestsGetAlgoInfo == NULL) return testInternalError; + NCCLCHECK(rcclTestsGetAlgoInfo(comm, ncclFuncAlltoAll, count, type, 0, 0, 1, algo, proto, nchannels)); + return testSuccess; +} + +testResult_t AlltoAllGetCollImplInfo(ncclComm_t comm, size_t count, ncclDataType_t type, ncclRedOp_t op, + const void* sendbuff, void* recvbuff, int graphCapturing, int* algo, int* proto, int* nchannels) { + if (rcclTestsGetCollImplInfo == NULL) return testInternalError; + NCCLCHECK(rcclTestsGetCollImplInfo(comm, ncclFuncAlltoAll, count, type, op, sendbuff, recvbuff, graphCapturing, algo, proto, nchannels)); + return testSuccess; +} + struct testColl alltoAllTest = { "AlltoAll", AlltoAllGetCollByteCount, AlltoAllInitData, AlltoAllGetBw, AlltoAllRunColl, + AlltoAllGetAlgoProtoChannels, + // No symk hook: the symmetric kernels cover AllReduce/AllGather/ReduceScatter + // only, so rcclSymKGetInfo can never describe an AllToAll. NULL, - NULL, - NULL, + AlltoAllGetCollImplInfo, AlltoAllDeviceTime }; diff --git a/projects/rccl-tests/src/common.h b/projects/rccl-tests/src/common.h index 61d35eb63f8a..4c7b015e04df 100644 --- a/projects/rccl-tests/src/common.h +++ b/projects/rccl-tests/src/common.h @@ -454,18 +454,30 @@ testResult_t testLaunchDeviceKernel(F kernel, void* sendbuff, size_t sendoffset, #define SPECIALIZE_KERNEL(kernel, type, op) nullptr #endif +// Must match librccl's internal ncclFunc_t (src/include/nccl_common.h): these +// values cross the rcclGetAlgoInfo / rcclSymKGetInfo / rcclGetCollImplInfo ABI, +// so a renumbering here silently mislabels results instead of failing to build. typedef enum { ncclFuncBroadcast = 0, ncclFuncReduce = 1, ncclFuncAllGather = 2, ncclFuncReduceScatter = 3, ncclFuncAllReduce = 4, - ncclFuncAllReduceWithBias = 5, - ncclFuncSendRecv = 6, - ncclFuncSend = 7, - ncclFuncRecv = 8, - ncclFuncAllToAllPivot = 9, - ncclNumFuncs = 10 + ncclFuncSendRecv = 5, + ncclFuncSend = 6, + ncclFuncRecv = 7, + ncclFuncAlltoAll = 8, + ncclFuncScatter = 9, + ncclFuncGather = 10, + ncclFuncAllToAllPivot = 11, + ncclFuncAlltoAllGda = 12, + ncclFuncAlltoAllvGda = 13, + ncclFuncAllGatherV = 14, + ncclFuncPutSignal = 15, + ncclFuncSignal = 16, + ncclFuncWaitSignal = 17, + ncclFuncAlltoAllv = 18, + ncclNumFuncs = 19 } ncclFunc_t; typedef ncclResult_t (*rcclTestsGetAlgoInfo_t)(struct ncclComm* comm, ncclFunc_t coll, uint64_t count, ncclDataType_t dataType, diff --git a/projects/rccl/CHANGELOG.md b/projects/rccl/CHANGELOG.md index 8d8124cfe3ef..a2b9692032b0 100644 --- a/projects/rccl/CHANGELOG.md +++ b/projects/rccl/CHANGELOG.md @@ -5,6 +5,13 @@ Full documentation for RCCL is available at [https://rccl.readthedocs.io](https: ## RCCL 2.31.2 for ROCm 10.0.0 (Unreleased) ### Added +* `RCCL_CE_AR_MAX_MSG_BYTES` (default `-1`): overrides the 2-shot AllReduce size cap from the arch table. Set to a positive value to override `ceNonRegMax[AR]`. +* `RCCL_CE_AR_REG_MAX_MSG_BYTES` (default `-1`): overrides the registered CE AllReduce size cap from the arch table. +* `RCCL_CE_AR_STAGING_BYTES` (default `-1`): overrides the CE AllReduce staging buffer allocation size; when unset, `NCCL_CE_AR_STAGING_BYTES` is used. +* Per-architecture dispatch table (`rcclArchThresholds`) centralizing algo/proto selection for DDA protocol, CE, and symmetric kernel per collective, replacing scattered hardcoded defaults. Tables are defined for gfx1250, gfx950, and gfx942; the lookup `rcclGetArchThresholds()` maps GCN arch strings to the appropriate entry. The `symMinR2[func]` field sets a per-collective lower bound for registered buffers. +* `RCCL_IGNORE_ARCH_TABLE` (default `0`): when set to `1`, bypasses the `rcclArchThresholds` dispatch table and falls back to compile-time constants for all DDA, CE, and symmetric-kernel thresholds. Has no effect on gfx942 and gfx950, which always use the arch table. +* New `rcclAddonAlgos_t` values: `RCCL_CE_SCRATCH` (CE via DDA scratch, distinct from `RCCL_CE_REGISTERED`), `RCCL_A2A_PIVOT`, `RCCL_A2A_GDA`, `RCCL_A2A_GIN_SDMA`, and `RCCL_DIRECT_ALLTOALL`, with corresponding `rcclGetAlgoName()` labels. +* rccl-tests: AlltoAll now reports algo/protocol in VERSION output, consistent with other collectives. * Compatibility with NCCL 2.31.2. * Per-collective configuration APIs (`ncclCollConfig_t` / `nccl*Config()` entry points) and the `ncclConfigExt_t` vendor extension list. Initialize configs with `NCCL_COLLCONFIG_INITIALIZER`. * Communicator config (`ncclConfig_v23100`) fields for implicit launch ordering (`launchOrderImplicit`), RMA signal count (`numRmaSig`), eager RMA init (`rmaEagerInit`), and host collective fault tolerance (`hostCftMode`). @@ -15,6 +22,21 @@ Full documentation for RCCL is available at [https://rccl.readthedocs.io](https: * Enabled hierarchical Copy Engine `ncclAllGather` and `ncclAlltoAll` on multi-node communicators: the intra-node phase runs over SDMA/CE across the LSA team and the inter-node phase over one-sided RMA through the CPU (GIN) proxy. The path is off by default and takes two opt-ins: `NCCL_CTA_POLICY=2` (equivalently `config.CTAPolicy = NCCL_CTA_POLICY_ZERO`), and symmetric registration of both the send and the receive buffer through `ncclCommWindowRegister`. On architectures other than gfx1250 (MI450) a third is needed, `NCCL_CUMEM_ENABLE=1`, because symmetric memory depends on cuMem and cuMem auto-detection enables itself only on gfx1250. `NCCL_GIN_ENABLE`, `NCCL_NUM_RMA_CTX`, `NCCL_WIN_ENABLE` and `NCCL_DMABUF_ENABLE` need no change, since their defaults already qualify; the inter-node rail draws from the internal RMA context pool (`NCCL_NUM_RMA_INT_CTX`, default 4) rather than the user-addressable contexts, so `numRmaCtx` only has to stay above zero. Selection also requires an available GIN backend and the same number of ranks on every node; communicators that do not qualify keep using the kernel path. Validated for correctness and path selection from 1 to 32 nodes. ### Changed + +* `rcclSelectAllGather` signature changed — it now takes a `cudaStream_t stream` parameter (for live dispatch) and a `bool query` mode that uses `graphCapturingHint` instead of probing the stream. The comment block was updated too. +* `RCCL_DDA_THRESHOLD`, `RCCL_DDA_LL_THRESHOLD`, and `RCCL_DDA_LL128_THRESHOLD` now default to `-1` (unset), resolved at runtime from the arch dispatch table instead of being hardcoded to 128 MiB and 64 KiB respectively (RCCL_DDA_LL128_THRESHOLD had no pre-table default; the LL128 tier was off by default). Setting these env vars explicitly still takes precedence over the table. +* `RCCL_DDA_LL128` now defaults to `-1` (auto): the LL128 tier is enabled only for architectures whose arch-table row has a non-zero `ddaLL128Max` for the collective (currently gfx1250 only). +* `RCCL_CE_ALLREDUCE` now defaults to `-1` (auto/enabled) instead of `0` (disabled). CE AllReduce is therefore on by default for gfx1250 communicators that meet all other eligibility criteria. +* CE-2-Shot AllReduce size cap is now resolved at runtime via `rcclCeAr2ShotMax()` (env var wins, then arch table `ceNonRegMax[AR]`, then the 256 MiB `NCCL_CE_AR_TMPBUF_DEFAULT_BYTES` fallback) rather than from a compile-time constant. +* `RCCL_FORCE_CE_ALLREDUCE=1` no longer overrides the CE AllReduce 2-shot selector cap. It bypasses the `CTA_POLICY_ZERO` check only; the selector cap is enforced regardless. +* `tuning_model_11` registered for gfx1250 single-node; gfx1250 added to the AINIC tuning index map. +* Extended `ncclFunc_t` enum in rccl-tests `common.h` to match the librccl internal ABI: renumbered existing values and added `ncclFuncAlltoAll` (8), `ncclFuncScatter` (9), `ncclFuncGather` (10), `ncclFuncAllToAllPivot` (11), `ncclFuncAlltoAllGda` (12), `ncclFuncAlltoAllvGda` (13), `ncclFuncAllGatherV` (14), `ncclFuncPutSignal` (15), `ncclFuncSignal` (16), `ncclFuncWaitSignal` (17), `ncclFuncAlltoAllv` (18); `ncclNumFuncs` updated to 19. +* AlltoAll index variables in `alltoall_dda.h` and `alltoall_dda_fabric.h` widened from `int` to `size_t` to prevent overflow on large message sizes. +* DDA fabric and IPC AlltoAll launchers refactored to derive launch geometry from a single inline helper (`ddaAllToAllFabricGeom`, `ddaAllToAllFabricLLGeom`, `ddaAllToAllFabricLL128Geom`, `ddaAllToAllIpcGeom`) shared between the dispatch path and the new block-count query functions. +* `rcclGetCollImplInfo` now covers ReduceScatter and AlltoAll (previously both fell back to `rcclGetAlgoInfo()`); rccl-tests reports accurate algo/proto for all four collective types. +* Algorithm selectors (`rcclSelectAllReduce`, `rcclSelectAllGather`, `rcclSelectReduceScatter`, `rcclSelectAlltoAll`) now tag the task with `rcclSymkExtract` (`ALLOW`/`DENY`) so `taskAppend` does not re-derive the symmetric kernel decision. Setting `NCCL_ALGO=Ring` now correctly prevents the symmetric kernel from being extracted even when both buffers are registered. + + * **Breaking: `NCCL_GIN_TYPE` values for AMD backends are not compatible with 2.30.7.** NCCL 2.31 inserted EFA GDA at value 5, so rocSHMEM GDA moved 5→6 and Anvil SDMA moved 6→7. The IB proxy remains `2`. Jobs that still set `NCCL_GIN_TYPE=6` now select rocSHMEM GDA, not Anvil SDMA. See `src/gin/README.md`. * One-sided RMA supports multiple contexts and signals; the previous restriction to context 0 and signal index 0 has been lifted (`numRmaCtx` / `numRmaSig`). * Updated the RMA plugin interface to v15. diff --git a/projects/rccl/docs/api-reference/env-variables.rst b/projects/rccl/docs/api-reference/env-variables.rst index 789b20a6a742..74532d1bd462 100644 --- a/projects/rccl/docs/api-reference/env-variables.rst +++ b/projects/rccl/docs/api-reference/env-variables.rst @@ -607,3 +607,118 @@ variables are collected in the following table. | disabled, events are dropped once a pool is exhausted. - | ``0``: Fixed-size pools. | ``1``: Pools grow on demand (default). + +Algorithm dispatch and tuning (gfx1250 / MI450) +================================================ + +The following variables control per-architecture algorithm and protocol dispatch, +including the DDA fabric tiers, Copy Engine (CE) AllReduce, and the per-architecture +threshold table introduced for gfx1250. + +.. list-table:: + :header-rows: 1 + :widths: 40,60 + + * - **Environment variable** + - **Values** + + * - | ``RCCL_IGNORE_ARCH_TABLE`` + | Controls whether the per-architecture dispatch table + (``rcclArchThresholds``) is used to set DDA, CE, and symmetric-kernel + thresholds, or whether pre-table compile-time constants are used instead. + When the table is active (``0``), each threshold variable below + (``RCCL_DDA_THRESHOLD``, ``RCCL_DDA_LL_THRESHOLD``, etc.) defaults to + ``-1`` (unset) and is resolved from the table at runtime; setting any of + those variables explicitly still overrides the table. + - | ``0``: Use the per-architecture table for threshold defaults (default). + | ``1``: Ignore the table; fall back to pre-table compile-time constants. + This matches the behavior before the arch table was introduced. + + * - | ``RCCL_DDA_THRESHOLD`` + | Upper bound in bytes for the DDA VMM/Simple tier per collective. + Messages above this size exit DDA and fall through to Ring, CE, or the + symmetric kernel. When set to ``-1`` (default), the value is resolved + from the per-architecture table (``ddaVmmMax`` field). Set + ``RCCL_IGNORE_ARCH_TABLE=0`` for the table to take effect. + - | ``-1``: Resolved from the per-arch table at runtime (default). + | ``0``: Disable the DDA VMM tier for all collectives. + | ``N`` (bytes): Use ``N`` as the VMM tier ceiling for all collectives, + overriding the table. Pre-table default: ``134217728`` (128 MiB). + + * - | ``RCCL_DDA_LL_THRESHOLD`` + | Upper bound in bytes for the DDA LL (low-latency) tier. Messages at or + below this size use the LL protocol; larger messages move to LL128 or VMM. + When ``-1`` (default), resolved from ``ddaLLMax`` in the arch table. + - | ``-1``: Resolved from the per-arch table at runtime (default). + | ``0``: Disable the DDA LL tier. + | ``N`` (bytes): Use ``N`` as the LL tier ceiling, overriding the table. + Pre-table default: ``65536`` (64 KiB). + + * - | ``RCCL_DDA_LL128`` + | Enables the DDA LL128 protocol tier. When ``-1`` (auto), LL128 is enabled + only for architectures whose per-arch table has a non-zero ``ddaLL128Max`` + entry (currently gfx1250 only). ``1`` forces the tier on but still reads + its ceiling from the arch table, so it yields the same effective cap as + ``-1`` in every current configuration; ``0`` disables the tier on all + architectures. + - | ``-1``: Auto-enabled only when the arch table has non-zero LL128 thresholds + (default). + | ``0``: Disabled on all architectures. + | ``1``: Force-enabled; ceiling is still read from the arch table (same cap + as ``-1`` in every current configuration). + * - | ``RCCL_DDA_LL128_THRESHOLD`` + | Upper bound in bytes for the DDA LL128 tier. Messages above this size move + to VMM/Simple or Ring. When ``-1`` (default), resolved from ``ddaLL128Max`` + in the arch table. When set explicitly, it takes precedence over ``RCCL_DDA_LL128=0`` (the env var is read before the flag check). + - | ``-1``: Resolved from the per-arch table at runtime (default). + | ``0``: Disable the DDA LL128 tier. + | ``N`` (bytes): Use ``N`` as the LL128 tier ceiling, overriding the table. + Pre-table default: ``0`` (LL128 tier was off before the arch table). + + * - | ``RCCL_CE_ALLREDUCE`` + | Enables the Copy Engine (CE) registered-window AllReduce path. When ``-1`` (auto), + CE AllReduce is on by default for gfx1250 communicators that meet all + eligibility criteria, and off for all other architectures. + - | ``-1``: Auto — enabled on gfx1250, disabled elsewhere (default). + | ``0``: Disabled on all architectures. + | ``1``: Force-enabled (subject to other eligibility checks such as buffer + registration and message size). + + * - | ``RCCL_FORCE_CE_ALLREDUCE`` + | Bypasses the ``NCCL_CTA_POLICY=2`` (``CTA_POLICY_ZERO``) requirement for + CE AllReduce, allowing CE to run without symmetric window registration. + Does not override the staging buffer size cap (``RCCL_CE_AR_MAX_MSG_BYTES`` + or the arch table ``ceNonRegMax[AR]``); that cap is enforced regardless. + When ``-1`` (auto), follows the same gfx1250-default logic as + ``RCCL_CE_ALLREDUCE``. + - | ``-1``: Auto — same default as ``RCCL_CE_ALLREDUCE`` (default). + | ``0``: Disabled. + | ``1``: Force-enabled (CTA_POLICY check bypassed). + + * - | ``RCCL_CE_AR_MAX_MSG_BYTES`` + | Overrides the CE 2-shot AllReduce message size cap. When ``-1`` (default), + the cap is read from ``ceNonRegMax[AllReduce]`` in the per-arch table. A + null or unknown-arch table restores the pre-table 256 MiB default. This + variable sizes the selector only; it does not affect the staging buffer + allocation (see ``RCCL_CE_AR_STAGING_BYTES``). + - | ``-1``: Resolved from the per-arch table (default). + | ``N`` (bytes): Use ``N`` as the 2-shot AllReduce size cap. + + * - | ``RCCL_CE_AR_REG_MAX_MSG_BYTES`` + | Overrides the CE registered-window AllReduce message size cap. When ``-1`` + (default), the cap is read from ``ceRegMax[AllReduce]`` in the per-arch + table. This is the upper bound for the path where both send and receive + buffers are in symmetric windows (``-R 2`` mode in rccl-tests). + - | ``-1``: Resolved from the per-arch table (default). + | ``N`` (bytes): Use ``N`` as the registered AllReduce size cap. + + * - | ``RCCL_CE_AR_STAGING_BYTES`` + | Overrides the total allocation size of the CE AllReduce staging buffer + (``ceARTmpBuf``). When ``-1`` (default), the buffer is allocated at the + compile-time constant ``NCCL_CE_AR_STAGING_BYTES`` (16 MiB). Increasing + this reduces pipelining overhead for large messages but raises per-rank + GPU memory usage. This variable sizes the buffer only; the selector cap + is controlled separately by ``RCCL_CE_AR_MAX_MSG_BYTES``. + - | ``-1``: Use the compile-time default of 16 MiB (default). + | ``N`` (bytes): Set the per-slot payload capacity to ``N``; ``ceARTmpBuf`` is ``NCCL_CE_NUM_SLOTS`` (2) times that. + diff --git a/projects/rccl/src/CMakeLists.txt b/projects/rccl/src/CMakeLists.txt index 851d73f9b89b..ca15ffaa8132 100644 --- a/projects/rccl/src/CMakeLists.txt +++ b/projects/rccl/src/CMakeLists.txt @@ -397,6 +397,7 @@ set(SRC_FILES include/proxy.h include/proxy_diag_counters.h include/ras.h + include/rccl_arch_thresholds.h include/rccl_common.h include/rccl_decision.h include/rccl_float8.h @@ -522,6 +523,7 @@ set(SRC_FILES ras/ras_param.cc ras/ras_param.h ras/rasnet.cc + rccl_arch_thresholds.cc rccl_wrap.cc register/coll_reg.cc register/register.cc diff --git a/projects/rccl/src/algorithms/dda/all_gather/dda_all_gather_fabric_ll.cu b/projects/rccl/src/algorithms/dda/all_gather/dda_all_gather_fabric_ll.cu index 10d60383cde4..fde723910fed 100644 --- a/projects/rccl/src/algorithms/dda/all_gather/dda_all_gather_fabric_ll.cu +++ b/projects/rccl/src/algorithms/dda/all_gather/dda_all_gather_fabric_ll.cu @@ -10,6 +10,7 @@ #include "algorithms/dda/all_gather/all_gather_dda_fabric_ll.h" #include "checks.h" #include "comm.h" +#include "rccl_common.h" #include "algorithms/dda/dda_init_detail.h" // nccl_dda_detail::kDdaLLAgMaxBlocksPerPeer #include "debug.h" #include "algorithms/dda/fabric/fabric_gpu_barrier.h" // dda::common::kDdaMaxNranks @@ -131,7 +132,7 @@ bool ncclAllGatherDdaFabricLLEligible(ncclComm* comm, const void* sendbuff, void return false; } - if (perRankBytes * (size_t)comm->nRanks > (size_t)rcclParamDdaLLThreshold()) { + if (perRankBytes * (size_t)comm->nRanks > rcclDdaLLThreshold(comm, ncclFuncAllGather)) { return false; } // One packet carries 8B of payload, so this rank's contribution has to fit the diff --git a/projects/rccl/src/algorithms/dda/all_gather/dda_all_gather_fabric_ll128.cu b/projects/rccl/src/algorithms/dda/all_gather/dda_all_gather_fabric_ll128.cu index 1e7048a84e37..a4518f654dad 100644 --- a/projects/rccl/src/algorithms/dda/all_gather/dda_all_gather_fabric_ll128.cu +++ b/projects/rccl/src/algorithms/dda/all_gather/dda_all_gather_fabric_ll128.cu @@ -12,7 +12,7 @@ #include "comm.h" #include "debug.h" #include "algorithms/dda/fabric/fabric_gpu_barrier.h" // dda::common::kDdaMaxNranks -#include "param.h" +#include "rccl_common.h" #include @@ -160,7 +160,7 @@ bool ncclAllGatherDdaFabricLL128Eligible(ncclComm* comm, const void* sendbuff, v if ((reinterpret_cast(sendbuff) % 16) != 0 || (reinterpret_cast(recvbuff) % 16) != 0) { return false; } - if (perRankBytes * (size_t)comm->nRanks > (size_t)rcclParamDdaLL128Threshold()) { + if (perRankBytes * (size_t)comm->nRanks > rcclDdaLL128Threshold(comm, ncclFuncAllGather)) { return false; } // Derived from the scratch allocation diff --git a/projects/rccl/src/algorithms/dda/all_reduce/dda_all_reduce_fabric_ll128.cu b/projects/rccl/src/algorithms/dda/all_reduce/dda_all_reduce_fabric_ll128.cu index 1a0ffcbbdc35..6f31d12bd4d7 100644 --- a/projects/rccl/src/algorithms/dda/all_reduce/dda_all_reduce_fabric_ll128.cu +++ b/projects/rccl/src/algorithms/dda/all_reduce/dda_all_reduce_fabric_ll128.cu @@ -16,6 +16,7 @@ #include "algorithms/dda/dda_init_detail.h" #include "algorithms/dda/fabric/fabric_gpu_barrier.h" // dda::common::kDdaMaxNranks #include "param.h" +#include "rccl_common.h" #include @@ -157,9 +158,10 @@ bool ncclAllReduceDdaFabricLL128Eligible_old(ncclComm* comm, const void* sendbuf if (bytes % 8 != 0) { return false; } - // Use the runtime LL128 threshold (RCCL_DDA_LL128_THRESHOLD) as the cap. - const int64_t ll128Thresh = rcclParamDdaLL128Threshold(); - if (ll128Thresh <= 0 || bytes > (size_t)ll128Thresh) { + // Cap by the resolved LL128 threshold (RCCL_DDA_LL128_THRESHOLD, else this + // arch's table, else the built-in default). + const size_t ll128Thresh = rcclDdaLL128Threshold(comm, ncclFuncAllReduce); + if (ll128Thresh == 0 || bytes > ll128Thresh) { return false; } // Scratch is sized from the actual message (compact per-call slot stride), so diff --git a/projects/rccl/src/algorithms/dda/alltoall/alltoall_dda.h b/projects/rccl/src/algorithms/dda/alltoall/alltoall_dda.h index 02873a8f9374..739aa28db19c 100644 --- a/projects/rccl/src/algorithms/dda/alltoall/alltoall_dda.h +++ b/projects/rccl/src/algorithms/dda/alltoall/alltoall_dda.h @@ -48,8 +48,8 @@ __launch_bounds__(512) #pragma unroll NRANKS for (int r = 0; r < NRANKS; ++r) { int srcRank = r; - int srcIdx = idx + selfRank * idxEnd; - int destIdx = idx + r * idxEnd; + size_t srcIdx = idx + selfRank * idxEnd; + size_t destIdx = idx + r * idxEnd; *reinterpret_cast(&recvbuff[destIdx]) = reinterpret_cast(&ipcbuffs[srcRank][srcIdx])[0]; } } diff --git a/projects/rccl/src/algorithms/dda/alltoall/alltoall_dda_fabric.h b/projects/rccl/src/algorithms/dda/alltoall/alltoall_dda_fabric.h index c81cb8791228..3404ce4b9f73 100644 --- a/projects/rccl/src/algorithms/dda/alltoall/alltoall_dda_fabric.h +++ b/projects/rccl/src/algorithms/dda/alltoall/alltoall_dda_fabric.h @@ -47,8 +47,8 @@ __launch_bounds__(512) #pragma unroll kUnroll for (int r = 0; r < nRanksEff; ++r) { int srcRank = r; - int srcIdx = idx + selfRank * idxEnd; - int destIdx = idx + r * idxEnd; + size_t srcIdx = idx + selfRank * idxEnd; + size_t destIdx = idx + r * idxEnd; *reinterpret_cast(&recvbuff[destIdx]) = reinterpret_cast(&ipcbuffs[srcRank][srcIdx])[0]; } } diff --git a/projects/rccl/src/algorithms/dda/alltoall/dda_alltoall.h b/projects/rccl/src/algorithms/dda/alltoall/dda_alltoall.h index bd7a64b6dbac..be2a5b89ea46 100644 --- a/projects/rccl/src/algorithms/dda/alltoall/dda_alltoall.h +++ b/projects/rccl/src/algorithms/dda/alltoall/dda_alltoall.h @@ -10,6 +10,8 @@ #include "nccl.h" +#include + struct ncclComm; /** @@ -24,6 +26,14 @@ bool ncclAllToAllDdaIpcEligible(ncclComm* comm, const void* sendbuff, void* recv ncclResult_t ncclAllToAllDdaIpc(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, ncclComm* comm, cudaStream_t stream); +// Total CTAs (grid blocks) each DDA alltoall launcher would use for the given +// operands. Delegates to the shared *Geom() helper, so block counts are +// guaranteed to match the actual launch grid. +uint32_t ncclAllToAllDdaIpcBlocks(ncclComm* comm, size_t count, ncclDataType_t datatype); +uint32_t ncclAllToAllDdaFabricBlocks(ncclComm* comm, size_t count, ncclDataType_t datatype); +uint32_t ncclAllToAllDdaFabricLLBlocks(ncclComm* comm, size_t count, ncclDataType_t datatype); +uint32_t ncclAllToAllDdaFabricLL128Blocks(ncclComm* comm, size_t count, ncclDataType_t datatype); + /** * Check if DDA alltoall is eligible for the fabric/VMM path (runtime nRanks * up to kDdaMaxNranks, single- or multi-node within an MNNVL clique). diff --git a/projects/rccl/src/algorithms/dda/alltoall/dda_alltoall_fabric.cu b/projects/rccl/src/algorithms/dda/alltoall/dda_alltoall_fabric.cu index 735492edf428..8c5e284a1819 100644 --- a/projects/rccl/src/algorithms/dda/alltoall/dda_alltoall_fabric.cu +++ b/projects/rccl/src/algorithms/dda/alltoall/dda_alltoall_fabric.cu @@ -18,11 +18,18 @@ #include #include +#include namespace { using nccl_dda_detail::DdaFabricBarrierState; +// Single source of the launch geometry: grid/block for a byte payload. The +// kernel is instantiated for int8_t, so `bytes` is the per-block element count. +static inline std::pair ddaAllToAllFabricGeom(ncclComm* comm, size_t bytes) { + return dda::common::getGridAndBlockDims(bytes, 1, comm->ddaFabricMaxBlocks); +} + template static ncclResult_t ncclAllToAllDdaFabricTyped(const void* sendbuff, void* recvbuff, size_t count, ncclComm* comm, cudaStream_t stream) { @@ -32,16 +39,16 @@ static ncclResult_t ncclAllToAllDdaFabricTyped(const void* sendbuff, void* recvb } const int nRanks = comm->nRanks; - const size_t totalCount = count * nRanks; - if (totalCount * sizeof(T) > comm->ddaScratchBytes) { - WARN("DDA fabric alltoall: total element count %zu needs %zu bytes; comm scratch is %zu bytes", totalCount, - totalCount * sizeof(T), comm->ddaScratchBytes); + const size_t totalBytes = count * nRanks; + if (totalBytes > comm->ddaScratchBytes) { + WARN("DDA fabric alltoall: total %zu bytes exceeds comm scratch %zu bytes", totalBytes, + comm->ddaScratchBytes); return ncclInvalidArgument; } - // Use the block cap chosen at init (barrier flag buffer is sized for it). - const int nBlocksMax = comm->ddaFabricMaxBlocks; - auto gridBlock = dda::common::getGridAndBlockDims(count, sizeof(T), nBlocksMax); + // Use the block cap chosen at init (barrier flag buffer is sized for it); + // count is already the byte count (kernel instantiated for int8_t). + auto gridBlock = ddaAllToAllFabricGeom(comm, count); const auto& grid = gridBlock.first; const auto& block = gridBlock.second; @@ -57,7 +64,7 @@ static ncclResult_t ncclAllToAllDdaFabricTyped(const void* sendbuff, void* recvb // Stage sendbuff into this rank's scratch before the peer exchange. A single // host-launched cudaMemcpyAsync avoids the per-block in-kernel copy race on // the fabric path. - CUDACHECK(cudaMemcpyAsync(comm->ddaScratch, sendbuff, totalCount * sizeof(T), cudaMemcpyDeviceToDevice, stream)); + CUDACHECK(cudaMemcpyAsync(comm->ddaScratch, sendbuff, totalBytes, cudaMemcpyDeviceToDevice, stream)); switch (nRanks) { case 4: @@ -120,6 +127,11 @@ bool ncclAllToAllDdaFabricEligible(ncclComm* comm, const void* sendbuff, void* r return true; } +uint32_t ncclAllToAllDdaFabricBlocks(ncclComm* comm, size_t count, ncclDataType_t datatype) { + const auto grid = ddaAllToAllFabricGeom(comm, count * ncclTypeSize(datatype)).first; + return grid.x * grid.y; +} + ncclResult_t ncclAllToAllDdaFabric(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, ncclComm* comm, cudaStream_t stream) { if (datatype != ncclFloat32 && datatype != ncclFloat16 && datatype != ncclBfloat16) { diff --git a/projects/rccl/src/algorithms/dda/alltoall/dda_alltoall_fabric_ll.cu b/projects/rccl/src/algorithms/dda/alltoall/dda_alltoall_fabric_ll.cu index cb482693a027..1c4fb1e6d597 100644 --- a/projects/rccl/src/algorithms/dda/alltoall/dda_alltoall_fabric_ll.cu +++ b/projects/rccl/src/algorithms/dda/alltoall/dda_alltoall_fabric_ll.cu @@ -20,6 +20,7 @@ #include #include +#include namespace { @@ -50,6 +51,14 @@ static inline int ddaLLA2ABlocksPerPeer(size_t perChunkBytes) { return (int)bpp; } +// Single source of the launch geometry: grid.x = peer (nRanks), grid.y = the +// per-peer packet split; 256 threads/block. +static inline std::pair ddaAllToAllFabricLLGeom(ncclComm* comm, size_t perChunkBytes) { + const unsigned threads = 256; + const int blocksPerPeer = ddaLLA2ABlocksPerPeer(perChunkBytes); + return std::make_pair(dim3((unsigned)comm->nRanks, (unsigned)blocksPerPeer), dim3(threads)); +} + template static ncclResult_t ncclAllToAllDdaFabricLLTyped( const void* sendbuff, void* recvbuff, @@ -58,10 +67,10 @@ static ncclResult_t ncclAllToAllDdaFabricLLTyped( const int nRanks = comm->nRanks; const size_t perChunkBytes = count * sizeof(T); - const unsigned threads = 256; - const int blocksPerPeer = ddaLLA2ABlocksPerPeer(perChunkBytes); - dim3 block(threads); - dim3 grid((unsigned)nRanks, (unsigned)blocksPerPeer); + auto gridBlock = ddaAllToAllFabricLLGeom(comm, perChunkBytes); + const dim3 grid = gridBlock.first; + const dim3 block = gridBlock.second; + const int blocksPerPeer = (int)grid.y; T** peers = reinterpret_cast(comm->ddaPeerPtrsDev); uint32_t* epochDev = comm->ddaLLEpochDev; @@ -131,6 +140,11 @@ bool ncclAllToAllDdaFabricLLEligible(ncclComm* comm, const void* sendbuff, void* return true; } +uint32_t ncclAllToAllDdaFabricLLBlocks(ncclComm* comm, size_t count, ncclDataType_t datatype) { + const auto grid = ddaAllToAllFabricLLGeom(comm, count * ncclTypeSize(datatype)).first; + return grid.x * grid.y; +} + ncclResult_t ncclAllToAllDdaFabricLL(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, ncclComm* comm, cudaStream_t stream) { if (datatype != ncclFloat32 && datatype != ncclFloat16 && datatype != ncclBfloat16) { diff --git a/projects/rccl/src/algorithms/dda/alltoall/dda_alltoall_fabric_ll128.cu b/projects/rccl/src/algorithms/dda/alltoall/dda_alltoall_fabric_ll128.cu index bc7522534003..dced800b4d38 100644 --- a/projects/rccl/src/algorithms/dda/alltoall/dda_alltoall_fabric_ll128.cu +++ b/projects/rccl/src/algorithms/dda/alltoall/dda_alltoall_fabric_ll128.cu @@ -22,6 +22,7 @@ #include #include #include // getenv (grid-shape tuning overrides) +#include // Runtime-adjustable LL128 AllToAll block size (threads/block). Must be a // multiple of 16 (lanes per 128B line) in [16, 1024]; invalid values fall back @@ -85,6 +86,19 @@ static inline int ddaLL128A2ABlocksPerPeer(size_t perChunkBytes) { return (int)bpp; } +// Single source of the launch geometry: grid.x = peer (nRanks), grid.y = the +// per-peer line split, clamped so flatBlockId (nRanks*bpp-1) stays within the +// device epoch array. +static inline std::pair ddaAllToAllFabricLL128Geom(ncclComm* comm, size_t perChunkBytes) { + const unsigned threads = ddaLL128A2AThreads(1024); // multiple of 16 (lanes/line) + int blocksPerPeer = ddaLL128A2ABlocksPerPeer(perChunkBytes); + if (comm->nRanks * blocksPerPeer > comm->ddaLLEpochLen) { + blocksPerPeer = comm->ddaLLEpochLen / comm->nRanks; + if (blocksPerPeer < 1) blocksPerPeer = 1; + } + return std::make_pair(dim3((unsigned)comm->nRanks, (unsigned)blocksPerPeer), dim3(threads)); +} + template static ncclResult_t ncclAllToAllDdaFabricLL128Typed( const void* sendbuff, void* recvbuff, @@ -93,21 +107,14 @@ static ncclResult_t ncclAllToAllDdaFabricLL128Typed( const int nRanks = comm->nRanks; const size_t perChunkBytes = count * sizeof(T); - const unsigned threads = ddaLL128A2AThreads(1024); // multiple of 16 (lanes/line) - int blocksPerPeer = ddaLL128A2ABlocksPerPeer(perChunkBytes); - T** peers = reinterpret_cast(comm->ddaPeerPtrsDev); uint32_t* epochDev = comm->ddaLLEpochDev; const int epochLen = comm->ddaLLEpochLen; - // Clamp so flatBlockId (nRanks*bpp-1) stays within the device epoch array. - if (nRanks * blocksPerPeer > epochLen) { - blocksPerPeer = epochLen / nRanks; - if (blocksPerPeer < 1) blocksPerPeer = 1; - } - - dim3 block(threads); - dim3 grid((unsigned)nRanks, (unsigned)blocksPerPeer); + auto gridBlock = ddaAllToAllFabricLL128Geom(comm, perChunkBytes); + const dim3 grid = gridBlock.first; + const dim3 block = gridBlock.second; + const int blocksPerPeer = (int)grid.y; INFO(NCCL_COLL, "DDA fabric AllToAll LL128: nRanks=%d perChunkBytes=%zu grid=%ux%u block=%u (block-per-peer, bpp=%d)", nRanks, perChunkBytes, grid.x, grid.y, block.x, blocksPerPeer); @@ -174,6 +181,11 @@ bool ncclAllToAllDdaFabricLL128Eligible(ncclComm* comm, const void* sendbuff, vo return true; } +uint32_t ncclAllToAllDdaFabricLL128Blocks(ncclComm* comm, size_t count, ncclDataType_t datatype) { + const auto grid = ddaAllToAllFabricLL128Geom(comm, count * ncclTypeSize(datatype)).first; + return grid.x * grid.y; +} + ncclResult_t ncclAllToAllDdaFabricLL128(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, ncclComm* comm, cudaStream_t stream) { if (datatype != ncclFloat32 && datatype != ncclFloat16 && datatype != ncclBfloat16) { diff --git a/projects/rccl/src/algorithms/dda/alltoall/dda_alltoall_ipc.cu b/projects/rccl/src/algorithms/dda/alltoall/dda_alltoall_ipc.cu index d2e52d9559fa..51eb29871a05 100644 --- a/projects/rccl/src/algorithms/dda/alltoall/dda_alltoall_ipc.cu +++ b/projects/rccl/src/algorithms/dda/alltoall/dda_alltoall_ipc.cu @@ -20,6 +20,7 @@ #include #include #include +#include namespace { @@ -27,6 +28,13 @@ using nccl_dda_detail::DdaIpcBarrierState; using nccl_dda_detail::ddaMaxNBlocksForScratch; using nccl_dda_detail::kDdaNranks; +// Single source of the launch geometry: grid/block for a byte payload (the +// kernel is instantiated for int8_t). The grid is sized from the per-rank-pair +// chunk, not the whole message. +static inline std::pair ddaAllToAllIpcGeom(size_t bytes) { + return dda::common::getGridAndBlockDims(bytes, 1, ddaMaxNBlocksForScratch()); +} + template static ncclResult_t ncclAllToAllDdaIpcTyped(const void* sendbuff, void* recvbuff, size_t count, ncclComm* comm, cudaStream_t stream) { @@ -35,16 +43,14 @@ static ncclResult_t ncclAllToAllDdaIpcTyped(const void* sendbuff, void* recvbuff return ncclInvalidUsage; } - const size_t totalCount = count * comm->nRanks; - if (totalCount * sizeof(T) > comm->ddaScratchBytes) { - WARN("DDA IPC alltoall: total element count %zu needs %zu bytes; comm scratch is %zu bytes", totalCount, - totalCount * sizeof(T), comm->ddaScratchBytes); + const size_t totalBytes = count * comm->nRanks; + if (totalBytes > comm->ddaScratchBytes) { + WARN("DDA IPC alltoall: total %zu bytes exceeds comm scratch %zu bytes", totalBytes, + comm->ddaScratchBytes); return ncclInvalidArgument; } - const int nBlocksMax = ddaMaxNBlocksForScratch(); - // For alltoall, we use count for grid calculation (data per rank pair) - auto gridBlock = dda::common::getGridAndBlockDims(count, sizeof(T), nBlocksMax); + auto gridBlock = ddaAllToAllIpcGeom(count); const auto& grid = gridBlock.first; const auto& block = gridBlock.second; @@ -58,7 +64,7 @@ static ncclResult_t ncclAllToAllDdaIpcTyped(const void* sendbuff, void* recvbuff dda::common::ddaAllToAllIpc<<>>( d_ipcbuffs, static_cast(recvbuff), count, static_cast(sendbuff), comm->rank, barrierHost); } else { - CUDACHECK(cudaMemcpyAsync(comm->ddaScratch, sendbuff, totalCount * sizeof(T), cudaMemcpyDeviceToDevice, stream)); + CUDACHECK(cudaMemcpyAsync(comm->ddaScratch, sendbuff, totalBytes, cudaMemcpyDeviceToDevice, stream)); dda::common::ddaAllToAllIpc<<>>( d_ipcbuffs, static_cast(recvbuff), count, static_cast(sendbuff), comm->rank, barrierHost); } @@ -105,6 +111,12 @@ bool ncclAllToAllDdaIpcEligible(ncclComm* comm, const void* sendbuff, void* recv return true; } +uint32_t ncclAllToAllDdaIpcBlocks(ncclComm* comm, size_t count, ncclDataType_t datatype) { + (void)comm; + const auto grid = ddaAllToAllIpcGeom(count * ncclTypeSize(datatype)).first; + return grid.x * grid.y; +} + ncclResult_t ncclAllToAllDdaIpc(const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, ncclComm* comm, cudaStream_t stream) { if (datatype != ncclFloat32 && datatype != ncclFloat16 && datatype != ncclBfloat16) { diff --git a/projects/rccl/src/algorithms/dda/dda_init_detail.h b/projects/rccl/src/algorithms/dda/dda_init_detail.h index 959447463163..e80c9c108cbf 100644 --- a/projects/rccl/src/algorithms/dda/dda_init_detail.h +++ b/projects/rccl/src/algorithms/dda/dda_init_detail.h @@ -33,20 +33,23 @@ constexpr int kDdaNranks = dda::common::NRANKS; // An explicit buffer-size override takes precedence over derived sizing. // // The derived size is: max(simpleCap, llFloor, ll128Floor) where: -// - simpleCap: DDA_THRESHOLD (default 128 MiB) +// - simpleCap: rcclDdaScratchPayloadCap() (max DDA/CE-scratch table/env cap, +// including graph VMM even when the comm never captures, or the pre-table +// DDA defaults when the arch table is ignored) // - llFloor: 2 banks * nRanks * kDdaLLMaxBytes (when LL enabled) // - ll128Floor: whole slices per rank to carry DDA_LL128_THRESHOLD, 2 banks // -// Collectives that need more scratch (e.g., LL128 AR with large messages) are -// bounded by the eligibility check (scratchNeeded > ddaScratchBytes), which -// causes them to fall through to Simple path. -inline size_t ddaFabricScratchSizing(int nRanks, int64_t overrideBytes, int64_t ddaEnabled, int64_t ddaThreshold, - int64_t llEnabled, int64_t ll128Enabled, int64_t ll128Threshold = 0) { +// simpleCap is the 1:1 payload footprint (VMM Simple, AG CE-Scratch). LL/LL128 +// slot arrays can still exceed that at high rank counts, which is why the +// floors remain. Kernel-internal slot caps (kDdaLLArMaxBytes, etc.) may still +// refuse a message even when scratch is large enough. +inline size_t ddaFabricScratchSizing(int nRanks, int64_t overrideBytes, int64_t ddaEnabled, size_t ddaThreshold, + int64_t llEnabled, int64_t ll128Enabled, size_t ll128Threshold = 0) { if (overrideBytes >= 0) { return overrideBytes > 0 ? (size_t)overrideBytes : 0; } - const size_t simpleCap = ddaEnabled && ddaThreshold > 0 ? (size_t)ddaThreshold : 0; + const size_t simpleCap = ddaEnabled && ddaThreshold > 0 ? ddaThreshold : 0; if (simpleCap == 0) { return 0; } @@ -60,7 +63,7 @@ inline size_t ddaFabricScratchSizing(int nRanks, int64_t overrideBytes, int64_t // nRanks slots, 2 banks. size_t ll128Floor = 0; if ((llEnabled || ll128Enabled) && ll128Threshold > 0) { - const size_t perRank = ((size_t)ll128Threshold + (size_t)nRanks - 1) / (size_t)nRanks; + const size_t perRank = (ll128Threshold + (size_t)nRanks - 1) / (size_t)nRanks; size_t slotSlices = ddaLL128Slices(perRank); slotSlices += slotSlices & 1; // even: the two-shot tier halves this slot ll128Floor = (size_t)2 * nRanks * slotSlices * (size_t)kDdaLL128WireBytesPerSlice; diff --git a/projects/rccl/src/algorithms/dda/device/CollCommon.h b/projects/rccl/src/algorithms/dda/device/CollCommon.h index 819b56e85dbc..cbf0c9426126 100644 --- a/projects/rccl/src/algorithms/dda/device/CollCommon.h +++ b/projects/rccl/src/algorithms/dda/device/CollCommon.h @@ -116,8 +116,8 @@ static inline __device__ void reduceScatter(T* const* __restrict__ ipcbuffs, T* for (size_t idx = idxStart; idx < idxEnd; idx += idxStride) { // pattern = 2 performs reduce (one-shot) // pattern = 1 performs reduce-scatter (two-shot) - size_t srcIdx = (pattern == 2) ? idx : (idx + selfRank * idxEnd); - size_t destIdx = (pattern == 1) ? (idx + selfRank * idxEnd) : idx; + size_t srcIdx = (pattern == 2) ? idx : (idx + static_cast(selfRank) * idxEnd); + size_t destIdx = (pattern == 1) ? (idx + static_cast(selfRank) * idxEnd) : idx; uint4 sum{0, 0, 0, 0}; if constexpr (hasAcc) { @@ -150,12 +150,12 @@ static inline __device__ void allGather(T* const* __restrict__ ipcbuffs, T* __re #pragma unroll kUnroll for (int r = 0; r < nRanks; ++r) { int srcRank = (selfRank + r) % nRanks; - int destIdx = idx + srcRank * idxEnd; - int srcIdx; + size_t destIdx = idx + static_cast(srcRank) * idxEnd; + size_t srcIdx; if (enable_offset) { srcIdx = destIdx; } else { - srcIdx = static_cast(idx); + srcIdx = idx; } *reinterpret_cast(&destbuff[destIdx]) = reinterpret_cast(&ipcbuffs[srcRank][srcIdx])[0]; } @@ -220,7 +220,7 @@ inline std::pair getGridAndBlockDims(size_t count, int typeSize, siz // checks derive their per-message caps from it. // Footprint = 2 banks * nRanks * (kDdaLLMaxBytes) // example: 128 MiB at 16 MiB for 4 ranks -constexpr size_t kDdaLLMaxBytes = (size_t)(16) * 1024 * 1024; // 16M +constexpr size_t kDdaLLMaxBytes = (size_t)(32) * 1024 * 1024; // Scratch is double buffered under a single epoch counter, so a bank is half of // it, floored to 16B. Every LL/LL128 tier that derives its slot stride from the diff --git a/projects/rccl/src/algorithms/dda/fabric/fabric_init.cu b/projects/rccl/src/algorithms/dda/fabric/fabric_init.cu index a9dd08204d22..57f2a7b7df90 100644 --- a/projects/rccl/src/algorithms/dda/fabric/fabric_init.cu +++ b/projects/rccl/src/algorithms/dda/fabric/fabric_init.cu @@ -60,10 +60,17 @@ ncclResult_t ncclDdaFabricCommInit(ncclComm* comm) { const int nRanks = comm->nRanks; const int64_t llEnabled = rcclParamDdaLL(); - const int64_t llThresh = rcclParamDdaLLThreshold(); const int64_t ll128Enabled = rcclParamDdaLL128(); - const int64_t ll128Thresh = rcclParamDdaLL128Threshold(); - const int64_t simpleThresh = rcclParamDdaThreshold(); + // Size scratch to the largest DDA/CE-scratch table cap (all collectives, + // including graph VMM), not AllReduce VMM alone. Otherwise AG CE-Scratch + // or AR LL128 can win the selector and then fail the ddaScratchBytes check. + // ddaVmmMaxGraph is in that max even if this comm never captures: scratch + // is allocated once and exported across the clique, so a later graph AR + // must still fit. On gfx1250 that is a silent 256 MiB tax vs eager VMM. + // RCCL_DDA_FABRIC_BUFFER_SIZE overrides. + const size_t llThresh = rcclDdaLLThreshold(comm, ncclFuncAllReduce); + const size_t ll128Thresh = rcclDdaLL128Threshold(comm, ncclFuncAllReduce); + const size_t simpleThresh = rcclDdaScratchPayloadCap(comm); const int64_t fabricScratchOverride = rcclParamDdaFabricBufferSizeForScratch(); // Right-sized from the DDA thresholds and nRanks (env-overridable) instead of @@ -114,6 +121,13 @@ ncclResult_t ncclDdaFabricCommInit(ncclComm* comm) { } } + const size_t eagerVmmAr = rcclDdaVmmThreshold(comm, ncclFuncAllReduce); + INFO(NCCL_INIT, + "ncclDdaFabricCommInit: allocating %zu-byte scratch; payload cap %lld includes graph VMM " + "(ddaVmmMaxGraph) even for eager-only comms; AR eager VMM=%zu. " + "Override with RCCL_DDA_FABRIC_BUFFER_SIZE.", + bytes, (long long)simpleThresh, eagerVmmAr); + // Owned resources: handed to comm on success, freed at `fail` otherwise. // All declared before any goto so the cleanup label is reachable without // jumping over an initialization. diff --git a/projects/rccl/src/algorithms/dda/reduce_scatter/dda_reduce_scatter_fabric_ll.cu b/projects/rccl/src/algorithms/dda/reduce_scatter/dda_reduce_scatter_fabric_ll.cu index a5cfe5f00a07..7573a48aef22 100644 --- a/projects/rccl/src/algorithms/dda/reduce_scatter/dda_reduce_scatter_fabric_ll.cu +++ b/projects/rccl/src/algorithms/dda/reduce_scatter/dda_reduce_scatter_fabric_ll.cu @@ -23,7 +23,7 @@ namespace { -using dda::common::kDdaLLMaxBytes; +using dda::common::kDdaLLRsMaxBytes; using dda::common::kDdaLLRsSlotStridePkts; using dda::common::LLPacket16; @@ -117,8 +117,8 @@ bool ncclReduceScatterDdaFabricLLEligible(ncclComm* comm, const void* sendbuff, if (bytes % 16 != 0) { return false; } - // expand from 8B to 16B - if (bytes * 2 > kDdaLLMaxBytes) { + // Slot holds kDdaLLRsMaxBytes of payload (stride = cap/8 packets). + if (bytes > kDdaLLRsMaxBytes) { return false; } if (ddaLLRsScratchSize(comm->nRanks) > comm->ddaScratchBytes) { diff --git a/projects/rccl/src/algorithms/dda/reduce_scatter/reduce_scatter_dda_fabric_ll.h b/projects/rccl/src/algorithms/dda/reduce_scatter/reduce_scatter_dda_fabric_ll.h index 0e9d147fc8c0..5259b8a0cd26 100644 --- a/projects/rccl/src/algorithms/dda/reduce_scatter/reduce_scatter_dda_fabric_ll.h +++ b/projects/rccl/src/algorithms/dda/reduce_scatter/reduce_scatter_dda_fabric_ll.h @@ -31,8 +31,14 @@ namespace dda::common { -// LL is for small-message, so the full payload is well under the staging cap. -constexpr size_t kDdaLLRsSlotStridePkts = kDdaLLMaxBytes / sizeof(LLPacket16); +// Per-shard staging slot capacity and hard per-shard cap (enforced in the +// eligibility check). Fixes the slot stride at compile time so the +// double-buffered layout is identical on every rank and call. +// Footprint = 2 banks * nRanks * (kDdaLLRsMaxBytes * 2) for the 8B->16B +// expansion; 16 MiB at 1 MiB/rank * 4 ranks * 2 banks * 2 (LL overhead), +// within the 10 GiB DDA fabric scratch on gfx1250. +constexpr size_t kDdaLLRsMaxBytes = 1ULL*1024*1024; // 1 MiB per-rank (= 4 MiB total at 4 ranks) +constexpr size_t kDdaLLRsSlotStridePkts = kDdaLLRsMaxBytes / 8; // 131072 // LL reduce-scatter kernel. 1D grid over packets (8B payload each) of the // per-rank shard (recvcount elements). diff --git a/projects/rccl/src/ce_coll.cc b/projects/rccl/src/ce_coll.cc index 21b5f038b035..5619bff77d48 100644 --- a/projects/rccl/src/ce_coll.cc +++ b/projects/rccl/src/ce_coll.cc @@ -42,7 +42,6 @@ ncclResult_t ncclCeLaunchPersistentReduce(const void* in, void* out, int nRanks, RCCL_PARAM(CeMultiStreams, "CE_MULTI_STREAMS", 0); RCCL_PARAM(CeBatchAsyncEnable, "CE_BATCH_ASYNC_ENABLE", -2); RCCL_PARAM(CeCoopLaunch, "CE_COOP_LAUNCH", 0); -RCCL_PARAM_DECLARE(CeAllReduce); #ifdef CE_BATCH_ASYNC_SUPPORTED // Runtime detection: does the running driver actually implement hipMemcpyBatchAsync? @@ -155,6 +154,12 @@ ncclResult_t ncclCeInit(struct ncclComm* comm) { ncclWindow_vidmem* sigWinDevHost = nullptr; size_t ceDevBaseSize = alignUp(comm->nRanks * sizeof(uint32_t), 16) * 2; size_t sigBufferSize = NUM_SLOTS * comm->nRanks * sizeof(uint32_t); + static int64_t paramMax = rcclParamCeArMaxMsgBytes(); + static int64_t paramStaging = rcclParamCeArStagingBytes(); + static constexpr size_t kCeArMaxDefault = 256ULL * 1024 * 1024; + comm->ceColl.ceArMaxBytes = (paramMax >= 0) ? (size_t)paramMax + : (comm->archThresholds != nullptr ? comm->archThresholds->ceNonRegMax[ncclFuncAllReduce] : kCeArMaxDefault); + comm->ceColl.ceArStagingBytes = (paramStaging >= 0) ? (size_t)paramStaging : (size_t)NCCL_CE_AR_STAGING_BYTES; int i = 0; int targetStreams = 0; uint32_t graphSyncValue = GRAPH_SYNC_VALUE; @@ -410,6 +415,12 @@ bool ncclCeScratchAvailable(struct ncclComm* comm, ncclFunc_t coll, int /*ncclDe TRACE(NCCL_TUNING, "Skipping CE collective: symmetric support is not enabled"); return false; } + // Scratch path writes output to ddaScratch, not the user recv buffer. + // Reject if recv is already a registered symmetric window. + if (winRegType == ncclSymSendRegRecvReg || winRegType == ncclSymSendNonregRecvReg) { + TRACE(NCCL_TUNING, "Skipping CE scratch: recv buffer is registered"); + return false; + } return true; } @@ -1247,6 +1258,12 @@ bool ncclHierCeAvailable(struct ncclComm* comm, ncclFunc_t coll, int /*ncclDevRe TRACE(NCCL_TUNING, "Skipping hierarchical CE collective: not multi-node"); return false; } + // Sub-comms are only built at nNodes >= 8; rcclHierarchicalAlgoInfo dereferences + // them unconditionally, so bail out here if they are not initialized. + if (!comm->hierarchicalCommsInitialized) { + TRACE(NCCL_TUNING, "Skipping hierarchical CE collective: hierarchical sub-comms not initialized"); + return false; + } // If LSA already spans the whole comm, use CE path instead if (ncclDevrIsOneLsaTeam(comm)) { TRACE(NCCL_TUNING, "Skipping hierarchical CE collective: LSA spans the comm; use CE path instead"); @@ -2114,7 +2131,7 @@ static ncclResult_t ncclCeEnsureAllReduceStaging(struct ncclComm* comm) { if (comm->ceColl.ceARTmpBuf != nullptr) return ncclSuccess; if (!rcclParamCeAllReduce()) return ncclSuccess; - maxChunkBytes = ncclCeAllReduceMaxChunkBytes(comm->nRanks); + maxChunkBytes = comm->ceColl.ceArStagingBytes / (size_t)comm->nRanks; ceARTmpBufSize = alignUp(NUM_SLOTS * comm->nRanks * maxChunkBytes, 16); NCCLCHECKGOTO(ncclMemAlloc((void**)&ceARTmpBuf, ceARTmpBufSize), ret, fail); NCCLCHECKGOTO(ncclDevrWindowRegisterInGroup(comm, ceARTmpBuf, ceARTmpBufSize, NCCL_WIN_COLL_SYMMETRIC, &arWinDev), @@ -2148,7 +2165,7 @@ ncclResult_t ncclCeAllReduce(struct ncclComm* comm, const void* sendbuff, void* const size_t shardElems = count / comm->nRanks; const size_t shardBytes = shardElems * eltSize; const size_t NUM_SLOTS = NCCL_CE_NUM_SLOTS; - const size_t slotChunkBytes = ncclCeAllReduceSlotChunkBytes(ncclCeAllReduceMaxChunkBytes(comm->nRanks)); + const size_t slotChunkBytes = ncclCeAllReduceSlotChunkBytes(comm->ceColl.ceArStagingBytes / (size_t)comm->nRanks); if (shardElems == 0 || slotChunkBytes < eltSize) { WARN("CE AllReduce: no valid chunk layout (count=%zu eltSize=%zu nRanks=%d)", count, eltSize, comm->nRanks); return ncclInvalidArgument; diff --git a/projects/rccl/src/collectives.cc b/projects/rccl/src/collectives.cc index 9526ea839fda..7fa04fa9ca9e 100644 --- a/projects/rccl/src/collectives.cc +++ b/projects/rccl/src/collectives.cc @@ -216,8 +216,6 @@ static inline ncclResult_t ncclAlltoAllConfigImpl(const void* sendbuff, void* re return ncclEnqueueCheck(&info); } -RCCL_PARAM_DECLARE(ForceCeAllReduce); - // rcclDdaEnabled() is now in rccl_wrap.cc (declared in rccl_common.h) // Decides whether ncclAllReduce_impl takes the DDA path for this call. Kept as a small named helper @@ -225,51 +223,68 @@ RCCL_PARAM_DECLARE(ForceCeAllReduce); // logic is identical to the guard at the AllReduce call site below. // // `ceAllReduceAllowed` is the caller's single source of truth for "will CE AllReduce actually service -// this call" -- computed once at the call site from rcclUseCeAllReduce() plus whatever additional +// this call" -- computed once at the call site from rcclUseCeAr2Shot() plus whatever additional // gating the CE AllReduce implementation requires (graph latch, ncclGroupDepth, force/symReg // eligibility, etc). This helper does not re-derive CE eligibility itself: threading the same boolean // through both the early CE return and this guard keeps the two decisions from silently drifting apart // as CE AllReduce's own eligibility rules evolve. bool rcclAllReduceShouldTakeDdaPath(const ncclComm* comm, size_t count, ncclDataType_t datatype, bool symEligible, - bool ceAllReduceAllowed) { + bool ceAllReduceAllowed, bool query) { const size_t msgBytes = count * ncclTypeSize(datatype); - // gfx1250 DDA fabric AR is bounded by rcclDdaEnabled (RCCL_DDA_THRESHOLD) and the per-tier - // thresholds, so it may claim the full range regardless of CE eligibility -- ddaFabricArch1250 - // forces this branch unconditionally. On other arches, yield to DDA whenever CE won't actually - // service this call; yielding on message size alone left comms without CE's prerequisites (e.g. - // gfx950 with symmetricSupport off) with no DDA and no CE, falling back to the generic ring/tree - // kernel across the whole 4 MiB+ range that DDA still wins. + // !symEligible is required on every arch: gfx1250 fabric does not override it. + // ddaFabricArch1250 only skips the CE yield -- fabric AR may still run when CE + // would also be eligible, bounded by rcclDdaEnabled (widest arch tier cap, + // else RCCL_DDA_THRESHOLD) and the per-tier thresholds. On other arches, yield to + // DDA whenever CE will not service this call; yielding on message size alone + // left comms without CE's prerequisites (e.g. gfx950 with symmetricSupport off) + // with no DDA and no CE, falling back to the generic ring/tree kernel across + // the whole 4 MiB+ range that DDA still wins. const bool ddaFabricArch1250 = IsArchMatch(comm->archName, "gfx1250"); - return !symEligible && (ddaFabricArch1250 || !ceAllReduceAllowed) && rcclDdaEnabled(comm, msgBytes, 8388608); + const bool result = !symEligible && (ddaFabricArch1250 || !ceAllReduceAllowed) && + rcclDdaEnabled(comm, msgBytes, rcclDdaEntryThreshold(comm, ncclFuncAllReduce)); + if (!result && !query) { + if (symEligible) + INFO(NCCL_TUNING, "AR DDA disqualified: symk eligible"); + else if (!rcclParamDdaEnable()) + INFO(NCCL_TUNING, "AR DDA disqualified: RCCL_DDA_ENABLE=0"); + else if (ceAllReduceAllowed && !ddaFabricArch1250) + INFO(NCCL_TUNING, "AR DDA disqualified: ceAllReduceAllowed=1 on non-gfx1250"); + else + INFO(NCCL_TUNING, "AR DDA disqualified: msgBytes=%zu > entryThreshold=%zu", + msgBytes, rcclDdaEntryThreshold(comm, ncclFuncAllReduce)); + } + return result; } bool rcclAlltoAllShouldTakeDdaPath(const ncclComm* comm, size_t totalBytes, bool ceAlltoAllAllowed) { // AlltoAll has no symmetric kernel, so DDA must yield here or registered-window // CE never dispatches. Full contract is on the declaration in rccl_common.h. - return !ceAlltoAllAllowed && - rcclDdaEnabled(comm, totalBytes, kDdaAlltoAllGfx942ThresholdBytes, kDdaAlltoAllGfx950ThresholdBytes, - kDdaAlltoAllGfx1250ThresholdBytes); + return !ceAlltoAllAllowed && rcclDdaEnabled(comm, totalBytes, rcclDdaEntryThreshold(comm, ncclFuncAlltoAll)); } -// Check if symmetric kernels are requested for this collective (local windows -// only). Cross-rank agreement belongs in ncclMakeSymmetricTaskList at launch: +// Check if symmetric kernels is requested for this collective. +// Win variant: caller has already looked up both windows (avoids redundant ncclDevrFindWindow calls). +bool isSymmetricKernelRequestedWin(ncclComm* comm, ncclFunc_t coll, int symkOp, ncclDataType_t datatype, + size_t nElts, ncclDevrWindow* sendWin, ncclDevrWindow* recvWin) { + if (comm == nullptr || !comm->symmetricSupport) return false; + if (ncclSymkInitOnce(comm) != ncclSuccess) return false; + if (!ncclSymkAvailable(comm, coll, symkOp, datatype, nElts)) return false; + return sendWin != nullptr && recvWin != nullptr && (sendWin->winFlags & NCCL_WIN_COLL_SYMMETRIC) && + (recvWin->winFlags & NCCL_WIN_COLL_SYMMETRIC); +} + +// Cross-rank agreement belongs in ncclMakeSymmetricTaskList at launch: // doing it here deadlocks ncclGroupStart because ranks enqueue one at a time. // Callers that report from a single rank must leave agreeAcrossRanks false // (the default) so they do not bootstrapAllGather alone. bool isSymmetricKernelRequested(ncclComm* comm, ncclFunc_t coll, int symkOp, ncclDataType_t datatype, size_t nElts, const void* sendbuff, void* recvbuff, bool agreeAcrossRanks) { if (comm == nullptr) return false; - - bool local = false; - if (comm->symmetricSupport && ncclSymkInitOnce(comm) == ncclSuccess && - ncclSymkAvailable(comm, coll, symkOp, datatype, nElts)) { - struct ncclDevrWindow* sendWin = nullptr; - struct ncclDevrWindow* recvWin = nullptr; - ncclDevrFindWindow(comm, sendbuff, &sendWin); - ncclDevrFindWindow(comm, recvbuff, &recvWin); - local = sendWin != nullptr && recvWin != nullptr && (sendWin->winFlags & NCCL_WIN_COLL_SYMMETRIC) && - (recvWin->winFlags & NCCL_WIN_COLL_SYMMETRIC); - } + struct ncclDevrWindow* sendWin = nullptr; + struct ncclDevrWindow* recvWin = nullptr; + ncclDevrFindWindow(comm, sendbuff, &sendWin); + ncclDevrFindWindow(comm, recvbuff, &recvWin); + bool local = isSymmetricKernelRequestedWin(comm, coll, symkOp, datatype, nElts, sendWin, recvWin); if (!agreeAcrossRanks || comm->nRanks < 2 || comm->bootstrap == nullptr) return local; // Every rank that opted into agreement must enter the allgather, including @@ -380,13 +395,17 @@ ncclResult_t ncclAllGather_impl(const void* sendbuff, void* recvbuff, size_t sen NCCLCHECK(Recorder::instance().record(rrAllGather, info)); - // Select the implementation once, in one place (rccl_wrap.cc). The same function - // backs rcclGetCollImplInfo so rccl-tests attributes numbers to the backend that - // actually ran. Symmetric-registered buffers are extracted downstream, so DDA is - // gated on !symEligible inside the decision, exactly as before. + // Select the implementation once, in one place (rccl_wrap.cc). The returned + // decision drives dispatch here (DDA / Direct / Hier return early) and is + // carried into taskAppend() via info so the CE-vs-kernel choice and + // graph-capture state are never recomputed. The same function backs + // rcclGetCollImplInfo so rccl-tests attributes numbers to the backend that + // actually ran. struct rcclCollDecision decision; - NCCLCHECK(rcclSelectAllGather(comm, sendbuff, recvbuff, sendcount, datatype, /*query=*/false, + NCCLCHECK(rcclSelectAllGather(comm, sendbuff, recvbuff, sendcount, datatype, stream, /*query=*/false, /*graphCapturingHint=*/false, &decision)); + info.decision = decision; + info.decisionValid = true; // Canonical selection line for addon backends (CE / DDA / Direct / Hier / // symmetric). Native kernels report via the enqueue.cc channel{Lo..Hi} tuning @@ -432,10 +451,12 @@ ncclResult_t ncclAllGather_impl(const void* sendbuff, void* recvbuff, size_t sen info.useDirect = true; return ncclEnqueueCheck(&info); case RCCL_CE_REGISTERED: + case RCCL_CE_SCRATCH: // CE dispatch happens in taskAppend(); just enqueue. return ncclEnqueueCheck(&info); default: - // RCCL_AG_RING / native kernel algorithms go through the standard enqueue path. + // RCCL_SYMMETRIC / native kernel algorithms go through the standard enqueue + // path; taskAppend() honors info->decision. return ncclEnqueueCheck(&info); } } @@ -469,106 +490,65 @@ ncclResult_t ncclAlltoAll_impl(const void* sendbuff, void* recvbuff, size_t coun NCCLCHECK(Recorder::instance().record(rrAllToAll, sendbuff, recvbuff, count, datatype, comm, stream)); - size_t rankOffset = count * ncclTypeSize(datatype); - size_t rankAlign = rankOffset & ((~rankOffset) + 1); - - struct ncclInfo info; - if (comm->topo->pivotA2AEnabled && comm->nChannels >= comm->topo->pivotA2ANumBiRings * 2 && - rankOffset >= 744 * 1024 && rankAlign != 4 && rcclParamAlltoAllPivotEnable()) { - info = {ncclFuncAlltoAllPivot, - "AlltoAllPivot", - sendbuff, - recvbuff, - count, - datatype, - ncclSum, - 0, - comm, - stream, /* Args */ - ALLTOALL_PIVOT_CHUNKSTEPS, - ALLTOALL_PIVOT_SLICESTEPS, - nullptr}; - } else { -#ifdef ENABLE_ROCSHMEM - size_t msgSize = count * ncclTypeSize(datatype) * comm->nRanks; - if (rcclUseAlltoAllGda(comm) && msgSize <= comm->rocshmemThreshold) { - struct ncclInfo info = {ncclFuncAlltoAllGda, - "AlltoAllGda", - sendbuff, - recvbuff, - count, - datatype, - ncclSum, - 0, - comm, - stream, - ALLTOALL_PIVOT_CHUNKSTEPS, - ALLTOALL_PIVOT_SLICESTEPS, - nullptr}; + struct rcclCollDecision decision; + NCCLCHECK(rcclSelectAlltoAll(comm, sendbuff, recvbuff, count, datatype, stream, /*query=*/false, + /*graphCapturingHint=*/false, &decision)); + + // Canonical selection line for addon backends (CE / DDA / Pivot / GDA / + // Direct). Native kernels report via the enqueue.cc channel{Lo..Hi} tuning + // line instead; this names the addon RCCL runs so rcclGetCollImplInfo can be + // checked against it. DDA launchers already log grid/block at launch. + if (comm->rank == 0 && decision.algo >= NCCL_NUM_ALGORITHMS) { + const char* an = nullptr; + rcclGetAlgoName(decision.algo, &an); + INFO(NCCL_COLL, "AlltoAll impl selected: algo %s", an ? an : "?"); + } + switch (decision.algo) { + case RCCL_A2A_PIVOT: { + struct ncclInfo info = {ncclFuncAlltoAllPivot, "AlltoAllPivot", + sendbuff, recvbuff, count, datatype, ncclSum, 0, comm, stream, + ALLTOALL_PIVOT_CHUNKSTEPS, ALLTOALL_PIVOT_SLICESTEPS, nullptr}; + return ncclEnqueueCheck(&info); + } +#ifdef ENABLE_ROCSHMEM + case RCCL_A2A_GDA: { + struct ncclInfo info = {ncclFuncAlltoAllGda, "AlltoAllGda", + sendbuff, recvbuff, count, datatype, ncclSum, 0, comm, stream, + ALLTOALL_PIVOT_CHUNKSTEPS, ALLTOALL_PIVOT_SLICESTEPS, nullptr}; return ncclEnqueueCheck(&info); } -#endif // ENABLE_ROCSHMEM +#endif #if defined(ENABLE_ROCSHMEM_GIN) - // GIN LSA/SDMA is checked before DDA on purpose, so an eligible call takes this path even below - // the DDA threshold. It measured at parity or better on small sizes. - if (ncclAllToAllGinSdmaEligible(comm, sendbuff, recvbuff, count, datatype)) { + case RCCL_A2A_GIN_SDMA: INFO(NCCL_COLL, "AllToAll: taking GIN-SDMA path: nRanks=%d count=%zu datatype=%d bytes=%zu inPlace=%d", comm->nRanks, count, (int)datatype, count * ncclTypeSize(datatype), sendbuff == recvbuff ? 1 : 0); NCCLCHECK(ncclAllToAllGinSdma(sendbuff, recvbuff, count, datatype, comm, stream)); return ncclSuccess; - } #endif - // Symmetric kernels are not supported for AlltoAll, so unlike AllGather we cannot - // gate DDA on !symEligible. When single-node registered-window CE would dispatch, - // DDA must not early-return. Skip the window/graph probes unless DDA is actually enabled - // for this size and CTA_POLICY_ZERO is set -- the default AlltoAll path pays nothing. - const size_t totalBytes = comm->nRanks * count * ncclTypeSize(datatype); - bool ceAlltoAllAllowed = false; - if ((comm->config.CTAPolicy & NCCL_CTA_POLICY_ZERO) && - rcclAlltoAllShouldTakeDdaPath(comm, totalBytes, /*ceAlltoAllAllowed=*/false)) { - NCCLCHECK(alltoAllRegisteredCeAllowed(comm, sendbuff, recvbuff, datatype, stream, &ceAlltoAllAllowed)); - if (ceAlltoAllAllowed) { - INFO(NCCL_COLL, "AllToAll: yielding DDA to CE (NCCL_CTA_POLICY_ZERO)"); - } - } - if (rcclAlltoAllShouldTakeDdaPath(comm, totalBytes, ceAlltoAllAllowed)) { - if (IsArchMatch(comm->archName, "gfx1250")) { - const int64_t llThresh = rcclParamDdaLLThreshold(); - const int64_t ll128Thresh = rcclParamDdaLL128Threshold(); - // Small-chunk fast lane: LL protocol (no GPU barrier). - if (rcclParamDdaLL() && llThresh > 0 && totalBytes <= (size_t)llThresh && - ncclAllToAllDdaFabricLLEligible(comm, sendbuff, recvbuff, count, datatype)) { - INFO(NCCL_COLL, "AllToAll: taking DDA fabric LL path: nRanks=%d nNodes=%d count=%zu datatype=%d bytes=%zu", - comm->nRanks, comm->nNodes, count, (int)datatype, totalBytes); - NCCLCHECK(ncclAllToAllDdaFabricLL(sendbuff, recvbuff, count, datatype, comm, stream)); - return ncclSuccess; - } - // Mid-chunk fast lane: LL128 protocol (128B lines, no GPU barrier). - if (rcclParamDdaLL128() && ll128Thresh > 0 && totalBytes <= (size_t)ll128Thresh && - ncclAllToAllDdaFabricLL128Eligible(comm, sendbuff, recvbuff, count, datatype)) { - INFO(NCCL_COLL, "AllToAll: taking DDA fabric LL128 path: nRanks=%d nNodes=%d count=%zu datatype=%d bytes=%zu", - comm->nRanks, comm->nNodes, count, (int)datatype, totalBytes); - NCCLCHECK(ncclAllToAllDdaFabricLL128(sendbuff, recvbuff, count, datatype, comm, stream)); - return ncclSuccess; - } - if (ncclAllToAllDdaFabricEligible(comm, sendbuff, recvbuff, count, datatype)) { - INFO(NCCL_COLL, "AllToAll: taking DDA fabric (VMM) path: nRanks=%d nNodes=%d count=%zu datatype=%d bytes=%zu", - comm->nRanks, comm->nNodes, count, (int)datatype, count * ncclTypeSize(datatype)); - NCCLCHECK(ncclAllToAllDdaFabric(sendbuff, recvbuff, count, datatype, comm, stream)); - return ncclSuccess; - } - } else if (ncclAllToAllDdaIpcEligible(comm, sendbuff, recvbuff, count, datatype)) { - NCCLCHECK(ncclAllToAllDdaIpc(sendbuff, recvbuff, count, datatype, comm, stream)); - return ncclSuccess; - } - } - - info = { - ncclFuncAlltoAll, "AlltoAll", sendbuff, recvbuff, count, datatype, ncclSum, 0, comm, stream, /* Args */ - ALLTOALL_CHUNKSTEPS, ALLTOALL_SLICESTEPS - }; + case RCCL_DDA_FABRIC_LL: + return ncclAllToAllDdaFabricLL(sendbuff, recvbuff, count, datatype, comm, stream); + case RCCL_DDA_FABRIC_LL128: + return ncclAllToAllDdaFabricLL128(sendbuff, recvbuff, count, datatype, comm, stream); + case RCCL_DDA_FABRIC_VMM: + return ncclAllToAllDdaFabric(sendbuff, recvbuff, count, datatype, comm, stream); + case RCCL_DDA_IPC: + return ncclAllToAllDdaIpc(sendbuff, recvbuff, count, datatype, comm, stream); + case RCCL_CE_REGISTERED: + case RCCL_CE_SCRATCH: + case RCCL_DIRECT_ALLTOALL: + default: + break; } + + // CE and Direct (per-peer Send/Recv) share this enqueue; taskAppend honors + // info.decision instead of re-selecting. + struct ncclInfo info = { + ncclFuncAlltoAll, "AlltoAll", sendbuff, recvbuff, count, datatype, ncclSum, 0, comm, stream, + ALLTOALL_CHUNKSTEPS, ALLTOALL_SLICESTEPS + }; + info.decision = decision; + info.decisionValid = true; return ncclEnqueueCheck(&info); } @@ -1146,8 +1126,19 @@ ncclResult_t ncclReduceScatter_impl(const void* sendbuff, void* recvbuff, size_t comm->enableDirectReduceScatter = 0; // Select once in rccl_wrap.cc; the same function backs rcclGetCollImplInfo. + // Carry the decision into taskAppend() so RCCL_SYMMETRIC vs ring is not + // re-derived (NCCL_ALGO=Ring must not still extract the symmetric kernel). struct rcclCollDecision decision; NCCLCHECK(rcclSelectReduceScatter(comm, sendbuff, recvbuff, recvcount, datatype, op, /*query=*/false, &decision)); + // rcclSelectReduceScatter has no stream parameter (RS has no CE paths), so + // probe capture state here so enqueue.cc reads a valid ceCapturing. + { + struct ncclCudaGraph ceGraph; + NCCLCHECK(ncclCudaGetCapturingGraph(&ceGraph, stream, comm->config.graphUsageMode)); + decision.ceCapturing = ncclCudaGraphValid(ceGraph); + } + info.decision = decision; + info.decisionValid = true; // Canonical line for addon backends; native kernels report via the enqueue tuning line. if (comm->rank == 0 && decision.algo >= NCCL_NUM_ALGORITHMS) { @@ -1185,7 +1176,7 @@ ncclResult_t ncclReduceScatter_impl(const void* sendbuff, void* recvbuff, size_t "sendbuff=%p recvbuff=%p", recvcount, msgSize, comm->rank, nRanks, comm->nNodes, comm, stream, sendbuff, recvbuff); return rcclDirectReduceScatter(sendbuff, recvbuff, recvcount, datatype, op, comm, stream, chunkSteps, sliceSteps); - default: // RCCL_SYMMETRIC / native go through the standard enqueue path. + default: // RCCL_SYMMETRIC / native: enqueue; taskAppend honors info->decision. return ncclEnqueueCheck(&info); } } diff --git a/projects/rccl/src/dev_runtime.cc b/projects/rccl/src/dev_runtime.cc index 88bc7118034b..0cd9f6e4eec3 100644 --- a/projects/rccl/src/dev_runtime.cc +++ b/projects/rccl/src/dev_runtime.cc @@ -38,6 +38,10 @@ NCCL_PARAM(ElasticBufferRegister, "ELASTIC_BUFFER_REGISTER", 1); NCCL_PARAM(SymReuseSysmemHandles, "SYM_REUSE_SYSMEM_HANDLES", 0); NCCL_PARAM(DevApiJit, "DEV_API_JIT", 0); +// Defined in init.cc, where comm->symmetricSupport is derived from it. Window +// registration honors the same opt-out (see ncclCommWindowRegister_impl). +extern int64_t ncclParamWinEnable(); + // Elastic buffers back a symmetric window with CPU memory. Upstream uses the // host-NUMA VMM location type, but HIP/CLR has no host-NUMA member and rejects // it; RCCL allocates host segments as CU_MEM_LOCATION_TYPE_HOST on AMD (see @@ -2027,6 +2031,16 @@ ncclResult_t ncclCommWindowRegister_impl(struct ncclComm* comm, void* userPtr, s return ncclInvalidArgument; } + // RCCL: NCCL_WIN_ENABLE=0 opts out of symmetric-window paths, and the + // non-sym fallback cannot back cuMem/VMM buffers (cudaIpcGetMemHandle rejects + // them). Decline the registration instead of failing callers that register + // symmetric buffers unconditionally; their collectives run on unregistered + // buffers. + if (!ncclParamWinEnable()) { + INFO(NCCL_INIT, "%s: NCCL_WIN_ENABLE=0, skipping registration of %p (%zu bytes)", __func__, userPtr, userSize); + return ncclSuccess; + } + if (!comm->symmetricSupport && !comm->hostRmaSupport) { return ncclSuccess; } diff --git a/projects/rccl/src/enqueue/enqueue.cc b/projects/rccl/src/enqueue/enqueue.cc index ce28407fb312..a79eb824b0eb 100644 --- a/projects/rccl/src/enqueue/enqueue.cc +++ b/projects/rccl/src/enqueue/enqueue.cc @@ -224,7 +224,6 @@ rccl_static int rcclKernelPackedChannels(struct ncclComm* comm, ncclFunc_t func, return (int)std::min((size_t)nMaxChannels, divUp(cells, cellsPerChannel)); } -RCCL_PARAM_DECLARE(DirectReduceScatterThreshold); /*****************************************************************************/ /* Launch system : synchronization and CUDA kernel launch */ /*****************************************************************************/ @@ -2730,8 +2729,12 @@ static ncclResult_t updateCollCostTable(struct ncclComm* comm, struct ncclTaskCo float** collCostTable) { float (*table)[NCCL_NUM_PROTOCOLS] = (float (*)[NCCL_NUM_PROTOCOLS])collCostTable; - if (comm->nRanks == 1 || info->func == ncclFuncAlltoAllPivot || info->func == ncclFuncAlltoAllGda || - info->func == ncclFuncAlltoAllvGda) { + // comm->latencies[]/bandwidths[] only hold the NCCL_NUM_FUNCTIONS kernel + // collectives, so ncclTopoGetAlgoTime() must never be reached with a func + // beyond that (AllToAll and friends run as p2p or addon kernels, not as a + // tuned ring/tree kernel); indexing the tables with those would read past + // their end. + if (comm->nRanks == 1 || (int)info->func >= NCCL_NUM_FUNCTIONS) { table[NCCL_ALGO_RING][NCCL_PROTO_SIMPLE] = 0.0; return ncclSuccess; } @@ -3796,6 +3799,15 @@ static ncclResult_t collTaskAppend(struct ncclComm* comm, struct ncclInfo* info, t->collApiEventHandle = ncclProfilerApiState.collApiEventHandle; t->opCount = comm->opCount; t->acc = info->acc; + // Honor rcclSelectAllReduce / rcclSelectAllGather / rcclSelectReduceScatter: + // collTaskAppend is used for both RCCL_SYMMETRIC and ring/tree. Without this, + // ncclMakeSymmetricTaskList would still extract -R 2 after the selector + // withdrew symk (NCCL_ALGO, or AllReduce/AllGather size windows). + t->symkExtract = RCCL_SYMK_EXTRACT_NONE; + if (info->decisionValid) { + t->symkExtract = + (info->decision.algo == RCCL_SYMMETRIC) ? RCCL_SYMK_EXTRACT_ALLOW : RCCL_SYMK_EXTRACT_DENY; + } planner->nTasksColl += 1; ncclTaskCollSorterInsert(&planner->collSorter, t, t->trafficBytes); @@ -4166,8 +4178,6 @@ static ncclResult_t rmaTaskAppend(struct ncclComm* comm, struct ncclInfo* info) return ncclSuccess; } -RCCL_PARAM_DECLARE(ForceCeAllReduce); -RCCL_PARAM_DECLARE(CeAllReduce); RCCL_PARAM(ForceCe, "FORCE_CE", 1); // TODO(raw task): move this raw task capture implementation into raw_task.cc // once the remaining enqueue-local profiler and red-op dependencies are split. @@ -4355,7 +4365,8 @@ static ncclResult_t taskAppend(struct ncclComm* comm, struct ncclInfo* info) { // DDA/symmetric/kernel paths. bool ceCapturing, ceArGraphAllowed; if (info->decisionValid) { - // Already computed by ncclAllReduce_impl() via rcclSelectAllReduce(). + // Already computed by ncclAllReduce_impl() / ncclAllGather_impl() / + // ncclAlltoAll_impl() via the corresponding rcclSelect*(). ceCapturing = info->decision.ceCapturing; ceArGraphAllowed = info->decision.ceArGraphAllowed; } else { @@ -4363,12 +4374,26 @@ static ncclResult_t taskAppend(struct ncclComm* comm, struct ncclInfo* info) { NCCLCHECK(ncclCudaGetCapturingGraph(&ceGraph, info->stream, comm->config.graphUsageMode)); ceCapturing = ncclCudaGraphValid(ceGraph); rcclCeAllReduceGraphLatchTick(comm, ceCapturing); - ceArGraphAllowed = rcclCeAllReduceAllowed(comm); + ceArGraphAllowed = rcclCeArGraphSafe(comm); } - // Window registration type and CE eligibility, hoisted above the CE-init trigger below, - // which now consults hierCeAvailable. ceAllReduceFits is the size gate for CE AllReduce - // without symmetric registration: ceARTmpBuf holds at most NCCL_CE_AR_MAX_MSG_BYTES. + // Trigger CE initialization on the first CE-capable collective. + // This covers collectives whose user buffers ARE registered (AllGather, + // AlltoAll, Scatter, Gather) as well as AllReduce, which may bypass the + // ceCollTaskAppend path when user buffers are not symmetrically registered. + // Without this trigger, CE AllReduce-only workloads would never initialize + // the CE runtime (ceARTmpBuf stays NULL). + if (!ceCapturing && ncclCeImplemented(info->coll, info->op, info->datatype) && comm->symmetricSupport && + comm->nNodes == 1 && comm->ceColl.baseUCSymReadyPtr == NULL && ncclIntruQueueEmpty(&comm->ceInitTaskQueue)) { + struct ncclCeInitTask* ceTask; + NCCLCHECK(ncclCalloc(&ceTask, 1)); + ceTask->comm = comm; + ncclIntruQueueEnqueue(&comm->ceInitTaskQueue, ceTask); + ncclGroupCommJoin(comm, ncclGroupTaskTypeSymRegister); + } + + // Size gate for unregistered (2-shot) CE AllReduce: table/env cap, then + // the allocated staging buffer (grown above the default if 2-shot asked). bool ceAllReduceFits = false; ncclSymRegType_t winRegType; NCCLCHECK(ncclGetSymRegType(sendWin, recvWin, &winRegType)); @@ -4406,17 +4431,18 @@ static ncclResult_t taskAppend(struct ncclComm* comm, struct ncclInfo* info) { // The CE kernels never read the bias buffer, so every CE AllReduce sub-path below // (registered-window ceAvailable path and the unregistered force-mode ceAllReduceFits // path) must be closed off when a bias buffer is present, or ncclAllReduceWithBias would - // silently drop it. Checked once here rather than inside rcclUseCeAllReduce(), since this + // silently drop it. Checked once here rather than inside rcclUseCeAr2Shot(), since this // function no longer calls it for the AllReduce case. if (info->acc != nullptr || !ceArGraphAllowed || !ceAllReduceOpSupported || - (info->count % (size_t)comm->nRanks != 0) || !rcclParamCeAllReduce()) { + (info->count % (size_t)comm->nRanks != 0) || !rcclCeAllReduceEnabled(comm)) { ceAvailable = false; } else if (ceAllReduceOpSupported) { - // check if we want to force CE AllReduce without symmetric window registration - // msgsize needs to be less than or equal to NCCL_CE_AR_MAX_MSG_BYTES + // Unregistered 2-shot: table/env cap of 0 disables it; otherwise the + // message must fit the 2-shot window and the allocated buffer. size_t totalBytes = info->count * ncclTypeSize(info->datatype); - if (totalBytes > (size_t)NCCL_CE_AR_MAX_MSG_BYTES || !rcclParamForceCeAllReduce() || - !comm->symmetricSupport || comm->nNodes > 1) { + const size_t twoShotMax = rcclCeAr2ShotMax(comm); + if (twoShotMax == 0 || totalBytes > twoShotMax || totalBytes > comm->ceColl.ceArMaxBytes || + !rcclForceCeAllReduceEnabled(comm) || !comm->symmetricSupport || comm->nNodes > 1) { ceAllReduceFits = false; } else { ceAllReduceFits = true; @@ -4429,8 +4455,17 @@ static ncclResult_t taskAppend(struct ncclComm* comm, struct ncclInfo* info) { !ceCapturing && ncclCeScratchAvailable(comm, info->coll, info->op, info->datatype, winRegType); // Uncapped sym-window CE AllReduce (-R 2), without flipping whole comm to CE mode bool ceArSymRegistered = - info->coll == ncclFuncAllReduce && rcclParamForceCeAllReduce() && ceAvailable && !hasSysmemSegment; + info->coll == ncclFuncAllReduce && rcclForceCeAllReduceEnabled(comm) && ceAvailable && !hasSysmemSegment; size_t recvBytes = (size_t)comm->nRanks * info->count * ncclTypeSize(info->datatype); + // Sym-window CE AllGather (-R 2) above the symk/CE crossover, without requiring + // CTAPolicy=ZERO to flip the whole comm to CE mode. Same predicate the + // rcclSelectAllGather Branch #3 reports, so selection and dispatch agree. + bool ceAgSymRegistered = + !rcclNcclAlgoEnvIsSet() && + info->coll == ncclFuncAllGather && ceAvailable && !hasSysmemSegment && + rcclAllGatherCeRegisteredWindow(comm, recvBytes, winRegType, ceCapturing); + const bool allGatherDecided = (info->coll == ncclFuncAllGather && info->decisionValid); + const bool alltoAllDecided = (info->coll == ncclFuncAlltoAll && info->decisionValid); if (info->coll == ncclFuncAllReduce && info->decisionValid) { // AllReduce's backend was already chosen once by rcclSelectAllReduce(); // honor it here instead of recomputing CE eligibility. rcclSelectAllReduce @@ -4447,15 +4482,38 @@ static ncclResult_t taskAppend(struct ncclComm* comm, struct ncclInfo* info) { } // hierCeAvailable is AllGather/AlltoAll-only (ncclHierCeAvailable rejects // AllReduce), so it never affects this AllReduce branch. - } else if (((comm->config.CTAPolicy & NCCL_CTA_POLICY_ZERO) && (ceAvailable || hierCeAvailable) && - !hasSysmemSegment) || - ceArSymRegistered) { + } else if ((allGatherDecided || alltoAllDecided) && + (info->decision.algo == RCCL_CE_REGISTERED || info->decision.algo == RCCL_CE_SCRATCH)) { + // AllGather / AlltoAll CE was chosen once by rcclSelect*(); honor it so + // decision and dispatch agree. Non-CE (Direct) falls through to the + // shared funnel below. + const char* collName = alltoAllDecided ? "AlltoAll" : "AllGather"; + if (info->decision.algo == RCCL_CE_REGISTERED) { + INFO(NCCL_INIT, "Taking CE collective path for %s via registered windows", collName); + NCCLCHECK(ceCollTaskAppend(comm, info, sendWin, recvWin, /*ddaRecvBase=*/nullptr, /*ddaPeerBases=*/nullptr, + opDev)); + } else { + INFO(NCCL_TUNING, "Taking CE collective path for %s via DDA scratch, count=%zu", collName, info->count); + NCCLCHECK(ceCollTaskAppend(comm, info, /*sendWin=*/nullptr, /*recvWin=*/nullptr, + comm->ddaScratch, comm->ddaPeerPtrsHost, opDev)); + } + } else if (info->coll == ncclFuncReduceScatter && info->decisionValid) { + // ReduceScatter has no CE; the selector already chose symmetric vs ring. + // Honor it here so NCCL_ALGO / !symEligible cannot be overridden by + // ncclMakeSymmetricTaskList (collTaskAppend sets symkExtract from decision). + INFO(NCCL_INIT, "Taking kernel-based collective path for ReduceScatter"); + NCCLCHECK(collTaskAppend(comm, info, opDev)); + } else if ((!allGatherDecided && !alltoAllDecided && + (comm->config.CTAPolicy & NCCL_CTA_POLICY_ZERO) && + !(rcclNcclAlgoEnvIsSet() && (info->coll == ncclFuncAllGather || info->coll == ncclFuncReduceScatter)) && + (ceAvailable || hierCeAvailable) && !hasSysmemSegment) || + ceArSymRegistered || (!allGatherDecided && ceAgSymRegistered)) { INFO(NCCL_INIT, "Taking CE collective path with symmetric registered windows for user buffers"); NCCLCHECK(ceCollTaskAppend(comm, info, sendWin, recvWin, /*ddaRecvBase=*/nullptr, /*ddaPeerBases=*/nullptr, opDev)); - } else if (rcclParamForceCe() && CeScratchAvailable && winRegType != ncclSymSendRegRecvReg && - winRegType != ncclSymSendNonregRecvReg && !hasSysmemSegment && comm->ddaScratch != nullptr && + } else if (!rcclNcclAlgoEnvIsSet() && !allGatherDecided && !alltoAllDecided && + rcclParamForceCe() && CeScratchAvailable && !hasSysmemSegment && comm->ddaScratch != nullptr && recvBytes <= comm->ddaScratchBytes && info->coll != ncclFuncAllReduce) { INFO(NCCL_TUNING, "Using DDA scratch for CE collective, count=%zu, recvBytes=%zu", info->count, recvBytes); NCCLCHECK(ceCollTaskAppend(comm, info, /*sendWin=*/nullptr, /*recvWin=*/nullptr, comm->ddaScratch, @@ -4548,7 +4606,7 @@ static ncclResult_t taskAppend(struct ncclComm* comm, struct ncclInfo* info) { } NCCLCHECK(p2pTaskAppend(comm, info, ncclFuncRecv, collAPI, (void*)info->recvbuff, info->count, info->datatype, info->root, allowUB)); - } else if (ceAvailable && comm->symmetricSupport && info->coll == ncclFuncAllGather && + } else if (!allGatherDecided && ceAvailable && comm->symmetricSupport && info->coll == ncclFuncAllGather && info->count > ncclParamSymCeThreshold() && comm->minCompCap >= 100 && comm->isAllDirectNvlink) { // Use CE for Allgather on Blackwell with size > 8MB NCCLCHECK(ceCollTaskAppend(comm, info, sendWin, recvWin, /*ddaRecvBase=*/nullptr, /*ddaPeerBases=*/nullptr, diff --git a/projects/rccl/src/graph/paths.cc b/projects/rccl/src/graph/paths.cc index 63241c0c112c..e02845e4ca97 100644 --- a/projects/rccl/src/graph/paths.cc +++ b/projects/rccl/src/graph/paths.cc @@ -864,24 +864,36 @@ static bool rcclPathOverride(struct ncclTopoSystem* system, uint64_t distance) { } } -// Rewrite GPU<->NIC paths of type `fromType` to `toType` when the two devices share a PCI domain. +// Rewrite DEV/GPU <-> NET paths of type `fromType` to `toType` when the two devices share a PCI domain. +// Keyed on the physical device (DEV node), not on the GPU partitions layered on top of it: reaching a +// NIC is a property of the PCI function, and ncclTopoGdrDistance reads the DEV path for partitioned +// GPUs. Rewriting only the partitions leaves that decision on the original distance, and the no-GDR +// diversion in ncclTopoComputePaths then undoes the rewrite. Partitions mirror their parent so the +// graph search and the GDR decision see the same distance. static void rcclRewriteSameDomainNetPaths(struct ncclTopoSystem* system, int fromType, int toType) { - for (int g = 0; g < system->nodes[GPU].count; g++) { - struct ncclTopoNode* gpu = system->nodes[GPU].nodes + g; - int64_t gpuBusId = NCCL_TOPO_ID_LOCAL_ID(gpu->id); - int64_t gpuDomain = NCCL_BUSID_DOMAIN(gpuBusId); + for (int d = 0; d < system->nodes[DEV].count; d++) { + struct ncclTopoNode* dev = system->nodes[DEV].nodes + d; + // MLOPart splitting overlays the partition index on the DEV id; strip it to recover the busId. + int64_t devBusId = NCCL_TOPO_ID_LOCAL_ID(dev->id) & ~NCCL_TOPO_MLOPART_MASK; + int64_t devDomain = NCCL_BUSID_DOMAIN(devBusId); for (int n = 0; n < system->nodes[NET].count; n++) { struct ncclTopoNode* net = system->nodes[NET].nodes + n; // Skip uninitialized/invalid busIds (raw id 0), but allow domain 0000: // it is a valid PCI domain and must still match. - if (gpuBusId == 0 || net->net.busId == 0) continue; - if (gpuDomain != NCCL_BUSID_DOMAIN(net->net.busId)) continue; - if (gpu->paths[NET] && gpu->paths[NET][n].type == fromType) { - gpu->paths[NET][n].type = toType; - INFO(NCCL_GRAPH, "Rewrote same-domain GPU %d -> NET %d path %d->%d (domain 0x%04lx)", g, n, fromType, toType, - (unsigned long)gpuDomain); + if (devBusId == 0 || net->net.busId == 0) continue; + if (devDomain != NCCL_BUSID_DOMAIN(net->net.busId)) continue; + if (dev->paths[NET] && dev->paths[NET][n].type == fromType) { + dev->paths[NET][n].type = toType; + INFO(NCCL_GRAPH, "Rewrote same-domain DEV %d -> NET %d path %d->%d (domain 0x%04lx)", d, n, fromType, toType, + (unsigned long)devDomain); + } + if (net->paths[DEV] && net->paths[DEV][d].type == fromType) net->paths[DEV][d].type = toType; + for (int g = 0; g < system->nodes[GPU].count; g++) { + struct ncclTopoNode* gpu = system->nodes[GPU].nodes + g; + if (gpu->gpu.parent != dev) continue; + if (gpu->paths[NET] && gpu->paths[NET][n].type == fromType) gpu->paths[NET][n].type = toType; + if (net->paths[GPU] && net->paths[GPU][g].type == fromType) net->paths[GPU][g].type = toType; } - if (net->paths[GPU] && net->paths[GPU][g].type == fromType) net->paths[GPU][g].type = toType; } } } diff --git a/projects/rccl/src/graph/tuning.cc b/projects/rccl/src/graph/tuning.cc index 8d5483a28625..d366eb26a045 100644 --- a/projects/rccl/src/graph/tuning.cc +++ b/projects/rccl/src/graph/tuning.cc @@ -1157,10 +1157,94 @@ static struct tuningModel tuning_model_10{ }, }; +// tuning_model_11: gfx1250 fabric (MNNVL topology). +// Placeholder values based on gfx950 (tuning_model_6) -- validate from sweep data (AICOMRCCL-1756). +static struct tuningModel tuning_model_11{ + .hwLat = + { + /* NVLINK */ + {/* Tree (LL/LL128/Simple)*/ {0.9, 0.9, 2.3}, /* Ring (LL/LL128/Simple)*/ {0.8, 0.8, 2.1}, + /* CollNetDirect (Simple)*/ {0.0, 0.0, 0.9}, /* CollNetChain (Simple)*/ {0.0, 0.0, 0.0}, /* NVLS */ {0, 0, 0}, + /* NVLS Tree */ {0, 0, 0}, /* PAT */ {0, 0, 0}}, + /* PCI */ + {/* Tree (LL/LL128/Simple)*/ {2.2, 2.2, 5.7}, /* Ring (LL/LL128/Simple)*/ {2.2, 2.2, 5.7}, + /* CollNetDirect (Simple)*/ {0.0, 0.0, 5.7}, /* CollNetChain (Simple)*/ {0.0, 0.0, 5.7}, /* NVLS */ {0, 0, 0}, + /* NVLS Tree */ {0, 0, 0}, /* PAT */ {0, 0, 5.7}}, + /* NET */ + {/* Tree (LL/LL128/Simple)*/ {10.5, 10.5, 25.0}, /* Ring (LL/LL128/Simple)*/ {9.5, 9.5, 320.0}, + /* CollNetDirect (Simple)*/ {0.0, 0.0, 10.5}, /* CollNetChain (Simple)*/ {0.0, 0.0, 0.0}, /* NVLS */ {0, 0, 0}, + /* NVLS Tree */ {0, 0, 0}, /* PAT */ {0, 0, 320.0}}, + }, + .bwRatio = + { + /* 2 nodes */ + {/* Tree (LL/LL128/Simple)*/ {0.06, 0.06, 0.11}, /* Ring (LL/LL128/Simple)*/ {0.08, 0.08, 1.00}, + /* CollNetDirect (Simple)*/ {0.00, 0.00, 1.00}, /* CollNetChain (Simple)*/ {0.00, 0.00, 1.00}, + /* NVLS */ {0, 0, 0}, /* NVLS Tree */ {0, 0, 0}, /* PAT */ {0, 0, 0}}, + /* more than 2 nodes */ + {/* Tree (LL/LL128/Simple)*/ {0.06, 0.06, 0.59}, /* Ring (LL/LL128/Simple)*/ {0.08, 0.08, 1.00}, + /* CollNetDirect (Simple)*/ {0.00, 0.00, 1.00}, /* CollNetChain (Simple)*/ {0.00, 0.00, 1.00}, + /* NVLS */ {0, 0, 0}, /* NVLS Tree */ {0, 0, 0}, /* PAT */ {0, 0, 1.00}}, + }, + // Correction factors derived from 8_29 (Helios) AllReduce sweep, R=0 G=0. + // Index i = log2(nBytes/64): [0..3]=<1KB [4]=1KB [8]=16KB [11]=128KB [13]=512KB + // [14]=1MB [15]=2MB [16]=4MB [17]=8MB [21]=128MB [22]=256MB [26]=4GB+ + // LL: Tree wins 0-512KB (idx 0-13), Ring wins 1MB-4MB (idx 14-16) + // LL128: Tree wins 1MB-2MB (idx 14-15), Ring wins 4MB-128MB (idx 16-21) + // Simple: Ring wins 256MB+ (idx 22-26) + // Values >1.0 boost, <1.0 suppress. 2.0=clear winner, 0.5=loser. + .treeCorrectionFactor = + { + // LL: Tree[0-13] Ring[14-16] + {2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, + 0.5, 0.5, 0.5, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0,}, + // LL128: Tree[14-15] Ring[16-21] + {1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, + 2.0, 2.0, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 1.0, 1.0, 1.0, 1.0, 1.0,}, + // Simple: Ring[22-26] + {1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, + 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.5, 0.5, 0.5, 0.5, 0.5,}, + }, + .ringCorrectionFactor = + { + // LL: Ring[14-16] Tree[0-13] + {0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, + 2.0, 2.0, 2.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0,}, + // LL128: Ring[16-21] Tree[14-15] + {1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, + 0.5, 0.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 1.0, 1.0, 1.0, 1.0, 1.0,}, + // Simple: Ring[22-26] + {1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, + 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 2.0, 2.0, 2.0, 2.0, 2.0,}, + }, + .llProtoRanges = + { + // sizePerRank = totalBytes / nRanks (4 ranks default on gfx1250). + // Breakpoints from 8_29 (Helios) sweep data. + /*ReduceScatter*/ + {/*LL (min/max/factor/thread_threshold)*/ {0, 1048576, 1, 16}, + /*LL128 (min/max/factor/thread_threshold)*/ {1048576, 4194304, 1, 64}}, + /*AllGather*/ + {/*LL (min/max/factor/thread_threshold)*/ {0, 1048576, 1, 16}, + /*LL128 (min/max/factor/thread_threshold)*/ {1048576, 16777216, 1, 64}}, + /*AllReduce*/ + {/*LL (min/max/factor/thread_threshold)*/ {0, 131072, 1, 0}, + /*LL128 (min/max/factor/thread_threshold)*/ {131072, 16777216, 3145728, 0}}, + /*Reduce*/ + {/*LL (min/max/factor/thread_threshold)*/ {0, 16383, 1, 0}, + /*LL128 (min/max/factor/thread_threshold)*/ {16383, 16777216, 1, 0}}, + /*Broadcast*/ + {/*LL (min/max/factor/thread_threshold)*/ {0, 2048, 1, 0}, + /*LL128 (min/max/factor/thread_threshold)*/ {2048, 16777216, 1, 0}}, + }, + // gfx1250 returns early from rcclOverrideChannels, so channelThresholds is unreachable. + .channelThresholds = {{{CHAN_THRESHOLDS_UNDEFINED}}}, +}; + static struct tuningModel rcclTuningModel[] = { tuning_model_0, tuning_model_1, tuning_model_2, tuning_model_3, tuning_model_4, tuning_model_5, tuning_model_6, tuning_model_7, tuning_model_8, tuning_model_9, - tuning_model_10, + tuning_model_10, tuning_model_11, }; #if !defined(__HIP_PLATFORM_AMD__) && !defined(__HIPCC__) @@ -1786,17 +1870,33 @@ ncclResult_t ncclTopoGetAlgoTime(struct ncclComm* comm, int coll, int algorithm, return ncclSuccess; } +#include "rccl_arch_thresholds.h" + +void rcclApplyTuningOverrides(struct ncclTopoSystem* system) { + auto nicInfo = rcclPrimaryNic(); + if (IsArchMatch(system->nodes[GPU].nodes[0].gpu.gcn, "gfx950")) { + system->tuning = 6; + } + + if (nicInfo.type == rcclIBNicTypeAINIC) { + if (IsArchMatch(system->nodes[GPU].nodes[0].gpu.gcn, "gfx942")) { + system->tuning = 8; + } + } +} + + /** * takes gfx arch name as C-style string and returns a tuning index to */ int rcclGetTuningIndexForArch(const char* gfxarch) { static const std::vector> tuningIndexMap = { - {"gfx906", 0}, {"gfx908", 0}, {"gfx90a", 0}, {"gfx942", 5}, {"gfx950", 6}, {"gfx1030", 0}, - {"gfx1100",10}, {"gfx1101", 10}, {"gfx1102", 10}, {"gfx1151", 9}, {"gfx1200", 7}, {"gfx1201", 7} + {"gfx906", 0}, {"gfx908", 0}, {"gfx90a", 0}, {"gfx942", 5}, {"gfx950", 6}, {"gfx1250", 11}, {"gfx1030", 0}, + {"gfx1100", 10}, {"gfx1101", 10}, {"gfx1102", 10}, {"gfx1151", 9}, {"gfx1200", 7}, {"gfx1201", 7} }; static const std::vector> tuningIndexMapAINIC = { - {"gfx906", 0}, {"gfx908", 0}, {"gfx90a", 0}, {"gfx942", 8}, {"gfx950", 6}, + {"gfx906", 0}, {"gfx908", 0}, {"gfx90a", 0}, {"gfx942", 8}, {"gfx950", 6}, {"gfx1250", 11}, {"gfx1030", 0}, {"gfx1100", 10}, {"gfx1102", 10}, {"gfx1200", 7}, {"gfx1201", 7} }; @@ -1812,16 +1912,3 @@ int rcclGetTuningIndexForArch(const char* gfxarch) { } return 0; } - -void rcclApplyTuningOverrides(struct ncclTopoSystem* system) { - auto nicInfo = rcclPrimaryNic(); - if (IsArchMatch(system->nodes[GPU].nodes[0].gpu.gcn, "gfx950")) { - system->tuning = 6; - } - - if (nicInfo.type == rcclIBNicTypeAINIC) { - if (IsArchMatch(system->nodes[GPU].nodes[0].gpu.gcn, "gfx942")) { - system->tuning = 8; - } - } -} diff --git a/projects/rccl/src/include/ce_coll.h b/projects/rccl/src/include/ce_coll.h index 10a2fee53863..3d7e063b8099 100644 --- a/projects/rccl/src/include/ce_coll.h +++ b/projects/rccl/src/include/ce_coll.h @@ -22,14 +22,17 @@ // scale-out MPI tests assert on this exact text. #define RCCL_CE_HIER_SELECTED_TAG "[Hierarchical CE]" -// Largest message eligible for the unregistered forced CE AllReduce path. -#define NCCL_CE_AR_MAX_MSG_BYTES (256ull * 1024 * 1024) - // Total payload capacity of one reusable CE AllReduce staging slot. Messages -// larger than this are pipelined; sizing each slot to NCCL_CE_AR_MAX_MSG_BYTES +// larger than this are pipelined; sizing each slot to ceArStagingBytes per-rank // wastes VMM (512 MiB with two slots) and fails late VA reservations on ROCm. #define NCCL_CE_AR_STAGING_BYTES (16ull * 1024 * 1024) +// Fallback 2-shot max cap for rcclCeAr2ShotMax() when no arch table is present. +// Independent of NCCL_CE_AR_STAGING_BYTES (which governs buffer allocation). +#ifndef NCCL_CE_AR_TMPBUF_DEFAULT_BYTES +#define NCCL_CE_AR_TMPBUF_DEFAULT_BYTES (256ULL * 1024 * 1024) +#endif + #ifndef NCCL_CE_REDUCE_MAX_BLOCKS #define NCCL_CE_REDUCE_MAX_BLOCKS 46 #endif @@ -38,7 +41,7 @@ #define NCCL_CE_NUM_SLOTS 2 #endif -// Per-rank staging capacity in ceARTmpBuf. +// Per-rank staging capacity in ceARTmpBuf (fixed default; use ceArStagingBytes for runtime value). inline size_t ncclCeAllReduceMaxChunkBytes(int nRanks) { return (size_t)NCCL_CE_AR_STAGING_BYTES / (size_t)nRanks; } @@ -101,6 +104,8 @@ struct ncclCeColl { // The reduced result is written straight into the user recvbuff (no scratch). uint8_t* ceARTmpBuf; struct ncclDevrWindow* ceARTmpWin; + size_t ceArMaxBytes; // 2-shot staging cap, resolved at init: env RCCL_CE_AR_MAX_MSG_BYTES > arch ceArMax + size_t ceArStagingBytes; // resolved at init: env var RCCL_CE_AR_STAGING_BYTES > NCCL_CE_AR_STAGING_BYTES uint32_t* signalBuffer; struct ncclDevrWindow* signalWin; // Global counter barrier for regular launch: [0]=arrival, [1]=completed generation. diff --git a/projects/rccl/src/include/comm.h b/projects/rccl/src/include/comm.h index e62fa8b37c67..d66d5a75197f 100644 --- a/projects/rccl/src/include/comm.h +++ b/projects/rccl/src/include/comm.h @@ -254,6 +254,7 @@ struct ncclTaskColl { struct ncclDevrWindow* sendWin; struct ncclDevrWindow* recvWin; ncclSymRegType_t winRegType; + rcclSymkExtract symkExtract; void* ddaUserRecvBuff; // user recvbuff (using DDA staging) or NULL otherwise (if recvbuffer is using symmetric windows) size_t ddaCopyBackBytes; // bytes to copy scratch -> user recvbuff @@ -818,6 +819,10 @@ struct ncclComm { [3]; // for each collective, set for 5 channel-counts: 32,40,48,56,64, the two values for min/max size-threshold struct ncclTuningContext_t tuningContext; + // Per-arch DDA/CE dispatch thresholds -- populated at comm init from rcclGetArchThresholds(). + // NULL on architectures without a dedicated threshold table (falls back to env-var params). + const struct rcclArchThresholds* archThresholds; + /* This attribute can indicate the states of communicators and return code of * asynchronous NCCL operations. */ ncclResult_t asyncResult; diff --git a/projects/rccl/src/include/graph.h b/projects/rccl/src/include/graph.h index 5fb8ecb20de6..77f84c460da9 100644 --- a/projects/rccl/src/include/graph.h +++ b/projects/rccl/src/include/graph.h @@ -244,4 +244,5 @@ ncclResult_t ncclTopoTuneModel(struct ncclComm* comm, int minCompCap, int maxCom ncclResult_t ncclTopoGetAlgoTime(struct ncclComm* comm, int coll, int algorithm, int protocol, size_t nBytes, int numPipeOps, float* time); int rcclGetTuningIndexForArch(const char* gfxarch); +#include "rccl_arch_thresholds.h" #endif diff --git a/projects/rccl/src/include/info.h b/projects/rccl/src/include/info.h index d66904e45b3c..0b1ecd55445d 100644 --- a/projects/rccl/src/include/info.h +++ b/projects/rccl/src/include/info.h @@ -47,10 +47,12 @@ struct ncclInfo { // A config copied from config passed by user so older user config can be safely accessed // during synchronous host scheduling (never at launch/replay). ncclCollConfig_t collConfig; - // Implementation decision precomputed by ncclAllReduce_impl() via - // rcclSelectAllReduce() and consumed by taskAppend() to avoid recomputing the - // CE-vs-kernel choice and graph-capture state. Valid only when decisionValid - // is true (false for non-AllReduce collectives and the WithBias path). + // Implementation decision precomputed by ncclAllReduce_impl() / + // ncclAllGather_impl() / ncclReduceScatter_impl() / ncclAlltoAll_impl() via + // the matching rcclSelect*() and consumed by taskAppend() so it does not + // recompute CE-vs-kernel, graph-capture, or symmetric-vs-ring. Valid only + // when decisionValid is true (false for collectives that have not been wired + // yet, and the AllReduce WithBias path). struct rcclCollDecision decision; bool decisionValid; }; diff --git a/projects/rccl/src/include/rccl_arch_thresholds.h b/projects/rccl/src/include/rccl_arch_thresholds.h new file mode 100644 index 000000000000..5093e0e95fdd --- /dev/null +++ b/projects/rccl/src/include/rccl_arch_thresholds.h @@ -0,0 +1,86 @@ +/************************************************************************* + * Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +#ifndef RCCL_ARCH_THRESHOLDS_H_ +#define RCCL_ARCH_THRESHOLDS_H_ + +// Per-architecture DDA/CE/Ring dispatch threshold tables. +// Extracted from graph.h to isolate RCCL-specific dispatch logic from +// NVIDIA-originated graph infrastructure, reducing merge-conflict surface. + +#include "nccl_common.h" // ncclFunc_t, ncclFuncAlltoAll +#include // size_t, SIZE_MAX + +// DDA arrays are indexed by ncclFunc_t through AlltoAll (ncclFuncAlltoAll == 8). +// All collectives compare total message bytes against the table values. +// 0 disables that tier for that collective. No parallel built-in defaults: +// an unset env var uses this table, and an arch with no table gets 0. +enum { RCCL_DDA_FUNC_COUNT = ncclFuncAlltoAll + 1 }; + +// Sentinel for symMaxR2/symMaxR2Graph: no upper-bound suppression (keep symk at all sizes). +// Use this instead of 0 in symMaxR2 arrays -- 0 in a DDA array means disabled, which is +// inconsistent with the no-suppression meaning needed for symMaxR2 unused/uncapped slots. +static constexpr size_t kThreshUnlimited = SIZE_MAX; + +struct rcclArchThresholds { + // DDA tier upper bounds, per collective. gfx1250 uses fabric LL/LL128/VMM; + // gfx942/gfx950 use ddaVmmMax as the DDA-IPC cap (LL/LL128 unused, stay 0). + // All arrays are indexed by ncclFunc_t; 0 disables that tier for that coll. + // All collectives compare total message bytes against the table values. + size_t ddaLLMax[RCCL_DDA_FUNC_COUNT]; // DDA LL tier: 0 .. ddaLLMax[func] + size_t ddaLL128Max[RCCL_DDA_FUNC_COUNT]; // DDA LL128 tier: ddaLLMax[func]+1 .. ddaLL128Max[func] + size_t ddaVmmMax[RCCL_DDA_FUNC_COUNT]; // DDA VMM/IPC cap: ddaLL128Max[func]+1 .. ddaVmmMax[func] + + // R2 variant: DDA VMM cap when recv buffer is registered (winRegType != + // ncclSymSendNonregRecvNonreg). Selectors still require !symEligible (AR: + // !symkRequested) before DDA; registered sum/avg windows never reach this + // cap. It only shortens DDA when recv is registered and DDA is still + // eligible (e.g. a non-sum RS). 0 means "use ddaVmmMax". + size_t ddaVmmMaxR2[RCCL_DDA_FUNC_COUNT]; + + // Graph-mode variant: DDA VMM cap to use when the comm is inside a graph + // capture (graphCapturingHint=true). CE AllReduce is blocked during graph + // captures, so DDA can fill the full window that CE would normally absorb. + // 0 means "use ddaVmmMax" (no graph-specific override). + size_t ddaVmmMaxGraph[RCCL_DDA_FUNC_COUNT]; + + // CE non-registered (scratch) window per collective, total bytes. + // CE-Scratch fires when ceNonRegMin[func] <= totalBytes <= ceNonRegMax[func]. + // ceNonRegMin[func] = 0 means no lower bound (fires immediately above DDA exit). + // ceNonRegMax[func] = 0 means CE-Scratch / CE 2-shot disabled for that collective. + // AllReduce 2-shot: this is a selector cap only. ceARTmpBuf is allocated at + // NCCL_CE_AR_TMPBUF_DEFAULT_BYTES and grown only when this entry (or + // RCCL_CE_AR_MAX_MSG_BYTES) is larger. + size_t ceNonRegMin[RCCL_DDA_FUNC_COUNT]; + size_t ceNonRegMax[RCCL_DDA_FUNC_COUNT]; + + // CE registered-window upper bound per collective, total bytes. + // For AllReduce: registered CE copies through user symmetric windows (tuning cap only). + // For AllGather: upper bound for CE-registered (R=2). 0 means no upper bound. + size_t ceRegMax[RCCL_DDA_FUNC_COUNT]; + + // Symmetric kernel upper-bound per collective when recv buffer is registered (R2). + // Above this size CE is faster than symk; setting this withdraws symk as the final + // choice in rcclSelectAllReduce so CE-registered can win. It does not unblock the + // CE 2-shot or DDA branches, which stay gated on whether symk was requested at all. + // kThreshUnlimited means no suppression (symk may win at any size for that collective). + // Unused collectives default to kThreshUnlimited. + size_t symMaxR2[RCCL_DDA_FUNC_COUNT]; + // Graph-mode variant of symMaxR2 (graphCapturingHint / stream capture). CE is + // graph-unsafe, so graph mode typically wants no suppression (keep symk). + // kThreshUnlimited means no suppression, same convention as symMaxR2 -- not "inherit eager". + size_t symMaxR2Graph[RCCL_DDA_FUNC_COUNT]; + // Symmetric kernel lower-bound per collective when recv buffer is registered (R2). + // Below this size DDA is faster than symk; setting this suppresses symk so DDA + // can win in that sub-range while symk still wins above it. + // 0 means no suppression (symk may win at any size for that collective). + size_t symMinR2[RCCL_DDA_FUNC_COUNT]; + +}; + +const rcclArchThresholds* rcclGetArchThresholds(const char* gcn); + +#endif // RCCL_ARCH_THRESHOLDS_H_ diff --git a/projects/rccl/src/include/rccl_common.h b/projects/rccl/src/include/rccl_common.h index b29dbf0e218c..e169028c48e2 100644 --- a/projects/rccl/src/include/rccl_common.h +++ b/projects/rccl/src/include/rccl_common.h @@ -26,6 +26,7 @@ THE SOFTWARE. #include "param.h" #include "core.h" #include "rccl_decision.h" +#include "sym_kernels.h" typedef enum RcclTunableColls { RCCL_UNSUPPORTED_TUNABLE = -1, @@ -75,15 +76,30 @@ typedef enum { #endif RCCL_SYMMETRIC, // symmetric-window kernel RCCL_CE_2SHOT, // eager Copy-Engine 2-shot AllReduce (staging buffer) - RCCL_CE_REGISTERED, // Copy-Engine via registered windows / CTA_POLICY_ZERO + RCCL_CE_REGISTERED, // Copy-Engine via registered symmetric windows (CTAPolicy=ZERO) + RCCL_CE_SCRATCH, // Copy-Engine via DDA scratch buffer (RCCL_FORCE_CE + unregistered) RCCL_DDA_FABRIC_LL, // DDA fabric, LL protocol (small-message fast lane) RCCL_DDA_FABRIC_LL128,// DDA fabric, LL128 protocol (mid-message fast lane) RCCL_DDA_FABRIC_VMM, // DDA fabric, VMM/Simple path RCCL_DDA_IPC, // DDA IPC (single-node, fixed nRanks) RCCL_GIN_SDMA, // GIN-SDMA AllReduce (scaleup LSA/GIN) + RCCL_A2A_PIVOT, // AlltoAll pivot algorithm (large, aligned messages) + RCCL_A2A_GDA, // AlltoAll via GDA/RocSHMEM + RCCL_A2A_GIN_SDMA, // AlltoAll via GIN LSA/SDMA + // Appended rather than grouped with the other Direct entries so existing + // values stay stable. + RCCL_DIRECT_ALLTOALL, // AlltoAll as per-peer Send/Recv (no collective kernel) RCCL_ALGO_COUNT } rcclAddonAlgos_t; +// Tag written by rcclSelect* into ncclTaskColl::symkExtract so the extractor +// (ncclMakeSymmetricTaskList) knows whether the selector already vetoed symk. +enum rcclSymkExtract : int8_t { + RCCL_SYMK_EXTRACT_DENY = -1, // selector chose non-symk backend; do not extract + RCCL_SYMK_EXTRACT_NONE = 0, // no opinion; extractor uses windows + ncclSymkAvailable + RCCL_SYMK_EXTRACT_ALLOW = 1, // rcclSelect* chose RCCL_SYMMETRIC; may extract +}; + #ifdef RCCL_EXPOSE_STATIC #define RCCL_STATIC_EXPOSE_CHECK() #else @@ -142,8 +158,8 @@ NCCL_API(ncclResult_t, rcclGetAlgoInfo, struct ncclComm* comm, ncclFunc_t coll, // the full backend RCCL would actually run (CE, DDA, symmetric, or kernel) for // the given operands, so rccl-tests can attribute numbers to the right label. // `algo` returns a native NCCL_ALGO_* or rcclAddonAlgos_t value; name it with -// rcclGetAlgoName(). Currently implemented for AllReduce and AllGather; other -// collectives fall back to rcclGetAlgoInfo(). +// rcclGetAlgoName(). Currently implemented for AllReduce, AllGather, +// ReduceScatter, and AlltoAll; other collectives fall back to rcclGetAlgoInfo(). // // graphCapturing: pass non-zero if the collective will execute under HIP/CUDA // graph capture. This query is normally issued outside capture (before/after the @@ -165,23 +181,39 @@ NCCL_API(ncclResult_t, rcclGetCollImplInfo, struct ncclComm* comm, ncclFunc_t co ncclResult_t rcclSelectAllReduce(struct ncclComm* comm, const void* sendbuff, void* recvbuff, size_t count, ncclDataType_t datatype, ncclRedOp_t op, cudaStream_t stream, bool query, bool graphCapturingHint, struct rcclCollDecision* decision); -// Single source of truth for AllGather selection: DDA -> hierarchical kernel -> -// CE (single-node or hierarchical CE) -> direct -> ring. query=true fills -// protocol/nMaxChannels for reporting. CE dispatch lives in taskAppend(), so live -// returns RCCL_CE_REGISTERED but enqueues normally. -// graphCapturingHint (query only) suppresses the graph-unsafe CE branch under capture. +// Single source of truth for AllGather selection: DDA -> hierarchical -> CE -> +// direct -> symmetric -> ring. CE dispatch lives in taskAppend(); live returns +// RCCL_CE_REGISTERED / RCCL_CE_SCRATCH and enqueues with that decision. +// query=false : live dispatch (ncclAllGather_impl). ceCapturing is probed from +// `stream`; graphCapturingHint is ignored. +// query=true : side-effect-free reporting. The stream is not probed; +// graphCapturingHint supplies capture so CE is reported as skipped. ncclResult_t rcclSelectAllGather(struct ncclComm* comm, const void* sendbuff, void* recvbuff, size_t sendcount, - ncclDataType_t datatype, bool query, bool graphCapturingHint, + ncclDataType_t datatype, cudaStream_t stream, bool query, bool graphCapturingHint, struct rcclCollDecision* decision); // Single source of truth for ReduceScatter selection: symmetric -> DDA fabric -// (LL/LL128/VMM) / DDA IPC -> hierarchical -> Direct -> native ring/pat kernel. RS has no CE. +// (LL/LL128/VMM) / DDA IPC -> hierarchical -> Direct -> native ring/pat kernel. +// RS has no CE. Live enqueue carries the decision so taskAppend honors +// RCCL_SYMMETRIC vs ring (symkExtract) instead of re-deriving it. ncclResult_t rcclSelectReduceScatter(struct ncclComm* comm, const void* sendbuff, void* recvbuff, size_t recvcount, ncclDataType_t datatype, ncclRedOp_t op, bool query, struct rcclCollDecision* decision); +// Single source of truth for AlltoAll selection: Pivot -> GDA -> DDA (LL/LL128/VMM/IPC) -> +// CE registered -> HierCE -> CE scratch -> Direct (per-peer Send/Recv). +// Live enqueue carries the decision so taskAppend honors CE vs Direct instead of +// re-deriving it. query=false probes `stream` for capture; query=true uses +// graphCapturingHint and does not probe the stream. +ncclResult_t rcclSelectAlltoAll(struct ncclComm* comm, const void* sendbuff, void* recvbuff, size_t count, + ncclDataType_t datatype, cudaStream_t stream, bool query, bool graphCapturingHint, + struct rcclCollDecision* decision); // Selection helpers shared between collectives.cc and the wrapped decision logic. // (rcclDdaEnabled is declared below, next to the DDA param decls.) bool isSymmetricKernelRequested(struct ncclComm* comm, ncclFunc_t coll, int symkOp, ncclDataType_t datatype, size_t nElts, const void* sendbuff, void* recvbuff, bool agreeAcrossRanks = false); +// Pre-window variant: caller has already called ncclDevrFindWindow for both +// pointers; this skips the redundant lookup. +bool isSymmetricKernelRequestedWin(struct ncclComm* comm, ncclFunc_t coll, int symkOp, ncclDataType_t datatype, + size_t nElts, struct ncclDevrWindow* sendWin, struct ncclDevrWindow* recvWin); NCCL_API(ncclResult_t, rcclSymKGetInfo, struct ncclComm* comm, ncclFunc_t coll, uint64_t count, ncclDataType_t dataType, ncclRedOp_t op, int* algo, int* protocol, int* maxChannels); NCCL_API(ncclResult_t, rcclGetAlgoName, int algo, const char** algoName); @@ -198,19 +230,44 @@ bool rcclUseAlltoAllGda(struct ncclComm* comm); // Returns true when the CE AllReduce path should be used instead of the standard ring/tree kernels. // Pass the bias buffer as acc (nullptr when the caller is plain AllReduce). // Does NOT check ceARTmpBuf initialization; the caller is responsible. -bool rcclUseCeAllReduce(struct ncclComm* comm, size_t count, ncclDataType_t datatype, ncclRedOp_t op, const void* acc); +bool rcclUseCeAr2Shot(struct ncclComm* comm, size_t count, ncclDataType_t datatype, ncclRedOp_t op, const void* acc); // Updates the CE AllReduce graph latch from this call's capture state. // Invoke once per collective (any type) at each CE AR decision point. void rcclCeAllReduceGraphLatchTick(struct ncclComm* comm, bool ceCapturing); // Pure query: is CE AllReduce currently allowed on this comm? -bool rcclCeAllReduceAllowed(struct ncclComm* comm); +bool rcclCeArGraphSafe(struct ncclComm* comm); +// CE AllReduce knobs (defined in rccl_wrap.cc). Both default to -1 (unset), which +// resolves to the per-arch default; use the resolvers below rather than the raw +// params so an unset env var does not read as "disabled". +RCCL_PARAM_DECLARE(CeAllReduce); +RCCL_PARAM_DECLARE(ForceCeAllReduce); +// Is CE AllReduce enabled for this comm? RCCL_CE_ALLREDUCE wins when set; +// otherwise CE AllReduce is on only for the arch it is tuned for (gfx1250) and +// off elsewhere, independent of that arch table's ceNonRegMax/ceRegMax entries. +bool rcclCeAllReduceEnabled(const struct ncclComm* comm); +// Same resolution for RCCL_FORCE_CE_ALLREDUCE, which lets CE AllReduce (2-shot +// and registered) run without CTAPolicy=ZERO. +bool rcclForceCeAllReduceEnabled(const struct ncclComm* comm); +RCCL_PARAM_DECLARE(CeArMaxMsgBytes); // -1 = use ceNonRegMax[AR] (2-shot) from arch table +RCCL_PARAM_DECLARE(CeArRegMaxMsgBytes); // -1 = use ceRegMax[AR] (registered) from arch table +RCCL_PARAM_DECLARE(CeArStagingBytes); // -1 = use NCCL_CE_AR_STAGING_BYTES; sizes ceARTmpBuf, not the cap +// 2-shot AllReduce size cap: env RCCL_CE_AR_MAX_MSG_BYTES if set, else +// ceNonRegMax[AR] from the arch table (0 = 2-shot disabled). A null table +// (RCCL_IGNORE_ARCH_TABLE or unknown arch) restores the pre-table 256 MiB +// window. Does not size ceARTmpBuf; that stays at +// NCCL_CE_AR_TMPBUF_DEFAULT_BYTES unless this cap is larger. +size_t rcclCeAr2ShotMax(const ncclComm* comm); +// True when NCCL_ALGO is set by the user. Used to skip CE/DDA/Symmetric in +// both the selector (rccl_wrap.cc) and taskAppend (enqueue.cc). +bool rcclNcclAlgoEnvIsSet(); +// Registered CE AllReduce AUTO size cap. Env RCCL_CE_AR_REG_MAX_MSG_BYTES wins // Decides whether ncclAllReduce_impl takes the DDA path for this call. Mirrors the guard in -// collectives.cc exactly: DDA runs when the buffers are not symmetric-kernel eligible, CE AllReduce -// will not service the call per the caller-computed `ceAllReduceAllowed` (non-gfx1250 only; gfx1250 -// always keeps the DDA fabric path), and DDA is enabled for this arch/size. Host-side and GPU-free so -// the dispatch decision can be unit tested. +// collectives.cc exactly: DDA requires !symEligible on every arch (gfx1250 fabric included). +// Non-gfx1250 also requires CE AllReduce not to service the call (`ceAllReduceAllowed`); +// gfx1250 may still take fabric DDA when CE is eligible. DDA must also be enabled for this +// arch/size. Host-side and GPU-free so the dispatch decision can be unit tested. bool rcclAllReduceShouldTakeDdaPath(const struct ncclComm* comm, size_t count, ncclDataType_t datatype, - bool symEligible, bool ceAllReduceAllowed); + bool symEligible, bool ceAllReduceAllowed, bool query); // Decides whether ncclAlltoAll_impl takes the DDA early-return. AlltoAll has no // symmetric kernel, so unlike AllGather it cannot gate DDA on !symEligible. // `ceAlltoAllAllowed` is single-node CE (ncclCeAvailable); hier CE does not yield DDA. @@ -229,6 +286,8 @@ RCCL_PARAM_DECLARE(DirectReduceScatterThreshold); RCCL_PARAM_DECLARE(HierarchicalAllGather); // Hierarchical ReduceScatter enabled RCCL_PARAM_DECLARE(HierarchicalReduceScatter); +// Pivot AlltoAll enabled (defined in collectives.cc) +RCCL_PARAM_DECLARE(AlltoAllPivotEnable); #define HIERARCHICAL_TEMP_BUFFER_SIZE (128 * 1024 * 1024) // 128MB // DDA threshold @@ -240,16 +299,55 @@ RCCL_PARAM_DECLARE(DdaLL128Threshold); RCCL_PARAM_DECLARE(DdaEnable); extern int64_t ncclParamP2pDisable(); -// Per-collective DDA AlltoAll thresholds (4 MiB for all supported archs). -constexpr size_t kDdaAlltoAllGfx942ThresholdBytes = 4194304; -constexpr size_t kDdaAlltoAllGfx950ThresholdBytes = 4194304; -constexpr size_t kDdaAlltoAllGfx1250ThresholdBytes = 4194304; +// Value of RCCL_DDA_THRESHOLD / RCCL_DDA_LL_THRESHOLD / RCCL_DDA_LL128_THRESHOLD +// meaning "the user did not set this". 0 already means "disable this tier", so +// unset needs a value of its own for the arch tables to act as defaults. +constexpr int64_t kDdaThresholdUnset = -1; +// Pre-arch-table env var defaults, used as fallbacks when RCCL_IGNORE_ARCH_TABLE=1. +constexpr size_t kDdaLLBaseDefault = 65536; // 64 KiB (develop default) +constexpr size_t kDdaLL128BaseDefault = 0; // off (develop default: DDA_LL128=0) +constexpr size_t kDdaVmmBaseDefault = 134217728; // 128 MiB + +// Per-tier DDA size caps for this collective: env var (when set) else arch table. +// RCCL_IGNORE_ARCH_TABLE or a NULL/unknown-arch table restores the pre-table +// defaults (kDdaLLBaseDefault / kDdaLL128BaseDefault / kDdaVmmBaseDefault), +// which apply to every collective the way the old global env vars did. A table +// entry of 0 still disables that collective. A NULL `comm` uses the defaults. +size_t rcclDdaLLThreshold(const ncclComm* comm, ncclFunc_t func); +size_t rcclDdaLL128Threshold(const ncclComm* comm, ncclFunc_t func); +size_t rcclDdaVmmThreshold(const ncclComm* comm, ncclFunc_t func); + +// Cap for the DDA entry gate: the max of the LL, LL128 and VMM caps above, +// skipping tiers turned off by RCCL_DDA_LL / RCCL_DDA_LL128. Pass this to +// rcclDdaEnabled() so a collective whose VMM tier is tuned off (ddaVmmMax = 0) +// can still reach its LL/LL128 tiers; each tier re-checks its own cap inside. +size_t rcclDdaEntryThreshold(const ncclComm* comm, ncclFunc_t func); + +// True when AllGather should run CE-registered rather than the symmetric kernel: +// recv buffer registered, message above symMaxR2[AG] (the symk/CE crossover) and +// within ceRegMax[AG]. symMaxR2[AG] = 0 keeps symk at every size. Shared by the +// selector (reporting) and taskAppend (dispatch) so the two cannot disagree. +bool rcclAllGatherCeRegisteredWindow(const ncclComm* comm, size_t totalBytes, ncclSymRegType_t winRegType, + bool graphMode); + +// Payload cap used to size comm->ddaScratch: max of every DDA protocol table +// entry (LL / LL128 / VMM, including R2 and graph VMM variants) plus +// ceNonRegMax for collectives that stage through ddaScratch (not AllReduce +// 2-shot, which uses ceARTmpBuf). Env DDA_*_THRESHOLD, when set, is also +// folded in. With no table, the same pre-table DDA defaults the selectors +// use are folded in so scratch is not smaller than the VMM/LL/LL128 window. +// All table values are total message bytes; no per-rank scaling is applied. +// +// Graph VMM (ddaVmmMaxGraph) is part of that max even for comms that never +// capture: scratch is allocated once at init so a later capture can still +// fit. On gfx1250 that is AllReduce 256 MiB whenever it is the largest entry. +size_t rcclDdaScratchPayloadCap(const ncclComm* comm); -// Returns true when the DDA fast path should be attempted for a collective. -// Per-arch defaults cap the threshold; when 0, gfx950/gfx1250 fall back to -// the user-configurable RCCL_DDA_THRESHOLD env var. -bool rcclDdaEnabled(const ncclComm* comm, size_t totalBytes, size_t gfx942Default, size_t gfx950Default = 0, - size_t gfx1250Default = 0); +// Returns true when the DDA fast path should be attempted for this arch/size. +// `threshold` is the per-collective cap from rcclDdaEntryThreshold(). 0 disables +// DDA for the call. +bool rcclDdaEnabled(const ncclComm* comm, size_t totalBytes, size_t threshold, + bool query = false, const char* prefix = nullptr); int getFirmwareVersion(); bool rcclIsArchSupportedForFunc(struct ncclTaskColl* info, char const* archName); diff --git a/projects/rccl/src/include/rccl_decision.h b/projects/rccl/src/include/rccl_decision.h index 13d3e749e6bc..0acd40e5442c 100644 --- a/projects/rccl/src/include/rccl_decision.h +++ b/projects/rccl/src/include/rccl_decision.h @@ -41,7 +41,7 @@ struct rcclCollDecision { int protocol; // NCCL_PROTO_* uint32_t nMaxChannels; // reporting: channels for the kernel path (0 = N/A) // Runtime bits computed once at the decision point and carried into - // taskAppend() so it never recomputes graph-capture state for AllReduce. + // taskAppend() so it never recomputes graph-capture state. bool ceCapturing; bool ceArGraphAllowed; }; diff --git a/projects/rccl/src/init.cc b/projects/rccl/src/init.cc index 64686af5a846..bad0507b38c8 100644 --- a/projects/rccl/src/init.cc +++ b/projects/rccl/src/init.cc @@ -224,7 +224,6 @@ RCCL_PARAM(TdmSimpleEnable, "TDM_SIMPLE_ENABLE", 0); * Used on gfx1151 (StrixHalo) to set the nChannels for ncclTopoPreset before determining number of nodes. */ RCCL_PARAM(InitChannels, "INIT_CHANNELS", -1); -RCCL_PARAM_DECLARE(ForceCeAllReduce); // Returns the process-wide NCCL_CTA_POLICY env override, or NCCL_CONFIG_UNDEF_INT when the env var // is unset or held no valid token. A UNDEF result means no env override was applied, so per-call and @@ -2926,6 +2925,7 @@ static ncclResult_t ncclCommInitRankFunc(struct ncclAsyncJob* job_) { // [RCCL] Host mirrors of device side NCCL_LL128_LINEELEMS / NCCL_LL128_DATAELEMS comm->ll128LineElems = rcclLL128LineElemsFromArch(comm->archName); comm->ll128DataElems = rcclLL128DataElemsFromArch(comm->archName); + comm->archThresholds = rcclGetArchThresholds(comm->archName); NCCLCHECKGOTO(initTransportsRank(comm, job->parent, timers), res, fail); diff --git a/projects/rccl/src/rccl_arch_thresholds.cc b/projects/rccl/src/rccl_arch_thresholds.cc new file mode 100644 index 000000000000..6e9616708518 --- /dev/null +++ b/projects/rccl/src/rccl_arch_thresholds.cc @@ -0,0 +1,253 @@ +/************************************************************************* + * Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +// Per-architecture DDA/CE dispatch threshold tables and tuning index lookup. +// Moved from graph/tuning.cc to isolate RCCL-specific dispatch data from +// NVIDIA-originated graph infrastructure. + +#include "rccl_arch_thresholds.h" +#include "archinfo.h" // IsArchMatch +#include "net.h" // rcclPrimaryNic, rcclIBNicTypeAINIC + +#include +#include + +// Per-arch DDA/CE dispatch threshold tables. +// DDA arrays: [Broadcast=0, Reduce=1, AllGather=2, ReduceScatter=3, AllReduce=4, +// SendRecv=5, Send=6, Recv=7, AlltoAll=8]. 0 disables that tier. +// All collectives compare total message bytes against the table values. + +// gfx1250 placeholders -- validate against sweep data (AICOMRCCL-1756). +// Index mapping: [Bcast=0, Reduce=1, AG=2, RS=3, AR=4, SR=5, Send=6, Recv=7, A2A=8] +static const rcclArchThresholds rcclArchThresholds_gfx1250 = { + // ddaLLMax: DDA LL tier ceiling per collective (fabric gfx1250 only). + .ddaLLMax = { + 0, // [0] Broadcast -- not used + 0, // [1] Reduce -- not used + 128ULL*1024, // [2] AllGather + 16ULL*1024*1024, // [3] ReduceScatter + 32ULL*1024*1024, // [4] AllReduce + 0, // [5] SendRecv -- not used + 0, // [6] Send -- not used + 0, // [7] Recv -- not used + 64ULL*1024, // [8] AlltoAll + }, + // ddaLL128Max: DDA LL128 tier ceiling per collective. + // RCCL_PARAM(DdaLL128, ...) defaults to 1 (enabled); set RCCL_DDA_LL128=0 to disable. + .ddaLL128Max = { + 0, // [0] Broadcast -- not used + 0, // [1] Reduce -- not used + 64ULL*1024*1024, // [2] AllGather + 2ULL*1024*1024, // [3] ReduceScatter + 32ULL*1024*1024, // [4] AllReduce + 0, // [5] SendRecv -- not used + 0, // [6] Send -- not used + 0, // [7] Recv -- not used + 1ULL*1024*1024, // [8] AlltoAll + }, + // ddaVmmMax: DDA VMM (fabric simple) tier ceiling per collective. + // Messages above this fall to Ring/CE (or sym kernel for R2). + .ddaVmmMax = { + 0, // [0] Broadcast -- not used + 0, // [1] Reduce -- not used + 0, // [2] AllGather -- 0 = disabled + 0, // [3] ReduceScatter -- 0 = disabled + 16ULL*1024*1024, // [4] AllReduce + 0, // [5] SendRecv -- not used + 0, // [6] Send -- not used + 0, // [7] Recv -- not used + 0, // [8] AlltoAll -- 0 = disabled + }, + // ddaVmmMaxR2: DDA VMM cap when recv buffer is registered (R2 mode). + // DDA is gated on !symEligible (AR: !symkRequested) on every arch, including + // gfx1250 fabric; R2 sum/avg never takes DDA. A non-zero entry only shortens + // DDA when recv is registered and that gate still passes. 0 means use ddaVmmMax. + .ddaVmmMaxR2 = { + 0, // [0] Broadcast -- not used + 0, // [1] Reduce -- not used + 0, // [2] AllGather -- not used + 0, // [3] ReduceScatter -- not used + 4ULL*1024*1024, // [4] AllReduce + 0, // [5] SendRecv -- not used + 0, // [6] Send -- not used + 0, // [7] Recv -- not used + 0, // [8] AlltoAll -- not used + }, + // ddaVmmMaxGraph: DDA VMM cap during graph capture (graphCapturingHint=true). + // CE AllReduce is blocked by graphModeSeen latch during graph captures. + // Extending DDA VMM for AR lets DDA fill the window CE would otherwise absorb. + // 0 means use ddaVmmMax (no graph-specific override for that collective). + .ddaVmmMaxGraph = { + 0, // [0] Broadcast -- not used + 0, // [1] Reduce -- not used + 0, // [2] AllGather -- not used + 0, // [3] ReduceScatter -- not used + 32ULL*1024*1024, // [4] AllReduce + 0, // [5] SendRecv -- not used + 0, // [6] Send -- not used + 0, // [7] Recv -- not used + 0, // [8] AlltoAll -- not used + }, + // ceNonRegMin: lower bound for CE-Scratch window per collective (-R 0). + // 0 = disabled for that collective. + .ceNonRegMin = { + 0, // [0] Broadcast -- not used + 0, // [1] Reduce -- not used + 0, // [2] AllGather -- 0 = disabled + 0, // [3] ReduceScatter -- not used + 0, // [4] AllReduce -- not used + 0, // [5] SendRecv -- not used + 0, // [6] Send -- not used + 0, // [7] Recv -- not used + 0, // [8] AlltoAll -- not used + }, + // ceNonRegMax: upper bound for CE-Scratch window per collective (-R 0). + // AR: 0 = 2-shot off. ceARTmpBuf stays at the default 256 MiB unless this entry is larger. + .ceNonRegMax = { + 0, // [0] Broadcast -- not used + 0, // [1] Reduce -- not used + 0, // [2] AllGather -- 0 = disabled + 0, // [3] ReduceScatter -- not used + 0, // [4] AllReduce -- 0 = 2-shot off + 0, // [5] SendRecv -- not used + 0, // [6] Send -- not used + 0, // [7] Recv -- not used + 0, // [8] AlltoAll -- not used + }, + // ceRegMax: registered CE upper bound per collective (-R 2). + .ceRegMax = { + 0, // [0] Broadcast -- not used + 0, // [1] Reduce -- not used + 8ULL*1024*1024*1024, // [2] AllGather + 0, // [3] ReduceScatter -- not used + 256ULL*1024*1024, // [4] AllReduce + 0, // [5] SendRecv -- not used + 0, // [6] Send -- not used + 0, // [7] Recv -- not used + 0, // [8] AlltoAll -- not used + }, + // symMaxR2: suppress symk in favour of CE-registered when recv is registered and + // msg > threshold. kThreshUnlimited = no suppression; non-zero literal = byte cap; + // 0 = always suppress (symk not used for this collective on gfx1250). + .symMaxR2 = { + 0, // [0] Broadcast -- not used on gfx1250 + 0, // [1] Reduce -- not used on gfx1250 + 2ULL*1024*1024, // [2] AllGather + kThreshUnlimited, // [3] ReduceScatter + 0, // [4] AllReduce + 0, // [5] SendRecv -- not used on gfx1250 + 0, // [6] Send -- not used on gfx1250 + 0, // [7] Recv -- not used on gfx1250 + 0, // [8] AlltoAll -- not used on gfx1250 + }, + // Graph capture: CE is blocked, so symk or DDA wins. kThreshUnlimited = no suppression; + // non-zero literal = byte cap above which symk is withdrawn so DDA can win instead; + // 0 = always suppress (symk not used for this collective on gfx1250). + .symMaxR2Graph = { + 0, // [0] Broadcast -- not used on gfx1250 + 0, // [1] Reduce -- not used on gfx1250 + 2ULL*1024*1024, // [2] AllGather + kThreshUnlimited, // [3] ReduceScatter + 0, // [4] AllReduce + 0, // [5] SendRecv -- not used on gfx1250 + 0, // [6] Send -- not used on gfx1250 + 0, // [7] Recv -- not used on gfx1250 + 0, // [8] AlltoAll -- not used on gfx1250 + }, + // symMinR2: suppress symk below this size for R2 buffers so DDA wins in that sub-range. + // kThreshUnlimited = always suppress (totalBytes < SIZE_MAX always true) for unused collectives. + .symMinR2 = { + kThreshUnlimited, // [0] Broadcast -- not used on gfx1250 + kThreshUnlimited, // [1] Reduce -- not used on gfx1250 + 128ULL*1024, // [2] AllGather + 2ULL*1024*1024, // [3] ReduceScatter + 8ULL*1024*1024, // [4] AllReduce + kThreshUnlimited, // [5] SendRecv -- not used on gfx1250 + kThreshUnlimited, // [6] Send -- not used on gfx1250 + kThreshUnlimited, // [7] Recv -- not used on gfx1250 + kThreshUnlimited, // [8] AlltoAll -- not used on gfx1250 + }, +}; + +// gfx950: DDA-IPC cap is 128 MiB for AR/AG/RS and 4 MiB for AlltoAll. No fabric LL/LL128. +static const rcclArchThresholds rcclArchThresholds_gfx950 = { + .ddaLLMax = {0, 0, 0, 0, 0, 0, 0, 0, 0}, + .ddaLL128Max = {0, 0, 0, 0, 0, 0, 0, 0, 0}, + .ddaVmmMax = {0, 0, 128ULL*1024*1024, 128ULL*1024*1024, 128ULL*1024*1024, 0, 0, 0, 4ULL*1024*1024}, + .ddaVmmMaxR2 = {0, 0, 0, 0, 0, 0, 0, 0, 0}, + .ddaVmmMaxGraph = {0, 0, 0, 0, 0, 0, 0, 0, 0}, + .ceNonRegMin = {0, 0, 0, 0, 0, 0, 0, 0, 0}, + .ceNonRegMax = {0, 0, 0, 0, 256ULL*1024*1024, 0, 0, 0, 0}, + .ceRegMax = {0, 0, 0, 0, 256ULL*1024*1024, 0, 0, 0, 0}, + .symMaxR2 = { + kThreshUnlimited, // [0] Broadcast -- not used + kThreshUnlimited, // [1] Reduce -- not used + kThreshUnlimited, // [2] AllGather + kThreshUnlimited, // [3] ReduceScatter + kThreshUnlimited, // [4] AllReduce + kThreshUnlimited, // [5] SendRecv -- not used + kThreshUnlimited, // [6] Send -- not used + kThreshUnlimited, // [7] Recv -- not used + kThreshUnlimited, // [8] AlltoAll -- not used + }, + .symMaxR2Graph = { + kThreshUnlimited, // [0] Broadcast -- not used + kThreshUnlimited, // [1] Reduce -- not used + kThreshUnlimited, // [2] AllGather + kThreshUnlimited, // [3] ReduceScatter + kThreshUnlimited, // [4] AllReduce + kThreshUnlimited, // [5] SendRecv -- not used + kThreshUnlimited, // [6] Send -- not used + kThreshUnlimited, // [7] Recv -- not used + kThreshUnlimited, // [8] AlltoAll -- not used + }, + .symMinR2 = {0, 0, 0, 0, 0, 0, 0, 0, 0}, +}; + +// gfx942: DDA-IPC cap is 8 MiB for AR/AG/RS and 4 MiB for AlltoAll. No fabric LL/LL128. +static const rcclArchThresholds rcclArchThresholds_gfx942 = { + .ddaLLMax = {0, 0, 0, 0, 0, 0, 0, 0, 0}, + .ddaLL128Max = {0, 0, 0, 0, 0, 0, 0, 0, 0}, + .ddaVmmMax = {0, 0, 8ULL*1024*1024, 8ULL*1024*1024, 8ULL*1024*1024, 0, 0, 0, 4ULL*1024*1024}, + .ddaVmmMaxR2 = {0, 0, 0, 0, 0, 0, 0, 0, 0}, + .ddaVmmMaxGraph = {0, 0, 0, 0, 0, 0, 0, 0, 0}, + .ceNonRegMin = {0, 0, 0, 0, 0, 0, 0, 0, 0}, + .ceNonRegMax = {0, 0, 0, 0, 256ULL*1024*1024, 0, 0, 0, 0}, + .ceRegMax = {0, 0, 0, 0, 256ULL*1024*1024, 0, 0, 0, 0}, + .symMaxR2 = { + kThreshUnlimited, // [0] Broadcast -- not used + kThreshUnlimited, // [1] Reduce -- not used + kThreshUnlimited, // [2] AllGather + kThreshUnlimited, // [3] ReduceScatter + kThreshUnlimited, // [4] AllReduce + kThreshUnlimited, // [5] SendRecv -- not used + kThreshUnlimited, // [6] Send -- not used + kThreshUnlimited, // [7] Recv -- not used + kThreshUnlimited, // [8] AlltoAll -- not used + }, + .symMaxR2Graph = { + kThreshUnlimited, // [0] Broadcast -- not used + kThreshUnlimited, // [1] Reduce -- not used + kThreshUnlimited, // [2] AllGather + kThreshUnlimited, // [3] ReduceScatter + kThreshUnlimited, // [4] AllReduce + kThreshUnlimited, // [5] SendRecv -- not used + kThreshUnlimited, // [6] Send -- not used + kThreshUnlimited, // [7] Recv -- not used + kThreshUnlimited, // [8] AlltoAll -- not used + }, + .symMinR2 = {0, 0, 0, 0, 0, 0, 0, 0, 0}, +}; + +const rcclArchThresholds* rcclGetArchThresholds(const char* gcn) { + if (gcn == nullptr) return nullptr; + if (IsArchMatch(gcn, "gfx1250")) return &rcclArchThresholds_gfx1250; + if (IsArchMatch(gcn, "gfx950")) return &rcclArchThresholds_gfx950; + if (IsArchMatch(gcn, "gfx942")) return &rcclArchThresholds_gfx942; + return nullptr; +} + + diff --git a/projects/rccl/src/rccl_wrap.cc b/projects/rccl/src/rccl_wrap.cc index b19939310306..3bd8083296dc 100644 --- a/projects/rccl/src/rccl_wrap.cc +++ b/projects/rccl/src/rccl_wrap.cc @@ -25,6 +25,7 @@ THE SOFTWARE. #include "graph/topo.h" #include "enqueue.h" #include +#include #include "debug.h" #include "net.h" #include "amdsmi_wrap.h" @@ -38,6 +39,8 @@ THE SOFTWARE. #if defined(ENABLE_ROCSHMEM_GIN) #include "algorithms/gin/gin_all_reduce.h" #endif +#include "algorithms/dda/alltoall/dda_alltoall.h" +#include "algorithms/gin/gin_alltoall.h" #include "group.h" #include "sym_kernels.h" #include "dev_runtime.h" @@ -56,10 +59,14 @@ RCCL_PARAM(DirectReduceScatterThreshold, "DIRECT_REDUCE_SCATTER_THRESHOLD", 8388 RCCL_PARAM(DirectReduceScatterDisable, "DIRECT_REDUCE_SCATTER_DISABLE", 0); constexpr int AinicMaxDirectAGScale = 8; RCCL_PARAM(DirectAllGatherDisable, "DIRECT_ALLGATHER_DISABLE", -1); -RCCL_PARAM(CeAllReduce, "CE_ALLREDUCE", 0); +// -1 = unset: fall back to the per-arch default (see rcclCeAllReduceEnabled). +RCCL_PARAM(CeAllReduce, "CE_ALLREDUCE", -1); RCCL_PARAM(ThreadsPerBlock, "THREADS_PER_BLOCK", -1); RCCL_PARAM(UnrollFactor, "UNROLL_FACTOR", -1); -RCCL_PARAM(ForceCeAllReduce, "FORCE_CE_ALLREDUCE", 0); +RCCL_PARAM(ForceCeAllReduce, "FORCE_CE_ALLREDUCE", -1); +RCCL_PARAM(CeArMaxMsgBytes, "CE_AR_MAX_MSG_BYTES", -1); // -1 = use ceArMax from arch table (2-shot) +RCCL_PARAM(CeArStagingBytes, "CE_AR_STAGING_BYTES", -1); // -1 = use NCCL_CE_AR_STAGING_BYTES default +RCCL_PARAM(CeArRegMaxMsgBytes, "CE_AR_REG_MAX_MSG_BYTES", -1); // -1 = use ceArRegMax (registered) // Common DDA protocol-tier knobs, shared by every fabric collective (no // per-collective variants). For a given collective's size: @@ -69,16 +76,35 @@ RCCL_PARAM(ForceCeAllReduce, "FORCE_CE_ALLREDUCE", 0); // Constraint: DdaLLThreshold <= DdaLL128Threshold <= DdaThreshold. Setting an // enable flag (or its threshold) to 0 disables that tier and falls through to // the next, so each protocol can be A/B'd in isolation at runtime. +// +// The three thresholds default to kDdaThresholdUnset (-1) rather than to their +// sizes: 0 is a meaningful setting (disable the tier), so "unset" needs a value +// of its own for the arch tables to be able to supply a default. Read them +// through rcclDda{LL,LL128,Vmm}Threshold(), which resolve unset against this +// comm's arch table and yield 0 (no DDA) for arches that have none. RCCL_PARAM(DdaEnable, "DDA_ENABLE", 1); -RCCL_PARAM(DdaThreshold, "DDA_THRESHOLD", (size_t)(134217728)); // 128 MiB +RCCL_PARAM(DdaThreshold, "DDA_THRESHOLD", kDdaThresholdUnset); RCCL_PARAM(DdaLL, "DDA_LL", 1); -RCCL_PARAM(DdaLLThreshold, "DDA_LL_THRESHOLD", (size_t)(65536)); // 64 KiB +RCCL_PARAM(DdaLLThreshold, "DDA_LL_THRESHOLD", kDdaThresholdUnset); RCCL_PARAM(DdaLLOneShotThreshold, "DDA_LL_ONESHOT_THRESHOLD", (size_t)(1) * 1024 * 1024); // 1 MiB RCCL_PARAM(DdaLLTwoShotThreshold, "DDA_LL_TWOSHOT_THRESHOLD", (size_t)(2) * 1024 * 1024); // 2 MiB RCCL_PARAM(DdaLL128OneShotThreshold, "DDA_LL128_ONESHOT_THRESHOLD", (size_t)(4) * 1024 * 1024); // 4 MiB RCCL_PARAM(DdaLL128TwoShotThreshold, "DDA_LL128_TWOSHOT_THRESHOLD", (size_t)(64) * 1024 * 1024); // 64 MiB -RCCL_PARAM(DdaLL128, "DDA_LL128", 0); -RCCL_PARAM(DdaLL128Threshold, "DDA_LL128_THRESHOLD", (size_t)(67108864)); // 64 MiB +RCCL_PARAM(DdaLL128, "DDA_LL128", kDdaThresholdUnset); +RCCL_PARAM(DdaLL128Threshold, "DDA_LL128_THRESHOLD", kDdaThresholdUnset); +// When set, bypass the per-arch tuning table entirely and use base-commit +// env-var defaults for all thresholds (DDA, CE, symMaxR2). Useful for +// isolating arch-table effects without rebuilding. +RCCL_PARAM(IgnoreArchTable, "IGNORE_ARCH_TABLE", 0); +// Returns true when the user has restricted the algorithm set via NCCL_ALGO. +// When true, CE / DDA / Symmetric dispatch is skipped so getAlgoInfo() reaches +// Ring/Tree exactly as the user requested. Cached to avoid repeated getenv(). +bool rcclNcclAlgoEnvIsSet() { + static int cached = -1; + if (cached == -1) cached = (ncclGetEnv("NCCL_ALGO") != nullptr) ? 1 : 0; + return cached == 1; +} + #ifdef ENABLE_WARP_SPEED RCCL_PARAM(WarpSpeedCuCount, "WARP_SPEED_CU_COUNT", 0); RCCL_PARAM(WarpSpeedAutoMode, "WARP_SPEED_AUTO", 1); @@ -169,7 +195,8 @@ void rcclUpdateCollectiveProtocol(struct ncclComm* comm, size_t const& nBytes, s if (comm->nNodes == 1) { info->protocol = rcclGetProtoForGfx110x(info->func, sizePerRank); } - } else if (!userProtocolInput && comm->nNodes >= 2 && + } else if (!userProtocolInput && + (comm->nNodes >= 2 || IsArchMatch(comm->topo->nodes[GPU].nodes[0].gpu.gcn, "gfx1250")) && (info->func == ncclFuncReduceScatter || info->func == ncclFuncAllGather || info->func == ncclFuncAllReduce || info->func == ncclFuncBroadcast || info->func == ncclFuncReduce)) { auto tunableIndex = rcclGetTunableIndex(info->func); @@ -238,6 +265,10 @@ extern int64_t rcclParamForceCe(); RCCL_PARAM(ChannelTuningEnable, "CHANNEL_TUNING_ENABLE", 1); ncclResult_t rcclOverrideChannels(struct ncclComm* comm, ncclFunc_t coll, size_t nBytes, int& nc) { + if (IsArchMatch(comm->topo->nodes[GPU].nodes[0].gpu.gcn, "gfx1250")) { + INFO(NCCL_TUNING, "RCCL Channel Tuning not applied for gfx1250 (pending sweep data, AICOMRCCL-1756)"); + return ncclSuccess; + } const bool isGfx_110x_120x = IsArchMatch(comm->topo->nodes[GPU].nodes[0].gpu.gcn, "gfx110") || IsArchMatch(comm->topo->nodes[GPU].nodes[0].gpu.gcn, "gfx120"); // Make an exception for gfx110x and gfx120x @@ -497,6 +528,15 @@ ncclResult_t rcclGetAlgoInfo(struct ncclComm* comm, ncclFunc_t coll, uint64_t co *maxChannels = comm->p2pnChannels; return ncclSuccess; } + // AlltoAll has no tuned ring/tree kernel: taskAppend() splits it into per-peer + // Send/Recv over the p2p channels, so report that shape. The backend-aware + // answer (DDA / CE / pivot for these operands) comes from rcclGetCollImplInfo(). + if (coll == ncclFuncAlltoAll) { + *algo = rcclAddonAlgos_t::RCCL_DIRECT_ALLTOALL; + *protocol = NCCL_PROTO_SIMPLE; + *maxChannels = comm->p2pnChannels; + return ncclSuccess; + } struct ncclTaskColl task = {}; task.func = coll; task.count = count; @@ -519,11 +559,11 @@ ncclResult_t rcclGetCollImplInfo(struct ncclComm* comm, ncclFunc_t coll, uint64_ RCCL_STATIC_EXPOSE_CHECK(); if (algo == nullptr || protocol == nullptr || maxChannels == nullptr) return ncclInvalidArgument; - // AllReduce and AllGather are wired to the unified decision. They return the - // actual backend (CE / DDA / symmetric / kernel) for these operands so - // rccl-tests can label numbers with the implementation that ran. graphCapturing - // lets the caller declare graph mode, which the (out-of-capture) query cannot - // detect on its own -- see the header comment. + // AllReduce, AllGather, ReduceScatter, and AlltoAll are wired to the unified + // decision. They return the actual backend (CE / DDA / symmetric / kernel) for + // these operands so rccl-tests can label numbers with the implementation that + // ran. graphCapturing lets the caller declare graph mode, which the + // (out-of-capture) query cannot detect on its own -- see the header comment. if (coll == ncclFuncAllReduce) { struct rcclCollDecision decision; NCCLCHECK(rcclSelectAllReduce(comm, sendbuff, recvbuff, (size_t)count, dataType, op, /*stream=*/nullptr, @@ -536,8 +576,8 @@ ncclResult_t rcclGetCollImplInfo(struct ncclComm* comm, ncclFunc_t coll, uint64_ if (coll == ncclFuncAllGather) { struct rcclCollDecision decision; - NCCLCHECK(rcclSelectAllGather(comm, sendbuff, recvbuff, (size_t)count, dataType, /*query=*/true, - /*graphCapturingHint=*/graphCapturing != 0, &decision)); + NCCLCHECK(rcclSelectAllGather(comm, sendbuff, recvbuff, (size_t)count, dataType, /*stream=*/nullptr, + /*query=*/true, /*graphCapturingHint=*/graphCapturing != 0, &decision)); *algo = decision.algo; *protocol = decision.protocol; *maxChannels = decision.nMaxChannels; @@ -554,6 +594,16 @@ ncclResult_t rcclGetCollImplInfo(struct ncclComm* comm, ncclFunc_t coll, uint64_ return ncclSuccess; } + if (coll == ncclFuncAlltoAll) { + struct rcclCollDecision decision; + NCCLCHECK(rcclSelectAlltoAll(comm, sendbuff, recvbuff, (size_t)count, dataType, /*stream=*/nullptr, + /*query=*/true, /*graphCapturingHint=*/graphCapturing != 0, &decision)); + *algo = decision.algo; + *protocol = decision.protocol; + *maxChannels = decision.nMaxChannels; + return ncclSuccess; + } + // Other collectives: fall back to the size/algo query until they are migrated // onto rcclSelectXxx(). return rcclGetAlgoInfo(comm, coll, count, dataType, /*collNetSupport=*/0, /*nvlsSupport=*/0, /*numPipeOps=*/1, algo, @@ -650,6 +700,9 @@ ncclResult_t rcclGetAlgoName(int algo, const char** algoName) { case rcclAddonAlgos_t::RCCL_DIRECT_REDUCESCATTER: *algoName = "Direct"; break; + case rcclAddonAlgos_t::RCCL_DIRECT_ALLTOALL: + *algoName = "Direct"; + break; case rcclAddonAlgos_t::RCCL_HIERARCHICAL_REDUCESCATTER: *algoName = "Hier"; break; @@ -667,6 +720,9 @@ ncclResult_t rcclGetAlgoName(int algo, const char** algoName) { case rcclAddonAlgos_t::RCCL_CE_REGISTERED: *algoName = "CE"; break; + case rcclAddonAlgos_t::RCCL_CE_SCRATCH: + *algoName = "CE-Scratch"; + break; // Fabric variants all report "DDA"; the protocol column distinguishes // LL / LL128 / Simple, so the name needn't repeat it. case rcclAddonAlgos_t::RCCL_DDA_FABRIC_LL: @@ -680,6 +736,15 @@ ncclResult_t rcclGetAlgoName(int algo, const char** algoName) { case rcclAddonAlgos_t::RCCL_GIN_SDMA: *algoName = "GIN-SDMA"; break; + case rcclAddonAlgos_t::RCCL_A2A_PIVOT: + *algoName = "A2A-Pivot"; + break; + case rcclAddonAlgos_t::RCCL_A2A_GDA: + *algoName = "A2A-GDA"; + break; + case rcclAddonAlgos_t::RCCL_A2A_GIN_SDMA: + *algoName = "A2A-GIN"; + break; default: WARN("Invalid algorithm value: %d", algo); return ncclInvalidArgument; @@ -699,31 +764,281 @@ ncclResult_t rcclGetProtocolName(int protocol, const char** protocolName) { return ncclSuccess; } -bool rcclDdaEnabled(const ncclComm* comm, size_t totalBytes, size_t gfx942Default, size_t gfx950Default, - size_t gfx1250Default) { +namespace { +// This comm's arch table, or NULL when the arch has no DDA tuning at all. +// comm->archThresholds is populated at init; the lookup covers comms assembled +// by hand (unit tests) so they resolve exactly like production ones. +inline const rcclArchThresholds* extAlgoArchTable(const ncclComm* comm) { + if (comm == nullptr) return nullptr; + if (rcclParamIgnoreArchTable() && !IsArchMatch(comm->archName, "gfx942") + && !IsArchMatch(comm->archName, "gfx950")) return nullptr; + return comm->archThresholds != nullptr ? comm->archThresholds : rcclGetArchThresholds(comm->archName); +} + +// kDdaThresholdUnset means the user left the env var alone; anything else is +// their choice and wins, including 0 to disable that tier. +inline bool ddaThresholdFromEnv(int64_t param, size_t* threshold) { + if (param == kDdaThresholdUnset) return false; + *threshold = param > 0 ? (size_t)param : 0; + return true; +} + +// Table entry for `func`. Collectives past AlltoAll have no slot and stay 0. +inline size_t funcThresholdFromTable(const size_t* caps, ncclFunc_t func) { + return (unsigned)func < RCCL_DDA_FUNC_COUNT ? caps[func] : 0; +} + +// R2 symmetric-kernel size cap. Graph capture uses symMaxR2Graph; eager uses +// symMaxR2. kThreshUnlimited (SIZE_MAX) means no suppression; non-zero literal = byte cap. +// Table entries use kThreshUnlimited explicitly -- 0 is not used in these arrays. +inline size_t rcclSymMaxR2CapTab(const rcclArchThresholds* table, ncclFunc_t func, bool graphMode) { + if (table == nullptr) return SIZE_MAX; + const size_t* caps = graphMode ? table->symMaxR2Graph : table->symMaxR2; + return funcThresholdFromTable(caps, func); +} + +// R2 symmetric-kernel lower-bound per collective. Below this size DDA is +// faster than symk; symk is suppressed so DDA can win. 0 = no suppression. +inline size_t rcclSymMinR2CapTab(const rcclArchThresholds* table, ncclFunc_t func) { + if (table == nullptr) return 0; + return funcThresholdFromTable(table->symMinR2, func); +} +} // namespace + +inline size_t rcclCeRegMaxTab(const rcclArchThresholds* table, ncclFunc_t func) { + const int64_t param = (func == ncclFuncAllReduce) ? rcclParamCeArRegMaxMsgBytes() : -1; + if (param >= 0) return (size_t)param; + if (table == nullptr) return 0; + return (size_t)func < RCCL_DDA_FUNC_COUNT ? table->ceRegMax[(size_t)func] : 0; +} + +inline size_t rcclCeNonRegMaxTab(const rcclArchThresholds* table, ncclFunc_t func) { + if (table == nullptr) + return func == ncclFuncAllReduce ? NCCL_CE_AR_TMPBUF_DEFAULT_BYTES : 0; + return (size_t)func < RCCL_DDA_FUNC_COUNT ? table->ceNonRegMax[(size_t)func] : 0; +} + +inline size_t rcclCeAr2ShotMaxTab(const rcclArchThresholds* table) { + const int64_t param = rcclParamCeArMaxMsgBytes(); + if (param >= 0) return (size_t)param; + if (table == nullptr) return NCCL_CE_AR_TMPBUF_DEFAULT_BYTES; + return table->ceNonRegMax[ncclFuncAllReduce]; +} +size_t rcclCeAr2ShotMax(const ncclComm* comm) { + return rcclCeAr2ShotMaxTab(extAlgoArchTable(comm)); +} + +// CE AllReduce is only tuned on gfx1250, so it is default-on there and stays off +// everywhere else. Without this gate the arch tables for gfx942/gfx950 (which set +// ceNonRegMax[AR] = 256 MiB) would let unregistered 2-shot, and -- via force -- +// registered CE, service AllReduce on arches that were never measured for it. +static inline bool rcclCeAllReduceArchDefault(const ncclComm* comm) { + if (comm == nullptr || !IsArchMatch(comm->archName, "gfx1250")) return false; + return extAlgoArchTable(comm) != nullptr; +} + +inline bool rcclCeAllReduceEnabledDef(bool archDefault) { + const int64_t param = rcclParamCeAllReduce(); + if (param >= 0) return param != 0; + return archDefault; +} +bool rcclCeAllReduceEnabled(const ncclComm* comm) { + return rcclCeAllReduceEnabledDef(rcclCeAllReduceArchDefault(comm)); +} + +inline bool rcclForceCeAllReduceEnabledDef(bool archDefault) { + const int64_t param = rcclParamForceCeAllReduce(); + if (param >= 0) return param != 0; + return archDefault; +} +bool rcclForceCeAllReduceEnabled(const ncclComm* comm) { + return rcclForceCeAllReduceEnabledDef(rcclCeAllReduceArchDefault(comm)); +} + +inline size_t rcclCeNonRegMinTab(const rcclArchThresholds* table, ncclFunc_t func) { + if (table == nullptr) return 0; + return (size_t)func < RCCL_DDA_FUNC_COUNT ? table->ceNonRegMin[(size_t)func] : 0; +} + +// Returns true when the message is within the CE-registered (2-shot) AllGather +// window. Does not check symk eligibility -- callers gate on !symEligible explicitly. +// CE-registered gate: recv buffer registered and message within ceRegMax[AG]. +// symMaxR2 arbitration is handled upstream via symEligible in the selectors; +// graphMode is accepted but unused here so enqueue.cc can pass ceCapturing +// without a separate overload. +inline bool rcclAllGatherCeRegisteredWindowTab(const rcclArchThresholds* table, size_t totalBytes, + ncclSymRegType_t winRegType, bool /*graphMode*/) { + const bool recvReg = (winRegType == ncclSymSendRegRecvReg || winRegType == ncclSymSendNonregRecvReg); + if (!recvReg) return false; + const size_t regMax = rcclCeRegMaxTab(table, ncclFuncAllGather); + return regMax == kThreshUnlimited || totalBytes <= regMax; +} +bool rcclAllGatherCeRegisteredWindow(const ncclComm* comm, size_t totalBytes, + ncclSymRegType_t winRegType, bool graphMode) { + return rcclAllGatherCeRegisteredWindowTab(extAlgoArchTable(comm), totalBytes, winRegType, graphMode); +} + + +inline size_t rcclDdaLLThresholdTab(const rcclArchThresholds* table, ncclFunc_t func) { + size_t threshold; + if (ddaThresholdFromEnv(rcclParamDdaLLThreshold(), &threshold)) return threshold; + if (table == nullptr) return kDdaLLBaseDefault; + return funcThresholdFromTable(table->ddaLLMax, func); +} +size_t rcclDdaLLThreshold(const ncclComm* comm, ncclFunc_t func) { + return rcclDdaLLThresholdTab(extAlgoArchTable(comm), func); +} + +// Returns the LL128 size cap for this collective, or 0 if the tier is disabled. +// RCCL_DDA_LL128 three-way logic: +// kDdaThresholdUnset (-1, default): auto — enabled iff arch table has non-zero +// ddaLL128Max for func; RCCL_IGNORE_ARCH_TABLE=1 (table=nullptr) → 0 (off). +// 0: explicitly disabled for all collectives. +// >=1: forced on; uses table value when available, else kDdaLL128BaseDefault. +// Callers check "> 0" to decide enablement, so no separate Active() helper needed. +inline size_t rcclDdaLL128ThresholdTab(const rcclArchThresholds* table, ncclFunc_t func) { + size_t threshold; + if (ddaThresholdFromEnv(rcclParamDdaLL128Threshold(), &threshold)) return threshold; + const int64_t param = rcclParamDdaLL128(); + if (param == 0) return 0; + if (param == kDdaThresholdUnset) { + if (table == nullptr) return 0; + return funcThresholdFromTable(table->ddaLL128Max, func); + } + // param > 0: user forced on + if (table == nullptr) return kDdaLL128BaseDefault; + return funcThresholdFromTable(table->ddaLL128Max, func); +} +size_t rcclDdaLL128Threshold(const ncclComm* comm, ncclFunc_t func) { + return rcclDdaLL128ThresholdTab(extAlgoArchTable(comm), func); +} + +inline size_t rcclDdaVmmThresholdTab(const rcclArchThresholds* table, ncclFunc_t func) { + size_t threshold; + if (ddaThresholdFromEnv(rcclParamDdaThreshold(), &threshold)) return threshold; + if (table == nullptr) return kDdaVmmBaseDefault; + return funcThresholdFromTable(table->ddaVmmMax, func); +} +size_t rcclDdaVmmThreshold(const ncclComm* comm, ncclFunc_t func) { + return rcclDdaVmmThresholdTab(extAlgoArchTable(comm), func); +} + +// Widest window any enabled DDA tier can serve for this collective. The entry +// gate only decides whether the per-tier checks run, so a tier tuned off (0) +// must not hide the tiers that are still on: gfx1250 ReduceScatter has +// ddaVmmMax = 0 with non-zero LL/LL128 caps, and ddaLL128Max[AllReduce] +// (32 MiB) sits above ddaVmmMax[AllReduce] (16 MiB). RCCL_DDA_LL / +// RCCL_DDA_LL128 = 0 drop their tier so disabling one cannot widen the gate. +// Each tier still applies its own cap at the call site, including the VMM/IPC +// branch, which checks rcclDdaVmmThreshold() there. +// ddaVmmMaxGraph is also folded in: it is 0 for all collectives except AR +// (256 MiB on gfx1250), so it only widens the gate for graph-mode AR calls +// and is a no-op for all other collectives. +inline size_t rcclDdaEntryThresholdTab(const rcclArchThresholds* table, ncclFunc_t func) { + size_t cap = rcclDdaVmmThresholdTab(table, func); + if (rcclParamDdaLL()) cap = std::max(cap, rcclDdaLLThresholdTab(table, func)); + cap = std::max(cap, rcclDdaLL128ThresholdTab(table, func)); + if (table != nullptr) cap = std::max(cap, funcThresholdFromTable(table->ddaVmmMaxGraph, func)); + return cap; +} +size_t rcclDdaEntryThreshold(const ncclComm* comm, ncclFunc_t func) { + return rcclDdaEntryThresholdTab(extAlgoArchTable(comm), func); +} + +size_t rcclDdaScratchPayloadCap(const ncclComm* comm) { + size_t cap = 0; + auto bump = [&](size_t v) { + if (v > cap) cap = v; + }; + size_t env = 0; + if (ddaThresholdFromEnv(rcclParamDdaThreshold(), &env)) bump(env); + if (ddaThresholdFromEnv(rcclParamDdaLLThreshold(), &env)) bump(env); + if (ddaThresholdFromEnv(rcclParamDdaLL128Threshold(), &env)) bump(env); + + const rcclArchThresholds* table = extAlgoArchTable(comm); + if (table == nullptr) { + // Same pre-table defaults rcclDda{LL,LL128,Vmm}Threshold return when the + // table is ignored, so fabric scratch covers any DDA path the selector + // can still pick. Env, when set (including 0 to disable a tier), already + // won above and is not replaced. + size_t unused; + if (!ddaThresholdFromEnv(rcclParamDdaLLThreshold(), &unused)) bump(kDdaLLBaseDefault); + if (!ddaThresholdFromEnv(rcclParamDdaLL128Threshold(), &unused)) bump(kDdaLL128BaseDefault); + if (!ddaThresholdFromEnv(rcclParamDdaThreshold(), &unused)) bump(kDdaVmmBaseDefault); + return cap; + } + + // All table entries are total message bytes; the selector compares totalBytes + // directly (rccl_wrap.cc rcclSelectReduceScatter), so no per-rank scaling here. + for (int i = 0; i < RCCL_DDA_FUNC_COUNT; ++i) { + bump(table->ddaLLMax[i]); + bump(table->ddaLL128Max[i]); + bump(table->ddaVmmMax[i]); + bump(table->ddaVmmMaxR2[i]); + // Graph VMM is included even for eager-only processes: one buffer must + // fit a later capture (gfx1250 AR graph cap is 256 MiB). + bump(table->ddaVmmMaxGraph[i]); + // AG CE-Scratch (and any other non-AR CE-scratch) copies the receive into + // ddaScratch, so that window must fit. AR 2-shot uses ceARTmpBuf. + if (i != (int)ncclFuncAllReduce) bump(table->ceNonRegMax[i]); + } + return cap; +} + +// Context-aware VMM threshold resolver. Callers pass the full winRegType so +// the function can apply the correct override without the caller having to +// interpret registration semantics. Policy: only recv registration shifts the +// DDA VMM cap (send-only registration does not change DDA/CE dispatch). +// When the recv buffer is registered (ncclSymSendNonregRecvReg or +// ncclSymSendRegRecvReg) and the arch table has a non-zero R2 override for +// this collective, that cap wins over the default ddaVmmMax. +// When inside a graph capture (graphMode=true) and the table has a non-zero +// graph-mode override, that cap wins. Graph-mode is checked first. +// Env var (RCCL_DDA_THRESHOLD) always wins over all context variants. +inline size_t rcclDdaVmmThresholdCtxTab(const rcclArchThresholds* table, ncclFunc_t func, + ncclSymRegType_t winRegType, bool graphMode) { + size_t threshold; + if (ddaThresholdFromEnv(rcclParamDdaThreshold(), &threshold)) return threshold; + if (table == nullptr) return kDdaVmmBaseDefault; + if (graphMode) { + size_t graphCap = funcThresholdFromTable(table->ddaVmmMaxGraph, func); + if (graphCap != 0) return graphCap; + } + const bool recvReg = (winRegType == ncclSymSendNonregRecvReg || + winRegType == ncclSymSendRegRecvReg); + if (recvReg) { + size_t r2Cap = funcThresholdFromTable(table->ddaVmmMaxR2, func); + if (r2Cap != 0) return r2Cap; + } + return funcThresholdFromTable(table->ddaVmmMax, func); +} + +bool rcclDdaEnabled(const ncclComm* comm, size_t totalBytes, size_t threshold, + bool query, const char* prefix) { // The environment parameter can be NCCL_CONFIG_UNDEF_INT when launch order // is configured per communicator. Use the resolved communicator value: // testing the raw sentinel as a boolean disables DDA by default, while // testing only the environment would ignore an explicit config value. if (!rcclParamDdaEnable() || comm->config.launchOrderImplicit == 1 || ncclGroupDepth != 0) { + if (!query && prefix) INFO(NCCL_TUNING, "%s DDA disqualified: RCCL_DDA_ENABLE=%d launchOrderImplicit=%d ncclGroupDepth=%d", prefix, (int)rcclParamDdaEnable(), comm->config.launchOrderImplicit, ncclGroupDepth); return false; } - size_t threshold; if (IsArchMatch(comm->archName, "gfx1250")) { - threshold = gfx1250Default ? gfx1250Default : static_cast(rcclParamDdaThreshold()); + // gfx1250 has no nRanks floor. } else if (IsArchMatch(comm->archName, "gfx942") || IsArchMatch(comm->archName, "gfx950")) { if (comm->nRanks < 8) return false; - if (IsArchMatch(comm->archName, "gfx942")) { - threshold = gfx942Default; - } else { - threshold = gfx950Default ? gfx950Default : static_cast(rcclParamDdaThreshold()); - } } else { return false; } - return threshold > 0 && totalBytes <= threshold; + const bool ok = threshold > 0 && totalBytes <= threshold; + if (!ok && !query && prefix) + INFO(NCCL_TUNING, "%s DDA disqualified: totalBytes=%zu > entryThreshold=%zu", + prefix, totalBytes, threshold); + return ok; } + + bool rcclUseAlltoAllGda(struct ncclComm* comm) { #ifdef ENABLE_ROCSHMEM if (comm->enableRocshmem && comm->nNodes > 1 && (comm->nRanks / comm->nNodes == 8) && @@ -822,15 +1137,16 @@ bool rcclUseAllGatherDirect(struct ncclComm* comm, size_t& msgSize) { return (comm->enableCustColl && (msgSize <= threshold) && (threshold != -1) && !rankMultiple); } -bool rcclUseCeAllReduce(struct ncclComm* comm, size_t count, ncclDataType_t datatype, ncclRedOp_t op, const void* acc) { - static int enabled = rcclParamCeAllReduce(); - static int force = rcclParamForceCeAllReduce(); +bool rcclUseCeAr2Shot(struct ncclComm* comm, size_t count, ncclDataType_t datatype, ncclRedOp_t op, + const void* acc) { + const bool enabled = rcclCeAllReduceEnabled(comm); + const bool force = rcclForceCeAllReduceEnabled(comm); if (!enabled) { // Log once per process, not on every eligibility check (called per AllReduce). static bool warnedDisabled = false; if (!warnedDisabled) { warnedDisabled = true; - INFO(NCCL_INIT, "CE AllReduce not enabled. Set RCCL_CE_ALLREDUCE=1 to enable."); + INFO(NCCL_INIT, "CE AllReduce not enabled on %s. Set RCCL_CE_ALLREDUCE=1 to enable.", comm->archName); } return false; } @@ -855,17 +1171,14 @@ bool rcclUseCeAllReduce(struct ncclComm* comm, size_t count, ncclDataType_t data return false; } - // Total message must fit within the pre-allocated staging buffer. + // 2-shot selector cap (table/env). 0 means 2-shot is tuned off; registered CE + // still uses the default ceARTmpBuf. Does not override the allocated buffer: + // ncclCeInit grows ceArMaxBytes when this cap is larger than the default. + const size_t twoShotMax = rcclCeAr2ShotMax(comm); + if (twoShotMax == 0) return false; size_t msgBytes = count * ncclTypeSize(datatype); - if (force) { - if (msgBytes > NCCL_CE_AR_MAX_MSG_BYTES) { - WARN("Skipping CE AllReduce despite RCCL_FORCE_CE_ALLREDUCE=1: msgBytes (%zu) > NCCL_CE_AR_MAX_MSG_BYTES (%zu)", - msgBytes, NCCL_CE_AR_MAX_MSG_BYTES); - return false; - } - } - if (msgBytes > NCCL_CE_AR_MAX_MSG_BYTES) { - WARN("Skipping CE AllReduce: msgBytes (%zu) > NCCL_CE_AR_MAX_MSG_BYTES (%zu)", msgBytes, NCCL_CE_AR_MAX_MSG_BYTES); + if (msgBytes > twoShotMax) { + WARN("Skipping CE AllReduce: msgBytes (%zu) > twoShotMax (%zu)", msgBytes, twoShotMax); return false; } @@ -911,7 +1224,40 @@ void rcclCeAllReduceGraphLatchTick(struct ncclComm* comm, bool ceCapturing) { } } -bool rcclCeAllReduceAllowed(struct ncclComm* comm) { +// Sets decision to Ring/Simple and, when query=true, refines proto/channels via +// getAlgoInfo() and rcclKernelPackedChannels(). Used by rcclSelectAllReduce, +// rcclSelectAllGather and rcclSelectReduceScatter for both the NCCL_ALGO env-var +// early-return and the Ring fall-through tail. +static ncclResult_t rcclRingFallback(struct ncclComm* comm, const void* sendbuff, + void* recvbuff, ncclFunc_t func, size_t count, + ncclDataType_t datatype, bool query, + struct rcclCollDecision* decision) { + decision->algo = NCCL_ALGO_RING; + decision->protocol = NCCL_PROTO_SIMPLE; + if (query) { + struct ncclTaskColl task; + memset(&task, 0, sizeof(task)); + task.func = func; + task.sendbuff = sendbuff; + task.recvbuff = recvbuff; + task.count = count; + task.datatype = datatype; + NCCLCHECK(getAlgoInfo(comm, &task, 0, 0, 1)); + decision->protocol = task.protocol; + int packed = rcclKernelPackedChannels(comm, func, count, datatype, + task.protocol, task.nMaxChannels); +#ifdef ENABLE_WARP_SPEED + decision->nMaxChannels = task.useWarpSpeed ? task.nMaxChannels / task.nWarps : packed; + decision->algo = task.useWarpSpeed ? rcclAddonAlgos_t::RCCL_WARP_SPEED : task.algorithm; +#else + decision->nMaxChannels = packed; + decision->algo = task.algorithm; +#endif + } + return ncclSuccess; +} + +bool rcclCeArGraphSafe(struct ncclComm* comm) { return !comm->ceColl.graphModeSeen; } @@ -928,26 +1274,8 @@ ncclResult_t rcclSelectAllReduce(struct ncclComm* comm, const void* sendbuff, vo decision->protocol = NCCL_PROTO_SIMPLE; decision->nMaxChannels = 0; - const size_t msgBytes = count * ncclTypeSize(datatype); - -#if defined(ENABLE_ROCSHMEM_GIN) - // GIN-SDMA scaleup AllReduce. Same gates as the previous early return in - // ncclAllReduce_impl (group depth 0 + eligibility). Graph-capture-safe: init - // runs off a private stream in relaxed mode; kernels re-read signal baselines. - // Must beat CE / DDA / symmetric so rcclGetCollImplInfo names the backend that ran. - if (ncclGroupDepth == 0 && ncclAllReduceGinSdmaEligible(comm, sendbuff, recvbuff, count, datatype, op)) { - decision->algo = RCCL_GIN_SDMA; - decision->nMaxChannels = kGinAllReduceLsaCtas; - return ncclSuccess; - } -#endif - - // (1) Symmetric-window kernel eligibility takes priority over CE / DDA, exactly - // as the pre-refactor collectives.cc path did. - const bool symEligible = (op == ncclSum) && isSymmetricKernelRequested(comm, ncclFuncAllReduce, (int)ncclDevSum, - datatype, count, sendbuff, recvbuff); - - // (2) CE AllReduce graph state. CE is graph-unsafe, so capture disables it. + // CE AllReduce graph state. CE is graph-unsafe, so capture disables it. + // Probed before the symMaxR2 gate so graph mode can pick symMaxR2Graph. // - Live dispatch (query=false): probe the real stream and tick the graph // latch, exactly as the inline code did. // - Reporting (query=true): the query runs outside capture, so the stream @@ -964,154 +1292,210 @@ ncclResult_t rcclSelectAllReduce(struct ncclComm* comm, const void* sendbuff, vo ceCapturing = ncclCudaGraphValid(ceGraph); rcclCeAllReduceGraphLatchTick(comm, ceCapturing); } - bool ceArGraphAllowed = rcclCeAllReduceAllowed(comm); + bool ceArGraphAllowed = rcclCeArGraphSafe(comm); if (query && ceCapturing) ceArGraphAllowed = false; decision->ceCapturing = ceCapturing; decision->ceArGraphAllowed = ceArGraphAllowed; + if (!query && ceCapturing) + INFO(NCCL_TUNING, "AR CE-2shot/CE-registered disqualified: graph capture active (graphModeSeen=%d ceArGraphAllowed=%d)", + (int)comm->ceColl.graphModeSeen, (int)rcclCeArGraphSafe(comm)); + const size_t msgBytes = count * ncclTypeSize(datatype); + if (!rcclNcclAlgoEnvIsSet()) { + #if defined(ENABLE_ROCSHMEM_GIN) + // GIN-SDMA scaleup AllReduce. Same gates as the previous early return in + // ncclAllReduce_impl (group depth 0 + eligibility). Graph-capture-safe: init + // runs off a private stream in relaxed mode; kernels re-read signal baselines. + // Must beat CE / DDA / symmetric so rcclGetCollImplInfo names the backend that ran. + if (ncclGroupDepth == 0 && ncclAllReduceGinSdmaEligible(comm, sendbuff, recvbuff, count, datatype, op)) { + decision->algo = RCCL_GIN_SDMA; + decision->nMaxChannels = kGinAllReduceLsaCtas; + return ncclSuccess; + } + #endif + + // Symmetric-window lookup hoisted ahead of the symk signals: winRegType is needed + // for the symMaxR2 gate below, and the lookup is unconditional regardless, so + // pulling it up eliminates the redundant ncclDevrFindWindow inside + // isSymmetricKernelRequested when symk turns out to be requested. + struct ncclDevrWindow* sendWin = nullptr; + struct ncclDevrWindow* recvWin = nullptr; + ncclDevrFindWindow(comm, sendbuff, &sendWin); + ncclDevrFindWindow(comm, recvbuff, &recvWin); + const bool hasSysmemSegment = + ncclDevrWindowHasSysmemSegment(sendWin) || ncclDevrWindowHasSysmemSegment(recvWin); + ncclSymRegType_t winRegType; + NCCLCHECK(ncclGetSymRegType(sendWin, recvWin, &winRegType)); + + const rcclArchThresholds* const archTable = extAlgoArchTable(comm); + + // (1) Symmetric-window kernel eligibility takes priority over CE / DDA. + // symkRequested is the raw "symk would run for these operands" signal and keeps + // gating the CE 2-shot and DDA branches below, so registered buffers still reach + // CE-registered at (5) rather than being claimed by a staging-buffer or fabric path. + // symMaxR2 / symMaxR2Graph from the arch table only withdraws symk as the final + // choice once the message exceeds the CE/symk crossover size, letting + // CE-registered win instead. kThreshUnlimited means no suppression. + // recvRegistered is not needed here: isSymmetricKernelRequestedWin requires + // both windows to carry NCCL_WIN_COLL_SYMMETRIC, so symkRequested=true already + // implies recv is registered; when symkRequested=false suppression is moot. + const size_t symMaxR2 = rcclSymMaxR2CapTab(archTable, ncclFuncAllReduce, ceCapturing); + const size_t symMinR2 = rcclSymMinR2CapTab(archTable, ncclFuncAllReduce); + const bool symSuppressedByMax = msgBytes > symMaxR2; + const bool symSuppressedByMin = symMinR2 > 0 && msgBytes < symMinR2; + const bool symkRequested = + (op == ncclSum) && + isSymmetricKernelRequestedWin(comm, ncclFuncAllReduce, (int)ncclDevSum, datatype, count, sendWin, recvWin); + // symSuppressedByMin: DDA wins below symMinR2[AR]; do not block it with symkRequested. + const bool symEligible = symkRequested && !symSuppressedByMin && !symSuppressedByMax; + INFO(NCCL_TUNING, + "rcclSelectAllReduce: graph=%d symkRequested=%d symSuppressedByMax=%d symEligible=%d symMaxR2=%zu", + (int)ceCapturing, (int)symkRequested, (int)symSuppressedByMax, (int)symEligible, symMaxR2); + if (!query && symkRequested && symSuppressedByMin) + INFO(NCCL_TUNING, "AR symk disqualified: msgBytes=%zu < symMinR2=%zu", msgBytes, symMinR2); + if (!query && symkRequested && symSuppressedByMax) + INFO(NCCL_TUNING, "AR symk disqualified: msgBytes=%zu > symMaxR2=%zu", msgBytes, symMaxR2); - // Symmetric-window lookup, hoisted so CE 2-shot / DDA / CE-registered all share - // it. sendWin/recvWin drive winRegType and the sysmem-segment guard below. - struct ncclDevrWindow* sendWin = nullptr; - struct ncclDevrWindow* recvWin = nullptr; - ncclDevrFindWindow(comm, sendbuff, &sendWin); - ncclDevrFindWindow(comm, recvbuff, &recvWin); - const bool hasSysmemSegment = ncclDevrWindowHasSysmemSegment(sendWin) || ncclDevrWindowHasSysmemSegment(recvWin); - ncclSymRegType_t winRegType; - NCCLCHECK(ncclGetSymRegType(sendWin, recvWin, &winRegType)); // develop's single "will CE AllReduce service this call" gate (collectives.cc // ncclAllReduce_impl). force = RCCL_FORCE_CE_ALLREDUCE; symReg probes whether the // buffers are CE-registrable symmetric windows (uses ncclDevSum, matching develop). - const bool force = rcclParamForceCeAllReduce() != 0; + const bool ceArArchDefault = rcclCeAllReduceArchDefault(comm); + const bool force = rcclForceCeAllReduceEnabledDef(ceArArchDefault); const bool symReg = ncclCeAvailable(comm, ncclFuncAllReduce, (int)ncclDevSum, datatype, winRegType, sendWin, recvWin); // This call site never carries a bias buffer (ncclAllReduceWithBias_impl bypasses it entirely // and goes straight to taskAppend), so /*acc=*/nullptr here is always correct. const bool ceAllReduceAllowed = ncclGroupDepth == 0 && ceArGraphAllowed && - rcclUseCeAllReduce(comm, count, datatype, op, /*acc=*/nullptr) && (force || symReg); - - // (3) Eager CE 2-shot (staging buffer). Requires !symEligible and an - // initialized ceARTmpBuf (first call, before init, falls through to enqueue). - if (!symEligible && ceAllReduceAllowed && comm->ceColl.ceARTmpBuf != NULL) { - decision->algo = RCCL_CE_2SHOT; - decision->nMaxChannels = ncclCeLocalReduceBlocks(datatype, count / comm->nRanks); - return ncclSuccess; - } + rcclUseCeAr2Shot(comm, count, datatype, op, /*acc=*/nullptr) && (force || symReg); + + // (3) Eager CE 2-shot (staging buffer). Requires !symkRequested and an + // initialized ceARTmpBuf (first call, before init, falls through to enqueue). + // Gated on the raw symk signal, not symEligible: symmetric-window operands copy + // through the user windows via CE-registered, so they must not be diverted into + // the staging buffer just because symMaxR2 withdrew symk. + if (!symkRequested && ceAllReduceAllowed && comm->ceColl.ceARTmpBuf != NULL) { + decision->algo = RCCL_CE_2SHOT; + decision->nMaxChannels = ncclCeLocalReduceBlocks(datatype, count / comm->nRanks); + return ncclSuccess; + } - // (4) DDA fast paths. develop's shared gate: !symEligible, and either gfx1250 - // (fabric, full range) or CE is not going to service this call (!ceAllReduceAllowed), - // subject to rcclDdaEnabled thresholds -- all folded into the helper. - // - // GIN AllReduce is selected first in this function and requires symmetric - // windows. By default it only claims messages >= 256 MiB, so DDA must still be - // allowed for smaller symmetric AllReduces (otherwise they would hit the - // symmetric kernel instead of DDA). FORCE_ENABLE=1 keeps the original - // !symEligible gate because GIN already returned above for those sizes. - bool ddaSymEligible = symEligible; -#if defined(ENABLE_ROCSHMEM_GIN) - if (ncclAllReduceGinSdmaYieldToDda(comm, sendbuff, recvbuff, count, datatype, op)) { - ddaSymEligible = false; - } -#endif - const bool ddaFabricArch1250 = IsArchMatch(comm->archName, "gfx1250"); - if (rcclAllReduceShouldTakeDdaPath(comm, count, datatype, ddaSymEligible, ceAllReduceAllowed)) { - if (ddaFabricArch1250) { - // Small-message fast lane: LL protocol (no GPU barrier). - if (ncclAllReduceDdaFabricLLEligible(comm, sendbuff, recvbuff, count, datatype, op)) { - decision->algo = RCCL_DDA_FABRIC_LL; - decision->protocol = NCCL_PROTO_LL; - decision->nMaxChannels = ncclAllReduceDdaFabricLLBlocks(comm, count, datatype); - return ncclSuccess; - } - // Mid-size fast lane: LL128 protocol (128B lines, no GPU barrier). - if (ncclAllReduceDdaFabricLL128Eligible(comm, sendbuff, recvbuff, count, datatype, op)) { - decision->algo = RCCL_DDA_FABRIC_LL128; - decision->protocol = NCCL_PROTO_LL128; - decision->nMaxChannels = ncclAllReduceDdaFabricLL128Blocks(comm, count, datatype); - return ncclSuccess; - } - if (ncclAllReduceDdaFabricEligible(comm, sendbuff, recvbuff, count, datatype, op)) { - decision->algo = RCCL_DDA_FABRIC_VMM; - decision->nMaxChannels = ncclAllReduceDdaFabricBlocks(comm, count, datatype); - return ncclSuccess; - } - } else { - if (ncclAllReduceDdaIpcEligible(comm, sendbuff, recvbuff, count, datatype, op)) { - decision->algo = RCCL_DDA_IPC; - decision->nMaxChannels = ncclAllReduceDdaIpcBlocks(comm, count, datatype); - return ncclSuccess; + // (4) DDA fast paths. Shared gate: !symkRequested on every arch, and either + // gfx1250 (fabric may run even if CE is also eligible) or CE is not going to + // service this call (!ceAllReduceAllowed), subject to rcclDdaEnabled -- all + // folded into the helper. Passes the raw symk signal for the same reason as + // (3): symMaxR2 chooses between symk and CE-registered, it does not hand + // registered operands to DDA. + // + // GIN AllReduce is selected first in this function and requires symmetric + // windows. By default it only claims messages >= 256 MiB, so DDA must still be + // allowed for smaller symmetric AllReduces (otherwise they would hit the + // symmetric kernel instead of DDA). FORCE_ENABLE=1 keeps the original + // !symEligible gate because GIN already returned above for those sizes. + // symSuppressedByMin: DDA wins below symMinR2[AR]; do not block it with symkRequested. + bool ddaSymEligible = symkRequested && !symSuppressedByMin; + #if defined(ENABLE_ROCSHMEM_GIN) + if (ncclAllReduceGinSdmaYieldToDda(comm, sendbuff, recvbuff, count, datatype, op)) { + ddaSymEligible = false; + } + #endif + const bool ddaFabricArch1250 = IsArchMatch(comm->archName, "gfx1250"); + const size_t arDdaVmmMax = rcclDdaVmmThresholdCtxTab(archTable, ncclFuncAllReduce, winRegType, ceCapturing); + const bool arShouldTakeDda = rcclAllReduceShouldTakeDdaPath(comm, count, datatype, ddaSymEligible, ceAllReduceAllowed, query); + if (arShouldTakeDda) { + if (ddaFabricArch1250) { + const size_t arDdaLLMax = rcclDdaLLThresholdTab(archTable, ncclFuncAllReduce); + const size_t arDdaLL128Max = rcclDdaLL128ThresholdTab(archTable, ncclFuncAllReduce); + // Small-message fast lane: LL protocol (no GPU barrier). + if (rcclParamDdaLL() && msgBytes <= arDdaLLMax && + ncclAllReduceDdaFabricLLEligible(comm, sendbuff, recvbuff, count, datatype, op)) { + decision->algo = RCCL_DDA_FABRIC_LL; + decision->protocol = NCCL_PROTO_LL; + decision->nMaxChannels = ncclAllReduceDdaFabricLLBlocks(comm, count, datatype); + return ncclSuccess; + } + // Mid-size fast lane: LL128 protocol (128B lines, no GPU barrier). + if (arDdaLL128Max > 0 && msgBytes <= arDdaLL128Max && + ncclAllReduceDdaFabricLL128Eligible(comm, sendbuff, recvbuff, count, datatype, op)) { + decision->algo = RCCL_DDA_FABRIC_LL128; + decision->protocol = NCCL_PROTO_LL128; + decision->nMaxChannels = ncclAllReduceDdaFabricLL128Blocks(comm, count, datatype); + return ncclSuccess; + } + // The entry gate now admits the widest tier, so VMM re-checks its own cap. + if (arDdaVmmMax != 0 && msgBytes <= arDdaVmmMax && + ncclAllReduceDdaFabricEligible(comm, sendbuff, recvbuff, count, datatype, op)) { + decision->algo = RCCL_DDA_FABRIC_VMM; + decision->nMaxChannels = ncclAllReduceDdaFabricBlocks(comm, count, datatype); + return ncclSuccess; + } + } else { + if (arDdaVmmMax != 0 && msgBytes <= arDdaVmmMax && + ncclAllReduceDdaIpcEligible(comm, sendbuff, recvbuff, count, datatype, op)) { + decision->algo = RCCL_DDA_IPC; + decision->nMaxChannels = ncclAllReduceDdaIpcBlocks(comm, count, datatype); + return ncclSuccess; + } } } - } // (5) Enqueue-bound backends: CE registered (Branch B) vs symmetric vs kernel. // Reproduce taskAppend()'s AllReduce CE decision exactly so both agree. - // NOTE: CE registered wins over the symmetric kernel when both are eligible, - // matching taskAppend (its CE branch is not gated on symEligible; symmetric - // extraction only sees tasks that fall through to collTaskAppend). - // // develop's taskAppend appends CE for AllReduce iff !hasSysmemSegment && ceAvailable - // && ((CTAPolicy & ZERO) || force): ceAvailable starts from ncclCeAvailable(op) then - // is cleared unless graph-allowed, op-supported, count-divisible and RCCL_CE_ALLREDUCE. - bool ceAvailable = !ceCapturing && ncclCeAvailable(comm, ncclFuncAllReduce, (int)op, datatype, winRegType, sendWin, recvWin); + // && ((CTAPolicy & ZERO) || force): ceAvailable is the conjunction of the four + // sub-conditions below; split out so the disqualification log can name the blocker. + const bool ceBufferOk = !ceCapturing && ncclCeAvailable(comm, ncclFuncAllReduce, (int)op, datatype, winRegType, sendWin, recvWin); const bool ceAllReduceOpSupported = (op == ncclSum || op == ncclProd || op == ncclMin || op == ncclMax); - if (!ceArGraphAllowed || !ceAllReduceOpSupported || (count % (size_t)comm->nRanks != 0) || !rcclParamCeAllReduce()) { - ceAvailable = false; - } - if (ceAvailable && !hasSysmemSegment && ((comm->config.CTAPolicy & NCCL_CTA_POLICY_ZERO) || force)) { + const bool ceCountDivisible = (count % (size_t)comm->nRanks == 0); + const bool ceEnabledByArch = rcclCeAllReduceEnabledDef(ceArArchDefault); + const bool ceAvailable = ceArGraphAllowed && ceBufferOk && ceAllReduceOpSupported && ceCountDivisible && ceEnabledByArch; + // Tuning cap only: registered CE has no staging allocation, so this does not + // size a buffer. kThreshUnlimited (or null table) = no upper bound; + // env var 0 returns 0, making ceRegInWindow false (disables registered CE). + // Independent of the 2-shot selector (ceNonRegMax/env, 0 = off) and ceARTmpBuf sizing. + const size_t ceArRegMax = rcclCeRegMaxTab(archTable, ncclFuncAllReduce); + const bool ceRegInWindow = ceArRegMax == kThreshUnlimited || msgBytes <= ceArRegMax; + if (!symEligible && ceRegInWindow && ceAvailable && !hasSysmemSegment && + ((comm->config.CTAPolicy & NCCL_CTA_POLICY_ZERO) || force)) { decision->algo = RCCL_CE_REGISTERED; decision->nMaxChannels = ncclCeLocalReduceBlocks(datatype, count / comm->nRanks); return ncclSuccess; } + if (!query && !symEligible) INFO(NCCL_TUNING, + "AR CE-registered disqualified: ceAvailable=%d(graphAllowed=%d bufOk=%d opOk=%d countDiv=%d archEnabled=%d)" + " ceRegInWindow=%d hasSysmem=%d CTAPolicy=%d force=%d", + (int)ceAvailable, (int)ceArGraphAllowed, (int)ceBufferOk, (int)ceAllReduceOpSupported, + (int)ceCountDivisible, (int)ceEnabledByArch, + (int)ceRegInWindow, (int)hasSysmemSegment, (int)comm->config.CTAPolicy, (int)force); if (symEligible) { - decision->algo = RCCL_SYMMETRIC; // Reporting only: fill the symk protocol/channels that will actually run. - // Skipped on the live path (taskAppend recomputes and dispatches symk). - if (query) { - int a, p, ch; - if (rcclSymkQuery(comm, ncclFuncAllReduce, count, datatype, op, &a, &p, &ch)) { - decision->protocol = p; - decision->nMaxChannels = ch; - } + // Live path: collTaskAppend tags the task (symkExtract=1) so + // ncclMakeSymmetricTaskList honors this choice instead of re-deriving it. + if (!query) { + decision->algo = RCCL_SYMMETRIC; + return ncclSuccess; } - return ncclSuccess; + int a, p, ch; + if (rcclSymkQuery(comm, ncclFuncAllReduce, count, datatype, op, &a, &p, &ch)) { + decision->algo = RCCL_SYMMETRIC; + decision->protocol = p; + decision->nMaxChannels = ch; + return ncclSuccess; + } + // symk query failed — fall through to next candidate } // (6) Standard ring/tree/pat kernel. Fill algo/protocol/channels for reporting // (query mode); on the live path taskAppend() recomputes these downstream, so // skip the getAlgoInfo() cost there and leave a valid non-CE placeholder. - decision->algo = NCCL_ALGO_RING; - decision->protocol = NCCL_PROTO_SIMPLE; - if (query) { - struct ncclTaskColl task; - memset(&task, 0, sizeof(task)); - task.func = ncclFuncAllReduce; - task.sendbuff = sendbuff; - task.recvbuff = recvbuff; - task.count = count; - task.datatype = datatype; - NCCLCHECK(getAlgoInfo(comm, &task, /*collNetSupport=*/0, /*nvlsSupport=*/0, /*numPipeOps=*/1, /*simInfo=*/nullptr)); - decision->protocol = task.protocol; - // Report the traffic-packed channel count the kernel actually runs on, not the - // tuning cap (task.nMaxChannels), matching the enqueue.cc channel{Lo..Hi} log. - int packed = rcclKernelPackedChannels(comm, ncclFuncAllReduce, count, datatype, task.protocol, task.nMaxChannels); -#ifdef ENABLE_WARP_SPEED - // WarpSpeed reports as RING* with channels scaled by nWarps, matching rcclGetAlgoInfo. - decision->nMaxChannels = task.useWarpSpeed ? task.nMaxChannels / task.nWarps : packed; - decision->algo = task.useWarpSpeed ? rcclAddonAlgos_t::RCCL_WARP_SPEED : task.algorithm; -#else - decision->nMaxChannels = packed; - decision->algo = task.algorithm; -#endif } - return ncclSuccess; + return rcclRingFallback(comm, sendbuff, recvbuff, ncclFuncAllReduce, count, datatype, query, decision); } // See the header comment on rcclSelectAllGather(). Faithful consolidation of // ncclAllGather_impl() (DDA) and rcclSelectAllGatherAlgo() (hierarchical / direct // / ring); the outcome for any given operands is identical. ncclResult_t rcclSelectAllGather(struct ncclComm* comm, const void* sendbuff, void* recvbuff, size_t sendcount, - ncclDataType_t datatype, bool query, bool graphCapturingHint, + ncclDataType_t datatype, cudaStream_t stream, bool query, bool graphCapturingHint, struct rcclCollDecision* decision) { memset(decision, 0, sizeof(*decision)); decision->algo = NCCL_ALGO_RING; @@ -1122,130 +1506,212 @@ ncclResult_t rcclSelectAllGather(struct ncclComm* comm, const void* sendbuff, vo const size_t totalBytes = (size_t)comm->nRanks * sendcount * typeSize; size_t msgSize = totalBytes; - // (1) DDA fast paths. Symmetric-registered buffers defer to the symmetric - // kernel (extracted downstream), so DDA is gated on !symEligible, as before. - const bool symEligible = - isSymmetricKernelRequested(comm, ncclFuncAllGather, (int)ncclDevSum, datatype, sendcount, sendbuff, recvbuff); - // symEligible gates DDA below; the symk report itself is deferred until after - // the CE-registered check so it loses to CE exactly as dispatch does - // (taskAppend appends the CE task before ncclMakeSymmetricTaskList runs, so - // symk never reclaims it), mirroring rcclSelectAllReduce. - if (!symEligible && rcclDdaEnabled(comm, totalBytes, 8388608)) { - if (IsArchMatch(comm->archName, "gfx1250")) { - if (ncclAllGatherDdaFabricLLEligible(comm, sendbuff, recvbuff, sendcount, datatype)) { - decision->algo = RCCL_DDA_FABRIC_LL; - decision->protocol = NCCL_PROTO_LL; - decision->nMaxChannels = ncclAllGatherDdaFabricLLBlocks(comm, sendcount, datatype); - return ncclSuccess; - } - if (ncclAllGatherDdaFabricLL128Eligible(comm, sendbuff, recvbuff, sendcount, datatype)) { - decision->algo = RCCL_DDA_FABRIC_LL128; - decision->protocol = NCCL_PROTO_LL128; - decision->nMaxChannels = ncclAllGatherDdaFabricLL128Blocks(comm, sendcount, datatype); - return ncclSuccess; - } - if (ncclAllGatherDdaFabricEligible(comm, sendbuff, recvbuff, sendcount, datatype)) { - decision->algo = RCCL_DDA_FABRIC_VMM; - decision->nMaxChannels = ncclAllGatherDdaFabricBlocks(comm, sendcount, datatype); - return ncclSuccess; - } - } else if (ncclAllGatherDdaIpcEligible(comm, sendbuff, recvbuff, sendcount, datatype)) { - decision->algo = RCCL_DDA_IPC; - decision->nMaxChannels = ncclAllGatherDdaIpcBlocks(comm, sendcount, datatype); - return ncclSuccess; - } - } - - // (2) Hierarchical AllGather. Live dispatch requires being outside a group - // (rcclSelectAllGatherAlgo); the reporting query always runs outside a group, so - // the same gate reproduces rcclGetAlgoInfo's group-agnostic reporting. - if (ncclGroupDepth == 0 && rcclUseHierarchicalAllGather(comm, msgSize)) { - decision->algo = RCCL_HIERARCHICAL_ALLGATHER; - if (query) { - // -A reports the inter-comm proto/channels; intra values are logged only. - ncclComm* interComm = comm->hierarchicalInterComm; - ncclComm* intraComm = comm->hierarchicalIntraComm; - int nNodes = interComm->nRanks; - size_t interMsgSize = sendcount * typeSize * nNodes; - if (nNodes <= 16 && rcclUseAllGatherDirect(interComm, interMsgSize)) { - decision->protocol = NCCL_PROTO_SIMPLE; - decision->nMaxChannels = interComm->p2pnChannels; - } else { - struct ncclTaskColl task; - task.func = ncclFuncAllGather; - task.count = sendcount; - task.datatype = datatype; - NCCLCHECK(getAlgoInfo(interComm, &task, 0, 0, 1)); - decision->protocol = task.protocol; - decision->nMaxChannels = task.nMaxChannels; - } - int intraProto, intraChan; - size_t intraCount = sendcount * nNodes; - size_t intraMsgSize = intraCount * typeSize * intraComm->nRanks; - if (rcclUseAllGatherDirect(intraComm, intraMsgSize)) { - intraProto = NCCL_PROTO_SIMPLE; - intraChan = intraComm->p2pnChannels; - } else { - struct ncclTaskColl task; - task.func = ncclFuncAllGather; - task.count = intraCount; - task.datatype = datatype; - NCCLCHECK(getAlgoInfo(intraComm, &task, 0, 0, 1)); - intraProto = task.protocol; - intraChan = task.nMaxChannels; - } - INFO(NCCL_COLL, "Hierarchical AG inter: proto=%d channels=%u, intra: proto=%d channels=%d", decision->protocol, - decision->nMaxChannels, intraProto, intraChan); - } - return ncclSuccess; + // Graph capture probe hoisted so symMaxR2 can pick symMaxR2Graph vs + // symMaxR2 depending on whether a capture is active. Mirrors AllReduce. + // Tick the latch on the live path so a capture containing only AG (and no + // AR) still sets graphModeSeen, preventing CE AR from firing on a later + // eager AllReduce that shares this comm (ce_coll.h:93). + bool ceCapturing; + if (query) { + ceCapturing = graphCapturingHint; + } else { + struct ncclCudaGraph ceGraph; + NCCLCHECK(ncclCudaGetCapturingGraph(&ceGraph, stream, comm->config.graphUsageMode)); + ceCapturing = ncclCudaGraphValid(ceGraph); + rcclCeAllReduceGraphLatchTick(comm, ceCapturing); } + decision->ceCapturing = ceCapturing; - // (3) CE AllGather. Mirrors taskAppend()'s live gates and outranks Direct, as - // taskAppend checks CE before the useDirect branch. Reporting only here; the - // live path re-decides and dispatches CE in taskAppend(). - { - const bool ceCapturing = query ? graphCapturingHint : false; + if (!rcclNcclAlgoEnvIsSet()) { + // Window registration type is needed for both symSuppressedByMax and CE + // branch gates below; hoist the lookup here so it is computed once. struct ncclDevrWindow* sendWin = nullptr; struct ncclDevrWindow* recvWin = nullptr; ncclDevrFindWindow(comm, sendbuff, &sendWin); ncclDevrFindWindow(comm, recvbuff, &recvWin); - const bool hasSysmemSegment = ncclDevrWindowHasSysmemSegment(sendWin) || ncclDevrWindowHasSysmemSegment(recvWin); ncclSymRegType_t winRegType; NCCLCHECK(ncclGetSymRegType(sendWin, recvWin, &winRegType)); - // Branch #2: FORCE_CE via DDA scratch (unregistered windows). - const bool ceScratch = - !ceCapturing && ncclCeScratchAvailable(comm, ncclFuncAllGather, (int)ncclSum, datatype, winRegType); - if (rcclParamForceCe() && ceScratch && winRegType != ncclSymSendRegRecvReg && - winRegType != ncclSymSendNonregRecvReg && !hasSysmemSegment && comm->ddaScratch != nullptr && - totalBytes <= (size_t)comm->ddaScratchBytes) { - decision->algo = RCCL_CE_REGISTERED; - return ncclSuccess; + const rcclArchThresholds* const archTable = extAlgoArchTable(comm); + // (1) DDA fast paths. Symmetric-registered buffers defer to the symmetric + // kernel (extracted downstream), so DDA is gated on !symEligible, as before. + const bool agSymkRequested = + isSymmetricKernelRequestedWin(comm, ncclFuncAllGather, (int)ncclDevSum, datatype, sendcount, sendWin, recvWin); + // symMaxR2[AG] withdraws symk above a size threshold so CE-registered can win + // (mirrors the AllReduce symSuppressedByMax pattern). + // symMinR2[AG] withdraws symk below a size threshold so DDA wins small messages + // for R2 buffers, mirroring the AllReduce and ReduceScatter treatment. + // agRecvRegistered dropped: isSymmetricKernelRequestedWin requires both windows + // to carry NCCL_WIN_COLL_SYMMETRIC, so agSymkRequested=true implies recv registered. + const size_t agSymMaxR2 = rcclSymMaxR2CapTab(archTable, ncclFuncAllGather, ceCapturing); + const size_t agSymMinR2 = rcclSymMinR2CapTab(archTable, ncclFuncAllGather); + const bool agSymSuppressedByMin = agSymkRequested && agSymMinR2 > 0 && totalBytes < agSymMinR2; + const bool agSymSuppressedBySize = agSymkRequested && totalBytes > agSymMaxR2; + const bool symEligible = agSymkRequested && !agSymSuppressedByMin && !agSymSuppressedBySize; + if (!query && agSymkRequested && agSymSuppressedByMin) + INFO(NCCL_TUNING, "AG symk disqualified: totalBytes=%zu < agSymMinR2=%zu", totalBytes, agSymMinR2); + if (!query && agSymkRequested && agSymSuppressedBySize) + INFO(NCCL_TUNING, "AG symk disqualified: totalBytes=%zu > agSymMaxR2=%zu", totalBytes, agSymMaxR2); + // symEligible gates DDA below; the symk report itself is deferred until after + // the CE-registered check so it loses to CE exactly as dispatch does + // (taskAppend appends the CE task before ncclMakeSymmetricTaskList runs, so + // symk never reclaims it), mirroring rcclSelectAllReduce. + const size_t agDdaVmmMax = rcclDdaVmmThresholdCtxTab(archTable, ncclFuncAllGather, winRegType, ceCapturing); + if (!symEligible && rcclDdaEnabled(comm, totalBytes, rcclDdaEntryThresholdTab(archTable, ncclFuncAllGather), query, "AG")) { + const bool agFabricArch = IsArchMatch(comm->archName, "gfx1250"); + if (agFabricArch) { + const size_t agDdaLLMax = rcclDdaLLThresholdTab(archTable, ncclFuncAllGather); + const size_t agDdaLL128Max = rcclDdaLL128ThresholdTab(archTable, ncclFuncAllGather); + if (rcclParamDdaLL() && msgSize <= agDdaLLMax && + ncclAllGatherDdaFabricLLEligible(comm, sendbuff, recvbuff, sendcount, datatype)) { + decision->algo = RCCL_DDA_FABRIC_LL; + decision->protocol = NCCL_PROTO_LL; + decision->nMaxChannels = ncclAllGatherDdaFabricLLBlocks(comm, sendcount, datatype); + return ncclSuccess; + } + if (agDdaLL128Max > 0 && msgSize <= agDdaLL128Max && + ncclAllGatherDdaFabricLL128Eligible(comm, sendbuff, recvbuff, sendcount, datatype)) { + decision->algo = RCCL_DDA_FABRIC_LL128; + decision->protocol = NCCL_PROTO_LL128; + decision->nMaxChannels = ncclAllGatherDdaFabricLL128Blocks(comm, sendcount, datatype); + return ncclSuccess; + } + // The entry gate now admits the widest tier, so VMM re-checks its own cap. + if (agDdaVmmMax != 0 && totalBytes <= agDdaVmmMax && + ncclAllGatherDdaFabricEligible(comm, sendbuff, recvbuff, sendcount, datatype)) { + decision->algo = RCCL_DDA_FABRIC_VMM; + decision->nMaxChannels = ncclAllGatherDdaFabricBlocks(comm, sendcount, datatype); + return ncclSuccess; + } + } else if (agDdaVmmMax != 0 && totalBytes <= agDdaVmmMax && + ncclAllGatherDdaIpcEligible(comm, sendbuff, recvbuff, sendcount, datatype)) { + decision->algo = RCCL_DDA_IPC; + decision->nMaxChannels = ncclAllGatherDdaIpcBlocks(comm, sendcount, datatype); + return ncclSuccess; + } } - // Match taskAppend's single-node and hierarchical CE gates for -A 1 reporting. - const bool ceAvailable = - !ceCapturing && ncclCeAvailable(comm, ncclFuncAllGather, (int)ncclSum, datatype, winRegType, sendWin, recvWin); - const bool hierCeAvailable = !ceCapturing && ncclHierCeAvailable(comm, ncclFuncAllGather, (int)ncclSum, datatype, - winRegType, sendWin, recvWin); - if ((ceAvailable || hierCeAvailable) && !hasSysmemSegment && (comm->config.CTAPolicy & NCCL_CTA_POLICY_ZERO)) { - decision->algo = RCCL_CE_REGISTERED; + + if (!query && symEligible) INFO(NCCL_TUNING, "AG DDA disqualified: symk eligible"); + // (2) Hierarchical AllGather. Live dispatch requires being outside a group + // (rcclSelectAllGatherAlgo); the reporting query always runs outside a group, so + // the same gate reproduces rcclGetAlgoInfo's group-agnostic reporting. + if (ncclGroupDepth == 0 && rcclUseHierarchicalAllGather(comm, msgSize)) { + decision->algo = RCCL_HIERARCHICAL_ALLGATHER; + if (query) { + // -A reports the inter-comm proto/channels; intra values are logged only. + ncclComm* interComm = comm->hierarchicalInterComm; + ncclComm* intraComm = comm->hierarchicalIntraComm; + int nNodes = interComm->nRanks; + size_t interMsgSize = sendcount * typeSize * nNodes; + if (nNodes <= 16 && rcclUseAllGatherDirect(interComm, interMsgSize)) { + decision->protocol = NCCL_PROTO_SIMPLE; + decision->nMaxChannels = interComm->p2pnChannels; + } else { + struct ncclTaskColl task; + task.func = ncclFuncAllGather; + task.count = sendcount; + task.datatype = datatype; + NCCLCHECK(getAlgoInfo(interComm, &task, 0, 0, 1)); + decision->protocol = task.protocol; + decision->nMaxChannels = task.nMaxChannels; + } + int intraProto, intraChan; + size_t intraCount = sendcount * nNodes; + size_t intraMsgSize = intraCount * typeSize * intraComm->nRanks; + if (rcclUseAllGatherDirect(intraComm, intraMsgSize)) { + intraProto = NCCL_PROTO_SIMPLE; + intraChan = intraComm->p2pnChannels; + } else { + struct ncclTaskColl task; + task.func = ncclFuncAllGather; + task.count = intraCount; + task.datatype = datatype; + NCCLCHECK(getAlgoInfo(intraComm, &task, 0, 0, 1)); + intraProto = task.protocol; + intraChan = task.nMaxChannels; + } + INFO(NCCL_COLL, "Hierarchical AG inter: proto=%d channels=%u, intra: proto=%d channels=%d", decision->protocol, + decision->nMaxChannels, intraProto, intraChan); + } return ncclSuccess; } - } - // (3.5) Symmetric-window kernel. Reported only after CE-registered so it loses - // to CE exactly as dispatch does; on the live path taskAppend recomputes and - // the symmetric extraction downstream dispatches symk. Mirrors rcclSelectAllReduce. - if (query && symEligible) { - int a, p, ch; - if (rcclSymkQuery(comm, ncclFuncAllGather, sendcount, datatype, ncclSum, &a, &p, &ch)) { - decision->algo = a; - decision->protocol = p; - decision->nMaxChannels = ch; - return ncclSuccess; + // (3) CE AllGather. Outranks Direct, matching taskAppend's CE-before-useDirect + // order. Live and query share these gates; taskAppend honors the decision. + { + const bool hasSysmemSegment = ncclDevrWindowHasSysmemSegment(sendWin) || ncclDevrWindowHasSysmemSegment(recvWin); + // Branch #2: CE via DDA scratch (unregistered windows). + // Fires either via RCCL_FORCE_CE or automatically when totalBytes falls in the + // [ceNonRegMin, ceNonRegMax] window from the arch table. The scratch buffer + // must be large enough to hold the receive (ddaScratchBytes >= totalBytes). + const bool ceScratch = + !ceCapturing && ncclCeScratchAvailable(comm, ncclFuncAllGather, (int)ncclSum, datatype, winRegType); + const size_t agCeNonRegMax = rcclCeNonRegMaxTab(archTable, ncclFuncAllGather); + const size_t agCeNonRegMin = rcclCeNonRegMinTab(archTable, ncclFuncAllGather); + const bool agCeNonRegWindow = agCeNonRegMax > 0 && + totalBytes >= agCeNonRegMin && + totalBytes <= agCeNonRegMax; + if ((rcclParamForceCe() || agCeNonRegWindow) && ceScratch && + winRegType == ncclSymSendNonregRecvNonreg && + !hasSysmemSegment && + comm->ddaScratch != nullptr && totalBytes <= (size_t)comm->ddaScratchBytes) { + decision->algo = RCCL_CE_SCRATCH; + return ncclSuccess; + } + if (!query) INFO(NCCL_TUNING, "AG CE-scratch disqualified: ceScratch=%d hasSysmem=%d ddaScratch=%p totalBytes=%zu ddaScratchBytes=%zu forceCe=%d agCeNonRegWindow=%d", + (int)ceScratch, (int)hasSysmemSegment, comm->ddaScratch, totalBytes, (size_t)comm->ddaScratchBytes, (int)rcclParamForceCe(), (int)agCeNonRegWindow); + // Branch #3: CE via registered symmetric windows. Taken when symk is not + // eligible and the size is within ceRegMax, or when CTAPolicy=ZERO forces + // CE for every size. + const bool ceAvailable = + !ceCapturing && ncclCeAvailable(comm, ncclFuncAllGather, (int)ncclSum, datatype, winRegType, sendWin, recvWin); + if (ceAvailable && !hasSysmemSegment && + ((comm->config.CTAPolicy & NCCL_CTA_POLICY_ZERO) || + (!symEligible && rcclAllGatherCeRegisteredWindowTab(archTable, totalBytes, winRegType, ceCapturing)))) { + decision->algo = RCCL_CE_REGISTERED; + return ncclSuccess; + } + if (!query) INFO(NCCL_TUNING, "AG CE-registered disqualified: ceAvailable=%d hasSysmem=%d symEligible=%d ceRegWindow=%d", + (int)ceAvailable, (int)hasSysmemSegment, (int)symEligible, + (int)rcclAllGatherCeRegisteredWindowTab(archTable, totalBytes, winRegType, ceCapturing)); + // Branch #3.5: Hierarchical CE (multi-node, both buffers registered). + // ceCollTaskAppend routes to ncclHierCeAllGather via ncclHierCeDispatch(comm), + // so RCCL_CE_REGISTERED is correct here — same as rcclSelectAlltoAll Branch #5. + const bool hierCeAvailable = + !ceCapturing && ncclHierCeAvailable(comm, ncclFuncAllGather, (int)ncclSum, datatype, winRegType, sendWin, recvWin); + if (hierCeAvailable && !hasSysmemSegment && + (comm->config.CTAPolicy & NCCL_CTA_POLICY_ZERO)) { + decision->algo = RCCL_CE_REGISTERED; + if (query) { + int a, p, ch; + NCCLCHECK(rcclHierarchicalAlgoInfo(comm, ncclFuncAllGather, sendcount, datatype, &a, &p, &ch)); + decision->protocol = p; + decision->nMaxChannels = ch; + } + return ncclSuccess; + } + } + + // (4) Symmetric kernel. Live path dispatches symk via the downstream extraction. + // Placed after CE so CE-registered wins for sizes above the symk/CE crossover + // (symSuppressedBySize), mirroring rcclSelectAllReduce and rcclSelectReduceScatter. + if (symEligible) { + if (!query) { + decision->algo = RCCL_SYMMETRIC; + return ncclSuccess; + } + int a, p, ch; + if (rcclSymkQuery(comm, ncclFuncAllGather, sendcount, datatype, ncclSum, &a, &p, &ch)) { + decision->algo = RCCL_SYMMETRIC; + decision->protocol = p; + decision->nMaxChannels = ch; + return ncclSuccess; + } + // symk query failed — fall through to next candidate } + } - // (4) Direct AllGather (per-peer Send/Recv). + // (5) Direct AllGather (per-peer Send/Recv). if (rcclUseAllGatherDirect(comm, msgSize)) { decision->algo = RCCL_DIRECT_ALLGATHER; decision->protocol = NCCL_PROTO_SIMPLE; @@ -1253,34 +1719,7 @@ ncclResult_t rcclSelectAllGather(struct ncclComm* comm, const void* sendbuff, vo return ncclSuccess; } - // (5) Standard ring kernel. Fill algo/protocol/channels for reporting; the live - // path recomputes these in taskAppend(), so only the query needs them. - decision->algo = NCCL_ALGO_RING; - decision->protocol = NCCL_PROTO_SIMPLE; - if (query) { - struct ncclTaskColl task; - memset(&task, 0, sizeof(task)); - task.func = ncclFuncAllGather; - task.sendbuff = sendbuff; - task.recvbuff = recvbuff; - task.count = sendcount; - task.datatype = datatype; - NCCLCHECK(getAlgoInfo(comm, &task, 0, 0, 1)); - decision->protocol = task.protocol; - // Report the traffic-packed channel count the kernel actually runs on, not the - // tuning cap (task.nMaxChannels), matching the enqueue.cc channel{Lo..Hi} log. - int packed = - rcclKernelPackedChannels(comm, ncclFuncAllGather, sendcount, datatype, task.protocol, task.nMaxChannels); -#ifdef ENABLE_WARP_SPEED - // WarpSpeed reports as RING* with channels scaled by nWarps, matching rcclGetAlgoInfo. - decision->nMaxChannels = task.useWarpSpeed ? task.nMaxChannels / task.nWarps : packed; - decision->algo = task.useWarpSpeed ? rcclAddonAlgos_t::RCCL_WARP_SPEED : task.algorithm; -#else - decision->nMaxChannels = packed; - decision->algo = task.algorithm; -#endif - } - return ncclSuccess; + return rcclRingFallback(comm, sendbuff, recvbuff, ncclFuncAllGather, sendcount, datatype, query, decision); } // See the header comment. Consolidates the backend picking formerly inlined in @@ -1295,106 +1734,312 @@ ncclResult_t rcclSelectReduceScatter(struct ncclComm* comm, const void* sendbuff const size_t typeSize = ncclTypeSize(datatype); const size_t totalBytes = (size_t)comm->nRanks * recvcount * typeSize; - const size_t rsShardBytes = recvcount * typeSize; - - // (1) Symmetric eligibility (sum/avg). Reported last but gates DDA IPC / Direct here. - const bool symEligible = - (op == ncclSum || op == ncclAvg) && - isSymmetricKernelRequested(comm, ncclFuncReduceScatter, (op == ncclAvg) ? (int)ncclDevSumPostDiv : (int)ncclDevSum, - datatype, recvcount, sendbuff, recvbuff); - - // (2) DDA fast paths. gfx1250 fabric may win over symmetric (cross-rank identical - // state); IPC keeps the strict !symEligible guard. No Blocks helpers -> nMaxChannels 0. - const bool ddaFabricArch = IsArchMatch(comm->archName, "gfx1250"); - if ((!symEligible || ddaFabricArch) && rcclDdaEnabled(comm, totalBytes, 8388608)) { - if (ddaFabricArch) { - if (rcclParamDdaLL() && rsShardBytes <= (size_t)rcclParamDdaLLThreshold() && - ncclReduceScatterDdaFabricLLEligible(comm, sendbuff, recvbuff, recvcount, datatype, op)) { + + if (!rcclNcclAlgoEnvIsSet()) { + // (1) Symmetric eligibility (sum/avg). Reported last but gates DDA / hierarchical / Direct here. + // symMinR2: below this threshold DDA beats symk for R2 buffers -- suppress symk so DDA wins. + // Window lookup hoisted above isSymmetricKernelRequested to avoid the + // redundant ncclDevrFindWindow pair inside that call. + struct ncclDevrWindow* rsSendWin = nullptr; + struct ncclDevrWindow* rsRecvWin = nullptr; + ncclDevrFindWindow(comm, sendbuff, &rsSendWin); + ncclDevrFindWindow(comm, recvbuff, &rsRecvWin); + ncclSymRegType_t rsWinRegType; + NCCLCHECK(ncclGetSymRegType(rsSendWin, rsRecvWin, &rsWinRegType)); + const rcclArchThresholds* const archTable = extAlgoArchTable(comm); + const bool symkRequested = + (op == ncclSum || op == ncclAvg) && + isSymmetricKernelRequestedWin(comm, ncclFuncReduceScatter, (op == ncclAvg) ? (int)ncclDevSumPostDiv : (int)ncclDevSum, + datatype, recvcount, rsSendWin, rsRecvWin); + const size_t rsSymMinR2 = rcclSymMinR2CapTab(archTable, ncclFuncReduceScatter); + const bool symSuppressedByMin = symkRequested && rsSymMinR2 > 0 && totalBytes < rsSymMinR2; + // rsRecvRegistered dropped: isSymmetricKernelRequestedWin requires both windows + // to carry NCCL_WIN_COLL_SYMMETRIC, so symkRequested=true implies recv registered. + const size_t rsSymMaxR2 = rcclSymMaxR2CapTab(archTable, ncclFuncReduceScatter, /*graphMode=*/false); + const bool symSuppressedByMax = symkRequested && totalBytes > rsSymMaxR2; + const bool symEligible = symkRequested && !symSuppressedByMin && !symSuppressedByMax; + if (!query && symkRequested && symSuppressedByMin) + INFO(NCCL_TUNING, "RS symk disqualified: totalBytes=%zu < rsSymMinR2=%zu", totalBytes, rsSymMinR2); + if (!query && symkRequested && symSuppressedByMax) + INFO(NCCL_TUNING, "RS symk disqualified: totalBytes=%zu > rsSymMaxR2=%zu", totalBytes, rsSymMaxR2); + + // (2) DDA fast paths. Symmetric wins when buffers are registered (-R 2); DDA + // enters only when symk is unavailable. No Blocks helpers -> nMaxChannels 0. + const size_t rsDdaVmmMax = rcclDdaVmmThresholdCtxTab(archTable, ncclFuncReduceScatter, rsWinRegType, /*graphMode=*/false); + if (!symEligible && + rcclDdaEnabled(comm, totalBytes, rcclDdaEntryThresholdTab(archTable, ncclFuncReduceScatter), query, "RS")) { + const bool ddaFabricArch = IsArchMatch(comm->archName, "gfx1250"); + if (ddaFabricArch) { + const size_t rsDdaLLMax = rcclDdaLLThresholdTab(archTable, ncclFuncReduceScatter); + const size_t rsDdaLL128Max = rcclDdaLL128ThresholdTab(archTable, ncclFuncReduceScatter); + if (rcclParamDdaLL() && totalBytes <= rsDdaLLMax && + ncclReduceScatterDdaFabricLLEligible(comm, sendbuff, recvbuff, recvcount, datatype, op)) { + decision->algo = RCCL_DDA_FABRIC_LL; + decision->protocol = NCCL_PROTO_LL; + return ncclSuccess; + } + if (rsDdaLL128Max > 0 && totalBytes <= rsDdaLL128Max && + ncclReduceScatterDdaFabricLL128Eligible(comm, sendbuff, recvbuff, recvcount, datatype, op)) { + decision->algo = RCCL_DDA_FABRIC_LL128; + decision->protocol = NCCL_PROTO_LL128; + return ncclSuccess; + } + // The entry gate now admits the widest tier, so VMM re-checks its own cap. + // ddaVmmMax[RS] = 0 on gfx1250 keeps this tier off while LL/LL128 run. + if (rsDdaVmmMax != 0 && totalBytes <= rsDdaVmmMax && + ncclReduceScatterDdaFabricEligible(comm, sendbuff, recvbuff, recvcount, datatype, op)) { + decision->algo = RCCL_DDA_FABRIC_VMM; + return ncclSuccess; + } + } else if (rsDdaVmmMax != 0 && totalBytes <= rsDdaVmmMax && + ncclReduceScatterDdaIpcEligible(comm, sendbuff, recvbuff, recvcount, datatype, op)) { + decision->algo = RCCL_DDA_IPC; + return ncclSuccess; + } + } + + if (!query && symEligible) INFO(NCCL_TUNING, "RS DDA disqualified: symk eligible"); + // (3) Hierarchical ReduceScatter (multi-node, sum only). Live dispatch requires + // being outside a group; the reporting query always runs outside a group, so the + // same gate reproduces rcclGetAlgoInfo's group-agnostic reporting. + if (!symEligible && ncclGroupDepth == 0 && op == ncclSum && rcclUseHierarchicalReduceScatter(comm, totalBytes)) { + decision->algo = RCCL_HIERARCHICAL_REDUCESCATTER; + if (query) { + int a, p, ch; + NCCLCHECK(rcclHierarchicalAlgoInfo(comm, ncclFuncReduceScatter, recvcount, datatype, &a, &p, &ch)); + decision->protocol = p; + decision->nMaxChannels = ch; + } + return ncclSuccess; + } + + // (4) Direct ReduceScatter (per-peer Send/Recv, native kernel finishes the reduce). + // That reduce runs with PreOpSrcs=0 / postOp=false, an unscaled sum, so ncclAvg and + // user-defined PreMulSum (op >= ncclNumOps) fall through to the ring kernel instead. + size_t directMsgSize = totalBytes; + if (!symEligible && ncclGroupDepth == 0 && op < ncclAvg && rcclUseReduceScatterDirect(comm, directMsgSize)) { + decision->algo = RCCL_DIRECT_REDUCESCATTER; + decision->protocol = NCCL_PROTO_SIMPLE; + decision->nMaxChannels = comm->p2pnChannels; + return ncclSuccess; + } + + // (5) Symmetric kernel. Live path dispatches symk via the downstream extraction. + if (symEligible) { + if (!query) { + decision->algo = RCCL_SYMMETRIC; + return ncclSuccess; + } + int a, p, ch; + if (rcclSymkQuery(comm, ncclFuncReduceScatter, recvcount, datatype, op, &a, &p, &ch)) { + decision->algo = RCCL_SYMMETRIC; + decision->protocol = p; + decision->nMaxChannels = ch; + return ncclSuccess; + } + // symk query failed — fall through to next candidate + } + } + + // (6) Standard ring/pat kernel. Only the query needs algo/proto/channels filled; + // the live path recomputes these in taskAppend(). + return rcclRingFallback(comm, sendbuff, recvbuff, ncclFuncReduceScatter, recvcount, datatype, query, decision); +} + +// Single source of truth for AlltoAll implementation selection. Runs the full +// priority chain (Pivot -> GDA -> DDA LL/LL128/VMM/IPC -> CE registered -> +// HierCE -> CE scratch -> Direct p2p) and returns the decision. +// query=false : live dispatch path (ncclAlltoAll_impl). ceCapturing is probed +// from `stream`; graphCapturingHint is ignored. +// query=true : side-effect-free reporting for rcclGetCollImplInfo. The stream +// is not probed; graphCapturingHint supplies capture so CE is +// reported as skipped. +ncclResult_t rcclSelectAlltoAll(struct ncclComm* comm, const void* sendbuff, void* recvbuff, size_t count, + ncclDataType_t datatype, cudaStream_t stream, bool query, bool graphCapturingHint, + struct rcclCollDecision* decision) { + memset(decision, 0, sizeof(*decision)); + decision->algo = NCCL_ALGO_RING; + decision->protocol = NCCL_PROTO_SIMPLE; + decision->nMaxChannels = 0; + + const size_t typeSize = ncclTypeSize(datatype); + const size_t rankOffset = count * typeSize; // bytes per peer + const size_t totalBytes = comm->nRanks * rankOffset; // total message bytes + + // (1) Pivot: large, cache-line-aligned messages on pivot-enabled comms. + const size_t rankAlign = rankOffset & ((~rankOffset) + 1); + if (comm->topo->pivotA2AEnabled && comm->nChannels >= comm->topo->pivotA2ANumBiRings * 2 && + rankOffset >= 744 * 1024 && rankAlign != 4 && rcclParamAlltoAllPivotEnable()) { + decision->algo = RCCL_A2A_PIVOT; + // Pivot is a ring kernel over the collective channels, not p2p Send/Recv, so + // ask the tuner the way the live path does: taskAppend() rewrites the task to + // a per-peer byte count on ncclInt8 before getAlgoInfo() sees it. The kernel + // is only specialized for RING/SIMPLE, so the protocol default stands. + if (query) { + struct ncclTaskColl task; + memset(&task, 0, sizeof(task)); + task.func = ncclFuncAlltoAllPivot; + task.sendbuff = sendbuff; + task.recvbuff = recvbuff; + task.count = rankOffset; + task.datatype = ncclInt8; + NCCLCHECK(getAlgoInfo(comm, &task, /*collNetSupport=*/0, /*nvlsSupport=*/0, /*numPipeOps=*/1)); + decision->nMaxChannels = rcclKernelPackedChannels(comm, ncclFuncAlltoAllPivot, rankOffset, ncclInt8, + decision->protocol, task.nMaxChannels); + } + return ncclSuccess; + } + + // (2) GDA (RocSHMEM) path. +#ifdef ENABLE_ROCSHMEM + { + size_t msgSize = totalBytes; + if (rcclUseAlltoAllGda(comm) && msgSize <= comm->rocshmemThreshold) { + decision->algo = RCCL_A2A_GDA; + decision->nMaxChannels = 1; // getAlgoInfo() pins the GDA kernels to one channel + return ncclSuccess; + } + } +#endif + + // GIN LSA/SDMA is checked before DDA so an eligible call takes this path even + // below the DDA threshold (parity or better on small sizes). +#if defined(ENABLE_ROCSHMEM_GIN) + if (ncclAllToAllGinSdmaEligible(comm, sendbuff, recvbuff, count, datatype)) { + decision->algo = RCCL_A2A_GIN_SDMA; + return ncclSuccess; + } +#endif + + // Hoist window lookup for symSuppressedByMax (needed before DDA and CE gates). + struct ncclDevrWindow* a2aSendWin = nullptr; + struct ncclDevrWindow* a2aRecvWin = nullptr; + ncclDevrFindWindow(comm, sendbuff, &a2aSendWin); + ncclDevrFindWindow(comm, recvbuff, &a2aRecvWin); + ncclSymRegType_t a2aWinRegType; + NCCLCHECK(ncclGetSymRegType(a2aSendWin, a2aRecvWin, &a2aWinRegType)); + const rcclArchThresholds* const archTable = extAlgoArchTable(comm); + // symMaxR2[A2A] withdraws symk above threshold so CE-registered can win. + // a2aRecvRegistered dropped: isSymmetricKernelRequestedWin requires both windows + // to carry NCCL_WIN_COLL_SYMMETRIC, so a2aSymkRequested=true implies recv registered. + const bool a2aSymkRequested = + isSymmetricKernelRequestedWin(comm, ncclFuncAlltoAll, (int)ncclDevSum, datatype, count, a2aSendWin, a2aRecvWin); + const size_t a2aSymMaxR2 = rcclSymMaxR2CapTab(archTable, ncclFuncAlltoAll, /*graphMode=*/false); + const bool a2aSymSuppressedBySize = a2aSymkRequested && totalBytes > a2aSymMaxR2; + const bool a2aSymEligible = a2aSymkRequested && !a2aSymSuppressedBySize; + if (!query && a2aSymkRequested && a2aSymSuppressedBySize) + INFO(NCCL_TUNING, "A2A symk disqualified: totalBytes=%zu > a2aSymMaxR2=%zu", totalBytes, a2aSymMaxR2); + + // (3) DDA fast paths. gfx1250 uses fabric tiers; other archs use IPC. + // Symmetric-registered buffers defer to the symmetric kernel; DDA gated on !a2aSymEligible. + const size_t a2aDdaMax = rcclDdaVmmThresholdTab(archTable, ncclFuncAlltoAll); + if (!a2aSymEligible && rcclDdaEnabled(comm, totalBytes, rcclDdaEntryThresholdTab(archTable, ncclFuncAlltoAll), query, "A2A")) { + const bool a2aFabricArch = IsArchMatch(comm->archName, "gfx1250"); + if (a2aFabricArch) { + const size_t llThresh = rcclDdaLLThresholdTab(archTable, ncclFuncAlltoAll); + const size_t ll128Thresh = rcclDdaLL128ThresholdTab(archTable, ncclFuncAlltoAll); + if (rcclParamDdaLL() && llThresh > 0 && totalBytes <= llThresh && + ncclAllToAllDdaFabricLLEligible(comm, sendbuff, recvbuff, count, datatype)) { decision->algo = RCCL_DDA_FABRIC_LL; decision->protocol = NCCL_PROTO_LL; + decision->nMaxChannels = ncclAllToAllDdaFabricLLBlocks(comm, count, datatype); return ncclSuccess; } - if (rcclParamDdaLL128() && rsShardBytes <= (size_t)rcclParamDdaLL128Threshold() && - ncclReduceScatterDdaFabricLL128Eligible(comm, sendbuff, recvbuff, recvcount, datatype, op)) { + if (ll128Thresh > 0 && totalBytes <= ll128Thresh && + ncclAllToAllDdaFabricLL128Eligible(comm, sendbuff, recvbuff, count, datatype)) { decision->algo = RCCL_DDA_FABRIC_LL128; decision->protocol = NCCL_PROTO_LL128; + decision->nMaxChannels = ncclAllToAllDdaFabricLL128Blocks(comm, count, datatype); return ncclSuccess; } - if (ncclReduceScatterDdaFabricEligible(comm, sendbuff, recvbuff, recvcount, datatype, op)) { + // The entry gate now admits the widest tier, so VMM re-checks its own cap. + if (a2aDdaMax != 0 && totalBytes <= a2aDdaMax && + ncclAllToAllDdaFabricEligible(comm, sendbuff, recvbuff, count, datatype)) { decision->algo = RCCL_DDA_FABRIC_VMM; + decision->nMaxChannels = ncclAllToAllDdaFabricBlocks(comm, count, datatype); return ncclSuccess; } - } else if (ncclReduceScatterDdaIpcEligible(comm, sendbuff, recvbuff, recvcount, datatype, op)) { + } else if (a2aDdaMax != 0 && totalBytes <= a2aDdaMax && + ncclAllToAllDdaIpcEligible(comm, sendbuff, recvbuff, count, datatype)) { decision->algo = RCCL_DDA_IPC; + decision->nMaxChannels = ncclAllToAllDdaIpcBlocks(comm, count, datatype); return ncclSuccess; } } - // (3) Hierarchical ReduceScatter (multi-node, sum only). Live dispatch requires - // being outside a group; the reporting query always runs outside a group, so the - // same gate reproduces rcclGetAlgoInfo's group-agnostic reporting. - if (!symEligible && ncclGroupDepth == 0 && op == ncclSum && rcclUseHierarchicalReduceScatter(comm, totalBytes)) { - decision->algo = RCCL_HIERARCHICAL_REDUCESCATTER; - if (query) { - int a, p, ch; - NCCLCHECK(rcclHierarchicalAlgoInfo(comm, ncclFuncReduceScatter, recvcount, datatype, &a, &p, &ch)); - decision->protocol = p; - decision->nMaxChannels = ch; - } - return ncclSuccess; - } - - // (4) Direct ReduceScatter (per-peer Send/Recv, native kernel finishes the reduce). - // That reduce runs with PreOpSrcs=0 / postOp=false, an unscaled sum, so ncclAvg and - // user-defined PreMulSum (op >= ncclNumOps) fall through to the ring kernel instead. - size_t directMsgSize = totalBytes; - if (!symEligible && ncclGroupDepth == 0 && op < ncclAvg && rcclUseReduceScatterDirect(comm, directMsgSize)) { - decision->algo = RCCL_DIRECT_REDUCESCATTER; - decision->protocol = NCCL_PROTO_SIMPLE; - decision->nMaxChannels = comm->p2pnChannels; - return ncclSuccess; + if (!query && a2aSymEligible) INFO(NCCL_TUNING, "A2A DDA disqualified: symk eligible"); + // CE is graph-unsafe. Live probes the stream; reporting uses graphCapturingHint. + // taskAppend honors this decision, so capture must be recorded here rather than + // re-probed at enqueue. + // Tick the latch on the live path so a capture containing only A2A (and no + // AR) still sets graphModeSeen, preventing CE AR from firing on a later + // eager AllReduce that shares this comm (ce_coll.h:93). + bool ceCapturing; + if (query) { + ceCapturing = graphCapturingHint; + } else { + struct ncclCudaGraph ceGraph; + NCCLCHECK(ncclCudaGetCapturingGraph(&ceGraph, stream, comm->config.graphUsageMode)); + ceCapturing = ncclCudaGraphValid(ceGraph); + rcclCeAllReduceGraphLatchTick(comm, ceCapturing); } + decision->ceCapturing = ceCapturing; - // (5) Symmetric kernel. Live path dispatches symk via the downstream extraction. - if (symEligible) { - decision->algo = RCCL_SYMMETRIC; - if (query) { - int a, p, ch; - if (rcclSymkQuery(comm, ncclFuncReduceScatter, recvcount, datatype, op, &a, &p, &ch)) { + // (4) CE registered: single-node, symmetric-registered buffers, CTA_POLICY_ZERO. + // Live returns RCCL_CE_REGISTERED / RCCL_CE_SCRATCH and enqueues; taskAppend + // dispatches from that algo (mirroring rcclSelectAllGather). + // Hoisted above the ceCapturing gate so both Branch (4) and HierCE (5) see it; + // ncclCeAvailable does not check sysmem segments itself (ce_coll.cc:301). + const bool a2aHasSysmem = ncclDevrWindowHasSysmemSegment(a2aSendWin) || + ncclDevrWindowHasSysmemSegment(a2aRecvWin); + if (!ceCapturing) { + // Probe real window registration on both paths so the reported decision and + // the dispatched one cannot disagree. The lookups are null-safe, so the + // buffer-less ABI (rcclSymKGetInfo) simply sees unregistered buffers. + if ((comm->config.CTAPolicy & NCCL_CTA_POLICY_ZERO) && !a2aHasSysmem && + ncclCeAvailable(comm, ncclFuncAlltoAll, ncclDevSum, datatype, a2aWinRegType, a2aSendWin, a2aRecvWin)) { + decision->algo = RCCL_CE_REGISTERED; + return ncclSuccess; + } + if (!query) INFO(NCCL_TUNING, "A2A CE-registered disqualified: CTAPolicy=%d hasSysmem=%d ceAvailable=%d", + (int)comm->config.CTAPolicy, (int)a2aHasSysmem, + (int)ncclCeAvailable(comm, ncclFuncAlltoAll, ncclDevSum, datatype, a2aWinRegType, a2aSendWin, a2aRecvWin)); + + // (5) Hierarchical CE: multi-node, non-LSA-spanning. + // Require CTA_POLICY_ZERO and no sysmem segment, matching the AllGather twin. + if ((comm->config.CTAPolicy & NCCL_CTA_POLICY_ZERO) && !a2aHasSysmem && + ncclHierCeAvailable(comm, ncclFuncAlltoAll, ncclDevSum, datatype, a2aWinRegType, a2aSendWin, a2aRecvWin)) { + decision->algo = RCCL_CE_REGISTERED; // reports as CE; hier dispatch in taskAppend + if (query) { + int a, p, ch; + NCCLCHECK(rcclHierarchicalAlgoInfo(comm, ncclFuncAlltoAll, count, datatype, &a, &p, &ch)); decision->protocol = p; decision->nMaxChannels = ch; } + return ncclSuccess; + } + + // (6) CE scratch: unregistered buffers, RCCL_FORCE_CE, recv fits in DDA scratch. + // Guard a2aWinRegType to reject registered recv windows (they must not be routed through + // ddaScratch; that would clobber the user recv buffer). Mirrors AllGather Branch #2 at + // line ~1643 and the taskAppend funnel guard at enqueue.cc:3978. + if (rcclParamForceCe() && !a2aHasSysmem && + comm->ddaScratch != nullptr && totalBytes <= comm->ddaScratchBytes && + ncclCeScratchAvailable(comm, ncclFuncAlltoAll, ncclDevSum, datatype, a2aWinRegType)) { + decision->algo = RCCL_CE_SCRATCH; + return ncclSuccess; } - return ncclSuccess; } - // (6) Standard ring/pat kernel. Only the query needs algo/proto/channels filled; - // the live path recomputes these in taskAppend(). - decision->algo = NCCL_ALGO_RING; + // (7) Direct (p2p) fallback. AllToAll has no collective kernel: taskAppend() + // decomposes it into one Send and one Recv task per peer, which run Simple + // over the p2p channels -- the same shape rcclSelectAllGather reports as + // Direct, so it is named that way rather than Ring. The ring/tree tuning model + // has no AllToAll entry, so querying it would only invent an algorithm this + // collective never runs. + decision->algo = RCCL_DIRECT_ALLTOALL; decision->protocol = NCCL_PROTO_SIMPLE; - if (query) { - struct ncclTaskColl task; - memset(&task, 0, sizeof(task)); - task.func = ncclFuncReduceScatter; - task.sendbuff = sendbuff; - task.recvbuff = recvbuff; - task.count = recvcount; - task.datatype = datatype; - NCCLCHECK(getAlgoInfo(comm, &task, /*collNetSupport=*/0, /*nvlsSupport=*/0, /*numPipeOps=*/1, /*simInfo=*/nullptr)); - decision->protocol = task.protocol; - // Report the traffic-packed channel count the kernel actually runs on, not the - // tuning cap (task.nMaxChannels), matching the enqueue.cc channel{Lo..Hi} log. - int packed = - rcclKernelPackedChannels(comm, ncclFuncReduceScatter, recvcount, datatype, task.protocol, task.nMaxChannels); -#ifdef ENABLE_WARP_SPEED - // WarpSpeed reports as RING* with channels scaled by nWarps, matching rcclGetAlgoInfo. - decision->nMaxChannels = task.useWarpSpeed ? task.nMaxChannels / task.nWarps : packed; - decision->algo = task.useWarpSpeed ? rcclAddonAlgos_t::RCCL_WARP_SPEED : task.algorithm; -#else - decision->nMaxChannels = packed; - decision->algo = task.algorithm; -#endif - } + decision->nMaxChannels = comm->p2pnChannels; return ncclSuccess; } diff --git a/projects/rccl/src/scheduler/symmetric_sched.cc b/projects/rccl/src/scheduler/symmetric_sched.cc index 5f2bd891b24c..a873f56b245b 100644 --- a/projects/rccl/src/scheduler/symmetric_sched.cc +++ b/projects/rccl/src/scheduler/symmetric_sched.cc @@ -104,7 +104,8 @@ ncclResult_t ncclMakeSymmetricTaskList(struct ncclComm* comm, struct ncclTaskCol int index; struct ncclTaskColl* next = task->next; ncclDevRedOp_t symkOp = symkRedOp(task->opHost, task->opDev.op); - bool symAvailable = ncclSymkAvailable(comm, task->func, symkOp, task->datatype, task->count); + bool symAvailable = task->symkExtract != RCCL_SYMK_EXTRACT_DENY && + ncclSymkAvailable(comm, task->func, symkOp, task->datatype, task->count); // Env (NCCL_ALGO/PROTO/SYM_KERNEL) is a global override that wins over per-call // algSelection for any function it forced. uint64_t effAlgMask = comm->tuningContext.forced[task->func] ? 0 : task->algMask; diff --git a/projects/rccl/test/CeAllReduceEligibilityTests.cpp b/projects/rccl/test/CeAllReduceEligibilityTests.cpp index 275a332791cb..68b8e69d2dfd 100644 --- a/projects/rccl/test/CeAllReduceEligibilityTests.cpp +++ b/projects/rccl/test/CeAllReduceEligibilityTests.cpp @@ -12,6 +12,7 @@ #include "gtest/gtest.h" #include "nccl.h" #include "rccl_common.h" +#include "graph.h" #include #include @@ -116,12 +117,12 @@ TEST_F(CeAllReduceEligibilityTest, ChunkLayout_SmallMessageSingleChunk) // ncclCeAllReduce() only ever sees counts the eligibility gate accepted: an // exact multiple of nRanks, and no larger than the staging buffer. ASSERT_EQ(count % static_cast(nRanks), 0u); - ASSERT_LE(count * sizeof(float), static_cast(NCCL_CE_AR_MAX_MSG_BYTES)); + ASSERT_LE(count * sizeof(float), kCeArMaxMsgBytesDefault); const size_t shardElems = count / nRanks; const size_t shardBytes = shardElems * sizeof(float); const size_t slotChunkBytes = - ncclCeAllReduceSlotChunkBytes(ncclCeAllReduceMaxChunkBytes(nRanks)); + ncclCeAllReduceSlotChunkBytes(ceAllReduceMaxChunkBytes(nRanks)); // A shard this small fits one slot, so ncclCeAllReduce() sends it as a single // chunk and never enters the pipelined path. @@ -132,7 +133,7 @@ TEST_F(CeAllReduceEligibilityTest, ChunkLayout_SmallMessageSingleChunk) // The host scatter addresses staging slots in bytes (rank * slotChunkBytes) while // the reduce kernel addresses them in elements (rank * slotChunkElems). If those // two strides disagree by even one byte, every rank but rank 0 reduces shifted -// data. NCCL_CE_AR_MAX_MSG_BYTES / nRanks only divides evenly for power-of-2 rank +// data. kCeArMaxMsgBytesDefault / nRanks only divides evenly for power-of-2 rank // counts, so those were the only ones that used to work. TEST_F(CeAllReduceEligibilityTest, ChunkLayout_SlotStridesAgreeForAnyRankCount) { @@ -142,13 +143,13 @@ TEST_F(CeAllReduceEligibilityTest, ChunkLayout_SlotStridesAgreeForAnyRankCount) for(int nRanks : rankCounts) { const size_t slotChunkBytes = - ncclCeAllReduceSlotChunkBytes(ncclCeAllReduceMaxChunkBytes(nRanks)); + ncclCeAllReduceSlotChunkBytes(ceAllReduceMaxChunkBytes(nRanks)); SCOPED_TRACE("nRanks=" + std::to_string(nRanks)); // Rank boundaries stay aligned for the kernel's 16B vector loads, and the // slots stay inside the buffer ncclCeInit() sized from the raw capacity. EXPECT_EQ(slotChunkBytes % 16, 0u); - EXPECT_LE(slotChunkBytes, ncclCeAllReduceMaxChunkBytes(nRanks)); + EXPECT_LE(slotChunkBytes, ceAllReduceMaxChunkBytes(nRanks)); for(size_t eltSize : elementSizes) { @@ -162,18 +163,18 @@ TEST_F(CeAllReduceEligibilityTest, ChunkLayout_SlotStridesAgreeForAnyRankCount) TEST_F(CeAllReduceEligibilityTest, ChunkLayout_LargeMessagePipelined) { - // A shard only spills past one slot when NCCL_CE_AR_MAX_MSG_BYTES / nRanks is + // A shard only spills past one slot when kCeArMaxMsgBytesDefault / nRanks is // not 16B-aligned, i.e. for a non-power-of-2 rank count at the message cap. constexpr int nRanks = 6; - const size_t shardElems = ncclCeAllReduceMaxChunkBytes(nRanks) / sizeof(float); + const size_t shardElems = ceAllReduceMaxChunkBytes(nRanks) / sizeof(float); const size_t count = shardElems * nRanks; // divisible by nRanks by construction // The gate would still accept this count, so the layout below is reachable. - ASSERT_LE(count * sizeof(float), static_cast(NCCL_CE_AR_MAX_MSG_BYTES)); + ASSERT_LE(count * sizeof(float), kCeArMaxMsgBytesDefault); const size_t shardBytes = shardElems * sizeof(float); const size_t slotChunkBytes = - ncclCeAllReduceSlotChunkBytes(ncclCeAllReduceMaxChunkBytes(nRanks)); + ncclCeAllReduceSlotChunkBytes(ceAllReduceMaxChunkBytes(nRanks)); ASSERT_GT(shardBytes, slotChunkBytes); // Same bookkeeping ncclCeAllReduce() does once it has picked a chunk size. @@ -201,8 +202,8 @@ TEST_F(CeAllReduceEligibilityTest, MaxStagingBytesPerRank) for(int nRanks : {2, 3, 4, 5, 6, 7, 8, 12, 16, 24}) { SCOPED_TRACE("nRanks=" + std::to_string(nRanks)); - EXPECT_LE(ncclCeAllReduceMaxChunkBytes(nRanks) * static_cast(nRanks), - static_cast(NCCL_CE_AR_MAX_MSG_BYTES)); + EXPECT_LE(ceAllReduceMaxChunkBytes(nRanks) * static_cast(nRanks), + kCeArMaxMsgBytesDefault); } } @@ -220,6 +221,7 @@ TEST(RcclCeAllReduceEligibility, RcclUseCeAllReduce_Isolated) ncclDataType_t datatype; bool expected; std::unordered_map extraEnv; + std::string archName; }; const std::unordered_map baseEnv = { @@ -227,6 +229,15 @@ TEST(RcclCeAllReduceEligibility, RcclUseCeAllReduce_Isolated) }; const std::vector cases = { + // Per-arch default for 2-shot (staging buffer): off on gfx1250 (ceNonRegMax[AR]=0; + // gfx1250 uses registered CE instead) and off on gfx950. No env override. + {"DefaultOff_2Shot_Gfx1250_Isolated", 4, 1, true, NCCL_CTA_POLICY_ZERO, 4096, ncclSum, ncclFloat32, false, {}, "gfx1250"}, + // RCCL_CE_ALLREDUCE=-1 (default) is auto-on for gfx1250; overriding + // RCCL_CE_AR_MAX_MSG_BYTES lifts the ceNonRegMax=0 cap so rcclUseCeAr2Shot + // returns true, confirming the default-on wiring. + {"DefaultOn_2Shot_Gfx1250_Isolated", 4, 1, true, NCCL_CTA_POLICY_ZERO, 4096, ncclSum, ncclFloat32, true, {{"RCCL_CE_AR_MAX_MSG_BYTES", "1048576"}}, "gfx1250"}, + {"DefaultOff_Gfx950_Isolated", 4, 1, true, NCCL_CTA_POLICY_ZERO, 4096, ncclSum, ncclFloat32, false, {}, "gfx950"}, + // Null archName (zero-initialised mock) also falls through to off. {"DisabledByDefault_Isolated", 4, 1, true, NCCL_CTA_POLICY_ZERO, 4096, ncclSum, ncclFloat32, false, {}}, {"EligibleFloat32Sum_Isolated", 4, 1, true, NCCL_CTA_POLICY_ZERO, 4096, ncclSum, ncclFloat32, true, baseEnv}, {"MultiNodeRejected_Isolated", 4, 2, true, NCCL_CTA_POLICY_ZERO, 4096, ncclSum, ncclFloat32, false, baseEnv}, @@ -241,7 +252,7 @@ TEST(RcclCeAllReduceEligibility, RcclUseCeAllReduce_Isolated) {"UnsupportedOpRejected_Isolated", 4, 1, true, NCCL_CTA_POLICY_ZERO, 4096, ncclAvg, ncclFloat32, false, baseEnv}, {"Float8Rejected_Isolated", 4, 1, true, NCCL_CTA_POLICY_ZERO, 4096, ncclSum, ncclFloat8e4m3, false, baseEnv}, {"MessageTooLargeRejected_Isolated", 4, 1, true, NCCL_CTA_POLICY_ZERO, - (NCCL_CE_AR_MAX_MSG_BYTES / sizeof(float)) + 4, ncclSum, ncclFloat32, false, baseEnv}, + (kCeArMaxMsgBytesDefault / sizeof(float)) + 4, ncclSum, ncclFloat32, false, baseEnv}, }; for(const auto& tc : cases) @@ -253,13 +264,14 @@ TEST(RcclCeAllReduceEligibility, RcclUseCeAllReduce_Isolated) [tc]() { CeAllReduceMockComm mock; + mock.reset(tc.archName.empty() ? nullptr : tc.archName.c_str()); mock.comm.nRanks = tc.nRanks; mock.comm.nNodes = tc.nNodes; mock.comm.symmetricSupport = tc.symmetricSupport; mock.comm.config.CTAPolicy = tc.ctaPolicy; const bool result = - rcclUseCeAllReduce(mock.get(), tc.count, tc.datatype, tc.op, /*acc=*/nullptr); + rcclUseCeAr2Shot(mock.get(), tc.count, tc.datatype, tc.op, /*acc=*/nullptr); EXPECT_EQ(result, tc.expected) << tc.name; }) .withEnvironment(env) @@ -273,5 +285,135 @@ TEST(RcclCeAllReduceEligibility, RcclUseCeAllReduce_Isolated) EXPECT_TRUE(ProcessIsolatedTestRunner::executeAllTests(options)); } + +// --------------------------------------------------------------------------- +// rcclCeAr2ShotMax / ncclCeInit staging-buffer growth. +// +// ncclCeInit (ce_coll.cc:111-114) grows ceArMaxBytes past +// NCCL_CE_AR_TMPBUF_DEFAULT_BYTES when rcclCeAr2ShotMax returns a larger +// value. Two sources can produce that: +// (a) table->ceNonRegMax[ncclFuncAllReduce] > default, read via archThresholds +// (b) RCCL_CE_AR_MAX_MSG_BYTES env var (rcclParamCeArMaxMsgBytes() >= 0) +// +// The tests below verify that the chunk-layout functions +// (ncclCeAllReduceSlotChunkBytes, ncclCeAllReduceChooseChunkBytes) remain +// correct when ceArMaxBytes is grown, and that rcclCeAr2ShotMax resolves each +// source correctly. + +static rcclArchThresholds MakeTableWithArMax(size_t arMax) +{ + rcclArchThresholds t{}; + t.ceNonRegMax[ncclFuncAllReduce] = arMax; + return t; +} + +// Table path: ceNonRegMax[AR] = 512 MiB -> rcclCeAr2ShotMax must return 512 MiB. +TEST(RcclCeAr2ShotMax, TablePathReturnsGrownValue) +{ + constexpr size_t kGrownMax = 512ull * 1024 * 1024; + rcclArchThresholds tbl = MakeTableWithArMax(kGrownMax); + + CeAllReduceMockComm mock; + mock.comm.archThresholds = &tbl; + + EXPECT_EQ(rcclCeAr2ShotMax(mock.get()), kGrownMax); +} + +// Table path at the default value: no spurious growth. +TEST(RcclCeAr2ShotMax, TablePathAtDefaultNoGrowth) +{ + rcclArchThresholds tbl = MakeTableWithArMax(NCCL_CE_AR_TMPBUF_DEFAULT_BYTES); + + CeAllReduceMockComm mock; + mock.comm.archThresholds = &tbl; + + EXPECT_EQ(rcclCeAr2ShotMax(mock.get()), NCCL_CE_AR_TMPBUF_DEFAULT_BYTES); +} + +// Null table: falls back to NCCL_CE_AR_TMPBUF_DEFAULT_BYTES. +TEST(RcclCeAr2ShotMax, NullTableFallsBackToDefault) +{ + CeAllReduceMockComm mock; + mock.comm.archThresholds = nullptr; + + EXPECT_EQ(rcclCeAr2ShotMax(mock.get()), NCCL_CE_AR_TMPBUF_DEFAULT_BYTES); +} + +// Slot-stride invariants must hold when ceArMaxBytes is grown to 512 MiB. +// Mirrors ChunkLayout_SlotStridesAgreeForAnyRankCount with the grown buffer. +TEST(RcclCeArGrownBuffer, SlotStridesAgreeForGrownCapacity) +{ + constexpr size_t kGrownMax = 512ull * 1024 * 1024; + + const std::vector rankCounts = {2, 3, 4, 5, 6, 7, 8, 12, 16, 24}; + const std::vector elementSizes = {1, 2, 4, 8}; + + for (int nRanks : rankCounts) + { + SCOPED_TRACE("nRanks=" + std::to_string(nRanks)); + const size_t slotChunkBytes = + ncclCeAllReduceSlotChunkBytes(ceAllReduceMaxChunkBytes(nRanks, kGrownMax)); + + EXPECT_EQ(slotChunkBytes % 16, 0u); + EXPECT_LE(slotChunkBytes, ceAllReduceMaxChunkBytes(nRanks, kGrownMax)); + + for (size_t eltSize : elementSizes) + { + EXPECT_EQ((slotChunkBytes / eltSize) * eltSize, slotChunkBytes) + << "eltSize=" << eltSize; + } + } +} + +// Total staging footprint (nRanks * per-rank cap) must not exceed the grown +// ceArMaxBytes -- the physical buffer ncclCeInit allocates. +TEST(RcclCeArGrownBuffer, MaxStagingBytesWithinGrownCapacity) +{ + constexpr size_t kGrownMax = 512ull * 1024 * 1024; + + for (int nRanks : {2, 3, 4, 5, 6, 7, 8, 12, 16, 24}) + { + SCOPED_TRACE("nRanks=" + std::to_string(nRanks)); + EXPECT_LE(ceAllReduceMaxChunkBytes(nRanks, kGrownMax) * static_cast(nRanks), + kGrownMax); + } +} + +// At 512 MiB a max-size message for a non-power-of-2 rank count (which stresses +// alignment) must still produce a valid pipeline layout: chunks cover the shard +// exactly and every chunk is 16-byte aligned. +// Mirrors ChunkLayout_LargeMessagePipelined with ceArMaxBytes = 512 MiB. +TEST(RcclCeArGrownBuffer, LargeMessagePipelinedWithGrownCapacity) +{ + constexpr size_t kGrownMax = 512ull * 1024 * 1024; + constexpr int nRanks = 6; + + const size_t shardElems = ceAllReduceMaxChunkBytes(nRanks, kGrownMax) / sizeof(float); + const size_t count = shardElems * nRanks; + const size_t shardBytes = shardElems * sizeof(float); + + ASSERT_LE(count * sizeof(float), kGrownMax); + + const size_t slotChunkBytes = + ncclCeAllReduceSlotChunkBytes(ceAllReduceMaxChunkBytes(nRanks, kGrownMax)); + + // Must spill past a single slot to actually exercise the pipelined path. + ASSERT_GT(shardBytes, slotChunkBytes); + + const size_t chunkBytes = ncclCeAllReduceChooseChunkBytes(shardBytes, slotChunkBytes); + const size_t baseChunkElems = chunkBytes / sizeof(float); + const size_t tailChunkElems = shardElems % baseChunkElems; + const size_t chunksPerShard = shardElems / baseChunkElems + (tailChunkElems != 0 ? 1 : 0); + const size_t lastChunkElems = tailChunkElems != 0 ? tailChunkElems : baseChunkElems; + + ASSERT_GT(chunksPerShard, 1u); + EXPECT_EQ(chunkBytes % 16, 0u); + EXPECT_LE(chunkBytes, slotChunkBytes); + + // Byte and element views of the pipeline must cover the shard exactly. + EXPECT_EQ((chunksPerShard - 1) * baseChunkElems + lastChunkElems, shardElems); + EXPECT_EQ((chunksPerShard - 1) * chunkBytes + lastChunkElems * sizeof(float), shardBytes); +} + } // namespace RcclUnitTesting diff --git a/projects/rccl/test/DdaAlltoAllThresholdTests.cpp b/projects/rccl/test/DdaAlltoAllThresholdTests.cpp index 11e1eb7139d6..46998c15a41b 100644 --- a/projects/rccl/test/DdaAlltoAllThresholdTests.cpp +++ b/projects/rccl/test/DdaAlltoAllThresholdTests.cpp @@ -18,31 +18,11 @@ class DdaAlltoAllThresholdTest : public ::testing::Test DdaAlltoAllMockComm mockComm_; }; -TEST_F(DdaAlltoAllThresholdTest, UndefinedLaunchOrderDoesNotDisableDda) -{ - mockComm_.reset("gfx950:sramecc+:xnack-"); - mockComm_.comm.config.launchOrderImplicit = NCCL_CONFIG_UNDEF_INT; - EXPECT_TRUE(testRcclDdaAlltoAllThresholdEnabled( - mockComm_.get(), kAlltoAllFloat32CountAt4MbThreshold, ncclFloat32)); -} - -TEST_F(DdaAlltoAllThresholdTest, ExplicitImplicitLaunchOrderDisablesDda) -{ - mockComm_.reset("gfx950:sramecc+:xnack-"); - mockComm_.comm.config.launchOrderImplicit = 1; - EXPECT_FALSE(testRcclDdaAlltoAllThresholdEnabled( - mockComm_.get(), kAlltoAllFloat32CountAt4MbThreshold, ncclFloat32)); -} - TEST_F(DdaAlltoAllThresholdTest, Gfx942_ExactlyAt4MbThreshold_Enabled) { mockComm_.reset("gfx942:sramecc+:xnack-"); - const size_t totalBytes = kDdaAlltoAllGfx942ThresholdBytes; - EXPECT_TRUE(rcclDdaEnabled( - mockComm_.get(), - totalBytes, - kDdaAlltoAllGfx942ThresholdBytes, - kDdaAlltoAllGfx950ThresholdBytes)); + const size_t totalBytes = rcclGetArchThresholds("gfx942")->ddaVmmMax[ncclFuncAlltoAll]; + EXPECT_TRUE(rcclDdaEnabled(mockComm_.get(), totalBytes, rcclGetArchThresholds("gfx942")->ddaVmmMax[ncclFuncAlltoAll])); EXPECT_TRUE(testRcclDdaAlltoAllThresholdEnabled( mockComm_.get(), kAlltoAllFloat32CountAt4MbThreshold, ncclFloat32)); } @@ -50,12 +30,8 @@ TEST_F(DdaAlltoAllThresholdTest, Gfx942_ExactlyAt4MbThreshold_Enabled) TEST_F(DdaAlltoAllThresholdTest, Gfx942_OneByteOverThreshold_Disabled) { mockComm_.reset("gfx942:sramecc+:xnack-"); - const size_t totalBytes = kDdaAlltoAllGfx942ThresholdBytes + 1; - EXPECT_FALSE(rcclDdaEnabled( - mockComm_.get(), - totalBytes, - kDdaAlltoAllGfx942ThresholdBytes, - kDdaAlltoAllGfx950ThresholdBytes)); + const size_t totalBytes = rcclGetArchThresholds("gfx942")->ddaVmmMax[ncclFuncAlltoAll] + 1; + EXPECT_FALSE(rcclDdaEnabled(mockComm_.get(), totalBytes, rcclGetArchThresholds("gfx942")->ddaVmmMax[ncclFuncAlltoAll])); EXPECT_FALSE(testRcclDdaAlltoAllThresholdEnabled( mockComm_.get(), kAlltoAllFloat32CountAt4MbThreshold + 1, ncclFloat32)); } @@ -63,12 +39,8 @@ TEST_F(DdaAlltoAllThresholdTest, Gfx942_OneByteOverThreshold_Disabled) TEST_F(DdaAlltoAllThresholdTest, Gfx950_ExactlyAt4MbThreshold_Enabled) { mockComm_.reset("gfx950:sramecc+:xnack-"); - const size_t totalBytes = kDdaAlltoAllGfx950ThresholdBytes; - EXPECT_TRUE(rcclDdaEnabled( - mockComm_.get(), - totalBytes, - kDdaAlltoAllGfx942ThresholdBytes, - kDdaAlltoAllGfx950ThresholdBytes)); + const size_t totalBytes = rcclGetArchThresholds("gfx950")->ddaVmmMax[ncclFuncAlltoAll]; + EXPECT_TRUE(rcclDdaEnabled(mockComm_.get(), totalBytes, rcclGetArchThresholds("gfx950")->ddaVmmMax[ncclFuncAlltoAll])); EXPECT_TRUE(testRcclDdaAlltoAllThresholdEnabled( mockComm_.get(), kAlltoAllFloat32CountAt4MbThreshold, ncclFloat32)); } @@ -76,56 +48,31 @@ TEST_F(DdaAlltoAllThresholdTest, Gfx950_ExactlyAt4MbThreshold_Enabled) TEST_F(DdaAlltoAllThresholdTest, Gfx950_AlltoAllIgnoresHighUserThreshold) { mockComm_.reset("gfx950:sramecc+:xnack-"); - const size_t overCap = kDdaAlltoAllGfx950ThresholdBytes + 1; - EXPECT_FALSE(rcclDdaEnabled( - mockComm_.get(), - overCap, - kDdaAlltoAllGfx942ThresholdBytes, - kDdaAlltoAllGfx950ThresholdBytes)); + const size_t overCap = rcclGetArchThresholds("gfx950")->ddaVmmMax[ncclFuncAlltoAll] + 1; + EXPECT_FALSE(rcclDdaEnabled(mockComm_.get(), overCap, rcclGetArchThresholds("gfx950")->ddaVmmMax[ncclFuncAlltoAll])); - // Other collectives on gfx950 still honor the user threshold when gfx950Default is 0. + // Other collectives on gfx950 keep the arch table's much larger cap. const size_t eightMb = 8 * 1024 * 1024; - EXPECT_TRUE(rcclDdaEnabled(mockComm_.get(), eightMb, 8388608, 0)); + EXPECT_TRUE(rcclDdaEnabled(mockComm_.get(), eightMb, + rcclDdaVmmThreshold(mockComm_.get(), ncclFuncAllReduce))); } -TEST_F(DdaAlltoAllThresholdTest, Gfx1250_ExactlyAt4MbThreshold_Enabled) +TEST_F(DdaAlltoAllThresholdTest, Gfx1250_ExactlyAtLL128Threshold_Enabled) { mockComm_.reset("gfx1250:sramecc+:xnack-"); - const size_t totalBytes = kDdaAlltoAllGfx1250ThresholdBytes; - EXPECT_TRUE(rcclDdaEnabled( - mockComm_.get(), - totalBytes, - kDdaAlltoAllGfx942ThresholdBytes, - kDdaAlltoAllGfx950ThresholdBytes, - kDdaAlltoAllGfx1250ThresholdBytes)); + const size_t threshold = rcclGetArchThresholds("gfx1250")->ddaLL128Max[ncclFuncAlltoAll]; + EXPECT_TRUE(rcclDdaEnabled(mockComm_.get(), threshold, threshold)); EXPECT_TRUE(testRcclDdaAlltoAllThresholdEnabled( - mockComm_.get(), kAlltoAllFloat32CountAt4MbThreshold, ncclFloat32)); + mockComm_.get(), kAlltoAllFloat32CountAt1MbLL128Threshold, ncclFloat32)); } -TEST_F(DdaAlltoAllThresholdTest, Gfx1250_OneByteOverThreshold_Disabled) +TEST_F(DdaAlltoAllThresholdTest, Gfx1250_OneByteOverLL128Threshold_Disabled) { mockComm_.reset("gfx1250:sramecc+:xnack-"); - const size_t totalBytes = kDdaAlltoAllGfx1250ThresholdBytes + 1; - EXPECT_FALSE(rcclDdaEnabled( - mockComm_.get(), - totalBytes, - kDdaAlltoAllGfx942ThresholdBytes, - kDdaAlltoAllGfx950ThresholdBytes, - kDdaAlltoAllGfx1250ThresholdBytes)); + const size_t threshold = rcclGetArchThresholds("gfx1250")->ddaLL128Max[ncclFuncAlltoAll]; + EXPECT_FALSE(rcclDdaEnabled(mockComm_.get(), threshold + 1, threshold)); EXPECT_FALSE(testRcclDdaAlltoAllThresholdEnabled( - mockComm_.get(), kAlltoAllFloat32CountAt4MbThreshold + 1, ncclFloat32)); -} - -TEST_F(DdaAlltoAllThresholdTest, Gfx1250_AlltoAllIgnoresHighUserThreshold) -{ - mockComm_.reset("gfx1250:sramecc+:xnack-"); - const size_t overCap = kDdaAlltoAllGfx1250ThresholdBytes + 1; - EXPECT_FALSE(rcclDdaEnabled( - mockComm_.get(), - overCap, - kDdaAlltoAllGfx942ThresholdBytes, - kDdaAlltoAllGfx950ThresholdBytes, - kDdaAlltoAllGfx1250ThresholdBytes)); + mockComm_.get(), kAlltoAllFloat32CountAt1MbLL128Threshold + 1, ncclFloat32)); } TEST_F(DdaAlltoAllThresholdTest, UnsupportedArch_Disabled) @@ -143,14 +90,11 @@ TEST_F(DdaAlltoAllThresholdTest, FewerThanEightRanks_Disabled) mockComm_.get(), kAlltoAllFloat32CountAt4MbThreshold, ncclFloat32)); } -TEST_F(DdaAlltoAllThresholdTest, SymmetricSupport_DoesNotDisableAlltoAll) +TEST_F(DdaAlltoAllThresholdTest, SymmetricSupport_Disabled) { - // AllReduce/AllGather DDA yields to symmetric kernels, but AlltoAll has no - // symmetric kernel (collectives.cc), so rcclDdaEnabled stays independent of - // comm->symmetricSupport. mockComm_.reset("gfx950:sramecc+:xnack-"); mockComm_.comm.symmetricSupport = 1; - EXPECT_TRUE(testRcclDdaAlltoAllThresholdEnabled( + EXPECT_FALSE(testRcclDdaAlltoAllThresholdEnabled( mockComm_.get(), kAlltoAllFloat32CountAt4MbThreshold, ncclFloat32)); } @@ -178,7 +122,7 @@ TEST_F(DdaAlltoAllThresholdTest, StagingBytesAtThresholdMatches4Mb) kAlltoAllFloat32CountAt4MbThreshold, nccl_dda_detail::kDdaNranks, sizeof(float)); - EXPECT_EQ(stagingBytes, kDdaAlltoAllGfx942ThresholdBytes); + EXPECT_EQ(stagingBytes, rcclGetArchThresholds("gfx942")->ddaVmmMax[ncclFuncAlltoAll]); } TEST(DdaAlltoAllThreshold, DdaEnableOff_Disabled) @@ -205,10 +149,11 @@ TEST(DdaAlltoAllThreshold, Gfx1250_FallbackToUserThreshold) mockComm.reset("gfx1250:sramecc+:xnack-"); const size_t eightMb = 8 * 1024 * 1024; const size_t twelveMb = 12 * 1024 * 1024; - // With RCCL_DDA_THRESHOLD=10MiB and gfx1250Default=0, 8MiB should pass. - EXPECT_TRUE(rcclDdaEnabled(mockComm.get(), eightMb, 8388608, 0, 0)); + // RCCL_DDA_THRESHOLD=10MiB overrides the arch table, so 8MiB passes. + const size_t cap = rcclDdaVmmThreshold(mockComm.get(), ncclFuncAllReduce); + EXPECT_TRUE(rcclDdaEnabled(mockComm.get(), eightMb, cap)); // But 12MiB should fail (over 10MiB threshold). - EXPECT_FALSE(rcclDdaEnabled(mockComm.get(), twelveMb, 8388608, 0, 0)); + EXPECT_FALSE(rcclDdaEnabled(mockComm.get(), twelveMb, cap)); }, {{"RCCL_DDA_THRESHOLD", "10485760"}}); // 10 MiB } @@ -223,10 +168,11 @@ TEST(DdaAlltoAllThreshold, Gfx950_FallbackToUserThreshold) mockComm.reset("gfx950:sramecc+:xnack-"); const size_t eightMb = 8 * 1024 * 1024; const size_t twelveMb = 12 * 1024 * 1024; - // With RCCL_DDA_THRESHOLD=10MiB and gfx950Default=0, 8MiB should pass. - EXPECT_TRUE(rcclDdaEnabled(mockComm.get(), eightMb, 8388608, 0)); + // RCCL_DDA_THRESHOLD=10MiB overrides the arch table, so 8MiB passes. + const size_t cap = rcclDdaVmmThreshold(mockComm.get(), ncclFuncAllReduce); + EXPECT_TRUE(rcclDdaEnabled(mockComm.get(), eightMb, cap)); // But 12MiB should fail (over 10MiB threshold). - EXPECT_FALSE(rcclDdaEnabled(mockComm.get(), twelveMb, 8388608, 0)); + EXPECT_FALSE(rcclDdaEnabled(mockComm.get(), twelveMb, cap)); }, {{"RCCL_DDA_THRESHOLD", "10485760"}}); // 10 MiB } @@ -235,28 +181,24 @@ TEST_F(DdaAlltoAllThresholdTest, Gfx1250_FourRanks_Enabled) { mockComm_.reset("gfx1250:sramecc+:xnack-"); mockComm_.comm.nRanks = 4; - const size_t totalBytes = kDdaAlltoAllGfx1250ThresholdBytes; + const size_t totalBytes = rcclGetArchThresholds("gfx1250")->ddaLLMax[ncclFuncAlltoAll]; // gfx1250 has no nRanks<8 gate, so DDA should be enabled at nRanks=4. EXPECT_TRUE(rcclDdaEnabled( mockComm_.get(), totalBytes, - kDdaAlltoAllGfx942ThresholdBytes, - kDdaAlltoAllGfx950ThresholdBytes, - kDdaAlltoAllGfx1250ThresholdBytes)); + rcclGetArchThresholds("gfx1250")->ddaLLMax[ncclFuncAlltoAll])); } TEST_F(DdaAlltoAllThresholdTest, Gfx942_FourRanks_Disabled) { mockComm_.reset("gfx942:sramecc+:xnack-"); mockComm_.comm.nRanks = 4; - const size_t totalBytes = kDdaAlltoAllGfx942ThresholdBytes; + const size_t totalBytes = rcclGetArchThresholds("gfx942")->ddaVmmMax[ncclFuncAlltoAll]; // gfx942 requires nRanks>=8, so DDA should be disabled at nRanks=4. EXPECT_FALSE(rcclDdaEnabled( mockComm_.get(), totalBytes, - kDdaAlltoAllGfx942ThresholdBytes, - kDdaAlltoAllGfx950ThresholdBytes, - kDdaAlltoAllGfx1250ThresholdBytes)); + rcclGetArchThresholds("gfx942")->ddaVmmMax[ncclFuncAlltoAll])); } } // namespace RcclUnitTesting diff --git a/projects/rccl/test/DdaFabricEligibilityTests.cpp b/projects/rccl/test/DdaFabricEligibilityTests.cpp index 12088f1e18c7..1dc48b878ffc 100644 --- a/projects/rccl/test/DdaFabricEligibilityTests.cpp +++ b/projects/rccl/test/DdaFabricEligibilityTests.cpp @@ -12,8 +12,11 @@ #include "algorithms/dda/all_reduce/dda_all_reduce.h" #include "algorithms/dda/alltoall/dda_alltoall.h" #include "algorithms/dda/reduce_scatter/dda_reduce_scatter.h" +#include "algorithms/dda/reduce_scatter/reduce_scatter_dda_fabric_ll.h" #include "algorithms/dda/fabric/fabric_gpu_barrier.h" +#include "graph.h" #include "gtest/gtest.h" +#include "rccl_common.h" namespace RcclUnitTesting { @@ -263,6 +266,41 @@ TEST(DdaFabricScratchSizingTest, LL128FloorDominatesAtHighRankCount) EXPECT_GT(sizing, (size_t)smallSimpleCap); } +TEST(DdaFabricScratchSizingTest, PayloadCapTakesMaxAcrossDdaAndCeScratchTables) +{ + rcclArchThresholds table{}; + table.ddaVmmMax[ncclFuncAllReduce] = 16ULL * 1024 * 1024; + table.ddaLL128Max[ncclFuncAllReduce] = 32ULL * 1024 * 1024; + table.ceNonRegMax[ncclFuncAllGather] = 32ULL * 1024 * 1024; + table.ddaVmmMaxGraph[ncclFuncAllReduce] = 256ULL * 1024 * 1024; + table.ddaLLMax[ncclFuncReduceScatter] = 1ULL * 1024 * 1024; + + DdaFabricMockComm mock; + mock.comm.nRanks = 4; + mock.comm.archThresholds = &table; + + // Graph VMM (256 MiB) dominates LL128/CE-Scratch (32 MiB) and RS LL + // (1 MiB/rank * 4 = 4 MiB). AR-only VMM (16 MiB) must not win. + EXPECT_EQ(rcclDdaScratchPayloadCap(mock.get()), 256ULL * 1024 * 1024); + + table.ddaVmmMaxGraph[ncclFuncAllReduce] = 0; + EXPECT_EQ(rcclDdaScratchPayloadCap(mock.get()), 32ULL * 1024 * 1024); +} + +TEST(DdaFabricScratchSizingTest, PayloadCapUsesPreTableDefaultsWhenNoArchTable) +{ + DdaFabricMockComm mock; + mock.comm.nRanks = 8; + mock.comm.archThresholds = nullptr; + mock.comm.archName = nullptr; + // Matches rcclDdaVmmThreshold() with a null table (128 MiB dominates + // the 32 KiB / 32 MiB LL defaults). + EXPECT_EQ(rcclDdaScratchPayloadCap(mock.get()), kDdaVmmBaseDefault); + EXPECT_EQ(rcclDdaVmmThreshold(mock.get(), ncclFuncAllReduce), kDdaVmmBaseDefault); + EXPECT_EQ(rcclDdaLLThreshold(mock.get(), ncclFuncAllReduce), kDdaLLBaseDefault); + EXPECT_EQ(rcclDdaLL128Threshold(mock.get(), ncclFuncAllReduce), kDdaLL128BaseDefault); +} + // --------------------------------------------------------------------------- // AllGather // --------------------------------------------------------------------------- @@ -985,6 +1023,8 @@ TEST_F(DdaFabricEligibilityTest, AllReduceLL_AtTwoShotThresholdEligible) // the rejection is attributable to the threshold and not to the shard shape rules. TEST_F(DdaFabricEligibilityTest, AllReduceLL_PastBothThresholds) { + // DDA_LL_TWOSHOT_THRESHOLD default 2 MiB; use 524320 float32 (~2.1 MiB) + // so both one-shot (1 MiB) and two-shot (2 MiB) limits are exceeded. EXPECT_FALSE(ncclAllReduceDdaFabricLLEligible( mockComm_.get(), sendbuff_, recvbuff_, 524320, ncclFloat32, ncclSum)); } @@ -1181,6 +1221,24 @@ TEST_F(DdaFabricEligibilityTest, AllReduceLL128TwoShot_HalvedSlotStillReachesFur mockComm_.get(), sendbuff_, recvbuff_, count, ncclFloat32, ncclSum)); } +// ncclAllReduceDdaFabricLL128Eligible threshold tests: at-cap and past-cap. +// These call the shape-eligibility function directly (which uses rcclParamDdaLL128OneShotThreshold / +// TwoShotThreshold, not the arch table), so they are independent of archName. +// The 32 MiB cap matches gfx1250 ddaLL128Max[AllReduce] from the arch table. +TEST_F(DdaFabricEligibilityTest, AllReduceLL128_AtCapEligible) +{ + // gfx1250 ddaLL128Max[AllReduce] = 32 MiB; 32 MiB / 4 bytes = 8388608 float32 + EXPECT_TRUE(ncclAllReduceDdaFabricLL128Eligible( + mockComm_.get(), sendbuff_, recvbuff_, 8388608, ncclFloat32, ncclSum)); +} + +TEST_F(DdaFabricEligibilityTest, AllReduceLL128_PastCapRejected) +{ + // 8388610 * 4 = 33554440 bytes = 32 MiB + 8 bytes; %8 == 0 + EXPECT_FALSE(ncclAllReduceDdaFabricLL128Eligible( + mockComm_.get(), sendbuff_, recvbuff_, 8388610, ncclFloat32, ncclSum)); +} + // --------------------------------------------------------------------------- // AllToAll // --------------------------------------------------------------------------- @@ -1432,7 +1490,8 @@ TEST_F(DdaFabricLL128EligibilityTest, AllGatherLL128_ScratchTooSmallForOneSlice) TEST_F(DdaFabricLL128EligibilityTest, AllGatherLL128_AtThresholdEligible) { - constexpr size_t totalCap = 64u * 1024u * 1024u; // DDA_LL128_THRESHOLD default + // fixture sets archName="gfx1250"; resolved threshold = ddaLL128Max[AG] = 64 MiB + constexpr size_t totalCap = 64u * 1024u * 1024u; const size_t perRankBytes = totalCap / (size_t)mockComm_.comm.nRanks; EXPECT_TRUE(ncclAllGatherDdaFabricLL128Eligible( mockComm_.get(), sendbuff_, recvbuff_, perRankBytes / sizeof(float), ncclFloat32)); @@ -1446,4 +1505,30 @@ TEST_F(DdaFabricLL128EligibilityTest, AllGatherLL128_PastThresholdRejected) mockComm_.get(), sendbuff_, recvbuff_, perRankBytes / sizeof(float), ncclFloat32)); } + +// ReduceScatter LL +// +// kDdaLLRsMaxBytes (1 MiB per-rank) replaced the old `bytes * 2 > kDdaLLMaxBytes` +// rule. ncclReduceScatterDdaFabricLLEligible had no test callers at all, so +// neither side of the boundary was pinned. + +TEST_F(DdaFabricEligibilityTest, ReduceScatterLL_AtCapEligible) +{ + // Per-rank shard exactly at kDdaLLRsMaxBytes must be accepted. + // kDdaLLRsMaxBytes is a multiple of 16, so the alignment gate passes. + const size_t perRankBytes = dda::common::kDdaLLRsMaxBytes; + const size_t recvcount = perRankBytes / sizeof(float); + EXPECT_TRUE(ncclReduceScatterDdaFabricLLEligible( + mockComm_.get(), sendbuff_, recvbuff_, recvcount, ncclFloat32, ncclSum)); +} + +TEST_F(DdaFabricEligibilityTest, ReduceScatterLL_PastCapRejected) +{ + // One 16-byte step above kDdaLLRsMaxBytes must be rejected. + const size_t perRankBytes = dda::common::kDdaLLRsMaxBytes + 16; + const size_t recvcount = perRankBytes / sizeof(float); + EXPECT_FALSE(ncclReduceScatterDdaFabricLLEligible( + mockComm_.get(), sendbuff_, recvbuff_, recvcount, ncclFloat32, ncclSum)); +} + } // namespace RcclUnitTesting diff --git a/projects/rccl/test/DdaIpcEligibilityTests.cpp b/projects/rccl/test/DdaIpcEligibilityTests.cpp index 12c3ae52a627..62887464e7eb 100644 --- a/projects/rccl/test/DdaIpcEligibilityTests.cpp +++ b/projects/rccl/test/DdaIpcEligibilityTests.cpp @@ -144,7 +144,7 @@ TEST_F(DdaIpcEligibilityTest, AllToAll_StagingBytesAtThresholdFitsScratch) kAlltoAllFloat32CountAt4MbThreshold, mockComm_.comm.nRanks, sizeof(float)); - EXPECT_EQ(stagingBytes, kDdaAlltoAllGfx950ThresholdBytes); + EXPECT_EQ(stagingBytes, rcclGetArchThresholds("gfx950")->ddaVmmMax[ncclFuncAlltoAll]); EXPECT_LE(stagingBytes, mockComm_.comm.ddaScratchBytes); } @@ -154,7 +154,7 @@ TEST_F(DdaIpcEligibilityTest, AllToAll_StagingBytesOneCountOverThresholdStillEli const size_t count = kAlltoAllFloat32CountAt4MbThreshold + 4; const size_t stagingBytes = testAlltoAllDdaIpcStagingBytes( count, mockComm_.comm.nRanks, sizeof(float)); - EXPECT_GT(stagingBytes, kDdaAlltoAllGfx950ThresholdBytes); + EXPECT_GT(stagingBytes, rcclGetArchThresholds("gfx950")->ddaVmmMax[ncclFuncAlltoAll]); EXPECT_TRUE(ncclAllToAllDdaIpcEligible( mockComm_.get(), sendbuff_, recvbuff_, count, ncclFloat32)); } diff --git a/projects/rccl/test/DevRuntimeTestsStubs.cc b/projects/rccl/test/DevRuntimeTestsStubs.cc index c163621dcd5b..bf6777ec81e7 100644 --- a/projects/rccl/test/DevRuntimeTestsStubs.cc +++ b/projects/rccl/test/DevRuntimeTestsStubs.cc @@ -111,6 +111,10 @@ int64_t ncclLoadParam(char const*, int64_t deftVal, int64_t, int64_t* cache, int } int64_t ncclParamEnqueueRearchEnable() { return 0; } +// Emitted by init.cc in the real build; dev_runtime.cc only declares it extern. +// Returns the NCCL_PARAM default so window registration is not opted out. +int64_t ncclParamWinEnable() { return 1; } + // --------------------------------------------------------------------------- // Proxy / mgmt task enqueue. // --------------------------------------------------------------------------- diff --git a/projects/rccl/test/RcclWrapTests.cpp b/projects/rccl/test/RcclWrapTests.cpp index eceef57e1e1d..4cff9c2363a6 100644 --- a/projects/rccl/test/RcclWrapTests.cpp +++ b/projects/rccl/test/RcclWrapTests.cpp @@ -326,10 +326,10 @@ TEST(Rcclwrap, RcclUpdateThreadThreshold_UserEnvSet) } else { - ncclComm comm; - comm.nRanks = 8; - comm.nNodes = 4; - memset(comm.minMaxLLRange, 0, sizeof(comm.minMaxLLRange)); + auto comm = std::make_unique(); + comm->nRanks = 8; + comm->nNodes = 4; + memset(comm->minMaxLLRange, 0, sizeof(comm->minMaxLLRange)); ncclTaskColl info; info.func = ncclFuncReduceScatter; @@ -337,7 +337,7 @@ TEST(Rcclwrap, RcclUpdateThreadThreshold_UserEnvSet) int threadThreshold = 5; // Any number should do, we should make // sure this number does not change - rcclUpdateThreadThreshold(&comm, 0, &info, threadThreshold); + rcclUpdateThreadThreshold(comm.get(), 0, &info, threadThreshold); EXPECT_EQ(threadThreshold, 5); } @@ -364,17 +364,17 @@ TEST(Rcclwrap, RcclUpdateThreadThreshold_MinNChannelsSet) } else { - ncclComm comm{}; + auto comm = std::make_unique(); ncclTaskColl info{}; int threadThreshold = 5; - comm.nRanks = 4; - comm.nNodes = 4; + comm->nRanks = 4; + comm->nNodes = 4; info.func = ncclFuncAllGather; info.protocol = 0; - memset(comm.minMaxLLRange, 0, sizeof(comm.minMaxLLRange)); + memset(comm->minMaxLLRange, 0, sizeof(comm->minMaxLLRange)); - rcclUpdateThreadThreshold(&comm, 0, &info, threadThreshold); + rcclUpdateThreadThreshold(comm.get(), 0, &info, threadThreshold); EXPECT_EQ(threadThreshold, 5); } @@ -401,17 +401,17 @@ TEST(Rcclwrap, RcclUpdateThreadThreshold_MaxChannelsSet) } else { - ncclComm comm{}; + auto comm = std::make_unique(); ncclTaskColl info{}; int threadThreshold = 5; - comm.nRanks = 4; - comm.nNodes = 4; + comm->nRanks = 4; + comm->nNodes = 4; info.func = ncclFuncAllGather; info.protocol = 0; - memset(comm.minMaxLLRange, 0, sizeof(comm.minMaxLLRange)); + memset(comm->minMaxLLRange, 0, sizeof(comm->minMaxLLRange)); - rcclUpdateThreadThreshold(&comm, 0, &info, threadThreshold); + rcclUpdateThreadThreshold(comm.get(), 0, &info, threadThreshold); EXPECT_EQ(threadThreshold, 5); } @@ -422,75 +422,75 @@ TEST(Rcclwrap, RcclUpdateThreadThreshold_MaxChannelsSet) TEST(Rcclwrap, RcclUpdateThreadThreshold_NoEnv_nNodesLessThan2) { - ncclComm comm{}; + auto comm = std::make_unique(); ncclTaskColl info{}; int threadThreshold = 5; - comm.nRanks = 4; - comm.nNodes = 1; // less than 2 + comm->nRanks = 4; + comm->nNodes = 1; // less than 2 info.func = ncclFuncReduceScatter; info.protocol = 0; - memset(comm.minMaxLLRange, 0, sizeof(comm.minMaxLLRange)); + memset(comm->minMaxLLRange, 0, sizeof(comm->minMaxLLRange)); - rcclUpdateThreadThreshold(&comm, 0, &info, threadThreshold); + rcclUpdateThreadThreshold(comm.get(), 0, &info, threadThreshold); EXPECT_EQ(threadThreshold, 5); // no change } TEST(Rcclwrap, RcclUpdateThreadThreshold_NoEnv_FuncUnsupported) { - ncclComm comm{}; + auto comm = std::make_unique(); ncclTaskColl info{}; int threadThreshold = 5; - comm.nRanks = 4; - comm.nNodes = 2; + comm->nRanks = 4; + comm->nNodes = 2; info.func = ncclFuncAllReduce; // unsupported func info.protocol = 0; - memset(comm.minMaxLLRange, 0, sizeof(comm.minMaxLLRange)); + memset(comm->minMaxLLRange, 0, sizeof(comm->minMaxLLRange)); - rcclUpdateThreadThreshold(&comm, 0, &info, threadThreshold); + rcclUpdateThreadThreshold(comm.get(), 0, &info, threadThreshold); EXPECT_EQ(threadThreshold, 5); } TEST(Rcclwrap, RcclUpdateThreadThreshold_NoEnv_UpdateOccurs) { - ncclComm comm{}; + auto comm = std::make_unique(); ncclTaskColl info{}; int threadThreshold = 5; - comm.nRanks = 4; - comm.nNodes = 2; + comm->nRanks = 4; + comm->nNodes = 2; info.func = ncclFuncReduceScatter; info.protocol = 0; - memset(comm.minMaxLLRange, 0, sizeof(comm.minMaxLLRange)); + memset(comm->minMaxLLRange, 0, sizeof(comm->minMaxLLRange)); int idx = rcclGetTunableIndex(info.func); - comm.minMaxLLRange[idx][info.protocol][RCCL_PROTOCOL_THREAD_THRESHOLD_IDX] = 10; + comm->minMaxLLRange[idx][info.protocol][RCCL_PROTOCOL_THREAD_THRESHOLD_IDX] = 10; - rcclUpdateThreadThreshold(&comm, 0, &info, threadThreshold); + rcclUpdateThreadThreshold(comm.get(), 0, &info, threadThreshold); EXPECT_EQ(threadThreshold, 40); // 10 * 4 } TEST(Rcclwrap, RcclUpdateThreadThreshold_NoEnv_ThresholdUndefined) { - ncclComm comm{}; + auto comm = std::make_unique(); ncclTaskColl info{}; int threadThreshold = 5; - comm.nRanks = 4; - comm.nNodes = 3; + comm->nRanks = 4; + comm->nNodes = 3; info.func = ncclFuncAllGather; info.protocol = 0; - memset(comm.minMaxLLRange, 0, sizeof(comm.minMaxLLRange)); + memset(comm->minMaxLLRange, 0, sizeof(comm->minMaxLLRange)); int idx = rcclGetTunableIndex(info.func); - comm.minMaxLLRange[idx][info.protocol][RCCL_PROTOCOL_THREAD_THRESHOLD_IDX] + comm->minMaxLLRange[idx][info.protocol][RCCL_PROTOCOL_THREAD_THRESHOLD_IDX] = RCCL_LL_LIMITS_UNDEFINED; - rcclUpdateThreadThreshold(&comm, 0, &info, threadThreshold); + rcclUpdateThreadThreshold(comm.get(), 0, &info, threadThreshold); EXPECT_EQ(threadThreshold, 5); } @@ -1905,26 +1905,26 @@ TEST(Rcclwrap, GetAlgoInfo_LlOverridePreservedWithoutDirectRs) TEST(RcclCeGraphLatch, SetsLatchOnFirstCapture) { - ncclComm comm{}; - comm.ceColl.graphModeSeen = false; - comm.localPersistentRefs = 0; + auto comm = std::make_unique(); + comm->ceColl.graphModeSeen = false; + comm->localPersistentRefs = 0; - rcclCeAllReduceGraphLatchTick(&comm, /*ceCapturing=*/true); + rcclCeAllReduceGraphLatchTick(comm.get(), /*ceCapturing=*/true); - EXPECT_TRUE(comm.ceColl.graphModeSeen); - EXPECT_FALSE(rcclCeAllReduceAllowed(&comm)); + EXPECT_TRUE(comm->ceColl.graphModeSeen); + EXPECT_FALSE(rcclCeArGraphSafe(comm.get())); } TEST(RcclCeGraphLatch, StaysSetAcrossRepeatedCaptureTicks) { - ncclComm comm{}; - comm.ceColl.graphModeSeen = false; - comm.localPersistentRefs = 0; + auto comm = std::make_unique(); + comm->ceColl.graphModeSeen = false; + comm->localPersistentRefs = 0; for(int i = 0; i < 3; ++i) { - rcclCeAllReduceGraphLatchTick(&comm, /*ceCapturing=*/true); - EXPECT_TRUE(comm.ceColl.graphModeSeen) << "iteration " << i; + rcclCeAllReduceGraphLatchTick(comm.get(), /*ceCapturing=*/true); + EXPECT_TRUE(comm->ceColl.graphModeSeen) << "iteration " << i; } } @@ -1932,65 +1932,65 @@ TEST(RcclCeGraphLatch, StaysSetAcrossRepeatedCaptureTicks) // CE AR while still capturing -- this previously caused a cross-rank deadlock. TEST(RcclCeGraphLatch, DoesNotClearMidCaptureEvenWithZeroRefs) { - ncclComm comm{}; - comm.ceColl.graphModeSeen = true; - comm.localPersistentRefs = 0; + auto comm = std::make_unique(); + comm->ceColl.graphModeSeen = true; + comm->localPersistentRefs = 0; - rcclCeAllReduceGraphLatchTick(&comm, /*ceCapturing=*/true); + rcclCeAllReduceGraphLatchTick(comm.get(), /*ceCapturing=*/true); - EXPECT_TRUE(comm.ceColl.graphModeSeen); - EXPECT_FALSE(rcclCeAllReduceAllowed(&comm)); + EXPECT_TRUE(comm->ceColl.graphModeSeen); + EXPECT_FALSE(rcclCeArGraphSafe(comm.get())); } TEST(RcclCeGraphLatch, ClearsWhenCaptureEndsAndNoRefsRemain) { - ncclComm comm{}; - comm.ceColl.graphModeSeen = true; - comm.localPersistentRefs = 0; + auto comm = std::make_unique(); + comm->ceColl.graphModeSeen = true; + comm->localPersistentRefs = 0; - rcclCeAllReduceGraphLatchTick(&comm, /*ceCapturing=*/false); + rcclCeAllReduceGraphLatchTick(comm.get(), /*ceCapturing=*/false); - EXPECT_FALSE(comm.ceColl.graphModeSeen); - EXPECT_TRUE(rcclCeAllReduceAllowed(&comm)); + EXPECT_FALSE(comm->ceColl.graphModeSeen); + EXPECT_TRUE(rcclCeArGraphSafe(comm.get())); } TEST(RcclCeGraphLatch, StaysSetWhileCapturedPlanStillReferencesComm) { - ncclComm comm{}; - comm.ceColl.graphModeSeen = true; - comm.localPersistentRefs = 1; + auto comm = std::make_unique(); + comm->ceColl.graphModeSeen = true; + comm->localPersistentRefs = 1; - rcclCeAllReduceGraphLatchTick(&comm, /*ceCapturing=*/false); + rcclCeAllReduceGraphLatchTick(comm.get(), /*ceCapturing=*/false); - EXPECT_TRUE(comm.ceColl.graphModeSeen) + EXPECT_TRUE(comm->ceColl.graphModeSeen) << "Latch must stay set until every captured plan is reclaimed"; - EXPECT_FALSE(rcclCeAllReduceAllowed(&comm)); + EXPECT_FALSE(rcclCeArGraphSafe(comm.get())); // Reclaim completes: the next tick clears the latch. - comm.localPersistentRefs = 0; - rcclCeAllReduceGraphLatchTick(&comm, /*ceCapturing=*/false); - EXPECT_FALSE(comm.ceColl.graphModeSeen); - EXPECT_TRUE(rcclCeAllReduceAllowed(&comm)); + comm->localPersistentRefs = 0; + rcclCeAllReduceGraphLatchTick(comm.get(), /*ceCapturing=*/false); + EXPECT_FALSE(comm->ceColl.graphModeSeen); + EXPECT_TRUE(rcclCeArGraphSafe(comm.get())); } TEST(RcclCeGraphLatch, AllowedQueryHasNoSideEffects) { - ncclComm comm{}; - comm.ceColl.graphModeSeen = true; + auto comm = std::make_unique(); + comm->ceColl.graphModeSeen = true; for(int i = 0; i < 5; ++i) { - EXPECT_FALSE(rcclCeAllReduceAllowed(&comm)); + EXPECT_FALSE(rcclCeArGraphSafe(comm.get())); } - EXPECT_TRUE(comm.ceColl.graphModeSeen) << "Query must be a pure read, not a state transition"; + EXPECT_TRUE(comm->ceColl.graphModeSeen) << "Query must be a pure read, not a state transition"; } TEST(RcclCeGraphLatch, NeverLatchedAllowsCeAllReduceByDefault) { - ncclComm comm{}; - comm.ceColl.graphModeSeen = false; + auto comm = std::make_unique(); + comm->ceColl.graphModeSeen = false; - EXPECT_TRUE(rcclCeAllReduceAllowed(&comm)); + EXPECT_TRUE(rcclCeArGraphSafe(comm.get())); } #ifdef ENABLE_WARP_SPEED @@ -2425,7 +2425,7 @@ static void CreateCeAllReduceEligibleComm( // A count of float32 divisible by the 8 mock ranks; well within NCCL_CE_AR_MAX_MSG_BYTES. static constexpr size_t kCeAllReduceCount = 4096; -// rcclUseCeAllReduce gates the Copy Engine AllReduce path. The CE kernels never +// rcclUseCeAr2Shot gates the Copy Engine AllReduce path. The CE kernels never // read the bias buffer, so an eligible-looking ncclAllReduceWithBias call must // still be refused; taking CE there silently drops the bias from the result. TEST(Rcclwrap, RcclUseCeAllReduce_BiasBuffer) @@ -2441,7 +2441,7 @@ TEST(Rcclwrap, RcclUseCeAllReduce_BiasBuffer) CreateCeAllReduceEligibleComm(comm, *topo, gpu, /*nRanks=*/8); int biasBuffer = 0; - EXPECT_FALSE(rcclUseCeAllReduce(comm, + EXPECT_FALSE(rcclUseCeAr2Shot(comm, kCeAllReduceCount, ncclFloat32, ncclSum, @@ -2464,7 +2464,7 @@ TEST(Rcclwrap, RcclUseCeAllReduce_BiasBuffer) struct ncclTopoNode gpu; CreateCeAllReduceEligibleComm(comm, *topo, gpu, /*nRanks=*/8); - EXPECT_TRUE(rcclUseCeAllReduce(comm, + EXPECT_TRUE(rcclUseCeAr2Shot(comm, kCeAllReduceCount, ncclFloat32, ncclSum, @@ -2487,7 +2487,7 @@ TEST(Rcclwrap, RcclUseCeAllReduce_BiasBuffer) // enabled for this arch and size. // // `ceAllReduceAllowed` is passed in directly rather than derived inside the -// test: the call site computes it once from rcclUseCeAllReduce() plus whatever +// test: the call site computes it once from rcclUseCeAr2Shot() plus whatever // additional gating CE AllReduce requires (graph latch, ncclGroupDepth, // force/symReg, op/dtype/size/divisibility support, etc). Driving it directly // keeps these cases independent of RCCL_CE_ALLREDUCE's default and of CE @@ -2519,75 +2519,75 @@ size_t CountForBytes(size_t bytes, ncclDataType_t dt) // 4 MiB CE minimum) must still take DDA rather than fall to the generic kernel. TEST(RcclAllReduceDdaDecision, Gfx950_SymOff_LargeMsg_TakesDda) { - ncclComm comm{}; - InitDdaDecisionComm(comm, "gfx950", 8, 1, /*symmetricSupport=*/false); + auto comm = std::make_unique(); + InitDdaDecisionComm(*comm, "gfx950", 8, 1, /*symmetricSupport=*/false); size_t count = CountForBytes(8ull * 1024 * 1024, ncclFloat32); - EXPECT_TRUE(rcclAllReduceShouldTakeDdaPath(&comm, count, ncclFloat32, - /*symEligible=*/false, /*ceAllReduceAllowed=*/false)); + EXPECT_TRUE(rcclAllReduceShouldTakeDdaPath(comm.get(), count, ncclFloat32, + /*symEligible=*/false, /*ceAllReduceAllowed=*/false, /*query=*/false)); } // gfx950, small message with CE unavailable: squarely in DDA's range, takes DDA. TEST(RcclAllReduceDdaDecision, Gfx950_SymOff_SmallMsg_TakesDda) { - ncclComm comm{}; - InitDdaDecisionComm(comm, "gfx950", 8, 1, /*symmetricSupport=*/false); + auto comm = std::make_unique(); + InitDdaDecisionComm(*comm, "gfx950", 8, 1, /*symmetricSupport=*/false); size_t count = CountForBytes(2ull * 1024 * 1024, ncclFloat32); - EXPECT_TRUE(rcclAllReduceShouldTakeDdaPath(&comm, count, ncclFloat32, - /*symEligible=*/false, /*ceAllReduceAllowed=*/false)); + EXPECT_TRUE(rcclAllReduceShouldTakeDdaPath(comm.get(), count, ncclFloat32, + /*symEligible=*/false, /*ceAllReduceAllowed=*/false, /*query=*/false)); } // gfx950 with symmetricSupport on and every CE prerequisite met: CE will service // the call, so the DDA guard must yield (returns false). TEST(RcclAllReduceDdaDecision, Gfx950_SymOn_CeEligible_YieldsToCe) { - ncclComm comm{}; - InitDdaDecisionComm(comm, "gfx950", 8, 1, /*symmetricSupport=*/true); + auto comm = std::make_unique(); + InitDdaDecisionComm(*comm, "gfx950", 8, 1, /*symmetricSupport=*/true); size_t count = CountForBytes(8ull * 1024 * 1024, ncclFloat32); // divisible by 8 ranks - EXPECT_FALSE(rcclAllReduceShouldTakeDdaPath(&comm, count, ncclFloat32, - /*symEligible=*/false, /*ceAllReduceAllowed=*/true)); + EXPECT_FALSE(rcclAllReduceShouldTakeDdaPath(comm.get(), count, ncclFloat32, + /*symEligible=*/false, /*ceAllReduceAllowed=*/true, /*query=*/false)); } // CE eligible by size/op/dtype but disabled by the graph latch (folded into the // caller's ceAllReduceAllowed=false): CE will not run, so DDA must reclaim the call. TEST(RcclAllReduceDdaDecision, Gfx950_SymOn_GraphLatched_TakesDda) { - ncclComm comm{}; - InitDdaDecisionComm(comm, "gfx950", 8, 1, /*symmetricSupport=*/true); + auto comm = std::make_unique(); + InitDdaDecisionComm(*comm, "gfx950", 8, 1, /*symmetricSupport=*/true); size_t count = CountForBytes(8ull * 1024 * 1024, ncclFloat32); - EXPECT_TRUE(rcclAllReduceShouldTakeDdaPath(&comm, count, ncclFloat32, - /*symEligible=*/false, /*ceAllReduceAllowed=*/false)); + EXPECT_TRUE(rcclAllReduceShouldTakeDdaPath(comm.get(), count, ncclFloat32, + /*symEligible=*/false, /*ceAllReduceAllowed=*/false, /*query=*/false)); } // CE declines on an unsupported op (folded into ceAllReduceAllowed=false) even // with symmetricSupport on, so DDA reclaims the call. TEST(RcclAllReduceDdaDecision, Gfx950_SymOn_UnsupportedOp_TakesDda) { - ncclComm comm{}; - InitDdaDecisionComm(comm, "gfx950", 8, 1, /*symmetricSupport=*/true); + auto comm = std::make_unique(); + InitDdaDecisionComm(*comm, "gfx950", 8, 1, /*symmetricSupport=*/true); size_t count = CountForBytes(8ull * 1024 * 1024, ncclFloat32); - EXPECT_TRUE(rcclAllReduceShouldTakeDdaPath(&comm, count, ncclFloat32, - /*symEligible=*/false, /*ceAllReduceAllowed=*/false)); + EXPECT_TRUE(rcclAllReduceShouldTakeDdaPath(comm.get(), count, ncclFloat32, + /*symEligible=*/false, /*ceAllReduceAllowed=*/false, /*query=*/false)); } // gfx942 with symmetricSupport off: a 6 MiB call is within the 8 MiB gfx942 DDA // cap and, with CE unavailable, takes DDA. TEST(RcclAllReduceDdaDecision, Gfx942_SymOff_MidMsg_TakesDda) { - ncclComm comm{}; - InitDdaDecisionComm(comm, "gfx942", 8, 1, /*symmetricSupport=*/false); + auto comm = std::make_unique(); + InitDdaDecisionComm(*comm, "gfx942", 8, 1, /*symmetricSupport=*/false); size_t count = CountForBytes(6ull * 1024 * 1024, ncclFloat32); - EXPECT_TRUE(rcclAllReduceShouldTakeDdaPath(&comm, count, ncclFloat32, - /*symEligible=*/false, /*ceAllReduceAllowed=*/false)); + EXPECT_TRUE(rcclAllReduceShouldTakeDdaPath(comm.get(), count, ncclFloat32, + /*symEligible=*/false, /*ceAllReduceAllowed=*/false, /*query=*/false)); } // gfx942 above its 8 MiB DDA cap: rcclDdaEnabled returns false, so no DDA. TEST(RcclAllReduceDdaDecision, Gfx942_SymOff_AboveCap_NoDda) { - ncclComm comm{}; - InitDdaDecisionComm(comm, "gfx942", 8, 1, /*symmetricSupport=*/false); + auto comm = std::make_unique(); + InitDdaDecisionComm(*comm, "gfx942", 8, 1, /*symmetricSupport=*/false); size_t count = CountForBytes(9ull * 1024 * 1024, ncclFloat32); - EXPECT_FALSE(rcclAllReduceShouldTakeDdaPath(&comm, count, ncclFloat32, - /*symEligible=*/false, /*ceAllReduceAllowed=*/false)); + EXPECT_FALSE(rcclAllReduceShouldTakeDdaPath(comm.get(), count, ncclFloat32, + /*symEligible=*/false, /*ceAllReduceAllowed=*/false, /*query=*/false)); } // gfx942 with symmetricSupport on and every CE prerequisite met: CE claims the call @@ -2595,11 +2595,11 @@ TEST(RcclAllReduceDdaDecision, Gfx942_SymOff_AboveCap_NoDda) // Mirror of Gfx942_SymOff_MidMsg_TakesDda: ceAllReduceAllowed flips the decision. TEST(RcclAllReduceDdaDecision, Gfx942_SymOn_CeEligible_YieldsToCe) { - ncclComm comm{}; - InitDdaDecisionComm(comm, "gfx942", 8, 1, /*symmetricSupport=*/true); + auto comm = std::make_unique(); + InitDdaDecisionComm(*comm, "gfx942", 8, 1, /*symmetricSupport=*/true); size_t count = CountForBytes(6ull * 1024 * 1024, ncclFloat32); // divisible by 8 ranks - EXPECT_FALSE(rcclAllReduceShouldTakeDdaPath(&comm, count, ncclFloat32, - /*symEligible=*/false, /*ceAllReduceAllowed=*/true)); + EXPECT_FALSE(rcclAllReduceShouldTakeDdaPath(comm.get(), count, ncclFloat32, + /*symEligible=*/false, /*ceAllReduceAllowed=*/true, /*query=*/false)); } // gfx942 with symmetricSupport on but CE declines on an unsupported op (folded @@ -2607,11 +2607,11 @@ TEST(RcclAllReduceDdaDecision, Gfx942_SymOn_CeEligible_YieldsToCe) // the 8 MiB gfx942 cap. TEST(RcclAllReduceDdaDecision, Gfx942_SymOn_UnsupportedOp_TakesDda) { - ncclComm comm{}; - InitDdaDecisionComm(comm, "gfx942", 8, 1, /*symmetricSupport=*/true); + auto comm = std::make_unique(); + InitDdaDecisionComm(*comm, "gfx942", 8, 1, /*symmetricSupport=*/true); size_t count = CountForBytes(6ull * 1024 * 1024, ncclFloat32); - EXPECT_TRUE(rcclAllReduceShouldTakeDdaPath(&comm, count, ncclFloat32, - /*symEligible=*/false, /*ceAllReduceAllowed=*/false)); + EXPECT_TRUE(rcclAllReduceShouldTakeDdaPath(comm.get(), count, ncclFloat32, + /*symEligible=*/false, /*ceAllReduceAllowed=*/false, /*query=*/false)); } // gfx1250 forces the DDA fabric path regardless of CE eligibility: the @@ -2620,21 +2620,21 @@ TEST(RcclAllReduceDdaDecision, Gfx942_SymOn_UnsupportedOp_TakesDda) // divisible), so the short-circuit is the only reason DDA is chosen here. TEST(RcclAllReduceDdaDecision, Gfx1250_CeEligible_StillTakesDda) { - ncclComm comm{}; - InitDdaDecisionComm(comm, "gfx1250", 8, 1, /*symmetricSupport=*/true); - size_t count = CountForBytes(64ull * 1024 * 1024, ncclFloat32); // CE-eligible size, divisible by 8 - EXPECT_TRUE(rcclAllReduceShouldTakeDdaPath(&comm, count, ncclFloat32, - /*symEligible=*/false, /*ceAllReduceAllowed=*/true)); + auto comm = std::make_unique(); + InitDdaDecisionComm(*comm, "gfx1250", 8, 1, /*symmetricSupport=*/true); + size_t count = CountForBytes(16ull * 1024 * 1024, ncclFloat32); // CE-eligible (4-256 MiB window) and within DDA entry cap (32 MiB), divisible by 8 + EXPECT_TRUE(rcclAllReduceShouldTakeDdaPath(comm.get(), count, ncclFloat32, + /*symEligible=*/false, /*ceAllReduceAllowed=*/true, /*query=*/false)); } // An arch DDA never runs on: rcclDdaEnabled returns false, so no DDA on any size. TEST(RcclAllReduceDdaDecision, UnsupportedArch_NoDda) { - ncclComm comm{}; - InitDdaDecisionComm(comm, "gfx90a", 8, 1, /*symmetricSupport=*/false); + auto comm = std::make_unique(); + InitDdaDecisionComm(*comm, "gfx90a", 8, 1, /*symmetricSupport=*/false); size_t count = CountForBytes(2ull * 1024 * 1024, ncclFloat32); - EXPECT_FALSE(rcclAllReduceShouldTakeDdaPath(&comm, count, ncclFloat32, - /*symEligible=*/false, /*ceAllReduceAllowed=*/false)); + EXPECT_FALSE(rcclAllReduceShouldTakeDdaPath(comm.get(), count, ncclFloat32, + /*symEligible=*/false, /*ceAllReduceAllowed=*/false, /*query=*/false)); } // --------------------------------------------------------------------------- @@ -2674,7 +2674,7 @@ TEST(Rcclwrap, AlltoAllDdaDecision_Gfx1250_AboveThreshold_NoDda) { ncclComm comm{}; InitDdaDecisionComm(comm, "gfx1250", 4, 1, /*symmetricSupport=*/true); - size_t totalBytes = kDdaAlltoAllGfx1250ThresholdBytes + 1; + size_t totalBytes = rcclDdaEntryThreshold(&comm, ncclFuncAlltoAll) + 1; EXPECT_FALSE(rcclAlltoAllShouldTakeDdaPath(&comm, totalBytes, /*ceAlltoAllAllowed=*/false)); } @@ -2689,22 +2689,22 @@ TEST(Rcclwrap, AlltoAllDdaDecision_Gfx950_TooFewRanks_NoDda) // gfx942/gfx950 DDA requires the full 8-GPU node; fewer ranks disables it. TEST(RcclAllReduceDdaDecision, Gfx950_TooFewRanks_NoDda) { - ncclComm comm{}; - InitDdaDecisionComm(comm, "gfx950", 4, 1, /*symmetricSupport=*/false); + auto comm = std::make_unique(); + InitDdaDecisionComm(*comm, "gfx950", 4, 1, /*symmetricSupport=*/false); size_t count = CountForBytes(2ull * 1024 * 1024, ncclFloat32); - EXPECT_FALSE(rcclAllReduceShouldTakeDdaPath(&comm, count, ncclFloat32, - /*symEligible=*/false, /*ceAllReduceAllowed=*/false)); + EXPECT_FALSE(rcclAllReduceShouldTakeDdaPath(comm.get(), count, ncclFloat32, + /*symEligible=*/false, /*ceAllReduceAllowed=*/false, /*query=*/false)); } // Symmetric-kernel eligible buffers win outright: the DDA guard yields (returns false) // regardless of arch/size. TEST(RcclAllReduceDdaDecision, SymEligible_YieldsToSymmetricKernel) { - ncclComm comm{}; - InitDdaDecisionComm(comm, "gfx950", 8, 1, /*symmetricSupport=*/false); + auto comm = std::make_unique(); + InitDdaDecisionComm(*comm, "gfx950", 8, 1, /*symmetricSupport=*/false); size_t count = CountForBytes(2ull * 1024 * 1024, ncclFloat32); - EXPECT_FALSE(rcclAllReduceShouldTakeDdaPath(&comm, count, ncclFloat32, - /*symEligible=*/true, /*ceAllReduceAllowed=*/true)); + EXPECT_FALSE(rcclAllReduceShouldTakeDdaPath(comm.get(), count, ncclFloat32, + /*symEligible=*/true, /*ceAllReduceAllowed=*/true, /*query=*/false)); } // --------------------------------------------------------------------------- @@ -2863,4 +2863,362 @@ TEST(Rcclwrap, UnrollFactor_AcceptsUsableUnroll) ); } +// Per-tier DDA caps: the arch table is the default, RCCL_DDA_* is the override. +// Isolated because the env lookups behind these accessors are cached per process. +// Pins the concrete gfx1250 AllReduce thresholds so that a silent zero-out +// of the table row is caught. Values must match rcclArchThresholds_gfx1250 +// in tuning.cc. +TEST(RcclDdaTierThresholds, Gfx1250_NoEnv_UsesArchTable) +{ + RUN_ISOLATED_TEST_WITH_ENV( + "Gfx1250_NoEnv_UsesArchTable", + []() + { + auto comm = std::make_unique(); + InitDdaDecisionComm(*comm, "gfx1250", 8, 1, /*symmetricSupport=*/false); + comm->archThresholds = rcclGetArchThresholds("gfx1250"); + ASSERT_NE(comm->archThresholds, nullptr); + // LL and LL128 ceilings for AR are both 32 MiB on gfx1250; + // VMM ceiling is 16 MiB (above this, Ring/CE takes over). + EXPECT_EQ(rcclDdaLLThreshold(comm.get(), ncclFuncAllReduce), + 32ULL * 1024 * 1024); + EXPECT_EQ(rcclDdaLL128Threshold(comm.get(), ncclFuncAllReduce), + 32ULL * 1024 * 1024); + EXPECT_EQ(rcclDdaVmmThreshold(comm.get(), ncclFuncAllReduce), + 16ULL * 1024 * 1024); + }, + {}); +} + +TEST(RcclDdaTierThresholds, Gfx1250_EnvOverridesArchTable) +{ + RUN_ISOLATED_TEST_WITH_ENV( + "Gfx1250_EnvOverridesArchTable", + []() + { + auto comm = std::make_unique(); + InitDdaDecisionComm(*comm, "gfx1250", 8, 1, /*symmetricSupport=*/false); + comm->archThresholds = rcclGetArchThresholds("gfx1250"); + ASSERT_NE(comm->archThresholds, nullptr); + + constexpr size_t llEnv = 16384; + constexpr size_t ll128Env = 1048576; + constexpr size_t vmmEnv = 4194304; + // Values differ from the table, so equality below can only come from the env. + ASSERT_NE(comm->archThresholds->ddaLLMax[ncclFuncAllReduce], llEnv); + ASSERT_NE(comm->archThresholds->ddaLL128Max[ncclFuncAllReduce], ll128Env); + ASSERT_NE(comm->archThresholds->ddaVmmMax[ncclFuncAllReduce], vmmEnv); + + EXPECT_EQ(rcclDdaLLThreshold(comm.get(), ncclFuncAllReduce), llEnv); + EXPECT_EQ(rcclDdaLL128Threshold(comm.get(), ncclFuncAllReduce), ll128Env); + EXPECT_EQ(rcclDdaVmmThreshold(comm.get(), ncclFuncAllReduce), vmmEnv); + }, + {{"RCCL_DDA_LL_THRESHOLD", "16384"}, + {"RCCL_DDA_LL128_THRESHOLD", "1048576"}, + {"RCCL_DDA_THRESHOLD", "4194304"}}); +} + +// An arch with no threshold table (e.g. gfx90a, not in the tuning table) +// still gets base defaults from the resolvers -- the null-table path in +// rcclDdaLLThresholdTab/rcclDdaLL128ThresholdTab/rcclDdaVmmThresholdTab +// returns kDdaLL*BaseDefault, not 0. rcclDdaEnabled() returns false +// because gfx90a is not in the recognised-arch list, independent of the +// threshold value. +TEST(RcclDdaTierThresholds, UntunedArch_NoEnv_NoThresholds) +{ + RUN_ISOLATED_TEST_WITH_ENV( + "UntunedArch_NoEnv_NoThresholds", + []() + { + auto comm = std::make_unique(); + InitDdaDecisionComm(*comm, "gfx90a", 8, 1, /*symmetricSupport=*/false); + ASSERT_EQ(rcclGetArchThresholds(comm->archName), nullptr); + EXPECT_EQ(rcclDdaLLThreshold(comm.get(), ncclFuncAllReduce), kDdaLLBaseDefault); + EXPECT_EQ(rcclDdaLL128Threshold(comm.get(), ncclFuncAllReduce), kDdaLL128BaseDefault); + EXPECT_EQ(rcclDdaVmmThreshold(comm.get(), ncclFuncAllReduce), kDdaVmmBaseDefault); + EXPECT_FALSE(rcclDdaEnabled(comm.get(), 1024, rcclDdaVmmThreshold(comm.get(), ncclFuncAllReduce))); + }, + {}); +} + +// A comm assembled without archThresholds (unit tests) still resolves through +// its arch's table, including AlltoAll which has its own table slot. +TEST(RcclDdaTierThresholds, NoAttachedTable_ResolvesViaArchName) +{ + RUN_ISOLATED_TEST_WITH_ENV( + "NoAttachedTable_ResolvesViaArchName", + []() + { + auto comm = std::make_unique(); + InitDdaDecisionComm(*comm, "gfx942", 8, 1, /*symmetricSupport=*/false); + ASSERT_EQ(comm->archThresholds, nullptr); + EXPECT_EQ(rcclDdaVmmThreshold(comm.get(), ncclFuncAllReduce), rcclGetArchThresholds("gfx942")->ddaVmmMax[ncclFuncAllReduce]); + EXPECT_EQ(rcclDdaVmmThreshold(comm.get(), ncclFuncAlltoAll), rcclGetArchThresholds("gfx942")->ddaVmmMax[ncclFuncAlltoAll]); + + InitDdaDecisionComm(*comm, "gfx1250", 8, 1, /*symmetricSupport=*/false); + const rcclArchThresholds* table = rcclGetArchThresholds("gfx1250"); + ASSERT_NE(table, nullptr); + EXPECT_EQ(rcclDdaLLThreshold(comm.get(), ncclFuncAlltoAll), table->ddaLLMax[ncclFuncAlltoAll]); + EXPECT_EQ(rcclDdaLL128Threshold(comm.get(), ncclFuncAlltoAll), table->ddaLL128Max[ncclFuncAlltoAll]); + EXPECT_EQ(rcclDdaVmmThreshold(comm.get(), ncclFuncAlltoAll), rcclGetArchThresholds("gfx1250")->ddaVmmMax[ncclFuncAlltoAll]); + }, + {}); +} + +// A tier the env disables must stay disabled even though the table sets it. +TEST(RcclDdaTierThresholds, Gfx1250_EnvZeroDisablesLlTier) +{ + RUN_ISOLATED_TEST_WITH_ENV( + "Gfx1250_EnvZeroDisablesLlTier", + []() + { + auto comm = std::make_unique(); + InitDdaDecisionComm(*comm, "gfx1250", 8, 1, /*symmetricSupport=*/false); + comm->archThresholds = rcclGetArchThresholds("gfx1250"); + ASSERT_NE(comm->archThresholds, nullptr); + ASSERT_GT(comm->archThresholds->ddaLLMax[ncclFuncAllReduce], 0ul); + EXPECT_EQ(rcclDdaLLThreshold(comm.get(), ncclFuncAllReduce), 0ul); + }, + {{"RCCL_DDA_LL_THRESHOLD", "0"}}); + +} + + +// --------------------------------------------------------------------------- +// rcclSelectAlltoAll: decision per size-band (query=true, no GPU required). +// +// Tests exercise the priority chain for gfx1250 (fabric LL/LL128/VMM tiers) +// and gfx950/gfx942 (DDA-IPC tier), then the direct p2p fallback for an arch +// that has no DDA. +// +// CE_REGISTERED is excluded: ncclCeAvailable calls ncclCeImplemented which +// calls hipDriverGetVersion() -- not available in a no-GPU unit-test process. +// That path is covered by CE integration tests. +// +// The helper extends InitDdaDecisionComm with the fabric-resource sentinels +// that the DDA eligibility predicates require. +namespace +{ +void InitA2ADecisionComm(ncclComm& comm, const char* arch, int nRanks) +{ + InitDdaDecisionComm(comm, arch, nRanks, /*nNodes=*/1, /*symmetricSupport=*/false); + comm.archThresholds = rcclGetArchThresholds(arch); + // rcclSelectAlltoAll dereferences comm->topo->pivotA2AEnabled as its first + // predicate (before any arch or DDA logic). Zero-initialised: pivotA2AEnabled=false, + // so the pivot branch is skipped and falls through to the DDA logic under test. + comm.topo = new ncclTopoSystem(); + // Sentinel pointers satisfy the non-null checks in the fabric eligibility + // predicates (the values are never dereferenced by query=true calls). + comm.bootstrap = reinterpret_cast(0x1); + comm.ddaFabricMemHandler = reinterpret_cast(0x2); + comm.ddaScratch = reinterpret_cast(0x3); + comm.ddaPeerPtrsDev = reinterpret_cast(0x4); + comm.ddaScratchBytes = std::numeric_limits::max(); + comm.ddaFabricMaxBlocks = DDA_FABRIC_MAXBLOCKS; + comm.ddaFabricBarrierState = + reinterpret_cast(0x5); + comm.ddaLLEpochDev = reinterpret_cast(0x6); + comm.ddaLLEpochLen = DDA_FABRIC_MAXBLOCKS; +} + +ncclResult_t SelectA2A(ncclComm& comm, size_t totalBytes, rcclCollDecision& out) +{ + // count is per-peer bytes / sizeof(float); totalBytes = count * sizeof(float) * nRanks + size_t count = totalBytes / sizeof(float) / comm.nRanks; + return rcclSelectAlltoAll(&comm, /*sendbuff=*/nullptr, /*recvbuff=*/nullptr, + count, ncclFloat32, /*stream=*/nullptr, + /*query=*/true, /*graphCapturingHint=*/false, &out); +} +} // namespace + +// gfx1250, message at the DDA-LL ceiling (ddaLLMax[A2A] = 64 KiB total): +// rcclSelectAlltoAll must choose RCCL_DDA_FABRIC_LL. +TEST(RcclAlltoAllDecision, Gfx1250_DdaLL_Band) +{ + auto comm = std::make_unique(); + InitA2ADecisionComm(*comm, "gfx1250", 4); + const rcclArchThresholds* tbl = rcclGetArchThresholds("gfx1250"); + ASSERT_NE(tbl, nullptr); + const size_t llMax = tbl->ddaLLMax[ncclFuncAlltoAll]; + ASSERT_GT(llMax, 0ul) << "gfx1250 must have a non-zero LL cap for AlltoAll"; + + // Message at the ceiling; per-rank chunk is 16-byte aligned (LL eligibility). + rcclCollDecision dec{}; + ASSERT_EQ(SelectA2A(*comm, llMax, dec), ncclSuccess); + EXPECT_EQ(dec.algo, RCCL_DDA_FABRIC_LL); +} + +// gfx1250, message just above the LL ceiling and within the LL128 band +// (ddaLL128Max[A2A] = 1 MiB total): +// rcclSelectAlltoAll must choose RCCL_DDA_FABRIC_LL128. +TEST(RcclAlltoAllDecision, Gfx1250_DdaLL128_Band) +{ + auto comm = std::make_unique(); + InitA2ADecisionComm(*comm, "gfx1250", 4); + const rcclArchThresholds* tbl = rcclGetArchThresholds("gfx1250"); + ASSERT_NE(tbl, nullptr); + const size_t llMax = tbl->ddaLLMax[ncclFuncAlltoAll]; + const size_t ll128Max = tbl->ddaLL128Max[ncclFuncAlltoAll]; + ASSERT_GT(ll128Max, llMax) << "gfx1250 LL128 cap must exceed LL cap for AlltoAll"; + + // One 16*nRanks step above the LL ceiling, still within LL128. + // The alignment ensures each per-rank chunk is a multiple of 16 bytes + // (required by both LL128 and LL eligibility predicates). + const size_t align = 16u * (size_t)comm->nRanks; + const size_t totalBytes = ((llMax + align) / align) * align; + ASSERT_LE(totalBytes, ll128Max); + + rcclCollDecision dec{}; + ASSERT_EQ(SelectA2A(*comm, totalBytes, dec), ncclSuccess); + EXPECT_EQ(dec.algo, RCCL_DDA_FABRIC_LL128); +} + +// gfx1250, message above all DDA caps (ddaVmmMax[A2A] = 0 disables VMM; +// ddaLL128Max[A2A] = 1 MiB): anything above 1 MiB falls to Direct p2p. +TEST(RcclAlltoAllDecision, Gfx1250_AboveDdaCap_DirectFallback) +{ + auto comm = std::make_unique(); + InitA2ADecisionComm(*comm, "gfx1250", 4); + const rcclArchThresholds* tbl = rcclGetArchThresholds("gfx1250"); + ASSERT_NE(tbl, nullptr); + ASSERT_EQ(tbl->ddaVmmMax[ncclFuncAlltoAll], 0ul) + << "gfx1250 VMM must be disabled for AlltoAll"; + const size_t ll128Max = tbl->ddaLL128Max[ncclFuncAlltoAll]; + + const size_t align = 16u * (size_t)comm->nRanks; + const size_t totalBytes = ((ll128Max + align) / align) * align; + + rcclCollDecision dec{}; + ASSERT_EQ(SelectA2A(*comm, totalBytes, dec), ncclSuccess); + EXPECT_EQ(dec.algo, RCCL_DIRECT_ALLTOALL); +} + +// gfx950, message at its DDA-IPC ceiling (ddaVmmMax[A2A] = 4 MiB): +// rcclSelectAlltoAll must choose RCCL_DDA_IPC. +TEST(RcclAlltoAllDecision, Gfx950_DdaIpc_Band) +{ + auto comm = std::make_unique(); + InitA2ADecisionComm(*comm, "gfx950", 8); + const rcclArchThresholds* tbl = rcclGetArchThresholds("gfx950"); + ASSERT_NE(tbl, nullptr); + const size_t ddaMax = tbl->ddaVmmMax[ncclFuncAlltoAll]; + ASSERT_GT(ddaMax, 0ul) << "gfx950 must have a non-zero DDA-IPC cap for AlltoAll"; + + rcclCollDecision dec{}; + ASSERT_EQ(SelectA2A(*comm, ddaMax, dec), ncclSuccess); + EXPECT_EQ(dec.algo, RCCL_DDA_IPC); +} + +// gfx950, message above its 4 MiB DDA-IPC cap: falls to RCCL_DIRECT_ALLTOALL. +TEST(RcclAlltoAllDecision, Gfx950_AboveDdaCap_DirectFallback) +{ + auto comm = std::make_unique(); + InitA2ADecisionComm(*comm, "gfx950", 8); + const rcclArchThresholds* tbl = rcclGetArchThresholds("gfx950"); + ASSERT_NE(tbl, nullptr); + const size_t ddaMax = tbl->ddaVmmMax[ncclFuncAlltoAll]; + + const size_t align = 16u * (size_t)comm->nRanks; + const size_t totalBytes = ((ddaMax + align) / align) * align; + + rcclCollDecision dec{}; + ASSERT_EQ(SelectA2A(*comm, totalBytes, dec), ncclSuccess); + EXPECT_EQ(dec.algo, RCCL_DIRECT_ALLTOALL); +} + +// An arch with no DDA support (gfx90a): any size must fall straight to +// RCCL_DIRECT_ALLTOALL because rcclDdaEnabled returns false for unknown archs. +TEST(RcclAlltoAllDecision, UnsupportedArch_DirectFallback) +{ + auto comm = std::make_unique(); + // rcclSelectAlltoAll dereferences comm->topo->pivotA2AEnabled as its first + // predicate, so a topo is required. InitA2ADecisionComm attaches a zero- + // initialised ncclTopoSystem (pivotA2AEnabled=false) and the DDA sentinels; + // gfx90a exits rcclDdaEnabled early, so the sentinels are never dereferenced. + InitA2ADecisionComm(*comm, "gfx90a", 8); + + rcclCollDecision dec{}; + ASSERT_EQ(rcclSelectAlltoAll(comm.get(), nullptr, nullptr, /*count=*/1024, + ncclFloat32, nullptr, /*query=*/true, + /*graphCapturingHint=*/false, &dec), + ncclSuccess); + EXPECT_EQ(dec.algo, RCCL_DIRECT_ALLTOALL); +} + + +// --------------------------------------------------------------------------- +// rcclAllGatherCeRegisteredWindow -- four-outcome gate test. +// +// The function is a pure CE-capability gate: recv registered AND totalBytes +// within ceRegMax[AG]. symMaxR2 arbitration is a selector-level concern +// (symEligible) and is intentionally absent here, matching AllReduce and +// AlltoAll. gfx1250 table: ceRegMax[AG] = 8 GiB. +// --------------------------------------------------------------------------- +namespace +{ +rcclArchThresholds MakeAgCeRegTable(size_t ceRegMaxAg) +{ + rcclArchThresholds t{}; + for(int i = 0; i < NCCL_NUM_FUNCTIONS; ++i) + t.ceRegMax[i] = SIZE_MAX; + t.ceRegMax[ncclFuncAllGather] = ceRegMaxAg; + return t; +} +} // namespace + +// Outcome 1: recv not registered -> false regardless of size. +TEST(RcclAllGatherCeRegisteredWindow, RecvNotRegistered_ReturnsFalse) +{ + constexpr size_t kRegMax = 8ull * 1024 * 1024 * 1024; + rcclArchThresholds tbl = MakeAgCeRegTable(kRegMax); + ncclComm mockComm{}; mockComm.archThresholds = &tbl; + constexpr size_t kMid = 4ull * 1024 * 1024; + + EXPECT_FALSE(rcclAllGatherCeRegisteredWindow(&mockComm, kMid, + ncclSymSendRegRecvNonreg, /*graphMode=*/false)); + EXPECT_FALSE(rcclAllGatherCeRegisteredWindow(&mockComm, kMid, + ncclSymSendNonregRecvNonreg, /*graphMode=*/false)); +} + +// Outcome 2: totalBytes > ceRegMax -> false (message too large for CE hardware). +TEST(RcclAllGatherCeRegisteredWindow, AboveCeRegMax_ReturnsFalse) +{ + constexpr size_t kRegMax = 8ull * 1024 * 1024 * 1024; + rcclArchThresholds tbl = MakeAgCeRegTable(kRegMax); + ncclComm mockComm{}; mockComm.archThresholds = &tbl; + + EXPECT_FALSE(rcclAllGatherCeRegisteredWindow(&mockComm, kRegMax + 1, + ncclSymSendNonregRecvReg, /*graphMode=*/false)); + EXPECT_FALSE(rcclAllGatherCeRegisteredWindow(&mockComm, kRegMax + 1, + ncclSymSendRegRecvReg, /*graphMode=*/false)); +} + +// Outcome 3: totalBytes <= ceRegMax, recv registered, small message -> true. +// (symMaxR2 suppression is a selector-level concern; this function does not check it.) +TEST(RcclAllGatherCeRegisteredWindow, BelowCeRegMax_RecvRegistered_ReturnsTrue) +{ + constexpr size_t kRegMax = 8ull * 1024 * 1024 * 1024; + rcclArchThresholds tbl = MakeAgCeRegTable(kRegMax); + ncclComm mockComm{}; mockComm.archThresholds = &tbl; + constexpr size_t kSmall = 2ull * 1024 * 1024; // 2 MiB -- below symMaxR2 crossover + + EXPECT_TRUE(rcclAllGatherCeRegisteredWindow(&mockComm, kSmall, + ncclSymSendNonregRecvReg, /*graphMode=*/false)); + EXPECT_TRUE(rcclAllGatherCeRegisteredWindow(&mockComm, kSmall, + ncclSymSendRegRecvReg, /*graphMode=*/false)); +} + +// Outcome 4: exactly at ceRegMax, recv registered -> true (boundary is inclusive). +TEST(RcclAllGatherCeRegisteredWindow, AtCeRegMax_RecvRegistered_ReturnsTrue) +{ + constexpr size_t kRegMax = 8ull * 1024 * 1024 * 1024; + rcclArchThresholds tbl = MakeAgCeRegTable(kRegMax); + ncclComm mockComm{}; mockComm.archThresholds = &tbl; + + EXPECT_TRUE(rcclAllGatherCeRegisteredWindow(&mockComm, kRegMax, + ncclSymSendNonregRecvReg, /*graphMode=*/false)); + EXPECT_TRUE(rcclAllGatherCeRegisteredWindow(&mockComm, kRegMax, + ncclSymSendRegRecvReg, /*graphMode=*/false)); +} + } // namespace RcclUnitTesting diff --git a/projects/rccl/test/ce/CeTestHelpers.hpp b/projects/rccl/test/ce/CeTestHelpers.hpp index ead0038534ac..283b69efd798 100644 --- a/projects/rccl/test/ce/CeTestHelpers.hpp +++ b/projects/rccl/test/ce/CeTestHelpers.hpp @@ -115,7 +115,7 @@ inline hipError_t ceFillRankScalarFloat(void* buf, size_t nElem, int rank) // True when NCCL logs indicate the CE AllReduce pipeline ran. // Match the execution INFO lines from ncclCeAllReduce(), not the -// "Skipping CE AllReduce: ..." WARN that rcclUseCeAllReduce() emits +// "Skipping CE AllReduce: ..." WARN that rcclUseCeAr2Shot() emits // when nNodes > 1 (that substring used to fail multi-node EightRanks). inline bool ceLogShowsAllReducePath(const std::string& log) { diff --git a/projects/rccl/test/common/CeAllReduceTestHelpers.hpp b/projects/rccl/test/common/CeAllReduceTestHelpers.hpp index e50b1c792bd6..9df078ae76ab 100644 --- a/projects/rccl/test/common/CeAllReduceTestHelpers.hpp +++ b/projects/rccl/test/common/CeAllReduceTestHelpers.hpp @@ -28,20 +28,35 @@ inline bool isCeRuntimeDriverSupported() (driverVer >= 70051831 && driverVer < 70060000); } -// The chunk-size helpers live in ce_coll.h (ncclCeAllReduceMaxChunkBytes, -// ncclCeAllReduceSlotChunkBytes, ncclCeAllReduceChooseChunkBytes) so these tests -// exercise the same code ncclCeAllReduce() uses instead of a copy that can drift. +// Default ceARTmpBuf capacity. Matches NCCL_CE_AR_TMPBUF_DEFAULT_BYTES. +// Per-rank chunk capacity is that size / nRanks, same as ncclCeInit. +constexpr size_t kCeArMaxMsgBytesDefault = NCCL_CE_AR_TMPBUF_DEFAULT_BYTES; + +inline size_t ceAllReduceMaxChunkBytes(int nRanks, + size_t ceArMaxBytes = kCeArMaxMsgBytesDefault) +{ + return ceArMaxBytes / static_cast(nRanks); +} // Minimal ncclComm stand-in for CE AllReduce eligibility unit tests. struct CeAllReduceMockComm { ncclComm comm{}; + char archNameBuf[64]{}; + + CeAllReduceMockComm(const CeAllReduceMockComm&) = delete; + CeAllReduceMockComm& operator=(const CeAllReduceMockComm&) = delete; CeAllReduceMockComm() { reset(); } - void reset() + void reset(const char* archName = nullptr) { std::memset(&comm, 0, sizeof(comm)); + std::memset(archNameBuf, 0, sizeof(archNameBuf)); + if (archName && *archName) { + std::strncpy(archNameBuf, archName, sizeof(archNameBuf) - 1); + } + comm.archName = archNameBuf[0] ? archNameBuf : nullptr; comm.nNodes = 1; comm.nRanks = 4; comm.rank = 0; @@ -51,6 +66,7 @@ struct CeAllReduceMockComm comm.devrState.bigSize = 1; comm.devrState.lsaSize = comm.nRanks; comm.devrState.lsaSelf = comm.rank; + comm.ceColl.ceArMaxBytes = kCeArMaxMsgBytesDefault; } ncclComm* get() { return &comm; } diff --git a/projects/rccl/test/common/DdaAlltoAllTestHelpers.hpp b/projects/rccl/test/common/DdaAlltoAllTestHelpers.hpp index c90e1479c3df..269b4626645f 100644 --- a/projects/rccl/test/common/DdaAlltoAllTestHelpers.hpp +++ b/projects/rccl/test/common/DdaAlltoAllTestHelpers.hpp @@ -21,12 +21,15 @@ namespace RcclUnitTesting { // Use the production rcclDdaEnabled() from rccl_common.h directly. -// Threshold constants kDdaAlltoAllGfx{942,950,1250}ThresholdBytes are also in rccl_common.h. inline size_t testAlltoAllTotalBytes(size_t count, int nRanks, ncclDataType_t datatype) { return static_cast(nRanks) * count * static_cast(ncclTypeSize(datatype)); } +inline size_t testDdaAlltoAllThreshold(const ncclComm* comm) { + return rcclDdaEntryThreshold(comm, ncclFuncAlltoAll); +} + inline bool testRcclDdaAlltoAllThresholdEnabled( const ncclComm* comm, size_t count, @@ -34,9 +37,7 @@ inline bool testRcclDdaAlltoAllThresholdEnabled( return rcclDdaEnabled( comm, testAlltoAllTotalBytes(count, comm->nRanks, datatype), - kDdaAlltoAllGfx942ThresholdBytes, - kDdaAlltoAllGfx950ThresholdBytes, - kDdaAlltoAllGfx1250ThresholdBytes); + testDdaAlltoAllThreshold(comm)); } // Mirrors dda_alltoall_ipc.cu: in-kernel staging copy on single-block launches only. @@ -55,6 +56,9 @@ struct DdaAlltoAllMockComm ncclComm comm{}; char archNameBuf[64]{}; + DdaAlltoAllMockComm(const DdaAlltoAllMockComm&) = delete; + DdaAlltoAllMockComm& operator=(const DdaAlltoAllMockComm&) = delete; + DdaAlltoAllMockComm() { reset("gfx950:sramecc+:xnack-"); } void reset(const char* archName) @@ -73,7 +77,12 @@ struct DdaAlltoAllMockComm // Largest float32 per-rank count whose 8-rank AlltoAll totals exactly 4 MiB. constexpr size_t kAlltoAllFloat32CountAt4MbThreshold = - kDdaAlltoAllGfx942ThresholdBytes / + 4194304UL / + (static_cast(nccl_dda_detail::kDdaNranks) * sizeof(float)); + +// Per-rank float32 count whose 8-rank AlltoAll totals exactly 1 MiB (gfx1250 LL128 ceiling). +constexpr size_t kAlltoAllFloat32CountAt1MbLL128Threshold = + 1048576UL / (static_cast(nccl_dda_detail::kDdaNranks) * sizeof(float)); // 4 KiB/rank float32: single-block grid on 8-rank IPC launch (in-kernel copy path). diff --git a/projects/rccl/test/common/DdaFabricTestHelpers.hpp b/projects/rccl/test/common/DdaFabricTestHelpers.hpp index 0a7809da07a5..509e923a807e 100644 --- a/projects/rccl/test/common/DdaFabricTestHelpers.hpp +++ b/projects/rccl/test/common/DdaFabricTestHelpers.hpp @@ -35,6 +35,7 @@ struct DdaFabricMockComm // with an exact small value when exercising scratch-size rejection. comm.ddaScratchBytes = std::numeric_limits::max(); comm.ddaFabricMaxBlocks = DDA_FABRIC_MAXBLOCKS; + comm.archName = const_cast("gfx1250"); setFabricResourcesPresent(true); } diff --git a/projects/rccl/test/graph/TopoTests.cpp b/projects/rccl/test/graph/TopoTests.cpp index 3f30d3ac95bd..fba866cf51ec 100644 --- a/projects/rccl/test/graph/TopoTests.cpp +++ b/projects/rccl/test/graph/TopoTests.cpp @@ -237,6 +237,47 @@ class TopoTest : public ::testing::Test { return gpuPci; } + // A GPU and a NIC in one PCI domain but on separate root ports, with no PCIe switch above + // either: their only common ancestor is the host bridge, so ncclTopoSetPaths() rates the pair + // PATH_PHB even though a single root complex holds both. This is the gfx1250 shape + // rcclRewriteSameDomainNetPaths() corrects. nParts > 0 makes the HIP devices MLOPart partitions + // sharing the one PCI function. + void addSameDomainGpuNic(struct ncclXmlNode* cpu, int domain, int dev, int baseRank, + const char* gcn, int nParts) { + char gpuBus[32], nicBus[32]; + snprintf(gpuBus, sizeof(gpuBus), "%04x:04:00.0", domain); + snprintf(nicBus, sizeof(nicBus), "%04x:03:00.0", domain); + struct ncclXmlNode* gpuPci = addGpuPci(cpu, gpuBus, gcn, baseRank, baseRank, + nParts > 0 ? 0 : NCCL_TOPO_UNDEF); + for (int p = 1; p < nParts; p++) addGpuUnderPci(gpuPci, gcn, baseRank + p, baseRank + p, p); + addNic(cpu, nicBus, dev); + } + + // The same pair, but hung directly off one shared PCIe switch instead of the host bridge, which + // ncclTopoSetPaths() rates PATH_PIX. That is nearer than the PATH_PXB the same-domain rewrite + // installs, so it is the shape that shows whether the rewrite's fromType guards hold. + void addSharedBridgeGpuNic(struct ncclXmlNode* cpu, int domain, int dev, int baseRank, + const char* gcn, int nParts) { + char switchBus[32], gpuBus[32], nicBus[32]; + snprintf(switchBus, sizeof(switchBus), "%04x:02:00.0", domain); + snprintf(gpuBus, sizeof(gpuBus), "%04x:04:00.0", domain); + snprintf(nicBus, sizeof(nicBus), "%04x:03:00.0", domain); + struct ncclXmlNode* pciSwitch = addPciBridge(cpu, switchBus); + struct ncclXmlNode* gpuPci = + addGpuPci(pciSwitch, gpuBus, gcn, baseRank, baseRank, nParts > 0 ? 0 : NCCL_TOPO_UNDEF); + for (int p = 1; p < nParts; p++) addGpuUnderPci(gpuPci, gcn, baseRank + p, baseRank + p, p); + addNic(pciSwitch, nicBus, dev); + } + + // NET nodes are numbered in XML traversal order, so locate one by the PCI domain of its NIC + // rather than assuming that order matches the order the domains were added in. + static int netIndexForDomain(struct ncclTopoSystem* s, int domain) { + for (int n = 0; n < s->nodes[NET].count; n++) { + if (NCCL_BUSID_DOMAIN(s->nodes[NET].nodes[n].net.busId) == (uint64_t)domain) return n; + } + return -1; + } + struct ncclTopoSystem* buildSystemWithPaths(uint64_t host) { struct ncclTopoSystem* built = buildSystem(host); if (built == nullptr) return nullptr; @@ -666,6 +707,202 @@ TEST_F(TopoTest, CheckGdr_NonMloPartGpuStillUsesOwnPath) { ncclTopoFree(built); } +// rcclRewriteSameDomainNetPaths() reclassifies a GPU/NIC pair sharing a PCI domain from PHB to PXB +// on gfx1250, where the domain is one root complex and the host-bridge hop the BFS walked is not +// the cost PHB stands for. It is keyed on the physical device because ncclTopoGdrDistance() reads +// the parent DEV entry for a partition: rewriting only the GPU nodes leaves the GDR verdict on the +// original PHB, and the no-GDR diversion in ncclTopoComputePaths() then puts the entry back to PHB. +TEST_F(TopoTest, SameDomainNetPaths_Gfx1250PromotesPhysicalDeviceAndPartitions) { + const uint64_t host = 0xf0; + struct ncclXmlNode* cpu = addSystemCpu(host); + addSameDomainGpuNic(cpu, /*domain=*/1, /*dev=*/0, /*baseRank=*/0, "gfx1250", /*nParts=*/2); + addSameDomainGpuNic(cpu, /*domain=*/2, /*dev=*/1, /*baseRank=*/2, "gfx1250", /*nParts=*/2); + + struct ncclTopoSystem* built = buildSystemWithPaths(host); + ASSERT_NE(built, nullptr); + ASSERT_EQ(built->nodes[GPU].count, 4); + ASSERT_EQ(built->nodes[NET].count, 2); + + for (int g = 0; g < built->nodes[GPU].count; g++) { + struct ncclTopoNode* gpu = built->nodes[GPU].nodes + g; + const int domain = (int)NCCL_BUSID_DOMAIN(NCCL_TOPO_ID_LOCAL_ID(gpu->id)); + const int local = netIndexForDomain(built, domain); + SCOPED_TRACE(testing::Message() << "gpu " << g << " domain " << domain); + ASSERT_GE(local, 0); + ASSERT_NE(gpu->gpu.parent, nullptr); + + const int dev = (int)(gpu->gpu.parent - built->nodes[DEV].nodes); + + // The physical device carries the promotion and the partition agrees with it, so the GDR + // decision and the graph search read the same distance. + EXPECT_EQ(gpu->gpu.parent->paths[NET][local].type, PATH_PXB); + EXPECT_EQ(gpu->paths[NET][local].type, PATH_PXB); + + // The NIC's own view has to match. ncclTopoGetLocal(system, NET, n, GPU, ...) and net.localGpu + // read NET->GPU when a NIC picks the GPU nearest to it, so a one-directional rewrite would + // promote the GPU's view while leaving the NIC still seeing a host-bridge hop. + EXPECT_EQ(built->nodes[NET].nodes[local].paths[GPU][g].type, PATH_PXB); + EXPECT_EQ(built->nodes[NET].nodes[local].paths[DEV][dev].type, PATH_PXB); + + // A NIC in another domain really is across the host bridge and has to stay PHB, otherwise the + // search loses the locality this rewrite exists to expose. + for (int n = 0; n < built->nodes[NET].count; n++) { + if (n == local) continue; + EXPECT_EQ(gpu->paths[NET][n].type, PATH_PHB); + EXPECT_EQ(gpu->gpu.parent->paths[NET][n].type, PATH_PHB); + EXPECT_EQ(built->nodes[NET].nodes[n].paths[GPU][g].type, PATH_PHB); + EXPECT_EQ(built->nodes[NET].nodes[n].paths[DEV][dev].type, PATH_PHB); + } + + // PXB is below the default netGdrLevel (PATH_P2C), so the promotion has to turn GDR on. + enum ncclTopoGdrMode mode = ncclTopoGdrModeDisable; + ASSERT_EQ(ncclTopoCheckGdr(built, gpu->gpu.rank, built->nodes[NET].nodes[local].id, /*read=*/0, + &mode), + ncclSuccess); + EXPECT_NE(mode, ncclTopoGdrModeDisable); + } + + ncclTopoFree(built); +} + +// The rewrite is gfx1250-only. Every other architecture keeps the distance the PCIe tree actually +// describes, so the same shape stays PHB and GDR stays off. +TEST_F(TopoTest, SameDomainNetPaths_OtherArchKeepsPhb) { + const uint64_t host = 0xf1; + struct ncclXmlNode* cpu = addSystemCpu(host); + addSameDomainGpuNic(cpu, /*domain=*/1, /*dev=*/0, /*baseRank=*/0, "gfx942", /*nParts=*/2); + + struct ncclTopoSystem* built = buildSystemWithPaths(host); + ASSERT_NE(built, nullptr); + ASSERT_EQ(built->nodes[GPU].count, 2); + ASSERT_EQ(built->nodes[NET].count, 1); + + for (int g = 0; g < built->nodes[GPU].count; g++) { + struct ncclTopoNode* gpu = built->nodes[GPU].nodes + g; + SCOPED_TRACE(testing::Message() << "gpu " << g); + ASSERT_NE(gpu->gpu.parent, nullptr); + // Pin the partition entry as well as the physical device's: ncclTopoGdrDistance() reads the + // parent for a partition, so a rewrite leaking onto the GPU nodes would otherwise go unseen. + // Both directions, since nothing else here would catch a leak onto the NIC's own entries. + const int dev = (int)(gpu->gpu.parent - built->nodes[DEV].nodes); + EXPECT_EQ(gpu->gpu.parent->paths[NET][0].type, PATH_PHB); + EXPECT_EQ(gpu->paths[NET][0].type, PATH_PHB); + EXPECT_EQ(built->nodes[NET].nodes[0].paths[GPU][g].type, PATH_PHB); + EXPECT_EQ(built->nodes[NET].nodes[0].paths[DEV][dev].type, PATH_PHB); + + enum ncclTopoGdrMode mode = ncclTopoGdrModeDisable; + ASSERT_EQ(ncclTopoCheckGdr(built, gpu->gpu.rank, built->nodes[NET].nodes[0].id, /*read=*/0, + &mode), + ncclSuccess); + EXPECT_EQ(mode, ncclTopoGdrModeDisable); + } + + ncclTopoFree(built); +} + +// An SPX gfx1250 device is one whole GPU rather than a set of CPX partitions, so its mloPart stays +// NCCL_TOPO_UNDEF and ncclTopoGdrDistance() reads the GPU's own NET entry instead of the parent +// DEV's. GDR therefore already worked for this shape before the rework; what the DEV-keyed rewrite +// adds is that the physical device agrees with its GPU, so the graph search and the GDR decision +// cannot read different distances for the same hardware. +// +// A second domain is required here specifically: for an SPX GPU, ncclTopoGdrDistance() reads the +// GPU's own entry rather than a parent's, so a rewrite that leaked past the domain match (e.g. the +// `gpu->gpu.parent != dev` guard in rcclRewriteSameDomainNetPaths() being dropped) would promote the +// far NIC's entry directly, and nothing downstream re-derives it from a parent to catch the leak. +// The single-domain form of this test cannot see that: there is no "wrong domain" NIC to check. +TEST_F(TopoTest, SameDomainNetPaths_Gfx1250UnpartitionedGpu) { + const uint64_t host = 0xf2; + struct ncclXmlNode* cpu = addSystemCpu(host); + addSameDomainGpuNic(cpu, /*domain=*/1, /*dev=*/0, /*baseRank=*/0, "gfx1250", /*nParts=*/0); + addSameDomainGpuNic(cpu, /*domain=*/2, /*dev=*/1, /*baseRank=*/1, "gfx1250", /*nParts=*/0); + + struct ncclTopoSystem* built = buildSystemWithPaths(host); + ASSERT_NE(built, nullptr); + // One DEV per GPU: ncclTopoAddGpuSub() creates the physical device whether or not the HIP + // devices are partitions, so the DEV-keyed rewrite has the same nodes to walk either way. + ASSERT_EQ(built->nodes[GPU].count, 2); + ASSERT_EQ(built->nodes[DEV].count, 2); + ASSERT_EQ(built->nodes[NET].count, 2); + + for (int g = 0; g < built->nodes[GPU].count; g++) { + struct ncclTopoNode* gpu = built->nodes[GPU].nodes + g; + const int domain = (int)NCCL_BUSID_DOMAIN(NCCL_TOPO_ID_LOCAL_ID(gpu->id)); + const int local = netIndexForDomain(built, domain); + SCOPED_TRACE(testing::Message() << "gpu " << g << " domain " << domain); + ASSERT_GE(local, 0); + ASSERT_EQ(gpu->gpu.mloPart, NCCL_TOPO_UNDEF); + ASSERT_NE(gpu->gpu.parent, nullptr); + + const int dev = (int)(gpu->gpu.parent - built->nodes[DEV].nodes); + + EXPECT_EQ(gpu->paths[NET][local].type, PATH_PXB); + EXPECT_EQ(gpu->gpu.parent->paths[NET][local].type, PATH_PXB); + EXPECT_EQ(built->nodes[NET].nodes[local].paths[GPU][g].type, PATH_PXB); + EXPECT_EQ(built->nodes[NET].nodes[local].paths[DEV][dev].type, PATH_PXB); + + enum ncclTopoGdrMode mode = ncclTopoGdrModeDisable; + ASSERT_EQ(ncclTopoCheckGdr(built, gpu->gpu.rank, built->nodes[NET].nodes[local].id, /*read=*/0, + &mode), + ncclSuccess); + EXPECT_NE(mode, ncclTopoGdrModeDisable); + + // The NIC in the other domain is not this GPU's, and there is no parent-DEV re-check to save a + // leaked promotion for an SPX GPU, so this pair has to stay untouched. + for (int n = 0; n < built->nodes[NET].count; n++) { + if (n == local) continue; + EXPECT_EQ(gpu->paths[NET][n].type, PATH_PHB); + EXPECT_EQ(gpu->gpu.parent->paths[NET][n].type, PATH_PHB); + EXPECT_EQ(built->nodes[NET].nodes[n].paths[GPU][g].type, PATH_PHB); + EXPECT_EQ(built->nodes[NET].nodes[n].paths[DEV][dev].type, PATH_PHB); + + enum ncclTopoGdrMode crossMode = ncclTopoGdrModeDefault; + ASSERT_EQ(ncclTopoCheckGdr(built, gpu->gpu.rank, built->nodes[NET].nodes[n].id, /*read=*/0, + &crossMode), + ncclSuccess); + EXPECT_EQ(crossMode, ncclTopoGdrModeDisable); + } + } + + ncclTopoFree(built); +} + +// The rewrite promotes PATH_PHB and nothing else. A gfx1250 GPU and NIC hanging off one shared PCIe +// switch are already PATH_PIX, which is nearer than the PATH_PXB the promotion installs, so the +// fromType guards have to leave the pair alone. Without them the rewrite would demote a +// switch-local pair to PXB purely because the two share a PCI domain, losing the locality the +// graph search reads out of the path type. GDR is on either way, so only the type catches this. +TEST_F(TopoTest, SameDomainNetPaths_Gfx1250KeepsNearerPixPath) { + const uint64_t host = 0xf3; + struct ncclXmlNode* cpu = addSystemCpu(host); + addSharedBridgeGpuNic(cpu, /*domain=*/1, /*dev=*/0, /*baseRank=*/0, "gfx1250", /*nParts=*/2); + + struct ncclTopoSystem* built = buildSystemWithPaths(host); + ASSERT_NE(built, nullptr); + ASSERT_EQ(built->nodes[GPU].count, 2); + ASSERT_EQ(built->nodes[NET].count, 1); + + for (int g = 0; g < built->nodes[GPU].count; g++) { + struct ncclTopoNode* gpu = built->nodes[GPU].nodes + g; + SCOPED_TRACE(testing::Message() << "gpu " << g); + ASSERT_NE(gpu->gpu.parent, nullptr); + const int dev = (int)(gpu->gpu.parent - built->nodes[DEV].nodes); + + EXPECT_EQ(gpu->paths[NET][0].type, PATH_PIX); + EXPECT_EQ(gpu->gpu.parent->paths[NET][0].type, PATH_PIX); + EXPECT_EQ(built->nodes[NET].nodes[0].paths[GPU][g].type, PATH_PIX); + EXPECT_EQ(built->nodes[NET].nodes[0].paths[DEV][dev].type, PATH_PIX); + + enum ncclTopoGdrMode mode = ncclTopoGdrModeDisable; + ASSERT_EQ(ncclTopoCheckGdr(built, gpu->gpu.rank, built->nodes[NET].nodes[0].id, /*read=*/0, + &mode), + ncclSuccess); + EXPECT_NE(mode, ncclTopoGdrModeDisable); + } + + ncclTopoFree(built); +} + // The ring search caps the GPU-to-NIC path type it accepts with this helper (ROCM-30906), so on a // rail-optimised node the answer has to be the switch-local NIC and not the NIC of another rail. TEST_F(TopoTest, GpuMaxLocalNetPath_RailSystemStopsAtTheLocalNic) { diff --git a/projects/rccl/test/host/CMakeLists.txt b/projects/rccl/test/host/CMakeLists.txt index 15dd1dc237b8..e0d808d97ba4 100644 --- a/projects/rccl/test/host/CMakeLists.txt +++ b/projects/rccl/test/host/CMakeLists.txt @@ -68,6 +68,7 @@ set(RCCL_MICRO_TEST_SOURCES fakes/ce_fakes.cc # src/ce_coll.cc availability seams fakes/collectives_fakes.cc # src/collectives.cc name helpers fakes/dda_fakes.cc # src/algorithms/dda/* selection seams + fakes/rccl_arch_thresholds_fakes.cc # src/rccl_arch_thresholds.cc fakes/strongstream_stubs.cc # src/misc/strongstream.cc capture seam fakes/sym_kernels_fakes.cc # src/sym_kernels.cc query seams fakes/transport_stubs.cc # transport/PXN/AINIC seams @@ -259,6 +260,7 @@ if(BUILD_TESTS) fakes/strongstream_stubs.cc # src/misc/strongstream.cc fakes/sym_kernels_fakes.cc # src/sym_kernels.cc fakes/tuning_fakes.cc # src/graph/tuning.cc + fakes/rccl_arch_thresholds_fakes.cc # src/rccl_arch_thresholds.cc ${PROJECT_BINARY_DIR}/hipify/src/misc/argcheck.cc # real PtrCheck/CommCheck ${PROJECT_BINARY_DIR}/hipify/src/misc/archinfo.cc # real IsArchMatch/GcnArchNameFormat ${PROJECT_BINARY_DIR}/hipify/src/misc/utils.cc # real busId/hash helpers @@ -379,16 +381,18 @@ if(BUILD_TESTS) # UNIT UNDER TEST defines; a target that needs a real value where the floor # aborts gets a seam in the owning TU's fakes file instead. # --------------------------------------------------------------------------- - # LINKS ONLY BECAUSE OF --gc-sections. Relinking without it leaves 34 undefined - # symbols: 33 ordinary externals of enqueue.cc that nothing in fakes/ defines at - # all -- among them hipFuncGetAttributes, netTransport, ncclGroupStartInternal - # and the 13-strong ncclProfiler* set -- plus ncclOsStrSep, an external of the - # hipified utils.cc oracle TU below. The control globals that used to make up - # most of this list are gone: each now lives in the per-production-TU fakes file - # that references it, so no *_stubs.cc leaves one dangling. All 34 that remain - # need fakes written. So a new test reaching one of those paths fails at LINK + # LINKS ONLY BECAUSE OF --gc-sections. Relinking without it leaves a few dozen + # undefined symbols: ordinary externals of enqueue.cc that nothing in fakes/ + # defines at all -- among them hipFuncGetAttributes, netTransport and most of + # the ncclProfiler* set -- plus ncclOsStrSep, an external of the hipified + # utils.cc oracle TU below. The control globals that used to make up most of + # this list are gone: each now lives in the per-production-TU fakes file that + # references it, so no *_stubs.cc leaves one dangling. Every one that remains + # needs a fake written. So a new test reaching one of those paths fails at LINK # time, not as an assertion: e.g. ncclInitKernelsForDevice (enqueue.cc:90) - # needs a hipFuncGetAttributes fake before it can be called at all. + # needs a hipFuncGetAttributes fake before it can be called at all. A symbol + # that gc-sections stops dropping (enqueue.cc grew a reference from retained + # code) is the same signal: write the fake in its owning TU's fakes file. # To recount: take this target's CMakeFiles/.../link.txt, swap --gc-sections for # --error-limit=0 (lld stops at 20 errors otherwise), and run it. set(TEST_MICRO_ENQUEUE_SOURCE_FILES @@ -402,10 +406,12 @@ if(BUILD_TESTS) fakes/hip_fakes.cc # Shared per-production-TU fakes (see MICROTEST_README.md for the map). fakes/ce_fakes.cc # src/ce_coll.cc + fakes/collconfig_fakes.cc # src/config/collconfig.cc fakes/collectives_fakes.cc # src/collectives.cc name tables/helpers fakes/comm_fakes.cc # src/init.cc comm lifecycle fakes/dev_runtime_fakes.cc # src/dev_runtime.cc fakes/env_fakes.cc # src/misc/param.cc + getenv interposition + fakes/group_fakes.cc # src/group.cc (ncclGroupCommJoin's thread-locals) fakes/os_fakes.cc # src/os/linux.cc aligned alloc fakes/proxy_fakes.cc # src/proxy.cc fakes/rccl_wrap_fakes.cc # src/rccl_wrap.cc @@ -415,6 +421,7 @@ if(BUILD_TESTS) fakes/strongstream_stubs.cc # src/misc/strongstream.cc fakes/sym_kernels_fakes.cc # src/sym_kernels.cc fakes/tuning_fakes.cc # src/graph/tuning.cc + fakes/rccl_arch_thresholds_fakes.cc # src/rccl_arch_thresholds.cc ${PROJECT_BINARY_DIR}/hipify/src/misc/argcheck.cc # real PtrCheck/CommCheck ${PROJECT_BINARY_DIR}/hipify/src/misc/archinfo.cc # real IsArchMatch ${PROJECT_BINARY_DIR}/hipify/src/misc/utils.cc # real busId/hash helpers @@ -904,6 +911,7 @@ set(TEST_MICRO_INIT_SOURCE_FILES fakes/strongstream_stubs.cc fakes/sym_kernels_fakes.cc fakes/tuning_fakes.cc + fakes/rccl_arch_thresholds_fakes.cc ${RCCL_HIPIFY_DIR}/src/misc/argcheck.cc ${RCCL_HIPIFY_DIR}/src/misc/archinfo.cc ${RCCL_HIPIFY_DIR}/src/misc/utils.cc @@ -974,10 +982,12 @@ set(TEST_MICRO_ENQUEUE_SOURCE_FILES ${CMAKE_CURRENT_SOURCE_DIR}/fakes/nccl_fakes.cc ${CMAKE_CURRENT_SOURCE_DIR}/fakes/hip_fakes.cc ${CMAKE_CURRENT_SOURCE_DIR}/fakes/ce_fakes.cc + ${CMAKE_CURRENT_SOURCE_DIR}/fakes/collconfig_fakes.cc ${CMAKE_CURRENT_SOURCE_DIR}/fakes/collectives_fakes.cc ${CMAKE_CURRENT_SOURCE_DIR}/fakes/comm_fakes.cc ${CMAKE_CURRENT_SOURCE_DIR}/fakes/dev_runtime_fakes.cc ${CMAKE_CURRENT_SOURCE_DIR}/fakes/env_fakes.cc + ${CMAKE_CURRENT_SOURCE_DIR}/fakes/group_fakes.cc ${CMAKE_CURRENT_SOURCE_DIR}/fakes/os_fakes.cc ${CMAKE_CURRENT_SOURCE_DIR}/fakes/proxy_fakes.cc ${CMAKE_CURRENT_SOURCE_DIR}/fakes/rccl_wrap_fakes.cc @@ -987,6 +997,7 @@ set(TEST_MICRO_ENQUEUE_SOURCE_FILES ${CMAKE_CURRENT_SOURCE_DIR}/fakes/strongstream_stubs.cc ${CMAKE_CURRENT_SOURCE_DIR}/fakes/sym_kernels_fakes.cc ${CMAKE_CURRENT_SOURCE_DIR}/fakes/tuning_fakes.cc + ${CMAKE_CURRENT_SOURCE_DIR}/fakes/rccl_arch_thresholds_fakes.cc ${RCCL_HIPIFY_DIR}/src/misc/argcheck.cc ${RCCL_HIPIFY_DIR}/src/misc/archinfo.cc ${RCCL_HIPIFY_DIR}/src/misc/utils.cc diff --git a/projects/rccl/test/host/enqueue-test.cc b/projects/rccl/test/host/enqueue-test.cc index 716c092c683f..45fc0a0862d1 100644 --- a/projects/rccl/test/host/enqueue-test.cc +++ b/projects/rccl/test/host/enqueue-test.cc @@ -3321,3 +3321,76 @@ TEST_F(EnqueueMicrotest, RedOpCreate_RecorderFailure_Propagates) { ncclRedOpCreatePreMulSum_impl(&op, &s, ncclFloat32, ncclScalarHostImmediate, rc.get())); } +// =========================================================================== +// collTaskAppend -- symkExtract tag (enqueue.cc:3508-3511) +// +// collTaskAppend is static inside enqueue.cc and in scope here via the +// ENQUEUE_CC_PATH textual include. The two cases below pin that a +// non-symmetric decision writes RCCL_SYMK_EXTRACT_DENY and RCCL_SYMMETRIC +// writes RCCL_SYMK_EXTRACT_ALLOW. A third case pins decisionValid=false → +// RCCL_SYMK_EXTRACT_NONE. The consumer (ncclMakeSymmetricTaskList) is a +// fail-loud stub in test/host/fakes/sched_stubs.cc and is not reachable here; +// only the tag written onto the task is checked. +// =========================================================================== + +// Minimal comm that satisfies collTaskAppend without entering the Broadcast / +// AllGatherV branch and without GPU allocation. +struct CollTaskComm { + std::unique_ptr storage{new ncclComm{}}; + CollTaskComm() { storage->nRanks = 4; } + ncclComm* get() { return storage.get(); } +}; + +static ncclInfo MakeCollInfo(ncclComm* comm, bool decisionValid, int algo) { + ncclInfo info{}; + info.comm = comm; + info.coll = ncclFuncAllReduce; + info.op = ncclSum; + info.datatype = ncclFloat32; + info.count = static_cast(comm->nRanks); + info.chunkSteps = 1; + info.sliceSteps = 1; + info.decisionValid = decisionValid; + info.decision.algo = algo; + return info; +} + +TEST_F(EnqueueMicrotest, CollTaskAppend_NonSymmetricDecision_WritesDeny) { + // decisionValid=true, algo != RCCL_SYMMETRIC → symkExtract must be DENY so + // ncclMakeSymmetricTaskList does not extract the task even when buffers are + // registered. + CollTaskComm cc; + ncclInfo info = MakeCollInfo(cc.get(), /*decisionValid=*/true, NCCL_ALGO_RING); + ncclDevRedOpFull opDev{}; + ASSERT_EQ(ncclSuccess, collTaskAppend(cc.get(), &info, opDev)); + ASSERT_EQ(1, cc.get()->planner.nTasksColl); + const ncclTaskColl* t = ncclTaskCollSorterDequeueAll(&cc.get()->planner.collSorter); + ASSERT_NE(nullptr, t); + EXPECT_EQ(RCCL_SYMK_EXTRACT_DENY, t->symkExtract); +} + +TEST_F(EnqueueMicrotest, CollTaskAppend_SymmetricDecision_WritesAllow) { + // decisionValid=true, algo == RCCL_SYMMETRIC → symkExtract must be ALLOW so + // ncclMakeSymmetricTaskList can extract the task. + CollTaskComm cc; + ncclInfo info = MakeCollInfo(cc.get(), /*decisionValid=*/true, RCCL_SYMMETRIC); + ncclDevRedOpFull opDev{}; + ASSERT_EQ(ncclSuccess, collTaskAppend(cc.get(), &info, opDev)); + ASSERT_EQ(1, cc.get()->planner.nTasksColl); + const ncclTaskColl* t = ncclTaskCollSorterDequeueAll(&cc.get()->planner.collSorter); + ASSERT_NE(nullptr, t); + EXPECT_EQ(RCCL_SYMK_EXTRACT_ALLOW, t->symkExtract); +} + +TEST_F(EnqueueMicrotest, CollTaskAppend_NoDecision_WritesNone) { + // decisionValid=false → symkExtract must stay NONE so the extractor uses + // window inspection rather than a stale/absent selector result. + CollTaskComm cc; + ncclInfo info = MakeCollInfo(cc.get(), /*decisionValid=*/false, NCCL_ALGO_RING); + ncclDevRedOpFull opDev{}; + ASSERT_EQ(ncclSuccess, collTaskAppend(cc.get(), &info, opDev)); + ASSERT_EQ(1, cc.get()->planner.nTasksColl); + const ncclTaskColl* t = ncclTaskCollSorterDequeueAll(&cc.get()->planner.collSorter); + ASSERT_NE(nullptr, t); + EXPECT_EQ(RCCL_SYMK_EXTRACT_NONE, t->symkExtract); +} diff --git a/projects/rccl/test/host/fakes/collconfig_fakes.cc b/projects/rccl/test/host/fakes/collconfig_fakes.cc new file mode 100644 index 000000000000..009eb9b20b3a --- /dev/null +++ b/projects/rccl/test/host/fakes/collconfig_fakes.cc @@ -0,0 +1,46 @@ +/************************************************************************* + * Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +// Fakes for the symbols defined by src/config/collconfig.cc. No state, so no +// reset. collTaskAppend reads a per-call config through these two, so a target +// that includes enqueue.cc and covers collTaskAppend references them. +// +// The isolation predicate is reproduced rather than stubbed: it is pure +// arithmetic over the config header, so a copy cannot drift in behaviour, only +// in the fields it knows about. The mask query is faithful for the +// no-selection case and fails loud otherwise, because honoring a selection +// needs the real parser (ncclAlgParse / ncclAlgValidForFuncMask, src/config/ +// algorithm_{parser,registry}.cc), which this binary does not link: a test that +// scripts config->algSelection must add those oracle TUs to the target rather +// than silently read back "automatic". + +#include + +#include "nccl.h" +#include "config/collconfig.h" + +#include "fail_loud.h" + +bool ncclCollConfigHasAlgSelection(const ncclCollConfig_t* config) { + if (config == NULL) return false; + return config->algSelection != NCCL_CONFIG_UNDEF_PTR && config->algSelection[0] != '\0'; +} + +bool ncclCollConfigNeedAggIsolate(const ncclCollConfig_t* config) { + if (config->size == 0) return false; // no user config passed + if (config->minCTAs != NCCL_CONFIG_UNDEF_INT || config->maxCTAs != NCCL_CONFIG_UNDEF_INT) return true; + if (config->nvlsCTAs != NCCL_CONFIG_UNDEF_INT) return true; + if (ncclCollConfigHasAlgSelection(config)) return true; + return false; +} + +ncclResult_t ncclCollConfigGetAlgMask(const ncclCollConfig_t* config, ncclFunc_t, uint64_t* outMask) { + if (ncclCollConfigHasAlgSelection(config)) { + FailLoud("collconfig_fakes", "ncclCollConfigGetAlgMask: config->algSelection needs the real parser"); + } + *outMask = 0; // no selection -> automatic + return ncclSuccess; +} diff --git a/projects/rccl/test/host/fakes/collectives_fakes.cc b/projects/rccl/test/host/fakes/collectives_fakes.cc index 95946f424ff0..4d6f107ef3f1 100644 --- a/projects/rccl/test/host/fakes/collectives_fakes.cc +++ b/projects/rccl/test/host/fakes/collectives_fakes.cc @@ -18,6 +18,7 @@ #include "device.h" #include "info.h" #include "nccl.h" +#include "nccl_fakes.h" // g_loadParam, for the RCCL_PARAM default this stands in for // Algorithm/protocol name tables. Passed to the RCCL tuning override hooks and // logged through. @@ -95,3 +96,8 @@ const char* ncclDatatypeToString(ncclDataType_t type) { // Red-op text is not consumed by this target; retain the existing placeholder. const char* ncclDevRedOpToString(ncclDevRedOp_t) { return "redop"; } + +// rcclSelectAlltoAll reads this to decide the pivot path, so it is referenced +// from outside collectives.cc and the redirected RCCL_PARAM does not cover it. +// Same env name and default (off) as collectives.cc:452. +int64_t rcclParamAlltoAllPivotEnable() { return g_loadParam("RCCL_ALL_TO_ALL_PIVOT_ENABLE", 0); } diff --git a/projects/rccl/test/host/fakes/comm_fakes.cc b/projects/rccl/test/host/fakes/comm_fakes.cc index f4c5c2171589..26ad609cfd46 100644 --- a/projects/rccl/test/host/fakes/comm_fakes.cc +++ b/projects/rccl/test/host/fakes/comm_fakes.cc @@ -9,6 +9,7 @@ // comm/util globals, and the pure-instrumentation recorder / nvtx no-ops. #include "nccl.h" +#include "nccl_fakes.h" // g_loadParam, for the NCCL_PARAM defaults this stands in for #include "comm.h" // also pulls recorder.h (no include guard) #include "utils.h" #include "roctx.h" @@ -44,6 +45,16 @@ void ResetCommFakes() { // --- Comm globals (real in init.cc) --------------------------------------- enum ncclLaunchMode ncclParamLaunchMode = ncclLaunchModeParallel; +// The per-launch resource params init.cc declares. collTaskAppend resolves the +// task's CTA counts through them (env > per-call > comm), so they are referenced +// from outside init.cc and the redirected NCCL_PARAM does not cover them. Same +// env names and defaults as init.cc, routed through g_loadParam so a fixture can +// script the env-override arm of NCCL_CONFIG_SET. +int64_t ncclParamMinCTAs() { return g_loadParam("MIN_CTAS", NCCL_CONFIG_UNDEF_INT); } // init.cc:2667 +int64_t ncclParamMaxCTAs() { return g_loadParam("MAX_CTAS", NCCL_CONFIG_UNDEF_INT); } // init.cc:2666 +int64_t ncclParamNvlsChannels() { return g_loadParam("NVLS_NCHANNELS", NCCL_CONFIG_UNDEF_INT); } // init.cc:135 +int64_t ncclParamCGAClusterSize() { return g_loadParam("CGA_CLUSTER_SIZE", NCCL_CONFIG_UNDEF_INT); } // init.cc:2664 + // ncclThreadSignalLocalInstance is owned by src/misc/utils.cc, not init.cc, and // lives in utils_fakes.cc: the init and enqueue targets compile the real // utils.cc as an oracle TU, so a copy here is a duplicate symbol for them. @@ -57,5 +68,8 @@ roctx_scoped_range_in::roctx_scoped_range_in(const char*) noexcept {} roctx_scoped_range_in::~roctx_scoped_range_in() {} thread_local ncclProfilerApiState_t ncclProfilerApiState = {}; +ncclResult_t ncclProfilerStartGroupApiEvent(struct ncclInfo*, bool) { return ncclSuccess; } ncclResult_t ncclProfilerRecordGroupApiEventState(ncclProfilerEventState_t) { return ncclSuccess; } ncclResult_t ncclProfilerStopGroupApiEvent() { return ncclSuccess; } +ncclResult_t ncclProfilerStartCollApiEvent(struct ncclInfo*, bool) { return ncclSuccess; } +ncclResult_t ncclProfilerStopCollApiEvent() { return ncclSuccess; } diff --git a/projects/rccl/test/host/fakes/dda_fakes.cc b/projects/rccl/test/host/fakes/dda_fakes.cc index 1f22ac20a2b6..5b0ea2a3f7b1 100644 --- a/projects/rccl/test/host/fakes/dda_fakes.cc +++ b/projects/rccl/test/host/fakes/dda_fakes.cc @@ -11,6 +11,7 @@ #include "algorithms/dda/all_gather/dda_all_gather.h" #include "algorithms/dda/all_reduce/dda_all_reduce.h" +#include "algorithms/dda/alltoall/dda_alltoall.h" #include "algorithms/dda/reduce_scatter/dda_reduce_scatter.h" #include "signature-drift.h" @@ -54,6 +55,17 @@ DEFINE_DDA_BLOCKS(allGatherDdaFabricBlocks, ncclAllGatherDdaFabricBlocks, 122) DEFINE_DDA_BLOCKS(allGatherDdaFabricLLBlocks, ncclAllGatherDdaFabricLLBlocks, 123) DEFINE_DDA_BLOCKS(allGatherDdaFabricLL128Blocks, ncclAllGatherDdaFabricLL128Blocks, 124) +// AlltoAll takes the same operand shape as AllGather (no reduction op), so it +// reuses that macro. +DEFINE_DDA_ALLGATHER_ELIGIBLE(allToAllDdaIpcEligible, ncclAllToAllDdaIpcEligible) +DEFINE_DDA_ALLGATHER_ELIGIBLE(allToAllDdaFabricEligible, ncclAllToAllDdaFabricEligible) +DEFINE_DDA_ALLGATHER_ELIGIBLE(allToAllDdaFabricLLEligible, ncclAllToAllDdaFabricLLEligible) +DEFINE_DDA_ALLGATHER_ELIGIBLE(allToAllDdaFabricLL128Eligible, ncclAllToAllDdaFabricLL128Eligible) +DEFINE_DDA_BLOCKS(allToAllDdaIpcBlocks, ncclAllToAllDdaIpcBlocks, 141) +DEFINE_DDA_BLOCKS(allToAllDdaFabricBlocks, ncclAllToAllDdaFabricBlocks, 142) +DEFINE_DDA_BLOCKS(allToAllDdaFabricLLBlocks, ncclAllToAllDdaFabricLLBlocks, 143) +DEFINE_DDA_BLOCKS(allToAllDdaFabricLL128Blocks, ncclAllToAllDdaFabricLL128Blocks, 144) + DEFINE_DDA_REDUCTION_ELIGIBLE(reduceScatterDdaIpcEligible, ncclReduceScatterDdaIpcEligible) DEFINE_DDA_REDUCTION_ELIGIBLE(reduceScatterDdaFabricEligible, ncclReduceScatterDdaFabricEligible) DEFINE_DDA_REDUCTION_ELIGIBLE(reduceScatterDdaFabricLLEligible, ncclReduceScatterDdaFabricLLEligible) diff --git a/projects/rccl/test/host/fakes/dda_fakes.h b/projects/rccl/test/host/fakes/dda_fakes.h index 0cb86fc2532f..5b5a6be56c88 100644 --- a/projects/rccl/test/host/fakes/dda_fakes.h +++ b/projects/rccl/test/host/fakes/dda_fakes.h @@ -38,6 +38,15 @@ extern std::function g_allGatherDda extern std::function g_allGatherDdaFabricLLBlocks; extern std::function g_allGatherDdaFabricLL128Blocks; +extern std::function g_allToAllDdaIpcEligible; +extern std::function g_allToAllDdaFabricEligible; +extern std::function g_allToAllDdaFabricLLEligible; +extern std::function g_allToAllDdaFabricLL128Eligible; +extern std::function g_allToAllDdaIpcBlocks; +extern std::function g_allToAllDdaFabricBlocks; +extern std::function g_allToAllDdaFabricLLBlocks; +extern std::function g_allToAllDdaFabricLL128Blocks; + extern std::function g_reduceScatterDdaIpcEligible; extern std::function diff --git a/projects/rccl/test/host/fakes/dev_runtime_micro_fakes.cc b/projects/rccl/test/host/fakes/dev_runtime_micro_fakes.cc index 04a52b84471e..6ddb0c93bb6d 100644 --- a/projects/rccl/test/host/fakes/dev_runtime_micro_fakes.cc +++ b/projects/rccl/test/host/fakes/dev_runtime_micro_fakes.cc @@ -149,6 +149,12 @@ int64_t ncclLoadParam(char const*, int64_t deftVal, int64_t, int64_t* cache, int return deftVal; } +// Emitted by init.cc, which this binary does not link, so the redirected +// NCCL_PARAM in dev_runtime.cc does not cover it. Routed through g_loadParam +// like the params dev_runtime.cc owns, so a test can drive the NCCL_WIN_ENABLE=0 +// opt-out in ncclCommWindowRegister_impl. +int64_t ncclParamWinEnable() { return g_loadParam("WIN_ENABLE", 1); } + // --------------------------------------------------------------------------- // Proxy. // --------------------------------------------------------------------------- diff --git a/projects/rccl/test/host/fakes/enqueue_fakes.cc b/projects/rccl/test/host/fakes/enqueue_fakes.cc index 21dab8a742f2..19c1301daf6f 100644 --- a/projects/rccl/test/host/fakes/enqueue_fakes.cc +++ b/projects/rccl/test/host/fakes/enqueue_fakes.cc @@ -17,6 +17,7 @@ void ResetEnqueueFakes() { ResetCeFakes(); ResetCommFakes(); ResetDevRuntimeFakes(); + ResetGroupFakes(); ResetProxyFakes(); ResetRcclWrapFakes(); ResetRecorderFakes(); diff --git a/projects/rccl/test/host/fakes/enqueue_fakes.h b/projects/rccl/test/host/fakes/enqueue_fakes.h index e003150edc8d..99ba5b9c94a6 100644 --- a/projects/rccl/test/host/fakes/enqueue_fakes.h +++ b/projects/rccl/test/host/fakes/enqueue_fakes.h @@ -20,6 +20,7 @@ #include "comm_fakes.h" // src/init.cc comm lifecycle #include "dev_runtime_fakes.h" // src/dev_runtime.cc #include "env_fakes.h" // src/misc/param.cc + getenv interposition +#include "group_fakes.h" // src/group.cc #include "hip_fakes.h" // HIP runtime seams #include "nccl_fakes.h" // reusable nccl* seams #include "nccl_stubs.h" // core/lifecycle functors; nccl_stubs.cc is linked into this binary too diff --git a/projects/rccl/test/host/fakes/group_fakes.cc b/projects/rccl/test/host/fakes/group_fakes.cc index a230938427b8..d38230649674 100644 --- a/projects/rccl/test/host/fakes/group_fakes.cc +++ b/projects/rccl/test/host/fakes/group_fakes.cc @@ -10,8 +10,16 @@ #include +#include "group.h" // the real declarations of the thread-locals defined below #include "nccl_fakes.h" // g_loadParam, for the NCCL_PARAM default this stands in for +// group.cc's thread-local group state. ncclGroupCommJoin is inline in group.h, +// so any TU that joins a comm to a group needs these even though it never calls +// into group.cc. ncclGroupDepth/ncclGroupError sit in nccl_stubs.cc because the +// fail-loud floor already needed them. +thread_local struct ncclComm* ncclGroupCommHead[ncclGroupTaskTypeNum] = {nullptr}; +thread_local int ncclGroupBlocking = -1; // group.cc's "default mode" sentinel + ncclResult_t ncclGroupStartInternal() { return ncclSuccess; } ncclResult_t ncclGroupEndInternal(ncclSimInfo_t*) { return ncclSuccess; } diff --git a/projects/rccl/test/host/fakes/rccl_arch_thresholds_fakes.cc b/projects/rccl/test/host/fakes/rccl_arch_thresholds_fakes.cc new file mode 100644 index 000000000000..e994d2ffaac5 --- /dev/null +++ b/projects/rccl/test/host/fakes/rccl_arch_thresholds_fakes.cc @@ -0,0 +1,17 @@ +/************************************************************************* + * Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. + * + * See LICENSE.txt for license information + ************************************************************************/ + +// Fake for src/rccl_arch_thresholds.cc. +// Host-only test binaries do not link the real TU (it pulls in net.h / archinfo +// symbols that are already provided by other stubs), so this stub satisfies the +// linker. Returning nullptr is the correct "no arch table" sentinel -- every +// caller in production null-checks before dereferencing. + +#include "rccl_arch_thresholds.h" + +const rcclArchThresholds* rcclGetArchThresholds(const char* /*gcn*/) { + return nullptr; +} diff --git a/projects/rccl/test/host/fakes/rccl_wrap_fakes.cc b/projects/rccl/test/host/fakes/rccl_wrap_fakes.cc index 8fa9ce229de2..c2b2a9e2bb24 100644 --- a/projects/rccl/test/host/fakes/rccl_wrap_fakes.cc +++ b/projects/rccl/test/host/fakes/rccl_wrap_fakes.cc @@ -81,8 +81,8 @@ bool rcclIsArchSupportedForFunc(struct ncclTaskColl*, char const*) { return g_rcclIsArchSupportedForFunc; } -bool g_rcclCeAllReduceAllowed = false; -bool rcclCeAllReduceAllowed(struct ncclComm*) { return g_rcclCeAllReduceAllowed; } +bool g_rcclCeArGraphSafe = false; +bool rcclCeArGraphSafe(struct ncclComm*) { return g_rcclCeArGraphSafe; } int g_rcclCeAllReduceGraphLatchTickCalls = 0; bool g_rcclCeAllReduceGraphLatchTickLastCapturing = false; void rcclCeAllReduceGraphLatchTick(struct ncclComm*, bool ceCapturing) { @@ -152,7 +152,7 @@ void ResetRcclWrapFakes() { g_rcclOverrideProtocolResult = ncclSuccess; g_rcclOverrideAlgorithmCalls = 0; g_rcclOverrideProtocolCalls = 0; - g_rcclCeAllReduceAllowed = false; + g_rcclCeArGraphSafe = false; g_rcclCeAllReduceGraphLatchTickCalls = 0; g_rcclCeAllReduceGraphLatchTickLastCapturing = false; g_rcclWarpSpeedSupported = false; diff --git a/projects/rccl/test/host/fakes/rccl_wrap_fakes.h b/projects/rccl/test/host/fakes/rccl_wrap_fakes.h index 82e62f4dfb0f..ce8cc5671103 100644 --- a/projects/rccl/test/host/fakes/rccl_wrap_fakes.h +++ b/projects/rccl/test/host/fakes/rccl_wrap_fakes.h @@ -49,9 +49,9 @@ extern int g_rcclOverrideAlgorithmCalls; extern int g_rcclOverrideProtocolCalls; // CE (copy-engine) allreduce gates (rccl_wrap.cc:834-855). -extern bool g_rcclCeAllReduceAllowed; -extern int g_rcclCeAllReduceGraphLatchTickCalls; -extern bool g_rcclCeAllReduceGraphLatchTickLastCapturing; +extern bool g_rcclCeArGraphSafe; // UNDRIVEN +extern int g_rcclCeAllReduceGraphLatchTickCalls; // UNDRIVEN +extern bool g_rcclCeAllReduceGraphLatchTickLastCapturing; // UNDRIVEN // ------------------------------------------------------------------------- // WARP_SPEED seams (rccl_wrap.cc:1448+). enqueue.cc calls these from five sites diff --git a/projects/rccl/test/host/fakes/wrap_fakes.cc b/projects/rccl/test/host/fakes/wrap_fakes.cc index bedbf131877c..3cd517b427da 100644 --- a/projects/rccl/test/host/fakes/wrap_fakes.cc +++ b/projects/rccl/test/host/fakes/wrap_fakes.cc @@ -194,17 +194,29 @@ bool isSymmetricKernelRequested(struct ncclComm* comm, ncclFunc_t coll, int symk return g_isSymmetricKernelRequested(comm, coll, symkOp, datatype, nElts, sendbuff, recvbuff, agreeAcrossRanks); } +// The window-taking twin: rcclSelectXxx hoists the ncclDevrFindWindow lookups +// and asks this one, so it drives `symEligible` on every path a fixture can +// reach. Routed through the SAME seam rather than a second hook so one +// ScopedHook still controls the decision; the windows stand in for the buffers, +// which no fixture inspects. agreeAcrossRanks is false because the production +// window overload is the rank-local check (its caller does the agreement). +bool isSymmetricKernelRequestedWin(struct ncclComm* comm, ncclFunc_t coll, int symkOp, ncclDataType_t datatype, + size_t nElts, struct ncclDevrWindow* sendWin, struct ncclDevrWindow* recvWin) { + return g_isSymmetricKernelRequested(comm, coll, symkOp, datatype, nElts, sendWin, recvWin, + /*agreeAcrossRanks=*/false); +} + // rcclAllReduceShouldTakeDdaPath: real body lives in collectives.cc (not // linked here), same abort-floor-turned-seam treatment as the rest. Default // false: DDA not taken, letting CE-registered/symmetric/plain-kernel run. -static bool DefaultAllReduceShouldTakeDdaPath(const struct ncclComm*, size_t, ncclDataType_t, bool, bool) { +static bool DefaultAllReduceShouldTakeDdaPath(const struct ncclComm*, size_t, ncclDataType_t, bool, bool, bool) { return false; } -std::function g_allReduceShouldTakeDdaPath = +std::function g_allReduceShouldTakeDdaPath = DefaultAllReduceShouldTakeDdaPath; bool rcclAllReduceShouldTakeDdaPath(const struct ncclComm* comm, size_t count, ncclDataType_t dt, bool symEligible, - bool ceAllReduceAllowed) { - return g_allReduceShouldTakeDdaPath(comm, count, dt, symEligible, ceAllReduceAllowed); + bool ceAllReduceAllowed, bool query) { + return g_allReduceShouldTakeDdaPath(comm, count, dt, symEligible, ceAllReduceAllowed, query); } // getAlgoInfo / rcclKernelPackedChannels: rccl_wrap.cc `extern`-declares both diff --git a/projects/rccl/test/host/fakes/wrap_fakes.h b/projects/rccl/test/host/fakes/wrap_fakes.h index bb5d67d815c3..182dd8e6c661 100644 --- a/projects/rccl/test/host/fakes/wrap_fakes.h +++ b/projects/rccl/test/host/fakes/wrap_fakes.h @@ -55,7 +55,7 @@ extern std::function g_paramP2pDisable; extern std::function g_isSymmetricKernelRequested; -extern std::function g_allReduceShouldTakeDdaPath; +extern std::function g_allReduceShouldTakeDdaPath; extern std::function g_getAlgoInfo; diff --git a/projects/rccl/test/host/wrap-test.cc b/projects/rccl/test/host/wrap-test.cc index 095923e42cad..a5265a2395e0 100644 --- a/projects/rccl/test/host/wrap-test.cc +++ b/projects/rccl/test/host/wrap-test.cc @@ -640,7 +640,7 @@ TEST(WrapMicrotest, HierarchicalTempBufferSize_NeitherFlagIsZero) { } // =========================================================================== -// rcclCeAllReduceGraphLatchTick / rcclCeAllReduceAllowed -- plain ncclComm +// rcclCeAllReduceGraphLatchTick / rcclCeArGraphSafe -- plain ncclComm // field access, no topology. rccl_wrap.cc:834-857. // =========================================================================== @@ -733,14 +733,14 @@ TEST(WrapMicrotestIsolated, CeAllReduceGraphLatchTick_BothTransitionsLogTheirOwn TEST(WrapMicrotest, CeAllReduceAllowed_TrueWhenLatchClear) { ncclComm* comm = MakeZeroedComm(); comm->ceColl.graphModeSeen = false; - EXPECT_TRUE(rcclCeAllReduceAllowed(comm)); + EXPECT_TRUE(rcclCeArGraphSafe(comm)); DeleteCommWithArch(comm); } TEST(WrapMicrotest, CeAllReduceAllowed_FalseWhenLatchSet) { ncclComm* comm = MakeZeroedComm(); comm->ceColl.graphModeSeen = true; - EXPECT_FALSE(rcclCeAllReduceAllowed(comm)); + EXPECT_FALSE(rcclCeArGraphSafe(comm)); DeleteCommWithArch(comm); } @@ -3000,7 +3000,7 @@ TEST(WrapMicrotest, OverrideChannels_UndefinedFirstBucketStopsBeforeLaterMatch) } // =========================================================================== -// rcclUseCeAllReduce -- rccl_wrap.cc:766-831. Caches rcclParamCeAllReduce +// rcclUseCeAr2Shot -- rccl_wrap.cc:766-831. Caches rcclParamCeAllReduce // ("enabled") and rcclParamForceCeAllReduce ("force") in function-local // statics -- rcclParamCeAllReduce's real default (0) fully blocked this // function before the g_loadParam seam existed. Isolated per case. @@ -3015,10 +3015,10 @@ TEST(WrapMicrotestIsolated, UseCeAllReduce_DisabledByDefaultWarnsOnce) { RcclUnitTesting::ScopedDebugLogging debugLogging(NCCL_LOG_INFO, NCCL_ALL); ncclComm* comm = MakeZeroedComm(); std::string log1 = RcclUnitTesting::CaptureLog( - [&]() { EXPECT_FALSE(rcclUseCeAllReduce(comm, 8, ncclFloat32, ncclSum, nullptr)); }); + [&]() { EXPECT_FALSE(rcclUseCeAr2Shot(comm, 8, ncclFloat32, ncclSum, nullptr)); }); EXPECT_NE(std::string::npos, log1.find("CE AllReduce not enabled")); std::string log2 = RcclUnitTesting::CaptureLog( - [&]() { EXPECT_FALSE(rcclUseCeAllReduce(comm, 8, ncclFloat32, ncclSum, nullptr)); }); + [&]() { EXPECT_FALSE(rcclUseCeAr2Shot(comm, 8, ncclFloat32, ncclSum, nullptr)); }); EXPECT_EQ(std::string::npos, log2.find("CE AllReduce not enabled")); // warn-once latch DeleteCommWithArch(comm); }); @@ -3031,7 +3031,7 @@ TEST(WrapMicrotestIsolated, UseCeAllReduce_BiasBufferPresentReturnsFalse) { g_loadParam = ForceParam("RCCL_CE_ALLREDUCE", int64_t(1)); ncclComm* comm = MakeZeroedComm(); int bias = 0; - EXPECT_FALSE(rcclUseCeAllReduce(comm, 8, ncclFloat32, ncclSum, &bias)); + EXPECT_FALSE(rcclUseCeAr2Shot(comm, 8, ncclFloat32, ncclSum, &bias)); DeleteCommWithArch(comm); }); } @@ -3044,7 +3044,7 @@ TEST(WrapMicrotestIsolated, UseCeAllReduce_NoSymmetricSupportReturnsFalse) { ncclComm* comm = MakeZeroedComm(); comm->symmetricSupport = 0; std::string log = RcclUnitTesting::CaptureLog( - [&]() { EXPECT_FALSE(rcclUseCeAllReduce(comm, 8, ncclFloat32, ncclSum, nullptr)); }); + [&]() { EXPECT_FALSE(rcclUseCeAr2Shot(comm, 8, ncclFloat32, ncclSum, nullptr)); }); EXPECT_NE(std::string::npos, log.find("symmetric support is not enabled")); DeleteCommWithArch(comm); }); @@ -3059,7 +3059,7 @@ TEST(WrapMicrotestIsolated, UseCeAllReduce_MultiNodeReturnsFalse) { comm->symmetricSupport = 1; comm->nNodes = 2; std::string log = RcclUnitTesting::CaptureLog( - [&]() { EXPECT_FALSE(rcclUseCeAllReduce(comm, 8, ncclFloat32, ncclSum, nullptr)); }); + [&]() { EXPECT_FALSE(rcclUseCeAr2Shot(comm, 8, ncclFloat32, ncclSum, nullptr)); }); EXPECT_NE(std::string::npos, log.find("nNodes is not 1")); DeleteCommWithArch(comm); }); @@ -3075,7 +3075,7 @@ TEST(WrapMicrotestIsolated, UseCeAllReduce_CountNotDivisibleByNRanksReturnsFalse comm->nNodes = 1; comm->nRanks = 4; std::string log = RcclUnitTesting::CaptureLog( - [&]() { EXPECT_FALSE(rcclUseCeAllReduce(comm, /*count=*/5, ncclFloat32, ncclSum, nullptr)); }); + [&]() { EXPECT_FALSE(rcclUseCeAr2Shot(comm, /*count=*/5, ncclFloat32, ncclSum, nullptr)); }); EXPECT_NE(std::string::npos, log.find("is not divisible by nRanks")); DeleteCommWithArch(comm); }); @@ -3093,7 +3093,7 @@ TEST(WrapMicrotestIsolated, UseCeAllReduce_MsgTooLargeReturnsFalse) { // count * sizeof(float) must exceed NCCL_CE_AR_MAX_MSG_BYTES (256MiB). size_t count = (256ull * 1024 * 1024 / 4) + 1; std::string log = RcclUnitTesting::CaptureLog( - [&]() { EXPECT_FALSE(rcclUseCeAllReduce(comm, count, ncclFloat32, ncclSum, nullptr)); }); + [&]() { EXPECT_FALSE(rcclUseCeAr2Shot(comm, count, ncclFloat32, ncclSum, nullptr)); }); EXPECT_NE(std::string::npos, log.find("msgBytes")); DeleteCommWithArch(comm); }); @@ -3110,7 +3110,7 @@ TEST(WrapMicrotestIsolated, UseCeAllReduce_NonZeroCtaPolicyWithoutForceReturnsFa comm->nRanks = 1; comm->config.CTAPolicy = NCCL_CTA_POLICY_DEFAULT; std::string log = RcclUnitTesting::CaptureLog( - [&]() { EXPECT_FALSE(rcclUseCeAllReduce(comm, 8, ncclFloat32, ncclSum, nullptr)); }); + [&]() { EXPECT_FALSE(rcclUseCeAr2Shot(comm, 8, ncclFloat32, ncclSum, nullptr)); }); EXPECT_NE(std::string::npos, log.find("CTA policy is not ZERO")); DeleteCommWithArch(comm); }); @@ -3126,7 +3126,7 @@ TEST(WrapMicrotestIsolated, UseCeAllReduce_UnsupportedOpReturnsFalse) { comm->nNodes = 1; comm->nRanks = 1; comm->config.CTAPolicy = NCCL_CTA_POLICY_ZERO; - EXPECT_FALSE(rcclUseCeAllReduce(comm, 8, ncclFloat32, ncclAvg, nullptr)); + EXPECT_FALSE(rcclUseCeAr2Shot(comm, 8, ncclFloat32, ncclAvg, nullptr)); DeleteCommWithArch(comm); }); } @@ -3141,7 +3141,7 @@ TEST(WrapMicrotestIsolated, UseCeAllReduce_Float8DatatypeReturnsFalse) { comm->nNodes = 1; comm->nRanks = 1; comm->config.CTAPolicy = NCCL_CTA_POLICY_ZERO; - EXPECT_FALSE(rcclUseCeAllReduce(comm, 8, ncclFloat8e4m3, ncclSum, nullptr)); + EXPECT_FALSE(rcclUseCeAr2Shot(comm, 8, ncclFloat8e4m3, ncclSum, nullptr)); DeleteCommWithArch(comm); }); } @@ -3160,7 +3160,7 @@ TEST(WrapMicrotestIsolated, UseCeAllReduce_ForceBypassesCtaPolicyAndAllValidRetu comm->nNodes = 1; comm->nRanks = 1; comm->config.CTAPolicy = NCCL_CTA_POLICY_DEFAULT; // not ZERO -- force must bypass this - EXPECT_TRUE(rcclUseCeAllReduce(comm, 8, ncclFloat32, ncclSum, nullptr)); + EXPECT_TRUE(rcclUseCeAr2Shot(comm, 8, ncclFloat32, ncclSum, nullptr)); DeleteCommWithArch(comm); }); } @@ -3186,9 +3186,9 @@ TEST(WrapMicrotestIsolated, UseCeAllReduce_NonLogOnceGuardWarnsOnEveryCall) { comm->symmetricSupport = 1; comm->nNodes = 2; // fails the "nNodes is not 1" guard, which has no log-once protection std::string log1 = RcclUnitTesting::CaptureLog( - [&]() { EXPECT_FALSE(rcclUseCeAllReduce(comm, 8, ncclFloat32, ncclSum, nullptr)); }); + [&]() { EXPECT_FALSE(rcclUseCeAr2Shot(comm, 8, ncclFloat32, ncclSum, nullptr)); }); std::string log2 = RcclUnitTesting::CaptureLog( - [&]() { EXPECT_FALSE(rcclUseCeAllReduce(comm, 8, ncclFloat32, ncclSum, nullptr)); }); + [&]() { EXPECT_FALSE(rcclUseCeAr2Shot(comm, 8, ncclFloat32, ncclSum, nullptr)); }); EXPECT_NE(std::string::npos, log1.find("nNodes is not 1")); EXPECT_NE(std::string::npos, log2.find("nNodes is not 1")) << "if this now finds nothing, the production WARN gained log-once protection -- " @@ -3198,34 +3198,17 @@ TEST(WrapMicrotestIsolated, UseCeAllReduce_NonLogOnceGuardWarnsOnEveryCall) { } // =========================================================================== -// rcclDdaEnabled -- rccl_wrap.cc:648-667. rcclParamDdaEnable's real default -// (1) and ncclGroupDepth's real default (0) both favor the "enabled" path, +// rcclDdaEnabled -- rcclParamDdaEnable's real default (1) and +// ncclGroupDepth's real default (0) both favor the "enabled" path, // so most branches are reachable without the seam; not cached, no isolation -// needed. +// needed. Threshold is passed explicitly by the caller (from the arch table). // =========================================================================== TEST(WrapMicrotest, DdaEnabled_DisabledByParamReturnsFalse) { ScopedHook loadParam(g_loadParam, ForceParam("RCCL_DDA_ENABLE", int64_t(0))); ncclComm* comm = MakeCommWithArch("gfx942"); comm->nRanks = 8; - EXPECT_FALSE(rcclDdaEnabled(comm, /*totalBytes=*/1024, /*gfx942Default=*/2048, 0, 0)); - DeleteCommWithArch(comm); -} - -TEST(WrapMicrotest, DdaEnabled_Gfx1250UsesCallerDefaultThreshold) { - ncclComm* comm = MakeCommWithArch("gfx1250"); - EXPECT_TRUE(rcclDdaEnabled(comm, /*totalBytes=*/1024, 0, 0, /*gfx1250Default=*/2048)); - DeleteCommWithArch(comm); -} - -// Same ternary-with-fallback pattern as the gfx950 test below -// (DdaEnabled_Gfx950FallsBackToParamThresholdWhenDefaultZero), but for -// gfx1250 -- previously only the caller-provides-a-real-default case was -// exercised; this closes the fallback-path gap. -TEST(WrapMicrotest, DdaEnabled_Gfx1250FallsBackToParamThresholdWhenDefaultZero) { - ncclComm* comm = MakeCommWithArch("gfx1250"); - // gfx1250Default == 0 -> falls back to rcclParamDdaThreshold()'s real default (128MiB); well within it. - EXPECT_TRUE(rcclDdaEnabled(comm, /*totalBytes=*/1024, 0, 0, /*gfx1250Default=*/0)); + EXPECT_FALSE(rcclDdaEnabled(comm, /*totalBytes=*/1024, /*threshold=*/2048)); DeleteCommWithArch(comm); } @@ -3238,7 +3221,7 @@ TEST(WrapMicrotest, DdaEnabled_ImplicitLaunchOrderReturnsFalse) { ncclComm* comm = MakeCommWithArch("gfx942"); comm->nRanks = 8; comm->config.launchOrderImplicit = 1; - EXPECT_FALSE(rcclDdaEnabled(comm, /*totalBytes=*/1024, /*gfx942Default=*/2048, 0, 0)); + EXPECT_FALSE(rcclDdaEnabled(comm, /*totalBytes=*/1024, /*threshold=*/2048)); DeleteCommWithArch(comm); } @@ -3250,7 +3233,7 @@ TEST(WrapMicrotest, DdaEnabled_InsideGroupReturnsFalse) { ncclComm* comm = MakeCommWithArch("gfx942"); comm->nRanks = 8; ncclGroupDepth = 1; - EXPECT_FALSE(rcclDdaEnabled(comm, /*totalBytes=*/1024, /*gfx942Default=*/2048, 0, 0)); + EXPECT_FALSE(rcclDdaEnabled(comm, /*totalBytes=*/1024, /*threshold=*/2048)); ncclGroupDepth = 0; // restore: other tests in this binary assume the default DeleteCommWithArch(comm); } @@ -3258,36 +3241,28 @@ TEST(WrapMicrotest, DdaEnabled_InsideGroupReturnsFalse) { TEST(WrapMicrotest, DdaEnabled_Gfx942BelowRankThresholdReturnsFalse) { ncclComm* comm = MakeCommWithArch("gfx942"); comm->nRanks = 7; // below the 8-rank floor - EXPECT_FALSE(rcclDdaEnabled(comm, /*totalBytes=*/1024, /*gfx942Default=*/2048, 0, 0)); + EXPECT_FALSE(rcclDdaEnabled(comm, /*totalBytes=*/1024, /*threshold=*/2048)); DeleteCommWithArch(comm); } TEST(WrapMicrotest, DdaEnabled_Gfx942WithinThresholdReturnsTrue) { ncclComm* comm = MakeCommWithArch("gfx942"); comm->nRanks = 8; - EXPECT_TRUE(rcclDdaEnabled(comm, /*totalBytes=*/2048, /*gfx942Default=*/2048, 0, 0)); + EXPECT_TRUE(rcclDdaEnabled(comm, /*totalBytes=*/2048, /*threshold=*/2048)); DeleteCommWithArch(comm); } TEST(WrapMicrotest, DdaEnabled_Gfx942AboveThresholdReturnsFalse) { ncclComm* comm = MakeCommWithArch("gfx942"); comm->nRanks = 8; - EXPECT_FALSE(rcclDdaEnabled(comm, /*totalBytes=*/2049, /*gfx942Default=*/2048, 0, 0)); - DeleteCommWithArch(comm); -} - -TEST(WrapMicrotest, DdaEnabled_Gfx950FallsBackToParamThresholdWhenDefaultZero) { - ncclComm* comm = MakeCommWithArch("gfx950"); - comm->nRanks = 8; - // gfx950Default == 0 -> falls back to rcclParamDdaThreshold()'s real default (128MiB); well within it. - EXPECT_TRUE(rcclDdaEnabled(comm, /*totalBytes=*/1024, 0, /*gfx950Default=*/0, 0)); + EXPECT_FALSE(rcclDdaEnabled(comm, /*totalBytes=*/2049, /*threshold=*/2048)); DeleteCommWithArch(comm); } TEST(WrapMicrotest, DdaEnabled_UnsupportedArchReturnsFalse) { ncclComm* comm = MakeCommWithArch("gfx90a"); comm->nRanks = 8; - EXPECT_FALSE(rcclDdaEnabled(comm, /*totalBytes=*/1024, /*gfx942Default=*/2048, /*gfx950Default=*/2048, 0)); + EXPECT_FALSE(rcclDdaEnabled(comm, /*totalBytes=*/1024, /*threshold=*/2048)); DeleteCommWithArch(comm); } @@ -3415,7 +3390,7 @@ TEST(WrapMicrotest, GetFirmwareVersion_FailureReturnsNegativeOne) { // rcclSelectAllReduce -- rccl_wrap.cc:864-1026. The master AllReduce decision // function: symmetric -> CE 2-shot -> DDA (fabric LL/LL128/VMM + IPC) -> CE // registered -> symmetric (reported) -> plain kernel, in that priority -// order. Every test is isolated: rcclUseCeAllReduce (called internally to +// order. Every test is isolated: rcclUseCeAr2Shot (called internally to // compute ceAllReduceAllowed) caches its enabled/force flags in // function-local statics, so any test touching RCCL_CE_ALLREDUCE/ // RCCL_FORCE_CE_ALLREDUCE needs a fresh process, and for consistency every @@ -3491,17 +3466,25 @@ TEST(WrapMicrotestIsolated, SelectAllReduce_SymmetricEligibleChoosesSymmetric) { // CE registered wins over symmetric when both are eligible, per the // production comment's documented precedence -- distinct from the test // above, which only proves symmetric wins when CE ISN'T eligible. -TEST(WrapMicrotestIsolated, SelectAllReduce_CeRegisteredBeatsSymmetricWhenBothEligible) { +TEST(WrapMicrotestIsolated, SelectAllReduce_SymmetricBeatesCeRegisteredWhenBothEligible) { RUN_ISOLATED_TEST( - "Wrap_SelectAllReduce_CeRegisteredBeatsSymmetricWhenBothEligible", + "Wrap_SelectAllReduce_SymmetricBeatesCeRegisteredWhenBothEligible", []() { - g_loadParam = ForceParam("RCCL_CE_ALLREDUCE", int64_t(1)); + g_loadParam = [](const char* env, int64_t def) -> int64_t { + if (std::strcmp(env, "RCCL_CE_ALLREDUCE") == 0) return 1; + if (std::strcmp(env, "RCCL_CE_AR_REG_MAX_MSG_BYTES") == 0) return INT64_MAX; + return def; + }; ScopedHook symRequested(g_isSymmetricKernelRequested, [](struct ncclComm*, ncclFunc_t, int, ncclDataType_t, size_t, const void*, void*, bool) { return true; }); ScopedHook ceAvailable( g_ceAvailable, [](struct ncclComm*, ncclFunc_t, int, ncclDataType_t, ncclSymRegType_t, struct ncclDevrWindow*, struct ncclDevrWindow*) { return true; }); + ScopedHook symkAvailable(g_symkAvailable, + [](struct ncclComm*, ncclFunc_t, int, ncclDataType_t, size_t) { return true; }); + ScopedHook tuningCompute(g_tuningCompute, + SelectSymkTuning(ncclSymkKernelId_AllReduce_RSxLD_AGxST, /*maxChannels=*/6)); ncclComm* comm = MakeSelectComm(); comm->symmetricSupport = 1; comm->config.CTAPolicy = NCCL_CTA_POLICY_ZERO; @@ -3509,9 +3492,7 @@ TEST(WrapMicrotestIsolated, SelectAllReduce_CeRegisteredBeatsSymmetricWhenBothEl EXPECT_EQ(ncclSuccess, rcclSelectAllReduce(comm, nullptr, nullptr, /*count=*/8, ncclFloat32, ncclSum, /*stream=*/nullptr, /*query=*/true, /*graphCapturingHint=*/false, &decision)); - EXPECT_EQ((int)rcclAddonAlgos_t::RCCL_CE_REGISTERED, decision.algo); - EXPECT_EQ(NCCL_PROTO_SIMPLE, decision.protocol); - EXPECT_EQ(1, decision.nMaxChannels); + EXPECT_EQ((int)rcclAddonAlgos_t::RCCL_SYMMETRIC, decision.algo); DeleteCommWithArch(comm); }); } @@ -3524,7 +3505,11 @@ TEST(WrapMicrotestIsolated, SelectAllReduce_CeRegisteredChosenWithProdOp) { RUN_ISOLATED_TEST( "Wrap_SelectAllReduce_CeRegisteredChosenWithProdOp", []() { - g_loadParam = ForceParam("RCCL_CE_ALLREDUCE", int64_t(1)); + g_loadParam = [](const char* env, int64_t def) -> int64_t { + if (std::strcmp(env, "RCCL_CE_ALLREDUCE") == 0) return 1; + if (std::strcmp(env, "RCCL_CE_AR_REG_MAX_MSG_BYTES") == 0) return INT64_MAX; + return def; + }; ScopedHook ceAvailable( g_ceAvailable, [](struct ncclComm*, ncclFunc_t, int, ncclDataType_t, ncclSymRegType_t, @@ -3603,7 +3588,11 @@ TEST(WrapMicrotestIsolated, SelectAllReduce_CeRegisteredChosenWhenAvailableAndPo RUN_ISOLATED_TEST( "Wrap_SelectAllReduce_CeRegisteredChosenWhenAvailableAndPolicyZero", []() { - g_loadParam = ForceParam("RCCL_CE_ALLREDUCE", int64_t(1)); + g_loadParam = [](const char* env, int64_t def) -> int64_t { + if (std::strcmp(env, "RCCL_CE_ALLREDUCE") == 0) return 1; + if (std::strcmp(env, "RCCL_CE_AR_REG_MAX_MSG_BYTES") == 0) return INT64_MAX; + return def; + }; ScopedHook ceAvailable( g_ceAvailable, [](struct ncclComm*, ncclFunc_t, int, ncclDataType_t, ncclSymRegType_t, @@ -3625,7 +3614,11 @@ TEST(WrapMicrotestIsolated, SelectAllReduce_CeRegisteredForwardsBlockCalculatorA RUN_ISOLATED_TEST( "Wrap_SelectAllReduce_CeRegisteredForwardsBlockCalculatorArguments", []() { - g_loadParam = ForceParam("RCCL_CE_ALLREDUCE", int64_t(1)); + g_loadParam = [](const char* env, int64_t def) -> int64_t { + if (std::strcmp(env, "RCCL_CE_ALLREDUCE") == 0) return 1; + if (std::strcmp(env, "RCCL_CE_AR_REG_MAX_MSG_BYTES") == 0) return INT64_MAX; + return def; + }; ScopedHook ceAvailable( g_ceAvailable, [](struct ncclComm*, ncclFunc_t, int, ncclDataType_t, ncclSymRegType_t, @@ -3697,10 +3690,10 @@ TEST(WrapMicrotestIsolated, SelectAllReduce_RecvWinSysmemSegmentBlocksCeRegister }); } -// CE 2-shot: needs ceAllReduceAllowed (rcclUseCeAllReduce eligible + +// CE 2-shot: needs ceAllReduceAllowed (rcclUseCeAr2Shot eligible + // force-or-symReg) AND a non-null ceARTmpBuf. Uses `force` (via // RCCL_FORCE_CE_ALLREDUCE) as the "force || symReg" side, matching -// rcclUseCeAllReduce's own ForceBypassesCtaPolicy test precedent. +// rcclUseCeAr2Shot's own ForceBypassesCtaPolicy test precedent. TEST(WrapMicrotestIsolated, SelectAllReduce_CeTwoShotChosenWhenEligibleAndStagingBufferReady) { RUN_ISOLATED_TEST( "Wrap_SelectAllReduce_CeTwoShotChosenWhenEligibleAndStagingBufferReady", @@ -3717,7 +3710,7 @@ TEST(WrapMicrotestIsolated, SelectAllReduce_CeTwoShotChosenWhenEligibleAndStagin }); ncclComm* comm = MakeSelectComm(); comm->symmetricSupport = 1; - // force bypasses this, matching rcclUseCeAllReduce's precedent + // force bypasses this, matching rcclUseCeAr2Shot's precedent comm->config.CTAPolicy = NCCL_CTA_POLICY_DEFAULT; uint8_t stagingBuf[16]; comm->ceColl.ceARTmpBuf = stagingBuf; @@ -3734,7 +3727,7 @@ TEST(WrapMicrotestIsolated, SelectAllReduce_CeTwoShotChosenWhenEligibleAndStagin // ceAllReduceAllowed's final conjunct is `force || symReg`; every prior // test drove it true via one side or the other, but never both false at -// once. Same setup as above (staging buffer ready, rcclUseCeAllReduce +// once. Same setup as above (staging buffer ready, rcclUseCeAr2Shot // otherwise eligible), but neither RCCL_FORCE_CE_ALLREDUCE nor a CE- // available seam is set -- CE 2-shot must not fire despite the buffer // being ready. @@ -3745,7 +3738,7 @@ TEST(WrapMicrotestIsolated, SelectAllReduce_CeTwoShotNotChosenWhenNeitherForceNo g_loadParam = ForceParam("RCCL_CE_ALLREDUCE", int64_t(1)); ncclComm* comm = MakeSelectComm(); comm->symmetricSupport = 1; - comm->config.CTAPolicy = NCCL_CTA_POLICY_ZERO; // rcclUseCeAllReduce itself still eligible + comm->config.CTAPolicy = NCCL_CTA_POLICY_ZERO; // rcclUseCeAr2Shot itself still eligible uint8_t stagingBuf[16]; comm->ceColl.ceARTmpBuf = stagingBuf; rcclCollDecision decision{}; @@ -3790,7 +3783,7 @@ TEST(WrapMicrotestIsolated, SelectAllReduce_DdaFabricLLChosenOnGfx1250) { "Wrap_SelectAllReduce_DdaFabricLLChosenOnGfx1250", []() { ScopedHook shouldTakeDda(g_allReduceShouldTakeDdaPath, - [](const struct ncclComm*, size_t, ncclDataType_t, bool, bool) { return true; }); + [](const struct ncclComm*, size_t, ncclDataType_t, bool, bool, bool) { return true; }); ScopedHook llEligible(g_allReduceDdaFabricLLEligible, [](ncclComm*, const void*, void*, size_t, ncclDataType_t, ncclRedOp_t) { return true; }); ncclComm* comm = MakeCommWithArch("gfx1250"); @@ -3815,9 +3808,13 @@ TEST(WrapMicrotestIsolated, SelectAllReduce_DdaFabricLL128ChosenWhenLLNotEligibl RUN_ISOLATED_TEST( "Wrap_SelectAllReduce_DdaFabricLL128ChosenWhenLLNotEligible", []() { - g_loadParam = ForceParam("RCCL_DDA_LL128", int64_t(1)); + g_loadParam = [](const char* env, int64_t def) -> int64_t { + if (std::strcmp(env, "RCCL_DDA_LL128") == 0) return 1; + if (std::strcmp(env, "RCCL_DDA_LL128_THRESHOLD") == 0) return INT64_MAX; + return def; + }; ScopedHook shouldTakeDda(g_allReduceShouldTakeDdaPath, - [](const struct ncclComm*, size_t, ncclDataType_t, bool, bool) { return true; }); + [](const struct ncclComm*, size_t, ncclDataType_t, bool, bool, bool) { return true; }); ScopedHook ll128Eligible(g_allReduceDdaFabricLL128Eligible, [](ncclComm*, const void*, void*, size_t, ncclDataType_t, ncclRedOp_t) { return true; @@ -3845,8 +3842,12 @@ TEST(WrapMicrotestIsolated, SelectAllReduce_DdaFabricLL128ChosenAboveLegacyThres RUN_ISOLATED_TEST( "Wrap_SelectAllReduce_DdaFabricLL128ChosenAboveLegacyThreshold", []() { + g_loadParam = [](const char* env, int64_t def) -> int64_t { + if (std::strcmp(env, "RCCL_DDA_LL128_THRESHOLD") == 0) return INT64_MAX; + return def; + }; ScopedHook shouldTakeDda(g_allReduceShouldTakeDdaPath, - [](const struct ncclComm*, size_t, ncclDataType_t, bool, bool) { return true; }); + [](const struct ncclComm*, size_t, ncclDataType_t, bool, bool, bool) { return true; }); ScopedHook ll128Eligible(g_allReduceDdaFabricLL128Eligible, [](ncclComm*, const void*, void*, size_t, ncclDataType_t, ncclRedOp_t) { return true; @@ -3875,7 +3876,7 @@ TEST(WrapMicrotestIsolated, SelectAllReduce_DdaFabricVmmChosenWhenNeitherLLNorLL "Wrap_SelectAllReduce_DdaFabricVmmChosenWhenNeitherLLNorLL128Eligible", []() { ScopedHook shouldTakeDda(g_allReduceShouldTakeDdaPath, - [](const struct ncclComm*, size_t, ncclDataType_t, bool, bool) { return true; }); + [](const struct ncclComm*, size_t, ncclDataType_t, bool, bool, bool) { return true; }); ScopedHook vmmEligible(g_allReduceDdaFabricEligible, [](ncclComm*, const void*, void*, size_t, ncclDataType_t, ncclRedOp_t) { return true; }); // g_allReduceDdaFabricLLEligible / LL128Eligible left at their default (false). @@ -3900,7 +3901,7 @@ TEST(WrapMicrotestIsolated, SelectAllReduce_DdaIpcChosenOnNonGfx1250Arch) { "Wrap_SelectAllReduce_DdaIpcChosenOnNonGfx1250Arch", []() { ScopedHook shouldTakeDda(g_allReduceShouldTakeDdaPath, - [](const struct ncclComm*, size_t, ncclDataType_t, bool, bool) { return true; }); + [](const struct ncclComm*, size_t, ncclDataType_t, bool, bool, bool) { return true; }); ScopedHook ipcEligible(g_allReduceDdaIpcEligible, [](ncclComm*, const void*, void*, size_t, ncclDataType_t, ncclRedOp_t) { return true; }); ncclComm* comm = MakeCommWithArch("gfx942"); // not gfx1250 @@ -4075,7 +4076,7 @@ TEST(WrapMicrotestIsolated, SelectAllReduce_LiveModeProbesCapturingGraph) { // rcclSelectAllGather -- rccl_wrap.cc:1031-1202. Same overall shape as // rcclSelectAllReduce, different priority order: DDA -> Hierarchical -> CE // (force-scratch + registered) -> symmetric (reported) -> Direct -> plain -// kernel. Isolated throughout for the same reason (rcclUseCeAllReduce is not +// kernel. Isolated throughout for the same reason (rcclUseCeAr2Shot is not // called here, but rcclUseAllGatherDirect's own disable-flag/threshold // statics are reached via the Direct branch and the Hierarchical branch's // sub-comm dispatch). @@ -4093,7 +4094,8 @@ TEST(WrapMicrotestIsolated, SelectAllGather_DdaFabricLLChosenOnGfx1250) { comm->nNodes = 1; rcclCollDecision decision{}; EXPECT_EQ(ncclSuccess, rcclSelectAllGather(comm, nullptr, nullptr, /*sendcount=*/8, ncclFloat32, - /*query=*/true, /*graphCapturingHint=*/false, &decision)); + /*stream=*/nullptr, /*query=*/true, /*graphCapturingHint=*/false, + &decision)); EXPECT_EQ((int)rcclAddonAlgos_t::RCCL_DDA_FABRIC_LL, decision.algo); EXPECT_EQ(123u, decision.nMaxChannels); // proves ncclAllGatherDdaFabricLLBlocks ran, not a sibling @@ -4106,7 +4108,11 @@ TEST(WrapMicrotestIsolated, SelectAllGather_DdaFabricLL128ChosenWhenLLNotEligibl RUN_ISOLATED_TEST( "Wrap_SelectAllGather_DdaFabricLL128ChosenWhenLLNotEligible", []() { - g_loadParam = ForceParam("RCCL_DDA_LL128", int64_t(1)); + g_loadParam = [](const char* env, int64_t def) -> int64_t { + if (std::strcmp(env, "RCCL_DDA_LL128") == 0) return 1; + if (std::strcmp(env, "RCCL_DDA_LL128_THRESHOLD") == 0) return INT64_MAX; + return def; + }; ScopedHook ll128Eligible(g_allGatherDdaFabricLL128Eligible, [](ncclComm*, const void*, void*, size_t, ncclDataType_t) { return true; }); // g_allGatherDdaFabricLLEligible left at its default (false). @@ -4115,7 +4121,8 @@ TEST(WrapMicrotestIsolated, SelectAllGather_DdaFabricLL128ChosenWhenLLNotEligibl comm->nNodes = 1; rcclCollDecision decision{}; EXPECT_EQ(ncclSuccess, rcclSelectAllGather(comm, nullptr, nullptr, /*sendcount=*/8, ncclFloat32, - /*query=*/true, /*graphCapturingHint=*/false, &decision)); + /*stream=*/nullptr, /*query=*/true, /*graphCapturingHint=*/false, + &decision)); EXPECT_EQ((int)rcclAddonAlgos_t::RCCL_DDA_FABRIC_LL128, decision.algo); EXPECT_EQ(124u, decision.nMaxChannels); // proves ncclAllGatherDdaFabricLL128Blocks ran, not a sibling @@ -4135,7 +4142,8 @@ TEST(WrapMicrotestIsolated, SelectAllGather_DdaFabricVmmChosenWhenNeitherLLNorLL comm->nNodes = 1; rcclCollDecision decision{}; EXPECT_EQ(ncclSuccess, rcclSelectAllGather(comm, nullptr, nullptr, /*sendcount=*/8, ncclFloat32, - /*query=*/true, /*graphCapturingHint=*/false, &decision)); + /*stream=*/nullptr, /*query=*/true, /*graphCapturingHint=*/false, + &decision)); EXPECT_EQ((int)rcclAddonAlgos_t::RCCL_DDA_FABRIC_VMM, decision.algo); EXPECT_EQ(122u, decision.nMaxChannels); // proves ncclAllGatherDdaFabricBlocks ran, not a sibling @@ -4156,7 +4164,8 @@ TEST(WrapMicrotestIsolated, SelectAllGather_DdaIpcChosenOnNonGfx1250Arch) { comm->nNodes = 1; rcclCollDecision decision{}; EXPECT_EQ(ncclSuccess, rcclSelectAllGather(comm, nullptr, nullptr, /*sendcount=*/8, ncclFloat32, - /*query=*/true, /*graphCapturingHint=*/false, &decision)); + /*stream=*/nullptr, /*query=*/true, /*graphCapturingHint=*/false, + &decision)); EXPECT_EQ((int)rcclAddonAlgos_t::RCCL_DDA_IPC, decision.algo); EXPECT_EQ(121u, decision.nMaxChannels); // proves ncclAllGatherDdaIpcBlocks ran, not a sibling @@ -4187,9 +4196,9 @@ TEST(WrapMicrotestIsolated, SelectAllGather_DdaGatedOnNotSymEligible) { comm->nNodes = 1; rcclCollDecision decision{}; EXPECT_EQ(ncclSuccess, rcclSelectAllGather(comm, nullptr, nullptr, /*sendcount=*/8, ncclFloat32, - /*query=*/false, /*graphCapturingHint=*/false, &decision)); + /*stream=*/nullptr, /*query=*/false, /*graphCapturingHint=*/false, + &decision)); EXPECT_NE((int)rcclAddonAlgos_t::RCCL_DDA_IPC, decision.algo); - EXPECT_EQ(NCCL_ALGO_RING, decision.algo); // fell all the way to the plain-kernel placeholder DeleteCommWithArch(comm); }); } @@ -4208,8 +4217,8 @@ TEST(WrapMicrotestIsolated, SelectAllGather_HierarchicalChosenLiveMode) { comm->hierarchicalCommsInitialized = true; rcclCollDecision decision{}; EXPECT_EQ(ncclSuccess, - rcclSelectAllGather(comm, nullptr, nullptr, /*sendcount=*/8, ncclFloat32, /*query=*/false, - /*graphCapturingHint=*/false, &decision)); + rcclSelectAllGather(comm, nullptr, nullptr, /*sendcount=*/8, ncclFloat32, /*stream=*/nullptr, + /*query=*/false, /*graphCapturingHint=*/false, &decision)); EXPECT_EQ((int)rcclAddonAlgos_t::RCCL_HIERARCHICAL_ALLGATHER, decision.algo); DeleteCommWithArch(comm); }); @@ -4226,8 +4235,8 @@ TEST(WrapMicrotestIsolated, SelectAllGather_InsideGroupExcludesHierarchical) { ncclGroupDepth = 1; rcclCollDecision decision{}; EXPECT_EQ(ncclSuccess, - rcclSelectAllGather(comm, nullptr, nullptr, /*sendcount=*/8, ncclFloat32, /*query=*/false, - /*graphCapturingHint=*/false, &decision)); + rcclSelectAllGather(comm, nullptr, nullptr, /*sendcount=*/8, ncclFloat32, /*stream=*/nullptr, + /*query=*/false, /*graphCapturingHint=*/false, &decision)); EXPECT_EQ(NCCL_ALGO_RING, decision.algo); DeleteCommWithArch(comm); }); @@ -4252,8 +4261,9 @@ TEST(WrapMicrotestIsolated, SelectAllGather_CeForceScratchChosen) { comm->ddaScratchBytes = sizeof(scratch); rcclCollDecision decision{}; EXPECT_EQ(ncclSuccess, rcclSelectAllGather(comm, nullptr, nullptr, /*sendcount=*/8, ncclFloat32, - /*query=*/true, /*graphCapturingHint=*/false, &decision)); - EXPECT_EQ((int)rcclAddonAlgos_t::RCCL_CE_REGISTERED, decision.algo); + /*stream=*/nullptr, /*query=*/true, /*graphCapturingHint=*/false, + &decision)); + EXPECT_EQ((int)rcclAddonAlgos_t::RCCL_CE_SCRATCH, decision.algo); DeleteCommWithArch(comm); }); } @@ -4270,7 +4280,8 @@ TEST(WrapMicrotestIsolated, SelectAllGather_CaptureExcludesCeForceScratch) { comm->ddaScratchBytes = sizeof(scratch); rcclCollDecision decision{}; EXPECT_EQ(ncclSuccess, rcclSelectAllGather(comm, nullptr, nullptr, /*sendcount=*/8, ncclFloat32, - /*query=*/true, /*graphCapturingHint=*/true, &decision)); + /*stream=*/nullptr, /*query=*/true, /*graphCapturingHint=*/true, + &decision)); EXPECT_EQ(NCCL_ALGO_RING, decision.algo); DeleteCommWithArch(comm); }); @@ -4288,7 +4299,8 @@ TEST(WrapMicrotestIsolated, SelectAllGather_CaptureExcludesCeRegistered) { comm->config.CTAPolicy = NCCL_CTA_POLICY_ZERO; rcclCollDecision decision{}; EXPECT_EQ(ncclSuccess, rcclSelectAllGather(comm, nullptr, nullptr, /*sendcount=*/8, ncclFloat32, - /*query=*/true, /*graphCapturingHint=*/true, &decision)); + /*stream=*/nullptr, /*query=*/true, /*graphCapturingHint=*/true, + &decision)); EXPECT_EQ(NCCL_ALGO_RING, decision.algo); DeleteCommWithArch(comm); }); @@ -4310,7 +4322,8 @@ TEST(WrapMicrotestIsolated, SelectAllGather_CeForceScratchRequiresForceCe) { comm->ddaScratchBytes = sizeof(scratch); rcclCollDecision decision{}; EXPECT_EQ(ncclSuccess, rcclSelectAllGather(comm, nullptr, nullptr, /*sendcount=*/8, ncclFloat32, - /*query=*/true, /*graphCapturingHint=*/false, &decision)); + /*stream=*/nullptr, /*query=*/true, /*graphCapturingHint=*/false, + &decision)); EXPECT_EQ(NCCL_ALGO_RING, decision.algo); DeleteCommWithArch(comm); }); @@ -4331,8 +4344,9 @@ TEST(WrapMicrotestIsolated, SelectAllGather_CeForceScratchRejectsFullyRegistered comm->ddaScratch = scratch; comm->ddaScratchBytes = sizeof(scratch); rcclCollDecision decision{}; - EXPECT_EQ(ncclSuccess, rcclSelectAllGather(comm, nullptr, nullptr, 8, ncclFloat32, true, false, &decision)); - EXPECT_EQ(NCCL_ALGO_RING, decision.algo); + EXPECT_EQ(ncclSuccess, rcclSelectAllGather(comm, nullptr, nullptr, 8, ncclFloat32, /*stream=*/nullptr, true, + false, &decision)); + EXPECT_NE((int)rcclAddonAlgos_t::RCCL_CE_SCRATCH, decision.algo); DeleteCommWithArch(comm); }); } @@ -4352,8 +4366,9 @@ TEST(WrapMicrotestIsolated, SelectAllGather_CeForceScratchRejectsRegisteredRecvW comm->ddaScratch = scratch; comm->ddaScratchBytes = sizeof(scratch); rcclCollDecision decision{}; - EXPECT_EQ(ncclSuccess, rcclSelectAllGather(comm, nullptr, nullptr, 8, ncclFloat32, true, false, &decision)); - EXPECT_EQ(NCCL_ALGO_RING, decision.algo); + EXPECT_EQ(ncclSuccess, rcclSelectAllGather(comm, nullptr, nullptr, 8, ncclFloat32, /*stream=*/nullptr, true, + false, &decision)); + EXPECT_NE((int)rcclAddonAlgos_t::RCCL_CE_SCRATCH, decision.algo); DeleteCommWithArch(comm); }); } @@ -4372,7 +4387,8 @@ TEST(WrapMicrotestIsolated, SelectAllGather_CeForceScratchRejectsSysmemSegment) comm->ddaScratchBytes = sizeof(scratch); rcclCollDecision decision{}; EXPECT_EQ(ncclSuccess, - rcclSelectAllGather(comm, &sendBuffer, &recvBuffer, 8, ncclFloat32, true, false, &decision)); + rcclSelectAllGather(comm, &sendBuffer, &recvBuffer, 8, ncclFloat32, /*stream=*/nullptr, true, false, + &decision)); EXPECT_EQ(NCCL_ALGO_RING, decision.algo); DeleteCommWithArch(comm); }); @@ -4390,8 +4406,9 @@ TEST(WrapMicrotestIsolated, SelectAllGather_CeForceScratchAcceptsExactCapacity) comm->ddaScratchBytes = sizeof(scratch); rcclCollDecision decision{}; // nRanks(1) * sendcount(16) * sizeof(float32)(4) == 64 bytes. - EXPECT_EQ(ncclSuccess, rcclSelectAllGather(comm, nullptr, nullptr, 16, ncclFloat32, true, false, &decision)); - EXPECT_EQ((int)rcclAddonAlgos_t::RCCL_CE_REGISTERED, decision.algo); + EXPECT_EQ(ncclSuccess, rcclSelectAllGather(comm, nullptr, nullptr, 16, ncclFloat32, /*stream=*/nullptr, true, + false, &decision)); + EXPECT_EQ((int)rcclAddonAlgos_t::RCCL_CE_SCRATCH, decision.algo); DeleteCommWithArch(comm); }); } @@ -4417,7 +4434,8 @@ TEST(WrapMicrotestIsolated, SelectAllGather_CeForceScratchNotChosenWhenProbeSays comm->ddaScratchBytes = sizeof(scratch); rcclCollDecision decision{}; EXPECT_EQ(ncclSuccess, rcclSelectAllGather(comm, nullptr, nullptr, /*sendcount=*/8, ncclFloat32, - /*query=*/true, /*graphCapturingHint=*/false, &decision)); + /*stream=*/nullptr, /*query=*/true, /*graphCapturingHint=*/false, + &decision)); EXPECT_NE((int)rcclAddonAlgos_t::RCCL_CE_REGISTERED, decision.algo) << "ncclCeScratchAvailable() returning false must block the force-scratch branch"; DeleteCommWithArch(comm); @@ -4444,7 +4462,8 @@ TEST(WrapMicrotestIsolated, SelectAllGather_CeRegisteredViaSymmetricWindowsChose comm->config.CTAPolicy = NCCL_CTA_POLICY_ZERO; rcclCollDecision decision{}; EXPECT_EQ(ncclSuccess, rcclSelectAllGather(comm, nullptr, nullptr, /*sendcount=*/8, ncclFloat32, - /*query=*/true, /*graphCapturingHint=*/false, &decision)); + /*stream=*/nullptr, /*query=*/true, /*graphCapturingHint=*/false, + &decision)); EXPECT_EQ((int)rcclAddonAlgos_t::RCCL_CE_REGISTERED, decision.algo); // ...and the CTA-policy conjunct itself: same everything, policy back @@ -4453,7 +4472,8 @@ TEST(WrapMicrotestIsolated, SelectAllGather_CeRegisteredViaSymmetricWindowsChose comm->config.CTAPolicy = 0; rcclCollDecision noPolicy{}; EXPECT_EQ(ncclSuccess, rcclSelectAllGather(comm, nullptr, nullptr, /*sendcount=*/8, ncclFloat32, - /*query=*/true, /*graphCapturingHint=*/false, &noPolicy)); + /*stream=*/nullptr, /*query=*/true, /*graphCapturingHint=*/false, + &noPolicy)); EXPECT_NE((int)rcclAddonAlgos_t::RCCL_CE_REGISTERED, noPolicy.algo) << "CE via symmetric windows must require CTA_POLICY_ZERO"; DeleteCommWithArch(comm); @@ -4477,7 +4497,8 @@ TEST(WrapMicrotestIsolated, SelectAllGather_SysmemSegmentBlocksCeRegistered) { TestDevrWindows registered(comm, &sendBuffer, true, &recvBuffer, false); rcclCollDecision decision{}; EXPECT_EQ(ncclSuccess, rcclSelectAllGather(comm, &sendBuffer, &recvBuffer, /*sendcount=*/8, ncclFloat32, - /*query=*/true, /*graphCapturingHint=*/false, &decision)); + /*stream=*/nullptr, /*query=*/true, /*graphCapturingHint=*/false, + &decision)); EXPECT_NE((int)rcclAddonAlgos_t::RCCL_CE_REGISTERED, decision.algo); DeleteCommWithArch(comm); }); @@ -4501,7 +4522,8 @@ TEST(WrapMicrotestIsolated, SelectAllGather_RecvWinSysmemSegmentBlocksCeRegister TestDevrWindows registered(comm, &sendSentinel, false, &recvSentinel, true); rcclCollDecision decision{}; EXPECT_EQ(ncclSuccess, rcclSelectAllGather(comm, &sendSentinel, &recvSentinel, /*sendcount=*/8, ncclFloat32, - /*query=*/true, /*graphCapturingHint=*/false, &decision)); + /*stream=*/nullptr, /*query=*/true, /*graphCapturingHint=*/false, + &decision)); EXPECT_NE((int)rcclAddonAlgos_t::RCCL_CE_REGISTERED, decision.algo); DeleteCommWithArch(comm); }); @@ -4523,7 +4545,8 @@ TEST(WrapMicrotestIsolated, SelectAllGather_ForceScratchBlockedWhenScratchBuffer // comm->ddaScratch left at its zero-init default (nullptr). rcclCollDecision decision{}; EXPECT_EQ(ncclSuccess, rcclSelectAllGather(comm, nullptr, nullptr, /*sendcount=*/8, ncclFloat32, - /*query=*/true, /*graphCapturingHint=*/false, &decision)); + /*stream=*/nullptr, /*query=*/true, /*graphCapturingHint=*/false, + &decision)); EXPECT_NE((int)rcclAddonAlgos_t::RCCL_CE_REGISTERED, decision.algo); DeleteCommWithArch(comm); }); @@ -4547,7 +4570,8 @@ TEST(WrapMicrotestIsolated, SelectAllGather_ForceScratchBlockedWhenMessageExceed rcclCollDecision decision{}; // totalBytes = nRanks(1) * sendcount(1024) * sizeof(float32)(4) = 4096, > 64. EXPECT_EQ(ncclSuccess, rcclSelectAllGather(comm, nullptr, nullptr, /*sendcount=*/1024, ncclFloat32, - /*query=*/true, /*graphCapturingHint=*/false, &decision)); + /*stream=*/nullptr, /*query=*/true, /*graphCapturingHint=*/false, + &decision)); EXPECT_NE((int)rcclAddonAlgos_t::RCCL_CE_REGISTERED, decision.algo); DeleteCommWithArch(comm); }); @@ -4573,7 +4597,8 @@ TEST(WrapMicrotestIsolated, SelectAllGather_SymmetricReportedWhenQueryAndEligibl comm->symmetricSupport = 1; rcclCollDecision decision{}; EXPECT_EQ(ncclSuccess, rcclSelectAllGather(comm, nullptr, nullptr, /*sendcount=*/8, ncclFloat32, - /*query=*/true, /*graphCapturingHint=*/false, &decision)); + /*stream=*/nullptr, /*query=*/true, /*graphCapturingHint=*/false, + &decision)); EXPECT_EQ((int)rcclAddonAlgos_t::RCCL_SYMMETRIC, decision.algo); EXPECT_EQ(NCCL_PROTO_LL, decision.protocol); EXPECT_EQ(4, decision.nMaxChannels); @@ -4598,7 +4623,8 @@ TEST(WrapMicrotestIsolated, SelectAllGather_SymmetricEligibleButSymkQueryFailsFa comm->symmetricSupport = 1; rcclCollDecision decision{}; EXPECT_EQ(ncclSuccess, rcclSelectAllGather(comm, nullptr, nullptr, /*sendcount=*/8, ncclFloat32, - /*query=*/true, /*graphCapturingHint=*/false, &decision)); + /*stream=*/nullptr, /*query=*/true, /*graphCapturingHint=*/false, + &decision)); EXPECT_NE((int)rcclAddonAlgos_t::RCCL_SYMMETRIC, decision.algo); EXPECT_EQ(NCCL_ALGO_RING, decision.algo); // plain-kernel placeholder, not overwritten by symk DeleteCommWithArch(comm); @@ -4616,7 +4642,8 @@ TEST(WrapMicrotestIsolated, SelectAllGather_DirectChosenWhenEligible) { comm->p2pnChannels = 17; rcclCollDecision decision{}; EXPECT_EQ(ncclSuccess, rcclSelectAllGather(comm, nullptr, nullptr, /*sendcount=*/8, ncclFloat32, - /*query=*/true, /*graphCapturingHint=*/false, &decision)); + /*stream=*/nullptr, /*query=*/true, /*graphCapturingHint=*/false, + &decision)); EXPECT_EQ((int)rcclAddonAlgos_t::RCCL_DIRECT_ALLGATHER, decision.algo); EXPECT_EQ(NCCL_PROTO_SIMPLE, decision.protocol); EXPECT_EQ(17, decision.nMaxChannels); @@ -4648,7 +4675,8 @@ TEST(WrapMicrotestIsolated, SelectAllGather_PlainKernelFallbackReportsGetAlgoInf ncclComm* comm = MakeSelectComm(); // not a DDA/Direct-eligible arch rcclCollDecision decision{}; EXPECT_EQ(ncclSuccess, rcclSelectAllGather(comm, nullptr, nullptr, /*sendcount=*/8, ncclFloat32, - /*query=*/true, /*graphCapturingHint=*/false, &decision)); + /*stream=*/nullptr, /*query=*/true, /*graphCapturingHint=*/false, + &decision)); EXPECT_EQ(NCCL_ALGO_TREE, decision.algo); EXPECT_EQ(NCCL_PROTO_LL128, decision.protocol); EXPECT_EQ(38, decision.nMaxChannels); @@ -4731,64 +4759,16 @@ TEST(WrapMicrotestIsolated, SelectReduceScatter_DdaFabricLLChosenOnGfx1250) { }); } -// The genuinely ReduceScatter-unique exception: on gfx1250, DDA fabric is -// allowed to preempt symmetric eligibility entirely (`!symEligible || -// ddaFabricArch`) -- distinct from every other gate in this function (and -// from AllReduce/AllGather's DDA, which are strictly !symEligible-gated -// with no such override). -TEST(WrapMicrotestIsolated, SelectReduceScatter_Gfx1250DdaPreemptsSymmetricEvenWhenEligible) { - RUN_ISOLATED_TEST( - "Wrap_SelectReduceScatter_Gfx1250DdaPreemptsSymmetricEvenWhenEligible", - []() { - ScopedHook symRequested(g_isSymmetricKernelRequested, [](struct ncclComm*, ncclFunc_t, int, ncclDataType_t, - size_t, const void*, void*, bool) { return true; }); - ScopedHook vmmEligible(g_reduceScatterDdaFabricEligible, - [](ncclComm*, const void*, void*, size_t, ncclDataType_t, ncclRedOp_t) { - return true; - }); - ncclComm* comm = MakeCommWithArch("gfx1250"); - comm->nRanks = 1; - comm->nNodes = 1; - rcclCollDecision decision{}; - EXPECT_EQ(ncclSuccess, rcclSelectReduceScatter(comm, nullptr, nullptr, /*recvcount=*/8, ncclFloat32, - ncclSum, /*query=*/false, &decision)); - EXPECT_EQ((int)rcclAddonAlgos_t::RCCL_DDA_FABRIC_VMM, decision.algo); // DDA won, not SYMMETRIC - DeleteCommWithArch(comm); - }); -} - -// Complementary proof: on a NON-gfx1250 arch, the same symEligible=true + -// DDA-eligible=true setup must NOT let DDA win -- the preemption is -// strictly gfx1250-specific. -TEST(WrapMicrotestIsolated, SelectReduceScatter_NonGfx1250DdaStrictlyGatedOnNotSymEligible) { - RUN_ISOLATED_TEST( - "Wrap_SelectReduceScatter_NonGfx1250DdaStrictlyGatedOnNotSymEligible", - []() { - ScopedHook symRequested(g_isSymmetricKernelRequested, [](struct ncclComm*, ncclFunc_t, int, ncclDataType_t, - size_t, const void*, void*, bool) { return true; }); - ScopedHook ipcEligible(g_reduceScatterDdaIpcEligible, - [](ncclComm*, const void*, void*, size_t, ncclDataType_t, ncclRedOp_t) { - return true; - }); - ncclComm* comm = MakeCommWithArch("gfx942"); // not gfx1250 - // 8, not 1: satisfies rcclDdaEnabled's own gfx942 rank floor so this test isolates - // the symEligible gate specifically, rather than piggybacking on an unrelated - // rank-count guard that would mask the same gate being broken. - comm->nRanks = 8; - comm->nNodes = 1; - rcclCollDecision decision{}; - EXPECT_EQ(ncclSuccess, rcclSelectReduceScatter(comm, nullptr, nullptr, /*recvcount=*/8, ncclFloat32, - ncclSum, /*query=*/false, &decision)); - EXPECT_EQ((int)rcclAddonAlgos_t::RCCL_SYMMETRIC, decision.algo); // symmetric won, not DDA_IPC - DeleteCommWithArch(comm); - }); -} TEST(WrapMicrotestIsolated, SelectReduceScatter_DdaFabricLL128ChosenWhenLLNotEligible) { RUN_ISOLATED_TEST( "Wrap_SelectReduceScatter_DdaFabricLL128ChosenWhenLLNotEligible", []() { - g_loadParam = ForceParam("RCCL_DDA_LL128", int64_t(1)); + g_loadParam = [](const char* env, int64_t def) -> int64_t { + if (std::strcmp(env, "RCCL_DDA_LL128") == 0) return 1; + if (std::strcmp(env, "RCCL_DDA_LL128_THRESHOLD") == 0) return INT64_MAX; + return def; + }; ScopedHook ll128Eligible(g_reduceScatterDdaFabricLL128Eligible, [](ncclComm*, const void*, void*, size_t, ncclDataType_t, ncclRedOp_t) { return true; @@ -5547,11 +5527,16 @@ TEST(WrapMicrotestIsolated, GetCollImplInfo_AllReduceDelegatesToSelectAllReduce) []() { ScopedHook symRequested(g_isSymmetricKernelRequested, [](struct ncclComm*, ncclFunc_t, int, ncclDataType_t, size_t, const void*, void*, bool) { return true; }); + ScopedHook symkAvailable(g_symkAvailable, + [](struct ncclComm*, ncclFunc_t, int, ncclDataType_t, size_t) { return true; }); + ScopedHook tuningCompute(g_tuningCompute, + SelectSymkTuning(ncclSymkKernelId_AllReduce_RSxLD_AGxST, /*maxChannels=*/6)); // Must have a real archName, not a zeroed comm: symEligible=true skips // rcclSelectAllReduce's CE-2-shot early return, so execution reaches its // unconditional IsArchMatch(comm->archName, "gfx1250") a few lines later -- // confirmed via a real crash (IsArchMatch doesn't null-check) before this fix. ncclComm* comm = MakeSelectComm(); + comm->symmetricSupport = 1; int algo, protocol, maxChannels; EXPECT_EQ(ncclSuccess, rcclGetCollImplInfo(comm, ncclFuncAllReduce, 8, ncclFloat32, ncclSum, nullptr, nullptr, /*graphCapturing=*/0, &algo, &protocol, &maxChannels)); @@ -5928,7 +5913,8 @@ TEST(WrapMicrotestIsolated, SelectAllGather_PlainKernelFallbackGetAlgoInfoFailur ncclComm* comm = MakeSelectComm(); // not DDA/Hierarchical/CE/symmetric/Direct eligible rcclCollDecision decision{}; EXPECT_EQ(ncclInternalError, rcclSelectAllGather(comm, nullptr, nullptr, /*sendcount=*/8, ncclFloat32, - /*query=*/true, /*graphCapturingHint=*/false, &decision)); + /*stream=*/nullptr, /*query=*/true, + /*graphCapturingHint=*/false, &decision)); DeleteCommWithArch(comm); }); } @@ -6206,11 +6192,11 @@ void ExerciseInfoCallSites(bool seedDirectDisabled) { DeleteCommWithArch(comm); ClearMicroEnv(); } - // rcclUseCeAllReduce: disabled-by-default (774). + // rcclUseCeAr2Shot: disabled-by-default (774). { g_loadParam = ForceParam("RCCL_CE_ALLREDUCE", int64_t(0)); ncclComm* comm = MakeCommWithArch("gfx942"); - rcclUseCeAllReduce(comm, /*count=*/8, ncclFloat32, ncclSum, /*acc=*/nullptr); + rcclUseCeAr2Shot(comm, /*count=*/8, ncclFloat32, ncclSum, /*acc=*/nullptr); DeleteCommWithArch(comm); } // rcclCeAllReduceGraphLatchTick: latch-set (837), latch-cleared (850). @@ -6354,7 +6340,8 @@ TEST(WrapMicrotestIsolated, SelectAllGather_HierarchicalQueryModeInterAndIntraDi comm->hierarchicalIntraComm = intraComm; rcclCollDecision decision{}; EXPECT_EQ(ncclSuccess, rcclSelectAllGather(comm, nullptr, nullptr, /*sendcount=*/8, ncclFloat32, - /*query=*/true, /*graphCapturingHint=*/false, &decision)); + /*stream=*/nullptr, /*query=*/true, /*graphCapturingHint=*/false, + &decision)); EXPECT_EQ((int)rcclAddonAlgos_t::RCCL_HIERARCHICAL_ALLGATHER, decision.algo); EXPECT_EQ(NCCL_PROTO_SIMPLE, decision.protocol); EXPECT_EQ(11, decision.nMaxChannels); // inter's p2pnChannels, not intra's 22 nor a getAlgoInfo value @@ -6394,7 +6381,8 @@ TEST(WrapMicrotestIsolated, SelectAllGather_HierarchicalQueryModeInterAboveNodeC comm->hierarchicalIntraComm = intraComm; rcclCollDecision decision{}; EXPECT_EQ(ncclSuccess, rcclSelectAllGather(comm, nullptr, nullptr, /*sendcount=*/8, ncclFloat32, - /*query=*/true, /*graphCapturingHint=*/false, &decision)); + /*stream=*/nullptr, /*query=*/true, /*graphCapturingHint=*/false, + &decision)); EXPECT_EQ(NCCL_PROTO_LL128, decision.protocol); EXPECT_EQ(42, decision.nMaxChannels); DeleteCommWithArch(comm); @@ -6428,7 +6416,8 @@ TEST(WrapMicrotestIsolated, SelectAllGather_HierarchicalQueryModeFallsToGetAlgoI RcclUnitTesting::ScopedDebugLogging debugLogging(NCCL_LOG_INFO, NCCL_ALL); // exercises the summary INFO too rcclCollDecision decision{}; EXPECT_EQ(ncclSuccess, rcclSelectAllGather(comm, nullptr, nullptr, /*sendcount=*/8, ncclFloat32, - /*query=*/true, /*graphCapturingHint=*/false, &decision)); + /*stream=*/nullptr, /*query=*/true, /*graphCapturingHint=*/false, + &decision)); EXPECT_EQ(NCCL_PROTO_LL128, decision.protocol); EXPECT_EQ(42, decision.nMaxChannels); DeleteCommWithArch(comm); diff --git a/projects/rccl/test/test_categories_fixtures_debug.yaml b/projects/rccl/test/test_categories_fixtures_debug.yaml index 77e6171d7396..119bfc7f7218 100644 --- a/projects/rccl/test/test_categories_fixtures_debug.yaml +++ b/projects/rccl/test/test_categories_fixtures_debug.yaml @@ -29,8 +29,16 @@ test_categories: - "DdaFabricEligibilityTest.*" - "DdaFabricMaxBlocksTest.*" - "DdaFabricScratchSizingTest.*" + - "RcclDdaTierThresholds.*" + - "RcclAllReduceDdaDecision.*" - "CeAllReduceEligibilityTest.*" - "RcclCeAllReduceEligibility.*" + - "RcclAllGatherCeRegisteredWindow.*" + - "RcclCeGraphLatch.*" + - "RcclAlltoAllDecision.*" + - "RcclCeAr2ShotMax.*" + - "RcclCeArGrownBuffer.*" + - "SkipPresetTopoMatching.*" - "CeAlltoAllvEligibilityTest.*" - "CeAlltoAllEligibilityTest.*" - "GinAllReduceEligibilityTest.*" diff --git a/projects/rccl/test/transport/NetIbMPI/GdrFlushTests.cpp b/projects/rccl/test/transport/NetIbMPI/GdrFlushTests.cpp index 78841759410c..3ac2e59d79b4 100644 --- a/projects/rccl/test/transport/NetIbMPI/GdrFlushTests.cpp +++ b/projects/rccl/test/transport/NetIbMPI/GdrFlushTests.cpp @@ -235,11 +235,11 @@ TEST_F(GdrFlushTest, ForcedScratchpadWrite_ReproducesFault) { if (!scratchpadFlushEnabled()) GTEST_SKIP() << "Requires the scratchpad flush enabled (RCCL_GDR_FLUSH_GPU_MEM_NO_RELAXED_ORDERING=1)"; AssertInitAndGetDevices(nullptr); - if (!gdrSupported()) GTEST_SKIP() << "GDR (NCCL_PTR_CUDA) not supported on this device"; + if (!(gdrPtrSupport() & NCCL_PTR_CUDA)) GTEST_SKIP() << "GDR (NCCL_PTR_CUDA) not supported on this device"; ncclResult_t forced = ncclSuccess; RunRecvFlushBurst(/*iterations=*/1, /*verifyData=*/false, - /*forceWrite=*/true, &forced); + &forced); if (MPIEnvironment::world_rank == 0) EXPECT_NE(forced, ncclSuccess) << "forced scratchpad RDMA_WRITE on a dma-buf buffer should fault the flush QP"; diff --git a/projects/rccl/tools/scripts/test_runner/configs/mi300_mellanox_ib_func.json b/projects/rccl/tools/scripts/test_runner/configs/mi300_mellanox_ib_func.json index 3a198a0de1a7..e7714cffb1f0 100644 --- a/projects/rccl/tools/scripts/test_runner/configs/mi300_mellanox_ib_func.json +++ b/projects/rccl/tools/scripts/test_runner/configs/mi300_mellanox_ib_func.json @@ -1279,6 +1279,26 @@ "name": "CeAlltoAllEligibilityTest", "description": "CE AlltoAll eligibility gating tests matching taskAppend (CTA_POLICY_ZERO, sysmem, capture, group depth)", "test_filter": "CeAlltoAllEligibilityTest.*" + }, + { + "name": "RcclAlltoAllDecision", + "description": "AlltoAll dispatch decision unit tests (DDA, CE, pivot selection)", + "test_filter": "RcclAlltoAllDecision.*" + }, + { + "name": "RcclCeAr2ShotMax", + "description": "CE AllReduce 2-shot arch threshold upper-bound tests", + "test_filter": "RcclCeAr2ShotMax.*" + }, + { + "name": "RcclCeArGrownBuffer", + "description": "CE AllReduce grown-buffer (ceRegMax) eligibility tests", + "test_filter": "RcclCeArGrownBuffer.*" + }, + { + "name": "SkipPresetTopoMatching", + "description": "SkipPreset arch topology matching and threshold lookup tests", + "test_filter": "SkipPresetTopoMatching.*" } ] }, diff --git a/projects/rccl/tools/scripts/test_runner/configs/mi300x_mellanox_ib.json b/projects/rccl/tools/scripts/test_runner/configs/mi300x_mellanox_ib.json index d74a233ecc7b..28c852087d11 100644 --- a/projects/rccl/tools/scripts/test_runner/configs/mi300x_mellanox_ib.json +++ b/projects/rccl/tools/scripts/test_runner/configs/mi300x_mellanox_ib.json @@ -980,7 +980,25 @@ { "name": "P2pMaxNchannels", "description": "NCCL_MAX_P2P_NCHANNELS resolution, p2pnChannels upper bound, and the multi-node maxP2pPeers per-peer reduction", - "test_filter": "P2pMaxNchannels*" + "test_filter": "P2pMaxNchannels*", + "name": "RcclAlltoAllDecision", + "description": "AlltoAll dispatch decision unit tests (DDA, CE, pivot selection)", + "test_filter": "RcclAlltoAllDecision.*" + }, + { + "name": "RcclCeAr2ShotMax", + "description": "CE AllReduce 2-shot arch threshold upper-bound tests", + "test_filter": "RcclCeAr2ShotMax.*" + }, + { + "name": "RcclCeArGrownBuffer", + "description": "CE AllReduce grown-buffer (ceRegMax) eligibility tests", + "test_filter": "RcclCeArGrownBuffer.*" + }, + { + "name": "SkipPresetTopoMatching", + "description": "SkipPreset arch topology matching and threshold lookup tests", + "test_filter": "SkipPresetTopoMatching.*" } ] }, diff --git a/projects/rccl/tools/scripts/test_runner/configs/mi355x_thor2_roce.json b/projects/rccl/tools/scripts/test_runner/configs/mi355x_thor2_roce.json index 29f251a2c25a..3da0640b6548 100644 --- a/projects/rccl/tools/scripts/test_runner/configs/mi355x_thor2_roce.json +++ b/projects/rccl/tools/scripts/test_runner/configs/mi355x_thor2_roce.json @@ -723,6 +723,26 @@ "name": "TimeoutTests", "description": "ncclTimeout host API unit tests: enum ordering, error string, async-error set/get/clear, argument validation", "test_filter": "TimeoutTests.*" + }, + { + "name": "RcclAlltoAllDecision", + "description": "AlltoAll dispatch decision unit tests (DDA, CE, pivot selection)", + "test_filter": "RcclAlltoAllDecision.*" + }, + { + "name": "RcclCeAr2ShotMax", + "description": "CE AllReduce 2-shot arch threshold upper-bound tests", + "test_filter": "RcclCeAr2ShotMax.*" + }, + { + "name": "RcclCeArGrownBuffer", + "description": "CE AllReduce grown-buffer (ceRegMax) eligibility tests", + "test_filter": "RcclCeArGrownBuffer.*" + }, + { + "name": "SkipPresetTopoMatching", + "description": "SkipPreset arch topology matching and threshold lookup tests", + "test_filter": "SkipPresetTopoMatching.*" } ] }, diff --git a/projects/rccl/tools/scripts/test_runner/configs/mi355x_thor2_roce_func.json b/projects/rccl/tools/scripts/test_runner/configs/mi355x_thor2_roce_func.json index 64cea2197c29..d0ad7d8d2b7b 100644 --- a/projects/rccl/tools/scripts/test_runner/configs/mi355x_thor2_roce_func.json +++ b/projects/rccl/tools/scripts/test_runner/configs/mi355x_thor2_roce_func.json @@ -1280,6 +1280,26 @@ "name": "CeAlltoAllEligibilityTest", "description": "CE AlltoAll eligibility gating tests matching taskAppend (CTA_POLICY_ZERO, sysmem, capture, group depth)", "test_filter": "CeAlltoAllEligibilityTest.*" + }, + { + "name": "RcclAlltoAllDecision", + "description": "AlltoAll dispatch decision unit tests (DDA, CE, pivot selection)", + "test_filter": "RcclAlltoAllDecision.*" + }, + { + "name": "RcclCeAr2ShotMax", + "description": "CE AllReduce 2-shot arch threshold upper-bound tests", + "test_filter": "RcclCeAr2ShotMax.*" + }, + { + "name": "RcclCeArGrownBuffer", + "description": "CE AllReduce grown-buffer (ceRegMax) eligibility tests", + "test_filter": "RcclCeArGrownBuffer.*" + }, + { + "name": "SkipPresetTopoMatching", + "description": "SkipPreset arch topology matching and threshold lookup tests", + "test_filter": "SkipPresetTopoMatching.*" } ] }, diff --git a/projects/rccl/tools/scripts/test_runner/configs/mi455_ainic_roce.json b/projects/rccl/tools/scripts/test_runner/configs/mi455_ainic_roce.json index 6d28f48e2d74..d2778d6e322b 100644 --- a/projects/rccl/tools/scripts/test_runner/configs/mi455_ainic_roce.json +++ b/projects/rccl/tools/scripts/test_runner/configs/mi455_ainic_roce.json @@ -1304,6 +1304,26 @@ "name": "CeAlltoAllEligibilityTest", "description": "CE AlltoAll eligibility gating tests matching taskAppend (CTA_POLICY_ZERO, sysmem, capture, group depth)", "test_filter": "CeAlltoAllEligibilityTest.*" + }, + { + "name": "RcclAlltoAllDecision", + "description": "AlltoAll dispatch decision unit tests (DDA, CE, pivot selection)", + "test_filter": "RcclAlltoAllDecision.*" + }, + { + "name": "RcclCeAr2ShotMax", + "description": "CE AllReduce 2-shot arch threshold upper-bound tests", + "test_filter": "RcclCeAr2ShotMax.*" + }, + { + "name": "RcclCeArGrownBuffer", + "description": "CE AllReduce grown-buffer (ceRegMax) eligibility tests", + "test_filter": "RcclCeArGrownBuffer.*" + }, + { + "name": "SkipPresetTopoMatching", + "description": "SkipPreset arch topology matching and threshold lookup tests", + "test_filter": "SkipPresetTopoMatching.*" } ] }, diff --git a/projects/rccl/tools/scripts/test_runner/configs/mi455_ainic_roce_func.json b/projects/rccl/tools/scripts/test_runner/configs/mi455_ainic_roce_func.json index 0380d4238278..601ccf335da7 100644 --- a/projects/rccl/tools/scripts/test_runner/configs/mi455_ainic_roce_func.json +++ b/projects/rccl/tools/scripts/test_runner/configs/mi455_ainic_roce_func.json @@ -704,7 +704,7 @@ "RCCL_IB_QP_SCHED_RESET_INTERVAL": "1000", "RCCL_IB_QP_SCHED_SPLIT_DATA_MIN": "2000", "NCCL_IB_RETURN_ASYNC_EVENTS": "1", - "RCCL_IB_FAULT_INJECTION": "1" + "RCCL_IB_FAULT_INJECTION": "1" }, "tests": [ { @@ -1274,6 +1274,74 @@ "name": "CeAlltoAllEligibilityTest", "description": "CE AlltoAll eligibility gating tests matching taskAppend (CTA_POLICY_ZERO, sysmem, capture, group depth)", "test_filter": "CeAlltoAllEligibilityTest.*" + }, + { + "name": "CeAllReduceEligibilityTest", + "description": "CE AllReduce eligibility gate tests. RCCL_CE_AR_MAX_MSG_BYTES activates 2-shot on gfx1250 (ceNonRegMax[AllReduce]=0 by table design).", + "test_filter": "CeAllReduceEligibilityTest.*:RcclCeAllReduceEligibility.*", + "env_variables": { + "RCCL_CE_AR_MAX_MSG_BYTES": "268435456" + } + }, + { + "name": "DdaFabricEligibilityTest", + "description": "DDA fabric eligibility tests for AllReduce, AllGather, ReduceScatter, AlltoAll across LL/LL128/VMM tiers on gfx1250", + "test_filter": "DdaFabricEligibilityTest.*:DdaFabricLL128EligibilityTest.*" + }, + { + "name": "DdaFabricMaxBlocksTest", + "description": "DDA fabric max-blocks CU cap computation and env-override tests", + "test_filter": "DdaFabricMaxBlocksTest.*" + }, + { + "name": "DdaFabricScratchSizingTest", + "description": "DDA fabric scratch-buffer sizing tests covering LL floor, LL128 floor, and CE scratch interplay", + "test_filter": "DdaFabricScratchSizingTest.*" + }, + { + "name": "DdaAlltoAllThresholdTest", + "description": "DDA AlltoAll threshold tests for gfx1250/gfx942/gfx950: entry gate, rank count, staging copy selection", + "test_filter": "DdaAlltoAllThresholdTest.*:DdaAlltoAllThreshold.*" + }, + { + "name": "RcclDdaTierThresholds", + "description": "Arch-table tier threshold resolution tests: gfx1250 table, env overrides, IGNORE_ARCH_TABLE fallback", + "test_filter": "RcclDdaTierThresholds.*" + }, + { + "name": "RcclAllReduceDdaDecision", + "description": "AllReduce DDA dispatch decision tests across gfx1250/gfx942/gfx950: symk yield, CE yield, rank gate", + "test_filter": "RcclAllReduceDdaDecision.*" + }, + { + "name": "RcclAllGatherCeRegisteredWindow", + "description": "AllGather CE-registered window eligibility tests: ceRegMax gate, recv-registered requirement", + "test_filter": "RcclAllGatherCeRegisteredWindow.*" + }, + { + "name": "RcclCeGraphLatch", + "description": "CE AllReduce graph-mode latch tests: set on capture, cleared when all plan refs dropped", + "test_filter": "RcclCeGraphLatch.*" + }, + { + "name": "RcclAlltoAllDecision", + "description": "AlltoAll dispatch decision unit tests (DDA, CE, pivot selection)", + "test_filter": "RcclAlltoAllDecision.*" + }, + { + "name": "RcclCeAr2ShotMax", + "description": "CE AllReduce 2-shot arch threshold upper-bound tests", + "test_filter": "RcclCeAr2ShotMax.*" + }, + { + "name": "RcclCeArGrownBuffer", + "description": "CE AllReduce grown-buffer (ceRegMax) eligibility tests", + "test_filter": "RcclCeArGrownBuffer.*" + }, + { + "name": "SkipPresetTopoMatching", + "description": "SkipPreset arch topology matching and threshold lookup tests", + "test_filter": "SkipPresetTopoMatching.*" } ] },