Repository navigation
rccl-tests/gin-sdma: cap A2A GIN put to 128 MiB to fix >1 GiB SDMA hang #9927
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
4df1c45
d24881b
fbc6ba7
852d20d
2401bde
3db8170
73e0a47
579861c
51c2af8
92b25f7
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,183 @@ | ||
| ################################################################################# | ||
| # Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. | ||
| # | ||
| # Permission is hereby granted, free of charge, to any person obtaining a copy | ||
| # of this software and associated documentation files (the "Software"), to deal | ||
| # in the Software without restriction, including without limitation the rights | ||
| # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell cop- | ||
| # ies of the Software, and to permit persons to whom the Software is furnished | ||
| # to do so, subject to the following conditions: | ||
| # | ||
| # The above copyright notice and this permission notice shall be included in all | ||
| # copies or substantial portions of the Software. | ||
| # | ||
| # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IM- | ||
| # PLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS | ||
| # FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR | ||
| # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER | ||
| # IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNE- | ||
| # CTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. | ||
| ################################################################################ | ||
|
|
||
| # GIN-SDMA AllToAll regression tests for the >1 GiB SDMA hang fix (PR #9927). | ||
| # | ||
| # These drive the real GinHybridAlltoAllKernel (deviceImpl 3, NCCL_GIN_TYPE=5) | ||
| # at per-peer transfer sizes that cross the two single-descriptor limits the | ||
| # 128 MiB SDMA copy clamp guards. The clamp now lives in the Anvil-SDMA backend | ||
| # Put (ncclGinApi_Put<NCCL_NET_DEVICE_GIN_ANVIL_SDMA>), which segments every | ||
| # gin.put() into <=128 MiB SDMA copies; the kernels just call plain gin.put(): | ||
| # | ||
| # 1. >128 MiB/peer -- exercises the new multi-segment put loop. | ||
| # 2. >1 GiB/peer -- crosses the 30-bit (1 GiB) count-field boundary, where | ||
| # an unclamped single put would silently truncate. | ||
| # 3. 2 GiB total -- the original hang repro (256 MiB/peer at 8 ranks); a | ||
| # subprocess timeout turns a reintroduced hang into a | ||
| # test failure instead of stalling the runner forever. | ||
| # | ||
| # Exact-integer datatypes are used so a truncated or stale tail cannot be masked | ||
| # by floating-point tolerance; the perf binary's built-in data check (-c 1) sets | ||
| # a non-zero exit code on any wrong element. | ||
| # | ||
| # These require an 8x MI355X (or similar) node, an MPI launcher, and a | ||
| # GIN-SDMA-capable RCCL build, so the module is skipped unless | ||
| # RCCL_TESTS_GIN_SDMA_A2A is set in the environment. Configuration is taken from | ||
| # environment variables (see below) with defaults matching the 8-GPU repro: | ||
| # RCCL_TESTS_A2A_NP MPI ranks (default: detected GPU count) | ||
| # RCCL_TESTS_MPI_LAUNCHER launcher binary (default: mpirun) | ||
| # RCCL_TESTS_MPI_OPTS extra launcher opts (e.g. --allow-run-as-root -mca ...) | ||
| # RCCL_TESTS_A2A_XENV extra "-x K=V" env the backend needs on this cluster | ||
| # (e.g. HSA_FORCE_FINE_GRAIN_PCIE=1 NCCL_CUMEM_ENABLE=1) | ||
| # RCCL_TESTS_A2A_EXE path to alltoall_perf (default: ../build/alltoall_perf) | ||
| # RCCL_TESTS_A2A_TIMEOUT_S per-run hang timeout in seconds (default: 900) | ||
| # | ||
| # Verified on 8x MI355X (ROCm 7.13, NCCL_GIN_TYPE=5, force1ch): 256 MiB/peer and | ||
| # 2 GiB/peer int32 and the 2 GiB-total guard all pass with #wrong=0, no hang | ||
| # (busbw ~424-428 GB/s). | ||
|
|
||
| import os | ||
| import shlex | ||
| import signal | ||
| import subprocess | ||
|
|
||
| import pytest | ||
|
|
||
| MiB = 1024 * 1024 | ||
| GiB = 1024 * MiB | ||
|
|
||
| path = os.path.dirname(os.path.abspath(__file__)) | ||
| # Path to alltoall_perf. Override with RCCL_TESTS_A2A_EXE when the binary is not | ||
| # at the default build/ location (e.g. an installed/container path). | ||
| executable = os.environ.get( | ||
| "RCCL_TESTS_A2A_EXE", os.path.join(path, "..", "build", "alltoall_perf")) | ||
|
|
||
| # Opt-in: only run where the GIN-SDMA backend + hardware are available. | ||
| _enabled = os.environ.get("RCCL_TESTS_GIN_SDMA_A2A", "") not in ("", "0", "false", "False") | ||
|
|
||
|
|
||
| def _detect_ngpus(): | ||
| if os.environ.get("ROCR_VISIBLE_DEVICES") is not None: | ||
| return len(os.environ["ROCR_VISIBLE_DEVICES"].split(",")) | ||
| if os.environ.get("HIP_VISIBLE_DEVICES") is not None: | ||
| return len(os.environ["HIP_VISIBLE_DEVICES"].split(",")) | ||
| try: | ||
| out = subprocess.check_output( | ||
| 'rocminfo | grep "Device Type:.\\s*.GPU" | wc -l', shell=True) | ||
| return int(out) | ||
| except Exception: | ||
| return 0 | ||
|
|
||
|
|
||
| NP = int(os.environ.get("RCCL_TESTS_A2A_NP", "0")) or _detect_ngpus() | ||
| LAUNCHER = os.environ.get("RCCL_TESTS_MPI_LAUNCHER", "mpirun") | ||
| CTAS = os.environ.get("RCCL_TESTS_A2A_CTAS", "16") | ||
| # Generous per-call timeout; a real hang spins forever, so a finite cap is what | ||
| # converts item (3) into a detectable failure. Scaled up for the multi-GiB case. | ||
| TIMEOUT_S = int(os.environ.get("RCCL_TESTS_A2A_TIMEOUT_S", "900")) | ||
|
|
||
| # Extra launcher options (e.g. "--allow-run-as-root -mca pml ob1 ...") and any | ||
| # deployment-specific env the GIN-SDMA backend needs beyond the essentials below | ||
| # (e.g. HSA_FORCE_FINE_GRAIN_PCIE=1 NCCL_CUMEM_ENABLE=1 RCCL_ENABLE_INTRANET=1). | ||
| # GIN enablement itself is site-specific, so it is injected here rather than | ||
| # hard-coded, keeping the test portable across clusters. | ||
| MPI_OPTS = shlex.split(os.environ.get("RCCL_TESTS_MPI_OPTS", "")) | ||
| XENV = shlex.split(os.environ.get("RCCL_TESTS_A2A_XENV", "")) | ||
|
|
||
| pytestmark = pytest.mark.skipif( | ||
| not _enabled, | ||
| reason="GIN-SDMA AllToAll tests are opt-in; set RCCL_TESTS_GIN_SDMA_A2A=1 on " | ||
| "a GIN-SDMA-capable (e.g. 8x MI355X) node to enable.") | ||
|
|
||
|
|
||
| def _run_a2a(request, total_bytes, dtype): | ||
| """Run one all-to-all at a fixed total (per-rank) size, forcing the GIN-SDMA | ||
| tier. Returns (returncode, stdout). A timeout (hang) fails the test.""" | ||
| if NP < 2: | ||
| pytest.skip("need >= 2 ranks/GPUs for AllToAll") | ||
|
|
||
| size = str(int(total_bytes)) | ||
| # Essentials to exercise the GIN-SDMA put path; force the SDMA (large) tier | ||
| # for every size. Deployment-specific env comes from RCCL_TESTS_A2A_XENV. | ||
| gin_env = [] | ||
| for kv in ["NCCL_GIN_ENABLE=1", "NCCL_GIN_TYPE=5", | ||
| "NCCL_GIN_ANVIL_SDMA_THRESHOLD=0", | ||
| "NCCL_GIN_ANVIL_SDMA_THRESHOLD_ALLTOALL=0"] + XENV: | ||
| gin_env += ["-x", kv] | ||
|
|
||
| hostfile = request.config.getoption("--hostfile") | ||
| launch = [LAUNCHER, "-np", str(NP)] + MPI_OPTS | ||
| if hostfile: | ||
| launch += ["-host", hostfile] | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. In Open MPI -host expects a comma separated list of hosts, not a hostfile, the option is --hostfile. |
||
|
|
||
| args = launch + gin_env + [ | ||
| executable, | ||
| "-b", size, "-e", size, | ||
| "-f", "2", | ||
| "-g", "1", | ||
| "-R", "2", | ||
| "-D", "3", | ||
| "-V", CTAS, | ||
| "-d", dtype, | ||
| "-c", "1", # data check: nonzero exit on any wrong element | ||
| "-w", "1", | ||
| "-n", "3", | ||
| ] | ||
| cmd = " ".join(shlex.quote(a) for a in args) | ||
| print(cmd) | ||
| # Run in a new session so a hang can be killed as a whole process group -- | ||
| # otherwise a wedged launcher leaves orphaned ranks pinning the GPUs (and | ||
| # exhausting SDMA queues for later tests). | ||
| proc = subprocess.Popen(cmd, shell=True, universal_newlines=True, | ||
| stdout=subprocess.PIPE, stderr=subprocess.STDOUT, | ||
| start_new_session=True) | ||
| try: | ||
| out, _ = proc.communicate(timeout=TIMEOUT_S) | ||
| except subprocess.TimeoutExpired: | ||
| try: | ||
| os.killpg(os.getpgid(proc.pid), signal.SIGKILL) | ||
| except ProcessLookupError: | ||
| pass | ||
| out, _ = proc.communicate() | ||
| pytest.fail( | ||
| "AllToAll GIN-SDMA HANG: no completion within {}s at total={} bytes " | ||
| "({} MiB/peer), dtype={}. Output tail:\n{}".format( | ||
| TIMEOUT_S, size, total_bytes // NP // MiB, dtype, (out or "")[-2000:])) | ||
| print(out) | ||
| return proc.returncode, out | ||
|
|
||
|
|
||
| # (item 1 + 2) Multi-segment loop and the 1 GiB truncation boundary. per_peer is | ||
| # the transfer to each peer; total per-rank bytes = per_peer * NP. | ||
| @pytest.mark.parametrize("per_peer_mib", [256, 2048]) # 256 MiB (2 seg), 2 GiB (16 seg) | ||
| @pytest.mark.parametrize("dtype", ["int32", "int64", "uint8"]) | ||
| def test_AllToAllGinSdmaLargeSegmented(request, per_peer_mib, dtype): | ||
| total = per_peer_mib * MiB * NP | ||
| rc, _ = _run_a2a(request, total, dtype) | ||
| assert rc == 0, "AllToAll data check failed (nonzero exit) at {} MiB/peer, dtype={}".format( | ||
| per_peer_mib, dtype) | ||
|
|
||
|
|
||
| # (item 3) The original 2 GiB-total hang repro, as a completion guard. At 8 ranks | ||
| # this is 256 MiB/peer -- the size that hung as a single unclamped put. | ||
| def test_AllToAllGinSdma2GiBTotalHangGuard(request): | ||
| rc, _ = _run_a2a(request, 2 * GiB, "int32") | ||
| assert rc == 0, "AllToAll 2 GiB-total data check failed (nonzero exit)" | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,108 @@ | ||
| /************************************************************************* | ||
| * Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. | ||
| * | ||
| * See LICENSE.txt for license information | ||
| ************************************************************************/ | ||
|
|
||
| // Put-size limits and segment math for the GIN Anvil-SDMA device backend. | ||
| // The single source of truth for the two per-put byte caps that | ||
| // ncclGinApi_Put<NCCL_NET_DEVICE_GIN_ANVIL_SDMA> segments every SDMA copy | ||
| // against, plus the pure segmentation arithmetic itself. Kept as a standalone, | ||
| // dependency-free header (no ncclDevComm, no GPU intrinsics, no getenv) so the | ||
| // same code is (a) called from the SDMA backend Put on device and (b) unit-tested | ||
| // on the host with GoogleTest (define GIN_SDMA_HOST_ONLY to drop the device | ||
| // attributes and compile it as plain host C++; see | ||
| // projects/rccl/test/gin/gin_sdma_policy_test.cpp). | ||
|
|
||
| #ifndef _NCCL_DEVICE_GIN_ANVIL_SDMA_PUT_POLICY_H_ | ||
| #define _NCCL_DEVICE_GIN_ANVIL_SDMA_PUT_POLICY_H_ | ||
|
|
||
| #include <cstddef> | ||
|
|
||
| // The device backend gets __host__ __device__; plain host builds (and the host | ||
| // unit test, which defines GIN_SDMA_HOST_ONLY) get no attribute so the header | ||
| // compiles as ordinary C++ with no device codegen. | ||
| #if (defined(__CUDACC__) || defined(__HIPCC__)) && !defined(GIN_SDMA_HOST_ONLY) | ||
| #define GIN_SDMA_HD __host__ __device__ | ||
| #else | ||
| #define GIN_SDMA_HD | ||
| #endif | ||
|
|
||
| namespace gin_sdma { | ||
|
|
||
| // Max bytes per single SDMA copy on the Anvil-SDMA backend. The SDMA linear-copy | ||
| // descriptor count field is 30 bits and 1-based (HW encodes count = bytes - 1, | ||
| // see rocr-runtime amd_blit_sdma.cpp / sdma_registers.h), so the largest single | ||
| // packet is (2^30 - 1) + 1 = 2^30 = exactly 1 GiB. A put of >1 GiB silently | ||
| // truncates: a 2 GiB put encodes count = (2^31-1) & 0x3FFFFFFF = 2^30-1 and the | ||
| // HW copies only 1 GiB, corrupting the transfer. Every SDMA copy must be split | ||
| // into segments of at most this size. | ||
| // | ||
| // Set to exactly 1 GiB: this is the hardware maximum, so the segmentation clamp | ||
| // (seg <= kGinPutMaxBytes) guarantees count = seg-1 <= 2^30-1 = 0x3FFFFFFF, which | ||
| // fills the 30-bit field exactly with no truncation. Zero margin by design; do | ||
| // NOT raise above 2^30. 1 GiB is a multiple of 32 B, satisfying the copy | ||
| // descriptor's 32 B length alignment. | ||
| static constexpr size_t kGinPutMaxBytes = 1024ull * 1024 * 1024; // 1 GiB (2^30, HW max) | ||
|
|
||
| // Max bytes per single SDMA copy that the Anvil-SDMA backend copies *reliably* | ||
| // on MI355X + ROCm 7.13 (NCCL_GIN_TYPE=5). This is SMALLER than kGinPutMaxBytes: | ||
| // the 30-bit count field bounds correctness at 1 GiB, but a single copy | ||
| // descriptor at/above 256 MiB (2^28) on the fused COPY_LINEAR_WAIT_SIGNAL_MI4 | ||
| // path stalls the SDMA engine, so the fused copy never lands AND its SignalInc | ||
| // never fires -> every rank spins forever in waitSignal (a HANG, not a data | ||
| // miscompare). Measured on 8x MI355X (2026-08-07, alltoall_perf -D 3): a single | ||
| // 256 MiB/peer put (AllToAll @ 2 GiB total) HANGS; capping each copy to 128 MiB | ||
| // (2 copies/peer) completes with identical bandwidth (busbw ~423 GB/s, unchanged | ||
| // vs the 1 GiB total case). 128 MiB is proven safe with zero measured perf loss; | ||
| // do not raise without re-measuring. The backend Put segments every SDMA copy to | ||
| // this size, so it protects ALL callers of gin.put() on the Anvil-SDMA backend. | ||
| static constexpr size_t kGinSdmaSafeCopyBytes = 128ull * 1024 * 1024; // 128 MiB (reliable single-copy max) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This is entirely empirical do we have any understanding why the limit is not 1G as well here?
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It's two different limits. 1 GiB ( |
||
|
|
||
| // The single per-copy segmentation limit the backend Put clamps to: the smaller | ||
| // of the two constants above (correctness bound AND reliability bound). Compile- | ||
| // time constant so the device loop and the host unit test share one value. | ||
| static constexpr size_t kGinPutSegBytes = | ||
| kGinSdmaSafeCopyBytes < kGinPutMaxBytes ? kGinSdmaSafeCopyBytes : kGinPutMaxBytes; | ||
|
|
||
| // ------------------------------ segment math ------------------------------ | ||
| // | ||
| // Pure arithmetic for splitting one peer transfer into <=kGinPutSegBytes copies. | ||
| // The Anvil-SDMA backend Put drives its ::sdma_anvil::put() loop entirely from | ||
| // these two functions, so the same code that runs on device is what the host | ||
| // unit test exercises. Every SDMA copy must satisfy: no segment exceeds the cap, | ||
| // the segments tile [0,bytes) contiguously and sum back to bytes, and exactly | ||
| // one segment is flagged final (it alone carries the caller's SignalInc). | ||
|
|
||
| // One SDMA-copy segment of a chunked transfer. | ||
| struct PutSegment { | ||
| size_t offset; // byte offset of this segment within the transfer | ||
| size_t bytes; // segment length in bytes (always <= maxSeg) | ||
| bool isFinal; // last segment: carries the caller's remote action (signal) | ||
| }; | ||
|
|
||
| // Number of copy segments a `bytes`-length transfer splits into when each copy | ||
| // is capped at `maxSeg`. A zero-length transfer still issues exactly one (empty) | ||
| // copy so its trailing signal fires; otherwise it is ceil(bytes/maxSeg). | ||
| // maxSeg == 0 is a caller error and yields 0 (callers pass a nonzero cap). | ||
| GIN_SDMA_HD inline size_t ginPutSegmentCount(size_t bytes, size_t maxSeg) { | ||
| if (maxSeg == 0) return 0; | ||
| if (bytes == 0) return 1; | ||
| return (bytes + maxSeg - 1) / maxSeg; | ||
| } | ||
|
|
||
| // The i-th (0-based, i < ginPutSegmentCount) segment of a `bytes`-length transfer | ||
| // capped at `maxSeg`. Segments tile [0,bytes) contiguously; the last one is | ||
| // flagged isFinal so the backend carries the remote action on it alone. | ||
| GIN_SDMA_HD inline PutSegment ginPutSegmentAt(size_t bytes, size_t maxSeg, size_t i) { | ||
| PutSegment s{0, 0, false}; | ||
| s.offset = i * maxSeg; | ||
| const size_t rem = bytes - s.offset; | ||
| s.bytes = rem > maxSeg ? maxSeg : rem; | ||
| s.isFinal = (s.offset + s.bytes >= bytes); | ||
| return s; | ||
| } | ||
|
|
||
| } // namespace gin_sdma | ||
|
|
||
| #endif // _NCCL_DEVICE_GIN_ANVIL_SDMA_PUT_POLICY_H_ | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
There is nothing specific to SDMA as far as I can tell in that tester and it could be used to validate the same feature in GDA (or other GIN transports for that matter), maybe scrubbing the ANVIL_SDMA from the comments and function names would be good.