From 0123e9aa490b5f8b7bc88ae8e128d3b3a6453411 Mon Sep 17 00:00:00 2001 From: TennyWang1223 Date: Mon, 1 Jun 2026 12:11:14 +0000 Subject: [PATCH 1/2] fea: reduce_scatter support all dim Signed-off-by: TennyWang1223 --- .../device_communicators/communicator_cuda.py | 7 +- .../device_communicators/custom_all_reduce.py | 75 ++++- aiter/dist/parallel_state.py | 61 ++-- aiter/ops/custom_all_reduce.py | 4 + csrc/include/custom_all_reduce.cuh | 283 ++++++++++++++-- csrc/include/custom_all_reduce.h | 11 + csrc/include/rocm_ops.hpp | 4 + csrc/kernels/custom_all_reduce.cu | 26 +- .../multigpu_tests/test_reduce_scatter.py | 302 ++++++++---------- 9 files changed, 519 insertions(+), 254 deletions(-) diff --git a/aiter/dist/device_communicators/communicator_cuda.py b/aiter/dist/device_communicators/communicator_cuda.py index 9749bbeee26..aa72ca81db6 100644 --- a/aiter/dist/device_communicators/communicator_cuda.py +++ b/aiter/dist/device_communicators/communicator_cuda.py @@ -663,12 +663,15 @@ def reduce_scatter( ): world_size = self.world_size ca_comm = self.ca_comm + # Custom kernel supports scatter on first/last/mid dims; gate via + # should_custom_rs which also rejects first-dim-non-vectorizable + # shapes (no naive fallback exists for that case, see C++ dispatch). if ( ca_comm is not None and not ca_comm.disabled - and ca_comm.should_custom_ar(input_) + and ca_comm.should_custom_rs(input_, dim) ): - ca_comm.custom_reduce_scatter(input_, output_) + ca_comm.custom_reduce_scatter(input_, output_, dim) else: pynccl_comm = self.pynccl_comm assert pynccl_comm is not None diff --git a/aiter/dist/device_communicators/custom_all_reduce.py b/aiter/dist/device_communicators/custom_all_reduce.py index b0f657b9fb7..d422f387c71 100644 --- a/aiter/dist/device_communicators/custom_all_reduce.py +++ b/aiter/dist/device_communicators/custom_all_reduce.py @@ -629,35 +629,100 @@ def custom_all_reduce( registered_input=False, ) + # reduce_scatter split_dim enum — must match `aiter::ReduceScatterSplitDim` + # in csrc/include/custom_all_reduce.cuh. + _RS_SPLIT_FIRST = 0 + _RS_SPLIT_LAST = 1 + _RS_SPLIT_MID = 2 + + @staticmethod + def _compute_rs_args(shape, dim: int, numel: int): + """Collapse `shape` around the scatter dim into the canonical + (m, n, k, split_dim) the C++ dispatcher expects. `dim` must be + already normalized to [0, len(shape)).""" + ndim = len(shape) + if dim == 0: + return 0, 0, numel, CustomAllreduce._RS_SPLIT_FIRST + if dim == ndim - 1: + n = 1 + for s in shape[:-1]: + n *= s + return 0, n, shape[-1], CustomAllreduce._RS_SPLIT_LAST + m = 1 + for s in shape[:dim]: + m *= s + k = 1 + for s in shape[dim + 1 :]: + k *= s + return m, shape[dim], k, CustomAllreduce._RS_SPLIT_MID + + def should_custom_rs(self, inp: torch.Tensor, dim: int) -> bool: + """Return True iff the custom reduce_scatter kernel can handle + (inp, dim). Mirrors the C++ dispatch's hard requirements: + + - all the should_custom_ar gates (size cap, contiguous, etc.) + - inp.shape[dim_normalized] % world_size == 0 + - for dim == 0 (first-dim split) the flattened input must be + vectorizable: numel % (world_size * pack_size) == 0; there is + no naive fallback for the first-dim kernel and the framework is + expected to route to an external lib in that case. + (last/mid-dim kernels have naive fallbacks built in.) + """ + if not self.should_custom_ar(inp): + return False + ndim = inp.dim() + if dim < 0: + dim += ndim + if dim < 0 or dim >= ndim: + return False + if inp.shape[dim] % self.world_size != 0: + return False + if dim == 0: + pack_size = 16 // inp.element_size() + if inp.numel() % (self.world_size * pack_size) != 0: + return False + return True + def reduce_scatter( self, inp: torch.Tensor, out: torch.Tensor, + dim: int = 0, *, registered: bool = False, ): assert is_weak_contiguous(out), "output tensor is not weak-contiguous" + ndim = inp.dim() + if dim < 0: + dim += ndim + m, n, k, split_dim = self._compute_rs_args(tuple(inp.shape), dim, inp.numel()) reg = 0 if registered else self._pool["input"].data_ptr reg_bytes = 0 if registered else self._pool["input"].max_size ops.reduce_scatter( self._ptr, inp, out, + m, + n, + k, + split_dim, reg, reg_bytes, ) def custom_reduce_scatter( - self, input: torch.Tensor, output: torch.Tensor + self, input: torch.Tensor, output: torch.Tensor, dim: int = 0 ) -> Optional[torch.Tensor]: - # when custom allreduce is disabled, this will be None - if self.disabled or not self.should_custom_ar(input): + # when custom allreduce is disabled or this shape/dim is unsupported, + # this will be None and the caller is expected to fall back to an + # external reduce_scatter implementation (NCCL / pynccl / torch.dist). + if self.disabled or not self.should_custom_rs(input, dim): return None if self._IS_CAPTURING: if torch.cuda.is_current_stream_capturing(): - return self.reduce_scatter(input, output, registered=True) + return self.reduce_scatter(input, output, dim, registered=True) else: - return self.reduce_scatter(input, output, registered=False) + return self.reduce_scatter(input, output, dim, registered=False) def _allgather_out_shape(self, inp: torch.Tensor, dim: int): ndim = inp.dim() diff --git a/aiter/dist/parallel_state.py b/aiter/dist/parallel_state.py index 0badd35bdef..eb3e99e03e5 100644 --- a/aiter/dist/parallel_state.py +++ b/aiter/dist/parallel_state.py @@ -532,14 +532,15 @@ def fused_allreduce_rmsnorm_quant_per_group( prefill_support: bool = False, emit_bf16: bool = False, ): - return self.fused_allreduce_rmsnorm_quant( + if self.device_communicator is None: + raise ValueError("No device communicator found") + return self.device_communicator.fused_allreduce_rmsnorm_quant_per_group( input_, residual_inp_, weight_, eps, + group_size, prefill_support, - quant_type="per_group", - group_size=group_size, emit_bf16=emit_bf16, ) @@ -550,32 +551,9 @@ def fused_qknorm_allreduce( k_w: torch.Tensor, eps: float, ): - return fused_qknorm_allreduce_( - qkv_in, - q_w, - k_w, - eps, - group_name=self.unique_name, - ) - - def fused_allreduce_rmsnorm_mxfp4_quant( - self, - input_: torch.Tensor, - residual_inp_: torch.Tensor, - weight_: torch.Tensor, - eps: float, - prefill_support: bool = False, - emit_bf16: bool = False, - ): - return self.fused_allreduce_rmsnorm_quant( - input_, - residual_inp_, - weight_, - eps, - prefill_support, - quant_type="mxfp4", - emit_bf16=emit_bf16, - ) + if self.device_communicator is None: + raise ValueError("No device communicator found") + return self.device_communicator.fused_qknorm_allreduce(qkv_in, q_w, k_w, eps) def _fused_allreduce_rmsnorm_out_place( self, @@ -659,14 +637,25 @@ def reduce_scatter_tensor( # return outplace_reduce_scatter(input_, group_name=self.unique_name, dim=dim) world_size = self.world_size assert world_size > 1, "error! world_size = 1" + ndim = input_.dim() assert ( - input_.numel() % world_size == 0 - ), "input shape error, input.numel() % world_size should equals to 0" - if input_.shape[0] % world_size == 0: - out_dim0 = input_.shape[0] // world_size - out_shape = (out_dim0,) + input_.shape[1:] - else: - out_shape = (input_.numel() // world_size,) + -ndim <= dim < ndim + ), f"Invalid dim ({dim}) for input tensor with shape {tuple(input_.shape)}" + if dim < 0: + dim += ndim + assert input_.shape[dim] % world_size == 0, ( + f"input shape error, input.shape[{dim}]={input_.shape[dim]} " + f"is not divisible by world_size={world_size}" + ) + # Output keeps the same rank/strides as input, only the scattered + # dim shrinks by world_size. Allocation is contiguous; the custom + # kernel writes elements in linear order into this layout, so no + # post-kernel reshape/copy is needed. + out_shape = ( + input_.shape[:dim] + + (input_.shape[dim] // world_size,) + + input_.shape[dim + 1 :] + ) output_ = torch.empty(out_shape, dtype=input_.dtype, device=input_.device) if use_custom: diff --git a/aiter/ops/custom_all_reduce.py b/aiter/ops/custom_all_reduce.py index 6f4741cf2e8..b002e31b718 100644 --- a/aiter/ops/custom_all_reduce.py +++ b/aiter/ops/custom_all_reduce.py @@ -39,6 +39,10 @@ def reduce_scatter( _fa: int, inp: torch.Tensor, out: torch.Tensor, + m: int, + n: int, + k: int, + split_dim: int, reg_ptr: int, reg_bytes: int, ) -> None: ... diff --git a/csrc/include/custom_all_reduce.cuh b/csrc/include/custom_all_reduce.cuh index dec526d8018..d84d78aaf69 100644 --- a/csrc/include/custom_all_reduce.cuh +++ b/csrc/include/custom_all_reduce.cuh @@ -30,6 +30,15 @@ namespace aiter { +// Selects which input dim is the scatter dim for reduce_scatter. +// Python framework collapses the tensor to one of these canonical shapes: +// kFirst : input flattened to (k,) ; only `k` used (= numel) +// kLast : input reshaped to (n, k) ; `m=0`, n/k used +// kMid : input reshaped to (m, n, k) ; all three used +// In all cases `n` (or `k` for kFirst) is the INPUT dim length; output's +// corresponding dim = input_dim / ngpus. +enum class ReduceScatterSplitDim : int { kFirst = 0, kLast = 1, kMid = 2 }; + constexpr int kMaxBlocks = 80; // note: we don't want to use atomics for signals because peer atomics are no // supported on PCIe links @@ -764,16 +773,22 @@ __global__ void __launch_bounds__(512, 1) allgather_lastdim(RankData* _dp, } } -/* - * reduce_scatter, at first dim - * range = size / (pack_size * ngpu) - * for case: - * input:(ngpus * n) -> output:(n) - * input:(ngpus * m, n, ...) -> output(m, n, ...) - * cond: size % (pack_size * ngpus) == 0 - * */ +// ========== reduce_scatter kernel start ========== +// +// reduce_scatter has 3 categories depending on where the scatter dim sits in +// the input tensor. Any higher-rank tensor can be collapsed into one of these +// shapes by flattening the dims before / after the scatter dim: +// - scatter dim is the first dim: input(ngpus*m, n[, ...]) -> output(m, n[, ...]) +// - scatter dim is the last dim: input(m, ngpus*n) -> output(m, n) +// - scatter dim is a middle dim: input(m, ngpus*n, k) -> output(m, n, k) +// Each non-first-dim case has a vectorized kernel (pack 16B per thread) and a +// naive fallback for shapes that cannot be vectorized along the inner dim. +// +// reduce_scatter, scatter on first dim. +// Note: `range` is the per-rank packed element count, i.e. total_elems / (ngpus * pack_size). +// cond: total_elems % (ngpus * pack_size) == 0 template -__global__ void __launch_bounds__(512, 1) reduce_scatter_first_dim( +__global__ void __launch_bounds__(512, 1) reduce_scatter_split_first_dim( RankData* _dp, RankSignals sg, Signal* self_sg, T* __restrict__ result, int rank, int range) { int tid = blockIdx.x * blockDim.x + threadIdx.x; @@ -799,6 +814,148 @@ __global__ void __launch_bounds__(512, 1) reduce_scatter_first_dim( } } +// reduce_scatter, scatter on last dim — naive (non-vectorized) fallback. +// Use when the last dim cannot be vectorized along pack_size +// (e.g. n % (ngpus * pack_size) != 0). Slower than the vectorized version. +// Params: `n` is the input's last-dim length; output's last dim = n / ngpus. +// cond: n % ngpus == 0 +// shape: input (m, n) -> output (m, n / ngpus) +template +__global__ void __launch_bounds__(256, 1) reduce_scatter_split_lastdim_naive( + RankData* _dp, RankSignals sg, Signal* self_sg, T* __restrict__ result, int rank, int m, int n +) +{ + int size = m * n / ngpus; + int splited_n = n / ngpus; + int index = blockIdx.x * blockDim.x + threadIdx.x; + const T* ptrs[ngpus]; +#pragma unroll + for (int i = 0; i < ngpus; ++i) + { + int target = (rank + i) % ngpus; + ptrs[i] = (const T*)_dp->ptrs[target]; + } + start_sync(sg, self_sg, rank); + for (int i = index; i < size; i += blockDim.x * gridDim.x) + { + int index_x = i % splited_n; + int index_y = i / splited_n; + int load_index = index_y * n + rank * splited_n + index_x; + opus::fp32_t rslt_reg = 0.0f; +#pragma unroll + for (int j = 0; j < ngpus; ++j) + { + rslt_reg += upcast_s(ptrs[j][load_index]); + } + result[i] = downcast_s(rslt_reg); + } +} + +// reduce_scatter, scatter on last dim — vectorized (16B / thread). +// Params: `n` is the input's last-dim length; output's last dim = n / ngpus. +// cond: n % (ngpus * pack_size) == 0 +// shape: input (m, n) -> output (m, n / ngpus) +template +__global__ void __launch_bounds__(256, 1) reduce_scatter_split_lastdim( + RankData* _dp, RankSignals sg, Signal* self_sg, T* __restrict__ result, int rank, int m, int n +) +{ + constexpr int pack_size = 16 / sizeof(T); + using P = typename opus::vector_t; + using A = typename opus::vector_t; + int size = m * n / (ngpus * pack_size); + int splited_n = n / (ngpus * pack_size); + int packed_dim_n = n / pack_size; + int index = blockIdx.x * blockDim.x + threadIdx.x; + const P* ptrs[ngpus]; +#pragma unroll + for (int i = 0; i < ngpus; ++i) + { + int target = (rank + i) % ngpus; + ptrs[i] = (const P*)_dp->ptrs[target]; + } + start_sync(sg, self_sg, rank); + for (int i = index; i < size; i += blockDim.x * gridDim.x) + { + int index_x = i % splited_n; + int index_y = i / splited_n; + int load_index = index_y * packed_dim_n + rank * splited_n + index_x; + *(reinterpret_cast(result) + i) = packed_reduce(ptrs, load_index); + } +} + +// reduce_scatter, scatter on middle dim — naive (non-vectorized) fallback. +// Use when the inner dim `k` cannot be vectorized (k % pack_size != 0). +// Params: `n` is the input's middle-dim length; output's middle dim = n / ngpus. +// cond: n % ngpus == 0 +// shape: input (m, n, k) -> output (m, n / ngpus, k) +template +__global__ void __launch_bounds__(256, 1) reduce_scatter_split_middim_naive( + RankData* _dp, RankSignals sg, Signal* self_sg, T* __restrict__ result, int rank, int m, int n, int k +) +{ + int size = m * n * k / ngpus; + int splited_n = n / ngpus; + int index = blockIdx.x * blockDim.x + threadIdx.x; + const T* ptrs[ngpus]; +#pragma unroll + for (int i = 0; i < ngpus; ++i) + { + int target = (rank + i) % ngpus; + ptrs[i] = (const T*)_dp->ptrs[target]; + } + start_sync(sg, self_sg, rank); + for (int i = index; i < size; i += blockDim.x * gridDim.x) + { + int index_m = i / (splited_n * k); + int index_n = (i % (splited_n * k)) / k; + int index_k = (i % (splited_n * k)) % k; + int load_index = index_m * (n * k) + (rank * splited_n + index_n) * k + index_k; + opus::fp32_t rslt_reg = 0.0f; +#pragma unroll + for (int j = 0; j < ngpus; ++j) + { + rslt_reg += upcast_s(ptrs[j][load_index]); + } + result[i] = downcast_s(rslt_reg); + } +} + +// reduce_scatter, scatter on middle dim — vectorized (16B / thread along k). +// Params: `n` is the input's middle-dim length; output's middle dim = n / ngpus. +// cond: n % ngpus == 0 && k % pack_size == 0 +// shape: input (m, n, k) -> output (m, n / ngpus, k) +template +__global__ void __launch_bounds__(256, 1) reduce_scatter_split_middim( + RankData* _dp, RankSignals sg, Signal* self_sg, T* __restrict__ result, int rank, int m, int n, int k +) +{ + constexpr int pack_size = 16 / sizeof(T); + using P = typename opus::vector_t; + using A = typename opus::vector_t; + int size = m * n * k / (pack_size * ngpus); + int splited_n = n / ngpus; + int packed_dim_k = k / pack_size; + int index = blockIdx.x * blockDim.x + threadIdx.x; + const P* ptrs[ngpus]; +#pragma unroll + for (int i = 0; i < ngpus; ++i) + { + int target = (rank + i) % ngpus; + ptrs[i] = (const P*)_dp->ptrs[target]; + } + start_sync(sg, self_sg, rank); + for (int i = index; i < size; i += blockDim.x * gridDim.x) + { + int index_m = i / (splited_n * packed_dim_k); + int index_n = (i % (splited_n * packed_dim_k)) / packed_dim_k; + int index_k = (i % (splited_n * packed_dim_k)) % packed_dim_k; + int load_index = index_m * (n * packed_dim_k) + (rank * splited_n + index_n) * packed_dim_k + index_k; + *(reinterpret_cast(result) + i) = packed_reduce(ptrs, load_index); + } +} +// ========== reduce_scatter kernel end ========== + // fp8 quant all-reduce code start template struct Fp16Filter @@ -3400,30 +3557,102 @@ class CustomAllreduce #undef KL } +// reduce_scatter dispatch. Python wrapper is responsible for: +// - normalizing dim < 0 +// - rejecting shapes where n (or k, for kFirst) % ngpus != 0 +// - falling back to an external library when scatter is on the first dim +// AND the flattened size cannot be vectorized (numel % (ngpus*pack) != 0) +// +// For kLast / kMid, this dispatcher auto-falls-back to the naive kernel when +// the inner dim cannot be vectorized; both kernels live in this file. template -void dispatchReduceScatter(hipStream_t stream, T* input, T* output, int size) +void dispatchReduceScatter(hipStream_t stream, T* input, T* output, + int m, int n, int k, + ReduceScatterSplitDim split_dim) { - RankData* ptrs = get_buffer_RD(stream, input); - auto d = 16 / sizeof(T); - int range = size / (world_size_ * d); - dim3 block(512); - int block_num = (range + 511) / 512; - dim3 grid(std::min(16, block_num)); - switch(world_size_) + RankData* ptrs = get_buffer_RD(stream, input); + constexpr int pack_size = 16 / sizeof(T); + constexpr int kGridCap = 80; // bounded by signal slots (see kMaxBlocks) + + switch(split_dim) { - case 8: - reduce_scatter_first_dim - <<>>(ptrs, sg_, self_sg_, output, rank_, range); + case ReduceScatterSplitDim::kFirst: { + // Python guarantees k % (ngpus * pack_size) == 0 here. + int range = k / (world_size_ * pack_size); + dim3 block(512); + dim3 grid(std::min(kGridCap, (range + 511) / 512)); + switch(world_size_) + { + case 8: + reduce_scatter_split_first_dim + <<>>(ptrs, sg_, self_sg_, output, rank_, range); + break; + case 4: + reduce_scatter_split_first_dim + <<>>(ptrs, sg_, self_sg_, output, rank_, range); + break; + case 2: + reduce_scatter_split_first_dim + <<>>(ptrs, sg_, self_sg_, output, rank_, range); + break; + default: printf("reduce_scatter world_size error!\n"); + } break; - case 4: - reduce_scatter_first_dim - <<>>(ptrs, sg_, self_sg_, output, rank_, range); + } + case ReduceScatterSplitDim::kLast: { + // Framework passes (n_f, k_f); kernel signature is (m, n) where + // kernel.m = n_f, kernel.n = k_f. n % ngpus is guaranteed by Python. + bool vec = (k % (world_size_ * pack_size) == 0); + int size = vec ? (n * k) / (world_size_ * pack_size) + : (n * k) / world_size_; + dim3 block(256); + dim3 grid(std::min(kGridCap, (size + 255) / 256)); +#define LAUNCH_LAST(NG) \ + do { \ + if(vec) \ + reduce_scatter_split_lastdim \ + <<>>(ptrs, sg_, self_sg_, output, rank_, n, k); \ + else \ + reduce_scatter_split_lastdim_naive \ + <<>>(ptrs, sg_, self_sg_, output, rank_, n, k); \ + } while(0) + switch(world_size_) + { + case 8: LAUNCH_LAST(8); break; + case 4: LAUNCH_LAST(4); break; + case 2: LAUNCH_LAST(2); break; + default: printf("reduce_scatter world_size error!\n"); + } +#undef LAUNCH_LAST break; - case 2: - reduce_scatter_first_dim - <<>>(ptrs, sg_, self_sg_, output, rank_, range); + } + case ReduceScatterSplitDim::kMid: { + // n is the input's middle-dim length; n % ngpus is guaranteed by Python. + bool vec = (k % pack_size == 0); + int size = vec ? (m * n * k) / (world_size_ * pack_size) + : (m * n * k) / world_size_; + dim3 block(256); + dim3 grid(std::min(kGridCap, (size + 255) / 256)); +#define LAUNCH_MID(NG) \ + do { \ + if(vec) \ + reduce_scatter_split_middim \ + <<>>(ptrs, sg_, self_sg_, output, rank_, m, n, k); \ + else \ + reduce_scatter_split_middim_naive \ + <<>>(ptrs, sg_, self_sg_, output, rank_, m, n, k); \ + } while(0) + switch(world_size_) + { + case 8: LAUNCH_MID(8); break; + case 4: LAUNCH_MID(4); break; + case 2: LAUNCH_MID(2); break; + default: printf("reduce_scatter world_size error!\n"); + } +#undef LAUNCH_MID break; - default: printf("reduce_scatter world_size error!\n"); + } + default: printf("reduce_scatter split_dim error!\n"); } } diff --git a/csrc/include/custom_all_reduce.h b/csrc/include/custom_all_reduce.h index 04370f40a5f..0c9c59b61bd 100644 --- a/csrc/include/custom_all_reduce.h +++ b/csrc/include/custom_all_reduce.h @@ -38,9 +38,20 @@ void all_reduce(fptr_t _fa, bool open_fp8_quant, int64_t reg_inp_ptr, int64_t reg_inp_bytes); +// reduce_scatter dispatcher. (m, n, k, split_dim) describe the canonical +// shape the Python wrapper collapsed the input to: +// split_dim = 0 (kFirst): only `k` (= numel) used +// split_dim = 1 (kLast) : input reshaped to (n, k); m=0 +// split_dim = 2 (kMid) : input reshaped to (m, n, k) +// In all cases the scattered dim length is the input dim (k for kFirst, +// n for kLast/kMid); output's scattered dim = that / ngpus. void reduce_scatter(fptr_t _fa, const aiter_tensor_t& inp, const aiter_tensor_t& out, + int64_t m, + int64_t n, + int64_t k, + int64_t split_dim, int64_t reg_ptr, int64_t reg_bytes); void all_gather_reg(fptr_t _fa, diff --git a/csrc/include/rocm_ops.hpp b/csrc/include/rocm_ops.hpp index fe53ffe11ac..b9a38a220c8 100644 --- a/csrc/include/rocm_ops.hpp +++ b/csrc/include/rocm_ops.hpp @@ -431,6 +431,10 @@ namespace py = pybind11; py::arg("_fa"), \ py::arg("inp"), \ py::arg("out"), \ + py::arg("m"), \ + py::arg("n"), \ + py::arg("k"), \ + py::arg("split_dim"), \ py::arg("reg_ptr"), \ py::arg("reg_bytes")); \ m.def("all_gather_reg", \ diff --git a/csrc/kernels/custom_all_reduce.cu b/csrc/kernels/custom_all_reduce.cu index e96c2c7750b..840e81f312c 100644 --- a/csrc/kernels/custom_all_reduce.cu +++ b/csrc/kernels/custom_all_reduce.cu @@ -118,7 +118,9 @@ static void _all_reduce(fptr_t _fa, void* inp, void* out, } static void _reduce_scatter(fptr_t _fa, void* inp, void* out, - int64_t size, AiterDtype dtype) + int m, int n, int k, + aiter::ReduceScatterSplitDim split_dim, + AiterDtype dtype) { hipStream_t stream = aiter::getCurrentHIPStream(); auto fa = reinterpret_cast(_fa); @@ -128,14 +130,14 @@ static void _reduce_scatter(fptr_t _fa, void* inp, void* out, fa->dispatchReduceScatter(stream, reinterpret_cast(inp), reinterpret_cast(out), - size); + m, n, k, split_dim); break; } case AITER_DTYPE_fp16: { fa->dispatchReduceScatter(stream, reinterpret_cast(inp), reinterpret_cast(out), - size); + m, n, k, split_dim); break; } #if (__CUDA_ARCH__ >= 800 || !defined(__CUDA_ARCH__)) @@ -143,7 +145,7 @@ static void _reduce_scatter(fptr_t _fa, void* inp, void* out, fa->dispatchReduceScatter(stream, reinterpret_cast(inp), reinterpret_cast(out), - size); + m, n, k, split_dim); break; } #endif @@ -436,13 +438,15 @@ void all_reduce(fptr_t _fa, void reduce_scatter(fptr_t _fa, const aiter_tensor_t& inp, const aiter_tensor_t& out, + int64_t m, int64_t n, int64_t k, + int64_t split_dim, int64_t reg_ptr, int64_t reg_bytes) { HipDeviceGuard device_guard(inp.device_id); hipStream_t stream = aiter::getCurrentHIPStream(); - auto dtype = inp.dtype(); - int64_t inp_numel = inp.numel(); - int64_t data_bytes = inp_numel * inp.element_size(); + auto dtype = inp.dtype(); + int64_t data_bytes = inp.numel() * inp.element_size(); + auto sd = static_cast(split_dim); if(reg_ptr != 0) { @@ -450,11 +454,15 @@ void reduce_scatter(fptr_t _fa, throw std::runtime_error("registered buffer is too small to contain the input"); HIP_CALL(hipMemcpyAsync((void*)reg_ptr, inp.data_ptr(), data_bytes, hipMemcpyDeviceToDevice, stream)); - _reduce_scatter(_fa, (void*)reg_ptr, out.data_ptr(), inp_numel, dtype); + _reduce_scatter(_fa, (void*)reg_ptr, out.data_ptr(), + static_cast(m), static_cast(n), static_cast(k), + sd, dtype); } else { - _reduce_scatter(_fa, inp.data_ptr(), out.data_ptr(), inp_numel, dtype); + _reduce_scatter(_fa, inp.data_ptr(), out.data_ptr(), + static_cast(m), static_cast(n), static_cast(k), + sd, dtype); } } diff --git a/op_tests/multigpu_tests/test_reduce_scatter.py b/op_tests/multigpu_tests/test_reduce_scatter.py index 7a5dec71cff..cd5d7d60145 100644 --- a/op_tests/multigpu_tests/test_reduce_scatter.py +++ b/op_tests/multigpu_tests/test_reduce_scatter.py @@ -14,17 +14,12 @@ init_distributed_environment, set_custom_all_reduce, get_tp_group, - graph_capture, destroy_model_parallel, destroy_distributed_environment, ) from aiter.dist.utils import get_open_port, get_distributed_init_method, get_ip from aiter.dist.communication_op import tensor_model_parallel_reduce_scatter -from aiter.test_common import ( - checkAllclose, - perftest, - benchmark, -) +from aiter.test_common import perftest from multiprocessing import set_start_method, Pool, freeze_support import logging @@ -38,13 +33,14 @@ def reduce_scatter( pp_size, rankID, x, - withGraph=False, + dim=0, use_custom=False, distributed_init_method: Optional[str] = None, ): + """Per-rank worker. Runs reduce_scatter on x with the given dim and + returns (output, per-call latency in us).""" device = torch.device(f"cuda:{rankID}") torch.cuda.set_device(device) - # init logger.info(f"RANK: {rankID} {tp_size} init_process_group...") set_custom_all_reduce(True) init_distributed_environment( @@ -54,35 +50,18 @@ def reduce_scatter( ) ensure_model_parallel_initialized(tp_size, pp_size) x = x.to(device) - # dist.barrier(device_ids=[i for i in range(tp_size)]) - # warmup and align all gpu + # warmup + barrier so the timing on first call isn't polluted. group = get_tp_group().device_group dist.all_reduce(torch.zeros(1).cuda(), group=group) torch.cuda.synchronize() - if withGraph: - graph = torch.cuda.CUDAGraph() - with graph_capture() as gc: - with torch.cuda.graph(graph, stream=gc.stream): - out = tensor_model_parallel_reduce_scatter(x, use_custom=use_custom) - out.fill_(0) - - @perftest() - def run_ca(): - graph.replay() - - _, us = run_ca() - out = (out, us) - else: - - @perftest() - def run_ca(x): - return tensor_model_parallel_reduce_scatter(x, use_custom=use_custom) + @perftest() + def run_ca(x): + return tensor_model_parallel_reduce_scatter(x, use_custom=use_custom, dim=dim) - out = run_ca(x) + out = run_ca(x) - # destroy if dist.is_initialized(): destroy_model_parallel() destroy_distributed_environment() @@ -90,148 +69,139 @@ def run_ca(x): return out -def get_reduce_scatter_output( +def _build_input(shape, dtype, tp_size, rand_seed): + """Deterministic per-rank input: rand_seed[i] repeats over a chunk so + each rank ends up with an identical tensor of shape `shape`. With all + ranks having identical input, sum_across_ranks = tp_size * input — that + gives us an analytic reference for any scatter dim (see _ref_output).""" + n = 1 + for s in shape: + n *= s + chunk_size = n // tp_size + return rand_seed.repeat_interleave(chunk_size).reshape(shape).to(dtype).contiguous() + + +def _ref_output(input_tensor, dim, rank, tp_size): + """Analytic reference for one rank's reduce_scatter output. Computed in + fp32 to avoid bf16 accumulation noise on the multiply.""" + ndim = input_tensor.dim() + if dim < 0: + dim += ndim + full_sum = tp_size * input_tensor.float() + chunk = input_tensor.shape[dim] // tp_size + out = full_sum.narrow(dim, rank * chunk, chunk).contiguous() + return out.to(input_tensor.dtype) + + +def run_reduce_scatter_parallel( tp_size, pp_size, shape, + dim, dtype, rand_seed, use_custom, - distributed_init_method: Optional[str] = None, + distributed_init_method, ): + """Spawn tp_size processes, each running one reduce_scatter call. + Returns list of (out, us) per rank.""" os.environ["MASTER_ADDR"] = "127.0.0.1" os.environ["MASTER_PORT"] = "49373" pool = Pool(processes=tp_size) rets = [] for i in range(tp_size): - # input = torch.randn(shape, dtype=dtype, device="cuda") - input_ = torch.ones(shape, dtype=dtype, device="cuda") - n = input_.numel() - chunk_size = n // 8 - input = rand_seed.repeat_interleave(chunk_size) + x = _build_input(shape, dtype, tp_size, rand_seed) rets.append( pool.apply_async( reduce_scatter, - args=( - tp_size, - pp_size, - i, - input, - False, - use_custom, - # 1, - distributed_init_method, - ), + args=(tp_size, pp_size, i, x, dim, use_custom, distributed_init_method), ) - # pool.apply_async(call_aiter_allgather_naive, args=(tp_size, pp_size, i, input, 1)) ) pool.close() pool.join() + return [el.get() for el in rets] - ar_rslt = [] - rets = [el.get() for el in rets] - for out, us in rets: - ar_rslt.append(out) - return ar_rslt +def run_case(label, shape, dim, dtype, tp_size, init_method_factory): + """End-to-end one case: spawn the custom run, compute accuracy against + the analytic PyTorch reference, collect latency. Returns one row for + the summary table. -def reduce_scatter_acctest( - tp_size, pp_size, shape, dtype, distributed_init_method: Optional[str] = None -): + No external-library comparison — other libs (torch.distributed / + pynccl) don't support scatter on non-zero dims, so latency-vs-them + isn't meaningful for the new kernels.""" rand_seed = torch.randint(1, 16, (tp_size,), dtype=dtype, device="cuda") - dist_rslt = get_reduce_scatter_output( - tp_size, pp_size, shape, dtype, rand_seed, False, distributed_init_method - ) - aiter_rslt = get_reduce_scatter_output( - tp_size, pp_size, shape, dtype, rand_seed, True, distributed_init_method + + custom_rets = run_reduce_scatter_parallel( + tp_size, + 1, + shape, + dim, + dtype, + rand_seed, + True, + init_method_factory(), ) - error = 0.0 - for i in range(len(dist_rslt)): - error += checkAllclose(dist_rslt[i], aiter_rslt[i]) - if error == 0: - print("accuracy pass") - else: - print("accuracy failed") + # Analytic reference vs each rank's output. + ref_input = _build_input(shape, dtype, tp_size, rand_seed) + max_err = 0.0 + mean_err = 0.0 + for rank, (out, _us) in enumerate(custom_rets): + ref = _ref_output(ref_input, dim, rank, tp_size).cpu() + diff = (out.cpu().float() - ref.float()).abs() + max_err = max(max_err, diff.max().item()) + # Use max-over-ranks for mean too, so a single bad rank shows up. + mean_err = max(mean_err, diff.mean().item()) + custom_us = [us for _, us in custom_rets] -@benchmark() -def reduce_scatter_perftest( - tp_size, - pp_size, - shape, - dtype, - withGraph=False, - use_custom=False, - distributed_init_method: Optional[str] = None, -): - print(f"run perf test, use custom allgather {use_custom}") - os.environ["MASTER_ADDR"] = "127.0.0.1" - os.environ["MASTER_PORT"] = "49373" - pool = Pool(processes=tp_size) - ref = torch.zeros(shape, dtype=dtype) - rets = [] - input_list = [] - for i in range(tp_size): - x = torch.randn(shape, dtype=dtype) - input_list.append(x) - rets.append( - pool.apply_async( - reduce_scatter, - args=( - tp_size, - pp_size, - i, - x, - withGraph, - use_custom, - distributed_init_method, - ), - ) - ) - pool.close() - pool.join() - ref = input_list[0] - for i in range(tp_size - 1): - ref = torch.concat((ref, input_list[i + 1]), -1) - - rets = [el.get() for el in rets] - all_us = [us for _, us in rets] - for out, us in rets: - msg = f"reduce_scatter (use custom {use_custom}): {shape=} {dtype=} {withGraph=} {us:>8.2f}" - print(msg) return { - "min_us": min(all_us), - "max_us": max(all_us), + "case": label, + "shape": str(tuple(shape)), + "dim": dim, + "dtype": str(dtype).split(".")[-1], + "max_abs_err": max_err, + "mean_abs_err": mean_err, + "min_us": min(custom_us), + "max_us": max(custom_us), } -l_dtype = ["bf16"] -l_shape = [ - # (4096, 2048) - (128, 8192), - (32768, 8192), - # (16, 512) +# Cases designed for tp_size=8 and bf16 (pack_size = 16 // 2 = 8). +# Each case targets a specific kernel branch in dispatchReduceScatter: +# +# first_dim_vec : numel % (ngpus * pack_size) == 0 → split_first_dim +# last_dim_vec : last % (ngpus * pack_size) == 0 → split_lastdim (vec) +# last_dim_naive : last % ngpus == 0 but % pack != 0 → split_lastdim_naive +# mid_dim_vec : k % pack_size == 0 → split_middim (vec) +# mid_dim_naive : k % pack_size != 0 → split_middim_naive +# +# All five satisfy shape[dim] % ngpus == 0 (Python wrapper's hard requirement). +l_cases = [ + ("first_dim_vec", (512, 1024), 0), + ("last_dim_vec", (256, 512), -1), + ("last_dim_naive", (256, 40), -1), + ("mid_dim_vec", (16, 64, 64), 1), + ("mid_dim_naive", (16, 64, 5), 1), ] -parser = argparse.ArgumentParser(description="config input of test") +l_dtype = ["bf16"] + +parser = argparse.ArgumentParser(description="reduce_scatter accuracy + latency test") parser.add_argument( "-d", "--dtype", type=str, choices=l_dtype, - nargs="?", - const=None, default=None, help="data type", ) parser.add_argument( - "-s", - "--shape", - type=dtypes.str2tuple, - nargs="?", - const=None, + "-c", + "--case", + type=str, default=None, - help="shape. e.g. -s 128,8192", + help="run only one case by label, e.g. mid_dim_naive", ) @@ -239,50 +209,32 @@ def reduce_scatter_perftest( freeze_support() args = parser.parse_args() if args.dtype is None: - l_dtype = [dtypes.d_dtypes[key] for key in l_dtype] + dtypes_to_run = [dtypes.d_dtypes[k] for k in l_dtype] + else: + dtypes_to_run = [dtypes.d_dtypes[args.dtype]] + if args.case is None: + cases_to_run = l_cases else: - l_dtype = [dtypes.d_dtypes[args.dtype]] - if args.shape is not None: - l_shape = [args.shape] - df = [] - for dtype in l_dtype: - for shape in l_shape: - print(f"accuracy test of dtype:{dtype}, shape:{shape}") - reduce_scatter_acctest( - 8, - 1, - shape, - dtype, - distributed_init_method=get_distributed_init_method( - get_ip(), get_open_port() - ), + cases_to_run = [c for c in l_cases if c[0] == args.case] + assert cases_to_run, f"no case named {args.case!r}" + + tp_size = 8 + + def init_method_factory(): + return get_distributed_init_method(get_ip(), get_open_port()) + + rows = [] + for dtype in dtypes_to_run: + for label, shape, dim in cases_to_run: + print(f"\n=== {label} shape={shape} dim={dim} dtype={dtype} ===") + row = run_case(label, shape, dim, dtype, tp_size, init_method_factory) + print( + f" max_abs_err={row['max_abs_err']:.4g} " + f"mean_abs_err={row['mean_abs_err']:.4g} " + f"latency={row['min_us']:.2f}-{row['max_us']:.2f}us" ) - print(f"perf test of dtype:{dtype}, shape:{shape}") - for use_custom in [True, False]: - ret = reduce_scatter_perftest( - 8, - 1, - shape, - dtype, - withGraph=False, - use_custom=use_custom, - distributed_init_method=get_distributed_init_method( - get_ip(), get_open_port() - ), - ) - df.append(ret) - df = pd.DataFrame(df) - show_cols = [ - "tp_size", - "shape", - "dtype", - "withGraph", - "use_custom", - "min_us", - "max_us", - ] - show_cols = [c for c in show_cols if c in df.columns] - logger.info( - "reduce scatter summary (markdown):\n%s", - df[show_cols].to_markdown(index=False), - ) + rows.append(row) + + df = pd.DataFrame(rows) + print("\n=== reduce_scatter summary ===") + print(df.to_markdown(index=False, floatfmt=".4g")) From 61998139464d47073f25d74b6dc1ff7c1e9e646c Mon Sep 17 00:00:00 2001 From: TennyWang1223 Date: Thu, 4 Jun 2026 04:01:43 +0000 Subject: [PATCH 2/2] [fix]: support dim!=0 case in torch distribute Signed-off-by: TennyWang1223 --- aiter/dist/parallel_state.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/aiter/dist/parallel_state.py b/aiter/dist/parallel_state.py index eb3e99e03e5..2bb0fb5e1b1 100644 --- a/aiter/dist/parallel_state.py +++ b/aiter/dist/parallel_state.py @@ -663,9 +663,20 @@ def reduce_scatter_tensor( input_, output_, group_name=self.unique_name, dim=dim ) else: - torch.distributed.reduce_scatter_tensor( - output_, input_, group=self.device_group - ) + if dim != 0: + input_ = input_.movedim(dim, 0).contiguous() + tmp_out_shape = (input_.shape[0] // world_size,) + input_.shape[1:] + tmp_output = torch.empty( + tmp_out_shape, dtype=input_.dtype, device=input_.device + ) + torch.distributed.reduce_scatter_tensor( + tmp_output, input_, group=self.device_group + ) + output_ = tmp_output.movedim(0, dim).contiguous() + else: + torch.distributed.reduce_scatter_tensor( + output_, input_, group=self.device_group + ) return output_ def all_gather(