Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions aiter/dist/device_communicators/communicator_cuda.py
Original file line number Diff line number Diff line change
Expand Up @@ -663,12 +663,15 @@ def reduce_scatter(
):
world_size = self.world_size
ca_comm = self.ca_comm
# Custom kernel supports scatter on first/last/mid dims; gate via
# should_custom_rs which also rejects first-dim-non-vectorizable
# shapes (no naive fallback exists for that case, see C++ dispatch).
if (
ca_comm is not None
and not ca_comm.disabled
and ca_comm.should_custom_ar(input_)
and ca_comm.should_custom_rs(input_, dim)
):
ca_comm.custom_reduce_scatter(input_, output_)
ca_comm.custom_reduce_scatter(input_, output_, dim)
else:
pynccl_comm = self.pynccl_comm
assert pynccl_comm is not None
Expand Down
75 changes: 70 additions & 5 deletions aiter/dist/device_communicators/custom_all_reduce.py
Original file line number Diff line number Diff line change
Expand Up @@ -629,35 +629,100 @@ def custom_all_reduce(
registered_input=False,
)

# reduce_scatter split_dim enum — must match `aiter::ReduceScatterSplitDim`
# in csrc/include/custom_all_reduce.cuh.
_RS_SPLIT_FIRST = 0
_RS_SPLIT_LAST = 1
_RS_SPLIT_MID = 2

@staticmethod
def _compute_rs_args(shape, dim: int, numel: int):
"""Collapse `shape` around the scatter dim into the canonical
(m, n, k, split_dim) the C++ dispatcher expects. `dim` must be
already normalized to [0, len(shape))."""
ndim = len(shape)
if dim == 0:
return 0, 0, numel, CustomAllreduce._RS_SPLIT_FIRST
if dim == ndim - 1:
n = 1
for s in shape[:-1]:
n *= s
return 0, n, shape[-1], CustomAllreduce._RS_SPLIT_LAST
m = 1
for s in shape[:dim]:
m *= s
k = 1
for s in shape[dim + 1 :]:
k *= s
return m, shape[dim], k, CustomAllreduce._RS_SPLIT_MID

def should_custom_rs(self, inp: torch.Tensor, dim: int) -> bool:
"""Return True iff the custom reduce_scatter kernel can handle
(inp, dim). Mirrors the C++ dispatch's hard requirements:

- all the should_custom_ar gates (size cap, contiguous, etc.)
- inp.shape[dim_normalized] % world_size == 0
- for dim == 0 (first-dim split) the flattened input must be
vectorizable: numel % (world_size * pack_size) == 0; there is
no naive fallback for the first-dim kernel and the framework is
expected to route to an external lib in that case.
(last/mid-dim kernels have naive fallbacks built in.)
"""
if not self.should_custom_ar(inp):
return False
ndim = inp.dim()
if dim < 0:
dim += ndim
if dim < 0 or dim >= ndim:
return False
if inp.shape[dim] % self.world_size != 0:
return False
if dim == 0:
pack_size = 16 // inp.element_size()
if inp.numel() % (self.world_size * pack_size) != 0:
return False
return True

def reduce_scatter(
self,
inp: torch.Tensor,
out: torch.Tensor,
dim: int = 0,
*,
registered: bool = False,
):
assert is_weak_contiguous(out), "output tensor is not weak-contiguous"
ndim = inp.dim()
if dim < 0:
dim += ndim
m, n, k, split_dim = self._compute_rs_args(tuple(inp.shape), dim, inp.numel())
reg = 0 if registered else self._pool["input"].data_ptr
reg_bytes = 0 if registered else self._pool["input"].max_size
ops.reduce_scatter(
self._ptr,
inp,
out,
m,
n,
k,
split_dim,
reg,
reg_bytes,
)

def custom_reduce_scatter(
self, input: torch.Tensor, output: torch.Tensor
self, input: torch.Tensor, output: torch.Tensor, dim: int = 0
) -> Optional[torch.Tensor]:
# when custom allreduce is disabled, this will be None
if self.disabled or not self.should_custom_ar(input):
# when custom allreduce is disabled or this shape/dim is unsupported,
# this will be None and the caller is expected to fall back to an
# external reduce_scatter implementation (NCCL / pynccl / torch.dist).
if self.disabled or not self.should_custom_rs(input, dim):
return None
if self._IS_CAPTURING:
if torch.cuda.is_current_stream_capturing():
return self.reduce_scatter(input, output, registered=True)
return self.reduce_scatter(input, output, dim, registered=True)
else:
return self.reduce_scatter(input, output, registered=False)
return self.reduce_scatter(input, output, dim, registered=False)

def _allgather_out_shape(self, inp: torch.Tensor, dim: int):
ndim = inp.dim()
Expand Down
78 changes: 39 additions & 39 deletions aiter/dist/parallel_state.py
Original file line number Diff line number Diff line change
Expand Up @@ -532,14 +532,15 @@ def fused_allreduce_rmsnorm_quant_per_group(
prefill_support: bool = False,
emit_bf16: bool = False,
):
return self.fused_allreduce_rmsnorm_quant(
if self.device_communicator is None:
raise ValueError("No device communicator found")
return self.device_communicator.fused_allreduce_rmsnorm_quant_per_group(
input_,
residual_inp_,
weight_,
eps,
group_size,
prefill_support,
quant_type="per_group",
group_size=group_size,
emit_bf16=emit_bf16,
)

Expand All @@ -550,32 +551,9 @@ def fused_qknorm_allreduce(
k_w: torch.Tensor,
eps: float,
):
return fused_qknorm_allreduce_(
qkv_in,
q_w,
k_w,
eps,
group_name=self.unique_name,
)

def fused_allreduce_rmsnorm_mxfp4_quant(
self,
input_: torch.Tensor,
residual_inp_: torch.Tensor,
weight_: torch.Tensor,
eps: float,
prefill_support: bool = False,
emit_bf16: bool = False,
):
return self.fused_allreduce_rmsnorm_quant(
input_,
residual_inp_,
weight_,
eps,
prefill_support,
quant_type="mxfp4",
emit_bf16=emit_bf16,
)
if self.device_communicator is None:
raise ValueError("No device communicator found")
return self.device_communicator.fused_qknorm_allreduce(qkv_in, q_w, k_w, eps)

def _fused_allreduce_rmsnorm_out_place(
self,
Expand Down Expand Up @@ -659,24 +637,46 @@ def reduce_scatter_tensor(
# return outplace_reduce_scatter(input_, group_name=self.unique_name, dim=dim)
world_size = self.world_size
assert world_size > 1, "error! world_size = 1"
ndim = input_.dim()
assert (
input_.numel() % world_size == 0
), "input shape error, input.numel() % world_size should equals to 0"
if input_.shape[0] % world_size == 0:
out_dim0 = input_.shape[0] // world_size
out_shape = (out_dim0,) + input_.shape[1:]
else:
out_shape = (input_.numel() // world_size,)
-ndim <= dim < ndim
), f"Invalid dim ({dim}) for input tensor with shape {tuple(input_.shape)}"
if dim < 0:
dim += ndim
assert input_.shape[dim] % world_size == 0, (
f"input shape error, input.shape[{dim}]={input_.shape[dim]} "
f"is not divisible by world_size={world_size}"
)
# Output keeps the same rank/strides as input, only the scattered
# dim shrinks by world_size. Allocation is contiguous; the custom
# kernel writes elements in linear order into this layout, so no
# post-kernel reshape/copy is needed.
out_shape = (
input_.shape[:dim]
+ (input_.shape[dim] // world_size,)
+ input_.shape[dim + 1 :]
)

output_ = torch.empty(out_shape, dtype=input_.dtype, device=input_.device)
if use_custom:
Comment thread
yitingw1 marked this conversation as resolved.
Resolved
outplace_reduce_scatter(
input_, output_, group_name=self.unique_name, dim=dim
)
else:
torch.distributed.reduce_scatter_tensor(
output_, input_, group=self.device_group
)
if dim != 0:
input_ = input_.movedim(dim, 0).contiguous()
tmp_out_shape = (input_.shape[0] // world_size,) + input_.shape[1:]
tmp_output = torch.empty(
tmp_out_shape, dtype=input_.dtype, device=input_.device
)
torch.distributed.reduce_scatter_tensor(
tmp_output, input_, group=self.device_group
)
output_ = tmp_output.movedim(0, dim).contiguous()
else:
torch.distributed.reduce_scatter_tensor(
output_, input_, group=self.device_group
)
return output_

def all_gather(
Expand Down
4 changes: 4 additions & 0 deletions aiter/ops/custom_all_reduce.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,10 @@ def reduce_scatter(
_fa: int,
inp: torch.Tensor,
out: torch.Tensor,
m: int,
n: int,
k: int,
split_dim: int,
reg_ptr: int,
reg_bytes: int,
) -> None: ...
Expand Down
Loading
Loading