Skip to content
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 38 additions & 0 deletions aiter/dist/device_communicators/custom_all_reduce.py
Original file line number Diff line number Diff line change
Expand Up @@ -912,6 +912,7 @@ def _select_ops(self):
self._ops_meta_size = ops.meta_size_gfx1250
self._ops_all_reduce = ops.all_reduce_gfx1250
self._ops_all_gather = ops.all_gather_gfx1250
self._ops_all_gather_lastdim_add = None
self._ops_reduce_scatter = ops.reduce_scatter_gfx1250
self._ops_dispose = ops.dispose_gfx1250
self._ops_get_graph_buffer_count = ops.get_graph_buffer_count_gfx1250
Expand All @@ -934,6 +935,7 @@ def _select_ops(self):
self._ops_init_custom_ar = ops.init_custom_ar
self._ops_all_reduce = ops.all_reduce
self._ops_all_gather = None
self._ops_all_gather_lastdim_add = ops.all_gather_lastdim_add
self._ops_reduce_scatter = ops.reduce_scatter
self._ops_dispose = ops.dispose
self._ops_register_input_buffer = ops.register_input_buffer
Expand Down Expand Up @@ -1888,6 +1890,42 @@ def all_gather_reg(self, inp: torch.Tensor, out: torch.Tensor = None, dim: int =
)
return out

def all_gather_lastdim_add(
self,
inp: torch.Tensor,
add_b: torch.Tensor,
add_c: torch.Tensor | None = None,
out: torch.Tensor | None = None,
) -> torch.Tensor:
"""out = bf16(bf16(allgather(inp, dim=-1) + add_b) [+ add_c]) in one
launch (the add3 double rounding), for 2D [rows, n] shards. Under graph
capture with enable_register_for_capturing, inp is read in place
(capture registers it); otherwise it is staged into the registered
input pool first."""
if self._ops_all_gather_lastdim_add is None:
raise NotImplementedError(
"all_gather_lastdim_add is not supported on gfx1250"
)
assert inp.dim() == 2, "all_gather_lastdim_add only supports 2D tensors"
shape = (inp.shape[0], inp.shape[1] * self.world_size)
if out is None:
out = torch.empty(shape, dtype=inp.dtype, device=inp.device)
for t in (add_b, add_c, out):
assert t is None or (
t.shape == shape and t.is_contiguous()
), f"all_gather_lastdim_add expects contiguous {shape} add/out tensors"
registered = (
self._IS_CAPTURING
and torch.cuda.is_current_stream_capturing()
and self.enable_register_for_capturing
)
reg = 0 if registered else self._pool["input"].data_ptr
reg_bytes = 0 if registered else self._pool["input"].max_size
self._ops_all_gather_lastdim_add(
self._ptr, inp, out, add_b, add_c, reg, reg_bytes
)
return out

def all_gather_unreg(
self, inp: torch.Tensor, out: torch.Tensor = None, dim: int = 0
):
Expand Down
13 changes: 13 additions & 0 deletions aiter/ops/custom_all_reduce.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@


import torch
from typing import Optional

from ..jit.core import compile_ops

Expand Down Expand Up @@ -58,6 +59,18 @@ def all_gather_reg(
) -> None: ...


@compile_ops("module_custom_all_reduce", develop=True)
def all_gather_lastdim_add(
_fa: int,
inp: torch.Tensor,
out: torch.Tensor,
add_b: torch.Tensor,
add_c: Optional[torch.Tensor] = None,
reg_ptr: int = 0,
reg_bytes: int = 0,
) -> None: ...


@compile_ops("module_custom_all_reduce", develop=True)
def all_gather_unreg(
_fa: int,
Expand Down
91 changes: 91 additions & 0 deletions csrc/include/custom_all_reduce.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -823,6 +823,60 @@ __global__ void __launch_bounds__(512, 1)
end_sync<ngpus, true>(sg, self_sg, rank);
}

// allgather along the last dim fused with the residual adds that follow it:
// result = T(T(gathered + b) [+ c]), the same double rounding as the unfused
// add-then-add (sglang add3). b and c are [rows, ngpus * last_dim] like result.
template <typename T, int ngpus, bool kHasC>
__global__ void __launch_bounds__(512, 1) allgather_lastdim_add(RankData* _dp,
RankSignals sg,
Signal* self_sg,
T* __restrict__ result,
int rank,
int size,
int last_dim_size,
const T* __restrict__ add_b,
const T* __restrict__ add_c)
{
constexpr int tnum_gpu = THREAD_NUM / ngpus;
constexpr int pack_size = 16 / sizeof(T);
using P = typename opus::vector_t<T, pack_size>;
using A = typename opus::vector_t<opus::fp32_t, pack_size>;
int warp_id = threadIdx.x / tnum_gpu;
int lane_id = threadIdx.x % tnum_gpu;
int tid = blockIdx.x * tnum_gpu + lane_id;
int stride = gridDim.x * tnum_gpu;

last_dim_size /= pack_size;
const P* ptrs[ngpus];

#pragma unroll
for(int i = 0; i < ngpus; ++i)
{
ptrs[i] = (const P*)_dp->ptrs[i];
}
const P* bp = reinterpret_cast<const P*>(add_b);
const P* cp = reinterpret_cast<const P*>(add_c);
start_sync<ngpus>(sg, self_sg, rank);

for(int idx = tid; idx < size; idx += stride)
{
int y = idx / last_dim_size;
int x = idx % last_dim_size;
int write_idx = (ngpus * y + warp_id) * last_dim_size + x;
A acc = upcast(ptrs[warp_id][idx]);
packed_assign_add<opus::fp32_t, pack_size>(acc, upcast(bp[write_idx]));
P out = downcast<P>(acc);
if constexpr(kHasC)
{
A acc2 = upcast(out);
packed_assign_add<opus::fp32_t, pack_size>(acc2, upcast(cp[write_idx]));
out = downcast<P>(acc2);
}
*(reinterpret_cast<P*>(&result[0]) + write_idx) = out;
}
end_sync<ngpus, true>(sg, self_sg, rank);
}

template <typename T, int ngpus>
__global__ void __launch_bounds__(512, 1) allgather_lastdim(RankData* _dp,
RankSignals sg,
Expand Down Expand Up @@ -5043,6 +5097,43 @@ void dispatchAllGatherV(hipStream_t stream,
}
}

template <typename T>
void dispatchAllGatherLastDimAdd(hipStream_t stream,
T* input,
T* output,
int size,
int last_dim_size,
const T* add_b,
const T* add_c)
{
RankData* ptrs = get_buffer_RD(stream, input);
auto d = 16 / sizeof(T);
if(size % d != 0 || last_dim_size % d != 0)
throw std::runtime_error("allgather_lastdim_add needs 16-byte aligned rows");
size /= d;
dim3 block(512);
int tnum_per_block = 512 / world_size_;
int block_num = (size + tnum_per_block - 1) / tnum_per_block;
dim3 grid(std::min(block_num, 80));
#define AG_ADD_CASE(ngpus) \
case ngpus: \
if(add_c != nullptr) \
allgather_lastdim_add<T, ngpus, true><<<grid, block, 0, stream>>>( \
ptrs, sg_, self_sg_, output, rank_, size, last_dim_size, add_b, add_c); \
else \
allgather_lastdim_add<T, ngpus, false><<<grid, block, 0, stream>>>( \
ptrs, sg_, self_sg_, output, rank_, size, last_dim_size, add_b, add_c); \
break;
switch(world_size_)
{
AG_ADD_CASE(8)
AG_ADD_CASE(4)
AG_ADD_CASE(2)
default: throw std::runtime_error("allgather_lastdim_add world_size error");
}
#undef AG_ADD_CASE
}

template <typename T>
void dispatchFusedAllReduceRMSNorm(hipStream_t stream,
T* input,
Expand Down
8 changes: 8 additions & 0 deletions csrc/include/custom_all_reduce.h
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
* limitations under the License.
*/
#include <cstdint>
#include <optional>
#include <vector>
#include "aiter_tensor.h"

Expand Down Expand Up @@ -58,6 +59,13 @@ void all_gather_reg(fptr_t _fa,
const aiter_tensor_t& inp,
const aiter_tensor_t& out,
int64_t dim);
void all_gather_lastdim_add(fptr_t _fa,
const aiter_tensor_t& inp,
const aiter_tensor_t& out,
const aiter_tensor_t& add_b,
std::optional<aiter_tensor_t> add_c,
int64_t reg_ptr,
int64_t reg_bytes);
void all_gather_unreg(fptr_t _fa,
const aiter_tensor_t& inp,
int64_t reg_buffer,
Expand Down
9 changes: 9 additions & 0 deletions csrc/include/rocm_ops.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -574,6 +574,15 @@ namespace py = pybind11;
py::arg("inp"), \
py::arg("out"), \
py::arg("dim")); \
m.def("all_gather_lastdim_add", \
&aiter::all_gather_lastdim_add, \
py::arg("_fa"), \
py::arg("inp"), \
py::arg("out"), \
py::arg("add_b"), \
py::arg("add_c") = std::nullopt, \
py::arg("reg_ptr") = 0, \
py::arg("reg_bytes") = 0); \
m.def("all_gather_unreg", \
&aiter::all_gather_unreg, \
py::arg("_fa"), \
Expand Down
39 changes: 39 additions & 0 deletions csrc/kernels/custom_all_reduce.cu
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <optional>
#include "custom_all_reduce.cuh"
#include "aiter_stream.h"
#include "aiter_tensor.h"
Expand Down Expand Up @@ -497,6 +498,44 @@ void all_gather_reg(fptr_t _fa,
last_dim_size, dim);
}

void all_gather_lastdim_add(fptr_t _fa,
const aiter_tensor_t& inp,
const aiter_tensor_t& out,
const aiter_tensor_t& add_b,
std::optional<aiter_tensor_t> add_c,
int64_t reg_ptr,
int64_t reg_bytes)
{
HipDeviceGuard device_guard(inp.device_id);
hipStream_t stream = aiter::getCurrentHIPStream();
auto fa = reinterpret_cast<aiter::CustomAllreduce*>(_fa);
int64_t last_dim_size = inp.size(-1);
if(inp.dtype() != AITER_DTYPE_bf16)
throw std::runtime_error("all_gather_lastdim_add supports bfloat16 only");
if(inp.dim() != 2 || out.dim() != 2 || add_b.dim() != 2 ||
(add_c.has_value() && add_c->dim() != 2))
throw std::runtime_error("all_gather_lastdim_add supports 2D tensors only");
void* src = inp.data_ptr();
if(reg_ptr != 0)
{
// unregistered input: stage it into the registered pool first
int64_t data_bytes = inp.numel() * inp.element_size();
if(data_bytes > reg_bytes)
throw std::runtime_error("registered buffer is too small to contain the input");
HIP_CALL(hipMemcpyAsync(
(void*)reg_ptr, inp.data_ptr(), data_bytes, hipMemcpyDeviceToDevice, stream));
src = (void*)reg_ptr;
}
fa->dispatchAllGatherLastDimAdd<opus::bf16_t>(
stream,
reinterpret_cast<opus::bf16_t*>(src),
reinterpret_cast<opus::bf16_t*>(out.data_ptr()),
inp.numel(),
last_dim_size,
reinterpret_cast<const opus::bf16_t*>(add_b.data_ptr()),
add_c.has_value() ? reinterpret_cast<const opus::bf16_t*>(add_c->data_ptr()) : nullptr);
}

void all_gather_unreg(fptr_t _fa,
const aiter_tensor_t& inp,
int64_t reg_buffer,
Expand Down
Loading
Loading