Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions aiter/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,7 @@ def getLogger():
)
from .ops.fused_qk_norm_rope_cache_quant import *
from .ops.fused_qk_rmsnorm_group_quant import *
from .ops.inverse_rope_group_quant import *
from .ops.groupnorm import *
from .ops.mhc import *
from .ops.causal_conv1d_update import *
Expand Down
1 change: 1 addition & 0 deletions aiter/jit/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -1075,6 +1075,7 @@ def _get_ck_exclude_modules():
"module_cache",
"module_fused_qk_norm_mrope_cache_quant_shuffle",
"module_fused_qk_norm_rope_cache_quant_shuffle",
"module_inverse_rope_group_quant",
"module_mla_metadata",
"module_mla_reduce",
"module_moe_asm",
Expand Down
16 changes: 16 additions & 0 deletions aiter/jit/optCompilerConfig.json
Original file line number Diff line number Diff line change
Expand Up @@ -1210,6 +1210,22 @@
"verbose": "False",
"blob_gen_cmd": "''"
},
"module_inverse_rope_group_quant": {
"srcs": [
"f'{AITER_CSRC_DIR}/pybind/inverse_rope_group_quant_pybind.cu'",
"f'{AITER_CSRC_DIR}/kernels/inverse_rope_group_quant.cu'"
],
"flags_extra_cc": [],
"flags_extra_hip": [
"'-DENABLE_FP8'"
],
"extra_ldflags": "None",
"extra_include": [
"f'{AITER_CSRC_DIR}/include/opus'"
],
"verbose": "False",
"blob_gen_cmd": "''"
},
"module_fmha_fwd_with_sink_asm": {
"srcs": [
"f'{AITER_CSRC_DIR}/py_itfs_cu/asm_fmha_fwd_with_sink.cu'"
Expand Down
113 changes: 113 additions & 0 deletions aiter/ops/inverse_rope_group_quant.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
# SPDX-License-Identifier: MIT
# Copyright (C) 2024-2026, Advanced Micro Devices, Inc. All rights reserved.

"""DeepSeek-V4 attention-output inverse RoPE fused with FP8 group quant.

Its own JIT module: this sits on the attention *output* path and shares no
kernel with the QK-norm/RoPE input-path ops, so folding it into their 6k-line
translation unit would only make every edit there rebuild it.
"""

import torch
from torch import Tensor

from ..jit.core import compile_ops
from ..utility.dtypes import get_dtype_fp8


@compile_ops(
"module_inverse_rope_group_quant",
fc_name="inverse_rope_group_quant",
develop=True,
)
def _inverse_rope_group_quant_kernel(
o: Tensor,
x_fp8: Tensor,
x_scale: Tensor,
positions: Tensor,
cos_cache: Tensor,
sin_cache: Tensor,
num_groups: int,
quant_group_size: int = 128,
scale_shuffle: bool = False,
) -> None: ...


def inverse_rope_group_quant(
o: Tensor,
positions: Tensor,
cos_cache: Tensor,
sin_cache: Tensor,
num_groups: int,
quant_group_size: int = 128,
scale_shuffle: bool = False,
x_fp8: Tensor | None = None,
x_scale: Tensor | None = None,
) -> tuple[Tensor, Tensor]:
"""Inverse-RoPE V4 attention output and emit FP8 group-quant rows.

Args:
o: ``[S, H, head_dim]`` bf16/fp16 attention output before inverse RoPE.
positions: ``[S]`` absolute positions.
cos_cache/sin_cache: ``[max_pos, rd//2]``.
num_groups: output-LoRA local groups ``G``.
quant_group_size: quant block along ``D``; V4 wo_a path uses 128.
scale_shuffle: when True, emit scale in MFMA tile-shuffled layout
``[G, S_pad, Ks_pad]`` for ``V_MFMA_SCALE_F32_16x16x128_F8``.
When False, emit row-major ``[S, G, Ks]``.

Returns:
``(x_fp8, x_scale)``.
"""
assert o.dim() == 3, f"o must be [S,H,Dh], got {tuple(o.shape)}"
S, H, head_dim = o.shape
assert (
H * head_dim
) % num_groups == 0, (
f"H*head_dim={H * head_dim} must be divisible by num_groups={num_groups}"
)
D = (H * head_dim) // num_groups
assert (
D % quant_group_size == 0
), f"per-group D={D} must be divisible by quant_group_size={quant_group_size}"

for name, cache in (("cos_cache", cos_cache), ("sin_cache", sin_cache)):
assert cache.dim() == 2, (
f"{name} must be 2D [max_pos, rd//2], got {tuple(cache.shape)}; "
"reshape a cache carrying singleton batch/head dims at the call site"
)

from .. import dtypes

if x_fp8 is None:
x_fp8 = torch.empty((S, num_groups, D), dtype=get_dtype_fp8(), device=o.device)
scale_groups = D // quant_group_size
if x_scale is None:
if scale_shuffle:
S_pad = ((S + 31) // 32) * 32
Ks_pad = ((scale_groups + 7) // 8) * 8
x_scale = torch.full(
(num_groups, S_pad, Ks_pad),
0x7F,
dtype=dtypes.fp8_e8m0,
device=o.device,
)
else:
x_scale = torch.empty(
(S, num_groups, scale_groups),
dtype=dtypes.fp8_e8m0,
device=o.device,
)

_inverse_rope_group_quant_kernel(
o,
x_fp8,
x_scale,
positions,
cos_cache,
sin_cache,
num_groups,
quant_group_size,
scale_shuffle,
)
return x_fp8, x_scale
34 changes: 34 additions & 0 deletions csrc/include/inverse_rope_group_quant.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
// SPDX-License-Identifier: MIT
// Copyright (C) 2024-2026, Advanced Micro Devices, Inc. All rights reserved.
#pragma once

#include "aiter_tensor.h"
#include <cstdint>

namespace aiter {

// DeepSeek-V4 output path helper:
// input o [S, H, head_dim] bf16/fp16, before inverse RoPE
// output x_fp8 [S, G, D] fp8, where D = H*head_dim/G
// output x_scale e8m0 scale bytes
// Applies GPT-J inverse RoPE to every head's rope tail, then group-quantizes the
// flattened per-group rows for the upcoming wo_a grouped FP8 BMM.
//
// scale_shuffle:
// false = row-major [S, G, Ks], Ks = D / quant_group_size
// true = MFMA tile-shuffled for V_MFMA_SCALE_F32_16x16x128_F8 (gfx950)
// Storage [G, S_pad, Ks_pad] flat with 256-byte tiles of [32_M, 8_K].
// Tile-internal: byte = lane*4 + iter, lane = (k%4)*16 + (m%16),
// iter = ((m/16)&1) + ((k/4)&1)*2. S_pad = ceil(S,32), Ks_pad = ceil(Ks,8).
void inverse_rope_group_quant(
aiter_tensor_t& o,
aiter_tensor_t& x_fp8,
aiter_tensor_t& x_scale,
aiter_tensor_t& positions,
aiter_tensor_t& cos_cache,
aiter_tensor_t& sin_cache,
int64_t num_groups,
int64_t quant_group_size = 128,
bool scale_shuffle = false);

} // namespace aiter
13 changes: 13 additions & 0 deletions csrc/include/rocm_ops.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -2027,6 +2027,19 @@ namespace py = pybind11;
py::arg("v_fp8"), \
py::arg("v_descale"));

#define INVERSE_ROPE_GROUP_QUANT_PYBIND \
m.def("inverse_rope_group_quant", \
&aiter::inverse_rope_group_quant, \
py::arg("o"), \
py::arg("x_fp8"), \
py::arg("x_scale"), \
py::arg("positions"), \
py::arg("cos_cache"), \
py::arg("sin_cache"), \
py::arg("num_groups"), \
py::arg("quant_group_size") = 128, \
py::arg("scale_shuffle") = false);

#define SMOOTHQUANT_PYBIND \
m.def("smoothquant_fwd", &smoothquant_fwd); \
m.def("moe_smoothquant_fwd", &moe_smoothquant_fwd);
Expand Down
Loading
Loading