Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
80 changes: 80 additions & 0 deletions tests/models/kimi_k3/test_amd_kda_group64.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

import torch
from torch import nn

import vllm.models.kimi_k3.amd.kda as amd_kda
from vllm.model_executor.layers.mamba.gdn.kimi_gdn_linear_attn import (
KimiGatedDeltaNetAttention as CommonKimiGatedDeltaNetAttention,
)
from vllm.models.kimi_k3.amd.kda import KimiGatedDeltaNetAttention
from vllm.models.kimi_k3.amd.ops.kda_input_projection import (
kda_input_projection,
)


class _Projection(nn.Module):
def forward(self, hidden_states: torch.Tensor):
return hidden_states + 1, None


def _make_amd_layer() -> KimiGatedDeltaNetAttention:
layer = object.__new__(KimiGatedDeltaNetAttention)
nn.Module.__init__(layer)
layer.in_proj_qkvgfab = _Projection()
layer.register_buffer("_kda_group64_weight", None, persistent=False)
layer.register_buffer("_kda_group64_scale", None, persistent=False)
return layer


def test_default_projection_seam_preserves_exact_result() -> None:
layer = object.__new__(CommonKimiGatedDeltaNetAttention)
nn.Module.__init__(layer)
layer.in_proj_qkvgfab = _Projection()
hidden_states = torch.arange(8, dtype=torch.float32).reshape(1, 8)

actual = CommonKimiGatedDeltaNetAttention._project_qkvgfab(layer, hidden_states)

assert torch.equal(actual, layer.in_proj_qkvgfab(hidden_states)[0])


def test_group64_dispatch_falls_back_without_prepacked_weight() -> None:
hidden_states = torch.arange(8, dtype=torch.float32).reshape(1, 8)
fallback_calls = 0

def fallback(value: torch.Tensor) -> torch.Tensor:
nonlocal fallback_calls
fallback_calls += 1
return value + 1

actual = kda_input_projection(hidden_states, fallback, None, None)

assert fallback_calls == 1
assert torch.equal(actual, hidden_states + 1)


def test_amd_projection_seam_preserves_fallback_result() -> None:
layer = _make_amd_layer()
hidden_states = torch.arange(8, dtype=torch.float32).reshape(1, 8)

actual = layer._project_qkvgfab(hidden_states)

assert torch.equal(actual, hidden_states + 1)


def test_post_load_hook_installs_prepacked_buffers(monkeypatch) -> None:
layer = _make_amd_layer()
layer.in_proj_qkvgfab = nn.Linear(8, 8, bias=False)
packed_weight = torch.empty(2, 2)
packed_scale = torch.empty(2)
monkeypatch.setattr(
amd_kda,
"prepack_kda_input_group64",
lambda weight: (packed_weight, packed_scale),
)

layer.process_weights_after_loading(torch.bfloat16)

assert layer._kda_group64_weight is packed_weight
assert layer._kda_group64_scale is packed_scale
Original file line number Diff line number Diff line change
Expand Up @@ -324,14 +324,18 @@ def rearrange_mixed_qkv(
qkv = qkv.permute(1, 0, 2, 3).contiguous().unsqueeze(1)
return qkv.unbind(0)

def _project_qkvgfab(self, hidden_states: torch.Tensor) -> torch.Tensor:
"""Projection seam for platform-specific subclasses."""
return self.in_proj_qkvgfab(hidden_states)[0]

def forward(
self,
hidden_states: torch.Tensor,
positions: torch.Tensor,
output: torch.Tensor,
) -> None:
num_tokens = hidden_states.size(0)
projected_qkvgfab = self.in_proj_qkvgfab(hidden_states)[0]
projected_qkvgfab = self._project_qkvgfab(hidden_states)
if self.use_full_rank_gate:
split_sizes = [
3 * self.local_projection_size,
Expand Down
51 changes: 51 additions & 0 deletions vllm/models/kimi_k3/amd/kda.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

"""AMD Kimi-K3 KDA integration."""

import torch

from vllm.config import VllmConfig
from vllm.model_executor.layers.mamba.gdn.kimi_gdn_linear_attn import (
KimiGatedDeltaNetAttention as _KimiGatedDeltaNetAttention,
)
from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig

from .ops.kda_input_projection import (
kda_input_projection,
prepack_kda_input_group64,
)


class KimiGatedDeltaNetAttention(_KimiGatedDeltaNetAttention):
"""Kimi GDN layer with the gfx950 group64 input projection."""

def __init__(
self,
config: KimiLinearConfig,
vllm_config: VllmConfig,
prefix: str = "",
) -> None:
super().__init__(config, vllm_config, prefix)
self.register_buffer("_kda_group64_weight", None, persistent=False)
self.register_buffer("_kda_group64_scale", None, persistent=False)

def process_weights_after_loading(self, act_dtype: torch.dtype) -> None:
del act_dtype
weight = getattr(self.in_proj_qkvgfab, "weight", None)
if not isinstance(weight, torch.Tensor):
return
packed = prepack_kda_input_group64(weight)
if packed is not None:
self._kda_group64_weight, self._kda_group64_scale = packed

def _project_qkvgfab(self, hidden_states: torch.Tensor) -> torch.Tensor:
return kda_input_projection(
hidden_states,
super()._project_qkvgfab,
self._kda_group64_weight,
self._kda_group64_scale,
)


__all__ = ["KimiGatedDeltaNetAttention"]
4 changes: 1 addition & 3 deletions vllm/models/kimi_k3/amd/linear.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,6 @@
RowParallelLinear,
)
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.mamba.gdn.kimi_gdn_linear_attn import (
KimiGatedDeltaNetAttention,
)
from vllm.model_executor.layers.mamba.mamba_utils import (
MambaStateCopyFunc,
MambaStateCopyFuncCalculator,
Expand Down Expand Up @@ -61,6 +58,7 @@
make_layers,
maybe_prefix,
)
from vllm.models.kimi_k3.amd.kda import KimiGatedDeltaNetAttention
from vllm.models.kimi_k3.amd.ops.attn_res import attn_res
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig
Expand Down
98 changes: 98 additions & 0 deletions vllm/models/kimi_k3/amd/ops/kda_input_projection.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

"""Fail-closed AMD-only Kimi-K3 KDA input-projection dispatch."""

from __future__ import annotations

from collections.abc import Callable

import torch

from vllm.platforms import current_platform
from vllm.utils.torch_utils import direct_register_custom_op


def _kda_input_group64_impl(
hidden_states: torch.Tensor,
packed_weight: torch.Tensor,
packed_scale: torch.Tensor,
) -> torch.Tensor:
from aiter.ops.flydsl.kimi_k3_kda_input_group64 import (
kimi_k3_kda_input_group64,
)

return kimi_k3_kda_input_group64(hidden_states, packed_weight, packed_scale)


def _kda_input_group64_fake(
hidden_states: torch.Tensor,
packed_weight: torch.Tensor,
packed_scale: torch.Tensor,
) -> torch.Tensor:
del packed_weight, packed_scale
return hidden_states.new_empty((hidden_states.shape[0], 6288))


direct_register_custom_op(
op_name="kimi_k3_kda_input_group64",
op_func=_kda_input_group64_impl,
mutates_args=[],
fake_impl=_kda_input_group64_fake,
dispatch_key=current_platform.dispatch_key,
)


def prepack_kda_input_group64(
bf16_weight: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor] | None:
"""Return a model-load-time pack, or ``None`` for exact fallback."""

try:
from aiter.ops.flydsl.kimi_k3_kda_input_group64 import (
quantize_kimi_k3_kda_input_group64,
)
except (ImportError, ModuleNotFoundError):
return None
if (
not torch.version.hip
or bf16_weight.dtype != torch.bfloat16
or tuple(bf16_weight.shape) != (6288, 7168)
or not bf16_weight.is_cuda
or not bf16_weight.is_contiguous()
):
return None
try:
properties = torch.cuda.get_device_properties(bf16_weight.device)
except (AssertionError, RuntimeError):
return None
if str(getattr(properties, "gcnArchName", "")).split(":", 1)[0] != "gfx950":
return None
return quantize_kimi_k3_kda_input_group64(bf16_weight)


def kda_input_projection(
hidden_states: torch.Tensor,
bf16_fallback: Callable[[torch.Tensor], torch.Tensor],
packed_weight: torch.Tensor | None,
packed_scale: torch.Tensor | None,
) -> torch.Tensor:
"""Use group64 only for the exact typed contract; otherwise BF16 fallback."""

if packed_weight is None or packed_scale is None:
return bf16_fallback(hidden_states)
try:
from aiter.ops.flydsl.kimi_k3_kda_input_group64 import (
supports_kimi_k3_kda_input_group64,
)
except (ImportError, ModuleNotFoundError):
return bf16_fallback(hidden_states)
if not supports_kimi_k3_kda_input_group64(
hidden_states, packed_weight, packed_scale
):
return bf16_fallback(hidden_states)
return torch.ops.vllm.kimi_k3_kda_input_group64(
hidden_states,
packed_weight,
packed_scale,
)
Loading