Skip to content
Merged
Show file tree
Hide file tree
Changes from 8 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 48 additions & 0 deletions tests/kernels/test_fused_inv_rope_fp8_quant.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,12 @@
import pytest
import torch

from vllm.model_executor.layers.quantization.utils.fp8_utils import (
deepgemm_post_process_fp8_weight_block,
)
from vllm.models.deepseek_v4.common.ops import fused_inv_rope_fp8_quant
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import fp8_einsum

# -- Default dimensions matching DeepSeek V3/V4 --------------------------
HEAD_DIM = 512
Expand All @@ -33,6 +38,49 @@
EPS = 1e-10


@pytest.mark.skipif(
not current_platform.is_device_capability_family(100), reason="Requires SM100/SM103"
)
@pytest.mark.parametrize("groups", [2, 4])
@pytest.mark.parametrize("tokens", [1, 6, 18, 32])
def test_dsv41_fused_wo_a_matches_deepgemm(groups, tokens):
"""Fused WO-A matches inverse RoPE + fp8_einsum + MXFP8 quantize."""
from flashinfer import mxfp8_quantize

from vllm.models.deepseek_v41.nvidia.ops.fused_wo_a import _FUSED_WO_A_KERNEL

torch.manual_seed(0)
n = groups * 1024
x = torch.randn(tokens, 64, 512, device="cuda", dtype=torch.bfloat16)[:, : n // 128]
positions = torch.randint(0, 128, (tokens,), device="cuda")
rope = make_cos_sin_cache(128, device="cuda")
wq = torch.randn(n, 4096, device="cuda").to(FP8_DTYPE)
scale = torch.exp2(torch.randint(-8, 0, (n, 128), device="cuda").float())
wq, ws = deepgemm_post_process_fp8_weight_block(
wq, scale, (1, 32), False, True, groups
)
q, sf = _FUSED_WO_A_KERNEL(
x=x, positions=positions, rope=rope, weight=wq, weight_scale=ws
)

a, sa = fused_inv_rope_fp8_quant(
x, positions, rope, groups, 8, quant_group_size=32, tma_aligned_scales=True
)
y = torch.empty(tokens, groups, 1024, device="cuda", dtype=torch.bfloat16)
fp8_einsum("bhr,hdr->bhd", (a, sa), (wq, ws), y, recipe=(1, 1, 32))
ref, ref_sf = mxfp8_quantize(y.flatten(1), backend="cute-dsl")

def rows(scales): # F8_128x4 bytes -> [128 rows, n / 32 blocks]
return scales.view(n // 128, 32, 4, 4).permute(2, 1, 0, 3).reshape(128, -1)

actual, expected = (
d.float().view(tokens, -1, 32) * torch.exp2(rows(s)[:tokens, :, None] - 127.0)
for d, s in ((q, sf), (ref, ref_sf))
)
assert (actual - expected).norm() / expected.norm() < 3e-3
assert not rows(sf)[tokens:].any(), "padded scale rows must be zero"


# =========================================================================
# Helpers
# =========================================================================
Expand Down
4 changes: 2 additions & 2 deletions vllm/models/deepseek_v41/attention.py
Original file line number Diff line number Diff line change
Expand Up @@ -697,7 +697,7 @@ def bind_gemm_rs(self) -> None:
else:
gemm_rs.warn_incompatible_projection()

def _wo_b_proj(self, z: torch.Tensor) -> torch.Tensor:
def _wo_b_proj(self, z: torch.Tensor | QuantizedActivation) -> torch.Tensor:
"""Apply ``wo_b``; with GEMM-RS bound, also reduce-scatter the result.

Every ``_o_proj`` implementation projects through this so the decoder
Expand All @@ -707,7 +707,7 @@ def _wo_b_proj(self, z: torch.Tensor) -> torch.Tensor:
"""
if self.gemm_rs is None:
return self.wo_b(z)
if self.gemm_rs.should_run(z):
if isinstance(z, torch.Tensor) and self.gemm_rs.should_run(z):
return self.gemm_rs.apply(z, self.wo_b)
# Small batches stay on the unfused path, which is faster there.
return sp_reduce_scatter(self.wo_b(z))
Expand Down
42 changes: 8 additions & 34 deletions vllm/models/deepseek_v41/nvidia/flashinfer_sparse.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,15 +9,16 @@
from vllm.config import VllmConfig
from vllm.config.cache import CacheDType
from vllm.forward_context import get_forward_context
from vllm.models.deepseek_v4.nvidia.ops.o_proj import (
compute_fp8_einsum_recipe,
deep_gemm_fp8_o_proj,
)
from vllm.models.deepseek_v4.nvidia.ops.o_proj import compute_fp8_einsum_recipe
from vllm.models.deepseek_v41.attention import DeepseekV4Attention
from vllm.models.deepseek_v41.common.ops import (
build_flashinfer_mixed_sparse_indices,
compute_global_topk_indices_and_lens,
)
from vllm.models.deepseek_v41.nvidia.ops.o_proj import (
dsv41_o_proj,
register_dsv41_o_proj_warmup,
)
from vllm.models.deepseek_v41.sparse_mla import (
DeepseekV4FlashMLAMetadata,
DeepseekV4SparseMLABackend,
Expand Down Expand Up @@ -242,27 +243,14 @@ def get_padded_num_q_heads(cls, num_heads: int) -> int:
return _pad_to_supported_q_heads(num_heads)

def _o_proj(self, attn_out: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
o = attn_out[:, : self.n_local_heads, :]
return deep_gemm_fp8_o_proj(
o,
positions,
self.rotary_emb.cos_sin_cache,
self.wo_a,
self._wo_b_proj,
n_groups=self.n_local_groups,
heads_per_group=self.n_local_heads // self.n_local_groups,
nope_dim=self.nope_head_dim,
rope_dim=self.rope_head_dim,
o_lora_rank=self.o_lora_rank,
einsum_recipe=self._einsum_recipe,
tma_aligned_scales=self._tma_aligned_scales,
)
return dsv41_o_proj(self, attn_out, positions)

def __init__(self, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
self._einsum_recipe, self._tma_aligned_scales = compute_fp8_einsum_recipe(
self._o_proj_block_size
)
register_dsv41_o_proj_warmup(self)
# Per-tensor FP8 scale buffers + precomputed scalar BMM scales. Only the
# per-tensor FP8 cache path consumes these; bf16 reads ``self.scale``.
if self.kv_cache_torch_dtype != torch.float8_e4m3fn:
Expand Down Expand Up @@ -623,21 +611,7 @@ def get_padded_num_q_heads(cls, num_heads: int) -> int:
return _pad_to_supported_q_heads(num_heads)

def _o_proj(self, attn_out: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
o = attn_out[:, : self.n_local_heads, :]
return deep_gemm_fp8_o_proj(
o,
positions,
self.rotary_emb.cos_sin_cache,
self.wo_a,
self._wo_b_proj,
n_groups=self.n_local_groups,
heads_per_group=self.n_local_heads // self.n_local_groups,
nope_dim=self.nope_head_dim,
rope_dim=self.rope_head_dim,
o_lora_rank=self.o_lora_rank,
einsum_recipe=self._einsum_recipe,
tma_aligned_scales=self._tma_aligned_scales,
)
return dsv41_o_proj(self, attn_out, positions)

def __init__(self, vllm_config: VllmConfig, *args, **kwargs) -> None:
super().__init__(vllm_config, *args, **kwargs)
Expand Down
26 changes: 7 additions & 19 deletions vllm/models/deepseek_v41/nvidia/flashmla.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,16 +6,17 @@
import torch

from vllm.forward_context import get_forward_context
from vllm.models.deepseek_v4.nvidia.ops.o_proj import (
compute_fp8_einsum_recipe,
deep_gemm_fp8_o_proj,
)
from vllm.models.deepseek_v4.nvidia.ops.o_proj import compute_fp8_einsum_recipe
from vllm.models.deepseek_v41.attention import DeepseekV4Attention
from vllm.models.deepseek_v41.common.ops import (
combine_topk_swa_indices,
compute_global_topk_indices_and_lens,
dequantize_and_gather_k_cache,
)
from vllm.models.deepseek_v41.nvidia.ops.o_proj import (
dsv41_o_proj,
register_dsv41_o_proj_warmup,
)
from vllm.models.deepseek_v41.sparse_mla import (
DeepseekV4FlashMLABackend,
DeepseekV4FlashMLAMetadata,
Expand Down Expand Up @@ -57,23 +58,10 @@ def __init__(self, *args, **kwargs) -> None:
self._einsum_recipe, self._tma_aligned_scales = compute_fp8_einsum_recipe(
self._o_proj_block_size
)
register_dsv41_o_proj_warmup(self)

def _o_proj(self, attn_out: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
o = attn_out[:, : self.n_local_heads, :]
return deep_gemm_fp8_o_proj(
o,
positions,
self.rotary_emb.cos_sin_cache,
self.wo_a,
self._wo_b_proj,
n_groups=self.n_local_groups,
heads_per_group=self.n_local_heads // self.n_local_groups,
nope_dim=self.nope_head_dim,
rope_dim=self.rope_head_dim,
o_lora_rank=self.o_lora_rank,
einsum_recipe=self._einsum_recipe,
tma_aligned_scales=self._tma_aligned_scales,
)
return dsv41_o_proj(self, attn_out, positions)

@classmethod
def get_padded_num_q_heads(cls, num_heads: int) -> int:
Expand Down
Loading
Loading