Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 26 additions & 19 deletions vllm/model_executor/layers/attention/mla_attention.py
Original file line number Diff line number Diff line change
Expand Up @@ -491,29 +491,36 @@ def __init__(
compilation_config.static_forward_context[prefix] = self

self.prefill_backend: MLAPrefillBackend | None
try:
prefill_backend_cls = get_mla_prefill_backend(vllm_config)
except ValueError:
if (
not self.impl.is_sparse
or vllm_config.attention_config.mla_prefill_backend is not None
):
raise
if self.impl.is_sparse and not self.impl.supports_dense_mha_prefill:
logger.warning_once(
"No MLA prefill backend supports this model; sparse MLA will use the "
"top-k MQA path only (no dense-MHA prefill)."
"Sparse MLA impl has no dense-MHA prefill path; using the top-k "
"MQA path only."
)
self.prefill_backend = None
else:
self.prefill_backend = prefill_backend_cls(
num_heads=self.num_heads,
scale=self.scale,
kv_lora_rank=self.kv_lora_rank,
qk_nope_head_dim=self.qk_nope_head_dim,
qk_rope_head_dim=self.qk_rope_head_dim,
v_head_dim=self.v_head_dim,
vllm_config=vllm_config,
)
try:
prefill_backend_cls = get_mla_prefill_backend(vllm_config)
except ValueError:
if (
not self.impl.is_sparse
or vllm_config.attention_config.mla_prefill_backend is not None
):
raise
logger.warning_once(
"No MLA prefill backend supports this model; sparse MLA will "
"use the top-k MQA path only (no dense-MHA prefill)."
)
self.prefill_backend = None
else:
self.prefill_backend = prefill_backend_cls(
num_heads=self.num_heads,
scale=self.scale,
kv_lora_rank=self.kv_lora_rank,
qk_nope_head_dim=self.qk_nope_head_dim,
qk_rope_head_dim=self.qk_rope_head_dim,
v_head_dim=self.v_head_dim,
vllm_config=vllm_config,
)

self.kv_cache = torch.tensor([])

Expand Down
9 changes: 7 additions & 2 deletions vllm/model_executor/models/deepseek_v2.py
Original file line number Diff line number Diff line change
Expand Up @@ -848,11 +848,16 @@ def _try_load_fp8_indexer_wk(
# We have both weight and scale: dequantize FP8 to BF16.
weight_fp8, scale_inv = entry["weight"], entry["scale"]
del buf[layer_prefix]
block_size = weight_fp8.shape[1] // scale_inv.shape[1]
if scale_inv.ndim == 1:
# Per-channel scale: one scale per row of [out, in]
group_shape = GroupShape(1, weight_fp8.shape[1])
else:
block_size = weight_fp8.shape[1] // scale_inv.shape[1]
group_shape = GroupShape(block_size, block_size)
weight_bf16 = scaled_dequantize(
weight_fp8,
scale_inv,
group_shape=GroupShape(block_size, block_size),
group_shape=group_shape,
out_dtype=torch.bfloat16,
)

Expand Down
4 changes: 4 additions & 0 deletions vllm/v1/attention/backend.py
Original file line number Diff line number Diff line change
Expand Up @@ -785,6 +785,10 @@ class AttentionImplBase(ABC, Generic[T]):
# route between the dense-MHA prefill and sparse-MQA paths.
is_sparse: ClassVar[bool] = False

# Whether this impl provides a dense-MHA prefill path (forward_mha). Sparse
# impls without one run the top-k MQA path for all requests.
supports_dense_mha_prefill: ClassVar[bool] = True

# Required attributes that all impls should have
num_heads: int
head_size: int
Expand Down
17 changes: 17 additions & 0 deletions vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
from vllm.v1.attention.backends.mla.rocm_aiter_mla import (
AiterMLAHelper,
)
from vllm.v1.attention.backends.utils import split_decodes_and_prefills
from vllm.v1.kv_cache_interface import AttentionSpec
from vllm.v1.worker.workspace import current_workspace_manager

Expand Down Expand Up @@ -333,6 +334,14 @@ class ROCMAiterMLASparseMetadata(AttentionMetadata):
block_size: int = 1
topk_tokens: int = 2048

# Fields read by the shared MLA forward. This impl has no dense-MHA prefill
# path (supports_dense_mha_prefill=False), so it always runs the MQA path;
num_decodes: int = 0
num_prefills: int = 0
num_decode_tokens: int = 0
prefill_max_seq_len: int = 0
prefill: object = None

# Persistent MLA metadata (only populated when persistent mode is enabled,
# i.e. when the aiter sparse decode kernel supports work-stealing splits).
work_meta_data: torch.Tensor | None = None
Expand Down Expand Up @@ -471,6 +480,10 @@ def build(
fast_build: bool = False,
) -> ROCMAiterMLASparseMetadata:
num_tokens = common_attn_metadata.num_actual_tokens
(num_decodes, num_prefills, num_decode_tokens, _) = split_decodes_and_prefills(
common_attn_metadata,
decode_threshold=self.reorder_batch_threshold or 1,
)
starts = np.asarray(common_attn_metadata.query_start_loc_cpu, dtype=np.int32)
seg_lengths = np.diff(starts)
req_id_per_token = np.repeat(
Expand Down Expand Up @@ -583,6 +596,9 @@ def build(
block_size=self.kv_cache_spec.block_size,
attn_out_dtype=self.model_dtype,
topk_tokens=self.topk_tokens,
num_decodes=num_decodes,
num_prefills=num_prefills,
num_decode_tokens=num_decode_tokens,
qo_indptr=qo_indptr,
paged_kv_last_page_len=paged_kv_last_page_len,
paged_kv_indices=paged_kv_indices,
Expand Down Expand Up @@ -628,6 +644,7 @@ def log2sumexp2(a: torch.Tensor, dim: int) -> torch.Tensor:

class ROCMAiterMLASparseImpl(MLAAttentionImpl[ROCMAiterMLASparseMetadata]):
is_sparse = True
supports_dense_mha_prefill = False

def __init__(
self,
Expand Down
Loading