Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 69 additions & 0 deletions tests/v1/attention/test_mla_backends.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,17 @@
MLAAttention,
QueryLenSupport,
_DecodeConcatQuantFP8,
_is_mla_bmm_requantization_allowed,
build_mla_chunked_context_metadata,
)
from vllm.model_executor.layers.linear import (
LinearBase,
UnquantizedLinearMethod,
)
from vllm.model_executor.layers.quantization.quark.quark import (
QuarkConfig,
QuarkLinearMethod,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
from vllm.platforms import current_platform
from vllm.utils.math_utils import cdiv
Expand Down Expand Up @@ -94,6 +103,66 @@ def test_mla_kv_cache_spec_uses_layer_cache_dtype(
assert spec.page_size_bytes == 64 * 656


@pytest.mark.parametrize(
("quant_config", "quant_method", "expected"),
[
pytest.param(None, UnquantizedLinearMethod(), True, id="fully-bf16"),
pytest.param(
object(),
UnquantizedLinearMethod(),
False,
id="explicitly-unquantized",
),
pytest.param(object(), object(), True, id="quantized"),
],
)
def test_mla_bmm_requantization_respects_linear_precision_provenance(
quant_config, quant_method, expected
):
kv_b_proj = SimpleNamespace(
quant_config=quant_config,
quant_method=quant_method,
)

assert _is_mla_bmm_requantization_allowed(kv_b_proj) is expected


@pytest.mark.parametrize(
"prefix",
[
"model.layers.3.self_attn.kv_b_proj",
"model.layers.78.self_attn.kv_b_proj",
],
)
def test_quark_excluded_mla_projection_denies_bmm_requantization(prefix):
quant_config = QuarkConfig({"exclude": [prefix]})
linear = LinearBase.__new__(LinearBase)
quant_method = quant_config.get_quant_method(linear, prefix)
kv_b_proj = SimpleNamespace(
quant_config=quant_config,
quant_method=quant_method,
)

assert isinstance(quant_method, UnquantizedLinearMethod)
assert not _is_mla_bmm_requantization_allowed(kv_b_proj)


def test_quark_dynamic_mla_projection_allows_bmm_requantization(monkeypatch):
prefix = "model.layers.3.self_attn.kv_b_proj"
quant_config = QuarkConfig({"exclude": [prefix]})
quant_config.dynamic_mxfp4_quant = True
monkeypatch.setattr(quant_config, "get_scheme", lambda **_: object())
linear = LinearBase.__new__(LinearBase)
quant_method = quant_config.get_quant_method(linear, prefix)
kv_b_proj = SimpleNamespace(
quant_config=quant_config,
quant_method=quant_method,
)

assert isinstance(quant_method, QuarkLinearMethod)
assert _is_mla_bmm_requantization_allowed(kv_b_proj)


@pytest.mark.cpu_test
def test_dcp_chunked_context_accepts_non_virtual_block_aligned_prefix():
context_lens_cpu = torch.tensor([65, 96], dtype=torch.int32)
Expand Down
17 changes: 16 additions & 1 deletion vllm/model_executor/layers/attention/mla_attention.py
Original file line number Diff line number Diff line change
Expand Up @@ -245,6 +245,7 @@
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
UnquantizedLinearMethod,
)
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8
Expand Down Expand Up @@ -310,6 +311,16 @@
_FP8_DTYPE = current_platform.fp8_dtype()


def _is_mla_bmm_requantization_allowed(kv_b_proj: torch.nn.Module) -> bool:
"""Whether online BMM weight quantization preserves checkpoint precision."""
return not (
getattr(kv_b_proj, "quant_config", None) is not None
and isinstance(
getattr(kv_b_proj, "quant_method", None), UnquantizedLinearMethod
)
)


def _detect_output_quant_key(
output: torch.Tensor,
output_scale: torch.Tensor | None,
Expand Down Expand Up @@ -617,11 +628,15 @@ def __init__(
use_pcp=self.use_pcp,
)

self.is_aiter_triton_fp8_bmm_enabled = rocm_aiter_ops.is_fp8bmm_enabled()
allow_bmm_requantization = _is_mla_bmm_requantization_allowed(self.kv_b_proj)
self.is_aiter_triton_fp8_bmm_enabled = (
rocm_aiter_ops.is_fp8bmm_enabled() and allow_bmm_requantization
)

# If kv_b_proj_weight is unquantized, quantize it to mxfp4 if supported
self.is_aiter_triton_fp4_bmm_enabled = (
rocm_aiter_ops.is_fp4bmm_enabled()
and allow_bmm_requantization
and hasattr(self.kv_b_proj, "weight")
and self.kv_b_proj.weight.dtype == torch.bfloat16
)
Expand Down
Loading