From 7b23ef780cf2ab32ce45f249c6f50efb05ac157a Mon Sep 17 00:00:00 2001 From: Peiyuan Zhou Date: Wed, 29 Jul 2026 12:03:19 +0800 Subject: [PATCH 01/15] [Model][NVIDIA] Route DSA models to the SM100 implementation Route GlmMoeDsaForCausalLM and the matching MTP architecture to the optimized deepseek_v32 implementation on SM100-family devices, keeping the generic deepseek_v2 fallback everywhere else. Default the KV cache to FP8 since this implementation requires a sparse FP8 cache. Split out of #48597 (reverted by #49768). Co-authored-by: Claude Signed-off-by: Peiyuan Zhou --- tests/models/registry.py | 6 +++++ vllm/config/speculative.py | 8 ++++++- vllm/model_executor/models/registry.py | 3 ++- vllm/models/deepseek_v32/__init__.py | 28 ++++++++++++++++++++---- vllm/models/deepseek_v32/attention.py | 6 +++++ vllm/v1/spec_decode/llm_base_proposer.py | 6 ++++- 6 files changed, 50 insertions(+), 7 deletions(-) diff --git a/tests/models/registry.py b/tests/models/registry.py index 16604ec7ff83..3f3a90d011a8 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -1673,6 +1673,12 @@ def check_available_online( speculative_model="luccafong/deepseek_mtp_draft_random", trust_remote_code=True, ), + "DeepseekV32MTPModel": _HfExamplesInfo( + "zai-org/GLM-5", + speculative_model="zai-org/GLM-5", + min_transformers_version="5.0.1", + is_available_online=False, + ), "DeepSeekV4MTPModel": _HfExamplesInfo( "deepseek-ai/DeepSeek-V4-Flash", speculative_model="deepseek-ai/DeepSeek-V4-Flash", diff --git a/vllm/config/speculative.py b/vllm/config/speculative.py index c3e0866b453d..c334fde44600 100644 --- a/vllm/config/speculative.py +++ b/vllm/config/speculative.py @@ -341,6 +341,7 @@ def compute_hash(self) -> str: @staticmethod def hf_config_override(hf_config: PretrainedConfig) -> PretrainedConfig: initial_architecture = hf_config.architectures[0] + use_sparse_mtp = hf_config.model_type == "glm_moe_dsa" if hf_config.model_type == "dots3_note": n_predict = getattr(hf_config, "num_nextn_predict_layers", 1) mtp_layer_types = getattr(hf_config, "mtp_layer_types", None) @@ -365,7 +366,12 @@ def hf_config_override(hf_config: PretrainedConfig) -> PretrainedConfig: if hf_config.model_type == "deepseek_mtp": n_predict = getattr(hf_config, "num_nextn_predict_layers", None) hf_config.update( - {"n_predict": n_predict, "architectures": ["DeepSeekMTPModel"]} + { + "n_predict": n_predict, + "architectures": [ + "DeepseekV32MTPModel" if use_sparse_mtp else "DeepSeekMTPModel" + ], + } ) if hf_config.model_type == "deepseek_v4": hf_config.model_type = "deepseek_mtp" diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index 1e4bf6731c96..63495d397864 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -115,7 +115,7 @@ "Glm4ForCausalLM": ("glm4", "Glm4ForCausalLM"), "Glm4MoeForCausalLM": ("glm4_moe", "Glm4MoeForCausalLM"), "Glm4MoeLiteForCausalLM": ("glm4_moe_lite", "Glm4MoeLiteForCausalLM"), - "GlmMoeDsaForCausalLM": ("deepseek_v2", "GlmMoeDsaForCausalLM"), + "GlmMoeDsaForCausalLM": ("vllm.models.deepseek_v32", "GlmMoeDsaForCausalLM"), "GptOssForCausalLM": ("gpt_oss", "GptOssForCausalLM"), "GPT2LMHeadModel": ("gpt2", "GPT2LMHeadModel"), "GPTJForCausalLM": ("gpt_j", "GPTJForCausalLM"), @@ -659,6 +659,7 @@ "Eagle3DeepseekV3ForCausalLM": ("deepseek_eagle3", "Eagle3DeepseekV2ForCausalLM"), "EagleDeepSeekMTPModel": ("deepseek_eagle", "EagleDeepseekV3ForCausalLM"), "DeepSeekMTPModel": ("deepseek_mtp", "DeepSeekMTP"), + "DeepseekV32MTPModel": ("vllm.models.deepseek_v32", "DeepseekV32MTP"), "Dots3NoteMTPModel": ("vllm.models.dots3_note", "Dots3NoteMTP"), "DeepSeekV4MTPModel": ("vllm.models.deepseek_v4", "DeepSeekV4MTP"), "BailingMoeV3MTPModel": ("bailing_moe_v3_mtp", "BailingMoeV3MTPModel"), diff --git a/vllm/models/deepseek_v32/__init__.py b/vllm/models/deepseek_v32/__init__.py index f3eba73142b4..7fa607c7ee61 100644 --- a/vllm/models/deepseek_v32/__init__.py +++ b/vllm/models/deepseek_v32/__init__.py @@ -6,21 +6,41 @@ MLA + a "lightning indexer" that selects the top-k tokens for a sparse MLA attend. The same model code serves any DSA checkpoint, including GLM-5.2 (``glm_moe_dsa``), which reuses this architecture. + +The kernels under ``nvidia/`` target the Blackwell (SM100) family. Pre-SM100 +CUDA (e.g. H100) falls back to the generic ``deepseek_v2`` implementation, which +already handles the DSA (index_topk) architecture and is ``torch.compile`` +-friendly there, so those devices keep working rather than hard-failing. """ from vllm.platforms import current_platform if current_platform.is_rocm(): + # GLM-5.2 keeps the generic implementation here, as it has on main; only + # DeepSeek V3.2 has an AMD DSA port. + from vllm.model_executor.models.deepseek_v2 import GlmMoeDsaForCausalLM + from .amd.model import DeepseekV32ForCausalLM from .amd.mtp import DeepseekV32MTP -elif current_platform.is_xpu(): - raise NotImplementedError("deepseek_v32 does not yet support XPU.") -else: - # Covers Blackwell (sm100) and all other CUDA devices. +elif current_platform.is_device_capability_family(100): from .nvidia.model import DeepseekV32ForCausalLM from .nvidia.mtp import DeepseekV32MTP + # GLM-5.2 (glm_moe_dsa) reuses the same optimized DSA module on SM100. + GlmMoeDsaForCausalLM = DeepseekV32ForCausalLM +else: + # Pre-SM100 CUDA, XPU and CPU. The generic implementation already handles + # the DSA architecture, so these keep serving instead of hard-failing -- + # which is what GLM-5.2 does on main, where its registry entry points + # straight at deepseek_v2 and never reaches this package. + from vllm.model_executor.models.deepseek_mtp import DeepSeekMTP as DeepseekV32MTP + from vllm.model_executor.models.deepseek_v2 import ( + DeepseekV3ForCausalLM as DeepseekV32ForCausalLM, + ) + from vllm.model_executor.models.deepseek_v2 import GlmMoeDsaForCausalLM + __all__ = [ "DeepseekV32ForCausalLM", "DeepseekV32MTP", + "GlmMoeDsaForCausalLM", ] diff --git a/vllm/models/deepseek_v32/attention.py b/vllm/models/deepseek_v32/attention.py index 253971b7bd05..3ffffa25d54d 100644 --- a/vllm/models/deepseek_v32/attention.py +++ b/vllm/models/deepseek_v32/attention.py @@ -183,6 +183,12 @@ def __init__( ) -> None: quant_config = vllm_config.quant_config cache_config = vllm_config.cache_config + if cache_config is not None and cache_config.cache_dtype == "auto": + # This implementation asserts an fp8 sparse cache below. Checkpoints + # that declare a kv_cache_scheme get "auto" resolved upstream, but + # plain fp8 checkpoints do not — default it here so they do not hit + # that assert. + cache_config.cache_dtype = "fp8" hidden_size = config.hidden_size qk_nope_head_dim = config.qk_nope_head_dim diff --git a/vllm/v1/spec_decode/llm_base_proposer.py b/vllm/v1/spec_decode/llm_base_proposer.py index 29a95caa88eb..7a67a00084ab 100644 --- a/vllm/v1/spec_decode/llm_base_proposer.py +++ b/vllm/v1/spec_decode/llm_base_proposer.py @@ -1018,7 +1018,11 @@ def model_returns_tuple(self) -> bool: # feedback into the next draft step. architectures = self.draft_model_config.hf_config.architectures or [] return bool( - {"DeepSeekMTPModel", "KimiK3MTPModel"}.intersection(architectures) + { + "DeepSeekMTPModel", + "DeepseekV32MTPModel", + "KimiK3MTPModel", + }.intersection(architectures) ) return self.method not in ("mtp", "draft_model", "dflash") From c0c72409e3bba1393ece60876eabf8f74925ad00 Mon Sep 17 00:00:00 2001 From: Peiyuan Zhou Date: Fri, 31 Jul 2026 11:29:05 +0800 Subject: [PATCH 02/15] [Model][NVIDIA] Alias the GLM-5.2 class instead of assigning it mypy rejects rebinding a class name with a plain assignment ("Cannot assign to a type"), which the other branches of this dispatch bind by import. Use an import alias so every branch defines the name the same way. Co-authored-by: Claude Opus 5 Signed-off-by: Peiyuan Zhou --- vllm/models/deepseek_v32/__init__.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/vllm/models/deepseek_v32/__init__.py b/vllm/models/deepseek_v32/__init__.py index 7fa607c7ee61..3a2c8e6da6ba 100644 --- a/vllm/models/deepseek_v32/__init__.py +++ b/vllm/models/deepseek_v32/__init__.py @@ -23,11 +23,10 @@ from .amd.model import DeepseekV32ForCausalLM from .amd.mtp import DeepseekV32MTP elif current_platform.is_device_capability_family(100): + # GLM-5.2 (glm_moe_dsa) reuses the same optimized DSA module on SM100. from .nvidia.model import DeepseekV32ForCausalLM + from .nvidia.model import DeepseekV32ForCausalLM as GlmMoeDsaForCausalLM from .nvidia.mtp import DeepseekV32MTP - - # GLM-5.2 (glm_moe_dsa) reuses the same optimized DSA module on SM100. - GlmMoeDsaForCausalLM = DeepseekV32ForCausalLM else: # Pre-SM100 CUDA, XPU and CPU. The generic implementation already handles # the DSA architecture, so these keep serving instead of hard-failing -- From be5107dfbd070e1dbe2953e6ed02a1d0dd38d6ad Mon Sep 17 00:00:00 2001 From: Peiyuan Zhou Date: Fri, 31 Jul 2026 21:07:54 +0800 Subject: [PATCH 03/15] [Model][NVIDIA] Let the SM100 DSA path serve an unquantized KV cache Routing GlmMoeDsaForCausalLM here changed what a default launch does. This implementation asserted an fp8 KV cache, so an unset --kv-cache-dtype was rewritten to fp8 from inside a per-layer constructor: the engine went on reporting kv_cache_dtype=auto while allocating an fp8 cache (2,935,232 KV tokens against main's 1,509,888 for the same launch), and the "Using ... data type to store kv cache" line never printed. The assert was stricter than anything below it needs. fused_norm_rope already writes an unquantized cache when the dtype is not fp8, fused_q already emits the bf16 (ql_nope, q_pe) query that the fp8_ds_mla layout uses, FlashInfer sparse accepts that tuple, and the ROCm subclass already derives the same two flags from the cache dtype. Only the query form (taken from the backend capability rather than the cache) and the unconditional fp8 view of the paged cache assumed fp8; both now come from the dtype, and nothing rewrites cache_dtype. Measured on 8xB300, GLM-5.2 block-fp8, TP8, MTP=5, 8192-in/1024-out at concurrency 1. Default launch: 1,513,024 KV tokens (bf16, matching main's 1,509,888), 362 tok/s vs main's 340 for the same config, GSM8K 0.950. --kv-cache-dtype fp8_e4m3 is unchanged at 2,935,232 tokens and 403 tok/s, GSM8K 0.938. Co-authored-by: Claude Opus 5 Signed-off-by: Peiyuan Zhou --- .../models/test_deepseek_v32_kv_cache_form.py | 52 ++++++ tests/models/test_deepseek_v32_routing.py | 150 ++++++++++++++++++ vllm/models/deepseek_v32/attention.py | 57 ++++--- 3 files changed, 239 insertions(+), 20 deletions(-) create mode 100644 tests/models/test_deepseek_v32_kv_cache_form.py create mode 100644 tests/models/test_deepseek_v32_routing.py diff --git a/tests/models/test_deepseek_v32_kv_cache_form.py b/tests/models/test_deepseek_v32_kv_cache_form.py new file mode 100644 index 000000000000..ca5cc9293c76 --- /dev/null +++ b/tests/models/test_deepseek_v32_kv_cache_form.py @@ -0,0 +1,52 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Query/cache form selection for the SM100 DSA attention layer. + +The fused kernels emit either a packed fp8 MQA query against an fp8 cache, or a +bf16 (ql_nope, q_pe) tuple against a cache used as-is. Picking the wrong one is +silent: an unquantized cache viewed as fp8 reinterprets its bytes. +""" + +import pytest + +from vllm.models.deepseek_v32.attention import select_query_and_cache_form + + +@pytest.mark.parametrize("kv_cache_dtype", ["fp8", "fp8_e4m3"]) +def test_per_tensor_fp8_packs_the_query_and_views_the_cache( + kv_cache_dtype: str, +) -> None: + assert select_query_and_cache_form(kv_cache_dtype, True) == (True, True) + + +def test_ds_mla_uses_the_bf16_tuple_and_raw_bytes() -> None: + """FlashMLA dequantizes internally, so the query stays bf16.""" + assert select_query_and_cache_form("fp8_ds_mla", False) == (False, False) + + +@pytest.mark.parametrize("kv_cache_dtype", ["auto", "bfloat16", "float16"]) +def test_unquantized_cache_is_never_viewed_as_fp8(kv_cache_dtype: str) -> None: + """The regression this guards: viewing a bf16 cache as fp8 reads garbage.""" + assert select_query_and_cache_form(kv_cache_dtype, True) == (False, False) + + +@pytest.mark.parametrize("kv_cache_dtype", ["fp8", "fp8_e4m3"]) +def test_fp8_cache_on_a_bf16_query_backend_is_rejected(kv_cache_dtype: str) -> None: + """Only the fp8_ds_mla layout pairs an fp8 cache with a bf16 query.""" + with pytest.raises(AssertionError, match="fp8_ds_mla"): + select_query_and_cache_form(kv_cache_dtype, False) + + +def test_unquantized_cache_does_not_depend_on_backend_query_support() -> None: + both = {select_query_and_cache_form("auto", s) for s in (True, False)} + assert both == {(False, False)} + + +@pytest.mark.parametrize("kv_cache_dtype", ["fp8_e5m2", "fp8_inc", "nvfp4"]) +def test_fp8_layouts_this_layer_cannot_address_are_rejected( + kv_cache_dtype: str, +) -> None: + """These count as quantized but are not e4m3, so reading them as e4m3 + would be silent corruption. A backend that allowed one should fail loudly.""" + with pytest.raises(AssertionError, match="cannot address"): + select_query_and_cache_form(kv_cache_dtype, True) diff --git a/tests/models/test_deepseek_v32_routing.py b/tests/models/test_deepseek_v32_routing.py new file mode 100644 index 000000000000..4836d8b89136 --- /dev/null +++ b/tests/models/test_deepseek_v32_routing.py @@ -0,0 +1,150 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Which DSA implementation and MTP draft an architecture resolves to. + +The optimized implementation under ``deepseek_v32/nvidia`` is SM100-only; +everywhere else the package re-exports the generic ``deepseek_v2`` classes. The +draft model has to follow the target, and the DeepSeek-family MTP returns a +(logit_hidden, recycle_hidden) tuple that the proposer has to know about. None +of that needs a GPU to check, and the initialization tests skip these models +(``is_available_online=False``), so it would otherwise go uncovered. +""" + +import importlib +import sys +from types import SimpleNamespace + +import pytest + +from vllm.config.speculative import SpeculativeConfig +from vllm.model_executor.models.registry import ( + _SPECULATIVE_DECODING_MODELS, + _TEXT_GENERATION_MODELS, +) +from vllm.v1.spec_decode.llm_base_proposer import SpecDecodeBaseProposer + + +def _reload_package( + monkeypatch: pytest.MonkeyPatch, *, sm100: bool, rocm: bool = False +): + """Re-import the package with the platform forced, as at process start.""" + from vllm.platforms import current_platform + + monkeypatch.setattr(current_platform, "is_rocm", lambda: rocm) + monkeypatch.setattr(current_platform, "is_xpu", lambda: False) + monkeypatch.setattr(current_platform, "is_cuda", lambda: not rocm) + monkeypatch.setattr( + current_platform, + "is_device_capability_family", + lambda family: sm100 and not rocm and family == 100, + ) + import vllm.models.deepseek_v32 as pkg + + return importlib.reload(pkg) + + +@pytest.fixture(autouse=True) +def _restore_package(): + """Drop the patched module so the next importer re-imports it for real. + + Reloading here instead would race monkeypatch teardown and could cache a + module built against a faked platform. + """ + yield + sys.modules.pop("vllm.models.deepseek_v32", None) + + +def test_sm100_uses_the_optimized_implementation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + pkg = _reload_package(monkeypatch, sm100=True) + + assert pkg.DeepseekV32ForCausalLM.__module__.endswith("deepseek_v32.nvidia.model") + assert pkg.DeepseekV32MTP.__module__.endswith("deepseek_v32.nvidia.mtp") + # GLM-5.2 reuses the same DSA module rather than getting its own. + assert pkg.GlmMoeDsaForCausalLM is pkg.DeepseekV32ForCausalLM + + +def test_pre_sm100_cuda_falls_back_to_generic(monkeypatch: pytest.MonkeyPatch) -> None: + """H100/CPU keep the deepseek_v2 path instead of failing to import.""" + pkg = _reload_package(monkeypatch, sm100=False) + + assert pkg.DeepseekV32ForCausalLM.__module__.endswith("models.deepseek_v2") + assert pkg.GlmMoeDsaForCausalLM.__module__.endswith("models.deepseek_v2") + assert pkg.DeepseekV32MTP.__module__.endswith("models.deepseek_mtp") + + +def test_rocm_keeps_its_own_dsa_port_but_generic_glm( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """DeepSeek V3.2 has an AMD DSA port; GLM-5.2 does not, so it stays generic.""" + pkg = _reload_package(monkeypatch, sm100=False, rocm=True) + + assert pkg.DeepseekV32ForCausalLM.__module__.endswith("deepseek_v32.amd.model") + assert pkg.DeepseekV32MTP.__module__.endswith("deepseek_v32.amd.mtp") + assert pkg.GlmMoeDsaForCausalLM.__module__.endswith("models.deepseek_v2") + + +def test_both_architectures_are_registered() -> None: + assert _TEXT_GENERATION_MODELS["GlmMoeDsaForCausalLM"] == ( + "vllm.models.deepseek_v32", + "GlmMoeDsaForCausalLM", + ) + assert _SPECULATIVE_DECODING_MODELS["DeepseekV32MTPModel"] == ( + "vllm.models.deepseek_v32", + "DeepseekV32MTP", + ) + + +class _HfConfigStub: + """The bits of PretrainedConfig that hf_config_override touches.""" + + def __init__(self, model_type: str, architecture: str) -> None: + self.architectures = [architecture] + self.model_type = model_type + self.num_nextn_predict_layers = 1 + + def update(self, values: dict) -> None: + self.__dict__.update(values) + + +def _mtp_arch(model_type: str, architecture: str = "SomeForCausalLM") -> list[str]: + hf_config = _HfConfigStub(model_type, architecture) + SpeculativeConfig.hf_config_override(hf_config) + return hf_config.architectures + + +def test_glm_selects_the_sparse_mtp() -> None: + assert _mtp_arch("glm_moe_dsa") == ["DeepseekV32MTPModel"] + + +@pytest.mark.parametrize("model_type", ["deepseek_v3", "deepseek_v32"]) +def test_other_deepseek_keeps_the_original_mtp(model_type: str) -> None: + """Only the DSA-sparse family gets the new draft; V3 must not move.""" + assert _mtp_arch(model_type) == ["DeepSeekMTPModel"] + + +@pytest.mark.parametrize( + "architectures,expected", + [ + (["DeepSeekMTPModel"], True), + (["DeepseekV32MTPModel"], True), + (["Glm4MoeMTPModel"], False), + ([], False), + ], +) +def test_tuple_return_contract_covers_the_new_architecture( + architectures: list[str], expected: bool +) -> None: + """The DSA MTP recycles the post-norm hidden, so it returns a 2-tuple. + + Miss this and the proposer feeds a tuple where a tensor is expected. + """ + proposer = SimpleNamespace( + method="mtp", + draft_model_config=SimpleNamespace( + hf_config=SimpleNamespace(architectures=architectures) + ), + ) + + assert SpecDecodeBaseProposer.model_returns_tuple(proposer) is expected diff --git a/vllm/models/deepseek_v32/attention.py b/vllm/models/deepseek_v32/attention.py index 3ffffa25d54d..8f8381bb7c08 100644 --- a/vllm/models/deepseek_v32/attention.py +++ b/vllm/models/deepseek_v32/attention.py @@ -47,6 +47,41 @@ from vllm.model_executor.layers.attention.mla_attention import MLACommonMetadata +# Per-tensor fp8 layouts this attention reads as float8_e4m3fn. Which cache +# dtypes are reachable at all is the sparse MLA backend's call +# (``supported_kv_cache_dtypes``); this only says how to address the ones that +# get here, so an e5m2 or nvfp4 cache raises instead of being read as e4m3. +_E4M3_KV_LAYOUTS = ("fp8", "fp8_e4m3") + + +def select_query_and_cache_form( + kv_cache_dtype: str, backend_supports_quant_query: bool +) -> tuple[bool, bool]: + """``(pack the MQA query as fp8, view the paged cache as fp8)``. + + An fp8 query only pairs with a per-tensor fp8 cache. Everything else takes + the bf16 ``(ql_nope, q_pe)`` query tuple that the fp8_ds_mla layout already + uses -- including an unquantized cache, so no dtype is forced on the user. + The fused kernels already emit both forms. + """ + if not is_quantized_kv_cache(kv_cache_dtype): + return False, False + if kv_cache_dtype == "fp8_ds_mla": + # Block-scaled fp8 NoPE + bf16 RoPE, addressed as raw bytes and + # dequantized inside FlashMLA. + return False, False + assert kv_cache_dtype in _E4M3_KV_LAYOUTS, ( + f"deepseek_v32 cannot address a {kv_cache_dtype} KV cache. " + f"Supported: unquantized, {', '.join(_E4M3_KV_LAYOUTS)}, fp8_ds_mla." + ) + assert backend_supports_quant_query, ( + "deepseek_v32 on a bf16-query sparse MLA backend (FlashMLA sparse) " + "requires the fp8_ds_mla KV cache layout. " + "Launch with --kv-cache-dtype fp8_ds_mla." + ) + return True, True + + class DeepseekV32Indexer(nn.Module): indexer_cache_cls = DeepseekV32IndexerCache @@ -183,13 +218,6 @@ def __init__( ) -> None: quant_config = vllm_config.quant_config cache_config = vllm_config.cache_config - if cache_config is not None and cache_config.cache_dtype == "auto": - # This implementation asserts an fp8 sparse cache below. Checkpoints - # that declare a kv_cache_scheme get "auto" resolved upstream, but - # plain fp8 checkpoints do not — default it here so they do not hit - # that assert. - cache_config.cache_dtype = "fp8" - hidden_size = config.hidden_size qk_nope_head_dim = config.qk_nope_head_dim qk_rope_head_dim = config.qk_rope_head_dim @@ -290,20 +318,9 @@ def __init__( ) if self.require_fp8_kv_cache: - assert is_quantized_kv_cache(self.kv_cache_dtype), ( - "deepseek_v32 (nvidia) requires an fp8 KV cache served by a sparse " - "MLA backend. Launch with --kv-cache-dtype fp8 (FlashInfer sparse) " - "or --kv-cache-dtype fp8_ds_mla (FlashMLA sparse)." + self._fp8_query, self._fp8_kv_needs_view = select_query_and_cache_form( + self.kv_cache_dtype, self.impl.supports_quant_query_input ) - self._fp8_query = self.impl.supports_quant_query_input - if not self._fp8_query: - assert self.kv_cache_dtype == "fp8_ds_mla", ( - "deepseek_v32 (nvidia) on a bf16-query sparse MLA backend " - "(FlashMLA sparse) requires the fp8_ds_mla KV cache layout. " - "Launch with --kv-cache-dtype fp8_ds_mla." - ) - - self._fp8_kv_needs_view = self.kv_cache_dtype != "fp8_ds_mla" self._index_rope_interleave = getattr(config, "indexer_rope_interleave", False) From a16fc42ceea3e8472ae785dc5e5d07a8fef79161 Mon Sep 17 00:00:00 2001 From: Woosuk Kwon Date: Wed, 19 Aug 2026 01:45:59 +0000 Subject: [PATCH 04/15] [Model] Default GLM-5.2 to MRV2 breakable graphs Choose Model Runner V2 for GLM-5.2 and auto-enable the non-compiled breakable CUDA graph path for the model and MTP architectures on every platform. The optimized SM100 model routing remains hardware-specific, while this serving default does not. Cover both NVFP4 and FP8 checkpoints with and without MTP. Co-authored-by: OpenAI Codex Signed-off-by: Woosuk Kwon --- tests/test_config.py | 82 ++++++++++++++++++++++++++++++++++++++++++++ vllm/config/vllm.py | 81 ++++++++++++++++++++++++------------------- 2 files changed, 128 insertions(+), 35 deletions(-) diff --git a/tests/test_config.py b/tests/test_config.py index a2797e52126a..bf7e1fae66a9 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -127,6 +127,88 @@ def test_rocm_defaults_deepseek_v4_to_mrv1(monkeypatch): default_v2_model_runner_architectures.cache_clear() +@pytest.mark.parametrize( + "model", + ["nvidia/GLM-5.2-NVFP4", "zai-org/GLM-5.2-FP8"], +) +@pytest.mark.parametrize("with_mtp", [False, True], ids=["no-mtp", "mtp"]) +def test_glm52_defaults_to_mrv2_and_breakable_cudagraph(monkeypatch, model, with_mtp): + from vllm.compilation.breakable_cudagraph import ( + is_breakable_cudagraph_enabled, + ) + from vllm.config.vllm import default_v2_model_runner_architectures + + monkeypatch.delenv("VLLM_USE_BREAKABLE_CUDAGRAPH", raising=False) + monkeypatch.delenv("VLLM_USE_V2_MODEL_RUNNER", raising=False) + monkeypatch.setattr(vllm_config_module, "HAS_TRITON", True) + default_v2_model_runner_architectures.cache_clear() + + model_config = SimpleNamespace( + model=model, + architectures=["GlmMoeDsaForCausalLM"], + runner_type="generate", + is_moe=True, + is_hybrid=False, + is_attention_free=False, + is_diffusion=False, + ) + config = SimpleNamespace( + model_config=model_config, + speculative_config=SimpleNamespace(method="mtp") if with_mtp else None, + parallel_config=SimpleNamespace(prefill_context_parallel_size=1), + compilation_config=CompilationConfig( + cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE + ), + ) + config._dflash_needs_multi_kv_group = lambda: False + config._is_default_v2_model_runner_model = lambda: ( + VllmConfig._is_default_v2_model_runner_model(config) + ) + config._get_v2_model_runner_unsupported_features = lambda: [] + config._uses_breakable_cudagraph_by_default = lambda: ( + VllmConfig._uses_breakable_cudagraph_by_default(config) + ) + + try: + assert VllmConfig.use_v2_model_runner.fget(config) + assert VllmConfig._maybe_enable_breakable_cudagraph(config) + assert is_breakable_cudagraph_enabled() + assert config.compilation_config.mode == CompilationMode.NONE + assert config.compilation_config.cudagraph_mode.has_piecewise_cudagraphs() + finally: + os.environ.pop("VLLM_USE_BREAKABLE_CUDAGRAPH", None) + default_v2_model_runner_architectures.cache_clear() + + +@pytest.mark.parametrize( + "architecture", ["GlmMoeDsaForCausalLM", "DeepseekV32MTPModel"] +) +def test_glm52_breakable_cudagraph_default_is_platform_independent( + monkeypatch, architecture +): + from vllm.platforms import current_platform + + monkeypatch.delenv("VLLM_USE_BREAKABLE_CUDAGRAPH", raising=False) + monkeypatch.setattr( + current_platform, + "is_device_capability_family", + lambda family: False, + ) + config = SimpleNamespace( + model_config=SimpleNamespace(architectures=[architecture]), + compilation_config=CompilationConfig(), + ) + config._uses_breakable_cudagraph_by_default = lambda: ( + VllmConfig._uses_breakable_cudagraph_by_default(config) + ) + + try: + assert VllmConfig._maybe_enable_breakable_cudagraph(config) + assert config.compilation_config.mode == CompilationMode.NONE + finally: + os.environ.pop("VLLM_USE_BREAKABLE_CUDAGRAPH", None) + + @pytest.mark.parametrize( ("use_v2_model_runner", "expected_capture_sizes"), [ diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index 5faf09fc347e..8c12a0199a94 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -70,6 +70,7 @@ { "DeepseekV2ForCausalLM", "DeepseekV4ForCausalLM", + "GlmMoeDsaForCausalLM", "GraniteMoeForCausalLM", "InklingForCausalLM", "InklingForConditionalGeneration", @@ -79,6 +80,22 @@ } ) +DEFAULT_BREAKABLE_CUDAGRAPH_ARCHITECTURES = frozenset( + { + "DeepseekV32MTPModel", + "DeepseekV4ForCausalLM", + "DeepSeekV4MTPModel", + "GlmMoeDsaForCausalLM", + "InklingForCausalLM", + "InklingForConditionalGeneration", + "KimiK3ForConditionalGeneration", + "KimiK3MTPModel", + "KimiLinearForCausalLM", + "MiniMaxM3SparseForCausalLM", + "MiniMaxM3SparseForConditionalGeneration", + } +) + @lru_cache def default_v2_model_runner_architectures() -> frozenset[str]: @@ -694,6 +711,34 @@ def _is_default_v2_model_runner_model(self) -> bool: return False return is_default_v2_architecture or not model_config.is_moe + def _uses_breakable_cudagraph_by_default(self) -> bool: + model_config = self.model_config + if model_config is None: + return False + + architectures = set(model_config.architectures) + return bool(architectures & DEFAULT_BREAKABLE_CUDAGRAPH_ARCHITECTURES) + + def _maybe_enable_breakable_cudagraph(self) -> bool: + if ( + "VLLM_USE_BREAKABLE_CUDAGRAPH" not in os.environ + and self._uses_breakable_cudagraph_by_default() + ): + os.environ["VLLM_USE_BREAKABLE_CUDAGRAPH"] = "1" + logger.info_once( + "Auto-enabling VLLM_USE_BREAKABLE_CUDAGRAPH=1. " + "Set VLLM_USE_BREAKABLE_CUDAGRAPH=0 to opt out." + ) + + from vllm.compilation.breakable_cudagraph import ( + is_breakable_cudagraph_enabled, + ) + + enabled = is_breakable_cudagraph_enabled() + if enabled: + self.compilation_config.mode = CompilationMode.NONE + return enabled + @property def needs_dp_coordinator(self) -> bool: """ @@ -1291,41 +1336,7 @@ def __post_init__(self): ) self.compilation_config.mode = CompilationMode.NONE - # For model classes don't carry @support_torch_compile — - # the breakable cudagraph is the supported PIECEWISE path. Auto-enable - # it unless the user has explicitly opted out via the env var. - if ( - self.model_config is not None - and "VLLM_USE_BREAKABLE_CUDAGRAPH" not in os.environ - and any( - a - in ( - "DeepseekV4ForCausalLM", - "DeepSeekV4MTPModel", - "InklingForCausalLM", - "InklingForConditionalGeneration", - "KimiK3ForConditionalGeneration", - "KimiK3MTPModel", - "KimiLinearForCausalLM", - "MiniMaxM3SparseForCausalLM", - "MiniMaxM3SparseForConditionalGeneration", - ) - for a in self.model_config.architectures - ) - ): - os.environ["VLLM_USE_BREAKABLE_CUDAGRAPH"] = "1" - logger.info_once( - "Auto-enabling VLLM_USE_BREAKABLE_CUDAGRAPH=1. " - "Set VLLM_USE_BREAKABLE_CUDAGRAPH=0 to opt out." - ) - - from vllm.compilation.breakable_cudagraph import ( - is_breakable_cudagraph_enabled, - ) - - breakable_cudagraph_enabled = is_breakable_cudagraph_enabled() - if breakable_cudagraph_enabled: - self.compilation_config.mode = CompilationMode.NONE + breakable_cudagraph_enabled = self._maybe_enable_breakable_cudagraph() if not breakable_cudagraph_enabled and ( self.compilation_config.backend == "eager" From 4f6dffefd7835693b4f0f90243fbd8e75b24b891 Mon Sep 17 00:00:00 2001 From: Woosuk Kwon Date: Wed, 19 Aug 2026 03:31:19 +0000 Subject: [PATCH 05/15] [Model][NVIDIA] Use the CUDA DSA path on all NVIDIA GPUs GLM-5.2 now defaults to the non-compiled MRV2 path, so route every CUDA device through the NVIDIA deepseek_v32 implementation. Capability-specific kernels continue to gate themselves and fall back when unavailable. Drop the standalone routing and KV-cache-form test files. Co-authored-by: OpenAI Codex Signed-off-by: Woosuk Kwon --- .../models/test_deepseek_v32_kv_cache_form.py | 52 ------ tests/models/test_deepseek_v32_routing.py | 150 ------------------ vllm/models/deepseek_v32/__init__.py | 17 +- 3 files changed, 7 insertions(+), 212 deletions(-) delete mode 100644 tests/models/test_deepseek_v32_kv_cache_form.py delete mode 100644 tests/models/test_deepseek_v32_routing.py diff --git a/tests/models/test_deepseek_v32_kv_cache_form.py b/tests/models/test_deepseek_v32_kv_cache_form.py deleted file mode 100644 index ca5cc9293c76..000000000000 --- a/tests/models/test_deepseek_v32_kv_cache_form.py +++ /dev/null @@ -1,52 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Query/cache form selection for the SM100 DSA attention layer. - -The fused kernels emit either a packed fp8 MQA query against an fp8 cache, or a -bf16 (ql_nope, q_pe) tuple against a cache used as-is. Picking the wrong one is -silent: an unquantized cache viewed as fp8 reinterprets its bytes. -""" - -import pytest - -from vllm.models.deepseek_v32.attention import select_query_and_cache_form - - -@pytest.mark.parametrize("kv_cache_dtype", ["fp8", "fp8_e4m3"]) -def test_per_tensor_fp8_packs_the_query_and_views_the_cache( - kv_cache_dtype: str, -) -> None: - assert select_query_and_cache_form(kv_cache_dtype, True) == (True, True) - - -def test_ds_mla_uses_the_bf16_tuple_and_raw_bytes() -> None: - """FlashMLA dequantizes internally, so the query stays bf16.""" - assert select_query_and_cache_form("fp8_ds_mla", False) == (False, False) - - -@pytest.mark.parametrize("kv_cache_dtype", ["auto", "bfloat16", "float16"]) -def test_unquantized_cache_is_never_viewed_as_fp8(kv_cache_dtype: str) -> None: - """The regression this guards: viewing a bf16 cache as fp8 reads garbage.""" - assert select_query_and_cache_form(kv_cache_dtype, True) == (False, False) - - -@pytest.mark.parametrize("kv_cache_dtype", ["fp8", "fp8_e4m3"]) -def test_fp8_cache_on_a_bf16_query_backend_is_rejected(kv_cache_dtype: str) -> None: - """Only the fp8_ds_mla layout pairs an fp8 cache with a bf16 query.""" - with pytest.raises(AssertionError, match="fp8_ds_mla"): - select_query_and_cache_form(kv_cache_dtype, False) - - -def test_unquantized_cache_does_not_depend_on_backend_query_support() -> None: - both = {select_query_and_cache_form("auto", s) for s in (True, False)} - assert both == {(False, False)} - - -@pytest.mark.parametrize("kv_cache_dtype", ["fp8_e5m2", "fp8_inc", "nvfp4"]) -def test_fp8_layouts_this_layer_cannot_address_are_rejected( - kv_cache_dtype: str, -) -> None: - """These count as quantized but are not e4m3, so reading them as e4m3 - would be silent corruption. A backend that allowed one should fail loudly.""" - with pytest.raises(AssertionError, match="cannot address"): - select_query_and_cache_form(kv_cache_dtype, True) diff --git a/tests/models/test_deepseek_v32_routing.py b/tests/models/test_deepseek_v32_routing.py deleted file mode 100644 index 4836d8b89136..000000000000 --- a/tests/models/test_deepseek_v32_routing.py +++ /dev/null @@ -1,150 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Which DSA implementation and MTP draft an architecture resolves to. - -The optimized implementation under ``deepseek_v32/nvidia`` is SM100-only; -everywhere else the package re-exports the generic ``deepseek_v2`` classes. The -draft model has to follow the target, and the DeepSeek-family MTP returns a -(logit_hidden, recycle_hidden) tuple that the proposer has to know about. None -of that needs a GPU to check, and the initialization tests skip these models -(``is_available_online=False``), so it would otherwise go uncovered. -""" - -import importlib -import sys -from types import SimpleNamespace - -import pytest - -from vllm.config.speculative import SpeculativeConfig -from vllm.model_executor.models.registry import ( - _SPECULATIVE_DECODING_MODELS, - _TEXT_GENERATION_MODELS, -) -from vllm.v1.spec_decode.llm_base_proposer import SpecDecodeBaseProposer - - -def _reload_package( - monkeypatch: pytest.MonkeyPatch, *, sm100: bool, rocm: bool = False -): - """Re-import the package with the platform forced, as at process start.""" - from vllm.platforms import current_platform - - monkeypatch.setattr(current_platform, "is_rocm", lambda: rocm) - monkeypatch.setattr(current_platform, "is_xpu", lambda: False) - monkeypatch.setattr(current_platform, "is_cuda", lambda: not rocm) - monkeypatch.setattr( - current_platform, - "is_device_capability_family", - lambda family: sm100 and not rocm and family == 100, - ) - import vllm.models.deepseek_v32 as pkg - - return importlib.reload(pkg) - - -@pytest.fixture(autouse=True) -def _restore_package(): - """Drop the patched module so the next importer re-imports it for real. - - Reloading here instead would race monkeypatch teardown and could cache a - module built against a faked platform. - """ - yield - sys.modules.pop("vllm.models.deepseek_v32", None) - - -def test_sm100_uses_the_optimized_implementation( - monkeypatch: pytest.MonkeyPatch, -) -> None: - pkg = _reload_package(monkeypatch, sm100=True) - - assert pkg.DeepseekV32ForCausalLM.__module__.endswith("deepseek_v32.nvidia.model") - assert pkg.DeepseekV32MTP.__module__.endswith("deepseek_v32.nvidia.mtp") - # GLM-5.2 reuses the same DSA module rather than getting its own. - assert pkg.GlmMoeDsaForCausalLM is pkg.DeepseekV32ForCausalLM - - -def test_pre_sm100_cuda_falls_back_to_generic(monkeypatch: pytest.MonkeyPatch) -> None: - """H100/CPU keep the deepseek_v2 path instead of failing to import.""" - pkg = _reload_package(monkeypatch, sm100=False) - - assert pkg.DeepseekV32ForCausalLM.__module__.endswith("models.deepseek_v2") - assert pkg.GlmMoeDsaForCausalLM.__module__.endswith("models.deepseek_v2") - assert pkg.DeepseekV32MTP.__module__.endswith("models.deepseek_mtp") - - -def test_rocm_keeps_its_own_dsa_port_but_generic_glm( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """DeepSeek V3.2 has an AMD DSA port; GLM-5.2 does not, so it stays generic.""" - pkg = _reload_package(monkeypatch, sm100=False, rocm=True) - - assert pkg.DeepseekV32ForCausalLM.__module__.endswith("deepseek_v32.amd.model") - assert pkg.DeepseekV32MTP.__module__.endswith("deepseek_v32.amd.mtp") - assert pkg.GlmMoeDsaForCausalLM.__module__.endswith("models.deepseek_v2") - - -def test_both_architectures_are_registered() -> None: - assert _TEXT_GENERATION_MODELS["GlmMoeDsaForCausalLM"] == ( - "vllm.models.deepseek_v32", - "GlmMoeDsaForCausalLM", - ) - assert _SPECULATIVE_DECODING_MODELS["DeepseekV32MTPModel"] == ( - "vllm.models.deepseek_v32", - "DeepseekV32MTP", - ) - - -class _HfConfigStub: - """The bits of PretrainedConfig that hf_config_override touches.""" - - def __init__(self, model_type: str, architecture: str) -> None: - self.architectures = [architecture] - self.model_type = model_type - self.num_nextn_predict_layers = 1 - - def update(self, values: dict) -> None: - self.__dict__.update(values) - - -def _mtp_arch(model_type: str, architecture: str = "SomeForCausalLM") -> list[str]: - hf_config = _HfConfigStub(model_type, architecture) - SpeculativeConfig.hf_config_override(hf_config) - return hf_config.architectures - - -def test_glm_selects_the_sparse_mtp() -> None: - assert _mtp_arch("glm_moe_dsa") == ["DeepseekV32MTPModel"] - - -@pytest.mark.parametrize("model_type", ["deepseek_v3", "deepseek_v32"]) -def test_other_deepseek_keeps_the_original_mtp(model_type: str) -> None: - """Only the DSA-sparse family gets the new draft; V3 must not move.""" - assert _mtp_arch(model_type) == ["DeepSeekMTPModel"] - - -@pytest.mark.parametrize( - "architectures,expected", - [ - (["DeepSeekMTPModel"], True), - (["DeepseekV32MTPModel"], True), - (["Glm4MoeMTPModel"], False), - ([], False), - ], -) -def test_tuple_return_contract_covers_the_new_architecture( - architectures: list[str], expected: bool -) -> None: - """The DSA MTP recycles the post-norm hidden, so it returns a 2-tuple. - - Miss this and the proposer feeds a tuple where a tensor is expected. - """ - proposer = SimpleNamespace( - method="mtp", - draft_model_config=SimpleNamespace( - hf_config=SimpleNamespace(architectures=architectures) - ), - ) - - assert SpecDecodeBaseProposer.model_returns_tuple(proposer) is expected diff --git a/vllm/models/deepseek_v32/__init__.py b/vllm/models/deepseek_v32/__init__.py index 3a2c8e6da6ba..c721dc9e1170 100644 --- a/vllm/models/deepseek_v32/__init__.py +++ b/vllm/models/deepseek_v32/__init__.py @@ -7,10 +7,9 @@ attend. The same model code serves any DSA checkpoint, including GLM-5.2 (``glm_moe_dsa``), which reuses this architecture. -The kernels under ``nvidia/`` target the Blackwell (SM100) family. Pre-SM100 -CUDA (e.g. H100) falls back to the generic ``deepseek_v2`` implementation, which -already handles the DSA (index_topk) architecture and is ``torch.compile`` --friendly there, so those devices keep working rather than hard-failing. +The CUDA implementation selects capability-specific kernels internally and +falls back when an optimization is unavailable. GLM-5.2 uses the non-compiled +MRV2 path, so the same implementation can serve all NVIDIA GPUs. """ from vllm.platforms import current_platform @@ -22,16 +21,14 @@ from .amd.model import DeepseekV32ForCausalLM from .amd.mtp import DeepseekV32MTP -elif current_platform.is_device_capability_family(100): - # GLM-5.2 (glm_moe_dsa) reuses the same optimized DSA module on SM100. +elif current_platform.is_cuda(): + # GLM-5.2 (glm_moe_dsa) reuses the CUDA DSA module. Individual optimized + # kernels remain gated on the device capabilities they support. from .nvidia.model import DeepseekV32ForCausalLM from .nvidia.model import DeepseekV32ForCausalLM as GlmMoeDsaForCausalLM from .nvidia.mtp import DeepseekV32MTP else: - # Pre-SM100 CUDA, XPU and CPU. The generic implementation already handles - # the DSA architecture, so these keep serving instead of hard-failing -- - # which is what GLM-5.2 does on main, where its registry entry points - # straight at deepseek_v2 and never reaches this package. + # XPU and CPU keep the generic implementation. from vllm.model_executor.models.deepseek_mtp import DeepSeekMTP as DeepseekV32MTP from vllm.model_executor.models.deepseek_v2 import ( DeepseekV3ForCausalLM as DeepseekV32ForCausalLM, From 098c61b21674469621afaadd8abb570070222886 Mon Sep 17 00:00:00 2001 From: Woosuk Kwon Date: Wed, 19 Aug 2026 03:41:18 +0000 Subject: [PATCH 06/15] [Model][NVIDIA] Route DeepSeek V3.2 through the CUDA DSA path Use the deepseek_v32 main and MTP implementations for DeepSeek V3.2, including NVIDIA NVFP4 checkpoints. Default both architectures to MRV2 with breakable CUDA graphs and remove the obsolete MTP eager override. Co-authored-by: OpenAI Codex Signed-off-by: Woosuk Kwon --- tests/models/registry.py | 5 ++- tests/test_config.py | 45 ++++++++++++++++++++++---- vllm/config/speculative.py | 8 ++--- vllm/config/vllm.py | 2 ++ vllm/model_executor/models/registry.py | 5 ++- 5 files changed, 49 insertions(+), 16 deletions(-) diff --git a/tests/models/registry.py b/tests/models/registry.py index 3f3a90d011a8..f8901beac316 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -1674,9 +1674,8 @@ def check_available_online( trust_remote_code=True, ), "DeepseekV32MTPModel": _HfExamplesInfo( - "zai-org/GLM-5", - speculative_model="zai-org/GLM-5", - min_transformers_version="5.0.1", + "nvidia/DeepSeek-V3.2-NVFP4", + speculative_model="nvidia/DeepSeek-V3.2-NVFP4", is_available_online=False, ), "DeepSeekV4MTPModel": _HfExamplesInfo( diff --git a/tests/test_config.py b/tests/test_config.py index bf7e1fae66a9..5b88ba80357f 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -128,11 +128,17 @@ def test_rocm_defaults_deepseek_v4_to_mrv1(monkeypatch): @pytest.mark.parametrize( - "model", - ["nvidia/GLM-5.2-NVFP4", "zai-org/GLM-5.2-FP8"], + ("model", "architecture"), + [ + ("nvidia/GLM-5.2-NVFP4", "GlmMoeDsaForCausalLM"), + ("zai-org/GLM-5.2-FP8", "GlmMoeDsaForCausalLM"), + ("nvidia/DeepSeek-V3.2-NVFP4", "DeepseekV32ForCausalLM"), + ], ) @pytest.mark.parametrize("with_mtp", [False, True], ids=["no-mtp", "mtp"]) -def test_glm52_defaults_to_mrv2_and_breakable_cudagraph(monkeypatch, model, with_mtp): +def test_dsa_models_default_to_mrv2_and_breakable_cudagraph( + monkeypatch, model, architecture, with_mtp +): from vllm.compilation.breakable_cudagraph import ( is_breakable_cudagraph_enabled, ) @@ -145,7 +151,7 @@ def test_glm52_defaults_to_mrv2_and_breakable_cudagraph(monkeypatch, model, with model_config = SimpleNamespace( model=model, - architectures=["GlmMoeDsaForCausalLM"], + architectures=[architecture], runner_type="generate", is_moe=True, is_hybrid=False, @@ -181,9 +187,14 @@ def test_glm52_defaults_to_mrv2_and_breakable_cudagraph(monkeypatch, model, with @pytest.mark.parametrize( - "architecture", ["GlmMoeDsaForCausalLM", "DeepseekV32MTPModel"] + "architecture", + [ + "DeepseekV32ForCausalLM", + "DeepseekV32MTPModel", + "GlmMoeDsaForCausalLM", + ], ) -def test_glm52_breakable_cudagraph_default_is_platform_independent( +def test_dsa_breakable_cudagraph_default_is_platform_independent( monkeypatch, architecture ): from vllm.platforms import current_platform @@ -209,6 +220,28 @@ def test_glm52_breakable_cudagraph_default_is_platform_independent( os.environ.pop("VLLM_USE_BREAKABLE_CUDAGRAPH", None) +@pytest.mark.parametrize( + ("model_type", "expected_architecture"), + [ + ("deepseek_v32", "DeepseekV32MTPModel"), + ("glm_moe_dsa", "DeepseekV32MTPModel"), + ("deepseek_v3", "DeepSeekMTPModel"), + ], +) +def test_dsa_models_select_matching_mtp(model_type, expected_architecture): + from transformers import PretrainedConfig + + hf_config = PretrainedConfig( + architectures=["DeepseekV32ForCausalLM"], + num_nextn_predict_layers=1, + ) + hf_config.model_type = model_type + + SpeculativeConfig.hf_config_override(hf_config) + + assert hf_config.architectures == [expected_architecture] + + @pytest.mark.parametrize( ("use_v2_model_runner", "expected_capture_sizes"), [ diff --git a/vllm/config/speculative.py b/vllm/config/speculative.py index c334fde44600..6cdba55263c3 100644 --- a/vllm/config/speculative.py +++ b/vllm/config/speculative.py @@ -341,7 +341,7 @@ def compute_hash(self) -> str: @staticmethod def hf_config_override(hf_config: PretrainedConfig) -> PretrainedConfig: initial_architecture = hf_config.architectures[0] - use_sparse_mtp = hf_config.model_type == "glm_moe_dsa" + use_v32_mtp = hf_config.model_type in ("deepseek_v32", "glm_moe_dsa") if hf_config.model_type == "dots3_note": n_predict = getattr(hf_config, "num_nextn_predict_layers", 1) mtp_layer_types = getattr(hf_config, "mtp_layer_types", None) @@ -369,7 +369,7 @@ def hf_config_override(hf_config: PretrainedConfig) -> PretrainedConfig: { "n_predict": n_predict, "architectures": [ - "DeepseekV32MTPModel" if use_sparse_mtp else "DeepSeekMTPModel" + "DeepseekV32MTPModel" if use_v32_mtp else "DeepSeekMTPModel" ], } ) @@ -765,10 +765,6 @@ def __post_init__(self): if self.method == "mtp": if self.target_model_config is None: raise ValueError("target_model_config must be present for mtp") - if self.target_model_config.hf_text_config.model_type == "deepseek_v32": - # FIXME(luccafong): cudagraph with v32 MTP is not supported, - # remove this when the issue is fixed. - self.enforce_eager = True # use the draft model from the same model: self.model = self.target_model_config.model # Align the quantization of draft model for cases such as diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index 8c12a0199a94..4e3495929634 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -69,6 +69,7 @@ DEFAULT_V2_MODEL_RUNNER_ARCHITECTURES = frozenset( { "DeepseekV2ForCausalLM", + "DeepseekV32ForCausalLM", "DeepseekV4ForCausalLM", "GlmMoeDsaForCausalLM", "GraniteMoeForCausalLM", @@ -83,6 +84,7 @@ DEFAULT_BREAKABLE_CUDAGRAPH_ARCHITECTURES = frozenset( { "DeepseekV32MTPModel", + "DeepseekV32ForCausalLM", "DeepseekV4ForCausalLM", "DeepSeekV4MTPModel", "GlmMoeDsaForCausalLM", diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index 63495d397864..00237f0b836c 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -92,7 +92,10 @@ "DeepseekForCausalLM": ("deepseek_v2", "DeepseekForCausalLM"), "DeepseekV2ForCausalLM": ("deepseek_v2", "DeepseekV2ForCausalLM"), "DeepseekV3ForCausalLM": ("deepseek_v2", "DeepseekV3ForCausalLM"), - "DeepseekV32ForCausalLM": ("deepseek_v2", "DeepseekV3ForCausalLM"), + "DeepseekV32ForCausalLM": ( + "vllm.models.deepseek_v32", + "DeepseekV32ForCausalLM", + ), "DeepseekV4ForCausalLM": ("vllm.models.deepseek_v4", "DeepseekV4ForCausalLM"), "Ernie4_5ForCausalLM": ("ernie45", "Ernie4_5ForCausalLM"), "Ernie4_5_MoeForCausalLM": ("ernie45_moe", "Ernie4_5_MoeForCausalLM"), From ef1627515d4c6ac94e9e2e2b77e5a7fab2c227b9 Mon Sep 17 00:00:00 2001 From: Woosuk Kwon Date: Wed, 19 Aug 2026 03:42:52 +0000 Subject: [PATCH 07/15] [Model][NVIDIA] Keep the V3.2 default route CUDA-only Resolve the registry through CUDA-specific aliases so NVIDIA uses the deepseek_v32 implementation while ROCm, XPU, and CPU retain their existing defaults. Keep the explicit AMD package exports available for opt-in use. Co-authored-by: OpenAI Codex Signed-off-by: Woosuk Kwon --- vllm/model_executor/models/registry.py | 4 ++-- vllm/models/deepseek_v32/__init__.py | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index 00237f0b836c..cbec957eb2c2 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -94,7 +94,7 @@ "DeepseekV3ForCausalLM": ("deepseek_v2", "DeepseekV3ForCausalLM"), "DeepseekV32ForCausalLM": ( "vllm.models.deepseek_v32", - "DeepseekV32ForCausalLM", + "CudaDeepseekV32ForCausalLM", ), "DeepseekV4ForCausalLM": ("vllm.models.deepseek_v4", "DeepseekV4ForCausalLM"), "Ernie4_5ForCausalLM": ("ernie45", "Ernie4_5ForCausalLM"), @@ -662,7 +662,7 @@ "Eagle3DeepseekV3ForCausalLM": ("deepseek_eagle3", "Eagle3DeepseekV2ForCausalLM"), "EagleDeepSeekMTPModel": ("deepseek_eagle", "EagleDeepseekV3ForCausalLM"), "DeepSeekMTPModel": ("deepseek_mtp", "DeepSeekMTP"), - "DeepseekV32MTPModel": ("vllm.models.deepseek_v32", "DeepseekV32MTP"), + "DeepseekV32MTPModel": ("vllm.models.deepseek_v32", "CudaDeepseekV32MTP"), "Dots3NoteMTPModel": ("vllm.models.dots3_note", "Dots3NoteMTP"), "DeepSeekV4MTPModel": ("vllm.models.deepseek_v4", "DeepSeekV4MTP"), "BailingMoeV3MTPModel": ("bailing_moe_v3_mtp", "BailingMoeV3MTPModel"), diff --git a/vllm/models/deepseek_v32/__init__.py b/vllm/models/deepseek_v32/__init__.py index c721dc9e1170..b977e857b888 100644 --- a/vllm/models/deepseek_v32/__init__.py +++ b/vllm/models/deepseek_v32/__init__.py @@ -17,6 +17,12 @@ if current_platform.is_rocm(): # GLM-5.2 keeps the generic implementation here, as it has on main; only # DeepSeek V3.2 has an AMD DSA port. + from vllm.model_executor.models.deepseek_mtp import ( + DeepSeekMTP as CudaDeepseekV32MTP, + ) + from vllm.model_executor.models.deepseek_v2 import ( + DeepseekV3ForCausalLM as CudaDeepseekV32ForCausalLM, + ) from vllm.model_executor.models.deepseek_v2 import GlmMoeDsaForCausalLM from .amd.model import DeepseekV32ForCausalLM @@ -25,17 +31,27 @@ # GLM-5.2 (glm_moe_dsa) reuses the CUDA DSA module. Individual optimized # kernels remain gated on the device capabilities they support. from .nvidia.model import DeepseekV32ForCausalLM + from .nvidia.model import DeepseekV32ForCausalLM as CudaDeepseekV32ForCausalLM from .nvidia.model import DeepseekV32ForCausalLM as GlmMoeDsaForCausalLM from .nvidia.mtp import DeepseekV32MTP + from .nvidia.mtp import DeepseekV32MTP as CudaDeepseekV32MTP else: # XPU and CPU keep the generic implementation. + from vllm.model_executor.models.deepseek_mtp import ( + DeepSeekMTP as CudaDeepseekV32MTP, + ) from vllm.model_executor.models.deepseek_mtp import DeepSeekMTP as DeepseekV32MTP + from vllm.model_executor.models.deepseek_v2 import ( + DeepseekV3ForCausalLM as CudaDeepseekV32ForCausalLM, + ) from vllm.model_executor.models.deepseek_v2 import ( DeepseekV3ForCausalLM as DeepseekV32ForCausalLM, ) from vllm.model_executor.models.deepseek_v2 import GlmMoeDsaForCausalLM __all__ = [ + "CudaDeepseekV32ForCausalLM", + "CudaDeepseekV32MTP", "DeepseekV32ForCausalLM", "DeepseekV32MTP", "GlmMoeDsaForCausalLM", From 5acdd90ac46b0b7146a3556852d34d1049fea1d1 Mon Sep 17 00:00:00 2001 From: Woosuk Kwon Date: Wed, 19 Aug 2026 03:53:42 +0000 Subject: [PATCH 08/15] [Model][NVIDIA] Dispatch V3.2 with canonical class names Move the CUDA-or-generic registry dispatch into a separate module that exports DeepseekV32ForCausalLM and DeepseekV32MTP without prefixed aliases. Keep ROCm on the generic compiled MRV1 path without automatic breakable graphs while preserving the opt-in AMD package exports. Co-authored-by: OpenAI Codex Signed-off-by: Woosuk Kwon --- tests/test_config.py | 48 +++++++++++++++++--------- vllm/config/vllm.py | 24 ++++++++++--- vllm/model_executor/models/registry.py | 6 ++-- vllm/models/deepseek_v32/__init__.py | 16 --------- vllm/models/deepseek_v32_cuda.py | 16 +++++++++ 5 files changed, 69 insertions(+), 41 deletions(-) create mode 100644 vllm/models/deepseek_v32_cuda.py diff --git a/tests/test_config.py b/tests/test_config.py index 5b88ba80357f..23aa64435be2 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -113,18 +113,29 @@ def test_v2_model_runner_env_tri_state(monkeypatch, env_value, expected): assert envs.VLLM_USE_V2_MODEL_RUNNER is expected -def test_rocm_defaults_deepseek_v4_to_mrv1(monkeypatch): - """ROCm keeps DeepSeek V4 on MRV1, which is still faster there.""" - from vllm.config.vllm import default_v2_model_runner_architectures +def test_rocm_keeps_compiled_deepseek_defaults(monkeypatch): + """ROCm keeps DeepSeek V3.2 and V4 on their compiled MRV1 paths.""" + from vllm.config.vllm import ( + default_breakable_cudagraph_architectures, + default_v2_model_runner_architectures, + ) from vllm.platforms import current_platform monkeypatch.setattr(current_platform, "is_rocm", lambda: True) # The lookup is lru_cached against a fixed platform. default_v2_model_runner_architectures.cache_clear() + default_breakable_cudagraph_architectures.cache_clear() try: - assert "DeepseekV4ForCausalLM" not in default_v2_model_runner_architectures() + v2_architectures = default_v2_model_runner_architectures() + breakable_architectures = default_breakable_cudagraph_architectures() + + assert "DeepseekV32ForCausalLM" not in v2_architectures + assert "DeepseekV4ForCausalLM" not in v2_architectures + assert "DeepseekV32ForCausalLM" not in breakable_architectures + assert "DeepseekV32MTPModel" not in breakable_architectures finally: default_v2_model_runner_architectures.cache_clear() + default_breakable_cudagraph_architectures.cache_clear() @pytest.mark.parametrize( @@ -187,24 +198,25 @@ def test_dsa_models_default_to_mrv2_and_breakable_cudagraph( @pytest.mark.parametrize( - "architecture", + ("architecture", "is_rocm", "expected"), [ - "DeepseekV32ForCausalLM", - "DeepseekV32MTPModel", - "GlmMoeDsaForCausalLM", + ("DeepseekV32ForCausalLM", False, True), + ("DeepseekV32ForCausalLM", True, False), + ("DeepseekV32MTPModel", False, True), + ("DeepseekV32MTPModel", True, False), + ("GlmMoeDsaForCausalLM", False, True), + ("GlmMoeDsaForCausalLM", True, True), ], ) -def test_dsa_breakable_cudagraph_default_is_platform_independent( - monkeypatch, architecture +def test_dsa_breakable_cudagraph_platform_default( + monkeypatch, architecture, is_rocm, expected ): + from vllm.config.vllm import default_breakable_cudagraph_architectures from vllm.platforms import current_platform monkeypatch.delenv("VLLM_USE_BREAKABLE_CUDAGRAPH", raising=False) - monkeypatch.setattr( - current_platform, - "is_device_capability_family", - lambda family: False, - ) + monkeypatch.setattr(current_platform, "is_rocm", lambda: is_rocm) + default_breakable_cudagraph_architectures.cache_clear() config = SimpleNamespace( model_config=SimpleNamespace(architectures=[architecture]), compilation_config=CompilationConfig(), @@ -214,10 +226,12 @@ def test_dsa_breakable_cudagraph_default_is_platform_independent( ) try: - assert VllmConfig._maybe_enable_breakable_cudagraph(config) - assert config.compilation_config.mode == CompilationMode.NONE + assert VllmConfig._maybe_enable_breakable_cudagraph(config) is expected + if expected: + assert config.compilation_config.mode == CompilationMode.NONE finally: os.environ.pop("VLLM_USE_BREAKABLE_CUDAGRAPH", None) + default_breakable_cudagraph_architectures.cache_clear() @pytest.mark.parametrize( diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index 4e3495929634..1e69c0d71805 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -105,13 +105,27 @@ def default_v2_model_runner_architectures() -> frozenset[str]: from vllm.platforms import current_platform if current_platform.is_rocm(): - # TODO(rocm): DeepSeek V4 is still faster on MRV1 on ROCm. The - # attention layer picks the eager cudagraph region MRV1 needs, so - # this is a perf default only; drop it once MRV2 catches up. - return DEFAULT_V2_MODEL_RUNNER_ARCHITECTURES - {"DeepseekV4ForCausalLM"} + # TODO(rocm): These models are still faster on the compiled MRV1 path. + return DEFAULT_V2_MODEL_RUNNER_ARCHITECTURES - { + "DeepseekV32ForCausalLM", + "DeepseekV4ForCausalLM", + } return DEFAULT_V2_MODEL_RUNNER_ARCHITECTURES +@lru_cache +def default_breakable_cudagraph_architectures() -> frozenset[str]: + """Architectures defaulting to breakable CUDA graphs on this platform.""" + from vllm.platforms import current_platform + + if current_platform.is_rocm(): + return DEFAULT_BREAKABLE_CUDAGRAPH_ARCHITECTURES - { + "DeepseekV32ForCausalLM", + "DeepseekV32MTPModel", + } + return DEFAULT_BREAKABLE_CUDAGRAPH_ARCHITECTURES + + class OptimizationLevel(IntEnum): """Optimization level enum.""" @@ -719,7 +733,7 @@ def _uses_breakable_cudagraph_by_default(self) -> bool: return False architectures = set(model_config.architectures) - return bool(architectures & DEFAULT_BREAKABLE_CUDAGRAPH_ARCHITECTURES) + return bool(architectures & default_breakable_cudagraph_architectures()) def _maybe_enable_breakable_cudagraph(self) -> bool: if ( diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index cbec957eb2c2..efbdfddae8cf 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -93,8 +93,8 @@ "DeepseekV2ForCausalLM": ("deepseek_v2", "DeepseekV2ForCausalLM"), "DeepseekV3ForCausalLM": ("deepseek_v2", "DeepseekV3ForCausalLM"), "DeepseekV32ForCausalLM": ( - "vllm.models.deepseek_v32", - "CudaDeepseekV32ForCausalLM", + "vllm.models.deepseek_v32_cuda", + "DeepseekV32ForCausalLM", ), "DeepseekV4ForCausalLM": ("vllm.models.deepseek_v4", "DeepseekV4ForCausalLM"), "Ernie4_5ForCausalLM": ("ernie45", "Ernie4_5ForCausalLM"), @@ -662,7 +662,7 @@ "Eagle3DeepseekV3ForCausalLM": ("deepseek_eagle3", "Eagle3DeepseekV2ForCausalLM"), "EagleDeepSeekMTPModel": ("deepseek_eagle", "EagleDeepseekV3ForCausalLM"), "DeepSeekMTPModel": ("deepseek_mtp", "DeepSeekMTP"), - "DeepseekV32MTPModel": ("vllm.models.deepseek_v32", "CudaDeepseekV32MTP"), + "DeepseekV32MTPModel": ("vllm.models.deepseek_v32_cuda", "DeepseekV32MTP"), "Dots3NoteMTPModel": ("vllm.models.dots3_note", "Dots3NoteMTP"), "DeepSeekV4MTPModel": ("vllm.models.deepseek_v4", "DeepSeekV4MTP"), "BailingMoeV3MTPModel": ("bailing_moe_v3_mtp", "BailingMoeV3MTPModel"), diff --git a/vllm/models/deepseek_v32/__init__.py b/vllm/models/deepseek_v32/__init__.py index b977e857b888..c721dc9e1170 100644 --- a/vllm/models/deepseek_v32/__init__.py +++ b/vllm/models/deepseek_v32/__init__.py @@ -17,12 +17,6 @@ if current_platform.is_rocm(): # GLM-5.2 keeps the generic implementation here, as it has on main; only # DeepSeek V3.2 has an AMD DSA port. - from vllm.model_executor.models.deepseek_mtp import ( - DeepSeekMTP as CudaDeepseekV32MTP, - ) - from vllm.model_executor.models.deepseek_v2 import ( - DeepseekV3ForCausalLM as CudaDeepseekV32ForCausalLM, - ) from vllm.model_executor.models.deepseek_v2 import GlmMoeDsaForCausalLM from .amd.model import DeepseekV32ForCausalLM @@ -31,27 +25,17 @@ # GLM-5.2 (glm_moe_dsa) reuses the CUDA DSA module. Individual optimized # kernels remain gated on the device capabilities they support. from .nvidia.model import DeepseekV32ForCausalLM - from .nvidia.model import DeepseekV32ForCausalLM as CudaDeepseekV32ForCausalLM from .nvidia.model import DeepseekV32ForCausalLM as GlmMoeDsaForCausalLM from .nvidia.mtp import DeepseekV32MTP - from .nvidia.mtp import DeepseekV32MTP as CudaDeepseekV32MTP else: # XPU and CPU keep the generic implementation. - from vllm.model_executor.models.deepseek_mtp import ( - DeepSeekMTP as CudaDeepseekV32MTP, - ) from vllm.model_executor.models.deepseek_mtp import DeepSeekMTP as DeepseekV32MTP - from vllm.model_executor.models.deepseek_v2 import ( - DeepseekV3ForCausalLM as CudaDeepseekV32ForCausalLM, - ) from vllm.model_executor.models.deepseek_v2 import ( DeepseekV3ForCausalLM as DeepseekV32ForCausalLM, ) from vllm.model_executor.models.deepseek_v2 import GlmMoeDsaForCausalLM __all__ = [ - "CudaDeepseekV32ForCausalLM", - "CudaDeepseekV32MTP", "DeepseekV32ForCausalLM", "DeepseekV32MTP", "GlmMoeDsaForCausalLM", diff --git a/vllm/models/deepseek_v32_cuda.py b/vllm/models/deepseek_v32_cuda.py new file mode 100644 index 000000000000..3b9029ca564c --- /dev/null +++ b/vllm/models/deepseek_v32_cuda.py @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""CUDA DSA models with generic fallbacks on other platforms.""" + +from vllm.platforms import current_platform + +if current_platform.is_cuda(): + from vllm.models.deepseek_v32.nvidia.model import DeepseekV32ForCausalLM + from vllm.models.deepseek_v32.nvidia.mtp import DeepseekV32MTP +else: + from vllm.model_executor.models.deepseek_mtp import DeepSeekMTP as DeepseekV32MTP + from vllm.model_executor.models.deepseek_v2 import ( + DeepseekV3ForCausalLM as DeepseekV32ForCausalLM, + ) + +__all__ = ["DeepseekV32ForCausalLM", "DeepseekV32MTP"] From 68f87f8ef21df30810d708a6d14782f3fd8fb2b2 Mon Sep 17 00:00:00 2001 From: Woosuk Kwon Date: Wed, 19 Aug 2026 03:56:22 +0000 Subject: [PATCH 09/15] Clarify ROCm MRV2 model exclusions Co-authored-by: OpenAI Codex Signed-off-by: Woosuk Kwon --- vllm/config/vllm.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index 1e69c0d71805..36fb08ba4208 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -105,7 +105,8 @@ def default_v2_model_runner_architectures() -> frozenset[str]: from vllm.platforms import current_platform if current_platform.is_rocm(): - # TODO(rocm): These models are still faster on the compiled MRV1 path. + # TODO(rocm): These models are either unsupported by MRV2 or slower with + # MRV2 on AMD GPUs. return DEFAULT_V2_MODEL_RUNNER_ARCHITECTURES - { "DeepseekV32ForCausalLM", "DeepseekV4ForCausalLM", From 6c909f1b45e7f77a770f023f719498c05c390bda Mon Sep 17 00:00:00 2001 From: Woosuk Kwon Date: Wed, 19 Aug 2026 04:01:41 +0000 Subject: [PATCH 10/15] Simplify DeepSeek V3.2 platform routing Co-authored-by: OpenAI Codex Signed-off-by: Woosuk Kwon --- vllm/model_executor/models/registry.py | 4 ++-- vllm/models/deepseek_v32/__init__.py | 17 +++++------------ vllm/models/deepseek_v32_cuda.py | 16 ---------------- 3 files changed, 7 insertions(+), 30 deletions(-) delete mode 100644 vllm/models/deepseek_v32_cuda.py diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index efbdfddae8cf..00237f0b836c 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -93,7 +93,7 @@ "DeepseekV2ForCausalLM": ("deepseek_v2", "DeepseekV2ForCausalLM"), "DeepseekV3ForCausalLM": ("deepseek_v2", "DeepseekV3ForCausalLM"), "DeepseekV32ForCausalLM": ( - "vllm.models.deepseek_v32_cuda", + "vllm.models.deepseek_v32", "DeepseekV32ForCausalLM", ), "DeepseekV4ForCausalLM": ("vllm.models.deepseek_v4", "DeepseekV4ForCausalLM"), @@ -662,7 +662,7 @@ "Eagle3DeepseekV3ForCausalLM": ("deepseek_eagle3", "Eagle3DeepseekV2ForCausalLM"), "EagleDeepSeekMTPModel": ("deepseek_eagle", "EagleDeepseekV3ForCausalLM"), "DeepSeekMTPModel": ("deepseek_mtp", "DeepSeekMTP"), - "DeepseekV32MTPModel": ("vllm.models.deepseek_v32_cuda", "DeepseekV32MTP"), + "DeepseekV32MTPModel": ("vllm.models.deepseek_v32", "DeepseekV32MTP"), "Dots3NoteMTPModel": ("vllm.models.dots3_note", "Dots3NoteMTP"), "DeepSeekV4MTPModel": ("vllm.models.deepseek_v4", "DeepSeekV4MTP"), "BailingMoeV3MTPModel": ("bailing_moe_v3_mtp", "BailingMoeV3MTPModel"), diff --git a/vllm/models/deepseek_v32/__init__.py b/vllm/models/deepseek_v32/__init__.py index c721dc9e1170..0ac154505f2e 100644 --- a/vllm/models/deepseek_v32/__init__.py +++ b/vllm/models/deepseek_v32/__init__.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""DeepSeek V3.2 (``deepseek_v32``) model — hardware-isolated entry point. +"""DeepSeek V3.2 (``deepseek_v32``) platform entry point. DeepSeek V3.2 introduced the DeepSeek Sparse Attention (DSA) architecture: MLA + a "lightning indexer" that selects the top-k tokens for a sparse MLA @@ -8,27 +8,20 @@ (``glm_moe_dsa``), which reuses this architecture. The CUDA implementation selects capability-specific kernels internally and -falls back when an optimization is unavailable. GLM-5.2 uses the non-compiled -MRV2 path, so the same implementation can serve all NVIDIA GPUs. +falls back when an optimization is unavailable. Other platforms use the +generic implementation by default. """ from vllm.platforms import current_platform -if current_platform.is_rocm(): - # GLM-5.2 keeps the generic implementation here, as it has on main; only - # DeepSeek V3.2 has an AMD DSA port. - from vllm.model_executor.models.deepseek_v2 import GlmMoeDsaForCausalLM - - from .amd.model import DeepseekV32ForCausalLM - from .amd.mtp import DeepseekV32MTP -elif current_platform.is_cuda(): +if current_platform.is_cuda(): # GLM-5.2 (glm_moe_dsa) reuses the CUDA DSA module. Individual optimized # kernels remain gated on the device capabilities they support. from .nvidia.model import DeepseekV32ForCausalLM from .nvidia.model import DeepseekV32ForCausalLM as GlmMoeDsaForCausalLM from .nvidia.mtp import DeepseekV32MTP else: - # XPU and CPU keep the generic implementation. + # ROCm, XPU, and CPU keep the generic implementation. from vllm.model_executor.models.deepseek_mtp import DeepSeekMTP as DeepseekV32MTP from vllm.model_executor.models.deepseek_v2 import ( DeepseekV3ForCausalLM as DeepseekV32ForCausalLM, diff --git a/vllm/models/deepseek_v32_cuda.py b/vllm/models/deepseek_v32_cuda.py deleted file mode 100644 index 3b9029ca564c..000000000000 --- a/vllm/models/deepseek_v32_cuda.py +++ /dev/null @@ -1,16 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""CUDA DSA models with generic fallbacks on other platforms.""" - -from vllm.platforms import current_platform - -if current_platform.is_cuda(): - from vllm.models.deepseek_v32.nvidia.model import DeepseekV32ForCausalLM - from vllm.models.deepseek_v32.nvidia.mtp import DeepseekV32MTP -else: - from vllm.model_executor.models.deepseek_mtp import DeepSeekMTP as DeepseekV32MTP - from vllm.model_executor.models.deepseek_v2 import ( - DeepseekV3ForCausalLM as DeepseekV32ForCausalLM, - ) - -__all__ = ["DeepseekV32ForCausalLM", "DeepseekV32MTP"] From 296a500d206c59df47f7954e380493d0d5cab094 Mon Sep 17 00:00:00 2001 From: Woosuk Kwon Date: Wed, 19 Aug 2026 04:07:10 +0000 Subject: [PATCH 11/15] Simplify DSA query and cache form selection Co-authored-by: OpenAI Codex Signed-off-by: Woosuk Kwon --- vllm/models/deepseek_v32/attention.py | 44 ++++----------------------- 1 file changed, 6 insertions(+), 38 deletions(-) diff --git a/vllm/models/deepseek_v32/attention.py b/vllm/models/deepseek_v32/attention.py index 8f8381bb7c08..38122cd8bde2 100644 --- a/vllm/models/deepseek_v32/attention.py +++ b/vllm/models/deepseek_v32/attention.py @@ -47,41 +47,6 @@ from vllm.model_executor.layers.attention.mla_attention import MLACommonMetadata -# Per-tensor fp8 layouts this attention reads as float8_e4m3fn. Which cache -# dtypes are reachable at all is the sparse MLA backend's call -# (``supported_kv_cache_dtypes``); this only says how to address the ones that -# get here, so an e5m2 or nvfp4 cache raises instead of being read as e4m3. -_E4M3_KV_LAYOUTS = ("fp8", "fp8_e4m3") - - -def select_query_and_cache_form( - kv_cache_dtype: str, backend_supports_quant_query: bool -) -> tuple[bool, bool]: - """``(pack the MQA query as fp8, view the paged cache as fp8)``. - - An fp8 query only pairs with a per-tensor fp8 cache. Everything else takes - the bf16 ``(ql_nope, q_pe)`` query tuple that the fp8_ds_mla layout already - uses -- including an unquantized cache, so no dtype is forced on the user. - The fused kernels already emit both forms. - """ - if not is_quantized_kv_cache(kv_cache_dtype): - return False, False - if kv_cache_dtype == "fp8_ds_mla": - # Block-scaled fp8 NoPE + bf16 RoPE, addressed as raw bytes and - # dequantized inside FlashMLA. - return False, False - assert kv_cache_dtype in _E4M3_KV_LAYOUTS, ( - f"deepseek_v32 cannot address a {kv_cache_dtype} KV cache. " - f"Supported: unquantized, {', '.join(_E4M3_KV_LAYOUTS)}, fp8_ds_mla." - ) - assert backend_supports_quant_query, ( - "deepseek_v32 on a bf16-query sparse MLA backend (FlashMLA sparse) " - "requires the fp8_ds_mla KV cache layout. " - "Launch with --kv-cache-dtype fp8_ds_mla." - ) - return True, True - - class DeepseekV32Indexer(nn.Module): indexer_cache_cls = DeepseekV32IndexerCache @@ -318,9 +283,12 @@ def __init__( ) if self.require_fp8_kv_cache: - self._fp8_query, self._fp8_kv_needs_view = select_query_and_cache_form( - self.kv_cache_dtype, self.impl.supports_quant_query_input + fp8_attention = ( + is_quantized_kv_cache(self.kv_cache_dtype) + and self.kv_cache_dtype != "fp8_ds_mla" ) + self._fp8_query = fp8_attention and self.impl.supports_quant_query_input + self._fp8_kv_needs_view = fp8_attention self._index_rope_interleave = getattr(config, "indexer_rope_interleave", False) @@ -586,7 +554,7 @@ def _sparse_indexer_and_attn( return if self._fp8_kv_needs_view: - kv_cache = kv_cache.view(torch.float8_e4m3fn) + kv_cache = kv_cache.view(current_platform.fp8_dtype()) if self._fp8_query: # FlashInfer sparse: single packed fp8 query. mqa_q_arg: torch.Tensor | tuple[torch.Tensor, torch.Tensor] = mqa_q[ From 5a10d32f530a16d4bb38a8ed60b75a08b252824d Mon Sep 17 00:00:00 2001 From: Woosuk Kwon Date: Wed, 19 Aug 2026 04:12:12 +0000 Subject: [PATCH 12/15] Keep existing DSA FP8 cache requirement Co-authored-by: OpenAI Codex Signed-off-by: Woosuk Kwon --- vllm/models/deepseek_v32/attention.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/vllm/models/deepseek_v32/attention.py b/vllm/models/deepseek_v32/attention.py index 38122cd8bde2..253971b7bd05 100644 --- a/vllm/models/deepseek_v32/attention.py +++ b/vllm/models/deepseek_v32/attention.py @@ -183,6 +183,7 @@ def __init__( ) -> None: quant_config = vllm_config.quant_config cache_config = vllm_config.cache_config + hidden_size = config.hidden_size qk_nope_head_dim = config.qk_nope_head_dim qk_rope_head_dim = config.qk_rope_head_dim @@ -283,12 +284,20 @@ def __init__( ) if self.require_fp8_kv_cache: - fp8_attention = ( - is_quantized_kv_cache(self.kv_cache_dtype) - and self.kv_cache_dtype != "fp8_ds_mla" + assert is_quantized_kv_cache(self.kv_cache_dtype), ( + "deepseek_v32 (nvidia) requires an fp8 KV cache served by a sparse " + "MLA backend. Launch with --kv-cache-dtype fp8 (FlashInfer sparse) " + "or --kv-cache-dtype fp8_ds_mla (FlashMLA sparse)." ) - self._fp8_query = fp8_attention and self.impl.supports_quant_query_input - self._fp8_kv_needs_view = fp8_attention + self._fp8_query = self.impl.supports_quant_query_input + if not self._fp8_query: + assert self.kv_cache_dtype == "fp8_ds_mla", ( + "deepseek_v32 (nvidia) on a bf16-query sparse MLA backend " + "(FlashMLA sparse) requires the fp8_ds_mla KV cache layout. " + "Launch with --kv-cache-dtype fp8_ds_mla." + ) + + self._fp8_kv_needs_view = self.kv_cache_dtype != "fp8_ds_mla" self._index_rope_interleave = getattr(config, "indexer_rope_interleave", False) @@ -554,7 +563,7 @@ def _sparse_indexer_and_attn( return if self._fp8_kv_needs_view: - kv_cache = kv_cache.view(current_platform.fp8_dtype()) + kv_cache = kv_cache.view(torch.float8_e4m3fn) if self._fp8_query: # FlashInfer sparse: single packed fp8 query. mqa_q_arg: torch.Tensor | tuple[torch.Tensor, torch.Tensor] = mqa_q[ From 143e72c53405279da5844a63ae409bdac4399090 Mon Sep 17 00:00:00 2001 From: Woosuk Kwon Date: Wed, 19 Aug 2026 05:07:40 +0000 Subject: [PATCH 13/15] Fix DSA platform and KV cache CI coverage Co-authored-by: OpenAI Codex Signed-off-by: Woosuk Kwon --- .../test_fused_deepseek_v32_norm_rope.py | 7 +++---- tests/test_config.py | 9 ++++++++- vllm/models/deepseek_v32/amd/rocm.py | 1 - vllm/models/deepseek_v32/attention.py | 19 +++---------------- vllm/models/deepseek_v32/common/kernels.py | 3 ++- 5 files changed, 16 insertions(+), 23 deletions(-) diff --git a/tests/kernels/test_fused_deepseek_v32_norm_rope.py b/tests/kernels/test_fused_deepseek_v32_norm_rope.py index 7a00d0c3d644..d3d5f3b0dc75 100644 --- a/tests/kernels/test_fused_deepseek_v32_norm_rope.py +++ b/tests/kernels/test_fused_deepseek_v32_norm_rope.py @@ -146,8 +146,8 @@ def assert_fp8(got: torch.Tensor, ref: torch.Tensor, msg: str): @pytest.mark.parametrize("num_tokens", [1, 4, 17, 512, 4096]) @pytest.mark.parametrize("index_interleave", [True, False]) -@pytest.mark.parametrize("mla_fp8", [False, True]) -def test_fused_norm_rope(num_tokens: int, index_interleave: bool, mla_fp8: bool): +@pytest.mark.parametrize("mla_dtype", ["auto", "bfloat16", "fp8"]) +def test_fused_norm_rope(num_tokens: int, index_interleave: bool, mla_dtype: str): torch.manual_seed(0) dev = "cuda" max_pos = 8192 @@ -167,13 +167,12 @@ def test_fused_norm_rope(num_tokens: int, index_interleave: bool, mla_fp8: bool) bs = max_pos # single block covering all tokens mla_dim = KV_LORA + ROPE_DIM + mla_fp8 = mla_dtype == "fp8" if mla_fp8: mla_cache = torch.zeros(1, bs, mla_dim, device=dev, dtype=torch.uint8) - mla_dtype = "fp8" mla_k_scale = torch.tensor([0.3], device=dev, dtype=torch.float32) else: mla_cache = torch.zeros(1, bs, mla_dim, device=dev, dtype=torch.bfloat16) - mla_dtype = "auto" mla_k_scale = None idx_row = INDEX_HEAD_DIM + INDEX_HEAD_DIM // 128 * 4 # 132 idx_cache = torch.zeros(1, bs, idx_row, device=dev, dtype=torch.uint8) diff --git a/tests/test_config.py b/tests/test_config.py index 23aa64435be2..666a73b7c6fe 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -153,12 +153,18 @@ def test_dsa_models_default_to_mrv2_and_breakable_cudagraph( from vllm.compilation.breakable_cudagraph import ( is_breakable_cudagraph_enabled, ) - from vllm.config.vllm import default_v2_model_runner_architectures + from vllm.config.vllm import ( + default_breakable_cudagraph_architectures, + default_v2_model_runner_architectures, + ) + from vllm.platforms import current_platform monkeypatch.delenv("VLLM_USE_BREAKABLE_CUDAGRAPH", raising=False) monkeypatch.delenv("VLLM_USE_V2_MODEL_RUNNER", raising=False) monkeypatch.setattr(vllm_config_module, "HAS_TRITON", True) + monkeypatch.setattr(current_platform, "is_rocm", lambda: False) default_v2_model_runner_architectures.cache_clear() + default_breakable_cudagraph_architectures.cache_clear() model_config = SimpleNamespace( model=model, @@ -195,6 +201,7 @@ def test_dsa_models_default_to_mrv2_and_breakable_cudagraph( finally: os.environ.pop("VLLM_USE_BREAKABLE_CUDAGRAPH", None) default_v2_model_runner_architectures.cache_clear() + default_breakable_cudagraph_architectures.cache_clear() @pytest.mark.parametrize( diff --git a/vllm/models/deepseek_v32/amd/rocm.py b/vllm/models/deepseek_v32/amd/rocm.py index 0297fc353863..6d8e7ea855b8 100644 --- a/vllm/models/deepseek_v32/amd/rocm.py +++ b/vllm/models/deepseek_v32/amd/rocm.py @@ -38,7 +38,6 @@ class DeepseekV32ROCmIndexer(DeepseekV32Indexer): class DeepseekV32MLAAttention(DeepseekV32Attention): - require_fp8_kv_cache: bool = False indexer_cls = DeepseekV32ROCmIndexer def __init__(self, vllm_config, config, prefix, topk_indices_buffer=None): diff --git a/vllm/models/deepseek_v32/attention.py b/vllm/models/deepseek_v32/attention.py index 253971b7bd05..02582810ca02 100644 --- a/vllm/models/deepseek_v32/attention.py +++ b/vllm/models/deepseek_v32/attention.py @@ -170,7 +170,6 @@ class DeepseekV32Attention(MLAAttention): indexer: "DeepseekV32Indexer | None" indexer_cls: "type[DeepseekV32Indexer]" = DeepseekV32Indexer - require_fp8_kv_cache: bool = True supports_dense_mha_prefill = False def __init__( @@ -283,21 +282,9 @@ def __init__( self.layer_name if enable_short_prefill_scoring_skip else "" ) - if self.require_fp8_kv_cache: - assert is_quantized_kv_cache(self.kv_cache_dtype), ( - "deepseek_v32 (nvidia) requires an fp8 KV cache served by a sparse " - "MLA backend. Launch with --kv-cache-dtype fp8 (FlashInfer sparse) " - "or --kv-cache-dtype fp8_ds_mla (FlashMLA sparse)." - ) - self._fp8_query = self.impl.supports_quant_query_input - if not self._fp8_query: - assert self.kv_cache_dtype == "fp8_ds_mla", ( - "deepseek_v32 (nvidia) on a bf16-query sparse MLA backend " - "(FlashMLA sparse) requires the fp8_ds_mla KV cache layout. " - "Launch with --kv-cache-dtype fp8_ds_mla." - ) - - self._fp8_kv_needs_view = self.kv_cache_dtype != "fp8_ds_mla" + fp8_attention = is_quantized_kv_cache(self.kv_cache_dtype) + self._fp8_query = fp8_attention and self.impl.supports_quant_query_input + self._fp8_kv_needs_view = fp8_attention and self.kv_cache_dtype != "fp8_ds_mla" self._index_rope_interleave = getattr(config, "indexer_rope_interleave", False) diff --git a/vllm/models/deepseek_v32/common/kernels.py b/vllm/models/deepseek_v32/common/kernels.py index c5c6c97d55a9..31315172482e 100644 --- a/vllm/models/deepseek_v32/common/kernels.py +++ b/vllm/models/deepseek_v32/common/kernels.py @@ -6,6 +6,7 @@ from vllm.platforms import current_platform from vllm.triton_utils import tl, triton +from vllm.utils.torch_utils import is_quantized_kv_cache # Cache of tiny 1-element dummy tensors (per device, dtype) reused by the # has_indexer=False path so the indexer args don't allocate every call. @@ -473,7 +474,7 @@ def fused_norm_rope( # --- MLA KV cache setup --- mla_cache_ds_mla = mla_kv_cache_dtype == "fp8_ds_mla" - mla_cache_fp8 = mla_kv_cache_dtype not in ("auto", "fp8_ds_mla") + mla_cache_fp8 = is_quantized_kv_cache(mla_kv_cache_dtype) and not mla_cache_ds_mla mla_num_tiles = 1 mla_ds_scale_view = torch.empty(0, dtype=torch.float32, device=device) mla_ds_rope_view = torch.empty(0, dtype=torch.bfloat16, device=device) From 0eaae2525012924516563a419a9a06fcacfdc3e6 Mon Sep 17 00:00:00 2001 From: Woosuk Kwon Date: Wed, 19 Aug 2026 05:43:09 +0000 Subject: [PATCH 14/15] Fix DeepSeek V3.2 startup artifact expectations Co-authored-by: OpenAI Codex Signed-off-by: Woosuk Kwon --- tests/compile/h100/test_startup.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/compile/h100/test_startup.py b/tests/compile/h100/test_startup.py index 075fc8e24972..76d9b466f664 100644 --- a/tests/compile/h100/test_startup.py +++ b/tests/compile/h100/test_startup.py @@ -133,15 +133,15 @@ class ModelStartupSpec(NamedTuple): id="gpt_oss_120b", ), # NOTE: DeepSeek-V3.2 requires sparse MLA (index_topk) which needs - # Hopper+ GPUs. This test must run on H100 (see pytorch.yaml). + # Hopper+ GPUs. This test must run on H100 (see pytorch.yaml). It defaults + # to breakable CUDA graphs without compilation. pytest.param( ModelStartupSpec( model="deepseek-ai/DeepSeek-V3.2", hf_overrides=_SMALL_MOE_OVERRIDES, - cold_artifacts_saved=9, - # https://github.com/vllm-project/vllm/issues/38051 - warm_artifacts_saved=0 if is_torch_equal_or_newer("2.12.0") else 9, - warm_artifacts_loaded=9 if is_torch_equal_or_newer("2.12.0") else 0, + cold_artifacts_saved=0, + warm_artifacts_saved=0, + warm_artifacts_loaded=0, ), id="deepseek_v3.2", ), From e73990b0a4069a759b5f9619b3d9844861727dfd Mon Sep 17 00:00:00 2001 From: Woosuk Kwon Date: Wed, 19 Aug 2026 05:47:44 +0000 Subject: [PATCH 15/15] Remove DeepSeek V3.2 compile test cases Co-authored-by: OpenAI Codex Signed-off-by: Woosuk Kwon --- tests/compile/fusions_e2e/conftest.py | 21 +---------------- tests/compile/fusions_e2e/models.py | 24 -------------------- tests/compile/fusions_e2e/test_tp1_quant.py | 6 ++--- tests/compile/fusions_e2e/test_tp2_ar_rms.py | 6 ++--- tests/compile/h100/test_startup.py | 13 ----------- 5 files changed, 5 insertions(+), 65 deletions(-) diff --git a/tests/compile/fusions_e2e/conftest.py b/tests/compile/fusions_e2e/conftest.py index 929b27c11c12..f3df3c358ea2 100644 --- a/tests/compile/fusions_e2e/conftest.py +++ b/tests/compile/fusions_e2e/conftest.py @@ -90,11 +90,8 @@ def run( backend_name = attn_backend.backend.name.lower() requires_mla = "deepseek" in model_name.lower() is_mla = "mla" in backend_name - # DeepSeek V3.2 uses sparse MLA - requires_sparse = "v3.2" in model_name.lower() - is_sparse = "sparse" in backend_name - if requires_mla != is_mla or requires_sparse != is_sparse: + if requires_mla != is_mla: pytest.skip( f"Incompatible model '{model_name}' and " f"attention backend '{attn_backend.backend.name}'" @@ -128,22 +125,6 @@ def run( # models (e.g. Llama-4-Scout-FP8) at 16384 tokens may trigger OOM. model_kwargs.setdefault("max_num_batched_tokens", 8192) - # Sparse MLA models (DSv3.2) hit an over-strict inductor assertion in - # decompose_auto_functionalized when +rotary_embedding is forced into - # the compile graph. Disable qk_norm+rope fusion (which auto-enables - # +rotary_embedding) for this combo to avoid the known torch bug. - # TODO: remove once upstream torch fix lands. - if requires_sparse: - if "pass_config" in compilation_config: - compilation_config["pass_config"].enable_qk_norm_rope_fusion = False - matches_check = [m for m in matches_check if m != "norm_rope_fusion"] - # DSv3.2 sparse indexer uses persistent_topk with k=config.index_topk - # (2048 for the default config). max_model_len must be >= index_topk - # or the topk kernel raises "k out of range" at runtime. - model_kwargs["max_model_len"] = max( - model_kwargs.get("max_model_len", 0), 2048 - ) - # Always compile the full graph instead of piecewise if not compilation_config["use_inductor_graph_partition"]: compilation_config["splitting_ops"] = [] diff --git a/tests/compile/fusions_e2e/models.py b/tests/compile/fusions_e2e/models.py index 2d407fbc3025..7879cbe65096 100644 --- a/tests/compile/fusions_e2e/models.py +++ b/tests/compile/fusions_e2e/models.py @@ -58,18 +58,6 @@ id="TRITON_MLA", ) -FLASHMLA_SPARSE_ATTN = pytest.param( - AttentionBackendCase( - backend=AttentionBackendEnum.FLASHMLA_SPARSE, - model_kwargs=dict(kv_cache_dtype="fp8_ds_mla"), - ), - id="FLASHMLA_SPARSE", - marks=pytest.mark.skipif( - not is_blackwell(), - reason="FlashMLA Sparse requires Blackwell", - ), -) - # Models llama3_8b = ModelFusionInfo( model_name="meta-llama/Llama-3.1-8B-Instruct", @@ -197,18 +185,6 @@ ), ) -deepseek_v32_fp4 = ModelFusionInfo( - model_name="nvidia/DeepSeek-V3.2-NVFP4", - matches=lambda n_layers: Matches( - rms_quant_fusion=0, - # silu+quant on dense layers only; MoE hides the act+quant site - act_quant_fusion=min(3, n_layers), - # MLA attn + NVFP4 output quant fuses on sparse MLA output path - attn_quant_fusion=n_layers, - ar_rms_fusion=n_layers * 2 + 1, - ), -) - gpt_oss_20b = ModelFusionInfo( model_name="openai/gpt-oss-20b", matches=lambda n_layers: Matches( diff --git a/tests/compile/fusions_e2e/test_tp1_quant.py b/tests/compile/fusions_e2e/test_tp1_quant.py index fbb382b4458d..3fab133f963e 100644 --- a/tests/compile/fusions_e2e/test_tp1_quant.py +++ b/tests/compile/fusions_e2e/test_tp1_quant.py @@ -18,7 +18,6 @@ from .models import ( FLASHINFER_ATTN, FLASHINFER_MLA_ATTN, - FLASHMLA_SPARSE_ATTN, ROCM_AITER_UNIFIED_ATTN, ROCM_ATTN, TRITON_ATTN, @@ -26,7 +25,6 @@ deepseek_coder_v2_lite_fp8, deepseek_r1_fp4, deepseek_v3_fp8, - deepseek_v32_fp4, llama3_8b_fp4, llama3_8b_fp8, llama4_scout_fp4, @@ -149,11 +147,11 @@ def test_tp1_fp8_fusions( @pytest.mark.parametrize( "model_name, matches_fn, model_kwargs, hf_overrides", - [llama3_8b_fp4, llama4_scout_fp4, deepseek_r1_fp4, deepseek_v32_fp4], + [llama3_8b_fp4, llama4_scout_fp4, deepseek_r1_fp4], ) @pytest.mark.parametrize( "attn_backend", - [FLASHINFER_ATTN, FLASHINFER_MLA_ATTN, FLASHMLA_SPARSE_ATTN], + [FLASHINFER_ATTN, FLASHINFER_MLA_ATTN], ) @pytest.mark.parametrize("n_layers", [6]) @pytest.mark.parametrize("custom_ops", custom_ops_combos("rms_norm")) diff --git a/tests/compile/fusions_e2e/test_tp2_ar_rms.py b/tests/compile/fusions_e2e/test_tp2_ar_rms.py index b6ad4e2e6e85..c88d47cd314f 100644 --- a/tests/compile/fusions_e2e/test_tp2_ar_rms.py +++ b/tests/compile/fusions_e2e/test_tp2_ar_rms.py @@ -18,14 +18,12 @@ from .models import ( FLASHINFER_ATTN, FLASHINFER_MLA_ATTN, - FLASHMLA_SPARSE_ATTN, ROCM_AITER_UNIFIED_ATTN, ROCM_ATTN, TRITON_ATTN, deepseek_coder_v2_lite_fp8, deepseek_r1_fp4, deepseek_v3_fp8, - deepseek_v32_fp4, gpt_oss_20b, llama3_8b, llama3_8b_fp4, @@ -120,11 +118,11 @@ def test_tp2_ar_rms_fp8_fusions( @multi_gpu_test(num_gpus=2) @pytest.mark.parametrize( "model_name, matches_fn, model_kwargs, hf_overrides", - [llama3_8b_fp4, llama4_scout_fp4, deepseek_r1_fp4, deepseek_v32_fp4], + [llama3_8b_fp4, llama4_scout_fp4, deepseek_r1_fp4], ) @pytest.mark.parametrize( "attn_backend", - [FLASHINFER_ATTN, FLASHINFER_MLA_ATTN, FLASHMLA_SPARSE_ATTN], + [FLASHINFER_ATTN, FLASHINFER_MLA_ATTN], ) @pytest.mark.parametrize("n_layers", [4]) @pytest.mark.parametrize("custom_ops", custom_ops_combos("rms_norm")) diff --git a/tests/compile/h100/test_startup.py b/tests/compile/h100/test_startup.py index 76d9b466f664..e57cf85b4a77 100644 --- a/tests/compile/h100/test_startup.py +++ b/tests/compile/h100/test_startup.py @@ -132,19 +132,6 @@ class ModelStartupSpec(NamedTuple): ), id="gpt_oss_120b", ), - # NOTE: DeepSeek-V3.2 requires sparse MLA (index_topk) which needs - # Hopper+ GPUs. This test must run on H100 (see pytorch.yaml). It defaults - # to breakable CUDA graphs without compilation. - pytest.param( - ModelStartupSpec( - model="deepseek-ai/DeepSeek-V3.2", - hf_overrides=_SMALL_MOE_OVERRIDES, - cold_artifacts_saved=0, - warm_artifacts_saved=0, - warm_artifacts_loaded=0, - ), - id="deepseek_v3.2", - ), pytest.param( ModelStartupSpec( model="moonshotai/Kimi-K2.5",