From 984782e11234fcaa0544f46ce13540a9373e347c Mon Sep 17 00:00:00 2001 From: NickLucche Date: Mon, 5 Oct 2026 08:52:31 +0000 Subject: [PATCH 1/3] [Model][GLM-5.3-Flash] Warm up the DCP top-k merge kernels for the kpool indexer Under DCP the kpool indexer merges per-rank top-k with the CuteDSL/Triton DCP merge kernels, but only SparseAttnIndexer registered them for JIT warmup, so GLM-5.x compiled PackDCPTopkCandidatesKernel during inference. Register them from SparseAttnIndexerKpool as well. Co-Authored-By: Claude Opus 5.5 Signed-off-by: NickLucche --- vllm/models/glm5next/nvidia/sparse_indexer.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/vllm/models/glm5next/nvidia/sparse_indexer.py b/vllm/models/glm5next/nvidia/sparse_indexer.py index 806b633c5836..17c1d84a1b1d 100644 --- a/vllm/models/glm5next/nvidia/sparse_indexer.py +++ b/vllm/models/glm5next/nvidia/sparse_indexer.py @@ -26,6 +26,7 @@ from vllm.models.glm5next.nvidia.ops import kpool_compress as kpool_ops from vllm.platforms import current_platform from vllm.utils.deep_gemm import has_deep_gemm +from vllm.utils.import_utils import has_cutedsl from vllm.utils.torch_utils import ( LayerNameType, _resolve_layer_name, @@ -717,6 +718,19 @@ def __init__( ) self.dcp_world_size = _parallel.decode_context_parallel_size self.dcp_rank = get_dcp_group().rank_in_group + if ( + _cfg is not None + and _cfg.kernel_config.enable_jit_warmup + and current_platform.is_cuda() + and has_cutedsl() + ): + from vllm.model_executor.kernels.attention.dsa.dcp_indexer_cutedsl import ( # noqa: E501 + _PACK_DCP_TOPK_CANDIDATES_KERNEL, + _STABLE_TOPK_FROM_GATHERED_CANDIDATES_KERNEL, + ) + + _PACK_DCP_TOPK_CANDIDATES_KERNEL.register_warmup() + _STABLE_TOPK_FROM_GATHERED_CANDIDATES_KERNEL.register_warmup() @property def cp_kv_cache_interleave_size(self) -> int: From b7067a9bb29c0e838ec142fee969ae6216c04f26 Mon Sep 17 00:00:00 2001 From: NickLucche Date: Mon, 5 Oct 2026 19:31:50 +0200 Subject: [PATCH 2/3] Add unit tests for kpool indexer DCP merge kernel JIT warmup Cover the warmup registration added to SparseAttnIndexerKpool.__init__: both DCP top-k merge kernels are registered under DCP with JIT warmup enabled, and neither is registered without DCP or with warmup disabled. Generated with Mistral Vibe. Co-Authored-By: Mistral Vibe Signed-off-by: NickLucche --- .../test_sparse_indexer_jit_warmup.py | 102 ++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 tests/models/glm5next/test_sparse_indexer_jit_warmup.py diff --git a/tests/models/glm5next/test_sparse_indexer_jit_warmup.py b/tests/models/glm5next/test_sparse_indexer_jit_warmup.py new file mode 100644 index 000000000000..c69b9eb93be0 --- /dev/null +++ b/tests/models/glm5next/test_sparse_indexer_jit_warmup.py @@ -0,0 +1,102 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for the GLM kpool indexer's DCP top-k merge kernel JIT warmup.""" + +import sys +from types import ModuleType, SimpleNamespace +from typing import Any + +import pytest +import torch + +from vllm.config import VllmConfig, set_current_vllm_config + +_CUTEDSL_MODULE = "vllm.model_executor.kernels.attention.dsa.dcp_indexer_cutedsl" + + +class _RecordingKernel: + """Stand-in for a JIT kernel singleton that records warmup registration.""" + + def __init__(self, name: str, registrations: list[str]) -> None: + self._name = name + self._registrations = registrations + + def register_warmup(self) -> None: + self._registrations.append(self._name) + + +def _build_kpool_indexer( + monkeypatch: pytest.MonkeyPatch, *, dcp: int, jit_warmup: bool +) -> list[str]: + """Build a kpool indexer with the DCP merge kernels stubbed out. + + Returns the names of the kernels whose register_warmup() ran. The + platform guards (CUDA, DeepGEMM, CuteDSL, DCP group) are faked so the + wiring is exercised on any machine. + """ + from vllm.models.glm5next.nvidia import sparse_indexer + + registrations: list[str] = [] + fake_cutedsl: Any = ModuleType(_CUTEDSL_MODULE) + fake_cutedsl._PACK_DCP_TOPK_CANDIDATES_KERNEL = _RecordingKernel( + "pack_dcp_topk_candidates", registrations + ) + fake_cutedsl._STABLE_TOPK_FROM_GATHERED_CANDIDATES_KERNEL = _RecordingKernel( + "stable_topk_from_gathered_candidates", registrations + ) + monkeypatch.setitem(sys.modules, _CUTEDSL_MODULE, fake_cutedsl) + monkeypatch.setattr( + sparse_indexer, "current_platform", SimpleNamespace(is_cuda=lambda: True) + ) + monkeypatch.setattr(sparse_indexer, "has_deep_gemm", lambda: True) + # Added by the warmup change under test; absent before it. + monkeypatch.setattr(sparse_indexer, "has_cutedsl", lambda: True, raising=False) + monkeypatch.setattr( + sparse_indexer, "get_dcp_group", lambda: SimpleNamespace(rank_in_group=0) + ) + + config = VllmConfig( + kernel_config={"enable_jit_warmup": jit_warmup}, + parallel_config={ + "tensor_parallel_size": dcp, + "decode_context_parallel_size": dcp, + }, + ) + with set_current_vllm_config(config): + sparse_indexer.SparseAttnIndexerKpool( + k_cache=None, + quant_block_size=128, + scale_fmt="ue8m0", + topk_tokens=2048, + head_dim=128, + max_pool_len=4096, + max_total_seq_len=8192, + topk_indices_buffer=torch.empty(8, 2176, dtype=torch.int32), + ) + return registrations + + +def test_kpool_indexer_registers_dcp_merge_kernels_for_jit_warmup( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Under DCP with JIT warmup enabled, the kpool indexer registers the two + DCP top-k merge kernels so they compile at startup instead of on the + first long-context request, like the base SparseAttnIndexer does.""" + registrations = _build_kpool_indexer(monkeypatch, dcp=2, jit_warmup=True) + assert registrations == [ + "pack_dcp_topk_candidates", + "stable_topk_from_gathered_candidates", + ] + + +@pytest.mark.parametrize( + "dcp,jit_warmup", + [(1, True), (2, False)], + ids=["no-dcp", "jit-warmup-disabled"], +) +def test_kpool_indexer_skips_dcp_merge_warmup_without_dcp_or_opt_in( + monkeypatch: pytest.MonkeyPatch, dcp: int, jit_warmup: bool +) -> None: + """Without DCP the merge kernels never run, and with warmup disabled they + must not be compiled eagerly: neither case may register them.""" + assert _build_kpool_indexer(monkeypatch, dcp=dcp, jit_warmup=jit_warmup) == [] From 28963bf13b201a0ae2bfb24046536d95fdec23f9 Mon Sep 17 00:00:00 2001 From: NickLucche Date: Wed, 7 Oct 2026 23:29:16 +0200 Subject: [PATCH 3/3] Simplify kpool DCP merge kernel warmup registration Address review feedback: drop the unit test and reduce the warmup condition to `current_platform.is_cuda() and has_cutedsl()`. The JIT warmup registry only compiles registered kernels when `enable_jit_warmup` is on (kernel_warmup gates registry.warmup()), and DCP > 1 is guaranteed by the surrounding branch, so the extra checks are redundant. Generated with Mistral Vibe. Co-Authored-By: Mistral Vibe Signed-off-by: NickLucche --- .../test_sparse_indexer_jit_warmup.py | 102 ------------------ vllm/models/glm5next/nvidia/sparse_indexer.py | 7 +- 2 files changed, 1 insertion(+), 108 deletions(-) delete mode 100644 tests/models/glm5next/test_sparse_indexer_jit_warmup.py diff --git a/tests/models/glm5next/test_sparse_indexer_jit_warmup.py b/tests/models/glm5next/test_sparse_indexer_jit_warmup.py deleted file mode 100644 index c69b9eb93be0..000000000000 --- a/tests/models/glm5next/test_sparse_indexer_jit_warmup.py +++ /dev/null @@ -1,102 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Tests for the GLM kpool indexer's DCP top-k merge kernel JIT warmup.""" - -import sys -from types import ModuleType, SimpleNamespace -from typing import Any - -import pytest -import torch - -from vllm.config import VllmConfig, set_current_vllm_config - -_CUTEDSL_MODULE = "vllm.model_executor.kernels.attention.dsa.dcp_indexer_cutedsl" - - -class _RecordingKernel: - """Stand-in for a JIT kernel singleton that records warmup registration.""" - - def __init__(self, name: str, registrations: list[str]) -> None: - self._name = name - self._registrations = registrations - - def register_warmup(self) -> None: - self._registrations.append(self._name) - - -def _build_kpool_indexer( - monkeypatch: pytest.MonkeyPatch, *, dcp: int, jit_warmup: bool -) -> list[str]: - """Build a kpool indexer with the DCP merge kernels stubbed out. - - Returns the names of the kernels whose register_warmup() ran. The - platform guards (CUDA, DeepGEMM, CuteDSL, DCP group) are faked so the - wiring is exercised on any machine. - """ - from vllm.models.glm5next.nvidia import sparse_indexer - - registrations: list[str] = [] - fake_cutedsl: Any = ModuleType(_CUTEDSL_MODULE) - fake_cutedsl._PACK_DCP_TOPK_CANDIDATES_KERNEL = _RecordingKernel( - "pack_dcp_topk_candidates", registrations - ) - fake_cutedsl._STABLE_TOPK_FROM_GATHERED_CANDIDATES_KERNEL = _RecordingKernel( - "stable_topk_from_gathered_candidates", registrations - ) - monkeypatch.setitem(sys.modules, _CUTEDSL_MODULE, fake_cutedsl) - monkeypatch.setattr( - sparse_indexer, "current_platform", SimpleNamespace(is_cuda=lambda: True) - ) - monkeypatch.setattr(sparse_indexer, "has_deep_gemm", lambda: True) - # Added by the warmup change under test; absent before it. - monkeypatch.setattr(sparse_indexer, "has_cutedsl", lambda: True, raising=False) - monkeypatch.setattr( - sparse_indexer, "get_dcp_group", lambda: SimpleNamespace(rank_in_group=0) - ) - - config = VllmConfig( - kernel_config={"enable_jit_warmup": jit_warmup}, - parallel_config={ - "tensor_parallel_size": dcp, - "decode_context_parallel_size": dcp, - }, - ) - with set_current_vllm_config(config): - sparse_indexer.SparseAttnIndexerKpool( - k_cache=None, - quant_block_size=128, - scale_fmt="ue8m0", - topk_tokens=2048, - head_dim=128, - max_pool_len=4096, - max_total_seq_len=8192, - topk_indices_buffer=torch.empty(8, 2176, dtype=torch.int32), - ) - return registrations - - -def test_kpool_indexer_registers_dcp_merge_kernels_for_jit_warmup( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Under DCP with JIT warmup enabled, the kpool indexer registers the two - DCP top-k merge kernels so they compile at startup instead of on the - first long-context request, like the base SparseAttnIndexer does.""" - registrations = _build_kpool_indexer(monkeypatch, dcp=2, jit_warmup=True) - assert registrations == [ - "pack_dcp_topk_candidates", - "stable_topk_from_gathered_candidates", - ] - - -@pytest.mark.parametrize( - "dcp,jit_warmup", - [(1, True), (2, False)], - ids=["no-dcp", "jit-warmup-disabled"], -) -def test_kpool_indexer_skips_dcp_merge_warmup_without_dcp_or_opt_in( - monkeypatch: pytest.MonkeyPatch, dcp: int, jit_warmup: bool -) -> None: - """Without DCP the merge kernels never run, and with warmup disabled they - must not be compiled eagerly: neither case may register them.""" - assert _build_kpool_indexer(monkeypatch, dcp=dcp, jit_warmup=jit_warmup) == [] diff --git a/vllm/models/glm5next/nvidia/sparse_indexer.py b/vllm/models/glm5next/nvidia/sparse_indexer.py index 17c1d84a1b1d..77fa4c409152 100644 --- a/vllm/models/glm5next/nvidia/sparse_indexer.py +++ b/vllm/models/glm5next/nvidia/sparse_indexer.py @@ -718,12 +718,7 @@ def __init__( ) self.dcp_world_size = _parallel.decode_context_parallel_size self.dcp_rank = get_dcp_group().rank_in_group - if ( - _cfg is not None - and _cfg.kernel_config.enable_jit_warmup - and current_platform.is_cuda() - and has_cutedsl() - ): + if current_platform.is_cuda() and has_cutedsl(): from vllm.model_executor.kernels.attention.dsa.dcp_indexer_cutedsl import ( # noqa: E501 _PACK_DCP_TOPK_CANDIDATES_KERNEL, _STABLE_TOPK_FROM_GATHERED_CANDIDATES_KERNEL,