From c1c4a6e35a498389470e4218670912e463a19c87 Mon Sep 17 00:00:00 2001 From: perzhang Date: Thu, 29 Oct 2026 22:11:17 -0500 Subject: [PATCH] fix(plugin): fix Kimi-K3's KDA autotune key in the plugin, not the vLLM image #2818 fixed Kimi-K3's import failure under Triton 3.8 ValueError: key contains names that are not kernel arguments: NC. by applying vllm-project/vllm#60446 as a patch on the vLLM build. That only reaches a deployment once its vLLM image is rebuilt: the nightly vllm-latest built before #2818 merged still fails every PR's Kimi-K3 cell. Fix it where the plugin fixes vLLM. register_model wraps triton.autotune: for a kernel defined under vllm.models.kimi_k3 it drops the key names that are not kernel arguments, which is what #60446 does by hand, and passes every other kernel through unchanged. The decorator runs at import, so the wrapper has to precede any import of vllm.models.kimi_k3; vLLM's model-inspection subprocess, where the failure surfaced, runs load_general_plugins and so this hook first. A vLLM that carries #60446 names no such key and nothing is dropped; an INFO line names each dropped key. The image patch and its dockerfile steps are removed; the vLLM recipe is back to its state before #2818. Co-Authored-By: Claude Opus 5.5 --- .../plugin/vllm/kimi_k3_kda_autotune_patch.py | 76 +++++++++++++++++++ atom/plugin/vllm/register.py | 8 ++ .../vllm-kimi-k3-kda-autotune-key.patch | 26 ------- docker/vllm_release.dockerfile | 5 -- tests/test_vllm_kimi_k3_kda_autotune_patch.py | 70 +++++++++++++++++ 5 files changed, 154 insertions(+), 31 deletions(-) create mode 100644 atom/plugin/vllm/kimi_k3_kda_autotune_patch.py delete mode 100644 docker/patches/vllm-kimi-k3-kda-autotune-key.patch create mode 100644 tests/test_vllm_kimi_k3_kda_autotune_patch.py diff --git a/atom/plugin/vllm/kimi_k3_kda_autotune_patch.py b/atom/plugin/vllm/kimi_k3_kda_autotune_patch.py new file mode 100644 index 0000000000..2d6df4032c --- /dev/null +++ b/atom/plugin/vllm/kimi_k3_kda_autotune_patch.py @@ -0,0 +1,76 @@ +"""Let vLLM's Kimi-K3 KDA kernels import under Triton 3.8. + +The pinned vLLM keys the autotune of ``chunk_kda_fwd_kernel_intra_sub_chunk`` +(``vllm/models/kimi_k3/{amd,nvidia}/ops/third_party/kda/chunk_intra.py``) on +``NC``, which is not one of the kernel's arguments. Triton 3.8 checks autotune +keys against the arguments when the decorator runs, so importing +``vllm.models.kimi_k3`` raises + + ValueError: key contains names that are not kernel arguments: NC. + +and every Kimi-K3 deployment fails model inspection. vllm-project/vllm#60446 +drops the key upstream, after the pinned commit. + +The decorator runs at import, so the kernel cannot be patched afterwards; the +fix has to be in place before vLLM's module is imported. This wraps +``triton.autotune``: for a kernel defined under ``vllm.models.kimi_k3`` it +drops the key names that are not arguments -- what #60446 does by hand -- and +it passes every other kernel through unchanged. A vLLM that carries #60446 +names no such key and the wrapper drops nothing; the INFO line naming a dropped +key is the sign this patch is still needed. +""" + +import functools +import logging + +logger = logging.getLogger("atom") + +_KIMI_K3_MODULE_PREFIX = "vllm.models.kimi_k3." + + +def _keys_that_are_arguments(fn, key): + """``key`` without the names ``fn`` does not take, for Kimi-K3 kernels only.""" + module = getattr(fn, "__module__", None) or "" + arg_names = getattr(fn, "arg_names", None) + if not module.startswith(_KIMI_K3_MODULE_PREFIX) or arg_names is None: + return key + dropped = [name for name in key if name not in arg_names] + if not dropped: + return key + logger.info( + "ATOM plugin: dropped autotune key(s) %s of %s.%s; they are not kernel " + "arguments, which Triton 3.8 rejects (vllm-project/vllm#60446).", + dropped, + module, + getattr(fn, "__name__", ""), + ) + return [name for name in key if name in arg_names] + + +def apply_vllm_kimi_k3_kda_autotune_patch() -> bool: + """Wrap ``triton.autotune``. Idempotent; returns whether this call installed it. + + Must run before anything imports ``vllm.models.kimi_k3``. It is installed + from ``register_model``, which vLLM's model-inspection subprocess also runs + (``load_general_plugins``) before it imports the model class. + """ + try: + import triton + except ImportError: + return False + original = triton.autotune + if getattr(original, "_atom_kimi_k3_autotune_keys_patched", False): + return False + + @functools.wraps(original) + def autotune(configs, key, *args, **kwargs): + def decorator(fn): + return original( + configs, _keys_that_are_arguments(fn, key), *args, **kwargs + )(fn) + + return decorator + + autotune._atom_kimi_k3_autotune_keys_patched = True + triton.autotune = autotune + return True diff --git a/atom/plugin/vllm/register.py b/atom/plugin/vllm/register.py index 0f0c5fdfc2..f5c0dc27f2 100644 --- a/atom/plugin/vllm/register.py +++ b/atom/plugin/vllm/register.py @@ -437,6 +437,14 @@ def register_model() -> None: return _set_plugin_mode() + # Before anything imports vllm.models.kimi_k3: its KDA kernels name an + # autotune key that is not an argument, which Triton 3.8 rejects at import. + # vLLM's model-inspection subprocess runs this hook too. + from atom.plugin.vllm.kimi_k3_kda_autotune_patch import ( + apply_vllm_kimi_k3_kda_autotune_patch, + ) + + apply_vllm_kimi_k3_kda_autotune_patch() # The general-plugin hook runs in the EngineCore process that owns the # scheduler/KVCacheManager; install these here as well as in the platform # hook, which vLLM does not always reach. diff --git a/docker/patches/vllm-kimi-k3-kda-autotune-key.patch b/docker/patches/vllm-kimi-k3-kda-autotune-key.patch deleted file mode 100644 index cb971a730e..0000000000 --- a/docker/patches/vllm-kimi-k3-kda-autotune-key.patch +++ /dev/null @@ -1,26 +0,0 @@ -diff --git a/vllm/models/kimi_k3/amd/ops/third_party/kda/chunk_intra.py b/vllm/models/kimi_k3/amd/ops/third_party/kda/chunk_intra.py -index 8b9aa86..07b4a38 100644 ---- a/vllm/models/kimi_k3/amd/ops/third_party/kda/chunk_intra.py -+++ b/vllm/models/kimi_k3/amd/ops/third_party/kda/chunk_intra.py -@@ -451,7 +451,7 @@ def chunk_kda_fwd_kernel_inter_solve_fused( - for num_warps in [1, 2, 4, 8] - for num_stages in [2, 3, 4] - ], -- key=["BK", "NC", "BT", "HV"], -+ key=["BK", "BT", "HV"], - ) - @triton.jit(do_not_specialize=["B", "T"]) - def chunk_kda_fwd_kernel_intra_sub_chunk( -diff --git a/vllm/models/kimi_k3/nvidia/ops/third_party/kda/chunk_intra.py b/vllm/models/kimi_k3/nvidia/ops/third_party/kda/chunk_intra.py -index 8b9aa86..07b4a38 100644 ---- a/vllm/models/kimi_k3/nvidia/ops/third_party/kda/chunk_intra.py -+++ b/vllm/models/kimi_k3/nvidia/ops/third_party/kda/chunk_intra.py -@@ -451,7 +451,7 @@ def chunk_kda_fwd_kernel_inter_solve_fused( - for num_warps in [1, 2, 4, 8] - for num_stages in [2, 3, 4] - ], -- key=["BK", "NC", "BT", "HV"], -+ key=["BK", "BT", "HV"], - ) - @triton.jit(do_not_specialize=["B", "T"]) - def chunk_kda_fwd_kernel_intra_sub_chunk( diff --git a/docker/vllm_release.dockerfile b/docker/vllm_release.dockerfile index 00f60fa295..685ce791bd 100644 --- a/docker/vllm_release.dockerfile +++ b/docker/vllm_release.dockerfile @@ -66,9 +66,6 @@ RUN echo "========== [OOT 2/7] Verify base packages (atom/aiter/mori) ========== ls /tmp/triton-base-backup/ COPY docker/patches/vllm-torch210-compat.patch /tmp/vllm-torch210-compat.patch -# Backport of vllm-project/vllm#60446: Triton >= 3.8 rejects the invalid NC -# autotune key at import, which fails Kimi-K3 model inspection. -COPY docker/patches/vllm-kimi-k3-kda-autotune-key.patch /tmp/vllm-kimi-k3-kda-autotune-key.patch RUN echo "========== [OOT 3/7] Clone and patch vLLM ==========" && \ test -n "${VLLM_COMMIT}" && \ @@ -78,8 +75,6 @@ RUN echo "========== [OOT 3/7] Clone and patch vLLM ==========" && \ git checkout "${VLLM_COMMIT}" && \ git apply --check /tmp/vllm-torch210-compat.patch && \ git apply /tmp/vllm-torch210-compat.patch && \ - git apply --check /tmp/vllm-kimi-k3-kda-autotune-key.patch && \ - git apply /tmp/vllm-kimi-k3-kda-autotune-key.patch && \ git submodule update --init --recursive && \ echo "vLLM commit:" && \ git rev-parse HEAD diff --git a/tests/test_vllm_kimi_k3_kda_autotune_patch.py b/tests/test_vllm_kimi_k3_kda_autotune_patch.py new file mode 100644 index 0000000000..e3005b3eec --- /dev/null +++ b/tests/test_vllm_kimi_k3_kda_autotune_patch.py @@ -0,0 +1,70 @@ +# SPDX-License-Identifier: MIT +"""The Kimi-K3 KDA autotune patch drops non-argument keys, and only for K3. + +vLLM's Kimi-K3 KDA kernel keys its autotune on `NC`, which is not an argument, +and Triton 3.8 rejects that when the decorator runs (vllm-project/vllm#60446). +The patch wraps `triton.autotune`; these check what the wrapper hands Triton, +with real Triton and no vLLM or GPU. The patch module is loaded from its file +so the `atom.plugin.vllm` package, which imports vLLM, is not. +""" + +import importlib.util +import pathlib + +import pytest + +triton = pytest.importorskip("triton") + +ROOT = pathlib.Path(__file__).resolve().parents[1] +PATCH = ROOT / "atom/plugin/vllm/kimi_k3_kda_autotune_patch.py" + +KERNELS = """ +import triton +import triton.language as tl + + +@triton.jit +def kernel(x, BK: tl.constexpr): + pass +""" + + +def _load(path, name): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@pytest.fixture +def patched(monkeypatch): + # Restores triton.autotune at teardown, whatever the patch installed. + monkeypatch.setattr(triton, "autotune", triton.autotune) + module = _load(PATCH, "atom_kimi_k3_kda_autotune_patch_under_test") + assert module.apply_vllm_kimi_k3_kda_autotune_patch() + assert not module.apply_vllm_kimi_k3_kda_autotune_patch() + return module + + +def _kernel(tmp_path, module_name): + # Triton reads the kernel's source, so it has to live in a file. + source = tmp_path / "kernels.py" + source.write_text(KERNELS) + return _load(source, module_name).kernel + + +def test_drops_the_non_argument_key_of_a_kimi_k3_kernel(patched, tmp_path): + kernel = _kernel(tmp_path, "vllm.models.kimi_k3.amd.ops.third_party.kda.probe") + tuned = triton.autotune(configs=[triton.Config({})], key=["BK", "NC"])(kernel) + assert list(tuned.keys) == ["BK"] + + +def test_passes_every_other_kernel_to_triton_unchanged(patched, tmp_path): + kernel = _kernel(tmp_path, "vllm.models.other.probe") + decorate = triton.autotune(configs=[triton.Config({})], key=["BK", "NC"]) + try: + tuned = decorate(kernel) + except ValueError as exc: # Triton >= 3.8 checks keys itself + assert "NC" in str(exc) + else: + assert list(tuned.keys) == ["BK", "NC"]