Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 76 additions & 0 deletions atom/plugin/vllm/kimi_k3_kda_autotune_patch.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
"""Let vLLM's Kimi-K3 KDA kernels import under Triton 3.8.

The pinned vLLM keys the autotune of ``chunk_kda_fwd_kernel_intra_sub_chunk``
(``vllm/models/kimi_k3/{amd,nvidia}/ops/third_party/kda/chunk_intra.py``) on
``NC``, which is not one of the kernel's arguments. Triton 3.8 checks autotune
keys against the arguments when the decorator runs, so importing
``vllm.models.kimi_k3`` raises

ValueError: key contains names that are not kernel arguments: NC.

and every Kimi-K3 deployment fails model inspection. vllm-project/vllm#60446
drops the key upstream, after the pinned commit.

The decorator runs at import, so the kernel cannot be patched afterwards; the
fix has to be in place before vLLM's module is imported. This wraps
``triton.autotune``: for a kernel defined under ``vllm.models.kimi_k3`` it
drops the key names that are not arguments -- what #60446 does by hand -- and
it passes every other kernel through unchanged. A vLLM that carries #60446
names no such key and the wrapper drops nothing; the INFO line naming a dropped
key is the sign this patch is still needed.
"""

import functools
import logging

logger = logging.getLogger("atom")

_KIMI_K3_MODULE_PREFIX = "vllm.models.kimi_k3."


def _keys_that_are_arguments(fn, key):
"""``key`` without the names ``fn`` does not take, for Kimi-K3 kernels only."""
module = getattr(fn, "__module__", None) or ""
arg_names = getattr(fn, "arg_names", None)
if not module.startswith(_KIMI_K3_MODULE_PREFIX) or arg_names is None:
return key
dropped = [name for name in key if name not in arg_names]
if not dropped:
return key
logger.info(
"ATOM plugin: dropped autotune key(s) %s of %s.%s; they are not kernel "
"arguments, which Triton 3.8 rejects (vllm-project/vllm#60446).",
dropped,
module,
getattr(fn, "__name__", "<kernel>"),
)
return [name for name in key if name in arg_names]


def apply_vllm_kimi_k3_kda_autotune_patch() -> bool:
"""Wrap ``triton.autotune``. Idempotent; returns whether this call installed it.

Must run before anything imports ``vllm.models.kimi_k3``. It is installed
from ``register_model``, which vLLM's model-inspection subprocess also runs
(``load_general_plugins``) before it imports the model class.
"""
try:
import triton
except ImportError:
return False
original = triton.autotune
if getattr(original, "_atom_kimi_k3_autotune_keys_patched", False):
return False

@functools.wraps(original)
def autotune(configs, key, *args, **kwargs):
def decorator(fn):
return original(
configs, _keys_that_are_arguments(fn, key), *args, **kwargs
)(fn)

return decorator

autotune._atom_kimi_k3_autotune_keys_patched = True
triton.autotune = autotune
return True
8 changes: 8 additions & 0 deletions atom/plugin/vllm/register.py
Original file line number Diff line number Diff line change
Expand Up @@ -437,6 +437,14 @@ def register_model() -> None:
return

_set_plugin_mode()
# Before anything imports vllm.models.kimi_k3: its KDA kernels name an
# autotune key that is not an argument, which Triton 3.8 rejects at import.
# vLLM's model-inspection subprocess runs this hook too.
from atom.plugin.vllm.kimi_k3_kda_autotune_patch import (
apply_vllm_kimi_k3_kda_autotune_patch,
)

apply_vllm_kimi_k3_kda_autotune_patch()
# The general-plugin hook runs in the EngineCore process that owns the
# scheduler/KVCacheManager; install these here as well as in the platform
# hook, which vLLM does not always reach.
Expand Down
26 changes: 0 additions & 26 deletions docker/patches/vllm-kimi-k3-kda-autotune-key.patch

This file was deleted.

5 changes: 0 additions & 5 deletions docker/vllm_release.dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -66,9 +66,6 @@ RUN echo "========== [OOT 2/7] Verify base packages (atom/aiter/mori) ==========
ls /tmp/triton-base-backup/

COPY docker/patches/vllm-torch210-compat.patch /tmp/vllm-torch210-compat.patch
# Backport of vllm-project/vllm#60446: Triton >= 3.8 rejects the invalid NC
# autotune key at import, which fails Kimi-K3 model inspection.
COPY docker/patches/vllm-kimi-k3-kda-autotune-key.patch /tmp/vllm-kimi-k3-kda-autotune-key.patch

RUN echo "========== [OOT 3/7] Clone and patch vLLM ==========" && \
test -n "${VLLM_COMMIT}" && \
Expand All @@ -78,8 +75,6 @@ RUN echo "========== [OOT 3/7] Clone and patch vLLM ==========" && \
git checkout "${VLLM_COMMIT}" && \
git apply --check /tmp/vllm-torch210-compat.patch && \
git apply /tmp/vllm-torch210-compat.patch && \
git apply --check /tmp/vllm-kimi-k3-kda-autotune-key.patch && \
git apply /tmp/vllm-kimi-k3-kda-autotune-key.patch && \
git submodule update --init --recursive && \
echo "vLLM commit:" && \
git rev-parse HEAD
Expand Down
70 changes: 70 additions & 0 deletions tests/test_vllm_kimi_k3_kda_autotune_patch.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
# SPDX-License-Identifier: MIT
"""The Kimi-K3 KDA autotune patch drops non-argument keys, and only for K3.

vLLM's Kimi-K3 KDA kernel keys its autotune on `NC`, which is not an argument,
and Triton 3.8 rejects that when the decorator runs (vllm-project/vllm#60446).
The patch wraps `triton.autotune`; these check what the wrapper hands Triton,
with real Triton and no vLLM or GPU. The patch module is loaded from its file
so the `atom.plugin.vllm` package, which imports vLLM, is not.
"""

import importlib.util
import pathlib

import pytest

triton = pytest.importorskip("triton")

ROOT = pathlib.Path(__file__).resolve().parents[1]
PATCH = ROOT / "atom/plugin/vllm/kimi_k3_kda_autotune_patch.py"

KERNELS = """
import triton
import triton.language as tl


@triton.jit
def kernel(x, BK: tl.constexpr):
pass
"""


def _load(path, name):
spec = importlib.util.spec_from_file_location(name, path)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module


@pytest.fixture
def patched(monkeypatch):
# Restores triton.autotune at teardown, whatever the patch installed.
monkeypatch.setattr(triton, "autotune", triton.autotune)
module = _load(PATCH, "atom_kimi_k3_kda_autotune_patch_under_test")
assert module.apply_vllm_kimi_k3_kda_autotune_patch()
assert not module.apply_vllm_kimi_k3_kda_autotune_patch()
return module


def _kernel(tmp_path, module_name):
# Triton reads the kernel's source, so it has to live in a file.
source = tmp_path / "kernels.py"
source.write_text(KERNELS)
return _load(source, module_name).kernel


def test_drops_the_non_argument_key_of_a_kimi_k3_kernel(patched, tmp_path):
kernel = _kernel(tmp_path, "vllm.models.kimi_k3.amd.ops.third_party.kda.probe")
tuned = triton.autotune(configs=[triton.Config({})], key=["BK", "NC"])(kernel)
assert list(tuned.keys) == ["BK"]


def test_passes_every_other_kernel_to_triton_unchanged(patched, tmp_path):
kernel = _kernel(tmp_path, "vllm.models.other.probe")
decorate = triton.autotune(configs=[triton.Config({})], key=["BK", "NC"])
try:
tuned = decorate(kernel)
except ValueError as exc: # Triton >= 3.8 checks keys itself
assert "NC" in str(exc)
else:
assert list(tuned.keys) == ["BK", "NC"]
Loading