Skip to content
Open
Show file tree
Hide file tree
Changes from 34 commits
Commits
Show all changes
41 commits
Select commit Hold shift + click to select a range
989c78c
[Quant][Prototype] Hoist NVFP4 input quant out of Linear apply
mgoin May 12, 2026
cc0e21c
[Quant][Prototype] Add static FP8 + Llama; gate manual fusion behind …
mgoin May 13, 2026
5165a8e
[Quant][Prototype] Wire fused rms_norm_static_fp8_quant on hoist path
mgoin May 13, 2026
5d6d041
[Quant][Prototype] Drop VLLM_HOIST_INPUT_QUANT gate
mgoin May 13, 2026
d49d677
[Quant][Prototype] Drop NVFP4 scope; let QuantizedActivation flow int…
mgoin May 13, 2026
4cb7ecc
[Quant][Prototype] Move RMSNorm+quant dispatch into quant_fusion
mgoin May 13, 2026
5c84221
[Quant][Prototype] Pass quant_key/input_scale explicitly at call site
mgoin May 13, 2026
1c6157f
Revert "[Quant][Prototype] Pass quant_key/input_scale explicitly at c…
mgoin May 13, 2026
2994e39
[Quant][Prototype] Add allreduce to manual fusion path; wire Qwen3
mgoin May 14, 2026
44c5ad1
[Quant][Prototype] Robust AR check + handle final norm
mgoin May 14, 2026
6f5dcaf
[Quant][Prototype] Wire FlashInfer fused AR+RMSNorm+FP8 single kernel
mgoin May 14, 2026
7321e88
[Quant][Prototype] Refactor to FusedAllReduceRMSQuant + NVFP4 impl
mgoin May 20, 2026
b11c165
Change to functional approach and no quant support
mgoin May 21, 2026
8e14d9e
Merge branch 'main' into fp8-manual-fusion-expand
mgoin May 26, 2026
34d692d
Query kernel.input_quant_key() instead of hard-coding it in schemes
mgoin May 26, 2026
ce8c613
Guard ar_rms_quant flashinfer paths for CUDA-without-FlashInfer
mgoin May 26, 2026
a2ffe3e
Derive flashinfer max_token_num locally; consolidate input_quant_key …
mgoin May 27, 2026
2f863ca
Split ar_rms_quant dispatch by capability, not by quant key
mgoin May 27, 2026
b0d1410
Rename consumer -> consumer_linear at fused_ar_rms_norm_quant call sites
mgoin May 27, 2026
c95561f
Fix up
mgoin May 28, 2026
18f04bd
Add no-residual fused AR path, RMSNorm guard, and QA quant-key checks
mgoin May 28, 2026
8d91162
Unify _flashinfer_ar_rms with the quant impls' residual handling
mgoin May 28, 2026
39b6bad
Merge branch 'main' into fp8-manual-fusion-expand
mgoin May 28, 2026
689bc13
Fall back to unfused AR+RMSNorm when batch exceeds flashinfer workspace
mgoin May 29, 2026
4dde8cd
Make AR+RMSNorm fusion opt-in to fix TP correctness in reused decoders
mgoin May 29, 2026
c3cb1c7
Tests
mgoin May 29, 2026
a3a3ac3
[Prototype] Manual TP fusion via a per-layer LayerCommunicator (RFC #…
mgoin May 29, 2026
5ec55bf
Crystallize the QuantizedActivation producer<->consumer contract
mgoin Jun 1, 2026
0e37c31
Merge branch 'main' into manual-fusion-ar-rms-quant
mgoin Jun 16, 2026
aaeb5dc
[Kernel] Unify manual fusion behind ResidualStream; convert Qwen3
mgoin Jun 16, 2026
d176caf
[Kernel] Fix Qwen3-VL final all-reduce; harden finalize_norm
mgoin Jun 16, 2026
b5c8382
[Kernel] Manual fusion cleanup: dedup buffers, type hint, test notes
mgoin Jun 16, 2026
6666496
Merge branch 'main' into manual-fusion-ar-rms-quant
jeejeelee Jun 26, 2026
41d68a5
Move forward
jeejeelee Jun 27, 2026
9f01b6f
Refactor
jeejeelee Jul 1, 2026
75f0ed2
Merge branch 'main' into manual-fusion-ar-rms-quant
jeejeelee Jul 1, 2026
f9f893d
FIX
jeejeelee Jul 1, 2026
e024f21
FIX
jeejeelee Jul 1, 2026
26c75ca
Merge branch 'main' into manual-fusion-ar-rms-quant
jeejeelee Jul 1, 2026
63bea01
Merge branch 'main' into manual-fusion-ar-rms-quant
jeejeelee Jul 2, 2026
9a7073e
Merge branch 'main' into manual-fusion-ar-rms-quant
jeejeelee Jul 2, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 8 additions & 11 deletions tests/compile/fusions_e2e/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,22 +74,20 @@
llama3_8b = ModelFusionInfo(
model_name="meta-llama/Llama-3.1-8B-Instruct",
matches=lambda n_layers: Matches(
ar_rms_fusion=n_layers * 2 + 1,
aiter_ar_rms_fusion=n_layers * 2,
sequence_parallel=n_layers * 2 + 1,
async_tp=n_layers * 4,
# AR+RMSNorm fused in model code; compiler catches 1 leftover.
# NOTE: manual fusion consumes the AR pattern before the compiler's
# sequence_parallel / async_tp / aiter passes run, so those drop to 0.
ar_rms_fusion=1,
),
)

llama3_8b_fp8 = ModelFusionInfo(
model_name="RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8",
matches=lambda n_layers: Matches(
rms_quant_fusion=n_layers * 2,
# RMSNorm+quant fused in model code; compiler finds none.
rms_quant_fusion=0,
act_quant_fusion=n_layers,
attn_quant_fusion=n_layers,
ar_rms_fusion=n_layers * 2 + 1,
sequence_parallel=n_layers * 2 + 1,
async_tp=n_layers * 4,
),
)

Expand All @@ -98,9 +96,8 @@
matches=lambda n_layers: Matches(
act_quant_fusion=n_layers,
attn_quant_fusion=n_layers,
ar_rms_fusion=n_layers * 2 + 1,
sequence_parallel=n_layers * 2 + 1,
async_tp=n_layers * 4,
# AR+RMSNorm+quant fused in model code; compiler catches 1 leftover.
ar_rms_fusion=1,
),
)

Expand Down
4 changes: 2 additions & 2 deletions tests/compile/fusions_e2e/test_tp2_ar_rms.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@
gpt_oss_20b,
llama3_8b,
llama3_8b_fp4,
llama3_8b_fp8,
llama4_scout_fp4,
llama4_scout_fp8,
qwen3_a3b,
Expand All @@ -45,8 +44,9 @@
@pytest.mark.parametrize(
"model_name, matches_fn, model_kwargs, hf_overrides",
# qwen3 & dsv3 should still fuse AR+rms even though group quant is not yet supported
# llama3_8b_fp8 is omitted: it is migrated to manual fusion, so its rms+quant is
# done in model code and the compiler AR+RMS pass has nothing left to match.
[
llama3_8b_fp8,
llama4_scout_fp8,
qwen3_a3b_fp8,
deepseek_coder_v2_lite_fp8,
Expand Down
18 changes: 10 additions & 8 deletions tests/compile/fusions_e2e/test_tp2_async_tp.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,20 +18,22 @@
from .models import (
FLASHINFER_ATTN,
TRITON_ATTN,
llama3_8b,
llama3_8b_fp4,
llama3_8b_fp8,
llama4_scout_fp8,
qwen3_a3b,
)

# NOTE: the migrated base-Llama models (llama3_8b*) are intentionally absent here.
# Manual fusion folds the all-reduce into model code, consuming the AR pattern
# before the sequence-parallel / async-TP passes run -- so those models no longer
# exercise async-TP fusion. Reconciling manual fusion with async-TP is tracked
# separately (RFC #43224); unmigrated models (llama4/qwen3) still cover the passes.
pytestmark = pytest.mark.skipif(not current_platform.is_cuda(), reason="Only test CUDA")


@multi_gpu_test(num_gpus=2)
@pytest.mark.parametrize(
"model_name, matches_fn, model_kwargs, hf_overrides",
[llama3_8b_fp8, llama4_scout_fp8],
[llama4_scout_fp8],
)
@pytest.mark.parametrize("attn_backend", [TRITON_ATTN, FLASHINFER_ATTN])
@pytest.mark.parametrize("n_layers", [4])
Expand Down Expand Up @@ -95,7 +97,7 @@ def test_tp2_async_tp_fp8_fusions(
@multi_gpu_test(num_gpus=2)
@pytest.mark.parametrize(
"model_name, matches_fn, model_kwargs, hf_overrides",
[llama3_8b_fp4],
[qwen3_a3b],
)
@pytest.mark.parametrize("attn_backend", [FLASHINFER_ATTN])
@pytest.mark.parametrize("n_layers", [4])
Expand Down Expand Up @@ -158,7 +160,7 @@ def test_tp2_async_tp_nvfp4_fusions(
@multi_gpu_test(num_gpus=2)
@pytest.mark.parametrize(
"model_name, matches_fn, model_kwargs, hf_overrides",
[llama3_8b, qwen3_a3b],
[qwen3_a3b],
)
@pytest.mark.parametrize("attn_backend", [TRITON_ATTN])
@pytest.mark.parametrize("n_layers", [4])
Expand Down Expand Up @@ -216,7 +218,7 @@ def test_tp2_async_tp_fusions(
@multi_gpu_test(num_gpus=2)
@pytest.mark.parametrize(
"model_name, matches_fn, model_kwargs, hf_overrides",
[llama3_8b_fp8, llama4_scout_fp8],
[llama4_scout_fp8],
)
@pytest.mark.parametrize("attn_backend", [TRITON_ATTN, FLASHINFER_ATTN])
@pytest.mark.parametrize("n_layers", [4])
Expand Down Expand Up @@ -280,7 +282,7 @@ def test_tp2_sp_ar_rms_fp8_fusions(
@multi_gpu_test(num_gpus=2)
@pytest.mark.parametrize(
"model_name, matches_fn, model_kwargs, hf_overrides",
[llama3_8b, qwen3_a3b],
[qwen3_a3b],
)
@pytest.mark.parametrize("attn_backend", [TRITON_ATTN])
@pytest.mark.parametrize("n_layers", [4])
Expand Down
83 changes: 83 additions & 0 deletions tests/fusion/conftest.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Fixtures for manual-fusion tests.

The harness lets a test prove that a manual fusion op actually runs in model
code, even when every compiler-driven fusion pass is disabled. The mechanism
is a ``TorchDispatchMode`` installed inside each worker via ``collective_rpc``
that counts every ATen / vLLM C++ op call, plus a per-worker registry of
Python-level wrappers (e.g. ``flashinfer_trtllm_fused_allreduce_norm``) that
TorchDispatchMode cannot see.
"""

import os
from collections import Counter
from contextlib import contextmanager

import pytest

# Required so collective_rpc can ship our counter install/fetch closures to
# workers (vLLM defaults to a strict msgpack codec that rejects functions).
os.environ.setdefault("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")


def _install_dispatch_counter(worker_self):
"""Install a TorchDispatchMode counter on this worker. Runs via RPC."""
from collections import Counter

from torch.utils._python_dispatch import TorchDispatchMode

class _OpCounter(TorchDispatchMode):
def __init__(self):
super().__init__()
self.counts: Counter[str] = Counter()

def __torch_dispatch__(self, func, types, args=(), kwargs=None):
self.counts[str(func)] += 1
return func(*args, **(kwargs or {}))

counter = _OpCounter()
counter.__enter__()
worker_self._fusion_op_counter = counter


def _fetch_dispatch_counts(worker_self) -> dict[str, int]:
"""Tear down the counter and return its dict. Runs via RPC."""
counter = getattr(worker_self, "_fusion_op_counter", None)
if counter is None:
return {}
counter.__exit__(None, None, None)
del worker_self._fusion_op_counter
return dict(counter.counts)


@pytest.fixture
def op_count_session():
"""Context manager that yields a Counter aggregated across workers.

Usage:

def test_x(op_count_session):
llm = LLM(...)
with op_count_session(llm) as counts:
llm.generate(...)
assert counts["_C.fused_add_rms_norm_static_fp8_quant.default"] > 0
"""

@contextmanager
def session(llm):
llm.collective_rpc(_install_dispatch_counter)
aggregated: Counter[str] = Counter()
try:
yield aggregated
finally:
per_worker = llm.collective_rpc(_fetch_dispatch_counts)
for d in per_worker:
aggregated.update(d)

return session


def count_matching(counts: Counter, *needles: str) -> int:
"""Sum counts whose op name contains any of the given substrings."""
return sum(c for op, c in counts.items() if any(n in op for n in needles))
107 changes: 107 additions & 0 deletions tests/fusion/test_ar_rms_quant_fusion.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Manual-fusion tests for ``fused_ar_rms_norm_quant``.

These tests verify that the manual fusion helper introduced in
``vllm.model_executor.layers.fusion.ar_rms_quant`` actually fires
inside model forward when every compiler-driven fusion pass is disabled. They
do not validate output correctness — that belongs in the accuracy harness.

Tracks the AR+RMS+Quant bullet under
https://github.com/vllm-project/vllm/issues/43224.
"""

import pytest

from vllm import LLM, SamplingParams
from vllm.config import (
CompilationConfig,
CompilationMode,
CUDAGraphMode,
PassConfig,
)
from vllm.platforms import current_platform

from .conftest import count_matching

pytestmark = pytest.mark.skipif(
not current_platform.is_cuda(), reason="CUDA-only fusion path"
)


def _build_llm(model: str, tp_size: int, n_layers: int = 4) -> LLM:
"""Build an LLM with every compiler fusion pass off.

Manual fusion must work without any help from the compiler. Forcing all
``PassConfig`` flags off (and disabling the compile cache) leaves the
fused op call sites in model code as the only thing that can produce a
fused kernel invocation.
"""
return LLM(
model=model,
tensor_parallel_size=tp_size,
load_format="dummy",
hf_overrides={"num_hidden_layers": n_layers},
max_model_len=1024,
# Stripped n_layers model is tiny; don't reserve the whole GPU so
# consecutive parametrized tests can run in the same process.
gpu_memory_utilization=0.2,
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
cudagraph_mode=CUDAGraphMode.NONE,
inductor_compile_config={"force_disable_caches": True},
pass_config=PassConfig(
fuse_norm_quant=False,
fuse_act_quant=False,
fuse_attn_quant=False,
fuse_allreduce_rms=False,
enable_qk_norm_rope_fusion=False,
enable_sp=False,
fuse_gemm_comms=False,
),
),
disable_custom_all_reduce=True,
)


def _generate(llm: LLM) -> None:
llm.generate(
["The capital of France is"],
SamplingParams(temperature=0, max_tokens=32),
)


@pytest.mark.parametrize(
"model,expected_ops",
[
# FP8 static-per-tensor: helper dispatches to the C++ norm+quant kernel
# for every decoder layer with a residual; the first layer (residual is
# None) takes the rms_norm_static_fp8_quant path. Lower-bound check
# only — exact count depends on layer count and decode steps.
pytest.param(
"RedHatAI/Llama-3.2-1B-Instruct-FP8",
("fused_add_rms_norm_static_fp8_quant",),
id="llama-3.2-1b-fp8",
),
],
)
def test_tp1_manual_fusion(op_count_session, model: str, expected_ops: tuple[str, ...]):
llm = _build_llm(model, tp_size=1)
with op_count_session(llm) as counts:
_generate(llm)

for op_substr in expected_ops:
assert count_matching(counts, op_substr) > 0, (
f"Manual fusion did not fire: no calls matching '{op_substr}'. "
f"counts={dict(counts)}"
)


# TODO(@mgoin): TP=2 variant.
#
# The TP>=2 + residual path dispatches to
# ``flashinfer_trtllm_fused_allreduce_norm``, which is a Python wrapper, not
# a torch op — TorchDispatchMode cannot see it. To assert it fired, add a
# parallel "python-call spy" alongside the dispatch counter that monkey-
# patches the name in ``vllm.model_executor.layers.fusion.ar_rms_quant``
# inside each worker, then check it was called at least once.
47 changes: 43 additions & 4 deletions tests/fusion/test_quant_activation_contract.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,14 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Contract tests for the QuantizedActivation linear-kernel integration."""
"""Contract tests for the ``QuantizedActivation`` manual-fusion integration.

Pure-Python introspection over the linear-kernel registry -- no model load, no
device -- pinning which backends consume a pre-quantized activation and how the
bridge / consumer enforce the key. Tracks the AR+RMS+Quant bullet under
https://github.com/vllm-project/vllm/issues/43224.
"""

from types import SimpleNamespace

import pytest
import torch
Expand Down Expand Up @@ -30,6 +38,15 @@
Int8ScaledMMLinearKernel,
Int8ScaledMMLinearLayerConfig,
)
from vllm.model_executor.layers.fusion.ar_rms_quant import (
_FUSED_AR_RMS_QUANT_IMPLS,
_FUSED_RMS_QUANT_IMPLS,
_flashinfer_ar_rms_fp8,
_flashinfer_ar_rms_nvfp4,
_rms_norm_fp8_static,
_to_fp8_qa,
_to_nvfp4_qa,
)
from vllm.model_executor.layers.fusion.quant_activation import (
QuantizedActivation,
as_quantized_activation,
Expand All @@ -41,7 +58,8 @@
)
from vllm.platforms import current_platform

# The only backends that consume a pre-quantized activation.
# The only backends that consume a pre-quantized activation. Anything else must
# quantize its own input, so the bridge must not mark its layer.
SUPPORTING = {
CutlassFP8ScaledMMLinearKernel,
FlashInferFP8ScaledMMLinearKernel,
Expand All @@ -64,8 +82,8 @@ def _all_kernel_classes() -> list[type]:


def _probe(cls: type):
"""A bare kernel instance with a plausible config, so input_quant_key()
can be queried without the hardware-gated constructor."""
"""A bare kernel instance with a plausible config, so ``input_quant_key()``
can be queried without the hardware-gated ``__init__``."""
obj = cls.__new__(cls) # type: ignore[call-overload]
if issubclass(cls, NvFp4LinearKernel):
obj.config = NvFp4LinearLayerConfig()
Expand Down Expand Up @@ -115,6 +133,27 @@ def test_bridge_marks_supporting_and_skips_others():
assert not hasattr(layer, "input_quant_key")


def test_fp8_producer_stamps_registered_key():
qa = _to_fp8_qa(
torch.zeros(2, 4, dtype=current_platform.fp8_dtype()),
torch.zeros(2, 4),
SimpleNamespace(input_scale=torch.tensor(1.0)),
)
assert qa.quant_key == kFp8StaticTensorSym
assert _FUSED_RMS_QUANT_IMPLS[kFp8StaticTensorSym] is _rms_norm_fp8_static
assert _FUSED_AR_RMS_QUANT_IMPLS[kFp8StaticTensorSym] is _flashinfer_ar_rms_fp8


def test_nvfp4_producer_stamps_registered_key():
qa = _to_nvfp4_qa(
torch.zeros(2, 2, dtype=torch.uint8), # 2 fp4 packed per byte
torch.zeros(2, 4, dtype=torch.uint8), # viewed as float8_e4m3fn
torch.zeros(2, 4),
)
assert qa.quant_key == kNvfp4Dynamic
assert _FUSED_AR_RMS_QUANT_IMPLS[kNvfp4Dynamic] is _flashinfer_ar_rms_nvfp4


def test_as_quantized_activation_validates_key():
qa = QuantizedActivation(
data=torch.zeros(2, 4, dtype=current_platform.fp8_dtype()),
Expand Down
Loading
Loading