Skip to content

[Bugfix][Quantization] Fix Humming FP8 linear for per-tensor/per-channel FP8 weights #60163

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yewentao256 merged 12 commits into from
Oct 29, 2026
Merged

[Bugfix][Quantization] Fix Humming FP8 linear for per-tensor/per-channel FP8 weights #60163

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
da8d30a
[Bugfix][Quantization] Fix Humming FP8 linear for per-tensor/per-chan…
Manfredss Oct 6, 2026
95db3aa
Merge branch 'main' into fix/fp8-humming-weight-layout
Manfredss Oct 6, 2026
671702b
Merge branch 'main' into fix/fp8-humming-weight-layout
Manfredss Oct 14, 2026
374400e
Merge branch 'main' into fix/fp8-humming-weight-layout
Manfredss Oct 26, 2026
970068e
Reject Humming FP8 for online PTPC, like Marlin
Manfredss Oct 26, 2026
606a1e6
Tag the (K, N) weight layout with processed_weight_dims
Manfredss Oct 26, 2026
ac4440d
Merge branch 'main' into fix/fp8-humming-weight-layout
mergify[bot] Oct 27, 2026
95d2bdc
Merge branch 'main' into fix/fp8-humming-weight-layout
Manfredss Oct 27, 2026
19a4774
Merge branch 'main' into fix/fp8-humming-weight-layout
yewentao256 Oct 28, 2026
df776f2
Merge branch 'main' into fix/fp8-humming-weight-layout
Manfredss Oct 28, 2026
833c5f3
Merge branch 'main' into fix/fp8-humming-weight-layout
vllm-agent Oct 28, 2026
d88d84d
Merge branch 'main' into fix/fp8-humming-weight-layout
Manfredss Oct 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions tests/quantization/test_fp8.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -579,14 +579,15 @@ def test_static_fp8_moe_input_scales_remain_scalar() -> None:
)
@pytest.mark.parametrize("model_id", MODELS)
@pytest.mark.parametrize(
"force_marlin", [True, False] if current_platform.is_cuda() else [False]
"linear_backend",
["auto", "marlin", "humming"] if current_platform.is_cuda() else ["auto"],
)
@pytest.mark.parametrize(
"use_rocm_aiter", [True, False] if current_platform.is_rocm() else [False]
)
def test_model_load_and_run(
model_id: str,
force_marlin: bool,
linear_backend: str,
use_rocm_aiter: bool,
monkeypatch,
dist_init,
Expand All @@ -596,8 +597,8 @@ def test_model_load_and_run(
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")

kernel_config = KernelConfig(
linear_backend="marlin" if force_marlin else "auto",
moe_backend="marlin" if force_marlin else "auto",
linear_backend=linear_backend,
moe_backend="marlin" if linear_backend == "marlin" else "auto",
)
model, vllm_config = load_model_without_vllm_runner(
model_id,
Expand Down
11 changes: 10 additions & 1 deletion vllm/model_executor/layers/quantization/fp8.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,9 @@
CutlassFP8ScaledMMLinearKernel,
MarlinFP8ScaledMMLinearKernel,
)
from vllm.model_executor.kernels.linear.scaled_mm.humming import (
HummingFP8ScaledMMLinearKernel,
)
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoEMethodBase,
Expand Down Expand Up @@ -406,6 +409,9 @@ def process_weights_after_loading(self, layer: torch.nn.Module) -> None:

# Update layer with new values.
replace_parameter(layer, "weight", weight.data)
# The helper returns (K, N); retag so layout-aware kernels see it.
layer.weight.input_dim = 0
layer.weight.output_dim = 1
replace_parameter(layer, "weight_scale", weight_scale.data)

if input_scale is not None:
Expand All @@ -432,7 +438,10 @@ def apply(
bias,
)
else:
if isinstance(self.fp8_linear, CutlassFP8ScaledMMLinearKernel):
if isinstance(
self.fp8_linear,
(CutlassFP8ScaledMMLinearKernel, HummingFP8ScaledMMLinearKernel),
):
return self.fp8_linear.apply_weights(layer, x, bias)

# per-tensor/channel: dequant to BF16 and run GEMM
Expand Down
19 changes: 17 additions & 2 deletions vllm/model_executor/layers/quantization/online/fp8.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@
CutlassFP8ScaledMMLinearKernel,
MarlinFP8ScaledMMLinearKernel,
)
from vllm.model_executor.kernels.linear.scaled_mm.humming import (
HummingFP8ScaledMMLinearKernel,
)
from vllm.model_executor.layers.fused_moe import RoutedExperts
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
Expand Down Expand Up @@ -220,6 +223,9 @@ def process_weights_after_loading(self, layer: Module) -> None:

# Update layer with new values.
replace_parameter(layer, "weight", qweight.t().data)
# Transposed to (K, N); retag so layout-aware kernels see it.
layer.weight.input_dim = 0
layer.weight.output_dim = 1
replace_parameter(layer, "weight_scale", weight_scale.data)

if self.use_marlin and hasattr(self.fp8_linear, "marlin_input_dtype"):
Expand All @@ -237,7 +243,10 @@ def apply(
) -> torch.Tensor:
# if batch invariant mode is enabled, use BF16 dequant
if envs.VLLM_BATCH_INVARIANT:
if isinstance(self.fp8_linear, CutlassFP8ScaledMMLinearKernel):
if isinstance(
self.fp8_linear,
(CutlassFP8ScaledMMLinearKernel, HummingFP8ScaledMMLinearKernel),
):
return self.fp8_linear.apply_weights(layer, x, bias)

weight_fp8 = layer.weight.to(torch.bfloat16)
Expand Down Expand Up @@ -403,6 +412,9 @@ def process_weights_after_loading(self, layer: Module) -> None:
qweight = _fp8_quant_per_channel(layer.weight, weight_scale)

replace_parameter(layer, "weight", qweight.t())
# Transposed to (K, N); retag so layout-aware kernels see it.
layer.weight.input_dim = 0
layer.weight.output_dim = 1

@yewentao256 yewentao256 Oct 25, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please take a look at this AI generated comment:

One question about the PTPC path: `Fp8PtpcOnlineLinearMethod` requires per-token FP8 activation quantization, but Humming appears to use a weight-only path with FP16/BF16 activations.

With this layout fix, PTPC can now successfully run through Humming. Does Humming preserve the requested activation quantization semantics here, or should we reject this combination as we already do for Marlin?

@Manfredss Manfredss Oct 26, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good catch, it doesn't. HummingFP8ScaledMMLinearKernel.process_weights_after_loading calls prepare_humming_linear_layer_config() without an input schema. Humming then uses its default HummingInputSchema() (a_dtype=None), which means 16-bit activations with input quantization disabled. That is W8A16, the same as Marlin FP8. With the layout fix alone (the earlier version of this PR), PTPC would silently run as weight-only FP8 whenever it lands on Humming. That happens in two cases:

  • on Ampere with VLLM_BATCH_INVARIANT=1, where the other FP8 kernels need SM89+ and Marlin opts out;
  • with --linear-backend humming.

970068e adds Humming to the existing Marlin check in Fp8PtpcOnlineLinearMethod.create_weights, so PTPC now raises the same ValueError, and the message names the selected kernel. The commit also drops the layout tag that the earlier version set in PTPC, since PTPC no longer reaches Humming. Fp8LinearMethod and fp8_per_tensor still run on Humming. They already fall back to W8A16 Marlin on GPUs without FP8 support, so Humming matches their existing behavior.

A40, nightly 00b6c4c5:

VLLM_BATCH_INVARIANT=1 vllm serve Qwen/Qwen3-8B --port 8000 --max-model-len 4096 --gpu-memory-utilization 0.9 --quantization fp8_per_channel

main

(EngineCore pid=5214) INFO 10-26 21:37:40 [__init__.py:817] Selected HummingFP8ScaledMMLinearKernel for Fp8PtpcOnlineLinearMethod
(EngineCore pid=5214) ERROR 10-26 21:37:42 [core.py:1574] RuntimeError: self.stride(-1) must be 1 to view Float8_e4m3fn as Int (different element sizes), but got 12288

layout fix applied to PTPC too, without the new check

(EngineCore pid=5749) INFO 10-26 21:38:22 [__init__.py:817] Selected HummingFP8ScaledMMLinearKernel for Fp8PtpcOnlineLinearMethod
(APIServer pid=5423) INFO 10-26 21:39:27 [entry.py:139] Starting vLLM server on http://0.0.0.0:8000

this PR

(EngineCore pid=6497) INFO 10-26 21:40:02 [__init__.py:817] Selected HummingFP8ScaledMMLinearKernel for Fp8PtpcOnlineLinearMethod
(EngineCore pid=6497) ERROR 10-26 21:40:03 [core.py:1574] ValueError: FP8 PTPC online quant requires a kernel that honors per-token activation quantization; HummingFP8ScaledMMLinearKernel is W8A16 weight-only. Requires SM89+ for Cutlass FP8 or ROCm MI3xx for rowwise scaled_mm.

this PR, without BI: vllm serve Qwen/Qwen3-8B --port 8000 --max-model-len 4096 --gpu-memory-utilization 0.9 --quantization fp8_per_channel --linear-backend humming

(EngineCore pid=7034) INFO 10-26 21:40:41 [__init__.py:817] Selected HummingFP8ScaledMMLinearKernel for Fp8PtpcOnlineLinearMethod
(EngineCore pid=7034) ERROR 10-26 21:40:42 [core.py:1574] ValueError: FP8 PTPC online quant requires a kernel that honors per-token activation quantization; HummingFP8ScaledMMLinearKernel is W8A16 weight-only. Requires SM89+ for Cutlass FP8 or ROCm MI3xx for rowwise scaled_mm.

main, same command with --linear-backend marlin, for comparison

(EngineCore pid=7536) INFO 10-26 21:41:17 [__init__.py:817] Selected MarlinFP8ScaledMMLinearKernel for Fp8PtpcOnlineLinearMethod
(EngineCore pid=7536) ERROR 10-26 21:41:18 [core.py:1574] ValueError: FP8 PTPC online quant requires a kernel that honors per-token activation quantization; MarlinFP8 is W8A16 weight-only. Requires SM89+ for Cutlass FP8 or ROCm MI3xx for rowwise scaled_mm.

replace_parameter(layer, "weight_scale", weight_scale)

self.fp8_linear.process_weights_after_loading(layer)
Expand All @@ -419,7 +431,10 @@ def apply(
# if batch invariant mode is enabled dequant
if (
envs.VLLM_BATCH_INVARIANT
and not isinstance(self.fp8_linear, CutlassFP8ScaledMMLinearKernel)
and not isinstance(
self.fp8_linear,
(CutlassFP8ScaledMMLinearKernel, HummingFP8ScaledMMLinearKernel),
)
and not isinstance(x, QuantizedActivation)
):
weight_dequant = (
Expand Down
Loading