From f6c01cfc81d35623d203c7caff2af63377c1b0f0 Mon Sep 17 00:00:00 2001 From: waizuichougou <2082431897@qq.com> Date: Sat, 12 Sep 2026 20:12:09 +0800 Subject: [PATCH 1/3] [Bugfix][Quantization] Fix online MoE conversion with Humming Signed-off-by: waizuichougou <2082431897@qq.com> --- tests/quantization/test_online.py | 78 +++++++++++++++++++ .../layers/quantization/online/fp8.py | 4 + .../layers/quantization/online/moe_base.py | 22 +++++- .../layers/quantization/online/mxfp4.py | 3 + .../layers/quantization/online/mxfp8.py | 9 +-- 5 files changed, 110 insertions(+), 6 deletions(-) diff --git a/tests/quantization/test_online.py b/tests/quantization/test_online.py index fbd5cbc56392..bb70a747dd53 100644 --- a/tests/quantization/test_online.py +++ b/tests/quantization/test_online.py @@ -36,6 +36,8 @@ ) from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.fused_moe import FusedMoEFactory +from vllm.model_executor.layers.fused_moe.oracle.fp8 import Fp8MoeBackend +from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import Mxfp4MoeBackend from vllm.model_executor.layers.linear import ( ColumnParallelLinear, LinearBase, @@ -75,6 +77,7 @@ ) from vllm.model_executor.layers.quantization.online.mxfp8 import ( Mxfp8OnlineLinearMethod, + Mxfp8OnlineMoEMethod, ) from vllm.model_executor.layers.quantization.online.nvfp4 import ( Nvfp4OnlineMoEMethod, @@ -124,6 +127,81 @@ def on_gfx942() -> bool: ) +@pytest.mark.parametrize( + "method_cls,backend_attr,backend,converter_path", + [ + pytest.param( + Fp8PerTensorOnlineMoEMethod, + "fp8_backend", + Fp8MoeBackend.HUMMING, + "vllm.model_executor.layers.fused_moe.oracle.fp8." + "convert_to_fp8_moe_kernel_format", + id="fp8", + ), + pytest.param( + Mxfp8OnlineMoEMethod, + "fp8_backend", + Fp8MoeBackend.HUMMING, + "vllm.model_executor.layers.fused_moe.oracle.fp8." + "convert_to_fp8_moe_kernel_format", + id="mxfp8", + ), + pytest.param( + Mxfp4OnlineMoEMethod, + "mxfp4_backend", + Mxfp4MoeBackend.HUMMING, + "vllm.model_executor.layers.quantization.online.mxfp4." + "convert_weight_to_mxfp4_moe_kernel_format", + id="mxfp4", + ), + ], +) +def test_online_moe_stages_quantized_weights_for_humming( + method_cls, + backend_attr: str, + backend, + converter_path: str, + monkeypatch, +) -> None: + method = object.__new__(method_cls) + setattr(method, backend_attr, backend) + method.weight_scale_name = "weight_scale" + method.experts_cls = None + method.get_fused_moe_quant_config = Mock(return_value=None) + + layer = torch.nn.Module() + layer.register_parameter( + "w13_weight", + torch.nn.Parameter(torch.empty(2, 8, 4, dtype=torch.bfloat16)), + ) + layer.register_parameter( + "w2_weight", + torch.nn.Parameter(torch.empty(2, 4, 4, dtype=torch.bfloat16)), + ) + is_mxfp4 = method_cls is Mxfp4OnlineMoEMethod + weight_dtype = torch.uint8 if is_mxfp4 else torch.float8_e4m3fn + scale_dtype = ( + torch.float32 if method_cls is Fp8PerTensorOnlineMoEMethod else torch.uint8 + ) + packed_factor = 2 if is_mxfp4 else 1 + w13 = torch.empty(2, 8, 4 // packed_factor, dtype=weight_dtype) + w2 = torch.empty(2, 4, 4 // packed_factor, dtype=weight_dtype) + w13_scale = torch.empty(2, 8, 1, dtype=scale_dtype) + w2_scale = torch.empty(2, 4, 1, dtype=scale_dtype) + + def convert(**kwargs): + state = layer.state_dict() + assert state["w13_weight"].data_ptr() == w13.data_ptr() + assert state["w2_weight"].data_ptr() == w2.data_ptr() + assert state["w13_weight_scale"].data_ptr() == w13_scale.data_ptr() + assert state["w2_weight_scale"].data_ptr() == w2_scale.data_ptr() + converted = (w13, w2, w13_scale, w2_scale) + return (*converted, None, None) if is_mxfp4 else converted + + monkeypatch.setattr(converter_path, convert) + method._setup_kernel(layer, w13, w2, w13_scale, w2_scale, None, None) + + def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed( monkeypatch, ) -> None: diff --git a/vllm/model_executor/layers/quantization/online/fp8.py b/vllm/model_executor/layers/quantization/online/fp8.py index 0234702c8bab..4fb33a47c207 100644 --- a/vllm/model_executor/layers/quantization/online/fp8.py +++ b/vllm/model_executor/layers/quantization/online/fp8.py @@ -25,6 +25,7 @@ from vllm.model_executor.layers.fused_moe import RoutedExperts from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig from vllm.model_executor.layers.fused_moe.oracle.fp8 import ( + Fp8MoeBackend, select_fp8_moe_backend, ) from vllm.model_executor.layers.linear import ( @@ -489,6 +490,9 @@ def _setup_kernel( make_fp8_moe_kernel, ) + if self.fp8_backend == Fp8MoeBackend.HUMMING: + self._stage_humming_quantized_weights(layer, w13, w2, w13_scale, w2_scale) + # Shuffle weights to runtime format. w13, w2, w13_scale, w2_scale = convert_to_fp8_moe_kernel_format( fp8_backend=self.fp8_backend, diff --git a/vllm/model_executor/layers/quantization/online/moe_base.py b/vllm/model_executor/layers/quantization/online/moe_base.py index 866c88b3d649..1c202c0d4e55 100644 --- a/vllm/model_executor/layers/quantization/online/moe_base.py +++ b/vllm/model_executor/layers/quantization/online/moe_base.py @@ -14,7 +14,7 @@ from vllm.model_executor.model_loader.reload.layerwise import ( initialize_online_processing, ) -from vllm.model_executor.utils import set_weight_attrs +from vllm.model_executor.utils import replace_parameter, set_weight_attrs class OnlineMoEMethodBase(FusedMoEMethodBase): @@ -126,6 +126,26 @@ def _zero_padding(self, layer: torch.nn.Module) -> None: ): layer.w2_bias[:, hidden_size:] = 0 + @staticmethod + def _stage_humming_quantized_weights( + layer: torch.nn.Module, + w13: torch.Tensor, + w2: torch.Tensor, + w13_scale: torch.Tensor, + w2_scale: torch.Tensor, + ) -> None: + """Expose online-quantized tensors to Humming's layer-based converter. + + Other MoE format converters consume the tensors passed to them directly. + Humming instead discovers its input through ``layer.state_dict()``. Stage + the canonical tensors on the layer so it does not inspect the original + full-precision online-loading parameters. + """ + replace_parameter(layer, "w13_weight", w13) + replace_parameter(layer, "w2_weight", w2) + replace_parameter(layer, "w13_weight_scale", w13_scale) + replace_parameter(layer, "w2_weight_scale", w2_scale) + @abstractmethod def process_weights_after_loading(self, layer: torch.nn.Module) -> None: pass diff --git a/vllm/model_executor/layers/quantization/online/mxfp4.py b/vllm/model_executor/layers/quantization/online/mxfp4.py index 0115988e9265..ce5cea7c179c 100644 --- a/vllm/model_executor/layers/quantization/online/mxfp4.py +++ b/vllm/model_executor/layers/quantization/online/mxfp4.py @@ -210,6 +210,9 @@ def _setup_kernel( w13_bias: torch.Tensor | None, w2_bias: torch.Tensor | None, ) -> None: + if self.mxfp4_backend == Mxfp4MoeBackend.HUMMING: + self._stage_humming_quantized_weights(layer, w13, w2, w13_scale, w2_scale) + w13, w2, w13_scale, w2_scale, w13_bias, w2_bias = ( convert_weight_to_mxfp4_moe_kernel_format( mxfp4_backend=self.mxfp4_backend, diff --git a/vllm/model_executor/layers/quantization/online/mxfp8.py b/vllm/model_executor/layers/quantization/online/mxfp8.py index 5165762d5b88..11a329ab3ad2 100644 --- a/vllm/model_executor/layers/quantization/online/mxfp8.py +++ b/vllm/model_executor/layers/quantization/online/mxfp8.py @@ -16,9 +16,9 @@ FusedMoEQuantConfig, RoutedExperts, ) - from vllm.model_executor.layers.fused_moe.oracle.fp8 import Fp8MoeBackend from vllm.model_executor.kernels.linear import init_mxfp8_linear_kernel +from vllm.model_executor.layers.fused_moe.oracle.fp8 import Fp8MoeBackend from vllm.model_executor.layers.fused_moe.oracle.mxfp8 import ( select_mxfp8_moe_backend, ) @@ -33,7 +33,6 @@ mxfp8_e4m3_quantize, ) from vllm.model_executor.utils import replace_parameter -from vllm.platforms import current_platform class Mxfp8OnlineLinearMethod(OnlineLinearBase): @@ -176,6 +175,9 @@ def _setup_kernel( make_fp8_moe_kernel, ) + if self.fp8_backend == Fp8MoeBackend.HUMMING: + self._stage_humming_quantized_weights(layer, w13, w2, w13_scale, w2_scale) + # Shuffle weights to runtime format. w13, w2, w13_scale, w2_scale = convert_to_fp8_moe_kernel_format( fp8_backend=self.fp8_backend, @@ -235,9 +237,6 @@ def process_weights_after_loading(self, layer: Module) -> None: if getattr(layer, "_already_called_process_weights_after_loading", False): return - fp8_dtype = current_platform.fp8_dtype() - w13 = torch.empty_like(layer.w13_weight, dtype=fp8_dtype) - w2 = torch.empty_like(layer.w2_weight, dtype=fp8_dtype) layer.w13_input_scale = None layer.w2_input_scale = None From 6074d2dce85414d59fe2c71e2ad49f53b7c2952e Mon Sep 17 00:00:00 2001 From: waizuichougou <2082431897@qq.com> Date: Thu, 24 Sep 2026 21:06:07 +0800 Subject: [PATCH 2/3] [Bugfix][Quantization] Fix online MoE import ordering Apply the Ruff import ordering required by the pre-commit checks. Co-authored-by: OpenAI Codex Signed-off-by: waizuichougou <2082431897@qq.com> --- tests/quantization/test_online.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/quantization/test_online.py b/tests/quantization/test_online.py index 1d7c9df98dd8..41e6ddedb4c1 100644 --- a/tests/quantization/test_online.py +++ b/tests/quantization/test_online.py @@ -36,9 +36,9 @@ ) from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.fused_moe import FusedMoEFactory +from vllm.model_executor.layers.fused_moe.oracle import nvfp4 as nvfp4_oracle from vllm.model_executor.layers.fused_moe.oracle.fp8 import Fp8MoeBackend from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import Mxfp4MoeBackend -from vllm.model_executor.layers.fused_moe.oracle import nvfp4 as nvfp4_oracle from vllm.model_executor.layers.linear import ( ColumnParallelLinear, LinearBase, From 6f80706b576a2e404e7cb42fb1a53daae2042ba9 Mon Sep 17 00:00:00 2001 From: waizuichougou <2082431897@qq.com> Date: Mon, 12 Oct 2026 18:03:15 +0800 Subject: [PATCH 3/3] Fix Humming prequantized MoE weight handling Signed-off-by: waizuichougou <2082431897@qq.com> --- tests/quantization/test_online.py | 1 + .../layers/fused_moe/oracle/fp8.py | 16 ---------------- 2 files changed, 1 insertion(+), 16 deletions(-) diff --git a/tests/quantization/test_online.py b/tests/quantization/test_online.py index a0c2ab47ee3e..a7720c5c93be 100644 --- a/tests/quantization/test_online.py +++ b/tests/quantization/test_online.py @@ -181,6 +181,7 @@ def test_online_moe_stages_quantized_weights_for_humming( setattr(method, backend_attr, backend) method.weight_scale_name = "weight_scale" method.experts_cls = None + method.moe = SimpleNamespace(use_mxfp4_w4a4_dsv4=False) method.get_fused_moe_quant_config = Mock(return_value=None) layer = torch.nn.Module() diff --git a/vllm/model_executor/layers/fused_moe/oracle/fp8.py b/vllm/model_executor/layers/fused_moe/oracle/fp8.py index 446825c59047..208fa529ded0 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/fp8.py +++ b/vllm/model_executor/layers/fused_moe/oracle/fp8.py @@ -38,7 +38,6 @@ kFp8Dynamic128Sym, kFp8Static128BlockSym, ) -from vllm.model_executor.utils import replace_parameter from vllm.platforms import current_platform from vllm.utils.math_utils import round_up @@ -610,21 +609,6 @@ def convert_to_fp8_moe_kernel_format( convert_to_humming_moe_kernel_format, ) - # Online quantization has not installed the quantized tensors yet. - scale_name = ( - "weight_scale_inv" - if hasattr(layer, "w13_weight_scale_inv") - else "weight_scale" - ) - for prefix, weight, scale, input_scale in ( - ("w13", w13, w13_scale, w13_input_scale), - ("w2", w2, w2_scale, w2_input_scale), - ): - replace_parameter(layer, f"{prefix}_weight", weight) - replace_parameter(layer, f"{prefix}_{scale_name}", scale) - if input_scale is not None: - replace_parameter(layer, f"{prefix}_input_scale", input_scale) - convert_to_humming_moe_kernel_format( layer, quant_config=_humming_fp8_weight_schema(layer, w13, w13_scale) )