diff --git a/tests/quantization/test_online.py b/tests/quantization/test_online.py index f18814dcd8ab..c0bdf9a4f3ac 100644 --- a/tests/quantization/test_online.py +++ b/tests/quantization/test_online.py @@ -44,6 +44,8 @@ from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.fused_moe import FusedMoEFactory, RoutedExperts from vllm.model_executor.layers.fused_moe.oracle import nvfp4 as nvfp4_oracle +from vllm.model_executor.layers.fused_moe.oracle.fp8 import Fp8MoeBackend +from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import Mxfp4MoeBackend from vllm.model_executor.layers.linear import ( ColumnParallelLinear, LinearBase, @@ -161,6 +163,82 @@ def on_gfx942() -> bool: MODELOPT_NVFP4_MOE_MODEL_NAME = "nvidia/Qwen3-30B-A3B-NVFP4" +@pytest.mark.parametrize( + "method_cls,backend_attr,backend,converter_path", + [ + pytest.param( + Fp8PerTensorOnlineMoEMethod, + "fp8_backend", + Fp8MoeBackend.HUMMING, + "vllm.model_executor.layers.fused_moe.oracle.fp8." + "convert_to_fp8_moe_kernel_format", + id="fp8", + ), + pytest.param( + Mxfp8OnlineMoEMethod, + "fp8_backend", + Fp8MoeBackend.HUMMING, + "vllm.model_executor.layers.fused_moe.oracle.fp8." + "convert_to_fp8_moe_kernel_format", + id="mxfp8", + ), + pytest.param( + Mxfp4OnlineMoEMethod, + "mxfp4_backend", + Mxfp4MoeBackend.HUMMING, + "vllm.model_executor.layers.quantization.online.mxfp4." + "convert_weight_to_mxfp4_moe_kernel_format", + id="mxfp4", + ), + ], +) +def test_online_moe_stages_quantized_weights_for_humming( + method_cls, + backend_attr: str, + backend, + converter_path: str, + monkeypatch, +) -> None: + method = object.__new__(method_cls) + setattr(method, backend_attr, backend) + method.weight_scale_name = "weight_scale" + method.experts_cls = None + method.moe = SimpleNamespace(use_mxfp4_w4a4_dsv4=False) + method.get_fused_moe_quant_config = Mock(return_value=None) + + layer = torch.nn.Module() + layer.register_parameter( + "w13_weight", + torch.nn.Parameter(torch.empty(2, 8, 4, dtype=torch.bfloat16)), + ) + layer.register_parameter( + "w2_weight", + torch.nn.Parameter(torch.empty(2, 4, 4, dtype=torch.bfloat16)), + ) + is_mxfp4 = method_cls is Mxfp4OnlineMoEMethod + weight_dtype = torch.uint8 if is_mxfp4 else torch.float8_e4m3fn + scale_dtype = ( + torch.float32 if method_cls is Fp8PerTensorOnlineMoEMethod else torch.uint8 + ) + packed_factor = 2 if is_mxfp4 else 1 + w13 = torch.empty(2, 8, 4 // packed_factor, dtype=weight_dtype) + w2 = torch.empty(2, 4, 4 // packed_factor, dtype=weight_dtype) + w13_scale = torch.empty(2, 8, 1, dtype=scale_dtype) + w2_scale = torch.empty(2, 4, 1, dtype=scale_dtype) + + def convert(**kwargs): + state = layer.state_dict() + assert state["w13_weight"].data_ptr() == w13.data_ptr() + assert state["w2_weight"].data_ptr() == w2.data_ptr() + assert state["w13_weight_scale"].data_ptr() == w13_scale.data_ptr() + assert state["w2_weight_scale"].data_ptr() == w2_scale.data_ptr() + converted = (w13, w2, w13_scale, w2_scale) + return (*converted, None, None) if is_mxfp4 else converted + + monkeypatch.setattr(converter_path, convert) + method._setup_kernel(layer, w13, w2, w13_scale, w2_scale, None, None) + + @pytest.mark.parametrize( "overrides", [ diff --git a/vllm/model_executor/layers/fused_moe/oracle/fp8.py b/vllm/model_executor/layers/fused_moe/oracle/fp8.py index 074acd2a0942..a46e6b5d133f 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/fp8.py +++ b/vllm/model_executor/layers/fused_moe/oracle/fp8.py @@ -38,7 +38,6 @@ kFp8Dynamic128Sym, kFp8Static128BlockSym, ) -from vllm.model_executor.utils import replace_parameter from vllm.platforms import current_platform from vllm.utils.math_utils import round_up @@ -612,21 +611,6 @@ def convert_to_fp8_moe_kernel_format( convert_to_humming_moe_kernel_format, ) - # Online quantization has not installed the quantized tensors yet. - scale_name = ( - "weight_scale_inv" - if hasattr(layer, "w13_weight_scale_inv") - else "weight_scale" - ) - for prefix, weight, scale, input_scale in ( - ("w13", w13, w13_scale, w13_input_scale), - ("w2", w2, w2_scale, w2_input_scale), - ): - replace_parameter(layer, f"{prefix}_weight", weight) - replace_parameter(layer, f"{prefix}_{scale_name}", scale) - if input_scale is not None: - replace_parameter(layer, f"{prefix}_input_scale", input_scale) - convert_to_humming_moe_kernel_format( layer, quant_config=_humming_fp8_weight_schema(layer, w13, w13_scale) ) diff --git a/vllm/model_executor/layers/quantization/online/fp8.py b/vllm/model_executor/layers/quantization/online/fp8.py index 905155f5d819..0709b8d3f756 100644 --- a/vllm/model_executor/layers/quantization/online/fp8.py +++ b/vllm/model_executor/layers/quantization/online/fp8.py @@ -25,6 +25,7 @@ from vllm.model_executor.layers.fused_moe import RoutedExperts from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig from vllm.model_executor.layers.fused_moe.oracle.fp8 import ( + Fp8MoeBackend, select_fp8_moe_backend, ) from vllm.model_executor.layers.fusion.quant_activation import ( @@ -531,6 +532,9 @@ def _setup_kernel( make_fp8_moe_kernel, ) + if self.fp8_backend == Fp8MoeBackend.HUMMING: + self._stage_humming_quantized_weights(layer, w13, w2, w13_scale, w2_scale) + # Shuffle weights to runtime format. w13, w2, w13_scale, w2_scale = convert_to_fp8_moe_kernel_format( fp8_backend=self.fp8_backend, diff --git a/vllm/model_executor/layers/quantization/online/moe_base.py b/vllm/model_executor/layers/quantization/online/moe_base.py index eb175ebd33d8..69af6202048b 100644 --- a/vllm/model_executor/layers/quantization/online/moe_base.py +++ b/vllm/model_executor/layers/quantization/online/moe_base.py @@ -18,7 +18,7 @@ from vllm.model_executor.model_loader.reload.layerwise import ( initialize_online_processing, ) -from vllm.model_executor.utils import set_weight_attrs +from vllm.model_executor.utils import replace_parameter, set_weight_attrs class OnlineMoEMethodBase(OnlineRequantizationMixin, FusedMoEMethodBase): @@ -218,6 +218,26 @@ def _zero_padding( ): layer.w2_bias[:, hidden_size:] = 0 + @staticmethod + def _stage_humming_quantized_weights( + layer: torch.nn.Module, + w13: torch.Tensor, + w2: torch.Tensor, + w13_scale: torch.Tensor, + w2_scale: torch.Tensor, + ) -> None: + """Expose online-quantized tensors to Humming's layer-based converter. + + Other MoE format converters consume the tensors passed to them directly. + Humming instead discovers its input through ``layer.state_dict()``. Stage + the canonical tensors on the layer so it does not inspect the original + full-precision online-loading parameters. + """ + replace_parameter(layer, "w13_weight", w13) + replace_parameter(layer, "w2_weight", w2) + replace_parameter(layer, "w13_weight_scale", w13_scale) + replace_parameter(layer, "w2_weight_scale", w2_scale) + @abstractmethod def process_weights_after_loading(self, layer: torch.nn.Module) -> None: pass diff --git a/vllm/model_executor/layers/quantization/online/mxfp4.py b/vllm/model_executor/layers/quantization/online/mxfp4.py index 70a64340f694..beb0ada46a50 100644 --- a/vllm/model_executor/layers/quantization/online/mxfp4.py +++ b/vllm/model_executor/layers/quantization/online/mxfp4.py @@ -212,6 +212,9 @@ def _setup_kernel( w13_bias: torch.Tensor | None, w2_bias: torch.Tensor | None, ) -> None: + if self.mxfp4_backend == Mxfp4MoeBackend.HUMMING: + self._stage_humming_quantized_weights(layer, w13, w2, w13_scale, w2_scale) + w13, w2, w13_scale, w2_scale, w13_bias, w2_bias = ( convert_weight_to_mxfp4_moe_kernel_format( mxfp4_backend=self.mxfp4_backend, diff --git a/vllm/model_executor/layers/quantization/online/mxfp8.py b/vllm/model_executor/layers/quantization/online/mxfp8.py index f16f2b9fbf24..039735e6c01d 100644 --- a/vllm/model_executor/layers/quantization/online/mxfp8.py +++ b/vllm/model_executor/layers/quantization/online/mxfp8.py @@ -17,9 +17,9 @@ FusedMoEQuantConfig, RoutedExperts, ) - from vllm.model_executor.layers.fused_moe.oracle.fp8 import Fp8MoeBackend from vllm.model_executor.kernels.linear import init_mxfp8_linear_kernel +from vllm.model_executor.layers.fused_moe.oracle.fp8 import Fp8MoeBackend from vllm.model_executor.layers.fused_moe.oracle.mxfp8 import ( select_mxfp8_moe_backend, ) @@ -178,6 +178,9 @@ def _setup_kernel( make_fp8_moe_kernel, ) + if self.fp8_backend == Fp8MoeBackend.HUMMING: + self._stage_humming_quantized_weights(layer, w13, w2, w13_scale, w2_scale) + # Shuffle weights to runtime format. w13, w2, w13_scale, w2_scale = convert_to_fp8_moe_kernel_format( fp8_backend=self.fp8_backend,