From 6247970c6dba0c5a9d2f5ee64c19444d5607d74c Mon Sep 17 00:00:00 2001 From: yiliu30 Date: Sun, 25 Oct 2026 07:46:55 -0700 Subject: [PATCH 1/6] feat(svdquant): support serialized MXFP4 checkpoints Load block-32 uint8 E8M0 scales through init_mxfp4_linear_kernel and keep the BF16 rank correction. NVFP4 remains SM103-only; MXFP4 is accepted on SM100/SM103. Signed-off-by: yiliu30 --- .../quantization/test_svdquant_config.py | 60 ++++++++- .../quantization/test_svdquant_linear.py | 82 ++++++++++-- .../quantization/test_svdquant_tp_loading.py | 60 ++++++++- vllm_omni/quantization/svdquant_config.py | 124 +++++++++++------- 4 files changed, 265 insertions(+), 61 deletions(-) diff --git a/tests/diffusion/quantization/test_svdquant_config.py b/tests/diffusion/quantization/test_svdquant_config.py index adbb88ed08e..8efda106ff3 100644 --- a/tests/diffusion/quantization/test_svdquant_config.py +++ b/tests/diffusion/quantization/test_svdquant_config.py @@ -43,7 +43,7 @@ def test_config_from_checkpoint_dict() -> None: ("kwargs", "message"), [ ({"rank": 0}, "rank must be positive"), - ({"precision": "int4"}, "NVFP4 checkpoints only"), + ({"precision": "int4"}, "supports serialized"), ({"act_unsigned": True}, "unsigned activations"), ], ) @@ -71,6 +71,14 @@ def test_factory_builds_svdquant_with_extra_checkpoint_fields() -> None: assert "svdquant" in SUPPORTED_QUANTIZATION_METHODS +def test_config_accepts_mxfp4_precision() -> None: + config = DiffusionSVDQuantConfig.from_config({"rank": 16, "precision": "mxfp4"}) + + assert config.rank == 16 + assert config.precision == "mxfp4" + assert config.get_supported_act_dtypes() == [torch.bfloat16] + + def test_skipped_and_non_linear_modules_remain_unquantized() -> None: config = DiffusionSVDQuantConfig(modules_to_not_convert=["blocks.0.adaln_proj.linear"]) linear = Mock(spec=LinearBase) @@ -85,7 +93,7 @@ def test_skipped_and_non_linear_modules_remain_unquantized() -> None: def test_linear_method_creates_canonical_partitioned_weights( monkeypatch: pytest.MonkeyPatch, ) -> None: - monkeypatch.setattr(svdquant_config, "_assert_supported", lambda: None) + monkeypatch.setattr(svdquant_config, "_assert_supported", lambda precision: None) method = DiffusionSVDQuantLinearMethod(DiffusionSVDQuantConfig(rank=32)) layer = torch.nn.Module() @@ -117,10 +125,37 @@ def test_linear_method_creates_canonical_partitioned_weights( assert layer.wcscales.output_dim == 0 +def test_linear_method_creates_mxfp4_block32_weights_without_global_scales( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(svdquant_config, "_assert_supported", lambda precision: None) + method = DiffusionSVDQuantLinearMethod(DiffusionSVDQuantConfig(rank=32, precision="mxfp4")) + layer = torch.nn.Module() + + method.create_weights( + layer, + input_size_per_partition=128, + output_partition_sizes=[64, 64], + input_size=128, + output_size=128, + params_dtype=torch.bfloat16, + ) + + assert layer.qweight.shape == (128, 64) + assert layer.qweight.dtype == torch.int8 + assert layer.wscales.shape == (4, 128) + assert layer.wscales.dtype == torch.uint8 + assert layer.proj_down.shape == (128, 32) + assert layer.proj_up.shape == (128, 32) + assert layer.smooth_factor.shape == (128,) + assert not hasattr(layer, "wcscales") + assert not hasattr(layer, "wtscale") + + def test_linear_method_rejects_misaligned_input_partition( monkeypatch: pytest.MonkeyPatch, ) -> None: - monkeypatch.setattr(svdquant_config, "_assert_supported", lambda: None) + monkeypatch.setattr(svdquant_config, "_assert_supported", lambda precision: None) method = DiffusionSVDQuantLinearMethod(DiffusionSVDQuantConfig()) with pytest.raises(ValueError, match="divisible by the block size 16"): @@ -134,10 +169,27 @@ def test_linear_method_rejects_misaligned_input_partition( ) +def test_mxfp4_rejects_input_partition_misaligned_to_block32( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(svdquant_config, "_assert_supported", lambda precision: None) + method = DiffusionSVDQuantLinearMethod(DiffusionSVDQuantConfig(precision="mxfp4")) + + with pytest.raises(ValueError, match="divisible by the block size 32"): + method.create_weights( + torch.nn.Module(), + input_size_per_partition=48, + output_partition_sizes=[32], + input_size=48, + output_size=32, + params_dtype=torch.bfloat16, + ) + + def test_active_linear_uses_svdquant_method( monkeypatch: pytest.MonkeyPatch, ) -> None: - monkeypatch.setattr(svdquant_config, "_assert_supported", lambda: None) + monkeypatch.setattr(svdquant_config, "_assert_supported", lambda precision: None) config = DiffusionSVDQuantConfig() linear = Mock(spec=LinearBase) diff --git a/tests/diffusion/quantization/test_svdquant_linear.py b/tests/diffusion/quantization/test_svdquant_linear.py index bdca88d9d18..7136681a9c8 100644 --- a/tests/diffusion/quantization/test_svdquant_linear.py +++ b/tests/diffusion/quantization/test_svdquant_linear.py @@ -46,11 +46,18 @@ def _register_parameter( def test_supports_only_validated_datacenter_blackwell() -> None: - assert not svdquant._supports_capability(SimpleNamespace(major=10, minor=0)) - assert svdquant._supports_capability(SimpleNamespace(major=10, minor=3)) - assert not svdquant._supports_capability(SimpleNamespace(major=11, minor=0)) - assert not svdquant._supports_capability(SimpleNamespace(major=12, minor=0)) - assert not svdquant._supports_capability(None) + assert not svdquant._supports_capability(SimpleNamespace(major=10, minor=0), svdquant._NVFP4_CAPABILITIES) + assert svdquant._supports_capability(SimpleNamespace(major=10, minor=3), svdquant._NVFP4_CAPABILITIES) + assert not svdquant._supports_capability(SimpleNamespace(major=11, minor=0), svdquant._NVFP4_CAPABILITIES) + assert not svdquant._supports_capability(SimpleNamespace(major=12, minor=0), svdquant._NVFP4_CAPABILITIES) + assert not svdquant._supports_capability(None, svdquant._NVFP4_CAPABILITIES) + + +def test_mxfp4_supports_blackwell_sm100_and_sm103() -> None: + assert svdquant._supports_capability(SimpleNamespace(major=10, minor=0), svdquant._MXFP4_CAPABILITIES) + assert svdquant._supports_capability(SimpleNamespace(major=10, minor=3), svdquant._MXFP4_CAPABILITIES) + assert not svdquant._supports_capability(SimpleNamespace(major=9, minor=0), svdquant._MXFP4_CAPABILITIES) + assert not svdquant._supports_capability(None, svdquant._MXFP4_CAPABILITIES) def test_rejects_nvfp4_backend_with_incompatible_layer_contract( @@ -73,7 +80,7 @@ def test_prepare_weights_uses_existing_nvfp4_layout_contract( monkeypatch: pytest.MonkeyPatch, ) -> None: kernel = _FakeNvfp4Kernel() - monkeypatch.setattr(svdquant, "_assert_supported", lambda: None) + monkeypatch.setattr(svdquant, "_assert_supported", lambda precision: None) monkeypatch.setattr(svdquant, "_nvfp4_kernel", lambda: kernel) method = svdquant.DiffusionSVDQuantLinearMethod(svdquant.DiffusionSVDQuantConfig()) layer = torch.nn.Module() @@ -105,7 +112,7 @@ def test_prepare_weights_drops_identity_output_scale( monkeypatch: pytest.MonkeyPatch, ) -> None: kernel = _FakeNvfp4Kernel() - monkeypatch.setattr(svdquant, "_assert_supported", lambda: None) + monkeypatch.setattr(svdquant, "_assert_supported", lambda precision: None) monkeypatch.setattr(svdquant, "_nvfp4_kernel", lambda: kernel) method = svdquant.DiffusionSVDQuantLinearMethod(svdquant.DiffusionSVDQuantConfig()) layer = torch.nn.Module() @@ -123,6 +130,63 @@ def test_prepare_weights_drops_identity_output_scale( assert layer.output_channel_scale is None +def test_prepare_weights_uses_mxfp4_block32_layout( + monkeypatch: pytest.MonkeyPatch, +) -> None: + kernel = _FakeNvfp4Kernel() + monkeypatch.setattr(svdquant, "_assert_supported", lambda precision: None) + monkeypatch.setattr(svdquant, "_mxfp4_kernel", lambda: kernel) + method = svdquant.DiffusionSVDQuantLinearMethod(svdquant.DiffusionSVDQuantConfig(precision="mxfp4")) + layer = torch.nn.Module() + qweight = torch.arange(16, dtype=torch.uint8).reshape(4, 4) + wscales = torch.tensor([[130, 131, 132, 133], [134, 135, 136, 137]], dtype=torch.uint8) + _register_parameter(layer, "qweight", qweight.view(torch.int8)) + _register_parameter(layer, "wscales", wscales) + + method.process_weights_after_loading(layer) + + assert kernel.processed + assert not hasattr(layer, "qweight") + assert not hasattr(layer, "wscales") + assert layer.weight.dtype == torch.uint8 + assert torch.equal(layer.weight, qweight) + assert layer.weight_scale.dtype == torch.uint8 + assert torch.equal(layer.weight_scale, wscales.transpose(0, 1)) + assert not hasattr(layer, "alpha") + assert not hasattr(layer, "input_global_scale_inv") + assert not hasattr(layer, "output_channel_scale") + + +def test_mxfp4_compatibility_path_applies_rank_correction( + monkeypatch: pytest.MonkeyPatch, +) -> None: + base_weight = torch.tensor( + [[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]], + dtype=torch.bfloat16, + ) + kernel = _FakeNvfp4Kernel(base_weight) + monkeypatch.setattr(svdquant, "_assert_supported", lambda precision: None) + monkeypatch.setattr(svdquant, "_mxfp4_kernel", lambda: kernel) + method = svdquant.DiffusionSVDQuantLinearMethod(svdquant.DiffusionSVDQuantConfig(precision="mxfp4")) + + layer = torch.nn.Module() + layer.smooth_factor = torch.tensor([2.0, 4.0], dtype=torch.bfloat16) + layer.proj_down = torch.tensor([[1.0], [2.0]], dtype=torch.bfloat16) + layer.proj_up = torch.tensor([[1.0], [2.0], [3.0]], dtype=torch.bfloat16) + layer.output_size_per_partition = 3 + x = torch.tensor([[2.0, 4.0], [4.0, 8.0]], dtype=torch.bfloat16) + bias = torch.tensor([1.0, 2.0, 3.0], dtype=torch.bfloat16) + + actual = method.apply(layer, x, bias) + + smoothed = x / layer.smooth_factor + expected = torch.mm(smoothed, base_weight) + expected = expected + torch.mm(torch.mm(x, layer.proj_down), layer.proj_up.T) + expected = expected + bias + torch.testing.assert_close(actual, expected) + torch.testing.assert_close(kernel.last_input, smoothed) + + def test_compatibility_path_applies_qkv_scale_and_rank_correction( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -131,7 +195,7 @@ def test_compatibility_path_applies_qkv_scale_and_rank_correction( dtype=torch.bfloat16, ) kernel = _FakeNvfp4Kernel(base_weight) - monkeypatch.setattr(svdquant, "_assert_supported", lambda: None) + monkeypatch.setattr(svdquant, "_assert_supported", lambda precision: None) monkeypatch.setattr(svdquant, "_nvfp4_kernel", lambda: kernel) method = svdquant.DiffusionSVDQuantLinearMethod(svdquant.DiffusionSVDQuantConfig()) @@ -158,7 +222,7 @@ def test_compatibility_path_applies_qkv_scale_and_rank_correction( def test_compatibility_path_rejects_fp16_activations( monkeypatch: pytest.MonkeyPatch, ) -> None: - monkeypatch.setattr(svdquant, "_assert_supported", lambda: None) + monkeypatch.setattr(svdquant, "_assert_supported", lambda precision: None) method = svdquant.DiffusionSVDQuantLinearMethod(svdquant.DiffusionSVDQuantConfig()) layer = torch.nn.Module() with pytest.raises(ValueError, match="requires BF16 activations"): diff --git a/tests/diffusion/quantization/test_svdquant_tp_loading.py b/tests/diffusion/quantization/test_svdquant_tp_loading.py index 14138dc1c19..cf2d54befce 100644 --- a/tests/diffusion/quantization/test_svdquant_tp_loading.py +++ b/tests/diffusion/quantization/test_svdquant_tp_loading.py @@ -27,10 +27,68 @@ def _values(shape: tuple[int, ...], dtype: torch.dtype) -> torch.Tensor: def _method(monkeypatch: pytest.MonkeyPatch): - monkeypatch.setattr(svdquant, "_assert_supported", lambda: None) + monkeypatch.setattr(svdquant, "_assert_supported", lambda precision: None) return svdquant.DiffusionSVDQuantLinearMethod(svdquant.DiffusionSVDQuantConfig(rank=8)) +def _mxfp4_method(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(svdquant, "_assert_supported", lambda precision: None) + return svdquant.DiffusionSVDQuantLinearMethod(svdquant.DiffusionSVDQuantConfig(rank=8, precision="mxfp4")) + + +def test_mxfp4_tp_loaders_shard_block32_scales( + monkeypatch: pytest.MonkeyPatch, +) -> None: + column = _linear_without_init(ColumnParallelLinear, tp_rank=1) + method = _mxfp4_method(monkeypatch) + method.create_weights( + column, + input_size_per_partition=128, + output_partition_sizes=[32], + input_size=128, + output_size=64, + params_dtype=torch.bfloat16, + weight_loader=column.weight_loader, + ) + + loaded_tensors = { + "qweight": _values((64, 64), torch.int8), + "wscales": _values((4, 64), torch.uint8), + "proj_up": _values((64, 8), torch.bfloat16), + } + for name, loaded in loaded_tensors.items(): + parameter = getattr(column, name) + parameter.weight_loader(parameter, loaded) + expected = loaded.narrow(parameter.output_dim, 32, 32) + torch.testing.assert_close(parameter, expected) + assert not hasattr(column, "wcscales") + assert not hasattr(column, "wtscale") + + row = _linear_without_init(RowParallelLinear, tp_rank=1) + method.create_weights( + row, + input_size_per_partition=64, + output_partition_sizes=[64], + input_size=128, + output_size=64, + params_dtype=torch.bfloat16, + weight_loader=row.weight_loader, + ) + + loaded_tensors = { + "qweight": _values((64, 64), torch.int8), + "wscales": _values((4, 64), torch.uint8), + "proj_down": _values((128, 8), torch.bfloat16), + "smooth_factor": _values((128,), torch.bfloat16), + } + for name, loaded in loaded_tensors.items(): + parameter = getattr(row, name) + parameter.weight_loader(parameter, loaded) + shard_size = parameter.shape[parameter.input_dim] + expected = loaded.narrow(parameter.input_dim, shard_size, shard_size) + torch.testing.assert_close(parameter, expected) + + def test_column_parallel_loader_shards_svdquant_outputs( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/vllm_omni/quantization/svdquant_config.py b/vllm_omni/quantization/svdquant_config.py index ad22592af22..9abc50afedd 100644 --- a/vllm_omni/quantization/svdquant_config.py +++ b/vllm_omni/quantization/svdquant_config.py @@ -1,9 +1,9 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project -"""Serialized SVDQuant NVFP4 support for diffusion transformers. +"""Serialized SVDQuant NVFP4 and MXFP4 support for diffusion transformers. -The checkpoint stores NVFP4 weights plus a rank-R correction for each -quantized linear. The four-bit GEMM uses vLLM's existing NVFP4 kernel +The checkpoint stores four-bit weights plus a rank-R correction for each +quantized linear. The four-bit GEMM uses vLLM's existing NVFP4 or MXFP4 kernel registry, while the rank correction uses ordinary BF16 matrix multiplication. Native SVDQuant fusion is a separate optimization and is not required to load or run the checkpoint. @@ -25,7 +25,9 @@ FlashInferCuteDslNvFp4LinearKernel, FlashInferCutlassNvFp4LinearKernel, FlashInferTrtllmNvFp4LinearKernel, + MxFp4LinearKernel, NvFp4LinearKernel, + init_mxfp4_linear_kernel, init_nvfp4_linear_kernel, ) from vllm.model_executor.layers.linear import ( @@ -39,6 +41,7 @@ ) from vllm.model_executor.layers.quantization.utils.quant_utils import ( is_layer_skipped, + kMxfp4Dynamic, ) from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.model_executor.utils import set_weight_attrs @@ -50,7 +53,11 @@ logger = init_logger(__name__) -_SUPPORTED_CAPABILITIES = {(10, 3)} +_NVFP4_BLOCK_SIZE = 16 +_MXFP4_BLOCK_SIZE = 32 +_SUPPORTED_PRECISIONS = ("nvfp4", "mxfp4") +_NVFP4_CAPABILITIES = {(10, 3)} +_MXFP4_CAPABILITIES = {(10, 0), (10, 3)} _COMPATIBLE_NVFP4_KERNELS = ( CutlassNvFp4LinearKernel, FbgemmNvFp4LinearKernel, @@ -62,25 +69,27 @@ ) -def _supports_capability(capability: DeviceCapability | None) -> bool: +def _supports_capability(capability: DeviceCapability | None, supported: set[tuple[int, int]]) -> bool: return ( capability is not None and ( capability.major, capability.minor, ) - in _SUPPORTED_CAPABILITIES + in supported ) -def _assert_supported() -> None: +def _assert_supported(precision: str) -> None: + supported = _MXFP4_CAPABILITIES if precision == "mxfp4" else _NVFP4_CAPABILITIES if not current_platform.is_cuda(): - raise RuntimeError("SVDQuant NVFP4 requires a CUDA device") + raise RuntimeError(f"SVDQuant {precision} requires a CUDA device") capability = current_platform.get_device_capability() - if not _supports_capability(capability): + if not _supports_capability(capability, supported): device = current_platform.device_name sm = capability.to_int() if capability is not None else "unknown" - raise RuntimeError(f"SVDQuant NVFP4 is validated on SM103 only; got {device!r} (SM{sm})") + expected = ", ".join(f"SM{major}{minor}" for major, minor in sorted(supported)) + raise RuntimeError(f"SVDQuant {precision} is validated on {expected} only; got {device!r} (SM{sm})") @functools.cache @@ -95,8 +104,14 @@ def _nvfp4_kernel() -> NvFp4LinearKernel: return kernel +@functools.cache +def _mxfp4_kernel() -> MxFp4LinearKernel: + """Select vLLM's MXFP4 backend for the current device (same path as online MXFP4).""" + return init_mxfp4_linear_kernel(activation_quant_key=kMxfp4Dynamic) + + class DiffusionSVDQuantConfig(QuantizationConfig): - """Configuration for serialized NVFP4 W4A4 plus low-rank correction.""" + """Configuration for serialized NVFP4 or MXFP4 W4A4 plus low-rank correction.""" def __init__( self, @@ -108,10 +123,8 @@ def __init__( super().__init__() if rank <= 0: raise ValueError(f"SVDQuant rank must be positive, got {rank}") - if precision != "nvfp4": - raise ValueError( - f"Phase 1 SVDQuant supports serialized NVFP4 checkpoints only; got precision={precision!r}" - ) + if precision not in _SUPPORTED_PRECISIONS: + raise ValueError(f"SVDQuant supports serialized {_SUPPORTED_PRECISIONS} checkpoints; got {precision!r}") if act_unsigned: raise ValueError("Phase 1 SVDQuant does not support unsigned activations") self.rank = rank @@ -131,7 +144,9 @@ def get_supported_act_dtypes(cls) -> list[torch.dtype]: @classmethod def get_min_capability(cls) -> int: - return 103 + # MXFP4 kernels exist from SM100; the per-precision check in + # _assert_supported() keeps NVFP4 restricted to SM103. + return 100 @classmethod def get_config_filenames(cls) -> list[str]: @@ -170,7 +185,7 @@ class DiffusionSVDQuantLinearMethod(LinearMethodBase): """Load and execute a serialized SVDQuant linear layer.""" def __init__(self, quant_config: DiffusionSVDQuantConfig) -> None: - _assert_supported() + _assert_supported(quant_config.precision) self.quant_config = quant_config def create_weights( @@ -188,10 +203,12 @@ def create_weights( "weight_loader", default_weight_loader, ) - if input_size_per_partition % 16 != 0: + precision = self.quant_config.precision + block_size = _MXFP4_BLOCK_SIZE if precision == "mxfp4" else _NVFP4_BLOCK_SIZE + if input_size_per_partition % block_size != 0: raise ValueError( - "SVDQuant NVFP4 requires each input partition to be divisible " - f"by the block size 16; got {input_size_per_partition}" + f"SVDQuant {precision} requires each input partition to be divisible " + f"by the block size {block_size}; got {input_size_per_partition}" ) output_size_per_partition = sum(output_partition_sizes) rank = self.quant_config.rank @@ -215,9 +232,9 @@ def create_weights( wscales = Parameter( torch.empty( - input_size_per_partition // 16, + input_size_per_partition // block_size, output_size_per_partition, - dtype=torch.float8_e4m3fn, + dtype=torch.uint8 if precision == "mxfp4" else torch.float8_e4m3fn, ), requires_grad=False, ) @@ -277,40 +294,43 @@ def create_weights( }, ) - wcscales = Parameter( - torch.ones( - output_size_per_partition, - dtype=torch.bfloat16, - ), - requires_grad=False, - ) - set_weight_attrs( - wcscales, - { - "output_dim": 0, - "weight_loader": weight_loader, - }, - ) + if precision == "nvfp4": + wcscales = Parameter( + torch.ones( + output_size_per_partition, + dtype=torch.bfloat16, + ), + requires_grad=False, + ) + set_weight_attrs( + wcscales, + { + "output_dim": 0, + "weight_loader": weight_loader, + }, + ) - wtscale = Parameter( - torch.ones(1, dtype=torch.bfloat16), - requires_grad=False, - ) - set_weight_attrs(wtscale, {"weight_loader": default_weight_loader}) + wtscale = Parameter( + torch.ones(1, dtype=torch.bfloat16), + requires_grad=False, + ) + set_weight_attrs(wtscale, {"weight_loader": default_weight_loader}) layer.register_parameter("qweight", qweight) layer.register_parameter("wscales", wscales) layer.register_parameter("proj_down", proj_down) layer.register_parameter("proj_up", proj_up) layer.register_parameter("smooth_factor", smooth_factor) - layer.register_parameter("wcscales", wcscales) - layer.register_parameter("wtscale", wtscale) + if precision == "nvfp4": + layer.register_parameter("wcscales", wcscales) + layer.register_parameter("wtscale", wtscale) del input_size, output_size layer.output_size_per_partition = output_size_per_partition def process_weights_after_loading(self, layer: torch.nn.Module) -> None: - """Adapt the canonical row-major checkpoint to vLLM's NVFP4 ABI.""" + """Adapt the canonical row-major checkpoint to the selected vLLM FP4 ABI.""" + precision = self.quant_config.precision qweight = layer.qweight wscales = layer.wscales del layer.qweight @@ -330,6 +350,15 @@ def process_weights_after_loading(self, layer: torch.nn.Module) -> None: ), ) + if precision == "mxfp4": + kernel = _mxfp4_kernel() + kernel.process_weights_after_loading(layer) + logger.info_once( + "SVDQuant MXFP4 is using %s; the rank correction is not fused into the GEMM.", + type(kernel).__name__, + ) + return + layer.register_parameter( "input_global_scale_inv", Parameter( @@ -370,9 +399,9 @@ def apply( x: torch.Tensor, bias: torch.Tensor | None = None, ) -> torch.Tensor: - """Compute the base NVFP4 GEMM plus the BF16 rank correction.""" + """Compute the base four-bit GEMM plus the BF16 rank correction.""" if x.dtype != torch.bfloat16: - raise ValueError(f"SVDQuant NVFP4 requires BF16 activations; got {x.dtype}") + raise ValueError(f"SVDQuant {self.quant_config.precision} requires BF16 activations; got {x.dtype}") original_shape = x.shape x_2d = x.reshape(-1, original_shape[-1]).contiguous() @@ -380,7 +409,8 @@ def apply( # The residual branch consumes the original activation. Only the # four-bit base GEMM consumes the smoothed activation. smoothed = x_2d / layer.smooth_factor - out = _nvfp4_kernel().apply_weights( + kernel = _mxfp4_kernel() if self.quant_config.precision == "mxfp4" else _nvfp4_kernel() + out = kernel.apply_weights( layer=layer, x=smoothed, bias=None, From 4294ef517dbd0c2507e6f8607fbdfd567bd37a3b Mon Sep 17 00:00:00 2001 From: yiliu30 Date: Sun, 25 Oct 2026 09:24:29 -0700 Subject: [PATCH 2/6] fix(minimax-h3): reorder serialized SVD QKV tensors Signed-off-by: yiliu30 --- .../test_minimax_h3_quantization.py | 60 +++++++++++++++++++ .../minimax_h3/minimax_h3_transformer.py | 28 +++++++++ 2 files changed, 88 insertions(+) diff --git a/tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py b/tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py index 0e6197c96d1..7f8dfd258af 100644 --- a/tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py +++ b/tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py @@ -161,6 +161,15 @@ def __init__(self, loader): self.weight.weight_loader = loader +class _MultiWeightTarget(nn.Module): + def __init__(self, loaders): + super().__init__() + for name, loader in loaders.items(): + parameter = nn.Parameter(torch.empty(1)) + parameter.weight_loader = loader + self.register_parameter(name, parameter) + + def test_model_load_weights_transforms_before_calling_vllm_loader(): from vllm_omni.diffusion.models.minimax_h3.minimax_h3_transformer import ( MiniMaxH3DiTArchConfig, @@ -212,6 +221,57 @@ def fc1_loader(param, loaded_weight, shard_id): ] +def test_model_load_weights_reorders_serialized_svd_qkv_tensors(): + from vllm_omni.diffusion.models.minimax_h3.minimax_h3_transformer import ( + MiniMaxH3DiTArchConfig, + MiniMaxH3DiTModel, + ) + + calls = {} + + def capture(name): + def loader(param, loaded_weight): + del param + calls[name] = loaded_weight.clone() + + return loader + + model = object.__new__(MiniMaxH3DiTModel) + nn.Module.__init__(model) + model.arch = MiniMaxH3DiTArchConfig( + hidden_size=1, + num_attention_heads=2, + attention_head_dim=1, + ffn_hidden_size=2, + ) + model.blocks = nn.ModuleList([nn.Module()]) + model.blocks[0].attn = nn.Module() + model.blocks[0].attn.qkv_proj = _MultiWeightTarget( + {name: capture(name) for name in ("qweight", "wscales", "proj_up", "bias", "wcscales")} + ) + + qweight = torch.arange(12, dtype=torch.int8).reshape(6, 2) + wscales = torch.arange(18, dtype=torch.uint8).reshape(3, 6) + proj_up = torch.arange(6, dtype=torch.bfloat16).reshape(6, 1) + bias = torch.arange(6, dtype=torch.bfloat16) + wcscales = torch.arange(6, dtype=torch.bfloat16) + names = ("qweight", "wscales", "proj_up", "bias", "wcscales") + loaded = model.load_weights( + [ + (f"blocks.0.attn.qkv_proj.{name}", tensor) + for name, tensor in zip(names, (qweight, wscales, proj_up, bias, wcscales)) + ] + ) + + assert loaded == {f"blocks.0.attn.qkv_proj.{name}" for name in names} + expected_rows = [0, 3, 1, 4, 2, 5] + assert calls["qweight"][:, 0].tolist() == [row * qweight.shape[1] for row in expected_rows] + assert calls["proj_up"][:, 0].tolist() == expected_rows + assert calls["bias"].tolist() == expected_rows + assert calls["wcscales"].tolist() == expected_rows + assert calls["wscales"][0, :].tolist() == expected_rows + + def test_loader_adapter_declares_equivalent_direct_mmap_transform(monkeypatch): from vllm_omni.diffusion.model_loader.checkpoint_adapters import ( get_direct_mmap_adapter, diff --git a/vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py b/vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py index 43c97e700ec..890e56f76d8 100644 --- a/vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py +++ b/vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py @@ -1786,6 +1786,34 @@ def load_weights( head_dim=self.arch.attention_head_dim, ) weight_loader(param, loaded_weight) + elif name.endswith( + ( + ".attn.qkv_proj.qweight", + ".attn.qkv_proj.proj_up", + ".attn.qkv_proj.bias", + ".attn.qkv_proj.wcscales", + ) + ): + # Serialized quantization checkpoints keep the same grouped + # Q/K/V row order as the BF16 checkpoint. Reorder every tensor + # with an output dimension before the QKV loader shards it. + loaded_weight = _reorder_grouped_qkv_to_qkv( + loaded_weight, + num_query_groups=self.arch.num_attention_heads, + heads_per_group=1, + head_dim=self.arch.attention_head_dim, + ) + weight_loader(param, loaded_weight) + elif name.endswith(".attn.qkv_proj.wscales"): + # E8M0 scales are stored [K // block, N], so reorder their N + # axis through the same grouped-QKV permutation. + loaded_weight = _reorder_grouped_qkv_to_qkv( + loaded_weight.T, + num_query_groups=self.arch.num_attention_heads, + heads_per_group=1, + head_dim=self.arch.attention_head_dim, + ).T.contiguous() + weight_loader(param, loaded_weight) elif name.endswith((".mlp.fc1.weight", ".mlp.fc1.weight_scale")): if loaded_weight.shape[0] % 2: raise ValueError( From 51e925f7e5ca67c450bcd889eeaa49db2e86aba8 Mon Sep 17 00:00:00 2001 From: yiliu30 Date: Sun, 25 Oct 2026 18:56:58 -0700 Subject: [PATCH 3/6] docs(svdquant): document MXFP4 checkpoint layout Signed-off-by: yiliu30 --- docs/user_guide/quantization/svdquant.md | 34 ++++++++++++++---------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/docs/user_guide/quantization/svdquant.md b/docs/user_guide/quantization/svdquant.md index 4fbd8040d7e..061c17e2d92 100644 --- a/docs/user_guide/quantization/svdquant.md +++ b/docs/user_guide/quantization/svdquant.md @@ -4,8 +4,8 @@ [SVDQuant](https://arxiv.org/abs/2411.05007) combines four-bit weights and activations with a small low-rank branch that corrects part of the -quantization error. vLLM-Omni consumes an offline-quantized checkpoint; -it does not calibrate the model while loading. +quantization error. vLLM-Omni consumes an offline-quantized NVFP4 or MXFP4 +checkpoint; it does not calibrate the model while loading. ## Checkpoint contract @@ -29,22 +29,28 @@ rank `R`, the checkpoint stores: | Suffix | Shape | dtype | | --- | --- | --- | | `qweight` | `(N, K / 2)` | `int8` (two packed FP4 values per byte) | -| `wscales` | `(K / 16, N)` | `float8_e4m3fn` | +| `wscales` (NVFP4) | `(K / 16, N)` | `float8_e4m3fn` | +| `wscales` (MXFP4) | `(K / 32, N)` | `uint8` (E8M0 scales) | | `proj_down` | `(K, R)` | `bfloat16` | | `proj_up` | `(N, R)` | `bfloat16` | | `smooth_factor` | `(K,)` | `bfloat16` | -| `wcscales` | `(N,)` | `bfloat16` | -| `wtscale` | `(1,)` | `bfloat16` | +| `wcscales` (NVFP4 only) | `(N,)` | `bfloat16` | +| `wtscale` (NVFP4 only) | `(1,)` | `bfloat16` | -`K` must be divisible by 16 on every tensor-parallel rank. Set `wcscales` to -ones when no per-output correction is needed. Modules listed in -`modules_to_not_convert` keep their checkpoint precision. +`K` must be divisible by 16 for NVFP4 or 32 for MXFP4 on every +tensor-parallel rank. Set `wcscales` to ones when no NVFP4 per-output +correction is needed. MXFP4 checkpoints do not contain `wcscales` or +`wtscale`. Modules listed in `modules_to_not_convert` keep their checkpoint +precision. ## Runtime support -The compatibility path accepts BF16 inputs and executes an NVFP4 GEMM followed -by the BF16 rank correction. It supports vLLM's FlashInfer, CUTLASS, and FBGEMM -NVFP4 tensor layouts; incompatible forced backends fail during model loading. -SM103 is the currently validated and enabled hardware target. Native fusion of -the NVFP4 GEMM and rank correction is separate from this checkpoint-loading -contract. +The compatibility path accepts BF16 inputs and executes a four-bit GEMM followed +by the BF16 rank correction. NVFP4 is enabled on SM103 and supports vLLM's +FlashInfer, CUTLASS, and FBGEMM tensor layouts; incompatible forced backends +fail during model loading. + +MXFP4 is enabled on SM100 and SM103 and uses vLLM's standard MXFP4 kernel +selection. FlashInfer provides W4A4 execution when its CuTe-DSL backend is +available; otherwise Marlin uses W4A16. Native fusion of the quantized GEMM and +rank correction is separate from this checkpoint-loading contract. From 6df271096996a5751752565295efeac30d65491d Mon Sep 17 00:00:00 2001 From: yiliu30 Date: Mon, 26 Oct 2026 05:38:10 -0700 Subject: [PATCH 4/6] fix(svdquant): handle Marlin bias and replicated loading Signed-off-by: yiliu30 --- .../quantization/test_svdquant_config.py | 4 + .../quantization/test_svdquant_linear.py | 40 ++++++++ .../quantization/test_svdquant_marlin_gpu.py | 91 +++++++++++++++++++ vllm_omni/quantization/svdquant_config.py | 11 ++- 4 files changed, 144 insertions(+), 2 deletions(-) create mode 100644 tests/diffusion/quantization/test_svdquant_marlin_gpu.py diff --git a/tests/diffusion/quantization/test_svdquant_config.py b/tests/diffusion/quantization/test_svdquant_config.py index 8efda106ff3..5149ba22ba0 100644 --- a/tests/diffusion/quantization/test_svdquant_config.py +++ b/tests/diffusion/quantization/test_svdquant_config.py @@ -116,6 +116,8 @@ def test_linear_method_creates_canonical_partitioned_weights( assert layer.wcscales.shape == (128,) assert layer.wtscale.shape == (1,) + assert layer.input_size_per_partition == 128 + assert layer.output_size_per_partition == 128 assert layer.qweight.input_dim == 1 assert layer.qweight.output_dim == 0 assert layer.wscales.input_dim == 0 @@ -150,6 +152,8 @@ def test_linear_method_creates_mxfp4_block32_weights_without_global_scales( assert layer.smooth_factor.shape == (128,) assert not hasattr(layer, "wcscales") assert not hasattr(layer, "wtscale") + assert layer.input_size_per_partition == 128 + assert layer.output_size_per_partition == 128 def test_linear_method_rejects_misaligned_input_partition( diff --git a/tests/diffusion/quantization/test_svdquant_linear.py b/tests/diffusion/quantization/test_svdquant_linear.py index 7136681a9c8..d924a10ecf4 100644 --- a/tests/diffusion/quantization/test_svdquant_linear.py +++ b/tests/diffusion/quantization/test_svdquant_linear.py @@ -34,6 +34,15 @@ def apply_weights( return torch.mm(x, self.base_weight) +class _FakeMarlinBiasKernel(_FakeNvfp4Kernel): + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + super().process_weights_after_loading(layer) + bias = getattr(layer, "bias", None) + if bias is not None: + padded = torch.nn.functional.pad(bias.detach(), (0, 1)) + layer.bias = torch.nn.Parameter(padded.flip(0).contiguous(), requires_grad=False) + + def _register_parameter( layer: torch.nn.Module, name: str, @@ -187,6 +196,37 @@ def test_mxfp4_compatibility_path_applies_rank_correction( torch.testing.assert_close(kernel.last_input, smoothed) +def test_mxfp4_preserves_logical_bias_when_kernel_repacks_it( + monkeypatch: pytest.MonkeyPatch, +) -> None: + base_weight = torch.tensor( + [[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]], + dtype=torch.bfloat16, + ) + kernel = _FakeMarlinBiasKernel(base_weight) + monkeypatch.setattr(svdquant, "_assert_supported", lambda precision: None) + monkeypatch.setattr(svdquant, "_mxfp4_kernel", lambda: kernel) + method = svdquant.DiffusionSVDQuantLinearMethod(svdquant.DiffusionSVDQuantConfig(precision="mxfp4")) + + layer = torch.nn.Module() + _register_parameter(layer, "qweight", torch.zeros((3, 1), dtype=torch.int8)) + _register_parameter(layer, "wscales", torch.zeros((2, 3), dtype=torch.uint8)) + _register_parameter(layer, "bias", torch.tensor([1.0, 2.0, 3.0], dtype=torch.bfloat16)) + layer.smooth_factor = torch.ones(2, dtype=torch.bfloat16) + layer.proj_down = torch.zeros((2, 1), dtype=torch.bfloat16) + layer.proj_up = torch.zeros((3, 1), dtype=torch.bfloat16) + layer.output_size_per_partition = 3 + + method.process_weights_after_loading(layer) + assert layer.bias.shape == (4,) + + x = torch.tensor([[2.0, 4.0]], dtype=torch.bfloat16) + actual = method.apply(layer, x, layer.bias) + expected = torch.mm(x, base_weight) + torch.tensor([1.0, 2.0, 3.0], dtype=torch.bfloat16) + + torch.testing.assert_close(actual, expected) + + def test_compatibility_path_applies_qkv_scale_and_rank_correction( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/diffusion/quantization/test_svdquant_marlin_gpu.py b/tests/diffusion/quantization/test_svdquant_marlin_gpu.py new file mode 100644 index 00000000000..1b11230d073 --- /dev/null +++ b/tests/diffusion/quantization/test_svdquant_marlin_gpu.py @@ -0,0 +1,91 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""Regression coverage for SVDQuant MXFP4 loading through Marlin.""" + +import pytest +import torch +from torch.nn import functional as F +from vllm.config import set_current_vllm_config +from vllm.distributed import ( + destroy_distributed_environment, + destroy_model_parallel, + init_distributed_environment, + initialize_model_parallel, +) +from vllm.model_executor.kernels.linear.mxfp4.base import MxFp4LinearLayerConfig +from vllm.model_executor.kernels.linear.mxfp4.marlin import MarlinMxFp4LinearKernel +from vllm.model_executor.layers.linear import ReplicatedLinear +from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import is_fp4_marlin_supported +from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp4Dynamic +from vllm.utils.network_utils import get_open_port +from vllm.utils.torch_utils import set_default_torch_dtype + +import vllm_omni.quantization.svdquant_config as svdquant +from tests.helpers.mark import hardware_test +from vllm_omni.diffusion.data import OmniDiffusionConfig +from vllm_omni.diffusion.vllm_config import create_diffusion_vllm_config +from vllm_omni.quantization.svdquant_config import DiffusionSVDQuantConfig + +pytestmark = [pytest.mark.core_model, pytest.mark.diffusion] + + +@hardware_test(res={"cuda": ["B200"]}, num_cards=1) +def test_mxfp4_replicated_linear_loads_with_forced_marlin(mocker): + """Marlin padding must not break replicated projection loading or bias.""" + if not is_fp4_marlin_supported(): + pytest.skip("Marlin FP4 is unavailable") + + config = DiffusionSVDQuantConfig(rank=8, precision="mxfp4") + od_config = OmniDiffusionConfig(model="test", dtype=torch.bfloat16, quantization_config=config) + vllm_config = create_diffusion_vllm_config(torch.device("cuda:0"), od_config) + + try: + init_distributed_environment( + world_size=1, + rank=0, + local_rank=0, + distributed_init_method=f"tcp://127.0.0.1:{get_open_port()}", + backend="nccl", + ) + with set_current_vllm_config(vllm_config), set_default_torch_dtype(torch.bfloat16), torch.device("cuda:0"): + initialize_model_parallel(tensor_model_parallel_size=1, pipeline_model_parallel_size=1) + kernel = MarlinMxFp4LinearKernel(MxFp4LinearLayerConfig(activation_quant_key=kMxfp4Dynamic)) + mocker.patch.object(svdquant, "_mxfp4_kernel", return_value=kernel) + + # N=96 forces Marlin to pad the output dimension to 128. + layer = ReplicatedLinear( + 128, + 96, + bias=True, + params_dtype=torch.bfloat16, + quant_config=config, + prefix="test.proj_mlp", + disable_tp=True, + ) + assert layer.input_size_per_partition == 128 + + generator = torch.Generator(device="cuda").manual_seed(0) + inputs = torch.randn(4, 128, generator=generator, dtype=torch.bfloat16) + bias = torch.randn(96, generator=generator, dtype=torch.bfloat16) + tensors = { + # Byte 0x22 encodes two E2M1 +1.0 values; E8M0 127 is 2**0. + "qweight": torch.full((96, 64), 0x22, device="cuda", dtype=torch.uint8).view(torch.int8), + "wscales": torch.full((4, 96), 127, device="cuda", dtype=torch.uint8), + "proj_down": torch.zeros((128, 8), device="cuda", dtype=torch.bfloat16), + "proj_up": torch.zeros((96, 8), device="cuda", dtype=torch.bfloat16), + "smooth_factor": torch.ones(128, device="cuda", dtype=torch.bfloat16), + "bias": bias, + } + for name, value in tensors.items(): + param = getattr(layer, name) + param.weight_loader(param, value) + + layer.quant_method.process_weights_after_loading(layer) + assert layer.bias.shape == (128,) + + actual = layer(inputs)[0] + expected = F.linear(inputs, torch.ones((96, 128), device="cuda", dtype=torch.bfloat16), bias) + torch.testing.assert_close(actual, expected, rtol=1e-2, atol=1e-2) + finally: + destroy_model_parallel() + destroy_distributed_environment() diff --git a/vllm_omni/quantization/svdquant_config.py b/vllm_omni/quantization/svdquant_config.py index 9abc50afedd..6b4a70d97c6 100644 --- a/vllm_omni/quantization/svdquant_config.py +++ b/vllm_omni/quantization/svdquant_config.py @@ -326,6 +326,7 @@ def create_weights( layer.register_parameter("wtscale", wtscale) del input_size, output_size + layer.input_size_per_partition = input_size_per_partition layer.output_size_per_partition = output_size_per_partition def process_weights_after_loading(self, layer: torch.nn.Module) -> None: @@ -351,6 +352,11 @@ def process_weights_after_loading(self, layer: torch.nn.Module) -> None: ) if precision == "mxfp4": + # Marlin replaces layer.bias with a padded and permuted Parameter. + # Keep the logical bias for the post-GEMM add in apply(). + bias = getattr(layer, "bias", None) + if bias is not None: + layer._svdquant_bias = bias.detach().clone() kernel = _mxfp4_kernel() kernel.process_weights_after_loading(layer) logger.info_once( @@ -428,8 +434,9 @@ def apply( correction_input, layer.proj_up.transpose(0, 1), ) - if bias is not None: - out.add_(bias) + bias_to_add = getattr(layer, "_svdquant_bias", bias) if bias is not None else None + if bias_to_add is not None: + out.add_(bias_to_add) return out.reshape( *original_shape[:-1], layer.output_size_per_partition, From 573a73c67b0ed04a3fb621bb6221dfb65afe9e9c Mon Sep 17 00:00:00 2001 From: yiliu30 Date: Mon, 26 Oct 2026 18:39:16 -0700 Subject: [PATCH 5/6] fix(svdquant): address MXFP4 review feedback Signed-off-by: yiliu30 --- .../test_minimax_h3_quantization.py | 8 ++- .../quantization/test_svdquant_linear.py | 54 +++++++++++++++++++ .../minimax_h3/minimax_h3_transformer.py | 1 - vllm_omni/quantization/svdquant_config.py | 20 ++++--- 4 files changed, 71 insertions(+), 12 deletions(-) diff --git a/tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py b/tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py index 7f8dfd258af..af136cdf000 100644 --- a/tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py +++ b/tests/diffusion/models/minimax_h3/test_minimax_h3_quantization.py @@ -247,19 +247,18 @@ def loader(param, loaded_weight): model.blocks = nn.ModuleList([nn.Module()]) model.blocks[0].attn = nn.Module() model.blocks[0].attn.qkv_proj = _MultiWeightTarget( - {name: capture(name) for name in ("qweight", "wscales", "proj_up", "bias", "wcscales")} + {name: capture(name) for name in ("qweight", "wscales", "proj_up", "wcscales")} ) qweight = torch.arange(12, dtype=torch.int8).reshape(6, 2) wscales = torch.arange(18, dtype=torch.uint8).reshape(3, 6) proj_up = torch.arange(6, dtype=torch.bfloat16).reshape(6, 1) - bias = torch.arange(6, dtype=torch.bfloat16) wcscales = torch.arange(6, dtype=torch.bfloat16) - names = ("qweight", "wscales", "proj_up", "bias", "wcscales") + names = ("qweight", "wscales", "proj_up", "wcscales") loaded = model.load_weights( [ (f"blocks.0.attn.qkv_proj.{name}", tensor) - for name, tensor in zip(names, (qweight, wscales, proj_up, bias, wcscales)) + for name, tensor in zip(names, (qweight, wscales, proj_up, wcscales)) ] ) @@ -267,7 +266,6 @@ def loader(param, loaded_weight): expected_rows = [0, 3, 1, 4, 2, 5] assert calls["qweight"][:, 0].tolist() == [row * qweight.shape[1] for row in expected_rows] assert calls["proj_up"][:, 0].tolist() == expected_rows - assert calls["bias"].tolist() == expected_rows assert calls["wcscales"].tolist() == expected_rows assert calls["wscales"][0, :].tolist() == expected_rows diff --git a/tests/diffusion/quantization/test_svdquant_linear.py b/tests/diffusion/quantization/test_svdquant_linear.py index d924a10ecf4..a28bc274e5d 100644 --- a/tests/diffusion/quantization/test_svdquant_linear.py +++ b/tests/diffusion/quantization/test_svdquant_linear.py @@ -69,6 +69,17 @@ def test_mxfp4_supports_blackwell_sm100_and_sm103() -> None: assert not svdquant._supports_capability(None, svdquant._MXFP4_CAPABILITIES) +def test_mxfp4_accepts_xpu_while_nvfp4_remains_cuda_only( + monkeypatch: pytest.MonkeyPatch, +) -> None: + platform = SimpleNamespace(is_xpu=lambda: True, is_cuda=lambda: False) + monkeypatch.setattr(svdquant, "current_platform", platform) + + svdquant._assert_supported("mxfp4") + with pytest.raises(RuntimeError, match="requires a CUDA device"): + svdquant._assert_supported("nvfp4") + + def test_rejects_nvfp4_backend_with_incompatible_layer_contract( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -219,6 +230,13 @@ def test_mxfp4_preserves_logical_bias_when_kernel_repacks_it( method.process_weights_after_loading(layer) assert layer.bias.shape == (4,) + assert "_svdquant_bias" in dict(layer.named_buffers()) + assert "_svdquant_bias" not in layer.state_dict() + assert layer._svdquant_bias is not layer.bias + torch.testing.assert_close( + layer._svdquant_bias, + torch.tensor([1.0, 2.0, 3.0], dtype=torch.bfloat16), + ) x = torch.tensor([[2.0, 4.0]], dtype=torch.bfloat16) actual = method.apply(layer, x, layer.bias) @@ -226,6 +244,42 @@ def test_mxfp4_preserves_logical_bias_when_kernel_repacks_it( torch.testing.assert_close(actual, expected) + layer.to(dtype=torch.float32) + assert layer._svdquant_bias.dtype == torch.float32 + + +def test_mxfp4_does_not_duplicate_bias_when_kernel_leaves_it_unchanged( + monkeypatch: pytest.MonkeyPatch, +) -> None: + base_weight = torch.tensor( + [[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]], + dtype=torch.bfloat16, + ) + kernel = _FakeNvfp4Kernel(base_weight) + monkeypatch.setattr(svdquant, "_assert_supported", lambda precision: None) + monkeypatch.setattr(svdquant, "_mxfp4_kernel", lambda: kernel) + method = svdquant.DiffusionSVDQuantLinearMethod(svdquant.DiffusionSVDQuantConfig(precision="mxfp4")) + + layer = torch.nn.Module() + _register_parameter(layer, "qweight", torch.zeros((3, 1), dtype=torch.int8)) + _register_parameter(layer, "wscales", torch.zeros((2, 3), dtype=torch.uint8)) + _register_parameter(layer, "bias", torch.tensor([1.0, 2.0, 3.0], dtype=torch.bfloat16)) + original_bias = layer.bias + layer.smooth_factor = torch.ones(2, dtype=torch.bfloat16) + layer.proj_down = torch.zeros((2, 1), dtype=torch.bfloat16) + layer.proj_up = torch.zeros((3, 1), dtype=torch.bfloat16) + layer.output_size_per_partition = 3 + + method.process_weights_after_loading(layer) + assert layer.bias is original_bias + assert not hasattr(layer, "_svdquant_bias") + + x = torch.tensor([[2.0, 4.0]], dtype=torch.bfloat16) + actual = method.apply(layer, x, layer.bias) + expected = torch.mm(x, base_weight) + original_bias + + torch.testing.assert_close(actual, expected) + def test_compatibility_path_applies_qkv_scale_and_rank_correction( monkeypatch: pytest.MonkeyPatch, diff --git a/vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py b/vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py index 890e56f76d8..a41c3437aae 100644 --- a/vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py +++ b/vllm_omni/diffusion/models/minimax_h3/minimax_h3_transformer.py @@ -1790,7 +1790,6 @@ def load_weights( ( ".attn.qkv_proj.qweight", ".attn.qkv_proj.proj_up", - ".attn.qkv_proj.bias", ".attn.qkv_proj.wcscales", ) ): diff --git a/vllm_omni/quantization/svdquant_config.py b/vllm_omni/quantization/svdquant_config.py index 6b4a70d97c6..91a9839d1bc 100644 --- a/vllm_omni/quantization/svdquant_config.py +++ b/vllm_omni/quantization/svdquant_config.py @@ -81,6 +81,8 @@ def _supports_capability(capability: DeviceCapability | None, supported: set[tup def _assert_supported(precision: str) -> None: + if precision == "mxfp4" and current_platform.is_xpu(): + return supported = _MXFP4_CAPABILITIES if precision == "mxfp4" else _NVFP4_CAPABILITIES if not current_platform.is_cuda(): raise RuntimeError(f"SVDQuant {precision} requires a CUDA device") @@ -352,13 +354,19 @@ def process_weights_after_loading(self, layer: torch.nn.Module) -> None: ) if precision == "mxfp4": - # Marlin replaces layer.bias with a padded and permuted Parameter. - # Keep the logical bias for the post-GEMM add in apply(). - bias = getattr(layer, "bias", None) - if bias is not None: - layer._svdquant_bias = bias.detach().clone() + bias_before = getattr(layer, "bias", None) + bias_snapshot = None if bias_before is None else bias_before.detach().clone() kernel = _mxfp4_kernel() kernel.process_weights_after_loading(layer) + bias_after = getattr(layer, "bias", None) + bias_replaced = bias_after is not bias_before + if bias_after is not None and bias_before is not None: + bias_replaced = bias_replaced or bias_after.shape != bias_before.shape + if bias_snapshot is not None and bias_replaced: + # Marlin replaces and permutes layer.bias. Keep the logical bias + # as a non-persistent buffer so module moves and offload hooks + # handle it, while FlashInfer/XPU retain the original parameter. + layer.register_buffer("_svdquant_bias", bias_snapshot, persistent=False) logger.info_once( "SVDQuant MXFP4 is using %s; the rank correction is not fused into the GEMM.", type(kernel).__name__, @@ -434,7 +442,7 @@ def apply( correction_input, layer.proj_up.transpose(0, 1), ) - bias_to_add = getattr(layer, "_svdquant_bias", bias) if bias is not None else None + bias_to_add = getattr(layer, "_svdquant_bias", bias) if bias_to_add is not None: out.add_(bias_to_add) return out.reshape( From 2dc0772f552f080f84aee5710b007379e5f0dbd8 Mon Sep 17 00:00:00 2001 From: yiliu30 Date: Tue, 27 Oct 2026 21:38:29 -0700 Subject: [PATCH 6/6] fix(svdquant): preserve caller bias ownership Signed-off-by: yiliu30 --- .../quantization/test_svdquant_linear.py | 91 ++++++++++++++++--- .../quantization/test_svdquant_marlin_gpu.py | 3 +- vllm_omni/quantization/svdquant_config.py | 21 ++--- 3 files changed, 90 insertions(+), 25 deletions(-) diff --git a/tests/diffusion/quantization/test_svdquant_linear.py b/tests/diffusion/quantization/test_svdquant_linear.py index a28bc274e5d..11f744f9c4c 100644 --- a/tests/diffusion/quantization/test_svdquant_linear.py +++ b/tests/diffusion/quantization/test_svdquant_linear.py @@ -5,6 +5,7 @@ import pytest import torch +from vllm.model_executor.layers.linear import RowParallelLinear from vllm_omni.quantization import svdquant_config as svdquant @@ -54,6 +55,37 @@ def _register_parameter( ) +def _make_mxfp4_row_parallel( + monkeypatch: pytest.MonkeyPatch, + *, + tp_rank: int, + skip_bias_add: bool = False, +) -> RowParallelLinear: + monkeypatch.setattr(svdquant, "_assert_supported", lambda precision: None) + method = svdquant.DiffusionSVDQuantLinearMethod(svdquant.DiffusionSVDQuantConfig(precision="mxfp4")) + + layer = object.__new__(RowParallelLinear) + torch.nn.Module.__init__(layer) + layer.tp_rank = tp_rank + layer.tp_size = 2 + layer.input_is_parallel = True + layer.reduce_results = True + layer.skip_bias_add = skip_bias_add + layer.return_bias = True + layer.output_size_per_partition = 3 + layer.quant_method = method + + _register_parameter(layer, "qweight", torch.zeros((3, 1), dtype=torch.int8)) + _register_parameter(layer, "wscales", torch.zeros((2, 3), dtype=torch.uint8)) + _register_parameter(layer, "bias", torch.tensor([1.0, 2.0, 3.0], dtype=torch.bfloat16)) + layer.smooth_factor = torch.ones(2, dtype=torch.bfloat16) + layer.proj_down = torch.zeros((2, 1), dtype=torch.bfloat16) + layer.proj_up = torch.zeros((3, 1), dtype=torch.bfloat16) + + method.process_weights_after_loading(layer) + return layer + + def test_supports_only_validated_datacenter_blackwell() -> None: assert not svdquant._supports_capability(SimpleNamespace(major=10, minor=0), svdquant._NVFP4_CAPABILITIES) assert svdquant._supports_capability(SimpleNamespace(major=10, minor=3), svdquant._NVFP4_CAPABILITIES) @@ -207,7 +239,7 @@ def test_mxfp4_compatibility_path_applies_rank_correction( torch.testing.assert_close(kernel.last_input, smoothed) -def test_mxfp4_preserves_logical_bias_when_kernel_repacks_it( +def test_mxfp4_restores_logical_bias_when_kernel_repacks_it( monkeypatch: pytest.MonkeyPatch, ) -> None: base_weight = torch.tensor( @@ -229,23 +261,58 @@ def test_mxfp4_preserves_logical_bias_when_kernel_repacks_it( layer.output_size_per_partition = 3 method.process_weights_after_loading(layer) - assert layer.bias.shape == (4,) - assert "_svdquant_bias" in dict(layer.named_buffers()) - assert "_svdquant_bias" not in layer.state_dict() - assert layer._svdquant_bias is not layer.bias - torch.testing.assert_close( - layer._svdquant_bias, - torch.tensor([1.0, 2.0, 3.0], dtype=torch.bfloat16), - ) + logical_bias = torch.tensor([1.0, 2.0, 3.0], dtype=torch.bfloat16) + assert layer.bias.shape == logical_bias.shape + assert not hasattr(layer, "_svdquant_bias") + torch.testing.assert_close(layer.bias, logical_bias) x = torch.tensor([[2.0, 4.0]], dtype=torch.bfloat16) - actual = method.apply(layer, x, layer.bias) - expected = torch.mm(x, base_weight) + torch.tensor([1.0, 2.0, 3.0], dtype=torch.bfloat16) + actual_without_bias = method.apply(layer, x, None) + torch.testing.assert_close(actual_without_bias, torch.mm(x, base_weight)) + actual = method.apply(layer, x, layer.bias) + expected = torch.mm(x, base_weight) + logical_bias torch.testing.assert_close(actual, expected) layer.to(dtype=torch.float32) - assert layer._svdquant_bias.dtype == torch.float32 + assert layer.bias.dtype == torch.float32 + + +def test_mxfp4_row_parallel_tp2_adds_bias_once( + monkeypatch: pytest.MonkeyPatch, +) -> None: + base_weight = torch.zeros((2, 3), dtype=torch.bfloat16) + kernel = _FakeMarlinBiasKernel(base_weight) + monkeypatch.setattr(svdquant, "_mxfp4_kernel", lambda: kernel) + monkeypatch.setattr("vllm.model_executor.layers.linear.tensor_model_parallel_all_reduce", lambda value: value) + + rank0 = _make_mxfp4_row_parallel(monkeypatch, tp_rank=0) + rank1 = _make_mxfp4_row_parallel(monkeypatch, tp_rank=1) + x = torch.tensor([[2.0, 4.0]], dtype=torch.bfloat16) + + output_rank0, _ = rank0(x) + output_rank1, _ = rank1(x) + combined = output_rank0 + output_rank1 + expected = torch.tensor([[1.0, 2.0, 3.0]], dtype=torch.bfloat16) + + torch.testing.assert_close(combined, expected) + + +def test_mxfp4_row_parallel_skip_bias_add_returns_logical_bias( + monkeypatch: pytest.MonkeyPatch, +) -> None: + base_weight = torch.zeros((2, 3), dtype=torch.bfloat16) + kernel = _FakeMarlinBiasKernel(base_weight) + monkeypatch.setattr(svdquant, "_mxfp4_kernel", lambda: kernel) + monkeypatch.setattr("vllm.model_executor.layers.linear.tensor_model_parallel_all_reduce", lambda value: value) + + layer = _make_mxfp4_row_parallel(monkeypatch, tp_rank=0, skip_bias_add=True) + x = torch.tensor([[2.0, 4.0]], dtype=torch.bfloat16) + output, output_bias = layer(x) + + torch.testing.assert_close(output, torch.zeros_like(output)) + assert output_bias is not None + torch.testing.assert_close(output_bias, torch.tensor([1.0, 2.0, 3.0], dtype=torch.bfloat16)) def test_mxfp4_does_not_duplicate_bias_when_kernel_leaves_it_unchanged( diff --git a/tests/diffusion/quantization/test_svdquant_marlin_gpu.py b/tests/diffusion/quantization/test_svdquant_marlin_gpu.py index 1b11230d073..baf0ed24ed5 100644 --- a/tests/diffusion/quantization/test_svdquant_marlin_gpu.py +++ b/tests/diffusion/quantization/test_svdquant_marlin_gpu.py @@ -81,7 +81,8 @@ def test_mxfp4_replicated_linear_loads_with_forced_marlin(mocker): param.weight_loader(param, value) layer.quant_method.process_weights_after_loading(layer) - assert layer.bias.shape == (128,) + assert layer.bias.shape == (96,) + torch.testing.assert_close(layer.bias, bias) actual = layer(inputs)[0] expected = F.linear(inputs, torch.ones((96, 128), device="cuda", dtype=torch.bfloat16), bias) diff --git a/vllm_omni/quantization/svdquant_config.py b/vllm_omni/quantization/svdquant_config.py index 91a9839d1bc..3a69fbd7069 100644 --- a/vllm_omni/quantization/svdquant_config.py +++ b/vllm_omni/quantization/svdquant_config.py @@ -358,15 +358,13 @@ def process_weights_after_loading(self, layer: torch.nn.Module) -> None: bias_snapshot = None if bias_before is None else bias_before.detach().clone() kernel = _mxfp4_kernel() kernel.process_weights_after_loading(layer) - bias_after = getattr(layer, "bias", None) - bias_replaced = bias_after is not bias_before - if bias_after is not None and bias_before is not None: - bias_replaced = bias_replaced or bias_after.shape != bias_before.shape - if bias_snapshot is not None and bias_replaced: - # Marlin replaces and permutes layer.bias. Keep the logical bias - # as a non-persistent buffer so module moves and offload hooks - # handle it, while FlashInfer/XPU retain the original parameter. - layer.register_buffer("_svdquant_bias", bias_snapshot, persistent=False) + if bias_before is not None and bias_snapshot is not None: + # MXFP4 kernels may pad or permute layer.bias for their internal + # GEMM layout, but apply() always passes bias=None to the kernel. + # Restore the checkpoint bias so callers retain control over + # whether it is added and skip_bias_add returns the logical tensor. + bias_before.data = bias_snapshot + layer.bias = bias_before logger.info_once( "SVDQuant MXFP4 is using %s; the rank correction is not fused into the GEMM.", type(kernel).__name__, @@ -442,9 +440,8 @@ def apply( correction_input, layer.proj_up.transpose(0, 1), ) - bias_to_add = getattr(layer, "_svdquant_bias", bias) - if bias_to_add is not None: - out.add_(bias_to_add) + if bias is not None: + out.add_(bias) return out.reshape( *original_shape[:-1], layer.output_size_per_partition,