Skip to content
34 changes: 20 additions & 14 deletions docs/user_guide/quantization/svdquant.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@

[SVDQuant](https://arxiv.org/abs/2411.05007) combines four-bit weights
and activations with a small low-rank branch that corrects part of the
quantization error. vLLM-Omni consumes an offline-quantized checkpoint;
it does not calibrate the model while loading.
quantization error. vLLM-Omni consumes an offline-quantized NVFP4 or MXFP4
checkpoint; it does not calibrate the model while loading.

## Checkpoint contract

Expand All @@ -29,22 +29,28 @@ rank `R`, the checkpoint stores:
| Suffix | Shape | dtype |
| --- | --- | --- |
| `qweight` | `(N, K / 2)` | `int8` (two packed FP4 values per byte) |
| `wscales` | `(K / 16, N)` | `float8_e4m3fn` |
| `wscales` (NVFP4) | `(K / 16, N)` | `float8_e4m3fn` |
| `wscales` (MXFP4) | `(K / 32, N)` | `uint8` (E8M0 scales) |
| `proj_down` | `(K, R)` | `bfloat16` |
| `proj_up` | `(N, R)` | `bfloat16` |
| `smooth_factor` | `(K,)` | `bfloat16` |
| `wcscales` | `(N,)` | `bfloat16` |
| `wtscale` | `(1,)` | `bfloat16` |
| `wcscales` (NVFP4 only) | `(N,)` | `bfloat16` |
| `wtscale` (NVFP4 only) | `(1,)` | `bfloat16` |

`K` must be divisible by 16 on every tensor-parallel rank. Set `wcscales` to
ones when no per-output correction is needed. Modules listed in
`modules_to_not_convert` keep their checkpoint precision.
`K` must be divisible by 16 for NVFP4 or 32 for MXFP4 on every
tensor-parallel rank. Set `wcscales` to ones when no NVFP4 per-output
correction is needed. MXFP4 checkpoints do not contain `wcscales` or
`wtscale`. Modules listed in `modules_to_not_convert` keep their checkpoint
precision.

## Runtime support

The compatibility path accepts BF16 inputs and executes an NVFP4 GEMM followed
by the BF16 rank correction. It supports vLLM's FlashInfer, CUTLASS, and FBGEMM
NVFP4 tensor layouts; incompatible forced backends fail during model loading.
SM103 is the currently validated and enabled hardware target. Native fusion of
the NVFP4 GEMM and rank correction is separate from this checkpoint-loading
contract.
The compatibility path accepts BF16 inputs and executes a four-bit GEMM followed
by the BF16 rank correction. NVFP4 is enabled on SM103 and supports vLLM's
FlashInfer, CUTLASS, and FBGEMM tensor layouts; incompatible forced backends
fail during model loading.

MXFP4 is enabled on SM100 and SM103 and uses vLLM's standard MXFP4 kernel
selection. FlashInfer provides W4A4 execution when its CuTe-DSL backend is
available; otherwise Marlin uses W4A16. Native fusion of the quantized GEMM and
rank correction is separate from this checkpoint-loading contract.
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,15 @@ def __init__(self, loader):
self.weight.weight_loader = loader


class _MultiWeightTarget(nn.Module):
def __init__(self, loaders):
super().__init__()
for name, loader in loaders.items():
parameter = nn.Parameter(torch.empty(1))
parameter.weight_loader = loader
self.register_parameter(name, parameter)


def test_model_load_weights_transforms_before_calling_vllm_loader():
from vllm_omni.diffusion.models.minimax_h3.minimax_h3_transformer import (
MiniMaxH3DiTArchConfig,
Expand Down Expand Up @@ -212,6 +221,57 @@ def fc1_loader(param, loaded_weight, shard_id):
]


def test_model_load_weights_reorders_serialized_svd_qkv_tensors():
from vllm_omni.diffusion.models.minimax_h3.minimax_h3_transformer import (
MiniMaxH3DiTArchConfig,
MiniMaxH3DiTModel,
)

calls = {}

def capture(name):
def loader(param, loaded_weight):
del param
calls[name] = loaded_weight.clone()

return loader

model = object.__new__(MiniMaxH3DiTModel)
nn.Module.__init__(model)
model.arch = MiniMaxH3DiTArchConfig(
hidden_size=1,
num_attention_heads=2,
attention_head_dim=1,
ffn_hidden_size=2,
)
model.blocks = nn.ModuleList([nn.Module()])
model.blocks[0].attn = nn.Module()
model.blocks[0].attn.qkv_proj = _MultiWeightTarget(
{name: capture(name) for name in ("qweight", "wscales", "proj_up", "bias", "wcscales")}
)

qweight = torch.arange(12, dtype=torch.int8).reshape(6, 2)
wscales = torch.arange(18, dtype=torch.uint8).reshape(3, 6)
proj_up = torch.arange(6, dtype=torch.bfloat16).reshape(6, 1)
bias = torch.arange(6, dtype=torch.bfloat16)
wcscales = torch.arange(6, dtype=torch.bfloat16)
names = ("qweight", "wscales", "proj_up", "bias", "wcscales")
loaded = model.load_weights(
[
(f"blocks.0.attn.qkv_proj.{name}", tensor)
for name, tensor in zip(names, (qweight, wscales, proj_up, bias, wcscales))
]
)

assert loaded == {f"blocks.0.attn.qkv_proj.{name}" for name in names}
expected_rows = [0, 3, 1, 4, 2, 5]
assert calls["qweight"][:, 0].tolist() == [row * qweight.shape[1] for row in expected_rows]
assert calls["proj_up"][:, 0].tolist() == expected_rows
assert calls["bias"].tolist() == expected_rows
assert calls["wcscales"].tolist() == expected_rows
assert calls["wscales"][0, :].tolist() == expected_rows


def test_loader_adapter_declares_equivalent_direct_mmap_transform(monkeypatch):
from vllm_omni.diffusion.model_loader.checkpoint_adapters import (
get_direct_mmap_adapter,
Expand Down
60 changes: 56 additions & 4 deletions tests/diffusion/quantization/test_svdquant_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ def test_config_from_checkpoint_dict() -> None:
("kwargs", "message"),
[
({"rank": 0}, "rank must be positive"),
({"precision": "int4"}, "NVFP4 checkpoints only"),
({"precision": "int4"}, "supports serialized"),
({"act_unsigned": True}, "unsigned activations"),
],
)
Expand Down Expand Up @@ -71,6 +71,14 @@ def test_factory_builds_svdquant_with_extra_checkpoint_fields() -> None:
assert "svdquant" in SUPPORTED_QUANTIZATION_METHODS


def test_config_accepts_mxfp4_precision() -> None:
config = DiffusionSVDQuantConfig.from_config({"rank": 16, "precision": "mxfp4"})

assert config.rank == 16
assert config.precision == "mxfp4"
assert config.get_supported_act_dtypes() == [torch.bfloat16]


def test_skipped_and_non_linear_modules_remain_unquantized() -> None:
config = DiffusionSVDQuantConfig(modules_to_not_convert=["blocks.0.adaln_proj.linear"])
linear = Mock(spec=LinearBase)
Expand All @@ -85,7 +93,7 @@ def test_skipped_and_non_linear_modules_remain_unquantized() -> None:
def test_linear_method_creates_canonical_partitioned_weights(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(svdquant_config, "_assert_supported", lambda: None)
monkeypatch.setattr(svdquant_config, "_assert_supported", lambda precision: None)
method = DiffusionSVDQuantLinearMethod(DiffusionSVDQuantConfig(rank=32))
layer = torch.nn.Module()

Expand Down Expand Up @@ -117,10 +125,37 @@ def test_linear_method_creates_canonical_partitioned_weights(
assert layer.wcscales.output_dim == 0


def test_linear_method_creates_mxfp4_block32_weights_without_global_scales(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(svdquant_config, "_assert_supported", lambda precision: None)
method = DiffusionSVDQuantLinearMethod(DiffusionSVDQuantConfig(rank=32, precision="mxfp4"))
layer = torch.nn.Module()

method.create_weights(
layer,
input_size_per_partition=128,
output_partition_sizes=[64, 64],
input_size=128,
output_size=128,
params_dtype=torch.bfloat16,
)

assert layer.qweight.shape == (128, 64)
assert layer.qweight.dtype == torch.int8
assert layer.wscales.shape == (4, 128)
assert layer.wscales.dtype == torch.uint8
assert layer.proj_down.shape == (128, 32)
assert layer.proj_up.shape == (128, 32)
assert layer.smooth_factor.shape == (128,)
assert not hasattr(layer, "wcscales")
assert not hasattr(layer, "wtscale")


def test_linear_method_rejects_misaligned_input_partition(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(svdquant_config, "_assert_supported", lambda: None)
monkeypatch.setattr(svdquant_config, "_assert_supported", lambda precision: None)
method = DiffusionSVDQuantLinearMethod(DiffusionSVDQuantConfig())

with pytest.raises(ValueError, match="divisible by the block size 16"):
Expand All @@ -134,10 +169,27 @@ def test_linear_method_rejects_misaligned_input_partition(
)


def test_mxfp4_rejects_input_partition_misaligned_to_block32(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(svdquant_config, "_assert_supported", lambda precision: None)
method = DiffusionSVDQuantLinearMethod(DiffusionSVDQuantConfig(precision="mxfp4"))

with pytest.raises(ValueError, match="divisible by the block size 32"):
method.create_weights(
torch.nn.Module(),
input_size_per_partition=48,
output_partition_sizes=[32],
input_size=48,
output_size=32,
params_dtype=torch.bfloat16,
)


def test_active_linear_uses_svdquant_method(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(svdquant_config, "_assert_supported", lambda: None)
monkeypatch.setattr(svdquant_config, "_assert_supported", lambda precision: None)
config = DiffusionSVDQuantConfig()
linear = Mock(spec=LinearBase)

Expand Down
82 changes: 73 additions & 9 deletions tests/diffusion/quantization/test_svdquant_linear.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,11 +46,18 @@ def _register_parameter(


def test_supports_only_validated_datacenter_blackwell() -> None:
assert not svdquant._supports_capability(SimpleNamespace(major=10, minor=0))
assert svdquant._supports_capability(SimpleNamespace(major=10, minor=3))
assert not svdquant._supports_capability(SimpleNamespace(major=11, minor=0))
assert not svdquant._supports_capability(SimpleNamespace(major=12, minor=0))
assert not svdquant._supports_capability(None)
assert not svdquant._supports_capability(SimpleNamespace(major=10, minor=0), svdquant._NVFP4_CAPABILITIES)
assert svdquant._supports_capability(SimpleNamespace(major=10, minor=3), svdquant._NVFP4_CAPABILITIES)
assert not svdquant._supports_capability(SimpleNamespace(major=11, minor=0), svdquant._NVFP4_CAPABILITIES)
assert not svdquant._supports_capability(SimpleNamespace(major=12, minor=0), svdquant._NVFP4_CAPABILITIES)
assert not svdquant._supports_capability(None, svdquant._NVFP4_CAPABILITIES)


def test_mxfp4_supports_blackwell_sm100_and_sm103() -> None:
assert svdquant._supports_capability(SimpleNamespace(major=10, minor=0), svdquant._MXFP4_CAPABILITIES)
assert svdquant._supports_capability(SimpleNamespace(major=10, minor=3), svdquant._MXFP4_CAPABILITIES)
assert not svdquant._supports_capability(SimpleNamespace(major=9, minor=0), svdquant._MXFP4_CAPABILITIES)
assert not svdquant._supports_capability(None, svdquant._MXFP4_CAPABILITIES)


def test_rejects_nvfp4_backend_with_incompatible_layer_contract(
Expand All @@ -73,7 +80,7 @@ def test_prepare_weights_uses_existing_nvfp4_layout_contract(
monkeypatch: pytest.MonkeyPatch,
) -> None:
kernel = _FakeNvfp4Kernel()
monkeypatch.setattr(svdquant, "_assert_supported", lambda: None)
monkeypatch.setattr(svdquant, "_assert_supported", lambda precision: None)
monkeypatch.setattr(svdquant, "_nvfp4_kernel", lambda: kernel)
method = svdquant.DiffusionSVDQuantLinearMethod(svdquant.DiffusionSVDQuantConfig())
layer = torch.nn.Module()
Expand Down Expand Up @@ -105,7 +112,7 @@ def test_prepare_weights_drops_identity_output_scale(
monkeypatch: pytest.MonkeyPatch,
) -> None:
kernel = _FakeNvfp4Kernel()
monkeypatch.setattr(svdquant, "_assert_supported", lambda: None)
monkeypatch.setattr(svdquant, "_assert_supported", lambda precision: None)
monkeypatch.setattr(svdquant, "_nvfp4_kernel", lambda: kernel)
method = svdquant.DiffusionSVDQuantLinearMethod(svdquant.DiffusionSVDQuantConfig())
layer = torch.nn.Module()
Expand All @@ -123,6 +130,63 @@ def test_prepare_weights_drops_identity_output_scale(
assert layer.output_channel_scale is None


def test_prepare_weights_uses_mxfp4_block32_layout(
monkeypatch: pytest.MonkeyPatch,
) -> None:
kernel = _FakeNvfp4Kernel()
monkeypatch.setattr(svdquant, "_assert_supported", lambda precision: None)
monkeypatch.setattr(svdquant, "_mxfp4_kernel", lambda: kernel)
method = svdquant.DiffusionSVDQuantLinearMethod(svdquant.DiffusionSVDQuantConfig(precision="mxfp4"))
layer = torch.nn.Module()
qweight = torch.arange(16, dtype=torch.uint8).reshape(4, 4)
wscales = torch.tensor([[130, 131, 132, 133], [134, 135, 136, 137]], dtype=torch.uint8)
_register_parameter(layer, "qweight", qweight.view(torch.int8))
_register_parameter(layer, "wscales", wscales)

method.process_weights_after_loading(layer)

assert kernel.processed
assert not hasattr(layer, "qweight")
assert not hasattr(layer, "wscales")
assert layer.weight.dtype == torch.uint8
assert torch.equal(layer.weight, qweight)
assert layer.weight_scale.dtype == torch.uint8
assert torch.equal(layer.weight_scale, wscales.transpose(0, 1))
assert not hasattr(layer, "alpha")
assert not hasattr(layer, "input_global_scale_inv")
assert not hasattr(layer, "output_channel_scale")


def test_mxfp4_compatibility_path_applies_rank_correction(
monkeypatch: pytest.MonkeyPatch,
) -> None:
base_weight = torch.tensor(
[[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]],
dtype=torch.bfloat16,
)
kernel = _FakeNvfp4Kernel(base_weight)
monkeypatch.setattr(svdquant, "_assert_supported", lambda precision: None)
monkeypatch.setattr(svdquant, "_mxfp4_kernel", lambda: kernel)
method = svdquant.DiffusionSVDQuantLinearMethod(svdquant.DiffusionSVDQuantConfig(precision="mxfp4"))

layer = torch.nn.Module()
layer.smooth_factor = torch.tensor([2.0, 4.0], dtype=torch.bfloat16)
layer.proj_down = torch.tensor([[1.0], [2.0]], dtype=torch.bfloat16)
layer.proj_up = torch.tensor([[1.0], [2.0], [3.0]], dtype=torch.bfloat16)
layer.output_size_per_partition = 3
x = torch.tensor([[2.0, 4.0], [4.0, 8.0]], dtype=torch.bfloat16)
bias = torch.tensor([1.0, 2.0, 3.0], dtype=torch.bfloat16)

actual = method.apply(layer, x, bias)

smoothed = x / layer.smooth_factor
expected = torch.mm(smoothed, base_weight)
expected = expected + torch.mm(torch.mm(x, layer.proj_down), layer.proj_up.T)
expected = expected + bias
torch.testing.assert_close(actual, expected)
torch.testing.assert_close(kernel.last_input, smoothed)


def test_compatibility_path_applies_qkv_scale_and_rank_correction(
monkeypatch: pytest.MonkeyPatch,
) -> None:
Expand All @@ -131,7 +195,7 @@ def test_compatibility_path_applies_qkv_scale_and_rank_correction(
dtype=torch.bfloat16,
)
kernel = _FakeNvfp4Kernel(base_weight)
monkeypatch.setattr(svdquant, "_assert_supported", lambda: None)
monkeypatch.setattr(svdquant, "_assert_supported", lambda precision: None)
monkeypatch.setattr(svdquant, "_nvfp4_kernel", lambda: kernel)
method = svdquant.DiffusionSVDQuantLinearMethod(svdquant.DiffusionSVDQuantConfig())

Expand All @@ -158,7 +222,7 @@ def test_compatibility_path_applies_qkv_scale_and_rank_correction(
def test_compatibility_path_rejects_fp16_activations(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(svdquant, "_assert_supported", lambda: None)
monkeypatch.setattr(svdquant, "_assert_supported", lambda precision: None)
method = svdquant.DiffusionSVDQuantLinearMethod(svdquant.DiffusionSVDQuantConfig())
layer = torch.nn.Module()
with pytest.raises(ValueError, match="requires BF16 activations"):
Expand Down
Loading
Loading