Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
08fc188
[Perf][GLM-5.3-Flash] DeepGEMM FP8 MegaMoE with fused shared experts
gcanlin Oct 9, 2026
8b5ccd7
[GLM-5.3-Flash] Run the FP8 MegaMoE through DeepGemmMegaMoEBackend
gcanlin Oct 9, 2026
be97fe0
Merge commit 'df3bd094ae' into perf/glm53-fp8-megamoe
gcanlin Oct 20, 2026
0d3243d
[GLM-5.3-Flash] Fix pre-commit: typed block size, no call in test def…
gcanlin Oct 20, 2026
5263538
[GLM-5.3-Flash] MegaMoE: requantize complete weight/scale pairs only
gcanlin Oct 20, 2026
2434fcd
[GLM-5.3-Flash] MegaMoE: requantize in process_weights_after_loading
gcanlin Oct 21, 2026
dd3cbc8
[GLM-5.3-Flash] MegaMoE: refuse layerwise reload until #57733
gcanlin Oct 22, 2026
5da5c7f
Merge commit '37e16689fa' into perf/glm53-fp8-megamoe
gcanlin Nov 2, 2026
b5315f2
[GLM-5.3-Flash] MegaMoE: reload through the model hook
gcanlin Nov 2, 2026
0c909ba
[GLM-5.3-Flash] MegaMoE: keep only the tests that catch silent errors
gcanlin Nov 2, 2026
0c4725d
[GLM-5.3-Flash] MegaMoE: keep the adapter out of the shared backend
gcanlin Nov 2, 2026
b80b925
[GLM-5.3-Flash] MegaMoE: read the shared-expert shard id like the bac…
gcanlin Nov 2, 2026
4dcadcb
[GLM-5.3-Flash] MegaMoE: make the shared-expert shard id a static method
gcanlin Nov 2, 2026
3454a5f
[GLM-5.3-Flash] MegaMoE: move the block-FP8 requant to mxfp8_utils
gcanlin Nov 2, 2026
0d99737
Merge branch 'main' into perf/glm53-fp8-megamoe
gcanlin Nov 3, 2026
e7586c4
[GLM-5.3-Flash] MegaMoE: warn when the requantization rounds the weights
gcanlin Nov 3, 2026
317550c
Merge branch 'main' into perf/glm53-fp8-megamoe
mergify[bot] Nov 4, 2026
f308eb6
[GLM-5.3-Flash] MegaMoE: inline the moe_backend check
gcanlin Nov 4, 2026
259d365
Merge branch 'main' into perf/glm53-fp8-megamoe
vllm-agent Nov 4, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
195 changes: 195 additions & 0 deletions tests/models/glm5next/test_mega_moe.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,195 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

from types import SimpleNamespace

import torch

from vllm.model_executor.layers.fused_moe.deep_gemm_mega_moe import (
DeepGemmSm100Fp8MegaMoEBackend,
requant_block_fp8_to_ue8m0,
)
from vllm.models.glm5next.nvidia.mega_moe import Glm5NextMegaMoEExperts

BLOCK = (128, 128)


def _block_fp8(real: torch.Tensor, power_of_two: bool = False):
"""Quantize ``(..., mn, k)`` to E4M3 with one float32 scale per 128x128."""
*lead, mn, k = real.shape
blocks = real.reshape(*lead, mn // 128, 128, k // 128, 128)
scale = blocks.abs().amax(dim=(-3, -1)) / 448.0
if power_of_two:
scale = torch.exp2(torch.ceil(torch.log2(scale)))
full = scale.repeat_interleave(128, -2).repeat_interleave(128, -1)
return (real / full).to(torch.float8_e4m3fn), scale


def _dequant_block(weight: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
return weight.float() * scale.repeat_interleave(128, -2).repeat_interleave(128, -1)


def test_requant_gives_power_of_two_scale_per_32_columns():
torch.manual_seed(0)
real = torch.randn(3, 256, 384) * 0.02
weight, scale = _block_fp8(real)

requant, sf = requant_block_fp8_to_ue8m0(weight, scale, chunk=2)

assert requant.dtype == torch.float8_e4m3fn
assert requant.shape == weight.shape
assert sf.shape == (3, 256, 384 // 32)
assert torch.equal(torch.log2(sf), torch.round(torch.log2(sf)))
assert requant.float().abs().max() <= 448.0
original = _dequant_block(weight, scale)
restored = requant.float() * sf.repeat_interleave(32, -1)
# One more E4M3 rounding of every element: a few percent, never garbage.
rel = ((restored - original).norm() / original.norm()).item()
assert 0 < rel < 0.05


def test_requant_is_exact_for_power_of_two_block_scales():
torch.manual_seed(1)
real = torch.randn(256, 256) * 0.05
weight, scale = _block_fp8(real, power_of_two=True)

requant, sf = requant_block_fp8_to_ue8m0(weight, scale)

# A finer power-of-two scale only shifts exponents, so nothing rounds.
assert torch.equal(
requant.float() * sf.repeat_interleave(32, -1), _dequant_block(weight, scale)
)


def _experts(**kwargs) -> Glm5NextMegaMoEExperts:
vllm_config = SimpleNamespace(
scheduler_config=SimpleNamespace(max_num_batched_tokens=64),
compilation_config=SimpleNamespace(static_forward_context={}),
)
return Glm5NextMegaMoEExperts(
vllm_config, # type: ignore[arg-type]
num_experts=4,
num_local_experts=2,
experts_start_idx=2,
top_k=2,
hidden_size=256,
intermediate_size=128,
weight_block_size=BLOCK,
num_shared_experts=1,
prefix="model.layers.3.mlp.experts",
**kwargs,
)


def test_loader_params_match_the_block_fp8_expert_mapping():
experts = _experts()

names = dict(experts.named_parameters())

assert names["routed_experts.w13_weight"].shape == (2, 256, 256)
assert names["routed_experts.w13_weight"].dtype == torch.float8_e4m3fn
assert names["routed_experts.w13_weight_scale_inv"].shape == (2, 2, 2)
assert names["routed_experts.w2_weight"].shape == (2, 256, 128)
assert names["routed_experts.w2_weight_scale_inv"].shape == (2, 2, 1)


def test_weight_loader_places_gate_then_up_for_local_experts_only():
experts = _experts()
loader = experts.routed_experts
assert loader is not None
gate = torch.full((128, 256), 2.0).to(torch.float8_e4m3fn)
up = torch.full((128, 256), 3.0).to(torch.float8_e4m3fn)
up_scale = torch.full((1, 2), 0.5)

def load(param, tensor, name, shard, expert):
return experts.weight_loader(
param, tensor, name, shard_id=shard, expert_id=expert, return_success=True
)

# Experts 0 and 1 live on another rank.
assert not load(
loader.w13_weight, gate, "experts.routed_experts.w13_weight", "w1", 1
)
assert load(loader.w13_weight, gate, "experts.routed_experts.w13_weight", "w1", 3)
assert load(loader.w13_weight, up, "experts.routed_experts.w13_weight", "w3", 3)
assert load(
loader.w13_weight_scale_inv,
up_scale,
"experts.routed_experts.w13_weight_scale_inv",
"w3",
3,
)

w13 = loader.w13_weight.data.float()
assert torch.all(w13[1, :128] == 2.0) and torch.all(w13[1, 128:] == 3.0)
assert torch.all(w13[0] == 0.0)
scale = loader.w13_weight_scale_inv.data
assert torch.all(scale[1, 1] == 0.5) and torch.all(scale[1, 0] == 0.0)


class _FakeDeepGemm:
"""Identity layout transforms, so the test sees what the backend passes."""

sf_calls: list[tuple] = []

@classmethod
def transform_sf_into_required_layout(cls, sf, mn, k, gran, num_groups):
cls.sf_calls.append((tuple(sf.shape), mn, k, gran, num_groups))
return sf

@staticmethod
def transform_weights_for_mega_moe(l1, l2):
return l1, l2


def test_fp8_backend_requantizes_routed_weights_to_1x32(monkeypatch):
torch.manual_seed(2)
monkeypatch.setattr("vllm.utils.deep_gemm._import_deep_gemm", lambda: _FakeDeepGemm)
_FakeDeepGemm.sf_calls = []
w13, s13 = _block_fp8(torch.randn(2, 256, 256) * 0.02)
w2, s2 = _block_fp8(torch.randn(2, 256, 128) * 0.02)

(l1, l1_sf), (l2, l2_sf) = DeepGemmSm100Fp8MegaMoEBackend().transform_weights(
w13_weight=w13,
w13_weight_scale=s13,
w2_weight=w2,
w2_weight_scale=s2,
num_local_experts=2,
hidden_size=256,
intermediate_size=128,
)

assert _FakeDeepGemm.sf_calls == [
((2, 256, 8), 256, 256, (1, 32), 2),
((2, 256, 4), 256, 128, (1, 32), 2),
]
for q, sf, w, s in ((l1, l1_sf, w13, s13), (l2, l2_sf, w2, s2)):
assert q.dtype == torch.float8_e4m3fn
restored = q.float() * sf.repeat_interleave(32, -1)
original = _dequant_block(w, s)
assert ((restored - original).norm() / original.norm()).item() < 0.05


def test_finalize_hands_the_checkpoint_weights_to_the_backend(monkeypatch):
experts = _experts()
seen = {}

class Backend:
mma_type = "fp8xfp8"

def transform_weights(self, **kwargs):
seen.update(kwargs)
return ("l1", "l1_sf"), ("l2", "l2_sf")

monkeypatch.setattr(experts, "_ensure_backend", lambda: Backend())
monkeypatch.setattr("vllm.utils.deep_gemm._import_deep_gemm", lambda: object())

experts.finalize_weights()

assert seen["w13_weight"].dtype == torch.float8_e4m3fn
assert seen["w13_weight_scale"].shape == (2, 2, 2)
assert seen["w2_weight_scale"].shape == (2, 2, 1)
assert experts._transformed_l1_weights == ("l1", "l1_sf")
# Only the kernel's tensors remain.
assert experts.routed_experts is None
assert dict(experts.named_parameters()) == {}
166 changes: 166 additions & 0 deletions vllm/model_executor/layers/fused_moe/deep_gemm_mega_moe.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,66 @@ def ue8m0_uint8_to_float(sf: torch.Tensor) -> torch.Tensor:
return (sf.to(torch.int32) << 23).view(torch.float32)


_FP8_MAX = 448.0
_SF_GRAN_K = 32


def requant_block_fp8_to_ue8m0(
weight: torch.Tensor,
scale: torch.Tensor,
chunk: int = 8,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Requantize block-FP8 weights to E4M3 with one UE8M0 scale per 1x32.

The SM100 MegaMoE feeds its scales to the block-scaled MMA, so they have to
be powers of two, one per row and 32 columns. A checkpoint with float32
scales per block (e.g. 128x128) is rounded once more to get there.

Args:
weight: ``(..., mn, k)`` E4M3 values.
scale: ``(..., mn / block_m, k / block_k)`` float32 dequantization
scales (``real = weight * scale``).
chunk: Leading-dimension entries converted at a time, bounding the
float32 workspace.

Returns:
``(weight, sf)``: the requantized E4M3 weights and their float32
power-of-two scales, ``(..., mn, k / 32)``.

"""
assert weight.dtype == torch.float8_e4m3fn, weight.dtype
assert scale.dtype == torch.float32, scale.dtype
*lead, mn, k = weight.shape
block_m, block_k = mn // scale.shape[-2], k // scale.shape[-1]
assert k % _SF_GRAN_K == 0, k
assert tuple(scale.shape) == (*lead, mn // block_m, k // block_k) and (
mn % block_m == 0 and k % block_k == 0
), (tuple(weight.shape), tuple(scale.shape))
w = weight.reshape(-1, mn, k)
s = scale.reshape(-1, scale.shape[-2], scale.shape[-1])
out = torch.empty_like(w)
sf = torch.empty(
w.shape[0], mn, k // _SF_GRAN_K, dtype=torch.float32, device=w.device
)
for start in range(0, w.shape[0], chunk):
end = min(start + chunk, w.shape[0])
s_full = (
s[start:end]
.repeat_interleave(block_m, dim=1)
.repeat_interleave(block_k, dim=2)
)
real = w[start:end].to(torch.float32) * s_full
groups = real.view(end - start, mn, k // _SF_GRAN_K, _SF_GRAN_K)
amax = groups.abs().amax(dim=-1).clamp_(min=1e-30)
# Smallest power of two holding the group's largest value in E4M3.
new_sf = torch.exp2(torch.ceil(torch.log2(amax / _FP8_MAX)))
out[start:end] = ((groups / new_sf.unsqueeze(-1)).view(end - start, mn, k)).to(
torch.float8_e4m3fn
)
sf[start:end] = new_sf
return out.view(*lead, mn, k), sf.view(*lead, mn, k // _SF_GRAN_K)


class DeepGemmMegaMoEBackend:
"""Abstract MegaMoE backend that hides deep_gemm-specific details.

Expand Down Expand Up @@ -433,13 +493,115 @@ def bf16_mega_gate(
)


class DeepGemmSm100Fp8MegaMoEBackend(DeepGemmSm100MegaMoEBackend):
"""SM100 MegaMoE over block-FP8 experts (``fp8_fp4_mega_moe`` with E4M3
weights). Routed and shared expert weights arrive with float32 block scales
and are requantized to the kernel's UE8M0 1x32 scales."""

mma_type = "fp8xfp8"

@staticmethod
def _to_kernel_format(
deep_gemm, weight: torch.Tensor, scale: torch.Tensor, num_groups: int
) -> tuple[torch.Tensor, torch.Tensor]:
weight, sf = requant_block_fp8_to_ue8m0(weight, scale)
mn, k = weight.shape[-2:]
sf = deep_gemm.transform_sf_into_required_layout(
sf.contiguous(), mn, k, (1, _SF_GRAN_K), num_groups
)
return weight, sf

def transform_weights(
self,
*,
w13_weight: torch.Tensor,
w13_weight_scale: torch.Tensor,
w2_weight: torch.Tensor,
w2_weight_scale: torch.Tensor,
num_local_experts: int,
hidden_size: int,
intermediate_size: int,
activation: str | None = None,
) -> tuple[
tuple[torch.Tensor, torch.Tensor],
tuple[torch.Tensor, torch.Tensor],
]:
from vllm.utils.deep_gemm import _import_deep_gemm

deep_gemm = _import_deep_gemm()
kwargs = {} if activation is None else {"activation": activation}
return deep_gemm.transform_weights_for_mega_moe(
self._to_kernel_format(
deep_gemm, w13_weight, w13_weight_scale, num_local_experts
),
self._to_kernel_format(
deep_gemm, w2_weight, w2_weight_scale, num_local_experts
),
**kwargs,
)

def transform_shared_expert_weights(
self,
*,
shared_experts: nn.Module,
num_shared_experts: int,
hidden_size: int,
intermediate_size: int,
prefix: str,
) -> (
tuple[
tuple[torch.Tensor, torch.Tensor],
tuple[torch.Tensor, torch.Tensor],
]
| None
):
from vllm.utils.deep_gemm import _import_deep_gemm

deep_gemm = _import_deep_gemm()
shared_size = intermediate_size * num_shared_experts
pairs = []
for linear, shape in (
(shared_experts.gate_up_proj, (2 * shared_size, hidden_size)),
(shared_experts.down_proj, (hidden_size, shared_size)),
):
weight = linear.weight
scale = getattr(linear, "weight_scale_inv", None)
if scale is None:
scale = getattr(linear, "weight_scale", None)
if (
scale is None
or weight.dtype != torch.float8_e4m3fn
or scale.dtype != torch.float32
or tuple(weight.shape) != shape
):
logger.warning(
"Disabling native MegaMoE shared-expert fusion for %s: "
"expected replicated block-FP8 weights of shape %s with "
"float32 block scales.",
prefix,
shape,
)
return None
# Requantized copies: the shared MLP keeps its own storage, which
# the generic post-load hooks may still rewrite.
w, sf = self._to_kernel_format(
deep_gemm, weight.data.unsqueeze(0), scale.data.unsqueeze(0), 1
)
pairs.append((w.squeeze(0), sf.squeeze(0)))
return deep_gemm.transform_weights_for_mega_moe(*pairs)


def get_deep_gemm_mega_moe_backend(
device: torch.device,
hidden_size: int,
intermediate_size: int,
mma_type: str = "fp8xfp4",
) -> DeepGemmMegaMoEBackend:
"""Return the DeepGEMM MegaMoE backend for ``device``.

``mma_type`` selects by expert weight format: ``fp8xfp4`` for MXFP4
experts, ``fp8xfp8`` for block-FP8 experts.

A successful return implies the backend can run on this device with the
given ``hidden_size`` / ``intermediate_size``. All runtime capability
checks (arch + shape divisibility) live here; there is no separate check
Expand All @@ -463,4 +625,8 @@ def get_deep_gemm_mega_moe_backend(
"DeepGEMM MegaMoE requires hidden and intermediate sizes "
"to be multiples of 128."
)
if mma_type == "fp8xfp8":
return DeepGemmSm100Fp8MegaMoEBackend()
if mma_type != "fp8xfp4":
raise ValueError(f"Unsupported DeepGEMM MegaMoE mma_type: {mma_type!r}")
return DeepGemmSm100MegaMoEBackend()
Loading
Loading