diff --git a/tests/models/glm5next/test_mega_moe.py b/tests/models/glm5next/test_mega_moe.py new file mode 100644 index 000000000000..506f467275d7 --- /dev/null +++ b/tests/models/glm5next/test_mega_moe.py @@ -0,0 +1,277 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""GLM-5.3-Flash MegaMoE: block-FP8 checkpoint shards requantized to the +MXFP8 MegaMoE kernel layout.""" + +from types import SimpleNamespace + +import pytest +import torch + +from vllm.model_executor.layers.fused_moe.deep_gemm_mega_moe import MegaMoEActivation +from vllm.model_executor.layers.quantization.utils.mxfp8_utils import ( + requant_block_fp8_to_ue8m0, +) +from vllm.model_executor.model_loader.reload import ( + finish_reload, + record_metadata_for_reloading, + start_reload, +) +from vllm.model_executor.model_loader.utils import process_weights_after_loading +from vllm.models.glm5next.nvidia.mega_moe import Glm5NextMegaMoEExperts + +BLOCK = (128, 128) +HIDDEN, INTER = 256, 128 +FP8 = torch.float8_e4m3fn +NUM_EXPERTS = 4 +# Global ids of the experts _experts() places on this rank. +LOCAL_EXPERTS = slice(2, 4) + + +@pytest.fixture(autouse=True) +def sm100_mega_moe_layout(monkeypatch): + """Build the SM100 kernel layout on any device.""" + monkeypatch.setattr( + "vllm.model_executor.layers.fused_moe.deep_gemm_mega_moe." + "check_deep_gemm_mega_moe_supported", + lambda *args: None, + ) + + +def _block_fp8(real: torch.Tensor, power_of_two: bool = False): + """Quantize ``(..., mn, k)`` to E4M3 with one float32 scale per 128x128.""" + *lead, mn, k = real.shape + blocks = real.reshape(*lead, mn // 128, 128, k // 128, 128) + scale = blocks.abs().amax(dim=(-3, -1)) / 448.0 + if power_of_two: + scale = torch.exp2(torch.ceil(torch.log2(scale))) + full = scale.repeat_interleave(128, -2).repeat_interleave(128, -1) + return (real / full).to(FP8), scale + + +def _dequant_block(weight: torch.Tensor, scale: torch.Tensor) -> torch.Tensor: + return weight.float() * scale.repeat_interleave(128, -2).repeat_interleave(128, -1) + + +def _dequant_ue8m0(weight: torch.Tensor, sf: torch.Tensor) -> torch.Tensor: + return weight.float() * torch.exp2(sf.float() - 127).repeat_interleave(32, -1) + + +def test_requant_is_exact_for_power_of_two_block_scales(): + torch.manual_seed(1) + real = torch.randn(1, 256, 256) * 0.05 + weight, scale = _block_fp8(real, power_of_two=True) + + requant, sf = requant_block_fp8_to_ue8m0(weight, scale) + + # A finer power-of-two scale only shifts exponents, so nothing rounds. + assert torch.equal(_dequant_ue8m0(requant, sf), _dequant_block(weight, scale)) + + +def _experts() -> Glm5NextMegaMoEExperts: + vllm_config = SimpleNamespace( + scheduler_config=SimpleNamespace(max_num_batched_tokens=64), + compilation_config=SimpleNamespace(static_forward_context={}), + ) + return Glm5NextMegaMoEExperts( + vllm_config, # type: ignore[arg-type] + num_experts=NUM_EXPERTS, + num_local_experts=2, + experts_start_idx=2, + top_k=2, + hidden_size=HIDDEN, + intermediate_size=INTER, + weight_block_size=BLOCK, + activation=MegaMoEActivation("swiglu", clamp=10.0), + num_shared_experts=1, + prefix="model.layers.3.mlp.experts", + ) + + +def _checkpoint(seed: int) -> dict[str, torch.Tensor]: + """Block-FP8 experts as stored: gate rows, then up rows, for w13.""" + generator = torch.Generator().manual_seed(seed) + + def rand(*shape): + return torch.randn(*shape, generator=generator) * 0.02 + + w13, s13 = _block_fp8(rand(NUM_EXPERTS, 2 * INTER, HIDDEN)) + w2, s2 = _block_fp8(rand(NUM_EXPERTS, HIDDEN, INTER)) + return { + "w13_weight": w13, + "w13_weight_scale_inv": s13, + "w2_weight": w2, + "w2_weight_scale_inv": s2, + } + + +def _load_shuffled(experts, ckpt, seed, suffix="") -> None: + """Load the checkpoint tensors whose names end in ``suffix``, shuffled.""" + shards = [] + for expert_id in range(NUM_EXPERTS): + for name, tensor in ckpt.items(): + if not name.endswith(suffix): + continue + if name.startswith("w2_"): + shards.append((name, tensor[expert_id], "w2", expert_id)) + else: + half = tensor.shape[1] // 2 + shards.append((name, tensor[expert_id, :half], "w1", expert_id)) + shards.append((name, tensor[expert_id, half:], "w3", expert_id)) + generator = torch.Generator().manual_seed(seed) + for i in torch.randperm(len(shards), generator=generator).tolist(): + name, loaded_weight, shard_id, expert_id = shards[i] + param = getattr(experts, name) + loaded = param.weight_loader( + param, + loaded_weight, + f"experts.{name}", + shard_id=shard_id, + expert_id=expert_id, + return_success=True, + ) + # Physical experts 2 and 3 are this rank's. + assert loaded == (expert_id >= 2) + + +def _reference_mega_moe_layout(w13, s13, w2, s2): + """CPU copy of DeepGEMM's transform_weights_for_mega_moe on packed scales.""" + + def interleave(t): + g, n, k = t.shape + halves = t.view(g, 2, n // 16, 8, k) + return torch.stack([halves[:, 0], halves[:, 1]], dim=2).reshape(g, n, k) + + def utccp(sf): + g, mn, k = sf.shape + return sf.reshape(g, -1, 4, 32, k).transpose(2, 3).reshape(g, mn, k) + + return ( + interleave(w13.view(torch.uint8)), + utccp(interleave(s13.view(torch.int32))), + w2.view(torch.uint8), + utccp(s2.view(torch.int32)), + ) + + +def _expected_routed(ckpt): + w13, s13 = requant_block_fp8_to_ue8m0( + ckpt["w13_weight"][LOCAL_EXPERTS], ckpt["w13_weight_scale_inv"][LOCAL_EXPERTS] + ) + w2, s2 = requant_block_fp8_to_ue8m0( + ckpt["w2_weight"][LOCAL_EXPERTS], ckpt["w2_weight_scale_inv"][LOCAL_EXPERTS] + ) + return _reference_mega_moe_layout(w13, s13, w2, s2) + + +def _assert_layout(actual, expected): + for tensor, reference in zip(actual, expected): + assert torch.equal(tensor.view(reference.dtype), reference) + + +def _routed(experts): + return (*experts.l1_weights, *experts.l2_weights) + + +@pytest.mark.parametrize("first", ["weight", "weight_scale_inv"]) +def test_weight_update_split_across_calls(first): + """A weight update may send weights and block scales in separate + load_weights calls. The finish step rejects a shard still missing its + other half and requantizes each shard once both are in.""" + experts = _experts() + _load_shuffled(experts, _checkpoint(seed=0), seed=0) + experts.requantize_block_fp8() + + ckpt = _checkpoint(seed=1) + second = "weight_scale_inv" if first == "weight" else "weight" + _load_shuffled(experts, ckpt, seed=1, suffix=first) + with pytest.raises(ValueError, match="without their weight or block scale"): + experts.requantize_block_fp8() + _load_shuffled(experts, ckpt, seed=2, suffix=second) + experts.requantize_block_fp8() + + _assert_layout(_routed(experts), _expected_routed(ckpt)) + + +def _shared_checkpoint(seed: int) -> dict[str, torch.Tensor]: + generator = torch.Generator().manual_seed(seed) + w13, s13 = _block_fp8(torch.randn(1, 2 * INTER, HIDDEN, generator=generator) * 0.02) + w2, s2 = _block_fp8(torch.randn(1, HIDDEN, INTER, generator=generator) * 0.02) + return {"w13": w13, "s13": s13, "w2": w2, "s2": s2} + + +def _load_shared(experts, ckpt) -> None: + w13, s13, w2, s2 = ckpt["w13"], ckpt["s13"], ckpt["w2"], ckpt["s2"] + params = {f"mlp.experts.{n}" for n, _ in experts.named_parameters()} + for name, data in [ + ("down_proj.weight_scale_inv", s2[0]), + ("gate_proj.weight", w13[0, :INTER]), + ("up_proj.weight_scale_inv", s13[0, 1:]), + ("down_proj.weight", w2[0]), + ("up_proj.weight", w13[0, INTER:]), + ("gate_proj.weight_scale_inv", s13[0, :1]), + ]: + loaded = experts.load_shared_expert_weight(f"mlp.shared_experts.{name}", data) + assert loaded in params + + +def _expected_shared(ckpt): + q13, sf13 = requant_block_fp8_to_ue8m0(ckpt["w13"], ckpt["s13"]) + q2, sf2 = requant_block_fp8_to_ue8m0(ckpt["w2"], ckpt["s2"]) + return [t[0] for t in _reference_mega_moe_layout(q13, sf13, q2, sf2)] + + +def _shared(experts): + return ( + experts.shared_l1_weight, + experts.shared_l1_weight_scale, + experts.shared_l2_weight, + experts.shared_l2_weight_scale, + ) + + +class _Model(torch.nn.Module): + """Finishes a load the way the GLM model, wrapper and MTP drafter do.""" + + def __init__(self): + super().__init__() + self.experts = _experts() + + def process_weights_after_loading(self): + self.experts.requantize_block_fp8() + + +@pytest.mark.parametrize("mode", ["layerwise", "direct"]) +def test_reload_requantizes_into_the_kernel_tensors(monkeypatch, mode): + """A reload writes the block-FP8 shards and finishes with the model's + process_weights_after_loading; the kernel tensors keep their storage.""" + monkeypatch.setenv("VLLM_RELOAD_MODE", mode) + model_config = SimpleNamespace( + word_embeddings_untied_by_checkpoint=False, quantization=None + ) + device = torch.get_default_device() + model = _Model() + experts = model.experts + record_metadata_for_reloading(model) + ckpt, shared_ckpt = _checkpoint(seed=0), _shared_checkpoint(seed=0) + _load_shuffled(experts, ckpt, seed=0) + _load_shared(experts, shared_ckpt) + process_weights_after_loading(model, model_config, device) # type: ignore[arg-type] + # Already requantized shards are left alone by a second call. + experts.requantize_block_fp8() + _assert_layout(_routed(experts), _expected_routed(ckpt)) + _assert_layout(_shared(experts), _expected_shared(shared_ckpt)) + storage = {n: p.data_ptr() for n, p in experts.named_parameters()} + + ckpt, shared_ckpt = _checkpoint(seed=1), _shared_checkpoint(seed=1) + start_reload(model) + _load_shuffled(experts, ckpt, seed=1) + _load_shared(experts, shared_ckpt) + # Every loaded tensor is in, so the layer is not meta any more: the derived + # kernel scales do not count as missing loads. + assert not experts.w13_weight.is_meta + finish_reload(model, model_config, device) # type: ignore[arg-type] + + _assert_layout(_routed(experts), _expected_routed(ckpt)) + _assert_layout(_shared(experts), _expected_shared(shared_ckpt)) + assert {n: p.data_ptr() for n, p in experts.named_parameters()} == storage diff --git a/vllm/model_executor/layers/quantization/utils/mxfp8_utils.py b/vllm/model_executor/layers/quantization/utils/mxfp8_utils.py index 9352a155a153..e49332a6ffaa 100644 --- a/vllm/model_executor/layers/quantization/utils/mxfp8_utils.py +++ b/vllm/model_executor/layers/quantization/utils/mxfp8_utils.py @@ -87,6 +87,32 @@ def _mxfp8_e4m3_quantize_torch( return x_fp8, scales_uint8 +def requant_block_fp8_to_ue8m0( + weight: torch.Tensor, scale: torch.Tensor +) -> tuple[torch.Tensor, torch.Tensor]: + """Requantize block-FP8 weights to E4M3 with one UE8M0 scale per 1x32. + + The SM100 MegaMoE feeds its scales to the block-scaled MMA, so they have to + be powers of two, one per row and 32 columns. A checkpoint with float32 + scales per block (e.g. 128x128) is rounded once more to get there. + + Args: + weight: ``(n, mn, k)`` E4M3 values. + scale: ``(n, mn / block_m, k / block_k)`` float32 dequantization + scales (``real = weight * scale``). + + Returns: + ``(weight, sf)``: the requantized E4M3 weights and their UE8M0 scale + bytes, ``(n, mn, k / 32)`` uint8. + + """ + n, mn, k = weight.shape + _, blocks_m, blocks_k = scale.shape + real = weight.to(torch.float32).view(n, blocks_m, -1, blocks_k, k // blocks_k) + real.mul_(scale[:, :, None, :, None]) + return _mxfp8_e4m3_quantize_torch(real.view(n, mn, k)) + + def _mxfp8_quant_triton_kernel(): """Lazily-built Triton kernel: per-32-block E8M0 scale + FP8-E4M3 quant. diff --git a/vllm/models/glm5next/common/model.py b/vllm/models/glm5next/common/model.py index e944ad38f151..e5f05e8af599 100644 --- a/vllm/models/glm5next/common/model.py +++ b/vllm/models/glm5next/common/model.py @@ -9,7 +9,8 @@ from transformers import Glm5NextTextConfig from vllm._aiter_ops import rocm_aiter_ops -from vllm.config import ParallelConfig, VllmConfig +from vllm.config import ParallelConfig, VllmConfig, get_current_vllm_config +from vllm.config.kernel import NATIVE_MEGA_MOE_BACKENDS from vllm.distributed import ( get_ep_group, get_pp_group, @@ -24,6 +25,9 @@ GateLinear, fused_moe_make_expert_params_mapping, ) +from vllm.model_executor.layers.fused_moe.router.fused_topk_bias_router import ( + fused_topk_bias, +) from vllm.model_executor.layers.fused_moe.utils import ( resolve_layer_fused_shared_expert, ) @@ -277,12 +281,21 @@ def __init__( ) swiglu_limit = config.swiglu_limit + vllm_config = get_current_vllm_config() + self.use_mega_moe = ( + vllm_config.kernel_config.moe_backend in NATIVE_MEGA_MOE_BACKENDS + ) self.is_fused_shared_expert_enabled = False - if config.n_shared_experts is not None: + if config.n_shared_experts is not None and not self.use_mega_moe: self.is_fused_shared_expert_enabled = resolve_layer_fused_shared_expert( quant_config, prefix ) and _fused_shared_experts_tuned(parallel_config) - if config.n_shared_experts is None or self.is_fused_shared_expert_enabled: + # MegaMoE owns the shared-expert weights itself. + if ( + config.n_shared_experts is None + or self.is_fused_shared_expert_enabled + or self.use_mega_moe + ): self.shared_experts = None else: intermediate_size = config.moe_intermediate_size * config.n_shared_experts @@ -298,6 +311,10 @@ def __init__( swiglu_limit=swiglu_limit, ) + if self.use_mega_moe: + self._init_mega_moe_experts(vllm_config, config, quant_config, prefix) + return + self.experts = FusedMoEFactory( shared_experts=self.shared_experts, gate=self.gate, @@ -327,11 +344,83 @@ def __init__( swiglu_limit=swiglu_limit, ) + def _init_mega_moe_experts( + self, + vllm_config: VllmConfig, + config: Glm5NextTextConfig, + quant_config: QuantizationConfig | None, + prefix: str, + ) -> None: + """DeepGEMM FP8xFP8 MegaMoE: the kernel dispatches, runs the routed and + shared experts and combines across the EP group.""" + from vllm.model_executor.layers.fused_moe.deep_gemm_mega_moe import ( + MegaMoEActivation, + ) + from vllm.models.glm5next.nvidia.mega_moe import Glm5NextMegaMoEExperts + + if not vllm_config.parallel_config.enable_expert_parallel: + raise NotImplementedError( + "GLM-5.3 MegaMoE requires expert parallel. Enable it with " + "--enable-expert-parallel, or pick a different moe backend." + ) + if self.tp_size != 1: + raise NotImplementedError( + "GLM-5.3 MegaMoE requires tensor_parallel_size=1 (DP + EP)." + ) + if (config.n_group or 1) > 1 or (config.topk_group or 1) > 1: + raise NotImplementedError("GLM-5.3 MegaMoE requires one expert group.") + weight_block_size = getattr(quant_config, "weight_block_size", None) + if weight_block_size is None: + raise NotImplementedError( + "GLM-5.3 MegaMoE requires a block-FP8 checkpoint " + "(quantization_config.weight_block_size)." + ) + if self.n_physical_experts % self.ep_size != 0: + raise ValueError( + f"n_physical_experts={self.n_physical_experts} must be divisible " + f"by ep_size={self.ep_size}." + ) + self.mega_renormalize = config.norm_topk_prob + self.mega_scoring_func = config.scoring_func + self.experts = Glm5NextMegaMoEExperts( + vllm_config, + num_experts=self.n_physical_experts, + num_local_experts=self.n_local_physical_experts, + experts_start_idx=self.physical_expert_start, + num_logical_experts=self.n_logical_experts, + top_k=config.num_experts_per_tok, + hidden_size=config.hidden_size, + intermediate_size=config.moe_intermediate_size, + weight_block_size=tuple(weight_block_size), + num_shared_experts=config.n_shared_experts or 0, + activation=MegaMoEActivation("swiglu", clamp=config.swiglu_limit), + prefix=f"{prefix}.experts", + ) + + def finalize_mega_moe_weights(self) -> None: + if self.use_mega_moe: + self.experts.requantize_block_fp8() + def forward( self, hidden_states: torch.Tensor, already_sequence_parallel: bool = False, ) -> torch.Tensor: + if self.use_mega_moe: + router_logits, _ = self.gate(hidden_states) + bias = self.gate.e_score_correction_bias + topk_weights, topk_ids = fused_topk_bias( + hidden_states=hidden_states, + gating_output=router_logits, + scoring_func=self.mega_scoring_func, + e_score_correction_bias=bias.data if bias is not None else None, + topk=self.experts.top_k, + renormalize=self.mega_renormalize, + indices_type=torch.int64, + routed_scaling_factor=self.routed_scaling_factor, + ) + return self.experts(hidden_states, topk_weights, topk_ids) + num_tokens, hidden_dim = hidden_states.shape # Chunk the hidden states so they aren't replicated across TP ranks. @@ -707,6 +796,9 @@ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): config = vllm_config.model_config.hf_text_config _validate_supported_config(config) self.config = config + self.use_mega_moe = ( + vllm_config.kernel_config.moe_backend in NATIVE_MEGA_MOE_BACKENDS + ) self.vocab_size = config.vocab_size self.device = current_platform.device_type @@ -909,6 +1001,7 @@ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: # NOTE: global despite per-layer FSE; non-fused layers skip the extras. num_experts=self.config.n_routed_experts + num_fused_shared, num_redundant_experts=num_redundant_experts, + routed_experts_prefix="" if self.use_mega_moe else "routed_experts", ) else: expert_params_mapping = [] @@ -936,6 +1029,12 @@ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: # Models trained using ColossalAI may include these tensors in # the checkpoint. Skip them. continue + if self.use_mega_moe and ".shared_experts." in name: + mlp = self.get_submodule(name.partition(".shared_experts.")[0]) + loaded_params.add( + mlp.experts.load_shared_expert_weight(name, loaded_weight) + ) + continue # Only parse the layer index when FSE is enabled. if ( "mlp.shared_experts." in name @@ -1151,6 +1250,11 @@ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: loader = AutoWeightsLoader(self) return loader.load_weights(weights) + def process_weights_after_loading(self) -> None: + for module in self.modules(): + if isinstance(module, Glm5NextMoE): + module.finalize_mega_moe_weights() + @MULTIMODAL_REGISTRY.register_processor( Glm5NextMultiModalProcessor, @@ -1251,6 +1355,9 @@ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): # so pipeline parallelism is gated off (consistent with the text-only # model) and we intentionally do not alias it here. + def process_weights_after_loading(self) -> None: + self.language_model.process_weights_after_loading() + def set_moe_parameters(self) -> None: self.moe_mlp_layers = [ layer.mlp diff --git a/vllm/models/glm5next/common/mtp.py b/vllm/models/glm5next/common/mtp.py index cc738fdf4d00..ff039eac47ce 100644 --- a/vllm/models/glm5next/common/mtp.py +++ b/vllm/models/glm5next/common/mtp.py @@ -8,6 +8,7 @@ from vllm._aiter_ops import rocm_aiter_ops from vllm.config import VllmConfig +from vllm.config.kernel import NATIVE_MEGA_MOE_BACKENDS from vllm.model_executor.layers.fused_moe import ( fused_moe_make_expert_params_mapping, ) @@ -211,6 +212,9 @@ def get_top_tokens( class Glm5NextMTP(nn.Module, DeepseekV2MixtureOfExperts): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() + self.use_mega_moe = ( + vllm_config.kernel_config.moe_backend in NATIVE_MEGA_MOE_BACKENDS + ) self.config = vllm_config.model_config.hf_text_config self.quant_config = vllm_config.quant_config self.model = Glm5NextMultiTokenPredictor( @@ -316,6 +320,7 @@ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: # NOTE: global despite per-layer FSE; non-fused layers skip the extras. num_experts=self.config.n_routed_experts + num_fused_shared, num_redundant_experts=self.num_redundant_experts, + routed_experts_prefix="" if self.use_mega_moe else "routed_experts", ) params_dict = dict(self.named_parameters()) @@ -339,6 +344,12 @@ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: if spec_layer is None: continue name = self._rewrite_spec_layer_name(spec_layer, name) + if self.use_mega_moe and ".shared_experts." in name: + mlp = self.get_submodule(name.partition(".shared_experts.")[0]) + loaded_params.add( + mlp.experts.load_shared_expert_weight(name, loaded_weight) + ) + continue # Only parse the layer index when FSE is enabled. if ( "mlp.shared_experts." in name @@ -447,3 +458,7 @@ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: f"missing from checkpoint." ) return loaded_params + + def process_weights_after_loading(self) -> None: + for mlp in self.moe_mlp_layers: + mlp.finalize_mega_moe_weights() diff --git a/vllm/models/glm5next/nvidia/mega_moe.py b/vllm/models/glm5next/nvidia/mega_moe.py new file mode 100644 index 000000000000..58f95af993a3 --- /dev/null +++ b/vllm/models/glm5next/nvidia/mega_moe.py @@ -0,0 +1,228 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""GLM-5.3-Flash adapter for the DeepGEMM MegaMoE kernel.""" + +import torch +from torch import nn + +from vllm.config import VllmConfig +from vllm.logger import init_logger +from vllm.model_executor.layers.fused_moe.deep_gemm_mega_moe import ( + MEGA_MOE_MXFP8, + DeepGemmMegaMoEExperts, + MegaMoEWeightSpec, +) +from vllm.model_executor.layers.quantization.utils.mxfp8_utils import ( + requant_block_fp8_to_ue8m0, +) + +logger = init_logger(__name__) + +# (local expert id, shard id); the fused shared expert has expert id None. +Shard = tuple[int | None, str] + + +class Glm5NextMegaMoEExperts(DeepGemmMegaMoEExperts): + """MXFP8 MegaMoE experts loaded from GLM-5.3-Flash's block-FP8 checkpoint. + + The checkpoint stores E4M3 weights with one float32 scale per + ``weight_block_size`` block; the kernel takes power-of-two UE8M0 scales per + 1x32. Loading writes each weight shard straight into the kernel layout and + stages its block scale in ``*_weight_scale_inv``. Once a load is complete, + the model's ``process_weights_after_loading`` calls + :meth:`requantize_block_fp8`, which rounds every newly loaded shard to the + kernel scales in place. The shared expert is fused the same way. + """ + + def __init__( + self, + vllm_config: VllmConfig, + *, + weight_block_size: tuple[int, int], + num_shared_experts: int = 0, + **kwargs, + ): + self.weight_block_size = weight_block_size + self._num_fused_shared_experts = num_shared_experts + # Shards whose weight / block scale arrived since the last requantize. + self._new_weights: set[Shard] = set() + self._new_scales: set[Shard] = set() + super().__init__(vllm_config, weight_format=MEGA_MOE_MXFP8, **kwargs) + + def _fusable_shared_experts( + self, shared_experts: nn.Module | None + ) -> tuple[int, MegaMoEWeightSpec | None]: + n = self._num_fused_shared_experts + return n, self.weight_format.routed if n else None + + def _create_weights(self) -> None: + super()._create_weights() + e, h, i = self.num_local_experts, self.hidden_size, self.intermediate_size + self._register_block_scale("w13", (e, 2 * i, h)) + self._register_block_scale("w2", (e, h, i)) + if self.has_fused_shared_experts: + s = i * self.num_shared_experts + self._register_block_scale("shared_l1", (2 * s, h)) + self._register_block_scale("shared_l2", (h, s)) + + def _register_block_scale(self, name: str, shape: tuple[int, ...]) -> None: + """Stage the checkpoint's float32 block scales of ``name``. The kernel's + ``{name}_weight_scale`` is derived from them and never loaded.""" + *outer, n, k = shape + block_m, block_k = self.weight_block_size + scale = torch.zeros(*outer, n // block_m, k // block_k, dtype=torch.float32) + self.register_parameter( + f"{name}_weight_scale_inv", nn.Parameter(scale, requires_grad=False) + ) + getattr(self, f"{name}_weight_scale").weight_loader_numel = 0 + + @staticmethod + def _shard_rows(t: torch.Tensor, shard_id: str) -> torch.Tensor: + """Rows of ``t`` (gate rows, then up rows, for w13) that hold ``shard_id``.""" + if shard_id == "w2": + return t + half = t.shape[0] // 2 + return t[:half] if shard_id == "w1" else t[half:] + + @staticmethod + def _shared_shard_id(name: str) -> str: + return "w1" if ".gate_proj." in name else "w3" if ".up_proj." in name else "w2" + + def weight_loader( + self, + param: nn.Parameter, + loaded_weight: torch.Tensor, + weight_name: str, + shard_id: str, + expert_id: int, + return_success: bool = False, + ) -> bool | None: + local_expert_ids = self._map_global_expert_id(expert_id) + if weight_name.endswith("weight_scale_inv"): + param_prefix = "w2_" if shard_id == "w2" else "w13_" + loaded = bool(local_expert_ids) and param_prefix in weight_name + for local_expert_id in local_expert_ids if loaded else (): + self._shard_rows(param.data[local_expert_id], shard_id).copy_( + loaded_weight + ) + new = self._new_scales + else: + loaded = bool( + super().weight_loader( + param, + loaded_weight, + weight_name, + shard_id, + expert_id, + return_success=True, + ) + ) + new = self._new_weights + if loaded: + new.update( + (local_expert_id, shard_id) for local_expert_id in local_expert_ids + ) + return loaded if return_success else None + + def load_shared_expert_weight(self, name: str, loaded_weight: torch.Tensor) -> str: + """Load one shared-expert checkpoint tensor; return the parameter name.""" + layer = "shared_l2" if self._shared_shard_id(name) == "w2" else "shared_l1" + suffix = "weight_scale_inv" if "weight_scale" in name else "weight" + param = getattr(self, f"{layer}_{suffix}") + param.weight_loader(param, loaded_weight, name) + return f"{name.partition('shared_experts.')[0]}experts.{layer}_{suffix}" + + def shared_weight_loader( + self, param: nn.Parameter, loaded_weight: torch.Tensor, weight_name: str + ) -> None: + shard_id = self._shared_shard_id(weight_name) + if "weight_scale" in weight_name: + self._shard_rows(param.data, shard_id).copy_(loaded_weight) + self._new_scales.add((None, shard_id)) + else: + self._load_weight(param.data, loaded_weight, shard_id, False) + self._new_weights.add((None, shard_id)) + + @torch.no_grad() + def requantize_block_fp8(self, chunk: int = 8) -> None: + """Round every shard loaded since the last call to UE8M0 1x32 scales, + in place. + + Runs once a load is complete. A weight update may send a weight and its + block scale in separate ``load_weights`` calls, but both have to be in + by then. + """ + unpaired = self._new_weights ^ self._new_scales + if unpaired: + raise ValueError( + f"{self.prefix}: block-FP8 shards (local expert, shard; None is " + "the shared expert) loaded without their weight or block scale: " + f"{sorted(unpaired, key=str)}" + ) + for shard_id in ("w1", "w3", "w2"): + experts = sorted( + e for e, s in self._new_weights if s == shard_id and e is not None + ) + for start in range(0, len(experts), chunk): + self._requantize_shards( + [ + self._shard_tensors(e, shard_id) + for e in experts[start : start + chunk] + ], + shard_id, + ) + if (None, shard_id) in self._new_weights: + self._requantize_shards([self._shard_tensors(None, shard_id)], shard_id) + self._new_weights.clear() + self._new_scales.clear() + + def _shard_tensors( + self, local_expert_id: int | None, shard_id: str + ) -> tuple[torch.Tensor, ...]: + """Kernel weight, kernel scale and staged block scale of one routed + expert, or of the shared expert for ``local_expert_id`` None.""" + if local_expert_id is None: + name = "shared_l2" if shard_id == "w2" else "shared_l1" + else: + name = "w2" if shard_id == "w2" else "w13" + tensors = tuple( + getattr(self, f"{name}_{suffix}").data + for suffix in ("weight", "weight_scale", "weight_scale_inv") + ) + if local_expert_id is None: + return tensors + return tuple(t[local_expert_id] for t in tensors) + + def _requantize_shards( + self, tensors: list[tuple[torch.Tensor, ...]], shard_id: str + ) -> None: + """Requantize ``shard_id`` of each kernel-layout weight in place.""" + # A shard's layout destination is its checkpoint rows; the source is unused. + raw = torch.stack( + [ + self._kernel_layout_shard(w, w, shard_id, False)[0].reshape( + -1, w.shape[-1] + ) + for w, _, _ in tensors + ] + ) + block_scale = torch.stack( + [self._shard_rows(s, shard_id) for _, _, s in tensors] + ) + # Power-of-two block scales requantize exactly; any other is rounded. + if not torch.equal(block_scale, block_scale.log2().round().exp2()): + logger.warning_once( + "GLM-5.3-Flash on the deep_gemm_mega_moe backend: the checkpoint's " + "%dx%d block-FP8 MoE scales are not all powers of two, so " + "requantizing them to MXFP8 (UE8M0 scales per 1x32) changes the " + "expert weights. Use another moe_backend to run the block-FP8 " + "weights as they are.", + *self.weight_block_size, + ) + weight, sf = requant_block_fp8_to_ue8m0(raw, block_scale) + for (dst, dst_scale, _), w, s in zip(tensors, weight, sf): + self._load_weight(dst, w, shard_id, False) + self._load_weight(dst_scale, s, shard_id, True) + + +Glm5NextMegaMoEExperts.weight_loader.supports_moe_loading = True # type: ignore[attr-defined]