From 4f7837204745533f828421af612e3f90dbf21400 Mon Sep 17 00:00:00 2001 From: yueming-yuan Date: Thu, 8 Oct 2026 17:42:31 -0700 Subject: [PATCH 01/11] Budget the EAGLE draft of models that name their MTP depth mtp_num_hidden_layers The target's KV pool sizing adds the EAGLE draft's layers to the per-token cost, taking the draft's depth from num_nextn_predict_layers. Qwen3.5 names it mtp_num_hidden_layers, so with the target's own MTP layer as draft the depth read as none, and the draft's pool, sized to the target's token count, was allocated outside mem_fraction_static: about a tenth more KV than budgeted on Qwen3.5-35B-A3B, and an out-of-memory at startup on a 4-layer cut of it, whose one full-attention layer makes the draft's pool as large as the target's (127.6 GB with 82 GB left at 0.7 on GB300). --- .../spec_aux_hidden_state.py | 5 +++ .../test_spec_aux_hidden_state.py | 31 +++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/python/sglang/srt/model_executor/model_runner_components/spec_aux_hidden_state.py b/python/sglang/srt/model_executor/model_runner_components/spec_aux_hidden_state.py index 950761105fae..a5d38150828e 100644 --- a/python/sglang/srt/model_executor/model_runner_components/spec_aux_hidden_state.py +++ b/python/sglang/srt/model_executor/model_runner_components/spec_aux_hidden_state.py @@ -94,6 +94,11 @@ def _resolve_eagle_aux_hidden_state( is_draft_model=True, ) num_nextn_predict_layers = draft_model_config.num_nextn_predict_layers + if num_nextn_predict_layers is None: + # Qwen3.5 names its MTP depth mtp_num_hidden_layers; models without MTP have neither + num_nextn_predict_layers = getattr( + draft_model_config.hf_text_config, "mtp_num_hidden_layers", None + ) if num_nextn_predict_layers is not None: config.eagle_draft_num_layers = int(num_nextn_predict_layers) elif get_spec().speculative_draft_model_path: diff --git a/test/registered/unit/model_executor/model_runner_components/test_spec_aux_hidden_state.py b/test/registered/unit/model_executor/model_runner_components/test_spec_aux_hidden_state.py index a48604188f99..07c6872cd6a3 100644 --- a/test/registered/unit/model_executor/model_runner_components/test_spec_aux_hidden_state.py +++ b/test/registered/unit/model_executor/model_runner_components/test_spec_aux_hidden_state.py @@ -3,9 +3,13 @@ import pytest +from sglang.srt.model_executor.model_runner_components import spec_aux_hidden_state from sglang.srt.model_executor.model_runner_components.spec_aux_hidden_state import ( + SpecAuxHiddenStateConfig, _map_muse_target_layer_ids, + _resolve_eagle_aux_hidden_state, ) +from sglang.srt.speculative.spec_info import SpeculativeAlgorithm from sglang.test.ci.ci_register import register_cpu_ci register_cpu_ci(est_time=11, suite="base-a-test-cpu") @@ -34,5 +38,32 @@ def test_muse_target_layer_id_mapping(target_model_type, draft_architecture, exp ) +def test_an_mtp_depth_named_mtp_num_hidden_layers_counts_as_the_eagle_draft(monkeypatch): + """Qwen3.5 names its MTP depth mtp_num_hidden_layers. Read as no draft, the target's + KV budget leaves the draft's pool, of the target's token count, outside + mem_fraction_static.""" + monkeypatch.setattr( + spec_aux_hidden_state, + "get_spec", + lambda: SimpleNamespace(speculative_draft_model_path=None), + ) + config = SpecAuxHiddenStateConfig() + + _resolve_eagle_aux_hidden_state( + config=config, + server_args=None, + model_config=SimpleNamespace( + num_nextn_predict_layers=None, + hf_text_config=SimpleNamespace(mtp_num_hidden_layers=1), + is_hybrid_swa=False, + is_deepseek_v4_arch=False, + ), + spec_algorithm=SpeculativeAlgorithm.EAGLE, + is_draft_worker=False, + ) + + assert config.eagle_draft_num_layers == 1 + + if __name__ == "__main__": sys.exit(pytest.main([__file__, "-v"])) From 56574c65795744b2ad3876ac433e8db552e5f135 Mon Sep 17 00:00:00 2001 From: yueming-yuan Date: Thu, 8 Oct 2026 17:43:04 -0700 Subject: [PATCH 02/11] Unpack UE8M0 scales of weights whose rows are not a multiple of 128 The packed UE8M0 layout holds one scale row per weight row, so a weight of 2624 rows (GLM-5's fused q_a + kv_a) ends in a partial block. The inverse viewed the rows as whole blocks of 128 and raised `shape '[20, 128, 48]' is invalid for input of size 125952`, which made the weight checker fail on every GLM-5 fp8 update on Blackwell. It now takes the first row of each block and checks every row of a block against it, the partial one included. --- .../sglang/srt/layers/quantization/fp8_utils.py | 16 ++++++++-------- .../unit/layers/quantization/test_fp8_utils.py | 17 +++++++++++------ 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/python/sglang/srt/layers/quantization/fp8_utils.py b/python/sglang/srt/layers/quantization/fp8_utils.py index fd86e97283fd..d1adb084511a 100755 --- a/python/sglang/srt/layers/quantization/fp8_utils.py +++ b/python/sglang/srt/layers/quantization/fp8_utils.py @@ -1936,24 +1936,24 @@ def _inverse_transform_scale_ue8m0_impl(sf_packed): assert len(sf_packed.shape) == 2, f"{sf_packed.shape=}" assert sf_packed.dtype == torch.int32 - mn_repeat_128, k_div_4 = sf_packed.shape - mn = mn_repeat_128 // block_size + # one row per weight row, so the last block is partial when the rows are not a multiple of 128 + num_rows, k_div_4 = sf_packed.shape + mn = -(-num_rows // block_size) k = k_div_4 * 4 # packed u8 -> fp32 - sf_u8 = sf_packed.contiguous().flatten().view(torch.uint8).view(mn_repeat_128, k) + sf_u8 = sf_packed.contiguous().flatten().view(torch.uint8).view(num_rows, k) sf_fp32 = (sf_u8.to(torch.int32) << 23).view(torch.float32) # remove repeat - sf_reshaped = sf_fp32.view(mn, block_size, k) - sf_unrepeated = sf_reshaped[:, 0:1, :] - if not torch.all(sf_unrepeated == sf_reshaped): + sf_unrepeated = sf_fp32[::block_size].contiguous() + sf_repeated = sf_unrepeated.repeat_interleave(block_size, dim=0)[:num_rows] + if not torch.all(sf_repeated == sf_fp32): from sglang.srt.debug_utils.dumper import get_tensor_info raise AssertionError( - f"sf_unrepeated != sf_reshaped ({get_tensor_info(sf_unrepeated)=} {get_tensor_info(sf_reshaped)=})" + f"sf_repeated != sf_fp32 ({get_tensor_info(sf_repeated)=} {get_tensor_info(sf_fp32)=})" ) - sf_unrepeated = sf_unrepeated.squeeze(1).contiguous() assert sf_unrepeated.shape == (mn, k) return sf_unrepeated diff --git a/test/registered/unit/layers/quantization/test_fp8_utils.py b/test/registered/unit/layers/quantization/test_fp8_utils.py index d0ed39ca3a1c..10626babd195 100644 --- a/test/registered/unit/layers/quantization/test_fp8_utils.py +++ b/test/registered/unit/layers/quantization/test_fp8_utils.py @@ -82,13 +82,18 @@ def is_deep_gemm(self): class TestInverseTransformScaleUe8m0(CustomTestCase): def test_round_trip(self): + for shape in ( + # DeepSeek V3 kv_b_proj + (32768, 512), + # GLM-5 fused q_a + kv_a: 2048 + 576 rows, the last block partial + (2624, 6144), + ): + with self.subTest(shape=shape): + self._check_round_trip(shape) + + def _check_round_trip(self, shape): for _ in range(100): - weight_bf16 = torch.randn( - # DeepSeek V3 kv_b_proj - (32768, 512), - dtype=torch.bfloat16, - device="cuda", - ) + weight_bf16 = torch.randn(shape, dtype=torch.bfloat16, device="cuda") weight_block_size = [128, 128] From 5fd76bbb5e89ca5448c602b1d9227c2023975af5 Mon Sep 17 00:00:00 2001 From: yueming-yuan Date: Thu, 8 Oct 2026 17:44:00 -0700 Subject: [PATCH 03/11] Dequantize mxfp8 indexer shards into the fused wk_weights_proj with their UE8M0 scales On CUDA the DSA indexer loads wk and weights_proj into one bf16 param, dequantizing quantized shards with block_quant_dequant, which multiplies by the scale. mxfp8 checkpoints store 1x32 UE8M0 scales as uint8 exponents, so an mxfp8 GLM-5.x checkpoint with a quantized wk (as miles' converter makes) started with indexer keys scaled by the raw integers: off by up to 5e4 against the bf16 the trainer's first update writes. Shards with uint8 scales now go through dequant_mxfp8_2d_to_bf16. --- .../deepseek_common/deepseek_weight_loader.py | 24 ++++++--- .../test_deepseek_fused_indexer_mxfp8.py | 53 +++++++++++++++++++ 2 files changed, 71 insertions(+), 6 deletions(-) create mode 100644 test/registered/unit/models/test_deepseek_fused_indexer_mxfp8.py diff --git a/python/sglang/srt/models/deepseek_common/deepseek_weight_loader.py b/python/sglang/srt/models/deepseek_common/deepseek_weight_loader.py index 774f325b7c75..58ec36d9a619 100644 --- a/python/sglang/srt/models/deepseek_common/deepseek_weight_loader.py +++ b/python/sglang/srt/models/deepseek_common/deepseek_weight_loader.py @@ -36,6 +36,7 @@ normalize_e4m3fn_to_e4m3fnuz, quant_weight_ue8m0, ) +from sglang.srt.layers.quantization.mxfp8_block_convert import dequant_mxfp8_2d_to_bf16 from sglang.srt.layers.quantization.int8_utils import ( block_dequant as int8_block_dequant, ) @@ -86,6 +87,19 @@ def _get_indexer_weight_block_size( return [128, 128] +def _dequant_fused_indexer_shard( + weight: torch.Tensor, + scale: torch.Tensor, + quant_config: Optional[QuantizationConfig], +) -> torch.Tensor: + # mxfp8 scales are UE8M0 exponents stored as uint8, not fp32 multipliers + if scale.dtype == torch.uint8: + return dequant_mxfp8_2d_to_bf16(weight, scale) + return block_quant_dequant( + weight, scale, _get_indexer_weight_block_size(quant_config), torch.bfloat16 + ) + + def _load_fused_indexer_wk( name: str, loaded_weight: torch.Tensor, @@ -119,9 +133,8 @@ def _load_fused_indexer_wk( ) if "weight" in entry and "scale" in entry: pending.pop(fused_name + ".weights_proj") - block_size = _get_indexer_weight_block_size(quant_config) - weights_bf16 = block_quant_dequant( - entry["weight"], entry["scale"], block_size, torch.bfloat16 + weights_bf16 = _dequant_fused_indexer_shard( + entry["weight"], entry["scale"], quant_config ) fused_param.data[-weights_bf16.shape[0] :].copy_(weights_bf16) return True @@ -139,9 +152,8 @@ def _load_fused_indexer_wk( ) if "weight" in entry and "scale" in entry: pending.pop(fused_name) - block_size = _get_indexer_weight_block_size(quant_config) - wk_bf16 = block_quant_dequant( - entry["weight"], entry["scale"], block_size, torch.bfloat16 + wk_bf16 = _dequant_fused_indexer_shard( + entry["weight"], entry["scale"], quant_config ) fused_param.data[: wk_bf16.shape[0]].copy_(wk_bf16) return True diff --git a/test/registered/unit/models/test_deepseek_fused_indexer_mxfp8.py b/test/registered/unit/models/test_deepseek_fused_indexer_mxfp8.py new file mode 100644 index 000000000000..860314fd1f57 --- /dev/null +++ b/test/registered/unit/models/test_deepseek_fused_indexer_mxfp8.py @@ -0,0 +1,53 @@ +"""An mxfp8 indexer wk / weights_proj must reach the fused bf16 wk_weights_proj dequantized with its UE8M0 scales.""" + +import unittest +from types import SimpleNamespace + +import torch + +from sglang.srt.layers.quantization.mxfp8_block_convert import ( + dequant_mxfp8_2d_to_bf16, +) +from sglang.srt.models.deepseek_common.deepseek_weight_loader import ( + _load_fused_indexer_wk, +) +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=1, suite="base-a-test-cpu") + +HEAD_DIM, N_HEADS, HIDDEN = 128, 32, 256 +PREFIX = "model.layers.0.self_attn.indexer" + + +class TestFusedIndexerMxfp8(CustomTestCase): + def test_mxfp8_shards_are_dequantized_with_their_ue8m0_scales(self): + """The scales are biased exponents: used as multipliers, the indexer keys + are off by orders of magnitude until a bf16 update overwrites them.""" + torch.manual_seed(0) + fused = torch.zeros(HEAD_DIM + N_HEADS, HIDDEN, dtype=torch.bfloat16) + params_dict = {f"{PREFIX}.wk_weights_proj.weight": fused} + quant_config = SimpleNamespace(weight_block_size=[1, 32]) + pending = {} + expected = {} + for shard, rows in (("wk", HEAD_DIM), ("weights_proj", N_HEADS)): + weight = torch.randn(rows, HIDDEN).to(torch.float8_e4m3fn) + scale = torch.randint(120, 130, (rows, HIDDEN // 32), dtype=torch.uint8) + expected[shard] = dequant_mxfp8_2d_to_bf16(weight, scale) + for name, tensor in (("weight", weight), ("weight_scale_inv", scale)): + self.assertTrue( + _load_fused_indexer_wk( + f"{PREFIX}.{shard}.{name}", + tensor, + params_dict, + pending, + quant_config, + ) + ) + + self.assertTrue(torch.equal(fused[:HEAD_DIM], expected["wk"])) + self.assertTrue(torch.equal(fused[-N_HEADS:], expected["weights_proj"])) + + +if __name__ == "__main__": + unittest.main() From 90352280bbca0bc17e731517b51008ed8fb40951 Mon Sep 17 00:00:00 2001 From: yueming-yuan Date: Thu, 8 Oct 2026 17:45:15 -0700 Subject: [PATCH 04/11] Leave the NVFP4 MoE activation input scales out of the weight check w13_input_scale and w2_input_scale are calibrated activation scales, like the KV cache scales: no weight update carries them (miles' NVFP4 quantizer emits weight, weight_scale and weight_scale_2 only). The checker reset them to random before an update and compared them after it, so every NVFP4 MoE update failed the check on them alone, whatever it wrote. They are now marked _skip_weight_check, as the KV cache scales are. --- .../srt/layers/quantization/modelopt_quant.py | 3 + .../test_modelopt_nvfp4_moe_weight_check.py | 81 +++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 test/registered/unit/layers/quantization/test_modelopt_nvfp4_moe_weight_check.py diff --git a/python/sglang/srt/layers/quantization/modelopt_quant.py b/python/sglang/srt/layers/quantization/modelopt_quant.py index 806c52bc25b8..7daad0580d31 100755 --- a/python/sglang/srt/layers/quantization/modelopt_quant.py +++ b/python/sglang/srt/layers/quantization/modelopt_quant.py @@ -2574,6 +2574,9 @@ def create_weights( ) w2_input_scale._sglang_require_global_experts = True layer.register_parameter("w2_input_scale", w2_input_scale) + # calibrated activation scales, as the KV cache scales: no weight update carries them + w13_input_scale._skip_weight_check = True + w2_input_scale._skip_weight_check = True def _build_mega_moe_weights(self, layer: torch.nn.Module) -> None: # Activations are quantized per token at dispatch, so w13_input_scale diff --git a/test/registered/unit/layers/quantization/test_modelopt_nvfp4_moe_weight_check.py b/test/registered/unit/layers/quantization/test_modelopt_nvfp4_moe_weight_check.py new file mode 100644 index 000000000000..df3d941fb93b --- /dev/null +++ b/test/registered/unit/layers/quantization/test_modelopt_nvfp4_moe_weight_check.py @@ -0,0 +1,81 @@ +"""The weight checker must leave the NVFP4 MoE activation input scales alone: no weight update carries them, so +a reset before an update leaves them random and the comparison after it fails whatever the update wrote. + +The platform check is stubbed so the layer is built on CPU, as in test_modelopt_nvfp4_moe_dispatch.py. +""" + +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=6, suite="base-a-test-cpu") + +import unittest +from types import SimpleNamespace +from unittest import mock + +import torch + +# Import modelopt_quant first (see test_modelopt_nvfp4_moe_scales.py for the circular-import reason). +# isort: off +from sglang.srt.layers.quantization import modelopt_quant +from sglang.srt.layers.quantization.modelopt_quant import ModelOptNvFp4FusedMoEMethod + +# isort: on +from sglang.srt.layers.moe.utils import MoeRunnerBackend +from sglang.srt.utils.weight_checker import _is_skip_weight_check +from sglang.test.test_utils import CustomTestCase + +NUM_EXPERTS = 8 +HIDDEN = 64 +INTERMEDIATE = 32 + + +def _nvfp4_moe_layer() -> torch.nn.Module: + with ( + mock.patch.object( + modelopt_quant, + "get_moe_runner_backend", + return_value=MoeRunnerBackend.FLASHINFER_TRTLLM_ROUTED, + ), + mock.patch.object( + modelopt_quant, + "get_platform", + return_value=SimpleNamespace(is_blackwell=True), + ), + mock.patch.object(modelopt_quant, "is_cuda", return_value=False), + ): + method = ModelOptNvFp4FusedMoEMethod( + SimpleNamespace( + group_size=16, + is_checkpoint_nvfp4_serialized=True, + use_per_token_activation=False, + get_name=lambda: "modelopt_fp4", + ) + ) + layer = torch.nn.Module() + layer.num_experts = NUM_EXPERTS + layer.num_local_experts = NUM_EXPERTS + layer.moe_ep_rank = 0 + layer.moe_runner_config = SimpleNamespace(is_gated=True) + method.create_weights( + layer, + num_experts=NUM_EXPERTS, + hidden_size=HIDDEN, + intermediate_size_per_partition=INTERMEDIATE, + params_dtype=torch.bfloat16, + weight_loader=lambda *args, **kwargs: None, + ) + return layer + + +class TestNvFp4MoeWeightCheck(CustomTestCase): + def test_the_checker_skips_the_input_scales_and_checks_the_weight_scales(self): + params = dict(_nvfp4_moe_layer().named_parameters()) + + for name in ("w13_input_scale", "w2_input_scale"): + self.assertTrue(_is_skip_weight_check(name, params[name]), name) + for name in ("w13_weight", "w13_weight_scale", "w13_weight_scale_2"): + self.assertFalse(_is_skip_weight_check(name, params[name]), name) + + +if __name__ == "__main__": + unittest.main() From 0b1623ac507839640e8757449bafab190dec35d3 Mon Sep 17 00:00:00 2001 From: yueming-yuan Date: Thu, 8 Oct 2026 17:48:25 -0700 Subject: [PATCH 05/11] Format --- .../srt/models/deepseek_common/deepseek_weight_loader.py | 2 +- .../model_runner_components/test_spec_aux_hidden_state.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/python/sglang/srt/models/deepseek_common/deepseek_weight_loader.py b/python/sglang/srt/models/deepseek_common/deepseek_weight_loader.py index 58ec36d9a619..3b4607b8bfd0 100644 --- a/python/sglang/srt/models/deepseek_common/deepseek_weight_loader.py +++ b/python/sglang/srt/models/deepseek_common/deepseek_weight_loader.py @@ -36,10 +36,10 @@ normalize_e4m3fn_to_e4m3fnuz, quant_weight_ue8m0, ) -from sglang.srt.layers.quantization.mxfp8_block_convert import dequant_mxfp8_2d_to_bf16 from sglang.srt.layers.quantization.int8_utils import ( block_dequant as int8_block_dequant, ) +from sglang.srt.layers.quantization.mxfp8_block_convert import dequant_mxfp8_2d_to_bf16 from sglang.srt.layers.utils import get_layer_id from sglang.srt.model_loader.utils import ( maybe_executor_submit, diff --git a/test/registered/unit/model_executor/model_runner_components/test_spec_aux_hidden_state.py b/test/registered/unit/model_executor/model_runner_components/test_spec_aux_hidden_state.py index 07c6872cd6a3..0b28912dd11e 100644 --- a/test/registered/unit/model_executor/model_runner_components/test_spec_aux_hidden_state.py +++ b/test/registered/unit/model_executor/model_runner_components/test_spec_aux_hidden_state.py @@ -38,7 +38,9 @@ def test_muse_target_layer_id_mapping(target_model_type, draft_architecture, exp ) -def test_an_mtp_depth_named_mtp_num_hidden_layers_counts_as_the_eagle_draft(monkeypatch): +def test_an_mtp_depth_named_mtp_num_hidden_layers_counts_as_the_eagle_draft( + monkeypatch, +): """Qwen3.5 names its MTP depth mtp_num_hidden_layers. Read as no draft, the target's KV budget leaves the draft's pool, of the target's token count, outside mem_fraction_static.""" From 05486babc4dca71ee71b62b50ac8a184d7d6aa0e Mon Sep 17 00:00:00 2001 From: yueming-yuan Date: Mon, 12 Oct 2026 12:39:17 -0700 Subject: [PATCH 06/11] Move the GemmaRMSNorm and GLM-lite NextN post-load fixes to #43251 Both rebuild model-specific derived state after a write that bypasses the model's own loader. sgl-project/sglang#43251 carries them with the other model- and format-specific fixes p2p needs, so this PR keeps the general engine support. --- python/sglang/srt/layers/layernorm.py | 5 +-- python/sglang/srt/model_loader/loader.py | 5 --- .../sglang/srt/models/glm4_moe_lite_nextn.py | 4 -- .../model_loader/test_default_model_loader.py | 19 +-------- .../test_glm4_moe_lite_nextn_post_load.py | 39 ------------------- 5 files changed, 2 insertions(+), 70 deletions(-) delete mode 100644 test/registered/unit/models/test_glm4_moe_lite_nextn_post_load.py diff --git a/python/sglang/srt/layers/layernorm.py b/python/sglang/srt/layers/layernorm.py index d5954344ea61..2d3367d1d4ff 100644 --- a/python/sglang/srt/layers/layernorm.py +++ b/python/sglang/srt/layers/layernorm.py @@ -1134,11 +1134,8 @@ def __init__( def _weight_loader(self, param: torch.Tensor, loaded_weight: torch.Tensor) -> None: assert param.size() == loaded_weight.size() param.data.copy_(loaded_weight) - self.refresh_gemma_weight() - - def refresh_gemma_weight(self) -> None: # Keep storage stable for CUDA graphs or fused paths that capture this buffer. - torch.add(self.weight.data, 1.0, out=self.gemma_weight) + torch.add(param.data, 1.0, out=self.gemma_weight) def _forward_impl( self, diff --git a/python/sglang/srt/model_loader/loader.py b/python/sglang/srt/model_loader/loader.py index 4be00fa62fe0..cd13ea193a84 100644 --- a/python/sglang/srt/model_loader/loader.py +++ b/python/sglang/srt/model_loader/loader.py @@ -85,7 +85,6 @@ from sglang.srt.distributed import ( model_parallel_is_initialized, ) -from sglang.srt.layers.layernorm import GemmaRMSNorm from sglang.srt.layers.modelopt_utils import QUANT_CFG_CHOICES from sglang.srt.layers.moe.utils import ( install_shared_experts_fusion_decision, @@ -326,10 +325,6 @@ def post_load_weights(model: nn.Module) -> None: # `is_nextn=True`, so the loader doesn't need to know. if hasattr(model, "post_load_weights"): model.post_load_weights() - # its weight loader derives the buffer, and these loaders bypass it - for module in model.modules(): - if isinstance(module, GemmaRMSNorm): - module.refresh_gemma_weight() def _modules_with_quant_method(model: nn.Module): diff --git a/python/sglang/srt/models/glm4_moe_lite_nextn.py b/python/sglang/srt/models/glm4_moe_lite_nextn.py index 86f6b9f50668..25e0b7c93a86 100644 --- a/python/sglang/srt/models/glm4_moe_lite_nextn.py +++ b/python/sglang/srt/models/glm4_moe_lite_nextn.py @@ -179,9 +179,5 @@ def forward( def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): super().load_weights(weights, is_nextn=True) - def post_load_weights(self, is_nextn=True, weight_names=None): - # loaders and weight-update sessions that bypass load_weights call it with no arguments - super().post_load_weights(is_nextn=True, weight_names=weight_names) - EntryClass = [Glm4MoeLiteForCausalLMNextN] diff --git a/test/registered/unit/model_loader/test_default_model_loader.py b/test/registered/unit/model_loader/test_default_model_loader.py index 5cfb2eeda675..2ed200d3a9e1 100644 --- a/test/registered/unit/model_loader/test_default_model_loader.py +++ b/test/registered/unit/model_loader/test_default_model_loader.py @@ -6,10 +6,9 @@ import torch import sglang.srt.model_loader.loader as loader_mod -from sglang.srt.layers.layernorm import GemmaRMSNorm from sglang.srt.layers.moe import MoeRunnerBackend from sglang.srt.layers.quantization.fp8 import Fp8Config -from sglang.srt.model_loader.loader import DefaultModelLoader, post_load_weights +from sglang.srt.model_loader.loader import DefaultModelLoader from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -229,21 +228,5 @@ def test_parameters_keep_their_subclass_attributes_and_ties(self): self.assertIs(model.lm_head.weight, model.embed.weight) -class TestPostLoadWeights(CustomTestCase): - def test_gemma_weight_follows_a_weight_written_without_its_loader(self): - """p2p weight updates and the remote-instance loader write params by address; fused allreduce kernels read - `gemma_weight`, so a stale one would run the old norm.""" - model = torch.nn.Module() - model.norm = GemmaRMSNorm(4) - gemma_weight_address = model.norm.gemma_weight.data_ptr() - new_weight = torch.tensor([0.5, -0.25, 0.0, 2.0]) - - model.norm.weight.data.copy_(new_weight) - post_load_weights(model) - - torch.testing.assert_close(model.norm.gemma_weight, new_weight + 1) - self.assertEqual(model.norm.gemma_weight.data_ptr(), gemma_weight_address) - - if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/models/test_glm4_moe_lite_nextn_post_load.py b/test/registered/unit/models/test_glm4_moe_lite_nextn_post_load.py deleted file mode 100644 index a48382fe3965..000000000000 --- a/test/registered/unit/models/test_glm4_moe_lite_nextn_post_load.py +++ /dev/null @@ -1,39 +0,0 @@ -"""The GLM-lite NextN draft must take the NextN branch of post_load_weights when called without arguments.""" - -import unittest -from unittest.mock import patch - -import torch - -from sglang.srt.model_loader.loader import post_load_weights -from sglang.srt.models.glm4_moe_lite import Glm4MoeLiteForCausalLM -from sglang.srt.models.glm4_moe_lite_nextn import Glm4MoeLiteForCausalLMNextN -from sglang.test.ci.ci_register import register_cpu_ci -from sglang.test.test_utils import CustomTestCase - -register_cpu_ci(est_time=5, suite="base-a-test-cpu") - - -class TestGlm4MoeLiteNextNPostLoad(CustomTestCase): - def test_a_bare_post_load_runs_the_nextn_branch(self): - """A weight-update session that writes weights without load_weights (p2p) finishes with a bare - post_load_weights; on the draft the main model's branch walks a layer range it does not have.""" - nextn_flags = [] - draft = Glm4MoeLiteForCausalLMNextN.__new__(Glm4MoeLiteForCausalLMNextN) - # an empty module, without the layers its __init__ would build - torch.nn.Module.__init__(draft) - - with patch.object( - Glm4MoeLiteForCausalLM, - "post_load_weights", - lambda self, is_nextn=False, weight_names=None: nextn_flags.append( - is_nextn - ), - ): - post_load_weights(draft) - - self.assertEqual(nextn_flags, [True]) - - -if __name__ == "__main__": - unittest.main() From 51dffcbdb682b0d3b45c7eb58e05361fe7ee9464 Mon Sep 17 00:00:00 2001 From: yueming-yuan Date: Thu, 8 Oct 2026 00:26:39 -0700 Subject: [PATCH 07/11] Run the NextN branch of post_load_weights on the GLM-lite NextN draft loader.post_load_weights calls model.post_load_weights() with no arguments and relies on NextN subclasses to pin is_nextn=True, as DeepseekV3ForCausalLMNextN does. Glm4MoeLiteForCausalLMNextN did not, so the main model's branch ran on the draft and raised AttributeError: 'Glm4MoeLiteModelNextN' object has no attribute 'start_layer'. It hits every weight-update session that writes without load_weights and covers the draft (p2p on GLM-4.7-Flash with EAGLE). --- .../sglang/srt/models/glm4_moe_lite_nextn.py | 4 +++ .../test_glm4_moe_lite_nextn_post_load.py | 35 +++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 test/registered/unit/models/test_glm4_moe_lite_nextn_post_load.py diff --git a/python/sglang/srt/models/glm4_moe_lite_nextn.py b/python/sglang/srt/models/glm4_moe_lite_nextn.py index 25e0b7c93a86..86f6b9f50668 100644 --- a/python/sglang/srt/models/glm4_moe_lite_nextn.py +++ b/python/sglang/srt/models/glm4_moe_lite_nextn.py @@ -179,5 +179,9 @@ def forward( def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]): super().load_weights(weights, is_nextn=True) + def post_load_weights(self, is_nextn=True, weight_names=None): + # loaders and weight-update sessions that bypass load_weights call it with no arguments + super().post_load_weights(is_nextn=True, weight_names=weight_names) + EntryClass = [Glm4MoeLiteForCausalLMNextN] diff --git a/test/registered/unit/models/test_glm4_moe_lite_nextn_post_load.py b/test/registered/unit/models/test_glm4_moe_lite_nextn_post_load.py new file mode 100644 index 000000000000..282f19040dae --- /dev/null +++ b/test/registered/unit/models/test_glm4_moe_lite_nextn_post_load.py @@ -0,0 +1,35 @@ +"""The GLM-lite NextN draft must take the NextN branch of post_load_weights when called without arguments.""" + +import unittest +from unittest.mock import patch + +from sglang.srt.model_loader.loader import post_load_weights +from sglang.srt.models.glm4_moe_lite import Glm4MoeLiteForCausalLM +from sglang.srt.models.glm4_moe_lite_nextn import Glm4MoeLiteForCausalLMNextN +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +class TestGlm4MoeLiteNextNPostLoad(CustomTestCase): + def test_a_bare_post_load_runs_the_nextn_branch(self): + """A weight-update session that writes weights without load_weights (p2p) finishes with a bare + post_load_weights; on the draft the main model's branch walks a layer range it does not have.""" + nextn_flags = [] + draft = Glm4MoeLiteForCausalLMNextN.__new__(Glm4MoeLiteForCausalLMNextN) + + with patch.object( + Glm4MoeLiteForCausalLM, + "post_load_weights", + lambda self, is_nextn=False, weight_names=None: nextn_flags.append( + is_nextn + ), + ): + post_load_weights(draft) + + self.assertEqual(nextn_flags, [True]) + + +if __name__ == "__main__": + unittest.main() From e93841802fdca5577173d1d9d6aefd87306bb927 Mon Sep 17 00:00:00 2001 From: yueming-yuan Date: Thu, 8 Oct 2026 11:14:43 -0700 Subject: [PATCH 08/11] Refresh GemmaRMSNorm's gemma_weight in post_load_weights GemmaRMSNorm's weight loader derives gemma_weight = weight + 1, which fused allreduce kernels read. Loaders and weight-update sessions that write params without load_weights (p2p, remote instance, sharded state, dummy) left it at its old value. --- python/sglang/srt/layers/layernorm.py | 5 ++- python/sglang/srt/model_loader/loader.py | 5 +++ .../model_loader/test_post_load_weights.py | 32 +++++++++++++++++++ 3 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 test/registered/unit/model_loader/test_post_load_weights.py diff --git a/python/sglang/srt/layers/layernorm.py b/python/sglang/srt/layers/layernorm.py index 2d3367d1d4ff..d5954344ea61 100644 --- a/python/sglang/srt/layers/layernorm.py +++ b/python/sglang/srt/layers/layernorm.py @@ -1134,8 +1134,11 @@ def __init__( def _weight_loader(self, param: torch.Tensor, loaded_weight: torch.Tensor) -> None: assert param.size() == loaded_weight.size() param.data.copy_(loaded_weight) + self.refresh_gemma_weight() + + def refresh_gemma_weight(self) -> None: # Keep storage stable for CUDA graphs or fused paths that capture this buffer. - torch.add(param.data, 1.0, out=self.gemma_weight) + torch.add(self.weight.data, 1.0, out=self.gemma_weight) def _forward_impl( self, diff --git a/python/sglang/srt/model_loader/loader.py b/python/sglang/srt/model_loader/loader.py index cd13ea193a84..4be00fa62fe0 100644 --- a/python/sglang/srt/model_loader/loader.py +++ b/python/sglang/srt/model_loader/loader.py @@ -85,6 +85,7 @@ from sglang.srt.distributed import ( model_parallel_is_initialized, ) +from sglang.srt.layers.layernorm import GemmaRMSNorm from sglang.srt.layers.modelopt_utils import QUANT_CFG_CHOICES from sglang.srt.layers.moe.utils import ( install_shared_experts_fusion_decision, @@ -325,6 +326,10 @@ def post_load_weights(model: nn.Module) -> None: # `is_nextn=True`, so the loader doesn't need to know. if hasattr(model, "post_load_weights"): model.post_load_weights() + # its weight loader derives the buffer, and these loaders bypass it + for module in model.modules(): + if isinstance(module, GemmaRMSNorm): + module.refresh_gemma_weight() def _modules_with_quant_method(model: nn.Module): diff --git a/test/registered/unit/model_loader/test_post_load_weights.py b/test/registered/unit/model_loader/test_post_load_weights.py new file mode 100644 index 000000000000..7ef80ad467b4 --- /dev/null +++ b/test/registered/unit/model_loader/test_post_load_weights.py @@ -0,0 +1,32 @@ +"""Loaders that write params without `load_weights` must leave the buffers weight loaders derive from them current.""" + +import unittest + +import torch + +from sglang.srt.layers.layernorm import GemmaRMSNorm +from sglang.srt.model_loader.loader import post_load_weights +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +class TestPostLoadWeights(CustomTestCase): + def test_gemma_weight_follows_a_weight_written_without_its_loader(self): + """p2p weight updates and the remote-instance loader write params by address; fused allreduce kernels read + `gemma_weight`, so a stale one would run the old norm.""" + model = torch.nn.Module() + model.norm = GemmaRMSNorm(4) + gemma_weight_address = model.norm.gemma_weight.data_ptr() + new_weight = torch.tensor([0.5, -0.25, 0.0, 2.0]) + + model.norm.weight.data.copy_(new_weight) + post_load_weights(model) + + torch.testing.assert_close(model.norm.gemma_weight, new_weight + 1) + self.assertEqual(model.norm.gemma_weight.data_ptr(), gemma_weight_address) + + +if __name__ == "__main__": + unittest.main() From 2fe26d14cf3473dd0c89595a002255cbd0c3c511 Mon Sep 17 00:00:00 2001 From: yueming-yuan Date: Thu, 8 Oct 2026 11:58:49 -0700 Subject: [PATCH 09/11] Initialize the module in the GLM-lite NextN post-load test post_load_weights now walks the model's modules to refresh gemma_weight, which an instance made with __new__ alone does not have. --- .../unit/models/test_glm4_moe_lite_nextn_post_load.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/registered/unit/models/test_glm4_moe_lite_nextn_post_load.py b/test/registered/unit/models/test_glm4_moe_lite_nextn_post_load.py index 282f19040dae..a48382fe3965 100644 --- a/test/registered/unit/models/test_glm4_moe_lite_nextn_post_load.py +++ b/test/registered/unit/models/test_glm4_moe_lite_nextn_post_load.py @@ -3,6 +3,8 @@ import unittest from unittest.mock import patch +import torch + from sglang.srt.model_loader.loader import post_load_weights from sglang.srt.models.glm4_moe_lite import Glm4MoeLiteForCausalLM from sglang.srt.models.glm4_moe_lite_nextn import Glm4MoeLiteForCausalLMNextN @@ -18,6 +20,8 @@ def test_a_bare_post_load_runs_the_nextn_branch(self): post_load_weights; on the draft the main model's branch walks a layer range it does not have.""" nextn_flags = [] draft = Glm4MoeLiteForCausalLMNextN.__new__(Glm4MoeLiteForCausalLMNextN) + # an empty module, without the layers its __init__ would build + torch.nn.Module.__init__(draft) with patch.object( Glm4MoeLiteForCausalLM, From 8eeedbbdb71488330bf4498fa190f1f2e914f064 Mon Sep 17 00:00:00 2001 From: yueming-yuan Date: Thu, 8 Oct 2026 12:48:15 -0700 Subject: [PATCH 10/11] Move the gemma_weight test into the loader test file The post_load_weights util lives in loader.py, whose tests already have this file and its fixtures; a separate file only adds an interpreter import to the CPU gate. --- .../model_loader/test_default_model_loader.py | 19 ++++++++++- .../model_loader/test_post_load_weights.py | 32 ------------------- 2 files changed, 18 insertions(+), 33 deletions(-) delete mode 100644 test/registered/unit/model_loader/test_post_load_weights.py diff --git a/test/registered/unit/model_loader/test_default_model_loader.py b/test/registered/unit/model_loader/test_default_model_loader.py index 2ed200d3a9e1..5cfb2eeda675 100644 --- a/test/registered/unit/model_loader/test_default_model_loader.py +++ b/test/registered/unit/model_loader/test_default_model_loader.py @@ -6,9 +6,10 @@ import torch import sglang.srt.model_loader.loader as loader_mod +from sglang.srt.layers.layernorm import GemmaRMSNorm from sglang.srt.layers.moe import MoeRunnerBackend from sglang.srt.layers.quantization.fp8 import Fp8Config -from sglang.srt.model_loader.loader import DefaultModelLoader +from sglang.srt.model_loader.loader import DefaultModelLoader, post_load_weights from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -228,5 +229,21 @@ def test_parameters_keep_their_subclass_attributes_and_ties(self): self.assertIs(model.lm_head.weight, model.embed.weight) +class TestPostLoadWeights(CustomTestCase): + def test_gemma_weight_follows_a_weight_written_without_its_loader(self): + """p2p weight updates and the remote-instance loader write params by address; fused allreduce kernels read + `gemma_weight`, so a stale one would run the old norm.""" + model = torch.nn.Module() + model.norm = GemmaRMSNorm(4) + gemma_weight_address = model.norm.gemma_weight.data_ptr() + new_weight = torch.tensor([0.5, -0.25, 0.0, 2.0]) + + model.norm.weight.data.copy_(new_weight) + post_load_weights(model) + + torch.testing.assert_close(model.norm.gemma_weight, new_weight + 1) + self.assertEqual(model.norm.gemma_weight.data_ptr(), gemma_weight_address) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/model_loader/test_post_load_weights.py b/test/registered/unit/model_loader/test_post_load_weights.py deleted file mode 100644 index 7ef80ad467b4..000000000000 --- a/test/registered/unit/model_loader/test_post_load_weights.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Loaders that write params without `load_weights` must leave the buffers weight loaders derive from them current.""" - -import unittest - -import torch - -from sglang.srt.layers.layernorm import GemmaRMSNorm -from sglang.srt.model_loader.loader import post_load_weights -from sglang.test.ci.ci_register import register_cpu_ci -from sglang.test.test_utils import CustomTestCase - -register_cpu_ci(est_time=5, suite="base-a-test-cpu") - - -class TestPostLoadWeights(CustomTestCase): - def test_gemma_weight_follows_a_weight_written_without_its_loader(self): - """p2p weight updates and the remote-instance loader write params by address; fused allreduce kernels read - `gemma_weight`, so a stale one would run the old norm.""" - model = torch.nn.Module() - model.norm = GemmaRMSNorm(4) - gemma_weight_address = model.norm.gemma_weight.data_ptr() - new_weight = torch.tensor([0.5, -0.25, 0.0, 2.0]) - - model.norm.weight.data.copy_(new_weight) - post_load_weights(model) - - torch.testing.assert_close(model.norm.gemma_weight, new_weight + 1) - self.assertEqual(model.norm.gemma_weight.data_ptr(), gemma_weight_address) - - -if __name__ == "__main__": - unittest.main() From db730e297ef4296d85bad93efb40b960d5db4187 Mon Sep 17 00:00:00 2001 From: yueming-yuan Date: Mon, 12 Oct 2026 12:40:26 -0700 Subject: [PATCH 11/11] Refresh gemma_weight at the end of every weight-update session post_load_weights refreshed GemmaRMSNorm's derived gemma_weight only in sessions where no load ran. A `direct` or custom load writes params without the norm's own loader yet marks the session loaded, so those sessions kept the old gemma_weight. --- .../model_runner_components/weight_updater.py | 3 +++ python/sglang/srt/model_loader/loader.py | 6 +++++- .../test_weight_updater_published_storage.py | 21 ++++++++++++++++++- 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/python/sglang/srt/model_executor/model_runner_components/weight_updater.py b/python/sglang/srt/model_executor/model_runner_components/weight_updater.py index 267f2c7e665c..072c97212c68 100644 --- a/python/sglang/srt/model_executor/model_runner_components/weight_updater.py +++ b/python/sglang/srt/model_executor/model_runner_components/weight_updater.py @@ -12,6 +12,7 @@ DefaultModelLoader, get_model_loader, post_load_weights, + refresh_gemma_weights, ) from sglang.srt.model_loader.utils import set_default_torch_dtype from sglang.srt.model_loader.weight_utils import default_weight_loader @@ -306,6 +307,8 @@ def begin_weight_update(self: WeightUpdater) -> None: def end_weight_update(self: WeightUpdater, *, run_post_load: bool) -> None: if run_post_load: post_load_weights(self.get_model()) + else: + refresh_gemma_weights(self.get_model()) DefaultModelLoader.postprocess_weights( self.get_model(), torch.device(self.device) ) diff --git a/python/sglang/srt/model_loader/loader.py b/python/sglang/srt/model_loader/loader.py index 4be00fa62fe0..ae8266aede23 100644 --- a/python/sglang/srt/model_loader/loader.py +++ b/python/sglang/srt/model_loader/loader.py @@ -326,7 +326,11 @@ def post_load_weights(model: nn.Module) -> None: # `is_nextn=True`, so the loader doesn't need to know. if hasattr(model, "post_load_weights"): model.post_load_weights() - # its weight loader derives the buffer, and these loaders bypass it + refresh_gemma_weights(model) + + +def refresh_gemma_weights(model: nn.Module) -> None: + # its weight loader derives the buffer, so writes that bypass the loader leave it stale for module in model.modules(): if isinstance(module, GemmaRMSNorm): module.refresh_gemma_weight() diff --git a/test/registered/unit/model_executor/model_runner_components/test_weight_updater_published_storage.py b/test/registered/unit/model_executor/model_runner_components/test_weight_updater_published_storage.py index b2de83768ef8..4cb81a6fb3b4 100644 --- a/test/registered/unit/model_executor/model_runner_components/test_weight_updater_published_storage.py +++ b/test/registered/unit/model_executor/model_runner_components/test_weight_updater_published_storage.py @@ -1,4 +1,5 @@ -"""A weight-update session must leave every parameter that peers write by address where it was published.""" +"""A weight-update session must leave every parameter that peers write by address where it was published, and +rebuild what the loaders derive from the weights it wrote.""" import unittest from contextlib import contextmanager @@ -8,6 +9,7 @@ import torch import sglang.srt.model_executor.model_runner_components.weight_updater as weight_updater_mod +from sglang.srt.layers.layernorm import GemmaRMSNorm from sglang.srt.model_executor.model_runner_components.weight_updater import ( WeightUpdater, ) @@ -96,5 +98,22 @@ def test_a_rank_that_published_nothing_is_not_checked(self): weight_updater.end_weight_update(run_post_load=False) +class TestDerivedState(CustomTestCase): + def test_gemma_weight_follows_a_norm_weight_loaded_without_its_loader(self): + """`direct` and custom loads write params without GemmaRMSNorm's loader yet mark the session loaded, so + fused allreduce kernels would read the old `gemma_weight`.""" + model = torch.nn.Module() + model.norm = GemmaRMSNorm(4) + weight_updater = _weight_updater(model, published_weight_info=None) + new_weight = torch.tensor([0.5, -0.25, 0.0, 2.0]) + + with _postprocess(lambda model, device: None): + weight_updater.begin_weight_update() + model.norm.weight.data.copy_(new_weight) + weight_updater.end_weight_update(run_post_load=False) + + torch.testing.assert_close(model.norm.gemma_weight, new_weight + 1) + + if __name__ == "__main__": unittest.main()