From 8c97a625ec16f4d936d7386278d24323bd40e7d1 Mon Sep 17 00:00:00 2001 From: Po-Han Huang Date: Tue, 8 Sep 2026 21:02:01 -0700 Subject: [PATCH 1/2] Fix mixed-precision GLM-5 ModelOpt loading --- python/sglang/srt/models/glm5_next.py | 19 ++++ .../unit/models/test_glm5_next_modelopt.py | 90 +++++++++++++++++++ 2 files changed, 109 insertions(+) create mode 100644 test/registered/unit/models/test_glm5_next_modelopt.py diff --git a/python/sglang/srt/models/glm5_next.py b/python/sglang/srt/models/glm5_next.py index f5253cfdf00d..c6ec6653003e 100644 --- a/python/sglang/srt/models/glm5_next.py +++ b/python/sglang/srt/models/glm5_next.py @@ -94,6 +94,7 @@ GlmOcrVisionPatchEmbed, GlmOcrVisionPatchMerger, ) +from sglang.srt.models.utils import WeightsMapper from sglang.srt.multimodal.mm_utils import ( run_dp_presharded_mrope_vision_model, run_dp_sharded_mrope_vision_model, @@ -1077,6 +1078,13 @@ def forward( class Glm5NextForConditionalGeneration(nn.Module): + hf_to_sglang_mapper = WeightsMapper( + orig_to_new_substr={ + "model.language_model.": "model.", + "model.visual": "visual", + } + ) + packed_modules_mapping = { "fused_qkv_a_proj_with_mqa": ["q_a_proj", "kv_a_proj_with_mqa"], "fused_qkvbfg_a_proj": [ @@ -1200,6 +1208,17 @@ def shared_experts_fusion_disable_reason(cls, hf_config, quant_config): text_config = getattr(hf_config, "text_config", hf_config) if not getattr(text_config, "n_shared_experts", None): return "No shared experts are defined in the config." + if quant_config is not None and quant_config.get_name() == "modelopt_fp4": + first_sparse_layer = getattr(text_config, "first_k_dense_replace", 0) + for layer_id in range(first_sparse_layer, text_config.num_hidden_layers): + moe_prefix = f"model.layers.{layer_id}.mlp" + if quant_config.is_layer_excluded( + f"{moe_prefix}.shared_experts" + ) and not quant_config.is_layer_excluded(f"{moe_prefix}.experts"): + return ( + "ModelOpt FP4 keeps shared experts unquantized while routed " + "experts are quantized." + ) if not _is_cuda: return "Shared experts fusion currently requires CUDA devices." if _device_sm is not None and _device_sm < 80: diff --git a/test/registered/unit/models/test_glm5_next_modelopt.py b/test/registered/unit/models/test_glm5_next_modelopt.py new file mode 100644 index 000000000000..04b3fb7af64e --- /dev/null +++ b/test/registered/unit/models/test_glm5_next_modelopt.py @@ -0,0 +1,90 @@ +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +from sglang.srt.layers.quantization.modelopt_quant import ModelOptFp4Config +from sglang.srt.models import glm5_next +from sglang.srt.models.glm5_next import Glm5NextForConditionalGeneration +from sglang.srt.runtime_context import get_parallel +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +class TestGlm5NextModelOpt(CustomTestCase): + def _config(self, exclude_modules): + config = ModelOptFp4Config( + is_checkpoint_nvfp4_serialized=True, + group_size=16, + exclude_modules=exclude_modules, + ) + config.apply_weight_name_mapper( + Glm5NextForConditionalGeneration.hf_to_sglang_mapper + ) + return config + + def _hf_config(self): + return SimpleNamespace( + text_config=SimpleNamespace( + first_k_dense_replace=3, + num_hidden_layers=45, + n_shared_experts=1, + ) + ) + + def test_checkpoint_exclusions_match_sglang_module_names(self): + config = self._config( + [ + "model.language_model.embed_tokens", + "model.language_model.layers.11.self_attn*", + "model.language_model.layers.11.mlp.shared_experts*", + "model.visual*", + ] + ) + + self.assertTrue(config.is_layer_excluded("model.embed_tokens")) + self.assertTrue( + config.is_layer_excluded("model.layers.11.self_attn.kv_b_proj") + ) + self.assertTrue( + config.is_layer_excluded( + "model.layers.11.mlp.shared_experts.gate_up_proj" + ) + ) + self.assertTrue(config.is_layer_excluded("visual.blocks.0.attn.qkv_proj")) + + def test_mixed_precision_shared_experts_disable_fusion(self): + config = self._config( + ["model.language_model.layers.3.mlp.shared_experts*"] + ) + + reason = Glm5NextForConditionalGeneration.shared_experts_fusion_disable_reason( + self._hf_config(), config + ) + + self.assertIn("shared experts unquantized", reason) + + def test_uniform_modelopt_fp4_does_not_disable_fusion(self): + config = self._config([]) + a2a_backend = SimpleNamespace(is_deepep=lambda: False) + + with ( + patch.object(glm5_next, "_is_cuda", True), + patch.object(glm5_next, "_device_sm", 100), + patch.object( + glm5_next, "get_moe_a2a_backend", return_value=a2a_backend + ), + get_parallel().override(moe_ep_size=1), + ): + reason = ( + Glm5NextForConditionalGeneration.shared_experts_fusion_disable_reason( + self._hf_config(), config + ) + ) + + self.assertIsNone(reason) + + +if __name__ == "__main__": + unittest.main() From d378591c7113671f5744b393f8f2eb1b52023727 Mon Sep 17 00:00:00 2001 From: Po-Han Huang Date: Tue, 8 Sep 2026 22:35:55 -0700 Subject: [PATCH 2/2] Test GLM-5 NVFP4 checkpoint variants --- .../unit/models/test_glm5_next_modelopt.py | 68 +++++++++++-------- 1 file changed, 41 insertions(+), 27 deletions(-) diff --git a/test/registered/unit/models/test_glm5_next_modelopt.py b/test/registered/unit/models/test_glm5_next_modelopt.py index 04b3fb7af64e..459805680d00 100644 --- a/test/registered/unit/models/test_glm5_next_modelopt.py +++ b/test/registered/unit/models/test_glm5_next_modelopt.py @@ -11,6 +11,27 @@ register_cpu_ci(est_time=5, suite="base-a-test-cpu") +NVIDIA_EXCLUDE_MODULES = [ + "model.language_model.embed_tokens", + "model.language_model.layers.11.self_attn*", + "model.language_model.layers.11.mlp.gate", + "model.language_model.layers.11.mlp.shared_experts*", + "model.visual*", +] + +RADIXARK_EXCLUDE_MODULES = [ + "model.language_model.embed_tokens", + "model.language_model.layers.11.self_attn*", + "model.language_model.layers.11.mlp.gate", + "model.visual*", + "model.embed_tokens", + "model.layers.11.self_attn*", + "model.layers.11.mlp.gate", + "model.language_model.layers.45*", + "model.layers.45*", + "visual*", +] + class TestGlm5NextModelOpt(CustomTestCase): def _config(self, exclude_modules): @@ -34,30 +55,25 @@ def _hf_config(self): ) def test_checkpoint_exclusions_match_sglang_module_names(self): - config = self._config( - [ - "model.language_model.embed_tokens", - "model.language_model.layers.11.self_attn*", - "model.language_model.layers.11.mlp.shared_experts*", - "model.visual*", - ] - ) + checkpoints = { + "nvidia/GLM-5.3-Flash-NVFP4": NVIDIA_EXCLUDE_MODULES, + "RadixArk/GLM-5.3-Flash-NVFP4": RADIXARK_EXCLUDE_MODULES, + } - self.assertTrue(config.is_layer_excluded("model.embed_tokens")) - self.assertTrue( - config.is_layer_excluded("model.layers.11.self_attn.kv_b_proj") - ) - self.assertTrue( - config.is_layer_excluded( - "model.layers.11.mlp.shared_experts.gate_up_proj" - ) - ) - self.assertTrue(config.is_layer_excluded("visual.blocks.0.attn.qkv_proj")) + for checkpoint, exclude_modules in checkpoints.items(): + with self.subTest(checkpoint=checkpoint): + config = self._config(exclude_modules) + self.assertTrue(config.is_layer_excluded("model.embed_tokens")) + self.assertTrue( + config.is_layer_excluded("model.layers.11.self_attn.kv_b_proj") + ) + self.assertTrue(config.is_layer_excluded("model.layers.11.mlp.gate")) + self.assertTrue( + config.is_layer_excluded("visual.blocks.0.attn.qkv_proj") + ) - def test_mixed_precision_shared_experts_disable_fusion(self): - config = self._config( - ["model.language_model.layers.3.mlp.shared_experts*"] - ) + def test_nvidia_mixed_precision_shared_experts_disable_fusion(self): + config = self._config(NVIDIA_EXCLUDE_MODULES) reason = Glm5NextForConditionalGeneration.shared_experts_fusion_disable_reason( self._hf_config(), config @@ -65,16 +81,14 @@ def test_mixed_precision_shared_experts_disable_fusion(self): self.assertIn("shared experts unquantized", reason) - def test_uniform_modelopt_fp4_does_not_disable_fusion(self): - config = self._config([]) + def test_radixark_uniform_fp4_shared_experts_keep_fusion(self): + config = self._config(RADIXARK_EXCLUDE_MODULES) a2a_backend = SimpleNamespace(is_deepep=lambda: False) with ( patch.object(glm5_next, "_is_cuda", True), patch.object(glm5_next, "_device_sm", 100), - patch.object( - glm5_next, "get_moe_a2a_backend", return_value=a2a_backend - ), + patch.object(glm5_next, "get_moe_a2a_backend", return_value=a2a_backend), get_parallel().override(moe_ep_size=1), ): reason = (