From 90c2a39bf725602db32650baf31c28069915ca05 Mon Sep 17 00:00:00 2001 From: Ranjan Debnath Date: Mon, 12 Oct 2026 19:06:30 +0000 Subject: [PATCH 1/7] glm4_moe_lite: skip model-side routed scaling on XPU (fix GLM-4.7-Flash accuracy) On XPU every MoE path applies routed_scaling_factor itself: sgl-kernel fused_experts (unquant, fp8, mxfp4) and the triton runner's moe_sum_reduce. Glm4MoeLiteSparseMoeBlock.forward_normal still multiplied on every non-CUDA device, so XPU scaled the routed output by routed_scaling_factor squared (1.8^2 for GLM-4.7-Flash) and the model answered wrongly on any attention backend (gsm8k 0.25 at limit 20). Exclude XPU like CUDA, matching DeepseekV2MoE.forward_normal. Verified on intel_xpu tp4 (Arc Pro B60): gsm8k 5-shot, full 1319, flexible-extract 0.8408, strict-match 0.8415 (H200 84.2). --- python/sglang/srt/models/glm4_moe_lite.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/sglang/srt/models/glm4_moe_lite.py b/python/sglang/srt/models/glm4_moe_lite.py index 3296a3bf5b56..e9763fe430e0 100644 --- a/python/sglang/srt/models/glm4_moe_lite.py +++ b/python/sglang/srt/models/glm4_moe_lite.py @@ -79,7 +79,7 @@ from sglang.srt.models.deepseek_common.deepseek_weight_loader import ( DeepseekV2WeightLoaderMixin, ) -from sglang.srt.models.deepseek_common.utils import _is_cuda, _use_aiter +from sglang.srt.models.deepseek_common.utils import _is_cuda, _is_xpu, _use_aiter from sglang.srt.models.deepseek_v2 import DeepseekV2AttentionMLA from sglang.srt.runtime_context import get_exec, get_parallel, get_stream from sglang.srt.utils import ( @@ -354,7 +354,7 @@ def forward_normal( topk_output = self.topk.empty_topk_output(hidden_states.device) final_hidden_states = self.experts(hidden_states, topk_output) - if not _is_cuda and not _use_aiter: + if not _is_cuda and not _is_xpu and not _use_aiter: final_hidden_states *= self.routed_scaling_factor if shared_output is not None: with use_symmetric_memory( From 7ebced6dcdb62e95a1a1d5a6881a5db3286761e6 Mon Sep 17 00:00:00 2001 From: Ranjan Debnath Date: Fri, 23 Oct 2026 07:34:30 +0000 Subject: [PATCH 2/7] Add GLM-4.7-Flash GSM8K nightly test on XPU (TP=4) Covers the glm4_moe_lite routed scaling fix: with the factor applied twice the model scores about 0.25 on gsm8k. --max-running-requests 4 is the B60 setting that keeps concurrent requests from losing the device. Arc Pro B60, tp 4, on main 6e667c4310 with the fix: 0.96 on the 50 questions, 1431 s. --- .../xpu/llm_models/test_xpu_glm_4_7_flash.py | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 test/registered/xpu/llm_models/test_xpu_glm_4_7_flash.py diff --git a/test/registered/xpu/llm_models/test_xpu_glm_4_7_flash.py b/test/registered/xpu/llm_models/test_xpu_glm_4_7_flash.py new file mode 100644 index 000000000000..438cb331f165 --- /dev/null +++ b/test/registered/xpu/llm_models/test_xpu_glm_4_7_flash.py @@ -0,0 +1,40 @@ +"""GLM-4.7-Flash GSM8K accuracy on Intel XPU (TP=4). + +Scored by ``simple_eval_gsm8k.GSM8KEval``. +""" + +import unittest + +import torch + +from sglang.test.ci.ci_register import register_xpu_ci +from sglang.test.test_utils import CustomTestCase +from sglang.test.xpu.simple_eval_gsm8k_xpu_mixin import SimpleEvalGSM8KXPUMixin + +register_xpu_ci(est_time=2400, suite="nightly-xpu-kernel-main-4-gpu", nightly=True) + + +@unittest.skipUnless( + torch.xpu.is_available(), + "Intel XPU not available (torch.xpu.is_available() returned False)", +) +class TestGLM4_7FlashXPU(SimpleEvalGSM8KXPUMixin, CustomTestCase): + model = "zai-org/GLM-4.7-Flash" + tp_size = 4 + accuracy = 0.90 + timeout_for_server_launch = 3600 + num_examples = 50 + num_threads = 4 + max_tokens = 8192 + + other_args = SimpleEvalGSM8KXPUMixin.other_args + [ + "--mem-fraction-static", + "0.85", + # More concurrent requests lose the device on Arc Pro B60. + "--max-running-requests", + "4", + ] + + +if __name__ == "__main__": + unittest.main() From 772599daf4654609ed27567092744277010ff814 Mon Sep 17 00:00:00 2001 From: Ranjan Debnath Date: Wed, 28 Oct 2026 09:44:31 +0000 Subject: [PATCH 3/7] Use sgl-eval for the GLM-4.7-Flash XPU GSM8K test Switch from SimpleEvalGSM8KXPUMixin, which calls the deprecated run_eval, to the shared GSM8KMixin with the sgl_eval backend and explicit generation settings from the GLM-4.7-Flash cookbook: thinking on (the chat template's default) with --reasoning-parser glm45, temperature 0.7, top_p 0.95 (sgl-eval's default) and max_tokens 2048. Arc Pro B60, tp 4, on main 830103398a with this PR: 0.92 on the 50 questions (2 truncated at 2048 tokens), 793 s. The threshold is 0.80; with the routed scaling applied twice the model scores about 0.25. --- .../xpu/llm_models/test_xpu_glm_4_7_flash.py | 69 ++++++++++++++----- 1 file changed, 51 insertions(+), 18 deletions(-) diff --git a/test/registered/xpu/llm_models/test_xpu_glm_4_7_flash.py b/test/registered/xpu/llm_models/test_xpu_glm_4_7_flash.py index 438cb331f165..ad9aed153853 100644 --- a/test/registered/xpu/llm_models/test_xpu_glm_4_7_flash.py +++ b/test/registered/xpu/llm_models/test_xpu_glm_4_7_flash.py @@ -1,15 +1,20 @@ """GLM-4.7-Flash GSM8K accuracy on Intel XPU (TP=4). -Scored by ``simple_eval_gsm8k.GSM8KEval``. +Scored by sgl-eval with the GLM-4.7-Flash cookbook sampling settings. """ import unittest import torch +from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_xpu_ci -from sglang.test.test_utils import CustomTestCase -from sglang.test.xpu.simple_eval_gsm8k_xpu_mixin import SimpleEvalGSM8KXPUMixin +from sglang.test.kits.eval_accuracy_kit import GSM8KMixin +from sglang.test.test_utils import ( + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) register_xpu_ci(est_time=2400, suite="nightly-xpu-kernel-main-4-gpu", nightly=True) @@ -18,22 +23,50 @@ torch.xpu.is_available(), "Intel XPU not available (torch.xpu.is_available() returned False)", ) -class TestGLM4_7FlashXPU(SimpleEvalGSM8KXPUMixin, CustomTestCase): +class TestGLM4_7FlashXPU(GSM8KMixin, CustomTestCase): model = "zai-org/GLM-4.7-Flash" - tp_size = 4 - accuracy = 0.90 - timeout_for_server_launch = 3600 - num_examples = 50 - num_threads = 4 - max_tokens = 8192 - - other_args = SimpleEvalGSM8KXPUMixin.other_args + [ - "--mem-fraction-static", - "0.85", - # More concurrent requests lose the device on Arc Pro B60. - "--max-running-requests", - "4", - ] + base_url = DEFAULT_URL_FOR_TEST + gsm8k_backend = "sgl_eval" + gsm8k_score_threshold = 0.80 + gsm8k_num_examples = 50 + gsm8k_num_threads = 4 + # Cookbook sampling; the chat template thinks by default. + gsm8k_thinking = True + gsm8k_temperature = 0.7 + gsm8k_top_p = 0.95 + gsm8k_max_tokens = 2048 + + @classmethod + def setUpClass(cls): + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=3600, + other_args=[ + "--device", + "xpu", + "--attention-backend", + "intel_xpu", + "--dtype", + "bfloat16", + "--tp-size", + "4", + "--trust-remote-code", + "--reasoning-parser", + "glm45", + "--mem-fraction-static", + "0.85", + # More concurrent requests lose the device on Arc Pro B60. + "--max-running-requests", + "4", + "--disable-overlap-schedule", + "--disable-radix-cache", + ], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) if __name__ == "__main__": From b7ba8c655fcee7115075eda344ac00f961bec879 Mon Sep 17 00:00:00 2001 From: Ranjan Debnath Date: Wed, 28 Oct 2026 19:19:50 +0000 Subject: [PATCH 4/7] Drop the XPU-availability skip from the GLM-4.7-Flash XPU test The file lives under test/registered/xpu and is registered with register_xpu_ci, so it only runs on XPU runners. --- test/registered/xpu/llm_models/test_xpu_glm_4_7_flash.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/test/registered/xpu/llm_models/test_xpu_glm_4_7_flash.py b/test/registered/xpu/llm_models/test_xpu_glm_4_7_flash.py index ad9aed153853..d10a54b54dac 100644 --- a/test/registered/xpu/llm_models/test_xpu_glm_4_7_flash.py +++ b/test/registered/xpu/llm_models/test_xpu_glm_4_7_flash.py @@ -5,8 +5,6 @@ import unittest -import torch - from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_xpu_ci from sglang.test.kits.eval_accuracy_kit import GSM8KMixin @@ -19,10 +17,6 @@ register_xpu_ci(est_time=2400, suite="nightly-xpu-kernel-main-4-gpu", nightly=True) -@unittest.skipUnless( - torch.xpu.is_available(), - "Intel XPU not available (torch.xpu.is_available() returned False)", -) class TestGLM4_7FlashXPU(GSM8KMixin, CustomTestCase): model = "zai-org/GLM-4.7-Flash" base_url = DEFAULT_URL_FOR_TEST From a48ecd584118c028dba2e3ef9d864d4a57b6ca17 Mon Sep 17 00:00:00 2001 From: Ranjan Debnath Date: Mon, 2 Nov 2026 20:10:57 +0000 Subject: [PATCH 5/7] Use the XPU sgl-eval GSM8K mixin for the GLM-4.7-Flash XPU test Main now scores every test in test/registered/xpu/llm_models/ with SimpleEvalGSM8KXPUMixin (sgl-eval gsm8k, #47056). Move this test onto it instead of the generic GSM8KMixin with its own server launch. Same launch flags, 50 examples, 4 threads, threshold 0.80 and cookbook sampling (temperature 0.7, top_p 0.95, max_tokens 2048, thinking on by template default). --- .../xpu/llm_models/test_xpu_glm_4_7_flash.py | 73 ++++++------------- 1 file changed, 23 insertions(+), 50 deletions(-) diff --git a/test/registered/xpu/llm_models/test_xpu_glm_4_7_flash.py b/test/registered/xpu/llm_models/test_xpu_glm_4_7_flash.py index d10a54b54dac..a10ee2da35a1 100644 --- a/test/registered/xpu/llm_models/test_xpu_glm_4_7_flash.py +++ b/test/registered/xpu/llm_models/test_xpu_glm_4_7_flash.py @@ -1,66 +1,39 @@ """GLM-4.7-Flash GSM8K accuracy on Intel XPU (TP=4). -Scored by sgl-eval with the GLM-4.7-Flash cookbook sampling settings. +Scored by sgl-eval's ``gsm8k`` benchmark with the GLM-4.7-Flash cookbook +sampling settings. """ import unittest -from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_xpu_ci -from sglang.test.kits.eval_accuracy_kit import GSM8KMixin -from sglang.test.test_utils import ( - DEFAULT_URL_FOR_TEST, - CustomTestCase, - popen_launch_server, -) +from sglang.test.test_utils import CustomTestCase +from sglang.test.xpu.simple_eval_gsm8k_xpu_mixin import SimpleEvalGSM8KXPUMixin register_xpu_ci(est_time=2400, suite="nightly-xpu-kernel-main-4-gpu", nightly=True) -class TestGLM4_7FlashXPU(GSM8KMixin, CustomTestCase): +class TestGLM4_7FlashXPU(SimpleEvalGSM8KXPUMixin, CustomTestCase): model = "zai-org/GLM-4.7-Flash" - base_url = DEFAULT_URL_FOR_TEST - gsm8k_backend = "sgl_eval" - gsm8k_score_threshold = 0.80 - gsm8k_num_examples = 50 - gsm8k_num_threads = 4 + tp_size = 4 + accuracy = 0.80 + timeout_for_server_launch = 3600 + num_examples = 50 + num_threads = 4 # Cookbook sampling; the chat template thinks by default. - gsm8k_thinking = True - gsm8k_temperature = 0.7 - gsm8k_top_p = 0.95 - gsm8k_max_tokens = 2048 - - @classmethod - def setUpClass(cls): - cls.process = popen_launch_server( - cls.model, - cls.base_url, - timeout=3600, - other_args=[ - "--device", - "xpu", - "--attention-backend", - "intel_xpu", - "--dtype", - "bfloat16", - "--tp-size", - "4", - "--trust-remote-code", - "--reasoning-parser", - "glm45", - "--mem-fraction-static", - "0.85", - # More concurrent requests lose the device on Arc Pro B60. - "--max-running-requests", - "4", - "--disable-overlap-schedule", - "--disable-radix-cache", - ], - ) - - @classmethod - def tearDownClass(cls): - kill_process_tree(cls.process.pid) + temperature = 0.7 + top_p = 0.95 + max_tokens = 2048 + + other_args = SimpleEvalGSM8KXPUMixin.other_args + [ + "--reasoning-parser", + "glm45", + "--mem-fraction-static", + "0.85", + # More concurrent requests lose the device on Arc Pro B60. + "--max-running-requests", + "4", + ] if __name__ == "__main__": From 0d5f866098e51fc4a52389fb072048874cead532 Mon Sep 17 00:00:00 2001 From: Ranjan Debnath Date: Tue, 3 Nov 2026 09:25:25 +0000 Subject: [PATCH 6/7] Run 20 GSM8K examples in the GLM-4.7-Flash XPU test --- test/registered/xpu/llm_models/test_xpu_glm_4_7_flash.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/registered/xpu/llm_models/test_xpu_glm_4_7_flash.py b/test/registered/xpu/llm_models/test_xpu_glm_4_7_flash.py index a10ee2da35a1..fe229a91efe8 100644 --- a/test/registered/xpu/llm_models/test_xpu_glm_4_7_flash.py +++ b/test/registered/xpu/llm_models/test_xpu_glm_4_7_flash.py @@ -18,7 +18,7 @@ class TestGLM4_7FlashXPU(SimpleEvalGSM8KXPUMixin, CustomTestCase): tp_size = 4 accuracy = 0.80 timeout_for_server_launch = 3600 - num_examples = 50 + num_examples = 20 num_threads = 4 # Cookbook sampling; the chat template thinks by default. temperature = 0.7 From 97b13568d22fabf8ee5a73ee38cd789a09e0843a Mon Sep 17 00:00:00 2001 From: Ranjan Debnath Date: Wed, 4 Nov 2026 09:05:06 +0000 Subject: [PATCH 7/7] Lower the GLM-4.7-Flash XPU test threshold to 0.75 Three runs at 20 examples on 4x Arc Pro B60 scored 0.85, 0.80 and 0.90 (sampling at temperature 0.7, 1-2 answers per run cut off at 2048 tokens). 0.80 left no margin below the lowest run; 0.75 leaves one question. --- test/registered/xpu/llm_models/test_xpu_glm_4_7_flash.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/registered/xpu/llm_models/test_xpu_glm_4_7_flash.py b/test/registered/xpu/llm_models/test_xpu_glm_4_7_flash.py index fe229a91efe8..9b04645a534d 100644 --- a/test/registered/xpu/llm_models/test_xpu_glm_4_7_flash.py +++ b/test/registered/xpu/llm_models/test_xpu_glm_4_7_flash.py @@ -16,7 +16,8 @@ class TestGLM4_7FlashXPU(SimpleEvalGSM8KXPUMixin, CustomTestCase): model = "zai-org/GLM-4.7-Flash" tp_size = 4 - accuracy = 0.80 + # 3 runs on 4x B60 scored 0.80-0.90; one question of margin below the lowest. + accuracy = 0.75 timeout_for_server_launch = 3600 num_examples = 20 num_threads = 4