[CI] [Intel] xpu-eval: Add sgl-eval to XPU test deps and move all GSM8K tests to it

.github/workflows/nightly-test-intel-wheel.yml

@@ -99,7 +99,7 @@ jobs:
           print("image pins:", pins)
           PY
           PIP_INSTALL="/opt/venv/bin/python3 -m pip install --no-cache-dir -c /tmp/image-pins.txt --extra-index-url https://download.pytorch.org/whl/xpu"
-          docker exec ci_sglang_xpu $PIP_INSTALL "lm-eval==0.4.9"
+          docker exec ci_sglang_xpu $PIP_INSTALL "sgl-eval==0.1.3" "antlr4-python3-runtime==4.9.3"
           # Test deps of the PR CI suites (mirrors pr-test-xpu.yml, minus the sglang reinstall).
           docker exec ci_sglang_xpu $PIP_INSTALL pytest expecttest ray huggingface_hub "lmcache>=0.3.9" accelerate
           # Diffusion extra's deps (needed by test_xpu_zimage_turbo).

.github/workflows/nightly-test-intel.yml

@@ -65,9 +65,10 @@ jobs:
           # Bridges the up-to-24h window between merge and nightly image rebuild.
           dest=$(docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \
                  "import sglang, os; print(os.path.dirname(sglang.__file__))")
-          docker exec ci_sglang_xpu cp \
-            /sglang-checkout/python/sglang/test/ci/ci_utils.py \
-            "$dest/test/ci/ci_utils.py"
+          for f in ci/ci_utils.py xpu/simple_eval_gsm8k_xpu_mixin.py; do
+            docker exec ci_sglang_xpu cp \
+              "/sglang-checkout/python/sglang/test/$f" "$dest/test/$f"
+          done
           docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \
             "import sglang.test.ci.ci_utils; print('overlay ok')"
 
@@ -76,7 +77,7 @@ jobs:
         run: |
           docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir tabulate
           docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir --no-deps xgrammar==0.1.33
-          docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir "lm-eval==0.4.9"
+          docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir "sgl-eval==0.1.3" "antlr4-python3-runtime==4.9.3"
           docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir pytest
           # Diffusion extras (needed by test_xpu_flux2_dev, test_xpu_zimage_turbo).
           # Pins mirror python/pyproject_xpu.toml [project.optional-dependencies.diffusion].
@@ -173,9 +174,10 @@ jobs:
           # Bridges the up-to-24h window between merge and nightly image rebuild.
           dest=$(docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \
                  "import sglang, os; print(os.path.dirname(sglang.__file__))")
-          docker exec ci_sglang_xpu cp \
-            /sglang-checkout/python/sglang/test/ci/ci_utils.py \
-            "$dest/test/ci/ci_utils.py"
+          for f in ci/ci_utils.py xpu/simple_eval_gsm8k_xpu_mixin.py; do
+            docker exec ci_sglang_xpu cp \
+              "/sglang-checkout/python/sglang/test/$f" "$dest/test/$f"
+          done
           docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \
             "import sglang.test.ci.ci_utils; print('overlay ok')"
 
@@ -184,7 +186,7 @@ jobs:
         run: |
           docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir tabulate
           docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir --no-deps xgrammar==0.1.33
-          docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir "lm-eval==0.4.9"
+          docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir "sgl-eval==0.1.3" "antlr4-python3-runtime==4.9.3"
           docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir pytest
           # Diffusion extras (needed by test_xpu_flux2_dev, test_xpu_zimage_turbo).
           # Pins mirror python/pyproject_xpu.toml [project.optional-dependencies.diffusion].
@@ -270,9 +272,10 @@ jobs:
           # Bridges the up-to-24h window between merge and nightly image rebuild.
           dest=$(docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \
                  "import sglang, os; print(os.path.dirname(sglang.__file__))")
-          docker exec ci_sglang_xpu cp \
-            /sglang-checkout/python/sglang/test/ci/ci_utils.py \
-            "$dest/test/ci/ci_utils.py"
+          for f in ci/ci_utils.py xpu/simple_eval_gsm8k_xpu_mixin.py; do
+            docker exec ci_sglang_xpu cp \
+              "/sglang-checkout/python/sglang/test/$f" "$dest/test/$f"
+          done
           docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \
             "import sglang.test.ci.ci_utils; print('overlay ok')"
 
@@ -281,7 +284,7 @@ jobs:
         run: |
           docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir tabulate
           docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir --no-deps xgrammar==0.1.33
-          docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir "lm-eval==0.4.9"
+          docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir "sgl-eval==0.1.3" "antlr4-python3-runtime==4.9.3"
           docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir pytest
           # Diffusion extras (needed by test_xpu_flux2_dev, test_xpu_zimage_turbo).
           # Pins mirror python/pyproject_xpu.toml [project.optional-dependencies.diffusion].
@@ -367,9 +370,10 @@ jobs:
           # Bridges the up-to-24h window between merge and nightly image rebuild.
           dest=$(docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \
                  "import sglang, os; print(os.path.dirname(sglang.__file__))")
-          docker exec ci_sglang_xpu cp \
-            /sglang-checkout/python/sglang/test/ci/ci_utils.py \
-            "$dest/test/ci/ci_utils.py"
+          for f in ci/ci_utils.py xpu/simple_eval_gsm8k_xpu_mixin.py; do
+            docker exec ci_sglang_xpu cp \
+              "/sglang-checkout/python/sglang/test/$f" "$dest/test/$f"
+          done
           docker exec ci_sglang_xpu /opt/venv/bin/python3 -c \
             "import sglang.test.ci.ci_utils; print('overlay ok')"
 
@@ -378,7 +382,7 @@ jobs:
         run: |
           docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir tabulate
           docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir --no-deps xgrammar==0.1.33
-          docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir "lm-eval==0.4.9"
+          docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir "sgl-eval==0.1.3" "antlr4-python3-runtime==4.9.3"
           docker exec ci_sglang_xpu /opt/venv/bin/python3 -m pip install --no-cache-dir pytest
           # Diffusion extras
           # Pins mirror python/pyproject_xpu.toml [project.optional-dependencies.diffusion].

python/pyproject_xpu.toml

@@ -115,12 +115,15 @@ test = [
   "jsonlines",
   "lm-eval[api]>=0.4.9.2",
   "matplotlib",
+  # sgl-eval's math_verify grader needs 4.9.3 (latex2sympy2_extended fails on 4.7.x).
+  "antlr4-python3-runtime==4.9.3",
   "pandas",
   "parameterized",
   "peft>=0.18.0",
   "polars",
   "pytest",
   "sentence_transformers",
+  "sgl-eval==0.1.3",
   "tabulate",
 ]
 

python/sglang/test/xpu/simple_eval_gsm8k_xpu_mixin.py

@@ -1,8 +1,8 @@
-"""simple-evals GSM8K accuracy mixin for Intel XPU nightly tests.
+"""sgl-eval GSM8K accuracy mixin for Intel XPU nightly tests.
 
-Launches an SGLang server with XPU flags, then calls ``sglang.test.run_eval``
-with ``eval_name="gsm8k"`` so the ``simple_eval_gsm8k.GSM8KEval`` evaluator
-scores the run.
+Launches an SGLang server with XPU flags, then calls
+``sglang.test.sgl_eval_utils.run_sgl_eval`` with ``eval_name="gsm8k"`` so the
+sgl-eval GSM8K benchmark (0-shot chat, boxed-answer grader) scores the run.
 
 Subclasses set ``model``, ``tp_size``, ``accuracy``, and may override
 ``other_args`` / ``env`` / ``num_examples`` / ``num_threads``.
@@ -16,7 +16,7 @@
 from types import SimpleNamespace
 
 from sglang.srt.utils import kill_process_tree
-from sglang.test.run_eval import run_eval
+from sglang.test.sgl_eval_utils import run_sgl_eval
 from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
@@ -52,6 +52,12 @@ class SimpleEvalGSM8KXPUMixin(ABC):
     num_threads: int = 1
     # Short generations reduce prefill->decode handoffs that trip the same wedge.
     max_tokens: int = 512
+    # None keeps sgl-eval's greedy default; thinking models that loop under
+    # greedy decoding set both to sample instead.
+    temperature: float | None = None
+    top_p: float | None = None
+    # Forwarded per request, e.g. {"enable_thinking": False}.
+    chat_template_kwargs: dict | None = None
 
     @classmethod
     def setUpClass(cls):
@@ -81,7 +87,7 @@ def test_gsm8k(self):
 
         model_metrics = {
             "server": self.server_cmd,
-            "client": "simple_eval_gsm8k",
+            "client": "sgl_eval_gsm8k",
             "accuracy_threshold": getattr(self, "accuracy", "N/A"),
             "output_throughput_threshold": getattr(self, "output_throughput", "N/A"),
             "num_prompts": self.num_examples,
@@ -97,8 +103,11 @@ def test_gsm8k(self):
                 num_examples=self.num_examples,
                 num_threads=self.num_threads,
                 max_tokens=self.max_tokens,
+                temperature=self.temperature,
+                top_p=self.top_p,
+                chat_template_kwargs=self.chat_template_kwargs,
             )
-            metrics = run_eval(args)
+            metrics = run_sgl_eval(args)
             model_metrics["accuracy"] = metrics["score"]
             model_metrics["output_throughput"] = metrics.get("output_throughput")
             model_metrics["latency"] = metrics["latency"]

test/registered/xpu/llm_models/test_xpu_gemma_4_26b_a4b.py

@@ -1,6 +1,6 @@
 """gemma-4-26B-A4B GSM8K accuracy on Intel XPU (TP=4).
 
-Scored by ``simple_eval_gsm8k.GSM8KEval``.
+Scored by sgl-eval's ``gsm8k`` benchmark.
 """
 
 import unittest
@@ -11,7 +11,7 @@
 from sglang.test.test_utils import CustomTestCase
 from sglang.test.xpu.simple_eval_gsm8k_xpu_mixin import SimpleEvalGSM8KXPUMixin
 
-register_xpu_ci(est_time=2400, suite="nightly-xpu-kernel-main-4-gpu", nightly=True)
+register_xpu_ci(est_time=3000, suite="nightly-xpu-kernel-main-4-gpu", nightly=True)
 
 
 @unittest.skipUnless(

test/registered/xpu/llm_models/test_xpu_llama_3_1_8b.py

@@ -3,7 +3,7 @@
 TP=4 wedges the Level Zero driver during the first prefill batch on Arc/BMG;
 TP=2 runs cleanly with the same model and serves at ~18 tok/s.
 
-Scored by ``simple_eval_gsm8k.GSM8KEval``.
+Scored by sgl-eval's ``gsm8k`` benchmark.
 """
 
 import unittest
@@ -24,7 +24,10 @@
 class TestLlama31_8BInstructXPU(SimpleEvalGSM8KXPUMixin, CustomTestCase):
     model = "meta-llama/Llama-3.1-8B-Instruct"
     tp_size = 2
-    accuracy = 0.80
+    # sgl-eval's 0-shot boxed grader scores 0.81-0.82 here (5-shot run_eval: 0.85).
+    accuracy = 0.77
+    # sgl-eval's 0-shot CoT runs past the mixin's 512-token default.
+    max_tokens = 1024
 
     other_args = SimpleEvalGSM8KXPUMixin.other_args + [
         "--max-total-tokens",

test/registered/xpu/llm_models/test_xpu_nemotron_3_nano_30b_a3b.py

@@ -1,6 +1,6 @@
 """NVIDIA-Nemotron-3-Nano-30B-A3B GSM8K accuracy on Intel XPU (TP=4).
 
-Scored by ``simple_eval_gsm8k.GSM8KEval``.
+Scored by sgl-eval's ``gsm8k`` benchmark.
 """
 
 import unittest
@@ -11,7 +11,7 @@
 from sglang.test.test_utils import CustomTestCase
 from sglang.test.xpu.simple_eval_gsm8k_xpu_mixin import SimpleEvalGSM8KXPUMixin
 
-register_xpu_ci(est_time=2400, suite="nightly-xpu-kernel-main-4-gpu", nightly=True)
+register_xpu_ci(est_time=4800, suite="nightly-xpu-kernel-main-4-gpu", nightly=True)
 
 
 @unittest.skipUnless(

test/registered/xpu/llm_models/test_xpu_qwen3_30b_a3b.py

@@ -1,6 +1,6 @@
 """Qwen3-30B-A3B GSM8K accuracy on Intel XPU (TP=4).
 
-Scored by ``simple_eval_gsm8k.GSM8KEval``.
+Scored by sgl-eval's ``gsm8k`` benchmark.
 """
 
 import unittest

test/registered/xpu/llm_models/test_xpu_qwen3_5_35b_a3b.py

@@ -1,6 +1,6 @@
 """Qwen3.5-35B-A3B GSM8K accuracy on Intel XPU (TP=4).
 
-Scored by ``simple_eval_gsm8k.GSM8KEval``.
+Scored by sgl-eval ``gsm8k`` with sampled thinking.
 """
 
 import unittest
@@ -11,7 +11,7 @@
 from sglang.test.test_utils import CustomTestCase
 from sglang.test.xpu.simple_eval_gsm8k_xpu_mixin import SimpleEvalGSM8KXPUMixin
 
-register_xpu_ci(est_time=2400, suite="nightly-xpu-kernel-main-4-gpu", nightly=True)
+register_xpu_ci(est_time=3300, suite="nightly-xpu-kernel-main-4-gpu", nightly=True)
 
 
 @unittest.skipUnless(
@@ -21,17 +21,22 @@
 class TestQwen3_5_35BA3BXPU(SimpleEvalGSM8KXPUMixin, CustomTestCase):
     model = "Qwen/Qwen3.5-35B-A3B"
     tp_size = 4
-    accuracy = 0.90
+    # Lowest score over 2 runs (0.85, 0.90) on B70; raise only on new data.
+    accuracy = 0.85
     timeout_for_server_launch = 3600
-    num_examples = 50
+    num_examples = 20
     num_threads = 4
-    max_tokens = 8192
+    # Greedy thinking loops re-checking sgl-eval's boxed answer, and
+    # non-thinking skips the CoT (0.42); sampling ends most loops.
+    temperature = 1.0
+    top_p = 0.95
+    max_tokens = 16384
 
     other_args = SimpleEvalGSM8KXPUMixin.other_args + [
         "--page-size",
         "128",
         "--max-total-tokens",
-        "65536",
+        "131072",
         "--mem-fraction-static",
         "0.85",
     ]

test/registered/xpu/llm_models/test_xpu_qwen3_5_9b.py

@@ -1,8 +1,8 @@
 """Qwen3.5-9B GSM8K accuracy on Intel XPU (TP=4).
 
-Scored by ``simple_eval_gsm8k.GSM8KEval``. Covers both the opt-in fused GDN
-SYCL kernel path (``--linear-attn-backend intel_xpu``) and the default Triton
-GDN path (``triton``, unchanged from other platforms).
+Scored by sgl-eval ``gsm8k`` with thinking disabled. Covers both the opt-in
+fused GDN SYCL kernel path (``--linear-attn-backend intel_xpu``) and the default
+Triton GDN path (``triton``, unchanged from other platforms).
 """
 
 import unittest
@@ -13,7 +13,7 @@
 from sglang.test.test_utils import CustomTestCase
 from sglang.test.xpu.simple_eval_gsm8k_xpu_mixin import SimpleEvalGSM8KXPUMixin
 
-register_xpu_ci(est_time=2400, suite="nightly-xpu-kernel-main-4-gpu", nightly=True)
+register_xpu_ci(est_time=900, suite="nightly-xpu-kernel-main-4-gpu", nightly=True)
 
 
 @unittest.skipUnless(
@@ -23,11 +23,14 @@
 class Qwen3_5_9BXPUBase(SimpleEvalGSM8KXPUMixin, CustomTestCase):
     model = "Qwen/Qwen3.5-9B"
     tp_size = 1
-    accuracy = 0.90
-    # max_tokens=8192 lets the GSM8K CoT complete under num_threads=4.
-    num_examples = 50
+    # Thinking loops re-checking sgl-eval's boxed answer until max_tokens;
+    # non-thinking scored 0.96 on 50 examples (3 runs, B70).
+    chat_template_kwargs = {"enable_thinking": False}
+    # Lowest score over 3 runs of all 3 classes (B70); raise only on new data.
+    accuracy = 0.95
+    num_examples = 20
     num_threads = 4
-    max_tokens = 8192
+    max_tokens = 2048
 
     other_args = SimpleEvalGSM8KXPUMixin.other_args + [
         "--page-size",

test/registered/xpu/test_xpu_dcp_gsm8k.py

@@ -13,7 +13,7 @@
     popen_launch_server,
 )
 
-register_xpu_ci(est_time=900, suite="nightly-xpu-kernel-main-2-gpu", nightly=True)
+register_xpu_ci(est_time=1800, suite="nightly-xpu-kernel-main-2-gpu", nightly=True)
 
 
 @unittest.skipUnless(torch.xpu.is_available(), "needs Intel XPU")
@@ -23,6 +23,12 @@ class TestXPUDCPGSM8K(GSM8KMixin, CustomTestCase):
     gsm8k_score_threshold = 0.80
     gsm8k_num_examples = 200
     gsm8k_num_threads = 32
+    gsm8k_backend = "sgl_eval"
+    gsm8k_max_tokens = 8192
+    # Greedy thinking loops re-checking the boxed answer; sample like the
+    # model's generation_config instead.
+    gsm8k_temperature = 1.0
+    gsm8k_top_p = 0.95
 
     @classmethod
     def setUpClass(cls):