[NPU] Add GitHub test summary and deduplicate test code. Part 1

python/sglang/test/ascend/gsm8k_ascend_mixin.py

@@ -3,17 +3,21 @@
 from types import SimpleNamespace
 
 from sglang.srt.utils import kill_process_tree
+from sglang.test.ascend.test_ascend_utils import write_results_to_github_step_summary
 from sglang.test.few_shot_gsm8k import run_eval
 from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
     popen_launch_server,
+    write_github_step_summary,
 )
 
 
 class GSM8KAscendMixin(ABC):
     model = ""
     accuracy = 0.00
+    output_throughput = 0.00
+
     timeout_for_server_launch = DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH
     other_args = [
         "--trust-remote-code",
@@ -24,47 +28,77 @@ class GSM8KAscendMixin(ABC):
         "--disable-cuda-graph",
     ]
     gsm8k_num_shots = 5
+    num_questions = 200
+
+    env = {
+        **os.environ,
+        "PYTORCH_NPU_ALLOC_CONF": "expandable_segments:True",
+        "ASCEND_MF_STORE_URL": "tcp://127.0.0.1:24666",
+        "HCCL_BUFFSIZE": "200",
+        "SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK": "24",
+        "USE_VLLM_CUSTOM_ALLREDUCE": "1",
+        "HCCL_EXEC_TIMEOUT": "200",
+        "STREAMS_PER_DEVICE": "32",
+        "SGLANG_ENBLE_TORCH_COMILE": "1",
+        "AUTO_USE_UC_MEMORY": "0",
+        "P2P_HCCL_BUFFSIZE": "20",
+    }
 
     @classmethod
     def setUpClass(cls):
         cls.base_url = DEFAULT_URL_FOR_TEST
-        os.environ["PYTORCH_NPU_ALLOC_CONF"] = "expandable_segments:True"
-        os.environ["ASCEND_MF_STORE_URL"] = "tcp://127.0.0.1:24666"
-        os.environ["HCCL_BUFFSIZE"] = "200"
-        os.environ["SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK"] = "24"
-        os.environ["USE_VLLM_CUSTOM_ALLREDUCE"] = "1"
-        os.environ["HCCL_EXEC_TIMEOUT"] = "200"
-        os.environ["STREAMS_PER_DEVICE"] = "32"
-        os.environ["SGLANG_ENBLE_TORCH_COMILE"] = "1"
-        os.environ["AUTO_USE_UC_MEMORY"] = "0"
-        os.environ["P2P_HCCL_BUFFSIZE"] = "20"
-        env = os.environ.copy()
-
-        cls.process = popen_launch_server(
-            cls.model,
-            cls.base_url,
-            timeout=cls.timeout_for_server_launch,
-            other_args=cls.other_args,
-            env=env,
-        )
+        try:
+            cls.process = popen_launch_server(
+                cls.model,
+                cls.base_url,
+                timeout=cls.timeout_for_server_launch,
+                other_args=cls.other_args,
+                env=cls.env,
+            )
+        except Exception as e:
+            write_github_step_summary(f"Failed to launch server for {cls.model}: {e}")
+            cls.fail(f"Test failed for {cls.model}: {e}")
 
     @classmethod
     def tearDownClass(cls):
         kill_process_tree(cls.process.pid)
 
     def test_gsm8k(self):
-        args = SimpleNamespace(
-            num_shots=self.gsm8k_num_shots,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
-        )
-        metrics = run_eval(args)
-        self.assertGreaterEqual(
-            metrics["accuracy"],
-            self.accuracy,
-            f'Accuracy of {self.model} is {str(metrics["accuracy"])}, is lower than {self.accuracy}',
-        )
+        model_metrics = {
+            "params": self.other_args,
+            "accuracy": "-",
+            "accuracy_threshold": self.accuracy,
+            "output_throughput": "-",
+            "output_throughput_threshold": self.output_throughput,
+            "latency": "-",
+            "latency_threshold": "N/A",
+        }
+
+        try:
+            args = SimpleNamespace(
+                num_shots=self.gsm8k_num_shots,
+                data_path=None,
+                num_questions=self.num_questions,
+                max_new_tokens=512,
+                parallel=128,
+                host="http://127.0.0.1",
+                port=int(self.base_url.split(":")[-1]),
+            )
+            metrics = run_eval(args)
+            model_metrics["accuracy"] = metrics["accuracy"]
+            model_metrics["output_throughput"] = metrics["output_throughput"]
+            self.assertGreaterEqual(
+                metrics["accuracy"],
+                self.accuracy,
+                f'Accuracy of {self.model} is {str(metrics["accuracy"])}, is lower than {self.accuracy}',
+            )
+            self.assertGreaterEqual(
+                metrics["output_throughput"],
+                self.output_throughput,
+                f'Output throughput of {self.model} is {str(metrics["output_throughput"])}, is lower than {self.output_throughput}',
+            )
+        except Exception as e:
+            model_metrics["error"] = e
+            self.fail(f"Test failed for {self.model}: {e}")
+        finally:
+            write_results_to_github_step_summary({self.model: model_metrics})

python/sglang/test/ascend/test_ascend_utils.py

@@ -24,7 +24,9 @@
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
     auto_config_device,
+    is_in_ci,
     popen_launch_server,
+    write_github_step_summary,
 )
 
 # Model weights storage directory
@@ -90,6 +92,9 @@
 LLAMA_4_SCOUT_17B_16E_INSTRUCT_WEIGHTS_PATH = os.path.join(
     MODEL_WEIGHTS_DIR, "meta-llama/Llama-4-Scout-17B-16E-Instruct"
 )
+LLaDA2_0_MINI_WEIGHTS_PATH = os.path.join(
+    MODEL_WEIGHTS_DIR, "inclusionAI/LLaDA2.0-mini"
+)
 META_LLAMA_3_1_8B_INSTRUCT = os.path.join(
     MODEL_WEIGHTS_DIR, "LLM-Research/Meta-Llama-3.1-8B-Instruct"
 )
@@ -549,3 +554,35 @@ async def _run():
 
     assert res["completed"] == num_prompts
     return res
+
+
+HEADER = """
+### Models
+| Model | Variant | Output Throughput | Expected Output Throughput | Latency | Expected Latency | Accuracy | Threshold | Status |
+| ----- | ------- | -------- | ------------------ | ------- | ---------------- | -------- | --------- | ------ |
+"""
+
+
+def write_results_to_github_step_summary(results: dict):
+    if not is_in_ci():
+        return
+
+    write_github_step_summary_once(HEADER)
+
+    summary = ""
+    for model, metrics in results.items():
+        error = metrics.get("error", "")
+        status = (
+            "✅"
+            if error != "" and metrics["accuracy"] >= metrics["expected_accuracy"]
+            else "❌ " + error
+        )
+        summary += f"| {model} | {metrics['params']} | {metrics['output_throughput']} | {metrics['output_throughput_threshold']} | {metrics['latency']} | {metrics['latency_threshold']} | {metrics['accuracy']:} | {metrics['accuracy_threshold']} | {status} |\n"
+    write_github_step_summary(summary)
+
+
+def write_github_step_summary_once(summary: str):
+    if getattr(write_github_step_summary_once, "has_written", False):
+        return
+    write_github_step_summary_once.has_written = True
+    write_github_step_summary(summary)

python/sglang/test/ascend/test_mmlu.py

@@ -0,0 +1,19 @@
+from types import SimpleNamespace
+
+from sglang.test.run_eval import run_eval
+
+
+class TestMMLU:
+    accuracy_mmlu = 0.00
+
+    def test_mmlu(self):
+        args = SimpleNamespace(
+            base_url=self.base_url,
+            model=self.model,
+            eval_name="mmlu",
+            num_examples=128,
+            num_threads=32,
+        )
+        print("Starting mmlu test...")
+        metrics = run_eval(args)
+        self.assertGreater(metrics["score"], self.accuracy_mmlu)

python/sglang/test/ascend/vlm_utils.py

@@ -4,6 +4,7 @@
 import subprocess
 
 from sglang.srt.utils import kill_process_tree
+from sglang.test.ascend.test_ascend_utils import write_results_to_github_step_summary
 from sglang.test.test_utils import (
     DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
     DEFAULT_URL_FOR_TEST,
@@ -115,8 +116,19 @@ def _run_vlm_mmmu_test(
         """
         print(f"\nTesting model: {self.model}{test_name}")
 
+        model_metrics = {
+            "params": self.other_args,
+            "accuracy": "-",
+            "accuracy_threshold": self.mmmu_accuracy,
+            "output_throughput": "-",
+            "output_throughput_threshold": "N/A",
+            "latency": "-",
+            "latency_threshold": "N/A",
+        }
+
         process = None
         server_output = ""
+        mmmu_accuracy = None
 
         try:
             # Prepare environment variables
@@ -163,6 +175,8 @@ def _run_vlm_mmmu_test(
             if capture_output and process:
                 server_output = self._read_output_from_files()
 
+            model_metrics["accuracy"] = mmmu_accuracy
+
             # Assert performance meets expected threshold
             self.assertGreaterEqual(
                 mmmu_accuracy,
@@ -175,8 +189,10 @@ def _run_vlm_mmmu_test(
         except Exception as e:
             print(f"Error testing {self.model}{test_name}: {e}")
             self.fail(f"Test failed for {self.model}{test_name}: {e}")
-
+            model_metrics["error"] = e
         finally:
+            write_results_to_github_step_summary({self.model: model_metrics})
+
             # Ensure process cleanup happens regardless of success/failure
             if process is not None and process.poll() is None:
                 print(f"Cleaning up process {process.pid}")

test/registered/ascend/basic_function/dllm/test_npu_llada2_mini.py

@@ -1,77 +1,41 @@
 import os
 import unittest
-from types import SimpleNamespace
 
-from sglang.srt.utils import kill_process_tree
+from sglang.test.ascend.gsm8k_ascend_mixin import GSM8KAscendMixin
+from sglang.test.ascend.test_ascend_utils import LLaDA2_0_MINI_WEIGHTS_PATH
 from sglang.test.ci.ci_register import register_npu_ci
-from sglang.test.few_shot_gsm8k import run_eval as run_eval_few_shot_gsm8k
 from sglang.test.send_one import BenchArgs, send_one_prompt
 from sglang.test.test_utils import (
-    DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
-    DEFAULT_URL_FOR_TEST,
     CustomTestCase,
     is_in_ci,
-    popen_launch_server,
     write_github_step_summary,
 )
 
 register_npu_ci(est_time=400, suite="stage-b-test-4-npu-a3", nightly=False)
 register_npu_ci(est_time=400, suite="nightly-1-npu-a3", nightly=True)
 
 
-class TestLLaDA2Mini(CustomTestCase):
-    @classmethod
-    def setUpClass(cls):
-        cls._old_disable_acl = os.environ.get("SGLANG_NPU_DISABLE_ACL_FORMAT_WEIGHT")
-        os.environ["SGLANG_NPU_DISABLE_ACL_FORMAT_WEIGHT"] = "1"
-
-        cls.model = "/root/.cache/modelscope/hub/models/inclusionAI/LLaDA2.0-mini"
-        cls.base_url = DEFAULT_URL_FOR_TEST
-
-        other_args = [
-            "--trust-remote-code",
-            "--disable-radix-cache",
-            "--mem-fraction-static",
-            "0.9",
-            "--max-running-requests",
-            "1",
-            "--attention-backend",
-            "ascend",
-            "--dllm-algorithm",
-            "LowConfidence",  # TODO: Add dLLM configurations
-        ]
-
-        cls.process = popen_launch_server(
-            cls.model,
-            cls.base_url,
-            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
-            other_args=other_args,
-        )
-
-    @classmethod
-    def tearDownClass(cls):
-        kill_process_tree(cls.process.pid)
-
-        if cls._old_disable_acl is None:
-            os.environ.pop("SGLANG_NPU_DISABLE_ACL_FORMAT_WEIGHT", None)
-        else:
-            os.environ["SGLANG_NPU_DISABLE_ACL_FORMAT_WEIGHT"] = cls._old_disable_acl
-
-    def test_gsm8k(self):
-        args = SimpleNamespace(
-            num_shots=5,
-            data_path=None,
-            num_questions=200,
-            max_new_tokens=512,
-            parallel=128,
-            host="http://127.0.0.1",
-            port=int(self.base_url.split(":")[-1]),
-        )
-        metrics = run_eval_few_shot_gsm8k(args)
-        print(f"{metrics=}")
-
-        self.assertGreater(metrics["accuracy"], 0.88)
-        self.assertGreater(metrics["output_throughput"], 70)
+class TestLLaDA2Mini(GSM8KAscendMixin, CustomTestCase):
+    model = LLaDA2_0_MINI_WEIGHTS_PATH
+
+    other_args = [
+        "--trust-remote-code",
+        "--disable-radix-cache",
+        "--mem-fraction-static",
+        "0.9",
+        "--max-running-requests",
+        "1",
+        "--attention-backend",
+        "ascend",
+        "--dllm-algorithm",
+        "LowConfidence",  # TODO: Add dLLM configurations
+    ]
+    env = {
+        **os.environ,
+        "SGLANG_NPU_DISABLE_ACL_FORMAT_WEIGHT": "1",
+    }
+    accuracy = 0.88
+    output_throughput = 70
 
     def test_bs_1_speed(self):
         args = BenchArgs(port=int(self.base_url.split(":")[-1]), max_new_tokens=2048)