From 0a87eeb3c81ea2b0d81bdc44bd460922b68c2d57 Mon Sep 17 00:00:00 2001 From: Brayden Zhong Date: Fri, 5 Dec 2025 13:42:04 -0800 Subject: [PATCH 1/3] more --- python/sglang/test/mmmu_vlm_mixin.py | 4 ++-- test/srt/models/test_vlm_models.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python/sglang/test/mmmu_vlm_mixin.py b/python/sglang/test/mmmu_vlm_mixin.py index e621a00873f5..c4c739f1508b 100644 --- a/python/sglang/test/mmmu_vlm_mixin.py +++ b/python/sglang/test/mmmu_vlm_mixin.py @@ -53,7 +53,7 @@ def run_mmmu_eval( model = "openai_compatible" tp = 1 tasks = "mmmu_val" - batch_size = 32 + batch_size = 64 log_suffix = "openai_compatible" os.makedirs(output_path, exist_ok=True) @@ -137,7 +137,7 @@ def _run_vlm_mmmu_test( other_args=[ "--trust-remote-code", "--cuda-graph-max-bs", - "32", + "64", "--enable-multimodal", "--mem-fraction-static", str(self.parsed_args.mem_fraction_static), # Use class variable diff --git a/test/srt/models/test_vlm_models.py b/test/srt/models/test_vlm_models.py index b26ff0831f7a..3650621a50d8 100644 --- a/test/srt/models/test_vlm_models.py +++ b/test/srt/models/test_vlm_models.py @@ -14,7 +14,7 @@ MODELS = [SimpleNamespace(model="openbmb/MiniCPM-V-2_6", mmmu_accuracy=0.4)] else: MODELS = [ - SimpleNamespace(model="google/gemma-3-27b-it", mmmu_accuracy=0.45), + SimpleNamespace(model="google/gemma-3-4b-it", mmmu_accuracy=0.38), SimpleNamespace(model="Qwen/Qwen2.5-VL-3B-Instruct", mmmu_accuracy=0.4), SimpleNamespace(model="openbmb/MiniCPM-V-2_6", mmmu_accuracy=0.4), ] From 96c6ee490bfe3c07148c875710e54dd858175a9c Mon Sep 17 00:00:00 2001 From: Brayden Zhong Date: Fri, 5 Dec 2025 13:43:26 -0800 Subject: [PATCH 2/3] more --- test/srt/run_suite.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/srt/run_suite.py b/test/srt/run_suite.py index e17c25fcdae6..3adca0ea41e1 100644 --- a/test/srt/run_suite.py +++ b/test/srt/run_suite.py @@ -29,7 +29,7 @@ TestFile("models/test_qwen_models.py", 90), TestFile("models/test_reward_models.py", 103), TestFile("models/test_transformers_models.py", 245), - TestFile("models/test_vlm_models.py", 282), + TestFile("models/test_vlm_models.py", 232), TestFile("openai_server/basic/test_openai_embedding.py", 70), TestFile("openai_server/basic/test_openai_server.py", 184), TestFile("openai_server/basic/test_protocol.py", 3), @@ -237,7 +237,7 @@ TestFile("models/test_qwen_models.py", 82), TestFile("models/test_reward_models.py", 132), TestFile("models/test_transformers_models.py", 320), - TestFile("models/test_vlm_models.py", 437), + TestFile("models/test_vlm_models.py", 387), TestFile("openai_server/basic/test_openai_embedding.py", 141), TestFile("openai_server/basic/test_openai_server.py", 149), TestFile("openai_server/basic/test_protocol.py", 10), From 69882dbe1d00f91ec6c3a2002f44ba6b4f41f4a0 Mon Sep 17 00:00:00 2001 From: Brayden Zhong Date: Fri, 5 Dec 2025 14:05:44 -0800 Subject: [PATCH 3/3] more --- test/srt/run_suite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/srt/run_suite.py b/test/srt/run_suite.py index 3adca0ea41e1..8f28f5c27ead 100644 --- a/test/srt/run_suite.py +++ b/test/srt/run_suite.py @@ -29,7 +29,7 @@ TestFile("models/test_qwen_models.py", 90), TestFile("models/test_reward_models.py", 103), TestFile("models/test_transformers_models.py", 245), - TestFile("models/test_vlm_models.py", 232), + TestFile("models/test_vlm_models.py", 270), TestFile("openai_server/basic/test_openai_embedding.py", 70), TestFile("openai_server/basic/test_openai_server.py", 184), TestFile("openai_server/basic/test_protocol.py", 3),