diff --git a/python/sglang/test/mmmu_vlm_mixin.py b/python/sglang/test/mmmu_vlm_mixin.py index e621a00873f5..c4c739f1508b 100644 --- a/python/sglang/test/mmmu_vlm_mixin.py +++ b/python/sglang/test/mmmu_vlm_mixin.py @@ -53,7 +53,7 @@ def run_mmmu_eval( model = "openai_compatible" tp = 1 tasks = "mmmu_val" - batch_size = 32 + batch_size = 64 log_suffix = "openai_compatible" os.makedirs(output_path, exist_ok=True) @@ -137,7 +137,7 @@ def _run_vlm_mmmu_test( other_args=[ "--trust-remote-code", "--cuda-graph-max-bs", - "32", + "64", "--enable-multimodal", "--mem-fraction-static", str(self.parsed_args.mem_fraction_static), # Use class variable diff --git a/test/srt/models/test_vlm_models.py b/test/srt/models/test_vlm_models.py index b26ff0831f7a..3650621a50d8 100644 --- a/test/srt/models/test_vlm_models.py +++ b/test/srt/models/test_vlm_models.py @@ -14,7 +14,7 @@ MODELS = [SimpleNamespace(model="openbmb/MiniCPM-V-2_6", mmmu_accuracy=0.4)] else: MODELS = [ - SimpleNamespace(model="google/gemma-3-27b-it", mmmu_accuracy=0.45), + SimpleNamespace(model="google/gemma-3-4b-it", mmmu_accuracy=0.38), SimpleNamespace(model="Qwen/Qwen2.5-VL-3B-Instruct", mmmu_accuracy=0.4), SimpleNamespace(model="openbmb/MiniCPM-V-2_6", mmmu_accuracy=0.4), ] diff --git a/test/srt/run_suite.py b/test/srt/run_suite.py index e17c25fcdae6..8f28f5c27ead 100644 --- a/test/srt/run_suite.py +++ b/test/srt/run_suite.py @@ -29,7 +29,7 @@ TestFile("models/test_qwen_models.py", 90), TestFile("models/test_reward_models.py", 103), TestFile("models/test_transformers_models.py", 245), - TestFile("models/test_vlm_models.py", 282), + TestFile("models/test_vlm_models.py", 270), TestFile("openai_server/basic/test_openai_embedding.py", 70), TestFile("openai_server/basic/test_openai_server.py", 184), TestFile("openai_server/basic/test_protocol.py", 3), @@ -237,7 +237,7 @@ TestFile("models/test_qwen_models.py", 82), TestFile("models/test_reward_models.py", 132), TestFile("models/test_transformers_models.py", 320), - TestFile("models/test_vlm_models.py", 437), + TestFile("models/test_vlm_models.py", 387), TestFile("openai_server/basic/test_openai_embedding.py", 141), TestFile("openai_server/basic/test_openai_server.py", 149), TestFile("openai_server/basic/test_protocol.py", 10),