From 378099a589af811955a1a62c7368a9c659b0bab9 Mon Sep 17 00:00:00 2001 From: Ravi Theja Desetty Date: Sun, 23 Mar 2025 14:53:29 +0530 Subject: [PATCH] Update MMMU Benchmark instructions --- benchmark/mmmu/README.md | 8 ++++++-- benchmark/mmmu/bench_sglang.py | 4 +++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/benchmark/mmmu/README.md b/benchmark/mmmu/README.md index fa23751f3b6..00cdf1606f3 100644 --- a/benchmark/mmmu/README.md +++ b/benchmark/mmmu/README.md @@ -2,12 +2,16 @@ ### Evaluate sglang +Host the VLM: + ``` -python -m sglang.launch_server --model-path Qwen/Qwen2-VL-7B-Instruct --port 30000 +python -m sglang.launch_server --model-path Qwen/Qwen2-VL-7B-Instruct --chat-template qwen2-vl --port 30000 ``` +Benchmark: + ``` -python benchmark/mmmu/bench_sglang.py --model-path Qwen/Qwen2-VL-7B-Instruct --chat-template qwen2-vl --port 30000 +python benchmark/mmmu/bench_sglang.py --port 30000 ``` It's recommended to reduce the memory usage by appending something ike `--mem-fraction-static 0.6` to the command above. diff --git a/benchmark/mmmu/bench_sglang.py b/benchmark/mmmu/bench_sglang.py index 8a24af2e0e4..19f60d66138 100644 --- a/benchmark/mmmu/bench_sglang.py +++ b/benchmark/mmmu/bench_sglang.py @@ -2,7 +2,9 @@ Bench the sglang-hosted vLM with benchmark MMMU Usage: - python benchmark/mmmu/bench_sglang.py --model-path Qwen/Qwen2-VL-7B-Instruct --chat-template qwen2-vl + Host the VLM: python -m sglang.launch_server --model-path Qwen/Qwen2-VL-7B-Instruct --chat-template qwen2-vl --port 30000 + + Benchmark: python benchmark/mmmu/bench_sglang.py --port 30000 The eval output will be logged """