From 7f8eb9674fd0d169a270f6d5d63076f400c914ea Mon Sep 17 00:00:00 2001 From: ishandhanani Date: Mon, 27 Apr 2026 10:52:52 -0500 Subject: [PATCH] refactor(gpqa): drop structured runner; ship `configs/gpqa/run.sh` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ports GPQA to the same `configs//` script-based pattern that AIME moved to in PR #91, per follow-up issue #92. Same nemo-run unquoting hazard motivates the move: any benchmark passing Hydra `++overrides` through `ns eval` is one backslash-bearing extract_regex away from silently-broken evaluation. Replaces `src/srtctl/benchmarks/gpqa.py` (sglang.test.run_eval-based) with a NeMo-Skills-driven `configs/gpqa/run.sh` that recipes wire up via `type: custom`. Defaults match the upstream reasoning-eval reference (--benchmarks=gpqa:32, max_tokens=400000, temperature=1.0); all knobs overridable via env. Script warns at startup if HF_TOKEN is unset since GPQA Diamond is HF-gated. Removed: - `src/srtctl/benchmarks/gpqa.py` (62 LOC) - `src/srtctl/benchmarks/scripts/gpqa/bench.sh` (53 LOC) - `BenchmarkType.GPQA` enum - `gpqa` registry assertion in `tests/test_benchmarks.py` - GPQA row + section in `docs/config-reference.md` - `gpqa` mention in `examples/example.yaml` and `docs/architecture.md` tree Updated: - `docs/accuracy.md` GPQA section → script-based runbook (recipe shape, HF gating note, reasoning-mode env vars pointer back to AIME) - `src/srtctl/benchmarks/__init__.py` — drop gpqa import + __all__ entry - 6 existing `type: "gpqa"` recipes migrated 1:1 to `type: custom` + `bash /configs/gpqa/run.sh`, preserving their MAX_TOKENS / REPEAT / NUM_THREADS as env-var overrides - 2 h200 recipes' commented-out `type: "gpqa"` block replaced with a one-line pointer to the new docs Backward compat: none. Recipes with `type: gpqa` will fail schema validation — migration is a 1:1 swap to the `type: custom` block above. Test plan: `make check` — 611 passed, 2 skipped (matches pre-change baseline). MMLU / longbenchv2 / gsm8k untouched per the issue's focused-PR-per-benchmark plan. Co-Authored-By: Claude Opus 4.7 (1M context) --- configs/gpqa/run.sh | 75 +++++++++++++++++++ docs/accuracy.md | 71 +++++++++++++++--- docs/architecture.md | 2 - docs/config-reference.md | 21 ------ examples/example.yaml | 2 +- recipes/h200/8k1k/bs64-2p3d-mtp.yaml | 7 +- recipes/h200/8k1k/bs64-2p3d.yaml | 7 +- .../mooncake/mtp_radix_off/1p1d-mtp-acc.yaml | 16 ++-- .../1p1d-mtp-acc-prefixcache-retraction.yaml | 16 ++-- .../1p1d-mtp-acc-prefixcache.yaml | 16 ++-- .../stp_prefix_off/1p1d-dep4tp4-gpqa.yaml | 16 ++-- .../1p1d-dep4tp4-staging-gpqa.yaml | 16 ++-- .../stp_prefix_off/1p1d-dep4-nixl-gpqa.yaml | 16 ++-- src/srtctl/benchmarks/__init__.py | 2 - src/srtctl/benchmarks/gpqa.py | 62 --------------- src/srtctl/benchmarks/gsm8k.py | 2 +- src/srtctl/benchmarks/scripts/gpqa/bench.sh | 53 ------------- src/srtctl/core/schema.py | 1 - tests/test_benchmarks.py | 1 - 19 files changed, 207 insertions(+), 195 deletions(-) create mode 100755 configs/gpqa/run.sh delete mode 100644 src/srtctl/benchmarks/gpqa.py delete mode 100644 src/srtctl/benchmarks/scripts/gpqa/bench.sh diff --git a/configs/gpqa/run.sh b/configs/gpqa/run.sh new file mode 100755 index 000000000..d1043abe9 --- /dev/null +++ b/configs/gpqa/run.sh @@ -0,0 +1,75 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# GPQA reasoning eval — runs inside the NeMo Skills container. +# +# Phase 1: ns prepare_data gpqa (downloads GPQA Diamond from HF; gated dataset +# → set HF_TOKEN in benchmark.env) +# Phase 2: ns eval --benchmarks=gpqa:$REPEAT (NeMo Skills' default multi-choice +# extractor; pass@k via REPEAT) +# +# Server endpoint, model, and dataset can be overridden via env. Tuning knobs +# (max_tokens, repeat, etc.) match the upstream reasoning-eval reference +# (--benchmarks=gpqa:32, temperature=1.0, max_tokens=400000). +# +# Same nemo-run unquoting hazard as AIME applies — do not pass Hydra ++overrides +# with backslash-bearing values (e.g. custom extract_regex). Post-process the +# cached output-rs.jsonl files in Python if you need a broader extractor. + +set -euo pipefail + +ENDPOINT="${ENDPOINT:-http://localhost:8000/v1}" +MODEL="${MODEL:-dspro}" +DATASET="${DATASET:-gpqa}" +REPEAT="${REPEAT:-32}" +MAX_TOKENS="${MAX_TOKENS:-400000}" +NUM_THREADS="${NUM_THREADS:-512}" +TEMPERATURE="${TEMPERATURE:-1.0}" +TOP_P="${TOP_P:-1.0}" +SEED="${SEED:-42}" +OUTPUT_DIR="${OUTPUT_DIR:-/logs/accuracy/${DATASET}}" + +export OPENAI_API_KEY="${OPENAI_API_KEY:-EMPTY}" + +echo "=== Config ===" +echo " endpoint: $ENDPOINT" +echo " model: $MODEL" +echo " dataset: $DATASET" +echo " repeat: $REPEAT" +echo " max_tokens: $MAX_TOKENS" +echo " num_threads: $NUM_THREADS" +echo " temperature: $TEMPERATURE" +echo " top_p: $TOP_P" +echo " seed: $SEED" +echo " output_dir: $OUTPUT_DIR" +echo + +if [ -z "${HF_TOKEN:-}" ]; then + echo "WARNING: HF_TOKEN is not set. GPQA Diamond is HF-gated; ns prepare_data" + echo " will fail unless the token is plumbed through benchmark.env." +fi + +mkdir -p "$OUTPUT_DIR" + +echo "=== Phase 1: prepare_data ===" +ns prepare_data "$DATASET" + +echo +echo "=== Phase 2: ns eval ===" +ns eval \ + --server_type=openai \ + --model="$MODEL" \ + --server_address="$ENDPOINT" \ + --benchmarks="${DATASET}:${REPEAT}" \ + --output_dir="$OUTPUT_DIR" \ + --starting_seed="$SEED" \ + "++inference.tokens_to_generate=${MAX_TOKENS}" \ + "++max_concurrent_requests=${NUM_THREADS}" \ + "++inference.temperature=${TEMPERATURE}" \ + "++inference.top_p=${TOP_P}" \ + "++inference.timeout=25000000" + +echo +echo "=== Done ===" +echo "Metrics: ${OUTPUT_DIR}/eval-results/${DATASET}/metrics.json" diff --git a/docs/accuracy.md b/docs/accuracy.md index e435c181c..188435a50 100644 --- a/docs/accuracy.md +++ b/docs/accuracy.md @@ -1,6 +1,6 @@ # Accuracy Benchmarks -In srt-slurm, users can run different accuracy benchmarks by setting the benchmark section in the config yaml file. Supported benchmarks include `mmlu`, `gpqa`, `longbenchv2`, and AIME (via the script under `configs/aime/`). +In srt-slurm, users can run different accuracy benchmarks by setting the benchmark section in the config yaml file. Supported benchmarks include `mmlu`, `longbenchv2`, AIME (via the script under `configs/aime/`), and GPQA (via the script under `configs/gpqa/`). ## Table of Contents @@ -154,16 +154,69 @@ MMLU evaluation complete ## GPQA -For GPQA dataset, the benchmark section in yaml file can be modified in the following way: -```bash + +GPQA runs in the official **NeMo Skills container** (`nvcr.io/nvidia/eval-factory/nemo-skills:26.03`), +side-by-side with the model server. There is no first-class `type: gpqa` runner — +the eval logic lives in `configs/gpqa/run.sh` and recipes wire it up via +`type: custom` (same pattern as AIME above). + +### Recipe shape + +```yaml benchmark: - type: "gpqa" - num_examples: 198 # Number of examples to run - max_tokens: 65536 # We need a larger output token number for GPQA - repeat: 8 # Number of repetition - num_threads: 128 # Number of parallel threads for running benchmark + type: custom + container_image: nemo-skills # alias defined in srtslurm.yaml `containers:` + # or the full nvcr.io URI for Pyxis auto-pull + env: + OPENAI_API_KEY: "EMPTY" # ns/litellm requires it set; value is unused + HF_TOKEN: "${HF_TOKEN}" # REQUIRED: GPQA Diamond is HF-gated + # Optional knob overrides — defaults match the upstream reasoning-eval reference: + # MODEL: "dspro" # must match served-model-name from sglang_config + # DATASET: "gpqa" # ns prepare_data target (gpqa → GPQA Diamond) + # REPEAT: "32" # pass@k samples per problem + # MAX_TOKENS: "400000" # generous ceiling for reasoning traces + # NUM_THREADS: "512" # client-side concurrency + # TEMPERATURE: "1.0" + # TOP_P: "1.0" + # SEED: "42" # --starting_seed for reproducibility + command: | + bash /configs/gpqa/run.sh ``` -The `context-length` argument here should be set to a value larger than `max_tokens`. + +Container alias setup is identical to AIME — see the AIME section above for the +`srtslurm.yaml` `containers:` entry and the `enroot import` pre-cache step. + +### HF gating + +GPQA Diamond is gated on Hugging Face. The recipe must propagate `HF_TOKEN` +through `benchmark.env` (already plumbed end-to-end). Without it, `ns +prepare_data gpqa` aborts during dataset download. The script prints a +warning at startup if `HF_TOKEN` is unset. + +### Reasoning-mode env vars (server side) + +For reasoning-capable models, set the same SGLang reasoning env vars as AIME +(see the AIME "Reasoning-mode env vars" section). Without them, GPQA pass@k +drops well below what the model can do. + +### What the script does + +1. `ns prepare_data $DATASET` — fetches GPQA from HF (gated) into the NeMo + Skills install. +2. `ns eval --benchmarks=${DATASET}:${REPEAT} ...` against + `http://localhost:8000/v1` (the in-job dynamo frontend) with the + upstream reasoning-eval reference's tuning defaults. NeMo Skills' default + multi-choice extractor scores the generations. + +Outputs land at `/logs/accuracy//eval-results//metrics.json` +with pass@1, pass@N, and majority@N. + +### Custom answer-extraction regex (not currently applied) + +Same nemo-run unquoting hazard as AIME — do not pass Hydra `++overrides` with +backslash-bearing values to `ns eval`. Post-process the cached +`output-rs.jsonl` files with a Python script (raw-string regex, no shell +layers) if you need a broader extractor. ## LongBench-V2 diff --git a/docs/architecture.md b/docs/architecture.md index aa451cb42..b97fd3573 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1087,9 +1087,7 @@ src/srtctl/ | |-- __init__.py # Registry and exports | |-- base.py # BenchmarkRunner ABC, register_benchmark | |-- sa_bench.py # SA-Bench throughput benchmark -| |-- aime.py # AIME math accuracy benchmark | |-- mmlu.py # MMLU accuracy benchmark -| |-- gpqa.py # GPQA benchmark | |-- longbenchv2.py # LongBench v2 benchmark | |-- router.py # Router benchmark | |-- mooncake_router.py # Mooncake router benchmark diff --git a/docs/config-reference.md b/docs/config-reference.md index 74f844e81..6f8151fa8 100644 --- a/docs/config-reference.md +++ b/docs/config-reference.md @@ -420,7 +420,6 @@ Benchmark configuration. The `type` field determines which benchmark runner is u | `sa-bench` | Throughput/latency serving benchmark | | `sglang-bench` | SGLang bench_serving benchmark | | `mmlu` | MMLU accuracy evaluation | -| `gpqa` | GPQA (Graduate-level science QA) evaluation | | `longbenchv2` | Long-context evaluation benchmark | | `router` | Router performance with prefix caching | | `mooncake-router` | KV-aware routing with Mooncake trace | @@ -498,26 +497,6 @@ benchmark: | `repeat` | int | No | 8 | Number of repeats | | `num_threads` | int | No | 512 | Concurrent threads | -### gpqa - -Graduate-level science QA evaluation using sglang.test.run_eval. - -```yaml -benchmark: - type: "gpqa" - num_examples: 198 # Optional: Number of examples - max_tokens: 32768 # Optional: Max tokens per response - repeat: 8 # Optional: Number of repeats - num_threads: 128 # Optional: Concurrent threads -``` - -| Field | Type | Required | Default | Description | -| -------------- | ---- | -------- | ------- | ---------------------------- | -| `num_examples` | int | No | 198 | Number of examples to run | -| `max_tokens` | int | No | 32768 | Max tokens per response | -| `repeat` | int | No | 8 | Number of repeats | -| `num_threads` | int | No | 128 | Concurrent threads | - ### longbenchv2 Long-context evaluation benchmark. diff --git a/examples/example.yaml b/examples/example.yaml index f87f9e0b0..77f1becca 100644 --- a/examples/example.yaml +++ b/examples/example.yaml @@ -63,7 +63,7 @@ backend: # Benchmark configuration benchmark: - type: "sa-bench" # sa-bench, mmlu, gpqa, custom, or "manual" (no auto-benchmark) + type: "sa-bench" # sa-bench, mmlu, custom, or "manual" (no auto-benchmark) isl: 1024 # Input sequence length osl: 1024 # Output sequence length concurrencies: [256, 512] # Concurrency levels to test diff --git a/recipes/h200/8k1k/bs64-2p3d-mtp.yaml b/recipes/h200/8k1k/bs64-2p3d-mtp.yaml index ed1232d16..a12b8d52d 100644 --- a/recipes/h200/8k1k/bs64-2p3d-mtp.yaml +++ b/recipes/h200/8k1k/bs64-2p3d-mtp.yaml @@ -117,9 +117,4 @@ benchmark: concurrencies: "32x64x128" req_rate: "inf" -# benchmark: -# type: "gpqa" -# num_examples: 198 -# repeat: 4 -# num_threads: 32 -# max_tokens: 64000 +# See configs/gpqa/run.sh + docs/accuracy.md for the script-based GPQA recipe. diff --git a/recipes/h200/8k1k/bs64-2p3d.yaml b/recipes/h200/8k1k/bs64-2p3d.yaml index 73aaacc30..84640ae06 100644 --- a/recipes/h200/8k1k/bs64-2p3d.yaml +++ b/recipes/h200/8k1k/bs64-2p3d.yaml @@ -107,9 +107,4 @@ benchmark: concurrencies: "32x64x128" req_rate: "inf" -# benchmark: -# type: "gpqa" -# num_examples: 198 -# repeat: 4 -# num_threads: 32 -# max_tokens: 64000 \ No newline at end of file +# See configs/gpqa/run.sh + docs/accuracy.md for the script-based GPQA recipe. \ No newline at end of file diff --git a/recipes/qwen3.5/fp8/disagg/mooncake/mtp_radix_off/1p1d-mtp-acc.yaml b/recipes/qwen3.5/fp8/disagg/mooncake/mtp_radix_off/1p1d-mtp-acc.yaml index a81e53c37..792d88aea 100644 --- a/recipes/qwen3.5/fp8/disagg/mooncake/mtp_radix_off/1p1d-mtp-acc.yaml +++ b/recipes/qwen3.5/fp8/disagg/mooncake/mtp_radix_off/1p1d-mtp-acc.yaml @@ -121,8 +121,14 @@ backend: reasoning-parser: qwen3 benchmark: - type: "gpqa" - num_examples: 198 - max_tokens: 65536 - repeat: 8 - num_threads: 32 + type: custom + container_image: nemo-skills + env: + OPENAI_API_KEY: "EMPTY" + HF_TOKEN: "${HF_TOKEN}" + MODEL: "Qwen/Qwen3.5-397B-A17B-FP8" + MAX_TOKENS: "65536" + REPEAT: "8" + NUM_THREADS: "32" + command: | + bash /configs/gpqa/run.sh diff --git a/recipes/qwen3.5/fp8/disagg/mooncake/mtp_radix_on/1p1d-mtp-acc-prefixcache-retraction.yaml b/recipes/qwen3.5/fp8/disagg/mooncake/mtp_radix_on/1p1d-mtp-acc-prefixcache-retraction.yaml index 6e8f4a8fb..da11d23b1 100644 --- a/recipes/qwen3.5/fp8/disagg/mooncake/mtp_radix_on/1p1d-mtp-acc-prefixcache-retraction.yaml +++ b/recipes/qwen3.5/fp8/disagg/mooncake/mtp_radix_on/1p1d-mtp-acc-prefixcache-retraction.yaml @@ -123,8 +123,14 @@ backend: reasoning-parser: qwen3 benchmark: - type: "gpqa" - num_examples: 198 - max_tokens: 65536 - repeat: 8 - num_threads: 128 + type: custom + container_image: nemo-skills + env: + OPENAI_API_KEY: "EMPTY" + HF_TOKEN: "${HF_TOKEN}" + MODEL: "Qwen/Qwen3.5-397B-A17B-FP8" + MAX_TOKENS: "65536" + REPEAT: "8" + NUM_THREADS: "128" + command: | + bash /configs/gpqa/run.sh diff --git a/recipes/qwen3.5/fp8/disagg/mooncake/mtp_radix_on/1p1d-mtp-acc-prefixcache.yaml b/recipes/qwen3.5/fp8/disagg/mooncake/mtp_radix_on/1p1d-mtp-acc-prefixcache.yaml index 0b6b0a91e..4662c29d7 100644 --- a/recipes/qwen3.5/fp8/disagg/mooncake/mtp_radix_on/1p1d-mtp-acc-prefixcache.yaml +++ b/recipes/qwen3.5/fp8/disagg/mooncake/mtp_radix_on/1p1d-mtp-acc-prefixcache.yaml @@ -119,8 +119,14 @@ backend: reasoning-parser: qwen3 benchmark: - type: "gpqa" - num_examples: 198 - max_tokens: 65536 - repeat: 1 - num_threads: 64 + type: custom + container_image: nemo-skills + env: + OPENAI_API_KEY: "EMPTY" + HF_TOKEN: "${HF_TOKEN}" + MODEL: "Qwen/Qwen3.5-397B-A17B-FP8" + MAX_TOKENS: "65536" + REPEAT: "1" + NUM_THREADS: "64" + command: | + bash /configs/gpqa/run.sh diff --git a/recipes/qwen3.5/fp8/disagg/mooncake/stp_prefix_off/1p1d-dep4tp4-gpqa.yaml b/recipes/qwen3.5/fp8/disagg/mooncake/stp_prefix_off/1p1d-dep4tp4-gpqa.yaml index 1895495b9..f86f3d741 100644 --- a/recipes/qwen3.5/fp8/disagg/mooncake/stp_prefix_off/1p1d-dep4tp4-gpqa.yaml +++ b/recipes/qwen3.5/fp8/disagg/mooncake/stp_prefix_off/1p1d-dep4tp4-gpqa.yaml @@ -110,8 +110,14 @@ backend: watchdog-timeout: 1000000 benchmark: - type: "gpqa" - num_examples: 198 - max_tokens: 65536 - repeat: 8 - num_threads: 128 + type: custom + container_image: nemo-skills + env: + OPENAI_API_KEY: "EMPTY" + HF_TOKEN: "${HF_TOKEN}" + MODEL: "Qwen/Qwen3.5-397B-A17B-FP8" + MAX_TOKENS: "65536" + REPEAT: "8" + NUM_THREADS: "128" + command: | + bash /configs/gpqa/run.sh diff --git a/recipes/qwen3.5/fp8/disagg/mooncake/stp_prefix_off/1p1d-dep4tp4-staging-gpqa.yaml b/recipes/qwen3.5/fp8/disagg/mooncake/stp_prefix_off/1p1d-dep4tp4-staging-gpqa.yaml index 0110f7ba2..f99198da7 100644 --- a/recipes/qwen3.5/fp8/disagg/mooncake/stp_prefix_off/1p1d-dep4tp4-staging-gpqa.yaml +++ b/recipes/qwen3.5/fp8/disagg/mooncake/stp_prefix_off/1p1d-dep4tp4-staging-gpqa.yaml @@ -114,8 +114,14 @@ backend: watchdog-timeout: 1000000 benchmark: - type: "gpqa" - num_examples: 198 - max_tokens: 65536 - repeat: 8 - num_threads: 128 + type: custom + container_image: nemo-skills + env: + OPENAI_API_KEY: "EMPTY" + HF_TOKEN: "${HF_TOKEN}" + MODEL: "Qwen/Qwen3.5-397B-A17B-FP8" + MAX_TOKENS: "65536" + REPEAT: "8" + NUM_THREADS: "128" + command: | + bash /configs/gpqa/run.sh diff --git a/recipes/qwen3.5/fp8/disagg/nixl/stp_prefix_off/1p1d-dep4-nixl-gpqa.yaml b/recipes/qwen3.5/fp8/disagg/nixl/stp_prefix_off/1p1d-dep4-nixl-gpqa.yaml index ae06a008d..ad42b7fcb 100644 --- a/recipes/qwen3.5/fp8/disagg/nixl/stp_prefix_off/1p1d-dep4-nixl-gpqa.yaml +++ b/recipes/qwen3.5/fp8/disagg/nixl/stp_prefix_off/1p1d-dep4-nixl-gpqa.yaml @@ -109,8 +109,14 @@ backend: watchdog-timeout: 1000000 benchmark: - type: "gpqa" - num_examples: 198 - max_tokens: 65536 - repeat: 8 - num_threads: 128 + type: custom + container_image: nemo-skills + env: + OPENAI_API_KEY: "EMPTY" + HF_TOKEN: "${HF_TOKEN}" + MODEL: "Qwen/Qwen3.5-397B-A17B-FP8" + MAX_TOKENS: "65536" + REPEAT: "8" + NUM_THREADS: "128" + command: | + bash /configs/gpqa/run.sh diff --git a/src/srtctl/benchmarks/__init__.py b/src/srtctl/benchmarks/__init__.py index 02cf0a1ad..c5ff42f32 100644 --- a/src/srtctl/benchmarks/__init__.py +++ b/src/srtctl/benchmarks/__init__.py @@ -6,7 +6,6 @@ # Import runners to trigger registration from srtctl.benchmarks import ( custom, - gpqa, gsm8k, longbenchv2, mmlu, @@ -33,7 +32,6 @@ "sa_bench", "sglang_bench", "mmlu", - "gpqa", "gsm8k", "longbenchv2", "router", diff --git a/src/srtctl/benchmarks/gpqa.py b/src/srtctl/benchmarks/gpqa.py deleted file mode 100644 index bab067604..000000000 --- a/src/srtctl/benchmarks/gpqa.py +++ /dev/null @@ -1,62 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""GPQA accuracy benchmark runner.""" - -from __future__ import annotations - -from typing import TYPE_CHECKING - -from srtctl.benchmarks.base import SCRIPTS_DIR, BenchmarkRunner, register_benchmark - -if TYPE_CHECKING: - from srtctl.core.runtime import RuntimeContext - from srtctl.core.schema import SrtConfig - - -@register_benchmark("gpqa") -class GPQARunner(BenchmarkRunner): - """GPQA (Graduate-level science QA) accuracy evaluation. - - Uses sglang.test.run_eval with gpqa task. - - Optional config fields: - - benchmark.num_examples: Number of examples (default: 198) - - benchmark.max_tokens: Max tokens per response (default: 32768) - - benchmark.repeat: Number of repeats (default: 8) - - benchmark.num_threads: Concurrent threads (default: 128) - """ - - @property - def name(self) -> str: - return "GPQA" - - @property - def script_path(self) -> str: - return "/srtctl-benchmarks/gpqa/bench.sh" - - @property - def local_script_dir(self) -> str: - return str(SCRIPTS_DIR / "gpqa") - - def validate_config(self, config: SrtConfig) -> list[str]: - # GPQA has sensible defaults - return [] - - def build_command( - self, - config: SrtConfig, - runtime: RuntimeContext, - ) -> list[str]: - b = config.benchmark - endpoint = f"http://localhost:{runtime.frontend_port}" - - return [ - "bash", - self.script_path, - endpoint, - str(b.num_examples or 198), - str(b.max_tokens or 32768), - str(b.repeat or 8), - str(b.num_threads or 128), - ] diff --git a/src/srtctl/benchmarks/gsm8k.py b/src/srtctl/benchmarks/gsm8k.py index b930c0c15..15cfdbef5 100644 --- a/src/srtctl/benchmarks/gsm8k.py +++ b/src/srtctl/benchmarks/gsm8k.py @@ -60,7 +60,7 @@ def build_command( ) -> list[str]: b = config.benchmark # TODO: support overriding endpoint via config to target external servers; - # mmlu/gpqa/longbenchv2 share the same limitation today. + # mmlu/longbenchv2 share the same limitation today. endpoint = f"http://localhost:{runtime.frontend_port}" return [ diff --git a/src/srtctl/benchmarks/scripts/gpqa/bench.sh b/src/srtctl/benchmarks/scripts/gpqa/bench.sh deleted file mode 100644 index ce442e35f..000000000 --- a/src/srtctl/benchmarks/scripts/gpqa/bench.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# GPQA accuracy evaluation -# Expects: endpoint [num_examples] [max_tokens] [repeat] [num_threads] - -set -e - -ENDPOINT=$1 -NUM_EXAMPLES=${2:-198} -MAX_TOKENS=${3:-32768} -REPEAT=${4:-8} -NUM_THREADS=${5:-128} - -# Auto-detect model name from /v1/models endpoint; fall back to default -MODEL_NAME=$(curl -s "${ENDPOINT}/v1/models" 2>/dev/null | python3 -c "import sys,json; print(json.load(sys.stdin)['data'][0]['id'])" 2>/dev/null || echo "") -if [ -z "${MODEL_NAME}" ]; then - MODEL_NAME="deepseek-ai/DeepSeek-R1" - echo "Warning: Could not auto-detect model name, using default: ${MODEL_NAME}" -fi - -echo "GPQA Config: endpoint=${ENDPOINT}; model=${MODEL_NAME}; num_examples=${NUM_EXAMPLES}; max_tokens=${MAX_TOKENS}; repeat=${REPEAT}; num_threads=${NUM_THREADS}" - -# Create results directory -result_dir="/logs/accuracy" -mkdir -p "$result_dir" - -# Set OPENAI_API_KEY if not set -export OPENAI_API_KEY="${OPENAI_API_KEY:-EMPTY}" - -echo "Running GPQA evaluation..." - -python3 -m sglang.test.run_eval \ - --base-url "${ENDPOINT}" \ - --model "${MODEL_NAME}" \ - --eval-name gpqa \ - --num-examples "${NUM_EXAMPLES}" \ - --max-tokens "${MAX_TOKENS}" \ - --repeat "${REPEAT}" \ - --num-threads "${NUM_THREADS}" - -# Copy result file -result_file=$(ls -t /tmp/gpqa_*.json 2>/dev/null | head -n1) -if [ -f "$result_file" ]; then - cp "$result_file" "$result_dir/" - echo "Results saved to: $result_dir/$(basename "$result_file")" -else - echo "Warning: Could not find result file in /tmp" -fi - -echo "GPQA evaluation complete" - diff --git a/src/srtctl/core/schema.py b/src/srtctl/core/schema.py index 1806e0db4..7f002d99f 100644 --- a/src/srtctl/core/schema.py +++ b/src/srtctl/core/schema.py @@ -227,7 +227,6 @@ class BenchmarkType(str, Enum): MOONCAKE_ROUTER = "mooncake-router" TRACE_REPLAY = "trace-replay" MMLU = "mmlu" - GPQA = "gpqa" GSM8K = "gsm8k" LONGBENCHV2 = "longbenchv2" diff --git a/tests/test_benchmarks.py b/tests/test_benchmarks.py index 79183306f..c12268035 100644 --- a/tests/test_benchmarks.py +++ b/tests/test_benchmarks.py @@ -19,7 +19,6 @@ def test_list_benchmarks(self): assert "sa-bench" in benchmarks assert "sglang-bench" in benchmarks assert "mmlu" in benchmarks - assert "gpqa" in benchmarks assert "gsm8k" in benchmarks assert "longbenchv2" in benchmarks assert "router" in benchmarks