diff --git a/.github/workflows/nightly-benchmark.yml b/.github/workflows/nightly-benchmark.yml index ce95062363..a32085cfb7 100644 --- a/.github/workflows/nightly-benchmark.yml +++ b/.github/workflows/nightly-benchmark.yml @@ -2,7 +2,13 @@ name: Nightly Benchmark on: schedule: - - cron: '0 0 * * 0' + - cron: '0 0 * * *' + push: + branches: [ main ] + paths-ignore: + - "docs/**" + - "mkdocs.yml" + - "*.md" workflow_dispatch: inputs: models: @@ -21,26 +27,42 @@ concurrency: permissions: contents: read +env: + RUSTC_WRAPPER: sccache + SCCACHE_GHA_ENABLED: "true" + jobs: # --------------------------------------------------------------------------- # Build wheel once and reuse across all jobs # --------------------------------------------------------------------------- build-wheel: - runs-on: 8-gpu-h200 + runs-on: k8s-runner-gpu permissions: contents: read steps: - name: Checkout code uses: actions/checkout@v6 + - name: Cache wheel artifacts + id: cache-wheel + uses: actions/cache@v5 + with: + path: bindings/python/dist/*.whl + key: nightly-wheel-${{ runner.os }}-${{ hashFiles('Cargo.lock', '**/Cargo.toml', 'auth/src/**', 'bindings/python/src/**', 'data_connector/src/**', 'grpc_client/src/**', 'kv_index/src/**', 'mcp/src/**', 'mesh/src/**', 'model_gateway/src/**', 'multimodal/src/**', 'protocols/src/**', 'reasoning_parser/src/**', 'tokenizer/src/**', 'tool_parser/src/**', 'wasm/src/**', 'workflow/src/**') }} + restore-keys: | + nightly-wheel-${{ runner.os }}- + - name: Setup Rust + if: steps.cache-wheel.outputs.cache-hit != 'true' uses: ./.github/actions/setup-rust - name: Setup Python venv + if: steps.cache-wheel.outputs.cache-hit != 'true' run: bash scripts/ci_setup_python_venv.sh - name: Build Python wheel + if: steps.cache-wheel.outputs.cache-hit != 'true' run: bash scripts/ci_build_wheel.sh - name: Upload wheel artifact @@ -50,33 +72,36 @@ jobs: path: bindings/python/dist/*.whl retention-days: 7 + - name: Show sccache stats + if: always() && steps.cache-wheel.outputs.cache-hit != 'true' + run: sccache --show-stats + # --------------------------------------------------------------------------- - # Single-worker: one model per job for better isolation and debugging + # Single-worker (H100): one model per job on k8s H100 runners. # --------------------------------------------------------------------------- single-worker: name: "${{ matrix.model.id }} / single-${{ matrix.variant.id }}" needs: build-wheel if: ${{ !cancelled() }} - runs-on: 8-gpu-h200 + runs-on: ["k8s-runner-gpu", "4-gpu-h100"] timeout-minutes: 1440 strategy: fail-fast: false - max-parallel: 1 + max-parallel: 8 matrix: model: - - { id: llama-8b, test_class: TestNightlyLlama8bSingle } - - { id: llama-1b, test_class: TestNightlyLlama1bSingle } - - { id: qwen-7b, test_class: TestNightlyQwen7bSingle } - - { id: qwen-14b, test_class: TestNightlyQwen14bSingle } - - { id: deepseek-7b, test_class: TestNightlyDeepseek7bSingle } - - { id: qwen-30b, test_class: TestNightlyQwen30bSingle } - - { id: mistral-7b, test_class: TestNightlyMistral7bSingle } - - { id: gpt-oss, test_class: TestNightlyGptOssSingle } - - { id: llama-4-maverick-17b, test_class: TestNightlyLlama4MaverickSingle } + - { id: meta-llama/Llama-3.1-8B-Instruct, slug: meta-llama-Llama-3.1-8B-Instruct, test_class: TestNightlyLlama8bSingle } + - { id: meta-llama/Llama-3.2-1B-Instruct, slug: meta-llama-Llama-3.2-1B-Instruct, test_class: TestNightlyLlama1bSingle } + - { id: Qwen/Qwen2.5-7B-Instruct, slug: Qwen-Qwen2.5-7B-Instruct, test_class: TestNightlyQwen7bSingle } + - { id: Qwen/Qwen2.5-14B-Instruct, slug: Qwen-Qwen2.5-14B-Instruct, test_class: TestNightlyQwen14bSingle } + - { id: deepseek-ai/DeepSeek-R1-Distill-Qwen-7B, slug: deepseek-ai-DeepSeek-R1-Distill-Qwen-7B, test_class: TestNightlyDeepseek7bSingle } + - { id: Qwen/Qwen3-30B-A3B, slug: Qwen-Qwen3-30B-A3B, test_class: TestNightlyQwen30bSingle } + - { id: mistralai/Mistral-7B-Instruct-v0.3, slug: mistralai-Mistral-7B-Instruct-v0.3, test_class: TestNightlyMistral7bSingle } + - { id: openai/gpt-oss-20b, slug: openai-gpt-oss-20b, test_class: TestNightlyGptOssSingle } variant: - - { id: sglang, runtime: sglang, grpc_only: "false" } - - { id: vllm, runtime: vllm, grpc_only: "true" } + - { id: sglang, runtime: sglang, grpc_only: "false", setup_vllm: false, setup_trtllm: false, extra_deps: "genai-bench" } + - { id: vllm, runtime: vllm, grpc_only: "true", setup_vllm: true, setup_trtllm: false, extra_deps: "genai-bench" } steps: - name: Check filters @@ -85,7 +110,7 @@ jobs: MODELS="${{ github.event.inputs.models || 'all' }}" RUNTIME="${{ github.event.inputs.runtime || 'all' }}" SKIP="false" - if [ "$MODELS" != "all" ] && ! echo ",$MODELS," | grep -q ",${{ matrix.model.id }},"; then + if [ "$MODELS" != "all" ] && ! echo ",$MODELS," | grep -Fq ",${{ matrix.model.id }},"; then SKIP="true" fi if [ "$RUNTIME" != "all" ] && [ "$RUNTIME" != "${{ matrix.variant.id }}" ]; then @@ -97,18 +122,17 @@ jobs: if: steps.filter.outputs.skip != 'true' uses: actions/checkout@v6 - - name: Install inference backend - if: steps.filter.outputs.skip != 'true' - env: - SGLANG_USE_LATEST_TAG: "1" - run: | - bash scripts/ci_setup_python_venv.sh - source .venv/bin/activate - if [ "${{ matrix.variant.runtime }}" == "vllm" ]; then - bash scripts/ci_install_vllm.sh - else - bash scripts/ci_install_sglang.sh - fi + - name: Setup vLLM backend + if: steps.filter.outputs.skip != 'true' && matrix.variant.setup_vllm + uses: ./.github/actions/setup-vllm + + - name: Setup TRT-LLM backend + if: steps.filter.outputs.skip != 'true' && matrix.variant.setup_trtllm + uses: ./.github/actions/setup-trtllm + + - name: Setup SGLang backend + if: steps.filter.outputs.skip != 'true' && !matrix.variant.setup_vllm && !matrix.variant.setup_trtllm + uses: ./.github/actions/setup-sglang - name: Download wheel artifact if: steps.filter.outputs.skip != 'true' @@ -122,12 +146,14 @@ jobs: run: | pip uninstall -y smg || true pip install wheel/*.whl - bash scripts/ci_install_e2e_deps.sh genai-bench + bash scripts/ci_install_e2e_deps.sh ${{ matrix.variant.extra_deps }} - name: Run benchmark if: steps.filter.outputs.skip != 'true' env: HF_TOKEN: ${{ secrets.HF_TOKEN }} + GPU_TYPE: H100 + E2E_MODEL_TP_OVERRIDES: '{"meta-llama/Llama-3.1-8B-Instruct":1,"meta-llama/Llama-3.2-1B-Instruct":1,"Qwen/Qwen2.5-7B-Instruct":1,"Qwen/Qwen2.5-14B-Instruct":1,"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B":1,"Qwen/Qwen3-30B-A3B":1,"mistralai/Mistral-7B-Instruct-v0.3":1,"openai/gpt-oss-20b":1}' run: | bash scripts/ci_killall_sglang.sh "nuk_gpus" @@ -146,7 +172,7 @@ jobs: if: always() && steps.filter.outputs.skip != 'true' uses: actions/upload-artifact@v6 with: - name: nightly-${{ matrix.model.id }}-single-${{ matrix.variant.id }}-${{ github.run_id }} + name: nightly-${{ matrix.model.slug }}-single-${{ matrix.variant.id }}-${{ github.run_id }} path: nightly_*/ retention-days: 90 @@ -155,32 +181,31 @@ jobs: run: bash scripts/ci_killall_sglang.sh || true # --------------------------------------------------------------------------- - # Multi-worker: one model per job (uses all 8 GPUs) + # Multi-worker (H100): one model per job on k8s H100 runners. # --------------------------------------------------------------------------- multi-worker: name: "${{ matrix.model.id }} / multi-${{ matrix.variant.id }}" - needs: [build-wheel, single-worker] + needs: build-wheel if: ${{ !cancelled() }} - runs-on: 8-gpu-h200 + runs-on: ["k8s-runner-gpu", "4-gpu-h100"] timeout-minutes: 1440 strategy: fail-fast: false - max-parallel: 1 + max-parallel: 8 matrix: model: - - { id: llama-8b, test_class: TestNightlyLlama8bMulti } - - { id: llama-1b, test_class: TestNightlyLlama1bMulti } - - { id: qwen-7b, test_class: TestNightlyQwen7bMulti } - - { id: qwen-14b, test_class: TestNightlyQwen14bMulti } - - { id: deepseek-7b, test_class: TestNightlyDeepseek7bMulti } - - { id: qwen-30b, test_class: TestNightlyQwen30bMulti } - - { id: mistral-7b, test_class: TestNightlyMistral7bMulti } - - { id: gpt-oss, test_class: TestNightlyGptOssMulti } - # - { id: llama-4-maverick-17b, test_class: TestNightlyLlama4MaverickMulti } # 1 worker uses all 8 GPUs (tp=8) + - { id: meta-llama/Llama-3.1-8B-Instruct, slug: meta-llama-Llama-3.1-8B-Instruct, test_class: TestNightlyLlama8bMulti } + - { id: meta-llama/Llama-3.2-1B-Instruct, slug: meta-llama-Llama-3.2-1B-Instruct, test_class: TestNightlyLlama1bMulti } + - { id: Qwen/Qwen2.5-7B-Instruct, slug: Qwen-Qwen2.5-7B-Instruct, test_class: TestNightlyQwen7bMulti } + - { id: Qwen/Qwen2.5-14B-Instruct, slug: Qwen-Qwen2.5-14B-Instruct, test_class: TestNightlyQwen14bMulti } + - { id: deepseek-ai/DeepSeek-R1-Distill-Qwen-7B, slug: deepseek-ai-DeepSeek-R1-Distill-Qwen-7B, test_class: TestNightlyDeepseek7bMulti } + - { id: Qwen/Qwen3-30B-A3B, slug: Qwen-Qwen3-30B-A3B, test_class: TestNightlyQwen30bMulti } + - { id: mistralai/Mistral-7B-Instruct-v0.3, slug: mistralai-Mistral-7B-Instruct-v0.3, test_class: TestNightlyMistral7bMulti } + - { id: openai/gpt-oss-20b, slug: openai-gpt-oss-20b, test_class: TestNightlyGptOssMulti } variant: - - { id: sglang, runtime: sglang, grpc_only: "false" } - - { id: vllm, runtime: vllm, grpc_only: "true" } + - { id: sglang, runtime: sglang, grpc_only: "false", setup_vllm: false, setup_trtllm: false, extra_deps: "genai-bench" } + - { id: vllm, runtime: vllm, grpc_only: "true", setup_vllm: true, setup_trtllm: false, extra_deps: "genai-bench" } steps: - name: Check filters @@ -189,7 +214,7 @@ jobs: MODELS="${{ github.event.inputs.models || 'all' }}" RUNTIME="${{ github.event.inputs.runtime || 'all' }}" SKIP="false" - if [ "$MODELS" != "all" ] && ! echo ",$MODELS," | grep -q ",${{ matrix.model.id }},"; then + if [ "$MODELS" != "all" ] && ! echo ",$MODELS," | grep -Fq ",${{ matrix.model.id }},"; then SKIP="true" fi if [ "$RUNTIME" != "all" ] && [ "$RUNTIME" != "${{ matrix.variant.id }}" ]; then @@ -201,19 +226,122 @@ jobs: if: steps.filter.outputs.skip != 'true' uses: actions/checkout@v6 - - name: Install inference backend + - name: Setup vLLM backend + if: steps.filter.outputs.skip != 'true' && matrix.variant.setup_vllm + uses: ./.github/actions/setup-vllm + + - name: Setup TRT-LLM backend + if: steps.filter.outputs.skip != 'true' && matrix.variant.setup_trtllm + uses: ./.github/actions/setup-trtllm + + - name: Setup SGLang backend + if: steps.filter.outputs.skip != 'true' && !matrix.variant.setup_vllm && !matrix.variant.setup_trtllm + uses: ./.github/actions/setup-sglang + + - name: Download wheel artifact + if: steps.filter.outputs.skip != 'true' + uses: actions/download-artifact@v7 + with: + name: smg-wheel + path: wheel/ + + - name: Install wheel and test dependencies + if: steps.filter.outputs.skip != 'true' + run: | + pip uninstall -y smg || true + pip install wheel/*.whl + bash scripts/ci_install_e2e_deps.sh ${{ matrix.variant.extra_deps }} + + - name: Run benchmark if: steps.filter.outputs.skip != 'true' env: - SGLANG_USE_LATEST_TAG: "1" + HF_TOKEN: ${{ secrets.HF_TOKEN }} + GPU_TYPE: H100 + E2E_MODEL_TP_OVERRIDES: '{"meta-llama/Llama-3.1-8B-Instruct":1,"meta-llama/Llama-3.2-1B-Instruct":1,"Qwen/Qwen2.5-7B-Instruct":1,"Qwen/Qwen2.5-14B-Instruct":1,"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B":1,"Qwen/Qwen3-30B-A3B":1,"mistralai/Mistral-7B-Instruct-v0.3":1,"openai/gpt-oss-20b":1}' run: | - bash scripts/ci_setup_python_venv.sh - source .venv/bin/activate - if [ "${{ matrix.variant.runtime }}" == "vllm" ]; then - bash scripts/ci_install_vllm.sh - else - bash scripts/ci_install_sglang.sh + bash scripts/ci_killall_sglang.sh "nuk_gpus" + + K_FILTER="${{ matrix.model.test_class }}" + if [ "${{ matrix.variant.grpc_only }}" == "true" ]; then + K_FILTER="${{ matrix.model.test_class }} and grpc" fi + E2E_RUNTIME=${{ matrix.variant.runtime }} \ + ROUTER_LOCAL_MODEL_PATH="/raid/models" \ + pytest e2e_test/benchmarks/test_nightly_perf.py \ + -k "$K_FILTER" \ + -s -vv -o log_cli=true --log-cli-level=INFO + + - name: Upload benchmark results + if: always() && steps.filter.outputs.skip != 'true' + uses: actions/upload-artifact@v6 + with: + name: nightly-${{ matrix.model.slug }}-multi-${{ matrix.variant.id }}-${{ github.run_id }} + path: nightly_*/ + retention-days: 90 + + - name: Cleanup + if: always() && steps.filter.outputs.skip != 'true' + run: bash scripts/ci_killall_sglang.sh || true + + # --------------------------------------------------------------------------- + # Single-worker (H200): models requiring 8-gpu-h200 runners. + # Separate job so H200 pool isn't blocked by H100 max-parallel limits. + # --------------------------------------------------------------------------- + + single-worker-h200: + name: "${{ matrix.model.id }} / single-${{ matrix.variant.id }}" + needs: build-wheel + if: ${{ !cancelled() }} + runs-on: ["8-gpu-h200"] + timeout-minutes: 1440 + strategy: + fail-fast: false + max-parallel: 2 + matrix: + model: + - { id: meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8, slug: meta-llama-Llama-4-Maverick-17B-128E-Instruct-FP8, test_class: TestNightlyLlama4MaverickSingle } + variant: + - { id: sglang, runtime: sglang, grpc_only: "false", setup_vllm: false, setup_trtllm: false, extra_deps: "genai-bench" } + - { id: vllm, runtime: vllm, grpc_only: "true", setup_vllm: true, setup_trtllm: false, extra_deps: "genai-bench" } + + steps: + - name: Check filters + id: filter + run: | + MODELS="${{ github.event.inputs.models || 'all' }}" + RUNTIME="${{ github.event.inputs.runtime || 'all' }}" + SKIP="false" + if [ "$MODELS" != "all" ] && ! echo ",$MODELS," | grep -Fq ",${{ matrix.model.id }},"; then + SKIP="true" + fi + if [ "$RUNTIME" != "all" ] && [ "$RUNTIME" != "${{ matrix.variant.id }}" ]; then + SKIP="true" + fi + echo "skip=$SKIP" >> "$GITHUB_OUTPUT" + + - name: Checkout code + if: steps.filter.outputs.skip != 'true' + uses: actions/checkout@v6 + + - name: Setup Python + if: steps.filter.outputs.skip != 'true' + uses: actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Setup vLLM backend + if: steps.filter.outputs.skip != 'true' && matrix.variant.setup_vllm + uses: ./.github/actions/setup-vllm + + - name: Setup TRT-LLM backend + if: steps.filter.outputs.skip != 'true' && matrix.variant.setup_trtllm + uses: ./.github/actions/setup-trtllm + + - name: Setup SGLang backend + if: steps.filter.outputs.skip != 'true' && !matrix.variant.setup_vllm && !matrix.variant.setup_trtllm + uses: ./.github/actions/setup-sglang + - name: Download wheel artifact if: steps.filter.outputs.skip != 'true' uses: actions/download-artifact@v7 @@ -226,12 +354,13 @@ jobs: run: | pip uninstall -y smg || true pip install wheel/*.whl - bash scripts/ci_install_e2e_deps.sh genai-bench + bash scripts/ci_install_e2e_deps.sh ${{ matrix.variant.extra_deps }} - name: Run benchmark if: steps.filter.outputs.skip != 'true' env: HF_TOKEN: ${{ secrets.HF_TOKEN }} + GPU_TYPE: H200 run: | bash scripts/ci_killall_sglang.sh "nuk_gpus" @@ -250,7 +379,7 @@ jobs: if: always() && steps.filter.outputs.skip != 'true' uses: actions/upload-artifact@v6 with: - name: nightly-${{ matrix.model.id }}-multi-${{ matrix.variant.id }}-${{ github.run_id }} + name: nightly-${{ matrix.model.slug }}-single-${{ matrix.variant.id }}-${{ github.run_id }} path: nightly_*/ retention-days: 90 @@ -258,12 +387,33 @@ jobs: if: always() && steps.filter.outputs.skip != 'true' run: bash scripts/ci_killall_sglang.sh || true + # --------------------------------------------------------------------------- + # Multi-worker (H200): models requiring 8-gpu-h200 runners. + # --------------------------------------------------------------------------- + + # multi-worker-h200: + # name: "${{ matrix.model.id }} / multi-${{ matrix.variant.id }}" + # needs: build-wheel + # if: ${{ !cancelled() }} + # runs-on: ["8-gpu-h200"] + # timeout-minutes: 1440 + # strategy: + # fail-fast: false + # max-parallel: 2 + # matrix: + # model: + # - { id: meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8, slug: meta-llama-Llama-4-Maverick-17B-128E-Instruct-FP8, test_class: TestNightlyLlama4MaverickMulti } + # variant: + # - { id: sglang, runtime: sglang, grpc_only: "false", setup_vllm: false, setup_trtllm: false, extra_deps: "genai-bench" } + # - { id: vllm, runtime: vllm, grpc_only: "true", setup_vllm: true, setup_trtllm: false, extra_deps: "genai-bench" } + # # tp=8, keep disabled for nightly + # --------------------------------------------------------------------------- # Summary # --------------------------------------------------------------------------- summarize-benchmarks: - needs: [single-worker, multi-worker] + needs: [single-worker, multi-worker, single-worker-h200] runs-on: ubuntu-latest if: always() permissions: diff --git a/docs/assets/stylesheets/extra.css b/docs/assets/stylesheets/extra.css index dbbfde9de0..9a41321436 100644 --- a/docs/assets/stylesheets/extra.css +++ b/docs/assets/stylesheets/extra.css @@ -58,7 +58,7 @@ /* Hero title: dark gradient text instead of white. background shorthand resets background-clip and background-size, - so all properties must be re-declared here. */ + so all properties must be redeclared here. */ [data-md-color-scheme="default"] .md-typeset .hero h1 { background: linear-gradient( 135deg, @@ -130,7 +130,7 @@ /* Stat values: dark gradient instead of white. background shorthand resets background-clip and background-size, - so all properties must be re-declared here. */ + so all properties must be redeclared here. */ [data-md-color-scheme="default"] .md-typeset .stat-value { background: linear-gradient(135deg, var(--md-code-fg-color) 0%, var(--mesh-purple) 50%, var(--mesh-cyan) 100%); background-size: 200% 200%; diff --git a/e2e_test/benchmarks/nightly_summarize.py b/e2e_test/benchmarks/nightly_summarize.py index d061223341..546ca9919b 100755 --- a/e2e_test/benchmarks/nightly_summarize.py +++ b/e2e_test/benchmarks/nightly_summarize.py @@ -61,17 +61,24 @@ def parse_folder_name(folder_name: str) -> dict: """Parse experiment info from folder name. Expected patterns (newest to oldest): - - nightly_llama-8b_http_sglang_single -> model=llama-8b, protocol=http, runtime=sglang, worker_type=single - - nightly_llama-8b_grpc_vllm_multi -> model=llama-8b, protocol=grpc, runtime=vllm, worker_type=multi - - nightly_llama-8b_http_sglang -> model=llama-8b, protocol=http, runtime=sglang (worker_type=single) - - nightly_llama-8b_http (legacy) -> model=llama-8b, protocol=http + - nightly_meta-llama/Llama-3.1-8B-Instruct_http_sglang_single + -> model=meta-llama/Llama-3.1-8B-Instruct, protocol=http, + runtime=sglang, worker_type=single + - nightly_meta-llama/Llama-3.1-8B-Instruct_grpc_vllm_multi + -> model=meta-llama/Llama-3.1-8B-Instruct, protocol=grpc, + runtime=vllm, worker_type=multi + - nightly_meta-llama/Llama-3.1-8B-Instruct_http_sglang + -> model=meta-llama/Llama-3.1-8B-Instruct, protocol=http, + runtime=sglang (worker_type=single) + - nightly_meta-llama/Llama-3.1-8B-Instruct_http (legacy) + -> model=meta-llama/Llama-3.1-8B-Instruct, protocol=http """ info = {"model": "unknown", "protocol": "unknown", "runtime": None, "worker_type": "single"} # Remove nightly_ prefix name = folder_name.replace("nightly_", "") - # Try newest format: model_protocol_runtime_worker_type (e.g., llama-8b_grpc_sglang_single) + # Try newest format: model_protocol_runtime_worker_type (e.g., meta-llama/Llama-3.1-8B-Instruct_grpc_sglang_single) parts = name.rsplit("_", 3) if len(parts) >= 4 and parts[-1] in ("single", "multi") and parts[-2] in ("sglang", "vllm"): diff --git a/e2e_test/benchmarks/test_go_bindings_perf.py b/e2e_test/benchmarks/test_go_bindings_perf.py index b25dad6c8f..82cc8461ab 100644 --- a/e2e_test/benchmarks/test_go_bindings_perf.py +++ b/e2e_test/benchmarks/test_go_bindings_perf.py @@ -17,7 +17,7 @@ @pytest.mark.e2e @pytest.mark.workers(count=4) @pytest.mark.gateway(policy="round_robin") -@pytest.mark.model("llama-1b") +@pytest.mark.model("meta-llama/Llama-3.2-1B-Instruct") class TestGoBindingsPerf: """Performance benchmark for Go OAI server. diff --git a/e2e_test/benchmarks/test_nightly_perf.py b/e2e_test/benchmarks/test_nightly_perf.py index 6e1a744ded..852c0c8951 100644 --- a/e2e_test/benchmarks/test_nightly_perf.py +++ b/e2e_test/benchmarks/test_nightly_perf.py @@ -1,6 +1,6 @@ """Nightly comprehensive benchmark tests. -Runs all models on an 8-GPU H200 node using genai-bench default scenarios +Runs models on k8s H100 or H200 runners using genai-bench default scenarios and concurrency levels. No performance thresholds — results are uploaded as artifacts for tracking over time. @@ -54,8 +54,9 @@ def _run_nightly(setup_backend, genai_bench_runner, model_id, worker_count=1, ** # Total GPU count = tp * workers gpu_count = tp_per_worker * worker_count - # Include runtime and worker type in folder name - experiment_folder = f"nightly_{model_id}_{backend}_{runtime}_{worker_type}" + # Keep folder names filesystem-safe while retaining the canonical HF model id in metadata. + safe_model_id = model_id.replace("/", "__") + experiment_folder = f"nightly_{safe_model_id}_{backend}_{runtime}_{worker_type}" if _TEST_MODE: genai_bench_runner( @@ -95,15 +96,21 @@ def _run_nightly(setup_backend, genai_bench_runner, model_id, worker_count=1, ** # --------------------------------------------------------------------------- _NIGHTLY_MODELS: list[tuple[str, str, int, list[str], dict]] = [ - ("llama-8b", "Llama8b", 8, ["http", "grpc"], {}), - ("llama-1b", "Llama1b", 8, ["http", "grpc"], {}), - ("qwen-7b", "Qwen7b", 8, ["http", "grpc"], {}), - ("qwen-14b", "Qwen14b", 4, ["http", "grpc"], {}), - ("deepseek-7b", "Deepseek7b", 8, ["http", "grpc"], {}), - ("qwen-30b", "Qwen30b", 2, ["http", "grpc"], {}), - ("mistral-7b", "Mistral7b", 8, ["http", "grpc"], {}), - ("gpt-oss", "GptOss", 4, ["http", "grpc"], {}), - ("llama-4-maverick-17b", "Llama4Maverick", 1, ["http", "grpc"], {}), + ("meta-llama/Llama-3.1-8B-Instruct", "Llama8b", 4, ["http", "grpc"], {}), + ("meta-llama/Llama-3.2-1B-Instruct", "Llama1b", 4, ["http", "grpc"], {}), + ("Qwen/Qwen2.5-7B-Instruct", "Qwen7b", 4, ["http", "grpc"], {}), + ("Qwen/Qwen2.5-14B-Instruct", "Qwen14b", 4, ["http", "grpc"], {}), + ("deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", "Deepseek7b", 4, ["http", "grpc"], {}), + ("Qwen/Qwen3-30B-A3B", "Qwen30b", 4, ["http", "grpc"], {}), + ("mistralai/Mistral-7B-Instruct-v0.3", "Mistral7b", 4, ["http", "grpc"], {}), + ("openai/gpt-oss-20b", "GptOss", 4, ["http", "grpc"], {}), + ( + "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8", + "Llama4Maverick", + 1, + ["http", "grpc"], + {}, + ), ] diff --git a/e2e_test/bindings_go/test_go_oai_server.py b/e2e_test/bindings_go/test_go_oai_server.py index 34d63c3aea..8d020e1d66 100644 --- a/e2e_test/bindings_go/test_go_oai_server.py +++ b/e2e_test/bindings_go/test_go_oai_server.py @@ -19,7 +19,7 @@ logger = logging.getLogger(__name__) -@pytest.mark.model("llama-1b") +@pytest.mark.model("meta-llama/Llama-3.2-1B-Instruct") class TestGoOAIServerBasic: """Basic tests for Go OAI server functionality.""" @@ -98,7 +98,7 @@ def test_streaming_multiple_messages(self, go_openai_client, go_oai_server): logger.info(f"Response to follow-up: {full_content}") -@pytest.mark.model("llama-1b") +@pytest.mark.model("meta-llama/Llama-3.2-1B-Instruct") @pytest.mark.xfail( reason="Llama-3.2-1B-Instruct doesn't reliably support tool calling with tool_choice=required", strict=False, # Allow tests to pass if model happens to work @@ -107,7 +107,7 @@ class TestGoOAIServerFunctionCalling: """Tests for function calling through Go OAI server. Note: Function calling requires the model to support tool use. - The llama-1b model may not reliably support tool_choice='required'. + The meta-llama/Llama-3.2-1B-Instruct model may not reliably support tool_choice='required'. These tests are marked as xfail to allow CI to pass while still testing the Go OAI server's tool calling proxy functionality. """ @@ -240,7 +240,7 @@ def test_function_calling_tool_choice_none(self, go_openai_client, go_oai_server assert tool_calls is None or len(tool_calls) == 0 -@pytest.mark.model("llama-1b") +@pytest.mark.model("meta-llama/Llama-3.2-1B-Instruct") class TestGoOAIServerParameters: """Tests for various generation parameters through Go OAI server.""" @@ -304,7 +304,7 @@ def test_top_p_parameter(self, go_openai_client, go_oai_server): assert response.choices[0].message.content is not None -@pytest.mark.model("llama-1b") +@pytest.mark.model("meta-llama/Llama-3.2-1B-Instruct") class TestGoOAIServerResponseValidation: """Tests for response field validation.""" @@ -465,7 +465,7 @@ def test_finish_reason_length(self, go_openai_client, go_oai_server): logger.info(f"Finish reason: {response.choices[0].finish_reason}") -@pytest.mark.model("llama-1b") +@pytest.mark.model("meta-llama/Llama-3.2-1B-Instruct") class TestGoOAIServerConcurrency: """Tests for concurrent request handling.""" @@ -536,7 +536,7 @@ def make_streaming_request(i: int): logger.info(f"Stream {i}: {content[:30]}...") -@pytest.mark.model("llama-1b") +@pytest.mark.model("meta-llama/Llama-3.2-1B-Instruct") class TestGoOAIServerEdgeCases: """Tests for edge cases and error handling.""" @@ -614,7 +614,7 @@ def test_system_message_only(self, go_openai_client, go_oai_server): logger.info(f"Pirate response: {response.choices[0].message.content}") -@pytest.mark.model("llama-1b") +@pytest.mark.model("meta-llama/Llama-3.2-1B-Instruct") @pytest.mark.xfail( reason="Go OAI server does not currently support n > 1 (multiple choices)", strict=False, diff --git a/e2e_test/chat_completions/test_enable_thinking.py b/e2e_test/chat_completions/test_enable_thinking.py index de79949a1b..ebd801c780 100644 --- a/e2e_test/chat_completions/test_enable_thinking.py +++ b/e2e_test/chat_completions/test_enable_thinking.py @@ -24,7 +24,7 @@ # ============================================================================= -@pytest.mark.model("qwen-30b") +@pytest.mark.model("Qwen/Qwen3-30B-A3B") @pytest.mark.gateway(extra_args=["--reasoning-parser", "qwen3", "--history-backend", "memory"]) @pytest.mark.parametrize("setup_backend", ["grpc"], indirect=True) class TestEnableThinking: diff --git a/e2e_test/chat_completions/test_function_calling.py b/e2e_test/chat_completions/test_function_calling.py index b03679d14f..b184d28b6f 100644 --- a/e2e_test/chat_completions/test_function_calling.py +++ b/e2e_test/chat_completions/test_function_calling.py @@ -102,7 +102,7 @@ @pytest.mark.skip_for_runtime( "trtllm", reason="TRT-LLM does not support guided decoding (json_schema)" ) -@pytest.mark.model("llama-1b") +@pytest.mark.model("meta-llama/Llama-3.2-1B-Instruct") @pytest.mark.gateway(extra_args=["--tool-call-parser", "llama", "--history-backend", "memory"]) @pytest.mark.parametrize("setup_backend", ["grpc"], indirect=True) class TestOpenAIServerFunctionCalling: @@ -719,7 +719,7 @@ def test_streaming_multiple_choices_without_tools(self, setup_backend): # ============================================================================= -@pytest.mark.model("llama-8b") +@pytest.mark.model("meta-llama/Llama-3.1-8B-Instruct") @pytest.mark.gateway(extra_args=["--tool-call-parser", "pythonic", "--history-backend", "memory"]) @pytest.mark.parametrize("setup_backend", ["grpc"], indirect=True) class TestOpenAIPythonicFunctionCalling: @@ -1514,7 +1514,7 @@ def test_conflicting_defs_required_tool_choice(self, setup_backend): @pytest.mark.skip_for_runtime( "trtllm", reason="TRT-LLM does not support guided decoding (json_schema)" ) -@pytest.mark.model("llama-1b") +@pytest.mark.model("meta-llama/Llama-3.2-1B-Instruct") @pytest.mark.gateway(extra_args=["--tool-call-parser", "llama", "--history-backend", "memory"]) @pytest.mark.parametrize("setup_backend", ["grpc"], indirect=True) class TestToolChoiceLlama(_TestToolChoiceBase): @@ -1535,7 +1535,7 @@ class TestToolChoiceLlama(_TestToolChoiceBase): @pytest.mark.skip_for_runtime( "trtllm", reason="TRT-LLM does not support guided decoding (json_schema)" ) -@pytest.mark.model("qwen-7b") +@pytest.mark.model("Qwen/Qwen2.5-7B-Instruct") @pytest.mark.gateway(extra_args=["--tool-call-parser", "qwen", "--history-backend", "memory"]) @pytest.mark.parametrize("setup_backend", ["grpc"], indirect=True) class TestToolChoiceQwen(_TestToolChoiceBase): @@ -1553,7 +1553,7 @@ class TestToolChoiceQwen(_TestToolChoiceBase): @pytest.mark.skip_for_runtime( "trtllm", reason="TRT-LLM does not support guided decoding (json_schema)" ) -@pytest.mark.model("mistral-7b") +@pytest.mark.model("mistralai/Mistral-7B-Instruct-v0.3") @pytest.mark.gateway(extra_args=["--tool-call-parser", "mistral", "--history-backend", "memory"]) @pytest.mark.parametrize("setup_backend", ["grpc"], indirect=True) class TestToolChoiceMistral(_TestToolChoiceBase): diff --git a/e2e_test/chat_completions/test_openai_server.py b/e2e_test/chat_completions/test_openai_server.py index c7c36bc31e..699f634026 100644 --- a/e2e_test/chat_completions/test_openai_server.py +++ b/e2e_test/chat_completions/test_openai_server.py @@ -21,7 +21,7 @@ # ============================================================================= -@pytest.mark.model("llama-8b") +@pytest.mark.model("meta-llama/Llama-3.1-8B-Instruct") @pytest.mark.gateway(extra_args=["--history-backend", "memory"]) @pytest.mark.parametrize("setup_backend", ["grpc"], indirect=True) class TestChatCompletion: @@ -318,7 +318,7 @@ def _run_chat_completion_stream(self, client, model, logprobs, parallel_sample_n # ============================================================================= -@pytest.mark.model("gpt-oss") +@pytest.mark.model("openai/gpt-oss-20b") @pytest.mark.gateway(extra_args=["--reasoning-parser=gpt-oss", "--history-backend", "memory"]) class TestChatCompletionGptOss(TestChatCompletion): """Tests for chat completions API with GPT-OSS model (Harmony). diff --git a/e2e_test/chat_completions/test_reasoning_content.py b/e2e_test/chat_completions/test_reasoning_content.py index be7a7ab722..bb90e8e8c1 100644 --- a/e2e_test/chat_completions/test_reasoning_content.py +++ b/e2e_test/chat_completions/test_reasoning_content.py @@ -22,7 +22,7 @@ @pytest.mark.skip_for_runtime( "trtllm", reason="TRT-LLM does not support reasoning content extraction" ) -@pytest.mark.model("deepseek-7b") +@pytest.mark.model("deepseek-ai/DeepSeek-R1-Distill-Qwen-7B") @pytest.mark.gateway( extra_args=["--reasoning-parser", "deepseek_r1", "--history-backend", "memory"] ) diff --git a/e2e_test/chat_completions/test_validation.py b/e2e_test/chat_completions/test_validation.py index 99bbdf39d1..ea23a4a36c 100644 --- a/e2e_test/chat_completions/test_validation.py +++ b/e2e_test/chat_completions/test_validation.py @@ -40,7 +40,7 @@ def get_tokenizer(model_path: str): @pytest.mark.skip_for_runtime("trtllm", reason="TRT-LLM does not support ignore_eos parameter") -@pytest.mark.model("llama-8b") +@pytest.mark.model("meta-llama/Llama-3.1-8B-Instruct") @pytest.mark.gateway(extra_args=["--history-backend", "memory"]) @pytest.mark.parametrize("setup_backend", ["grpc"], indirect=True) class TestIgnoreEOS: @@ -108,7 +108,7 @@ def test_ignore_eos(self, setup_backend): # ============================================================================= -@pytest.mark.model("llama-8b") +@pytest.mark.model("meta-llama/Llama-3.1-8B-Instruct") @pytest.mark.gateway(extra_args=["--history-backend", "memory"]) @pytest.mark.parametrize("setup_backend", ["grpc"], indirect=True) class TestLargeMaxNewTokens: diff --git a/e2e_test/conftest.py b/e2e_test/conftest.py index d1e83d0dd8..5e03ac6526 100644 --- a/e2e_test/conftest.py +++ b/e2e_test/conftest.py @@ -20,7 +20,7 @@ Specify which model to use for the test. Args: - name: Model ID from MODEL_SPECS (e.g., "llama-8b", "qwen-7b") + name: Model ID from MODEL_SPECS (e.g., "meta-llama/Llama-3.1-8B-Instruct", "Qwen/Qwen2.5-7B-Instruct") GPU Resource Management: When GPUs are limited (e.g., 4 GPUs, 6 models), the model pool uses @@ -31,8 +31,8 @@ 3. The needed model is then launched on-demand Examples: - @pytest.mark.model("llama-8b") - @pytest.mark.model("qwen-72b") + @pytest.mark.model("meta-llama/Llama-3.1-8B-Instruct") + @pytest.mark.model("Qwen/Qwen2.5-14B-Instruct") @pytest.mark.workers(count=1, prefill=None, decode=None) Configure worker topology for the test. @@ -95,7 +95,7 @@ def test_chat(self, setup_backend): Test with specific model and multiple backends: @pytest.mark.e2e - @pytest.mark.model("qwen-7b") + @pytest.mark.model("Qwen/Qwen2.5-7B-Instruct") @pytest.mark.parametrize("setup_backend", ["grpc", "http"], indirect=True) class TestQwen: def test_generate(self, setup_backend): diff --git a/e2e_test/fixtures/hooks.py b/e2e_test/fixtures/hooks.py index ef46e6aa85..71bb59cb98 100644 --- a/e2e_test/fixtures/hooks.py +++ b/e2e_test/fixtures/hooks.py @@ -88,7 +88,6 @@ def pytest_collection_modifyitems( from infra import ( DEFAULT_MODEL, - MODEL_SPECS, PARAM_MODEL, PARAM_SETUP_BACKEND, ConnectionMode, @@ -108,9 +107,13 @@ def track_worker( def calculate_test_gpus(model_id: str, prefill: int, decode: int, regular: int) -> int: """Calculate GPU requirement for a single test.""" - if model_id not in MODEL_SPECS: - return 0 - tp = MODEL_SPECS[model_id].get("tp", 1) + from infra.model_specs import get_model_spec + + try: + spec = get_model_spec(model_id) + except KeyError as exc: + raise pytest.UsageError(f"Unknown model in test markers: {model_id}") from exc + tp = spec.get("tp", 1) return tp * (prefill + decode + regular) # Filter items based on -k expression to only scan tests that will actually run diff --git a/e2e_test/fixtures/pool.py b/e2e_test/fixtures/pool.py index 066438fdc1..809c3d5ba7 100644 --- a/e2e_test/fixtures/pool.py +++ b/e2e_test/fixtures/pool.py @@ -63,7 +63,7 @@ def model_pool(request: pytest.FixtureRequest) -> ModelPool: - @pytest.mark.workers(prefill=N, decode=N) for PD workers Environment variable overrides: - - E2E_MODELS: Comma-separated model IDs (e.g., "llama-8b,qwen-7b") + - E2E_MODELS: Comma-separated model IDs (e.g., "meta-llama/Llama-3.1-8B-Instruct,Qwen/Qwen2.5-7B-Instruct") - E2E_BACKENDS: Comma-separated backends (e.g., "grpc,http") - SKIP_MODEL_POOL: Set to "1" to skip worker startup """ @@ -174,7 +174,7 @@ def model_client( """Get OpenAI client for the model specified by @pytest.mark.model(). Usage: - @pytest.mark.model("llama-8b") + @pytest.mark.model("meta-llama/Llama-3.1-8B-Instruct") def test_chat(model_client): response = model_client.chat.completions.create(...) """ @@ -214,7 +214,7 @@ def model_base_url( """Get the base URL for the model specified by @pytest.mark.model(). Usage: - @pytest.mark.model("llama-8b") + @pytest.mark.model("meta-llama/Llama-3.1-8B-Instruct") def test_direct_http(model_base_url): response = httpx.get(f"{model_base_url}/health") """ diff --git a/e2e_test/infra/constants.py b/e2e_test/infra/constants.py index 1b368bff38..960d096f9e 100644 --- a/e2e_test/infra/constants.py +++ b/e2e_test/infra/constants.py @@ -41,7 +41,7 @@ class Runtime(StrEnum): PARAM_MODEL = "model" # Default model -DEFAULT_MODEL = "llama-8b" +DEFAULT_MODEL = "meta-llama/Llama-3.1-8B-Instruct" # Default runtime for gRPC tests DEFAULT_RUNTIME = "sglang" diff --git a/e2e_test/infra/model_pool.py b/e2e_test/infra/model_pool.py index 86ac91313e..eac9262c5e 100644 --- a/e2e_test/infra/model_pool.py +++ b/e2e_test/infra/model_pool.py @@ -45,10 +45,10 @@ class WorkerIdentity: Each worker is uniquely identified by (model_id, mode, worker_type, index). For example: - - llama-8b:http (regular worker, index 0) - - llama-8b:http:prefill_0 (first prefill worker) - - llama-8b:http:prefill_1 (second prefill worker) - - llama-8b:http:decode_0 (first decode worker) + - meta-llama/Llama-3.1-8B-Instruct:http (regular worker, index 0) + - meta-llama/Llama-3.1-8B-Instruct:http:prefill_0 (first prefill worker) + - meta-llama/Llama-3.1-8B-Instruct:http:prefill_1 (second prefill worker) + - meta-llama/Llama-3.1-8B-Instruct:http:decode_0 (first decode worker) Frozen/hashable so it can be used in sets and as dict keys for deduplication. """ @@ -102,7 +102,7 @@ class ModelInstance: port: int process: subprocess.Popen gpu_slot: GPUSlot | None - key: str # Unique instance key (e.g., "llama-8b:http:prefill_0") + key: str # Unique instance key (e.g., "meta-llama/Llama-3.1-8B-Instruct:http:prefill_0") worker_type: WorkerType = WorkerType.REGULAR bootstrap_port: int | None = None # For prefill workers in PD mode last_used: float = 0.0 # Timestamp for MRU eviction @@ -310,8 +310,8 @@ class ModelPool: - The needed model is then launched on-demand Instance keys: - - Regular workers: "model_id:mode" (e.g., "llama-8b:http") - - PD workers: "model_id:mode:worker_type" (e.g., "llama-8b:http:prefill") + - Regular workers: "model_id:mode" (e.g., "meta-llama/Llama-3.1-8B-Instruct:http") + - PD workers: "model_id:mode:worker_type" (e.g., "meta-llama/Llama-3.1-8B-Instruct:http:prefill") Limitations: - Currently one worker instance per (model_id, mode) combination @@ -320,8 +320,8 @@ class ModelPool: Usage: pool = ModelPool() - pool.startup(requirements=[("llama-8b", ConnectionMode.HTTP)]) - instance = pool.get("llama-8b", "http") # Pre-launched or on-demand + pool.startup(requirements=[("meta-llama/Llama-3.1-8B-Instruct", ConnectionMode.HTTP)]) + instance = pool.get("meta-llama/Llama-3.1-8B-Instruct", "http") # Pre-launched or on-demand """ def __init__(self, allocator: GPUAllocator | None = None): @@ -807,7 +807,7 @@ def get( Caller MUST call release() on the instance when done. Args: - model_id: The model ID (e.g., "llama-8b") + model_id: The model ID (e.g., "meta-llama/Llama-3.1-8B-Instruct") mode: The mode (ConnectionMode.HTTP or ConnectionMode.GRPC, or string) worker_type: The worker type (REGULAR, PREFILL, DECODE). Defaults to REGULAR. wait_for_gpus: If True, wait for GPUs to become available when all diff --git a/e2e_test/infra/model_specs.py b/e2e_test/infra/model_specs.py index 47e1f06e5b..d5bc8e4574 100644 --- a/e2e_test/infra/model_specs.py +++ b/e2e_test/infra/model_specs.py @@ -9,6 +9,7 @@ from __future__ import annotations +import json import os # Environment variable for local model paths (CI uses local copies for speed) @@ -26,28 +27,28 @@ def _resolve_model_path(hf_path: str) -> str: MODEL_SPECS: dict[str, dict] = { # Primary chat model - used for most tests - "llama-8b": { + "meta-llama/Llama-3.1-8B-Instruct": { "model": _resolve_model_path("meta-llama/Llama-3.1-8B-Instruct"), "memory_gb": 16, "tp": 1, "features": ["chat", "streaming", "function_calling"], }, # Small model for quick tests - "llama-1b": { + "meta-llama/Llama-3.2-1B-Instruct": { "model": _resolve_model_path("meta-llama/Llama-3.2-1B-Instruct"), "memory_gb": 4, "tp": 1, "features": ["chat", "streaming", "tool_choice"], }, # Function calling specialist - "qwen-7b": { + "Qwen/Qwen2.5-7B-Instruct": { "model": _resolve_model_path("Qwen/Qwen2.5-7B-Instruct"), "memory_gb": 14, "tp": 1, "features": ["chat", "streaming", "function_calling", "pythonic_tools"], }, # Function calling specialist (larger, for Response API tests) - "qwen-14b": { + "Qwen/Qwen2.5-14B-Instruct": { "model": _resolve_model_path("Qwen/Qwen2.5-14B-Instruct"), "memory_gb": 28, "tp": 2, @@ -55,21 +56,21 @@ def _resolve_model_path(hf_path: str) -> str: "worker_args": ["--context-length=16384"], # Faster startup, prevents memory issues }, # Reasoning model - "deepseek-7b": { + "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B": { "model": _resolve_model_path("deepseek-ai/DeepSeek-R1-Distill-Qwen-7B"), "memory_gb": 14, "tp": 1, "features": ["chat", "streaming", "reasoning"], }, # Thinking/reasoning model (larger) - "qwen-30b": { + "Qwen/Qwen3-30B-A3B": { "model": _resolve_model_path("Qwen/Qwen3-30B-A3B"), "memory_gb": 60, "tp": 4, "features": ["chat", "streaming", "thinking", "reasoning"], }, # Mistral for function calling - "mistral-7b": { + "mistralai/Mistral-7B-Instruct-v0.3": { "model": _resolve_model_path("mistralai/Mistral-7B-Instruct-v0.3"), "memory_gb": 14, "tp": 1, @@ -83,14 +84,14 @@ def _resolve_model_path(hf_path: str) -> str: "features": ["embedding"], }, # GPT-OSS model (Harmony) - "gpt-oss": { + "openai/gpt-oss-20b": { "model": _resolve_model_path("openai/gpt-oss-20b"), "memory_gb": 40, "tp": 2, "features": ["chat", "streaming", "reasoning", "harmony"], }, # Llama-4-Maverick (17B with 128 experts, FP8) - Nightly benchmarks - "llama-4-maverick-17b": { + "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8": { "model": _resolve_model_path("meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8"), "memory_gb": 34, # ~1 byte per parameter for FP8 quantized 17B params "tp": 8, # Tensor parallelism across 8 GPUs @@ -123,7 +124,19 @@ def get_model_spec(model_id: str) -> dict: """Get spec for a specific model, raising KeyError if not found.""" if model_id not in MODEL_SPECS: raise KeyError(f"Unknown model: {model_id}. Available: {list(MODEL_SPECS.keys())}") - return MODEL_SPECS[model_id] + spec = dict(MODEL_SPECS[model_id]) + tp_overrides_json = os.environ.get("E2E_MODEL_TP_OVERRIDES") + if tp_overrides_json: + try: + tp_overrides = json.loads(tp_overrides_json) + if isinstance(tp_overrides, dict): + override = tp_overrides.get(model_id) + if isinstance(override, int) and override > 0: + spec["tp"] = override + except json.JSONDecodeError: + # Ignore malformed override config and fall back to canonical specs. + pass + return spec # Convenience groupings for test parametrization @@ -137,13 +150,15 @@ def get_model_spec(model_id: str) -> dict: # Default model path constants (for backward compatibility with existing tests) # ============================================================================= -DEFAULT_MODEL_PATH = MODEL_SPECS["llama-8b"]["model"] -DEFAULT_SMALL_MODEL_PATH = MODEL_SPECS["llama-1b"]["model"] -DEFAULT_REASONING_MODEL_PATH = MODEL_SPECS["deepseek-7b"]["model"] -DEFAULT_ENABLE_THINKING_MODEL_PATH = MODEL_SPECS["qwen-30b"]["model"] -DEFAULT_QWEN_FUNCTION_CALLING_MODEL_PATH = MODEL_SPECS["qwen-7b"]["model"] -DEFAULT_MISTRAL_FUNCTION_CALLING_MODEL_PATH = MODEL_SPECS["mistral-7b"]["model"] -DEFAULT_GPT_OSS_MODEL_PATH = MODEL_SPECS["gpt-oss"]["model"] +DEFAULT_MODEL_PATH = MODEL_SPECS["meta-llama/Llama-3.1-8B-Instruct"]["model"] +DEFAULT_SMALL_MODEL_PATH = MODEL_SPECS["meta-llama/Llama-3.2-1B-Instruct"]["model"] +DEFAULT_REASONING_MODEL_PATH = MODEL_SPECS["deepseek-ai/DeepSeek-R1-Distill-Qwen-7B"]["model"] +DEFAULT_ENABLE_THINKING_MODEL_PATH = MODEL_SPECS["Qwen/Qwen3-30B-A3B"]["model"] +DEFAULT_QWEN_FUNCTION_CALLING_MODEL_PATH = MODEL_SPECS["Qwen/Qwen2.5-7B-Instruct"]["model"] +DEFAULT_MISTRAL_FUNCTION_CALLING_MODEL_PATH = MODEL_SPECS["mistralai/Mistral-7B-Instruct-v0.3"][ + "model" +] +DEFAULT_GPT_OSS_MODEL_PATH = MODEL_SPECS["openai/gpt-oss-20b"]["model"] DEFAULT_EMBEDDING_MODEL_PATH = MODEL_SPECS["embedding"]["model"] diff --git a/e2e_test/responses/test_builtin_tools.py b/e2e_test/responses/test_builtin_tools.py index 8f6314a64b..5fff91fdb3 100644 --- a/e2e_test/responses/test_builtin_tools.py +++ b/e2e_test/responses/test_builtin_tools.py @@ -130,7 +130,7 @@ def gateway_with_mcp_config_grpc(require_brave_server, mcp_config_file, model_po # Get a gRPC worker from the pool try: - instance = model_pool.get("gpt-oss", ConnectionMode.GRPC) + instance = model_pool.get("openai/gpt-oss-20b", ConnectionMode.GRPC) except (KeyError, RuntimeError) as e: pytest.skip(f"gRPC worker not available: {e}") @@ -251,7 +251,7 @@ def test_mixed_builtin_and_function_tools(self, setup_backend): @pytest.mark.e2e -@pytest.mark.model("gpt-oss") +@pytest.mark.model("openai/gpt-oss-20b") @pytest.mark.gateway(extra_args=["--reasoning-parser=gpt-oss", "--history-backend", "memory"]) @pytest.mark.parametrize("setup_backend", ["grpc"], indirect=True) class TestBuiltinToolsLocalBackend: diff --git a/e2e_test/responses/test_state_management.py b/e2e_test/responses/test_state_management.py index 1a2dc45e0c..7a0d03160e 100644 --- a/e2e_test/responses/test_state_management.py +++ b/e2e_test/responses/test_state_management.py @@ -155,7 +155,7 @@ def test_mutually_exclusive_parameters(self, setup_backend): @pytest.mark.e2e -@pytest.mark.model("qwen-14b") +@pytest.mark.model("Qwen/Qwen2.5-14B-Instruct") @pytest.mark.gateway(extra_args=["--tool-call-parser", "qwen", "--history-backend", "memory"]) @pytest.mark.parametrize("setup_backend", ["grpc"], indirect=True) class TestStateManagementLocal: @@ -250,7 +250,7 @@ def test_mutually_exclusive_parameters(self, setup_backend): @pytest.mark.e2e -@pytest.mark.model("gpt-oss") +@pytest.mark.model("openai/gpt-oss-20b") @pytest.mark.gateway(extra_args=["--reasoning-parser=gpt-oss", "--history-backend", "memory"]) @pytest.mark.parametrize("setup_backend", ["grpc"], indirect=True) class TestStateManagementHarmony: diff --git a/e2e_test/responses/test_streaming_events.py b/e2e_test/responses/test_streaming_events.py index a8f9253666..5d71f8192a 100644 --- a/e2e_test/responses/test_streaming_events.py +++ b/e2e_test/responses/test_streaming_events.py @@ -23,7 +23,7 @@ @pytest.mark.e2e -@pytest.mark.model("qwen-14b") +@pytest.mark.model("Qwen/Qwen2.5-14B-Instruct") @pytest.mark.gateway(extra_args=["--tool-call-parser", "qwen", "--history-backend", "memory"]) @pytest.mark.parametrize("setup_backend", ["grpc"], indirect=True) class TestStreamingEventsLocal: @@ -106,7 +106,7 @@ def test_output_item_event_emitted(self, setup_backend): @pytest.mark.e2e -@pytest.mark.model("gpt-oss") +@pytest.mark.model("openai/gpt-oss-20b") @pytest.mark.gateway(extra_args=["--reasoning-parser=gpt-oss", "--history-backend", "memory"]) @pytest.mark.parametrize("setup_backend", ["grpc"], indirect=True) class TestStreamingEventsHarmony: diff --git a/e2e_test/responses/test_structured_output.py b/e2e_test/responses/test_structured_output.py index 72d3eddd14..f4c1caad3b 100644 --- a/e2e_test/responses/test_structured_output.py +++ b/e2e_test/responses/test_structured_output.py @@ -115,7 +115,7 @@ def test_structured_output_json_schema(self, setup_backend): @pytest.mark.e2e -@pytest.mark.model("gpt-oss") +@pytest.mark.model("openai/gpt-oss-20b") @pytest.mark.gateway(extra_args=["--reasoning-parser=gpt-oss", "--history-backend", "memory"]) @pytest.mark.parametrize("setup_backend", ["grpc"], indirect=True) class TestStructuredOutputHarmony: @@ -212,7 +212,7 @@ def test_structured_output_json_schema(self, setup_backend): @pytest.mark.e2e -@pytest.mark.model("qwen-14b") +@pytest.mark.model("Qwen/Qwen2.5-14B-Instruct") @pytest.mark.gateway(extra_args=["--tool-call-parser", "qwen", "--history-backend", "memory"]) @pytest.mark.parametrize("setup_backend", ["grpc"], indirect=True) class TestSimpleSchemaStructuredOutput: diff --git a/e2e_test/responses/test_tools_call.py b/e2e_test/responses/test_tools_call.py index 82077b521f..d845b51b19 100644 --- a/e2e_test/responses/test_tools_call.py +++ b/e2e_test/responses/test_tools_call.py @@ -423,7 +423,7 @@ def test_mcp_multi_server_tool_call_streaming(self, setup_backend): @pytest.mark.e2e -@pytest.mark.model("gpt-oss") +@pytest.mark.model("openai/gpt-oss-20b") @pytest.mark.gateway(extra_args=["--reasoning-parser=gpt-oss", "--history-backend", "memory"]) @pytest.mark.parametrize("setup_backend", ["grpc"], indirect=True) class TestToolChoiceHarmony: @@ -798,7 +798,7 @@ def test_mcp_multi_server_tool_call_streaming(self, setup_backend): @pytest.mark.e2e -@pytest.mark.model("qwen-14b") +@pytest.mark.model("Qwen/Qwen2.5-14B-Instruct") @pytest.mark.gateway(extra_args=["--tool-call-parser", "qwen", "--history-backend", "memory"]) @pytest.mark.parametrize("setup_backend", ["grpc"], indirect=True) class TestToolChoiceLocal: diff --git a/e2e_test/router/test_worker_api.py b/e2e_test/router/test_worker_api.py index d045c6ee54..265a71c6ca 100644 --- a/e2e_test/router/test_worker_api.py +++ b/e2e_test/router/test_worker_api.py @@ -90,7 +90,7 @@ def test_igw_start_empty(self, model_pool: ModelPool): def test_igw_add_worker(self, model_pool: ModelPool): """Test adding a worker to IGW gateway.""" - http_instance = model_pool.get("llama-8b", ConnectionMode.HTTP) + http_instance = model_pool.get("meta-llama/Llama-3.1-8B-Instruct", ConnectionMode.HTTP) gateway = Gateway() gateway.start(igw_mode=True) @@ -115,7 +115,7 @@ def test_igw_add_worker(self, model_pool: ModelPool): def test_igw_add_and_remove_worker(self, model_pool: ModelPool): """Test adding and removing workers dynamically.""" - http_instance = model_pool.get("llama-8b", ConnectionMode.HTTP) + http_instance = model_pool.get("meta-llama/Llama-3.1-8B-Instruct", ConnectionMode.HTTP) gateway = Gateway() gateway.start(igw_mode=True) @@ -142,8 +142,8 @@ def test_igw_add_and_remove_worker(self, model_pool: ModelPool): def test_igw_multiple_workers(self, model_pool: ModelPool): """Test adding multiple workers (HTTP + gRPC) to IGW gateway.""" - http_instance = model_pool.get("llama-8b", ConnectionMode.HTTP) - grpc_instance = model_pool.get("llama-8b", ConnectionMode.GRPC) + http_instance = model_pool.get("meta-llama/Llama-3.1-8B-Instruct", ConnectionMode.HTTP) + grpc_instance = model_pool.get("meta-llama/Llama-3.1-8B-Instruct", ConnectionMode.GRPC) gateway = Gateway() gateway.start(igw_mode=True) @@ -174,7 +174,7 @@ class TestDisableHealthCheck: def test_disable_health_check_workers_immediately_healthy(self, model_pool: ModelPool): """Test that workers are immediately healthy when health checks are disabled.""" - http_instance = model_pool.get("llama-8b", ConnectionMode.HTTP) + http_instance = model_pool.get("meta-llama/Llama-3.1-8B-Instruct", ConnectionMode.HTTP) gateway = Gateway() gateway.start( diff --git a/multimodal/scripts/generate_vision_golden.py b/multimodal/scripts/generate_vision_golden.py index e4d452dc16..0ea960068a 100755 --- a/multimodal/scripts/generate_vision_golden.py +++ b/multimodal/scripts/generate_vision_golden.py @@ -195,9 +195,7 @@ def generate_golden_llava_next(image_path: str, output_dir: str) -> dict: print("LlavaNextImageProcessor not available, skipping llava_next") return None - processor = LlavaNextImageProcessor.from_pretrained( - "llava-hf/llava-v1.6-mistral-7b-hf" - ) + processor = LlavaNextImageProcessor.from_pretrained("llava-hf/llava-v1.6-mistral-7b-hf") image = Image.open(image_path).convert("RGB") original_size = image.size @@ -330,9 +328,7 @@ def generate_golden_qwen3_vl(image_path: str, output_dir: str) -> dict: """ from transformers import AutoProcessor - processor = AutoProcessor.from_pretrained( - "Qwen/Qwen3-VL-8B-Instruct", trust_remote_code=True - ) + processor = AutoProcessor.from_pretrained("Qwen/Qwen3-VL-8B-Instruct", trust_remote_code=True) image = Image.open(image_path).convert("RGB") original_size = image.size @@ -654,9 +650,7 @@ def main(): parser = argparse.ArgumentParser( description="Generate golden outputs for vision processor testing" ) - parser.add_argument( - "--model", "-m", help="Specific model to generate (default: all)" - ) + parser.add_argument("--model", "-m", help="Specific model to generate (default: all)") parser.add_argument("--image", "-i", action="append", help="Specific image path(s)") parser.add_argument( "--output-dir",