Repository navigation
e2e: canonical HF model IDs and nightly benchmark runner split #373
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 5 commits
fb578b3
c87aa0d
ff6847c
7bc6e7d
1174dba
daf4569
6b0132e
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,8 +1,17 @@ | ||
| name: Nightly Benchmark | ||
|
|
||
| on: | ||
| schedule: | ||
| - cron: '0 0 * * 0' | ||
| # schedule: | ||
| # - cron: '0 0 * * 0' | ||
| push: | ||
| branches: [ main ] | ||
| paths-ignore: | ||
| - "docs/**" | ||
| - "mkdocs.yml" | ||
| - "*.md" | ||
| pull_request: | ||
| branches: [ main ] | ||
| types: [ opened, synchronize, reopened, labeled ] | ||
| workflow_dispatch: | ||
| inputs: | ||
| models: | ||
|
|
@@ -21,26 +30,42 @@ concurrency: | |
| permissions: | ||
| contents: read | ||
|
|
||
| env: | ||
| RUSTC_WRAPPER: sccache | ||
| SCCACHE_GHA_ENABLED: "true" | ||
|
|
||
| jobs: | ||
| # --------------------------------------------------------------------------- | ||
| # Build wheel once and reuse across all jobs | ||
| # --------------------------------------------------------------------------- | ||
|
|
||
| build-wheel: | ||
| runs-on: 8-gpu-h200 | ||
| runs-on: k8s-runner-gpu | ||
| permissions: | ||
| contents: read | ||
| steps: | ||
| - name: Checkout code | ||
| uses: actions/checkout@v6 | ||
|
|
||
| - name: Cache wheel artifacts | ||
| id: cache-wheel | ||
| uses: actions/cache@v5 | ||
| with: | ||
| path: bindings/python/dist/*.whl | ||
| key: nightly-wheel-${{ runner.os }}-${{ hashFiles('Cargo.lock', '**/Cargo.toml', 'auth/src/**', 'bindings/python/src/**', 'data_connector/src/**', 'grpc_client/src/**', 'kv_index/src/**', 'mcp/src/**', 'mesh/src/**', 'model_gateway/src/**', 'multimodal/src/**', 'protocols/src/**', 'reasoning_parser/src/**', 'tokenizer/src/**', 'tool_parser/src/**', 'wasm/src/**', 'workflow/src/**') }} | ||
| restore-keys: | | ||
| nightly-wheel-${{ runner.os }}- | ||
|
|
||
| - name: Setup Rust | ||
| if: steps.cache-wheel.outputs.cache-hit != 'true' | ||
| uses: ./.github/actions/setup-rust | ||
|
|
||
| - name: Setup Python venv | ||
| if: steps.cache-wheel.outputs.cache-hit != 'true' | ||
| run: bash scripts/ci_setup_python_venv.sh | ||
|
|
||
| - name: Build Python wheel | ||
| if: steps.cache-wheel.outputs.cache-hit != 'true' | ||
| run: bash scripts/ci_build_wheel.sh | ||
|
|
||
| - name: Upload wheel artifact | ||
|
|
@@ -50,33 +75,38 @@ jobs: | |
| path: bindings/python/dist/*.whl | ||
| retention-days: 7 | ||
|
|
||
| - name: Show sccache stats | ||
| if: always() && steps.cache-wheel.outputs.cache-hit != 'true' | ||
| run: sccache --show-stats | ||
|
|
||
| # --------------------------------------------------------------------------- | ||
| # Single-worker: one model per job for better isolation and debugging | ||
| # Single-worker: one model per job for better isolation and debugging. | ||
| # Llama-4 Maverick runs on H200; all other models run on k8s H100. | ||
| # --------------------------------------------------------------------------- | ||
|
|
||
| single-worker: | ||
| name: "${{ matrix.model.id }} / single-${{ matrix.variant.id }}" | ||
| needs: build-wheel | ||
| if: ${{ !cancelled() }} | ||
| runs-on: 8-gpu-h200 | ||
| runs-on: ${{ fromJson(matrix.model.runs_on) }} | ||
| timeout-minutes: 1440 | ||
| strategy: | ||
| fail-fast: false | ||
| max-parallel: 1 | ||
| max-parallel: 8 | ||
| matrix: | ||
| model: | ||
| - { id: llama-8b, test_class: TestNightlyLlama8bSingle } | ||
| - { id: llama-1b, test_class: TestNightlyLlama1bSingle } | ||
| - { id: qwen-7b, test_class: TestNightlyQwen7bSingle } | ||
| - { id: qwen-14b, test_class: TestNightlyQwen14bSingle } | ||
| - { id: deepseek-7b, test_class: TestNightlyDeepseek7bSingle } | ||
| - { id: qwen-30b, test_class: TestNightlyQwen30bSingle } | ||
| - { id: mistral-7b, test_class: TestNightlyMistral7bSingle } | ||
| - { id: gpt-oss, test_class: TestNightlyGptOssSingle } | ||
| - { id: llama-4-maverick-17b, test_class: TestNightlyLlama4MaverickSingle } | ||
| - { id: meta-llama/Llama-3.1-8B-Instruct, slug: meta-llama-Llama-3.1-8B-Instruct, test_class: TestNightlyLlama8bSingle, runs_on: '["k8s-runner-gpu","4-gpu-h100"]', gpu_type: H100 } | ||
| - { id: meta-llama/Llama-3.2-1B-Instruct, slug: meta-llama-Llama-3.2-1B-Instruct, test_class: TestNightlyLlama1bSingle, runs_on: '["k8s-runner-gpu","4-gpu-h100"]', gpu_type: H100 } | ||
| - { id: Qwen/Qwen2.5-7B-Instruct, slug: Qwen-Qwen2.5-7B-Instruct, test_class: TestNightlyQwen7bSingle, runs_on: '["k8s-runner-gpu","4-gpu-h100"]', gpu_type: H100 } | ||
| - { id: Qwen/Qwen2.5-14B-Instruct, slug: Qwen-Qwen2.5-14B-Instruct, test_class: TestNightlyQwen14bSingle, runs_on: '["k8s-runner-gpu","4-gpu-h100"]', gpu_type: H100 } | ||
| - { id: deepseek-ai/DeepSeek-R1-Distill-Qwen-7B, slug: deepseek-ai-DeepSeek-R1-Distill-Qwen-7B, test_class: TestNightlyDeepseek7bSingle, runs_on: '["k8s-runner-gpu","4-gpu-h100"]', gpu_type: H100 } | ||
| - { id: Qwen/Qwen3-30B-A3B, slug: Qwen-Qwen3-30B-A3B, test_class: TestNightlyQwen30bSingle, runs_on: '["k8s-runner-gpu","4-gpu-h100"]', gpu_type: H100 } | ||
| - { id: mistralai/Mistral-7B-Instruct-v0.3, slug: mistralai-Mistral-7B-Instruct-v0.3, test_class: TestNightlyMistral7bSingle, runs_on: '["k8s-runner-gpu","4-gpu-h100"]', gpu_type: H100 } | ||
| - { id: openai/gpt-oss-20b, slug: openai-gpt-oss-20b, test_class: TestNightlyGptOssSingle, runs_on: '["k8s-runner-gpu","4-gpu-h100"]', gpu_type: H100 } | ||
| - { id: meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8, slug: meta-llama-Llama-4-Maverick-17B-128E-Instruct-FP8, test_class: TestNightlyLlama4MaverickSingle, runs_on: '["8-gpu-h200"]', gpu_type: H200 } | ||
| variant: | ||
| - { id: sglang, runtime: sglang, grpc_only: "false" } | ||
| - { id: vllm, runtime: vllm, grpc_only: "true" } | ||
| - { id: sglang, runtime: sglang, grpc_only: "false", setup_vllm: false, setup_trtllm: false, extra_deps: "genai-bench" } | ||
| - { id: vllm, runtime: vllm, grpc_only: "true", setup_vllm: true, setup_trtllm: false, extra_deps: "genai-bench" } | ||
|
|
||
| steps: | ||
| - name: Check filters | ||
|
|
@@ -85,7 +115,7 @@ jobs: | |
| MODELS="${{ github.event.inputs.models || 'all' }}" | ||
| RUNTIME="${{ github.event.inputs.runtime || 'all' }}" | ||
| SKIP="false" | ||
| if [ "$MODELS" != "all" ] && ! echo ",$MODELS," | grep -q ",${{ matrix.model.id }},"; then | ||
| if [ "$MODELS" != "all" ] && ! echo ",$MODELS," | grep -Fq ",${{ matrix.model.id }},"; then | ||
| SKIP="true" | ||
| fi | ||
| if [ "$RUNTIME" != "all" ] && [ "$RUNTIME" != "${{ matrix.variant.id }}" ]; then | ||
|
slin1237 marked this conversation as resolved.
Resolved
|
||
|
|
@@ -97,18 +127,23 @@ jobs: | |
| if: steps.filter.outputs.skip != 'true' | ||
| uses: actions/checkout@v6 | ||
|
|
||
| - name: Install inference backend | ||
| if: steps.filter.outputs.skip != 'true' | ||
| env: | ||
| SGLANG_USE_LATEST_TAG: "1" | ||
| run: | | ||
| bash scripts/ci_setup_python_venv.sh | ||
| source .venv/bin/activate | ||
| if [ "${{ matrix.variant.runtime }}" == "vllm" ]; then | ||
| bash scripts/ci_install_vllm.sh | ||
| else | ||
| bash scripts/ci_install_sglang.sh | ||
| fi | ||
| - name: Setup Python (H200 runners) | ||
| if: steps.filter.outputs.skip != 'true' && matrix.model.gpu_type == 'H200' | ||
| uses: actions/setup-python@v6 | ||
| with: | ||
| python-version: "3.12" | ||
|
|
||
| - name: Setup vLLM backend | ||
| if: steps.filter.outputs.skip != 'true' && matrix.variant.setup_vllm | ||
| uses: ./.github/actions/setup-vllm | ||
|
|
||
| - name: Setup TRT-LLM backend | ||
| if: steps.filter.outputs.skip != 'true' && matrix.variant.setup_trtllm | ||
| uses: ./.github/actions/setup-trtllm | ||
|
|
||
| - name: Setup SGLang backend | ||
| if: steps.filter.outputs.skip != 'true' && !matrix.variant.setup_vllm && !matrix.variant.setup_trtllm | ||
| uses: ./.github/actions/setup-sglang | ||
|
|
||
| - name: Download wheel artifact | ||
| if: steps.filter.outputs.skip != 'true' | ||
|
|
@@ -122,12 +157,14 @@ jobs: | |
| run: | | ||
| pip uninstall -y smg || true | ||
| pip install wheel/*.whl | ||
| bash scripts/ci_install_e2e_deps.sh genai-bench | ||
| bash scripts/ci_install_e2e_deps.sh ${{ matrix.variant.extra_deps }} | ||
|
|
||
| - name: Run benchmark | ||
| if: steps.filter.outputs.skip != 'true' | ||
| env: | ||
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | ||
| GPU_TYPE: ${{ matrix.model.gpu_type }} | ||
| E2E_MODEL_TP_OVERRIDES: '{"meta-llama/Llama-3.1-8B-Instruct":1,"meta-llama/Llama-3.2-1B-Instruct":1,"Qwen/Qwen2.5-7B-Instruct":1,"Qwen/Qwen2.5-14B-Instruct":1,"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B":1,"Qwen/Qwen3-30B-A3B":1,"mistralai/Mistral-7B-Instruct-v0.3":1,"openai/gpt-oss-20b":1}' | ||
| run: | | ||
| bash scripts/ci_killall_sglang.sh "nuk_gpus" | ||
|
|
||
|
|
@@ -146,7 +183,7 @@ jobs: | |
| if: always() && steps.filter.outputs.skip != 'true' | ||
| uses: actions/upload-artifact@v6 | ||
| with: | ||
| name: nightly-${{ matrix.model.id }}-single-${{ matrix.variant.id }}-${{ github.run_id }} | ||
| name: nightly-${{ matrix.model.slug }}-single-${{ matrix.variant.id }}-${{ github.run_id }} | ||
| path: nightly_*/ | ||
| retention-days: 90 | ||
|
|
||
|
|
@@ -155,32 +192,33 @@ jobs: | |
| run: bash scripts/ci_killall_sglang.sh || true | ||
|
|
||
| # --------------------------------------------------------------------------- | ||
| # Multi-worker: one model per job (uses all 8 GPUs) | ||
| # Multi-worker: one model per job. | ||
| # Llama-4 Maverick runs on H200; all other models run on k8s H100. | ||
| # --------------------------------------------------------------------------- | ||
|
|
||
| multi-worker: | ||
| name: "${{ matrix.model.id }} / multi-${{ matrix.variant.id }}" | ||
| needs: [build-wheel, single-worker] | ||
| needs: build-wheel | ||
| if: ${{ !cancelled() }} | ||
| runs-on: 8-gpu-h200 | ||
| runs-on: ${{ fromJson(matrix.model.runs_on) }} | ||
| timeout-minutes: 1440 | ||
| strategy: | ||
| fail-fast: false | ||
| max-parallel: 1 | ||
| max-parallel: 8 | ||
| matrix: | ||
| model: | ||
| - { id: llama-8b, test_class: TestNightlyLlama8bMulti } | ||
| - { id: llama-1b, test_class: TestNightlyLlama1bMulti } | ||
| - { id: qwen-7b, test_class: TestNightlyQwen7bMulti } | ||
| - { id: qwen-14b, test_class: TestNightlyQwen14bMulti } | ||
| - { id: deepseek-7b, test_class: TestNightlyDeepseek7bMulti } | ||
| - { id: qwen-30b, test_class: TestNightlyQwen30bMulti } | ||
| - { id: mistral-7b, test_class: TestNightlyMistral7bMulti } | ||
| - { id: gpt-oss, test_class: TestNightlyGptOssMulti } | ||
| # - { id: llama-4-maverick-17b, test_class: TestNightlyLlama4MaverickMulti } # 1 worker uses all 8 GPUs (tp=8) | ||
| - { id: meta-llama/Llama-3.1-8B-Instruct, slug: meta-llama-Llama-3.1-8B-Instruct, test_class: TestNightlyLlama8bMulti, runs_on: '["k8s-runner-gpu","4-gpu-h100"]', gpu_type: H100 } | ||
| - { id: meta-llama/Llama-3.2-1B-Instruct, slug: meta-llama-Llama-3.2-1B-Instruct, test_class: TestNightlyLlama1bMulti, runs_on: '["k8s-runner-gpu","4-gpu-h100"]', gpu_type: H100 } | ||
| - { id: Qwen/Qwen2.5-7B-Instruct, slug: Qwen-Qwen2.5-7B-Instruct, test_class: TestNightlyQwen7bMulti, runs_on: '["k8s-runner-gpu","4-gpu-h100"]', gpu_type: H100 } | ||
| - { id: Qwen/Qwen2.5-14B-Instruct, slug: Qwen-Qwen2.5-14B-Instruct, test_class: TestNightlyQwen14bMulti, runs_on: '["k8s-runner-gpu","4-gpu-h100"]', gpu_type: H100 } | ||
| - { id: deepseek-ai/DeepSeek-R1-Distill-Qwen-7B, slug: deepseek-ai-DeepSeek-R1-Distill-Qwen-7B, test_class: TestNightlyDeepseek7bMulti, runs_on: '["k8s-runner-gpu","4-gpu-h100"]', gpu_type: H100 } | ||
| - { id: Qwen/Qwen3-30B-A3B, slug: Qwen-Qwen3-30B-A3B, test_class: TestNightlyQwen30bMulti, runs_on: '["k8s-runner-gpu","4-gpu-h100"]', gpu_type: H100 } | ||
| - { id: mistralai/Mistral-7B-Instruct-v0.3, slug: mistralai-Mistral-7B-Instruct-v0.3, test_class: TestNightlyMistral7bMulti, runs_on: '["k8s-runner-gpu","4-gpu-h100"]', gpu_type: H100 } | ||
| - { id: openai/gpt-oss-20b, slug: openai-gpt-oss-20b, test_class: TestNightlyGptOssMulti, runs_on: '["k8s-runner-gpu","4-gpu-h100"]', gpu_type: H100 } | ||
| # - { id: meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8, test_class: TestNightlyLlama4MaverickMulti } # tp=8, keep disabled for nightly | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Commented matrix entry still incomplete. The past review flagged that this commented entry is missing # - { id: meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8, slug: meta-llama-Llama-4-Maverick-17B-128E-Instruct-FP8, test_class: TestNightlyLlama4MaverickMulti, runs_on: '["8-gpu-h200"]', gpu_type: H200 }🤖 Prompt for AI Agents |
||
| variant: | ||
| - { id: sglang, runtime: sglang, grpc_only: "false" } | ||
| - { id: vllm, runtime: vllm, grpc_only: "true" } | ||
| - { id: sglang, runtime: sglang, grpc_only: "false", setup_vllm: false, setup_trtllm: false, extra_deps: "genai-bench" } | ||
| - { id: vllm, runtime: vllm, grpc_only: "true", setup_vllm: true, setup_trtllm: false, extra_deps: "genai-bench" } | ||
|
|
||
| steps: | ||
| - name: Check filters | ||
|
|
@@ -189,7 +227,7 @@ jobs: | |
| MODELS="${{ github.event.inputs.models || 'all' }}" | ||
| RUNTIME="${{ github.event.inputs.runtime || 'all' }}" | ||
| SKIP="false" | ||
| if [ "$MODELS" != "all" ] && ! echo ",$MODELS," | grep -q ",${{ matrix.model.id }},"; then | ||
| if [ "$MODELS" != "all" ] && ! echo ",$MODELS," | grep -Fq ",${{ matrix.model.id }},"; then | ||
| SKIP="true" | ||
| fi | ||
| if [ "$RUNTIME" != "all" ] && [ "$RUNTIME" != "${{ matrix.variant.id }}" ]; then | ||
|
|
@@ -201,18 +239,23 @@ jobs: | |
| if: steps.filter.outputs.skip != 'true' | ||
| uses: actions/checkout@v6 | ||
|
|
||
| - name: Install inference backend | ||
| if: steps.filter.outputs.skip != 'true' | ||
| env: | ||
| SGLANG_USE_LATEST_TAG: "1" | ||
| run: | | ||
| bash scripts/ci_setup_python_venv.sh | ||
| source .venv/bin/activate | ||
| if [ "${{ matrix.variant.runtime }}" == "vllm" ]; then | ||
| bash scripts/ci_install_vllm.sh | ||
| else | ||
| bash scripts/ci_install_sglang.sh | ||
| fi | ||
| - name: Setup Python (H200 runners) | ||
| if: steps.filter.outputs.skip != 'true' && matrix.model.gpu_type == 'H200' | ||
| uses: actions/setup-python@v6 | ||
| with: | ||
| python-version: "3.12" | ||
|
|
||
| - name: Setup vLLM backend | ||
| if: steps.filter.outputs.skip != 'true' && matrix.variant.setup_vllm | ||
| uses: ./.github/actions/setup-vllm | ||
|
|
||
| - name: Setup TRT-LLM backend | ||
| if: steps.filter.outputs.skip != 'true' && matrix.variant.setup_trtllm | ||
| uses: ./.github/actions/setup-trtllm | ||
|
|
||
| - name: Setup SGLang backend | ||
| if: steps.filter.outputs.skip != 'true' && !matrix.variant.setup_vllm && !matrix.variant.setup_trtllm | ||
| uses: ./.github/actions/setup-sglang | ||
|
|
||
| - name: Download wheel artifact | ||
| if: steps.filter.outputs.skip != 'true' | ||
|
|
@@ -226,12 +269,14 @@ jobs: | |
| run: | | ||
| pip uninstall -y smg || true | ||
| pip install wheel/*.whl | ||
| bash scripts/ci_install_e2e_deps.sh genai-bench | ||
| bash scripts/ci_install_e2e_deps.sh ${{ matrix.variant.extra_deps }} | ||
|
|
||
| - name: Run benchmark | ||
| if: steps.filter.outputs.skip != 'true' | ||
| env: | ||
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | ||
| GPU_TYPE: ${{ matrix.model.gpu_type }} | ||
| E2E_MODEL_TP_OVERRIDES: '{"meta-llama/Llama-3.1-8B-Instruct":1,"meta-llama/Llama-3.2-1B-Instruct":1,"Qwen/Qwen2.5-7B-Instruct":1,"Qwen/Qwen2.5-14B-Instruct":1,"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B":1,"Qwen/Qwen3-30B-A3B":1,"mistralai/Mistral-7B-Instruct-v0.3":1,"openai/gpt-oss-20b":1}' | ||
| run: | | ||
| bash scripts/ci_killall_sglang.sh "nuk_gpus" | ||
|
|
||
|
|
@@ -250,7 +295,7 @@ jobs: | |
| if: always() && steps.filter.outputs.skip != 'true' | ||
| uses: actions/upload-artifact@v6 | ||
| with: | ||
| name: nightly-${{ matrix.model.id }}-multi-${{ matrix.variant.id }}-${{ github.run_id }} | ||
| name: nightly-${{ matrix.model.slug }}-multi-${{ matrix.variant.id }}-${{ github.run_id }} | ||
| path: nightly_*/ | ||
| retention-days: 90 | ||
|
|
||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.