Skip to content
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
169 changes: 107 additions & 62 deletions .github/workflows/nightly-benchmark.yml
Original file line number Diff line number Diff line change
@@ -1,8 +1,17 @@
name: Nightly Benchmark

on:
schedule:
- cron: '0 0 * * 0'
# schedule:
# - cron: '0 0 * * 0'
push:
branches: [ main ]
paths-ignore:
- "docs/**"
- "mkdocs.yml"
- "*.md"
pull_request:
branches: [ main ]
types: [ opened, synchronize, reopened, labeled ]
workflow_dispatch:
inputs:
models:
Expand All @@ -21,26 +30,42 @@ concurrency:
permissions:
contents: read

env:
RUSTC_WRAPPER: sccache
SCCACHE_GHA_ENABLED: "true"

jobs:
# ---------------------------------------------------------------------------
# Build wheel once and reuse across all jobs
# ---------------------------------------------------------------------------

build-wheel:
runs-on: 8-gpu-h200
runs-on: k8s-runner-gpu
permissions:
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
contents: read
steps:
- name: Checkout code
uses: actions/checkout@v6

- name: Cache wheel artifacts
id: cache-wheel
uses: actions/cache@v5
with:
path: bindings/python/dist/*.whl
key: nightly-wheel-${{ runner.os }}-${{ hashFiles('Cargo.lock', '**/Cargo.toml', 'auth/src/**', 'bindings/python/src/**', 'data_connector/src/**', 'grpc_client/src/**', 'kv_index/src/**', 'mcp/src/**', 'mesh/src/**', 'model_gateway/src/**', 'multimodal/src/**', 'protocols/src/**', 'reasoning_parser/src/**', 'tokenizer/src/**', 'tool_parser/src/**', 'wasm/src/**', 'workflow/src/**') }}
restore-keys: |
nightly-wheel-${{ runner.os }}-

- name: Setup Rust
if: steps.cache-wheel.outputs.cache-hit != 'true'
uses: ./.github/actions/setup-rust

- name: Setup Python venv
if: steps.cache-wheel.outputs.cache-hit != 'true'
run: bash scripts/ci_setup_python_venv.sh

- name: Build Python wheel
if: steps.cache-wheel.outputs.cache-hit != 'true'
run: bash scripts/ci_build_wheel.sh

- name: Upload wheel artifact
Expand All @@ -50,33 +75,38 @@ jobs:
path: bindings/python/dist/*.whl
retention-days: 7

- name: Show sccache stats
if: always() && steps.cache-wheel.outputs.cache-hit != 'true'
run: sccache --show-stats

# ---------------------------------------------------------------------------
# Single-worker: one model per job for better isolation and debugging
# Single-worker: one model per job for better isolation and debugging.
# Llama-4 Maverick runs on H200; all other models run on k8s H100.
# ---------------------------------------------------------------------------

single-worker:
name: "${{ matrix.model.id }} / single-${{ matrix.variant.id }}"
needs: build-wheel
if: ${{ !cancelled() }}
runs-on: 8-gpu-h200
runs-on: ${{ fromJson(matrix.model.runs_on) }}
timeout-minutes: 1440
strategy:
fail-fast: false
max-parallel: 1
max-parallel: 8
matrix:
model:
- { id: llama-8b, test_class: TestNightlyLlama8bSingle }
- { id: llama-1b, test_class: TestNightlyLlama1bSingle }
- { id: qwen-7b, test_class: TestNightlyQwen7bSingle }
- { id: qwen-14b, test_class: TestNightlyQwen14bSingle }
- { id: deepseek-7b, test_class: TestNightlyDeepseek7bSingle }
- { id: qwen-30b, test_class: TestNightlyQwen30bSingle }
- { id: mistral-7b, test_class: TestNightlyMistral7bSingle }
- { id: gpt-oss, test_class: TestNightlyGptOssSingle }
- { id: llama-4-maverick-17b, test_class: TestNightlyLlama4MaverickSingle }
- { id: meta-llama/Llama-3.1-8B-Instruct, slug: meta-llama-Llama-3.1-8B-Instruct, test_class: TestNightlyLlama8bSingle, runs_on: '["k8s-runner-gpu","4-gpu-h100"]', gpu_type: H100 }
- { id: meta-llama/Llama-3.2-1B-Instruct, slug: meta-llama-Llama-3.2-1B-Instruct, test_class: TestNightlyLlama1bSingle, runs_on: '["k8s-runner-gpu","4-gpu-h100"]', gpu_type: H100 }
- { id: Qwen/Qwen2.5-7B-Instruct, slug: Qwen-Qwen2.5-7B-Instruct, test_class: TestNightlyQwen7bSingle, runs_on: '["k8s-runner-gpu","4-gpu-h100"]', gpu_type: H100 }
- { id: Qwen/Qwen2.5-14B-Instruct, slug: Qwen-Qwen2.5-14B-Instruct, test_class: TestNightlyQwen14bSingle, runs_on: '["k8s-runner-gpu","4-gpu-h100"]', gpu_type: H100 }
- { id: deepseek-ai/DeepSeek-R1-Distill-Qwen-7B, slug: deepseek-ai-DeepSeek-R1-Distill-Qwen-7B, test_class: TestNightlyDeepseek7bSingle, runs_on: '["k8s-runner-gpu","4-gpu-h100"]', gpu_type: H100 }
- { id: Qwen/Qwen3-30B-A3B, slug: Qwen-Qwen3-30B-A3B, test_class: TestNightlyQwen30bSingle, runs_on: '["k8s-runner-gpu","4-gpu-h100"]', gpu_type: H100 }
- { id: mistralai/Mistral-7B-Instruct-v0.3, slug: mistralai-Mistral-7B-Instruct-v0.3, test_class: TestNightlyMistral7bSingle, runs_on: '["k8s-runner-gpu","4-gpu-h100"]', gpu_type: H100 }
- { id: openai/gpt-oss-20b, slug: openai-gpt-oss-20b, test_class: TestNightlyGptOssSingle, runs_on: '["k8s-runner-gpu","4-gpu-h100"]', gpu_type: H100 }
- { id: meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8, slug: meta-llama-Llama-4-Maverick-17B-128E-Instruct-FP8, test_class: TestNightlyLlama4MaverickSingle, runs_on: '["8-gpu-h200"]', gpu_type: H200 }
variant:
- { id: sglang, runtime: sglang, grpc_only: "false" }
- { id: vllm, runtime: vllm, grpc_only: "true" }
- { id: sglang, runtime: sglang, grpc_only: "false", setup_vllm: false, setup_trtllm: false, extra_deps: "genai-bench" }
- { id: vllm, runtime: vllm, grpc_only: "true", setup_vllm: true, setup_trtllm: false, extra_deps: "genai-bench" }

steps:
- name: Check filters
Expand All @@ -85,7 +115,7 @@ jobs:
MODELS="${{ github.event.inputs.models || 'all' }}"
RUNTIME="${{ github.event.inputs.runtime || 'all' }}"
SKIP="false"
if [ "$MODELS" != "all" ] && ! echo ",$MODELS," | grep -q ",${{ matrix.model.id }},"; then
if [ "$MODELS" != "all" ] && ! echo ",$MODELS," | grep -Fq ",${{ matrix.model.id }},"; then
SKIP="true"
fi
if [ "$RUNTIME" != "all" ] && [ "$RUNTIME" != "${{ matrix.variant.id }}" ]; then
Comment thread
slin1237 marked this conversation as resolved.
Resolved
Expand All @@ -97,18 +127,23 @@ jobs:
if: steps.filter.outputs.skip != 'true'
uses: actions/checkout@v6

- name: Install inference backend
if: steps.filter.outputs.skip != 'true'
env:
SGLANG_USE_LATEST_TAG: "1"
run: |
bash scripts/ci_setup_python_venv.sh
source .venv/bin/activate
if [ "${{ matrix.variant.runtime }}" == "vllm" ]; then
bash scripts/ci_install_vllm.sh
else
bash scripts/ci_install_sglang.sh
fi
- name: Setup Python (H200 runners)
if: steps.filter.outputs.skip != 'true' && matrix.model.gpu_type == 'H200'
uses: actions/setup-python@v6
with:
python-version: "3.12"

- name: Setup vLLM backend
if: steps.filter.outputs.skip != 'true' && matrix.variant.setup_vllm
uses: ./.github/actions/setup-vllm

- name: Setup TRT-LLM backend
if: steps.filter.outputs.skip != 'true' && matrix.variant.setup_trtllm
uses: ./.github/actions/setup-trtllm

- name: Setup SGLang backend
if: steps.filter.outputs.skip != 'true' && !matrix.variant.setup_vllm && !matrix.variant.setup_trtllm
uses: ./.github/actions/setup-sglang

- name: Download wheel artifact
if: steps.filter.outputs.skip != 'true'
Expand All @@ -122,12 +157,14 @@ jobs:
run: |
pip uninstall -y smg || true
pip install wheel/*.whl
bash scripts/ci_install_e2e_deps.sh genai-bench
bash scripts/ci_install_e2e_deps.sh ${{ matrix.variant.extra_deps }}

- name: Run benchmark
if: steps.filter.outputs.skip != 'true'
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
GPU_TYPE: ${{ matrix.model.gpu_type }}
E2E_MODEL_TP_OVERRIDES: '{"meta-llama/Llama-3.1-8B-Instruct":1,"meta-llama/Llama-3.2-1B-Instruct":1,"Qwen/Qwen2.5-7B-Instruct":1,"Qwen/Qwen2.5-14B-Instruct":1,"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B":1,"Qwen/Qwen3-30B-A3B":1,"mistralai/Mistral-7B-Instruct-v0.3":1,"openai/gpt-oss-20b":1}'
run: |
bash scripts/ci_killall_sglang.sh "nuk_gpus"

Expand All @@ -146,7 +183,7 @@ jobs:
if: always() && steps.filter.outputs.skip != 'true'
uses: actions/upload-artifact@v6
with:
name: nightly-${{ matrix.model.id }}-single-${{ matrix.variant.id }}-${{ github.run_id }}
name: nightly-${{ matrix.model.slug }}-single-${{ matrix.variant.id }}-${{ github.run_id }}
path: nightly_*/
retention-days: 90

Expand All @@ -155,32 +192,33 @@ jobs:
run: bash scripts/ci_killall_sglang.sh || true

# ---------------------------------------------------------------------------
# Multi-worker: one model per job (uses all 8 GPUs)
# Multi-worker: one model per job.
# Llama-4 Maverick runs on H200; all other models run on k8s H100.
# ---------------------------------------------------------------------------

multi-worker:
name: "${{ matrix.model.id }} / multi-${{ matrix.variant.id }}"
needs: [build-wheel, single-worker]
needs: build-wheel
if: ${{ !cancelled() }}
runs-on: 8-gpu-h200
runs-on: ${{ fromJson(matrix.model.runs_on) }}
timeout-minutes: 1440
strategy:
fail-fast: false
max-parallel: 1
max-parallel: 8
matrix:
model:
- { id: llama-8b, test_class: TestNightlyLlama8bMulti }
- { id: llama-1b, test_class: TestNightlyLlama1bMulti }
- { id: qwen-7b, test_class: TestNightlyQwen7bMulti }
- { id: qwen-14b, test_class: TestNightlyQwen14bMulti }
- { id: deepseek-7b, test_class: TestNightlyDeepseek7bMulti }
- { id: qwen-30b, test_class: TestNightlyQwen30bMulti }
- { id: mistral-7b, test_class: TestNightlyMistral7bMulti }
- { id: gpt-oss, test_class: TestNightlyGptOssMulti }
# - { id: llama-4-maverick-17b, test_class: TestNightlyLlama4MaverickMulti } # 1 worker uses all 8 GPUs (tp=8)
- { id: meta-llama/Llama-3.1-8B-Instruct, slug: meta-llama-Llama-3.1-8B-Instruct, test_class: TestNightlyLlama8bMulti, runs_on: '["k8s-runner-gpu","4-gpu-h100"]', gpu_type: H100 }
- { id: meta-llama/Llama-3.2-1B-Instruct, slug: meta-llama-Llama-3.2-1B-Instruct, test_class: TestNightlyLlama1bMulti, runs_on: '["k8s-runner-gpu","4-gpu-h100"]', gpu_type: H100 }
- { id: Qwen/Qwen2.5-7B-Instruct, slug: Qwen-Qwen2.5-7B-Instruct, test_class: TestNightlyQwen7bMulti, runs_on: '["k8s-runner-gpu","4-gpu-h100"]', gpu_type: H100 }
- { id: Qwen/Qwen2.5-14B-Instruct, slug: Qwen-Qwen2.5-14B-Instruct, test_class: TestNightlyQwen14bMulti, runs_on: '["k8s-runner-gpu","4-gpu-h100"]', gpu_type: H100 }
- { id: deepseek-ai/DeepSeek-R1-Distill-Qwen-7B, slug: deepseek-ai-DeepSeek-R1-Distill-Qwen-7B, test_class: TestNightlyDeepseek7bMulti, runs_on: '["k8s-runner-gpu","4-gpu-h100"]', gpu_type: H100 }
- { id: Qwen/Qwen3-30B-A3B, slug: Qwen-Qwen3-30B-A3B, test_class: TestNightlyQwen30bMulti, runs_on: '["k8s-runner-gpu","4-gpu-h100"]', gpu_type: H100 }
- { id: mistralai/Mistral-7B-Instruct-v0.3, slug: mistralai-Mistral-7B-Instruct-v0.3, test_class: TestNightlyMistral7bMulti, runs_on: '["k8s-runner-gpu","4-gpu-h100"]', gpu_type: H100 }
- { id: openai/gpt-oss-20b, slug: openai-gpt-oss-20b, test_class: TestNightlyGptOssMulti, runs_on: '["k8s-runner-gpu","4-gpu-h100"]', gpu_type: H100 }
# - { id: meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8, test_class: TestNightlyLlama4MaverickMulti } # tp=8, keep disabled for nightly

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟡 Minor

Commented matrix entry still incomplete.

The past review flagged that this commented entry is missing slug, runs_on, and gpu_type fields. While marked as addressed, the current code still shows the incomplete format. If re-enabled without these fields, fromJson(matrix.model.runs_on) and artifact naming will fail.

# - { id: meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8, slug: meta-llama-Llama-4-Maverick-17B-128E-Instruct-FP8, test_class: TestNightlyLlama4MaverickMulti, runs_on: '["8-gpu-h200"]', gpu_type: H200 }
🤖 Prompt for AI Agents
In @.github/workflows/nightly-benchmark.yml at line 218, Update the commented
matrix entry for id "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8" to
include the missing fields so downstream parsing
(fromJson(matrix.model.runs_on)) and artifact naming work: add a slug (e.g.,
meta-llama-Llama-4-Maverick-17B-128E-Instruct-FP8), a runs_on JSON string (e.g.,
'["8-gpu-h200"]'), and gpu_type (e.g., H200) to the commented YAML line so it
becomes a complete matrix entry if re-enabled.

variant:
- { id: sglang, runtime: sglang, grpc_only: "false" }
- { id: vllm, runtime: vllm, grpc_only: "true" }
- { id: sglang, runtime: sglang, grpc_only: "false", setup_vllm: false, setup_trtllm: false, extra_deps: "genai-bench" }
- { id: vllm, runtime: vllm, grpc_only: "true", setup_vllm: true, setup_trtllm: false, extra_deps: "genai-bench" }

steps:
- name: Check filters
Expand All @@ -189,7 +227,7 @@ jobs:
MODELS="${{ github.event.inputs.models || 'all' }}"
RUNTIME="${{ github.event.inputs.runtime || 'all' }}"
SKIP="false"
if [ "$MODELS" != "all" ] && ! echo ",$MODELS," | grep -q ",${{ matrix.model.id }},"; then
if [ "$MODELS" != "all" ] && ! echo ",$MODELS," | grep -Fq ",${{ matrix.model.id }},"; then
SKIP="true"
fi
if [ "$RUNTIME" != "all" ] && [ "$RUNTIME" != "${{ matrix.variant.id }}" ]; then
Expand All @@ -201,18 +239,23 @@ jobs:
if: steps.filter.outputs.skip != 'true'
uses: actions/checkout@v6

- name: Install inference backend
if: steps.filter.outputs.skip != 'true'
env:
SGLANG_USE_LATEST_TAG: "1"
run: |
bash scripts/ci_setup_python_venv.sh
source .venv/bin/activate
if [ "${{ matrix.variant.runtime }}" == "vllm" ]; then
bash scripts/ci_install_vllm.sh
else
bash scripts/ci_install_sglang.sh
fi
- name: Setup Python (H200 runners)
if: steps.filter.outputs.skip != 'true' && matrix.model.gpu_type == 'H200'
uses: actions/setup-python@v6
with:
python-version: "3.12"

- name: Setup vLLM backend
if: steps.filter.outputs.skip != 'true' && matrix.variant.setup_vllm
uses: ./.github/actions/setup-vllm

- name: Setup TRT-LLM backend
if: steps.filter.outputs.skip != 'true' && matrix.variant.setup_trtllm
uses: ./.github/actions/setup-trtllm

- name: Setup SGLang backend
if: steps.filter.outputs.skip != 'true' && !matrix.variant.setup_vllm && !matrix.variant.setup_trtllm
uses: ./.github/actions/setup-sglang

- name: Download wheel artifact
if: steps.filter.outputs.skip != 'true'
Expand All @@ -226,12 +269,14 @@ jobs:
run: |
pip uninstall -y smg || true
pip install wheel/*.whl
bash scripts/ci_install_e2e_deps.sh genai-bench
bash scripts/ci_install_e2e_deps.sh ${{ matrix.variant.extra_deps }}

- name: Run benchmark
if: steps.filter.outputs.skip != 'true'
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
GPU_TYPE: ${{ matrix.model.gpu_type }}
E2E_MODEL_TP_OVERRIDES: '{"meta-llama/Llama-3.1-8B-Instruct":1,"meta-llama/Llama-3.2-1B-Instruct":1,"Qwen/Qwen2.5-7B-Instruct":1,"Qwen/Qwen2.5-14B-Instruct":1,"deepseek-ai/DeepSeek-R1-Distill-Qwen-7B":1,"Qwen/Qwen3-30B-A3B":1,"mistralai/Mistral-7B-Instruct-v0.3":1,"openai/gpt-oss-20b":1}'
run: |
bash scripts/ci_killall_sglang.sh "nuk_gpus"

Expand All @@ -250,7 +295,7 @@ jobs:
if: always() && steps.filter.outputs.skip != 'true'
uses: actions/upload-artifact@v6
with:
name: nightly-${{ matrix.model.id }}-multi-${{ matrix.variant.id }}-${{ github.run_id }}
name: nightly-${{ matrix.model.slug }}-multi-${{ matrix.variant.id }}-${{ github.run_id }}
path: nightly_*/
retention-days: 90

Expand Down
4 changes: 2 additions & 2 deletions docs/assets/stylesheets/extra.css
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@

/* Hero title: dark gradient text instead of white.
background shorthand resets background-clip and background-size,
so all properties must be re-declared here. */
so all properties must be redeclared here. */
[data-md-color-scheme="default"] .md-typeset .hero h1 {
background: linear-gradient(
135deg,
Expand Down Expand Up @@ -130,7 +130,7 @@

/* Stat values: dark gradient instead of white.
background shorthand resets background-clip and background-size,
so all properties must be re-declared here. */
so all properties must be redeclared here. */
[data-md-color-scheme="default"] .md-typeset .stat-value {
background: linear-gradient(135deg, var(--md-code-fg-color) 0%, var(--mesh-purple) 50%, var(--mesh-cyan) 100%);
background-size: 200% 200%;
Expand Down
17 changes: 12 additions & 5 deletions e2e_test/benchmarks/nightly_summarize.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,17 +61,24 @@ def parse_folder_name(folder_name: str) -> dict:
"""Parse experiment info from folder name.

Expected patterns (newest to oldest):
- nightly_llama-8b_http_sglang_single -> model=llama-8b, protocol=http, runtime=sglang, worker_type=single
- nightly_llama-8b_grpc_vllm_multi -> model=llama-8b, protocol=grpc, runtime=vllm, worker_type=multi
- nightly_llama-8b_http_sglang -> model=llama-8b, protocol=http, runtime=sglang (worker_type=single)
- nightly_llama-8b_http (legacy) -> model=llama-8b, protocol=http
- nightly_meta-llama/Llama-3.1-8B-Instruct_http_sglang_single
-> model=meta-llama/Llama-3.1-8B-Instruct, protocol=http,
runtime=sglang, worker_type=single
- nightly_meta-llama/Llama-3.1-8B-Instruct_grpc_vllm_multi
-> model=meta-llama/Llama-3.1-8B-Instruct, protocol=grpc,
runtime=vllm, worker_type=multi
- nightly_meta-llama/Llama-3.1-8B-Instruct_http_sglang
-> model=meta-llama/Llama-3.1-8B-Instruct, protocol=http,
runtime=sglang (worker_type=single)
- nightly_meta-llama/Llama-3.1-8B-Instruct_http (legacy)
-> model=meta-llama/Llama-3.1-8B-Instruct, protocol=http
"""
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
info = {"model": "unknown", "protocol": "unknown", "runtime": None, "worker_type": "single"}

# Remove nightly_ prefix
name = folder_name.replace("nightly_", "")

# Try newest format: model_protocol_runtime_worker_type (e.g., llama-8b_grpc_sglang_single)
# Try newest format: model_protocol_runtime_worker_type (e.g., meta-llama/Llama-3.1-8B-Instruct_grpc_sglang_single)
Comment thread
slin1237 marked this conversation as resolved.
Resolved
parts = name.rsplit("_", 3)

if len(parts) >= 4 and parts[-1] in ("single", "multi") and parts[-2] in ("sglang", "vllm"):
Expand Down
2 changes: 1 addition & 1 deletion e2e_test/benchmarks/test_go_bindings_perf.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
@pytest.mark.e2e
@pytest.mark.workers(count=4)
@pytest.mark.gateway(policy="round_robin")
@pytest.mark.model("llama-1b")
@pytest.mark.model("meta-llama/Llama-3.2-1B-Instruct")
class TestGoBindingsPerf:
"""Performance benchmark for Go OAI server.

Expand Down
Loading
Loading