From e3b68aba348cc7269dc6d10ace702b47b3e287e7 Mon Sep 17 00:00:00 2001 From: paxiaatucsdedu Date: Tue, 24 Mar 2026 15:28:24 -0700 Subject: [PATCH 01/14] Add Llama-4 and Llama-3.3 to nightly benchmarks Include three Llama variants in nightly performance runs: meta-llama/Llama-4-Scout-17B-16E-Instruct, meta-llama/Llama-3.3-70B-Instruct, and RedHatAI/Llama-3.3-70B-Instruct-FP8-dynamic. Signed-off-by: paxiaatucsdedu --- .github/workflows/nightly-benchmark.yml | 5 +++ e2e_test/benchmarks/test_nightly_perf.py | 21 ++++++++++ e2e_test/infra/model_specs.py | 51 ++++++++++++++++++++++++ 3 files changed, 77 insertions(+) diff --git a/.github/workflows/nightly-benchmark.yml b/.github/workflows/nightly-benchmark.yml index a7f12a0b68..f5c468a195 100644 --- a/.github/workflows/nightly-benchmark.yml +++ b/.github/workflows/nightly-benchmark.yml @@ -121,6 +121,9 @@ jobs: - { id: Qwen/Qwen2.5-7B-Instruct, slug: Qwen-Qwen2.5-7B-Instruct, test_class: TestNightlyQwen7bSingle } - { id: Qwen/Qwen3-30B-A3B, slug: Qwen-Qwen3-30B-A3B, test_class: TestNightlyQwen30bSingle } - { id: openai/gpt-oss-20b, slug: openai-gpt-oss-20b, test_class: TestNightlyGptOss20bSingle } + - { id: meta-llama/Llama-4-Scout-17B-16E-Instruct, slug: meta-llama-Llama-4-Scout-17B-16E-Instruct, test_class: TestNightlyLlama4ScoutSingle } + - { id: meta-llama/Llama-3.3-70B-Instruct, slug: meta-llama-Llama-3.3-70B-Instruct, test_class: TestNightlyLlama70bSingle } + - { id: RedHatAI/Llama-3.3-70B-Instruct-FP8-dynamic, slug: RedHatAI-Llama-3.3-70B-Instruct-FP8-dynamic, test_class: TestNightlyLlama70bFp8Single } variant: - { id: sglang, runtime: sglang, grpc_only: "false", setup_vllm: false, setup_trtllm: false } - { id: vllm, runtime: vllm, grpc_only: "false", setup_vllm: true, setup_trtllm: false } @@ -188,6 +191,8 @@ jobs: E2E_NIGHTLY: "1" E2E_MODEL_TP_OVERRIDES: '{"meta-llama/Llama-3.1-8B-Instruct":1,"Qwen/Qwen2.5-7B-Instruct":1,"Qwen/Qwen3-30B-A3B":1}' E2E_LOG_DIR: nightly_gateway_logs + HF_TOKEN: ${{ secrets.HF_TOKEN }} + HF_HOME: /raid/models run: | mkdir -p nightly_gateway_logs bash scripts/ci_killall_sglang.sh "nuk_gpus" diff --git a/e2e_test/benchmarks/test_nightly_perf.py b/e2e_test/benchmarks/test_nightly_perf.py index ef439eebb3..51b3d290a0 100644 --- a/e2e_test/benchmarks/test_nightly_perf.py +++ b/e2e_test/benchmarks/test_nightly_perf.py @@ -109,6 +109,27 @@ def _run_nightly(setup_backend, genai_bench_runner, model_id, worker_count=1, ** ["http", "grpc"], {}, ), + ( + "meta-llama/Llama-4-Scout-17B-16E-Instruct", + "Llama4Scout", + 1, + ["http", "grpc"], + {}, + ), + ( + "meta-llama/Llama-3.3-70B-Instruct", + "Llama70b", + 1, + ["http", "grpc"], + {}, + ), + ( + "RedHatAI/Llama-3.3-70B-Instruct-FP8-dynamic", + "Llama70bFp8", + 1, + ["http", "grpc"], + {}, + ), ] diff --git a/e2e_test/infra/model_specs.py b/e2e_test/infra/model_specs.py index 31ec82591b..d62de3f7dc 100644 --- a/e2e_test/infra/model_specs.py +++ b/e2e_test/infra/model_specs.py @@ -127,6 +127,57 @@ def _resolve_model_path(hf_path: str) -> str: "--attention-backend=FLASHINFER", # FLASHINFER attention backend ], }, + # Llama-4-Scout (17B with 16 experts) - Nightly benchmarks + "meta-llama/Llama-4-Scout-17B-16E-Instruct": { + "model": _resolve_model_path("meta-llama/Llama-4-Scout-17B-16E-Instruct"), + "tp": 4, + "features": ["chat", "streaming", "function_calling", "moe"], + "worker_args": [ + "--trust-remote-code", + "--context-length=196608", + "--attention-backend=fa3", + "--cuda-graph-max-bs=256", + "--max-running-requests=300", + "--mem-fraction-static=0.85", + ], + "vllm_args": [ + "--trust-remote-code", + "--max-model-len=196608", + "--attention-backend=FLASHINFER", + ], + }, + # Llama-3.3-70B - Nightly benchmarks + "meta-llama/Llama-3.3-70B-Instruct": { + "model": _resolve_model_path("meta-llama/Llama-3.3-70B-Instruct"), + "tp": 4, + "features": ["chat", "streaming", "function_calling"], + "worker_args": [ + "--trust-remote-code", + "--mem-frac=0.9", + ], + "vllm_args": [ + "--trust-remote-code", + "--max-model-len=131072", + "--gpu-memory-utilization=0.9", + "--enable-chunked-prefill", + ], + }, + # Llama-3.3-70B FP8 - Nightly benchmarks + "RedHatAI/Llama-3.3-70B-Instruct-FP8-dynamic": { + "model": _resolve_model_path("RedHatAI/Llama-3.3-70B-Instruct-FP8-dynamic"), + "tp": 4, + "features": ["chat", "streaming", "function_calling"], + "worker_args": [ + "--trust-remote-code", + "--mem-frac=0.9", + ], + "vllm_args": [ + "--trust-remote-code", + "--max-model-len=131072", + "--gpu-memory-utilization=0.9", + "--enable-chunked-prefill", + ], + }, } From 922aa748796dc5faffbc536c98613f0d1445a739 Mon Sep 17 00:00:00 2001 From: paxiaatucsdedu Date: Tue, 24 Mar 2026 15:55:16 -0700 Subject: [PATCH 02/14] Replace mem-frac arg Signed-off-by: paxiaatucsdedu --- e2e_test/infra/model_specs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e_test/infra/model_specs.py b/e2e_test/infra/model_specs.py index d62de3f7dc..8d66014cad 100644 --- a/e2e_test/infra/model_specs.py +++ b/e2e_test/infra/model_specs.py @@ -153,7 +153,7 @@ def _resolve_model_path(hf_path: str) -> str: "features": ["chat", "streaming", "function_calling"], "worker_args": [ "--trust-remote-code", - "--mem-frac=0.9", + "--mem-fraction-static=0.9", ], "vllm_args": [ "--trust-remote-code", @@ -169,7 +169,7 @@ def _resolve_model_path(hf_path: str) -> str: "features": ["chat", "streaming", "function_calling"], "worker_args": [ "--trust-remote-code", - "--mem-frac=0.9", + "--mem-fraction-static=0.9", ], "vllm_args": [ "--trust-remote-code", From 812cf0dcd1d894fd5acdf6f1472d7ca93171143a Mon Sep 17 00:00:00 2001 From: paxiaatucsdedu Date: Tue, 24 Mar 2026 16:14:10 -0700 Subject: [PATCH 03/14] Drop HF_TOKEN and HF_HOME env vars Signed-off-by: paxiaatucsdedu --- .github/workflows/nightly-benchmark.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/nightly-benchmark.yml b/.github/workflows/nightly-benchmark.yml index f5c468a195..e6012ede2c 100644 --- a/.github/workflows/nightly-benchmark.yml +++ b/.github/workflows/nightly-benchmark.yml @@ -191,8 +191,6 @@ jobs: E2E_NIGHTLY: "1" E2E_MODEL_TP_OVERRIDES: '{"meta-llama/Llama-3.1-8B-Instruct":1,"Qwen/Qwen2.5-7B-Instruct":1,"Qwen/Qwen3-30B-A3B":1}' E2E_LOG_DIR: nightly_gateway_logs - HF_TOKEN: ${{ secrets.HF_TOKEN }} - HF_HOME: /raid/models run: | mkdir -p nightly_gateway_logs bash scripts/ci_killall_sglang.sh "nuk_gpus" From 8bdbbf2c6002d2d448b29eec14465ea3b23286cd Mon Sep 17 00:00:00 2001 From: paxiaatucsdedu Date: Tue, 24 Mar 2026 18:31:36 -0700 Subject: [PATCH 04/14] Select genai-bench api backend from E2E_RUNTIME Signed-off-by: paxiaatucsdedu --- e2e_test/benchmarks/conftest.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/e2e_test/benchmarks/conftest.py b/e2e_test/benchmarks/conftest.py index 818ef8083d..0b769142c1 100644 --- a/e2e_test/benchmarks/conftest.py +++ b/e2e_test/benchmarks/conftest.py @@ -72,12 +72,16 @@ def _build_command( if os.path.isdir(local_tokenizer): tokenizer_path = local_tokenizer + # Map runtime to the correct genai-bench --api-backend value. + runtime = os.environ.get("E2E_RUNTIME", "").lower() + api_backend = {"sglang": "sglang", "vllm": "vllm"}.get(runtime, "openai") + cmd.extend( [ image, "benchmark", "--api-backend", - "openai", + api_backend, "--api-base", router_url, "--api-key", From a091cecde75caac1efe095b46e8a5e95b2430cdf Mon Sep 17 00:00:00 2001 From: key4ng Date: Tue, 24 Mar 2026 19:31:45 -0700 Subject: [PATCH 05/14] Temply test nighly bench in pr[do not merge] - Added pull request trigger for specific paths in the nightly benchmark workflow. - Commented out previous model configurations to streamline the benchmark process. - Set job conditions to false for multi-worker and single-worker jobs to prevent execution. Signed-off-by: key4ng --- .github/workflows/nightly-benchmark.yml | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/nightly-benchmark.yml b/.github/workflows/nightly-benchmark.yml index e6012ede2c..a29fda46c8 100644 --- a/.github/workflows/nightly-benchmark.yml +++ b/.github/workflows/nightly-benchmark.yml @@ -13,6 +13,11 @@ on: description: 'Runtime: sglang, vllm, or all (default: all)' required: false default: 'all' + pull_request: + branches: [ main ] + paths: + - 'e2e_test/benchmarks/test_nightly_perf.py' + - '.github/workflows/nightly-benchmark.yml' concurrency: group: nightly-benchmark-${{ github.event_name }} @@ -117,10 +122,10 @@ jobs: max-parallel: 8 matrix: model: - - { id: meta-llama/Llama-3.1-8B-Instruct, slug: meta-llama-Llama-3.1-8B-Instruct, test_class: TestNightlyLlama8bSingle } - - { id: Qwen/Qwen2.5-7B-Instruct, slug: Qwen-Qwen2.5-7B-Instruct, test_class: TestNightlyQwen7bSingle } - - { id: Qwen/Qwen3-30B-A3B, slug: Qwen-Qwen3-30B-A3B, test_class: TestNightlyQwen30bSingle } - - { id: openai/gpt-oss-20b, slug: openai-gpt-oss-20b, test_class: TestNightlyGptOss20bSingle } + # - { id: meta-llama/Llama-3.1-8B-Instruct, slug: meta-llama-Llama-3.1-8B-Instruct, test_class: TestNightlyLlama8bSingle } + # - { id: Qwen/Qwen2.5-7B-Instruct, slug: Qwen-Qwen2.5-7B-Instruct, test_class: TestNightlyQwen7bSingle } + # - { id: Qwen/Qwen3-30B-A3B, slug: Qwen-Qwen3-30B-A3B, test_class: TestNightlyQwen30bSingle } + # - { id: openai/gpt-oss-20b, slug: openai-gpt-oss-20b, test_class: TestNightlyGptOss20bSingle } - { id: meta-llama/Llama-4-Scout-17B-16E-Instruct, slug: meta-llama-Llama-4-Scout-17B-16E-Instruct, test_class: TestNightlyLlama4ScoutSingle } - { id: meta-llama/Llama-3.3-70B-Instruct, slug: meta-llama-Llama-3.3-70B-Instruct, test_class: TestNightlyLlama70bSingle } - { id: RedHatAI/Llama-3.3-70B-Instruct-FP8-dynamic, slug: RedHatAI-Llama-3.3-70B-Instruct-FP8-dynamic, test_class: TestNightlyLlama70bFp8Single } @@ -225,7 +230,7 @@ jobs: multi-worker: name: "${{ matrix.model.id }} / multi-${{ matrix.variant.id }}" needs: build-wheel - if: ${{ !cancelled() }} + if: false runs-on: 4-gpu-h100 timeout-minutes: 1440 strategy: @@ -339,7 +344,7 @@ jobs: single-worker-h200: name: "${{ matrix.model.id }} / single-${{ matrix.variant.id }}" needs: build-wheel - if: ${{ !cancelled() }} + if: false runs-on: ["8-gpu-h200"] timeout-minutes: 1440 strategy: From d72149641cdfb34f068e3ed78e1fc58933fc3e63 Mon Sep 17 00:00:00 2001 From: key4ng Date: Wed, 25 Mar 2026 14:42:22 -0700 Subject: [PATCH 06/14] Add startup_timeout for large MoE models in model_specs.py Signed-off-by: key4ng --- e2e_test/infra/model_specs.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/e2e_test/infra/model_specs.py b/e2e_test/infra/model_specs.py index 8d66014cad..3ddba08e16 100644 --- a/e2e_test/infra/model_specs.py +++ b/e2e_test/infra/model_specs.py @@ -126,6 +126,7 @@ def _resolve_model_path(hf_path: str) -> str: "--max-model-len=163840", # 160K context length (vLLM) "--attention-backend=FLASHINFER", # FLASHINFER attention backend ], + "startup_timeout": 1200, # Large MoE model may need extra download/load time }, # Llama-4-Scout (17B with 16 experts) - Nightly benchmarks "meta-llama/Llama-4-Scout-17B-16E-Instruct": { @@ -145,6 +146,7 @@ def _resolve_model_path(hf_path: str) -> str: "--max-model-len=196608", "--attention-backend=FLASHINFER", ], + "startup_timeout": 1200, # Large MoE model may need extra download/load time }, # Llama-3.3-70B - Nightly benchmarks "meta-llama/Llama-3.3-70B-Instruct": { From 90543e6ee957d7e0801ed4e1ae89a7093949408a Mon Sep 17 00:00:00 2001 From: key4ng Date: Wed, 25 Mar 2026 17:42:35 -0700 Subject: [PATCH 07/14] Update attention backend in model_specs.py from FLASHINFER to FLASH_ATTN Signed-off-by: key4ng --- e2e_test/infra/model_specs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e_test/infra/model_specs.py b/e2e_test/infra/model_specs.py index 3ddba08e16..1022987c72 100644 --- a/e2e_test/infra/model_specs.py +++ b/e2e_test/infra/model_specs.py @@ -144,7 +144,7 @@ def _resolve_model_path(hf_path: str) -> str: "vllm_args": [ "--trust-remote-code", "--max-model-len=196608", - "--attention-backend=FLASHINFER", + "--attention-backend=FLASH_ATTN", ], "startup_timeout": 1200, # Large MoE model may need extra download/load time }, From a8ae4a622deaf954e2c91710e6659fae9c615352 Mon Sep 17 00:00:00 2001 From: key4ng Date: Wed, 25 Mar 2026 18:27:18 -0700 Subject: [PATCH 08/14] test done, revert changes for testing Signed-off-by: key4ng --- .github/workflows/nightly-benchmark.yml | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/.github/workflows/nightly-benchmark.yml b/.github/workflows/nightly-benchmark.yml index a29fda46c8..e6012ede2c 100644 --- a/.github/workflows/nightly-benchmark.yml +++ b/.github/workflows/nightly-benchmark.yml @@ -13,11 +13,6 @@ on: description: 'Runtime: sglang, vllm, or all (default: all)' required: false default: 'all' - pull_request: - branches: [ main ] - paths: - - 'e2e_test/benchmarks/test_nightly_perf.py' - - '.github/workflows/nightly-benchmark.yml' concurrency: group: nightly-benchmark-${{ github.event_name }} @@ -122,10 +117,10 @@ jobs: max-parallel: 8 matrix: model: - # - { id: meta-llama/Llama-3.1-8B-Instruct, slug: meta-llama-Llama-3.1-8B-Instruct, test_class: TestNightlyLlama8bSingle } - # - { id: Qwen/Qwen2.5-7B-Instruct, slug: Qwen-Qwen2.5-7B-Instruct, test_class: TestNightlyQwen7bSingle } - # - { id: Qwen/Qwen3-30B-A3B, slug: Qwen-Qwen3-30B-A3B, test_class: TestNightlyQwen30bSingle } - # - { id: openai/gpt-oss-20b, slug: openai-gpt-oss-20b, test_class: TestNightlyGptOss20bSingle } + - { id: meta-llama/Llama-3.1-8B-Instruct, slug: meta-llama-Llama-3.1-8B-Instruct, test_class: TestNightlyLlama8bSingle } + - { id: Qwen/Qwen2.5-7B-Instruct, slug: Qwen-Qwen2.5-7B-Instruct, test_class: TestNightlyQwen7bSingle } + - { id: Qwen/Qwen3-30B-A3B, slug: Qwen-Qwen3-30B-A3B, test_class: TestNightlyQwen30bSingle } + - { id: openai/gpt-oss-20b, slug: openai-gpt-oss-20b, test_class: TestNightlyGptOss20bSingle } - { id: meta-llama/Llama-4-Scout-17B-16E-Instruct, slug: meta-llama-Llama-4-Scout-17B-16E-Instruct, test_class: TestNightlyLlama4ScoutSingle } - { id: meta-llama/Llama-3.3-70B-Instruct, slug: meta-llama-Llama-3.3-70B-Instruct, test_class: TestNightlyLlama70bSingle } - { id: RedHatAI/Llama-3.3-70B-Instruct-FP8-dynamic, slug: RedHatAI-Llama-3.3-70B-Instruct-FP8-dynamic, test_class: TestNightlyLlama70bFp8Single } @@ -230,7 +225,7 @@ jobs: multi-worker: name: "${{ matrix.model.id }} / multi-${{ matrix.variant.id }}" needs: build-wheel - if: false + if: ${{ !cancelled() }} runs-on: 4-gpu-h100 timeout-minutes: 1440 strategy: @@ -344,7 +339,7 @@ jobs: single-worker-h200: name: "${{ matrix.model.id }} / single-${{ matrix.variant.id }}" needs: build-wheel - if: false + if: ${{ !cancelled() }} runs-on: ["8-gpu-h200"] timeout-minutes: 1440 strategy: From 43247f5348f37f1e0f4d9d3cc9a3d00fe6fdaa36 Mon Sep 17 00:00:00 2001 From: paxiaatucsdedu Date: Thu, 26 Mar 2026 09:25:38 -0700 Subject: [PATCH 09/14] feat(e2e): add HiCache/flashinfer args for Llama-70B and disable ignore_eos for SGLang Signed-off-by: paxiaatucsdedu --- e2e_test/benchmarks/conftest.py | 5 +++++ e2e_test/infra/model_specs.py | 14 ++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/e2e_test/benchmarks/conftest.py b/e2e_test/benchmarks/conftest.py index 0b769142c1..b4261f315e 100644 --- a/e2e_test/benchmarks/conftest.py +++ b/e2e_test/benchmarks/conftest.py @@ -112,6 +112,11 @@ def _build_command( cmd.extend(["--server-gpu-type", gpu_type]) if gpu_count: cmd.extend(["--server-gpu-count", str(gpu_count)]) + # Disable ignore_eos for SGLang: its scheduler was specifically tuned for + # ignore_eos, so normal workloads (without it) are actually slower. + if api_backend == "sglang": + cmd.extend(["--additional-request-params", '{"ignore_eos": false}']) + log_dir = os.environ.get("E2E_LOG_DIR") if log_dir: cmd.extend(["--log-dir", log_dir]) diff --git a/e2e_test/infra/model_specs.py b/e2e_test/infra/model_specs.py index 1022987c72..dc4782fc28 100644 --- a/e2e_test/infra/model_specs.py +++ b/e2e_test/infra/model_specs.py @@ -156,6 +156,13 @@ def _resolve_model_path(hf_path: str) -> str: "worker_args": [ "--trust-remote-code", "--mem-fraction-static=0.9", + "--enable-hierarchical-cache", + "--hicache-size=80", + "--enable-cache-report", + "--prefill-attention-backend=flashinfer", + "--decode-attention-backend=flashinfer", + "--hicache-io-backend=kernel", + "--hicache-mem-layout=layer_first", ], "vllm_args": [ "--trust-remote-code", @@ -172,6 +179,13 @@ def _resolve_model_path(hf_path: str) -> str: "worker_args": [ "--trust-remote-code", "--mem-fraction-static=0.9", + "--enable-hierarchical-cache", + "--hicache-size=80", + "--enable-cache-report", + "--prefill-attention-backend=flashinfer", + "--decode-attention-backend=flashinfer", + "--hicache-io-backend=kernel", + "--hicache-mem-layout=layer_first", ], "vllm_args": [ "--trust-remote-code", From a4175e25cf948db9b0e3848ed9a89583040fcf02 Mon Sep 17 00:00:00 2001 From: paxiaatucsdedu Date: Thu, 26 Mar 2026 11:03:26 -0700 Subject: [PATCH 10/14] test(e2e): enable PR trigger for benchmark testing [do not merge] - Add pull_request trigger for benchmark workflow (per b0cc8fd pattern) - Comment out existing models to reduce resource usage during testing - Set multi-worker and single-worker-h200 jobs to if:false Signed-off-by: paxiaatucsdedu --- .github/workflows/nightly-benchmark.yml | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/nightly-benchmark.yml b/.github/workflows/nightly-benchmark.yml index e6012ede2c..a29fda46c8 100644 --- a/.github/workflows/nightly-benchmark.yml +++ b/.github/workflows/nightly-benchmark.yml @@ -13,6 +13,11 @@ on: description: 'Runtime: sglang, vllm, or all (default: all)' required: false default: 'all' + pull_request: + branches: [ main ] + paths: + - 'e2e_test/benchmarks/test_nightly_perf.py' + - '.github/workflows/nightly-benchmark.yml' concurrency: group: nightly-benchmark-${{ github.event_name }} @@ -117,10 +122,10 @@ jobs: max-parallel: 8 matrix: model: - - { id: meta-llama/Llama-3.1-8B-Instruct, slug: meta-llama-Llama-3.1-8B-Instruct, test_class: TestNightlyLlama8bSingle } - - { id: Qwen/Qwen2.5-7B-Instruct, slug: Qwen-Qwen2.5-7B-Instruct, test_class: TestNightlyQwen7bSingle } - - { id: Qwen/Qwen3-30B-A3B, slug: Qwen-Qwen3-30B-A3B, test_class: TestNightlyQwen30bSingle } - - { id: openai/gpt-oss-20b, slug: openai-gpt-oss-20b, test_class: TestNightlyGptOss20bSingle } + # - { id: meta-llama/Llama-3.1-8B-Instruct, slug: meta-llama-Llama-3.1-8B-Instruct, test_class: TestNightlyLlama8bSingle } + # - { id: Qwen/Qwen2.5-7B-Instruct, slug: Qwen-Qwen2.5-7B-Instruct, test_class: TestNightlyQwen7bSingle } + # - { id: Qwen/Qwen3-30B-A3B, slug: Qwen-Qwen3-30B-A3B, test_class: TestNightlyQwen30bSingle } + # - { id: openai/gpt-oss-20b, slug: openai-gpt-oss-20b, test_class: TestNightlyGptOss20bSingle } - { id: meta-llama/Llama-4-Scout-17B-16E-Instruct, slug: meta-llama-Llama-4-Scout-17B-16E-Instruct, test_class: TestNightlyLlama4ScoutSingle } - { id: meta-llama/Llama-3.3-70B-Instruct, slug: meta-llama-Llama-3.3-70B-Instruct, test_class: TestNightlyLlama70bSingle } - { id: RedHatAI/Llama-3.3-70B-Instruct-FP8-dynamic, slug: RedHatAI-Llama-3.3-70B-Instruct-FP8-dynamic, test_class: TestNightlyLlama70bFp8Single } @@ -225,7 +230,7 @@ jobs: multi-worker: name: "${{ matrix.model.id }} / multi-${{ matrix.variant.id }}" needs: build-wheel - if: ${{ !cancelled() }} + if: false runs-on: 4-gpu-h100 timeout-minutes: 1440 strategy: @@ -339,7 +344,7 @@ jobs: single-worker-h200: name: "${{ matrix.model.id }} / single-${{ matrix.variant.id }}" needs: build-wheel - if: ${{ !cancelled() }} + if: false runs-on: ["8-gpu-h200"] timeout-minutes: 1440 strategy: From 17da2dc8f54dfd5ab85cac5e352f987f545c9913 Mon Sep 17 00:00:00 2001 From: paxiaatucsdedu Date: Thu, 26 Mar 2026 14:32:56 -0700 Subject: [PATCH 11/14] Auto-answer genai-bench prompt for sglang Signed-off-by: paxiaatucsdedu --- e2e_test/benchmarks/conftest.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/e2e_test/benchmarks/conftest.py b/e2e_test/benchmarks/conftest.py index b4261f315e..2da9f10949 100644 --- a/e2e_test/benchmarks/conftest.py +++ b/e2e_test/benchmarks/conftest.py @@ -4,6 +4,7 @@ import logging import os +import shlex import shutil import subprocess import time @@ -114,12 +115,20 @@ def _build_command( cmd.extend(["--server-gpu-count", str(gpu_count)]) # Disable ignore_eos for SGLang: its scheduler was specifically tuned for # ignore_eos, so normal workloads (without it) are actually slower. + # genai-bench prompts interactively when ignore_eos=false; pipe "N" via + # stdin so CI auto-answers "No" (keep ignore_eos=false). + needs_stdin_pipe = False if api_backend == "sglang": cmd.extend(["--additional-request-params", '{"ignore_eos": false}']) + needs_stdin_pipe = True log_dir = os.environ.get("E2E_LOG_DIR") if log_dir: cmd.extend(["--log-dir", log_dir]) + + if needs_stdin_pipe: + cmd = ["bash", "-c", f"echo N | {shlex.join(cmd)}"] + return cmd From e73a6c595e35bd36caead1def7230d606d8770ad Mon Sep 17 00:00:00 2001 From: paxiaatucsdedu Date: Fri, 27 Mar 2026 10:28:58 -0700 Subject: [PATCH 12/14] fix(e2e): revert conftest.py to main, remove HiCache flags, fix duplicate Scout entry Signed-off-by: paxiaatucsdedu --- e2e_test/benchmarks/conftest.py | 20 +------------------- e2e_test/infra/model_specs.py | 27 ++------------------------- 2 files changed, 3 insertions(+), 44 deletions(-) diff --git a/e2e_test/benchmarks/conftest.py b/e2e_test/benchmarks/conftest.py index 2da9f10949..818ef8083d 100644 --- a/e2e_test/benchmarks/conftest.py +++ b/e2e_test/benchmarks/conftest.py @@ -4,7 +4,6 @@ import logging import os -import shlex import shutil import subprocess import time @@ -73,16 +72,12 @@ def _build_command( if os.path.isdir(local_tokenizer): tokenizer_path = local_tokenizer - # Map runtime to the correct genai-bench --api-backend value. - runtime = os.environ.get("E2E_RUNTIME", "").lower() - api_backend = {"sglang": "sglang", "vllm": "vllm"}.get(runtime, "openai") - cmd.extend( [ image, "benchmark", "--api-backend", - api_backend, + "openai", "--api-base", router_url, "--api-key", @@ -113,22 +108,9 @@ def _build_command( cmd.extend(["--server-gpu-type", gpu_type]) if gpu_count: cmd.extend(["--server-gpu-count", str(gpu_count)]) - # Disable ignore_eos for SGLang: its scheduler was specifically tuned for - # ignore_eos, so normal workloads (without it) are actually slower. - # genai-bench prompts interactively when ignore_eos=false; pipe "N" via - # stdin so CI auto-answers "No" (keep ignore_eos=false). - needs_stdin_pipe = False - if api_backend == "sglang": - cmd.extend(["--additional-request-params", '{"ignore_eos": false}']) - needs_stdin_pipe = True - log_dir = os.environ.get("E2E_LOG_DIR") if log_dir: cmd.extend(["--log-dir", log_dir]) - - if needs_stdin_pipe: - cmd = ["bash", "-c", f"echo N | {shlex.join(cmd)}"] - return cmd diff --git a/e2e_test/infra/model_specs.py b/e2e_test/infra/model_specs.py index dc4782fc28..c445dd97ed 100644 --- a/e2e_test/infra/model_specs.py +++ b/e2e_test/infra/model_specs.py @@ -102,14 +102,6 @@ def _resolve_model_path(hf_path: str) -> str: "tp": 1, "features": ["chat", "streaming", "multimodal"], }, - # Llama-4-Scout (17B with 16 experts) - Multimodal tests - "meta-llama/Llama-4-Scout-17B-16E-Instruct": { - "model": _resolve_model_path("meta-llama/Llama-4-Scout-17B-16E-Instruct"), - "tp": 4, - "features": ["chat", "streaming", "multimodal", "moe"], - "vllm_args": ["--max-model-len", "196608"], - "startup_timeout": 1200, - }, # Llama-4-Maverick (17B with 128 experts, FP8) - Nightly benchmarks "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8": { "model": _resolve_model_path("meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8"), @@ -128,11 +120,11 @@ def _resolve_model_path(hf_path: str) -> str: ], "startup_timeout": 1200, # Large MoE model may need extra download/load time }, - # Llama-4-Scout (17B with 16 experts) - Nightly benchmarks + # Llama-4-Scout (17B with 16 experts) - Nightly benchmarks and Multimodal tests "meta-llama/Llama-4-Scout-17B-16E-Instruct": { "model": _resolve_model_path("meta-llama/Llama-4-Scout-17B-16E-Instruct"), "tp": 4, - "features": ["chat", "streaming", "function_calling", "moe"], + "features": ["chat", "streaming", "function_calling", "multimodal", "moe"], "worker_args": [ "--trust-remote-code", "--context-length=196608", @@ -144,7 +136,6 @@ def _resolve_model_path(hf_path: str) -> str: "vllm_args": [ "--trust-remote-code", "--max-model-len=196608", - "--attention-backend=FLASH_ATTN", ], "startup_timeout": 1200, # Large MoE model may need extra download/load time }, @@ -156,13 +147,6 @@ def _resolve_model_path(hf_path: str) -> str: "worker_args": [ "--trust-remote-code", "--mem-fraction-static=0.9", - "--enable-hierarchical-cache", - "--hicache-size=80", - "--enable-cache-report", - "--prefill-attention-backend=flashinfer", - "--decode-attention-backend=flashinfer", - "--hicache-io-backend=kernel", - "--hicache-mem-layout=layer_first", ], "vllm_args": [ "--trust-remote-code", @@ -179,13 +163,6 @@ def _resolve_model_path(hf_path: str) -> str: "worker_args": [ "--trust-remote-code", "--mem-fraction-static=0.9", - "--enable-hierarchical-cache", - "--hicache-size=80", - "--enable-cache-report", - "--prefill-attention-backend=flashinfer", - "--decode-attention-backend=flashinfer", - "--hicache-io-backend=kernel", - "--hicache-mem-layout=layer_first", ], "vllm_args": [ "--trust-remote-code", From 91d1775cc6a3f90a9893f20a02f4a70e7b8bff8c Mon Sep 17 00:00:00 2001 From: paxiaatucsdedu Date: Fri, 27 Mar 2026 11:56:14 -0700 Subject: [PATCH 13/14] test(e2e): remove unnecessary --trust-remote-code from Llama model specs SGLang v0.5.9 and vLLM v0.18.0 both have native support for Llama-4 and Llama-3.3 architectures in their model registries, so --trust-remote-code is not needed for these models. Removed from: - meta-llama/Llama-4-Scout-17B-16E-Instruct (worker_args + vllm_args) - meta-llama/Llama-3.3-70B-Instruct (worker_args + vllm_args) - RedHatAI/Llama-3.3-70B-Instruct-FP8-dynamic (worker_args + vllm_args) Signed-off-by: paxiaatucsdedu --- e2e_test/infra/model_specs.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/e2e_test/infra/model_specs.py b/e2e_test/infra/model_specs.py index c445dd97ed..64ee4ee1f5 100644 --- a/e2e_test/infra/model_specs.py +++ b/e2e_test/infra/model_specs.py @@ -126,7 +126,6 @@ def _resolve_model_path(hf_path: str) -> str: "tp": 4, "features": ["chat", "streaming", "function_calling", "multimodal", "moe"], "worker_args": [ - "--trust-remote-code", "--context-length=196608", "--attention-backend=fa3", "--cuda-graph-max-bs=256", @@ -134,7 +133,6 @@ def _resolve_model_path(hf_path: str) -> str: "--mem-fraction-static=0.85", ], "vllm_args": [ - "--trust-remote-code", "--max-model-len=196608", ], "startup_timeout": 1200, # Large MoE model may need extra download/load time @@ -145,11 +143,9 @@ def _resolve_model_path(hf_path: str) -> str: "tp": 4, "features": ["chat", "streaming", "function_calling"], "worker_args": [ - "--trust-remote-code", "--mem-fraction-static=0.9", ], "vllm_args": [ - "--trust-remote-code", "--max-model-len=131072", "--gpu-memory-utilization=0.9", "--enable-chunked-prefill", @@ -161,11 +157,9 @@ def _resolve_model_path(hf_path: str) -> str: "tp": 4, "features": ["chat", "streaming", "function_calling"], "worker_args": [ - "--trust-remote-code", "--mem-fraction-static=0.9", ], "vllm_args": [ - "--trust-remote-code", "--max-model-len=131072", "--gpu-memory-utilization=0.9", "--enable-chunked-prefill", From 02a457e1345ff7386488c34477e27357e0d20c89 Mon Sep 17 00:00:00 2001 From: paxiaatucsdedu Date: Fri, 27 Mar 2026 13:22:03 -0700 Subject: [PATCH 14/14] revert: remove PR trigger for benchmark testing (revert e3afaad1) Signed-off-by: paxiaatucsdedu --- .github/workflows/nightly-benchmark.yml | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/.github/workflows/nightly-benchmark.yml b/.github/workflows/nightly-benchmark.yml index a29fda46c8..e6012ede2c 100644 --- a/.github/workflows/nightly-benchmark.yml +++ b/.github/workflows/nightly-benchmark.yml @@ -13,11 +13,6 @@ on: description: 'Runtime: sglang, vllm, or all (default: all)' required: false default: 'all' - pull_request: - branches: [ main ] - paths: - - 'e2e_test/benchmarks/test_nightly_perf.py' - - '.github/workflows/nightly-benchmark.yml' concurrency: group: nightly-benchmark-${{ github.event_name }} @@ -122,10 +117,10 @@ jobs: max-parallel: 8 matrix: model: - # - { id: meta-llama/Llama-3.1-8B-Instruct, slug: meta-llama-Llama-3.1-8B-Instruct, test_class: TestNightlyLlama8bSingle } - # - { id: Qwen/Qwen2.5-7B-Instruct, slug: Qwen-Qwen2.5-7B-Instruct, test_class: TestNightlyQwen7bSingle } - # - { id: Qwen/Qwen3-30B-A3B, slug: Qwen-Qwen3-30B-A3B, test_class: TestNightlyQwen30bSingle } - # - { id: openai/gpt-oss-20b, slug: openai-gpt-oss-20b, test_class: TestNightlyGptOss20bSingle } + - { id: meta-llama/Llama-3.1-8B-Instruct, slug: meta-llama-Llama-3.1-8B-Instruct, test_class: TestNightlyLlama8bSingle } + - { id: Qwen/Qwen2.5-7B-Instruct, slug: Qwen-Qwen2.5-7B-Instruct, test_class: TestNightlyQwen7bSingle } + - { id: Qwen/Qwen3-30B-A3B, slug: Qwen-Qwen3-30B-A3B, test_class: TestNightlyQwen30bSingle } + - { id: openai/gpt-oss-20b, slug: openai-gpt-oss-20b, test_class: TestNightlyGptOss20bSingle } - { id: meta-llama/Llama-4-Scout-17B-16E-Instruct, slug: meta-llama-Llama-4-Scout-17B-16E-Instruct, test_class: TestNightlyLlama4ScoutSingle } - { id: meta-llama/Llama-3.3-70B-Instruct, slug: meta-llama-Llama-3.3-70B-Instruct, test_class: TestNightlyLlama70bSingle } - { id: RedHatAI/Llama-3.3-70B-Instruct-FP8-dynamic, slug: RedHatAI-Llama-3.3-70B-Instruct-FP8-dynamic, test_class: TestNightlyLlama70bFp8Single } @@ -230,7 +225,7 @@ jobs: multi-worker: name: "${{ matrix.model.id }} / multi-${{ matrix.variant.id }}" needs: build-wheel - if: false + if: ${{ !cancelled() }} runs-on: 4-gpu-h100 timeout-minutes: 1440 strategy: @@ -344,7 +339,7 @@ jobs: single-worker-h200: name: "${{ matrix.model.id }} / single-${{ matrix.variant.id }}" needs: build-wheel - if: false + if: ${{ !cancelled() }} runs-on: ["8-gpu-h200"] timeout-minutes: 1440 strategy: