From 3732dc767a20e2d00ddeb8932b6dbb62640e352c Mon Sep 17 00:00:00 2001 From: Nishanth Prakash Date: Wed, 1 Apr 2026 17:05:06 -0700 Subject: [PATCH] feat(ci): add Devstral 2 nightly benchmark Signed-off-by: Nishanth Prakash --- .github/workflows/nightly-benchmark.yml | 3 +++ e2e_test/benchmarks/test_nightly_perf.py | 1 + e2e_test/infra/model_specs.py | 9 +++++++++ 3 files changed, 13 insertions(+) diff --git a/.github/workflows/nightly-benchmark.yml b/.github/workflows/nightly-benchmark.yml index fba3023ad4..973ba83d1e 100644 --- a/.github/workflows/nightly-benchmark.yml +++ b/.github/workflows/nightly-benchmark.yml @@ -348,6 +348,9 @@ jobs: matrix: model: - { id: meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8, slug: meta-llama-Llama-4-Maverick-17B-128E-Instruct-FP8, test_class: TestNightlyLlama4MaverickSingle } + - id: mistralai/Devstral-2-123B-Instruct-2512 + slug: mistralai-Devstral-2-123B-Instruct-2512 + test_class: TestNightlyDevstral123bSingle # Pausing MiniMax M2 nightly benchmark # - { id: minimaxai/minimax-m2, slug: minimaxai-minimax-m2, test_class: TestNightlyMinimaxM2Single } variant: diff --git a/e2e_test/benchmarks/test_nightly_perf.py b/e2e_test/benchmarks/test_nightly_perf.py index 51b3d290a0..d987fee104 100644 --- a/e2e_test/benchmarks/test_nightly_perf.py +++ b/e2e_test/benchmarks/test_nightly_perf.py @@ -102,6 +102,7 @@ def _run_nightly(setup_backend, genai_bench_runner, model_id, worker_count=1, ** ("Qwen/Qwen3-30B-A3B", "Qwen30b", 4, ["http", "grpc"], {}), ("openai/gpt-oss-20b", "GptOss20b", 1, ["http", "grpc"], {}), ("minimaxai/minimax-m2", "MinimaxM2", 1, ["http", "grpc"], {}), + ("mistralai/Devstral-2-123B-Instruct-2512", "Devstral123b", 1, ["http", "grpc"], {}), ( "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8", "Llama4Maverick", diff --git a/e2e_test/infra/model_specs.py b/e2e_test/infra/model_specs.py index 5898e34c5b..adc898160b 100644 --- a/e2e_test/infra/model_specs.py +++ b/e2e_test/infra/model_specs.py @@ -77,6 +77,15 @@ def _resolve_model_path(hf_path: str) -> str: '{"disable_any_whitespace": true, "backend": "xgrammar"}', ], }, + # Devstral 2 - nightly benchmarks + "mistralai/Devstral-2-123B-Instruct-2512": { + "model": _resolve_model_path("mistralai/Devstral-2-123B-Instruct-2512"), + "tp": 8, + "features": ["chat", "streaming", "function_calling"], + "worker_args": ["--tool-call-parser", "mistral"], + "vllm_args": ["--tool-call-parser", "mistral", "--enable-auto-tool-choice"], + "startup_timeout": 1200, + }, # Embedding model "intfloat/e5-mistral-7b-instruct": { "model": _resolve_model_path("intfloat/e5-mistral-7b-instruct"),