diff --git a/.github/workflows/nightly-benchmark.yml b/.github/workflows/nightly-benchmark.yml index fba3023ad4..4089312b7d 100644 --- a/.github/workflows/nightly-benchmark.yml +++ b/.github/workflows/nightly-benchmark.yml @@ -350,6 +350,7 @@ jobs: - { id: meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8, slug: meta-llama-Llama-4-Maverick-17B-128E-Instruct-FP8, test_class: TestNightlyLlama4MaverickSingle } # Pausing MiniMax M2 nightly benchmark # - { id: minimaxai/minimax-m2, slug: minimaxai-minimax-m2, test_class: TestNightlyMinimaxM2Single } + - { id: mistralai/Devstral-2-123B-Instruct-2512, slug: mistralai-Devstral-2-123B-Instruct-2512, test_class: TestNightlyDevstral2Single } variant: - { id: sglang, runtime: sglang, grpc_only: "false", setup_vllm: false, setup_trtllm: false, extra_deps: "genai-bench" } - { id: vllm, runtime: vllm, grpc_only: "false", setup_vllm: true, setup_trtllm: false, extra_deps: "genai-bench" } diff --git a/e2e_test/benchmarks/test_nightly_perf.py b/e2e_test/benchmarks/test_nightly_perf.py index 51b3d290a0..525470748c 100644 --- a/e2e_test/benchmarks/test_nightly_perf.py +++ b/e2e_test/benchmarks/test_nightly_perf.py @@ -102,6 +102,13 @@ def _run_nightly(setup_backend, genai_bench_runner, model_id, worker_count=1, ** ("Qwen/Qwen3-30B-A3B", "Qwen30b", 4, ["http", "grpc"], {}), ("openai/gpt-oss-20b", "GptOss20b", 1, ["http", "grpc"], {}), ("minimaxai/minimax-m2", "MinimaxM2", 1, ["http", "grpc"], {}), + ( + "mistralai/Devstral-2-123B-Instruct-2512", + "Devstral2", + 1, + ["http", "grpc"], + {}, + ), ( "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8", "Llama4Maverick", diff --git a/e2e_test/infra/model_specs.py b/e2e_test/infra/model_specs.py index 3951eef0b7..36338868e4 100644 --- a/e2e_test/infra/model_specs.py +++ b/e2e_test/infra/model_specs.py @@ -114,6 +114,15 @@ def _resolve_model_path(hf_path: str) -> str: "sglang_args": ["--trust-remote-code"], "vllm_args": ["--trust-remote-code"], }, + # Devstral 2 123B - Nightly benchmarks + "mistralai/Devstral-2-123B-Instruct-2512": { + "model": _resolve_model_path("mistralai/Devstral-2-123B-Instruct-2512"), + "tp": 4, + "features": ["chat", "streaming", "function_calling", "reasoning"], + "startup_timeout": 1200, + "sglang_args": ["--trust-remote-code"], + "vllm_args": ["--trust-remote-code"], + }, # Vision-language model for multimodal benchmarks (MMMU) "Qwen/Qwen3-VL-8B-Instruct": { "model": _resolve_model_path("Qwen/Qwen3-VL-8B-Instruct"),