diff --git a/models/deepseek-ai/DeepSeek-V4.1-Flash.yaml b/models/deepseek-ai/DeepSeek-V4.1-Flash.yaml index 5b059cb8..32e0648a 100644 --- a/models/deepseek-ai/DeepSeek-V4.1-Flash.yaml +++ b/models/deepseek-ai/DeepSeek-V4.1-Flash.yaml @@ -5,7 +5,7 @@ meta: provider: "DeepSeek" description: "DeepSeek V4.1 Flash vision-language MoE (552B backbone; 8B active per prompt token, 16B per output token) combining sliding-window plus compressed sparse attention with a two-level indexer, engram n-gram memory, hyper-connections, and a DSpark multi-token draft head." date_added: 2026-09-09 - date_updated: 2026-09-12 + date_updated: 2026-09-13 difficulty: advanced tasks: - text @@ -20,6 +20,7 @@ meta: gb200: verified gb300: verified mi350x: verified + mi355x: verified model: model_id: "deepseek-ai/DeepSeek-V4.1-Flash" @@ -27,7 +28,10 @@ model: default_frontend: rust docker_image: nvidia: "vllm/vllm-openai:deepseekv41-flash-0909" - amd: "vllm/vllm-openai-rocm:deepseekv41-flash-0909" + # ROCm nightly rather than the 0909 tag: 0909 predates + # vllm-project/vllm#56503, which moves the mHC delayed pre block off the + # eager Torch reference and onto AITER. + amd: "vllm/vllm-openai-rocm:nightly-eed1f3d0c6043bd494424a22443ee198dd56f657" architecture: moe parameter_count: "552B" active_parameters: "8-16B" @@ -39,7 +43,7 @@ model: VLLM_ENGINE_READY_TIMEOUT_S: "3600" install: docker: - note: "Serve from the dedicated vllm/vllm-openai:deepseekv41-flash-0909 image — no pip wheel carries the DeepSeek-V4.1 architecture." + note: "Serve from the pinned image for your GPU vendor — no pip wheel carries the DeepSeek-V4.1 architecture. NVIDIA pins the 0909 release tag; AMD pins a ROCm nightly (see Prerequisites)." pip: false dependencies: [] @@ -111,11 +115,18 @@ hardware_overrides: extra_args: - "--gpu-memory-utilization" - "0.9" + # Plain "aiter" opens vLLM's full MoE priority list, whose head is the + # Composable Kernel backend, giving the a8w4 experts. Naming + # aiter_triton_mxfp4_bf16 instead pins the Triton W4A16 _moe_gemm_a16w4. - "--moe-backend" - - "aiter_triton_mxfp4_bf16" + - "aiter" extra_env: VLLM_ROCM_USE_AITER: "1" VLLM_ROCM_USE_AITER_MOE: "1" + # aiter.ops.triton warns on every call that Gluon is unavailable and it is + # falling back to Triton. Gluon is gfx1250-only, so on gfx950 that is a + # fixed property, and it was 98% of the lines in a gsm8k server log. + AITER_TRITON_LOG_LEVEL: "ERROR" # DeepseekV41ForCausalLM does not support torch.compile. The ROCm sparse # SWA backend only reports UNIFORM_BATCH, so default FULL_AND_PIECEWISE # cannot start unless breakable CUDA graphs are on. @@ -123,8 +134,17 @@ hardware_overrides: strategy_overrides: single_node_tp: - # Verified at TP4 (one GB200 NVL4 tray; 4 of 8 GPUs on an MI350X node). + # Verified at TP4 (one GB200 NVL4 tray; 4 of 8 GPUs on an MI350X or + # MI355X node). tp: 4 + hardware_overrides: + mi355x: + # vllm serve defaults max_num_seqs to 1024 on >=160 GiB GPUs, but the + # MI355X run this recipe is verified against served 128. Pinned so the + # published command reproduces that run, and only for this pairing. + extra_args: + - "--max-num-seqs" + - "128" pd_cluster: # Verified 1P1D on GB200 NVL4: one tray per role, TP4 each, NIXL transfer. env: @@ -303,12 +323,18 @@ guide: | ## Prerequisites - - The `vllm/vllm-openai:deepseekv41-flash-0909` image (vLLM 0.30.0+). No pip wheel serves - this architecture, so the Install block only offers Docker. + - An image (vLLM 0.30.0+); no pip wheel serves this architecture, so the Install block + only offers Docker. On NVIDIA that is `vllm/vllm-openai:deepseekv41-flash-0909`. On AMD + it is `vllm/vllm-openai-rocm:nightly-eed1f3d0c6043bd494424a22443ee198dd56f657` — a ROCm + nightly rather than the 0909 tag, because 0909 predates + [vllm-project/vllm#56503](https://github.com/vllm-project/vllm/pull/56503), which moves + the mHC delayed pre block off the eager Torch reference and onto AITER. - The Rust OpenAI frontend is selected by default in the command builder. Switch to Python if you encounter unsupported features or compatibility issues. - Expect a long first load: `VLLM_ENGINE_READY_TIMEOUT_S=3600` is set for that reason. - On AMD, the generated command sets `VLLM_USE_BREAKABLE_CUDAGRAPH=1`. DeepSeek-V4.1-Flash does not support `torch.compile`, and the ROCm sparse SWA backend only supports uniform-batch CUDA graphs. Without breakable CUDA graphs, default `FULL_AND_PIECEWISE` dies at capture. + - On AMD, the generated command passes `--moe-backend aiter` rather than naming a kernel. Naming `aiter_triton_mxfp4_bf16` pins the Triton W4A16 `_moe_gemm_a16w4` kernel; the plain name lets vLLM select the Composable Kernel a8w4 experts, which measured 4–9% faster per decode step at 131k context on MI355X, with larger gains at higher concurrency. It also sets `AITER_TRITON_LOG_LEVEL=ERROR` to drop AITER's per-call "Gluon unavailable" warning, which is constant on gfx950 and dominated the server log. + - On MI355X, **Tensor Parallel** additionally pins `--max-num-seqs 128`, the value the verified run served. Left unset, `vllm serve` defaults to 1024 on GPUs with at least 160 GiB. Raise it if your traffic needs more scheduler capacity than the verified configuration. - On AMD, ticking **Speculative decoding** also sets `enable_adaptive_verification:false`. vLLM currently refuses the true flag: `DeepseekV41IndexerBackend.supports_device_cpu_query_lens_mismatch()` is False, and `DeepseekV41ROCMAiterSparseSWABackend` reports `UNIFORM_BATCH` rather than `ALWAYS`. DSpark still drafts 5 tokens per round. ## Verifying