diff --git a/models/deepseek-ai/DeepSeek-V4-Flash.yaml b/models/deepseek-ai/DeepSeek-V4-Flash.yaml index 19de0c85c..a6f025902 100644 --- a/models/deepseek-ai/DeepSeek-V4-Flash.yaml +++ b/models/deepseek-ai/DeepSeek-V4-Flash.yaml @@ -3,7 +3,7 @@ meta: slug: "deepseek-v4-flash" provider: "DeepSeek" description: "DeepSeek V4 MoE model with hybrid CSA+HCA attention, manifold-constrained hyper-connections, and three-tier reasoning (Non-think / Think High / Think Max)." - date_updated: 2026-05-28 + date_updated: 2026-05-01 difficulty: hard tasks: - text @@ -18,11 +18,13 @@ meta: dgx_station_gb300: verified mi300x: unsupported mi325x: unsupported - mi355x: unsupported + mi355x: verified model: model_id: "deepseek-ai/DeepSeek-V4-Flash" min_vllm_version: "0.20.0" + docker_image: + amd: "vllm/vllm-openai-rocm:nightly" architecture: moe parameter_count: "284B" active_parameters: "13B" @@ -91,6 +93,20 @@ hardware_overrides: - "--attention_config.use_fp4_indexer_cache=True" - "--moe-backend" - "deep_gemm_mega_moe" + amd: + extra_args: + - "--distributed-executor-backend" + - "mp" + - "--gpu-memory-utilization" + - "0.9" + - "--max-num-seqs" + - "512" + - "--max-num-batched-tokens" + - "8192" + - "--compilation-config" + - '{"mode": 3, "cudagraph_mode": "FULL_DECODE_ONLY"}' + extra_env: + VLLM_ROCM_USE_AITER: "1" strategy_overrides: single_node_tp: @@ -98,15 +114,17 @@ strategy_overrides: # strategy doesn't add --enable-expert-parallel) and no MoE mega-kernel # backend on Blackwell. Autotune is disabled to minimize startup time. tp: 8 - extra_args: - - "--no-enable-flashinfer-autotune" hardware_overrides: + hopper: + extra_args: + - "--no-enable-flashinfer-autotune" blackwell: # Replaces recipe-level blackwell override: keeps the FP4 indexer cache # but drops --moe-backend deep_gemm_mega_moe (TP-only path uses the # default MoE backend). extra_args: - "--attention_config.use_fp4_indexer_cache=True" + - "--no-enable-flashinfer-autotune" single_node_dep: # Force DP=4 on all hardware (fills GB200's 4-GPU tray; on 8-GPU nodes use # half the GPUs per replica). dedupeArgs (last-wins) shadows the default @@ -244,6 +262,55 @@ guide: | --speculative-config '{"method":"mtp","num_speculative_tokens":3}' ``` + ### MI355X (4×288GB) + + If you want to further lower the latency, you can run with `--tensor-parallel-size 8`. + + ```bash + export VLLM_ROCM_USE_AITER=1 + + vllm serve deepseek-ai/DeepSeek-V4-Flash \ + --host localhost \ + --port 8001 \ + --dtype auto \ + --kv-cache-dtype fp8 \ + --tensor-parallel-size 4 \ + --max-num-seqs 512 \ + --max-num-batched-tokens 8192 \ + --distributed-executor-backend mp \ + --trust-remote-code \ + --gpu-memory-utilization 0.9 \ + --tokenizer-mode deepseek_v4 \ + --reasoning-parser deepseek_v4 \ + --tool-call-parser deepseek_v4 \ + --enable-auto-tool-choice \ + --compilation-config '{"mode": 3, "cudagraph_mode": "FULL_DECODE_ONLY"}' + ``` + + #### MI355X is validated on GSM8K dataset: + + ##### Launch command + + ```bash + MODEL=deepseek-ai/DeepSeek-V4-Flash + lm_eval --model local-completions \ + --model_args model=$MODEL,base_url=http://0.0.0.0:8001/v1/completions,num_concurrent=128,max_retries=10,max_gen_toks=2048,timeout=60000 \ + --batch_size auto \ + --tasks gsm8k \ + --num_fewshot 8 \ + --output_path . 2>&1 | tee -a eval.log + ``` + + ##### Reported result + + ```text + local-completions ({'model': 'deepseek-ai/DeepSeek-V4-Flash', 'base_url': 'http://0.0.0.0:8001/v1/completions', 'num_concurrent': 128, 'max_retries': 10, 'max_gen_toks': 2048, 'timeout': 60000}), gen_kwargs: ({}), limit: None, num_fewshot: 8, batch_size: auto + |Tasks|Version| Filter |n-shot| Metric | |Value | |Stderr| + |-----|------:|----------------|-----:|-----------|---|-----:|---|-----:| + |gsm8k| 3|flexible-extract| 8|exact_match|↑ |0.9439|± |0.0063| + | | |strict-match | 8|exact_match|↑ |0.9431|± |0.0064| + ``` + ### H200 Single-Node PD (Mooncake) Single-host disaggregated serving: 4 prefill GPUs + 4 decode GPUs on one 8-GPU H200 node, diff --git a/models/deepseek-ai/DeepSeek-V4-Pro.yaml b/models/deepseek-ai/DeepSeek-V4-Pro.yaml index 3abd3d827..a5adb136e 100644 --- a/models/deepseek-ai/DeepSeek-V4-Pro.yaml +++ b/models/deepseek-ai/DeepSeek-V4-Pro.yaml @@ -3,7 +3,7 @@ meta: slug: "deepseek-v4-pro" provider: "DeepSeek" description: "DeepSeek V4 flagship MoE (1.6T total / 49B active) with hybrid CSA+HCA attention, manifold-constrained hyper-connections, Muon-trained on 32T+ tokens, and three-tier reasoning." - date_updated: 2026-04-24 + date_updated: 2026-06-09 difficulty: hard tasks: - text @@ -17,11 +17,13 @@ meta: gb300: verified mi300x: unsupported mi325x: unsupported - mi355x: unsupported + mi355x: verified model: model_id: "deepseek-ai/DeepSeek-V4-Pro" min_vllm_version: "0.20.0" + docker_image: + amd: "vllm/vllm-openai-rocm:nightly" architecture: moe parameter_count: "1600B" active_parameters: "49B" @@ -103,21 +105,37 @@ hardware_overrides: - "--attention_config.use_fp4_indexer_cache=True" - "--moe-backend" - "deep_gemm_mega_moe" + amd: + extra_args: + - "--distributed-executor-backend" + - "mp" + - "--gpu-memory-utilization" + - "0.9" + - "--max-num-seqs" + - "512" + - "--max-num-batched-tokens" + - "8192" + - "--compilation-config" + - '{"mode": 3, "cudagraph_mode": "FULL_DECODE_ONLY"}' + extra_env: + VLLM_ROCM_USE_AITER: "1" strategy_overrides: single_node_tp: # Latency-oriented TP-only deployment: no expert parallelism (single_node_tp # strategy doesn't add --enable-expert-parallel) and no MoE mega-kernel # backend on Blackwell. Autotune is disabled to minimize startup time. - extra_args: - - "--no-enable-flashinfer-autotune" hardware_overrides: + hopper: + extra_args: + - "--no-enable-flashinfer-autotune" blackwell: # Replaces recipe-level blackwell override: keeps the FP4 indexer cache # but drops --moe-backend deep_gemm_mega_moe (TP-only path uses the # default MoE backend). extra_args: - "--attention_config.use_fp4_indexer_cache=True" + - "--no-enable-flashinfer-autotune" single_node_tep: extra_args: - "--compilation-config" @@ -246,6 +264,56 @@ guide: | - **H200 (8× GPU)**: DP + EP with `--data-parallel-size 8`. Context is capped at 800K tokens (`--max-model-len 800000`) to leave KV headroom with dense params replicated across ranks — applies to both single-node and multi-node H200. + - **MI355X (8× GPU)**: validated with ROCm + AITER (`VLLM_ROCM_USE_AITER=1`), + `--gpu-memory-utilization 0.9`, `--max-num-seqs 128`, + `--max-num-batched-tokens 8192`, and `--distributed-executor-backend mp`. - **GB200 NVL4 (4× GPU per tray)**: the ~960 GB mixed-precision checkpoint does not fit on one tray; run multi-node DP + EP across **2 trays** (8 GPUs total) with `--data-parallel-size 8`. Pick the "Multi-Node" tab and set nodes to 2. + + ### MI355X (8×288GB) + + ```bash + export VLLM_ROCM_USE_AITER=1 + + vllm serve deepseek-ai/DeepSeek-V4-Pro \ + --host localhost \ + --port 8001 \ + --dtype auto \ + --kv-cache-dtype fp8 \ + --tensor-parallel-size 8 \ + --max-num-seqs 512 \ + --max-num-batched-tokens 8192 \ + --distributed-executor-backend mp \ + --trust-remote-code \ + --gpu-memory-utilization 0.9 \ + --tokenizer-mode deepseek_v4 \ + --reasoning-parser deepseek_v4 \ + --tool-call-parser deepseek_v4 \ + --enable-auto-tool-choice \ + --compilation-config '{"mode": 3, "cudagraph_mode": "FULL_DECODE_ONLY"}' + ``` + + #### MI355X is validated on GSM8K dataset: + + ##### Launch command + + ```bash + MODEL=deepseek-ai/DeepSeek-V4-Pro + lm_eval --model local-completions \ + --model_args model=$MODEL,base_url=http://0.0.0.0:8001/v1/completions,num_concurrent=128,max_retries=10,max_gen_toks=2048,timeout=60000 \ + --batch_size auto \ + --tasks gsm8k \ + --num_fewshot 8 \ + --output_path . 2>&1 | tee -a eval.log + ``` + + ##### Reported result + + ```text + local-completions ({'model': 'deepseek-ai/DeepSeek-V4-Pro', 'base_url': 'http://0.0.0.0:8001/v1/completions', 'num_concurrent': 128, 'max_retries': 10, 'max_gen_toks': 2048, 'timeout': 60000}), gen_kwargs: ({}), limit: None, num_fewshot: 8, batch_size: auto + |Tasks|Version| Filter |n-shot| Metric | |Value | |Stderr| + |-----|------:|----------------|-----:|-----------|---|-----:|---|-----:| + |gsm8k| 3|flexible-extract| 8|exact_match|↑ |0.9538|± |0.0058| + | | |strict-match | 8|exact_match|↑ |0.9545|± |0.0057| + ```