diff --git a/.github/configs/nvidia-master.yaml b/.github/configs/nvidia-master.yaml index fdf705be71..463b2e83ad 100644 --- a/.github/configs/nvidia-master.yaml +++ b/.github/configs/nvidia-master.yaml @@ -2515,7 +2515,7 @@ dsv4-fp8-h200-vllm: # field, so dp-attn=true is used as the existing vLLM script switch for DP4 # layouts on 4 allocated GPUs. dsv4-fp4-b300-vllm: - image: vllm/vllm-openai:deepseekv4-cu130 + image: vllm/vllm-openai:v0.20.0-cu130 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: b300 diff --git a/benchmarks/single_node/dsv4_fp4_b300_vllm.sh b/benchmarks/single_node/dsv4_fp4_b300_vllm.sh index 6bb5b90496..3dfa992d78 100755 --- a/benchmarks/single_node/dsv4_fp4_b300_vllm.sh +++ b/benchmarks/single_node/dsv4_fp4_b300_vllm.sh @@ -42,8 +42,15 @@ if [ "${EP_SIZE:-1}" -gt 1 ]; then EP_ARGS=(--enable-expert-parallel) fi +# Mega-MoE backend and the lower GMU only kick in on the DP-attn path, +# per the vLLM v0.20.0 DeepSeek-V4-Pro recipe. All configs share the +# FULL_AND_PIECEWISE compilation config. +GMU_ARGS=() +MOE_ARGS=() if [ "${DP_ATTENTION}" = "true" ]; then MAX_NUM_BATCHED_TOKENS=2048 + GMU_ARGS=(--gpu-memory-utilization 0.85) + MOE_ARGS=(--moe-backend deep_gemm_mega_moe) else MAX_NUM_BATCHED_TOKENS=$(( ISL * 2 )) fi @@ -66,15 +73,16 @@ start_gpu_monitor set -x vllm serve "$MODEL" --host 0.0.0.0 --port "$PORT" \ - "${PARALLEL_ARGS[@]}" \ - --pipeline-parallel-size 1 \ - --kv-cache-dtype fp8 \ --trust-remote-code \ + --kv-cache-dtype fp8 \ --block-size 256 \ --no-enable-prefix-caching \ + "${PARALLEL_ARGS[@]}" \ "${EP_ARGS[@]}" \ + "${GMU_ARGS[@]}" \ + "${MOE_ARGS[@]}" \ --compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' \ - --attention_config.use_fp4_indexer_cache True \ + --attention_config.use_fp4_indexer_cache=True \ --tokenizer-mode deepseek_v4 \ --tool-call-parser deepseek_v4 \ --enable-auto-tool-choice \ diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 2be38aefe8..025b1c0c2a 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -1969,3 +1969,12 @@ - "Keeps the three validated 8k/1k points: low-latency 1P/1D TP8 conc=1, mid-curve 1P/1D DEP8 conc=256, and max-tpt 3P/1D DEP8 conc=4096" - "All three recipes run NATS/etcd on a dedicated infra node and use compute-node local NVMe model weights via /mnt/numa1/models/deepseek-v4-pro/" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1163 + +- config-keys: + - dsv4-fp4-b300-vllm + description: + - "Pin image to vllm/vllm-openai:v0.20.0-cu130 (was floating deepseekv4-cu130 tag); DeepGEMM is preinstalled in this image" + - "Use --attention_config.use_fp4_indexer_cache=True and --compilation-config {\"cudagraph_mode\": \"FULL_AND_PIECEWISE\", \"custom_ops\": [\"all\"]} for all configs" + - "Gate --moe-backend deep_gemm_mega_moe and --gpu-memory-utilization 0.85 on DP_ATTENTION=true per the v0.20.0 recipe" + - "Drop --pipeline-parallel-size 1; keep --no-enable-prefix-caching and --max-cudagraph-capture-size 2048" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1220