diff --git a/.github/configs/nvidia-master.yaml b/.github/configs/nvidia-master.yaml index 39e299cb08..efdc4d68ea 100644 --- a/.github/configs/nvidia-master.yaml +++ b/.github/configs/nvidia-master.yaml @@ -2513,7 +2513,7 @@ dsv4-fp8-h200-vllm: # field, so dp-attn=true is used as the existing vLLM script switch for DP4 # layouts on 4 allocated GPUs. dsv4-fp4-b300-vllm: - image: vllm/vllm-openai:deepseekv4-cu130 + image: vllm/vllm-openai:v0.20.0-cu130 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: b300 diff --git a/benchmarks/single_node/dsv4_fp4_b300_vllm.sh b/benchmarks/single_node/dsv4_fp4_b300_vllm.sh index 6bb5b90496..bf051c4f4f 100755 --- a/benchmarks/single_node/dsv4_fp4_b300_vllm.sh +++ b/benchmarks/single_node/dsv4_fp4_b300_vllm.sh @@ -39,7 +39,7 @@ fi EP_ARGS=() if [ "${EP_SIZE:-1}" -gt 1 ]; then - EP_ARGS=(--enable-expert-parallel) + EP_ARGS=(--enable-expert-parallel --moe-backend deep_gemm_mega_moe) fi if [ "${DP_ATTENTION}" = "true" ]; then @@ -66,20 +66,18 @@ start_gpu_monitor set -x vllm serve "$MODEL" --host 0.0.0.0 --port "$PORT" \ - "${PARALLEL_ARGS[@]}" \ - --pipeline-parallel-size 1 \ - --kv-cache-dtype fp8 \ --trust-remote-code \ + --kv-cache-dtype fp8 \ --block-size 256 \ --no-enable-prefix-caching \ + "${PARALLEL_ARGS[@]}" \ "${EP_ARGS[@]}" \ - --compilation-config '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}' \ - --attention_config.use_fp4_indexer_cache True \ + --compilation-config '{"mode": 0, "cudagraph_mode": "FULL_DECODE_ONLY"}' \ + --attention_config.use_fp4_indexer_cache=True \ --tokenizer-mode deepseek_v4 \ --tool-call-parser deepseek_v4 \ --enable-auto-tool-choice \ --reasoning-parser deepseek_v4 \ - --max-cudagraph-capture-size 2048 \ --max-model-len "$SERVE_MAX_MODEL_LEN" \ --max-num-batched-tokens "$MAX_NUM_BATCHED_TOKENS" > "$SERVER_LOG" 2>&1 & diff --git a/perf-changelog.yaml b/perf-changelog.yaml index a29c278f20..c64254c752 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -1918,3 +1918,12 @@ - "Three CONC bands: A=TP8 (1-8), B=TP4 (16-128), C=DP4 dp-attn (64-512); B/C overlap at conc 64,128" - "Configs: 1k1k and 8k1k, no validation.py / launcher / yaml-field changes (knob-free)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1180 + +- config-keys: + - dsv4-fp4-b300-vllm + description: + - "Pin image to vllm/vllm-openai:v0.20.0-cu130 (was floating deepseekv4-cu130 tag); DeepGEMM is preinstalled in this image" + - "Switch compilation-config to {\"mode\": 0, \"cudagraph_mode\": \"FULL_DECODE_ONLY\"} and use --attention_config.use_fp4_indexer_cache=True" + - "Bundle --moe-backend deep_gemm_mega_moe into EP_ARGS so it only applies when EP is enabled (EP_SIZE>1)" + - "Drop --pipeline-parallel-size 1 and --max-cudagraph-capture-size 2048 (no longer in v0.20.0 recipe); keep --no-enable-prefix-caching" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1206