diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm_mtp.sh index 2002c28025..704a2a2cbf 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_b200_vllm_mtp.sh @@ -2,24 +2,25 @@ set -eo pipefail set -x -# Agentic trace replay benchmark for DeepSeek-V4-Pro FP4 on B200 using vLLM, -# with MTP speculative decoding (num_speculative_tokens=3): synthetic acceptance -# length 2.49 for throughput, real target verification for the EVAL_ONLY eval. +# Agentic trace replay benchmark for DeepSeek-V4-Pro-0813 FP4 on B200 using +# vLLM DSpark (num_speculative_tokens=6, probabilistic drafting): synthetic +# acceptance length 3.77 for throughput, real target verification for eval. # -# This MTP-only recipe keeps the established engine args and agentic AIPerf rig, -# with two speculative-decoding behaviors: -# --speculative-config: synthetic acceptance length 2.49 (throughput) vs real MTP (EVAL_ONLY); see the SPEC_CONFIG block +# This DSpark recipe keeps the established engine args and agentic AIPerf rig, +# with two verification behaviors: +# --speculative-config: synthetic acceptance length 3.77 (throughput) vs real +# DSpark verification (EVAL_ONLY); see the SPEC_CONFIG block # cudagraph capture sizes expressed in TOKENS (see the capture block below). # -# The throughput sweep uses DEP8 with SimpleCPUOffloadConnector only. The recipe -# uses FP8 KV cache, sparse DeepSeek-V4 FlashInfer attention with an FP4 indexer -# cache, mega-MoE, long-prefill chunking, and FULL_DECODE_ONLY CUDA graphs with -# every decode batch captured explicitly. +# The throughput sweep uses TP8 and DEP8 with SimpleCPUOffloadConnector. The +# recipe uses FP8 KV cache, sparse DeepSeek-V4 FlashInfer attention with an FP4 +# indexer cache, mega-MoE, long-prefill chunking, and explicit CUDA graph sizes. # # Required env vars: # MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR # -# DEP8 offloads KV to host DRAM with KV_OFFLOAD_BACKEND=vllm-simple. +# Both TP8 and DEP8 offload KV to host DRAM with +# KV_OFFLOAD_BACKEND=vllm-simple. source "$(dirname "$0")/../../benchmark_lib.sh" @@ -93,8 +94,8 @@ fi export AIPERF_SERVER_METRICS_URLS="http://localhost:${VLLM_BACKEND_PORT}/metrics" export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="vllm:" -# DeepSeek-V4-Pro weights are large; engine startup can exceed default 600s. -export VLLM_ENGINE_READY_TIMEOUT_S=3600 +# DeepSeek-V4-Pro weights are large; engine startup can exceed one hour. +export VLLM_ENGINE_READY_TIMEOUT_S=7200 # vllm-project/vllm#43447 keeps local SWA prefix-cache tails sparsely, while # vllm-project/vllm#44774 applies the same reachability policy to Mooncake's @@ -219,7 +220,7 @@ if [ "$EP_SIZE" -gt 1 ]; then MODE_ARGS+=( --enable-expert-parallel --enable-ep-weight-filter - --moe-backend deep_gemm_amxf4_mega_moe + --moe-backend deep_gemm_mega_moe ) fi if [ "$DP_ATTENTION" = "true" ]; then @@ -228,7 +229,6 @@ if [ "$DP_ATTENTION" = "true" ]; then MODE_ARGS+=( --prefill-schedule-interval 8 --long-prefill-token-threshold 512 - --max-num-batched-tokens 8192 ) fi @@ -241,21 +241,30 @@ else MAX_NUM_SEQS=$((2 * CONC)) fi -# MTP: cudagraph capture sizes are in TOKENS. With num_speculative_tokens=N, +# DSpark: cudagraph capture sizes are in TOKENS. With num_speculative_tokens=N, # every uniform decode batch of S seqs verifies S*(1+N) tokens, so capture the # explicit multiples (1+N), 2*(1+N), ..., MAX_NUM_SEQS*(1+N). vLLM rounds # configured sizes up to multiples of (1+N) and deduplicates them; a plain # 1..MAX_NUM_SEQS list would cover only MAX_NUM_SEQS/(1+N) decode sequences. -NUM_SPEC_TOKENS=3 +NUM_SPEC_TOKENS=6 TOKENS_PER_SEQ=$((1 + NUM_SPEC_TOKENS)) -# Throughput pins synthetic MTP acceptance to the dsv4-pro golden AL (thinking_on, -# num_speculative_tokens=3, golden_al_distribution/dsv4_mtp.yaml). The EVAL_ONLY -# accuracy run uses real target verification instead -- synthetic acceptance -# bypasses verification and corrupts the SWE-bench eval (0.0000 score). +# Throughput pins synthetic DSpark acceptance to the DeepSeek-V4-Pro-0813 +# golden AL (thinking_on, probabilistic drafting, num_speculative_tokens=6, +# golden_al_distribution/dsv4-pro-0813-dspark.yaml). The EVAL_ONLY accuracy run +# uses real target verification instead -- synthetic acceptance bypasses +# verification and corrupts the SWE-bench eval. if [ "${EVAL_ONLY:-false}" = "true" ]; then - SPEC_CONFIG="{\"method\": \"mtp\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS}" + SPEC_CONFIG="{\"method\": \"dspark\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"draft_sample_method\": \"probabilistic\"}" else - SPEC_CONFIG="{\"method\": \"mtp\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"rejection_sample_method\": \"synthetic\", \"synthetic_acceptance_length\": 2.49}" + SPEC_CONFIG="{\"method\": \"dspark\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"draft_sample_method\": \"probabilistic\", \"rejection_sample_method\": \"synthetic\", \"synthetic_acceptance_length\": 3.77}" +fi +# DSpark reserves a verification slot for every target and draft token on each +# runnable DEP sequence. Add those slots on top of the established 8192-token +# prefill budget so enabling the deeper drafter cannot make the scheduler's +# effective token budget negative. +if [ "$DP_ATTENTION" = "true" ]; then + MAX_NUM_BATCHED_TOKENS=$((8192 + MAX_NUM_SEQS * TOKENS_PER_SEQ)) + MODE_ARGS+=(--max-num-batched-tokens "$MAX_NUM_BATCHED_TOKENS") fi CAPTURE_SIZE_LIST=() for ((num_seqs = 1; num_seqs <= MAX_NUM_SEQS; num_seqs++)); do diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 1d0144da48..ffed54819b 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -962,8 +962,8 @@ dsv4-fp4-b200-vllm: - { tp: 8, ep: 8, dp-attn: true, conc-start: 64, conc-end: 1024 } dsv4-fp4-b200-vllm-agentic-mtp: - image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-426e59f - model: deepseek-ai/DeepSeek-V4-Pro + image: vllm/vllm-openai:v0.28.0 + model: deepseek-ai/DeepSeek-V4-Pro-0813 model-prefix: dsv4 runner: cluster:b200-nscale precision: fp4 @@ -971,12 +971,12 @@ dsv4-fp4-b200-vllm-agentic-mtp: multinode: false scenarios: agentic-coding: - - dram-utilization: 0.95 + - dram-utilization: 0.90 search-space: - # TP8 SimpleCPU + MTP - - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: mtp, conc-list: [1, 4, 8, 14] } - # DEP8 SimpleCPU + MTP - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: mtp, conc-list: [32, 64, 96, 128, 160, 196], router: { name: vllm-router, version: "0.1.14" } } + # TP8 SimpleCPU + DSpark6 + - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: draft_model, conc-list: [1, 4, 8, 14] } + # DEP8 SimpleCPU + DSpark6 + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: draft_model, conc-list: [32, 64, 96, 128, 160, 196], router: { name: vllm-router, version: "0.1.14" } } dsv4-fp4-b200-trt: image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 9dfeb4a02a..f3079a6241 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6984,3 +6984,11 @@ - "Export OMP_NUM_THREADS=1 from all three scripts. vLLM clamps torch threads itself, but only after the weights are loaded: the 'Reducing Torch threads from 16 to 1 for serving' line is emitted after 'Model loading took', so the checkpoint is read with 16 torch threads per worker across 8 workers. Setting it externally caps threads from process start, which is what that log line recommends." - "Weight loading currently costs 5760-5946 s (96-99 min) per job on this runner, measured across the c1, c4, c56 and c64 AgentX jobs of run 34087124574 - roughly half of each ~3 h job. It does not affect any measured number: loading completes before warmup and the profiling phase. The effect is also only observable on this runner. A local EXT4 box serves the checkpoint from page cache and loads in ~40 s with or without the cap, so it never exercises the cold-read path. CI reads cold from WEKAFS, where each of the 8 ranks streams the full 805 GiB to slice its own shards - roughly 6.4 TB of aggregate network reads per job - and that is where oversubscribed reader threads cost real time." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2851 + +- config-keys: + - dsv4-fp4-b200-vllm-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Use DeepSeek-V4-Pro-0813 with DSpark6 on vLLM v0.28.0; set DRAM utilization to 0.90 and update startup settings." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2302