[WIP] Test DeepSeek-V4 GB300 Dynamo AgentX recipes / 测试 DeepSeek-V4 GB300 Dynamo AgentX 配置

benchmarks/single_node/agentic/dsv4_fp4_b200_vllm_mtp.sh

@@ -2,24 +2,25 @@
 set -eo pipefail
 set -x
 
-# Agentic trace replay benchmark for DeepSeek-V4-Pro FP4 on B200 using vLLM,
-# with MTP speculative decoding (num_speculative_tokens=3): synthetic acceptance
-# length 2.49 for throughput, real target verification for the EVAL_ONLY eval.
+# Agentic trace replay benchmark for DeepSeek-V4-Pro-0813 FP4 on B200 using
+# vLLM DSpark (num_speculative_tokens=6, probabilistic drafting): synthetic
+# acceptance length 3.77 for throughput, real target verification for eval.
 #
-# This MTP-only recipe keeps the established engine args and agentic AIPerf rig,
-# with two speculative-decoding behaviors:
-#   --speculative-config: synthetic acceptance length 2.49 (throughput) vs real MTP (EVAL_ONLY); see the SPEC_CONFIG block
+# This DSpark recipe keeps the established engine args and agentic AIPerf rig,
+# with two verification behaviors:
+#   --speculative-config: synthetic acceptance length 3.77 (throughput) vs real
+#       DSpark verification (EVAL_ONLY); see the SPEC_CONFIG block
 #   cudagraph capture sizes expressed in TOKENS (see the capture block below).
 #
-# The throughput sweep uses DEP8 with SimpleCPUOffloadConnector only. The recipe
-# uses FP8 KV cache, sparse DeepSeek-V4 FlashInfer attention with an FP4 indexer
-# cache, mega-MoE, long-prefill chunking, and FULL_DECODE_ONLY CUDA graphs with
-# every decode batch captured explicitly.
+# The throughput sweep uses TP8 and DEP8 with SimpleCPUOffloadConnector. The
+# recipe uses FP8 KV cache, sparse DeepSeek-V4 FlashInfer attention with an FP4
+# indexer cache, mega-MoE, long-prefill chunking, and explicit CUDA graph sizes.
 #
 # Required env vars:
 #   MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR
 #
-# DEP8 offloads KV to host DRAM with KV_OFFLOAD_BACKEND=vllm-simple.
+# Both TP8 and DEP8 offload KV to host DRAM with
+# KV_OFFLOAD_BACKEND=vllm-simple.
 
 source "$(dirname "$0")/../../benchmark_lib.sh"
 
@@ -93,8 +94,8 @@ fi
 export AIPERF_SERVER_METRICS_URLS="http://localhost:${VLLM_BACKEND_PORT}/metrics"
 export AIPERF_REQUIRED_SERVER_METRIC_PREFIX="vllm:"
 
-# DeepSeek-V4-Pro weights are large; engine startup can exceed default 600s.
-export VLLM_ENGINE_READY_TIMEOUT_S=3600
+# DeepSeek-V4-Pro weights are large; engine startup can exceed one hour.
+export VLLM_ENGINE_READY_TIMEOUT_S=7200
 
 # vllm-project/vllm#43447 keeps local SWA prefix-cache tails sparsely, while
 # vllm-project/vllm#44774 applies the same reachability policy to Mooncake's
@@ -219,7 +220,7 @@ if [ "$EP_SIZE" -gt 1 ]; then
     MODE_ARGS+=(
         --enable-expert-parallel
         --enable-ep-weight-filter
-        --moe-backend deep_gemm_amxf4_mega_moe
+        --moe-backend deep_gemm_mega_moe
     )
 fi
 if [ "$DP_ATTENTION" = "true" ]; then
@@ -228,7 +229,6 @@ if [ "$DP_ATTENTION" = "true" ]; then
     MODE_ARGS+=(
         --prefill-schedule-interval 8
         --long-prefill-token-threshold 512
-        --max-num-batched-tokens 8192
     )
 fi
 
@@ -241,21 +241,30 @@ else
     MAX_NUM_SEQS=$((2 * CONC))
 fi
 
-# MTP: cudagraph capture sizes are in TOKENS. With num_speculative_tokens=N,
+# DSpark: cudagraph capture sizes are in TOKENS. With num_speculative_tokens=N,
 # every uniform decode batch of S seqs verifies S*(1+N) tokens, so capture the
 # explicit multiples (1+N), 2*(1+N), ..., MAX_NUM_SEQS*(1+N). vLLM rounds
 # configured sizes up to multiples of (1+N) and deduplicates them; a plain
 # 1..MAX_NUM_SEQS list would cover only MAX_NUM_SEQS/(1+N) decode sequences.
-NUM_SPEC_TOKENS=3
+NUM_SPEC_TOKENS=6
 TOKENS_PER_SEQ=$((1 + NUM_SPEC_TOKENS))
-# Throughput pins synthetic MTP acceptance to the dsv4-pro golden AL (thinking_on,
-# num_speculative_tokens=3, golden_al_distribution/dsv4_mtp.yaml). The EVAL_ONLY
-# accuracy run uses real target verification instead -- synthetic acceptance
-# bypasses verification and corrupts the SWE-bench eval (0.0000 score).
+# Throughput pins synthetic DSpark acceptance to the DeepSeek-V4-Pro-0813
+# golden AL (thinking_on, probabilistic drafting, num_speculative_tokens=6,
+# golden_al_distribution/dsv4-pro-0813-dspark.yaml). The EVAL_ONLY accuracy run
+# uses real target verification instead -- synthetic acceptance bypasses
+# verification and corrupts the SWE-bench eval.
 if [ "${EVAL_ONLY:-false}" = "true" ]; then
-    SPEC_CONFIG="{\"method\": \"mtp\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS}"
+    SPEC_CONFIG="{\"method\": \"dspark\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"draft_sample_method\": \"probabilistic\"}"
 else
-    SPEC_CONFIG="{\"method\": \"mtp\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"rejection_sample_method\": \"synthetic\", \"synthetic_acceptance_length\": 2.49}"
+    SPEC_CONFIG="{\"method\": \"dspark\", \"num_speculative_tokens\": $NUM_SPEC_TOKENS, \"draft_sample_method\": \"probabilistic\", \"rejection_sample_method\": \"synthetic\", \"synthetic_acceptance_length\": 3.77}"
+fi
+# DSpark reserves a verification slot for every target and draft token on each
+# runnable DEP sequence. Add those slots on top of the established 8192-token
+# prefill budget so enabling the deeper drafter cannot make the scheduler's
+# effective token budget negative.
+if [ "$DP_ATTENTION" = "true" ]; then
+    MAX_NUM_BATCHED_TOKENS=$((8192 + MAX_NUM_SEQS * TOKENS_PER_SEQ))
+    MODE_ARGS+=(--max-num-batched-tokens "$MAX_NUM_BATCHED_TOKENS")
 fi
 CAPTURE_SIZE_LIST=()
 for ((num_seqs = 1; num_seqs <= MAX_NUM_SEQS; num_seqs++)); do

configs/nvidia-master.yaml

@@ -962,21 +962,21 @@ dsv4-fp4-b200-vllm:
       - { tp: 8, ep: 8, dp-attn: true, conc-start: 64, conc-end: 1024 }
 
 dsv4-fp4-b200-vllm-agentic-mtp:
-  image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-426e59f
-  model: deepseek-ai/DeepSeek-V4-Pro
+  image: vllm/vllm-openai:v0.28.0
+  model: deepseek-ai/DeepSeek-V4-Pro-0813
   model-prefix: dsv4
   runner: cluster:b200-nscale
   precision: fp4
   framework: vllm
   multinode: false
   scenarios:
     agentic-coding:
-    - dram-utilization: 0.95
+    - dram-utilization: 0.90
       search-space:
-      # TP8 SimpleCPU + MTP
-      - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: mtp, conc-list: [1, 4, 8, 14] }
-      # DEP8 SimpleCPU + MTP
-      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: mtp, conc-list: [32, 64, 96, 128, 160, 196], router: { name: vllm-router, version: "0.1.14" } }
+      # TP8 SimpleCPU + DSpark6
+      - { tp: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: draft_model, conc-list: [1, 4, 8, 14] }
+      # DEP8 SimpleCPU + DSpark6
+      - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: draft_model, conc-list: [32, 64, 96, 128, 160, 196], router: { name: vllm-router, version: "0.1.14" } }
 
 dsv4-fp4-b200-trt:
   image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066

perf-changelog.yaml

@@ -6984,3 +6984,11 @@
     - "Export OMP_NUM_THREADS=1 from all three scripts. vLLM clamps torch threads itself, but only after the weights are loaded: the 'Reducing Torch threads from 16 to 1 for serving' line is emitted after 'Model loading took', so the checkpoint is read with 16 torch threads per worker across 8 workers. Setting it externally caps threads from process start, which is what that log line recommends."
     - "Weight loading currently costs 5760-5946 s (96-99 min) per job on this runner, measured across the c1, c4, c56 and c64 AgentX jobs of run 34087124574 - roughly half of each ~3 h job. It does not affect any measured number: loading completes before warmup and the profiling phase. The effect is also only observable on this runner. A local EXT4 box serves the checkpoint from page cache and loads in ~40 s with or without the cap, so it never exercises the cold-read path. CI reads cold from WEKAFS, where each of the 8 ranks streams the full 805 GiB to slice its own shards - roughly 6.4 TB of aggregate network reads per job - and that is where oversubscribed reader threads cost real time."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2851
+
+- config-keys:
+    - dsv4-fp4-b200-vllm-agentic-mtp
+  scenario-type:
+    - agentic-coding
+  description:
+    - "Use DeepSeek-V4-Pro-0813 with DSpark6 on vLLM v0.28.0; set DRAM utilization to 0.90 and update startup settings."
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2302