diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm_mtp.sh index 9fdafe8314..7aa4384cf3 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm_mtp.sh @@ -18,8 +18,8 @@ set -x # Required env vars: # MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR # -# TP4, TP8, and DEP8 (TP8 + DP-attention) are GPU-resident (KV_OFFLOADING=none). -# DEP4 uses KV_OFFLOADING=dram with KV_OFFLOAD_BACKEND=vllm-simple or mooncake. +# TP8 and TP4 c8 are GPU-resident. TP4 c16, DEP4, and DEP8 use DRAM offload +# with KV_OFFLOAD_BACKEND=vllm-simple or mooncake. source "$(dirname "$0")/../../benchmark_lib.sh" @@ -49,9 +49,9 @@ if [ "$DP_ATTENTION" = "true" ] && [ $((2 * CONC % TP)) -ne 0 ]; then exit 1 fi -# DEP8 (TP8 + DP-attention) is a GPU-resident, high-concurrency arm that is -# tuned separately from the smaller DEP4 arm (larger prefill token budget, -# long-prefill chunking, and a lower GPU-memory-utilization headroom). +# DEP8 (TP8 + DP-attention) is a high-concurrency SimpleCPU arm tuned separately +# from DEP4 with a larger prefill token budget and lower GPU-memory-utilization +# headroom. Both DEP arms chunk long prefills. IS_DEP8=false if [ "$DP_ATTENTION" = "true" ] && [ "$TP" -eq 8 ]; then IS_DEP8=true @@ -119,8 +119,8 @@ ROUTER_PID="" MOONCAKE_MASTER_PID="" # The generated TOTAL_CPU_DRAM_GB budget is proportional to allocated GPUs. -# On cluster:b300-nv, dram-utilization=0.80 and DEP4 resolve to roughly the -# source recipe's 280 GiB per DP rank. TP4 remains GPU-resident. +# On cluster:b300-nv, dram-utilization=0.95 gives both DEP4 and DEP8 356 GB per +# DP rank (1,424 GB and 2,849 GB total, respectively). TP arms remain GPU-resident. OFFLOAD_ARGS=() case "$KV_OFFLOAD_BACKEND" in "") @@ -238,14 +238,14 @@ if [ "$EP_SIZE" -gt 1 ]; then ) fi if [ "$DP_ATTENTION" = "true" ]; then - MODE_ARGS+=(--prefill-schedule-interval 8) + MODE_ARGS+=( + --prefill-schedule-interval 8 + --long-prefill-token-threshold 512 + ) if [ "$IS_DEP8" = "true" ]; then - # GPU-resident DEP8 gets a larger prefill token budget and chunks long - # prefills so decode latency stays bounded at high concurrency. - MODE_ARGS+=( - --max-num-batched-tokens 16384 - --long-prefill-token-threshold 4096 - ) + # DEP8 gets a larger prefill token budget; the shared long-prefill + # threshold keeps decode latency bounded under load. + MODE_ARGS+=(--max-num-batched-tokens 16384) else MODE_ARGS+=(--max-num-batched-tokens 8192) fi diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 263dec901e..f916be18dc 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -1725,7 +1725,7 @@ dsv4-fp4-b300-vllm: - { tp: 8, ep: 8, dp-attn: true, conc-start: 2048, conc-end: 2048 } dsv4-fp4-b300-vllm-agentic-mtp: - image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-904e4ec + image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-426e59f model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:b300-nv @@ -1734,18 +1734,18 @@ dsv4-fp4-b300-vllm-agentic-mtp: multinode: false scenarios: agentic-coding: - - dram-utilization: 0.80 + - dram-utilization: 0.95 search-space: # TP8 GPU-resident + MTP (num_speculative_tokens=3) - - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 6, 8] } + - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4] } # TP4 GPU-resident + MTP (num_speculative_tokens=3) - - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 6, 8, 12, 16, 20] } + - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [8] } # TP4 SimpleCPU + MTP (num_speculative_tokens=3) - - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: mtp, conc-list: [20, 24, 28, 32, 36, 40] } + - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: mtp, conc-list: [16] } # DEP4 SimpleCPU + MTP (num_speculative_tokens=3) - - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: mtp, conc-list: [32, 40, 48, 56], router: { name: vllm-router, version: "0.1.14" } } + - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: mtp, conc-list: [48, 64], router: { name: vllm-router, version: "0.1.14" } } # DEP8 SimpleCPU + MTP (num_speculative_tokens=3) - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [64, 96, 112, 128, 144, 160, 176, 192, 224], router: { name: vllm-router, version: "0.1.14" } } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: mtp, conc-list: [128, 256, 384, 512, 576], router: { name: vllm-router, version: "0.1.14" } } dsv4-fp4-b300-trt: image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 64f87b4479..b89b34e5ff 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5770,3 +5770,10 @@ - "models.yaml: add mtp_flags (--speculative-algorithm EAGLE --speculative-eagle-topk 1) to DeepSeek-V4-Pro-AgentX; num-steps/num-draft-tokens are derived from DECODE_MTP_SIZE in server_sglang.sh. DeepSeek-V4's MTP head is a native NextN/EAGLE draft shipped with the model, so no --speculative-draft-model-path is needed." - "Two search-space arms: TP8/EP1 no-DP at conc-list [2,4,8,16,32], and TP8/EP8/DPA (ep=8, dp-attn=true) at conc-list [64,96,128], both on image lmsysorg/sglang-rocm:v0.5.15.post1-rocm720-mi35x-20260719." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2309 + +- config-keys: + - dsv4-fp4-b300-vllm-agentic-mtp + description: + - "Update the B300 DSV4 AgentX MTP image and SimpleCPU sweep through c576." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2536 +