From 5945eeb4b4c2a4ae2393787182a0047e7dc45779 Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Sat, 8 Aug 2026 21:11:09 +0000 Subject: [PATCH 1/5] perf(agentx): retune DSV4 B300 MTP sweep MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update the vLLM image and concurrency grid, raise the DEP4 host-memory budget, and set long-prefill-token-threshold to 512 for DEP4 and DEP8. 中文：更新 vLLM 镜像与并发配置，提高 DEP4 主机内存预算，并为 DEP4 和 DEP8 将 long-prefill-token-threshold 设为 512。 --- .../agentic/dsv4_fp4_b300_vllm_mtp.sh | 22 +++++++++---------- configs/nvidia-master.yaml | 16 ++++++-------- perf-changelog.yaml | 6 +++++ 3 files changed, 24 insertions(+), 20 deletions(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm_mtp.sh index 9fdafe8314..e56fceed6f 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm_mtp.sh @@ -50,8 +50,8 @@ if [ "$DP_ATTENTION" = "true" ] && [ $((2 * CONC % TP)) -ne 0 ]; then fi # DEP8 (TP8 + DP-attention) is a GPU-resident, high-concurrency arm that is -# tuned separately from the smaller DEP4 arm (larger prefill token budget, -# long-prefill chunking, and a lower GPU-memory-utilization headroom). +# tuned separately from the smaller DEP4 arm with a larger prefill token budget +# and lower GPU-memory-utilization headroom. Both DEP arms chunk long prefills. IS_DEP8=false if [ "$DP_ATTENTION" = "true" ] && [ "$TP" -eq 8 ]; then IS_DEP8=true @@ -119,8 +119,8 @@ ROUTER_PID="" MOONCAKE_MASTER_PID="" # The generated TOTAL_CPU_DRAM_GB budget is proportional to allocated GPUs. -# On cluster:b300-nv, dram-utilization=0.80 and DEP4 resolve to roughly the -# source recipe's 280 GiB per DP rank. TP4 remains GPU-resident. +# On cluster:b300-nv, dram-utilization=0.95 and DEP4 resolve to 1,424 GB total, +# or 356 GB per DP rank. TP4 remains GPU-resident. OFFLOAD_ARGS=() case "$KV_OFFLOAD_BACKEND" in "") @@ -238,14 +238,14 @@ if [ "$EP_SIZE" -gt 1 ]; then ) fi if [ "$DP_ATTENTION" = "true" ]; then - MODE_ARGS+=(--prefill-schedule-interval 8) + MODE_ARGS+=( + --prefill-schedule-interval 8 + --long-prefill-token-threshold 512 + ) if [ "$IS_DEP8" = "true" ]; then - # GPU-resident DEP8 gets a larger prefill token budget and chunks long - # prefills so decode latency stays bounded at high concurrency. - MODE_ARGS+=( - --max-num-batched-tokens 16384 - --long-prefill-token-threshold 4096 - ) + # GPU-resident DEP8 gets a larger prefill token budget; the shared + # long-prefill threshold keeps decode latency bounded under load. + MODE_ARGS+=(--max-num-batched-tokens 16384) else MODE_ARGS+=(--max-num-batched-tokens 8192) fi diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 393babf91b..e4233ec655 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -1725,7 +1725,7 @@ dsv4-fp4-b300-vllm: - { tp: 8, ep: 8, dp-attn: true, conc-start: 2048, conc-end: 2048 } dsv4-fp4-b300-vllm-agentic-mtp: - image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-904e4ec + image: vllm/vllm-openai:nightly-dev-x86_64-cu13.0.1-426e59f model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:b300-nv @@ -1734,18 +1734,16 @@ dsv4-fp4-b300-vllm-agentic-mtp: multinode: false scenarios: agentic-coding: - - dram-utilization: 0.80 + - dram-utilization: 0.95 search-space: # TP8 GPU-resident + MTP (num_speculative_tokens=3) - - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 6, 8] } + - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4] } # TP4 GPU-resident + MTP (num_speculative_tokens=3) - - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 6, 8, 12, 16, 20] } - # TP4 SimpleCPU + MTP (num_speculative_tokens=3) - - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: mtp, conc-list: [20, 24, 28, 32, 36, 40] } + - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [8, 12, 16] } # DEP4 SimpleCPU + MTP (num_speculative_tokens=3) - - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: mtp, conc-list: [32, 40, 48, 56], router: { name: vllm-router, version: "0.1.14" } } - # DEP8 SimpleCPU + MTP (num_speculative_tokens=3) - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [64, 96, 112, 128, 144, 160, 176, 192, 224], router: { name: vllm-router, version: "0.1.14" } } + - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: mtp, conc-list: [40, 48, 64], router: { name: vllm-router, version: "0.1.14" } } + # DEP8 GPU-resident + MTP (num_speculative_tokens=3) + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [64, 128, 192, 256, 320, 384], router: { name: vllm-router, version: "0.1.14" } } dsv4-fp4-b300-trt: image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 471981f60c..987e89f3cd 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5709,3 +5709,9 @@ - "Enable SGLang metrics on every aggregate, prefill, and decode engine." - "Use supported header-based Dynamo session routing with the in-repo AIPerf build." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2520 + +- config-keys: + - dsv4-fp4-b300-vllm-agentic-mtp + description: + - "Update the B300 DSV4 AgentX MTP image, TP/DEP sweep, and DEP prefill tuning." + pr-link: TBD From 052b5b340359aa0455c738dd20f66338924f88c4 Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Sat, 8 Aug 2026 21:12:03 +0000 Subject: [PATCH 2/5] docs(changelog): link DSV4 B300 MTP PR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the temporary changelog placeholder with draft PR #2536. 中文：将变更日志中的临时占位符替换为草稿 PR #2536 的链接。 --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 987e89f3cd..e4caca0ea7 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5714,4 +5714,4 @@ - dsv4-fp4-b300-vllm-agentic-mtp description: - "Update the B300 DSV4 AgentX MTP image, TP/DEP sweep, and DEP prefill tuning." - pr-link: TBD + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2536 From a33496451bdebb33902bc36678f5a37771acc193 Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Sun, 9 Aug 2026 00:07:13 +0000 Subject: [PATCH 3/5] perf(agentx): extend DEP8 SimpleCPU sweep MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move every DEP8 point to vLLM SimpleCPU offload, widen the concurrency grid through c512, and drop DEP4 c40. 中文：将全部 DEP8 配置切换到 vLLM SimpleCPU 卸载，将并发扫描扩展至 c512，并移除 DEP4 c40。 --- .../agentic/dsv4_fp4_b300_vllm_mtp.sh | 18 +++++++++--------- configs/nvidia-master.yaml | 6 +++--- perf-changelog.yaml | 2 +- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm_mtp.sh index e56fceed6f..a1226fe753 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm_mtp.sh @@ -18,8 +18,8 @@ set -x # Required env vars: # MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR # -# TP4, TP8, and DEP8 (TP8 + DP-attention) are GPU-resident (KV_OFFLOADING=none). -# DEP4 uses KV_OFFLOADING=dram with KV_OFFLOAD_BACKEND=vllm-simple or mooncake. +# TP4 and TP8 are GPU-resident (KV_OFFLOADING=none). DEP4 and DEP8 use +# KV_OFFLOADING=dram with KV_OFFLOAD_BACKEND=vllm-simple or mooncake. source "$(dirname "$0")/../../benchmark_lib.sh" @@ -49,9 +49,9 @@ if [ "$DP_ATTENTION" = "true" ] && [ $((2 * CONC % TP)) -ne 0 ]; then exit 1 fi -# DEP8 (TP8 + DP-attention) is a GPU-resident, high-concurrency arm that is -# tuned separately from the smaller DEP4 arm with a larger prefill token budget -# and lower GPU-memory-utilization headroom. Both DEP arms chunk long prefills. +# DEP8 (TP8 + DP-attention) is a high-concurrency SimpleCPU arm tuned separately +# from DEP4 with a larger prefill token budget and lower GPU-memory-utilization +# headroom. Both DEP arms chunk long prefills. IS_DEP8=false if [ "$DP_ATTENTION" = "true" ] && [ "$TP" -eq 8 ]; then IS_DEP8=true @@ -119,8 +119,8 @@ ROUTER_PID="" MOONCAKE_MASTER_PID="" # The generated TOTAL_CPU_DRAM_GB budget is proportional to allocated GPUs. -# On cluster:b300-nv, dram-utilization=0.95 and DEP4 resolve to 1,424 GB total, -# or 356 GB per DP rank. TP4 remains GPU-resident. +# On cluster:b300-nv, dram-utilization=0.95 gives both DEP4 and DEP8 356 GB per +# DP rank (1,424 GB and 2,849 GB total, respectively). TP arms remain GPU-resident. OFFLOAD_ARGS=() case "$KV_OFFLOAD_BACKEND" in "") @@ -243,8 +243,8 @@ if [ "$DP_ATTENTION" = "true" ]; then --long-prefill-token-threshold 512 ) if [ "$IS_DEP8" = "true" ]; then - # GPU-resident DEP8 gets a larger prefill token budget; the shared - # long-prefill threshold keeps decode latency bounded under load. + # DEP8 gets a larger prefill token budget; the shared long-prefill + # threshold keeps decode latency bounded under load. MODE_ARGS+=(--max-num-batched-tokens 16384) else MODE_ARGS+=(--max-num-batched-tokens 8192) diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index e4233ec655..53ad3f837d 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -1741,9 +1741,9 @@ dsv4-fp4-b300-vllm-agentic-mtp: # TP4 GPU-resident + MTP (num_speculative_tokens=3) - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [8, 12, 16] } # DEP4 SimpleCPU + MTP (num_speculative_tokens=3) - - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: mtp, conc-list: [40, 48, 64], router: { name: vllm-router, version: "0.1.14" } } - # DEP8 GPU-resident + MTP (num_speculative_tokens=3) - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [64, 128, 192, 256, 320, 384], router: { name: vllm-router, version: "0.1.14" } } + - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: mtp, conc-list: [48, 64], router: { name: vllm-router, version: "0.1.14" } } + # DEP8 SimpleCPU + MTP (num_speculative_tokens=3) + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: mtp, conc-list: [64, 128, 256, 384, 512], router: { name: vllm-router, version: "0.1.14" } } dsv4-fp4-b300-trt: image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index e4caca0ea7..5a8c83f823 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5713,5 +5713,5 @@ - config-keys: - dsv4-fp4-b300-vllm-agentic-mtp description: - - "Update the B300 DSV4 AgentX MTP image, TP/DEP sweep, and DEP prefill tuning." + - "Update the B300 DSV4 AgentX MTP image and TP/DEP sweep; use SimpleCPU for all DEP arms and extend DEP8 to c512." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2536 From 63ea0c95722c6833f0e72b62317f5f53393669ec Mon Sep 17 00:00:00 2001 From: Yifan Qiao Date: Sun, 9 Aug 2026 00:19:52 +0000 Subject: [PATCH 4/5] perf(agentx): finalize SimpleCPU sweep grid MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move TP4 c16 to SimpleCPU offload, skip TP4 c12 and DEP8 c64, and extend the DEP8 sweep through c576. 中文：将 TP4 c16 切换到 SimpleCPU 卸载，移除 TP4 c12 与 DEP8 c64，并将 DEP8 扫描扩展至 c576。 --- benchmarks/single_node/agentic/dsv4_fp4_b300_vllm_mtp.sh | 4 ++-- configs/nvidia-master.yaml | 6 ++++-- perf-changelog.yaml | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm_mtp.sh index a1226fe753..7aa4384cf3 100755 --- a/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_b300_vllm_mtp.sh @@ -18,8 +18,8 @@ set -x # Required env vars: # MODEL, TP, CONC, KV_OFFLOADING, TOTAL_CPU_DRAM_GB, RESULT_DIR # -# TP4 and TP8 are GPU-resident (KV_OFFLOADING=none). DEP4 and DEP8 use -# KV_OFFLOADING=dram with KV_OFFLOAD_BACKEND=vllm-simple or mooncake. +# TP8 and TP4 c8 are GPU-resident. TP4 c16, DEP4, and DEP8 use DRAM offload +# with KV_OFFLOAD_BACKEND=vllm-simple or mooncake. source "$(dirname "$0")/../../benchmark_lib.sh" diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 53ad3f837d..4627d5d7eb 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -1739,11 +1739,13 @@ dsv4-fp4-b300-vllm-agentic-mtp: # TP8 GPU-resident + MTP (num_speculative_tokens=3) - { tp: 8, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4] } # TP4 GPU-resident + MTP (num_speculative_tokens=3) - - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [8, 12, 16] } + - { tp: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [8] } + # TP4 SimpleCPU + MTP (num_speculative_tokens=3) + - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: mtp, conc-list: [16] } # DEP4 SimpleCPU + MTP (num_speculative_tokens=3) - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: mtp, conc-list: [48, 64], router: { name: vllm-router, version: "0.1.14" } } # DEP8 SimpleCPU + MTP (num_speculative_tokens=3) - - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: mtp, conc-list: [64, 128, 256, 384, 512], router: { name: vllm-router, version: "0.1.14" } } + - { tp: 8, ep: 8, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, spec-decoding: mtp, conc-list: [128, 256, 384, 512, 576], router: { name: vllm-router, version: "0.1.14" } } dsv4-fp4-b300-trt: image: ghcr.io#semianalysisai/trtllm-deepseek-v4:feat-deepseek_v4-c185066 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 5a8c83f823..23f0d7618d 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5713,5 +5713,5 @@ - config-keys: - dsv4-fp4-b300-vllm-agentic-mtp description: - - "Update the B300 DSV4 AgentX MTP image and TP/DEP sweep; use SimpleCPU for all DEP arms and extend DEP8 to c512." + - "Update the B300 DSV4 AgentX MTP image and SimpleCPU sweep through c576." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2536 From 38fff03d63708a3dbf1a998f29aff38c5b2ef48b Mon Sep 17 00:00:00 2001 From: Cameron Quilici Date: Tue, 11 Aug 2026 11:56:02 -0500 Subject: [PATCH 5/5] Update perf-changelog.yaml --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 7f7980253c..b89b34e5ff 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5776,4 +5776,4 @@ description: - "Update the B300 DSV4 AgentX MTP image and SimpleCPU sweep through c576." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2536 - \ No newline at end of file +