Skip to content

[NVIDIA][AgentX] Refresh GB200 DeepSeek-V4.1-Flash SGLang aggregate curve / [NVIDIA][AgentX] 更新 GB200 DeepSeek-V4.1-Flash SGLang 聚合曲线 #4112

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
nvpohanh merged 3 commits into from
Oct 16, 2026
Merged

[NVIDIA][AgentX] Refresh GB200 DeepSeek-V4.1-Flash SGLang aggregate curve / [NVIDIA][AgentX] 更新 GB200 DeepSeek-V4.1-Flash SGLang 聚合曲线 #4112

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
95a3d51
perf(dsv41flash): retune GB200 SGLang AgentX curve on 20261014 nightly
shyeh25 Oct 15, 2026
b260d0e
chore(dsv41flash): drop search-space comments from the GB200 SGLang e…
shyeh25 Oct 15, 2026
0be0fe0
Merge remote-tracking branch 'origin/main' into dsv41flash-fp4-gb200-…
nvpohanh Oct 16, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
244 changes: 199 additions & 45 deletions ...e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb200-fp4-mtp/agentic.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ base:
# Draft passes under long-context load outlast the 1800 s default watchdog.
watchdog-timeout: 3600
enable-metrics: true
tokenizer-backend: fastokens
env:
PYTHONNOUSERSITE: '1'
PYTHONUNBUFFERED: '1'
Expand All @@ -52,51 +53,62 @@ base:
AIPERF_HTTP_TCP_USER_TIMEOUT: '900000'
AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:'

override_tp4_c1:
override_tp4ep1_c1:
roles:
agg:
gpus: 4
args:
tensor-parallel-size: 4
expert-parallel-size: 4
mem-fraction-static: 0.76
expert-parallel-size: 1
chunked-prefill-size: 8192
cuda-graph-max-bs-decode: 1
max-running-requests: 1
mem-fraction-static: 0.88
swa-prefix-tails: 128
max-running-requests: 2
env:
SGLANG_OPT_DSV41_MEGA_MHC_PREFILL: '1'
benchmark:
env:
CONC: '1'
AGENTIC_WARMUP_GRACE_PERIOD: '1800'

override_tp4_c4:
override_tp4ep1_c4:
roles:
agg:
gpus: 4
args:
tensor-parallel-size: 4
expert-parallel-size: 4
mem-fraction-static: 0.76
expert-parallel-size: 1
chunked-prefill-size: 8192
swa-prefix-tails: 128
max-running-requests: 8
prefill-decode-interval: 384
mem-fraction-static: 0.76
prefill-decode-interval: 64
schedule-policy: shortest-prefill-first
swa-prefix-tails: 128
env:
SGLANG_OPT_DSV41_MEGA_MHC_PREFILL: '1'
benchmark:
env:
CONC: '4'
AGENTIC_WARMUP_GRACE_PERIOD: '1800'

override_tp4_c8:
override_tp4ep1_c8:
roles:
agg:
gpus: 4
args:
tensor-parallel-size: 4
expert-parallel-size: 4
mem-fraction-static: 0.76
expert-parallel-size: 1
chunked-prefill-size: 8192
swa-prefix-tails: 256
cuda-graph-backend-prefill: breakable
enable-decoder-swa-bounded-replay: true
max-running-requests: 16
prefill-decode-interval: 384
mem-fraction-static: 0.76
prefill-decode-interval: 96
schedule-policy: shortest-prefill-first
swa-prefix-tails: 256
env:
SGLANG_OPT_DSV41_MEGA_MHC_PREFILL: '1'
benchmark:
env:
CONC: '8'
Expand All @@ -109,17 +121,39 @@ override_tp4_c16:
args:
tensor-parallel-size: 4
expert-parallel-size: 4
mem-fraction-static: 0.76
chunked-prefill-size: 8192
swa-prefix-tails: 512
max-running-requests: 32
mem-fraction-static: 0.76
prefill-decode-interval: 96
schedule-policy: shortest-prefill-first
swa-prefix-tails: 512
benchmark:
env:
CONC: '16'
AGENTIC_WARMUP_GRACE_PERIOD: '1800'

override_dep4_c32:
override_tp4ep1_c32:
roles:
agg:
gpus: 4
args:
tensor-parallel-size: 4
expert-parallel-size: 1
chunked-prefill-size: 8192
dsv4-attn-backend: trtllm
max-running-requests: 64
mem-fraction-static: 0.76
prefill-decode-interval: 80
schedule-policy: shortest-prefill-first
swa-prefix-tails: 1024
env:
SGLANG_OPT_DSV41_MEGA_MHC_PREFILL: '1'
benchmark:
env:
CONC: '32'
AGENTIC_WARMUP_GRACE_PERIOD: '1800'

override_dep4_hicache_c64:
frontend:
type: sglang-router
args:
Expand All @@ -138,22 +172,39 @@ override_dep4_c32:
data-parallel-size: 4
expert-parallel-size: 4
enable-dp-attention: true
enable-dp-lm-head: true
chunked-prefill-size: 65536
cuda-graph-backend-prefill: disabled
dsv4-attn-backend: trtllm
enable-dp-attention-local-control-broadcast: true
enable-dp-lm-head: true
enable-hierarchical-cache: true
enable-symm-mem: true
hicache-io-backend: direct
hicache-mem-layout: page_first_direct
hicache-ratio: 2
hicache-write-policy: write_back
load-balance-method: total_requests
tokenizer-worker-num: 8
max-running-requests: 384
mem-fraction-static: 0.76
chunked-prefill-size: 32768
swa-prefix-tails: 2048
max-running-requests: 256
prefill-decode-interval: 88
moe-a2a-backend: megamoe
prefill-decode-interval: 72
schedule-policy: shortest-prefill-first
swa-prefix-tails: 3072
tokenizer-worker-num: 8
env:
SGLANG_DP_USE_GATHERV: '1'
SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '16384'
SGLANG_OPT_DSV41_MEGA_MHC_PREFILL: '0'
SGLANG_RAGGED_VERIFY_MODE: static
benchmark:
env:
CONC: '32'
CONC: '64'
AGENTIC_WARMUP_GRACE_PERIOD: '3600'
AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true'
KV_OFFLOADING: dram
TOTAL_CPU_DRAM_GB: '721'

override_dep4_c64:
override_dep4_hicache_c96:
frontend:
type: sglang-router
args:
Expand All @@ -172,22 +223,39 @@ override_dep4_c64:
data-parallel-size: 4
expert-parallel-size: 4
enable-dp-attention: true
enable-dp-lm-head: true
chunked-prefill-size: 65536
cuda-graph-backend-prefill: disabled
dsv4-attn-backend: trtllm
enable-dp-attention-local-control-broadcast: true
enable-dp-lm-head: true
enable-hierarchical-cache: true
enable-symm-mem: true
hicache-io-backend: direct
hicache-mem-layout: page_first_direct
hicache-ratio: 2
hicache-write-policy: write_back
load-balance-method: total_requests
tokenizer-worker-num: 8
max-running-requests: 384
mem-fraction-static: 0.76
chunked-prefill-size: 65536
swa-prefix-tails: 2048
max-running-requests: 256
prefill-decode-interval: 72
moe-a2a-backend: megamoe
prefill-decode-interval: 56
schedule-policy: shortest-prefill-first
swa-prefix-tails: 3072
tokenizer-worker-num: 8
env:
SGLANG_DP_USE_GATHERV: '1'
SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '16384'
SGLANG_OPT_DSV41_MEGA_MHC_PREFILL: '0'
SGLANG_RAGGED_VERIFY_MODE: static
benchmark:
env:
CONC: '64'
CONC: '96'
AGENTIC_WARMUP_GRACE_PERIOD: '3600'
AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true'
KV_OFFLOADING: dram
TOTAL_CPU_DRAM_GB: '721'

override_dep4_c128:
override_dep4_hicache_c128:
frontend:
type: sglang-router
args:
Expand All @@ -206,22 +274,39 @@ override_dep4_c128:
data-parallel-size: 4
expert-parallel-size: 4
enable-dp-attention: true
enable-dp-lm-head: true
chunked-prefill-size: 65536
cuda-graph-backend-prefill: disabled
dsv4-attn-backend: trtllm
enable-dp-attention-local-control-broadcast: true
enable-dp-lm-head: true
enable-hierarchical-cache: true
enable-symm-mem: true
hicache-io-backend: direct
hicache-mem-layout: page_first_direct
hicache-ratio: 2
hicache-write-policy: write_back
load-balance-method: total_requests
tokenizer-worker-num: 8
max-running-requests: 384
mem-fraction-static: 0.76
chunked-prefill-size: 65536
swa-prefix-tails: 2048
max-running-requests: 256
prefill-decode-interval: 24
moe-a2a-backend: megamoe
prefill-decode-interval: 48
schedule-policy: shortest-prefill-first
swa-prefix-tails: 3072
tokenizer-worker-num: 8
env:
SGLANG_DP_USE_GATHERV: '1'
SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '16384'
SGLANG_OPT_DSV41_MEGA_MHC_PREFILL: '0'
SGLANG_RAGGED_VERIFY_MODE: static
benchmark:
env:
CONC: '128'
AGENTIC_WARMUP_GRACE_PERIOD: '3600'
AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true'
KV_OFFLOADING: dram
TOTAL_CPU_DRAM_GB: '721'

override_dep4_c192:
override_dep4_hicache_c160:
frontend:
type: sglang-router
args:
Expand All @@ -240,17 +325,86 @@ override_dep4_c192:
data-parallel-size: 4
expert-parallel-size: 4
enable-dp-attention: true
enable-dp-lm-head: true
chunked-prefill-size: 65536
cuda-graph-backend-prefill: disabled
dsv4-attn-backend: trtllm
enable-dp-attention-local-control-broadcast: true
enable-dp-lm-head: true
enable-hierarchical-cache: true
enable-symm-mem: true
hicache-io-backend: direct
hicache-mem-layout: page_first_direct
hicache-ratio: 2
hicache-write-policy: write_back
load-balance-method: total_requests
tokenizer-worker-num: 8
max-running-requests: 384
mem-fraction-static: 0.76
chunked-prefill-size: 65536
moe-a2a-backend: megamoe
prefill-decode-interval: 40
schedule-policy: shortest-prefill-first
swa-prefix-tails: 3072
tokenizer-worker-num: 8
env:
SGLANG_DP_USE_GATHERV: '1'
SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '16384'
SGLANG_OPT_DSV41_MEGA_MHC_PREFILL: '0'
SGLANG_RAGGED_VERIFY_MODE: static
benchmark:
env:
CONC: '160'
AGENTIC_WARMUP_GRACE_PERIOD: '3600'
AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true'
KV_OFFLOADING: dram
TOTAL_CPU_DRAM_GB: '721'

override_dep4_hicache_c256:
frontend:
type: sglang-router
args:
policy: cache_aware
request-id-headers: x-correlation-id
dp-aware: true
connect-timeout-secs: 900
request-timeout-secs: 14400
disable-health-check: true
disable-retries: true
roles:
agg:
gpus: 4
args:
tensor-parallel-size: 4
data-parallel-size: 4
expert-parallel-size: 4
enable-dp-attention: true
chunked-prefill-size: 65536
cuda-graph-backend-prefill: disabled
dsv4-attn-backend: trtllm
enable-dp-attention-local-control-broadcast: true
enable-dp-lm-head: true
enable-hierarchical-cache: true
enable-symm-mem: true
hicache-io-backend: direct
hicache-mem-layout: page_first_direct
hicache-ratio: 2
hicache-write-policy: write_back
load-balance-method: total_requests
max-running-requests: 384
prefill-decode-interval: 12
mem-fraction-static: 0.76
moe-a2a-backend: megamoe
moe-runner-backend: flashinfer_mxfp4
prefill-decode-interval: 24
schedule-policy: shortest-prefill-first
swa-prefix-tails: 3072
tokenizer-worker-num: 8
env:
SGLANG_DP_USE_GATHERV: '1'
SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '16384'
SGLANG_OPT_DSV41_MEGA_MHC_PREFILL: '0'
SGLANG_RAGGED_VERIFY_MODE: static
benchmark:
env:
CONC: '192'
CONC: '256'
AGENTIC_WARMUP_GRACE_PERIOD: '3600'
AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true'
KV_OFFLOADING: dram
TOTAL_CPU_DRAM_GB: '721'
18 changes: 8 additions & 10 deletions inferencex-e2e/configs/nvidia-master.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8583,17 +8583,14 @@ dsv41flash-fp4-gb200-vllm-agentic-dspark:

# SGLang arm for DeepSeek-V4.1-Flash AgentX on GB200, from the SGLang cookbook
# (https://lmsysorg.mintlify.app/cookbook/autoregressive/DeepSeek/DeepSeek-V4_1).
# Official CUDA 13 nightly, verified 2026-10-11 (amd64 + arm64):
# manifest sha256:8346c975515af7ec6b2692faadd5332f789001f78f958335175d4d34e170537c
# arm64 sha256:4793eb4075b33cec16c8de140c0e6eeaa439a1986eaac1085f270e52bd50e52d.
# Official CUDA 13 nightly, verified 2026-10-14 (amd64 + arm64):
# manifest sha256:ddae0e399d3d94f7827f16183301cf8dac4cf0eae0ef25590b7dac5881d713d8
# arm64 sha256:2b9d5dd760cbaf6a521b1c56d2896dfbb5c6ed76521ac3df5e84ba6872362be2.
# Native five-token DSpark (--speculative-algorithm DSPARK
# --speculative-dspark-block-size 5), thinking on, golden AL 3.51 for
# throughput, real verification for evals. The KV cache stays GPU-resident.
# TP4/EP4 from C1 to C16; aggregated TP4/EP4 with DP attention behind
# sglang-router from C32 to C192. Native backends resolve automatically
# (dsv4 / flashinfer_mxfp4 / flashinfer_cutedsl).
# throughput, real verification for evals.
dsv41flash-fp4-gb200-sglang-agentic-dspark:
image: lmsysorg/sglang:nightly-dev-cu13-20261011-d32d8fa0@sha256:8346c975515af7ec6b2692faadd5332f789001f78f958335175d4d34e170537c
image: lmsysorg/sglang:nightly-dev-cu13-20261014-684fea8a@sha256:ddae0e399d3d94f7827f16183301cf8dac4cf0eae0ef25590b7dac5881d713d8
model: deepseek-ai/DeepSeek-V4.1-Flash
model-prefix: dsv41flash
runner: cluster:gb200-nv
Expand All @@ -8605,8 +8602,9 @@ dsv41flash-fp4-gb200-sglang-agentic-dspark:
agentic-coding:
- dram-utilization: 0.80
search-space:
- { tp: 4, ep: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4, 8, 16], srt-recipe: agentic.yaml }
- { tp: 4, ep: 4, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [32, 64, 128, 192], router: { name: sglang-router, version: "0.3.2" }, srt-recipe: agentic.yaml }
- { tp: 4, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4, 8, 32], srt-recipe: agentic.yaml }
- { tp: 4, ep: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [16], srt-recipe: agentic.yaml }
- { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: mtp, conc-list: [64, 96, 128, 160, 256], router: { name: sglang-router, version: "0.3.2" }, srt-recipe: agentic.yaml }

dsv41flash-fp4-b300-vllm-agentic-dspark:
image: vllm/vllm-openai:nightly-dev-x86_64-cu130-ac9126e58aa7
Expand Down
Loading
Loading