diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b300-fp4-mtp/agentic.yaml index 61192ac5ea..9645801cc9 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b300-fp4-mtp/agentic.yaml @@ -1,12 +1,19 @@ -# DeepSeek-V4.1-Flash AgentX on B300 with SGLang native DSpark. The KV cache -# is GPU-resident. base: schema: 2 name: dsv41flash-fp4-b300-sglang-agentic model: path: hf:deepseek-ai/DeepSeek-V4.1-Flash - container: lmsysorg/sglang:nightly-dev-cu13-20260922-582389ce@sha256:35ea4d321b0735051dcce2599fc495853ad1ef30f6a1908f0a75e74204362d14 + container: lmsysorg/sglang:nightly-dev-cu13-20261004-295a53c9@sha256:7cfd4dd8e0e902d2a3c0560565963c4e172091e1586aaed15ff5a0a131394d34 precision: fp4 + identity: + model: + repo: deepseek-ai/DeepSeek-V4.1-Flash + revision: dba1be0a40aa45a94ad051997016db3960a90277 + container: + image: lmsysorg/sglang:nightly-dev-cu13-20261004-295a53c9@sha256:7cfd4dd8e0e902d2a3c0560565963c4e172091e1586aaed15ff5a0a131394d34 + frameworks: + sglang: 0.0.0.dev1+g295a53c90 + sglang-router: 0.3.2 resources: gpu_type: b300 gpus_per_node: 8 @@ -27,13 +34,13 @@ base: trust-remote-code: true # Overlap checkpoint reads with weight loading from local NVMe. weight-loader-prefetch-checkpoints: true - chunked-prefill-size: 4096 + chunked-prefill-size: 16384 # Keep active decode requests progressing while long prefixes are queued. prefill-decode-interval: 16 # DSpark is the checkpoint's bundled draft; the block size is its only knob. speculative-algorithm: DSPARK speculative-dspark-block-size: 5 - # Admission never exceeds the captured decode graph batch. + # Capture decode graphs up to batch 64. cuda-graph-max-bs-decode: 64 reasoning-parser: auto tool-call-parser: auto @@ -60,17 +67,15 @@ base: AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' -# One variant per point. Admission is 2x CONC, capped at the graph batch. SWA -# prefix tails are 64x CONC within [128, 4096]. TP2 raises the static fraction -# (0.80 from c16, 0.85 from c32) and interleaves decode more often from c32. -# Saturation points get a longer warmup drain. +# TP arms admit up to 2x CONC, capped at 64; DP4 C192 admits 384 globally. +# SWA prefix tails are 64x per-rank CONC, with a minimum of 128. override_tp4_c1: roles: agg: gpus: 4 args: tensor-parallel-size: 4 - expert-parallel-size: 4 + expert-parallel-size: 1 mem-fraction-static: 0.7 swa-prefix-tails: 128 max-running-requests: 2 @@ -79,118 +84,29 @@ override_tp4_c1: CONC: '1' AGENTIC_WARMUP_GRACE_PERIOD: '1800' -override_tp4_c2: - roles: - agg: - gpus: 4 - args: - tensor-parallel-size: 4 - expert-parallel-size: 4 - mem-fraction-static: 0.7 - swa-prefix-tails: 128 - max-running-requests: 4 - benchmark: - env: - CONC: '2' - AGENTIC_WARMUP_GRACE_PERIOD: '1800' - override_tp4_c4: roles: agg: gpus: 4 args: tensor-parallel-size: 4 - expert-parallel-size: 4 + expert-parallel-size: 1 mem-fraction-static: 0.7 swa-prefix-tails: 256 max-running-requests: 8 + prefill-decode-interval: 192 benchmark: env: CONC: '4' AGENTIC_WARMUP_GRACE_PERIOD: '1800' -override_tp4_c8: - roles: - agg: - gpus: 4 - args: - tensor-parallel-size: 4 - expert-parallel-size: 4 - mem-fraction-static: 0.7 - swa-prefix-tails: 512 - max-running-requests: 16 - benchmark: - env: - CONC: '8' - AGENTIC_WARMUP_GRACE_PERIOD: '1800' - -override_tp4_c16: - roles: - agg: - gpus: 4 - args: - tensor-parallel-size: 4 - expert-parallel-size: 4 - mem-fraction-static: 0.7 - swa-prefix-tails: 1024 - max-running-requests: 32 - benchmark: - env: - CONC: '16' - AGENTIC_WARMUP_GRACE_PERIOD: '1800' - -override_tp4_c32: - roles: - agg: - gpus: 4 - args: - tensor-parallel-size: 4 - expert-parallel-size: 4 - mem-fraction-static: 0.7 - swa-prefix-tails: 2048 - max-running-requests: 64 - benchmark: - env: - CONC: '32' - AGENTIC_WARMUP_GRACE_PERIOD: '3600' - -override_tp4_c64: - roles: - agg: - gpus: 4 - args: - tensor-parallel-size: 4 - expert-parallel-size: 4 - mem-fraction-static: 0.7 - swa-prefix-tails: 4096 - max-running-requests: 64 - benchmark: - env: - CONC: '64' - AGENTIC_WARMUP_GRACE_PERIOD: '3600' - -override_tp4_c128: - roles: - agg: - gpus: 4 - args: - tensor-parallel-size: 4 - expert-parallel-size: 4 - mem-fraction-static: 0.7 - swa-prefix-tails: 4096 - max-running-requests: 64 - benchmark: - env: - CONC: '128' - AGENTIC_WARMUP_GRACE_PERIOD: '3600' - override_tp2_c1: roles: agg: gpus: 2 args: tensor-parallel-size: 2 - expert-parallel-size: 2 + expert-parallel-size: 1 mem-fraction-static: 0.7 swa-prefix-tails: 128 max-running-requests: 2 @@ -199,65 +115,53 @@ override_tp2_c1: CONC: '1' AGENTIC_WARMUP_GRACE_PERIOD: '1800' -override_tp2_c2: - roles: - agg: - gpus: 2 - args: - tensor-parallel-size: 2 - expert-parallel-size: 2 - mem-fraction-static: 0.7 - swa-prefix-tails: 128 - max-running-requests: 4 - benchmark: - env: - CONC: '2' - AGENTIC_WARMUP_GRACE_PERIOD: '1800' - override_tp2_c4: roles: agg: gpus: 2 args: tensor-parallel-size: 2 - expert-parallel-size: 2 + expert-parallel-size: 1 mem-fraction-static: 0.7 swa-prefix-tails: 256 max-running-requests: 8 + prefill-decode-interval: 96 benchmark: env: CONC: '4' AGENTIC_WARMUP_GRACE_PERIOD: '1800' -override_tp2_c8: +override_tp2_c16: roles: agg: gpus: 2 args: tensor-parallel-size: 2 - expert-parallel-size: 2 - mem-fraction-static: 0.7 - swa-prefix-tails: 512 - max-running-requests: 16 + expert-parallel-size: 1 + mem-fraction-static: 0.8 + swa-prefix-tails: 1024 + max-running-requests: 32 + prefill-decode-interval: 192 benchmark: env: - CONC: '8' + CONC: '16' AGENTIC_WARMUP_GRACE_PERIOD: '1800' -override_tp2_c16: +override_tp2_c24: roles: agg: gpus: 2 args: tensor-parallel-size: 2 - expert-parallel-size: 2 + expert-parallel-size: 1 mem-fraction-static: 0.8 - swa-prefix-tails: 1024 - max-running-requests: 32 + swa-prefix-tails: 1536 + max-running-requests: 48 + prefill-decode-interval: 96 benchmark: env: - CONC: '16' - AGENTIC_WARMUP_GRACE_PERIOD: '1800' + CONC: '24' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' override_tp2_c32: roles: @@ -265,44 +169,62 @@ override_tp2_c32: gpus: 2 args: tensor-parallel-size: 2 - expert-parallel-size: 2 + expert-parallel-size: 1 mem-fraction-static: 0.85 swa-prefix-tails: 2048 max-running-requests: 64 - prefill-decode-interval: 4 + prefill-decode-interval: 64 benchmark: env: CONC: '32' AGENTIC_WARMUP_GRACE_PERIOD: '3600' -override_tp2_c64: +override_tp2_c48: roles: agg: gpus: 2 args: tensor-parallel-size: 2 - expert-parallel-size: 2 - mem-fraction-static: 0.85 - swa-prefix-tails: 4096 + expert-parallel-size: 1 + mem-fraction-static: 0.8 + swa-prefix-tails: 3072 max-running-requests: 64 - prefill-decode-interval: 4 + prefill-decode-interval: 64 benchmark: env: - CONC: '64' + CONC: '48' AGENTIC_WARMUP_GRACE_PERIOD: '3600' -override_tp2_c128: +override_dp4_c192: + frontend: + type: sglang-router + enable_multiple_frontends: false + args: + policy: cache_aware + request-id-headers: x-correlation-id + dp-aware: true + connect-timeout-secs: 900 + request-timeout-secs: 14400 + disable-health-check: true + disable-retries: true roles: agg: - gpus: 2 + gpus: 4 args: - tensor-parallel-size: 2 - expert-parallel-size: 2 - mem-fraction-static: 0.85 - swa-prefix-tails: 4096 - max-running-requests: 64 - prefill-decode-interval: 4 - benchmark: - env: - CONC: '128' + tensor-parallel-size: 4 + expert-parallel-size: 4 + data-parallel-size: 4 + enable-dp-attention: true + enable-dp-lm-head: true + enable-dp-attention-local-control-broadcast: true + load-balance-method: total_requests + tokenizer-worker-num: 8 + chunked-prefill-size: 32768 + mem-fraction-static: 0.76 + swa-prefix-tails: 3072 + max-running-requests: 384 + benchmark: + env: + CONC: '192' AGENTIC_WARMUP_GRACE_PERIOD: '3600' + AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true' diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 0d57071448..2028bf148d 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -9207,11 +9207,9 @@ qwen3.5-fp8-b300-dynamo-sglang-agentic-disagg: ep: 2 dp-attn: false -# Official 2026-09-21 CUDA 13 nightly, amd64 manifest pinned for B300. -# Latest multiarch index: sha256:987c7e4bd26918647211a5dcad72a2bdf2a5f394ac1469eff730e7517fc139be. -# Uses the cookbook's automatic backends and the existing B200 AgentX memory bounds. +# Official 2026-10-04 CUDA 13 nightly, multiarch manifest pinned for B300. dsv41flash-fp4-b300-sglang-agentic-dspark: - image: lmsysorg/sglang:nightly-dev-cu13-20260922-582389ce@sha256:35ea4d321b0735051dcce2599fc495853ad1ef30f6a1908f0a75e74204362d14 + image: lmsysorg/sglang:nightly-dev-cu13-20261004-295a53c9@sha256:7cfd4dd8e0e902d2a3c0560565963c4e172091e1586aaed15ff5a0a131394d34 model: deepseek-ai/DeepSeek-V4.1-Flash model-prefix: dsv41flash runner: cluster:b300-dsxe @@ -9222,8 +9220,9 @@ dsv41flash-fp4-b300-sglang-agentic-dspark: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, ep: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b300-fp4-mtp/agentic.yaml } - - { tp: 2, ep: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 2, 4, 8, 16, 32, 64, 128], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b300-fp4-mtp/agentic.yaml } + - { tp: 4, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b300-fp4-mtp/agentic.yaml } + - { tp: 2, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4, 16, 24, 32, 48], srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b300-fp4-mtp/agentic.yaml } + - { tp: 4, ep: 4, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [192], router: { name: sglang-router, version: "0.3.2" }, srt-recipe: benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/b300-fp4-mtp/agentic.yaml } # Official SGLang nightly; TP4 C1/C2 use GPU Engram, other points use host DRAM. dsv41flash-fp4-gb300-sglang-agentic-dspark: diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index f292e9c263..0559184813 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9507,3 +9507,13 @@ - "Add Qwen3.5 FP8 GB300 AgentX PP4 prefill and DEP16 decode variants with MTP6, HiCache, and decode sequence-length splits 1." - "Select five disaggregated configurations and three existing aggregate configurations using the SGLang nightly image and pinned Dynamo revision." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3840 + +- config-keys: + - dsv41flash-fp4-b300-sglang-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Repin DeepSeek-V4.1-Flash FP4 B300 SGLang AgentX from nightly-dev-cu13-20260922-582389ce to nightly-dev-cu13-20261004-295a53c9, pinned by multiarch digest, and publish nine points: TP4/EP1 at concurrency 1 and 4, TP2/EP1 at concurrency 1, 4, 16, 24, 32 and 48, and aggregated TP4/EP4/DP4 at concurrency 192." + - "For the TP arms, switch to EP1, use 16K prefill chunks, and set point-specific prefill-decode scheduling." + - "Add an aggregated DP4 C192 point with four internal TP4/EP4 data-parallel ranks behind sglang-router 0.3.2, using cache-aware routing and correlation-ID affinity." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3744