From 95a3d514753349cffc252af957d6559adeebe258 Mon Sep 17 00:00:00 2001 From: shyeh25 <206795756+shyeh25@users.noreply.github.com> Date: Thu, 15 Oct 2026 05:54:59 -0700 Subject: [PATCH 1/2] perf(dsv41flash): retune GB200 SGLang AgentX curve on 20261014 nightly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move dsv41flash-fp4-gb200-sglang-agentic-dspark to lmsysorg/sglang:nightly-dev-cu13-20261014-684fea8a (multiarch digest pinned) and replace the eight points with ten tuned points: TP4/EP1 at C1/C4/C8/C32, TP4/EP4 at C16, and aggregated TP4/EP4 with DP attention behind sglang-router at C64/C96/C128/C160/C256 using the MegaMoE all-to-all backend and HiCache host-DRAM KV offload. The recipe carries the per-point schedule policy, attention backend, prefill-decode interval, prefill chunking, admission, memory and SWA prefix-tail settings. 将 dsv41flash-fp4-gb200-sglang-agentic-dspark 更新为 lmsysorg/sglang:nightly-dev-cu13-20261014-684fea8a（固定 multiarch digest），并以十个调优点取代原本的八个点：TP4/EP1 的 C1/C4/C8/C32、 TP4/EP4 的 C16，以及在 sglang-router 之后启用 DP attention 的聚合式 TP4/EP4 的 C64/C96/C128/C160/C256（使用 MegaMoE all-to-all 后端与 HiCache 主机 DRAM KV offload）。recipe 包含各点的调度策略、attention 后端、prefill-decode interval、 prefill chunk、运行请求上限、显存与 SWA 前缀尾部设置。 Co-Authored-By: Claude Opus 5.5 --- .../sglang/gb200-fp4-mtp/agentic.yaml | 244 ++++++++++++++---- inferencex-e2e/configs/nvidia-master.yaml | 22 +- inferencex-e2e/perf-changelog.yaml | 11 + 3 files changed, 222 insertions(+), 55 deletions(-) diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb200-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb200-fp4-mtp/agentic.yaml index 639b6c3993..029403c1bb 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb200-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb200-fp4-mtp/agentic.yaml @@ -30,6 +30,7 @@ base: # Draft passes under long-context load outlast the 1800 s default watchdog. watchdog-timeout: 3600 enable-metrics: true + tokenizer-backend: fastokens env: PYTHONNOUSERSITE: '1' PYTHONUNBUFFERED: '1' @@ -52,51 +53,62 @@ base: AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' -override_tp4_c1: +override_tp4ep1_c1: roles: agg: gpus: 4 args: tensor-parallel-size: 4 - expert-parallel-size: 4 - mem-fraction-static: 0.76 + expert-parallel-size: 1 chunked-prefill-size: 8192 + cuda-graph-max-bs-decode: 1 + max-running-requests: 1 + mem-fraction-static: 0.88 swa-prefix-tails: 128 - max-running-requests: 2 + env: + SGLANG_OPT_DSV41_MEGA_MHC_PREFILL: '1' benchmark: env: CONC: '1' AGENTIC_WARMUP_GRACE_PERIOD: '1800' -override_tp4_c4: +override_tp4ep1_c4: roles: agg: gpus: 4 args: tensor-parallel-size: 4 - expert-parallel-size: 4 - mem-fraction-static: 0.76 + expert-parallel-size: 1 chunked-prefill-size: 8192 - swa-prefix-tails: 128 max-running-requests: 8 - prefill-decode-interval: 384 + mem-fraction-static: 0.76 + prefill-decode-interval: 64 + schedule-policy: shortest-prefill-first + swa-prefix-tails: 128 + env: + SGLANG_OPT_DSV41_MEGA_MHC_PREFILL: '1' benchmark: env: CONC: '4' AGENTIC_WARMUP_GRACE_PERIOD: '1800' -override_tp4_c8: +override_tp4ep1_c8: roles: agg: gpus: 4 args: tensor-parallel-size: 4 - expert-parallel-size: 4 - mem-fraction-static: 0.76 + expert-parallel-size: 1 chunked-prefill-size: 8192 - swa-prefix-tails: 256 + cuda-graph-backend-prefill: breakable + enable-decoder-swa-bounded-replay: true max-running-requests: 16 - prefill-decode-interval: 384 + mem-fraction-static: 0.76 + prefill-decode-interval: 96 + schedule-policy: shortest-prefill-first + swa-prefix-tails: 256 + env: + SGLANG_OPT_DSV41_MEGA_MHC_PREFILL: '1' benchmark: env: CONC: '8' @@ -109,17 +121,39 @@ override_tp4_c16: args: tensor-parallel-size: 4 expert-parallel-size: 4 - mem-fraction-static: 0.76 chunked-prefill-size: 8192 - swa-prefix-tails: 512 max-running-requests: 32 + mem-fraction-static: 0.76 prefill-decode-interval: 96 + schedule-policy: shortest-prefill-first + swa-prefix-tails: 512 benchmark: env: CONC: '16' AGENTIC_WARMUP_GRACE_PERIOD: '1800' -override_dep4_c32: +override_tp4ep1_c32: + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + expert-parallel-size: 1 + chunked-prefill-size: 8192 + dsv4-attn-backend: trtllm + max-running-requests: 64 + mem-fraction-static: 0.76 + prefill-decode-interval: 80 + schedule-policy: shortest-prefill-first + swa-prefix-tails: 1024 + env: + SGLANG_OPT_DSV41_MEGA_MHC_PREFILL: '1' + benchmark: + env: + CONC: '32' + AGENTIC_WARMUP_GRACE_PERIOD: '1800' + +override_dep4_hicache_c64: frontend: type: sglang-router args: @@ -138,22 +172,39 @@ override_dep4_c32: data-parallel-size: 4 expert-parallel-size: 4 enable-dp-attention: true - enable-dp-lm-head: true + chunked-prefill-size: 65536 + cuda-graph-backend-prefill: disabled + dsv4-attn-backend: trtllm enable-dp-attention-local-control-broadcast: true + enable-dp-lm-head: true + enable-hierarchical-cache: true + enable-symm-mem: true + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + hicache-ratio: 2 + hicache-write-policy: write_back load-balance-method: total_requests - tokenizer-worker-num: 8 + max-running-requests: 384 mem-fraction-static: 0.76 - chunked-prefill-size: 32768 - swa-prefix-tails: 2048 - max-running-requests: 256 - prefill-decode-interval: 88 + moe-a2a-backend: megamoe + prefill-decode-interval: 72 + schedule-policy: shortest-prefill-first + swa-prefix-tails: 3072 + tokenizer-worker-num: 8 + env: + SGLANG_DP_USE_GATHERV: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '16384' + SGLANG_OPT_DSV41_MEGA_MHC_PREFILL: '0' + SGLANG_RAGGED_VERIFY_MODE: static benchmark: env: - CONC: '32' + CONC: '64' AGENTIC_WARMUP_GRACE_PERIOD: '3600' AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '721' -override_dep4_c64: +override_dep4_hicache_c96: frontend: type: sglang-router args: @@ -172,22 +223,39 @@ override_dep4_c64: data-parallel-size: 4 expert-parallel-size: 4 enable-dp-attention: true - enable-dp-lm-head: true + chunked-prefill-size: 65536 + cuda-graph-backend-prefill: disabled + dsv4-attn-backend: trtllm enable-dp-attention-local-control-broadcast: true + enable-dp-lm-head: true + enable-hierarchical-cache: true + enable-symm-mem: true + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + hicache-ratio: 2 + hicache-write-policy: write_back load-balance-method: total_requests - tokenizer-worker-num: 8 + max-running-requests: 384 mem-fraction-static: 0.76 - chunked-prefill-size: 65536 - swa-prefix-tails: 2048 - max-running-requests: 256 - prefill-decode-interval: 72 + moe-a2a-backend: megamoe + prefill-decode-interval: 56 + schedule-policy: shortest-prefill-first + swa-prefix-tails: 3072 + tokenizer-worker-num: 8 + env: + SGLANG_DP_USE_GATHERV: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '16384' + SGLANG_OPT_DSV41_MEGA_MHC_PREFILL: '0' + SGLANG_RAGGED_VERIFY_MODE: static benchmark: env: - CONC: '64' + CONC: '96' AGENTIC_WARMUP_GRACE_PERIOD: '3600' AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '721' -override_dep4_c128: +override_dep4_hicache_c128: frontend: type: sglang-router args: @@ -206,22 +274,39 @@ override_dep4_c128: data-parallel-size: 4 expert-parallel-size: 4 enable-dp-attention: true - enable-dp-lm-head: true + chunked-prefill-size: 65536 + cuda-graph-backend-prefill: disabled + dsv4-attn-backend: trtllm enable-dp-attention-local-control-broadcast: true + enable-dp-lm-head: true + enable-hierarchical-cache: true + enable-symm-mem: true + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + hicache-ratio: 2 + hicache-write-policy: write_back load-balance-method: total_requests - tokenizer-worker-num: 8 + max-running-requests: 384 mem-fraction-static: 0.76 - chunked-prefill-size: 65536 - swa-prefix-tails: 2048 - max-running-requests: 256 - prefill-decode-interval: 24 + moe-a2a-backend: megamoe + prefill-decode-interval: 48 + schedule-policy: shortest-prefill-first + swa-prefix-tails: 3072 + tokenizer-worker-num: 8 + env: + SGLANG_DP_USE_GATHERV: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '16384' + SGLANG_OPT_DSV41_MEGA_MHC_PREFILL: '0' + SGLANG_RAGGED_VERIFY_MODE: static benchmark: env: CONC: '128' AGENTIC_WARMUP_GRACE_PERIOD: '3600' AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '721' -override_dep4_c192: +override_dep4_hicache_c160: frontend: type: sglang-router args: @@ -240,17 +325,86 @@ override_dep4_c192: data-parallel-size: 4 expert-parallel-size: 4 enable-dp-attention: true - enable-dp-lm-head: true + chunked-prefill-size: 65536 + cuda-graph-backend-prefill: disabled + dsv4-attn-backend: trtllm enable-dp-attention-local-control-broadcast: true + enable-dp-lm-head: true + enable-hierarchical-cache: true + enable-symm-mem: true + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + hicache-ratio: 2 + hicache-write-policy: write_back load-balance-method: total_requests - tokenizer-worker-num: 8 + max-running-requests: 384 mem-fraction-static: 0.76 - chunked-prefill-size: 65536 + moe-a2a-backend: megamoe + prefill-decode-interval: 40 + schedule-policy: shortest-prefill-first swa-prefix-tails: 3072 + tokenizer-worker-num: 8 + env: + SGLANG_DP_USE_GATHERV: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '16384' + SGLANG_OPT_DSV41_MEGA_MHC_PREFILL: '0' + SGLANG_RAGGED_VERIFY_MODE: static + benchmark: + env: + CONC: '160' + AGENTIC_WARMUP_GRACE_PERIOD: '3600' + AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '721' + +override_dep4_hicache_c256: + frontend: + type: sglang-router + args: + policy: cache_aware + request-id-headers: x-correlation-id + dp-aware: true + connect-timeout-secs: 900 + request-timeout-secs: 14400 + disable-health-check: true + disable-retries: true + roles: + agg: + gpus: 4 + args: + tensor-parallel-size: 4 + data-parallel-size: 4 + expert-parallel-size: 4 + enable-dp-attention: true + chunked-prefill-size: 65536 + cuda-graph-backend-prefill: disabled + dsv4-attn-backend: trtllm + enable-dp-attention-local-control-broadcast: true + enable-dp-lm-head: true + enable-hierarchical-cache: true + enable-symm-mem: true + hicache-io-backend: direct + hicache-mem-layout: page_first_direct + hicache-ratio: 2 + hicache-write-policy: write_back + load-balance-method: total_requests max-running-requests: 384 - prefill-decode-interval: 12 + mem-fraction-static: 0.76 + moe-a2a-backend: megamoe + moe-runner-backend: flashinfer_mxfp4 + prefill-decode-interval: 24 + schedule-policy: shortest-prefill-first + swa-prefix-tails: 3072 + tokenizer-worker-num: 8 + env: + SGLANG_DP_USE_GATHERV: '1' + SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK: '16384' + SGLANG_OPT_DSV41_MEGA_MHC_PREFILL: '0' + SGLANG_RAGGED_VERIFY_MODE: static benchmark: env: - CONC: '192' + CONC: '256' AGENTIC_WARMUP_GRACE_PERIOD: '3600' AIPERF_HTTP_X_SMG_ROUTING_KEY_FROM_CORRELATION_ID: 'true' + KV_OFFLOADING: dram + TOTAL_CPU_DRAM_GB: '721' diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index b50c15d71b..2e134c37f8 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -8583,17 +8583,14 @@ dsv41flash-fp4-gb200-vllm-agentic-dspark: # SGLang arm for DeepSeek-V4.1-Flash AgentX on GB200, from the SGLang cookbook # (https://lmsysorg.mintlify.app/cookbook/autoregressive/DeepSeek/DeepSeek-V4_1). -# Official CUDA 13 nightly, verified 2026-10-11 (amd64 + arm64): -# manifest sha256:8346c975515af7ec6b2692faadd5332f789001f78f958335175d4d34e170537c -# arm64 sha256:4793eb4075b33cec16c8de140c0e6eeaa439a1986eaac1085f270e52bd50e52d. +# Official CUDA 13 nightly, verified 2026-10-14 (amd64 + arm64): +# manifest sha256:ddae0e399d3d94f7827f16183301cf8dac4cf0eae0ef25590b7dac5881d713d8 +# arm64 sha256:2b9d5dd760cbaf6a521b1c56d2896dfbb5c6ed76521ac3df5e84ba6872362be2. # Native five-token DSpark (--speculative-algorithm DSPARK # --speculative-dspark-block-size 5), thinking on, golden AL 3.51 for -# throughput, real verification for evals. The KV cache stays GPU-resident. -# TP4/EP4 from C1 to C16; aggregated TP4/EP4 with DP attention behind -# sglang-router from C32 to C192. Native backends resolve automatically -# (dsv4 / flashinfer_mxfp4 / flashinfer_cutedsl). +# throughput, real verification for evals. dsv41flash-fp4-gb200-sglang-agentic-dspark: - image: lmsysorg/sglang:nightly-dev-cu13-20261011-d32d8fa0@sha256:8346c975515af7ec6b2692faadd5332f789001f78f958335175d4d34e170537c + image: lmsysorg/sglang:nightly-dev-cu13-20261014-684fea8a@sha256:ddae0e399d3d94f7827f16183301cf8dac4cf0eae0ef25590b7dac5881d713d8 model: deepseek-ai/DeepSeek-V4.1-Flash model-prefix: dsv41flash runner: cluster:gb200-nv @@ -8605,8 +8602,13 @@ dsv41flash-fp4-gb200-sglang-agentic-dspark: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 4, ep: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4, 8, 16], srt-recipe: agentic.yaml } - - { tp: 4, ep: 4, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [32, 64, 128, 192], router: { name: sglang-router, version: "0.3.2" }, srt-recipe: agentic.yaml } + # TP4 with tensor-parallel experts (EP1) at C1-C8 and C32; the KV cache stays GPU-resident. + - { tp: 4, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4, 8, 32], srt-recipe: agentic.yaml } + # TP4/EP4 at C16; the KV cache stays GPU-resident. + - { tp: 4, ep: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [16], srt-recipe: agentic.yaml } + # TP4/EP4 with DP attention and MegaMoE behind sglang-router; HiCache extends the + # prefix cache into host DRAM. + - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: mtp, conc-list: [64, 96, 128, 160, 256], router: { name: sglang-router, version: "0.3.2" }, srt-recipe: agentic.yaml } dsv41flash-fp4-b300-vllm-agentic-dspark: image: vllm/vllm-openai:nightly-dev-x86_64-cu130-ac9126e58aa7 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index ab7f22149d..fb7bfa7044 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9843,3 +9843,14 @@ - "Extend TP2 AgentX to concurrency 32 so TP-only overlaps DP2 at c32." - "Enable SGLANG_OPT_HIP_FLYDSL_MXFP8=1." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4050 + +- config-keys: + - dsv41flash-fp4-gb200-sglang-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Update the image from lmsysorg/sglang:nightly-dev-cu13-20261011-d32d8fa0 to lmsysorg/sglang:nightly-dev-cu13-20261014-684fea8a, pinned by multiarch digest (sha256:ddae0e399d3d94f7827f16183301cf8dac4cf0eae0ef25590b7dac5881d713d8)." + - "Run TP4/EP1 at concurrency 1, 4, 8 and 32 and TP4/EP4 at concurrency 16; run TP4/EP4 with DP attention (DP4) behind sglang-router 0.3.2 at concurrency 64, 96, 128, 160 and 256 with the MegaMoE all-to-all backend and HiCache host-DRAM KV offload." + - "Use the shortest-prefill-first schedule policy from concurrency 4, TRT-LLM sparse attention from concurrency 32, SWA bounded replay with breakable prefill CUDA graphs at concurrency 8, and point-specific prefill-decode interval, prefill chunking, admission, memory and SWA prefix-tail settings from the single-node recipe." + - "No data-type or precision change to the DSpark draft: the bundled draft of deepseek-ai/DeepSeek-V4.1-Flash runs as shipped with block size 5." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX From b260d0e4ef31e76bacba68cce9a8107ba4922c72 Mon Sep 17 00:00:00 2001 From: shyeh25 <206795756+shyeh25@users.noreply.github.com> Date: Thu, 15 Oct 2026 07:11:37 -0700 Subject: [PATCH 2/2] chore(dsv41flash): drop search-space comments from the GB200 SGLang entry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove the per-entry comments that restated the topology and concurrency of each search-space line. The configuration is unchanged. 删除 GB200 SGLang 条目中逐行复述拓扑与并发的 search-space 注释，配置不变。 Co-Authored-By: Claude Opus 5.5 --- inferencex-e2e/configs/nvidia-master.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 2e134c37f8..c6a043a9f0 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -8602,12 +8602,8 @@ dsv41flash-fp4-gb200-sglang-agentic-dspark: agentic-coding: - dram-utilization: 0.80 search-space: - # TP4 with tensor-parallel experts (EP1) at C1-C8 and C32; the KV cache stays GPU-resident. - { tp: 4, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4, 8, 32], srt-recipe: agentic.yaml } - # TP4/EP4 at C16; the KV cache stays GPU-resident. - { tp: 4, ep: 4, kv-offloading: none, spec-decoding: mtp, conc-list: [16], srt-recipe: agentic.yaml } - # TP4/EP4 with DP attention and MegaMoE behind sglang-router; HiCache extends the - # prefix cache into host DRAM. - { tp: 4, ep: 4, dp-attn: true, kv-offloading: dram, kv-offload-backend: { name: hicache }, spec-decoding: mtp, conc-list: [64, 96, 128, 160, 256], router: { name: sglang-router, version: "0.3.2" }, srt-recipe: agentic.yaml } dsv41flash-fp4-b300-vllm-agentic-dspark: