diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml index 5e2915d307..1d56ebd861 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/dsv41flash/sglang/gb300-fp4-mtp/agentic.yaml @@ -29,6 +29,7 @@ base: cuda-graph-max-bs-decode: 64 reasoning-parser: auto tool-call-parser: auto + tokenizer-backend: fastokens # Draft passes under long-context load outlast the 1800 s default watchdog. watchdog-timeout: 3600 enable-metrics: true @@ -56,15 +57,6 @@ base: AIPERF_HTTP_TCP_USER_TIMEOUT: '900000' AIPERF_REQUIRED_SERVER_METRIC_PREFIX: 'sglang:' -# One variant per point. Admission is 2x CONC, capped at the graph batch. SWA -# prefix tails are 64x CONC up to 4096. TP2 keeps row-sharded Engram tables in -# host DRAM and runs 16K chunks with decoder SWA bounded replay and a longer -# prefill-decode interval. c64 and c128 run DEP2 (DP attention with MegaMoE) -# behind the SGLang router with shortest-prefill-first scheduling. TP4 uses -# static ragged verify and 16K chunks; c1 keeps Engram in HBM with no -# prefill-decode interval. TP4/EP1 c4 and c16 keep Engram in host DRAM with 4096 -# SWA tails and a 128 graph batch; c16 adds a 192-step interval. Saturation -# points get a longer warmup drain. override_tp2_c16: roles: agg: @@ -137,6 +129,7 @@ override_dep2_c64: tokenizer-worker-num: 4 chunked-prefill-size: 16384 swa-prefix-tails: 4096 + enable-decoder-swa-bounded-replay: true prefill-decode-interval: 24 max-running-requests: 128 env: &dep2_env @@ -169,6 +162,24 @@ override_dep2_c128: <<: *dep2_client CONC: '128' +override_dep2_c160: + frontend: *dep2_frontend + roles: + agg: + gpus: 2 + args: + <<: *dep2_args + prefill-decode-interval: 8 + mem-fraction-static: 0.85 + max-running-requests: 256 + cuda-graph-max-bs-decode: 128 + tokenizer-backend: huggingface + env: *dep2_env + benchmark: + env: + <<: *dep2_client + CONC: '160' + override_tp4_c1: roles: agg: @@ -178,7 +189,9 @@ override_tp4_c1: expert-parallel-size: 1 chunked-prefill-size: 16384 prefill-decode-interval: 0 - max-running-requests: 2 + max-running-requests: 1 + enable-decoder-swa-bounded-replay: true + cuda-graph-backend-prefill: disabled env: SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE: '0' SGLANG_RAGGED_VERIFY_MODE: static diff --git a/inferencex-e2e/configs/nvidia-master.yaml b/inferencex-e2e/configs/nvidia-master.yaml index 69dc0904bd..610f44b40f 100644 --- a/inferencex-e2e/configs/nvidia-master.yaml +++ b/inferencex-e2e/configs/nvidia-master.yaml @@ -8946,7 +8946,7 @@ dsv41flash-fp4-b300-sglang-agentic-dspark: # Official SGLang nightly; TP4 C1 uses GPU Engram, other points use host DRAM. dsv41flash-fp4-gb300-sglang-agentic-dspark: - image: lmsysorg/sglang:nightly-dev-cu13-20261012-e6072935@sha256:b5c488694d74c13587af6057b90220498c4869571c8e1e3c43dd6f43ae721306 + image: lmsysorg/sglang:nightly-dev-cu13-20261014-684fea8a@sha256:ddae0e399d3d94f7827f16183301cf8dac4cf0eae0ef25590b7dac5881d713d8 model: deepseek-ai/DeepSeek-V4.1-Flash model-prefix: dsv41flash runner: cluster:gb300-nv @@ -8959,7 +8959,7 @@ dsv41flash-fp4-gb300-sglang-agentic-dspark: - dram-utilization: 0.80 search-space: - { tp: 2, ep: 2, kv-offloading: none, spec-decoding: mtp, conc-list: [16, 32], srt-recipe: agentic.yaml } - - { tp: 2, ep: 2, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [64, 128], router: { name: sglang-router, version: "0.3.2" }, srt-recipe: agentic.yaml } + - { tp: 2, ep: 2, dp-attn: true, kv-offloading: none, spec-decoding: mtp, conc-list: [64, 128, 160], router: { name: sglang-router, version: "0.3.2" }, srt-recipe: agentic.yaml } - { tp: 4, ep: 1, kv-offloading: none, spec-decoding: mtp, conc-list: [1, 4, 16], srt-recipe: agentic.yaml } dsv41flash-fp4-gb300-dynamo-sglang-agentic-agg: image: lmsysorg/sglang:nightly-dev-20260928-81f27fb3@sha256:d9e4917808cfaa4b3be033a0c85a3a73d71eb93c17accbfa2ac3e96f551f3a33 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index fefd7900d2..32157d3f58 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -10022,3 +10022,15 @@ - "Append GB300 DeepSeek-V4.1-Flash Dynamo + vLLM AgentX c5120 (8P1D) and c6656 (10P1D) points using the agentx-harness 1.0.10 release already pinned on main." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4126 append-only: true + +- config-keys: + - dsv41flash-fp4-gb300-sglang-agentic-dspark + scenario-type: + - agentic-coding + description: + - "Bump the image to lmsysorg/sglang:nightly-dev-cu13-20261014-684fea8a, which ships the fastokens tokenizer and decoder SWA bounded replay with DP attention." + - "Use tokenizer-backend fastokens on every point except DEP2 c160, which keeps the HuggingFace tokenizer." + - "Enable decoder SWA bounded replay on DEP2 c64 and c128." + - "TP4 c1 admits one request at a time, enables decoder SWA bounded replay and disables the prefill CUDA graph." + - "Add DEP2 c160 with the c128 settings, a prefill-decode interval of 8, mem-fraction-static 0.85, 256 max running requests and a 128 decode graph batch." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/4033