From a3feffecaf82c2d58ca09f18cd15ee8a7506a2b4 Mon Sep 17 00:00:00 2001 From: Karan Verma Date: Sun, 23 Aug 2026 02:31:19 -0500 Subject: [PATCH 1/4] Move the DSv4 MI355X SGLang AgentX arm to the sgl-dev v0.5.18 image Ports the image and serving flags validated on the AMD fork onto this key. Runner and search space are unchanged. - image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260813 -> rocm/sgl-dev:v0.5.18-rocm720-mi35x-20260822 - enable the DSv4 kernel fusions the new image carries, all default off in environ.py: SGLANG_OPT_USE_AITER_BATCHED_GEMM, SGLANG_OPT_NATIVE_BPRESHUFFLE_SCALE and SGLANG_OPT_FUSE_COMPRESS_NORM_ROPE - mem-fraction-static 0.85 -> 0.89 to enlarge the full-attention KV pool, keeping enough remainder for the compressor state pools that are allocated after it - --disable-shared-experts-fusion -> --enforce-shared-experts-fusion; the tuned MoE tables in this image cover the resulting (inter_dim 384, expert 385, topk 7) shape at the token sizes this workload hits The stale comment claiming the image matches the disagg AgentX entry is updated; that key is still on v0.5.17-20260817. Co-authored-by: Cursor --- .../agentic/dsv4_fp4_mi355x_sglang_mtp.sh | 14 +++++++++++--- configs/amd-master.yaml | 7 ++++--- perf-changelog.yaml | 11 +++++++++++ 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh index 61add3eafa..c0f957a7c1 100644 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh @@ -73,6 +73,12 @@ export SGLANG_DSV4_REASONING_EFFORT=high export SGLANG_USE_ROCM700A=0 export SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton export AITER_BF16_FP8_MOE_BOUND=0 +# DSv4 kernel fusions carried by the sgl-dev image: aiter batched GEMM for the +# absorbed MLA projections, native b-preshuffle scale handling, and a fused +# compress+norm+rope epilogue. All three default off in environ.py. +export SGLANG_OPT_USE_AITER_BATCHED_GEMM=1 +export SGLANG_OPT_NATIVE_BPRESHUFFLE_SCALE=1 +export SGLANG_OPT_FUSE_COMPRESS_NORM_ROPE=1 # Unified radix tree: per-component (full-attn / SWA) cache management for # hybrid-attention models, plus proactive release of out-of-window SWA KV @@ -130,8 +136,10 @@ SGLANG_BACKEND_PORT="$PORT" # cap; same value the multi-node DeepSeek-V4-Pro-AgentX no_dp profile uses. CHUNKED_PREFILL_SIZE=8192 # MTP adds a draft KV pool and extra graph captures on top of the spec-none -# footprint, which ran at 0.90. -MEM_FRACTION_STATIC=0.85 +# footprint, which ran at 0.90. 0.89 recovers most of that: the DSv4 compressor +# state pools are sized from the full-attention pool and allocated after it, +# outside this budget, so the remainder has to stay large enough to cover them. +MEM_FRACTION_STATIC=0.89 PARALLEL_ARGS=(--tensor-parallel-size "$TP") if [ "$DP_ATTENTION" = "true" ]; then USE_SGLANG_ROUTER=true @@ -213,7 +221,7 @@ SGLANG_CMD=( --page-size 256 --swa-full-tokens-ratio 0.10 --kv-cache-dtype fp8_e4m3 - --disable-shared-experts-fusion + --enforce-shared-experts-fusion --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --chunked-prefill-size "$CHUNKED_PREFILL_SIZE" diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 67a0605398..4803850d7f 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1722,12 +1722,13 @@ glm5.2-fp4-mi355x-atom-agentic-mtp: # SGLang AgentX coverage removed in de493d859 (PR #2531, which deleted the # spec-none dsv4-fp4-mi355x-sglang-agentic-hicache key and its script) so the # aggregated SGLang path is comparable with dsv4-fp4-mi355x-vllm-agentic-mtp -# and dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp. The image matches the -# already-green disagg AgentX entry. Pure TP8 only: DSA + dp-attention hangs a +# and dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp. The image is the +# sgl-dev v0.5.18 build, which carries the DSv4 kernel fusions this key enables +# via SGLANG_OPT_*. Pure TP8 only: DSA + dp-attention hangs a # collective under long-context prefill, so no DEP arm ships until that path is # validated. conc 16 appears on both arms to isolate the host KV tier's gain. dsv4-fp4-mi355x-sglang-agentic-mtp: - image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260813 + image: rocm/sgl-dev:v0.5.18-rocm720-mi35x-20260822 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:mi355x-amds diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 98eef2c543..26bc066899 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6378,3 +6378,14 @@ - "Serve the TP8 arm with FULL_AND_PIECEWISE cudagraphs, capturing the mixed prefill/decode batches as well as the uniform decode batches. The DEP8 arm is unchanged." - "Replace the TP8 concurrency list [1, 4, 8, 16, 24] with [1, 4, 8, 14], dropping the published concurrency 16 and 24 points and adding concurrency 14." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2707 + +- config-keys: + - dsv4-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Update image from lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260813 to rocm/sgl-dev:v0.5.18-rocm720-mi35x-20260822." + - "Enable the DSv4 kernel fusions the new image carries, all of which default off: SGLANG_OPT_USE_AITER_BATCHED_GEMM=1 (aiter batched GEMM for the absorbed MLA projections), SGLANG_OPT_NATIVE_BPRESHUFFLE_SCALE=1, SGLANG_OPT_FUSE_COMPRESS_NORM_ROPE=1." + - "Raise mem-fraction-static from 0.85 to 0.89, enlarging the full-attention KV pool. The DSv4 compressor state pools are sized from that pool and allocated after it, outside this budget, so the remainder still has to cover them." + - "Switch --disable-shared-experts-fusion to --enforce-shared-experts-fusion, which fuses the single shared expert into the routed list. The tuned MoE tables in this image cover the resulting (inter_dim 384, expert 385, topk 7) shape at the token sizes this workload hits." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX From 5823400933df12c9cbcc37f66eb7d43f0107acc2 Mon Sep 17 00:00:00 2001 From: Karan Verma Date: Sun, 23 Aug 2026 02:33:53 -0500 Subject: [PATCH 2/4] Split the DSv4 MI355X SGLang AgentX search space by concurrency TP4 owns the low-concurrency end at [1, 2, 4, 8, 10] with no host KV tier, where halving the GPUs per replica raises throughput per GPU. The TP8 no-offload arm drops from [1, 2, 4, 8, 16] to [16], so TP8 picks up where TP4 leaves off instead of duplicating those points. conc 16 still appears on both TP8 arms to isolate the host KV tier's gain. This removes the published TP8 concurrency 1, 2, 4 and 8 points and adds TP4 concurrency 1, 2, 4, 8 and 10. TP4 doubles per-rank weights and leaves far less room for KV, which is why it stops at concurrency 10 and ships no hicache variant. The comment's "pure TP8 only" phrasing is updated to "tensor-parallel only", which is what it meant - the exclusion is DP-attention, not TP4. Co-authored-by: Cursor --- configs/amd-master.yaml | 13 +++++++++---- perf-changelog.yaml | 1 + 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 4803850d7f..4cfce21535 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1724,9 +1724,13 @@ glm5.2-fp4-mi355x-atom-agentic-mtp: # aggregated SGLang path is comparable with dsv4-fp4-mi355x-vllm-agentic-mtp # and dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp. The image is the # sgl-dev v0.5.18 build, which carries the DSv4 kernel fusions this key enables -# via SGLANG_OPT_*. Pure TP8 only: DSA + dp-attention hangs a -# collective under long-context prefill, so no DEP arm ships until that path is -# validated. conc 16 appears on both arms to isolate the host KV tier's gain. +# via SGLANG_OPT_*. Tensor-parallel only: DSA + dp-attention hangs a collective +# under long-context prefill, so no DEP arm ships until that path is validated. +# TP4 owns the low-concurrency end, where halving the GPUs per replica raises +# throughput per GPU, and TP8 picks up from conc 16; conc 16 appears on both TP8 +# arms to isolate the host KV tier's gain. TP4 doubles per-rank weights, leaving +# far less room for KV, which is why it stops at conc 10 and carries no host +# tier. dsv4-fp4-mi355x-sglang-agentic-mtp: image: rocm/sgl-dev:v0.5.18-rocm720-mi35x-20260822 model: deepseek-ai/DeepSeek-V4-Pro @@ -1739,5 +1743,6 @@ dsv4-fp4-mi355x-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, conc-list: [1, 2, 4, 8, 16], spec-decoding: mtp } + - { tp: 4, ep: 1, dp-attn: false, kv-offloading: none, conc-list: [1, 2, 4, 8, 10], spec-decoding: mtp } + - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, conc-list: [16], spec-decoding: mtp } - { tp: 8, ep: 1, dp-attn: false, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 32, 48], spec-decoding: mtp } diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 26bc066899..dd0cffd0b2 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6388,4 +6388,5 @@ - "Enable the DSv4 kernel fusions the new image carries, all of which default off: SGLANG_OPT_USE_AITER_BATCHED_GEMM=1 (aiter batched GEMM for the absorbed MLA projections), SGLANG_OPT_NATIVE_BPRESHUFFLE_SCALE=1, SGLANG_OPT_FUSE_COMPRESS_NORM_ROPE=1." - "Raise mem-fraction-static from 0.85 to 0.89, enlarging the full-attention KV pool. The DSv4 compressor state pools are sized from that pool and allocated after it, outside this budget, so the remainder still has to cover them." - "Switch --disable-shared-experts-fusion to --enforce-shared-experts-fusion, which fuses the single shared expert into the routed list. The tuned MoE tables in this image cover the resulting (inter_dim 384, expert 385, topk 7) shape at the token sizes this workload hits." + - "Split the parallelism by concurrency: add a TP4 arm at [1, 2, 4, 8, 10] with no host KV tier, where halving the GPUs per replica raises throughput per GPU, and reduce the TP8 no-offload arm from [1, 2, 4, 8, 16] to [16]. This drops the published TP8 concurrency 1, 2, 4 and 8 points and adds TP4 concurrency 1, 2, 4, 8 and 10." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX From 9a0194ccf81aca01a3a18798a67c09ce11a165e7 Mon Sep 17 00:00:00 2001 From: Karan Verma Date: Sun, 23 Aug 2026 02:51:06 -0500 Subject: [PATCH 3/4] Point the changelog entry at PR #2710 Co-authored-by: Cursor --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index dd0cffd0b2..a226799e1c 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6389,4 +6389,4 @@ - "Raise mem-fraction-static from 0.85 to 0.89, enlarging the full-attention KV pool. The DSv4 compressor state pools are sized from that pool and allocated after it, outside this budget, so the remainder still has to cover them." - "Switch --disable-shared-experts-fusion to --enforce-shared-experts-fusion, which fuses the single shared expert into the routed list. The tuned MoE tables in this image cover the resulting (inter_dim 384, expert 385, topk 7) shape at the token sizes this workload hits." - "Split the parallelism by concurrency: add a TP4 arm at [1, 2, 4, 8, 10] with no host KV tier, where halving the GPUs per replica raises throughput per GPU, and reduce the TP8 no-offload arm from [1, 2, 4, 8, 16] to [16]. This drops the published TP8 concurrency 1, 2, 4 and 8 points and adds TP4 concurrency 1, 2, 4, 8 and 10." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2710 From 4a1f75d030eacb8127ff08ac6639126e32be3e05 Mon Sep 17 00:00:00 2001 From: karverma-amd Date: Sun, 23 Aug 2026 20:38:40 -0500 Subject: [PATCH 4/4] Update amd-master.yaml --- configs/amd-master.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 4cfce21535..563d3ffa1c 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1732,7 +1732,7 @@ glm5.2-fp4-mi355x-atom-agentic-mtp: # far less room for KV, which is why it stops at conc 10 and carries no host # tier. dsv4-fp4-mi355x-sglang-agentic-mtp: - image: rocm/sgl-dev:v0.5.18-rocm720-mi35x-20260822 + image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260822 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:mi355x-amds