From 0541dbcfd79a60eac1b89b41ebfe1b881581784f Mon Sep 17 00:00:00 2001 From: Jiejing Zhang Date: Sun, 13 Sep 2026 18:31:01 +0000 Subject: [PATCH 1/4] Bump GLM-5.2 MI355X SGLang daily image MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将 GLM-5.2 MI355X SGLang daily 镜像更新至 20260913，并为 ROCm 启用 v2 fused top-k。 --- .../single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh | 7 ++----- configs/amd-master.yaml | 2 +- perf-changelog.yaml | 9 +++++++++ 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh index f4d318d2e..f46ad956e 100644 --- a/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh @@ -61,11 +61,8 @@ export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 # inter-turn idle gaps can reuse a socket exactly as the server closes it. # Outlast the client pool so the race cannot occur. export SGLANG_TIMEOUT_KEEP_ALIVE=900 -# The DSA indexer's top-k v2 kernel (default since v0.5.14) is JIT-compiled -# from CUDA-only source (cooperative_groups.h) and cannot build for gfx950; -# v1 dispatches to the precompiled HIP op in sgl-kernel (upstream MI355X CI -# runs DSA models the same way). -export SGLANG_OPT_USE_TOPK_V2=false +# PR #36684 & PR #36851 turned the v2 fused top-k on for GLM-5.x on ROCm. +export SGLANG_OPT_USE_TOPK_V2=true # HiCache L2 (host DRAM), optionally extended with Mooncake L3. # KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=hicache or mooncake. diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 54c789ba5..c2e3ff138 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1704,7 +1704,7 @@ minimaxm3-fp4-mi355x-vllm-agentic-mtp: # c6 and c8 removed after sweep validation: dominated by TP4/EP4 arm. # SA selects the Pareto-optimal arm per concurrency point. glm5.2-fp4-mi355x-sglang-agentic-mtp: - image: lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260910 + image: lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260913 model: amd/GLM-5.2-MXFP4 model-prefix: glm5.2 runner: cluster:mi355x-amds diff --git a/perf-changelog.yaml b/perf-changelog.yaml index ef367b14a..63dd9d342 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -7503,3 +7503,12 @@ - "Enable required power for the H200 Kimi-K3 latency recipe and route their launch, concurrency injection, failure handling and result validation through the pinned producer and shared collector." - "仅为 H200 Kimi-K3 latency 配方启用必需实测功耗。" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3044 + +- config-keys: + - glm5.2-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Update the SGLang ROCm image from lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260910 to lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260913. Enable the v2 fused top-k path for GLM-5.x on ROCm after sgl-project/sglang#36684 and #36851." + - "将 SGLang ROCm 镜像从 lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260910 更新到 lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260913；在 sgl-project/sglang#36684 和 #36851 合入后，为 ROCm 上的 GLM-5.x 启用 v2 fused top-k 路径。" + pr-link: TBD From ce81f2e445403f1cf4ef4c5607b9cea88e639812 Mon Sep 17 00:00:00 2001 From: Jiejing Zhang Date: Sun, 13 Sep 2026 18:31:40 +0000 Subject: [PATCH 2/4] Update changelog link for PR #3081 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将性能变更记录链接更新为新建的 PR #3081。 --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 63dd9d342..ea39bf716 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -7511,4 +7511,4 @@ description: - "Update the SGLang ROCm image from lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260910 to lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260913. Enable the v2 fused top-k path for GLM-5.x on ROCm after sgl-project/sglang#36684 and #36851." - "将 SGLang ROCm 镜像从 lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260910 更新到 lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260913；在 sgl-project/sglang#36684 和 #36851 合入后，为 ROCm 上的 GLM-5.x 启用 v2 fused top-k 路径。" - pr-link: TBD + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3081 From 38fb8a7a0f591d81724559fd49276f24e13c7d5b Mon Sep 17 00:00:00 2001 From: Jiejing Zhang Date: Sun, 13 Sep 2026 21:19:18 +0000 Subject: [PATCH 3/4] Disable HiCache for low-concurrency GLM-5.2 runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 在 GLM-5.2 TP4/EP4 的并发 1 和 2 配置中关闭 HiCache，并为更高并发保留 DRAM offloading。 --- configs/amd-master.yaml | 11 +++++++---- perf-changelog.yaml | 2 ++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index c2e3ff138..f9ef870c7 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1695,10 +1695,12 @@ minimaxm3-fp4-mi355x-vllm-agentic-mtp: - { tp: 2, kv-offloading: none, conc-list: [1, 2, 5], spec-decoding: mtp } # GLM-5.2 FP4 agentic-coding benchmark on MI355X via SGLang with MTP speculative -# decoding. Two arms: -# (1) TP4/EP4 with HiCache KV offloading to DRAM, concurrency sweep [1, 2, 4, 8, 10, 12]; +# decoding. Three arms: +# (1) TP4/EP4 without KV offloading at concurrency [1, 2], where the working +# set fits in GPU memory and does not need the HiCache host tier. +# (2) TP4/EP4 with HiCache KV offloading to DRAM at concurrency [4, 8, 10, 12]; # serves the throughput-dominant regime (conc ≥ 8). -# (2) TP8/EP1 without KV offloading, concurrency sweep [4, 10]; +# (3) TP8/EP1 without KV offloading, concurrency sweep [1, 2, 4, 10]; # EP=1 eliminates the MoE all-to-all collective, reducing ITL at low-to-mid # concurrency; GPU-resident KV keeps latency independent of DRAM bandwidth. # c6 and c8 removed after sweep validation: dominated by TP4/EP4 arm. @@ -1715,7 +1717,8 @@ glm5.2-fp4-mi355x-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.8 search-space: - - { tp: 4, ep: 4, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 2, 4, 8, 10, 12], spec-decoding: mtp } + - { tp: 4, ep: 4, kv-offloading: none, conc-list: [1, 2], spec-decoding: mtp } + - { tp: 4, ep: 4, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [4, 8, 10, 12], spec-decoding: mtp } - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 10], spec-decoding: mtp } # GLM-5.2 FP4 agentic-coding benchmark on MI355X via ATOM with MTP speculative # decoding. TP4 uses LMCache DRAM offload; TP8 is GPU-resident with no KV offload. diff --git a/perf-changelog.yaml b/perf-changelog.yaml index b8ed632ca..87a9061cf 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -7538,4 +7538,6 @@ description: - "Update the SGLang ROCm image from lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260910 to lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260913. Enable the v2 fused top-k path for GLM-5.x on ROCm after sgl-project/sglang#36684 and #36851." - "将 SGLang ROCm 镜像从 lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260910 更新到 lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260913；在 sgl-project/sglang#36684 和 #36851 合入后，为 ROCm 上的 GLM-5.x 启用 v2 fused top-k 路径。" + - "Disable HiCache for the TP4/EP4 arm at concurrency 1 and 2; keep HiCache DRAM offloading for concurrency 4, 8, 10, and 12." + - "在 TP4/EP4 的并发 1 和 2 配置中关闭 HiCache；并发 4、8、10 和 12 继续使用 HiCache DRAM offloading。" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3081 From 556c20bdeb871bb8d4b6c5bbc6022c5ed25f72ba Mon Sep 17 00:00:00 2001 From: Jiejing Zhang Date: Sun, 13 Sep 2026 21:19:56 +0000 Subject: [PATCH 4/4] Run low-concurrency GLM-5.2 on TP8 without HiCache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 并发 1 和 2 仅运行 TP8/EP1，并关闭 HiCache；TP4/EP4 HiCache 从并发 4 开始。 --- configs/amd-master.yaml | 9 +++------ perf-changelog.yaml | 4 ++-- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index f9ef870c7..a35dc4552 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1695,12 +1695,10 @@ minimaxm3-fp4-mi355x-vllm-agentic-mtp: - { tp: 2, kv-offloading: none, conc-list: [1, 2, 5], spec-decoding: mtp } # GLM-5.2 FP4 agentic-coding benchmark on MI355X via SGLang with MTP speculative -# decoding. Three arms: -# (1) TP4/EP4 without KV offloading at concurrency [1, 2], where the working -# set fits in GPU memory and does not need the HiCache host tier. -# (2) TP4/EP4 with HiCache KV offloading to DRAM at concurrency [4, 8, 10, 12]; +# decoding. Two arms: +# (1) TP4/EP4 with HiCache KV offloading to DRAM at concurrency [4, 8, 10, 12]; # serves the throughput-dominant regime (conc ≥ 8). -# (3) TP8/EP1 without KV offloading, concurrency sweep [1, 2, 4, 10]; +# (2) TP8/EP1 without KV offloading, concurrency sweep [1, 2, 4, 10]; # EP=1 eliminates the MoE all-to-all collective, reducing ITL at low-to-mid # concurrency; GPU-resident KV keeps latency independent of DRAM bandwidth. # c6 and c8 removed after sweep validation: dominated by TP4/EP4 arm. @@ -1717,7 +1715,6 @@ glm5.2-fp4-mi355x-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.8 search-space: - - { tp: 4, ep: 4, kv-offloading: none, conc-list: [1, 2], spec-decoding: mtp } - { tp: 4, ep: 4, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [4, 8, 10, 12], spec-decoding: mtp } - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 10], spec-decoding: mtp } # GLM-5.2 FP4 agentic-coding benchmark on MI355X via ATOM with MTP speculative diff --git a/perf-changelog.yaml b/perf-changelog.yaml index b970e9099..dfd382ae0 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -7548,6 +7548,6 @@ description: - "Update the SGLang ROCm image from lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260910 to lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260913. Enable the v2 fused top-k path for GLM-5.x on ROCm after sgl-project/sglang#36684 and #36851." - "将 SGLang ROCm 镜像从 lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260910 更新到 lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260913；在 sgl-project/sglang#36684 和 #36851 合入后，为 ROCm 上的 GLM-5.x 启用 v2 fused top-k 路径。" - - "Disable HiCache for the TP4/EP4 arm at concurrency 1 and 2; keep HiCache DRAM offloading for concurrency 4, 8, 10, and 12." - - "在 TP4/EP4 的并发 1 和 2 配置中关闭 HiCache；并发 4、8、10 和 12 继续使用 HiCache DRAM offloading。" + - "Remove concurrency 1 and 2 from the TP4/EP4 HiCache arm so those low-concurrency points run only on TP8/EP1 with KV offloading disabled; keep TP4/EP4 HiCache DRAM offloading for concurrency 4, 8, 10, and 12." + - "从 TP4/EP4 HiCache arm 中移除并发 1 和 2，使这些低并发点仅使用关闭 KV offloading 的 TP8/EP1；TP4/EP4 的并发 4、8、10 和 12 继续使用 HiCache DRAM offloading。" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3081