diff --git a/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh index f4d318d2e..f46ad956e 100644 --- a/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh @@ -61,11 +61,8 @@ export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 # inter-turn idle gaps can reuse a socket exactly as the server closes it. # Outlast the client pool so the race cannot occur. export SGLANG_TIMEOUT_KEEP_ALIVE=900 -# The DSA indexer's top-k v2 kernel (default since v0.5.14) is JIT-compiled -# from CUDA-only source (cooperative_groups.h) and cannot build for gfx950; -# v1 dispatches to the precompiled HIP op in sgl-kernel (upstream MI355X CI -# runs DSA models the same way). -export SGLANG_OPT_USE_TOPK_V2=false +# PR #36684 & PR #36851 turned the v2 fused top-k on for GLM-5.x on ROCm. +export SGLANG_OPT_USE_TOPK_V2=true # HiCache L2 (host DRAM), optionally extended with Mooncake L3. # KV_OFFLOADING=dram requires KV_OFFLOAD_BACKEND=hicache or mooncake. diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index e9a6dbe57..925723bf1 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1697,15 +1697,15 @@ minimaxm3-fp4-mi355x-vllm-agentic-mtp: # GLM-5.2 FP4 agentic-coding benchmark on MI355X via SGLang with MTP speculative # decoding. Two arms: -# (1) TP4/EP4 with HiCache KV offloading to DRAM, concurrency sweep [1, 2, 4, 8, 10, 12]; +# (1) TP4/EP4 with HiCache KV offloading to DRAM at concurrency [4, 8, 10, 12]; # serves the throughput-dominant regime (conc ≥ 8). -# (2) TP8/EP1 without KV offloading, concurrency sweep [4, 10]; +# (2) TP8/EP1 without KV offloading, concurrency sweep [1, 2, 4, 10]; # EP=1 eliminates the MoE all-to-all collective, reducing ITL at low-to-mid # concurrency; GPU-resident KV keeps latency independent of DRAM bandwidth. # c6 and c8 removed after sweep validation: dominated by TP4/EP4 arm. # SA selects the Pareto-optimal arm per concurrency point. glm5.2-fp4-mi355x-sglang-agentic-mtp: - image: lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260910 + image: lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260913 model: amd/GLM-5.2-MXFP4 model-prefix: glm5.2 runner: cluster:mi355x-amds @@ -1716,7 +1716,7 @@ glm5.2-fp4-mi355x-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.8 search-space: - - { tp: 4, ep: 4, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 2, 4, 8, 10, 12], spec-decoding: mtp } + - { tp: 4, ep: 4, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [4, 8, 10, 12], spec-decoding: mtp } - { tp: 8, ep: 1, kv-offloading: none, conc-list: [1, 2, 4, 10], spec-decoding: mtp } # GLM-5.2 FP4 agentic-coding benchmark on MI355X via ATOM with MTP speculative # decoding. TP4 uses LMCache DRAM offload; TP8 is GPU-resident with no KV offload. diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 8f75cf851..c154e3880 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -7680,3 +7680,14 @@ - "Use lmsysorg/sglang:nightly-dev-20260910-00840301 with its bundled Dynamo 1.5.0.dev20260909, EAGLE speculative decoding, and 110 GB HiCache write-back DRAM offload per prefill rank." - "Allow up to 3600 seconds for NIXL UCX engine initialization in the 1P(DEP8)4D(TP4) configuration while registering its larger set of HiCache pools." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2995 + +- config-keys: + - glm5.2-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Update the SGLang ROCm image from lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260910 to lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260913. Enable the v2 fused top-k path for GLM-5.x on ROCm after sgl-project/sglang#36684 and #36851." + - "将 SGLang ROCm 镜像从 lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260910 更新到 lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260913;在 sgl-project/sglang#36684 和 #36851 合入后,为 ROCm 上的 GLM-5.x 启用 v2 fused top-k 路径。" + - "Remove concurrency 1 and 2 from the TP4/EP4 HiCache arm so those low-concurrency points run only on TP8/EP1 with KV offloading disabled; keep TP4/EP4 HiCache DRAM offloading for concurrency 4, 8, 10, and 12." + - "从 TP4/EP4 HiCache arm 中移除并发 1 和 2,使这些低并发点仅使用关闭 KV offloading 的 TP8/EP1;TP4/EP4 的并发 4、8、10 和 12 继续使用 HiCache DRAM offloading。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3081