From 3050458b59972ddf27db526d0ed05f844d7b780e Mon Sep 17 00:00:00 2001 From: LI MOU Date: Tue, 8 Sep 2026 09:22:16 +0000 Subject: [PATCH 01/13] update config for glm5.2-mxfp4 on mi355x gpu --- .../agentic/glm5.2_fp4_mi355x_sglang_mtp.sh | 6 +++--- configs/amd-master.yaml | 2 +- perf-changelog.yaml | 10 ++++++++++ 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh index 11e40a70eb..b744c1c48c 100644 --- a/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh @@ -91,9 +91,9 @@ if agentic_kv_offload_enabled; then # env-var override for maximum throughput on nodes with >4 TB DRAM. HICACHE_RATIO="${HICACHE_RATIO:-1.5}" fi - # write_through_selective skips DRAM writes for non-reusable KV blocks, - # reducing host-bus traffic without affecting the cache hit rate. - HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through_selective}" + # Keep write_through as the validated baseline. Override + # HICACHE_WRITE_POLICY explicitly for selective-write experiments. + HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through}" HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first_direct}" case "$KV_OFFLOAD_BACKEND" in diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 2510dc3b8d..588f12e743 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1708,7 +1708,7 @@ minimaxm3-fp4-mi355x-vllm-agentic-mtp: # c6 and c8 removed after sweep validation: dominated by TP4/EP4 arm. # SA selects the Pareto-optimal arm per concurrency point. glm5.2-fp4-mi355x-sglang-agentic-mtp: - image: lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260728 + image: lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260907 model: amd/GLM-5.2-MXFP4 model-prefix: glm5.2 runner: cluster:mi355x-amds diff --git a/perf-changelog.yaml b/perf-changelog.yaml index e0ef448195..fefacd7614 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6947,3 +6947,13 @@ - "Update vLLM ROCm image from vllm/vllm-openai-rocm:v0.27.1 (v0.27.1 release) to vllm/vllm-openai-rocm:nightly-d9105ea8001e0a6d77a96327d17515bb5791fb36 (2026-09-07 upstream ROCm nightly, digest sha256:74d4a95f3ae672ecddf9acb7296917def82d9eca51687fa2862ae72b03ff1907, tag commit vllm-project/vllm@d9105ea8; Docker Hub last pushed 2026-09-07T05:26:48Z). benchmarks/single_node/agentic/minimaxm3_fp8_mi300x_mtp.sh is unchanged: TRITON_ATTN attention, fp8 KV, block-size 128, EAGLE3 speculative decoding with the Inferact MiniMax-M3 EAGLE3-GQA draft and the committed golden synthetic acceptance length, minimax_m3 tool-call and reasoning parsers; the search space is unchanged. The upstream commit-pinned ROCm nightly is the same vllm commit the B200 MiniMax-M3 AgentX recipe moved to in #2860. Note that vllm-openai-rocm commit-nightly tags have expired from Docker Hub within days in the past; node squash caches keep merged configs running, but a re-pin to a durable tag may be needed later." - "Add --compilation-config cudagraph_mode=FULL_DECODE_ONLY to the serve command. The upstream nightly does not torch-compile MiniMaxM3SparseForConditionalGeneration, so with VLLM_USE_BREAKABLE_CUDAGRAPH=0 the default FULL_AND_PIECEWISE mode aborts at engine init with piecewise CUDA graphs unavailable (first sweep, run 34174124043, eval cell); full decode-only graphs are what the MI355X MiniMax-M3 sibling runs on its nightly (#2825)." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2873 + +- config-keys: + - glm5.2-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Update the SGLang ROCm image from lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260728 to lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260907." + - "Pick up our recent SGLang main-branch optimizations for GLM-5.2-MXFP4 serving." + - "Restore HiCache write_through as the default write policy to optimize GLM-5.2-MXFP4 output interactivity and per-GPU throughput in the MI355X AgentX configuration." + pr-link: TBD From 5361b955f8451e45f23b23a9fdc4533c35898628 Mon Sep 17 00:00:00 2001 From: LI MOU Date: Tue, 8 Sep 2026 10:19:28 +0000 Subject: [PATCH 02/13] update pr link --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index fefacd7614..6e46e8f34e 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6956,4 +6956,4 @@ - "Update the SGLang ROCm image from lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260728 to lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260907." - "Pick up our recent SGLang main-branch optimizations for GLM-5.2-MXFP4 serving." - "Restore HiCache write_through as the default write policy to optimize GLM-5.2-MXFP4 output interactivity and per-GPU throughput in the MI355X AgentX configuration." - pr-link: TBD + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2887 From c8c7ca70bd4f5f143936f57615012110f8926278 Mon Sep 17 00:00:00 2001 From: Jiejing Zhang Date: Tue, 8 Sep 2026 16:56:03 +0000 Subject: [PATCH 03/13] Update changelog link for replacement PR Co-authored-by: Cursor --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index ba2c470587..8ea75ce86a 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6956,7 +6956,7 @@ - "Update the SGLang ROCm image from lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260728 to lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260907." - "Pick up our recent SGLang main-branch optimizations for GLM-5.2-MXFP4 serving." - "Restore HiCache write_through as the default write policy to optimize GLM-5.2-MXFP4 output interactivity and per-GPU throughput in the MI355X AgentX configuration." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2887 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2890 - config-keys: - qwen3.5-fp8-b200-sglang From 49371652fc403e8e549dd64399fe533c25e28774 Mon Sep 17 00:00:00 2001 From: Jiejing Zhang Date: Tue, 8 Sep 2026 22:29:04 +0000 Subject: [PATCH 04/13] Update changelog link for upstream PR Co-authored-by: Cursor --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 8ea75ce86a..9325fae024 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6956,7 +6956,7 @@ - "Update the SGLang ROCm image from lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260728 to lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260907." - "Pick up our recent SGLang main-branch optimizations for GLM-5.2-MXFP4 serving." - "Restore HiCache write_through as the default write policy to optimize GLM-5.2-MXFP4 output interactivity and per-GPU throughput in the MI355X AgentX configuration." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2890 + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2900 - config-keys: - qwen3.5-fp8-b200-sglang From fe078eb6ab6ec91e8c625f831a0b6f5e975032aa Mon Sep 17 00:00:00 2001 From: Jiejing Zhang Date: Thu, 10 Sep 2026 06:55:19 +0000 Subject: [PATCH 05/13] Raise SGLang /health timeout so eval-only GSM8K survives AITER JIT. Cold mha_varlen compile after ready is ~40s; the default 20s generate health check marked the server UnHealthy and lm_eval hit a dead port. Co-authored-by: Cursor --- .../single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh index b744c1c48c..c22d933651 100644 --- a/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh @@ -61,6 +61,10 @@ export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 # inter-turn idle gaps can reuse a socket exactly as the server closes it. # Outlast the client pool so the race cannot occur. export SGLANG_TIMEOUT_KEEP_ALIVE=900 +# /health does a 1-token generate. First real-MTP (eval-only) request can +# JIT mha_varlen for ~40s; the default 20s timeout marks the server +# UnHealthy and GSM8K then hits a dead port (mi355x-amds_09 2026-09-09). +export SGLANG_HEALTH_CHECK_TIMEOUT=120 # The DSA indexer's top-k v2 kernel (default since v0.5.14) is JIT-compiled # from CUDA-only source (cooperative_groups.h) and cannot build for gfx950; # v1 dispatches to the precompiled HIP op in sgl-kernel (upstream MI355X CI From 10ffc71e3b4f7502e429ce47c39f1f62d86845eb Mon Sep 17 00:00:00 2001 From: Jiejing Zhang Date: Fri, 11 Sep 2026 03:05:26 +0000 Subject: [PATCH 06/13] Bump the GLM-5.2 MI355X SGLang daily image to 20260910. Co-authored-by: Cursor --- configs/amd-master.yaml | 2 +- perf-changelog.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index ed3b587535..7ce3749b09 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1704,7 +1704,7 @@ minimaxm3-fp4-mi355x-vllm-agentic-mtp: # c6 and c8 removed after sweep validation: dominated by TP4/EP4 arm. # SA selects the Pareto-optimal arm per concurrency point. glm5.2-fp4-mi355x-sglang-agentic-mtp: - image: lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260907 + image: lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260910 model: amd/GLM-5.2-MXFP4 model-prefix: glm5.2 runner: cluster:mi355x-amds diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 3cc20b1132..5ec1d5e3ae 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6972,7 +6972,7 @@ scenario-type: - agentic-coding description: - - "Update the SGLang ROCm image from lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260728 to lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260907." + - "Update the SGLang ROCm image from lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260728 to lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260910." - "Pick up our recent SGLang main-branch optimizations for GLM-5.2-MXFP4 serving." - "Restore HiCache write_through as the default write policy to optimize GLM-5.2-MXFP4 output interactivity and per-GPU throughput in the MI355X AgentX configuration." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2900 From a47d5aeb336374c45b8701375db710740eb778ea Mon Sep 17 00:00:00 2001 From: Jiejing Zhang Date: Fri, 11 Sep 2026 05:39:57 +0000 Subject: [PATCH 07/13] Lower the TP HiCache host-pool ratio from 1.5 to 1.0. Eval and sweep keep the same ratio-based sizing so a green evals-only run at ratio 1.0 remains comparable for this sweep PR. TP4 pinned host DRAM drops from ~679 GB to ~453 GB. Co-authored-by: Cursor --- .../agentic/glm5.2_fp4_mi355x_sglang_mtp.sh | 35 +++++++------------ perf-changelog.yaml | 1 + 2 files changed, 13 insertions(+), 23 deletions(-) diff --git a/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh index 490799950e..70ca62ec47 100644 --- a/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh @@ -76,11 +76,10 @@ export SGLANG_OPT_USE_TOPK_V2=false # # Per-arm L2 ratio (sizing rationale below) applies to both backends unless # overridden via HICACHE_RATIO. TP arm (182.7 GB/rank device pool): the -# agentic-coding corpus saturates any fixed DRAM pool at conc ≥ 10; ratio 1.5 -# (~2.9 TB pinned) is the safe default for cluster:mi355x-amds nodes (~3.0 TB -# available DRAM per runners.yaml). ratio=2.5 (~4.8 TB) yields higher -# throughput at conc 10-12 but exceeds physical DRAM on these nodes and must -# be set via HICACHE_RATIO env-var override on nodes that can accommodate it. +# agentic-coding corpus saturates any fixed DRAM pool at conc ≥ 10; ratio 1.0 +# (~453 GB pinned at TP4) is the default. Larger ratios trade DRAM headroom +# for host-tier capacity and must be set via the HICACHE_RATIO env-var +# override on nodes that can accommodate them. # The DP-attention arm (159.4 GB/rank) only runs at conc >= 32, where the host # tier just absorbs overflow - ratio 0.5 (~1.2 TB pinned, ~1.8 TB of load # headroom) at negligible hit-rate cost (ratio 1.5 OOMs the host mid-storm at @@ -90,20 +89,9 @@ if agentic_kv_offload_enabled; then if [ "$DP_ATTENTION" = "true" ]; then HICACHE_RATIO="${HICACHE_RATIO:-0.5}" else - # ratio=1.5 (~2.9 TB pinned): safe default within the ~3.0 TB DRAM - # available on cluster:mi355x-amds nodes. Set HICACHE_RATIO=2.5 via - # env-var override for maximum throughput on nodes with >4 TB DRAM. - HICACHE_RATIO="${HICACHE_RATIO:-1.5}" - fi - - # GSM8K never fills the agentic host pool; ratio 1.5 OOMs the TP4 DRAM share. - if [ "${EVAL_ONLY:-false}" = "true" ]; then - HICACHE_EVAL_SIZE_GB="${HICACHE_EVAL_SIZE_GB:-16}" - HICACHE_POOL_ARGS=(--hicache-size "$HICACHE_EVAL_SIZE_GB") - HICACHE_POOL_DESC="size=${HICACHE_EVAL_SIZE_GB} GB/rank (eval-only)" - else - HICACHE_POOL_ARGS=(--hicache-ratio "$HICACHE_RATIO" --hicache-size 0) - HICACHE_POOL_DESC="ratio=$HICACHE_RATIO" + # ratio=1.0 (~113 GB/rank, ~453 GB pinned at TP4). Raise via the + # HICACHE_RATIO env-var override on nodes with more DRAM headroom. + HICACHE_RATIO="${HICACHE_RATIO:-1.0}" fi HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through}" @@ -112,10 +100,10 @@ if agentic_kv_offload_enabled; then HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first_direct}" case "$KV_OFFLOAD_BACKEND" in hicache) - echo "HiCache (GPU+host DRAM only): $HICACHE_POOL_DESC, write_policy=$HICACHE_WRITE_POLICY, io_backend=$HICACHE_IO_BACKEND, mem_layout=$HICACHE_MEM_LAYOUT" + echo "HiCache (GPU+host DRAM only): ratio=$HICACHE_RATIO, write_policy=$HICACHE_WRITE_POLICY, io_backend=$HICACHE_IO_BACKEND, mem_layout=$HICACHE_MEM_LAYOUT" CACHE_ARGS=( --enable-hierarchical-cache - "${HICACHE_POOL_ARGS[@]}" + --hicache-ratio "$HICACHE_RATIO" --hicache-write-policy "$HICACHE_WRITE_POLICY" --hicache-io-backend "$HICACHE_IO_BACKEND" --hicache-mem-layout "$HICACHE_MEM_LAYOUT" @@ -146,10 +134,11 @@ EOF MOONCAKE_MASTER_PID=$! sleep 2 kill -0 "$MOONCAKE_MASTER_PID" - echo "HiCache+Mooncake: $HICACHE_POOL_DESC, l3_per_rank=${L3_PER_RANK_GB} GB, dram_budget=${TOTAL_CPU_DRAM_GB} GB" + echo "HiCache+Mooncake: ratio=$HICACHE_RATIO, l3_per_rank=${L3_PER_RANK_GB} GB, dram_budget=${TOTAL_CPU_DRAM_GB} GB" CACHE_ARGS=( --enable-hierarchical-cache - "${HICACHE_POOL_ARGS[@]}" + --hicache-ratio "$HICACHE_RATIO" + --hicache-size 0 --hicache-write-policy "$HICACHE_WRITE_POLICY" --hicache-io-backend "$HICACHE_IO_BACKEND" --hicache-mem-layout "$HICACHE_MEM_LAYOUT" diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 5ec1d5e3ae..3c4d4aaeec 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6975,6 +6975,7 @@ - "Update the SGLang ROCm image from lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260728 to lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260910." - "Pick up our recent SGLang main-branch optimizations for GLM-5.2-MXFP4 serving." - "Restore HiCache write_through as the default write policy to optimize GLM-5.2-MXFP4 output interactivity and per-GPU throughput in the MI355X AgentX configuration." + - "Lower the TP HiCache host-pool ratio from 1.5 to 1.0 (DP-attention arm stays at 0.5). Eval and sweep still share the same ratio-based sizing; the smaller pool cuts pinned host DRAM on the TP4 arm (~453 GB vs ~679 GB)." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2900 - config-keys: From 8e7a1322b617ff6224112aa7d5ff3d672f2ba555 Mon Sep 17 00:00:00 2001 From: Jiejing Zhang Date: Fri, 11 Sep 2026 06:00:52 +0000 Subject: [PATCH 08/13] Keep HiCache write_through_selective instead of restoring write_through. Matches main and the ratio-1.0 eval PR so a green evals-only run is the same host-tier write path as this sweep. Co-authored-by: Cursor --- .../single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh | 4 +++- perf-changelog.yaml | 3 +-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh index 70ca62ec47..7c6bbf249f 100644 --- a/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh @@ -94,7 +94,9 @@ if agentic_kv_offload_enabled; then HICACHE_RATIO="${HICACHE_RATIO:-1.0}" fi - HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through}" + # write_through_selective skips DRAM writes for non-reusable KV blocks, + # reducing host-bus traffic without affecting the cache hit rate. + HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through_selective}" HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first_direct}" diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 3c4d4aaeec..85c620ecfa 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6974,8 +6974,7 @@ description: - "Update the SGLang ROCm image from lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260728 to lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260910." - "Pick up our recent SGLang main-branch optimizations for GLM-5.2-MXFP4 serving." - - "Restore HiCache write_through as the default write policy to optimize GLM-5.2-MXFP4 output interactivity and per-GPU throughput in the MI355X AgentX configuration." - - "Lower the TP HiCache host-pool ratio from 1.5 to 1.0 (DP-attention arm stays at 0.5). Eval and sweep still share the same ratio-based sizing; the smaller pool cuts pinned host DRAM on the TP4 arm (~453 GB vs ~679 GB)." + - "Lower the TP HiCache host-pool ratio from 1.5 to 1.0 (DP-attention arm stays at 0.5). Eval and sweep still share the same ratio-based sizing; the smaller pool cuts pinned host DRAM on the TP4 arm (~453 GB vs ~679 GB). Write policy remains write_through_selective." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2900 - config-keys: From 81d4111ef8885199e3ae713e5c6bb9c3898aa480 Mon Sep 17 00:00:00 2001 From: Jiejing Zhang Date: Fri, 11 Sep 2026 06:52:56 +0000 Subject: [PATCH 09/13] Rename --cuda-graph-max-bs for the v0.5.19 image. The 20260910 tag splits that flag into prefill and decode names, so the unsuffixed form is rejected as ambiguous and the server never starts. Co-authored-by: Cursor --- .../single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh | 5 ++++- perf-changelog.yaml | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh index 7c6bbf249f..6fa7422882 100644 --- a/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh @@ -209,6 +209,8 @@ MAX_RUNNING_REQUESTS=$((2 * CONC)) [ "$MAX_RUNNING_REQUESTS" -gt 256 ] && MAX_RUNNING_REQUESTS=256 # SGLang interpolates a bs list [1..max_bs] automatically; cap at 64 to # keep graph-capture memory bounded without giving up coverage. +# v0.5.19 split --cuda-graph-max-bs into prefill/decode; the un-suffixed +# flag is ambiguous and exits at argparse. CUDA_GRAPH_MAX_BS=$(( MAX_RUNNING_REQUESTS < 64 ? MAX_RUNNING_REQUESTS : 64 )) if [ "${EVAL_ONLY:-false}" != "true" ]; then @@ -236,7 +238,8 @@ SGLANG_CMD=( --chunked-prefill-size "$CHUNKED_PREFILL_SIZE" --mem-fraction-static "$MEM_FRACTION_STATIC" --max-running-requests "$MAX_RUNNING_REQUESTS" - --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS" + --cuda-graph-max-bs-prefill "$CUDA_GRAPH_MAX_BS" + --cuda-graph-max-bs-decode "$CUDA_GRAPH_MAX_BS" --speculative-algorithm EAGLE --speculative-num-steps 5 --speculative-eagle-topk 1 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 85c620ecfa..03a4ffef9c 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6975,6 +6975,7 @@ - "Update the SGLang ROCm image from lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260728 to lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260910." - "Pick up our recent SGLang main-branch optimizations for GLM-5.2-MXFP4 serving." - "Lower the TP HiCache host-pool ratio from 1.5 to 1.0 (DP-attention arm stays at 0.5). Eval and sweep still share the same ratio-based sizing; the smaller pool cuts pinned host DRAM on the TP4 arm (~453 GB vs ~679 GB). Write policy remains write_through_selective." + - "Replace --cuda-graph-max-bs with --cuda-graph-max-bs-prefill and --cuda-graph-max-bs-decode (same cap, min(2*CONC, 64)). v0.5.19 split the unsuffixed flag, so argparse rejects it as ambiguous against the two new names and the 20260910 image exits before serving." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2900 - config-keys: From aed71d7dc40f619a86f9a449b8dd5bed555dde96 Mon Sep 17 00:00:00 2001 From: Jiejing Zhang Date: Fri, 11 Sep 2026 07:07:32 +0000 Subject: [PATCH 10/13] Restore HiCache write-through policy Co-authored-by: Cursor --- .../single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh | 8 +------- perf-changelog.yaml | 3 ++- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh index 6fa7422882..eb48f02963 100644 --- a/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh @@ -61,10 +61,6 @@ export AIPERF_HTTP_TCP_USER_TIMEOUT=900000 # inter-turn idle gaps can reuse a socket exactly as the server closes it. # Outlast the client pool so the race cannot occur. export SGLANG_TIMEOUT_KEEP_ALIVE=900 -# /health does a 1-token generate. First real-MTP (eval-only) request can -# JIT mha_varlen for ~40s; the default 20s timeout marks the server -# UnHealthy and GSM8K then hits a dead port (mi355x-amds_09 2026-09-09). -export SGLANG_HEALTH_CHECK_TIMEOUT=120 # The DSA indexer's top-k v2 kernel (default since v0.5.14) is JIT-compiled # from CUDA-only source (cooperative_groups.h) and cannot build for gfx950; # v1 dispatches to the precompiled HIP op in sgl-kernel (upstream MI355X CI @@ -94,9 +90,7 @@ if agentic_kv_offload_enabled; then HICACHE_RATIO="${HICACHE_RATIO:-1.0}" fi - # write_through_selective skips DRAM writes for non-reusable KV blocks, - # reducing host-bus traffic without affecting the cache hit rate. - HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through_selective}" + HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through}" HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first_direct}" diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 03a4ffef9c..a503ad18a8 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6974,7 +6974,8 @@ description: - "Update the SGLang ROCm image from lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260728 to lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260910." - "Pick up our recent SGLang main-branch optimizations for GLM-5.2-MXFP4 serving." - - "Lower the TP HiCache host-pool ratio from 1.5 to 1.0 (DP-attention arm stays at 0.5). Eval and sweep still share the same ratio-based sizing; the smaller pool cuts pinned host DRAM on the TP4 arm (~453 GB vs ~679 GB). Write policy remains write_through_selective." + - "Lower the TP HiCache host-pool ratio from 1.5 to 1.0 (DP-attention arm stays at 0.5). Eval and sweep still share the same ratio-based sizing; the smaller pool cuts pinned host DRAM on the TP4 arm (~453 GB vs ~679 GB). Keep the write policy at write_through: evals-only run 34569310922 passed with ratio 1.0 and write_through, so write_through_selective was unrelated to the earlier failure." + - "Remove the SGLANG_HEALTH_CHECK_TIMEOUT=120 override and its stale first-request JIT explanation; successful evals-only run 34569310922 used the default health-check timeout." - "Replace --cuda-graph-max-bs with --cuda-graph-max-bs-prefill and --cuda-graph-max-bs-decode (same cap, min(2*CONC, 64)). v0.5.19 split the unsuffixed flag, so argparse rejects it as ambiguous against the two new names and the 20260910 image exits before serving." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2900 From b43627e157cc414e2c095325e779cf44e17b3a76 Mon Sep 17 00:00:00 2001 From: Jiejing Zhang Date: Fri, 11 Sep 2026 07:21:06 +0000 Subject: [PATCH 11/13] Use canonical decode CUDA graph limit Co-authored-by: Cursor --- .../agentic/glm5.2_fp4_mi355x_sglang_mtp.sh | 13 ++++++------- perf-changelog.yaml | 2 +- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh index ff4923bb33..b10646264f 100644 --- a/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh @@ -199,11 +199,11 @@ fi # than CONC sessions; headroom prevents scheduler stalls under burst. MAX_RUNNING_REQUESTS=$((2 * CONC)) [ "$MAX_RUNNING_REQUESTS" -gt 256 ] && MAX_RUNNING_REQUESTS=256 -# SGLang interpolates a bs list [1..max_bs] automatically; cap at 64 to -# keep graph-capture memory bounded without giving up coverage. -# v0.5.19 split --cuda-graph-max-bs into prefill/decode; the un-suffixed -# flag is ambiguous and exits at argparse. -CUDA_GRAPH_MAX_BS=$(( MAX_RUNNING_REQUESTS < 64 ? MAX_RUNNING_REQUESTS : 64 )) +# SGLang interpolates the decode bs list automatically; cap at 64 to keep +# graph-capture memory bounded without giving up coverage. +# --cuda-graph-max-bs was a deprecated alias for the decode setting only. +# The 20260910 image removed that alias, making its old spelling ambiguous. +CUDA_GRAPH_MAX_BS_DECODE=$(( MAX_RUNNING_REQUESTS < 64 ? MAX_RUNNING_REQUESTS : 64 )) if [ "${EVAL_ONLY:-false}" != "true" ]; then export SGLANG_SIMULATE_ACC_LEN=3.61 @@ -230,8 +230,7 @@ SGLANG_CMD=( --chunked-prefill-size "$CHUNKED_PREFILL_SIZE" --mem-fraction-static "$MEM_FRACTION_STATIC" --max-running-requests "$MAX_RUNNING_REQUESTS" - --cuda-graph-max-bs-prefill "$CUDA_GRAPH_MAX_BS" - --cuda-graph-max-bs-decode "$CUDA_GRAPH_MAX_BS" + --cuda-graph-max-bs-decode "$CUDA_GRAPH_MAX_BS_DECODE" --speculative-algorithm EAGLE --speculative-num-steps 5 --speculative-eagle-topk 1 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index cca8342baa..ec06d9561a 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6976,7 +6976,7 @@ - "Pick up our recent SGLang main-branch optimizations for GLM-5.2-MXFP4 serving." - "Lower the TP HiCache host-pool ratio from 1.5 to 1.0 (DP-attention arm stays at 0.5). Eval and sweep still share the same ratio-based sizing; the smaller pool cuts pinned host DRAM on the TP4 arm (~453 GB vs ~679 GB). Keep the write policy at write_through: evals-only run 34569310922 passed with ratio 1.0 and write_through, so write_through_selective was unrelated to the earlier failure." - "Remove the SGLANG_HEALTH_CHECK_TIMEOUT=120 override and its stale first-request JIT explanation; successful evals-only run 34569310922 used the default health-check timeout." - - "Replace --cuda-graph-max-bs with --cuda-graph-max-bs-prefill and --cuda-graph-max-bs-decode (same cap, min(2*CONC, 64)). v0.5.19 split the unsuffixed flag, so argparse rejects it as ambiguous against the two new names and the 20260910 image exits before serving." + - "Replace the removed decode-only alias --cuda-graph-max-bs with its canonical name --cuda-graph-max-bs-decode, retaining the same min(2*CONC, 64) cap. The 20260910 image includes sgl-project/sglang#38375, so argparse interprets the removed spelling as an ambiguous prefix of the decode and prefill options and exits before serving. Leave --cuda-graph-max-bs-prefill unset: it controls a separate prefill token-capture ceiling, not request concurrency." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2900 - config-keys: From 45c7ebf79e65f0ee0e5954b164a246de884b7b59 Mon Sep 17 00:00:00 2001 From: Jiejing Zhang Date: Fri, 11 Sep 2026 09:08:04 +0000 Subject: [PATCH 12/13] Switch GLM-5.2 DSA prefill/decode to Triton. sgl-project/sglang#30575 exposes the Fast Triton Sparse MLA path as an explicit backend; the 20260910 image already contains it. Co-authored-by: Cursor --- .../single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh | 4 ++-- perf-changelog.yaml | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh index b10646264f..f4d318d2ee 100644 --- a/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh @@ -220,8 +220,8 @@ SGLANG_CMD=( --trust-remote-code "${PARALLEL_ARGS[@]}" --kv-cache-dtype fp8_e4m3 - --dsa-prefill-backend tilelang - --dsa-decode-backend tilelang + --dsa-prefill-backend triton + --dsa-decode-backend triton # GLM-5.2 emits the GLM-4.7-style tool-call format; glm47 is required for # structured message.tool_calls (SWE-bench agentic evals die without it). # The glm45 reasoning parser keeps hybrid thinking in reasoning_content. diff --git a/perf-changelog.yaml b/perf-changelog.yaml index ec06d9561a..a8ede0cfd9 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6977,6 +6977,7 @@ - "Lower the TP HiCache host-pool ratio from 1.5 to 1.0 (DP-attention arm stays at 0.5). Eval and sweep still share the same ratio-based sizing; the smaller pool cuts pinned host DRAM on the TP4 arm (~453 GB vs ~679 GB). Keep the write policy at write_through: evals-only run 34569310922 passed with ratio 1.0 and write_through, so write_through_selective was unrelated to the earlier failure." - "Remove the SGLANG_HEALTH_CHECK_TIMEOUT=120 override and its stale first-request JIT explanation; successful evals-only run 34569310922 used the default health-check timeout." - "Replace the removed decode-only alias --cuda-graph-max-bs with its canonical name --cuda-graph-max-bs-decode, retaining the same min(2*CONC, 64) cap. The 20260910 image includes sgl-project/sglang#38375, so argparse interprets the removed spelling as an ambiguous prefix of the decode and prefill options and exits before serving. Leave --cuda-graph-max-bs-prefill unset: it controls a separate prefill token-capture ceiling, not request concurrency." + - "Switch DSA prefill and decode from tilelang to triton (sgl-project/sglang#30575 Fast Triton Sparse MLA). The 20260910 image already contains that backend; kv-cache-dtype stays fp8_e4m3, which the Triton DSA path requires." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2900 - config-keys: From f8e4fa65b9667a8ac9757bad8cf006d5df1b72bb Mon Sep 17 00:00:00 2001 From: Chun Fang Date: Sun, 13 Sep 2026 01:54:52 +0000 Subject: [PATCH 13/13] Fix changelog --- perf-changelog.yaml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index ce180524a6..1cc4df6fc7 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6967,19 +6967,6 @@ - "Add --compilation-config cudagraph_mode=FULL_DECODE_ONLY to the serve command. The upstream nightly does not torch-compile MiniMaxM3SparseForConditionalGeneration, so with VLLM_USE_BREAKABLE_CUDAGRAPH=0 the default FULL_AND_PIECEWISE mode aborts at engine init with piecewise CUDA graphs unavailable (first sweep, run 34174124043, eval cell); full decode-only graphs are what the MI355X MiniMax-M3 sibling runs on its nightly (#2825)." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2873 -- config-keys: - - glm5.2-fp4-mi355x-sglang-agentic-mtp - scenario-type: - - agentic-coding - description: - - "Update the SGLang ROCm image from lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260728 to lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260910." - - "Pick up our recent SGLang main-branch optimizations for GLM-5.2-MXFP4 serving." - - "Lower the TP HiCache host-pool ratio from 1.5 to 1.0 (DP-attention arm stays at 0.5). Eval and sweep still share the same ratio-based sizing; the smaller pool cuts pinned host DRAM on the TP4 arm (~453 GB vs ~679 GB). Keep the write policy at write_through: evals-only run 34569310922 passed with ratio 1.0 and write_through, so write_through_selective was unrelated to the earlier failure." - - "Remove the SGLANG_HEALTH_CHECK_TIMEOUT=120 override and its stale first-request JIT explanation; successful evals-only run 34569310922 used the default health-check timeout." - - "Replace the removed decode-only alias --cuda-graph-max-bs with its canonical name --cuda-graph-max-bs-decode, retaining the same min(2*CONC, 64) cap. The 20260910 image includes sgl-project/sglang#38375, so argparse interprets the removed spelling as an ambiguous prefix of the decode and prefill options and exits before serving. Leave --cuda-graph-max-bs-prefill unset: it controls a separate prefill token-capture ceiling, not request concurrency." - - "Switch DSA prefill and decode from tilelang to triton (sgl-project/sglang#30575 Fast Triton Sparse MLA). The 20260910 image already contains that backend; kv-cache-dtype stays fp8_e4m3, which the Triton DSA path requires." - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2900 - - config-keys: - qwen3.5-fp8-b200-sglang scenario-type: @@ -7441,3 +7428,16 @@ - "Add representative AMD coverage for fixed-sequence request diagnostics and multinode concurrency filename processing; shared processor changes also affect unlisted fixed-sequence recipes." - "增加固定序列请求诊断与多节点并发文件名处理的代表性 AMD 覆盖；共享处理器变更也影响未列出的固定序列配置。" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3026 + +- config-keys: + - glm5.2-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Update the SGLang ROCm image from lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260728 to lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260910." + - "Pick up our recent SGLang main-branch optimizations for GLM-5.2-MXFP4 serving." + - "Lower the TP HiCache host-pool ratio from 1.5 to 1.0 (DP-attention arm stays at 0.5). Eval and sweep still share the same ratio-based sizing; the smaller pool cuts pinned host DRAM on the TP4 arm (~453 GB vs ~679 GB). Keep the write policy at write_through: evals-only run 34569310922 passed with ratio 1.0 and write_through, so write_through_selective was unrelated to the earlier failure." + - "Remove the SGLANG_HEALTH_CHECK_TIMEOUT=120 override and its stale first-request JIT explanation; successful evals-only run 34569310922 used the default health-check timeout." + - "Replace the removed decode-only alias --cuda-graph-max-bs with its canonical name --cuda-graph-max-bs-decode, retaining the same min(2*CONC, 64) cap. The 20260910 image includes sgl-project/sglang#38375, so argparse interprets the removed spelling as an ambiguous prefix of the decode and prefill options and exits before serving. Leave --cuda-graph-max-bs-prefill unset: it controls a separate prefill token-capture ceiling, not request concurrency." + - "Switch DSA prefill and decode from tilelang to triton (sgl-project/sglang#30575 Fast Triton Sparse MLA). The 20260910 image already contains that backend; kv-cache-dtype stays fp8_e4m3, which the Triton DSA path requires." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2900