diff --git a/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh index 11e40a70e..f4d318d2e 100644 --- a/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/glm5.2_fp4_mi355x_sglang_mtp.sh @@ -72,11 +72,10 @@ export SGLANG_OPT_USE_TOPK_V2=false # # Per-arm L2 ratio (sizing rationale below) applies to both backends unless # overridden via HICACHE_RATIO. TP arm (182.7 GB/rank device pool): the -# agentic-coding corpus saturates any fixed DRAM pool at conc ≥ 10; ratio 1.5 -# (~2.9 TB pinned) is the safe default for cluster:mi355x-amds nodes (~3.0 TB -# available DRAM per runners.yaml). ratio=2.5 (~4.8 TB) yields higher -# throughput at conc 10-12 but exceeds physical DRAM on these nodes and must -# be set via HICACHE_RATIO env-var override on nodes that can accommodate it. +# agentic-coding corpus saturates any fixed DRAM pool at conc ≥ 10; ratio 1.0 +# (~453 GB pinned at TP4) is the default. Larger ratios trade DRAM headroom +# for host-tier capacity and must be set via the HICACHE_RATIO env-var +# override on nodes that can accommodate them. # The DP-attention arm (159.4 GB/rank) only runs at conc >= 32, where the host # tier just absorbs overflow - ratio 0.5 (~1.2 TB pinned, ~1.8 TB of load # headroom) at negligible hit-rate cost (ratio 1.5 OOMs the host mid-storm at @@ -86,14 +85,11 @@ if agentic_kv_offload_enabled; then if [ "$DP_ATTENTION" = "true" ]; then HICACHE_RATIO="${HICACHE_RATIO:-0.5}" else - # ratio=1.5 (~2.9 TB pinned): safe default within the ~3.0 TB DRAM - # available on cluster:mi355x-amds nodes. Set HICACHE_RATIO=2.5 via - # env-var override for maximum throughput on nodes with >4 TB DRAM. - HICACHE_RATIO="${HICACHE_RATIO:-1.5}" + # ratio=1.0 (~113 GB/rank, ~453 GB pinned at TP4). Raise via the + # HICACHE_RATIO env-var override on nodes with more DRAM headroom. + HICACHE_RATIO="${HICACHE_RATIO:-1.0}" fi - # write_through_selective skips DRAM writes for non-reusable KV blocks, - # reducing host-bus traffic without affecting the cache hit rate. - HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through_selective}" + HICACHE_WRITE_POLICY="${HICACHE_WRITE_POLICY:-write_through}" HICACHE_IO_BACKEND="${HICACHE_IO_BACKEND:-direct}" HICACHE_MEM_LAYOUT="${HICACHE_MEM_LAYOUT:-page_first_direct}" case "$KV_OFFLOAD_BACKEND" in @@ -203,9 +199,11 @@ fi # than CONC sessions; headroom prevents scheduler stalls under burst. MAX_RUNNING_REQUESTS=$((2 * CONC)) [ "$MAX_RUNNING_REQUESTS" -gt 256 ] && MAX_RUNNING_REQUESTS=256 -# SGLang interpolates a bs list [1..max_bs] automatically; cap at 64 to -# keep graph-capture memory bounded without giving up coverage. -CUDA_GRAPH_MAX_BS=$(( MAX_RUNNING_REQUESTS < 64 ? MAX_RUNNING_REQUESTS : 64 )) +# SGLang interpolates the decode bs list automatically; cap at 64 to keep +# graph-capture memory bounded without giving up coverage. +# --cuda-graph-max-bs was a deprecated alias for the decode setting only. +# The 20260910 image removed that alias, making its old spelling ambiguous. +CUDA_GRAPH_MAX_BS_DECODE=$(( MAX_RUNNING_REQUESTS < 64 ? MAX_RUNNING_REQUESTS : 64 )) if [ "${EVAL_ONLY:-false}" != "true" ]; then export SGLANG_SIMULATE_ACC_LEN=3.61 @@ -222,8 +220,8 @@ SGLANG_CMD=( --trust-remote-code "${PARALLEL_ARGS[@]}" --kv-cache-dtype fp8_e4m3 - --dsa-prefill-backend tilelang - --dsa-decode-backend tilelang + --dsa-prefill-backend triton + --dsa-decode-backend triton # GLM-5.2 emits the GLM-4.7-style tool-call format; glm47 is required for # structured message.tool_calls (SWE-bench agentic evals die without it). # The glm45 reasoning parser keeps hybrid thinking in reasoning_content. @@ -232,7 +230,7 @@ SGLANG_CMD=( --chunked-prefill-size "$CHUNKED_PREFILL_SIZE" --mem-fraction-static "$MEM_FRACTION_STATIC" --max-running-requests "$MAX_RUNNING_REQUESTS" - --cuda-graph-max-bs "$CUDA_GRAPH_MAX_BS" + --cuda-graph-max-bs-decode "$CUDA_GRAPH_MAX_BS_DECODE" --speculative-algorithm EAGLE --speculative-num-steps 5 --speculative-eagle-topk 1 diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 3d9a3dcca..54c789ba5 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1704,7 +1704,7 @@ minimaxm3-fp4-mi355x-vllm-agentic-mtp: # c6 and c8 removed after sweep validation: dominated by TP4/EP4 arm. # SA selects the Pareto-optimal arm per concurrency point. glm5.2-fp4-mi355x-sglang-agentic-mtp: - image: lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260728 + image: lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260910 model: amd/GLM-5.2-MXFP4 model-prefix: glm5.2 runner: cluster:mi355x-amds diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 75b77afe1..1cc4df6fc 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -7428,3 +7428,16 @@ - "Add representative AMD coverage for fixed-sequence request diagnostics and multinode concurrency filename processing; shared processor changes also affect unlisted fixed-sequence recipes." - "增加固定序列请求诊断与多节点并发文件名处理的代表性 AMD 覆盖;共享处理器变更也影响未列出的固定序列配置。" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3026 + +- config-keys: + - glm5.2-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Update the SGLang ROCm image from lmsysorg/sglang-rocm:v0.5.16-rocm720-mi35x-20260728 to lmsysorg/sglang-rocm:v0.5.19-rocm720-mi35x-20260910." + - "Pick up our recent SGLang main-branch optimizations for GLM-5.2-MXFP4 serving." + - "Lower the TP HiCache host-pool ratio from 1.5 to 1.0 (DP-attention arm stays at 0.5). Eval and sweep still share the same ratio-based sizing; the smaller pool cuts pinned host DRAM on the TP4 arm (~453 GB vs ~679 GB). Keep the write policy at write_through: evals-only run 34569310922 passed with ratio 1.0 and write_through, so write_through_selective was unrelated to the earlier failure." + - "Remove the SGLANG_HEALTH_CHECK_TIMEOUT=120 override and its stale first-request JIT explanation; successful evals-only run 34569310922 used the default health-check timeout." + - "Replace the removed decode-only alias --cuda-graph-max-bs with its canonical name --cuda-graph-max-bs-decode, retaining the same min(2*CONC, 64) cap. The 20260910 image includes sgl-project/sglang#38375, so argparse interprets the removed spelling as an ambiguous prefix of the decode and prefill options and exits before serving. Leave --cuda-graph-max-bs-prefill unset: it controls a separate prefill token-capture ceiling, not request concurrency." + - "Switch DSA prefill and decode from tilelang to triton (sgl-project/sglang#30575 Fast Triton Sparse MLA). The 20260910 image already contains that backend; kv-cache-dtype stays fp8_e4m3, which the Triton DSA path requires." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2900