diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh index 61add3eafa..c0f957a7c1 100644 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_sglang_mtp.sh @@ -73,6 +73,12 @@ export SGLANG_DSV4_REASONING_EFFORT=high export SGLANG_USE_ROCM700A=0 export SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton export AITER_BF16_FP8_MOE_BOUND=0 +# DSv4 kernel fusions carried by the sgl-dev image: aiter batched GEMM for the +# absorbed MLA projections, native b-preshuffle scale handling, and a fused +# compress+norm+rope epilogue. All three default off in environ.py. +export SGLANG_OPT_USE_AITER_BATCHED_GEMM=1 +export SGLANG_OPT_NATIVE_BPRESHUFFLE_SCALE=1 +export SGLANG_OPT_FUSE_COMPRESS_NORM_ROPE=1 # Unified radix tree: per-component (full-attn / SWA) cache management for # hybrid-attention models, plus proactive release of out-of-window SWA KV @@ -130,8 +136,10 @@ SGLANG_BACKEND_PORT="$PORT" # cap; same value the multi-node DeepSeek-V4-Pro-AgentX no_dp profile uses. CHUNKED_PREFILL_SIZE=8192 # MTP adds a draft KV pool and extra graph captures on top of the spec-none -# footprint, which ran at 0.90. -MEM_FRACTION_STATIC=0.85 +# footprint, which ran at 0.90. 0.89 recovers most of that: the DSv4 compressor +# state pools are sized from the full-attention pool and allocated after it, +# outside this budget, so the remainder has to stay large enough to cover them. +MEM_FRACTION_STATIC=0.89 PARALLEL_ARGS=(--tensor-parallel-size "$TP") if [ "$DP_ATTENTION" = "true" ]; then USE_SGLANG_ROUTER=true @@ -213,7 +221,7 @@ SGLANG_CMD=( --page-size 256 --swa-full-tokens-ratio 0.10 --kv-cache-dtype fp8_e4m3 - --disable-shared-experts-fusion + --enforce-shared-experts-fusion --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --chunked-prefill-size "$CHUNKED_PREFILL_SIZE" diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 67a0605398..563d3ffa1c 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1722,12 +1722,17 @@ glm5.2-fp4-mi355x-atom-agentic-mtp: # SGLang AgentX coverage removed in de493d859 (PR #2531, which deleted the # spec-none dsv4-fp4-mi355x-sglang-agentic-hicache key and its script) so the # aggregated SGLang path is comparable with dsv4-fp4-mi355x-vllm-agentic-mtp -# and dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp. The image matches the -# already-green disagg AgentX entry. Pure TP8 only: DSA + dp-attention hangs a -# collective under long-context prefill, so no DEP arm ships until that path is -# validated. conc 16 appears on both arms to isolate the host KV tier's gain. +# and dsv4-fp4-mi355x-sglang-disagg-agentic-hicache-mtp. The image is the +# sgl-dev v0.5.18 build, which carries the DSv4 kernel fusions this key enables +# via SGLANG_OPT_*. Tensor-parallel only: DSA + dp-attention hangs a collective +# under long-context prefill, so no DEP arm ships until that path is validated. +# TP4 owns the low-concurrency end, where halving the GPUs per replica raises +# throughput per GPU, and TP8 picks up from conc 16; conc 16 appears on both TP8 +# arms to isolate the host KV tier's gain. TP4 doubles per-rank weights, leaving +# far less room for KV, which is why it stops at conc 10 and carries no host +# tier. dsv4-fp4-mi355x-sglang-agentic-mtp: - image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260813 + image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260822 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:mi355x-amds @@ -1738,5 +1743,6 @@ dsv4-fp4-mi355x-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, conc-list: [1, 2, 4, 8, 16], spec-decoding: mtp } + - { tp: 4, ep: 1, dp-attn: false, kv-offloading: none, conc-list: [1, 2, 4, 8, 10], spec-decoding: mtp } + - { tp: 8, ep: 1, dp-attn: false, kv-offloading: none, conc-list: [16], spec-decoding: mtp } - { tp: 8, ep: 1, dp-attn: false, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [16, 32, 48], spec-decoding: mtp } diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 98eef2c543..a226799e1c 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6378,3 +6378,15 @@ - "Serve the TP8 arm with FULL_AND_PIECEWISE cudagraphs, capturing the mixed prefill/decode batches as well as the uniform decode batches. The DEP8 arm is unchanged." - "Replace the TP8 concurrency list [1, 4, 8, 16, 24] with [1, 4, 8, 14], dropping the published concurrency 16 and 24 points and adding concurrency 14." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2707 + +- config-keys: + - dsv4-fp4-mi355x-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Update image from lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260813 to rocm/sgl-dev:v0.5.18-rocm720-mi35x-20260822." + - "Enable the DSv4 kernel fusions the new image carries, all of which default off: SGLANG_OPT_USE_AITER_BATCHED_GEMM=1 (aiter batched GEMM for the absorbed MLA projections), SGLANG_OPT_NATIVE_BPRESHUFFLE_SCALE=1, SGLANG_OPT_FUSE_COMPRESS_NORM_ROPE=1." + - "Raise mem-fraction-static from 0.85 to 0.89, enlarging the full-attention KV pool. The DSv4 compressor state pools are sized from that pool and allocated after it, outside this budget, so the remainder still has to cover them." + - "Switch --disable-shared-experts-fusion to --enforce-shared-experts-fusion, which fuses the single shared expert into the routed list. The tuned MoE tables in this image cover the resulting (inter_dim 384, expert 385, topk 7) shape at the token sizes this workload hits." + - "Split the parallelism by concurrency: add a TP4 arm at [1, 2, 4, 8, 10] with no host KV tier, where halving the GPUs per replica raises throughput per GPU, and reduce the TP8 no-offload arm from [1, 2, 4, 8, 16] to [16]. This drops the published TP8 concurrency 1, 2, 4 and 8 points and adds TP4 concurrency 1, 2, 4, 8 and 10." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2710