From 348e594d5ba260930b08fbdfa09b6c7beeb06945 Mon Sep 17 00:00:00 2001 From: thomawan Date: Mon, 6 Jul 2026 18:14:11 +0800 Subject: [PATCH 1/5] Update config and change log --- .../fixed_seq_len/dsv4_fp4_mi355x_sglang.sh | 25 ++++++------------- configs/amd-master.yaml | 2 +- perf-changelog.yaml | 9 +++++++ 3 files changed, 17 insertions(+), 19 deletions(-) diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_sglang.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_sglang.sh index 2ced868365..b5844cee29 100755 --- a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_sglang.sh +++ b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_sglang.sh @@ -26,27 +26,9 @@ if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi export SGLANG_DEFAULT_THINKING=1 export SGLANG_DSV4_REASONING_EFFORT=max -export SGLANG_OPT_DEEPGEMM_HC_PRENORM=false -export SGLANG_USE_AITER=1 export SGLANG_USE_ROCM700A=0 -export SGLANG_DP_USE_GATHERV=1 -export SGLANG_OPT_USE_FUSED_COMPRESS=true export SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton -export SGLANG_OPT_FP8_WO_A_GEMM=false -export SGLANG_OPT_USE_JIT_INDEXER_METADATA=false -export SGLANG_OPT_USE_TOPK_V2=false -export SGLANG_OPT_USE_AITER_INDEXER=true -export SGLANG_OPT_USE_TILELANG_INDEXER=false -export SGLANG_OPT_USE_TILELANG_MHC_PRE=false -export SGLANG_OPT_USE_TILELANG_MHC_POST=false -export SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1 -export SGLANG_OPT_USE_FUSED_COMPRESS_TRITON=true export AITER_BF16_FP8_MOE_BOUND=0 -export SGLANG_EAGER_INPUT_NO_COPY=true - -# multi-stream -export SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=false -export SGLANG_ROCM_USE_MULTI_STREAM=false SERVER_LOG=/workspace/server.log @@ -63,12 +45,19 @@ PARALLEL_ARGS=( ) CHUNKED_PREFILL_SIZE=$ISL if [ "${DP_ATTENTION}" = "true" ]; then + export GPU_MAX_HW_QUEUES=5 + export SGLANG_SHARED_EXPERT_TP1=1 + export SGLANG_DP_SHARED_EXPERT_LOCAL=1 + export SGLANG_DP_USE_GATHERV=1 + export SGLANG_DP_USE_REDUCE_SCATTER=1 + CHUNKED_PREFILL_SIZE=$((ISL * TP)) PARALLEL_ARGS+=( --dp "$TP" --enable-dp-attention --enable-prefill-delayer --prefill-delayer-max-delay-ms 5000 + --enable-two-batch-overlap ) fi if [ "${EP_SIZE:-1}" -gt 1 ]; then diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index a982f6699a..ba10fe5700 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1834,7 +1834,7 @@ dsr1-fp4-mi355x-sglang-disagg-8k1k-mtp: - "DECODE_MTP_SIZE=1" dsv4-fp4-mi355x-sglang: - image: lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260618 + image: lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260706 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: mi355x diff --git a/perf-changelog.yaml b/perf-changelog.yaml index f01199eb1a..e6c7e18f31 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4556,3 +4556,12 @@ description: - "Add high concurrency configs" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1994 + +- config-keys: + - dsv4-fp4-mi355x-sglang + description: + - "Bump image to lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260706" + - "Clean the export envs" + - "Enable two batch overlap" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2093 + From 08713fb7db07ff24489695b0aed05773b6fa4415 Mon Sep 17 00:00:00 2001 From: thomawan Date: Mon, 6 Jul 2026 18:18:31 +0800 Subject: [PATCH 2/5] Fix format --- .../single_node/fixed_seq_len/dsv4_fp4_mi355x_sglang.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_sglang.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_sglang.sh index b5844cee29..8eb793f118 100755 --- a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_sglang.sh +++ b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_sglang.sh @@ -56,8 +56,8 @@ if [ "${DP_ATTENTION}" = "true" ]; then --dp "$TP" --enable-dp-attention --enable-prefill-delayer - --prefill-delayer-max-delay-ms 5000 - --enable-two-batch-overlap + --prefill-delayer-max-delay-ms 5000 + --enable-two-batch-overlap ) fi if [ "${EP_SIZE:-1}" -gt 1 ]; then From 5bdd0f56a61a4d2a5572f5a512d28a76c9ebf66b Mon Sep 17 00:00:00 2001 From: Thomas Wang <1am9trash@gmail.com> Date: Tue, 7 Jul 2026 09:11:24 +0800 Subject: [PATCH 3/5] Update config --- .../single_node/fixed_seq_len/dsv4_fp4_mi355x_sglang.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_sglang.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_sglang.sh index 8eb793f118..ecdac864f2 100755 --- a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_sglang.sh +++ b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_sglang.sh @@ -45,7 +45,6 @@ PARALLEL_ARGS=( ) CHUNKED_PREFILL_SIZE=$ISL if [ "${DP_ATTENTION}" = "true" ]; then - export GPU_MAX_HW_QUEUES=5 export SGLANG_SHARED_EXPERT_TP1=1 export SGLANG_DP_SHARED_EXPERT_LOCAL=1 export SGLANG_DP_USE_GATHERV=1 @@ -56,9 +55,11 @@ if [ "${DP_ATTENTION}" = "true" ]; then --dp "$TP" --enable-dp-attention --enable-prefill-delayer - --prefill-delayer-max-delay-ms 5000 - --enable-two-batch-overlap ) + if [ "$ISL" -gt 1024 ]; then + export GPU_MAX_HW_QUEUES=5 + PARALLEL_ARGS+=(--enable-two-batch-overlap) + fi fi if [ "${EP_SIZE:-1}" -gt 1 ]; then PARALLEL_ARGS+=(--ep-size "$EP_SIZE") @@ -72,10 +73,12 @@ sglang serve \ --trust-remote-code \ --disable-radix-cache \ --attention-backend dsv4 \ + --cuda-graph-max-bs ${CONC} \ --max-running-requests ${CONC} \ --mem-fraction-static 0.90 \ --swa-full-tokens-ratio 0.15 \ --page-size 256 \ + --kv-cache-dtype fp8_e4m3 \ --context-length $MAX_MODEL_LEN \ --chunked-prefill-size $CHUNKED_PREFILL_SIZE \ --disable-shared-experts-fusion \ From 32feb7d82f44d87fddf3fffc4716474b564d6b04 Mon Sep 17 00:00:00 2001 From: Thomas Wang <1am9trash@gmail.com> Date: Tue, 7 Jul 2026 11:07:47 +0800 Subject: [PATCH 4/5] Update dsv4_fp4_mi355x_sglang.sh --- .../single_node/fixed_seq_len/dsv4_fp4_mi355x_sglang.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_sglang.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_sglang.sh index ecdac864f2..79c5832db9 100755 --- a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_sglang.sh +++ b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_sglang.sh @@ -49,17 +49,15 @@ if [ "${DP_ATTENTION}" = "true" ]; then export SGLANG_DP_SHARED_EXPERT_LOCAL=1 export SGLANG_DP_USE_GATHERV=1 export SGLANG_DP_USE_REDUCE_SCATTER=1 + export GPU_MAX_HW_QUEUES=5 CHUNKED_PREFILL_SIZE=$((ISL * TP)) PARALLEL_ARGS+=( --dp "$TP" --enable-dp-attention --enable-prefill-delayer + --enable-two-batch-overlap ) - if [ "$ISL" -gt 1024 ]; then - export GPU_MAX_HW_QUEUES=5 - PARALLEL_ARGS+=(--enable-two-batch-overlap) - fi fi if [ "${EP_SIZE:-1}" -gt 1 ]; then PARALLEL_ARGS+=(--ep-size "$EP_SIZE") From ceecf93738ef8f35f8b3ca69f1b0d69d4b556288 Mon Sep 17 00:00:00 2001 From: Thomas Wang <1am9trash@gmail.com> Date: Wed, 8 Jul 2026 11:36:41 +0800 Subject: [PATCH 5/5] Fix changelog format --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index c7addabbf7..6e3dc244ec 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4596,4 +4596,4 @@ - "Bump image to lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260706" - "Clean the export envs" - "Enable two batch overlap" - pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2093 \ No newline at end of file + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2093