From 348e594d5ba260930b08fbdfa09b6c7beeb06945 Mon Sep 17 00:00:00 2001 From: thomawan Date: Mon, 6 Jul 2026 18:14:11 +0800 Subject: [PATCH 1/6] Update config and change log --- .../fixed_seq_len/dsv4_fp4_mi355x_sglang.sh | 25 ++++++------------- configs/amd-master.yaml | 2 +- perf-changelog.yaml | 9 +++++++ 3 files changed, 17 insertions(+), 19 deletions(-) diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_sglang.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_sglang.sh index 2ced868365..b5844cee29 100755 --- a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_sglang.sh +++ b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_sglang.sh @@ -26,27 +26,9 @@ if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi export SGLANG_DEFAULT_THINKING=1 export SGLANG_DSV4_REASONING_EFFORT=max -export SGLANG_OPT_DEEPGEMM_HC_PRENORM=false -export SGLANG_USE_AITER=1 export SGLANG_USE_ROCM700A=0 -export SGLANG_DP_USE_GATHERV=1 -export SGLANG_OPT_USE_FUSED_COMPRESS=true export SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton -export SGLANG_OPT_FP8_WO_A_GEMM=false -export SGLANG_OPT_USE_JIT_INDEXER_METADATA=false -export SGLANG_OPT_USE_TOPK_V2=false -export SGLANG_OPT_USE_AITER_INDEXER=true -export SGLANG_OPT_USE_TILELANG_INDEXER=false -export SGLANG_OPT_USE_TILELANG_MHC_PRE=false -export SGLANG_OPT_USE_TILELANG_MHC_POST=false -export SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1 -export SGLANG_OPT_USE_FUSED_COMPRESS_TRITON=true export AITER_BF16_FP8_MOE_BOUND=0 -export SGLANG_EAGER_INPUT_NO_COPY=true - -# multi-stream -export SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=false -export SGLANG_ROCM_USE_MULTI_STREAM=false SERVER_LOG=/workspace/server.log @@ -63,12 +45,19 @@ PARALLEL_ARGS=( ) CHUNKED_PREFILL_SIZE=$ISL if [ "${DP_ATTENTION}" = "true" ]; then + export GPU_MAX_HW_QUEUES=5 + export SGLANG_SHARED_EXPERT_TP1=1 + export SGLANG_DP_SHARED_EXPERT_LOCAL=1 + export SGLANG_DP_USE_GATHERV=1 + export SGLANG_DP_USE_REDUCE_SCATTER=1 + CHUNKED_PREFILL_SIZE=$((ISL * TP)) PARALLEL_ARGS+=( --dp "$TP" --enable-dp-attention --enable-prefill-delayer --prefill-delayer-max-delay-ms 5000 + --enable-two-batch-overlap ) fi if [ "${EP_SIZE:-1}" -gt 1 ]; then diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index a982f6699a..ba10fe5700 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1834,7 +1834,7 @@ dsr1-fp4-mi355x-sglang-disagg-8k1k-mtp: - "DECODE_MTP_SIZE=1" dsv4-fp4-mi355x-sglang: - image: lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260618 + image: lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260706 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: mi355x diff --git a/perf-changelog.yaml b/perf-changelog.yaml index f01199eb1a..e6c7e18f31 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4556,3 +4556,12 @@ description: - "Add high concurrency configs" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1994 + +- config-keys: + - dsv4-fp4-mi355x-sglang + description: + - "Bump image to lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260706" + - "Clean the export envs" + - "Enable two batch overlap" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2093 + From 08713fb7db07ff24489695b0aed05773b6fa4415 Mon Sep 17 00:00:00 2001 From: thomawan Date: Mon, 6 Jul 2026 18:18:31 +0800 Subject: [PATCH 2/6] Fix format --- .../single_node/fixed_seq_len/dsv4_fp4_mi355x_sglang.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_sglang.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_sglang.sh index b5844cee29..8eb793f118 100755 --- a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_sglang.sh +++ b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_sglang.sh @@ -56,8 +56,8 @@ if [ "${DP_ATTENTION}" = "true" ]; then --dp "$TP" --enable-dp-attention --enable-prefill-delayer - --prefill-delayer-max-delay-ms 5000 - --enable-two-batch-overlap + --prefill-delayer-max-delay-ms 5000 + --enable-two-batch-overlap ) fi if [ "${EP_SIZE:-1}" -gt 1 ]; then From 5bdd0f56a61a4d2a5572f5a512d28a76c9ebf66b Mon Sep 17 00:00:00 2001 From: Thomas Wang <1am9trash@gmail.com> Date: Tue, 7 Jul 2026 09:11:24 +0800 Subject: [PATCH 3/6] Update config --- .../single_node/fixed_seq_len/dsv4_fp4_mi355x_sglang.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_sglang.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_sglang.sh index 8eb793f118..ecdac864f2 100755 --- a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_sglang.sh +++ b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_sglang.sh @@ -45,7 +45,6 @@ PARALLEL_ARGS=( ) CHUNKED_PREFILL_SIZE=$ISL if [ "${DP_ATTENTION}" = "true" ]; then - export GPU_MAX_HW_QUEUES=5 export SGLANG_SHARED_EXPERT_TP1=1 export SGLANG_DP_SHARED_EXPERT_LOCAL=1 export SGLANG_DP_USE_GATHERV=1 @@ -56,9 +55,11 @@ if [ "${DP_ATTENTION}" = "true" ]; then --dp "$TP" --enable-dp-attention --enable-prefill-delayer - --prefill-delayer-max-delay-ms 5000 - --enable-two-batch-overlap ) + if [ "$ISL" -gt 1024 ]; then + export GPU_MAX_HW_QUEUES=5 + PARALLEL_ARGS+=(--enable-two-batch-overlap) + fi fi if [ "${EP_SIZE:-1}" -gt 1 ]; then PARALLEL_ARGS+=(--ep-size "$EP_SIZE") @@ -72,10 +73,12 @@ sglang serve \ --trust-remote-code \ --disable-radix-cache \ --attention-backend dsv4 \ + --cuda-graph-max-bs ${CONC} \ --max-running-requests ${CONC} \ --mem-fraction-static 0.90 \ --swa-full-tokens-ratio 0.15 \ --page-size 256 \ + --kv-cache-dtype fp8_e4m3 \ --context-length $MAX_MODEL_LEN \ --chunked-prefill-size $CHUNKED_PREFILL_SIZE \ --disable-shared-experts-fusion \ From 32feb7d82f44d87fddf3fffc4716474b564d6b04 Mon Sep 17 00:00:00 2001 From: Thomas Wang <1am9trash@gmail.com> Date: Tue, 7 Jul 2026 11:07:47 +0800 Subject: [PATCH 4/6] Update dsv4_fp4_mi355x_sglang.sh --- .../single_node/fixed_seq_len/dsv4_fp4_mi355x_sglang.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_sglang.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_sglang.sh index ecdac864f2..79c5832db9 100755 --- a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_sglang.sh +++ b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_sglang.sh @@ -49,17 +49,15 @@ if [ "${DP_ATTENTION}" = "true" ]; then export SGLANG_DP_SHARED_EXPERT_LOCAL=1 export SGLANG_DP_USE_GATHERV=1 export SGLANG_DP_USE_REDUCE_SCATTER=1 + export GPU_MAX_HW_QUEUES=5 CHUNKED_PREFILL_SIZE=$((ISL * TP)) PARALLEL_ARGS+=( --dp "$TP" --enable-dp-attention --enable-prefill-delayer + --enable-two-batch-overlap ) - if [ "$ISL" -gt 1024 ]; then - export GPU_MAX_HW_QUEUES=5 - PARALLEL_ARGS+=(--enable-two-batch-overlap) - fi fi if [ "${EP_SIZE:-1}" -gt 1 ]; then PARALLEL_ARGS+=(--ep-size "$EP_SIZE") From 2c75a9e7ff09d94e76166a42d167f2991e439947 Mon Sep 17 00:00:00 2001 From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com> Date: Tue, 14 Jul 2026 16:27:11 +0800 Subject: [PATCH 5/6] chore: prepare PR 2093 ingest recovery From 8e1a92cb0761fd7879f1db55cd012faf046b9ef1 Mon Sep 17 00:00:00 2001 From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com> Date: Tue, 14 Jul 2026 16:28:20 +0800 Subject: [PATCH 6/6] =?UTF-8?q?fix:=20recover=20PR=202093=20ingest=20/=20?= =?UTF-8?q?=E6=81=A2=E5=A4=8D=20PR=202093=20=E7=BB=93=E6=9E=9C=E5=85=A5?= =?UTF-8?q?=E5=BA=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- perf-changelog.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 57d510dd15..9edbad035f 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4750,3 +4750,11 @@ - "Image: lmsysorg/sglang:nightly-dev-cu13-20260709-074bb928" - "6 topologies across 1k/1k and 8k/1k: 1P1D TP4 STP + wide-EP (DEP4 prefill / DEP16 decode) from 1P1D up to 8P1D, recipes under benchmarks/multi_node/srt-slurm-recipes/sglang/qwen3.5/gb300-fp8/" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2137 + +- config-keys: + - dsv4-fp4-mi355x-sglang + description: + - "Bump image to lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260706" + - "Clean the export envs" + - "Enable two batch overlap" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2197