diff --git a/.github/configs/amd-master.yaml b/.github/configs/amd-master.yaml index 5e5452c4c8..04df9e3a58 100644 --- a/.github/configs/amd-master.yaml +++ b/.github/configs/amd-master.yaml @@ -1874,7 +1874,7 @@ dsr1-fp4-mi355x-sglang-disagg-8k1k-mtp: - "DECODE_MTP_SIZE=1" dsv4-fp4-mi355x-sglang: - image: lmsysorg/sglang-rocm:v0.5.13-rocm720-mi35x-20260612 + image: lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260618 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: mi355x diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_sglang.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_sglang.sh index ade8d96fec..cc1bf774fc 100755 --- a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_sglang.sh +++ b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_sglang.sh @@ -29,6 +29,7 @@ export SGLANG_DSV4_REASONING_EFFORT=max export SGLANG_OPT_DEEPGEMM_HC_PRENORM=false export SGLANG_USE_AITER=1 export SGLANG_USE_ROCM700A=0 +export SGLANG_DP_USE_GATHERV=1 export SGLANG_OPT_USE_FUSED_COMPRESS=true export SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton export SGLANG_OPT_FP8_WO_A_GEMM=false @@ -60,9 +61,9 @@ start_gpu_monitor PARALLEL_ARGS=( --tensor-parallel-size "$TP" ) -CHUNKED_PREFILL_SIZE=8192 +CHUNKED_PREFILL_SIZE=$ISL if [ "${DP_ATTENTION}" = "true" ]; then - CHUNKED_PREFILL_SIZE=$((8192 * TP)) + CHUNKED_PREFILL_SIZE=$((ISL * TP)) PARALLEL_ARGS+=( --dp "$TP" --enable-dp-attention diff --git a/perf-changelog.yaml b/perf-changelog.yaml index eb47ba6ae2..404224648f 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4014,3 +4014,10 @@ - "1k/1k: 1p1d-dep4-dep8 (conc 4096,12288), 1p4d-dep4-tp8 (conc 4-128), 1p1d-dep4-dep16 (conc 4096,6144)" - "8k/1k: 1p4d-dep4-tep4 (conc 128), 1p4d-dep4-tp8 (conc 4-256), 3p1d-dep4-dep16 (conc 1024), 6p1d-dep4-dep16 (conc 3072), 8p1d-dep4-dep16 (conc 6144)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1862 + +- config-keys: + - dsv4-fp4-mi355x-sglang + description: + - "Bump image to lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260618." + - "Enable SGLANG_DP_USE_GATHERV=1. Change to use all_gatherv + reduce_scatterv for tp + dp config." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1824