diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_sglang.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_sglang.sh index 2ced868365..79c5832db9 100755 --- a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_sglang.sh +++ b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_sglang.sh @@ -26,27 +26,9 @@ if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi export SGLANG_DEFAULT_THINKING=1 export SGLANG_DSV4_REASONING_EFFORT=max -export SGLANG_OPT_DEEPGEMM_HC_PRENORM=false -export SGLANG_USE_AITER=1 export SGLANG_USE_ROCM700A=0 -export SGLANG_DP_USE_GATHERV=1 -export SGLANG_OPT_USE_FUSED_COMPRESS=true export SGLANG_HACK_FLASHMLA_BACKEND=unified_kv_triton -export SGLANG_OPT_FP8_WO_A_GEMM=false -export SGLANG_OPT_USE_JIT_INDEXER_METADATA=false -export SGLANG_OPT_USE_TOPK_V2=false -export SGLANG_OPT_USE_AITER_INDEXER=true -export SGLANG_OPT_USE_TILELANG_INDEXER=false -export SGLANG_OPT_USE_TILELANG_MHC_PRE=false -export SGLANG_OPT_USE_TILELANG_MHC_POST=false -export SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1 -export SGLANG_OPT_USE_FUSED_COMPRESS_TRITON=true export AITER_BF16_FP8_MOE_BOUND=0 -export SGLANG_EAGER_INPUT_NO_COPY=true - -# multi-stream -export SGLANG_OPT_USE_MULTI_STREAM_OVERLAP=false -export SGLANG_ROCM_USE_MULTI_STREAM=false SERVER_LOG=/workspace/server.log @@ -63,12 +45,18 @@ PARALLEL_ARGS=( ) CHUNKED_PREFILL_SIZE=$ISL if [ "${DP_ATTENTION}" = "true" ]; then + export SGLANG_SHARED_EXPERT_TP1=1 + export SGLANG_DP_SHARED_EXPERT_LOCAL=1 + export SGLANG_DP_USE_GATHERV=1 + export SGLANG_DP_USE_REDUCE_SCATTER=1 + export GPU_MAX_HW_QUEUES=5 + CHUNKED_PREFILL_SIZE=$((ISL * TP)) PARALLEL_ARGS+=( --dp "$TP" --enable-dp-attention --enable-prefill-delayer - --prefill-delayer-max-delay-ms 5000 + --enable-two-batch-overlap ) fi if [ "${EP_SIZE:-1}" -gt 1 ]; then @@ -83,10 +71,12 @@ sglang serve \ --trust-remote-code \ --disable-radix-cache \ --attention-backend dsv4 \ + --cuda-graph-max-bs ${CONC} \ --max-running-requests ${CONC} \ --mem-fraction-static 0.90 \ --swa-full-tokens-ratio 0.15 \ --page-size 256 \ + --kv-cache-dtype fp8_e4m3 \ --context-length $MAX_MODEL_LEN \ --chunked-prefill-size $CHUNKED_PREFILL_SIZE \ --disable-shared-experts-fusion \ diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index f9a2a99e07..a2cb69db9e 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1736,7 +1736,7 @@ dsr1-fp4-mi355x-sglang-disagg-8k1k-mtp: - "DECODE_MTP_SIZE=1" dsv4-fp4-mi355x-sglang: - image: lmsysorg/sglang-rocm:v0.5.13.post1-rocm720-mi35x-20260618 + image: lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260706 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: mi355x diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 806996178a..f5e0ff4284 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4684,3 +4684,11 @@ - "Image: lmsysorg/sglang:v0.5.11-cu130" - "11 topologies across 1k/1k and 8k/1k (prefill TP4 + decode wide-EP: 8k/1k DEP16/24/32/40, 1k/1k DEP48/56; per-node TP4 low-latency); MTP flags: speculative-algorithm EAGLE, num-steps 2, eagle-topk 1, num-draft-tokens 3" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1907 + +- config-keys: + - dsv4-fp4-mi355x-sglang + description: + - "Bump image to lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260706" + - "Clean the export envs" + - "Enable two batch overlap" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2093