diff --git a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_mi355x.sh b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_mi355x.sh index b036e060e1..87809f54b6 100644 --- a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_mi355x.sh +++ b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_mi355x.sh @@ -20,6 +20,7 @@ if [[ "$MODEL" != /* ]]; then hf download "$MODEL"; fi export SGLANG_USE_AITER=1 export SGLANG_USE_AITER_UNIFIED_ATTN=1 export AITER_FLYDSL_FORCE=1 +export SGLANG_MAMBA_SSM_DTYPE=bfloat16 SERVER_LOG=/workspace/server.log MEM_FRAC_STATIC=${MEM_FRAC_STATIC:-0.8} diff --git a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_mi355x_mtp.sh b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_mi355x_mtp.sh index 8081b824e1..16e86e5443 100755 --- a/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_mi355x_mtp.sh +++ b/benchmarks/single_node/fixed_seq_len/qwen3.5_fp4_mi355x_mtp.sh @@ -20,6 +20,7 @@ hf download "$MODEL" export SGLANG_USE_AITER=1 export SGLANG_USE_AITER_UNIFIED_ATTN=1 export AITER_FLYDSL_FORCE=1 +export SGLANG_MAMBA_SSM_DTYPE=bfloat16 SERVER_LOG=/workspace/server.log MEM_FRAC_STATIC=${MEM_FRAC_STATIC:-0.8} diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 4e1d9a64d7..3e3170dac6 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -470,7 +470,7 @@ qwen3.5-fp8-mi355x-sglang-disagg: - "DECODE_MTP_SIZE=0" qwen3.5-fp4-mi355x-sglang: - image: lmsysorg/sglang-rocm:v0.5.13-rocm720-mi35x-20260612 + image: lmsysorg/sglang-rocm:v0.5.15-rocm720-mi35x-20260713 model: amd/Qwen3.5-397B-A17B-MXFP4 model-prefix: qwen3.5 runner: mi355x @@ -512,7 +512,7 @@ qwen3.5-fp4-mi355x-atom: - { tp: 4, conc-start: 4, conc-end: 16 } qwen3.5-fp4-mi355x-sglang-mtp: - image: lmsysorg/sglang-rocm:v0.5.13-rocm720-mi35x-20260612 + image: lmsysorg/sglang-rocm:v0.5.15-rocm720-mi35x-20260713 model: amd/Qwen3.5-397B-A17B-MXFP4 model-prefix: qwen3.5 runner: mi355x diff --git a/perf-changelog.yaml b/perf-changelog.yaml index e8e9078c2e..fe36101866 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -4833,7 +4833,6 @@ - "Add GLM-5.1 FP8 GB200 disaggregated dynamo-sglang MTP config covering 8k1k, high-throughput and low-latency topologies" - "Image: lmsysorg/sglang:v0.5.13.post1-cu130" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2123 - - config-keys: - dsv4-fp8-mi325x-vllm @@ -4860,3 +4859,11 @@ - "Add GB300 DeepSeek-V4 dynamo-sglang agentic recipes (agg-tp4 + dep8 disagg pareto)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2157 +- config-keys: + - qwen3.5-fp4-mi355x-sglang + - qwen3.5-fp4-mi355x-sglang-mtp + description: + - "Bump image to lmsysorg/sglang-rocm:v0.5.15-rocm720-mi35x-20260713" + - "Add SGLANG_MAMBA_SSM_DTYPE=bfloat16 in both non-MTP and MTP benchmark scripts" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2201 +