diff --git a/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_mi355x.sh b/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_mi355x.sh index d4616143ac..ecd3d0e67a 100755 --- a/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_mi355x.sh +++ b/benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_mi355x.sh @@ -35,6 +35,10 @@ if [ "${EVAL_ONLY}" = "true" ]; then MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN" fi +if [ "$MAX_MODEL_LEN" -lt 9472 ]; then + MAX_MODEL_LEN=9472 +fi + # If the machine runs a MEC FW older than 177, RCCL # cannot reclaim some memory. # Disable that features to avoid crashes. @@ -47,6 +51,12 @@ fi export VLLM_ROCM_USE_AITER=1 export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 +export VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 +export VLLM_ROCM_USE_SKINNY_GEMM=0 +export AITER_MXFP4_INTERMEDIATE=1 +export AITER_BYPASS_TUNE_CONFIG=0 +export AITER_MOE_SORT_BACKEND=auto +export OMP_NUM_THREADS=1 # Disable AITER RMSNorm for TP < 8 due to accuracy issues if [ "${TP}" -lt 8 ]; then @@ -59,9 +69,6 @@ else EP=" " fi -# following AMD andy luo's recipe -# https://x.com/linluo77/status/2017024513595301985 - # Start GPU monitoring (power, temperature, clocks every second) start_gpu_monitor @@ -69,10 +76,13 @@ set -x vllm serve $MODEL --port $PORT \ --tensor-parallel-size=$TP \ $EP \ ---gpu-memory-utilization 0.90 \ +--gpu-memory-utilization 0.85 \ --max-model-len $MAX_MODEL_LEN \ ---block-size=1 \ ---no-enable-prefix-caching \ +--kv-cache-dtype fp8 \ +--block-size 16 \ +--max-num-batched-tokens 16384 \ +--max-num-seqs 512 \ +--async-scheduling \ --trust-remote-code \ --no-enable-prefix-caching \ --mm-encoder-tp-mode data > $SERVER_LOG 2>&1 & diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index b15ae5bccd..b8f6f4fc31 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -509,7 +509,7 @@ kimik2.5-int4-mi300x-vllm: - { tp: 8, conc-start: 4, conc-end: 64 } kimik2.5-fp4-mi355x-vllm: - image: vllm/vllm-openai-rocm:v0.24.0 + image: vllm/vllm-openai-rocm:nightly-387189c42997b27e2c04b5d97ef8190ffa2bf909 model: amd/Kimi-K2.5-MXFP4 model-prefix: kimik2.5 runner: mi355x @@ -521,8 +521,8 @@ kimik2.5-fp4-mi355x-vllm: - isl: 8192 osl: 1024 search-space: - - { tp: 8, conc-start: 4, conc-end: 64 } - - { tp: 4, conc-start: 4, conc-end: 64 } + - { tp: 8, conc-start: 4, conc-end: 128 } + - { tp: 4, conc-start: 4, conc-end: 128 } kimik2.5-fp4-mi355x-vllm-agentic: image: vllm/vllm-openai-rocm:v0.22.0 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 4c22ed9198..6eebd97070 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5347,3 +5347,11 @@ - "Add seven checked-in srt-slurm recipes selected one-to-one by the 8K/1K prefill/decode topology and concurrency entries." - "Image: vllm/vllm-openai:v0.25.1" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2360 + +- config-keys: + - kimik2.5-fp4-mi355x-vllm + description: + - "Set the vLLM ROCm image to the immutable vllm/vllm-openai-rocm:nightly-387189c42997b27e2c04b5d97ef8190ffa2bf909 build (the latest official ROCm nightly, commit-pinned), which ships the AITER MXFP4 MoE bump (>= v0.1.16.post5 from vllm-project/vllm#48683) including the ROCm/aiter#3832 gfx950 MXFP4 MoE backend" + - "Apply the accuracy-gated Kimi-K2.5 MXFP4 settings: tuned AITER MXFP4 MoE, fused shared experts, FP8 KV cache, block size 16, 16384 batched tokens, 512 sequences, async scheduling, gpu-memory-utilization 0.85 (headroom for CUDA-graph capture on MI355X), and the AITER BF16 GEMM path" + - "Extend the TP4 and TP8 8k1k concurrency sweep from 64 to 128 (1k1k deprecated per #2263)" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2213