[AMD] Tune Kimi-K2.5 MXFP4 MI355X vLLM toward ATOM parity / 调优 Kimi-K2.5 MXFP4 MI355X vLLM，缩小与 ATOM 的性能差距

benchmarks/single_node/fixed_seq_len/kimik2.5_fp4_mi355x.sh

@@ -35,6 +35,10 @@ if [ "${EVAL_ONLY}" = "true" ]; then
     MAX_MODEL_LEN="$EVAL_MAX_MODEL_LEN"
 fi
 
+if [ "$MAX_MODEL_LEN" -lt 9472 ]; then
+    MAX_MODEL_LEN=9472
+fi
+
 # If the machine runs a MEC FW older than 177, RCCL
 # cannot reclaim some memory.
 # Disable that features to avoid crashes.
@@ -47,6 +51,12 @@ fi
 
 export VLLM_ROCM_USE_AITER=1
 export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4
+export VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1
+export VLLM_ROCM_USE_SKINNY_GEMM=0
+export AITER_MXFP4_INTERMEDIATE=1
+export AITER_BYPASS_TUNE_CONFIG=0
+export AITER_MOE_SORT_BACKEND=auto
+export OMP_NUM_THREADS=1
 
 # Disable AITER RMSNorm for TP < 8 due to accuracy issues
 if [ "${TP}" -lt 8 ]; then
@@ -59,20 +69,20 @@ else
   EP=" "
 fi
 
-# following AMD andy luo's recipe
-# https://x.com/linluo77/status/2017024513595301985
-
 # Start GPU monitoring (power, temperature, clocks every second)
 start_gpu_monitor
 
 set -x
 vllm serve $MODEL --port $PORT \
 --tensor-parallel-size=$TP \
 $EP \
---gpu-memory-utilization 0.90 \
+--gpu-memory-utilization 0.85 \
 --max-model-len $MAX_MODEL_LEN \
---block-size=1 \
---no-enable-prefix-caching \
+--kv-cache-dtype fp8 \
+--block-size 16 \
+--max-num-batched-tokens 16384 \
+--max-num-seqs 512 \
+--async-scheduling \
 --trust-remote-code \
 --no-enable-prefix-caching \
 --mm-encoder-tp-mode data > $SERVER_LOG 2>&1 &

configs/amd-master.yaml

@@ -509,7 +509,7 @@ kimik2.5-int4-mi300x-vllm:
       - { tp: 8, conc-start: 4, conc-end: 64 }
 
 kimik2.5-fp4-mi355x-vllm:
-  image: vllm/vllm-openai-rocm:v0.24.0
+  image: vllm/vllm-openai-rocm:nightly-387189c42997b27e2c04b5d97ef8190ffa2bf909
   model: amd/Kimi-K2.5-MXFP4
   model-prefix: kimik2.5
   runner: mi355x
@@ -521,8 +521,8 @@ kimik2.5-fp4-mi355x-vllm:
     - isl: 8192
       osl: 1024
       search-space:
-      - { tp: 8, conc-start: 4, conc-end: 64 }
-      - { tp: 4, conc-start: 4, conc-end: 64 }
+      - { tp: 8, conc-start: 4, conc-end: 128 }
+      - { tp: 4, conc-start: 4, conc-end: 128 }
 
 kimik2.5-fp4-mi355x-vllm-agentic:
   image: vllm/vllm-openai-rocm:v0.22.0

perf-changelog.yaml

@@ -5347,3 +5347,11 @@
     - "Add seven checked-in srt-slurm recipes selected one-to-one by the 8K/1K prefill/decode topology and concurrency entries."
     - "Image: vllm/vllm-openai:v0.25.1"
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2360
+
+- config-keys:
+    - kimik2.5-fp4-mi355x-vllm
+  description:
+    - "Set the vLLM ROCm image to the immutable vllm/vllm-openai-rocm:nightly-387189c42997b27e2c04b5d97ef8190ffa2bf909 build (the latest official ROCm nightly, commit-pinned), which ships the AITER MXFP4 MoE bump (>= v0.1.16.post5 from vllm-project/vllm#48683) including the ROCm/aiter#3832 gfx950 MXFP4 MoE backend"
+    - "Apply the accuracy-gated Kimi-K2.5 MXFP4 settings: tuned AITER MXFP4 MoE, fused shared experts, FP8 KV cache, block size 16, 16384 batched tokens, 512 sequences, async scheduling, gpu-memory-utilization 0.85 (headroom for CUDA-graph capture on MI355X), and the AITER BF16 GEMM path"
+    - "Extend the TP4 and TP8 8k1k concurrency sweep from 64 to 128 (1k1k deprecated per #2263)"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2213