diff --git a/benchmarks/multi_node/amd_utils/job.slurm b/benchmarks/multi_node/amd_utils/job.slurm index 0ff5a2b169..863d9d7635 100755 --- a/benchmarks/multi_node/amd_utils/job.slurm +++ b/benchmarks/multi_node/amd_utils/job.slurm @@ -337,7 +337,7 @@ export DOCKER_CONT_NAME="container_${ENGINE}_${SANITIZED_USER}_${MODEL_NAME}_${S # vLLM external router container. # NOTE: vllm/vllm-router only retains ~16 recent nightlies on Docker Hub; older # dated tags are garbage-collected (manifest unknown) -VLLM_ROUTER_IMAGE="${VLLM_ROUTER_IMAGE:-vllm/vllm-router:nightly-20260629-e667ebb}" +VLLM_ROUTER_IMAGE="${VLLM_ROUTER_IMAGE:-vllm/vllm-router:nightly-20260716-1fbcde7}" ROUTER_CONT_NAME="router_vllm_${SANITIZED_USER}_${SLURM_JOB_ID}" # Separate agentic benchmark-client container (see CLIENT_IMAGE handling below). CLIENT_CONT_NAME="container_${ENGINE}_${SANITIZED_USER}_client_${SLURM_JOB_ID}" diff --git a/benchmarks/multi_node/amd_utils/models_vllm.yaml b/benchmarks/multi_node/amd_utils/models_vllm.yaml index 9c046b4cf8..6e2f505edf 100644 --- a/benchmarks/multi_node/amd_utils/models_vllm.yaml +++ b/benchmarks/multi_node/amd_utils/models_vllm.yaml @@ -25,9 +25,9 @@ amd-Llama-3.3-70B-Instruct-FP8-KV: env: "VLLM_USE_V1=1 VLLM_V1_USE_PREFILL_DECODE_ATTENTION=1 AMDGCN_USE_BUFFER_OPS=1 VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_RMSNORM=1 VLLM_USE_AITER_TRITON_ROPE=1 TRITON_HIP_ASYNC_COPY_BYPASS_PERMUTE=1 TRITON_HIP_USE_ASYNC_COPY=1 TRITON_HIP_USE_BLOCK_PINGPONG=1 TRITON_HIP_ASYNC_FAST_SWIZZLE=1" Kimi-K2.5-MXFP4: - prefill_flags: "--tensor-parallel-size 8 --compilation-config '{\"cudagraph_mode\":\"PIECEWISE\"}' --no-enable-prefix-caching --block-size 1 --gpu-memory-utilization 0.90 --mm-encoder-tp-mode data" - decode_flags: "--tensor-parallel-size 8 --enable-expert-parallel --all2all-backend mori_low_latency --compilation-config '{\"cudagraph_mode\":\"PIECEWISE\"}' --no-enable-prefix-caching --block-size 1 --gpu-memory-utilization 0.90 --mm-encoder-tp-mode data" - env: "VLLM_USE_V1=1 VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_PAGED_ATTN=0 VLLM_ROCM_USE_AITER_RMSNORM=1 VLLM_USE_AITER_TRITON_SILU_MUL=0 VLLM_ENGINE_READY_TIMEOUT_S=3600" + prefill_flags: "--tensor-parallel-size 8 --no-enable-prefix-caching --block-size 1 --gpu-memory-utilization 0.90 --max-model-len 32768 --mm-encoder-tp-mode data --kv-cache-dtype fp8 --max-num-seqs 256 --max-num-batched-tokens 32768" + decode_flags: "--tensor-parallel-size 8 --all2all-backend mori_low_latency --no-enable-prefix-caching --block-size 1 --gpu-memory-utilization 0.90 --max-model-len 32768 --mm-encoder-tp-mode data --kv-cache-dtype fp8 --max-num-seqs 256 --max-num-batched-tokens 32768" + env: "VLLM_USE_V1=1 VLLM_ROCM_USE_AITER=1 VLLM_ROCM_USE_AITER_RMSNORM=1 VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 HSA_NO_SCRATCH_RECLAIM=1 VLLM_ENGINE_READY_TIMEOUT_S=3600" hf_dir: "models--amd--Kimi-K2.5-MXFP4" MiniMax-M2.5: diff --git a/benchmarks/multi_node/amd_utils/submit.sh b/benchmarks/multi_node/amd_utils/submit.sh index 262fd25aa0..f125eb0955 100755 --- a/benchmarks/multi_node/amd_utils/submit.sh +++ b/benchmarks/multi_node/amd_utils/submit.sh @@ -166,7 +166,7 @@ fi # Optional: exclude specific nodes (e.g. nodes with broken Docker sockets). # Set SLURM_EXCLUDE_NODES env var to a comma-separated list of hostnames. EXCLUDE_OPT=() -SLURM_EXCLUDE_NODES="${SLURM_EXCLUDE_NODES:-mia1-p01-g11,mia1-p01-g12,mia1-p01-g15}" +SLURM_EXCLUDE_NODES="${SLURM_EXCLUDE_NODES:-mia1-p01-g09,mia1-p01-g11,mia1-p01-g12,mia1-p01-g14,mia1-p01-g15}" if [[ -n "${SLURM_EXCLUDE_NODES:-}" ]]; then EXCLUDE_OPT=(--exclude "$SLURM_EXCLUDE_NODES") fi diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index ebe0cb29d4..0f547cba86 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -798,7 +798,7 @@ dsr1-fp8-mi355x-sglang-disagg-mtp: - "DECODE_MTP_SIZE=2" kimik2.5-fp4-mi355x-vllm-disagg: - image: vllm/vllm-openai-rocm:v0.24.0 + image: vllm/vllm-openai-rocm:nightly-2afa3f7e950264bb179d030c23a1ed1f46558fd9 model: amd/Kimi-K2.5-MXFP4 model-prefix: kimik2.5 runner: mi355x-disagg @@ -810,22 +810,43 @@ kimik2.5-fp4-mi355x-vllm-disagg: disagg: true scenarios: fixed-seq-len: + # All workers TP4 (real-weight sweep: TP8 decode is no better than TP4). + # Split across P/D topologies: 1D is stable at low conc; the high-conc tail + # runs on 2D so decode load is spread across two engines. - isl: 8192 osl: 1024 search-space: + # 1P(TP4) 1D(TP4) = 2 nodes. Low concurrency only. + - spec-decoding: "none" + conc-list: [ 1, 2, 4 ] + prefill: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "PREFILL_NODES=1" + decode: + num-worker: 1 + tp: 4 + ep: 1 + dp-attn: false + additional-settings: + - "DECODE_NODES=1" + # 1P(TP4) 2D(TP4) = 3 nodes. High concurrency (decode KV/load spread over 2D). - spec-decoding: "none" conc-list: [ 8, 16, 32, 64, 128, 256, 512 ] prefill: num-worker: 1 - tp: 8 + tp: 4 ep: 1 dp-attn: false additional-settings: - "PREFILL_NODES=1" decode: num-worker: 2 - tp: 8 - ep: 8 + tp: 4 + ep: 1 dp-attn: false additional-settings: - "DECODE_NODES=2" diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 75f85985c9..335245a6c3 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5049,3 +5049,14 @@ - "Topologies: 1p4d-dep4-tep8 (conc 4/24), 1p5d-dep4-tep4 (conc 5/30/60/115/195), 1p1d-dep4-dep8 (conc 308), 2p1d-dep4-dep8 (conc 615), 3p1d-dep4-dep8 (conc 1127), 4p1d-dep4-dep8 (conc 2151)" - "Runner updated to clone srt-slurm at sa-submission-q2-2026 and copy local recipes into recipes/trtllm/kimi-k25-nvfp4/b200-fp4/" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2249 + +- config-keys: + - kimik2.5-fp4-mi355x-vllm-disagg + description: + - "Bump image to vllm/vllm-openai-rocm:nightly-2afa3f7e950264bb179d030c23a1ed1f46558fd9" + - "All-TP4 prefill/decode (drop TP8): a real-weight sweep showed TP8 decode is no better than TP4 on tok/s/GPU; expert parallelism off (ep:1, single-node TP8 sweep showed EP -14% to -27% slower than dense)" + - "Split the 8k/1k P/D topology by concurrency: low conc (1,2,4) on 1P(TP4)/1D(TP4); high conc (8,16,32,64,128,256,512) on 1P(TP4)/2D(TP4) so decode load/KV is spread across two decode engines (the single-decode engine is unstable in the high-conc tail)" + - "Sync per-worker vLLM serve flags/env with the single-node recipe: --kv-cache-dtype fp8, --max-model-len 32768, --max-num-seqs 256, --max-num-batched-tokens 32768; VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4, HSA_NO_SCRATCH_RECLAIM=1, AITER defaults; drop the --compilation-config PIECEWISE pin (use vLLM default FULL_AND_PIECEWISE)" + - "Re-pin VLLM_ROUTER_IMAGE to vllm/vllm-router:nightly-20260716-1fbcde7 (previous nightly-20260629-e667ebb was garbage-collected from Docker Hub)" + - "Exclude known-bad nodes mia1-p01-g09,g14 from the disagg node pool" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2301