diff --git a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh index 0455010593..daa0704e56 100644 --- a/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh +++ b/benchmarks/single_node/agentic/dsv4_fp4_mi355x_vllm_mtp.sh @@ -404,6 +404,8 @@ export VLLM_ROCM_USE_AITER=1 export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 export VLLM_ROCM_USE_AITER_MOE=1 export VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 +# vLLM only clamps torch threads after weight loading; cap from process start. +export OMP_NUM_THREADS=1 sleep 180 diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_vllm.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_vllm.sh index da6a213882..f01047a7d4 100755 --- a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_vllm.sh +++ b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_vllm.sh @@ -46,10 +46,11 @@ fi export VLLM_ROCM_USE_AITER=1 export VLLM_ROCM_USE_AITER_MOE=1 -# Keep the upstream ROCm recipe knobs explicit. The shared-expert fusion -# self-disables when this checkpoint's shared-expert path is not eligible. +# Fuses the native-FP8 shared expert into the MXFP4 routed-expert AITER kernel. export VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 +# vLLM only clamps torch threads after weight loading; cap from process start. +export OMP_NUM_THREADS=1 SERVER_LOG=/workspace/server.log diff --git a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_vllm_mtp.sh b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_vllm_mtp.sh index e052b3a451..1e5de8ee3d 100755 --- a/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_vllm_mtp.sh +++ b/benchmarks/single_node/fixed_seq_len/dsv4_fp4_mi355x_vllm_mtp.sh @@ -41,10 +41,11 @@ fi export VLLM_ROCM_USE_AITER=1 export VLLM_ROCM_USE_AITER_MOE=1 -# Keep the upstream ROCm recipe knobs explicit. The shared-expert fusion -# self-disables when this checkpoint's shared-expert path is not eligible. +# Fuses the native-FP8 shared expert into the MXFP4 routed-expert AITER kernel. export VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 export VLLM_ROCM_QUICK_REDUCE_QUANTIZATION=INT4 +# vLLM only clamps torch threads after weight loading; cap from process start. +export OMP_NUM_THREADS=1 SERVER_LOG=/workspace/server.log PORT=${PORT:-8888} diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 2510dc3b8d..0940c2495a 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1198,10 +1198,11 @@ dsv4-fp4-mi355x-sglang-mtp: # and atom DSv4 mi355x entries below. The serving recipe enables AITER, # shared-expert fusion, and INT4 quick-reduce explicitly; the pinned nightly # then auto-selects its ROCm DeepSeek-V4 mHC, sparse-indexer, attention, and -# compressor-GEMM optimizations. Shared-expert fusion self-disables when the -# checkpoint path is not eligible. +# compressor-GEMM optimizations. Shared-expert fusion (FHMoE) needs +# vllm-project/vllm#53161, first shipped in this nightly; on earlier builds +# it self-disabled because this checkpoint path was not eligible. dsv4-fp4-mi355x-vllm: - image: vllm/vllm-openai-rocm:nightly-7c5dc571cbd1064ecc8a9b1045637ff647aa22cb + image: vllm/vllm-openai-rocm:nightly-d9105ea8001e0a6d77a96327d17515bb5791fb36 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: mi355x @@ -1224,7 +1225,7 @@ dsv4-fp4-mi355x-vllm: # above ~conc32 (-37% @ conc32). Image reuses the base entry's pinned ROCm # nightly, which includes the MTP path and subsequent DSV4 kernel updates. dsv4-fp4-mi355x-vllm-mtp: - image: vllm/vllm-openai-rocm:nightly-7c5dc571cbd1064ecc8a9b1045637ff647aa22cb + image: vllm/vllm-openai-rocm:nightly-d9105ea8001e0a6d77a96327d17515bb5791fb36 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: mi355x @@ -1315,7 +1316,7 @@ qwen3.5-fp8-mi325x-sglang-mtp: - { tp: 8, ep: 1, conc-start: 4, conc-end: 64, spec-decoding: mtp } dsv4-fp4-mi355x-vllm-agentic-mtp: - image: vllm/vllm-openai-rocm:nightly-e962733e08d10f7ca65dac4df99e116460b8b174 + image: vllm/vllm-openai-rocm:nightly-d9105ea8001e0a6d77a96327d17515bb5791fb36 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:mi355x-amds diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 390afc2ecf..9dfeb4a02a 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6969,3 +6969,18 @@ - "Use SGLang's current W4A4 MegaMoE and DP LM-head flags for DP-attention instead of the deprecated MegaMoE environment variables." - "Resolve draft-model jobs through the existing B200 SGLang speculative recipe and the staged DeepSeek-V4-Pro-0813 checkpoint." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2821 + +- config-keys: + - dsv4-fp4-mi355x-vllm + - dsv4-fp4-mi355x-vllm-mtp + - dsv4-fp4-mi355x-vllm-agentic-mtp + scenario-type: + - fixed-seq-len + - agentic-coding + description: + - "Repin all three DSv4 MI355X vLLM entries to vllm/vllm-openai-rocm:nightly-d9105ea8001e0a6d77a96327d17515bb5791fb36 (Docker Hub pushed 2026-09-07T05:26:48Z): the 8K/1K STP and MTP entries from nightly-7c5dc571cbd1064ecc8a9b1045637ff647aa22cb (#2792) and AgentX from nightly-e962733e08d10f7ca65dac4df99e116460b8b174 (#2841). d9105ea8 is the first ROCm nightly that ships vllm-project/vllm#53161 (vllm-project/vllm@de69e821), which fuses the DeepSeek-V4 native-FP8 shared expert into the MXFP4 routed-expert AITER kernel (FHMoE). The tag commit is 7 commits ahead of de69e821; the two superseded pins are 266 and 28 commits behind it, so both predate FHMoE." + - "No new serving flags. All three scripts already export VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS=1 (#2792 for 8K/1K STP and MTP, #2841 for AgentX), but on every nightly before de69e821 the request was inert: vLLM's eligibility check rejected this mixed FP4+FP8 checkpoint and the fusion self-disabled at startup. This pin is what makes the existing flag take effect, so the comments above it in the two 8K/1K scripts and on the two 8K/1K entries in configs/amd-master.yaml now say what the fusion does and which upstream commit it needs, instead of why it self-disables." + - "Fusion eligibility is gated on gfx950, tensor parallelism 8, data parallelism 1, no expert parallelism and no EPLB, --moe-backend aiter, BF16 model dtype, and the DeepSeek-V4 block-128 FP8 shared-expert quantization config. Both 8K/1K entries and the AgentX TP8 arm satisfy it; the AgentX dp-attn conc-64 arm runs data parallelism 8 and keeps the unfused path." + - "Export OMP_NUM_THREADS=1 from all three scripts. vLLM clamps torch threads itself, but only after the weights are loaded: the 'Reducing Torch threads from 16 to 1 for serving' line is emitted after 'Model loading took', so the checkpoint is read with 16 torch threads per worker across 8 workers. Setting it externally caps threads from process start, which is what that log line recommends." + - "Weight loading currently costs 5760-5946 s (96-99 min) per job on this runner, measured across the c1, c4, c56 and c64 AgentX jobs of run 34087124574 - roughly half of each ~3 h job. It does not affect any measured number: loading completes before warmup and the profiling phase. The effect is also only observable on this runner. A local EXT4 box serves the checkpoint from page cache and loads in ~40 s with or without the cap, so it never exercises the cold-read path. CI reads cold from WEKAFS, where each of the 8 ranks streams the full 805 GiB to slice its own shards - roughly 6.4 TB of aggregate network reads per job - and that is where oversubscribed reader threads cost real time." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2851