diff --git a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/atom/mi355x-fp4-mtp/agentic.yaml b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/atom/mi355x-fp4-mtp/agentic.yaml index ac3fc41258..60520959b6 100644 --- a/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/atom/mi355x-fp4-mtp/agentic.yaml +++ b/inferencex-e2e/benchmarks/single_node/srt-slurm-recipes/minimaxm3/atom/mi355x-fp4-mtp/agentic.yaml @@ -1,13 +1,13 @@ # MiniMax-M3 MXFP4 AgentX on MI355X with ATOM EAGLE3 (GQA draft, three tokens) # at TP2 and TP4, with GPU-resident KV or an in-process LMCache DRAM tier. # The DRAM variants add ATOM's lmcache_offload connector through -# extra-kv-connectors (runners/srt-slurm/patches/507-lmcache-server-atom-sglang.patch). +# srtctl's extra-kv-connectors (NVIDIA/srt-slurm#507, in srt-slurm since v2.36.0). base: schema: 2 name: minimaxm3-fp4-mi355x-atom-agentic model: path: hf:amd/MiniMax-M3-MXFP4 - container: rocm/atom-dev:nightly_202609281543 + container: rocm/atom-dev:nightly_202610070333 precision: fp4 resources: gpu_type: mi355x @@ -60,9 +60,9 @@ base: # One variant per point. Admission is 2x CONC. TP4 shards the indexer across # ranks at the measured points (15, 20, 24, 28, and the DRAM points 40, 48). -# The GPU-resident indexer-CP points turn off ATOM's default MiniMax-M3 mono -# decode (ATOM_MONO_ENABLE=0): with it on they hung in an NCCL broadcast -# mid-replay (run 36493321464). The LMCache points already run without mono. +# ATOM's MiniMax-M3 mono decode is on by default at every TP4 point and +# serves decode steps of up to 16 tokens (four requests with EAGLE3 K3); +# ATOM builds it for TP4 only, so TP2 runs the unfused path. override_tp4_c1: roles: agg: @@ -141,7 +141,6 @@ override_tp4_c15: max-num-seqs: 30 env: ATOM_M3_INDEXER_CP: '1' - ATOM_MONO_ENABLE: '0' benchmark: env: CONC: '15' @@ -154,7 +153,6 @@ override_tp4_c20: max-num-seqs: 40 env: ATOM_M3_INDEXER_CP: '1' - ATOM_MONO_ENABLE: '0' benchmark: env: CONC: '20' @@ -167,7 +165,6 @@ override_tp4_c24: max-num-seqs: 48 env: ATOM_M3_INDEXER_CP: '1' - ATOM_MONO_ENABLE: '0' benchmark: env: CONC: '24' @@ -180,7 +177,6 @@ override_tp4_c28: max-num-seqs: 56 env: ATOM_M3_INDEXER_CP: '1' - ATOM_MONO_ENABLE: '0' benchmark: env: CONC: '28' @@ -205,6 +201,16 @@ override_tp2_c2: env: CONC: '2' +override_tp2_c4: + roles: + agg: + gpus: 2 + args: + max-num-seqs: 8 + benchmark: + env: + CONC: '4' + # DRAM variants: LMCache's in-process CPU tier with ATOM's segmented LRU and # all-rank lookup, sized at TOTAL_CPU_DRAM_GB / TP (257 GB) per rank. # PYTHONHASHSEED keeps every rank's chunk keys identical; without it the diff --git a/inferencex-e2e/configs/amd-master.yaml b/inferencex-e2e/configs/amd-master.yaml index 10a264bc70..db08533962 100644 --- a/inferencex-e2e/configs/amd-master.yaml +++ b/inferencex-e2e/configs/amd-master.yaml @@ -738,7 +738,7 @@ kimik3-fp4-mi355x-vllm-agentic-mtp: - { tp: 8, ep: 1, dcp-size: 8, kv-offloading: dram, kv-offload-backend: { name: vllm-simple }, conc-list: [44, 48, 70], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/kimik3/vllm/mi355x-fp4-mtp/agentic.yaml } minimaxm3-fp4-mi355x-atom-agentic-mtp: - image: rocm/atom-dev:nightly_202609281543 + image: rocm/atom-dev:nightly_202610070333 model: amd/MiniMax-M3-MXFP4 model-prefix: minimaxm3 runner: cluster:mi355x-amds @@ -752,9 +752,9 @@ minimaxm3-fp4-mi355x-atom-agentic-mtp: - dram-utilization: 0.687 search-space: - { tp: 4, kv-offloading: none, conc-list: [1, 2, 4, 5, 8, 10, 12, 15, 20, 24, 28], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/atom/mi355x-fp4-mtp/agentic.yaml } - - { tp: 2, kv-offloading: none, conc-list: [1, 2], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/atom/mi355x-fp4-mtp/agentic.yaml } - - { tp: 2, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.6.dev98+g05fc77a0" }, conc-list: [20, 25, 30], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/atom/mi355x-fp4-mtp/agentic.yaml } - - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.6.dev98+g05fc77a0" }, conc-list: [40, 48], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/atom/mi355x-fp4-mtp/agentic.yaml } + - { tp: 2, kv-offloading: none, conc-list: [1, 2, 4], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/atom/mi355x-fp4-mtp/agentic.yaml } + - { tp: 2, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.6.dev139+gf22dec28" }, conc-list: [20, 25, 30], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/atom/mi355x-fp4-mtp/agentic.yaml } + - { tp: 4, kv-offloading: dram, kv-offload-backend: { name: lmcache, version: "0.5.6.dev139+gf22dec28" }, conc-list: [40, 48], spec-decoding: mtp, srt-recipe: benchmarks/single_node/srt-slurm-recipes/minimaxm3/atom/mi355x-fp4-mtp/agentic.yaml } dsr1-fp4-mi355x-sglang-disagg: image: lmsysorg/sglang-rocm:v0.5.17-rocm720-mi35x-20260809 diff --git a/inferencex-e2e/perf-changelog.yaml b/inferencex-e2e/perf-changelog.yaml index 3cb75805e5..bb30c845c3 100644 --- a/inferencex-e2e/perf-changelog.yaml +++ b/inferencex-e2e/perf-changelog.yaml @@ -9475,3 +9475,16 @@ - "Refresh the aggregated curve with TP4/EP4 points at concurrency 1, 4, 8, 16, 24 and 32, plus TP4/EP4/DP4 points at concurrency 48 and 128 behind sglang-router." - "Apply the point-specific prefill chunking, scheduling, admission, memory and SWA prefix-tail settings from the single-node recipe." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3745 + +- config-keys: + - minimaxm3-fp4-mi355x-atom-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Update the image from rocm/atom-dev:nightly_202609281543 (ATOM 0.1.7.dev43+gaa0c5c3a1) to nightly_202610070333 (digest sha256:2f605a3e8b459a0385f784edb0335c11892277fee650fe42044e4af39e36dd01; ATOM 0.1.7.dev60+g7fc0926c6, ROCm 7.2.4). ROCm/ATOM#2479 moves MiniMax-M3 mono decode onto the shared mono framework, publishes every mono mailbox flag only after the stores it announces have drained, and rewrites the sparse selection as exact top-k by counting; ROCm/ATOM#2483 rebuilds the per-step mailbox clear and cross-rank fence as one FlyDSL launch, replacing a Triton fence that Triton 3.7 on gfx950 compiled into a loop that could spin for ever; ROCm/ATOM#2422 fixes the mono long-context decision. The upstream recipes/MiniMax-M3-Agentic-InferenceX.md is unchanged." + - "Enable MiniMax-M3 mono decode at every TP4 point by removing ATOM_MONO_ENABLE=0 from the indexer-CP points C15, C20, C24 and C28. Mono serves decode steps of up to 16 tokens (four requests with EAGLE3 K3), covering all of TP4 C1, C2 and C4. ATOM builds mono for TP4 only, so TP2 runs the unfused path." + - "Add TP2 concurrency 4 (max-num-seqs 8); the GPU-resident TP2 points are now C1, C2 and C4." + - "Record LMCache 0.5.6.dev139+gf22dec28, the wheel in the new image, for both DRAM-offload arms; the offload settings are unchanged." + - "No data-type or precision change to the EAGLE3 draft model (Inferact/MiniMax-M3-EAGLE3-GQA): online_quant_config (ptpc_fp8), kv_cache_dtype fp8 and index-cache-dtype fp8 are unchanged; mono decode changes the target model's decode kernels, not draft precision." + - "更新镜像至 rocm/atom-dev:nightly_202610070333(ATOM 0.1.7.dev60+g7fc0926c6,含 ROCm/ATOM#2483 的 mono step fence 修复);去掉 TP4 C15/C20/C24/C28 的 ATOM_MONO_ENABLE=0,所有 TP4 点位启用 MiniMax-M3 mono decode(每步最多 16 个 token,即 EAGLE3 K3 下 4 个请求,覆盖 C1/C2/C4);ATOM 只为 TP4 构建 mono,TP2 走非融合路径。新增 TP2 C4,LMCache 版本记录为 0.5.6.dev139+gf22dec28。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3750