From cb24aecb927e4f2ce678069399c86faf8a9687bd Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 27 Aug 2026 11:22:53 -0500 Subject: [PATCH 1/5] feat(amd): port DSV4 ATOM to srt-slurm --- .../mi355x/disagg-1p1d-tp8-8k1k.yaml | 118 +++++++++++++++++ .../mi355x/disagg-2p1d-dpa-tp8-8k1k.yaml | 120 ++++++++++++++++++ configs/amd-master.yaml | 55 ++++++++ perf-changelog.yaml | 12 ++ 4 files changed, 305 insertions(+) create mode 100644 benchmarks/multi_node/srt-slurm-recipes/atom/deepseek-v4/mi355x/disagg-1p1d-tp8-8k1k.yaml create mode 100644 benchmarks/multi_node/srt-slurm-recipes/atom/deepseek-v4/mi355x/disagg-2p1d-dpa-tp8-8k1k.yaml diff --git a/benchmarks/multi_node/srt-slurm-recipes/atom/deepseek-v4/mi355x/disagg-1p1d-tp8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/atom/deepseek-v4/mi355x/disagg-1p1d-tp8-8k1k.yaml new file mode 100644 index 0000000000..6d4acc74e0 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/atom/deepseek-v4/mi355x/disagg-1p1d-tp8-8k1k.yaml @@ -0,0 +1,118 @@ +# DeepSeek-V4-Pro 1P1D ATOM P/D serving on MI355X. Native AToMesh +# routes prefill/decode traffic while ATOM transfers KV state through +# Mooncake TCP. The custom benchmark preserves the legacy 8k/1k search. + +name: "mi355x-atom-dsv4-disagg-1p1d-tp8-8k1k" + +model: + path: "hf:deepseek-ai/DeepSeek-V4-Pro" + container: "atom-nightly-202608251555" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "rocm/atom-dev:nightly_202608251555" + frameworks: + atom: "0.1.6rc1.dev336+g4e0848dcd" + atomesh: "4e0848dcdf94f3d4be0c1e3ce51db99bc93f8b8c" + +slurm: + time_limit: "08:00:00" + +resources: + gpu_type: "mi355x" + gpus_per_node: 8 + prefill_nodes: 1 + decode_nodes: 1 + prefill_workers: 1 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +frontend: + type: atomesh + enable_multiple_frontends: false + args: + policy: random + +backend: + type: atom + connector: mooncake + # The stable image predates Mooncake's ROCm DMA-BUF registration fix. TCP + # is the hardware-proven correctness transport; RDMA is validated separately + # when the fixed ATOM/Mooncake image is available. + mooncake_protocol: tcp + prefill_environment: &worker_environment + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache/hub" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache/hub" + PYTHONUNBUFFERED: "1" + OMP_NUM_THREADS: "1" + ATOM_MOE_GU_ITLV: "1" + AITER_BF16_FP8_MOE_BOUND: "0" + decode_environment: *worker_environment + atom_config: + prefill: + kv_cache_dtype: fp8 + gpu-memory-utilization: 0.85 + max-num-seqs: 256 + block-size: 16 + no-enable_prefix_caching: true + trust-remote-code: true + decode: + kv_cache_dtype: fp8 + gpu-memory-utilization: 0.85 + max-num-seqs: 128 + block-size: 16 + no-enable_prefix_caching: true + trust-remote-code: true + +srun_options: + container-writable: "" + container-remap-root: "" + mem: "0" + +health_check: + max_attempts: 720 + interval_seconds: 5 + +benchmark: + type: custom + command: | + set -euo pipefail + result_root="/results/${SLURM_JOB_ID}" + mkdir -p "${result_root}/fixed-seq" + archive_runtime_logs() { + tar -C /logs -czf "${result_root}/runtime-logs.tar.gz" . 2>/dev/null || true + } + trap archive_runtime_logs EXIT + for concurrency in 4 8 16 32 64 128; do + num_prompts=$((concurrency * 10)) + if (( num_prompts < 16 )); then num_prompts=16; fi + python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ + --backend openai \ + --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \ + --endpoint /v1/completions \ + --model deepseek-ai/DeepSeek-V4-Pro \ + --tokenizer deepseek-ai/DeepSeek-V4-Pro \ + --dataset-name random \ + --random-input-len 8192 \ + --random-output-len 1024 \ + --random-range-ratio 1.0 \ + --num-warmups "$((concurrency * 2))" \ + --num-prompts "${num_prompts}" \ + --max-concurrency "${concurrency}" \ + --request-rate inf \ + --ignore-eos \ + --trust-remote-code \ + --percentile-metrics ttft,tpot,itl,e2el \ + --save-result \ + --result-dir "${result_root}/fixed-seq" \ + --result-filename "dsv4-atom-mi355x-disagg-1p1d-tp8-isl8192-osl1024-c${concurrency}.json" + done + env: + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache/hub + HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub diff --git a/benchmarks/multi_node/srt-slurm-recipes/atom/deepseek-v4/mi355x/disagg-2p1d-dpa-tp8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/atom/deepseek-v4/mi355x/disagg-2p1d-dpa-tp8-8k1k.yaml new file mode 100644 index 0000000000..d062929763 --- /dev/null +++ b/benchmarks/multi_node/srt-slurm-recipes/atom/deepseek-v4/mi355x/disagg-2p1d-dpa-tp8-8k1k.yaml @@ -0,0 +1,120 @@ +# DeepSeek-V4-Pro 2P1D ATOM P/D serving on MI355X. Each logical endpoint +# occupies one TP8 node; the two prefill endpoints and decode endpoint use +# data-parallel attention/TBO as in the legacy production configuration. + +name: "mi355x-atom-dsv4-disagg-2p1d-dpa-tp8-8k1k" + +model: + path: "hf:deepseek-ai/DeepSeek-V4-Pro" + container: "atom-nightly-202608251555" + precision: "fp4" + +identity: + model: + repo: "deepseek-ai/DeepSeek-V4-Pro" + container: + image: "rocm/atom-dev:nightly_202608251555" + frameworks: + atom: "0.1.6rc1.dev336+g4e0848dcd" + atomesh: "4e0848dcdf94f3d4be0c1e3ce51db99bc93f8b8c" + +slurm: + time_limit: "08:00:00" + +resources: + gpu_type: "mi355x" + gpus_per_node: 8 + prefill_nodes: 2 + decode_nodes: 1 + prefill_workers: 2 + decode_workers: 1 + gpus_per_prefill: 8 + gpus_per_decode: 8 + +frontend: + type: atomesh + enable_multiple_frontends: false + args: + policy: random + +backend: + type: atom + connector: mooncake + mooncake_protocol: tcp + prefill_environment: &worker_environment + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache/hub" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache/hub" + PYTHONUNBUFFERED: "1" + OMP_NUM_THREADS: "1" + ATOM_MOE_GU_ITLV: "1" + AITER_BF16_FP8_MOE_BOUND: "0" + GPU_MAX_HW_QUEUES: "5" + ATOM_CPU_AFFINITY: "1" + decode_environment: *worker_environment + atom_config: + prefill: + kv_cache_dtype: fp8 + gpu-memory-utilization: 0.85 + max-num-seqs: 256 + block-size: 16 + no-enable_prefix_caching: true + enable-dp-attention: true + enable-tbo: true + trust-remote-code: true + decode: + kv_cache_dtype: fp8 + gpu-memory-utilization: 0.85 + max-num-seqs: 2048 + block-size: 16 + no-enable_prefix_caching: true + enable-dp-attention: true + enable-tbo: true + trust-remote-code: true + +srun_options: + container-writable: "" + container-remap-root: "" + mem: "0" + +health_check: + max_attempts: 720 + interval_seconds: 5 + +benchmark: + type: custom + command: | + set -euo pipefail + result_root="/results/${SLURM_JOB_ID}" + mkdir -p "${result_root}/fixed-seq" + archive_runtime_logs() { + tar -C /logs -czf "${result_root}/runtime-logs.tar.gz" . 2>/dev/null || true + } + trap archive_runtime_logs EXIT + for concurrency in 256 512 768 1024 2048; do + num_prompts=$((concurrency * 10)) + python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ + --backend openai \ + --base-url "http://${SRT_FRONTEND_HOST}:${SRT_FRONTEND_PORT}" \ + --endpoint /v1/completions \ + --model deepseek-ai/DeepSeek-V4-Pro \ + --tokenizer deepseek-ai/DeepSeek-V4-Pro \ + --dataset-name random \ + --random-input-len 8192 \ + --random-output-len 1024 \ + --random-range-ratio 1.0 \ + --num-warmups "$((concurrency * 2))" \ + --num-prompts "${num_prompts}" \ + --max-concurrency "${concurrency}" \ + --request-rate inf \ + --ignore-eos \ + --trust-remote-code \ + --percentile-metrics ttft,tpot,itl,e2el \ + --save-result \ + --result-dir "${result_root}/fixed-seq" \ + --result-filename "dsv4-atom-mi355x-disagg-2p1d-dpa-tp8-isl8192-osl1024-c${concurrency}.json" + done + env: + HF_HOME: /hf_hub_cache + HF_HUB_CACHE: /hf_hub_cache/hub + HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 0d5dbbec00..94f6f6127b 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -1063,3 +1063,58 @@ qwen3-0.6b-fp16-mi355x-sglang-srt-disagg: # Production-scale migration of the existing Qwen3.5 FP8 MI355X P/D lane to # srt-slurm. One full TP8 node serves prefill, one TP8 node serves decode, and # the request/KV planes remain native SGLang Router + AMD MoRI. + +dsv4-fp4-mi355x-atom-disagg: + image: rocm/atom-dev:nightly_202608251555 + model: deepseek-ai/DeepSeek-V4-Pro + model-prefix: dsv4 + runner: cluster:mi355x-amds + precision: fp4 + framework: atom-disagg + router: { name: atomesh, version: "4e0848dcdf94f3d4be0c1e3ce51db99bc93f8b8c" } + kv-p2p-transfer: mooncake + multinode: true + disagg: true + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + # The custom srt-slurm recipes own the exact concurrency loops. Keeping + # one matrix row per topology avoids launching the complete loop once per + # concurrency while preserving every original measurement point. + # 2P1D DPA+TP8 + - conc-list: [ 1 ] + prefill: + num-worker: 2 + tp: 8 + ep: 1 + dp-attn: true + additional-settings: + - "PREFILL_NODES=2" + - "CONFIG_FILE=recipes/atom/deepseek-v4/mi355x/disagg-2p1d-dpa-tp8-8k1k.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: true + additional-settings: + - "DECODE_NODES=1" + # 1P1D TP8 + - conc-list: [ 1 ] + prefill: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "PREFILL_NODES=1" + - "CONFIG_FILE=recipes/atom/deepseek-v4/mi355x/disagg-1p1d-tp8-8k1k.yaml" + decode: + num-worker: 1 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "DECODE_NODES=1" + # 1P1D TP8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 71631f96e9..6cdf0d7437 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6847,3 +6847,15 @@ - "Keep the workflow's host-only Python bytecode-cache prefix out of Slurm submissions, matching the legacy explicit Docker environment without changing serving arguments." - "Restore the legacy unlimited RDMA memory-lock limit and pin ATOM's literal worker model identity for evaluation routing." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2542 + +- config-keys: + - dsv4-fp4-mi355x-atom-disagg + description: + - "Port the DeepSeek-V4-Pro FP4 MI355X ATOM disaggregated 8k/1k submission from the configuration-specific AMD launcher to srt-slurm ATOM orchestration and native AToMesh routing." + - "Preserve all original search points: 1P1D TP8 at concurrency 4, 8, 16, 32, 64, and 128; 2P1D DPA+TP8 at concurrency 256, 512, 768, 1024, and 2048." + - "Use the unchanged InferenceX benchmark_serving.py through the custom benchmark contract and remove the exact legacy wrapper plus its obsolete deprecated 1k/1k entry." + - "Use the hardware-proven Mooncake TCP transport for the initial correctness sweep; validate the updated ROCm RDMA path separately before changing production transport." + - "Use the ATOM image's bundled native AToMesh binary directly, without Infera source overlays or Infera routing, and archive the complete runtime logs from the custom benchmark." + - "Run the current rocm/atom-dev nightly_202608251555 image at ATOM commit 4e0848dcdf94f3d4be0c1e3ce51db99bc93f8b8c, whose bundled native AToMesh and Mooncake implementation are used without downstream source patches." + - "Pin SemiAnalysisAI/srt-slurm PR #7 at 10a58d2ebb4d756c423424049c736b909b72e14f so this ATOM/AToMesh port shares the latest validated AMD orchestration base, including native-router data-parallel rank preservation and ATOM recipe flag normalization." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2627 From d0e4f8ed82a16882c94afed28e10b76480dcbabc Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 27 Aug 2026 12:38:26 -0500 Subject: [PATCH 2/5] fix(amd): align ATOM srt-slurm provenance --- perf-changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 6cdf0d7437..fcda6d68eb 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6857,5 +6857,5 @@ - "Use the hardware-proven Mooncake TCP transport for the initial correctness sweep; validate the updated ROCm RDMA path separately before changing production transport." - "Use the ATOM image's bundled native AToMesh binary directly, without Infera source overlays or Infera routing, and archive the complete runtime logs from the custom benchmark." - "Run the current rocm/atom-dev nightly_202608251555 image at ATOM commit 4e0848dcdf94f3d4be0c1e3ce51db99bc93f8b8c, whose bundled native AToMesh and Mooncake implementation are used without downstream source patches." - - "Pin SemiAnalysisAI/srt-slurm PR #7 at 10a58d2ebb4d756c423424049c736b909b72e14f so this ATOM/AToMesh port shares the latest validated AMD orchestration base, including native-router data-parallel rank preservation and ATOM recipe flag normalization." + - "Pin SemiAnalysisAI/srt-slurm PR #7 at c87d7b34b009be920896126013ad6dc74c5a99d5 so this ATOM/AToMesh port shares the latest validated AMD orchestration base, including native-router data-parallel rank preservation and ATOM recipe flag normalization." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2627 From 79fa5a79e09efd022f03e49ecf1710123fdc686e Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 2 Sep 2026 13:11:20 -0500 Subject: [PATCH 3/5] fix(amd): preserve upstream DSV4 ATOM recipe semantics --- .../mi355x/disagg-1p1d-tp8-8k1k.yaml | 54 +++++++++++------ .../mi355x/disagg-2p1d-dpa-tp8-8k1k.yaml | 48 ++++++++++----- configs/amd-master.yaml | 59 +------------------ perf-changelog.yaml | 5 +- 4 files changed, 73 insertions(+), 93 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/atom/deepseek-v4/mi355x/disagg-1p1d-tp8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/atom/deepseek-v4/mi355x/disagg-1p1d-tp8-8k1k.yaml index 6d4acc74e0..e735952b9f 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/atom/deepseek-v4/mi355x/disagg-1p1d-tp8-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/atom/deepseek-v4/mi355x/disagg-1p1d-tp8-8k1k.yaml @@ -1,22 +1,22 @@ # DeepSeek-V4-Pro 1P1D ATOM P/D serving on MI355X. Native AToMesh # routes prefill/decode traffic while ATOM transfers KV state through -# Mooncake TCP. The custom benchmark preserves the legacy 8k/1k search. +# Mooncake's default RDMA transport. The custom benchmark preserves the +# legacy 8k/1k search and request-length distribution. name: "mi355x-atom-dsv4-disagg-1p1d-tp8-8k1k" model: path: "hf:deepseek-ai/DeepSeek-V4-Pro" - container: "atom-nightly-202608251555" + container: "rocm/atom-dev:nightly_202606101403" precision: "fp4" identity: model: repo: "deepseek-ai/DeepSeek-V4-Pro" container: - image: "rocm/atom-dev:nightly_202608251555" + image: "rocm/atom-dev:nightly_202606101403" frameworks: - atom: "0.1.6rc1.dev336+g4e0848dcd" - atomesh: "4e0848dcdf94f3d4be0c1e3ce51db99bc93f8b8c" + atomesh: "087b82d9c1f630e79149ba37e6213257ec9a76f8" slurm: time_limit: "08:00:00" @@ -36,20 +36,30 @@ frontend: enable_multiple_frontends: false args: policy: random + log-level: info + disable-health-check: true + disable-circuit-breaker: true + prometheus-port: 29100 + +environment: &runtime_environment + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache/hub" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache/hub" + # Same Mooncake/ROCm lookup order as env_atom.sh in the original image. + LD_LIBRARY_PATH: "/opt/venv/lib/python3.10/site-packages/mooncake:/opt/rocm/lib" + PYTHONUNBUFFERED: "1" + PYTHONDONTWRITEBYTECODE: "1" + SAFETENSORS_FAST_GPU: "1" + VLLM_LOG_LEVEL: "WARNING" + ATOM_LOG_LEVEL: "WARNING" + AITER_LOG_LEVEL: "WARNING" + LOG_LEVEL: "WARNING" + LOGLEVEL: "WARNING" backend: type: atom connector: mooncake - # The stable image predates Mooncake's ROCm DMA-BUF registration fix. TCP - # is the hardware-proven correctness transport; RDMA is validated separately - # when the fixed ATOM/Mooncake image is available. - mooncake_protocol: tcp prefill_environment: &worker_environment - HF_HOME: "/hf_hub_cache" - HF_HUB_CACHE: "/hf_hub_cache/hub" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache/hub" - PYTHONUNBUFFERED: "1" - OMP_NUM_THREADS: "1" ATOM_MOE_GU_ITLV: "1" AITER_BF16_FP8_MOE_BOUND: "0" decode_environment: *worker_environment @@ -62,6 +72,7 @@ backend: no-enable_prefix_caching: true trust-remote-code: true decode: + cudagraph-capture-sizes: "[1,2,4,8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256]" kv_cache_dtype: fp8 gpu-memory-utilization: 0.85 max-num-seqs: 128 @@ -69,6 +80,13 @@ backend: no-enable_prefix_caching: true trust-remote-code: true +# The legacy exclusive allocation exposed the full 128-core, unlimited-memory +# node; do not inherit the shared small-model profile's 32-core / 128G limits. +sbatch_directives: + cpus-per-task: "128" + mem: "0" + exclusive: "" + srun_options: container-writable: "" container-remap-root: "" @@ -88,7 +106,7 @@ benchmark: tar -C /logs -czf "${result_root}/runtime-logs.tar.gz" . 2>/dev/null || true } trap archive_runtime_logs EXIT - for concurrency in 4 8 16 32 64 128; do + for concurrency in ${CONC_LIST}; do num_prompts=$((concurrency * 10)) if (( num_prompts < 16 )); then num_prompts=16; fi python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ @@ -100,7 +118,7 @@ benchmark: --dataset-name random \ --random-input-len 8192 \ --random-output-len 1024 \ - --random-range-ratio 1.0 \ + --random-range-ratio "${RANDOM_RANGE_RATIO:-0.8}" \ --num-warmups "$((concurrency * 2))" \ --num-prompts "${num_prompts}" \ --max-concurrency "${concurrency}" \ @@ -113,6 +131,4 @@ benchmark: --result-filename "dsv4-atom-mi355x-disagg-1p1d-tp8-isl8192-osl1024-c${concurrency}.json" done env: - HF_HOME: /hf_hub_cache - HF_HUB_CACHE: /hf_hub_cache/hub - HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub + <<: *runtime_environment diff --git a/benchmarks/multi_node/srt-slurm-recipes/atom/deepseek-v4/mi355x/disagg-2p1d-dpa-tp8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/atom/deepseek-v4/mi355x/disagg-2p1d-dpa-tp8-8k1k.yaml index d062929763..753096ed0e 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/atom/deepseek-v4/mi355x/disagg-2p1d-dpa-tp8-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/atom/deepseek-v4/mi355x/disagg-2p1d-dpa-tp8-8k1k.yaml @@ -6,17 +6,16 @@ name: "mi355x-atom-dsv4-disagg-2p1d-dpa-tp8-8k1k" model: path: "hf:deepseek-ai/DeepSeek-V4-Pro" - container: "atom-nightly-202608251555" + container: "rocm/atom-dev:nightly_202606101403" precision: "fp4" identity: model: repo: "deepseek-ai/DeepSeek-V4-Pro" container: - image: "rocm/atom-dev:nightly_202608251555" + image: "rocm/atom-dev:nightly_202606101403" frameworks: - atom: "0.1.6rc1.dev336+g4e0848dcd" - atomesh: "4e0848dcdf94f3d4be0c1e3ce51db99bc93f8b8c" + atomesh: "087b82d9c1f630e79149ba37e6213257ec9a76f8" slurm: time_limit: "08:00:00" @@ -36,17 +35,30 @@ frontend: enable_multiple_frontends: false args: policy: random + log-level: info + disable-health-check: true + disable-circuit-breaker: true + prometheus-port: 29100 + +environment: &runtime_environment + HF_HOME: "/hf_hub_cache" + HF_HUB_CACHE: "/hf_hub_cache/hub" + HUGGINGFACE_HUB_CACHE: "/hf_hub_cache/hub" + # Same Mooncake/ROCm lookup order as env_atom.sh in the original image. + LD_LIBRARY_PATH: "/opt/venv/lib/python3.10/site-packages/mooncake:/opt/rocm/lib" + PYTHONUNBUFFERED: "1" + PYTHONDONTWRITEBYTECODE: "1" + SAFETENSORS_FAST_GPU: "1" + VLLM_LOG_LEVEL: "WARNING" + ATOM_LOG_LEVEL: "WARNING" + AITER_LOG_LEVEL: "WARNING" + LOG_LEVEL: "WARNING" + LOGLEVEL: "WARNING" backend: type: atom connector: mooncake - mooncake_protocol: tcp prefill_environment: &worker_environment - HF_HOME: "/hf_hub_cache" - HF_HUB_CACHE: "/hf_hub_cache/hub" - HUGGINGFACE_HUB_CACHE: "/hf_hub_cache/hub" - PYTHONUNBUFFERED: "1" - OMP_NUM_THREADS: "1" ATOM_MOE_GU_ITLV: "1" AITER_BF16_FP8_MOE_BOUND: "0" GPU_MAX_HW_QUEUES: "5" @@ -63,6 +75,7 @@ backend: enable-tbo: true trust-remote-code: true decode: + cudagraph-capture-sizes: "[1,2,4,8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256]" kv_cache_dtype: fp8 gpu-memory-utilization: 0.85 max-num-seqs: 2048 @@ -72,6 +85,13 @@ backend: enable-tbo: true trust-remote-code: true +# The legacy exclusive allocation exposed the full 128-core, unlimited-memory +# node; do not inherit the shared small-model profile's 32-core / 128G limits. +sbatch_directives: + cpus-per-task: "128" + mem: "0" + exclusive: "" + srun_options: container-writable: "" container-remap-root: "" @@ -91,7 +111,7 @@ benchmark: tar -C /logs -czf "${result_root}/runtime-logs.tar.gz" . 2>/dev/null || true } trap archive_runtime_logs EXIT - for concurrency in 256 512 768 1024 2048; do + for concurrency in ${CONC_LIST}; do num_prompts=$((concurrency * 10)) python3 /infmax-workspace/utils/bench_serving/benchmark_serving.py \ --backend openai \ @@ -102,7 +122,7 @@ benchmark: --dataset-name random \ --random-input-len 8192 \ --random-output-len 1024 \ - --random-range-ratio 1.0 \ + --random-range-ratio "${RANDOM_RANGE_RATIO:-0.8}" \ --num-warmups "$((concurrency * 2))" \ --num-prompts "${num_prompts}" \ --max-concurrency "${concurrency}" \ @@ -115,6 +135,4 @@ benchmark: --result-filename "dsv4-atom-mi355x-disagg-2p1d-dpa-tp8-isl8192-osl1024-c${concurrency}.json" done env: - HF_HOME: /hf_hub_cache - HF_HUB_CACHE: /hf_hub_cache/hub - HUGGINGFACE_HUB_CACHE: /hf_hub_cache/hub + <<: *runtime_environment diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 94f6f6127b..5b3c566a6a 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -799,7 +799,7 @@ dsv4-fp4-mi355x-atom-disagg: image: rocm/atom-dev:nightly_202606101403 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 - runner: mi355x + runner: cluster:mi355x-amds precision: fp4 framework: atom-disagg router: { name: atomesh, version: "087b82d9c1f630e79149ba37e6213257ec9a76f8" } @@ -820,6 +820,7 @@ dsv4-fp4-mi355x-atom-disagg: dp-attn: true additional-settings: - "PREFILL_NODES=2" + - "CONFIG_FILE=recipes/atom/deepseek-v4/mi355x/disagg-2p1d-dpa-tp8-8k1k.yaml" decode: num-worker: 1 tp: 8 @@ -836,6 +837,7 @@ dsv4-fp4-mi355x-atom-disagg: dp-attn: false additional-settings: - "PREFILL_NODES=1" + - "CONFIG_FILE=recipes/atom/deepseek-v4/mi355x/disagg-1p1d-tp8-8k1k.yaml" decode: num-worker: 1 tp: 8 @@ -1063,58 +1065,3 @@ qwen3-0.6b-fp16-mi355x-sglang-srt-disagg: # Production-scale migration of the existing Qwen3.5 FP8 MI355X P/D lane to # srt-slurm. One full TP8 node serves prefill, one TP8 node serves decode, and # the request/KV planes remain native SGLang Router + AMD MoRI. - -dsv4-fp4-mi355x-atom-disagg: - image: rocm/atom-dev:nightly_202608251555 - model: deepseek-ai/DeepSeek-V4-Pro - model-prefix: dsv4 - runner: cluster:mi355x-amds - precision: fp4 - framework: atom-disagg - router: { name: atomesh, version: "4e0848dcdf94f3d4be0c1e3ce51db99bc93f8b8c" } - kv-p2p-transfer: mooncake - multinode: true - disagg: true - scenarios: - fixed-seq-len: - - isl: 8192 - osl: 1024 - search-space: - # The custom srt-slurm recipes own the exact concurrency loops. Keeping - # one matrix row per topology avoids launching the complete loop once per - # concurrency while preserving every original measurement point. - # 2P1D DPA+TP8 - - conc-list: [ 1 ] - prefill: - num-worker: 2 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - - "PREFILL_NODES=2" - - "CONFIG_FILE=recipes/atom/deepseek-v4/mi355x/disagg-2p1d-dpa-tp8-8k1k.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: true - additional-settings: - - "DECODE_NODES=1" - # 1P1D TP8 - - conc-list: [ 1 ] - prefill: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "PREFILL_NODES=1" - - "CONFIG_FILE=recipes/atom/deepseek-v4/mi355x/disagg-1p1d-tp8-8k1k.yaml" - decode: - num-worker: 1 - tp: 8 - ep: 1 - dp-attn: false - additional-settings: - - "DECODE_NODES=1" - # 1P1D TP8 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index fcda6d68eb..b978966ed1 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6854,8 +6854,7 @@ - "Port the DeepSeek-V4-Pro FP4 MI355X ATOM disaggregated 8k/1k submission from the configuration-specific AMD launcher to srt-slurm ATOM orchestration and native AToMesh routing." - "Preserve all original search points: 1P1D TP8 at concurrency 4, 8, 16, 32, 64, and 128; 2P1D DPA+TP8 at concurrency 256, 512, 768, 1024, and 2048." - "Use the unchanged InferenceX benchmark_serving.py through the custom benchmark contract and remove the exact legacy wrapper plus its obsolete deprecated 1k/1k entry." - - "Use the hardware-proven Mooncake TCP transport for the initial correctness sweep; validate the updated ROCm RDMA path separately before changing production transport." + - "Keep the original image, Mooncake default RDMA transport, random length ratio 0.8, decode graph capture sizes, router options, and TP8 / DP-attention / TBO settings." - "Use the ATOM image's bundled native AToMesh binary directly, without Infera source overlays or Infera routing, and archive the complete runtime logs from the custom benchmark." - - "Run the current rocm/atom-dev nightly_202608251555 image at ATOM commit 4e0848dcdf94f3d4be0c1e3ce51db99bc93f8b8c, whose bundled native AToMesh and Mooncake implementation are used without downstream source patches." - - "Pin SemiAnalysisAI/srt-slurm PR #7 at c87d7b34b009be920896126013ad6dc74c5a99d5 so this ATOM/AToMesh port shares the latest validated AMD orchestration base, including native-router data-parallel rank preservation and ATOM recipe flag normalization." + - "Keep rocm/atom-dev:nightly_202606101403 and the original AToMesh revision 087b82d9c1f630e79149ba37e6213257ec9a76f8; use the shared launcher and srt-slurm PR #7 from base PR #2542." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2627 From 1fb90feab35c283a4a2bc780eedf2e3e6d751b39 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Wed, 2 Sep 2026 13:13:56 -0500 Subject: [PATCH 4/5] fix(atom): preserve workflow-derived context limit --- .../atom/deepseek-v4/mi355x/disagg-1p1d-tp8-8k1k.yaml | 3 +++ .../atom/deepseek-v4/mi355x/disagg-2p1d-dpa-tp8-8k1k.yaml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/benchmarks/multi_node/srt-slurm-recipes/atom/deepseek-v4/mi355x/disagg-1p1d-tp8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/atom/deepseek-v4/mi355x/disagg-1p1d-tp8-8k1k.yaml index e735952b9f..97aa5452f6 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/atom/deepseek-v4/mi355x/disagg-1p1d-tp8-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/atom/deepseek-v4/mi355x/disagg-1p1d-tp8-8k1k.yaml @@ -65,6 +65,8 @@ backend: decode_environment: *worker_environment atom_config: prefill: + # Upstream workflow MAX_MODEL_LEN = ISL + OSL + 256. + max-model-len: 9472 kv_cache_dtype: fp8 gpu-memory-utilization: 0.85 max-num-seqs: 256 @@ -72,6 +74,7 @@ backend: no-enable_prefix_caching: true trust-remote-code: true decode: + max-model-len: 9472 cudagraph-capture-sizes: "[1,2,4,8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256]" kv_cache_dtype: fp8 gpu-memory-utilization: 0.85 diff --git a/benchmarks/multi_node/srt-slurm-recipes/atom/deepseek-v4/mi355x/disagg-2p1d-dpa-tp8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/atom/deepseek-v4/mi355x/disagg-2p1d-dpa-tp8-8k1k.yaml index 753096ed0e..d1a1dbc11c 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/atom/deepseek-v4/mi355x/disagg-2p1d-dpa-tp8-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/atom/deepseek-v4/mi355x/disagg-2p1d-dpa-tp8-8k1k.yaml @@ -66,6 +66,8 @@ backend: decode_environment: *worker_environment atom_config: prefill: + # Upstream workflow MAX_MODEL_LEN = ISL + OSL + 256. + max-model-len: 9472 kv_cache_dtype: fp8 gpu-memory-utilization: 0.85 max-num-seqs: 256 @@ -75,6 +77,7 @@ backend: enable-tbo: true trust-remote-code: true decode: + max-model-len: 9472 cudagraph-capture-sizes: "[1,2,4,8,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256]" kv_cache_dtype: fp8 gpu-memory-utilization: 0.85 From b1fc78055a73514fb828b746ad982a48226a8756 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Thu, 3 Sep 2026 20:15:54 -0500 Subject: [PATCH 5/5] fix(atom): use the official nightly with native token-limit support --- .../atom/deepseek-v4/mi355x/disagg-1p1d-tp8-8k1k.yaml | 9 ++++----- .../deepseek-v4/mi355x/disagg-2p1d-dpa-tp8-8k1k.yaml | 9 ++++----- configs/amd-master.yaml | 4 ++-- perf-changelog.yaml | 9 +++++++++ 4 files changed, 19 insertions(+), 12 deletions(-) diff --git a/benchmarks/multi_node/srt-slurm-recipes/atom/deepseek-v4/mi355x/disagg-1p1d-tp8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/atom/deepseek-v4/mi355x/disagg-1p1d-tp8-8k1k.yaml index 97aa5452f6..f3cd36eb02 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/atom/deepseek-v4/mi355x/disagg-1p1d-tp8-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/atom/deepseek-v4/mi355x/disagg-1p1d-tp8-8k1k.yaml @@ -7,16 +7,16 @@ name: "mi355x-atom-dsv4-disagg-1p1d-tp8-8k1k" model: path: "hf:deepseek-ai/DeepSeek-V4-Pro" - container: "rocm/atom-dev:nightly_202606101403" + container: "rocm/atom-dev:nightly_202609031453" precision: "fp4" identity: model: repo: "deepseek-ai/DeepSeek-V4-Pro" container: - image: "rocm/atom-dev:nightly_202606101403" + image: "rocm/atom-dev:nightly_202609031453" frameworks: - atomesh: "087b82d9c1f630e79149ba37e6213257ec9a76f8" + atomesh: "f95ef3ec30a16876a676e0af96d60f8e0605c7c5" slurm: time_limit: "08:00:00" @@ -45,8 +45,7 @@ environment: &runtime_environment HF_HOME: "/hf_hub_cache" HF_HUB_CACHE: "/hf_hub_cache/hub" HUGGINGFACE_HUB_CACHE: "/hf_hub_cache/hub" - # Same Mooncake/ROCm lookup order as env_atom.sh in the original image. - LD_LIBRARY_PATH: "/opt/venv/lib/python3.10/site-packages/mooncake:/opt/rocm/lib" + # Use the official image's ROCm/Mooncake library paths (Python 3.12). PYTHONUNBUFFERED: "1" PYTHONDONTWRITEBYTECODE: "1" SAFETENSORS_FAST_GPU: "1" diff --git a/benchmarks/multi_node/srt-slurm-recipes/atom/deepseek-v4/mi355x/disagg-2p1d-dpa-tp8-8k1k.yaml b/benchmarks/multi_node/srt-slurm-recipes/atom/deepseek-v4/mi355x/disagg-2p1d-dpa-tp8-8k1k.yaml index d1a1dbc11c..22c7ad775b 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/atom/deepseek-v4/mi355x/disagg-2p1d-dpa-tp8-8k1k.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/atom/deepseek-v4/mi355x/disagg-2p1d-dpa-tp8-8k1k.yaml @@ -6,16 +6,16 @@ name: "mi355x-atom-dsv4-disagg-2p1d-dpa-tp8-8k1k" model: path: "hf:deepseek-ai/DeepSeek-V4-Pro" - container: "rocm/atom-dev:nightly_202606101403" + container: "rocm/atom-dev:nightly_202609031453" precision: "fp4" identity: model: repo: "deepseek-ai/DeepSeek-V4-Pro" container: - image: "rocm/atom-dev:nightly_202606101403" + image: "rocm/atom-dev:nightly_202609031453" frameworks: - atomesh: "087b82d9c1f630e79149ba37e6213257ec9a76f8" + atomesh: "f95ef3ec30a16876a676e0af96d60f8e0605c7c5" slurm: time_limit: "08:00:00" @@ -44,8 +44,7 @@ environment: &runtime_environment HF_HOME: "/hf_hub_cache" HF_HUB_CACHE: "/hf_hub_cache/hub" HUGGINGFACE_HUB_CACHE: "/hf_hub_cache/hub" - # Same Mooncake/ROCm lookup order as env_atom.sh in the original image. - LD_LIBRARY_PATH: "/opt/venv/lib/python3.10/site-packages/mooncake:/opt/rocm/lib" + # Use the official image's ROCm/Mooncake library paths (Python 3.12). PYTHONUNBUFFERED: "1" PYTHONDONTWRITEBYTECODE: "1" SAFETENSORS_FAST_GPU: "1" diff --git a/configs/amd-master.yaml b/configs/amd-master.yaml index 5b3c566a6a..3937a8f008 100644 --- a/configs/amd-master.yaml +++ b/configs/amd-master.yaml @@ -796,13 +796,13 @@ dsr1-fp4-mi355x-sglang-disagg-mtp: # https://github.com/vllm-project/recipes/commit/2a3728ed9892debfd767a72a58ebc90b33f186e5 # MXFP8 runs from TP=4 on gfx950; block size 128 is mandatory for MSA. dsv4-fp4-mi355x-atom-disagg: - image: rocm/atom-dev:nightly_202606101403 + image: rocm/atom-dev:nightly_202609031453 model: deepseek-ai/DeepSeek-V4-Pro model-prefix: dsv4 runner: cluster:mi355x-amds precision: fp4 framework: atom-disagg - router: { name: atomesh, version: "087b82d9c1f630e79149ba37e6213257ec9a76f8" } + router: { name: atomesh, version: "f95ef3ec30a16876a676e0af96d60f8e0605c7c5" } kv-p2p-transfer: mooncake multinode: true disagg: true diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 24aee84148..787d1049e0 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -6897,3 +6897,12 @@ - "Use the ATOM image's bundled native AToMesh binary directly, without Infera source overlays or Infera routing, and archive the complete runtime logs from the custom benchmark." - "Keep rocm/atom-dev:nightly_202606101403 and the original AToMesh revision 087b82d9c1f630e79149ba37e6213257ec9a76f8; use the shared launcher and srt-slurm PR #7 from base PR #2542." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2627 + +- config-keys: + - dsv4-fp4-mi355x-atom-disagg + description: + - "Use the unmodified official rocm/atom-dev:nightly_202609031453 image and its native AToMesh revision f95ef3ec30a16876a676e0af96d60f8e0605c7c5; no source patches, overlays, or custom image." + - "The June image ignored max_completion_tokens after AToMesh normalized max_tokens, causing the 2P1D GSM8K decode worker to exceed its context capacity. The upstream ATOM API fix is included in this nightly." + - "Use the image-provided ROCm/Mooncake library paths instead of the retired Python 3.10 path; keep all explicit engine options, topologies, concurrency points, benchmark inputs, and eval thresholds unchanged." + - "Validate the shared single-entrypoint launcher and native Pyxis lifecycle without host-repair, RDMA-preflight, or GPU-drain scripts." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2627