From 13dd1d49f8f2a73f3e1f1d644114525b4c15ed59 Mon Sep 17 00:00:00 2001 From: seungrokj Date: Mon, 23 Mar 2026 13:56:40 +0000 Subject: [PATCH 1/4] slurm command fix Signed-off-by: seungrokj --- runners/launch_mi300x-amds.sh | 2 +- runners/launch_mi325x-amd.sh | 2 +- runners/launch_mi355x-amds.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/runners/launch_mi300x-amds.sh b/runners/launch_mi300x-amds.sh index 4faf073386..0e47384ab5 100644 --- a/runners/launch_mi300x-amds.sh +++ b/runners/launch_mi300x-amds.sh @@ -9,7 +9,7 @@ LOCK_FILE="${SQUASH_FILE}.lock" set -x -JOB_ID=$(salloc --partition=$PARTITION --gres=gpu:$TP --cpus-per-task=256 --time=180 --no-shell --job-name="$RUNNER_NAME" 2>&1 | tee /dev/stderr | grep -oP 'Granted job allocation \K[0-9]+') +JOB_ID=$(salloc --partition=$PARTITION --gres=gpu:$TP --exclusive --cpus-per-task=256 --time=180 --no-shell --job-name="$RUNNER_NAME" 2>&1 | tee /dev/stderr | grep -oP 'Granted job allocation \K[0-9]+') if [ -z "$JOB_ID" ]; then echo "ERROR: salloc failed to allocate a job" diff --git a/runners/launch_mi325x-amd.sh b/runners/launch_mi325x-amd.sh index 67f93a309f..ede4eef6ca 100644 --- a/runners/launch_mi325x-amd.sh +++ b/runners/launch_mi325x-amd.sh @@ -9,7 +9,7 @@ LOCK_FILE="${SQUASH_FILE}.lock" set -x -JOB_ID=$(salloc --partition=$PARTITION --gres=gpu:$TP --cpus-per-task=256 --time=480 --no-shell --job-name="$RUNNER_NAME" 2>&1 | tee /dev/stderr | grep -oP 'Granted job allocation \K[0-9]+') +JOB_ID=$(salloc --partition=$PARTITION --gres=gpu:$TP --exclusive --cpus-per-task=256 --time=480 --no-shell --job-name="$RUNNER_NAME" 2>&1 | tee /dev/stderr | grep -oP 'Granted job allocation \K[0-9]+') if [ -z "$JOB_ID" ]; then echo "ERROR: salloc failed to allocate a job" diff --git a/runners/launch_mi355x-amds.sh b/runners/launch_mi355x-amds.sh index fc04f5bb32..52e28e9b87 100644 --- a/runners/launch_mi355x-amds.sh +++ b/runners/launch_mi355x-amds.sh @@ -159,7 +159,7 @@ else LOCK_FILE="${SQUASH_FILE}.lock" set -x - salloc --partition=$PARTITION --gres=gpu:$TP --cpus-per-task=128 --time=180 --no-shell --job-name="$RUNNER_NAME" + salloc --partition=$PARTITION --gres=gpu:$TP --exclusive --cpus-per-task=128 --time=180 --no-shell --job-name="$RUNNER_NAME" JOB_ID=$(squeue --name="$RUNNER_NAME" -h -o %A | head -n1) srun --jobid=$JOB_ID bash -c "docker stop \$(docker ps -a -q)" From e4f812f14550fe79f174c9ae86c8a808c55ae879 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 23 Mar 2026 09:14:14 -0500 Subject: [PATCH 2/4] update perf changelog --- perf-changelog.yaml | 7 +++++++ runners/launch_mi300x-amds.sh | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index bf9df36148..f28e47a4f9 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -1007,3 +1007,10 @@ - "EAGLE speculative decoding: num-steps 3, draft-tokens 4, topk 1" - "New script: benchmarks/single_node/qwen3.5_fp8_b200_mtp.sh" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/898 + +- config-keys: + - "*mi355x*" + - "*mi325x*" + description: + - "Add --exclusive option to salloc commands on MI325X and MI355X nodes which may yield performance improvements" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/929 diff --git a/runners/launch_mi300x-amds.sh b/runners/launch_mi300x-amds.sh index 0e47384ab5..4faf073386 100644 --- a/runners/launch_mi300x-amds.sh +++ b/runners/launch_mi300x-amds.sh @@ -9,7 +9,7 @@ LOCK_FILE="${SQUASH_FILE}.lock" set -x -JOB_ID=$(salloc --partition=$PARTITION --gres=gpu:$TP --exclusive --cpus-per-task=256 --time=180 --no-shell --job-name="$RUNNER_NAME" 2>&1 | tee /dev/stderr | grep -oP 'Granted job allocation \K[0-9]+') +JOB_ID=$(salloc --partition=$PARTITION --gres=gpu:$TP --cpus-per-task=256 --time=180 --no-shell --job-name="$RUNNER_NAME" 2>&1 | tee /dev/stderr | grep -oP 'Granted job allocation \K[0-9]+') if [ -z "$JOB_ID" ]; then echo "ERROR: salloc failed to allocate a job" From 26574896bf57a4dc4211ebf5938a0abfd02f2cb8 Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 23 Mar 2026 09:21:21 -0500 Subject: [PATCH 3/4] add --exclusive to mi355x multinode --- benchmarks/multi_node/amd_utils/submit.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/benchmarks/multi_node/amd_utils/submit.sh b/benchmarks/multi_node/amd_utils/submit.sh index 44bd130f30..8021063501 100755 --- a/benchmarks/multi_node/amd_utils/submit.sh +++ b/benchmarks/multi_node/amd_utils/submit.sh @@ -129,6 +129,7 @@ fi sbatch_cmd=( sbatch --parsable + --exclusive -N "$NUM_NODES" -n "$NUM_NODES" "${NODELIST_OPT[@]}" From 3328ac68c672aed442efda842d8b505c021fc2be Mon Sep 17 00:00:00 2001 From: Cam Quilici Date: Mon, 23 Mar 2026 09:57:09 -0500 Subject: [PATCH 4/4] fix: add --exclusive to MI355X multi-node sbatch for accurate benchmarks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Without --exclusive, SLURM can co-schedule other jobs on the same nodes, causing memory bandwidth, XGMI/Infinity Fabric, and NUMA contention that degrades benchmark results — especially for non-TP8 disaggregated serving configs where cross-node RDMA traffic is latency-sensitive. Co-Authored-By: Claude Opus 4.6 (1M context) --- perf-changelog.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/perf-changelog.yaml b/perf-changelog.yaml index f28e47a4f9..f616949dd8 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -1,3 +1,20 @@ +- config-keys: + - dsr1-fp4-mi355x-atom + - dsr1-fp4-mi355x-atom-mtp + - dsr1-fp4-mi355x-sglang + - dsr1-fp4-mi355x-sglang-disagg + - dsr1-fp4-mi355x-sglang-disagg-mtp + - dsr1-fp8-mi355x-sglang + - dsr1-fp8-mi355x-sglang-disagg + - dsr1-fp8-mi355x-sglang-disagg-mtp + - gptoss-fp4-mi355x-atom + - gptoss-fp4-mi355x-vllm + - minimaxm2.5-fp8-mi355x-vllm + description: + - "Add --exclusive flag to MI355X multi-node sbatch to prevent node sharing during benchmarks" + - "Only non-TP8 configs are affected; TP8 already uses all GPUs on the node" + pr-link: TBD + - config-keys: - dsr1-fp8-b200-dynamo-trt - dsr1-fp8-h200-dynamo-trt