diff --git a/benchmarks/multi_node/amd_utils/submit.sh b/benchmarks/multi_node/amd_utils/submit.sh index 44bd130f30..8021063501 100755 --- a/benchmarks/multi_node/amd_utils/submit.sh +++ b/benchmarks/multi_node/amd_utils/submit.sh @@ -129,6 +129,7 @@ fi sbatch_cmd=( sbatch --parsable + --exclusive -N "$NUM_NODES" -n "$NUM_NODES" "${NODELIST_OPT[@]}" diff --git a/perf-changelog.yaml b/perf-changelog.yaml index bf9df36148..f616949dd8 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -1,3 +1,20 @@ +- config-keys: + - dsr1-fp4-mi355x-atom + - dsr1-fp4-mi355x-atom-mtp + - dsr1-fp4-mi355x-sglang + - dsr1-fp4-mi355x-sglang-disagg + - dsr1-fp4-mi355x-sglang-disagg-mtp + - dsr1-fp8-mi355x-sglang + - dsr1-fp8-mi355x-sglang-disagg + - dsr1-fp8-mi355x-sglang-disagg-mtp + - gptoss-fp4-mi355x-atom + - gptoss-fp4-mi355x-vllm + - minimaxm2.5-fp8-mi355x-vllm + description: + - "Add --exclusive flag to MI355X multi-node sbatch to prevent node sharing during benchmarks" + - "Only non-TP8 configs are affected; TP8 already uses all GPUs on the node" + pr-link: TBD + - config-keys: - dsr1-fp8-b200-dynamo-trt - dsr1-fp8-h200-dynamo-trt @@ -1007,3 +1024,10 @@ - "EAGLE speculative decoding: num-steps 3, draft-tokens 4, topk 1" - "New script: benchmarks/single_node/qwen3.5_fp8_b200_mtp.sh" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/898 + +- config-keys: + - "*mi355x*" + - "*mi325x*" + description: + - "Add --exclusive option to salloc commands on MI325X and MI355X nodes which may yield performance improvements" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/929 diff --git a/runners/launch_mi325x-amd.sh b/runners/launch_mi325x-amd.sh index 67f93a309f..ede4eef6ca 100644 --- a/runners/launch_mi325x-amd.sh +++ b/runners/launch_mi325x-amd.sh @@ -9,7 +9,7 @@ LOCK_FILE="${SQUASH_FILE}.lock" set -x -JOB_ID=$(salloc --partition=$PARTITION --gres=gpu:$TP --cpus-per-task=256 --time=480 --no-shell --job-name="$RUNNER_NAME" 2>&1 | tee /dev/stderr | grep -oP 'Granted job allocation \K[0-9]+') +JOB_ID=$(salloc --partition=$PARTITION --gres=gpu:$TP --exclusive --cpus-per-task=256 --time=480 --no-shell --job-name="$RUNNER_NAME" 2>&1 | tee /dev/stderr | grep -oP 'Granted job allocation \K[0-9]+') if [ -z "$JOB_ID" ]; then echo "ERROR: salloc failed to allocate a job" diff --git a/runners/launch_mi355x-amds.sh b/runners/launch_mi355x-amds.sh index fc04f5bb32..52e28e9b87 100644 --- a/runners/launch_mi355x-amds.sh +++ b/runners/launch_mi355x-amds.sh @@ -159,7 +159,7 @@ else LOCK_FILE="${SQUASH_FILE}.lock" set -x - salloc --partition=$PARTITION --gres=gpu:$TP --cpus-per-task=128 --time=180 --no-shell --job-name="$RUNNER_NAME" + salloc --partition=$PARTITION --gres=gpu:$TP --exclusive --cpus-per-task=128 --time=180 --no-shell --job-name="$RUNNER_NAME" JOB_ID=$(squeue --name="$RUNNER_NAME" -h -o %A | head -n1) srun --jobid=$JOB_ID bash -c "docker stop \$(docker ps -a -q)"