diff --git a/.github/configs/amd-master.yaml b/.github/configs/amd-master.yaml index 8ebc249e07..447b52e21b 100644 --- a/.github/configs/amd-master.yaml +++ b/.github/configs/amd-master.yaml @@ -46,6 +46,30 @@ dsr1-fp4-mi355x-atom: - { tp: 4, ep: 1, conc-start: 4, conc-end: 128 } - { tp: 8, ep: 1, conc-start: 4, conc-end: 4 } +dsr1-fp4-mi355x-atom-mtp: + image: rocm/atom:rocm7.2.0-ubuntu24.04-pytorch2.9-atom0.1.1 + model: amd/DeepSeek-R1-0528-MXFP4 + model-prefix: dsr1 + runner: mi355x + precision: fp4 + # WIP framwork (no customers yet) + framework: atom + multinode: false + seq-len-configs: + - isl: 1024 + osl: 1024 + search-space: + - { tp: 4, conc-start: 4, conc-end: 256, spec-decoding: mtp } + - { tp: 8, conc-start: 4, conc-end: 256, spec-decoding: mtp } + - isl: 1024 + osl: 8192 + search-space: + - { tp: 8, conc-start: 256, conc-end: 256, spec-decoding: mtp } + - isl: 8192 + osl: 1024 + search-space: + - { tp: 8, conc-start: 4, conc-end: 256, spec-decoding: mtp } + dsr1-fp8-mi300x-sglang: image: lmsysorg/sglang:v0.5.7-rocm700-mi30x model: deepseek-ai/DeepSeek-R1-0528 @@ -250,6 +274,29 @@ dsr1-fp8-mi355x-atom: search-space: - { tp: 8, conc-start: 4, conc-end: 128 } +dsr1-fp8-mi355x-atom-mtp: + image: rocm/atom:rocm7.2.0-ubuntu24.04-pytorch2.9-atom0.1.1 + model: deepseek-ai/DeepSeek-R1-0528 + model-prefix: dsr1 + runner: mi355x + precision: fp8 + # WIP framwork (no customers yet) + framework: atom + multinode: false + seq-len-configs: + - isl: 1024 + osl: 1024 + search-space: + - { tp: 8, conc-start: 4, conc-end: 256, spec-decoding: mtp } + - isl: 1024 + osl: 8192 + search-space: + - { tp: 8, conc-start: 4, conc-end: 256, spec-decoding: mtp } + - isl: 8192 + osl: 1024 + search-space: + - { tp: 8, conc-start: 4, conc-end: 256, spec-decoding: mtp } + dsr1-fp8-mi355x-sglang-disagg: image: rocm/sgl-dev:sglang-0.5.6.post1-rocm700-mi35x-mori-0113 model: deepseek-ai/DeepSeek-R1-0528 diff --git a/benchmarks/dsr1_fp4_mi355x_atom_mtp.sh b/benchmarks/dsr1_fp4_mi355x_atom_mtp.sh new file mode 100644 index 0000000000..0f5695e686 --- /dev/null +++ b/benchmarks/dsr1_fp4_mi355x_atom_mtp.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash + +source "$(dirname "$0")/benchmark_lib.sh" + +check_env_vars \ + MODEL \ + TP \ + CONC \ + ISL \ + OSL \ + RANDOM_RANGE_RATIO \ + RESULT_FILENAME \ + EP_SIZE \ + DP_ATTENTION + +if [[ -n "$SLURM_JOB_ID" ]]; then + echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" +fi + +echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" + +SERVER_LOG=/workspace/server.log +PORT=${PORT:-8888} + +export OMP_NUM_THREADS=1 + +# Calculate max-model-len based on ISL and OSL +if [ "$ISL" = "1024" ] && [ "$OSL" = "1024" ]; then + CALCULATED_MAX_MODEL_LEN="" +else + CALCULATED_MAX_MODEL_LEN=" --max-model-len 10240 " +fi + +if [ "$EP_SIZE" -gt 1 ]; then + EP=" --enable-expert-parallel" +else + EP=" " +fi + +set -x + +export AMDGCN_USE_BUFFER_OPS=1 + +python3 -m atom.entrypoints.openai_server \ + --model $MODEL \ + --server-port $PORT \ + -tp $TP \ + --kv_cache_dtype fp8 $CALCULATED_MAX_MODEL_LEN $EP \ + --method mtp \ + > $SERVER_LOG 2>&1 & + +SERVER_PID=$! + +# Wait for server to be ready +wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" + +run_benchmark_serving \ + --model "$MODEL" \ + --port "$PORT" \ + --backend vllm \ + --input-len "$ISL" \ + --output-len "$OSL" \ + --random-range-ratio "$RANDOM_RANGE_RATIO" \ + --num-prompts "$((CONC * 10))" \ + --max-concurrency "$CONC" \ + --result-filename "$RESULT_FILENAME" \ + --result-dir /workspace/ \ + --use-chat-template + +# After throughput, run evaluation only if RUN_EVAL is true +if [ "${RUN_EVAL}" = "true" ]; then + run_eval --framework lm-eval --port "$PORT" --concurrent-requests $CONC + append_lm_eval_summary +fi +set +x diff --git a/benchmarks/dsr1_fp8_mi355x_atom_mtp.sh b/benchmarks/dsr1_fp8_mi355x_atom_mtp.sh new file mode 100644 index 0000000000..d85eb31828 --- /dev/null +++ b/benchmarks/dsr1_fp8_mi355x_atom_mtp.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash + +source "$(dirname "$0")/benchmark_lib.sh" + +check_env_vars \ + MODEL \ + TP \ + CONC \ + ISL \ + OSL \ + RANDOM_RANGE_RATIO \ + RESULT_FILENAME \ + EP_SIZE \ + DP_ATTENTION + +if [[ -n "$SLURM_JOB_ID" ]]; then + echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" +fi + +echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL, EP_SIZE: $EP_SIZE, DP_ATTENTION: $DP_ATTENTION" + +SERVER_LOG=/workspace/server.log +PORT=${PORT:-8888} + +export OMP_NUM_THREADS=1 + +# Calculate max-model-len based on ISL and OSL +if [ "$ISL" = "1024" ] && [ "$OSL" = "1024" ]; then + CALCULATED_MAX_MODEL_LEN="" +else + CALCULATED_MAX_MODEL_LEN=" --max-model-len 10240 " +fi + +if [ "$EP_SIZE" -gt 1 ]; then + EP=" --enable-expert-parallel" +else + EP=" " +fi + +set -x + +python3 -m atom.entrypoints.openai_server \ + --model $MODEL \ + --server-port $PORT \ + -tp $TP \ + --kv_cache_dtype fp8 $CALCULATED_MAX_MODEL_LEN $EP \ + --method mtp \ + > $SERVER_LOG 2>&1 & + +SERVER_PID=$! + +# Wait for server to be ready +wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" + +export PYTHONDONTWRITEBYTECODE=1 +run_benchmark_serving \ + --model "$MODEL" \ + --port "$PORT" \ + --backend vllm \ + --input-len "$ISL" \ + --output-len "$OSL" \ + --random-range-ratio "$RANDOM_RANGE_RATIO" \ + --num-prompts "$((CONC * 10))" \ + --max-concurrency "$CONC" \ + --result-filename "$RESULT_FILENAME" \ + --result-dir /workspace/ \ + --use-chat-template + +# After throughput, run evaluation only if RUN_EVAL is true +if [ "${RUN_EVAL}" = "true" ]; then + run_eval --framework lm-eval --port "$PORT" --concurrent-requests $CONC + append_lm_eval_summary +fi +set +x + +set -x +rm -rf ./utils/bench_serving\ diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 4de5a2f3d2..ae58c3c91d 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -491,3 +491,12 @@ - "Image: lmsysorg/sglang:v0.5.8.post1-cu130-amd64" - "9 recipes: 4x 1k1k + 5x 8k1k, low-latency and max-throughput with EAGLE speculative decoding" pr-link: https://github.com/InferenceMAX/InferenceMAX/pull/667 + +- config-keys: + - dsr1-fp8-mi355x-atom-mtp + - dsr1-fp4-mi355x-atom-mtp + description: + - "Add DSR1 FP8/FP4 MI355X ATOM with MTP configuration" + - "Image: rocm/atom:rocm7.2.0-ubuntu24.04-pytorch2.9-atom0.1.1" + - "Deepseek R1 with speculative decoding: 1k1k, 1k8k, 8k1k" + pr-link: https://github.com/InferenceMAX/InferenceMAX/pull/673 diff --git a/runners/launch_mi355x-amds.sh b/runners/launch_mi355x-amds.sh index 2a4759e189..6e687014a7 100644 --- a/runners/launch_mi355x-amds.sh +++ b/runners/launch_mi355x-amds.sh @@ -141,6 +141,7 @@ else export PORT_OFFSET=${RUNNER_NAME: -1} export PORT=$(( 8888 + ${PORT_OFFSET} )) FRAMEWORK_SUFFIX=$([[ "$FRAMEWORK" == "atom" ]] && printf '_atom' || printf '') + SPEC_SUFFIX=$([[ "$SPEC_DECODING" == "mtp" ]] && printf '_mtp' || printf '') PARTITION="compute" SQUASH_FILE="/var/lib/squash/$(echo "$IMAGE" | sed 's/[\/:@#]/_/g').sqsh" @@ -169,7 +170,7 @@ else --container-writable \ --container-workdir=/workspace/ \ --no-container-entrypoint --export=ALL \ - bash benchmarks/${EXP_NAME%%_*}_${PRECISION}_mi355x${FRAMEWORK_SUFFIX}.sh + bash benchmarks/${EXP_NAME%%_*}_${PRECISION}_mi355x${FRAMEWORK_SUFFIX}${SPEC_SUFFIX}.sh scancel $JOB_ID