diff --git a/.github/configs/amd-master.yaml b/.github/configs/amd-master.yaml index 3837e5d757..087660dfb4 100644 --- a/.github/configs/amd-master.yaml +++ b/.github/configs/amd-master.yaml @@ -316,9 +316,9 @@ dsr1-fp8-mi355x-sglang-disagg: conc-list: [ 1024, 2048 ] prefill: num-worker: 1 - tp: 1 - ep: 8 - dp-attn: true + tp: 8 + ep: 1 + dp-attn: false additional-settings: - "PREFILL_NODES=1" decode: @@ -335,9 +335,9 @@ dsr1-fp8-mi355x-sglang-disagg: conc-list: [ 1536, 1024, 512, 256 ] prefill: num-worker: 1 - tp: 1 - ep: 8 - dp-attn: true + tp: 8 + ep: 1 + dp-attn: false additional-settings: - "PREFILL_NODES=1" decode: @@ -376,9 +376,9 @@ dsr1-fp8-mi355x-sglang-disagg: conc-list: [ 1024, 2048 ] prefill: num-worker: 1 - tp: 1 - ep: 8 - dp-attn: true + tp: 8 + ep: 1 + dp-attn: false additional-settings: - "PREFILL_NODES=1" decode: @@ -395,9 +395,9 @@ dsr1-fp8-mi355x-sglang-disagg: conc-list: [ 1536, 1024, 512, 256 ] prefill: num-worker: 1 - tp: 1 - ep: 8 - dp-attn: true + tp: 8 + ep: 1 + dp-attn: false additional-settings: - "PREFILL_NODES=1" decode: @@ -434,6 +434,24 @@ dsr1-fp8-mi355x-sglang-disagg: osl: 1024 search-space: # MTP configurations + - spec-decoding: "mtp" + conc-list: [ 512, 1024 ] + prefill: + num-worker: 2 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "PREFILL_NODES=2" + decode: + num-worker: 1 + tp: 1 + ep: 8 + dp-attn: true + additional-settings: + - "DECODE_NODES=1" + - "DECODE_MTP_SIZE=1" + # "Top of curve" (1 prefill worker at DEP8 and 1 decode worker at DEP16) - spec-decoding: "mtp" conc-list: [ 1024, 2048 ] @@ -455,7 +473,7 @@ dsr1-fp8-mi355x-sglang-disagg: # "Middle of curve" (1 prefill worker at DEP8 and 2 decode workers each at DEP8) - spec-decoding: "mtp" - conc-list: [ 1536, 1024, 512, 256 ] + conc-list: [ 1536, 1024 ] prefill: num-worker: 1 tp: 1 @@ -494,6 +512,24 @@ dsr1-fp8-mi355x-sglang-disagg: - "DECODE_MTP_SIZE=2" # non-MTP configurations + - spec-decoding: "none" + conc-list: [ 512, 1024 ] + prefill: + num-worker: 2 + tp: 8 + ep: 1 + dp-attn: false + additional-settings: + - "PREFILL_NODES=2" + decode: + num-worker: 1 + tp: 1 + ep: 8 + dp-attn: true + additional-settings: + - "DECODE_NODES=1" + - "DECODE_MTP_SIZE=0" + # "Top of curve" (1 prefill workers each at DEP8 and 1 decode workers at DEP16) - spec-decoding: "none" conc-list: [ 1024, 2048 ] @@ -515,7 +551,7 @@ dsr1-fp8-mi355x-sglang-disagg: # "Middle of curve" (1 prefill workers each at DEP8 and 2 decode workers at DEP8) - spec-decoding: "none" - conc-list: [ 1536, 1024, 512, 256 ] + conc-list: [ 1536, 1024 ] prefill: num-worker: 1 tp: 1 diff --git a/benchmarks/dsr1_fp8_mi355x_sglang-disagg.sh b/benchmarks/dsr1_fp8_mi355x_sglang-disagg.sh index b748957154..c36a24a599 100644 --- a/benchmarks/dsr1_fp8_mi355x_sglang-disagg.sh +++ b/benchmarks/dsr1_fp8_mi355x_sglang-disagg.sh @@ -29,7 +29,7 @@ fi set -x AMD_DISAGG_REPO="https://github.com/billishyahao/sglang_disagg.git" -COMMIT_ID="21c60a1f979f78e0459913bc37dd88a19344cea8" +COMMIT_ID="953f7c512b713683ac3a7182f78f393dcdf3768c" git clone ${AMD_DISAGG_REPO} \ && cd sglang_disagg \ diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 08da6aa877..1c8a4285c4 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -621,3 +621,12 @@ description: - "Add min-latency configurations for H200 Dynamo TRT" pr-link: https://github.com/InferenceMAX/InferenceMAX/pull/698 + +- config-keys: + - dsr1-fp8-mi355x-sglang-disagg + description: + - "Bump MI355X disagg FP8 recipe commit to 953f7c5 (bug fixes in sglang_disagg fork)" + - "1k1k: Switch prefill workers from DEP (tp:1, ep:8, dp-attn) to Pure TP (tp:8, ep:1) for MTP and non-MTP middle-of-curve 1P2D configs" + - "1k1k: Extend middle-of-curve concurrency range by adding conc=128 for both MTP and non-MTP 1P2D configs" + - "8k1k: Add new 2P1D (2-prefill, 1-decode) configs at conc [512, 1024] for both MTP (DECODE_MTP_SIZE=0) and non-MTP, with Pure TP prefill and DEP decode" + pr-link: https://github.com/InferenceMAX/InferenceMAX/pull/700