diff --git a/.github/configs/nvidia-master.yaml b/.github/configs/nvidia-master.yaml index 56147c185a..6c036dce4b 100644 --- a/.github/configs/nvidia-master.yaml +++ b/.github/configs/nvidia-master.yaml @@ -2976,29 +2976,59 @@ dsr1-fp8-h100-dynamo-sglang: - isl: 1024 osl: 1024 search-space: - # Max throughput TEP (1 prefill, 2 decode) - - conc-list: [1, 2, 4, 8, 16, 32, 64, 128] + # # STP: Max throughput TEP (1 prefill, 2 decode) + # - conc-list: [1, 2, 4, 8, 16, 32, 64, 128] + # prefill: + # num-worker: 1 + # tp: 16 + # ep: 1 + # dp-attn: false + # additional-settings: + # - "CONFIG_FILE=recipes/h100/1k1k/stp/h100-fp8-1p2d-max-tp.yaml" + # decode: + # num-worker: 2 + # tp: 16 + # ep: 1 + # dp-attn: false + # # STP: Max throughput DEP (1 prefill, 1 decode, dp-attention) + # - conc-list: [1, 2, 4, 8, 16, 32, 64] + # prefill: + # num-worker: 1 + # tp: 16 + # ep: 1 + # dp-attn: false + # additional-settings: + # - "CONFIG_FILE=recipes/h100/1k1k/stp/h100-fp8-1p1d-max-dep.yaml" + # decode: + # num-worker: 1 + # tp: 16 + # ep: 16 + # dp-attn: true + # MTP: Max throughput TEP (1 prefill, 2 decode) + - spec-decoding: "mtp" + conc-list: [1, 2, 4, 8, 16, 32, 64, 128] prefill: num-worker: 1 tp: 16 ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/h100/1k1k/stp/h100-fp8-1p2d-max-tp.yaml" + - "CONFIG_FILE=recipes/h100/1k1k/mtp/h100-fp8-1p2d-max-tp-mtp.yaml" decode: num-worker: 2 tp: 16 ep: 1 dp-attn: false - # Max throughput DEP (1 prefill, 1 decode, dp-attention) - - conc-list: [1, 2, 4, 8, 16, 32, 64] + # MTP: Max throughput DEP (1 prefill, 1 decode, dp-attention) + - spec-decoding: "mtp" + conc-list: [1, 2, 4, 8, 16, 32, 64] prefill: num-worker: 1 tp: 16 ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/h100/1k1k/stp/h100-fp8-1p1d-max-dep.yaml" + - "CONFIG_FILE=recipes/h100/1k1k/mtp/h100-fp8-1p1d-max-dep-mtp.yaml" decode: num-worker: 1 tp: 16 @@ -3007,29 +3037,59 @@ dsr1-fp8-h100-dynamo-sglang: - isl: 1024 osl: 8192 search-space: - # Max throughput TEP (1 prefill, 2 decode) - - conc-list: [1, 2, 4, 8, 16, 32] + # # STP: Max throughput TEP (1 prefill, 2 decode) + # - conc-list: [1, 2, 4, 8, 16, 32] + # prefill: + # num-worker: 1 + # tp: 16 + # ep: 1 + # dp-attn: false + # additional-settings: + # - "CONFIG_FILE=recipes/h100/1k8k/stp/h100-fp8-1p2d-max-tp.yaml" + # decode: + # num-worker: 2 + # tp: 16 + # ep: 1 + # dp-attn: false + # # STP: Max throughput DEP (1 prefill, 1 decode, dp-attention) + # - conc-list: [1, 2, 4, 8] + # prefill: + # num-worker: 1 + # tp: 16 + # ep: 1 + # dp-attn: false + # additional-settings: + # - "CONFIG_FILE=recipes/h100/1k8k/stp/h100-fp8-1p1d-max-dep.yaml" + # decode: + # num-worker: 1 + # tp: 16 + # ep: 16 + # dp-attn: true + # MTP: Max throughput TEP (1 prefill, 2 decode) + - spec-decoding: "mtp" + conc-list: [1, 2, 4, 8, 16, 32, 64] prefill: num-worker: 1 tp: 16 ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/h100/1k8k/stp/h100-fp8-1p2d-max-tp.yaml" + - "CONFIG_FILE=recipes/h100/1k8k/mtp/h100-fp8-1p2d-max-tp-mtp.yaml" decode: num-worker: 2 tp: 16 ep: 1 dp-attn: false - # Max throughput DEP (1 prefill, 1 decode, dp-attention) - - conc-list: [1, 2, 4, 8] + # MTP: Max throughput DEP (1 prefill, 1 decode, dp-attention) + - spec-decoding: "mtp" + conc-list: [1, 2, 4, 8, 16, 32, 64] prefill: num-worker: 1 tp: 16 ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/h100/1k8k/stp/h100-fp8-1p1d-max-dep.yaml" + - "CONFIG_FILE=recipes/h100/1k8k/mtp/h100-fp8-1p1d-max-dep-mtp.yaml" decode: num-worker: 1 tp: 16 @@ -3038,29 +3098,59 @@ dsr1-fp8-h100-dynamo-sglang: - isl: 8192 osl: 1024 search-space: - # Max throughput TEP (1 prefill, 1 decode) - - conc-list: [1, 2, 4, 8, 16, 32, 64, 128] + # # STP: Max throughput TEP (1 prefill, 1 decode) + # - conc-list: [1, 2, 4, 8, 16, 32, 64, 128] + # prefill: + # num-worker: 1 + # tp: 16 + # ep: 1 + # dp-attn: false + # additional-settings: + # - "CONFIG_FILE=recipes/h100/8k1k/stp/h100-fp8-1p1d-max-tp.yaml" + # decode: + # num-worker: 1 + # tp: 16 + # ep: 1 + # dp-attn: false + # # STP: Max throughput DEP (1 prefill, 1 decode, dp-attention) + # - conc-list: [1, 2, 4, 8, 16, 32, 64] + # prefill: + # num-worker: 1 + # tp: 16 + # ep: 1 + # dp-attn: false + # additional-settings: + # - "CONFIG_FILE=recipes/h100/8k1k/stp/h100-fp8-1p1d-max-dep.yaml" + # decode: + # num-worker: 1 + # tp: 16 + # ep: 16 + # dp-attn: true + # MTP: Max throughput TEP (1 prefill, 1 decode) + - spec-decoding: "mtp" + conc-list: [1, 2, 4, 8, 16, 32, 64, 128] prefill: num-worker: 1 tp: 16 ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/h100/8k1k/stp/h100-fp8-1p1d-max-tp.yaml" + - "CONFIG_FILE=recipes/h100/8k1k/mtp/h100-fp8-1p1d-max-tp-mtp.yaml" decode: num-worker: 1 tp: 16 ep: 1 dp-attn: false - # Max throughput DEP (1 prefill, 1 decode, dp-attention) - - conc-list: [1, 2, 4, 8, 16, 32, 64] + # MTP: Max throughput DEP (1 prefill, 1 decode, dp-attention) + - spec-decoding: "mtp" + conc-list: [1, 2, 4, 8, 16, 32, 64] prefill: num-worker: 1 tp: 16 ep: 1 dp-attn: false additional-settings: - - "CONFIG_FILE=recipes/h100/8k1k/stp/h100-fp8-1p1d-max-dep.yaml" + - "CONFIG_FILE=recipes/h100/8k1k/mtp/h100-fp8-1p1d-max-dep-mtp.yaml" decode: num-worker: 1 tp: 16 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 97228d7bba..524f18cceb 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -494,6 +494,24 @@ - "9 recipes: 4x 1k1k + 5x 8k1k, low-latency and max-throughput with EAGLE speculative decoding" pr-link: https://github.com/InferenceMAX/InferenceMAX/pull/667 +- config-keys: + - dsr1-fp8-h100-dynamo-sglang + description: + - "Add DeepSeek-R1 FP8 H100 Dynamo SGLang STP disaggregated multinode configurations" + - "Image: lmsysorg/sglang:v0.5.8-cu130" + - "1k1k, 1k8k, 8k1k sequence lengths" + - "Two modes per seq-len: Max throughput TEP (1P2D) and Max throughput DEP (1P1D with dp-attention)" + pr-link: https://github.com/InferenceMAX/InferenceMAX/pull/643 + +- config-keys: + - dsr1-fp8-h100-dynamo-sglang + description: + - "Add DeepSeek-R1 FP8 H100 Dynamo SGLang MTP disaggregated multinode configurations" + - "Image: lmsysorg/sglang:v0.5.8-cu130" + - "1k1k, 1k8k, 8k1k sequence lengths with MTP speculative decoding" + - "Two modes per seq-len: Max throughput TEP (1P2D) and Max throughput DEP (1P1D with dp-attention)" + pr-link: https://github.com/InferenceMAX/InferenceMAX/pull/644 + - config-keys: - dsr1-fp8-mi355x-atom-mtp - dsr1-fp4-mi355x-atom-mtp @@ -527,14 +545,14 @@ - "Add conc 4/8/16" - "Use Pure TP with MTP=2 for 1k1k conc smaller than 128 and reduce MTP to 1 for DEP configs" pr-link: https://github.com/InferenceMAX/InferenceMAX/pull/674 - + - config-keys: - dsr1-fp4-b200-dynamo-sglang-mtp description: - "Add B200 configs for DSR1 FP4 SGLANG MTP mode for 1k1k and 8k1k" - "Image: lmsysorg/sglang:v0.5.8.post1-cu130" pr-link: https://github.com/InferenceMAX/InferenceMAX/pull/683 - + - config-keys: - dsr1-fp4-gb300-dynamo-sglang description: @@ -564,14 +582,14 @@ description: - "Add more sweep points for DSR1 FP8 both MTP and non-MTP 1k1k, 8k1k" pr-link: https://github.com/InferenceMAX/InferenceMAX/pull/689 - + - config-keys: - dsr1-fp8-b200-dynamo-trt description: - "Update max_num_tokens and max_batch_size for min-latency decode workers" - "See srt-slurm recipe changes: https://github.com/ishandhanani/srt-slurm/pull/173" pr-link: https://github.com/InferenceMAX/InferenceMAX/pull/686 - + - config-keys: - dsr1-fp8-mi325x-sglang description: diff --git a/runners/launch_h100-dgxc-slurm.sh b/runners/launch_h100-dgxc-slurm.sh index 2112cc1d4f..0d6f718bfa 100644 --- a/runners/launch_h100-dgxc-slurm.sh +++ b/runners/launch_h100-dgxc-slurm.sh @@ -81,7 +81,7 @@ cat > srtslurm.yaml <