diff --git a/docs/source/features/speculative-decoding.md b/docs/source/features/speculative-decoding.md index 5364d9b6fcd7..456627dd373e 100644 --- a/docs/source/features/speculative-decoding.md +++ b/docs/source/features/speculative-decoding.md @@ -114,8 +114,7 @@ MTP is currently only supported by Deepseek. MTP can be tuned with the following ```python from tensorrt_llm.llmapi import MTPDecodingConfig -speculative_config = MTPDecodingConfig( - max_draft_len=3, num_nextn_predict_layers=3) +speculative_config = MTPDecodingConfig(max_draft_len=3) llm = LLM("/path/to/deepseek_model", speculative_config=speculative_config) ``` diff --git a/examples/configs/curated/deepseek-r1-deepgemm.yaml b/examples/configs/curated/deepseek-r1-deepgemm.yaml index 3c00b146e160..9e815e8318ce 100644 --- a/examples/configs/curated/deepseek-r1-deepgemm.yaml +++ b/examples/configs/curated/deepseek-r1-deepgemm.yaml @@ -12,7 +12,7 @@ kv_cache_config: stream_interval: 10 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 moe_config: backend: DEEPGEMM max_num_tokens: 3200 diff --git a/examples/configs/curated/deepseek-r1-latency.yaml b/examples/configs/curated/deepseek-r1-latency.yaml index 35ce5864dbaf..fffc6338a427 100644 --- a/examples/configs/curated/deepseek-r1-latency.yaml +++ b/examples/configs/curated/deepseek-r1-latency.yaml @@ -9,7 +9,7 @@ moe_config: backend: TRTLLM speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 use_relaxed_acceptance_for_thinking: true relaxed_topk: 10 relaxed_delta: 0.6 diff --git a/examples/configs/curated/deepseek-r1-throughput.yaml b/examples/configs/curated/deepseek-r1-throughput.yaml index 13566c7fc258..a51737b77bb9 100644 --- a/examples/configs/curated/deepseek-r1-throughput.yaml +++ b/examples/configs/curated/deepseek-r1-throughput.yaml @@ -12,4 +12,4 @@ kv_cache_config: stream_interval: 10 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc1.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc1.yaml index 301fbed8e3fd..65e76b8bbce8 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc1.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc1.yaml @@ -11,7 +11,7 @@ moe_config: backend: TRTLLM speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc1024.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc1024.yaml index 377332809aa5..6beb4dca3658 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc1024.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc1024.yaml @@ -15,7 +15,7 @@ attention_dp_config: timeout_iters: 60 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc128.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc128.yaml index 377332809aa5..6beb4dca3658 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc128.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc128.yaml @@ -15,7 +15,7 @@ attention_dp_config: timeout_iters: 60 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc16.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc16.yaml index 08b49ecf6d92..658b6c6b98dd 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc16.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc16.yaml @@ -11,7 +11,7 @@ moe_config: backend: TRTLLM speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc2.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc2.yaml index 360b1cf04248..6d3d4a12795c 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc2.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc2.yaml @@ -11,7 +11,7 @@ moe_config: backend: TRTLLM speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc2048.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc2048.yaml index 694e892291b8..85b625b5253c 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc2048.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc2048.yaml @@ -16,7 +16,7 @@ attention_dp_config: timeout_iters: 60 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc256.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc256.yaml index 6af6e55bd714..346562c8b490 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc256.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc256.yaml @@ -16,7 +16,7 @@ attention_dp_config: timeout_iters: 60 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc32.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc32.yaml index 136850f88fda..5d7f74990a3a 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc32.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc32.yaml @@ -11,7 +11,7 @@ moe_config: backend: TRTLLM speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc4.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc4.yaml index e482a9630e81..eabf01716d3f 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc4.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc4.yaml @@ -11,7 +11,7 @@ moe_config: backend: TRTLLM speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc512.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc512.yaml index 694e892291b8..85b625b5253c 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc512.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc512.yaml @@ -16,7 +16,7 @@ attention_dp_config: timeout_iters: 60 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc64.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc64.yaml index 463935d093b3..a734cd456cf1 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc64.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc64.yaml @@ -16,7 +16,7 @@ attention_dp_config: timeout_iters: 60 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc8.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc8.yaml index 9233d06f220f..830b9f517845 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc8.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc8.yaml @@ -11,7 +11,7 @@ moe_config: backend: TRTLLM speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k8k_tp8_conc1.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k8k_tp8_conc1.yaml index 2b71f4c170ce..6a28fe1c611e 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k8k_tp8_conc1.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k8k_tp8_conc1.yaml @@ -11,7 +11,7 @@ moe_config: backend: TRTLLM speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k8k_tp8_conc1024.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k8k_tp8_conc1024.yaml index 236a8dd62547..6c0bda6b7248 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k8k_tp8_conc1024.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k8k_tp8_conc1024.yaml @@ -16,7 +16,7 @@ attention_dp_config: timeout_iters: 60 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k8k_tp8_conc128.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k8k_tp8_conc128.yaml index 4aa4675de3d7..1ed7e61f3307 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k8k_tp8_conc128.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k8k_tp8_conc128.yaml @@ -16,7 +16,7 @@ attention_dp_config: timeout_iters: 60 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k8k_tp8_conc16.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k8k_tp8_conc16.yaml index 7658d3dac46f..468a3a40e8fb 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k8k_tp8_conc16.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k8k_tp8_conc16.yaml @@ -11,7 +11,7 @@ moe_config: backend: TRTLLM speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k8k_tp8_conc2.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k8k_tp8_conc2.yaml index 4d9ba43c2490..8cd897308d40 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k8k_tp8_conc2.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k8k_tp8_conc2.yaml @@ -11,7 +11,7 @@ moe_config: backend: TRTLLM speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k8k_tp8_conc2048.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k8k_tp8_conc2048.yaml index 15c0e0df70bf..ed2bf0d20a75 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k8k_tp8_conc2048.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k8k_tp8_conc2048.yaml @@ -16,7 +16,7 @@ attention_dp_config: timeout_iters: 60 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k8k_tp8_conc256.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k8k_tp8_conc256.yaml index ea88c9ca3908..c499df206d2d 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k8k_tp8_conc256.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k8k_tp8_conc256.yaml @@ -16,7 +16,7 @@ attention_dp_config: timeout_iters: 60 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k8k_tp8_conc32.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k8k_tp8_conc32.yaml index 03ef1ec07fc4..f996ebdfbc45 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k8k_tp8_conc32.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k8k_tp8_conc32.yaml @@ -11,7 +11,7 @@ moe_config: backend: TRTLLM speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k8k_tp8_conc4.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k8k_tp8_conc4.yaml index e8567eaf4d5f..59b473ed1cf7 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k8k_tp8_conc4.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k8k_tp8_conc4.yaml @@ -11,7 +11,7 @@ moe_config: backend: TRTLLM speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k8k_tp8_conc512.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k8k_tp8_conc512.yaml index 4f0b13d4007b..35efba108341 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k8k_tp8_conc512.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k8k_tp8_conc512.yaml @@ -16,7 +16,7 @@ attention_dp_config: timeout_iters: 60 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k8k_tp8_conc64.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k8k_tp8_conc64.yaml index a942df557eeb..c34e667bf660 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k8k_tp8_conc64.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k8k_tp8_conc64.yaml @@ -11,7 +11,7 @@ moe_config: backend: DEEPGEMM speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k8k_tp8_conc8.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k8k_tp8_conc8.yaml index 3f444b992acb..cc7c37b587b8 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k8k_tp8_conc8.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k8k_tp8_conc8.yaml @@ -11,7 +11,7 @@ moe_config: backend: TRTLLM speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc1.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc1.yaml index 684c70d6f3ab..f0c7009ff6a2 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc1.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc1.yaml @@ -11,7 +11,7 @@ moe_config: backend: TRTLLM speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc1024.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc1024.yaml index 323b2e7b0382..5a6395212464 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc1024.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc1024.yaml @@ -16,7 +16,7 @@ attention_dp_config: timeout_iters: 60 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc128.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc128.yaml index 8b3a969b9ae3..5824818862fa 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc128.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc128.yaml @@ -15,7 +15,7 @@ attention_dp_config: timeout_iters: 100 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc16.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc16.yaml index 08321903ddc4..2d890bbfce6b 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc16.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc16.yaml @@ -11,7 +11,7 @@ moe_config: backend: TRTLLM speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc2.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc2.yaml index 874825142ef6..6d0316b9636b 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc2.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc2.yaml @@ -11,7 +11,7 @@ moe_config: backend: TRTLLM speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc2048.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc2048.yaml index 5cebfad0cf36..088d028f15c8 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc2048.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc2048.yaml @@ -16,7 +16,7 @@ attention_dp_config: timeout_iters: 100 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc256.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc256.yaml index af6f9d1510e5..324707f9ce53 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc256.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc256.yaml @@ -16,7 +16,7 @@ attention_dp_config: timeout_iters: 100 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc32.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc32.yaml index 56dc431e7707..dd393fe55a9d 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc32.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc32.yaml @@ -16,7 +16,7 @@ attention_dp_config: timeout_iters: 100 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc4.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc4.yaml index 1cb322bb7ced..6a6b9fb25c63 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc4.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc4.yaml @@ -11,7 +11,7 @@ moe_config: backend: TRTLLM speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc64.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc64.yaml index bc8aa6e1a6e1..60e6bd6b7b44 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc64.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc64.yaml @@ -16,7 +16,7 @@ attention_dp_config: timeout_iters: 100 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc8.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc8.yaml index f2765b967225..b5a9acc4af79 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc8.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc8.yaml @@ -11,7 +11,7 @@ moe_config: backend: TRTLLM speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc1.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc1.yaml index aa1a5afb0572..8733e264ca9b 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc1.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc1.yaml @@ -11,7 +11,7 @@ moe_config: backend: CUTLASS speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc1024.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc1024.yaml index 1e3c68485303..79ce56db076e 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc1024.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc1024.yaml @@ -16,7 +16,7 @@ attention_dp_config: timeout_iters: 60 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc128.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc128.yaml index ce151e4c0e43..c715fb113f55 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc128.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc128.yaml @@ -15,7 +15,7 @@ attention_dp_config: timeout_iters: 60 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc16.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc16.yaml index 6f9d307a6299..dec48088fcfe 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc16.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc16.yaml @@ -11,7 +11,7 @@ moe_config: backend: CUTLASS speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc2.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc2.yaml index 89f385d5332d..16edce0afae0 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc2.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc2.yaml @@ -11,7 +11,7 @@ moe_config: backend: CUTLASS speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc2048.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc2048.yaml index 571d5997b809..df63e28b924b 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc2048.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc2048.yaml @@ -16,7 +16,7 @@ attention_dp_config: timeout_iters: 60 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc256.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc256.yaml index 1e3c68485303..79ce56db076e 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc256.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc256.yaml @@ -16,7 +16,7 @@ attention_dp_config: timeout_iters: 60 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc32.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc32.yaml index ef2b8a5c1bba..e32a3da7c0ee 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc32.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc32.yaml @@ -11,7 +11,7 @@ moe_config: backend: CUTLASS speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc4.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc4.yaml index ef2b8a5c1bba..e32a3da7c0ee 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc4.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc4.yaml @@ -11,7 +11,7 @@ moe_config: backend: CUTLASS speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc512.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc512.yaml index 1e3c68485303..79ce56db076e 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc512.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc512.yaml @@ -16,7 +16,7 @@ attention_dp_config: timeout_iters: 60 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc64.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc64.yaml index 13b9be4411c8..a82ef3c8a2fd 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc64.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc64.yaml @@ -11,7 +11,7 @@ moe_config: backend: CUTLASS speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc8.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc8.yaml index 96513a86f229..aa4be84fecc2 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc8.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc8.yaml @@ -11,7 +11,7 @@ moe_config: backend: CUTLASS speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k8k_tp8_conc1.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k8k_tp8_conc1.yaml index 03ba438d62db..1325858b69c9 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k8k_tp8_conc1.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k8k_tp8_conc1.yaml @@ -11,7 +11,7 @@ moe_config: backend: CUTLASS speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k8k_tp8_conc128.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k8k_tp8_conc128.yaml index 34ff41d6522e..18bdfac4c97e 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k8k_tp8_conc128.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k8k_tp8_conc128.yaml @@ -10,7 +10,7 @@ moe_config: backend: CUTLASS speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k8k_tp8_conc16.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k8k_tp8_conc16.yaml index af865f73633d..98acb2d24b94 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k8k_tp8_conc16.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k8k_tp8_conc16.yaml @@ -11,7 +11,7 @@ moe_config: backend: CUTLASS speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k8k_tp8_conc2.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k8k_tp8_conc2.yaml index fe92675c2c4f..eddcfa32ba95 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k8k_tp8_conc2.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k8k_tp8_conc2.yaml @@ -11,7 +11,7 @@ moe_config: backend: CUTLASS speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k8k_tp8_conc256.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k8k_tp8_conc256.yaml index 671925474080..c96310225cbb 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k8k_tp8_conc256.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k8k_tp8_conc256.yaml @@ -11,7 +11,7 @@ moe_config: backend: CUTLASS speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k8k_tp8_conc32.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k8k_tp8_conc32.yaml index d6807b446ae7..b430879ed2f9 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k8k_tp8_conc32.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k8k_tp8_conc32.yaml @@ -11,7 +11,7 @@ moe_config: backend: CUTLASS speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k8k_tp8_conc4.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k8k_tp8_conc4.yaml index 61493a2b9305..a69b0a726b1e 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k8k_tp8_conc4.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k8k_tp8_conc4.yaml @@ -11,7 +11,7 @@ moe_config: backend: CUTLASS speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k8k_tp8_conc512.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k8k_tp8_conc512.yaml index 02b526c379c1..0b6410202abb 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k8k_tp8_conc512.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k8k_tp8_conc512.yaml @@ -15,7 +15,7 @@ attention_dp_config: timeout_iters: 60 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k8k_tp8_conc64.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k8k_tp8_conc64.yaml index 8b547183b4c1..2e07c5a9d5a2 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k8k_tp8_conc64.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k8k_tp8_conc64.yaml @@ -11,7 +11,7 @@ moe_config: backend: CUTLASS speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k8k_tp8_conc8.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k8k_tp8_conc8.yaml index da1b564ef598..340d7ad76966 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k8k_tp8_conc8.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k8k_tp8_conc8.yaml @@ -11,7 +11,7 @@ moe_config: backend: CUTLASS speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc1.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc1.yaml index 589a9efecc40..94f6c878c0ca 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc1.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc1.yaml @@ -11,7 +11,7 @@ moe_config: backend: CUTLASS speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc128.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc128.yaml index 67d6d0607c7e..e6ebc5a352cc 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc128.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc128.yaml @@ -15,7 +15,7 @@ attention_dp_config: timeout_iters: 60 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc16.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc16.yaml index 6d48f11a01d2..2ad647b23e13 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc16.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc16.yaml @@ -11,7 +11,7 @@ moe_config: backend: CUTLASS speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc2.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc2.yaml index 6472d412a8e0..a3de0a2c5d59 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc2.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc2.yaml @@ -11,7 +11,7 @@ moe_config: backend: CUTLASS speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc256.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc256.yaml index 89c55d173de4..e7b5f9ae85f8 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc256.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc256.yaml @@ -11,7 +11,7 @@ moe_config: backend: CUTLASS speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc32.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc32.yaml index ba5477157a55..77705d4fbe33 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc32.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc32.yaml @@ -16,7 +16,7 @@ attention_dp_config: timeout_iters: 60 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc4.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc4.yaml index 174491a53992..c554796c614a 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc4.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc4.yaml @@ -11,7 +11,7 @@ moe_config: backend: CUTLASS speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc64.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc64.yaml index 2d837c41984f..2afb54fce7e8 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc64.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc64.yaml @@ -11,7 +11,7 @@ moe_config: backend: CUTLASS speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc8.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc8.yaml index 174491a53992..c554796c614a 100644 --- a/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc8.yaml +++ b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc8.yaml @@ -11,7 +11,7 @@ moe_config: backend: CUTLASS speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc1.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc1.yaml index 86b9fc38a30b..91267779f5de 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc1.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc1.yaml @@ -11,7 +11,7 @@ moe_config: backend: TRTLLM speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc1024.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc1024.yaml index 488657a4f68c..8d78025a3513 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc1024.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc1024.yaml @@ -15,7 +15,7 @@ attention_dp_config: timeout_iters: 60 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc128.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc128.yaml index 60b6effde258..ea4b03e17984 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc128.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc128.yaml @@ -16,7 +16,7 @@ attention_dp_config: timeout_iters: 60 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc16.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc16.yaml index 61b887373dd9..1cf94e168776 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc16.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc16.yaml @@ -11,7 +11,7 @@ moe_config: backend: TRTLLM speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc2.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc2.yaml index bfe7d741d8e0..389397735bfa 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc2.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc2.yaml @@ -11,7 +11,7 @@ moe_config: backend: TRTLLM speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc256.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc256.yaml index 4cb99341e6d0..9031655d28fd 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc256.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc256.yaml @@ -16,7 +16,7 @@ attention_dp_config: timeout_iters: 60 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc32.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc32.yaml index 136850f88fda..5d7f74990a3a 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc32.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc32.yaml @@ -11,7 +11,7 @@ moe_config: backend: TRTLLM speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc4.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc4.yaml index 06bcedefd1e5..68eab71800d9 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc4.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc4.yaml @@ -11,7 +11,7 @@ moe_config: backend: TRTLLM speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc512.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc512.yaml index 4708d9921aca..e8aba9b63a43 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc512.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc512.yaml @@ -15,7 +15,7 @@ attention_dp_config: timeout_iters: 60 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc64.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc64.yaml index 269017ea42ab..c239238e8f0d 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc64.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc64.yaml @@ -15,7 +15,7 @@ attention_dp_config: timeout_iters: 60 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc8.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc8.yaml index 4553a6c5b6f3..9129bd2dd8ef 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc8.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc8.yaml @@ -11,7 +11,7 @@ moe_config: backend: TRTLLM speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k8k_tp4_conc2048.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k8k_tp4_conc2048.yaml index 14fe2b902897..c7f4ed84f1f4 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k8k_tp4_conc2048.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k8k_tp4_conc2048.yaml @@ -16,7 +16,7 @@ attention_dp_config: timeout_iters: 60 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k8k_tp8_conc1.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k8k_tp8_conc1.yaml index 580fee48c6bd..9aa253b5dcab 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k8k_tp8_conc1.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k8k_tp8_conc1.yaml @@ -11,7 +11,7 @@ moe_config: backend: TRTLLM speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k8k_tp8_conc1024.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k8k_tp8_conc1024.yaml index f2c0e1ffefc1..4a1ab0308187 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k8k_tp8_conc1024.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k8k_tp8_conc1024.yaml @@ -16,7 +16,7 @@ attention_dp_config: timeout_iters: 60 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k8k_tp8_conc128.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k8k_tp8_conc128.yaml index 714ea9d6c54e..5e10eb4bdbf1 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k8k_tp8_conc128.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k8k_tp8_conc128.yaml @@ -15,7 +15,7 @@ attention_dp_config: timeout_iters: 60 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k8k_tp8_conc16.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k8k_tp8_conc16.yaml index 7658d3dac46f..468a3a40e8fb 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k8k_tp8_conc16.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k8k_tp8_conc16.yaml @@ -11,7 +11,7 @@ moe_config: backend: TRTLLM speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k8k_tp8_conc2.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k8k_tp8_conc2.yaml index 580fee48c6bd..9aa253b5dcab 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k8k_tp8_conc2.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k8k_tp8_conc2.yaml @@ -11,7 +11,7 @@ moe_config: backend: TRTLLM speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k8k_tp8_conc256.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k8k_tp8_conc256.yaml index 171385ddf9c5..22f084c0d047 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k8k_tp8_conc256.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k8k_tp8_conc256.yaml @@ -16,7 +16,7 @@ attention_dp_config: timeout_iters: 60 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k8k_tp8_conc32.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k8k_tp8_conc32.yaml index 580fee48c6bd..9aa253b5dcab 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k8k_tp8_conc32.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k8k_tp8_conc32.yaml @@ -11,7 +11,7 @@ moe_config: backend: TRTLLM speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k8k_tp8_conc4.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k8k_tp8_conc4.yaml index 7cb12fae343c..3fc621265552 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k8k_tp8_conc4.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k8k_tp8_conc4.yaml @@ -11,7 +11,7 @@ moe_config: backend: TRTLLM speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k8k_tp8_conc512.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k8k_tp8_conc512.yaml index 6ef36b70d956..46f19e26b0de 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k8k_tp8_conc512.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k8k_tp8_conc512.yaml @@ -14,7 +14,7 @@ attention_dp_config: timeout_iters: 60 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k8k_tp8_conc64.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k8k_tp8_conc64.yaml index f342b2939446..c4720d40b87b 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k8k_tp8_conc64.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k8k_tp8_conc64.yaml @@ -15,7 +15,7 @@ attention_dp_config: timeout_iters: 60 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k8k_tp8_conc8.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k8k_tp8_conc8.yaml index 3f444b992acb..cc7c37b587b8 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k8k_tp8_conc8.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k8k_tp8_conc8.yaml @@ -11,7 +11,7 @@ moe_config: backend: TRTLLM speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc1024.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc1024.yaml index ce35cbe7e16c..5ed468b4018e 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc1024.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc1024.yaml @@ -16,7 +16,7 @@ attention_dp_config: timeout_iters: 60 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc2048.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc2048.yaml index ce35cbe7e16c..5ed468b4018e 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc2048.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc2048.yaml @@ -16,7 +16,7 @@ attention_dp_config: timeout_iters: 60 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc1.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc1.yaml index 684c70d6f3ab..f0c7009ff6a2 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc1.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc1.yaml @@ -11,7 +11,7 @@ moe_config: backend: TRTLLM speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc128.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc128.yaml index 471c75443d6a..baaa644ca539 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc128.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc128.yaml @@ -16,7 +16,7 @@ attention_dp_config: timeout_iters: 60 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc16.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc16.yaml index 08321903ddc4..2d890bbfce6b 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc16.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc16.yaml @@ -11,7 +11,7 @@ moe_config: backend: TRTLLM speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc2.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc2.yaml index 874825142ef6..6d0316b9636b 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc2.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc2.yaml @@ -11,7 +11,7 @@ moe_config: backend: TRTLLM speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc256.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc256.yaml index 5dfdebedb273..93df61d6fa56 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc256.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc256.yaml @@ -16,7 +16,7 @@ attention_dp_config: timeout_iters: 60 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc32.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc32.yaml index 68eaf2181a64..7ed8eaa9395a 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc32.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc32.yaml @@ -11,7 +11,7 @@ moe_config: backend: TRTLLM speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc4.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc4.yaml index 1cb322bb7ced..6a6b9fb25c63 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc4.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc4.yaml @@ -11,7 +11,7 @@ moe_config: backend: TRTLLM speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc512.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc512.yaml index fc82d57f15c0..8cde1f4be315 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc512.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc512.yaml @@ -14,7 +14,7 @@ attention_dp_config: timeout_iters: 60 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc64.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc64.yaml index 5364e184c32e..b39f07478407 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc64.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc64.yaml @@ -16,7 +16,7 @@ attention_dp_config: timeout_iters: 60 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc8.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc8.yaml index f2765b967225..b5a9acc4af79 100644 --- a/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc8.yaml +++ b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc8.yaml @@ -11,7 +11,7 @@ moe_config: backend: TRTLLM speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/examples/disaggregated/slurm/benchmark/config.yaml b/examples/disaggregated/slurm/benchmark/config.yaml index e4bda5707d07..92a275c40d1f 100644 --- a/examples/disaggregated/slurm/benchmark/config.yaml +++ b/examples/disaggregated/slurm/benchmark/config.yaml @@ -107,7 +107,7 @@ worker_config: num_postprocess_workers: 4 speculative_config: # when mtp_size is 0, remove this section. decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 ctx: max_batch_size: 4 max_num_tokens: 4608 @@ -130,4 +130,4 @@ worker_config: backend: DEFAULT speculative_config: # when mtp_size is 0, remove this section. decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 diff --git a/examples/disaggregated/slurm/benchmark/submit.py b/examples/disaggregated/slurm/benchmark/submit.py index ba470d71c7c0..5842be21276f 100644 --- a/examples/disaggregated/slurm/benchmark/submit.py +++ b/examples/disaggregated/slurm/benchmark/submit.py @@ -443,9 +443,10 @@ def submit_job(config, log_dir, dry_run): load_balancer_config = yaml.safe_load(f) eplb_num_slots = load_balancer_config.get('num_slots', 0) - # Get mtp_size from gen config's speculative_config + # Get mtp_size from gen config's speculative_config. Leave it unset when + # max_draft_len is omitted so auto-MTP runs are not mislabeled as mtp0. mtp_size = worker_config['gen'].get('speculative_config', - {}).get('num_nextn_predict_layers', 0) + {}).get('max_draft_len') # Create base log directory path if 'log_dir' in env_config and env_config['log_dir']: @@ -456,11 +457,13 @@ def submit_job(config, log_dir, dry_run): date_prefix = datetime.now().strftime("%Y%m%d-%H%M%S") log_base = os.path.join(log_base, f"{date_prefix}/{isl}-{osl}") + mtp_suffix = "" if mtp_size is None else f"_mtp{mtp_size}" + # Determine directory suffix based on attention_dp if gen_enable_attention_dp: - dir_suffix = f"disagg_ctx{ctx_num}_gen{gen_num}_dep{gen_tp_size}_batch{gen_batch_size}_eplb{eplb_num_slots}_mtp{mtp_size}" + dir_suffix = f"disagg_ctx{ctx_num}_gen{gen_num}_dep{gen_tp_size}_batch{gen_batch_size}_eplb{eplb_num_slots}{mtp_suffix}" else: - dir_suffix = f"disagg_ctx{ctx_num}_gen{gen_num}_tep{gen_tp_size}_batch{gen_batch_size}_eplb{eplb_num_slots}_mtp{mtp_size}" + dir_suffix = f"disagg_ctx{ctx_num}_gen{gen_num}_tep{gen_tp_size}_batch{gen_batch_size}_eplb{eplb_num_slots}{mtp_suffix}" # Create full log directory path log_dir = os.path.join(log_base, dir_suffix) diff --git a/examples/llm-api/llm_speculative_decoding.py b/examples/llm-api/llm_speculative_decoding.py index 4f19dd6df330..af24a78cd4ea 100644 --- a/examples/llm-api/llm_speculative_decoding.py +++ b/examples/llm-api/llm_speculative_decoding.py @@ -16,8 +16,7 @@ def run_MTP(model: Optional[str] = None): - spec_config = MTPDecodingConfig(num_nextn_predict_layers=1, - use_relaxed_acceptance_for_thinking=True, + spec_config = MTPDecodingConfig(use_relaxed_acceptance_for_thinking=True, relaxed_topk=10, relaxed_delta=0.01) diff --git a/examples/llm-api/quickstart_advanced.py b/examples/llm-api/quickstart_advanced.py index ffc620feddce..8a3c63866de8 100644 --- a/examples/llm-api/quickstart_advanced.py +++ b/examples/llm-api/quickstart_advanced.py @@ -278,7 +278,7 @@ def setup_llm(args, **kwargs): if not args.use_one_model: print("Running MTP eagle with two model style.") spec_config = MTPDecodingConfig( - num_nextn_predict_layers=args.spec_decode_max_draft_len, + max_draft_len=args.spec_decode_max_draft_len, use_relaxed_acceptance_for_thinking=args. use_relaxed_acceptance_for_thinking, relaxed_topk=args.relaxed_topk, diff --git a/examples/models/core/nemotron/README_nemotron_super_v3.md b/examples/models/core/nemotron/README_nemotron_super_v3.md index 7ffd82f04945..0c9637a42256 100644 --- a/examples/models/core/nemotron/README_nemotron_super_v3.md +++ b/examples/models/core/nemotron/README_nemotron_super_v3.md @@ -143,7 +143,7 @@ kv_cache_config: mamba_ssm_philox_rounds: 5 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 5 + max_draft_len: 5 allow_advanced_sampling: true cuda_graph_config: max_batch_size: 64 diff --git a/examples/wide_ep/slurm_scripts/config.yaml b/examples/wide_ep/slurm_scripts/config.yaml index 5c5b4441b4ed..d34a3cc08ca4 100644 --- a/examples/wide_ep/slurm_scripts/config.yaml +++ b/examples/wide_ep/slurm_scripts/config.yaml @@ -93,7 +93,7 @@ worker_config: num_postprocess_workers: 4 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 ctx: max_batch_size: 1 max_num_tokens: 8448 @@ -114,4 +114,4 @@ worker_config: backend: DEFAULT speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 diff --git a/tensorrt_llm/_torch/model_config.py b/tensorrt_llm/_torch/model_config.py index 9d3afb55e278..666041f4ee95 100644 --- a/tensorrt_llm/_torch/model_config.py +++ b/tensorrt_llm/_torch/model_config.py @@ -695,6 +695,10 @@ def ceil_div(a, b): is_disagg, kv_cache_config, spec_config) if (self.spec_config is not None and self.spec_config.spec_dec_mode.is_mtp_one_model()): + assert self.spec_config.num_nextn_predict_layers is not None, ( + "num_nextn_predict_layers must be set from model config before building ModelConfig. " + "Ensure update_spec_config_from_model_config() has been called." + ) num_layers += self.spec_config.num_nextn_predict_layers num_attention_layers += self.spec_config.num_nextn_predict_layers diff --git a/tensorrt_llm/_torch/models/modeling_deepseekv3.py b/tensorrt_llm/_torch/models/modeling_deepseekv3.py index 7b3c2b40ab02..1e9c65c8ba81 100755 --- a/tensorrt_llm/_torch/models/modeling_deepseekv3.py +++ b/tensorrt_llm/_torch/models/modeling_deepseekv3.py @@ -1835,7 +1835,7 @@ def __init__(self, model_config: ModelConfig[PretrainedConfig]): self.model_nextn = 0 if model_config.spec_config is not None and model_config.spec_config.spec_dec_mode.is_mtp_one_model( ): - model_nextn = model_config.spec_config.num_nextn_predict_layers + model_nextn = self.config.num_nextn_predict_layers ckpt_nextn = self.config.num_nextn_predict_layers self.num_hidden_layers = self.config.num_hidden_layers assert ckpt_nextn > 0, "There is not MTP modules in the checkpoint." diff --git a/tensorrt_llm/_torch/models/modeling_exaone_moe.py b/tensorrt_llm/_torch/models/modeling_exaone_moe.py index 5a796f926c4d..dc8f306238dc 100644 --- a/tensorrt_llm/_torch/models/modeling_exaone_moe.py +++ b/tensorrt_llm/_torch/models/modeling_exaone_moe.py @@ -674,7 +674,7 @@ def __init__( model_config.spec_config is not None and model_config.spec_config.spec_dec_mode.is_mtp_one_model() ): - model_nextn = model_config.spec_config.num_nextn_predict_layers + model_nextn = self.config.num_nextn_predict_layers ckpt_nextn = self.config.num_nextn_predict_layers self.num_hidden_layers = self.config.num_hidden_layers if ckpt_nextn == 0: diff --git a/tensorrt_llm/_torch/models/modeling_glm.py b/tensorrt_llm/_torch/models/modeling_glm.py index ccbcbdfec58e..293510b65099 100644 --- a/tensorrt_llm/_torch/models/modeling_glm.py +++ b/tensorrt_llm/_torch/models/modeling_glm.py @@ -1024,7 +1024,7 @@ def __init__(self, model_config: ModelConfig[PretrainedConfig]): model_config.spec_config is not None and model_config.spec_config.spec_dec_mode.is_mtp_one_model() ): - model_nextn = model_config.spec_config.num_nextn_predict_layers + model_nextn = self.config.num_nextn_predict_layers ckpt_nextn = self.config.num_nextn_predict_layers self.num_hidden_layers = self.config.num_hidden_layers assert ckpt_nextn > 0, "There is not MTP modules in the checkpoint." diff --git a/tensorrt_llm/_torch/models/modeling_nemotron_h.py b/tensorrt_llm/_torch/models/modeling_nemotron_h.py index 86c562a2da3b..f6a1e5a4919f 100644 --- a/tensorrt_llm/_torch/models/modeling_nemotron_h.py +++ b/tensorrt_llm/_torch/models/modeling_nemotron_h.py @@ -775,7 +775,7 @@ def __init__( self.model_nextn = 0 if (model_config.spec_config is not None and model_config.spec_config.spec_dec_mode.is_mtp_one_model()): - model_nextn = model_config.spec_config.num_nextn_predict_layers + model_nextn = self.config.num_nextn_predict_layers ckpt_nextn = self.config.num_nextn_predict_layers self.num_hidden_layers = self.config.num_hidden_layers assert ckpt_nextn > 0, "There are not MTP modules in the checkpoint." @@ -874,7 +874,7 @@ def __init__( self.model_nextn = 0 if (model_config.spec_config is not None and model_config.spec_config.spec_dec_mode.is_mtp_one_model()): - model_nextn = model_config.spec_config.num_nextn_predict_layers + model_nextn = self.config.num_nextn_predict_layers ckpt_nextn = self.config.num_nextn_predict_layers self.num_hidden_layers = self.config.num_hidden_layers assert ckpt_nextn > 0, "There is not MTP modules in the checkpoint." diff --git a/tensorrt_llm/_torch/models/modeling_speculative.py b/tensorrt_llm/_torch/models/modeling_speculative.py index 8473f89b6714..faae007200f7 100755 --- a/tensorrt_llm/_torch/models/modeling_speculative.py +++ b/tensorrt_llm/_torch/models/modeling_speculative.py @@ -1241,11 +1241,16 @@ def __init__( spec_dec_mode = model_config.spec_config.spec_dec_mode assert spec_dec_mode.is_mtp_one_model() - mtp_num_layers = 1 if spec_dec_mode.is_mtp_eagle_one_model( - ) else model_config.spec_config.num_nextn_predict_layers + checkpoint_mtp_num_layers = model_config.pretrained_config.num_nextn_predict_layers + if spec_dec_mode.is_mtp_eagle_one_model(): + mtp_num_layers = 1 + mtp_repeat_count = model_config.spec_config.max_draft_len + else: + mtp_num_layers = min(model_config.spec_config.max_draft_len, + checkpoint_mtp_num_layers) + mtp_repeat_count = 1 - moe_load_balancer_set_repeated_for_next_layer( - model_config.spec_config.num_nextn_predict_layers // mtp_num_layers) + moe_load_balancer_set_repeated_for_next_layer(mtp_repeat_count) self.mtp_layers = nn.ModuleList([ mtp_layer(model_config, layer_idx + start_layer_idx, diff --git a/tensorrt_llm/_torch/pyexecutor/_util.py b/tensorrt_llm/_torch/pyexecutor/_util.py index a7e49d9d369c..5b0e071f7d0f 100644 --- a/tensorrt_llm/_torch/pyexecutor/_util.py +++ b/tensorrt_llm/_torch/pyexecutor/_util.py @@ -1807,9 +1807,8 @@ def init_feature_status(llm_args) -> Dict[str, bool]: feature_status[ "disaggregated_serving"] = llm_args.cache_transceiver_config is not None feature_status["chunked_prefill"] = llm_args.enable_chunked_prefill - feature_status["mtp"] = ( - isinstance(llm_args.speculative_config, MTPDecodingConfig) - and llm_args.speculative_config.num_nextn_predict_layers > 0) + feature_status["mtp"] = isinstance(llm_args.speculative_config, + MTPDecodingConfig) feature_status["eagle3_one_model"] = ( isinstance(llm_args.speculative_config, EagleDecodingConfig) and llm_args.speculative_config.eagle3_one_model) diff --git a/tensorrt_llm/_torch/speculative/mtp.py b/tensorrt_llm/_torch/speculative/mtp.py index 588a22e4e002..c310cf57816c 100644 --- a/tensorrt_llm/_torch/speculative/mtp.py +++ b/tensorrt_llm/_torch/speculative/mtp.py @@ -41,7 +41,7 @@ def __init__(self, max_num_requests: int, sa_manager=None): self.dtype = dtype - self.num_nextn_predict_layers = config.num_nextn_predict_layers + self.num_draft_slots = config.max_draft_len self.hidden_size = hidden_size self.max_num_requests = max_num_requests self.use_relaxed_acceptance_for_thinking = config.use_relaxed_acceptance_for_thinking @@ -54,12 +54,12 @@ def __init__(self, # Since golden token's hidden state will always be generated after target model self.mtp_past_hidden_states_pool = torch.zeros( - (slot_pool_size, self.num_nextn_predict_layers, self.hidden_size), + (slot_pool_size, self.num_draft_slots, self.hidden_size), device='cuda', dtype=self.dtype, ) self.mtp_past_tokens_pool = torch.zeros( - (slot_pool_size, self.num_nextn_predict_layers), + (slot_pool_size, self.num_draft_slots), device='cuda', dtype=torch.int, ) @@ -279,7 +279,7 @@ def __init__(self, @property def max_draft_len(self) -> int: - return self.spec_config.num_nextn_predict_layers + return self.spec_config.max_draft_len def forward( self, @@ -506,7 +506,7 @@ def skip_forward( resource_manager=None, ): batch_size = attn_metadata.num_seqs - mtp_num_modules = self.spec_config.num_nextn_predict_layers + mtp_num_modules = self.spec_config.max_draft_len accepted_tokens = torch.empty((batch_size, (mtp_num_modules + 1)), dtype=torch.int, device=logits.device) @@ -589,7 +589,7 @@ def unpack_sequence(packed_seq_cuda, seq_lens_cuda, seq_lens_cpu): seq_lens = attn_metadata.seq_lens_cuda seq_lens_cpu = attn_metadata.seq_lens hidden_size = hidden_states.shape[-1] - mtp_num_modules = self.spec_config.num_nextn_predict_layers + mtp_num_modules = self.spec_config.max_draft_len if self.is_thop: _, _ = torch.ops.trtllm.mtp_update_hidden_states_op( @@ -761,7 +761,7 @@ def sample_and_accept_draft_tokens( batch_size = attn_metadata.num_seqs num_contexts = attn_metadata.num_contexts num_gens = batch_size - num_contexts - mtp_num_modules = self.spec_config.num_nextn_predict_layers + mtp_num_modules = self.spec_config.max_draft_len if logits.dim() == 1: logits = logits.unsqueeze(0) @@ -864,7 +864,7 @@ def change_attn_metadata(self, num_accepted_tokens: torch.Tensor, attn_metadata: AttentionMetadata): self._prepare_attn_metadata_for_spec_dec(attn_metadata) batch_size = attn_metadata.num_seqs - mtp_num_modules = self.spec_config.num_nextn_predict_layers + mtp_num_modules = self.spec_config.max_draft_len num_contexts = attn_metadata.num_contexts attn_metadata._seq_lens[num_contexts:batch_size] -= 1 @@ -955,7 +955,7 @@ def prepare_drafter_inputs( num_gens = batch_size - num_contexts mtp_past_hidden_states_pool = spec_metadata.mtp_hidden_states_manager.mtp_past_hidden_states_pool mtp_past_tokens_pool = spec_metadata.mtp_hidden_states_manager.mtp_past_tokens_pool - mtp_num_modules = self.spec_config.num_nextn_predict_layers + mtp_num_modules = self.spec_config.max_draft_len if self.is_thop: # Temporary buffer @@ -1120,7 +1120,7 @@ def __init__(self, use_separate_draft_kv_cache: bool = False): super().__init__(spec_config, model_config, use_separate_draft_kv_cache) self.model_config = model_config - self.mtp_num_modules = spec_config.num_nextn_predict_layers + self.mtp_num_modules = spec_config.max_draft_len self._is_mamba_hybrid_cache = None @torch.compile(options={"max-autotune": True}) diff --git a/tensorrt_llm/_torch/speculative/utils.py b/tensorrt_llm/_torch/speculative/utils.py index 65049dab24a0..acabe338d66a 100644 --- a/tensorrt_llm/_torch/speculative/utils.py +++ b/tensorrt_llm/_torch/speculative/utils.py @@ -4,6 +4,8 @@ import torch +from tensorrt_llm.logger import logger + if TYPE_CHECKING: from tensorrt_llm.llmapi.llm_args import DecodingBaseConfig @@ -43,7 +45,7 @@ def get_spec_metadata(spec_config, max_draft_len=spec_config.max_draft_len, max_total_draft_tokens=spec_config.tokens_per_gen_step - 1, spec_dec_mode=spec_config.spec_dec_mode, - mtp_num_modules=spec_config.num_nextn_predict_layers, + mtp_num_modules=spec_config.max_draft_len, max_num_requests=max_num_requests, mtp_hidden_states_manager=spec_resource_manager, allow_advanced_sampling=spec_config.allow_advanced_sampling, @@ -259,8 +261,7 @@ def get_spec_decoder( spec_config: "DecodingBaseConfig", ): if spec_config.spec_dec_mode.is_mtp_one_model(): - return MTPSampler(sampler_args, - nextn=spec_config.num_nextn_predict_layers) + return MTPSampler(sampler_args, nextn=spec_config.max_draft_len) if spec_config.spec_dec_mode.is_eagle3( ) or spec_config.spec_dec_mode.is_mtp_eagle(): # TorchSampler handles Eagle3 gracefully, by integrating d2t into the sampling process @@ -374,11 +375,32 @@ def get_draft_kv_cache_manager(spec_config, resource_manager): def update_spec_config_from_model_config(spec_config, model_config): - if spec_config.spec_dec_mode.is_mtp_one_model(): - # Use `max_draft_len` for several low-level APIs. TODO: Remove this after distinguishing them. - spec_config.max_draft_len = spec_config.num_nextn_predict_layers - # Use `num_nextn_predict_layers_from_model_config` to decide decoding mode MTP / MTP_EAGLE. - spec_config.num_nextn_predict_layers_from_model_config = model_config.num_nextn_predict_layers + from tensorrt_llm.llmapi.llm_args import MTPDecodingConfig + if not isinstance(spec_config, MTPDecodingConfig): + return + # Read num_nextn_predict_layers from the model's pretrained config. + # This determines the actual MTP layer count in the checkpoint and drives + # the spec_dec_mode decision (EAGLE vs vanilla MTP). + spec_config.num_nextn_predict_layers = model_config.num_nextn_predict_layers + # For vanilla MTP (>1 MTP layers in the checkpoint): set max_draft_len to the + # minimum of the user-requested value and the model's layer count. + # If the user explicitly requested fewer draft tokens than the model has layers, + # honour that and warn. Otherwise default to using all model layers. + if spec_config.spec_dec_mode.is_mtp_vanilla(): + model_layers = spec_config.num_nextn_predict_layers + user_set = 'max_draft_len' in spec_config.model_fields_set + if user_set and spec_config.max_draft_len < model_layers: + logger.warning( + f"MTP: max_draft_len ({spec_config.max_draft_len}) is less than the model's " + f"num_nextn_predict_layers ({model_layers}). " + f"Using max_draft_len={spec_config.max_draft_len} draft tokens." + ) + # Keep the user-set max_draft_len as-is + else: + spec_config.max_draft_len = model_layers + spec_config.max_total_draft_tokens = spec_config.max_draft_len + # For Eagle MTP (1 MTP layer): max_draft_len controls how many times the + # single layer is run. It was already set by the user (defaults to 1). @dataclass diff --git a/tensorrt_llm/bench/dataclasses/reporting.py b/tensorrt_llm/bench/dataclasses/reporting.py index b11a06c57d03..e72b7a6a91e4 100755 --- a/tensorrt_llm/bench/dataclasses/reporting.py +++ b/tensorrt_llm/bench/dataclasses/reporting.py @@ -778,8 +778,7 @@ def get_max_draft_len(self) -> int: spec_config = self.kwargs["speculative_config"] # Handle both dict (from YAML) and object types if isinstance(spec_config, dict): - draft_len = (spec_config.get("max_draft_len") - or spec_config.get("num_nextn_predict_layers")) + draft_len = spec_config.get("max_draft_len") return draft_len or 0 return spec_config.max_draft_len or 0 diff --git a/tensorrt_llm/llmapi/llm_args.py b/tensorrt_llm/llmapi/llm_args.py index c845cf438e20..81b49c561b17 100644 --- a/tensorrt_llm/llmapi/llm_args.py +++ b/tensorrt_llm/llmapi/llm_args.py @@ -1539,11 +1539,6 @@ def spec_dec_mode(self): class MTPDecodingConfig(DecodingBaseConfig): decoding_type: Literal["MTP"] = "MTP" - num_nextn_predict_layers: PositiveInt = Field( - default=1, - description= - "Number of MTP modules. Each module predicts the next token, so N modules produce N draft tokens." - ) use_relaxed_acceptance_for_thinking: bool = Field( default=False, description= @@ -1576,14 +1571,14 @@ class MTPDecodingConfig(DecodingBaseConfig): description="Optional Suffix Automaton configuration. When set, " "combines SA drafting with MTP speculative decoding.") - # TODO: remove this after distinguishing `max_draft_len` and `num_nextn_predict_layers` - # Now we need a flag when MTPDecodingConfig is updated by PyTorchModelEngine. - num_nextn_predict_layers_from_model_config: int = Field( - default=1, + # Internal field: number of MTP layers in the model checkpoint. + # Auto-populated from pretrained_config by update_spec_config_from_model_config. + # Do not set manually. + num_nextn_predict_layers: Optional[int] = Field( + default=None, init=False, - description= - "Internal field storing MTP layer count from model config. Used to decide decoding mode: " - "when model has 1 layer and use_mtp_vanilla=False, uses faster EAGLE-style MTP instead of vanilla MTP." + description="Number of MTP layers in the model checkpoint. " + "Auto-populated from the model's pretrained config. Do not set manually." ) begin_thinking_phase_token: int = Field( @@ -1597,10 +1592,31 @@ class MTPDecodingConfig(DecodingBaseConfig): "Token ID marking end of thinking phase. Strict acceptance resumes after this." ) + @model_validator(mode="before") + @classmethod + def _remap_deprecated_num_nextn_predict_layers(cls, data): + if isinstance(data, dict) and "num_nextn_predict_layers" in data: + logger.warning( + "MTPDecodingConfig: 'num_nextn_predict_layers' is deprecated and will be " + "removed in a future release. Use 'max_draft_len' instead.") + if "max_draft_len" not in data: + data = dict(data) + data["max_draft_len"] = data.pop("num_nextn_predict_layers") + else: + data = dict(data) + data.pop("num_nextn_predict_layers") + return data + @model_validator(mode="after") def set_max_total_draft_tokens(self): - self.max_draft_len = self.num_nextn_predict_layers - self.max_total_draft_tokens = self.num_nextn_predict_layers # Current MTP only supports linear tree + # Default max_draft_len to 1 if not set by the user. + # For vanilla MTP, update_spec_config_from_model_config will override this + # with the actual num_nextn_predict_layers from the model. + if self.max_draft_len is None: + self.max_draft_len = 1 + elif self.max_draft_len <= 0: + raise ValueError("max_draft_len must be > 0 for MTP") + self.max_total_draft_tokens = self.max_draft_len # Current MTP only supports linear tree return self @model_validator(mode="after") @@ -1614,19 +1630,21 @@ def log_two_model_deprecation_warning(self): def supports_backend(self, backend: str) -> bool: return backend in ("pytorch", "_autodeploy") - @functools.cached_property + @property def num_capture_layers(self) -> int: - if not self.use_mtp_vanilla and not self.mtp_eagle_one_model: - return 1 - return 0 + return 1 if self.spec_dec_mode.is_mtp_eagle() else 0 - @functools.cached_property + @property def spec_dec_mode(self): from tensorrt_llm._torch.speculative.interface import \ SpeculativeDecodingMode as TorchSpeculativeDecodingMode - if self.num_nextn_predict_layers_from_model_config == 1 and not self.use_mtp_vanilla and self.mtp_eagle_one_model: + + # num_nextn_predict_layers is set from the model's pretrained config by + # update_spec_config_from_model_config. Treat None (before model load) as 1. + n = self.num_nextn_predict_layers if self.num_nextn_predict_layers is not None else 1 + if n == 1 and not self.use_mtp_vanilla and self.mtp_eagle_one_model: return TorchSpeculativeDecodingMode.MTP_EAGLE_ONE_MODEL - elif self.num_nextn_predict_layers_from_model_config == 1 and not self.use_mtp_vanilla and not self.mtp_eagle_one_model: + elif n == 1 and not self.use_mtp_vanilla and not self.mtp_eagle_one_model: return TorchSpeculativeDecodingMode.MTP_EAGLE return TorchSpeculativeDecodingMode.MTP diff --git a/tests/integration/defs/accuracy/references/gsm8k.yaml b/tests/integration/defs/accuracy/references/gsm8k.yaml index f7ffcaf91e73..19d02a210d0b 100644 --- a/tests/integration/defs/accuracy/references/gsm8k.yaml +++ b/tests/integration/defs/accuracy/references/gsm8k.yaml @@ -416,7 +416,7 @@ nvidia/Nemotron-Super-V3: - quant_algo: NVFP4 kv_cache_quant_algo: FP8 mtp_enabled: true - num_nextn_predict_layers: 3 + max_draft_len: 3 accuracy: 80.85 - spec_dec_algo: MTP accuracy: 92.70 diff --git a/tests/integration/defs/accuracy/references/mmlu.yaml b/tests/integration/defs/accuracy/references/mmlu.yaml index ffb2e27a0534..b9488dd869c6 100644 --- a/tests/integration/defs/accuracy/references/mmlu.yaml +++ b/tests/integration/defs/accuracy/references/mmlu.yaml @@ -427,7 +427,7 @@ nvidia/Nemotron-Super-V3: - quant_algo: NVFP4 kv_cache_quant_algo: FP8 mtp_enabled: true - num_nextn_predict_layers: 3 + max_draft_len: 3 accuracy: 77.56 nvidia/Nemotron-3-Nano: - accuracy: 73.85 diff --git a/tests/integration/defs/accuracy/test_disaggregated_serving.py b/tests/integration/defs/accuracy/test_disaggregated_serving.py index 84deaeb223d2..c614ca74504c 100644 --- a/tests/integration/defs/accuracy/test_disaggregated_serving.py +++ b/tests/integration/defs/accuracy/test_disaggregated_serving.py @@ -1059,11 +1059,11 @@ def test_auto_dtype(self, overlap_scheduler, mtp_nextn): if mtp_nextn > 0: ctx_server_config["speculative_config"] = { "decoding_type": "MTP", - "num_nextn_predict_layers": mtp_nextn + "max_draft_len": mtp_nextn } gen_server_config["speculative_config"] = { "decoding_type": "MTP", - "num_nextn_predict_layers": mtp_nextn + "max_draft_len": mtp_nextn } disaggregated_server_config = { "hostname": "localhost", @@ -1209,11 +1209,11 @@ def test_guided_decoding(self, backend: str, mtp_nextn: int, mocker): if mtp_nextn > 0: ctx_server_config["speculative_config"] = { "decoding_type": "MTP", - "num_nextn_predict_layers": mtp_nextn + "max_draft_len": mtp_nextn } gen_server_config["speculative_config"] = { "decoding_type": "MTP", - "num_nextn_predict_layers": mtp_nextn + "max_draft_len": mtp_nextn } disaggregated_server_config = { "hostname": "localhost", diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py index 47287b7bb32c..19b781938d11 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py @@ -1693,7 +1693,7 @@ def test_bfloat16(self, mtp_nextn, attention_dp, cuda_graph, ) mtp_config = None if mtp_nextn > 0: - mtp_config = MTPDecodingConfig(num_nextn_predict_layers=mtp_nextn) + mtp_config = MTPDecodingConfig(max_draft_len=mtp_nextn) with LLM(self.MODEL_PATH, kv_cache_config=kv_cache_config, enable_chunked_prefill=enable_chunked_prefill, @@ -1722,7 +1722,7 @@ def test_bfloat16_python_scheduler(self, mtp_nextn, attention_dp, ) mtp_config = None if mtp_nextn > 0: - mtp_config = MTPDecodingConfig(num_nextn_predict_layers=mtp_nextn) + mtp_config = MTPDecodingConfig(max_draft_len=mtp_nextn) with LLM(self.MODEL_PATH, kv_cache_config=kv_cache_config, enable_chunked_prefill=enable_chunked_prefill, @@ -1740,7 +1740,7 @@ def test_bfloat16_2_model_mtp(self): disable_overlap_scheduler=True, cuda_graph_config=CudaGraphConfig(), ) - mtp_config = MTPDecodingConfig(num_nextn_predict_layers=3, + mtp_config = MTPDecodingConfig(max_draft_len=3, mtp_eagle_one_model=False, speculative_model=self.MODEL_PATH) with LLM(self.MODEL_PATH, @@ -1760,8 +1760,7 @@ def test_bfloat16_mtp_sa(self): disable_overlap_scheduler=False, cuda_graph_config=CudaGraphConfig(), ) - mtp_config = MTPDecodingConfig(num_nextn_predict_layers=2, - sa_config=SAEnhancerConfig()) + mtp_config = MTPDecodingConfig(max_draft_len=2, use_sa_spec=True) with LLM(self.MODEL_PATH, kv_cache_config=kv_cache_config, max_num_tokens=8192, @@ -1782,7 +1781,7 @@ def test_bfloat16_mtp_sa_global_pool(self): enable_padding=True), ) mtp_config = MTPDecodingConfig( - num_nextn_predict_layers=2, + max_draft_len=2, sa_config=SAEnhancerConfig(enable_global_pool=True)) with LLM(self.MODEL_PATH, kv_cache_config=kv_cache_config, @@ -1804,7 +1803,7 @@ def test_bfloat16_4gpus_kv_cache_aware_routing(self, mtp_nextn): ) mtp_config = None if mtp_nextn > 0: - mtp_config = MTPDecodingConfig(num_nextn_predict_layers=mtp_nextn) + mtp_config = MTPDecodingConfig(max_draft_len=mtp_nextn) attention_dp_config = AttentionDpConfig( enable_kv_cache_aware_routing=True, ) with LLM(self.MODEL_PATH, @@ -1848,7 +1847,7 @@ def test_bfloat16_4gpus(self, tp_size, pp_size, ep_size, mtp_nextn, ) mtp_config = None if mtp_nextn > 0: - mtp_config = MTPDecodingConfig(num_nextn_predict_layers=mtp_nextn) + mtp_config = MTPDecodingConfig(max_draft_len=mtp_nextn) with LLM(self.MODEL_PATH, tensor_parallel_size=tp_size, pipeline_parallel_size=pp_size, @@ -1873,7 +1872,7 @@ def test_bfloat16_4gpus_python_scheduler(self, tp_size, pp_size, ep_size, pytorch_config = dict(cuda_graph_config=CudaGraphConfig(), ) mtp_config = None if mtp_nextn > 0: - mtp_config = MTPDecodingConfig(num_nextn_predict_layers=mtp_nextn) + mtp_config = MTPDecodingConfig(max_draft_len=mtp_nextn) with LLM(self.MODEL_PATH, tensor_parallel_size=tp_size, pipeline_parallel_size=pp_size, @@ -1916,9 +1915,9 @@ def test_fp8_block_scales(self, mtp, fp8kv, attention_dp, cuda_graph, mtp_config = None mtp_nextn = 2 if mtp == "eagle": - mtp_config = MTPDecodingConfig(num_nextn_predict_layers=mtp_nextn) + mtp_config = MTPDecodingConfig(max_draft_len=mtp_nextn) elif mtp == "vanilla": - mtp_config = MTPDecodingConfig(num_nextn_predict_layers=mtp_nextn, + mtp_config = MTPDecodingConfig(max_draft_len=mtp_nextn, use_mtp_vanilla=True) with LLM(f"{llm_models_root()}/DeepSeek-V3-Lite/fp8", @@ -1964,7 +1963,7 @@ def test_cute_dsl_fp8_block_scales( mtp_config = None if mtp_nextn > 0: - mtp_config = MTPDecodingConfig(num_nextn_predict_layers=mtp_nextn) + mtp_config = MTPDecodingConfig(max_draft_len=mtp_nextn) with LLM( f"{llm_models_root()}/DeepSeek-V3-Lite/fp8", @@ -2021,7 +2020,7 @@ def test_fp8_block_scales_cuda_graph_padding(self, mtp_nextn): kv_cache_config = KvCacheConfig(free_gpu_memory_fraction=0.75) mtp_config = None if mtp_nextn > 0: - mtp_config = MTPDecodingConfig(num_nextn_predict_layers=mtp_nextn) + mtp_config = MTPDecodingConfig(max_draft_len=mtp_nextn) pytorch_config = dict( disable_overlap_scheduler=False, cuda_graph_config=CudaGraphConfig( @@ -2048,7 +2047,7 @@ def test_fp8_block_scales_cuda_graph_padding_4gpus(self, mtp_nextn, kv_cache_config = KvCacheConfig(free_gpu_memory_fraction=0.75) mtp_config = None if mtp_nextn > 0: - mtp_config = MTPDecodingConfig(num_nextn_predict_layers=mtp_nextn) + mtp_config = MTPDecodingConfig(max_draft_len=mtp_nextn) pytorch_config = dict( disable_overlap_scheduler=False, cuda_graph_config=CudaGraphConfig(enable_padding=True), @@ -2103,7 +2102,7 @@ def test_fp8_block_scales_4gpus(self, tp_size, pp_size, ep_size, mtp_nextn, mtp_config = None if mtp_nextn > 0: - mtp_config = MTPDecodingConfig(num_nextn_predict_layers=mtp_nextn) + mtp_config = MTPDecodingConfig(max_draft_len=mtp_nextn) with LLM(f"{llm_models_root()}/DeepSeek-V3-Lite/fp8", tensor_parallel_size=tp_size, @@ -2164,7 +2163,7 @@ def test_cute_dsl_fp8_block_scales_4gpus( mtp_config = None if mtp_nextn > 0: - mtp_config = MTPDecodingConfig(num_nextn_predict_layers=mtp_nextn) + mtp_config = MTPDecodingConfig(max_draft_len=mtp_nextn) with LLM( f"{llm_models_root()}/DeepSeek-V3-Lite/fp8", @@ -2275,7 +2274,7 @@ def test_bfloat16_4gpus_online_eplb(self, moe_backend, mtp_nextn): load_balancer=eplb_config)) mtp_config = None if mtp_nextn > 0: - mtp_config = MTPDecodingConfig(num_nextn_predict_layers=mtp_nextn) + mtp_config = MTPDecodingConfig(max_draft_len=mtp_nextn) with LLM(self.MODEL_PATH, tensor_parallel_size=4, moe_expert_parallel_size=4, @@ -2340,7 +2339,7 @@ def test_nvfp4(self, fp8kv, attention_dp, cuda_graph, overlap_scheduler, moe_config=MoeConfig(backend=moe_backend)) mtp_config = None if mtp_nextn > 0: - mtp_config = MTPDecodingConfig(num_nextn_predict_layers=mtp_nextn) + mtp_config = MTPDecodingConfig(max_draft_len=mtp_nextn) if fp8kv: kv_cache_config.dtype = "fp8" @@ -2479,7 +2478,7 @@ def test_nvfp4_batch_waiting(self, torch_compile, fp8kv, cuda_graph, moe_config=MoeConfig(backend=moe_backend)) mtp_config = None if mtp_nextn > 0: - mtp_config = MTPDecodingConfig(num_nextn_predict_layers=mtp_nextn) + mtp_config = MTPDecodingConfig(max_draft_len=mtp_nextn) if fp8kv: kv_cache_config.dtype = "fp8" with LLM(f"{llm_models_root()}/DeepSeek-V3-Lite/nvfp4_moe_only_mtp", @@ -2539,7 +2538,7 @@ def test_nvfp4_4gpus(self, fp8kv, attention_dp, cuda_graph, mtp_config = None if mtp_nextn > 0: - mtp_config = MTPDecodingConfig(num_nextn_predict_layers=mtp_nextn) + mtp_config = MTPDecodingConfig(max_draft_len=mtp_nextn) if fp8kv: kv_cache_config.dtype = "fp8" @@ -2591,7 +2590,7 @@ def test_no_kv_cache_reuse(self, quant_dtype, mtp_nextn, fp8kv, ) mtp_config = None if mtp_nextn > 0: - mtp_config = MTPDecodingConfig(num_nextn_predict_layers=mtp_nextn) + mtp_config = MTPDecodingConfig(max_draft_len=mtp_nextn) if quant_dtype == "none": assert not fp8kv @@ -2674,7 +2673,7 @@ def test_guided_decoding(self, backend: str, mtp_nextn: int, mocker): cuda_graph_config = CudaGraphConfig(enable_padding=True) mtp_config = None if mtp_nextn > 0: - mtp_config = MTPDecodingConfig(num_nextn_predict_layers=mtp_nextn) + mtp_config = MTPDecodingConfig(max_draft_len=mtp_nextn) llm = LLM(self.MODEL_PATH, guided_decoding_backend=backend, kv_cache_config=kv_cache_config, @@ -2696,7 +2695,7 @@ def test_guided_decoding_4gpus(self, backend: str, mtp_nextn: int, mocker): cuda_graph_config = CudaGraphConfig(enable_padding=True) mtp_config = None if mtp_nextn > 0: - mtp_config = MTPDecodingConfig(num_nextn_predict_layers=mtp_nextn) + mtp_config = MTPDecodingConfig(max_draft_len=mtp_nextn) llm = LLM(self.MODEL_PATH, tensor_parallel_size=4, moe_expert_parallel_size=4, @@ -2893,7 +2892,7 @@ def test_nvfp4_multi_gpus(self, tp_size, pp_size, ep_size, mtp_nextn, fp8kv, mtp_config = None if mtp_nextn > 0: - mtp_config = MTPDecodingConfig(num_nextn_predict_layers=mtp_nextn) + mtp_config = MTPDecodingConfig(max_draft_len=mtp_nextn) with LLM(f"{llm_models_root()}/DeepSeek-R1/DeepSeek-R1-FP4", max_batch_size=max_batch_size, tensor_parallel_size=tp_size, @@ -3031,7 +3030,7 @@ def test_nvfp4_multi_gpus_sm120(self, tp_size, pp_size, ep_size, mtp_nextn, mtp_config = None if mtp_nextn > 0: - mtp_config = MTPDecodingConfig(num_nextn_predict_layers=mtp_nextn) + mtp_config = MTPDecodingConfig(max_draft_len=mtp_nextn) with LLM(f"{llm_models_root()}/DeepSeek-R1/DeepSeek-R1-0528-FP4-v2", max_batch_size=max_batch_size, tensor_parallel_size=tp_size, @@ -3101,7 +3100,7 @@ def test_nvfp4_multi_gpus_chunked_prefill(self, tp_size, pp_size, ep_size, mtp_config = None if mtp_nextn > 0: - mtp_config = MTPDecodingConfig(num_nextn_predict_layers=mtp_nextn) + mtp_config = MTPDecodingConfig(max_draft_len=mtp_nextn) with LLM(f"{llm_models_root()}/DeepSeek-R1/DeepSeek-R1-FP4", max_batch_size=max_batch_size, tensor_parallel_size=tp_size, @@ -3139,7 +3138,7 @@ def test_nvfp4_multi_gpus_corner_case(self): enable_padding=True, max_batch_size=1024), moe_config=MoeConfig(backend="TRTLLM")) - mtp_config = MTPDecodingConfig(num_nextn_predict_layers=1) + mtp_config = MTPDecodingConfig(max_draft_len=1) with LLM(f"{llm_models_root()}/DeepSeek-R1/DeepSeek-R1-FP4", tensor_parallel_size=8, pipeline_parallel_size=1, @@ -3194,7 +3193,7 @@ def test_fp8_blockscale(self, tp_size, pp_size, ep_size, mtp_nextn, fp8kv, mtp_config = None if mtp_nextn > 0: - mtp_config = MTPDecodingConfig(num_nextn_predict_layers=mtp_nextn) + mtp_config = MTPDecodingConfig(max_draft_len=mtp_nextn) with LLM(f"{llm_models_root()}/DeepSeek-R1/DeepSeek-R1", max_batch_size=max_batch_size, tensor_parallel_size=tp_size, @@ -3241,7 +3240,7 @@ def test_fp8_blockscale_chunked_prefill(self, tp_size, pp_size, ep_size, mtp_config = None if mtp_nextn > 0: - mtp_config = MTPDecodingConfig(num_nextn_predict_layers=mtp_nextn) + mtp_config = MTPDecodingConfig(max_draft_len=mtp_nextn) with LLM(f"{llm_models_root()}/DeepSeek-R1/DeepSeek-R1", max_batch_size=max_batch_size, tensor_parallel_size=tp_size, @@ -3383,7 +3382,7 @@ def test_fp8_blockscale(self, tp_size, pp_size, ep_size, mtp_nextn, fp8kv, mtp_config = None if mtp_nextn > 0: - mtp_config = MTPDecodingConfig(num_nextn_predict_layers=mtp_nextn) + mtp_config = MTPDecodingConfig(max_draft_len=mtp_nextn) with LLM(f"{llm_models_root()}/DeepSeek-V3.2-Exp-hf", max_batch_size=max_batch_size, tensor_parallel_size=tp_size, @@ -3448,7 +3447,7 @@ def test_dsa_host_cache_offload(self, tp_size, pp_size, ep_size, mtp_nextn, mtp_config = None if mtp_nextn > 0: - mtp_config = MTPDecodingConfig(num_nextn_predict_layers=mtp_nextn) + mtp_config = MTPDecodingConfig(max_draft_len=mtp_nextn) with LLM(self.MODEL_PATH, max_batch_size=max_batch_size, @@ -3505,7 +3504,7 @@ def test_nvfp4_multi_gpus_piecewise_cuda_graph(self, tp_size, pp_size, mtp_config = None if mtp_nextn > 0: - mtp_config = MTPDecodingConfig(num_nextn_predict_layers=mtp_nextn) + mtp_config = MTPDecodingConfig(max_draft_len=mtp_nextn) llm_kwargs = dict( max_batch_size=max_batch_size, @@ -3575,7 +3574,7 @@ def test_nvfp4_multi_gpus(self, tp_size, pp_size, ep_size, mtp_nextn, fp8kv, mtp_config = None if mtp_nextn > 0: - mtp_config = MTPDecodingConfig(num_nextn_predict_layers=mtp_nextn) + mtp_config = MTPDecodingConfig(max_draft_len=mtp_nextn) with LLM(f"{llm_models_root()}/DeepSeek-V3.2-Exp-FP4-v2", max_batch_size=max_batch_size, tensor_parallel_size=tp_size, @@ -3651,7 +3650,7 @@ def test_nvfp4_multi_gpus_chunked_prefill(self, tp_size, pp_size, ep_size, kv_cache_config.dtype = "fp8" mtp_config = None if mtp_nextn > 0: - mtp_config = MTPDecodingConfig(num_nextn_predict_layers=mtp_nextn) + mtp_config = MTPDecodingConfig(max_draft_len=mtp_nextn) dsa_config = None if q_split_threshold is not None: @@ -3707,7 +3706,7 @@ def test_bfloat16_4gpus(self, tp_size, ep_size, mtp_nextn, mtp_config = None if mtp_nextn > 0: - mtp_config = MTPDecodingConfig(num_nextn_predict_layers=mtp_nextn) + mtp_config = MTPDecodingConfig(max_draft_len=mtp_nextn) with LLM(self.MODEL_PATH, max_batch_size=max_batch_size, @@ -3741,7 +3740,7 @@ def test_nvfp4_multi_gpus(self, tp_size, pp_size, mtp_nextn, cuda_graph, mtp_config = None if mtp_nextn > 0: - mtp_config = MTPDecodingConfig(num_nextn_predict_layers=mtp_nextn) + mtp_config = MTPDecodingConfig(max_draft_len=mtp_nextn) with LLM(f"{llm_models_root()}/glm-4.6-fp4", max_batch_size=max_batch_size, tensor_parallel_size=tp_size, @@ -3772,7 +3771,7 @@ def test_nvfp4_2_model_mtp(self, tp_size, cuda_graph, overlap_scheduler, cuda_graph_config=CudaGraphConfig() if cuda_graph else None, moe_config=MoeConfig(backend=moe_backend)) - mtp_config = MTPDecodingConfig(num_nextn_predict_layers=3, + mtp_config = MTPDecodingConfig(max_draft_len=3, mtp_eagle_one_model=False, speculative_model=model_path) @@ -3794,6 +3793,37 @@ class TestGLM4_5Air(LlmapiAccuracyTestHarness): MODEL_NAME = "zai-org/GLM-4.5-Air" MODEL_PATH = f"{llm_models_root()}/GLM-4.5-Air" + @pytest.mark.timeout(14400) + @pytest.mark.skip_less_device_memory(80000) + @pytest.mark.skip_less_device(2) + @parametrize_with_ids("mtp_nextn", [0, 2]) + @parametrize_with_ids("overlap_scheduler", [False, True]) + @parametrize_with_ids("tp_size, ep_size", [(2, 2), (2, 1)]) + @parametrize_with_ids("max_batch_size, moe_backend", [(4, "CUTLASS")]) + def test_bfloat16_2gpus(self, tp_size, ep_size, mtp_nextn, + overlap_scheduler, max_batch_size, moe_backend): + pytorch_config = dict( + disable_overlap_scheduler=not overlap_scheduler, + moe_config=MoeConfig(backend=moe_backend), + ) + kv_cache_config = KvCacheConfig(free_gpu_memory_fraction=0.70) + + mtp_config = None + if mtp_nextn > 0: + mtp_config = MTPDecodingConfig(max_draft_len=mtp_nextn) + + with LLM(self.MODEL_PATH, + max_batch_size=max_batch_size, + tensor_parallel_size=tp_size, + moe_expert_parallel_size=ep_size, + kv_cache_config=kv_cache_config, + enable_chunked_prefill=True, + max_num_tokens=512, + **pytorch_config, + speculative_config=mtp_config) as llm: + task = GSM8K(self.MODEL_NAME) + task.evaluate(llm) + @pytest.mark.skip_less_device(2) @pytest.mark.parametrize( "tp_size,pp_size,mtp_nextn,cuda_graph,overlap_scheduler,chunked_prefill,max_batch_size,moe_backend", @@ -3814,7 +3844,7 @@ def test_nvfp4_multi_gpus(self, tp_size, pp_size, mtp_nextn, cuda_graph, mtp_config = None if mtp_nextn > 0: - mtp_config = MTPDecodingConfig(num_nextn_predict_layers=mtp_nextn) + mtp_config = MTPDecodingConfig(max_draft_len=mtp_nextn) with LLM(f"{llm_models_root()}/glm-4.5-air-fp4", max_batch_size=max_batch_size, @@ -3847,7 +3877,7 @@ def test_nvfp4_2_model_mtp(self, tp_size, cuda_graph, overlap_scheduler, cuda_graph_config=CudaGraphConfig() if cuda_graph else None, moe_config=MoeConfig(backend=moe_backend)) - mtp_config = MTPDecodingConfig(num_nextn_predict_layers=3, + mtp_config = MTPDecodingConfig(max_draft_len=3, mtp_eagle_one_model=False, speculative_model_dir=model_path) @@ -6357,7 +6387,7 @@ def test_fp8_8gpus(self): cuda_graph_config = CudaGraphConfig(enable_padding=True, max_batch_size=32) - mtp_config = MTPDecodingConfig(num_nextn_predict_layers=3) + mtp_config = MTPDecodingConfig(max_draft_len=3) moe_config = MoeConfig(backend='DEEPGEMM', max_num_tokens=32000) @@ -6400,7 +6430,7 @@ def test_nvfp4_4gpus(self): cuda_graph_config = CudaGraphConfig(enable_padding=True, max_batch_size=32) - mtp_config = MTPDecodingConfig(num_nextn_predict_layers=3) + mtp_config = MTPDecodingConfig(max_draft_len=3) pytorch_config = dict(cuda_graph_config=cuda_graph_config, kv_cache_config=kv_cache_config, @@ -6974,7 +7004,7 @@ def test_nvfp4_8gpus_mtp(self): # Test MTP (Multi-Token Prediction) accuracy with nvfp4-fp8kv model. # This test uses MTP with max_draft_len=3 and one_model mode. mtp_config = MTPDecodingConfig( - num_nextn_predict_layers=3, + max_draft_len=3, mtp_eagle_one_model=True, ) model_path = f"{llm_models_root()}/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4" @@ -7009,7 +7039,7 @@ def test_nvfp4_8gpus_mtp(self): def test_nvfp4_4gpu_mtp_ar(self): max_draft_len = 7 mtp_config = MTPDecodingConfig( - num_nextn_predict_layers=max_draft_len, + max_draft_len=max_draft_len, mtp_eagle_one_model=True, ) model_path = f"{llm_models_root()}/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4" @@ -7074,7 +7104,7 @@ def test_nvfp4_4gpu_mtp_ar(self): def test_bf16_4gpu_mtp_ar(self): max_draft_len = 7 mtp_config = MTPDecodingConfig( - num_nextn_predict_layers=max_draft_len, + max_draft_len=max_draft_len, mtp_eagle_one_model=True, ) model_path = f"{llm_models_root()}/NVIDIA-Nemotron-3-Super-120B-A12B-BF16" @@ -7281,7 +7311,7 @@ def test_8gpus(self, tp_size, ep_size): cuda_graph_config=CudaGraphConfig(max_batch_size=128, enable_padding=True), moe_config=MoeConfig(backend="DEEPGEMM"), - speculative_config=MTPDecodingConfig(num_nextn_predict_layers=1), + speculative_config=MTPDecodingConfig(), enable_chunked_prefill=True, custom_tokenizer="glm_moe_dsa", ) diff --git a/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp1_gentp1_deepseek_v3_lite_one_mtp.yaml b/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp1_gentp1_deepseek_v3_lite_one_mtp.yaml index a7ecc70fedd1..43081ce00a83 100644 --- a/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp1_gentp1_deepseek_v3_lite_one_mtp.yaml +++ b/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp1_gentp1_deepseek_v3_lite_one_mtp.yaml @@ -6,7 +6,7 @@ cuda_graph_config: null disable_overlap_scheduler: true speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 context_servers: num_instances: 1 tensor_parallel_size: 1 diff --git a/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp1_gentp1_deepseek_v3_lite_one_mtp_attention_dp_overlap.yaml b/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp1_gentp1_deepseek_v3_lite_one_mtp_attention_dp_overlap.yaml index 1c3ed4091a24..77ebcc6a0585 100644 --- a/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp1_gentp1_deepseek_v3_lite_one_mtp_attention_dp_overlap.yaml +++ b/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp1_gentp1_deepseek_v3_lite_one_mtp_attention_dp_overlap.yaml @@ -5,7 +5,7 @@ backend: pytorch cuda_graph_config: null speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 context_servers: num_instances: 1 tensor_parallel_size: 1 diff --git a/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp1_gentp1_deepseek_v3_lite_one_mtp_ctxpp2_gentp2.yaml b/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp1_gentp1_deepseek_v3_lite_one_mtp_ctxpp2_gentp2.yaml index f75e014e858b..efbdc97c4f38 100644 --- a/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp1_gentp1_deepseek_v3_lite_one_mtp_ctxpp2_gentp2.yaml +++ b/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp1_gentp1_deepseek_v3_lite_one_mtp_ctxpp2_gentp2.yaml @@ -11,7 +11,7 @@ context_servers: enable_attention_dp: false speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 cache_transceiver_config: backend: DEFAULT generation_servers: @@ -21,6 +21,6 @@ generation_servers: enable_attention_dp: false speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 cache_transceiver_config: backend: DEFAULT diff --git a/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp1_gentp1_deepseek_v3_lite_two_mtp.yaml b/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp1_gentp1_deepseek_v3_lite_two_mtp.yaml index 154d65be9e7a..18cb600debcd 100644 --- a/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp1_gentp1_deepseek_v3_lite_two_mtp.yaml +++ b/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp1_gentp1_deepseek_v3_lite_two_mtp.yaml @@ -6,7 +6,7 @@ cuda_graph_config: null disable_overlap_scheduler: true speculative_config: decoding_type: MTP - num_nextn_predict_layers: 2 + max_draft_len: 2 context_servers: num_instances: 1 tensor_parallel_size: 1 diff --git a/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_deepseek_v3_lite_attention_dp_one_mtp.yaml b/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_deepseek_v3_lite_attention_dp_one_mtp.yaml index 7c44075406f7..3aa588aaa79a 100644 --- a/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_deepseek_v3_lite_attention_dp_one_mtp.yaml +++ b/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2_gentp2_deepseek_v3_lite_attention_dp_one_mtp.yaml @@ -6,7 +6,7 @@ cuda_graph_config: null disable_overlap_scheduler: true speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 context_servers: num_instances: 1 tensor_parallel_size: 2 diff --git a/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2ep2pp2_gentp4_deepseek_v3_lite_one_mtp_block_reuse.yaml b/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2ep2pp2_gentp4_deepseek_v3_lite_one_mtp_block_reuse.yaml index 8089d057e0b8..18eeab37fb73 100644 --- a/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2ep2pp2_gentp4_deepseek_v3_lite_one_mtp_block_reuse.yaml +++ b/tests/integration/defs/disaggregated/test_configs/disagg_config_ctxtp2ep2pp2_gentp4_deepseek_v3_lite_one_mtp_block_reuse.yaml @@ -12,7 +12,7 @@ context_servers: disable_overlap_scheduler: true speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 kv_cache_config: enable_block_reuse: true cache_transceiver_config: @@ -25,7 +25,7 @@ generation_servers: enable_attention_dp: true speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 kv_cache_config: enable_block_reuse: true cache_transceiver_config: diff --git a/tests/integration/defs/kv_cache/test_kv_cache_v2_scheduler.py b/tests/integration/defs/kv_cache/test_kv_cache_v2_scheduler.py index 6d971073a359..0d262a1a1466 100644 --- a/tests/integration/defs/kv_cache/test_kv_cache_v2_scheduler.py +++ b/tests/integration/defs/kv_cache/test_kv_cache_v2_scheduler.py @@ -506,7 +506,7 @@ def _compare(self, prompts, max_tokens=32, kv_extra=None, **llm_kwargs): prompts, sampling_params, kv_extra=kv_extra, - speculative_config=MTPDecodingConfig(num_nextn_predict_layers=2), + speculative_config=MTPDecodingConfig(max_draft_len=2), tensor_parallel_size=self.TP_SIZE, assert_outputs_match=False, **llm_kwargs, diff --git a/tests/integration/defs/perf/disagg/test_configs/README.md b/tests/integration/defs/perf/disagg/test_configs/README.md index 59c2a414bad3..32dd67f4e8ea 100644 --- a/tests/integration/defs/perf/disagg/test_configs/README.md +++ b/tests/integration/defs/perf/disagg/test_configs/README.md @@ -83,7 +83,7 @@ worker_config: max_batch_size: 32 max_seq_len: 2251 speculative_config: - num_nextn_predict_layers: 3 # MTP layers + max_draft_len: 3 # MTP layers ctx: tensor_parallel_size: 4 max_batch_size: 4 @@ -106,7 +106,7 @@ worker_config: | Tensor parallel size | `worker_config.gen.tensor_parallel_size` | | Batch size | `worker_config.gen.max_batch_size` | | Attention DP | `worker_config.gen.enable_attention_dp` | -| MTP layers | `worker_config.gen.speculative_config.num_nextn_predict_layers` | +| MTP layers | `worker_config.gen.speculative_config.max_draft_len` | | Backend | `worker_config.gen.cache_transceiver_config.backend` | | Concurrency levels | `benchmark.concurrency_list` | diff --git a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/Qwen3-235B-A22B-FP4_1k1k_ctx1_gen1_dep16_bs64_eplb0_mtp3_ccb-NIXL.yaml b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/Qwen3-235B-A22B-FP4_1k1k_ctx1_gen1_dep16_bs64_eplb0_mtp3_ccb-NIXL.yaml index 51dfec2283cb..a1981f0d1054 100644 --- a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/Qwen3-235B-A22B-FP4_1k1k_ctx1_gen1_dep16_bs64_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/Qwen3-235B-A22B-FP4_1k1k_ctx1_gen1_dep16_bs64_eplb0_mtp3_ccb-NIXL.yaml @@ -81,7 +81,7 @@ worker_config: num_postprocess_workers: 4 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 ctx: max_batch_size: 4 max_num_tokens: 4608 @@ -102,4 +102,4 @@ worker_config: backend: NIXL speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 diff --git a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/Qwen3-235B-A22B-FP4_1k1k_ctx1_gen1_dep16_bs64_eplb0_mtp3_ccb-UCX.yaml b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/Qwen3-235B-A22B-FP4_1k1k_ctx1_gen1_dep16_bs64_eplb0_mtp3_ccb-UCX.yaml index b35f58c08910..d046bf0cc683 100644 --- a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/Qwen3-235B-A22B-FP4_1k1k_ctx1_gen1_dep16_bs64_eplb0_mtp3_ccb-UCX.yaml +++ b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/Qwen3-235B-A22B-FP4_1k1k_ctx1_gen1_dep16_bs64_eplb0_mtp3_ccb-UCX.yaml @@ -81,7 +81,7 @@ worker_config: num_postprocess_workers: 4 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 ctx: max_batch_size: 4 max_num_tokens: 4608 @@ -102,4 +102,4 @@ worker_config: backend: UCX speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 diff --git a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/Qwen3-235B-A22B-FP4_1k1k_ctx1_gen1_dep32_bs16_eplb0_mtp3_ccb-NIXL.yaml b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/Qwen3-235B-A22B-FP4_1k1k_ctx1_gen1_dep32_bs16_eplb0_mtp3_ccb-NIXL.yaml index 327fb0cc9cea..2a68d991dabc 100644 --- a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/Qwen3-235B-A22B-FP4_1k1k_ctx1_gen1_dep32_bs16_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/Qwen3-235B-A22B-FP4_1k1k_ctx1_gen1_dep32_bs16_eplb0_mtp3_ccb-NIXL.yaml @@ -81,7 +81,7 @@ worker_config: num_postprocess_workers: 4 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 ctx: max_batch_size: 4 max_num_tokens: 4608 @@ -102,4 +102,4 @@ worker_config: backend: NIXL speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 diff --git a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/Qwen3-235B-A22B-FP4_1k1k_ctx1_gen1_dep32_bs16_eplb0_mtp3_ccb-UCX.yaml b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/Qwen3-235B-A22B-FP4_1k1k_ctx1_gen1_dep32_bs16_eplb0_mtp3_ccb-UCX.yaml index a94894ec11d2..a45aa96f9a20 100644 --- a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/Qwen3-235B-A22B-FP4_1k1k_ctx1_gen1_dep32_bs16_eplb0_mtp3_ccb-UCX.yaml +++ b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/Qwen3-235B-A22B-FP4_1k1k_ctx1_gen1_dep32_bs16_eplb0_mtp3_ccb-UCX.yaml @@ -81,7 +81,7 @@ worker_config: num_postprocess_workers: 4 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 ctx: max_batch_size: 4 max_num_tokens: 4608 @@ -102,4 +102,4 @@ worker_config: backend: UCX speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 diff --git a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/Qwen3-235B-A22B-FP4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp1_ccb-NIXL.yaml b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/Qwen3-235B-A22B-FP4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp1_ccb-NIXL.yaml index ec68a438e933..4448b2d1ccba 100644 --- a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/Qwen3-235B-A22B-FP4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/Qwen3-235B-A22B-FP4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp1_ccb-NIXL.yaml @@ -81,7 +81,7 @@ worker_config: num_postprocess_workers: 4 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 ctx: max_batch_size: 4 max_num_tokens: 4608 @@ -102,4 +102,4 @@ worker_config: backend: NIXL speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 diff --git a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/Qwen3-235B-A22B-FP4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp1_ccb-UCX.yaml b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/Qwen3-235B-A22B-FP4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp1_ccb-UCX.yaml index 9595a89f0428..3dbdfafe92ce 100644 --- a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/Qwen3-235B-A22B-FP4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp1_ccb-UCX.yaml +++ b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/Qwen3-235B-A22B-FP4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp1_ccb-UCX.yaml @@ -81,7 +81,7 @@ worker_config: num_postprocess_workers: 4 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 ctx: max_batch_size: 4 max_num_tokens: 4608 @@ -102,4 +102,4 @@ worker_config: backend: UCX speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 diff --git a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx1_pp4_gen6_tep8_bs1_eplb0_mtp3-Default.yaml b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx1_pp4_gen6_tep8_bs1_eplb0_mtp3-Default.yaml index d42c95a5d9c2..bbc5d1719ddd 100644 --- a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx1_pp4_gen6_tep8_bs1_eplb0_mtp3-Default.yaml +++ b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx1_pp4_gen6_tep8_bs1_eplb0_mtp3-Default.yaml @@ -78,7 +78,7 @@ worker_config: num_postprocess_workers: 4 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 allreduce_strategy: MNNVL ctx: max_batch_size: 1 @@ -100,6 +100,6 @@ worker_config: backend: DEFAULT speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 moe_config: backend: TRTLLM diff --git a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx1_pp8_gen1_dep16_bs1_eplb0_mtp3-Default.yaml b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx1_pp8_gen1_dep16_bs1_eplb0_mtp3-Default.yaml index 596d59eaded8..822197bc4c21 100644 --- a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx1_pp8_gen1_dep16_bs1_eplb0_mtp3-Default.yaml +++ b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx1_pp8_gen1_dep16_bs1_eplb0_mtp3-Default.yaml @@ -78,7 +78,7 @@ worker_config: num_postprocess_workers: 4 speculative_config: &id001 decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 ctx: max_batch_size: 1 max_num_tokens: 131104 diff --git a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx1_pp8_gen1_dep8_bs4_eplb0_mtp2-Default.yaml b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx1_pp8_gen1_dep8_bs4_eplb0_mtp2-Default.yaml index 59a51d8bd4ea..d12ed5452f24 100644 --- a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx1_pp8_gen1_dep8_bs4_eplb0_mtp2-Default.yaml +++ b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx1_pp8_gen1_dep8_bs4_eplb0_mtp2-Default.yaml @@ -78,7 +78,7 @@ worker_config: num_postprocess_workers: 4 speculative_config: &id001 decoding_type: MTP - num_nextn_predict_layers: 2 + max_draft_len: 2 ctx: max_batch_size: 1 max_num_tokens: 131104 diff --git a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx1_pp8_gen1_tep8_bs1_eplb0_mtp3-Default.yaml b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx1_pp8_gen1_tep8_bs1_eplb0_mtp3-Default.yaml index 23905d6eb27e..ad29ac42bacf 100644 --- a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx1_pp8_gen1_tep8_bs1_eplb0_mtp3-Default.yaml +++ b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx1_pp8_gen1_tep8_bs1_eplb0_mtp3-Default.yaml @@ -79,7 +79,7 @@ worker_config: allreduce_strategy: MNNVL speculative_config: &id001 decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 ctx: max_batch_size: 1 max_num_tokens: 131104 diff --git a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx1_pp8_gen1_tep8_bs2_eplb0_mtp3-Default.yaml b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx1_pp8_gen1_tep8_bs2_eplb0_mtp3-Default.yaml index 92b602b04bce..68669dcbdc75 100644 --- a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx1_pp8_gen1_tep8_bs2_eplb0_mtp3-Default.yaml +++ b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx1_pp8_gen1_tep8_bs2_eplb0_mtp3-Default.yaml @@ -79,7 +79,7 @@ worker_config: allreduce_strategy: MNNVL speculative_config: &id001 decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 ctx: max_batch_size: 1 max_num_tokens: 131104 diff --git a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx1_pp8_gen5_tep8_bs2_eplb0_mtp3-Default.yaml b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx1_pp8_gen5_tep8_bs2_eplb0_mtp3-Default.yaml index e9ac8a6f35a0..86385f6cb1b2 100644 --- a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx1_pp8_gen5_tep8_bs2_eplb0_mtp3-Default.yaml +++ b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx1_pp8_gen5_tep8_bs2_eplb0_mtp3-Default.yaml @@ -78,7 +78,7 @@ worker_config: allreduce_strategy: MNNVL speculative_config: &id001 decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 ctx: max_batch_size: 1 max_num_tokens: 131104 diff --git a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx1_pp8_gen7_tep4_bs2_eplb0_mtp2-Default.yaml b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx1_pp8_gen7_tep4_bs2_eplb0_mtp2-Default.yaml index 3561bf7ef826..2ba162a1974a 100644 --- a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx1_pp8_gen7_tep4_bs2_eplb0_mtp2-Default.yaml +++ b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx1_pp8_gen7_tep4_bs2_eplb0_mtp2-Default.yaml @@ -78,7 +78,7 @@ worker_config: allreduce_strategy: MNNVL speculative_config: &id001 decoding_type: MTP - num_nextn_predict_layers: 2 + max_draft_len: 2 ctx: max_batch_size: 1 max_num_tokens: 131104 diff --git a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx2_pp4_gen7_tep8_bs2_eplb0_mtp3-Default.yaml b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx2_pp4_gen7_tep8_bs2_eplb0_mtp3-Default.yaml index d2267b31541f..fe166f4f4a08 100644 --- a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx2_pp4_gen7_tep8_bs2_eplb0_mtp3-Default.yaml +++ b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx2_pp4_gen7_tep8_bs2_eplb0_mtp3-Default.yaml @@ -77,7 +77,7 @@ worker_config: num_postprocess_workers: 4 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 allreduce_strategy: MNNVL ctx: max_batch_size: 1 @@ -99,6 +99,6 @@ worker_config: backend: DEFAULT speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 moe_config: backend: TRTLLM diff --git a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx3_pp4_gen1_dep8_bs16_eplb0_mtp1-Default.yaml b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx3_pp4_gen1_dep8_bs16_eplb0_mtp1-Default.yaml index 7a58ab33bd8e..a3ec09af18ef 100644 --- a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx3_pp4_gen1_dep8_bs16_eplb0_mtp1-Default.yaml +++ b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx3_pp4_gen1_dep8_bs16_eplb0_mtp1-Default.yaml @@ -77,7 +77,7 @@ worker_config: num_postprocess_workers: 4 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 ctx: max_batch_size: 1 max_num_tokens: 131104 @@ -98,6 +98,6 @@ worker_config: backend: DEFAULT speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 moe_config: backend: TRTLLM diff --git a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx3_pp8_gen1_dep16_bs8_eplb0_mtp2-Default.yaml b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx3_pp8_gen1_dep16_bs8_eplb0_mtp2-Default.yaml index cc2910d0f3a0..5252a2baacc8 100644 --- a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx3_pp8_gen1_dep16_bs8_eplb0_mtp2-Default.yaml +++ b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx3_pp8_gen1_dep16_bs8_eplb0_mtp2-Default.yaml @@ -78,7 +78,7 @@ worker_config: num_postprocess_workers: 4 speculative_config: &id001 decoding_type: MTP - num_nextn_predict_layers: 2 + max_draft_len: 2 ctx: max_batch_size: 1 max_num_tokens: 131104 diff --git a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx3_pp8_gen1_dep32_bs2_eplb0_mtp3-Default.yaml b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx3_pp8_gen1_dep32_bs2_eplb0_mtp3-Default.yaml index 3ab7efb3eb74..0099bde7eb04 100644 --- a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx3_pp8_gen1_dep32_bs2_eplb0_mtp3-Default.yaml +++ b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx3_pp8_gen1_dep32_bs2_eplb0_mtp3-Default.yaml @@ -78,7 +78,7 @@ worker_config: num_postprocess_workers: 4 speculative_config: &id001 decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 ctx: max_batch_size: 1 max_num_tokens: 131104 diff --git a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx5_pp4_gen1_dep16_bs8_eplb0_mtp3-Default.yaml b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx5_pp4_gen1_dep16_bs8_eplb0_mtp3-Default.yaml index c28023ab7904..9909be5c83b8 100644 --- a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx5_pp4_gen1_dep16_bs8_eplb0_mtp3-Default.yaml +++ b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx5_pp4_gen1_dep16_bs8_eplb0_mtp3-Default.yaml @@ -78,7 +78,7 @@ worker_config: num_postprocess_workers: 4 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 ctx: max_batch_size: 1 max_num_tokens: 131104 @@ -99,6 +99,6 @@ worker_config: backend: DEFAULT speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 moe_config: backend: TRTLLM diff --git a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx5_pp4_gen1_dep32_bs2_eplb0_mtp3-Default.yaml b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx5_pp4_gen1_dep32_bs2_eplb0_mtp3-Default.yaml index 7ee380ef5e08..a719b585b012 100644 --- a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx5_pp4_gen1_dep32_bs2_eplb0_mtp3-Default.yaml +++ b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx5_pp4_gen1_dep32_bs2_eplb0_mtp3-Default.yaml @@ -78,7 +78,7 @@ worker_config: num_postprocess_workers: 4 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 ctx: max_batch_size: 1 max_num_tokens: 131104 @@ -99,6 +99,6 @@ worker_config: backend: DEFAULT speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 moe_config: backend: TRTLLM diff --git a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx7_pp4_gen1_dep16_bs16_eplb0_mtp1-Default.yaml b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx7_pp4_gen1_dep16_bs16_eplb0_mtp1-Default.yaml index 858552e3bcd9..9259a06a4400 100644 --- a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx7_pp4_gen1_dep16_bs16_eplb0_mtp1-Default.yaml +++ b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx7_pp4_gen1_dep16_bs16_eplb0_mtp1-Default.yaml @@ -78,7 +78,7 @@ worker_config: num_postprocess_workers: 4 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 ctx: max_batch_size: 1 max_num_tokens: 131104 @@ -99,6 +99,6 @@ worker_config: backend: DEFAULT speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 moe_config: backend: TRTLLM diff --git a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx8_pp4_gen1_dep16_bs32_eplb0_mtp1-Default.yaml b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx8_pp4_gen1_dep16_bs32_eplb0_mtp1-Default.yaml index ee5116aca14c..bf9b04aa1e02 100644 --- a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx8_pp4_gen1_dep16_bs32_eplb0_mtp1-Default.yaml +++ b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx8_pp4_gen1_dep16_bs32_eplb0_mtp1-Default.yaml @@ -78,7 +78,7 @@ worker_config: num_postprocess_workers: 4 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 ctx: max_batch_size: 1 max_num_tokens: 131104 @@ -99,6 +99,6 @@ worker_config: backend: DEFAULT speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 moe_config: backend: TRTLLM diff --git a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx8_pp4_gen1_dep32_bs4_eplb0_mtp3-Default.yaml b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx8_pp4_gen1_dep32_bs4_eplb0_mtp3-Default.yaml index f21e99012a7e..f02fc1688c05 100644 --- a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx8_pp4_gen1_dep32_bs4_eplb0_mtp3-Default.yaml +++ b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx8_pp4_gen1_dep32_bs4_eplb0_mtp3-Default.yaml @@ -78,7 +78,7 @@ worker_config: num_postprocess_workers: 4 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 ctx: max_batch_size: 1 max_num_tokens: 131104 @@ -99,6 +99,6 @@ worker_config: backend: DEFAULT speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 moe_config: backend: TRTLLM diff --git a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx8_pp4_gen1_dep32_bs8_eplb0_mtp3-Default.yaml b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx8_pp4_gen1_dep32_bs8_eplb0_mtp3-Default.yaml index 63fa58073619..d2a804e18dc9 100644 --- a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx8_pp4_gen1_dep32_bs8_eplb0_mtp3-Default.yaml +++ b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_128k8k_ctx8_pp4_gen1_dep32_bs8_eplb0_mtp3-Default.yaml @@ -78,7 +78,7 @@ worker_config: num_postprocess_workers: 4 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 ctx: max_batch_size: 1 max_num_tokens: 131104 @@ -99,6 +99,6 @@ worker_config: backend: DEFAULT speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 moe_config: backend: TRTLLM diff --git a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_1k1k_ctx1_gen4_tep8_bs32_eplb0_mtp3_ccb-NIXL.yaml b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_1k1k_ctx1_gen4_tep8_bs32_eplb0_mtp3_ccb-NIXL.yaml index f1bd56bbf1ca..7f8a5a4c1507 100644 --- a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_1k1k_ctx1_gen4_tep8_bs32_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_1k1k_ctx1_gen4_tep8_bs32_eplb0_mtp3_ccb-NIXL.yaml @@ -81,7 +81,7 @@ worker_config: num_postprocess_workers: 4 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 allreduce_strategy: MNNVL ctx: max_batch_size: 4 @@ -103,4 +103,4 @@ worker_config: backend: NIXL speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 diff --git a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_1k1k_ctx1_gen4_tep8_bs32_eplb0_mtp3_ccb-UCX.yaml b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_1k1k_ctx1_gen4_tep8_bs32_eplb0_mtp3_ccb-UCX.yaml index 806b10c51973..3e79d7f58b33 100644 --- a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_1k1k_ctx1_gen4_tep8_bs32_eplb0_mtp3_ccb-UCX.yaml +++ b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_1k1k_ctx1_gen4_tep8_bs32_eplb0_mtp3_ccb-UCX.yaml @@ -81,7 +81,7 @@ worker_config: num_postprocess_workers: 4 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 allreduce_strategy: MNNVL ctx: max_batch_size: 4 @@ -103,4 +103,4 @@ worker_config: backend: UCX speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 diff --git a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp3_ccb-NIXL.yaml b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp3_ccb-NIXL.yaml index 3cd56a5e25ae..aa544221ca76 100644 --- a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp3_ccb-NIXL.yaml @@ -81,7 +81,7 @@ worker_config: num_postprocess_workers: 4 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 ctx: max_batch_size: 4 max_num_tokens: 4608 @@ -102,4 +102,4 @@ worker_config: backend: NIXL speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 diff --git a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp3_ccb-UCX.yaml b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp3_ccb-UCX.yaml index 3c9278738e4c..b6a775a1903b 100644 --- a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp3_ccb-UCX.yaml +++ b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb0_mtp3_ccb-UCX.yaml @@ -81,7 +81,7 @@ worker_config: num_postprocess_workers: 4 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 ctx: max_batch_size: 4 max_num_tokens: 4608 @@ -102,4 +102,4 @@ worker_config: backend: UCX speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 diff --git a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_8k1k_ctx1_gen1_dep32_bs128_eplb0_mtp3_ccb-UCX.yaml b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_8k1k_ctx1_gen1_dep32_bs128_eplb0_mtp3_ccb-UCX.yaml index 2afc97ccceb4..2138d8884642 100644 --- a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_8k1k_ctx1_gen1_dep32_bs128_eplb0_mtp3_ccb-UCX.yaml +++ b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_8k1k_ctx1_gen1_dep32_bs128_eplb0_mtp3_ccb-UCX.yaml @@ -94,7 +94,7 @@ worker_config: num_postprocess_workers: 4 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 ctx: max_batch_size: 2 max_num_tokens: 16896 @@ -116,4 +116,4 @@ worker_config: backend: UCX speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 diff --git a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_8k1k_ctx1_gen3_tep8_bs16_eplb0_mtp3_ccb-NIXL.yaml b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_8k1k_ctx1_gen3_tep8_bs16_eplb0_mtp3_ccb-NIXL.yaml index e995c5e8077f..41918c3bd0e6 100644 --- a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_8k1k_ctx1_gen3_tep8_bs16_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_8k1k_ctx1_gen3_tep8_bs16_eplb0_mtp3_ccb-NIXL.yaml @@ -81,7 +81,7 @@ worker_config: num_postprocess_workers: 4 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 allreduce_strategy: MNNVL ctx: max_batch_size: 1 @@ -103,4 +103,4 @@ worker_config: backend: NIXL speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 diff --git a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_8k1k_ctx1_gen3_tep8_bs16_eplb0_mtp3_ccb-UCX.yaml b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_8k1k_ctx1_gen3_tep8_bs16_eplb0_mtp3_ccb-UCX.yaml index 98a9170b2184..c94b02793caa 100644 --- a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_8k1k_ctx1_gen3_tep8_bs16_eplb0_mtp3_ccb-UCX.yaml +++ b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_8k1k_ctx1_gen3_tep8_bs16_eplb0_mtp3_ccb-UCX.yaml @@ -81,7 +81,7 @@ worker_config: num_postprocess_workers: 4 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 allreduce_strategy: MNNVL ctx: max_batch_size: 1 @@ -103,4 +103,4 @@ worker_config: backend: UCX speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 diff --git a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_8k1k_ctx8_gen1_dep32_bs16_eplb0_mtp3_ccb-NIXL.yaml b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_8k1k_ctx8_gen1_dep32_bs16_eplb0_mtp3_ccb-NIXL.yaml index 29fa7e3eb95c..f6e164e55161 100644 --- a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_8k1k_ctx8_gen1_dep32_bs16_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_8k1k_ctx8_gen1_dep32_bs16_eplb0_mtp3_ccb-NIXL.yaml @@ -81,7 +81,7 @@ worker_config: num_postprocess_workers: 4 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 ctx: max_batch_size: 1 max_num_tokens: 8448 @@ -102,4 +102,4 @@ worker_config: backend: NIXL speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 diff --git a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_8k1k_ctx8_gen1_dep32_bs16_eplb0_mtp3_ccb-UCX.yaml b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_8k1k_ctx8_gen1_dep32_bs16_eplb0_mtp3_ccb-UCX.yaml index e9aa9d4ebb6b..2160f3669a46 100644 --- a/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_8k1k_ctx8_gen1_dep32_bs16_eplb0_mtp3_ccb-UCX.yaml +++ b/tests/integration/defs/perf/disagg/test_configs/disagg/perf/deepseek-r1-fp4_8k1k_ctx8_gen1_dep32_bs16_eplb0_mtp3_ccb-UCX.yaml @@ -81,7 +81,7 @@ worker_config: num_postprocess_workers: 4 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 ctx: max_batch_size: 1 max_num_tokens: 8448 @@ -102,4 +102,4 @@ worker_config: backend: UCX speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 diff --git a/tests/integration/defs/perf/disagg/test_configs/wideep/accuracy/deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_ccb-NIXL.yaml b/tests/integration/defs/perf/disagg/test_configs/wideep/accuracy/deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_ccb-NIXL.yaml index 4212ac4b68d3..3ff82079fc1e 100644 --- a/tests/integration/defs/perf/disagg/test_configs/wideep/accuracy/deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_ccb-NIXL.yaml +++ b/tests/integration/defs/perf/disagg/test_configs/wideep/accuracy/deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_ccb-NIXL.yaml @@ -100,7 +100,7 @@ worker_config: num_postprocess_workers: 4 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 ctx: enable_layerwise_nvtx_marker: true max_batch_size: 4 @@ -122,4 +122,4 @@ worker_config: backend: NIXL speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 diff --git a/tests/integration/defs/perf/disagg/test_configs/wideep/perf/Qwen3-235B-A22B-FP4_1k1k_ctx1_gen1_dep16_bs64_eplb288_mtp3_ccb-NIXL.yaml b/tests/integration/defs/perf/disagg/test_configs/wideep/perf/Qwen3-235B-A22B-FP4_1k1k_ctx1_gen1_dep16_bs64_eplb288_mtp3_ccb-NIXL.yaml index 51b0aad684cd..1b86768c105d 100644 --- a/tests/integration/defs/perf/disagg/test_configs/wideep/perf/Qwen3-235B-A22B-FP4_1k1k_ctx1_gen1_dep16_bs64_eplb288_mtp3_ccb-NIXL.yaml +++ b/tests/integration/defs/perf/disagg/test_configs/wideep/perf/Qwen3-235B-A22B-FP4_1k1k_ctx1_gen1_dep16_bs64_eplb288_mtp3_ccb-NIXL.yaml @@ -88,7 +88,7 @@ worker_config: num_postprocess_workers: 4 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 ctx: enable_layerwise_nvtx_marker: true max_batch_size: 4 @@ -110,4 +110,4 @@ worker_config: backend: NIXL speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 diff --git a/tests/integration/defs/perf/disagg/test_configs/wideep/perf/Qwen3-235B-A22B-FP4_1k1k_ctx1_gen1_dep16_bs64_eplb288_mtp3_ccb-UCX.yaml b/tests/integration/defs/perf/disagg/test_configs/wideep/perf/Qwen3-235B-A22B-FP4_1k1k_ctx1_gen1_dep16_bs64_eplb288_mtp3_ccb-UCX.yaml index 7ba7346da166..f7c781d5465e 100644 --- a/tests/integration/defs/perf/disagg/test_configs/wideep/perf/Qwen3-235B-A22B-FP4_1k1k_ctx1_gen1_dep16_bs64_eplb288_mtp3_ccb-UCX.yaml +++ b/tests/integration/defs/perf/disagg/test_configs/wideep/perf/Qwen3-235B-A22B-FP4_1k1k_ctx1_gen1_dep16_bs64_eplb288_mtp3_ccb-UCX.yaml @@ -88,7 +88,7 @@ worker_config: num_postprocess_workers: 4 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 ctx: enable_layerwise_nvtx_marker: true max_batch_size: 4 @@ -110,4 +110,4 @@ worker_config: backend: UCX speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 diff --git a/tests/integration/defs/perf/disagg/test_configs/wideep/perf/Qwen3-235B-A22B-FP4_1k1k_ctx1_gen1_dep32_bs16_eplb288_mtp3_ccb-NIXL.yaml b/tests/integration/defs/perf/disagg/test_configs/wideep/perf/Qwen3-235B-A22B-FP4_1k1k_ctx1_gen1_dep32_bs16_eplb288_mtp3_ccb-NIXL.yaml index aad5cb0fcbb9..88e96cbb9526 100644 --- a/tests/integration/defs/perf/disagg/test_configs/wideep/perf/Qwen3-235B-A22B-FP4_1k1k_ctx1_gen1_dep32_bs16_eplb288_mtp3_ccb-NIXL.yaml +++ b/tests/integration/defs/perf/disagg/test_configs/wideep/perf/Qwen3-235B-A22B-FP4_1k1k_ctx1_gen1_dep32_bs16_eplb288_mtp3_ccb-NIXL.yaml @@ -88,7 +88,7 @@ worker_config: num_postprocess_workers: 4 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 ctx: enable_layerwise_nvtx_marker: true max_batch_size: 4 @@ -110,4 +110,4 @@ worker_config: backend: NIXL speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 diff --git a/tests/integration/defs/perf/disagg/test_configs/wideep/perf/Qwen3-235B-A22B-FP4_1k1k_ctx1_gen1_dep32_bs16_eplb288_mtp3_ccb-UCX.yaml b/tests/integration/defs/perf/disagg/test_configs/wideep/perf/Qwen3-235B-A22B-FP4_1k1k_ctx1_gen1_dep32_bs16_eplb288_mtp3_ccb-UCX.yaml index 8f24dfc36069..ac97eb47a268 100644 --- a/tests/integration/defs/perf/disagg/test_configs/wideep/perf/Qwen3-235B-A22B-FP4_1k1k_ctx1_gen1_dep32_bs16_eplb288_mtp3_ccb-UCX.yaml +++ b/tests/integration/defs/perf/disagg/test_configs/wideep/perf/Qwen3-235B-A22B-FP4_1k1k_ctx1_gen1_dep32_bs16_eplb288_mtp3_ccb-UCX.yaml @@ -88,7 +88,7 @@ worker_config: num_postprocess_workers: 4 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 ctx: enable_layerwise_nvtx_marker: true max_batch_size: 4 @@ -110,4 +110,4 @@ worker_config: backend: UCX speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 diff --git a/tests/integration/defs/perf/disagg/test_configs/wideep/perf/Qwen3-235B-A22B-FP4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp1_ccb-NIXL.yaml b/tests/integration/defs/perf/disagg/test_configs/wideep/perf/Qwen3-235B-A22B-FP4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp1_ccb-NIXL.yaml index f6650f6a4368..a0f9cdd9ee5d 100644 --- a/tests/integration/defs/perf/disagg/test_configs/wideep/perf/Qwen3-235B-A22B-FP4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp1_ccb-NIXL.yaml +++ b/tests/integration/defs/perf/disagg/test_configs/wideep/perf/Qwen3-235B-A22B-FP4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp1_ccb-NIXL.yaml @@ -88,7 +88,7 @@ worker_config: num_postprocess_workers: 4 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 ctx: enable_layerwise_nvtx_marker: true max_batch_size: 4 @@ -110,4 +110,4 @@ worker_config: backend: NIXL speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 diff --git a/tests/integration/defs/perf/disagg/test_configs/wideep/perf/Qwen3-235B-A22B-FP4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp1_ccb-UCX.yaml b/tests/integration/defs/perf/disagg/test_configs/wideep/perf/Qwen3-235B-A22B-FP4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp1_ccb-UCX.yaml index 82f35fbe0fe5..84da1584e3db 100644 --- a/tests/integration/defs/perf/disagg/test_configs/wideep/perf/Qwen3-235B-A22B-FP4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp1_ccb-UCX.yaml +++ b/tests/integration/defs/perf/disagg/test_configs/wideep/perf/Qwen3-235B-A22B-FP4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp1_ccb-UCX.yaml @@ -88,7 +88,7 @@ worker_config: num_postprocess_workers: 4 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 ctx: enable_layerwise_nvtx_marker: true max_batch_size: 4 @@ -110,4 +110,4 @@ worker_config: backend: UCX speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 diff --git a/tests/integration/defs/perf/disagg/test_configs/wideep/perf/deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_ccb-NIXL.yaml b/tests/integration/defs/perf/disagg/test_configs/wideep/perf/deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_ccb-NIXL.yaml index 61fc9afc9862..a64a07551b3e 100644 --- a/tests/integration/defs/perf/disagg/test_configs/wideep/perf/deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_ccb-NIXL.yaml +++ b/tests/integration/defs/perf/disagg/test_configs/wideep/perf/deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_ccb-NIXL.yaml @@ -87,7 +87,7 @@ worker_config: num_postprocess_workers: 4 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 ctx: enable_layerwise_nvtx_marker: true max_batch_size: 4 @@ -109,4 +109,4 @@ worker_config: backend: NIXL speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 diff --git a/tests/integration/defs/perf/disagg/test_configs/wideep/perf/deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_ccb-UCX.yaml b/tests/integration/defs/perf/disagg/test_configs/wideep/perf/deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_ccb-UCX.yaml index 4ddae59d3838..c9aa1d9424b2 100644 --- a/tests/integration/defs/perf/disagg/test_configs/wideep/perf/deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_ccb-UCX.yaml +++ b/tests/integration/defs/perf/disagg/test_configs/wideep/perf/deepseek-r1-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_ccb-UCX.yaml @@ -87,7 +87,7 @@ worker_config: num_postprocess_workers: 4 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 ctx: enable_layerwise_nvtx_marker: true max_batch_size: 4 @@ -109,4 +109,4 @@ worker_config: backend: UCX speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 diff --git a/tests/integration/defs/perf/disagg/test_configs/wideep/perf/deepseek-r1-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_ccb-DEFAULT.yaml b/tests/integration/defs/perf/disagg/test_configs/wideep/perf/deepseek-r1-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_ccb-DEFAULT.yaml index 5902675ed7a2..d9ec4f718329 100644 --- a/tests/integration/defs/perf/disagg/test_configs/wideep/perf/deepseek-r1-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_ccb-DEFAULT.yaml +++ b/tests/integration/defs/perf/disagg/test_configs/wideep/perf/deepseek-r1-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_ccb-DEFAULT.yaml @@ -88,7 +88,7 @@ worker_config: num_postprocess_workers: 4 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 ctx: enable_layerwise_nvtx_marker: true max_batch_size: 1 @@ -110,4 +110,4 @@ worker_config: backend: DEFAULT speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 diff --git a/tests/integration/defs/perf/disagg/test_configs/wideep/perf/deepseek-r1-fp4_8k1k_ctx8_gen1_dep32_bs16_eplb288_mtp3_ccb-NIXL.yaml b/tests/integration/defs/perf/disagg/test_configs/wideep/perf/deepseek-r1-fp4_8k1k_ctx8_gen1_dep32_bs16_eplb288_mtp3_ccb-NIXL.yaml index 95e1bc24edc4..aebc98fac87b 100644 --- a/tests/integration/defs/perf/disagg/test_configs/wideep/perf/deepseek-r1-fp4_8k1k_ctx8_gen1_dep32_bs16_eplb288_mtp3_ccb-NIXL.yaml +++ b/tests/integration/defs/perf/disagg/test_configs/wideep/perf/deepseek-r1-fp4_8k1k_ctx8_gen1_dep32_bs16_eplb288_mtp3_ccb-NIXL.yaml @@ -87,7 +87,7 @@ worker_config: num_postprocess_workers: 4 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 ctx: enable_layerwise_nvtx_marker: true max_batch_size: 1 @@ -109,4 +109,4 @@ worker_config: backend: NIXL speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 diff --git a/tests/integration/defs/perf/disagg/test_configs/wideep/perf/deepseek-r1-fp4_8k1k_ctx8_gen1_dep32_bs16_eplb288_mtp3_ccb-UCX.yaml b/tests/integration/defs/perf/disagg/test_configs/wideep/perf/deepseek-r1-fp4_8k1k_ctx8_gen1_dep32_bs16_eplb288_mtp3_ccb-UCX.yaml index 2c41527735b5..92314d994eb2 100644 --- a/tests/integration/defs/perf/disagg/test_configs/wideep/perf/deepseek-r1-fp4_8k1k_ctx8_gen1_dep32_bs16_eplb288_mtp3_ccb-UCX.yaml +++ b/tests/integration/defs/perf/disagg/test_configs/wideep/perf/deepseek-r1-fp4_8k1k_ctx8_gen1_dep32_bs16_eplb288_mtp3_ccb-UCX.yaml @@ -87,7 +87,7 @@ worker_config: num_postprocess_workers: 4 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 ctx: enable_layerwise_nvtx_marker: true max_batch_size: 1 @@ -109,4 +109,4 @@ worker_config: backend: UCX speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 diff --git a/tests/integration/defs/perf/disagg/test_configs/wideep/perf/deepseek-v32-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_ccb-NIXL.yaml b/tests/integration/defs/perf/disagg/test_configs/wideep/perf/deepseek-v32-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_ccb-NIXL.yaml index b4ca10f1e930..762bee22a322 100644 --- a/tests/integration/defs/perf/disagg/test_configs/wideep/perf/deepseek-v32-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_ccb-NIXL.yaml +++ b/tests/integration/defs/perf/disagg/test_configs/wideep/perf/deepseek-v32-fp4_1k1k_ctx2_gen1_dep16_bs128_eplb288_mtp3_ccb-NIXL.yaml @@ -94,7 +94,7 @@ worker_config: num_postprocess_workers: 4 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 ctx: max_batch_size: 4 max_num_tokens: 4608 @@ -117,4 +117,4 @@ worker_config: backend: NIXL speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 diff --git a/tests/integration/defs/perf/disagg/test_configs/wideep/perf/deepseek-v32-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_ccb-DEFAULT.yaml b/tests/integration/defs/perf/disagg/test_configs/wideep/perf/deepseek-v32-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_ccb-DEFAULT.yaml index ddbb4bf48d6a..58fd6a32e711 100644 --- a/tests/integration/defs/perf/disagg/test_configs/wideep/perf/deepseek-v32-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_ccb-DEFAULT.yaml +++ b/tests/integration/defs/perf/disagg/test_configs/wideep/perf/deepseek-v32-fp4_8k1k_ctx2_gen1_dep32_bs128_eplb288_mtp3_ccb-DEFAULT.yaml @@ -95,7 +95,7 @@ worker_config: num_postprocess_workers: 4 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 ctx: max_batch_size: 1 max_num_tokens: 8448 @@ -118,4 +118,4 @@ worker_config: backend: DEFAULT speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 diff --git a/tests/integration/defs/perf/disagg/test_configs/wideep/perf/deepseek-v32-fp4_8k1k_ctx8_gen1_dep32_bs16_eplb288_mtp3_ccb-NIXL.yaml b/tests/integration/defs/perf/disagg/test_configs/wideep/perf/deepseek-v32-fp4_8k1k_ctx8_gen1_dep32_bs16_eplb288_mtp3_ccb-NIXL.yaml index df5cbff3e240..1cc7c69cad8b 100644 --- a/tests/integration/defs/perf/disagg/test_configs/wideep/perf/deepseek-v32-fp4_8k1k_ctx8_gen1_dep32_bs16_eplb288_mtp3_ccb-NIXL.yaml +++ b/tests/integration/defs/perf/disagg/test_configs/wideep/perf/deepseek-v32-fp4_8k1k_ctx8_gen1_dep32_bs16_eplb288_mtp3_ccb-NIXL.yaml @@ -94,7 +94,7 @@ worker_config: num_postprocess_workers: 4 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 ctx: max_batch_size: 1 max_num_tokens: 8448 @@ -117,4 +117,4 @@ worker_config: backend: NIXL speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 diff --git a/tests/integration/defs/perf/disagg/utils/common.py b/tests/integration/defs/perf/disagg/utils/common.py index 48eb30581b82..09c759fa7f00 100644 --- a/tests/integration/defs/perf/disagg/utils/common.py +++ b/tests/integration/defs/perf/disagg/utils/common.py @@ -322,7 +322,8 @@ def extract_config_fields(config_data: dict) -> dict: gen_config = config_data["worker_config"]["gen"] mtp_size = 0 if "speculative_config" in gen_config: - mtp_size = gen_config["speculative_config"].get("num_nextn_predict_layers", 0) + spec_config = gen_config["speculative_config"] + mtp_size = spec_config.get("max_draft_len", 0) return { "isl": isl, diff --git a/tests/integration/defs/perf/pytorch_model_config.py b/tests/integration/defs/perf/pytorch_model_config.py index 22d3997d6f6a..d79c3974b965 100644 --- a/tests/integration/defs/perf/pytorch_model_config.py +++ b/tests/integration/defs/perf/pytorch_model_config.py @@ -77,7 +77,7 @@ def get_model_yaml_config(model_label: str, 'cuda_graph_config': {}, 'speculative_config': { 'decoding_type': 'MTP', - 'num_nextn_predict_layers': 3 + 'max_draft_len': 3 } } }, @@ -97,7 +97,7 @@ def get_model_yaml_config(model_label: str, }, 'speculative_config': { 'decoding_type': 'MTP', - 'num_nextn_predict_layers': 3 + 'max_draft_len': 3 }, 'disable_overlap_scheduler': True, 'enable_autotuner': True, diff --git a/tests/integration/defs/perf/test_perf_sanity.py b/tests/integration/defs/perf/test_perf_sanity.py index 3ce76105bd0c..b50c98860494 100644 --- a/tests/integration/defs/perf/test_perf_sanity.py +++ b/tests/integration/defs/perf/test_perf_sanity.py @@ -296,7 +296,12 @@ def __init__(self, server_config_data: dict, env_vars: str = ""): # speculative_config speculative_config = server_config_data.get("speculative_config", {}) self.spec_decoding_type = speculative_config.get("decoding_type", "") - self.num_nextn_predict_layers = speculative_config.get("num_nextn_predict_layers", 0) + # MTP user config migrated from num_nextn_predict_layers to max_draft_len. + # Keep both DB field names populated for perf/baseline compatibility while + # the surrounding reporting pipeline transitions to l_max_draft_len. + self.max_draft_len = speculative_config.get( + "max_draft_len", speculative_config.get("num_nextn_predict_layers", 0) + ) eagle3_value = speculative_config.get("eagle3_layers_to_capture", []) if isinstance(eagle3_value, int): self.eagle3_layers_to_capture = [eagle3_value] @@ -304,7 +309,6 @@ def __init__(self, server_config_data: dict, env_vars: str = ""): self.eagle3_layers_to_capture = eagle3_value else: self.eagle3_layers_to_capture = [] - self.max_draft_len = speculative_config.get("max_draft_len", 0) self.speculative_model = speculative_config.get("speculative_model", "") self.eagle3_one_model = speculative_config.get("eagle3_one_model", False) @@ -372,6 +376,9 @@ def to_match_keys(self) -> List[str]: # cache_transceiver_config "s_cache_transceiver_backend", # speculative_config + # Keep baseline matching on the legacy key during DB migration. + # l_max_draft_len is written to DB but not used for matching until + # backfill completes. "s_spec_decoding_type", "l_num_nextn_predict_layers", ] @@ -423,7 +430,7 @@ def to_db_data(self) -> dict: "l_cache_transceiver_max_tokens_in_buffer": self.cache_transceiver_max_tokens_in_buffer, # speculative_config "s_spec_decoding_type": self.spec_decoding_type, - "l_num_nextn_predict_layers": self.num_nextn_predict_layers, + "l_num_nextn_predict_layers": self.max_draft_len, "s_eagle3_layers_to_capture": ",".join(map(str, self.eagle3_layers_to_capture)), "l_max_draft_len": self.max_draft_len, "s_speculative_model_dir": self.speculative_model, diff --git a/tests/integration/defs/test_e2e.py b/tests/integration/defs/test_e2e.py index 3166ca140520..65b4ccedfd4a 100644 --- a/tests/integration/defs/test_e2e.py +++ b/tests/integration/defs/test_e2e.py @@ -1811,7 +1811,7 @@ def test_deepseek_r1_mtp_bench(llm_root, llm_venv): }, "speculative_config": { "decoding_type": "MTP", - "num_nextn_predict_layers": 1, + "max_draft_len": 1, }, } diff --git a/tests/scripts/perf-sanity/aggregated/config_database_b200_nvl.yaml b/tests/scripts/perf-sanity/aggregated/config_database_b200_nvl.yaml index 2a3e8b3549ee..f751033a0b6e 100644 --- a/tests/scripts/perf-sanity/aggregated/config_database_b200_nvl.yaml +++ b/tests/scripts/perf-sanity/aggregated/config_database_b200_nvl.yaml @@ -17,7 +17,7 @@ server_configs: backend: TRTLLM speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true @@ -50,7 +50,7 @@ server_configs: backend: TRTLLM speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true @@ -88,7 +88,7 @@ server_configs: timeout_iters: 60 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true @@ -121,7 +121,7 @@ server_configs: backend: TRTLLM speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true @@ -154,7 +154,7 @@ server_configs: backend: TRTLLM speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true @@ -191,7 +191,7 @@ server_configs: timeout_iters: 60 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true @@ -224,7 +224,7 @@ server_configs: backend: TRTLLM speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true @@ -262,7 +262,7 @@ server_configs: timeout_iters: 100 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true @@ -300,7 +300,7 @@ server_configs: timeout_iters: 100 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true @@ -333,7 +333,7 @@ server_configs: backend: TRTLLM speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true @@ -365,7 +365,7 @@ server_configs: backend: TRTLLM speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true @@ -432,7 +432,7 @@ server_configs: backend: TRTLLM speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true @@ -465,7 +465,7 @@ server_configs: backend: TRTLLM speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true @@ -503,7 +503,7 @@ server_configs: timeout_iters: 60 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true @@ -540,7 +540,7 @@ server_configs: timeout_iters: 60 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true @@ -572,7 +572,7 @@ server_configs: backend: TRTLLM speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true @@ -605,7 +605,7 @@ server_configs: backend: TRTLLM speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true @@ -641,7 +641,7 @@ server_configs: timeout_iters: 60 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true @@ -679,7 +679,7 @@ server_configs: timeout_iters: 60 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true @@ -717,7 +717,7 @@ server_configs: timeout_iters: 60 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 tensor_parallel_size: 4 moe_expert_parallel_size: 4 trust_remote_code: true diff --git a/tests/scripts/perf-sanity/aggregated/config_database_h200_sxm.yaml b/tests/scripts/perf-sanity/aggregated/config_database_h200_sxm.yaml index 9179ca664585..d28014bc0027 100644 --- a/tests/scripts/perf-sanity/aggregated/config_database_h200_sxm.yaml +++ b/tests/scripts/perf-sanity/aggregated/config_database_h200_sxm.yaml @@ -17,7 +17,7 @@ server_configs: backend: CUTLASS speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true @@ -50,7 +50,7 @@ server_configs: backend: CUTLASS speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true @@ -88,7 +88,7 @@ server_configs: timeout_iters: 60 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true @@ -121,7 +121,7 @@ server_configs: backend: CUTLASS speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true @@ -154,7 +154,7 @@ server_configs: backend: CUTLASS speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true @@ -191,7 +191,7 @@ server_configs: timeout_iters: 60 speculative_config: decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true @@ -224,7 +224,7 @@ server_configs: backend: CUTLASS speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true @@ -257,7 +257,7 @@ server_configs: backend: CUTLASS speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true @@ -290,7 +290,7 @@ server_configs: backend: CUTLASS speculative_config: decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 tensor_parallel_size: 8 moe_expert_parallel_size: 8 trust_remote_code: true diff --git a/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp4_v2_2_nodes_grace_blackwell.yaml b/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp4_v2_2_nodes_grace_blackwell.yaml index 0e194a0b1997..b3bee56a5a8a 100644 --- a/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp4_v2_2_nodes_grace_blackwell.yaml +++ b/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp4_v2_2_nodes_grace_blackwell.yaml @@ -32,7 +32,7 @@ server_configs: free_gpu_memory_fraction: 0.8 speculative_config: decoding_type: 'MTP' - num_nextn_predict_layers: 1 + max_draft_len: 1 client_configs: - name: "con1024_iter10_1k1k" concurrency: 1024 @@ -69,7 +69,7 @@ server_configs: free_gpu_memory_fraction: 0.8 speculative_config: decoding_type: 'MTP' - num_nextn_predict_layers: 1 + max_draft_len: 1 client_configs: - name: "con1024_iter10_8k1k" concurrency: 1024 @@ -100,7 +100,7 @@ server_configs: free_gpu_memory_fraction: 0.5 speculative_config: decoding_type: 'MTP' - num_nextn_predict_layers: 3 + max_draft_len: 3 client_configs: - name: "con32_iter12_1k1k" concurrency: 32 diff --git a/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp4_v2_blackwell.yaml b/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp4_v2_blackwell.yaml index df123fbc5df5..f92f3d80f876 100644 --- a/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp4_v2_blackwell.yaml +++ b/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp4_v2_blackwell.yaml @@ -26,7 +26,7 @@ server_configs: free_gpu_memory_fraction: 0.8 speculative_config: decoding_type: 'MTP' - num_nextn_predict_layers: 3 + max_draft_len: 3 client_configs: - name: "con2_iter10_1k1k" concurrency: 2 @@ -62,7 +62,7 @@ server_configs: free_gpu_memory_fraction: 0.8 speculative_config: decoding_type: 'MTP' - num_nextn_predict_layers: 1 + max_draft_len: 1 client_configs: - name: "con1024_iter10_1k1k" concurrency: 1024 @@ -93,7 +93,7 @@ server_configs: free_gpu_memory_fraction: 0.8 speculative_config: decoding_type: 'MTP' - num_nextn_predict_layers: 3 + max_draft_len: 3 client_configs: - name: "con4_iter10_8k1k" concurrency: 4 @@ -129,7 +129,7 @@ server_configs: free_gpu_memory_fraction: 0.8 speculative_config: decoding_type: 'MTP' - num_nextn_predict_layers: 1 + max_draft_len: 1 client_configs: - name: "con256_iter10_8k1k" concurrency: 256 diff --git a/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp4_v2_grace_blackwell.yaml b/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp4_v2_grace_blackwell.yaml index 621a7542a60d..fad67e46ba77 100644 --- a/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp4_v2_grace_blackwell.yaml +++ b/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp4_v2_grace_blackwell.yaml @@ -32,7 +32,7 @@ server_configs: free_gpu_memory_fraction: 0.8 speculative_config: decoding_type: 'MTP' - num_nextn_predict_layers: 1 + max_draft_len: 1 client_configs: - name: "con1024_iter10_1k1k" concurrency: 1024 @@ -63,7 +63,7 @@ server_configs: free_gpu_memory_fraction: 0.8 speculative_config: decoding_type: 'MTP' - num_nextn_predict_layers: 3 + max_draft_len: 3 client_configs: - name: "con32_iter10_1k1k" concurrency: 32 @@ -94,7 +94,7 @@ server_configs: free_gpu_memory_fraction: 0.8 speculative_config: decoding_type: 'MTP' - num_nextn_predict_layers: 3 + max_draft_len: 3 client_configs: - name: "con2_iter10_1k1k" concurrency: 2 @@ -130,7 +130,7 @@ server_configs: free_gpu_memory_fraction: 0.8 speculative_config: decoding_type: 'MTP' - num_nextn_predict_layers: 1 + max_draft_len: 1 client_configs: - name: "con256_iter10_8k1k" concurrency: 256 @@ -161,7 +161,7 @@ server_configs: free_gpu_memory_fraction: 0.8 speculative_config: decoding_type: 'MTP' - num_nextn_predict_layers: 3 + max_draft_len: 3 client_configs: - name: "con32_iter10_8k1k" concurrency: 32 @@ -192,7 +192,7 @@ server_configs: free_gpu_memory_fraction: 0.8 speculative_config: decoding_type: 'MTP' - num_nextn_predict_layers: 3 + max_draft_len: 3 client_configs: - name: "con2_iter10_8k1k" concurrency: 2 @@ -228,7 +228,7 @@ server_configs: free_gpu_memory_fraction: 0.8 speculative_config: decoding_type: 'MTP' - num_nextn_predict_layers: 1 + max_draft_len: 1 client_configs: - name: "con2048_iter5_1k8k" concurrency: 2048 @@ -259,7 +259,7 @@ server_configs: free_gpu_memory_fraction: 0.8 speculative_config: decoding_type: 'MTP' - num_nextn_predict_layers: 3 + max_draft_len: 3 client_configs: - name: "con32_iter10_1k8k" concurrency: 32 @@ -290,7 +290,7 @@ server_configs: free_gpu_memory_fraction: 0.8 speculative_config: decoding_type: 'MTP' - num_nextn_predict_layers: 3 + max_draft_len: 3 client_configs: - name: "con4_iter10_1k8k" concurrency: 4 diff --git a/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp8_blackwell.yaml b/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp8_blackwell.yaml index 7e83196a86d7..22dc57e9c93d 100644 --- a/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp8_blackwell.yaml +++ b/tests/scripts/perf-sanity/aggregated/deepseek_r1_fp8_blackwell.yaml @@ -26,7 +26,7 @@ server_configs: free_gpu_memory_fraction: 0.8 speculative_config: decoding_type: 'MTP' - num_nextn_predict_layers: 3 + max_draft_len: 3 client_configs: - name: "con4_iter10_1k1k" concurrency: 4 @@ -62,7 +62,7 @@ server_configs: free_gpu_memory_fraction: 0.8 speculative_config: decoding_type: 'MTP' - num_nextn_predict_layers: 1 + max_draft_len: 1 client_configs: - name: "con1024_iter10_1k1k" concurrency: 1024 @@ -93,7 +93,7 @@ server_configs: free_gpu_memory_fraction: 0.8 speculative_config: decoding_type: 'MTP' - num_nextn_predict_layers: 3 + max_draft_len: 3 client_configs: - name: "con4_iter10_8k1k" concurrency: 4 @@ -129,7 +129,7 @@ server_configs: free_gpu_memory_fraction: 0.8 speculative_config: decoding_type: 'MTP' - num_nextn_predict_layers: 1 + max_draft_len: 1 client_configs: - name: "con256_iter10_8k1k" concurrency: 256 diff --git a/tests/scripts/perf-sanity/aggregated/deepseek_v32_fp4_blackwell.yaml b/tests/scripts/perf-sanity/aggregated/deepseek_v32_fp4_blackwell.yaml index b45d899d9e41..a8407ed03c8c 100644 --- a/tests/scripts/perf-sanity/aggregated/deepseek_v32_fp4_blackwell.yaml +++ b/tests/scripts/perf-sanity/aggregated/deepseek_v32_fp4_blackwell.yaml @@ -26,7 +26,7 @@ server_configs: free_gpu_memory_fraction: 0.8 speculative_config: decoding_type: 'MTP' - num_nextn_predict_layers: 3 + max_draft_len: 3 client_configs: - name: "con2_iter10_8k1k" concurrency: 2 @@ -62,7 +62,7 @@ server_configs: free_gpu_memory_fraction: 0.8 speculative_config: decoding_type: 'MTP' - num_nextn_predict_layers: 1 + max_draft_len: 1 client_configs: - name: "con256_iter10_8k1k" concurrency: 256 diff --git a/tests/scripts/perf-sanity/aggregated/deepseek_v32_fp4_grace_blackwell.yaml b/tests/scripts/perf-sanity/aggregated/deepseek_v32_fp4_grace_blackwell.yaml index d8b9d1735540..a6f7206d5054 100644 --- a/tests/scripts/perf-sanity/aggregated/deepseek_v32_fp4_grace_blackwell.yaml +++ b/tests/scripts/perf-sanity/aggregated/deepseek_v32_fp4_grace_blackwell.yaml @@ -26,7 +26,7 @@ server_configs: free_gpu_memory_fraction: 0.8 speculative_config: decoding_type: 'MTP' - num_nextn_predict_layers: 3 + max_draft_len: 3 client_configs: - name: "con2_iter10_1k1k" concurrency: 2 @@ -62,7 +62,7 @@ server_configs: free_gpu_memory_fraction: 0.8 speculative_config: decoding_type: 'MTP' - num_nextn_predict_layers: 1 + max_draft_len: 1 client_configs: - name: "con1024_iter10_1k1k" concurrency: 1024 @@ -93,7 +93,7 @@ server_configs: free_gpu_memory_fraction: 0.8 speculative_config: decoding_type: 'MTP' - num_nextn_predict_layers: 3 + max_draft_len: 3 client_configs: - name: "con2_iter10_8k1k" concurrency: 2 @@ -129,7 +129,7 @@ server_configs: free_gpu_memory_fraction: 0.8 speculative_config: decoding_type: 'MTP' - num_nextn_predict_layers: 1 + max_draft_len: 1 client_configs: - name: "con256_iter10_8k1k" concurrency: 256 diff --git a/tests/scripts/perf-sanity/aggregated/gb300_deepseek_r1_fp4_v2_2_nodes_grace_blackwell.yaml b/tests/scripts/perf-sanity/aggregated/gb300_deepseek_r1_fp4_v2_2_nodes_grace_blackwell.yaml index 465a4abf8a4c..2d4822a3e121 100644 --- a/tests/scripts/perf-sanity/aggregated/gb300_deepseek_r1_fp4_v2_2_nodes_grace_blackwell.yaml +++ b/tests/scripts/perf-sanity/aggregated/gb300_deepseek_r1_fp4_v2_2_nodes_grace_blackwell.yaml @@ -32,7 +32,7 @@ server_configs: free_gpu_memory_fraction: 0.8 speculative_config: decoding_type: 'MTP' - num_nextn_predict_layers: 1 + max_draft_len: 1 client_configs: - name: "con1024_iter10_1k1k" concurrency: 1024 @@ -69,7 +69,7 @@ server_configs: free_gpu_memory_fraction: 0.8 speculative_config: decoding_type: 'MTP' - num_nextn_predict_layers: 1 + max_draft_len: 1 client_configs: - name: "con1024_iter10_8k1k" concurrency: 1024 @@ -100,7 +100,7 @@ server_configs: free_gpu_memory_fraction: 0.5 speculative_config: decoding_type: 'MTP' - num_nextn_predict_layers: 3 + max_draft_len: 3 client_configs: - name: "con32_iter12_1k1k" concurrency: 32 diff --git a/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index 94e8e0e4b4a2..51ecc9a9593f 100644 --- a/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -69,7 +69,7 @@ worker_config: disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 num_postprocess_workers: 4 stream_interval: 20 ctx: diff --git a/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_1k1k_con2048_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_1k1k_con2048_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml index ac5c36f16665..e069a98b1a99 100644 --- a/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_1k1k_con2048_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_1k1k_con2048_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml @@ -71,7 +71,7 @@ worker_config: disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 num_postprocess_workers: 4 stream_interval: 20 ctx: diff --git a/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_1k1k_con256_ctx1_dep4_gen1_dep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_1k1k_con256_ctx1_dep4_gen1_dep8_eplb0_mtp3_ccb-NIXL.yaml index 27ce86b321ef..7e32761afbc0 100644 --- a/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_1k1k_con256_ctx1_dep4_gen1_dep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_1k1k_con256_ctx1_dep4_gen1_dep8_eplb0_mtp3_ccb-NIXL.yaml @@ -71,7 +71,7 @@ worker_config: disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 num_postprocess_workers: 4 stream_interval: 20 ctx: diff --git a/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_8k1k_con1536_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_8k1k_con1536_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml index f986eeda6983..9026428e15b3 100644 --- a/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_8k1k_con1536_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_8k1k_con1536_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml @@ -71,7 +71,7 @@ worker_config: disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 num_postprocess_workers: 4 stream_interval: 20 ctx: diff --git a/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index 2b39be1b676d..2adda4bac501 100644 --- a/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -69,7 +69,7 @@ worker_config: disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 num_postprocess_workers: 4 stream_interval: 20 ctx: diff --git a/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_8k1k_con256_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_8k1k_con256_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml index c2f518f836c0..5a15c598d61d 100644 --- a/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_8k1k_con256_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/b200_deepseek-r1-fp4_8k1k_con256_ctx1_dep4_gen1_dep8_eplb0_mtp1_ccb-NIXL.yaml @@ -71,7 +71,7 @@ worker_config: disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 num_postprocess_workers: 4 stream_interval: 20 ctx: diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_128k8k_con1_ctx1_pp8_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_128k8k_con1_ctx1_pp8_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index eafb048e68b9..e37b60c77438 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_128k8k_con1_ctx1_pp8_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_128k8k_con1_ctx1_pp8_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -69,7 +69,7 @@ worker_config: disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 num_postprocess_workers: 4 stream_interval: 20 allreduce_strategy: MNNVL diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_128k8k_con64_ctx1_pp8_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_128k8k_con64_ctx1_pp8_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml index 0117cf4d0b5a..e115acaa05af 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_128k8k_con64_ctx1_pp8_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_128k8k_con64_ctx1_pp8_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml @@ -71,7 +71,7 @@ worker_config: disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 num_postprocess_workers: 4 stream_interval: 20 ctx: diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml index aa31b144efbc..6b0245deb307 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml @@ -71,7 +71,7 @@ worker_config: disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 num_postprocess_workers: 4 stream_interval: 20 ctx: diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index ce7d3b5be45d..5ea03a618905 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -69,7 +69,7 @@ worker_config: disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 num_postprocess_workers: 4 stream_interval: 20 allreduce_strategy: MNNVL diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con2048_ctx2_dep4_gen1_dep16_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con2048_ctx2_dep4_gen1_dep16_eplb0_mtp3_ccb-NIXL.yaml index 73dae2e6d765..718129fe2476 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con2048_ctx2_dep4_gen1_dep16_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con2048_ctx2_dep4_gen1_dep16_eplb0_mtp3_ccb-NIXL.yaml @@ -71,7 +71,7 @@ worker_config: disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 num_postprocess_workers: 4 stream_interval: 20 ctx: diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con2048_ctx2_dep4_gen1_dep16_eplb288_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con2048_ctx2_dep4_gen1_dep16_eplb288_mtp3_ccb-NIXL.yaml index 3370d1faa8cb..492c69b220e0 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con2048_ctx2_dep4_gen1_dep16_eplb288_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_1k1k_con2048_ctx2_dep4_gen1_dep16_eplb288_mtp3_ccb-NIXL.yaml @@ -74,7 +74,7 @@ worker_config: disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 num_postprocess_workers: 4 stream_interval: 20 ctx: diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml index 8c08fcde53d2..d161caea68eb 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml @@ -71,7 +71,7 @@ worker_config: disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 num_postprocess_workers: 4 stream_interval: 20 ctx: diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index 60bec288c4a2..2d449f57a7a6 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-r1-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -69,7 +69,7 @@ worker_config: disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 num_postprocess_workers: 4 stream_interval: 20 allreduce_strategy: MNNVL diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml index fcfbe0616235..216d38bcc41b 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml @@ -75,7 +75,7 @@ worker_config: disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 num_postprocess_workers: 4 stream_interval: 20 nvfp4_gemm_config: diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index 2c3b2b19759d..cfd1a7826312 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_1k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -70,7 +70,7 @@ worker_config: disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 num_postprocess_workers: 4 stream_interval: 20 nvfp4_gemm_config: diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index 2a91f3633c0d..a0b88f589cf8 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -73,7 +73,7 @@ worker_config: disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 num_postprocess_workers: 4 stream_interval: 20 nvfp4_gemm_config: diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml index 99002c9f2558..2651f02c2b57 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_32k4k_con256_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml @@ -74,7 +74,7 @@ worker_config: disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 num_postprocess_workers: 4 stream_interval: 20 nvfp4_gemm_config: diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml index e40afbe44267..194e61864a20 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_8k1k_con1024_ctx1_dep4_gen1_dep32_eplb256_mtp3_ccb-NIXL.yaml @@ -75,7 +75,7 @@ worker_config: disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 num_postprocess_workers: 4 stream_interval: 20 nvfp4_gemm_config: diff --git a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml index 0a7b3a052e37..562c388d8450 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb200_deepseek-v32-fp4_8k1k_con1_ctx1_dep4_gen1_tep8_eplb0_mtp3_ccb-NIXL.yaml @@ -70,7 +70,7 @@ worker_config: disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 num_postprocess_workers: 4 stream_interval: 20 nvfp4_gemm_config: diff --git a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml index 2cdb9a868988..c2bc0fee249b 100644 --- a/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf-sanity/disaggregated/gb300_deepseek-r1-fp4_1k1k_con1024_ctx1_dep4_gen1_dep32_eplb0_mtp3_ccb-NIXL.yaml @@ -71,7 +71,7 @@ worker_config: disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 num_postprocess_workers: 4 stream_interval: 20 ctx: diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-UCX.yaml index 28f8296df512..355a7625bd3d 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-UCX.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_128k8k_con128_ctx1_pp8_gen1_dep16_eplb0_mtp1_ccb-UCX.yaml @@ -71,7 +71,7 @@ worker_config: disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 num_postprocess_workers: 4 stream_interval: 20 ctx: diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con2048_ctx2_dep4_gen1_dep16_eplb0_mtp3_ccb-NIXL.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con2048_ctx2_dep4_gen1_dep16_eplb0_mtp3_ccb-NIXL.yaml index 73dae2e6d765..718129fe2476 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con2048_ctx2_dep4_gen1_dep16_eplb0_mtp3_ccb-NIXL.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con2048_ctx2_dep4_gen1_dep16_eplb0_mtp3_ccb-NIXL.yaml @@ -71,7 +71,7 @@ worker_config: disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP - num_nextn_predict_layers: 3 + max_draft_len: 3 num_postprocess_workers: 4 stream_interval: 20 ctx: diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml index c6133efa8011..1c56c251756d 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_1k1k_con3072_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml @@ -71,7 +71,7 @@ worker_config: disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 num_postprocess_workers: 4 stream_interval: 20 ctx: diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-UCX.yaml index 1ed2d2cff294..7e77615fae38 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-UCX.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-r1-fp4_8k1k_con4096_ctx1_dep4_gen1_dep16_eplb0_mtp1_ccb-UCX.yaml @@ -71,7 +71,7 @@ worker_config: disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 num_postprocess_workers: 4 stream_interval: 20 ctx: diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml index 86b2ca6ea9ba..80602cb3df0a 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_1k1k_con2048_ctx1_dep4_gen1_dep4_eplb0_mtp1_ccb-UCX.yaml @@ -72,7 +72,7 @@ worker_config: disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 num_postprocess_workers: 4 stream_interval: 20 nvfp4_gemm_config: diff --git a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-UCX.yaml b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-UCX.yaml index a2b8f7b1a8f4..71a4e13ca5be 100644 --- a/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-UCX.yaml +++ b/tests/scripts/perf/disaggregated/gb200_deepseek-v32-fp4_32k4k_con2048_ctx1_dep4_gen1_dep32_eplb288_mtp1_ccb-UCX.yaml @@ -77,7 +77,7 @@ worker_config: disable_overlap_scheduler: true speculative_config: &id001 decoding_type: MTP - num_nextn_predict_layers: 1 + max_draft_len: 1 num_postprocess_workers: 4 stream_interval: 20 nvfp4_gemm_config: diff --git a/tests/unittest/_torch/speculative/hw_agnostic/test_mtp.py b/tests/unittest/_torch/speculative/hw_agnostic/test_mtp.py index 7fcb142d587c..acb34fc704e6 100644 --- a/tests/unittest/_torch/speculative/hw_agnostic/test_mtp.py +++ b/tests/unittest/_torch/speculative/hw_agnostic/test_mtp.py @@ -352,7 +352,9 @@ def test_sample_and_accept_draft_tokens( ref_num_accepted_tokens, ): batch_size = len(draft_len) - spec_config = MTPDecodingConfig(num_nextn_predict_layers=mtp_num_modules) + spec_config = MTPDecodingConfig(max_draft_len=mtp_num_modules) + # Simulate update_spec_config_from_model_config for vanilla MTP tests + spec_config.num_nextn_predict_layers = mtp_num_modules # attention metedata attn_metadata = TrtllmAttentionMetadata( @@ -1075,7 +1077,9 @@ def test_mtp_update_mtp_hidden_states( batch_size = len(request_ids) batch_size - num_context_request hidden_size = hidden_states.shape[1] - spec_config = MTPDecodingConfig(num_nextn_predict_layers=num_nextn_predict_layers) + spec_config = MTPDecodingConfig(max_draft_len=num_nextn_predict_layers) + # Simulate update_spec_config_from_model_config for vanilla MTP tests + spec_config.num_nextn_predict_layers = num_nextn_predict_layers attn_metadata = TrtllmAttentionMetadata( max_num_requests=batch_size, max_num_tokens=1024, kv_cache_manager=None @@ -1660,7 +1664,9 @@ def test_prepare_drafter_inputs( hidden_size = previous_layer_hidden_states.shape[1] else: hidden_size = 10 - spec_config = MTPDecodingConfig(num_nextn_predict_layers=num_nextn_predict_layers) + spec_config = MTPDecodingConfig(max_draft_len=num_nextn_predict_layers) + # Simulate update_spec_config_from_model_config for vanilla MTP tests + spec_config.num_nextn_predict_layers = num_nextn_predict_layers if attn_metadata is None: attn_metadata = TrtllmAttentionMetadata(