From 00a8aa9a23e924b5f38f143bd3b151d361190388 Mon Sep 17 00:00:00 2001 From: namgyu-youn Date: Mon, 31 Aug 2026 12:39:32 +0900 Subject: [PATCH] Gate spec_decoding off multi_node_tp_pp MTP/dspark/eagle3 draft models don't implement SupportsPP, so PP + spec_decoding fails at startup. Same fix already applied to Kimi-K3; extends it to the 20 other recipes offering both. Signed-off-by: namgyu-youn --- models/MiniMaxAI/MiniMax-M3.yaml | 11 +++++++++++ models/Qwen/Qwen3-Next-80B-A3B-Instruct.yaml | 10 ++++++++++ models/Qwen/Qwen3.5-397B-A17B.yaml | 11 +++++++++++ models/Qwen/Qwen3.6-35B-A3B.yaml | 10 ++++++++++ models/Qwen/Qwen3.8-2.4T-A95B.yaml | 12 ++++++++++++ models/deepseek-ai/DeepSeek-V3.2.yaml | 11 +++++++++++ models/mindlab-research/Macaron-V1-Coding-Venti.yaml | 10 ++++++++++ models/moonshotai/Kimi-K2.5.yaml | 11 +++++++++++ models/moonshotai/Kimi-K2.6.yaml | 11 +++++++++++ models/moonshotai/Kimi-K2.7-Code.yaml | 11 +++++++++++ .../NVIDIA-Nemotron-3-Super-120B-A12B-BF16.yaml | 8 ++++++++ models/openai/gpt-oss-120b.yaml | 8 ++++++++ models/stepfun-ai/Step-3.5-Flash.yaml | 11 +++++++++++ models/zai-org/GLM-4.5.yaml | 11 +++++++++++ models/zai-org/GLM-4.6.yaml | 11 +++++++++++ models/zai-org/GLM-4.7.yaml | 11 +++++++++++ models/zai-org/GLM-5.1.yaml | 10 ++++++++++ models/zai-org/GLM-5.2.yaml | 10 ++++++++++ models/zai-org/GLM-5.3-Flash.yaml | 11 +++++++++++ models/zai-org/GLM-5.3.yaml | 10 ++++++++++ models/zai-org/GLM-5.yaml | 10 ++++++++++ 21 files changed, 219 insertions(+) diff --git a/models/MiniMaxAI/MiniMax-M3.yaml b/models/MiniMaxAI/MiniMax-M3.yaml index 9be777675..82af3c8e7 100644 --- a/models/MiniMaxAI/MiniMax-M3.yaml +++ b/models/MiniMaxAI/MiniMax-M3.yaml @@ -55,6 +55,17 @@ features: - "minimax_m3" spec_decoding: description: "Eagle3 speculative decoding with an Inferact MiniMax-M3 draft head — accelerates decoding. Choose between the standard MHA head and the GQA head (16× smaller draft KV cache — better for long-context or high-batch deployments)." + # eagle3 is a separate-draft-model method, which doesn't compose with + # pipeline parallelism yet (vllm-project/vllm#50098), so the feature is + # gated off multi_node_tp_pp. Every other strategy keeps it. + strategies: + - single_node_tp + - single_node_tep + - single_node_dep + - multi_node_tp + - multi_node_tep + - multi_node_dep + - pd_cluster default_mode: eagle3 modes: eagle3: diff --git a/models/Qwen/Qwen3-Next-80B-A3B-Instruct.yaml b/models/Qwen/Qwen3-Next-80B-A3B-Instruct.yaml index c617b1db1..8ac019529 100644 --- a/models/Qwen/Qwen3-Next-80B-A3B-Instruct.yaml +++ b/models/Qwen/Qwen3-Next-80B-A3B-Instruct.yaml @@ -35,6 +35,16 @@ features: - "hermes" spec_decoding: description: "Multi-token prediction speculative decoding for lower latency" + # qwen3_next_mtp's draft model doesn't implement SupportsPP yet + # (vllm-project/vllm#50098), so the feature is gated off multi_node_tp_pp. + # Every other strategy keeps it. + strategies: + - single_node_tp + - single_node_tep + - single_node_dep + - multi_node_tp + - multi_node_dep + - multi_node_tep args: - "--speculative-config" - '{"method":"qwen3_next_mtp","num_speculative_tokens":2}' diff --git a/models/Qwen/Qwen3.5-397B-A17B.yaml b/models/Qwen/Qwen3.5-397B-A17B.yaml index a941ff721..a6ccf0b23 100644 --- a/models/Qwen/Qwen3.5-397B-A17B.yaml +++ b/models/Qwen/Qwen3.5-397B-A17B.yaml @@ -46,6 +46,17 @@ features: - "qwen3" spec_decoding: description: "Multi-token prediction speculative decoding for lower latency" + # mtp's draft model doesn't implement SupportsPP yet + # (vllm-project/vllm#50098), so the feature is gated off multi_node_tp_pp. + # Every other strategy keeps it. + strategies: + - single_node_tp + - single_node_tep + - single_node_dep + - multi_node_tp + - multi_node_dep + - multi_node_tep + - pd_cluster args: - "--speculative-config" - '{"method":"mtp","num_speculative_tokens":3}' diff --git a/models/Qwen/Qwen3.6-35B-A3B.yaml b/models/Qwen/Qwen3.6-35B-A3B.yaml index 6cc0d61b3..05a5faa0a 100644 --- a/models/Qwen/Qwen3.6-35B-A3B.yaml +++ b/models/Qwen/Qwen3.6-35B-A3B.yaml @@ -50,6 +50,16 @@ features: - "qwen3" spec_decoding: description: "Multi-token prediction speculative decoding for lower latency" + # mtp's draft model doesn't implement SupportsPP yet + # (vllm-project/vllm#50098), so the feature is gated off multi_node_tp_pp. + # Every other strategy keeps it. + strategies: + - single_node_tp + - single_node_tep + - single_node_dep + - multi_node_tp + - multi_node_dep + - multi_node_tep modes: mtp: args: diff --git a/models/Qwen/Qwen3.8-2.4T-A95B.yaml b/models/Qwen/Qwen3.8-2.4T-A95B.yaml index 69f8471b3..4a88c0627 100644 --- a/models/Qwen/Qwen3.8-2.4T-A95B.yaml +++ b/models/Qwen/Qwen3.8-2.4T-A95B.yaml @@ -80,6 +80,18 @@ features: - "qwen3" spec_decoding: description: "Multi-token prediction using the draft head shipped inside the checkpoint — no separate speculator repo needed." + # mtp's draft model doesn't implement SupportsPP yet + # (vllm-project/vllm#50098), so the feature is gated off multi_node_tp_pp. + # Every other strategy keeps it. + strategies: + - single_node_tp + - single_node_tep + - single_node_dep + - multi_node_tp + - multi_node_tep + - multi_node_dep + - multi_node_tp_dp + - pd_cluster # config.json sets mtp_num_hidden_layers: 1 and the weight index carries `mtp.*` # tensors, so the draft head is present. Depth is 3, NOT 1: the single draft layer is # run autoregressively, and depth is a throughput/latency knob rather than a property diff --git a/models/deepseek-ai/DeepSeek-V3.2.yaml b/models/deepseek-ai/DeepSeek-V3.2.yaml index c5977cee7..be3046560 100644 --- a/models/deepseek-ai/DeepSeek-V3.2.yaml +++ b/models/deepseek-ai/DeepSeek-V3.2.yaml @@ -50,6 +50,17 @@ features: - "deepseek_v3" spec_decoding: description: "Multi-Token Prediction speculative decoding with 3 speculative tokens." + # mtp's draft model doesn't implement SupportsPP yet + # (vllm-project/vllm#50098), so the feature is gated off multi_node_tp_pp. + # Every other strategy keeps it. + strategies: + - single_node_tp + - single_node_tep + - single_node_dep + - multi_node_tp + - multi_node_dep + - multi_node_tep + - pd_cluster args: - "--speculative-config" - '{"method":"mtp","num_speculative_tokens":3}' diff --git a/models/mindlab-research/Macaron-V1-Coding-Venti.yaml b/models/mindlab-research/Macaron-V1-Coding-Venti.yaml index a43b87425..b842acd0a 100644 --- a/models/mindlab-research/Macaron-V1-Coding-Venti.yaml +++ b/models/mindlab-research/Macaron-V1-Coding-Venti.yaml @@ -43,6 +43,16 @@ features: - "glm45" spec_decoding: description: "Multi-Token Prediction speculative decoding (up to 5 draft tokens)" + # mtp/dspark's draft models don't implement SupportsPP yet + # (vllm-project/vllm#50098), so the feature is gated off multi_node_tp_pp. + # Every other strategy keeps it. + strategies: + - single_node_tp + - single_node_tep + - multi_node_tp + - multi_node_tep + - multi_node_dep + - pd_cluster default_mode: mtp modes: mtp: diff --git a/models/moonshotai/Kimi-K2.5.yaml b/models/moonshotai/Kimi-K2.5.yaml index cdae33650..8cfe147de 100644 --- a/models/moonshotai/Kimi-K2.5.yaml +++ b/models/moonshotai/Kimi-K2.5.yaml @@ -46,6 +46,17 @@ features: - "kimi_k2" spec_decoding: description: "Eagle3 speculative decoding for accelerated inference (requires vLLM >= 0.18.0)" + # eagle3 is a separate-draft-model method, which doesn't compose with + # pipeline parallelism yet (vllm-project/vllm#50098), so the feature is + # gated off multi_node_tp_pp. Every other strategy keeps it. + strategies: + - single_node_tp + - single_node_tep + - single_node_dep + - multi_node_tp + - multi_node_dep + - multi_node_tep + - pd_cluster args: - "--speculative-config" - '{"model":"lightseekorg/kimi-k2.5-eagle3-mla","method":"eagle3","num_speculative_tokens":3}' diff --git a/models/moonshotai/Kimi-K2.6.yaml b/models/moonshotai/Kimi-K2.6.yaml index 31f82f414..6c6504dde 100644 --- a/models/moonshotai/Kimi-K2.6.yaml +++ b/models/moonshotai/Kimi-K2.6.yaml @@ -50,6 +50,17 @@ features: - "kimi_k2" spec_decoding: description: "Eagle3 speculative decoding for accelerated inference" + # eagle3 is a separate-draft-model method, which doesn't compose with + # pipeline parallelism yet (vllm-project/vllm#50098), so the feature is + # gated off multi_node_tp_pp. Every other strategy keeps it. + strategies: + - single_node_tp + - single_node_tep + - single_node_dep + - multi_node_tp + - multi_node_dep + - multi_node_tep + - pd_cluster args: - "--speculative-config" - '{"model":"lightseekorg/kimi-k2.6-eagle3-mla","method":"eagle3","num_speculative_tokens":3}' diff --git a/models/moonshotai/Kimi-K2.7-Code.yaml b/models/moonshotai/Kimi-K2.7-Code.yaml index 63fc54a73..6467c09f8 100644 --- a/models/moonshotai/Kimi-K2.7-Code.yaml +++ b/models/moonshotai/Kimi-K2.7-Code.yaml @@ -50,6 +50,17 @@ features: - "kimi_k2" spec_decoding: description: "Eagle3 speculative decoding for accelerated inference" + # eagle3 is a separate-draft-model method, which doesn't compose with + # pipeline parallelism yet (vllm-project/vllm#50098), so the feature is + # gated off multi_node_tp_pp. Every other strategy keeps it. + strategies: + - single_node_tp + - single_node_tep + - single_node_dep + - multi_node_tp + - multi_node_dep + - multi_node_tep + - pd_cluster args: - "--speculative-config" - '{"model":"lightseekorg/kimi-k2.7-coder-eagle3.1-mla","method":"eagle3","num_speculative_tokens":3}' diff --git a/models/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16.yaml b/models/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16.yaml index e331ffc20..3bac9e028 100644 --- a/models/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16.yaml +++ b/models/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16.yaml @@ -43,6 +43,14 @@ features: - "nemotron_v3" spec_decoding: description: "MTP speculative decoding for accelerated inference" + # mtp's draft model doesn't implement SupportsPP yet + # (vllm-project/vllm#50098), so the feature is gated off multi_node_tp_pp. + # Every other strategy keeps it. + strategies: + - single_node_tp + - multi_node_tp + - multi_node_tep + - multi_node_dep args: - "--speculative-config" - '{"method":"mtp","num_speculative_tokens":3}' diff --git a/models/openai/gpt-oss-120b.yaml b/models/openai/gpt-oss-120b.yaml index c90c20e03..cb288fddb 100644 --- a/models/openai/gpt-oss-120b.yaml +++ b/models/openai/gpt-oss-120b.yaml @@ -37,6 +37,14 @@ features: - "--enable-auto-tool-choice" spec_decoding: description: "EAGLE3 speculative decoding for accelerated inference" + # eagle3 is a separate-draft-model method, which doesn't compose with + # pipeline parallelism yet (vllm-project/vllm#50098), so the feature is + # gated off multi_node_tp_pp. Every other strategy keeps it. + strategies: + - single_node_tp + - multi_node_tp + - multi_node_tep + - multi_node_dep args: - "--speculative-config" - '{"model":"nvidia/gpt-oss-120b-Eagle3-v3","num_speculative_tokens":7,"method":"eagle3","draft_tensor_parallel_size":1}' diff --git a/models/stepfun-ai/Step-3.5-Flash.yaml b/models/stepfun-ai/Step-3.5-Flash.yaml index 14de165e3..5d5b55a52 100644 --- a/models/stepfun-ai/Step-3.5-Flash.yaml +++ b/models/stepfun-ai/Step-3.5-Flash.yaml @@ -39,6 +39,17 @@ features: - "step3p5" spec_decoding: description: "Multi-Token Prediction speculative decoding with the step3p5_mtp method" + # step3p5_mtp's draft model doesn't implement SupportsPP yet + # (vllm-project/vllm#50098), so the feature is gated off multi_node_tp_pp. + # Every other strategy keeps it. + strategies: + - single_node_tp + - single_node_tep + - single_node_dep + - multi_node_tp + - multi_node_tep + - multi_node_dep + - pd_cluster args: - "--hf-overrides" - '{"num_nextn_predict_layers": 1}' diff --git a/models/zai-org/GLM-4.5.yaml b/models/zai-org/GLM-4.5.yaml index fc1739cfe..2eae1a132 100644 --- a/models/zai-org/GLM-4.5.yaml +++ b/models/zai-org/GLM-4.5.yaml @@ -41,6 +41,17 @@ features: - "glm45" spec_decoding: description: "Multi-Token Prediction speculative decoding using the model's built-in MTP layers" + # mtp's draft model doesn't implement SupportsPP yet + # (vllm-project/vllm#50098), so the feature is gated off multi_node_tp_pp. + # Every other strategy keeps it. + strategies: + - single_node_tp + - single_node_tep + - single_node_dep + - multi_node_tp + - multi_node_dep + - multi_node_tep + - pd_cluster args: - "--speculative-config.method" - "mtp" diff --git a/models/zai-org/GLM-4.6.yaml b/models/zai-org/GLM-4.6.yaml index 1ccf5f6b6..28e1d8287 100644 --- a/models/zai-org/GLM-4.6.yaml +++ b/models/zai-org/GLM-4.6.yaml @@ -41,6 +41,17 @@ features: - "glm45" spec_decoding: description: "Multi-Token Prediction speculative decoding using the model's built-in MTP layers" + # mtp's draft model doesn't implement SupportsPP yet + # (vllm-project/vllm#50098), so the feature is gated off multi_node_tp_pp. + # Every other strategy keeps it. + strategies: + - single_node_tp + - single_node_tep + - single_node_dep + - multi_node_tp + - multi_node_dep + - multi_node_tep + - pd_cluster args: - "--speculative-config.method" - "mtp" diff --git a/models/zai-org/GLM-4.7.yaml b/models/zai-org/GLM-4.7.yaml index e0e798a63..b2c425ed8 100644 --- a/models/zai-org/GLM-4.7.yaml +++ b/models/zai-org/GLM-4.7.yaml @@ -47,6 +47,17 @@ features: - "glm45" spec_decoding: description: "Multi-Token Prediction speculative decoding using the model's built-in MTP layers" + # mtp's draft model doesn't implement SupportsPP yet + # (vllm-project/vllm#50098), so the feature is gated off multi_node_tp_pp. + # Every other strategy keeps it. + strategies: + - single_node_tp + - single_node_tep + - single_node_dep + - multi_node_tp + - multi_node_dep + - multi_node_tep + - pd_cluster args: - "--speculative-config.method" - "mtp" diff --git a/models/zai-org/GLM-5.1.yaml b/models/zai-org/GLM-5.1.yaml index 0d306113c..e28d9c43e 100644 --- a/models/zai-org/GLM-5.1.yaml +++ b/models/zai-org/GLM-5.1.yaml @@ -40,6 +40,16 @@ features: - "glm45" spec_decoding: description: "Multi-Token Prediction speculative decoding (3 draft tokens) — not yet supported on the AMD MXFP4 checkpoint" + # mtp's draft model doesn't implement SupportsPP yet + # (vllm-project/vllm#50098), so the feature is gated off multi_node_tp_pp. + # Every other strategy keeps it. + strategies: + - single_node_tp + - single_node_tep + - multi_node_tp + - multi_node_tep + - multi_node_dep + - pd_cluster default_mode: mtp modes: mtp: diff --git a/models/zai-org/GLM-5.2.yaml b/models/zai-org/GLM-5.2.yaml index b9a23a73d..aba0bed90 100644 --- a/models/zai-org/GLM-5.2.yaml +++ b/models/zai-org/GLM-5.2.yaml @@ -47,6 +47,16 @@ features: - "glm45" spec_decoding: description: "Multi-Token Prediction speculative decoding (up to 7 draft tokens)" + # mtp/dspark's draft models don't implement SupportsPP yet + # (vllm-project/vllm#50098), so the feature is gated off multi_node_tp_pp. + # Every other strategy keeps it. + strategies: + - single_node_tp + - single_node_tep + - multi_node_tp + - multi_node_tep + - multi_node_dep + - pd_cluster default_mode: mtp modes: mtp: diff --git a/models/zai-org/GLM-5.3-Flash.yaml b/models/zai-org/GLM-5.3-Flash.yaml index f3fad8424..05800548a 100644 --- a/models/zai-org/GLM-5.3-Flash.yaml +++ b/models/zai-org/GLM-5.3-Flash.yaml @@ -53,6 +53,17 @@ features: - "glm45" spec_decoding: description: "Use the checkpoint's single MTP layer to draft one token" + # mtp's draft model doesn't implement SupportsPP yet + # (vllm-project/vllm#50098), so the feature is gated off multi_node_tp_pp. + # Every other strategy keeps it. + strategies: + - single_node_tp + - single_node_tep + - single_node_dep + - multi_node_tp + - multi_node_tep + - multi_node_dep + - pd_cluster args: - "--speculative-config" - '{"method":"mtp","num_speculative_tokens":5}' diff --git a/models/zai-org/GLM-5.3.yaml b/models/zai-org/GLM-5.3.yaml index 14fbd8957..f3ed1fbc2 100644 --- a/models/zai-org/GLM-5.3.yaml +++ b/models/zai-org/GLM-5.3.yaml @@ -47,6 +47,16 @@ features: - "glm45" spec_decoding: description: "Built-in Multi-Token Prediction — 5 draft tokens on NVIDIA/AMD, 1 on Ascend (draft stays eager)." + # mtp's draft model doesn't implement SupportsPP yet + # (vllm-project/vllm#50098), so the feature is gated off multi_node_tp_pp. + # Every other strategy keeps it. + strategies: + - single_node_tp + - single_node_tep + - multi_node_tp + - multi_node_tep + - multi_node_dep + - pd_cluster default_mode: mtp modes: mtp: diff --git a/models/zai-org/GLM-5.yaml b/models/zai-org/GLM-5.yaml index 1ce1c22d6..e281f9cb4 100644 --- a/models/zai-org/GLM-5.yaml +++ b/models/zai-org/GLM-5.yaml @@ -39,6 +39,16 @@ features: - "glm45" spec_decoding: description: "Multi-Token Prediction speculative decoding (3 draft tokens)" + # mtp's draft model doesn't implement SupportsPP yet + # (vllm-project/vllm#50098), so the feature is gated off multi_node_tp_pp. + # Every other strategy keeps it. + strategies: + - single_node_tp + - single_node_tep + - multi_node_tp + - multi_node_tep + - multi_node_dep + - pd_cluster args: - "--speculative-config.method" - "mtp"