Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions models/MiniMaxAI/MiniMax-M3.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,17 @@ features:
- "minimax_m3"
spec_decoding:
description: "Eagle3 speculative decoding with an Inferact MiniMax-M3 draft head — accelerates decoding. Choose between the standard MHA head and the GQA head (16× smaller draft KV cache — better for long-context or high-batch deployments)."
# eagle3 is a separate-draft-model method, which doesn't compose with
# pipeline parallelism yet (vllm-project/vllm#50098), so the feature is
# gated off multi_node_tp_pp. Every other strategy keeps it.
strategies:
- single_node_tp
- single_node_tep
- single_node_dep
- multi_node_tp
- multi_node_tep
- multi_node_dep
- pd_cluster
default_mode: eagle3
modes:
eagle3:
Expand Down
10 changes: 10 additions & 0 deletions models/Qwen/Qwen3-Next-80B-A3B-Instruct.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,16 @@ features:
- "hermes"
spec_decoding:
description: "Multi-token prediction speculative decoding for lower latency"
# qwen3_next_mtp's draft model doesn't implement SupportsPP yet
# (vllm-project/vllm#50098), so the feature is gated off multi_node_tp_pp.
# Every other strategy keeps it.
strategies:
- single_node_tp
- single_node_tep
- single_node_dep
- multi_node_tp
- multi_node_dep
- multi_node_tep
args:
- "--speculative-config"
- '{"method":"qwen3_next_mtp","num_speculative_tokens":2}'
Expand Down
11 changes: 11 additions & 0 deletions models/Qwen/Qwen3.5-397B-A17B.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,17 @@ features:
- "qwen3"
spec_decoding:
description: "Speculative decoding — pick native MTP or the Ascend MTP mode."
# Both modes' draft models don't implement SupportsPP yet
# (vllm-project/vllm#50098), so the feature is gated off multi_node_tp_pp.
# Every other strategy keeps it.
strategies:
- single_node_tp
- single_node_tep
- single_node_dep
- multi_node_tp
- multi_node_dep
- multi_node_tep
- pd_cluster
default_mode: mtp
modes:
mtp:
Expand Down
10 changes: 10 additions & 0 deletions models/Qwen/Qwen3.6-35B-A3B.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,16 @@ features:
- "qwen3"
spec_decoding:
description: "Multi-token prediction speculative decoding for lower latency"
# mtp's draft model doesn't implement SupportsPP yet
# (vllm-project/vllm#50098), so the feature is gated off multi_node_tp_pp.
# Every other strategy keeps it.
strategies:
- single_node_tp
- single_node_tep
- single_node_dep
- multi_node_tp
- multi_node_dep
- multi_node_tep
modes:
mtp:
args:
Expand Down
12 changes: 12 additions & 0 deletions models/Qwen/Qwen3.8-2.4T-A95B.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,18 @@ features:
- "qwen3"
spec_decoding:
description: "Multi-token prediction using the draft head shipped inside the checkpoint — no separate speculator repo needed."
# mtp's draft model doesn't implement SupportsPP yet
# (vllm-project/vllm#50098), so the feature is gated off multi_node_tp_pp.
# Every other strategy keeps it.
strategies:
- single_node_tp
- single_node_tep
- single_node_dep
- multi_node_tp
- multi_node_tep
- multi_node_dep
- multi_node_tp_dp
- pd_cluster
# config.json sets mtp_num_hidden_layers: 1 and the weight index carries `mtp.*`
# tensors, so the draft head is present. Depth is 3, NOT 1: the single draft layer is
# run autoregressively, and depth is a throughput/latency knob rather than a property
Expand Down
11 changes: 11 additions & 0 deletions models/deepseek-ai/DeepSeek-V3.2.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,17 @@ features:
- "deepseek_v3"
spec_decoding:
description: "Multi-Token Prediction speculative decoding with 3 speculative tokens."
# mtp's draft model doesn't implement SupportsPP yet
# (vllm-project/vllm#50098), so the feature is gated off multi_node_tp_pp.
# Every other strategy keeps it.
strategies:
- single_node_tp
- single_node_tep
- single_node_dep
- multi_node_tp
- multi_node_dep
- multi_node_tep
- pd_cluster
args:
- "--speculative-config"
- '{"method":"mtp","num_speculative_tokens":3}'
Expand Down
10 changes: 10 additions & 0 deletions models/mindlab-research/Macaron-V1-Coding-Venti.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,16 @@ features:
- "glm45"
spec_decoding:
description: "Multi-Token Prediction speculative decoding (up to 5 draft tokens)"
# mtp/dspark's draft models don't implement SupportsPP yet
# (vllm-project/vllm#50098), so the feature is gated off multi_node_tp_pp.
# Every other strategy keeps it.
strategies:
- single_node_tp
- single_node_tep
- multi_node_tp
- multi_node_tep
- multi_node_dep
- pd_cluster
default_mode: mtp
modes:
mtp:
Expand Down
11 changes: 11 additions & 0 deletions models/moonshotai/Kimi-K2.5.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,17 @@ features:
- "kimi_k2"
spec_decoding:
description: "Eagle3 speculative decoding for accelerated inference (requires vLLM >= 0.18.0)"
# eagle3 is a separate-draft-model method, which doesn't compose with
# pipeline parallelism yet (vllm-project/vllm#50098), so the feature is
# gated off multi_node_tp_pp. Every other strategy keeps it.
strategies:
- single_node_tp
- single_node_tep
- single_node_dep
- multi_node_tp
- multi_node_dep
- multi_node_tep
- pd_cluster
args:
- "--speculative-config"
- '{"model":"lightseekorg/kimi-k2.5-eagle3-mla","method":"eagle3","num_speculative_tokens":3}'
Expand Down
11 changes: 11 additions & 0 deletions models/moonshotai/Kimi-K2.6.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,17 @@ features:
- "kimi_k2"
spec_decoding:
description: "Eagle3 speculative decoding for accelerated inference"
# eagle3 is a separate-draft-model method, which doesn't compose with
# pipeline parallelism yet (vllm-project/vllm#50098), so the feature is
# gated off multi_node_tp_pp. Every other strategy keeps it.
strategies:
- single_node_tp
- single_node_tep
- single_node_dep
- multi_node_tp
- multi_node_dep
- multi_node_tep
- pd_cluster
args:
- "--speculative-config"
- '{"model":"lightseekorg/kimi-k2.6-eagle3-mla","method":"eagle3","num_speculative_tokens":3}'
Expand Down
11 changes: 11 additions & 0 deletions models/moonshotai/Kimi-K2.7-Code.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,17 @@ features:
- "kimi_k2"
spec_decoding:
description: "Eagle3 speculative decoding for accelerated inference"
# eagle3 is a separate-draft-model method, which doesn't compose with
# pipeline parallelism yet (vllm-project/vllm#50098), so the feature is
# gated off multi_node_tp_pp. Every other strategy keeps it.
strategies:
- single_node_tp
- single_node_tep
- single_node_dep
- multi_node_tp
- multi_node_dep
- multi_node_tep
- pd_cluster
args:
- "--speculative-config"
- '{"model":"lightseekorg/kimi-k2.7-coder-eagle3.1-mla","method":"eagle3","num_speculative_tokens":3}'
Expand Down
8 changes: 8 additions & 0 deletions models/nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,14 @@ features:
- "nemotron_v3"
spec_decoding:
description: "MTP speculative decoding for accelerated inference"
# mtp's draft model doesn't implement SupportsPP yet
# (vllm-project/vllm#50098), so the feature is gated off multi_node_tp_pp.
# Every other strategy keeps it.
strategies:
- single_node_tp
- multi_node_tp
- multi_node_tep
- multi_node_dep
args:
- "--speculative-config"
- '{"method":"mtp","num_speculative_tokens":3}'
Expand Down
8 changes: 8 additions & 0 deletions models/openai/gpt-oss-120b.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,14 @@ features:
- "--enable-auto-tool-choice"
spec_decoding:
description: "EAGLE3 speculative decoding for accelerated inference"
# eagle3 is a separate-draft-model method, which doesn't compose with
# pipeline parallelism yet (vllm-project/vllm#50098), so the feature is
# gated off multi_node_tp_pp. Every other strategy keeps it.
strategies:
- single_node_tp
- multi_node_tp
- multi_node_tep
- multi_node_dep
args:
- "--speculative-config"
- '{"model":"nvidia/gpt-oss-120b-Eagle3-v3","num_speculative_tokens":7,"method":"eagle3","draft_tensor_parallel_size":1}'
Expand Down
11 changes: 11 additions & 0 deletions models/stepfun-ai/Step-3.5-Flash.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,17 @@ features:
- "step3p5"
spec_decoding:
description: "Multi-Token Prediction speculative decoding with the step3p5_mtp method"
# step3p5_mtp's draft model doesn't implement SupportsPP yet
# (vllm-project/vllm#50098), so the feature is gated off multi_node_tp_pp.
# Every other strategy keeps it.
strategies:
- single_node_tp
- single_node_tep
- single_node_dep
- multi_node_tp
- multi_node_tep
- multi_node_dep
- pd_cluster
args:
- "--hf-overrides"
- '{"num_nextn_predict_layers": 1}'
Expand Down
11 changes: 11 additions & 0 deletions models/zai-org/GLM-4.5.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,17 @@ features:
- "glm45"
spec_decoding:
description: "Multi-Token Prediction speculative decoding using the model's built-in MTP layers"
# mtp's draft model doesn't implement SupportsPP yet
# (vllm-project/vllm#50098), so the feature is gated off multi_node_tp_pp.
# Every other strategy keeps it.
strategies:
- single_node_tp
- single_node_tep
- single_node_dep
- multi_node_tp
- multi_node_dep
- multi_node_tep
- pd_cluster
args:
- "--speculative-config.method"
- "mtp"
Expand Down
11 changes: 11 additions & 0 deletions models/zai-org/GLM-4.6.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,17 @@ features:
- "glm45"
spec_decoding:
description: "Multi-Token Prediction speculative decoding using the model's built-in MTP layers"
# mtp's draft model doesn't implement SupportsPP yet
# (vllm-project/vllm#50098), so the feature is gated off multi_node_tp_pp.
# Every other strategy keeps it.
strategies:
- single_node_tp
- single_node_tep
- single_node_dep
- multi_node_tp
- multi_node_dep
- multi_node_tep
- pd_cluster
args:
- "--speculative-config.method"
- "mtp"
Expand Down
11 changes: 11 additions & 0 deletions models/zai-org/GLM-4.7.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,17 @@ features:
- "glm45"
spec_decoding:
description: "Multi-Token Prediction speculative decoding using the model's built-in MTP layers"
# mtp's draft model doesn't implement SupportsPP yet
# (vllm-project/vllm#50098), so the feature is gated off multi_node_tp_pp.
# Every other strategy keeps it.
strategies:
- single_node_tp
- single_node_tep
- single_node_dep
- multi_node_tp
- multi_node_dep
- multi_node_tep
- pd_cluster
args:
- "--speculative-config.method"
- "mtp"
Expand Down
10 changes: 10 additions & 0 deletions models/zai-org/GLM-5.1.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,16 @@ features:
- "glm45"
spec_decoding:
description: "Multi-Token Prediction speculative decoding (3 draft tokens) — not yet supported on the AMD MXFP4 checkpoint"
# mtp's draft model doesn't implement SupportsPP yet
# (vllm-project/vllm#50098), so the feature is gated off multi_node_tp_pp.
# Every other strategy keeps it.
strategies:
- single_node_tp
- single_node_tep
- multi_node_tp
- multi_node_tep
- multi_node_dep
- pd_cluster
default_mode: mtp
modes:
mtp:
Expand Down
10 changes: 10 additions & 0 deletions models/zai-org/GLM-5.2.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,16 @@ features:
- "glm45"
spec_decoding:
description: "Multi-Token Prediction speculative decoding (up to 7 draft tokens)"
# mtp/dspark's draft models don't implement SupportsPP yet
# (vllm-project/vllm#50098), so the feature is gated off multi_node_tp_pp.
# Every other strategy keeps it.
strategies:
- single_node_tp
- single_node_tep
- multi_node_tp
- multi_node_tep
- multi_node_dep
- pd_cluster
default_mode: mtp
modes:
mtp:
Expand Down
11 changes: 11 additions & 0 deletions models/zai-org/GLM-5.3-Flash.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,17 @@ features:
- "glm45"
spec_decoding:
description: "Built-in MTP or an external DFlash2 drafter. Pick one method — only that --speculative-config is emitted."
# Both mtp and dflash are separate-draft-model methods, which don't compose
# with pipeline parallelism yet (vllm-project/vllm#50098), so the feature
# is gated off multi_node_tp_pp. Every other strategy keeps it.
strategies:
- single_node_tp
- single_node_tep
- single_node_dep
- multi_node_tp
- multi_node_tep
- multi_node_dep
- pd_cluster
default_mode: mtp
modes:
mtp:
Expand Down
10 changes: 10 additions & 0 deletions models/zai-org/GLM-5.3.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,16 @@ features:
- "glm45"
spec_decoding:
description: "Built-in Multi-Token Prediction — 5 draft tokens on NVIDIA/AMD, 1 on Ascend (draft stays eager)."
# mtp's draft model doesn't implement SupportsPP yet
# (vllm-project/vllm#50098), so the feature is gated off multi_node_tp_pp.
# Every other strategy keeps it.
strategies:
- single_node_tp
- single_node_tep
- multi_node_tp
- multi_node_tep
- multi_node_dep
- pd_cluster
default_mode: mtp
modes:
mtp:
Expand Down
10 changes: 10 additions & 0 deletions models/zai-org/GLM-5.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,16 @@ features:
- "glm45"
spec_decoding:
description: "Multi-Token Prediction speculative decoding (3 draft tokens)"
# mtp's draft model doesn't implement SupportsPP yet
# (vllm-project/vllm#50098), so the feature is gated off multi_node_tp_pp.
# Every other strategy keeps it.
strategies:
- single_node_tp
- single_node_tep
- multi_node_tp
- multi_node_tep
- multi_node_dep
- pd_cluster
args:
- "--speculative-config.method"
- "mtp"
Expand Down