Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion rust/src/chat/src/multimodal.rs
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,12 @@ impl ResolvedPlaceholder {
let marker_token_id = context.tokenizer().token_to_id(&token).ok_or_else(|| {
multimodal!("placeholder token `{token}` is not in the tokenizer vocabulary")
})?;
let embed_token_id = raw.placeholder_token_id_for(&metadata, modality)? as u32;
let embed_token_id = raw.placeholder_token_id_for(&metadata, modality)?;
let embed_token_id = u32::try_from(embed_token_id).map_err(|_| {
multimodal!(
"placeholder token id `{embed_token_id}` for `{modality}` must be non-negative"
)
})?;

Ok(Self {
token,
Expand Down
4 changes: 2 additions & 2 deletions tests/config/test_speculative_draft_hf_overrides.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ def record(hf_config: PretrainedConfig) -> PretrainedConfig:


@pytest.mark.cpu_test
def test_inkling_override_exposes_only_first_mtp_depth():
def test_inkling_override_exposes_checkpoint_mtp_depths():
text_config = _make_hf_config(
architectures=["InklingForCausalLM"],
model_type="inkling_model",
Expand All @@ -107,7 +107,7 @@ def test_inkling_override_exposes_only_first_mtp_depth():
assert out is text_config
assert out.model_type == "inkling_mtp"
assert out.architectures == ["InklingMTPModel"]
assert out.n_predict == 1
assert out.n_predict == 8
assert out.num_nextn_predict_layers == 8
assert out.chain_hidden_post_norm is False
assert out.local_layer_ids == [0, 2, 4]
Expand Down
17 changes: 8 additions & 9 deletions vllm/config/compilation.py
Original file line number Diff line number Diff line change
Expand Up @@ -1422,16 +1422,15 @@ def resolve_cudagraph_mode_and_sizes(
"and make sure compilation mode is VLLM_COMPILE"
)

# MRV1 adjusts cudagraph sizes to be a multiple of uniform_decode_query_len
# to avoid: https://github.com/vllm-project/vllm/issues/28207 and temp-fix:
# https://github.com/vllm-project/vllm/issues/28207#issuecomment-3504004536
# Will be removed in the near future when we have separate cudagraph capture
# sizes for decode and mixed prefill-decode.
# MRV2 handles cudagraph capture sizing in cudagraph_utils.py
# and doesn't need below: https://github.com/vllm-project/vllm/pull/45953
# Round cudagraph sizes to a multiple of uniform_decode_query_len so a
# uniform decode batch cannot dispatch to a "mixed" descriptor whose token
# count is not a multiple of the decode span, which corrupts per-request
# conv/attention state (https://github.com/vllm-project/vllm/issues/28207).
# Enabled for MRV2 as well: its cudagraph_utils.py sizing does not on its
# own keep uniform batches off mixed descriptors for multi-module MTP.
# Temporary until decode and mixed prefill-decode have separate sizes.
if (
not use_v2_model_runner
and cudagraph_mode.decode_mode() == CUDAGraphMode.FULL
cudagraph_mode.decode_mode() == CUDAGraphMode.FULL
and uniform_decode_query_len > 1
):
self.adjust_cudagraph_sizes_for_spec_decode(
Expand Down
24 changes: 14 additions & 10 deletions vllm/config/speculative.py
Original file line number Diff line number Diff line change
Expand Up @@ -555,6 +555,8 @@ def hf_config_override(hf_config: PretrainedConfig) -> PretrainedConfig:
)

if hf_config.model_type in ("inkling_mm_model", "inkling_model"):
# MTP weights live on the text backbone. Promote text_config and
# expose the nested MTP fields on the flat draft config.
mtp_config = getattr(hf_config, "mtp_config", None) or {}
hf_config = getattr(hf_config, "text_config", hf_config)
checkpoint_depths = mtp_config.get("num_nextn_predict_layers", 0)
Expand All @@ -563,12 +565,13 @@ def hf_config_override(hf_config: PretrainedConfig) -> PretrainedConfig:
hf_config.model_type = "inkling_mtp"
hf_config.update(
{
# Inkling currently exposes only the first checkpoint depth.
"n_predict": 1,
"n_predict": checkpoint_depths,
"num_nextn_predict_layers": checkpoint_depths,
"chain_hidden_post_norm": mtp_config.get(
"chain_hidden_post_norm", False
),
# The MTP depth blocks carry their own sliding-window
# pattern, which differs from the backbone's layer pattern.
"local_layer_ids": mtp_config.get("local_layer_ids", []),
"architectures": ["InklingMTPModel"],
}
Expand Down Expand Up @@ -974,14 +977,6 @@ def __post_init__(self):
"`num_speculative_tokens` was not provided"
)

if (
self.draft_model_config.hf_config.model_type == "inkling_mtp"
and self.num_speculative_tokens != 1
):
raise ValueError(
"Inkling MTP currently supports exactly one speculative token"
)

if self.method == "dspark":
# DSpark is a semi-autoregressive *block* drafter. A
# speculative length smaller than the checkpoint's block
Expand Down Expand Up @@ -1308,6 +1303,15 @@ def use_eagle(self) -> bool:
# TODO(ben): Refactor this so the naming is clearer
return self.method in ("eagle", "eagle3", "mtp", "dflash", "dspark")

def num_speculative_prefill_steps(self) -> int:
if self.method == "mtp" and self.draft_model_config is not None:
n_predict = getattr(
self.draft_model_config.hf_config, "num_nextn_predict_layers", None
)
if n_predict:
return min(int(n_predict), self.num_speculative_tokens)
return 1

def use_dflash(self) -> bool:
return self.method == "dflash"

Expand Down
16 changes: 9 additions & 7 deletions vllm/models/inkling/nvidia/model.py
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,7 @@ def forward(
# Caller folds mlp_output (pre-reduce, pre-sconv) into the next
# fused sconv+add+rmsnorm.
return hidden_states, (mlp_output, self.mlp_sconv)
# Standalone (MTP) tail: finish the sublayer without a norm.
return _sconv_add_norm(
mlp_output, hidden_states, self.mlp_sconv, None, positions
)[1]
Expand All @@ -236,10 +237,12 @@ class InklingReplicatedEmbedding(nn.Module):
"""Full-vocab embedding table replicated on every TP rank.

Trades the full table per rank (~2.3 GiB at V=201k / H=6144 bf16, vs a
1/tp shard) for no masked lookup or per-lookup TP all-reduce, and keeps the
full table on-rank for the fused gather+norm kernel. Bit-exact vs
vocab-parallel: the all-reduce there only ever summed one real row against
exact zeros. The LM head stays vocab-sharded.
1/tp shard) for no masked lookup and no per-lookup TP all-reduce — one
all-reduce per MTP draft step plus one per verify pass — and keeps the
full table on-rank for the fused gather+norm kernels (``embed_rmsnorm``,
``embed_dual_rmsnorm_cat``). Bit-exact vs vocab-parallel: the all-reduce
there only ever summed one real row against exact zeros. The LM head
stays vocab-sharded.
"""

def __init__(self, num_embeddings: int, embedding_dim: int) -> None:
Expand Down Expand Up @@ -611,7 +614,8 @@ def embed_input_ids(
def get_language_model(self) -> nn.Module:
# This class IS the causal LM (the towers are side branches), so the
# language model is self — callers expect a module exposing ``.model``
# and ``.lm_head``.
# / ``.lm_head`` (e.g. the MTP/eagle loader shares embeddings via
# ``get_language_model().model.embed_tokens``).
return self


Expand Down Expand Up @@ -672,8 +676,6 @@ def _iter_loadable_weights() -> Iterable[tuple[str, torch.Tensor]]:

yield name, weight

# The release checkpoint also carries auxiliary prediction-head weights;
# they are not part of the causal LM served by this implementation.
loader = AutoWeightsLoader(module, skip_prefixes=["model.mtp."])
loaded |= loader.load_weights(_iter_loadable_weights())

Expand Down
6 changes: 3 additions & 3 deletions vllm/models/inkling/nvidia/moe.py
Original file line number Diff line number Diff line change
Expand Up @@ -475,9 +475,9 @@ def forward(self, x: torch.Tensor) -> torch.Tensor | None:
lambda: self.sink_experts(x, gammas),
self._sink_events[0],
self._sink_events[1],
self._sink_stream
if num_tokens <= envs.VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD
else None,
self._sink_stream,
enable_parallel=num_tokens
<= envs.VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD,
)
self._routed_sel = None

Expand Down
55 changes: 42 additions & 13 deletions vllm/models/inkling/nvidia/mtp.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,13 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Inkling MTP (Multi-Token Prediction) draft model (NVIDIA).

Implements the first MTP depth from the reference ``mtp_model.py`` shipped with
the checkpoint. It owns ``hidden_norm`` / ``embed_norm`` RMSNorms, a ``2H -> H``
input projection, and a full Inkling transformer block with a dense bf16 MLP.
Mirrors the reference ``mtp_model.py`` shipped with the checkpoint: each MTP
depth ``i`` owns ``hidden_norm`` / ``embed_norm`` RMSNorms, an ``input_proj``
(``2H -> H``) and a full Inkling transformer block (dense bf16 MLP, with the
same short convolutions as the backbone; its attention is full or sliding-window
per depth, selected by ``mtp_config.local_layer_ids``). When enabled, a shared
``chain_norm`` is applied after every depth; its output is both the logits input
and the previous hidden state fed to the next depth.

The draft shares the target's token embedding table and LM head
(``load_eagle_model`` wires those references) and applies the backbone
Expand Down Expand Up @@ -50,6 +54,16 @@ def _mtp_depth_from_name(name: str) -> int | None:
return int(m.group(1)) if m else None


def _select_mtp_depth_count(n_predict: int, num_spec: int | None) -> int:
num_layers = min(n_predict, num_spec) if num_spec else n_predict
if num_layers <= 0:
raise ValueError(
"Inkling MTP requires num_nextn_predict_layers and "
"num_speculative_tokens to select at least one depth layer."
)
return num_layers


class InklingMTPDepthLayer(nn.Module):
"""One MTP depth: norm both inputs, fuse (2H->H), run a Inkling block."""

Expand Down Expand Up @@ -94,14 +108,30 @@ def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
vllm_config.speculative_config.draft_model_config.hf_config
)
self.config = config
if vllm_config.speculative_config.num_speculative_tokens != 1:
raise ValueError(
"Inkling MTP currently supports exactly one speculative token"
)
# The checkpoint ships num_nextn_predict_layers depth blocks, but only
# the first ``num_speculative_tokens`` are exercised (step i uses depth
# i). Build only those to save memory — each depth is a full Inkling block
# with its own (large) full-history sconv caches and KV cache.
n_predict = config.num_nextn_predict_layers
num_spec = vllm_config.speculative_config.num_speculative_tokens
self.num_mtp_layers = _select_mtp_depth_count(n_predict, num_spec)
self.chain_hidden_post_norm = config.chain_hidden_post_norm

# Depth blocks whose attention is sliding-window (swa_* head config)
# rather than full; keyed by MTP depth via the checkpoint's
# mtp_config.local_layer_ids (promoted onto the draft config). Mirrors
# InklingModel's local_ids split, but over MTP depths, not backbone
# layers.
local_ids = set(config.local_layer_ids)

# Keyed by depth index (str) to mirror the checkpoint layout.
self.layers = nn.ModuleDict(
{"0": InklingMTPDepthLayer(config, f"{prefix}.layers.0", 0 in local_ids)}
{
str(idx): InklingMTPDepthLayer(
config, f"{prefix}.layers.{idx}", idx in local_ids
)
for idx in range(self.num_mtp_layers)
}
)
self.chain_norm = (
InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
Expand Down Expand Up @@ -199,9 +229,8 @@ def forward(
# auto-enumerated as a draft attention layer); its per-token metadata is
# built by the speculator's build_attn_metadata and read from the
# forward context, so nothing extra is threaded here.
if spec_step_idx != 0:
raise ValueError("Inkling MTP only supports spec_step_idx=0")
layer = self.layers["0"]
depth = spec_step_idx % self.num_mtp_layers
layer = self.layers[str(depth)]
combined = self.fused_input_cat(
layer, previous_hidden_states, input_ids, inputs_embeds
)
Expand Down Expand Up @@ -353,8 +382,8 @@ def _load(name: str, weight: torch.Tensor, shard_id: object = None) -> bool:
# Only consume the MTP weights; everything else belongs to the target.
if ".mtp." not in name:
continue
# Only the first checkpoint depth is used for MTP=1.
if depth is not None and depth != 0:
# Skip depth blocks beyond the ones we built (num_speculative_tokens).
if depth is not None and depth >= module.model.num_mtp_layers:
continue
# model.mtp.chain_norm.weight -> model.chain_norm.weight
# model.mtp.layers.{i}.X -> model.layers.{i}.X
Expand Down
4 changes: 3 additions & 1 deletion vllm/utils/multi_stream_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ def maybe_execute_in_parallel(
event0: torch.cuda.Event,
event1: torch.cuda.Event,
aux_stream: torch.cuda.Stream | None = None,
enable_parallel: bool = True,
) -> tuple[Any, Any]:
"""Run two functions potentially in parallel on separate CUDA streams.

Expand All @@ -40,11 +41,12 @@ def maybe_execute_in_parallel(
event1: CUDA event recorded after fn1 so default stream can wait.
aux_stream: The second CUDA stream for fn1.
Multi-stream is disabled when aux_stream is None.
enable_parallel: Opt-in switch for the multi-stream path. Defaults to True.

Returns:
Tuple of (fn0_result, fn1_result).
"""
if aux_stream is not None:
if aux_stream is not None and enable_parallel:
event0.record()
result0 = fn0()
with torch.cuda.stream(aux_stream):
Expand Down
23 changes: 20 additions & 3 deletions vllm/v1/core/kv_cache_coordinator.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ def __init__(
for g in kv_cache_config.kv_cache_groups
)
self.scheduler_block_size = scheduler_block_size
self.num_spec_prefill_steps = 1

self.block_pool = BlockPool(
num_gpu_blocks=kv_cache_config.num_blocks,
Expand Down Expand Up @@ -281,9 +282,14 @@ def cache_blocks(self, request: Request, num_computed_tokens: int) -> None:
(including tokens that are already cached).
"""
for manager in self.single_type_managers:
# Only cache tokens with finalized KV. The last num_spec_prefill_steps - 1
# tokens can be re-prefilled by speculative modules.
num_tokens_to_cache = max(
0, num_computed_tokens - (self.num_spec_prefill_steps - 1)
)
manager.cache_blocks(
request,
num_computed_tokens,
num_tokens_to_cache,
retention_interval=self.retention_interval,
)

Expand Down Expand Up @@ -661,9 +667,20 @@ def cache_blocks(self, request: Request, num_computed_tokens: int) -> None:
# EAGLE groups match one block past each aligned boundary and drop
# it, so make that lookahead block eligible to be cached.
if manager.use_eagle and aligned_num_computed_tokens > 0:
# Only cache tokens with finalized KV. The last
# num_spec_prefill_steps - 1 tokens can be re-prefilled by
# speculative modules.
num_finalized_computed_tokens = max(
0, num_computed_tokens - (self.num_spec_prefill_steps - 1)
)
aligned_num_finalized_computed_tokens = (
num_finalized_computed_tokens
// self.scheduler_block_size
* self.scheduler_block_size
)
num_tokens_to_cache = min(
num_computed_tokens,
aligned_num_computed_tokens + manager.block_size,
num_finalized_computed_tokens,
aligned_num_finalized_computed_tokens + manager.block_size,
)
# The manager already knows the fine hit granularity
# (``scheduler_block_size``); retention is passed separately so it
Expand Down
2 changes: 2 additions & 0 deletions vllm/v1/core/kv_cache_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,7 @@ def __init__(
max_in_flight_tokens: int | None = None,
enable_caching: bool = True,
use_eagle: bool = False,
num_spec_prefill_steps: int = 1,
log_stats: bool = False,
enable_kv_cache_events: bool = False,
dcp_world_size: int = 1,
Expand Down Expand Up @@ -158,6 +159,7 @@ def __init__(
hash_block_size=hash_block_size,
metrics_collector=self.metrics_collector,
)
self.coordinator.num_spec_prefill_steps = num_spec_prefill_steps
self.num_kv_cache_groups = len(kv_cache_config.kv_cache_groups)
self.block_pool = self.coordinator.block_pool
self.kv_cache_config = kv_cache_config
Expand Down
20 changes: 20 additions & 0 deletions vllm/v1/core/kv_cache_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -2086,6 +2086,26 @@ def get_kv_cache_configs(
# Check if the KV cache specs are registered correctly.
# This is to prevent that some layers are initialized with unregistered specs.
KVCacheSpecRegistry.check_kv_cache_spec_registry(merged_kv_cache_specs)
# Under multi-module MTP with prefix caching, tag every sliding-window spec
# with the store-side retention lag so pool sizing, the runtime admission
# cap, and block eviction all agree that these groups retain a few extra
# trailing blocks (single source of truth: the spec field). No-op (lag 0)
# otherwise, so non-MTP models are unchanged. Done before grouping so the lag
# flows through spec unification (which preserves it via ``replace``).

# When speculating with more than 1 speculative module (e.g. multi-layered MTP)
# tag every SlidingWindowSpec with how many extra tokens to retain in the window.
extra_retained_tokens = (
vllm_config.speculative_config.num_speculative_prefill_steps() - 1
if vllm_config.speculative_config is not None
else 0
)
for layer_name, layer_spec in merged_kv_cache_specs.items():
if isinstance(layer_spec, SlidingWindowSpec):
merged_kv_cache_specs[layer_name] = replace(
layer_spec, extra_retained_tokens=extra_retained_tokens
)

# Get global KV cache groups. This also handles spec unification for
# hybrid models when disable_hybrid_kv_cache_manager is enabled.
# After this call, merged_kv_cache_specs may be modified in-place.
Expand Down
Loading
Loading