diff --git a/vllm/model_executor/models/llama_eagle.py b/vllm/model_executor/models/llama_eagle.py index 99a69adf1fc3..13bfb85770c4 100644 --- a/vllm/model_executor/models/llama_eagle.py +++ b/vllm/model_executor/models/llama_eagle.py @@ -19,6 +19,7 @@ maybe_remap_kv_scale_name, ) from vllm.model_executor.models.llama import LlamaDecoderLayer, LlamaForCausalLM +from vllm.v1.spec_decode.eagle_mixin import EagleMixin from .utils import ( AutoWeightsLoader, @@ -161,7 +162,7 @@ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: return loaded_params -class EagleLlamaForCausalLM(LlamaForCausalLM): +class EagleLlamaForCausalLM(EagleMixin, LlamaForCausalLM): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): nn.Module.__init__(self) self.config = vllm_config.speculative_config.draft_model_config.hf_config diff --git a/vllm/model_executor/models/llama_eagle3.py b/vllm/model_executor/models/llama_eagle3.py index 5f66716d5454..22b0e5235610 100644 --- a/vllm/model_executor/models/llama_eagle3.py +++ b/vllm/model_executor/models/llama_eagle3.py @@ -24,6 +24,7 @@ ) from vllm.model_executor.models.llama import LlamaDecoderLayer, LlamaForCausalLM from vllm.multimodal.inputs import NestedTensors +from vllm.v1.spec_decode.eagle_mixin import Eagle3Mixin from .utils import ( AutoWeightsLoader, @@ -262,7 +263,7 @@ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: return loaded_params -class Eagle3LlamaForCausalLM(LlamaForCausalLM): +class Eagle3LlamaForCausalLM(Eagle3Mixin, LlamaForCausalLM): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): nn.Module.__init__(self) self.config = vllm_config.speculative_config.draft_model_config.hf_config @@ -326,30 +327,6 @@ def forward( ) -> tuple[torch.Tensor, torch.Tensor]: return self.model(input_ids, positions, hidden_states, inputs_embeds) - def compute_logits( - self, - hidden_states: torch.Tensor, - ) -> torch.Tensor | None: - logits = self.logits_processor(self.lm_head, hidden_states) - if self.draft_id_to_target_id is None: - assert logits.shape[1] == self.config.vocab_size, ( - "Expected logits to have shape " - f"(*, {self.config.vocab_size}), but got {logits.shape}" - ) - return logits - - base = torch.arange(self.config.draft_vocab_size, device=logits.device) - targets = base + self.draft_id_to_target_id - logits_new = logits.new_full( - ( - logits.shape[0], - self.config.vocab_size, - ), - float("-inf"), - ) - logits_new[:, targets] = logits - return logits_new - def combine_hidden_states( self, hidden_states: torch.Tensor, diff --git a/vllm/v1/spec_decode/eagle.py b/vllm/v1/spec_decode/eagle.py index e53de6a1de4f..9ce1e25ef848 100644 --- a/vllm/v1/spec_decode/eagle.py +++ b/vllm/v1/spec_decode/eagle.py @@ -372,12 +372,6 @@ def initialize_cudagraph_keys(self, cudagraph_mode: CUDAGraphMode) -> None: self.cudagraph_dispatcher.initialize_cudagraph_keys(eagle_cudagraph_mode) - def _greedy_sample(self, hidden_states: torch.Tensor) -> torch.Tensor: - """Greedy-sample draft tokens from hidden states.""" - if self.use_local_argmax_reduction: - return self.model.get_top_tokens(hidden_states) - return self.model.compute_logits(hidden_states).argmax(dim=-1) - def propose( self, # [num_tokens] @@ -496,7 +490,10 @@ def propose( # Early exit if there is only one draft token to be generated. if self.num_speculative_tokens == 1 or self.parallel_drafting: - draft_token_ids = self._greedy_sample(sample_hidden_states) + draft_token_ids = self.model.sample_chain( + sample_hidden_states, + use_local_argmax_reduction=self.use_local_argmax_reduction, + ) return draft_token_ids.view(-1, self.num_speculative_tokens) if self.uses_mrope: @@ -527,7 +524,10 @@ def propose( # [batch_size, num_tree_tokens] return torch.cat(draft_token_ids_list, dim=1) - draft_token_ids = self._greedy_sample(sample_hidden_states) + draft_token_ids = self.model.sample_chain( + sample_hidden_states, + use_local_argmax_reduction=self.use_local_argmax_reduction, + ) if self.allowed_attn_types is not None and not isinstance( attn_metadata, self.allowed_attn_types @@ -685,7 +685,10 @@ def propose( last_hidden_states, hidden_states = ret_hidden_states hidden_states = hidden_states[:batch_size] - draft_token_ids = self._greedy_sample(last_hidden_states[:batch_size]) + draft_token_ids = self.model.sample_chain( + last_hidden_states[:batch_size], + use_local_argmax_reduction=self.use_local_argmax_reduction, + ) draft_token_ids_list.append(draft_token_ids) # [batch_size, num_speculative_tokens] diff --git a/vllm/v1/spec_decode/eagle_mixin.py b/vllm/v1/spec_decode/eagle_mixin.py new file mode 100644 index 000000000000..3748398fa5ff --- /dev/null +++ b/vllm/v1/spec_decode/eagle_mixin.py @@ -0,0 +1,78 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from typing import Any + +import torch + + +class EagleMixin: + lm_head: Any + logits_processor: Any + compute_logits: Any + + def sample_chain( + self, hidden_states: torch.Tensor, use_local_argmax_reduction: bool = False + ) -> torch.Tensor: + """Greedy-sample draft tokens from hidden states.""" + if use_local_argmax_reduction: + return self.logits_processor.get_top_tokens(self.lm_head, hidden_states) + return self.compute_logits(hidden_states).argmax(dim=-1) + + +class Eagle3Mixin: + lm_head: Any + logits_processor: Any + draft_id_to_target_id: Any | None + config: Any + + def compute_draft_logits( + self, + hidden_states: torch.Tensor, + ) -> torch.Tensor: + # Computes draft logits from hidden states. + logits = self.logits_processor(self.lm_head, hidden_states) + return logits + + def compute_logits( + self, + hidden_states: torch.Tensor, + ) -> torch.Tensor | None: + # Maps draft logits to target logits. + logits = self.compute_draft_logits(hidden_states) + if self.draft_id_to_target_id is None: + assert logits.shape[1] == self.config.vocab_size, ( + "Expected logits to have shape " + f"(*, {self.config.vocab_size}), but got {logits.shape}" + ) + return logits + + base = torch.arange(self.config.draft_vocab_size, device=logits.device) + targets = base + self.draft_id_to_target_id + logits_new = logits.new_full( + ( + logits.shape[0], + self.config.vocab_size, + ), + float("-inf"), + ) + logits_new[:, targets] = logits + return logits_new + + def sample_chain( + self, hidden_states: torch.Tensor, use_local_argmax_reduction: bool = False + ) -> torch.Tensor: + if use_local_argmax_reduction and not ( + hasattr(self, "draft_id_to_target_id") + and self.draft_id_to_target_id is not None + ): + return self.logits_processor.get_top_tokens(self.lm_head, hidden_states) + else: + logits = self.compute_draft_logits(hidden_states) + draft_token_ids = logits.argmax(dim=-1) + + if self.draft_id_to_target_id is None: + return draft_token_ids + + offset = self.draft_id_to_target_id[draft_token_ids] + draft_token_ids += offset + return draft_token_ids