Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion vllm/config/parallel.py
Original file line number Diff line number Diff line change
Expand Up @@ -652,7 +652,6 @@ def use_sequence_parallel_moe(self) -> bool:
)
and self.enable_expert_parallel
and self.tensor_parallel_size > 1
and self.data_parallel_size > 1
)

@property
Expand Down
24 changes: 24 additions & 0 deletions vllm/model_executor/models/deepseek_mtp.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
from vllm._aiter_ops import rocm_aiter_ops
from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig
from vllm.distributed import tensor_model_parallel_all_gather
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
fused_moe_make_expert_params_mapping,
Expand Down Expand Up @@ -40,6 +41,24 @@
logger = init_logger(__name__)


def _restore_full_token_layout_if_needed(
hidden_states: torch.Tensor,
residual: torch.Tensor,
num_tokens: int,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Restore full token rows for the MTP proposer after SP MoE layers."""
if hidden_states.shape[0] == num_tokens:
return hidden_states, residual

combined_states = torch.cat([hidden_states, residual], dim=-1)
combined_states = tensor_model_parallel_all_gather(combined_states, 0)
combined_states = combined_states[:num_tokens]
hidden_states, residual = combined_states.split(
[hidden_states.shape[-1], residual.shape[-1]], dim=-1
)
return hidden_states, residual


class SharedHead(nn.Module):
def __init__(
self,
Expand Down Expand Up @@ -119,6 +138,11 @@ def forward(
hidden_states=hidden_states,
residual=None,
)
hidden_states, residual = _restore_full_token_layout_if_needed(
hidden_states,
residual,
positions.shape[0],
)
hidden_states = residual + hidden_states # pre-final-norm (logits hidden)
# Recycle the post-final-norm hidden into the next draft step.
# compute_logits applies shared_head (== final norm) to the pre-norm
Expand Down
12 changes: 10 additions & 2 deletions vllm/models/deepseek_v32/nvidia/mtp.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,10 @@
default_weight_loader,
maybe_remap_kv_scale_name,
)
from vllm.model_executor.models.deepseek_mtp import SharedHead
from vllm.model_executor.models.deepseek_mtp import (
SharedHead,
_restore_full_token_layout_if_needed,
)
from vllm.model_executor.models.deepseek_v2 import (
DeepseekV2MixtureOfExperts,
DeepseekV2MoE,
Expand Down Expand Up @@ -89,7 +92,12 @@ def forward(
hidden_states, residual = self.mtp_block(
positions=positions, hidden_states=hidden_states, residual=None
)
# mtp_block's MoE output is left un-reduced (reduce_results=False); the
hidden_states, residual = _restore_full_token_layout_if_needed(
hidden_states,
residual,
positions.shape[0],
)
# mtp_block's MoE output is left un-reduced (skip_final_all_reduce); the
# main model fuses that all-reduce into the next norm, but here the
# recycle hidden is consumed directly, so reduce it now.
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
Expand Down
Loading