Skip to content
Merged
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 13 additions & 3 deletions vllm/models/minimax_m3/nvidia/model.py
Original file line number Diff line number Diff line change
Expand Up @@ -241,8 +241,7 @@ def _use_sequence_parallel(vllm_config: VllmConfig) -> bool:
return False
parallel_config = vllm_config.parallel_config
return (
parallel_config.pipeline_parallel_size == 1
and parallel_config.enable_expert_parallel
parallel_config.enable_expert_parallel
and parallel_config.tensor_parallel_size > 1
)

Expand Down Expand Up @@ -449,7 +448,8 @@ def __init__(
config.hidden_size,
bias=False,
quant_config=quant_config,
reduce_results=reduce_results,
# Under SP the forward reduce-scatters the output itself.
reduce_results=reduce_results and not use_sequence_parallel,
disable_tp=disable_tp,
prefix=f"{prefix}.down_proj",
)
Expand Down Expand Up @@ -1609,6 +1609,8 @@ def forward(
hidden_states,
)
hidden_states = sp_shard(hidden_states)
if residual is not None:
residual = sp_shard(residual)

remote_aux = self.collect_remote_aux_hidden_states(intermediate_tensors)

Expand Down Expand Up @@ -1637,6 +1639,14 @@ def forward(
self._maybe_add_hidden_state(
aux_hidden_states, self.end_layer, hidden_states, residual
)
if self.use_sequence_parallel:
# PP transport expects tensors replicated across TP ranks.
hidden_size = hidden_states.shape[-1]
packed = torch.cat([hidden_states, residual, *aux_hidden_states], -1)
packed = sp_all_gather(packed)[:num_tokens]
hidden_states, residual, *aux_hidden_states = packed.split(
hidden_size, dim=-1
)
return IntermediateTensors(
{
"hidden_states": hidden_states,
Expand Down
Loading