Skip to content
Merged
Show file tree
Hide file tree
Changes from 7 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 62 additions & 6 deletions tests/models/test_deepseek_v41_decoder_replay_layers.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,26 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""DecoderReplayLayers runs the replay layers on the forward's replay batch."""

from unittest.mock import MagicMock

import pytest
import torch

from vllm.config import CompilationConfig, CUDAGraphMode
from vllm.forward_context import (
BatchDescriptor,
ForwardContext,
get_forward_context,
override_forward_context,
)
from vllm.models.deepseek_v41.decoder_replay_layers import DecoderReplayLayers
from vllm.models.deepseek_v41.decoder_replay_layers import (
DecoderReplayLayers,
ReplayBatch,
)
from vllm.models.deepseek_v41.nvidia.decoder_replay_cudagraph import (
DecoderReplayCudaGraphManager,
)
from vllm.v1.worker.gpu import cudagraph_utils

pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")

Expand All @@ -32,9 +43,9 @@ def _context(metadata):


def _set_replay_batch(layers, rows, metadata):
layers.rows = torch.tensor(rows, device=DEVICE)
layers.forward_context = ForwardContext(
no_compile_layers={}, attn_metadata=metadata, slot_mapping={}
layers.replay_batch = ReplayBatch(
torch.tensor(rows, device=DEVICE),
ForwardContext(no_compile_layers={}, attn_metadata=metadata, slot_mapping={}),
)


Expand Down Expand Up @@ -69,7 +80,7 @@ def run_layers(hidden_states, positions, input_ids, pre_mix, post, res, residual
seen["is_padding"] = replay_context.is_padding
return residual, pre_mix

layers = DecoderReplayLayers(WINDOW, run_layers, [topk, candidates])
layers = DecoderReplayLayers(WINDOW, run_layers, [topk, candidates], set())
states = _states(NUM_TOKENS)
hidden, residual = states[0], states[-1]
full, sub = object(), object()
Expand All @@ -96,9 +107,54 @@ def run_layers(hidden_states, *rest):
seen["attn_metadata"] = get_forward_context().attn_metadata
return (hidden_states,)

layers = DecoderReplayLayers(WINDOW, run_layers, [])
layers = DecoderReplayLayers(WINDOW, run_layers, [], set())
states = _states(NUM_TOKENS)
full = object()
with override_forward_context(_context(full)):
(output,) = layers(*states)
assert output is states[0] and seen["attn_metadata"] is full


def _graph_context(num_tokens):
return ForwardContext(
no_compile_layers={},
attn_metadata={},
slot_mapping={},
cudagraph_runtime_mode=CUDAGraphMode.PIECEWISE,
batch_descriptor=BatchDescriptor(num_tokens=num_tokens),
)


def test_replay_graph_matches_eager(monkeypatch):
"""The replay graph of the next captured size pads and runs the rows."""
monkeypatch.setattr(cudagraph_utils, "get_pp_group", MagicMock)
capture = torch.cuda.stream(torch.cuda.Stream())
monkeypatch.setattr(cudagraph_utils, "graph_capture", lambda device: capture)
compilation = CompilationConfig(decoder_replay_cudagraph_capture_sizes=[4])
cfg = MagicMock(compilation_config=compilation, speculative_config=None)
cfg.scheduler_config = MagicMock(max_num_seqs=2, max_num_batched_tokens=64)
cfg.cache_config.use_kda_recoverssm = False
cfg.model_config.hf_config = MagicMock(hc_mult=HC, hidden_size=HIDDEN)
cfg.model_config.dtype = torch.float32

def run_layers(hidden, positions, ids, pre, post, mix, residual):
return residual + hidden[:, None] + positions[:, None, None], pre + ids[:, None]

layers = DecoderReplayLayers(4, run_layers, [], set())
manager = DecoderReplayCudaGraphManager(cfg, DEVICE, layers)

def prepare(desc):
rows = torch.arange(desc.num_tokens)
layers.replay_batch = ReplayBatch(rows, _graph_context(desc.num_tokens))

manager.capture_replay_graphs(prepare)
states = [
torch.randn(12, *b.shape[1:], device=DEVICE).to(b.dtype)
for b in manager.input_buffers
]
rows = torch.tensor([11, 2, 7], device=DEVICE)
desc = manager.dispatch(2, 3, None, 0)
with override_forward_context(_graph_context(desc.num_tokens)):
actual = manager.run(rows, states)
for a, e in zip(actual, run_layers(*(t[rows] for t in states))):
torch.testing.assert_close(a, e, rtol=0, atol=0)
45 changes: 31 additions & 14 deletions tests/models/test_deepseek_v41_replay_batch.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@

from vllm.config import CUDAGraphMode
from vllm.models.deepseek_v41.nvidia.model_state import DeepseekV41ModelState
from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
from vllm.v1.worker.gpu.model_states.default import DefaultModelState

Expand Down Expand Up @@ -57,7 +58,8 @@ def state(monkeypatch):
cfg.scheduler_config.max_num_batched_tokens = 1024
cfg.parallel_config.data_parallel_size = 1
cfg.compilation_config.fast_moe_cold_start = False
layers = SimpleNamespace(window=WINDOW, rows=None, forward_context=None)
cfg.compilation_config.cudagraph_mode = CUDAGraphMode.NONE
layers = SimpleNamespace(window=WINDOW, replay_batch=None, metadata_prefixes=set())
model = SimpleNamespace(token_lookback_depth=0, decoder_replay_layers=layers)
builds: list = []

Expand Down Expand Up @@ -127,13 +129,8 @@ def _prepare(state, batch, cg_mode):
[],
GROUPS,
)
layers = state.decoder_replay_layers
if layers.rows is None:
assert layers.forward_context is None
return None, None
return SimpleNamespace(
rows=layers.rows, forward_context=layers.forward_context
), state.builds[-1]
replay = state.decoder_replay_layers.replay_batch
return (None, None) if replay is None else (replay, state.builds[-1])


def test_replay_batch_keeps_each_request_window(state):
Expand Down Expand Up @@ -191,9 +188,9 @@ def test_replay_batch_keeps_adaptive_verification_query_bound(state):
assert replay is not None and build.batch.max_query_len == 200


def test_only_eager_steps_trim(state):
"""A CUDA graph keeps the layers on its whole batch; an eager step trims a
padded batch to its real rows."""
def test_graph_steps_trim_only_piecewise_at_threshold(state):
"""A CUDA graph keeps the layers on its whole batch, unless it is a PIECEWISE
one at the trim threshold or above; an eager step trims a batch to its rows."""
batch = _input_batch(QUERY_LENS, SEQ_LENS, PREFILLING, num_tokens_after_padding=512)
for cg_mode in (CUDAGraphMode.PIECEWISE, CUDAGraphMode.FULL):
assert _prepare(state, batch, cg_mode) == (None, None)
Expand All @@ -202,6 +199,10 @@ def test_only_eager_steps_trim(state):
assert replay.rows.tolist() == REPLAY_ROWS
assert build.cg_mode == CUDAGraphMode.NONE
assert build.batch.num_tokens_after_padding == len(REPLAY_ROWS)
state._trim_threshold = 256
replay, _ = _prepare(state, batch, CUDAGraphMode.PIECEWISE)
assert replay is not None and replay.rows.tolist() == REPLAY_ROWS
assert _prepare(state, batch, CUDAGraphMode.FULL) == (None, None)


def test_whole_batch_forwards_get_no_replay_batch(state):
Expand Down Expand Up @@ -251,9 +252,25 @@ def test_dp_ranks_replay_together(dp_state):


def test_idle_dp_rank_dummy_trims_with_its_peers(dp_state):
dummy = _input_batch([512], [512], [False])
"""An idle rank's dummy keeps one row, so peers do not wait on its replay."""
dummy = _input_batch([256, 256], [256, 256], [False, False])
dp_state.other = (True, 129)
replay, _ = _prepare(dp_state, dummy, CUDAGraphMode.NONE)
assert replay is not None and replay.rows.shape[0] == WINDOW
assert replay is not None and replay.rows.shape[0] == 1
dp_metadata = replay.forward_context.dp_metadata
assert dp_metadata.num_tokens_across_dp_cpu.tolist() == [WINDOW, 129]
assert dp_metadata.num_tokens_across_dp_cpu.tolist() == [1, 129]


def test_dp_ranks_pad_to_one_replay_graph(dp_state):
"""Every rank pads its replay batch to the graph fitting the largest rank's."""
dp_state.replay_cudagraphs = MagicMock()
dp_state.replay_cudagraphs.dispatch.return_value = BatchExecutionDescriptor(
CUDAGraphMode.PIECEWISE, 512, None
)
dp_state.other = (True, 300)
batch = _input_batch(QUERY_LENS, SEQ_LENS, PREFILLING)
replay, _ = _prepare(dp_state, batch, CUDAGraphMode.NONE)
assert dp_state.replay_cudagraphs.dispatch.call_args.args[:2] == (3, 300)
assert replay.forward_context.batch_descriptor.num_tokens == 512
dp_metadata = replay.forward_context.dp_metadata
assert dp_metadata.num_tokens_across_dp_cpu.tolist() == [512, 512]
25 changes: 25 additions & 0 deletions vllm/config/compilation.py
Original file line number Diff line number Diff line change
Expand Up @@ -570,6 +570,26 @@ class CompilationConfig:
If we limit the video count per prompt to `0`, it will also be set to `0`
(i.e., fall back to image-only mode)."""

cudagraph_decoder_replay: bool = True
"""Run the decoder replay layers of YOCO models (e.g. DeepSeek-V4.1) in CUDA
graphs of their own, and trim them in PIECEWISE graph steps. Requires
breakable PIECEWISE graphs; off under LoRA, prompt embeddings and non-first
PP ranks."""

decoder_replay_cudagraph_capture_sizes: list[int] = field(default_factory=list)
"""Decoder replay CUDA graph sizes for YOCO models (e.g. DeepSeek-V4.1).
If empty, capture 8 sizes per power-of-two interval (e.g. step 128 in
[1024, 2048), step 256 in [2048, 4096)) up to
min(max_cudagraph_capture_size, max_num_seqs * sliding_window), including
that upper bound. Explicit sizes override this inference and must not exceed
max_num_batched_tokens. Larger replay batches run eagerly."""

decoder_replay_trim_threshold: int = Field(default=768, ge=0)
"""For YOCO models (e.g. DeepSeek-V4.1), PIECEWISE graphs with at least this
many padded tokens trim the decoder replay batch; must exceed the replay
window (128), below which nothing trims. Independent of the replay graph
capture sizes; eager steps still trim."""

# Inductor capture
compile_sizes: list[int | str] | None = None
"""Sizes to compile for inductor. In addition
Expand Down Expand Up @@ -1033,6 +1053,11 @@ def __post_init__(self) -> None:
f"Invalid backend for piecewise compilation: {self.backend}"
)

if any(s <= 0 for s in self.decoder_replay_cudagraph_capture_sizes):
raise ValueError(
"All decoder_replay_cudagraph_capture_sizes must be positive"
)

# Validate encoder CUDA graph configuration
if (
self.cudagraph_mm_encoder
Expand Down
77 changes: 60 additions & 17 deletions vllm/models/deepseek_v41/decoder_replay_layers.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,58 +7,101 @@
``DeepseekV41ModelState`` prepares those rows as a sub-batch with attention
metadata and a forward context of its own, like a microbatch;
``DecoderReplayLayers`` gathers the layer inputs by its rows, runs the layers
under that context and scatters the outputs back to batch rows. Steps that run
in a CUDA graph keep the layers on the whole batch, inside the graph.
under that context and scatters the outputs back to batch rows. FULL graphs and
small PIECEWISE ones keep the layers on the whole batch; larger PIECEWISE graphs
break out to the replay batch, which may run in a graph of its own
(decoder_replay_cudagraph.py).
"""

from collections.abc import Callable
from dataclasses import dataclass

import torch

from vllm.forward_context import ForwardContext, override_forward_context
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.forward_context import (
ForwardContext,
in_piecewise_cudagraph,
override_forward_context,
)
from vllm.model_executor.layers.fused_moe.moe_output import MoEOutput


@dataclass
class ReplayBatch:
"""One forward's replay batch; ``run_graph`` is set when it runs in a graph."""

rows: torch.Tensor
forward_context: ForwardContext
run_graph: Callable[..., tuple[torch.Tensor, ...]] | None = None


class DecoderReplayLayers:
"""Runs the replay layers on the step's replay batch.

``run_layers`` takes a batch's layer inputs and returns its per-row
outputs. ``row_buffers`` hold per-row results the source layer's indexer
publishes for the layers after it; they are compacted to the replay rows
in place.
in place. ``metadata_prefixes`` are the attention metadata keys the layers
read, so the replay batch builds only those.
"""

def __init__(
self,
window: int,
run_layers: Callable[..., tuple[torch.Tensor, ...]],
row_buffers: list[torch.Tensor],
metadata_prefixes: set[str],
) -> None:
self.window = window
self.run_layers = run_layers
self.row_buffers = row_buffers
# The replay batch, set by the model state every step: its rows of the
# batch and its forward context. None runs the layers on the batch.
self.rows: torch.Tensor | None = None
self.forward_context: ForwardContext | None = None
self.metadata_prefixes = metadata_prefixes
# Set by the model state every step; None runs the layers on the batch.
self.replay_batch: ReplayBatch | None = None
self._break_outputs: list[torch.Tensor] | None = None

def __call__(
self, hidden_states: torch.Tensor | MoEOutput, *states: torch.Tensor | None
) -> tuple[torch.Tensor, ...]:
rows = self.rows
if rows is None:
if self.replay_batch is not None and in_piecewise_cudagraph():
return self._run_in_graph_break(hidden_states, *states)
return self._run(hidden_states, *states)

@eager_break_during_capture
def _run_in_graph_break(
self, hidden_states: torch.Tensor | MoEOutput, *states: torch.Tensor | None
) -> tuple[torch.Tensor, ...]:
"""A graph break: the outputs land in fixed buffers the next segment reads."""
outputs = self._run(hidden_states, *states)
if self._break_outputs is None:
# Sized by the first call: the largest model graph is captured first.
self._break_outputs = [torch.empty_like(out) for out in outputs]
return tuple(
buf[: out.shape[0]].copy_(out)
for buf, out in zip(self._break_outputs, outputs)
)

def _run(
self, hidden_states: torch.Tensor | MoEOutput, *states: torch.Tensor | None
) -> tuple[torch.Tensor, ...]:
replay_batch = self.replay_batch
if replay_batch is None:
return self.run_layers(hidden_states, *states)
# A trimming step holds a prefill longer than the window, more tokens
# than any step whose MoE leaves its finalize to the next layer.
# Replay batch steps hold more than `window` tokens: no deferred MoE finalize.
assert isinstance(hidden_states, torch.Tensor)
rows = replay_batch.rows
num_rows = rows.shape[0]
for buf in self.row_buffers:
buf[:num_rows].copy_(buf.index_select(0, rows))
with override_forward_context(self.forward_context):
row_outputs = self.run_layers(
hidden_states.index_select(0, rows),
*(None if t is None else t.index_select(0, rows) for t in states),
)
inputs = (hidden_states, *states)
with override_forward_context(replay_batch.forward_context):
if replay_batch.run_graph is not None:
row_outputs = replay_batch.run_graph(rows, inputs)
else:
row_outputs = self.run_layers(
*(None if t is None else t.index_select(0, rows) for t in inputs)
)
# The trimmed rows' outputs stay zero; nothing reads them.
num_tokens = hidden_states.shape[0]
return tuple(
Expand Down
Loading
Loading