From c834870b3e4967e978bcf22d3e33a93c2fe1e431 Mon Sep 17 00:00:00 2001 From: Juntian Liu Date: Thu, 1 Oct 2026 04:18:52 +0000 Subject: [PATCH 1/8] [Perf][DSv4.1] Run decoder replay layers in CUDA graphs and trim in PIECEWISE graphs Run the decoder-side SWA bounded replay layers on the replay batch in breakable PIECEWISE graphs of their own, captured by a CudaGraphManager owned by DeepseekV41ModelState. PIECEWISE model graphs above VLLM_DSV41_DECODER_REPLAY_TRIM_THRESHOLD tokens (1024 by default) break out to those layers, so they trim too instead of only eager steps. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Juntian Liu --- ...test_deepseek_v41_decoder_replay_layers.py | 33 ++++- .../models/test_deepseek_v41_replay_batch.py | 75 +++++++++-- vllm/envs.py | 6 + .../deepseek_v41/decoder_replay_layers.py | 75 +++++++++-- .../nvidia/decoder_replay_cudagraph.py | 71 ++++++++++ .../models/deepseek_v41/nvidia/model_state.py | 127 +++++++++++++++--- vllm/v1/worker/gpu/model_runner.py | 6 + vllm/v1/worker/gpu/model_states/interface.py | 13 +- 8 files changed, 359 insertions(+), 47 deletions(-) create mode 100644 vllm/models/deepseek_v41/nvidia/decoder_replay_cudagraph.py diff --git a/tests/models/test_deepseek_v41_decoder_replay_layers.py b/tests/models/test_deepseek_v41_decoder_replay_layers.py index 4af797a1e6c8..b49ddd1f50d8 100644 --- a/tests/models/test_deepseek_v41_decoder_replay_layers.py +++ b/tests/models/test_deepseek_v41_decoder_replay_layers.py @@ -5,12 +5,17 @@ import pytest import torch +from vllm.config import CUDAGraphMode from vllm.forward_context import ( + BatchDescriptor, ForwardContext, get_forward_context, override_forward_context, ) -from vllm.models.deepseek_v41.decoder_replay_layers import DecoderReplayLayers +from vllm.models.deepseek_v41.decoder_replay_layers import ( + DecoderReplayLayers, + ReplayBatch, +) pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA") @@ -32,9 +37,9 @@ def _context(metadata): def _set_replay_batch(layers, rows, metadata): - layers.rows = torch.tensor(rows, device=DEVICE) - layers.forward_context = ForwardContext( - no_compile_layers={}, attn_metadata=metadata, slot_mapping={} + layers.replay_batch = ReplayBatch( + torch.tensor(rows, device=DEVICE), + ForwardContext(no_compile_layers={}, attn_metadata=metadata, slot_mapping={}), ) @@ -102,3 +107,23 @@ def run_layers(hidden_states, *rest): with override_forward_context(_context(full)): (output,) = layers(*states) assert output is states[0] and seen["attn_metadata"] is full + + +def test_graph_break_writes_fixed_buffers(): + """Above the trim threshold, a PIECEWISE graph's replay layers write fixed + buffers, which the graph's next segment reads on every replay.""" + layers = DecoderReplayLayers(WINDOW, lambda hidden, *rest: (hidden * 2,), []) + layers.trim_threshold = NUM_TOKENS - 1 + context = ForwardContext( + no_compile_layers={}, + attn_metadata=object(), + slot_mapping={}, + cudagraph_runtime_mode=CUDAGraphMode.PIECEWISE, + batch_descriptor=BatchDescriptor(num_tokens=NUM_TOKENS), + ) + states = _states(NUM_TOKENS) + with override_forward_context(context): + (first,) = layers(*states) + (second,) = layers(states[0] * 3, *states[1:]) + assert first.data_ptr() == second.data_ptr() + assert torch.equal(second, states[0] * 6) diff --git a/tests/models/test_deepseek_v41_replay_batch.py b/tests/models/test_deepseek_v41_replay_batch.py index f6efe5c51608..86ba421ef9c6 100644 --- a/tests/models/test_deepseek_v41_replay_batch.py +++ b/tests/models/test_deepseek_v41_replay_batch.py @@ -12,6 +12,8 @@ from vllm.config import CUDAGraphMode from vllm.models.deepseek_v41.nvidia.model_state import DeepseekV41ModelState +from vllm.v1.attention.backends.utils import PAD_SLOT_ID +from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers from vllm.v1.worker.gpu.model_states.default import DefaultModelState @@ -57,7 +59,7 @@ def state(monkeypatch): cfg.scheduler_config.max_num_batched_tokens = 1024 cfg.parallel_config.data_parallel_size = 1 cfg.compilation_config.fast_moe_cold_start = False - layers = SimpleNamespace(window=WINDOW, rows=None, forward_context=None) + layers = SimpleNamespace(window=WINDOW, replay_batch=None, trim_threshold=None) model = SimpleNamespace(token_lookback_depth=0, decoder_replay_layers=layers) builds: list = [] @@ -127,13 +129,8 @@ def _prepare(state, batch, cg_mode): [], GROUPS, ) - layers = state.decoder_replay_layers - if layers.rows is None: - assert layers.forward_context is None - return None, None - return SimpleNamespace( - rows=layers.rows, forward_context=layers.forward_context - ), state.builds[-1] + replay = state.decoder_replay_layers.replay_batch + return (None, None) if replay is None else (replay, state.builds[-1]) def test_replay_batch_keeps_each_request_window(state): @@ -204,6 +201,23 @@ def test_only_eager_steps_trim(state): assert build.batch.num_tokens_after_padding == len(REPLAY_ROWS) +def test_piecewise_graphs_above_the_trim_threshold_replay(state): + """A PIECEWISE graph above the trim threshold breaks out to the replay layers, + which run on a replay batch even when nothing trims.""" + state.decoder_replay_layers.trim_threshold = 256 + batch = _input_batch(QUERY_LENS, SEQ_LENS, PREFILLING, num_tokens_after_padding=512) + replay, _ = _prepare(state, batch, CUDAGraphMode.PIECEWISE) + assert replay is not None and replay.rows.tolist() == REPLAY_ROWS + assert _prepare(state, batch, CUDAGraphMode.FULL) == (None, None) + short = _input_batch( + [100, 100], [100, 100], [True, True], num_tokens_after_padding=512 + ) + replay, _ = _prepare(state, short, CUDAGraphMode.PIECEWISE) + assert replay is not None and replay.rows.tolist() == list(range(200)) + state.decoder_replay_layers.trim_threshold = 512 + assert _prepare(state, batch, CUDAGraphMode.PIECEWISE) == (None, None) + + def test_whole_batch_forwards_get_no_replay_batch(state): short = _input_batch([100, 100], [100, 100], [True, True]) for cg_mode in (CUDAGraphMode.NONE, CUDAGraphMode.PIECEWISE, CUDAGraphMode.FULL): @@ -257,3 +271,48 @@ def test_idle_dp_rank_dummy_trims_with_its_peers(dp_state): assert replay is not None and replay.rows.shape[0] == WINDOW dp_metadata = replay.forward_context.dp_metadata assert dp_metadata.num_tokens_across_dp_cpu.tolist() == [WINDOW, 129] + + +def _graphs(num_tokens: int): + desc = BatchExecutionDescriptor(CUDAGraphMode.PIECEWISE, num_tokens, None) + return SimpleNamespace( + run=object(), + dispatch=MagicMock(return_value=desc), + capture_replay_graphs=lambda f: f(desc), + ) + + +def test_replay_batch_pads_to_its_graph_on_every_rank(dp_state): + """An eager step whose replay batch fits a graph pads it to the graph that + fits the largest rank's batch.""" + dp_state.replay_graphs = _graphs(512) + dp_state.other = (True, 300) + replay, build = _prepare( + dp_state, _input_batch(QUERY_LENS, SEQ_LENS, PREFILLING), CUDAGraphMode.NONE + ) + assert dp_state.replay_graphs.dispatch.call_args.args[:2] == (3, 300) + assert replay.run_graph is dp_state.replay_graphs.run + context = replay.forward_context + assert context.cudagraph_runtime_mode == CUDAGraphMode.PIECEWISE + assert context.batch_descriptor.num_tokens == 512 + assert context.dp_metadata.num_tokens_across_dp_cpu.tolist() == [512, 512] + assert replay.rows.tolist() == REPLAY_ROWS + assert build.batch.num_tokens == len(REPLAY_ROWS) + assert build.batch.num_tokens_after_padding == 512 + assert (build.slot_mappings[:, len(REPLAY_ROWS) :] == PAD_SLOT_ID).all() + + +def test_capture_replays_a_whole_dummy_batch(state): + """A capture builds its graph's replay batch from a dummy batch that keeps + every row.""" + _prepare(state, _input_batch(QUERY_LENS, SEQ_LENS, PREFILLING), CUDAGraphMode.NONE) + state.replay_graphs = _graphs(64) + block_tables = SimpleNamespace( + get_dummy_block_tables=lambda n: tuple(t[:n] for t in BLOCK_TABLES), + get_dummy_slot_mappings=lambda n: _slot_mappings(n).fill_(PAD_SLOT_ID), + ) + state.capture_cudagraphs(InputBuffers(16, 1024, DEVICE), block_tables, [], GROUPS) + replay = state.decoder_replay_layers.replay_batch + assert replay.rows.tolist() == list(range(64)) + assert replay.forward_context.batch_descriptor.num_tokens == 64 + assert state.builds[-1].batch.num_reqs == 16 diff --git a/vllm/envs.py b/vllm/envs.py index 5daa4fd3e67e..0b63cfd5be11 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -305,6 +305,7 @@ VLLM_DISABLE_DSV4_MEGAMOE_SHARED_EXPERT_FUSION: bool = False VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD: int = 256 VLLM_MULTI_STREAM_GEMM_TOKEN_THRESHOLD: int = 1024 + VLLM_DSV41_DECODER_REPLAY_TRIM_THRESHOLD: int = 1024 VLLM_COMPILE_CACHE_SAVE_FORMAT: Literal["binary", "unpacked"] = "binary" VLLM_USE_V2_MODEL_RUNNER: bool | None = None VLLM_LOG_MODEL_INSPECTION: bool = False @@ -2105,6 +2106,11 @@ def _resolve_rust_cli_path() -> str | None: "VLLM_MULTI_STREAM_GEMM_TOKEN_THRESHOLD": lambda: int( os.getenv("VLLM_MULTI_STREAM_GEMM_TOKEN_THRESHOLD", "1024") ), + # DeepSeek-V4.1 PIECEWISE CUDA graph steps of more tokens than this trim the + # decoder replay layers too, which then run in CUDA graphs of their own. + "VLLM_DSV41_DECODER_REPLAY_TRIM_THRESHOLD": lambda: int( + os.getenv("VLLM_DSV41_DECODER_REPLAY_TRIM_THRESHOLD", "1024") + ), # Format for saving torch.compile cache artifacts # - "binary": saves as binary file # Safe for multiple vllm serve processes accessing the same torch compile cache. diff --git a/vllm/models/deepseek_v41/decoder_replay_layers.py b/vllm/models/deepseek_v41/decoder_replay_layers.py index 869029aa7484..8d929162ebe2 100644 --- a/vllm/models/deepseek_v41/decoder_replay_layers.py +++ b/vllm/models/deepseek_v41/decoder_replay_layers.py @@ -8,17 +8,35 @@ metadata and a forward context of its own, like a microbatch; ``DecoderReplayLayers`` gathers the layer inputs by its rows, runs the layers under that context and scatters the outputs back to batch rows. Steps that run -in a CUDA graph keep the layers on the whole batch, inside the graph. +in a CUDA graph keep the layers on the whole batch, inside the graph; an eager +step's replay batch may run in a graph of its own (decoder_replay_cudagraph.py). """ from collections.abc import Callable +from dataclasses import dataclass import torch -from vllm.forward_context import ForwardContext, override_forward_context +from vllm.compilation.breakable_cudagraph import eager_break_during_capture +from vllm.forward_context import ( + ForwardContext, + get_forward_context, + in_piecewise_cudagraph, + override_forward_context, +) from vllm.model_executor.layers.fused_moe.moe_output import MoEOutput +@dataclass +class ReplayBatch: + """The replay layers' batch of one forward: its rows of the batch, its + forward context and, when it runs in a CUDA graph, the graph's runner.""" + + rows: torch.Tensor + forward_context: ForwardContext + run_graph: Callable[..., tuple[torch.Tensor, ...]] | None = None + + class DecoderReplayLayers: """Runs the replay layers on the step's replay batch. @@ -37,28 +55,59 @@ def __init__( self.window = window self.run_layers = run_layers self.row_buffers = row_buffers - # The replay batch, set by the model state every step: its rows of the - # batch and its forward context. None runs the layers on the batch. - self.rows: torch.Tensor | None = None - self.forward_context: ForwardContext | None = None + # Set by the model state every step; None runs the layers on the batch. + self.replay_batch: ReplayBatch | None = None + # PIECEWISE model graphs of more tokens break out of the graph to run the + # layers on the replay batch; set with the replay graphs. + self.trim_threshold: int | None = None + self._graph_outputs: list[torch.Tensor] | None = None def __call__( self, hidden_states: torch.Tensor | MoEOutput, *states: torch.Tensor | None ) -> tuple[torch.Tensor, ...]: - rows = self.rows - if rows is None: + if self.trim_threshold is not None and in_piecewise_cudagraph(): + batch_descriptor = get_forward_context().batch_descriptor + assert batch_descriptor is not None + if batch_descriptor.num_tokens > self.trim_threshold: + return self._run_in_graph_break(hidden_states, *states) + return self._run(hidden_states, *states) + + @eager_break_during_capture + def _run_in_graph_break( + self, hidden_states: torch.Tensor | MoEOutput, *states: torch.Tensor | None + ) -> tuple[torch.Tensor, ...]: + """An eager break of the model's graph: the outputs land in fixed buffers, + which the graph's next segment reads.""" + outputs = self._run(hidden_states, *states) + if self._graph_outputs is None: + # The largest model graph is captured first. + self._graph_outputs = [torch.empty_like(out) for out in outputs] + return tuple( + buf[: out.shape[0]].copy_(out) + for buf, out in zip(self._graph_outputs, outputs) + ) + + def _run( + self, hidden_states: torch.Tensor | MoEOutput, *states: torch.Tensor | None + ) -> tuple[torch.Tensor, ...]: + replay_batch = self.replay_batch + if replay_batch is None: return self.run_layers(hidden_states, *states) # A trimming step holds a prefill longer than the window, more tokens # than any step whose MoE leaves its finalize to the next layer. assert isinstance(hidden_states, torch.Tensor) + rows = replay_batch.rows num_rows = rows.shape[0] for buf in self.row_buffers: buf[:num_rows].copy_(buf.index_select(0, rows)) - with override_forward_context(self.forward_context): - row_outputs = self.run_layers( - hidden_states.index_select(0, rows), - *(None if t is None else t.index_select(0, rows) for t in states), - ) + inputs = (hidden_states, *states) + with override_forward_context(replay_batch.forward_context): + if replay_batch.run_graph is not None: + row_outputs = replay_batch.run_graph(rows, inputs) + else: + row_outputs = self.run_layers( + *(None if t is None else t.index_select(0, rows) for t in inputs) + ) # The trimmed rows' outputs stay zero; nothing reads them. num_tokens = hidden_states.shape[0] return tuple( diff --git a/vllm/models/deepseek_v41/nvidia/decoder_replay_cudagraph.py b/vllm/models/deepseek_v41/nvidia/decoder_replay_cudagraph.py new file mode 100644 index 000000000000..156fdb7d60de --- /dev/null +++ b/vllm/models/deepseek_v41/nvidia/decoder_replay_cudagraph.py @@ -0,0 +1,71 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""CUDA graphs of the decoder replay layers on a trimming step's replay batch.""" + +from collections.abc import Callable +from dataclasses import replace + +import torch + +from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphWrapper +from vllm.config import CUDAGraphMode, VllmConfig +from vllm.config.kernel import MEGA_MOE_BACKENDS +from vllm.forward_context import get_forward_context, override_forward_context +from vllm.models.deepseek_v41.decoder_replay_layers import DecoderReplayLayers +from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor as Desc +from vllm.v1.worker.gpu.cudagraph_utils import CudaGraphManager + + +class DecoderReplayCudaGraphManager(CudaGraphManager): + """Breakable PIECEWISE graphs of the replay layers keyed by the replay batch's + padded size. They read the layer inputs from static buffers.""" + + def __init__(self, vllm_config: VllmConfig, device, layers: DecoderReplayLayers): + super().__init__(vllm_config, device, CUDAGraphMode.PIECEWISE, 1) + self.layers = layers + self.breakable_cg_runner: BreakableCUDAGraphWrapper = BreakableCUDAGraphWrapper( + layers.run_layers, vllm_config + ) + config, act = vllm_config.model_config.hf_config, vllm_config.model_config.dtype + assert isinstance(act, torch.dtype) + hc, h, f32 = config.hc_mult, config.hidden_size, torch.float32 + mega_moe = vllm_config.kernel_config.moe_backend in MEGA_MOE_BACKENDS + ids = torch.int64 if mega_moe else torch.int32 + # hidden_states, positions, input_ids, pre_mix, post_mix, res_mix, residual + shapes = [(h,), (), (), (hc,), (hc, 1), (hc, hc), (hc, h)] + dtypes = [act, torch.int64, ids, f32, f32, f32, act] + n = self.compilation_config.max_cudagraph_capture_size + self.inputs = [ + torch.zeros(n, *s, dtype=d, device=device) for s, d in zip(shapes, dtypes) + ] + + def capture_replay_graphs(self, prepare: Callable[[Desc], None]) -> None: + """Capture each size on the replay batch ``prepare`` sets on the layers.""" + + def create_forward_fn(desc: Desc, warmup: bool): + prepare(desc) + assert self.layers.replay_batch is not None + context = self.layers.replay_batch.forward_context + inputs = [buf[: desc.num_tokens] for buf in self.inputs] + + def forward_fn(cg_mode: CUDAGraphMode) -> None: + with override_forward_context( + replace(context, cudagraph_runtime_mode=cg_mode) + ): + self.breakable_cg_runner(*inputs) + + return forward_fn + + self.capture(create_forward_fn, "Capturing decoder replay CUDA graphs") + self.layers.replay_batch = None + + def run(self, rows: torch.Tensor, inputs: tuple) -> tuple[torch.Tensor, ...]: + """Replay the forward context's graph on ``rows`` of ``inputs``.""" + num_rows = rows.shape[0] + for buf, t in zip(self.inputs, inputs): + torch.index_select(t, 0, rows, out=buf[:num_rows]) + batch_descriptor = get_forward_context().batch_descriptor + assert batch_descriptor is not None + num_tokens = batch_descriptor.num_tokens + outputs = self.breakable_cg_runner(*(buf[:num_tokens] for buf in self.inputs)) + return tuple(out[:num_rows] for out in outputs) diff --git a/vllm/models/deepseek_v41/nvidia/model_state.py b/vllm/models/deepseek_v41/nvidia/model_state.py index 30b287db22ee..1f320c0e18fd 100644 --- a/vllm/models/deepseek_v41/nvidia/model_state.py +++ b/vllm/models/deepseek_v41/nvidia/model_state.py @@ -8,10 +8,18 @@ import torch.distributed as dist import torch.nn as nn +import vllm.envs as envs +from vllm.compilation.breakable_cudagraph import is_breakable_cudagraph_enabled from vllm.config import CUDAGraphMode, VllmConfig -from vllm.distributed.parallel_state import get_dp_group -from vllm.forward_context import DPMetadata, create_forward_context -from vllm.models.deepseek_v41.decoder_replay_layers import DecoderReplayLayers +from vllm.distributed.parallel_state import get_dp_group, get_pp_group +from vllm.forward_context import BatchDescriptor, DPMetadata, create_forward_context +from vllm.models.deepseek_v41.decoder_replay_layers import ( + DecoderReplayLayers, + ReplayBatch, +) +from vllm.models.deepseek_v41.nvidia.decoder_replay_cudagraph import ( + DecoderReplayCudaGraphManager, +) from vllm.triton_utils import tl, triton from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadataBuilder from vllm.v1.attention.backends.utils import PAD_SLOT_ID @@ -19,8 +27,10 @@ from vllm.v1.kv_cache_interface import KVCacheConfig from vllm.v1.worker.dp_utils import should_skip_dp_coordination from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer +from vllm.v1.worker.gpu.block_table import BlockTables from vllm.v1.worker.gpu.buffer_utils import UvaBufferPool -from vllm.v1.worker.gpu.input_batch import InputBatch +from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor +from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache from vllm.v1.worker.gpu.model_states.default import DefaultModelState from vllm.v1.worker.gpu.model_states.interface import ModelSpecificAttnMetadata @@ -230,6 +240,22 @@ def __init__( self.max_num_reqs, dtype=torch.int32, device=device ) self._replay_attn_groups: list[list[AttentionGroup]] | None = None + layers = self.decoder_replay_layers + self.replay_graphs: DecoderReplayCudaGraphManager | None = None + if ( + layers is not None + and vllm_config.compilation_config.cudagraph_mode.has_piecewise_cudagraphs() + and is_breakable_cudagraph_enabled() + and get_pp_group().is_first_rank + # Neither the trimmed rows' LoRA mapping nor optional input ids are + # captured. + and vllm_config.lora_config is None + and not vllm_config.model_config.enable_prompt_embeds + ): + self.replay_graphs = DecoderReplayCudaGraphManager( + vllm_config, device, layers + ) + layers.trim_threshold = envs.VLLM_DSV41_DECODER_REPLAY_TRIM_THRESHOLD def add_request(self, req_index: int, new_req_data: NewRequestData) -> None: super().add_request(req_index, new_req_data) @@ -409,15 +435,25 @@ def _prepare_replay_batch( attn_groups: list[list[AttentionGroup]], kv_cache_config: KVCacheConfig, replay_start: torch.Tensor, + capture_desc: BatchExecutionDescriptor | None = None, ) -> None: """Set the replay layers' batch for this forward, or none when nothing - trims. Only eager steps trim: a CUDA graph keeps the layers on the whole - batch, and ranks share the graph mode. Under data parallelism every rank - replays if any does.""" + trims. Eager steps trim, and so do PIECEWISE graph steps above the trim + threshold, whose graph breaks out to the replay layers; FULL graphs and + smaller PIECEWISE ones keep the layers on the whole batch. Ranks share + the graph mode and size; under data parallelism every rank replays if + any does.""" layers = self.decoder_replay_layers assert layers is not None - layers.rows = layers.forward_context = None - if cudagraph_mode != CUDAGraphMode.NONE: + layers.replay_batch = None + # Captures and PIECEWISE graphs above the trim threshold (which break out + # to the replay layers) run the layers on a replay batch, trimmed or not. + replays = capture_desc is not None or ( + cudagraph_mode == CUDAGraphMode.PIECEWISE + and layers.trim_threshold is not None + and input_batch.num_tokens_after_padding > layers.trim_threshold + ) + if cudagraph_mode != CUDAGraphMode.NONE and not replays: return num_reqs = input_batch.num_reqs @@ -429,7 +465,7 @@ def _prepare_replay_batch( ) # Dummy batches (captures, an idle DP rank's) are not prefills: they # keep their rows unless a rank trims. - trims = bool( + trims = replays or bool( (kept_lens < query_lens)[input_batch.is_prefilling_np[:num_reqs]].any() ) num_tokens = int(kept_lens.sum()) @@ -439,8 +475,23 @@ def _prepare_replay_batch( if not trims: return + graphs, run_graph = self.replay_graphs, None + num_padded, cg_mode = num_tokens, CUDAGraphMode.NONE + dp_tokens = dp_metadata.num_tokens_across_dp_cpu if dp_metadata else None + if graphs is not None: + # Every rank runs the graph that fits the largest rank's batch. + max_tokens = num_tokens if dp_tokens is None else int(dp_tokens.max()) + desc = capture_desc or graphs.dispatch(num_reqs, max_tokens, None, 0) + if desc.cg_mode == CUDAGraphMode.PIECEWISE: + num_padded, cg_mode = desc.num_tokens, desc.cg_mode + run_graph = graphs.run + if dp_tokens is not None: + dp_tokens.fill_(num_padded) + dp_metadata = DPMetadata.make( + self.vllm_config.parallel_config, num_padded, dp_tokens + ) kept_batch, kept_slot_mappings = self._kept_input_batch( - input_batch, slot_mappings, replay_start, kept_lens + input_batch, slot_mappings, replay_start, kept_lens, num_padded ) attn_metadata = super().prepare_attn( kept_batch, @@ -453,22 +504,55 @@ def _prepare_replay_batch( self._kept_kv_start[:num_reqs] ), ) - layers.rows = self._kept_rows[:num_tokens] - layers.forward_context = create_forward_context( - attn_metadata, - self.vllm_config, - dp_metadata=dp_metadata, - slot_mapping=build_slot_mappings_by_layer( - kept_slot_mappings, kv_cache_config + layers.replay_batch = ReplayBatch( + self._kept_rows[:num_tokens], + create_forward_context( + attn_metadata, + self.vllm_config, + dp_metadata=dp_metadata, + cudagraph_runtime_mode=cg_mode, + batch_descriptor=BatchDescriptor(num_tokens=num_padded), + slot_mapping=build_slot_mappings_by_layer( + kept_slot_mappings, kv_cache_config + ), ), + run_graph, ) + def capture_cudagraphs( + self, + input_buffers: InputBuffers, + block_tables: BlockTables, + attn_groups: list[list[AttentionGroup]], + kv_cache_config: KVCacheConfig, + ) -> None: + def prepare(desc: BatchExecutionDescriptor) -> None: + # The runtime path on a dummy batch: the graph reads the buffers a + # real step's replay batch fills. + num_reqs = min(desc.num_tokens, self.max_num_reqs) + self._prepare_replay_batch( + InputBatch.make_dummy(num_reqs, desc.num_tokens, input_buffers), + CUDAGraphMode.NONE, + block_tables.get_dummy_block_tables(num_reqs), + block_tables.get_dummy_slot_mappings(desc.num_tokens), + attn_groups, + kv_cache_config, + self._replay_start[:num_reqs].zero_(), + capture_desc=desc, + ) + + if self.replay_graphs is not None: + # Profiling captures run on throwaway groups: build their own builders. + self._replay_attn_groups = None + self.replay_graphs.capture_replay_graphs(prepare) + def _kept_input_batch( self, input_batch: InputBatch, slot_mappings: torch.Tensor, replay_start: torch.Tensor, kept_lens: np.ndarray, + num_padded: int, ) -> tuple[InputBatch, torch.Tensor]: """Build the sub-``InputBatch`` of each request's last ``kept_lens`` rows, and its slot mappings. @@ -486,7 +570,8 @@ def _kept_input_batch( input_batch.query_start_loc_np[: num_reqs + 1] - kept_query_start_loc_np ) assert self._kept_slot_mappings is not None - kept_slot_mappings = self._kept_slot_mappings[:, :num_tokens] + kept_slot_mappings = self._kept_slot_mappings[:, :num_padded] + kept_slot_mappings[:, num_tokens:].fill_(PAD_SLOT_ID) _gather_replay_batch_kernel[(num_reqs,)]( input_batch.query_start_loc, dropped_before, @@ -511,11 +596,11 @@ def _kept_input_batch( kept_batch = replace( input_batch, num_tokens=num_tokens, - num_tokens_after_padding=num_tokens, + num_tokens_after_padding=num_padded, query_start_loc=self._kept_query_start_loc[: num_reqs + 1], query_start_loc_np=kept_query_start_loc_np, max_query_len=max_query_len, - positions=self._kept_positions[:num_tokens], + positions=self._kept_positions[:num_padded], fast_prefill=None, ) return kept_batch, kept_slot_mappings diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index fed9e80d0c26..b201358aea0b 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -1065,6 +1065,12 @@ def capture_model(self, *, profile_only: bool = False) -> int: self.lora_config, self ), ) + self.model_state.capture_cudagraphs( + input_buffers, + self.block_tables, + self.attn_groups, + self.kv_cache_config, + ) if self.speculator is not None: with use_workspace_lane(self._draft_workspace_lane): self.speculator.capture() diff --git a/vllm/v1/worker/gpu/model_states/interface.py b/vllm/v1/worker/gpu/model_states/interface.py index ee3e519a6e74..9ac7706de880 100644 --- a/vllm/v1/worker/gpu/model_states/interface.py +++ b/vllm/v1/worker/gpu/model_states/interface.py @@ -17,7 +17,8 @@ from vllm.v1.core.sched.output import NewRequestData from vllm.v1.kv_cache_interface import KVCacheConfig from vllm.v1.worker.encoder_cudagraph import EncoderCudaGraphManager -from vllm.v1.worker.gpu.input_batch import InputBatch +from vllm.v1.worker.gpu.block_table import BlockTables +from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache from vllm.v1.worker.gpu.mm.encoder_runner import EncoderRunner from vllm.v1.worker.gpu.states import RequestState @@ -126,6 +127,16 @@ def remove_request(self, req_id: str) -> None: def apply_staged_writes(self) -> None: return None + def capture_cudagraphs( + self, + input_buffers: InputBuffers, + block_tables: BlockTables, + attn_groups: list[list[AttentionGroup]], + kv_cache_config: KVCacheConfig, + ) -> None: + """Capture the CUDA graphs this state runs inside the model's forward.""" + return None + def get_additional_cg_support(self) -> tuple[AttentionCGSupport, str | None]: """Cudagraph support of attention groups this ModelState builds outside ``init_attn_backend`` (e.g. encoder-only layers). From df6c92b0253e4a1b0396d069ed7f6c6c2bbba648 Mon Sep 17 00:00:00 2001 From: Juntian Liu Date: Thu, 1 Oct 2026 06:06:56 +0000 Subject: [PATCH 2/8] add decoder cudagraph configs Signed-off-by: Juntian Liu --- tests/compile/test_config.py | 20 +++ ...test_deepseek_v41_decoder_replay_layers.py | 139 +++++++++++++++++- .../models/test_deepseek_v41_replay_batch.py | 30 ++++ vllm/config/compilation.py | 17 +++ vllm/envs.py | 6 - .../nvidia/decoder_replay_cudagraph.py | 26 +++- .../models/deepseek_v41/nvidia/model_state.py | 5 +- 7 files changed, 231 insertions(+), 12 deletions(-) diff --git a/tests/compile/test_config.py b/tests/compile/test_config.py index 24cb816b5d9d..d6ac2ce4e609 100644 --- a/tests/compile/test_config.py +++ b/tests/compile/test_config.py @@ -1366,3 +1366,23 @@ def test_inductor_asserts_user_override(monkeypatch): assert config.inductor_compile_config.get("size_asserts") is True if not _is_torch_equal_or_newer(torch.__version__, "2.12.0.dev"): assert config.inductor_compile_config.get("alignment_asserts") is False + + +@pytest.mark.parametrize( + "kwargs", + [ + {"decoder_replay_cudagraph_capture_sizes": [0]}, + {"decoder_replay_cudagraph_capture_sizes": [128, -1]}, + {"decoder_replay_trim_threshold": -1}, + ], +) +def test_decoder_replay_config_rejects_invalid_values(kwargs): + with pytest.raises(ValueError, match="decoder_replay"): + CompilationConfig(**kwargs) + + +def test_decoder_replay_config_changes_compilation_hash(): + default = CompilationConfig() + always_trim = CompilationConfig(decoder_replay_trim_threshold=0) + custom_sizes = CompilationConfig(decoder_replay_cudagraph_capture_sizes=[128, 1024]) + assert len({c.compute_hash() for c in (default, always_trim, custom_sizes)}) == 3 diff --git a/tests/models/test_deepseek_v41_decoder_replay_layers.py b/tests/models/test_deepseek_v41_decoder_replay_layers.py index b49ddd1f50d8..e5dabf310038 100644 --- a/tests/models/test_deepseek_v41_decoder_replay_layers.py +++ b/tests/models/test_deepseek_v41_decoder_replay_layers.py @@ -2,10 +2,12 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """DecoderReplayLayers runs the replay layers on the forward's replay batch.""" +from types import SimpleNamespace + import pytest import torch -from vllm.config import CUDAGraphMode +from vllm.config import CompilationConfig, CUDAGraphMode from vllm.forward_context import ( BatchDescriptor, ForwardContext, @@ -16,6 +18,10 @@ DecoderReplayLayers, ReplayBatch, ) +from vllm.models.deepseek_v41.nvidia.decoder_replay_cudagraph import ( + DecoderReplayCudaGraphManager, +) +from vllm.v1.worker.gpu import cudagraph_utils as gpu_cudagraph_utils pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA") @@ -127,3 +133,134 @@ def test_graph_break_writes_fixed_buffers(): (second,) = layers(states[0] * 3, *states[1:]) assert first.data_ptr() == second.data_ptr() assert torch.equal(second, states[0] * 6) + + +@pytest.fixture +def replay_config(monkeypatch): + monkeypatch.setattr( + gpu_cudagraph_utils, + "get_pp_group", + lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True), + ) + return SimpleNamespace( + compilation_config=CompilationConfig(), + scheduler_config=SimpleNamespace(max_num_seqs=8, max_num_batched_tokens=8192), + parallel_config=SimpleNamespace(tensor_parallel_size=1), + speculative_config=None, + cache_config=SimpleNamespace(use_kda_recoverssm=False), + model_config=SimpleNamespace( + hf_config=SimpleNamespace(hc_mult=HC, hidden_size=HIDDEN), + dtype=torch.float32, + ), + kernel_config=SimpleNamespace(moe_backend="auto"), + ) + + +@pytest.mark.parametrize( + "max_num_seqs,main_sizes,replay_sizes,expected", + [ + (8, [6, 1020, 2046, 8190], [], [6, 1020, 1024]), + (128, [6, 1020, 2046, 8190], [], [6, 1020, 2046, 8190]), + (8, [6, 510], [], [6, 510]), + (8, [6, 1020], [2048, 256, 2048], [256, 2048]), + ], +) +def test_decoder_replay_graph_sizes( + replay_config, max_num_seqs, main_sizes, replay_sizes, expected +): + """Replay captures, dispatch and buffers agree without changing model sizes.""" + cfg = replay_config + cfg.scheduler_config.max_num_seqs = max_num_seqs + cfg.compilation_config.cudagraph_capture_sizes = main_sizes + cfg.compilation_config.max_cudagraph_capture_size = main_sizes[-1] + cfg.compilation_config.decoder_replay_cudagraph_capture_sizes = replay_sizes + layers = DecoderReplayLayers(128, lambda *args: args, []) + manager = DecoderReplayCudaGraphManager(cfg, torch.device("cpu"), layers) + assert ( + sorted(d.num_tokens for d in manager._capture_descs[CUDAGraphMode.PIECEWISE]) + == expected + ) + assert all(buf.shape[0] == expected[-1] for buf in manager.inputs) + manager._graphs_captured = True + for tokens in (1, expected[-1] - 1, expected[-1]): + desc = manager.dispatch(max_num_seqs, tokens, None, 0) + assert desc.cg_mode == CUDAGraphMode.PIECEWISE + assert desc.num_tokens == next(s for s in expected if s >= tokens) + assert manager.dispatch(max_num_seqs, expected[-1] + 1, None, 0).cg_mode == ( + CUDAGraphMode.NONE + ) + assert cfg.compilation_config.cudagraph_capture_sizes == main_sizes + assert cfg.compilation_config.max_cudagraph_capture_size == main_sizes[-1] + assert cfg.compilation_config.decoder_replay_cudagraph_capture_sizes == replay_sizes + + +def test_decoder_replay_graph_sizes_fit_runner_buffers(replay_config): + cfg = replay_config + cfg.compilation_config.decoder_replay_cudagraph_capture_sizes = [ + cfg.scheduler_config.max_num_batched_tokens + 1 + ] + with pytest.raises(ValueError, match="must not exceed max_num_batched_tokens"): + DecoderReplayCudaGraphManager( + cfg, torch.device("cpu"), DecoderReplayLayers(128, lambda *args: args, []) + ) + + +@torch.inference_mode() +def test_bounded_replay_cuda_graph_matches_eager(replay_config, monkeypatch): + """Changing replay rows and padding reuses captured graphs with eager results.""" + cfg = replay_config + cfg.compilation_config.cudagraph_capture_sizes = [4, 8, 16] + cfg.compilation_config.max_cudagraph_capture_size = 16 + cfg.scheduler_config.max_num_seqs = 2 + monkeypatch.setattr( + gpu_cudagraph_utils, + "graph_capture", + lambda device: torch.cuda.stream(torch.cuda.Stream(device=device)), + ) + monkeypatch.setattr(gpu_cudagraph_utils, "is_global_first_rank", lambda: False) + + def run_layers(hidden, positions, ids, pre, post, mix, residual): + return residual + hidden[:, None, :] + positions[:, None, None], pre + ids[ + :, None + ] + + layers = DecoderReplayLayers(4, run_layers, []) + manager = DecoderReplayCudaGraphManager(cfg, DEVICE, layers) + + def prepare(desc): + layers.replay_batch = ReplayBatch( + torch.arange(desc.num_tokens, device=DEVICE), + ForwardContext( + no_compile_layers={}, + attn_metadata={}, + slot_mapping={}, + batch_descriptor=BatchDescriptor(num_tokens=desc.num_tokens), + ), + ) + + torch.accelerator.synchronize() + manager.capture_replay_graphs(prepare) + torch.accelerator.synchronize() + assert {d.num_tokens for d in manager.breakable_cg_runner.entries} == {4, 8} + assert layers.replay_batch is None + states = tuple( + torch.randn(12, *buf.shape[1:], device=DEVICE).to(buf.dtype) + for buf in manager.inputs + ) + for row_ids in ([11, 2, 7], [1, 2, 3, 4, 5, 6, 7, 8], [9, 8, 7, 6, 5], [3]): + rows = torch.tensor(row_ids, device=DEVICE) + desc = manager.dispatch(2, len(row_ids), None, 0) + context = ForwardContext( + no_compile_layers={}, + attn_metadata={}, + slot_mapping={}, + cudagraph_runtime_mode=desc.cg_mode, + batch_descriptor=BatchDescriptor(num_tokens=desc.num_tokens), + ) + with override_forward_context(context): + actual = manager.run(rows, states) + expected = run_layers(*(t[rows] for t in states)) + for a, e in zip(actual, expected): + torch.testing.assert_close(a, e, rtol=0, atol=0) + states[0].add_(1) + assert len(manager.breakable_cg_runner.entries) == 2 diff --git a/tests/models/test_deepseek_v41_replay_batch.py b/tests/models/test_deepseek_v41_replay_batch.py index 86ba421ef9c6..1cf387fa9b98 100644 --- a/tests/models/test_deepseek_v41_replay_batch.py +++ b/tests/models/test_deepseek_v41_replay_batch.py @@ -59,6 +59,7 @@ def state(monkeypatch): cfg.scheduler_config.max_num_batched_tokens = 1024 cfg.parallel_config.data_parallel_size = 1 cfg.compilation_config.fast_moe_cold_start = False + cfg.compilation_config.cudagraph_mode = CUDAGraphMode.NONE layers = SimpleNamespace(window=WINDOW, replay_batch=None, trim_threshold=None) model = SimpleNamespace(token_lookback_depth=0, decoder_replay_layers=layers) builds: list = [] @@ -316,3 +317,32 @@ def test_capture_replays_a_whole_dummy_batch(state): assert replay.rows.tolist() == list(range(64)) assert replay.forward_context.batch_descriptor.num_tokens == 64 assert state.builds[-1].batch.num_reqs == 16 + + +@pytest.mark.parametrize("keep_rows,other_tokens", [(False, 500), (True, 200)]) +def test_replay_above_graph_capacity_falls_back_without_dropping_rows( + dp_state, keep_rows, other_tokens +): + """An oversized local or peer batch uses eager replay with unpadded metadata.""" + dp_state._keeps_rows_np[1] = keep_rows + dp_state.other = (True, other_tokens) + expected = [0, *range(1 if keep_rows else 301 - WINDOW, 301), *range(301, 401)] + dp_state.replay_graphs = _graphs(256) + dp_state.replay_graphs.dispatch.return_value = BatchExecutionDescriptor( + CUDAGraphMode.NONE, max(len(expected), other_tokens), None + ) + replay, build = _prepare( + dp_state, _input_batch(QUERY_LENS, SEQ_LENS, PREFILLING), CUDAGraphMode.NONE + ) + assert dp_state.replay_graphs.dispatch.call_args.args[:2] == ( + 3, + max(len(expected), other_tokens), + ) + assert replay.run_graph is None + assert replay.rows.tolist() == expected + assert replay.forward_context.cudagraph_runtime_mode == CUDAGraphMode.NONE + assert replay.forward_context.dp_metadata.num_tokens_across_dp_cpu.tolist() == [ + len(expected), + other_tokens, + ] + assert build.batch.num_tokens_after_padding == len(expected) diff --git a/vllm/config/compilation.py b/vllm/config/compilation.py index eefcf1f49e24..70f26a0d9e3d 100644 --- a/vllm/config/compilation.py +++ b/vllm/config/compilation.py @@ -570,6 +570,18 @@ class CompilationConfig: If we limit the video count per prompt to `0`, it will also be set to `0` (i.e., fall back to image-only mode).""" + decoder_replay_cudagraph_capture_sizes: list[int] = field(default_factory=list) + """Decoder replay CUDA graph sizes for YOCO models (e.g. DeepSeek-V4.1). + If empty, use the model's capture sizes up to + min(max_cudagraph_capture_size, max_num_seqs * sliding_window), including + that upper bound. Explicit sizes override this inference and must not exceed + max_num_batched_tokens. Larger replay batches run eagerly.""" + + decoder_replay_trim_threshold: int = Field(default=1024, ge=0) + """For YOCO models (e.g. DeepSeek-V4.1), PIECEWISE graphs with more padded + tokens than this trim the decoder replay batch. Set to 0 to trim at every size. + Independent of the replay graph capture sizes; eager steps still trim.""" + # Inductor capture compile_sizes: list[int | str] | None = None """Sizes to compile for inductor. In addition @@ -1033,6 +1045,11 @@ def __post_init__(self) -> None: f"Invalid backend for piecewise compilation: {self.backend}" ) + if any(s <= 0 for s in self.decoder_replay_cudagraph_capture_sizes): + raise ValueError( + "All decoder_replay_cudagraph_capture_sizes must be positive" + ) + # Validate encoder CUDA graph configuration if ( self.cudagraph_mm_encoder diff --git a/vllm/envs.py b/vllm/envs.py index 0b63cfd5be11..5daa4fd3e67e 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -305,7 +305,6 @@ VLLM_DISABLE_DSV4_MEGAMOE_SHARED_EXPERT_FUSION: bool = False VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD: int = 256 VLLM_MULTI_STREAM_GEMM_TOKEN_THRESHOLD: int = 1024 - VLLM_DSV41_DECODER_REPLAY_TRIM_THRESHOLD: int = 1024 VLLM_COMPILE_CACHE_SAVE_FORMAT: Literal["binary", "unpacked"] = "binary" VLLM_USE_V2_MODEL_RUNNER: bool | None = None VLLM_LOG_MODEL_INSPECTION: bool = False @@ -2106,11 +2105,6 @@ def _resolve_rust_cli_path() -> str | None: "VLLM_MULTI_STREAM_GEMM_TOKEN_THRESHOLD": lambda: int( os.getenv("VLLM_MULTI_STREAM_GEMM_TOKEN_THRESHOLD", "1024") ), - # DeepSeek-V4.1 PIECEWISE CUDA graph steps of more tokens than this trim the - # decoder replay layers too, which then run in CUDA graphs of their own. - "VLLM_DSV41_DECODER_REPLAY_TRIM_THRESHOLD": lambda: int( - os.getenv("VLLM_DSV41_DECODER_REPLAY_TRIM_THRESHOLD", "1024") - ), # Format for saving torch.compile cache artifacts # - "binary": saves as binary file # Safe for multiple vllm serve processes accessing the same torch compile cache. diff --git a/vllm/models/deepseek_v41/nvidia/decoder_replay_cudagraph.py b/vllm/models/deepseek_v41/nvidia/decoder_replay_cudagraph.py index 156fdb7d60de..5daff505e303 100644 --- a/vllm/models/deepseek_v41/nvidia/decoder_replay_cudagraph.py +++ b/vllm/models/deepseek_v41/nvidia/decoder_replay_cudagraph.py @@ -3,6 +3,7 @@ """CUDA graphs of the decoder replay layers on a trimming step's replay batch.""" from collections.abc import Callable +from copy import copy from dataclasses import replace import torch @@ -21,10 +22,29 @@ class DecoderReplayCudaGraphManager(CudaGraphManager): padded size. They read the layer inputs from static buffers.""" def __init__(self, vllm_config: VllmConfig, device, layers: DecoderReplayLayers): - super().__init__(vllm_config, device, CUDAGraphMode.PIECEWISE, 1) + compilation = copy(vllm_config.compilation_config) + scheduler = vllm_config.scheduler_config + sizes = compilation.decoder_replay_cudagraph_capture_sizes + if not sizes: + bound = min( + compilation.max_cudagraph_capture_size, + scheduler.max_num_seqs * layers.window, + ) + sizes = [min(s, bound) for s in compilation.cudagraph_capture_sizes] + sizes = sorted(set(sizes)) + if sizes and max(sizes) > scheduler.max_num_batched_tokens: + raise ValueError( + "decoder_replay_cudagraph_capture_sizes must not exceed " + "max_num_batched_tokens" + ) + compilation.cudagraph_capture_sizes = sizes + compilation.max_cudagraph_capture_size = max(sizes, default=0) + replay_config = copy(vllm_config) + replay_config.compilation_config = compilation + super().__init__(replay_config, device, CUDAGraphMode.PIECEWISE, 1) self.layers = layers self.breakable_cg_runner: BreakableCUDAGraphWrapper = BreakableCUDAGraphWrapper( - layers.run_layers, vllm_config + layers.run_layers, replay_config ) config, act = vllm_config.model_config.hf_config, vllm_config.model_config.dtype assert isinstance(act, torch.dtype) @@ -34,7 +54,7 @@ def __init__(self, vllm_config: VllmConfig, device, layers: DecoderReplayLayers) # hidden_states, positions, input_ids, pre_mix, post_mix, res_mix, residual shapes = [(h,), (), (), (hc,), (hc, 1), (hc, hc), (hc, h)] dtypes = [act, torch.int64, ids, f32, f32, f32, act] - n = self.compilation_config.max_cudagraph_capture_size + n = compilation.max_cudagraph_capture_size self.inputs = [ torch.zeros(n, *s, dtype=d, device=device) for s, d in zip(shapes, dtypes) ] diff --git a/vllm/models/deepseek_v41/nvidia/model_state.py b/vllm/models/deepseek_v41/nvidia/model_state.py index 1f320c0e18fd..10cf1533c6ab 100644 --- a/vllm/models/deepseek_v41/nvidia/model_state.py +++ b/vllm/models/deepseek_v41/nvidia/model_state.py @@ -8,7 +8,6 @@ import torch.distributed as dist import torch.nn as nn -import vllm.envs as envs from vllm.compilation.breakable_cudagraph import is_breakable_cudagraph_enabled from vllm.config import CUDAGraphMode, VllmConfig from vllm.distributed.parallel_state import get_dp_group, get_pp_group @@ -255,7 +254,9 @@ def __init__( self.replay_graphs = DecoderReplayCudaGraphManager( vllm_config, device, layers ) - layers.trim_threshold = envs.VLLM_DSV41_DECODER_REPLAY_TRIM_THRESHOLD + layers.trim_threshold = ( + vllm_config.compilation_config.decoder_replay_trim_threshold + ) def add_request(self, req_index: int, new_req_data: NewRequestData) -> None: super().add_request(req_index, new_req_data) From c7220f0f5db57efe1ba28f8277ebf1f6cc8bc43e Mon Sep 17 00:00:00 2001 From: Juntian Liu Date: Thu, 1 Oct 2026 07:52:08 +0000 Subject: [PATCH 3/8] [Perf][DSv4.1] Build only the replay layers' metadata; trim from 768 tokens The replay batch builds attention metadata only for the groups the replay layers read, skipping the kv sources' compressor and their other compressed-KV and indexer groups (about 0.55 ms of CPU per trimming step). PIECEWISE graphs of at least 768 padded tokens now trim: the 384 and 576 graphs lost ~3 ms per mixed decode step to the replay overhead, while 768 and 1020 gained 18-25%. Trim the tests to the regression checks. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Juntian Liu --- tests/compile/test_config.py | 20 --- ...test_deepseek_v41_decoder_replay_layers.py | 166 ++++-------------- .../models/test_deepseek_v41_replay_batch.py | 101 ++--------- vllm/config/compilation.py | 6 +- .../deepseek_v41/decoder_replay_layers.py | 9 +- vllm/models/deepseek_v41/nvidia/model.py | 10 ++ .../models/deepseek_v41/nvidia/model_state.py | 24 +-- vllm/v1/worker/gpu/model_runner.py | 2 +- vllm/v1/worker/gpu/model_states/interface.py | 2 +- 9 files changed, 85 insertions(+), 255 deletions(-) diff --git a/tests/compile/test_config.py b/tests/compile/test_config.py index d6ac2ce4e609..24cb816b5d9d 100644 --- a/tests/compile/test_config.py +++ b/tests/compile/test_config.py @@ -1366,23 +1366,3 @@ def test_inductor_asserts_user_override(monkeypatch): assert config.inductor_compile_config.get("size_asserts") is True if not _is_torch_equal_or_newer(torch.__version__, "2.12.0.dev"): assert config.inductor_compile_config.get("alignment_asserts") is False - - -@pytest.mark.parametrize( - "kwargs", - [ - {"decoder_replay_cudagraph_capture_sizes": [0]}, - {"decoder_replay_cudagraph_capture_sizes": [128, -1]}, - {"decoder_replay_trim_threshold": -1}, - ], -) -def test_decoder_replay_config_rejects_invalid_values(kwargs): - with pytest.raises(ValueError, match="decoder_replay"): - CompilationConfig(**kwargs) - - -def test_decoder_replay_config_changes_compilation_hash(): - default = CompilationConfig() - always_trim = CompilationConfig(decoder_replay_trim_threshold=0) - custom_sizes = CompilationConfig(decoder_replay_cudagraph_capture_sizes=[128, 1024]) - assert len({c.compute_hash() for c in (default, always_trim, custom_sizes)}) == 3 diff --git a/tests/models/test_deepseek_v41_decoder_replay_layers.py b/tests/models/test_deepseek_v41_decoder_replay_layers.py index e5dabf310038..b6bb99afcea8 100644 --- a/tests/models/test_deepseek_v41_decoder_replay_layers.py +++ b/tests/models/test_deepseek_v41_decoder_replay_layers.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """DecoderReplayLayers runs the replay layers on the forward's replay batch.""" -from types import SimpleNamespace +from unittest.mock import MagicMock import pytest import torch @@ -21,7 +21,7 @@ from vllm.models.deepseek_v41.nvidia.decoder_replay_cudagraph import ( DecoderReplayCudaGraphManager, ) -from vllm.v1.worker.gpu import cudagraph_utils as gpu_cudagraph_utils +from vllm.v1.worker.gpu import cudagraph_utils pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA") @@ -80,7 +80,7 @@ def run_layers(hidden_states, positions, input_ids, pre_mix, post, res, residual seen["is_padding"] = replay_context.is_padding return residual, pre_mix - layers = DecoderReplayLayers(WINDOW, run_layers, [topk, candidates]) + layers = DecoderReplayLayers(WINDOW, run_layers, [topk, candidates], set()) states = _states(NUM_TOKENS) hidden, residual = states[0], states[-1] full, sub = object(), object() @@ -107,7 +107,7 @@ def run_layers(hidden_states, *rest): seen["attn_metadata"] = get_forward_context().attn_metadata return (hidden_states,) - layers = DecoderReplayLayers(WINDOW, run_layers, []) + layers = DecoderReplayLayers(WINDOW, run_layers, [], set()) states = _states(NUM_TOKENS) full = object() with override_forward_context(_context(full)): @@ -115,152 +115,56 @@ def run_layers(hidden_states, *rest): assert output is states[0] and seen["attn_metadata"] is full -def test_graph_break_writes_fixed_buffers(): - """Above the trim threshold, a PIECEWISE graph's replay layers write fixed - buffers, which the graph's next segment reads on every replay.""" - layers = DecoderReplayLayers(WINDOW, lambda hidden, *rest: (hidden * 2,), []) - layers.trim_threshold = NUM_TOKENS - 1 - context = ForwardContext( +def _graph_context(num_tokens): + return ForwardContext( no_compile_layers={}, - attn_metadata=object(), + attn_metadata={}, slot_mapping={}, cudagraph_runtime_mode=CUDAGraphMode.PIECEWISE, - batch_descriptor=BatchDescriptor(num_tokens=NUM_TOKENS), + batch_descriptor=BatchDescriptor(num_tokens=num_tokens), ) + + +def test_graph_break_writes_fixed_buffers(): + layers = DecoderReplayLayers(WINDOW, lambda hidden, *rest: (hidden * 2,), [], set()) + layers.trim_threshold = NUM_TOKENS - 1 states = _states(NUM_TOKENS) - with override_forward_context(context): + with override_forward_context(_graph_context(NUM_TOKENS)): (first,) = layers(*states) (second,) = layers(states[0] * 3, *states[1:]) assert first.data_ptr() == second.data_ptr() assert torch.equal(second, states[0] * 6) -@pytest.fixture -def replay_config(monkeypatch): - monkeypatch.setattr( - gpu_cudagraph_utils, - "get_pp_group", - lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True), - ) - return SimpleNamespace( - compilation_config=CompilationConfig(), - scheduler_config=SimpleNamespace(max_num_seqs=8, max_num_batched_tokens=8192), - parallel_config=SimpleNamespace(tensor_parallel_size=1), - speculative_config=None, - cache_config=SimpleNamespace(use_kda_recoverssm=False), - model_config=SimpleNamespace( - hf_config=SimpleNamespace(hc_mult=HC, hidden_size=HIDDEN), - dtype=torch.float32, - ), - kernel_config=SimpleNamespace(moe_backend="auto"), - ) - - -@pytest.mark.parametrize( - "max_num_seqs,main_sizes,replay_sizes,expected", - [ - (8, [6, 1020, 2046, 8190], [], [6, 1020, 1024]), - (128, [6, 1020, 2046, 8190], [], [6, 1020, 2046, 8190]), - (8, [6, 510], [], [6, 510]), - (8, [6, 1020], [2048, 256, 2048], [256, 2048]), - ], -) -def test_decoder_replay_graph_sizes( - replay_config, max_num_seqs, main_sizes, replay_sizes, expected -): - """Replay captures, dispatch and buffers agree without changing model sizes.""" - cfg = replay_config - cfg.scheduler_config.max_num_seqs = max_num_seqs - cfg.compilation_config.cudagraph_capture_sizes = main_sizes - cfg.compilation_config.max_cudagraph_capture_size = main_sizes[-1] - cfg.compilation_config.decoder_replay_cudagraph_capture_sizes = replay_sizes - layers = DecoderReplayLayers(128, lambda *args: args, []) - manager = DecoderReplayCudaGraphManager(cfg, torch.device("cpu"), layers) - assert ( - sorted(d.num_tokens for d in manager._capture_descs[CUDAGraphMode.PIECEWISE]) - == expected - ) - assert all(buf.shape[0] == expected[-1] for buf in manager.inputs) - manager._graphs_captured = True - for tokens in (1, expected[-1] - 1, expected[-1]): - desc = manager.dispatch(max_num_seqs, tokens, None, 0) - assert desc.cg_mode == CUDAGraphMode.PIECEWISE - assert desc.num_tokens == next(s for s in expected if s >= tokens) - assert manager.dispatch(max_num_seqs, expected[-1] + 1, None, 0).cg_mode == ( - CUDAGraphMode.NONE - ) - assert cfg.compilation_config.cudagraph_capture_sizes == main_sizes - assert cfg.compilation_config.max_cudagraph_capture_size == main_sizes[-1] - assert cfg.compilation_config.decoder_replay_cudagraph_capture_sizes == replay_sizes - - -def test_decoder_replay_graph_sizes_fit_runner_buffers(replay_config): - cfg = replay_config - cfg.compilation_config.decoder_replay_cudagraph_capture_sizes = [ - cfg.scheduler_config.max_num_batched_tokens + 1 - ] - with pytest.raises(ValueError, match="must not exceed max_num_batched_tokens"): - DecoderReplayCudaGraphManager( - cfg, torch.device("cpu"), DecoderReplayLayers(128, lambda *args: args, []) - ) - - -@torch.inference_mode() -def test_bounded_replay_cuda_graph_matches_eager(replay_config, monkeypatch): - """Changing replay rows and padding reuses captured graphs with eager results.""" - cfg = replay_config - cfg.compilation_config.cudagraph_capture_sizes = [4, 8, 16] - cfg.compilation_config.max_cudagraph_capture_size = 16 - cfg.scheduler_config.max_num_seqs = 2 - monkeypatch.setattr( - gpu_cudagraph_utils, - "graph_capture", - lambda device: torch.cuda.stream(torch.cuda.Stream(device=device)), - ) - monkeypatch.setattr(gpu_cudagraph_utils, "is_global_first_rank", lambda: False) +def test_replay_graph_matches_eager(monkeypatch): + monkeypatch.setattr(cudagraph_utils, "get_pp_group", MagicMock) + capture = torch.cuda.stream(torch.cuda.Stream()) + monkeypatch.setattr(cudagraph_utils, "graph_capture", lambda device: capture) + compilation = CompilationConfig(decoder_replay_cudagraph_capture_sizes=[4, 8]) + cfg = MagicMock(compilation_config=compilation, speculative_config=None) + cfg.scheduler_config = MagicMock(max_num_seqs=2, max_num_batched_tokens=64) + cfg.cache_config.use_kda_recoverssm = False + cfg.model_config.hf_config = MagicMock(hc_mult=HC, hidden_size=HIDDEN) + cfg.model_config.dtype = torch.float32 def run_layers(hidden, positions, ids, pre, post, mix, residual): - return residual + hidden[:, None, :] + positions[:, None, None], pre + ids[ - :, None - ] + return residual + hidden[:, None] + positions[:, None, None], pre + ids[:, None] - layers = DecoderReplayLayers(4, run_layers, []) + layers = DecoderReplayLayers(4, run_layers, [], set()) manager = DecoderReplayCudaGraphManager(cfg, DEVICE, layers) def prepare(desc): - layers.replay_batch = ReplayBatch( - torch.arange(desc.num_tokens, device=DEVICE), - ForwardContext( - no_compile_layers={}, - attn_metadata={}, - slot_mapping={}, - batch_descriptor=BatchDescriptor(num_tokens=desc.num_tokens), - ), - ) + rows = torch.arange(desc.num_tokens) + layers.replay_batch = ReplayBatch(rows, _graph_context(desc.num_tokens)) - torch.accelerator.synchronize() manager.capture_replay_graphs(prepare) - torch.accelerator.synchronize() - assert {d.num_tokens for d in manager.breakable_cg_runner.entries} == {4, 8} - assert layers.replay_batch is None - states = tuple( - torch.randn(12, *buf.shape[1:], device=DEVICE).to(buf.dtype) - for buf in manager.inputs - ) - for row_ids in ([11, 2, 7], [1, 2, 3, 4, 5, 6, 7, 8], [9, 8, 7, 6, 5], [3]): + states = [ + torch.randn(12, *b.shape[1:], device=DEVICE).to(b.dtype) for b in manager.inputs + ] + for row_ids in ([11, 2, 7], [1, 2, 3, 4, 5, 6, 7, 8]): rows = torch.tensor(row_ids, device=DEVICE) desc = manager.dispatch(2, len(row_ids), None, 0) - context = ForwardContext( - no_compile_layers={}, - attn_metadata={}, - slot_mapping={}, - cudagraph_runtime_mode=desc.cg_mode, - batch_descriptor=BatchDescriptor(num_tokens=desc.num_tokens), - ) - with override_forward_context(context): + with override_forward_context(_graph_context(desc.num_tokens)): actual = manager.run(rows, states) - expected = run_layers(*(t[rows] for t in states)) - for a, e in zip(actual, expected): - torch.testing.assert_close(a, e, rtol=0, atol=0) - states[0].add_(1) - assert len(manager.breakable_cg_runner.entries) == 2 + for a, e in zip(actual, run_layers(*(t[rows] for t in states))): + torch.testing.assert_close(a, e, rtol=0, atol=0) diff --git a/tests/models/test_deepseek_v41_replay_batch.py b/tests/models/test_deepseek_v41_replay_batch.py index 1cf387fa9b98..07459208aaec 100644 --- a/tests/models/test_deepseek_v41_replay_batch.py +++ b/tests/models/test_deepseek_v41_replay_batch.py @@ -12,7 +12,6 @@ from vllm.config import CUDAGraphMode from vllm.models.deepseek_v41.nvidia.model_state import DeepseekV41ModelState -from vllm.v1.attention.backends.utils import PAD_SLOT_ID from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers from vllm.v1.worker.gpu.model_states.default import DefaultModelState @@ -60,7 +59,9 @@ def state(monkeypatch): cfg.parallel_config.data_parallel_size = 1 cfg.compilation_config.fast_moe_cold_start = False cfg.compilation_config.cudagraph_mode = CUDAGraphMode.NONE - layers = SimpleNamespace(window=WINDOW, replay_batch=None, trim_threshold=None) + layers = SimpleNamespace( + window=WINDOW, replay_batch=None, trim_threshold=None, metadata_prefixes=set() + ) model = SimpleNamespace(token_lookback_depth=0, decoder_replay_layers=layers) builds: list = [] @@ -190,8 +191,8 @@ def test_replay_batch_keeps_adaptive_verification_query_bound(state): def test_only_eager_steps_trim(state): - """A CUDA graph keeps the layers on its whole batch; an eager step trims a - padded batch to its real rows.""" + """A CUDA graph keeps the layers on its whole batch, unless it is a PIECEWISE + one at the trim threshold or above; an eager step trims a batch to its rows.""" batch = _input_batch(QUERY_LENS, SEQ_LENS, PREFILLING, num_tokens_after_padding=512) for cg_mode in (CUDAGraphMode.PIECEWISE, CUDAGraphMode.FULL): assert _prepare(state, batch, cg_mode) == (None, None) @@ -200,23 +201,10 @@ def test_only_eager_steps_trim(state): assert replay.rows.tolist() == REPLAY_ROWS assert build.cg_mode == CUDAGraphMode.NONE assert build.batch.num_tokens_after_padding == len(REPLAY_ROWS) - - -def test_piecewise_graphs_above_the_trim_threshold_replay(state): - """A PIECEWISE graph above the trim threshold breaks out to the replay layers, - which run on a replay batch even when nothing trims.""" state.decoder_replay_layers.trim_threshold = 256 - batch = _input_batch(QUERY_LENS, SEQ_LENS, PREFILLING, num_tokens_after_padding=512) replay, _ = _prepare(state, batch, CUDAGraphMode.PIECEWISE) assert replay is not None and replay.rows.tolist() == REPLAY_ROWS assert _prepare(state, batch, CUDAGraphMode.FULL) == (None, None) - short = _input_batch( - [100, 100], [100, 100], [True, True], num_tokens_after_padding=512 - ) - replay, _ = _prepare(state, short, CUDAGraphMode.PIECEWISE) - assert replay is not None and replay.rows.tolist() == list(range(200)) - state.decoder_replay_layers.trim_threshold = 512 - assert _prepare(state, batch, CUDAGraphMode.PIECEWISE) == (None, None) def test_whole_batch_forwards_get_no_replay_batch(state): @@ -274,75 +262,16 @@ def test_idle_dp_rank_dummy_trims_with_its_peers(dp_state): assert dp_metadata.num_tokens_across_dp_cpu.tolist() == [WINDOW, 129] -def _graphs(num_tokens: int): - desc = BatchExecutionDescriptor(CUDAGraphMode.PIECEWISE, num_tokens, None) - return SimpleNamespace( - run=object(), - dispatch=MagicMock(return_value=desc), - capture_replay_graphs=lambda f: f(desc), +def test_dp_ranks_pad_to_one_replay_graph(dp_state): + """Every rank pads its replay batch to the graph fitting the largest rank's.""" + dp_state.replay_graphs = MagicMock() + dp_state.replay_graphs.dispatch.return_value = BatchExecutionDescriptor( + CUDAGraphMode.PIECEWISE, 512, None ) - - -def test_replay_batch_pads_to_its_graph_on_every_rank(dp_state): - """An eager step whose replay batch fits a graph pads it to the graph that - fits the largest rank's batch.""" - dp_state.replay_graphs = _graphs(512) dp_state.other = (True, 300) - replay, build = _prepare( - dp_state, _input_batch(QUERY_LENS, SEQ_LENS, PREFILLING), CUDAGraphMode.NONE - ) + batch = _input_batch(QUERY_LENS, SEQ_LENS, PREFILLING) + replay, _ = _prepare(dp_state, batch, CUDAGraphMode.NONE) assert dp_state.replay_graphs.dispatch.call_args.args[:2] == (3, 300) - assert replay.run_graph is dp_state.replay_graphs.run - context = replay.forward_context - assert context.cudagraph_runtime_mode == CUDAGraphMode.PIECEWISE - assert context.batch_descriptor.num_tokens == 512 - assert context.dp_metadata.num_tokens_across_dp_cpu.tolist() == [512, 512] - assert replay.rows.tolist() == REPLAY_ROWS - assert build.batch.num_tokens == len(REPLAY_ROWS) - assert build.batch.num_tokens_after_padding == 512 - assert (build.slot_mappings[:, len(REPLAY_ROWS) :] == PAD_SLOT_ID).all() - - -def test_capture_replays_a_whole_dummy_batch(state): - """A capture builds its graph's replay batch from a dummy batch that keeps - every row.""" - _prepare(state, _input_batch(QUERY_LENS, SEQ_LENS, PREFILLING), CUDAGraphMode.NONE) - state.replay_graphs = _graphs(64) - block_tables = SimpleNamespace( - get_dummy_block_tables=lambda n: tuple(t[:n] for t in BLOCK_TABLES), - get_dummy_slot_mappings=lambda n: _slot_mappings(n).fill_(PAD_SLOT_ID), - ) - state.capture_cudagraphs(InputBuffers(16, 1024, DEVICE), block_tables, [], GROUPS) - replay = state.decoder_replay_layers.replay_batch - assert replay.rows.tolist() == list(range(64)) - assert replay.forward_context.batch_descriptor.num_tokens == 64 - assert state.builds[-1].batch.num_reqs == 16 - - -@pytest.mark.parametrize("keep_rows,other_tokens", [(False, 500), (True, 200)]) -def test_replay_above_graph_capacity_falls_back_without_dropping_rows( - dp_state, keep_rows, other_tokens -): - """An oversized local or peer batch uses eager replay with unpadded metadata.""" - dp_state._keeps_rows_np[1] = keep_rows - dp_state.other = (True, other_tokens) - expected = [0, *range(1 if keep_rows else 301 - WINDOW, 301), *range(301, 401)] - dp_state.replay_graphs = _graphs(256) - dp_state.replay_graphs.dispatch.return_value = BatchExecutionDescriptor( - CUDAGraphMode.NONE, max(len(expected), other_tokens), None - ) - replay, build = _prepare( - dp_state, _input_batch(QUERY_LENS, SEQ_LENS, PREFILLING), CUDAGraphMode.NONE - ) - assert dp_state.replay_graphs.dispatch.call_args.args[:2] == ( - 3, - max(len(expected), other_tokens), - ) - assert replay.run_graph is None - assert replay.rows.tolist() == expected - assert replay.forward_context.cudagraph_runtime_mode == CUDAGraphMode.NONE - assert replay.forward_context.dp_metadata.num_tokens_across_dp_cpu.tolist() == [ - len(expected), - other_tokens, - ] - assert build.batch.num_tokens_after_padding == len(expected) + assert replay.forward_context.batch_descriptor.num_tokens == 512 + dp_metadata = replay.forward_context.dp_metadata + assert dp_metadata.num_tokens_across_dp_cpu.tolist() == [512, 512] diff --git a/vllm/config/compilation.py b/vllm/config/compilation.py index 70f26a0d9e3d..48c3686e06cb 100644 --- a/vllm/config/compilation.py +++ b/vllm/config/compilation.py @@ -577,9 +577,9 @@ class CompilationConfig: that upper bound. Explicit sizes override this inference and must not exceed max_num_batched_tokens. Larger replay batches run eagerly.""" - decoder_replay_trim_threshold: int = Field(default=1024, ge=0) - """For YOCO models (e.g. DeepSeek-V4.1), PIECEWISE graphs with more padded - tokens than this trim the decoder replay batch. Set to 0 to trim at every size. + decoder_replay_trim_threshold: int = Field(default=768, ge=0) + """For YOCO models (e.g. DeepSeek-V4.1), PIECEWISE graphs with at least this + many padded tokens trim the decoder replay batch. Set to 0 to trim at every size. Independent of the replay graph capture sizes; eager steps still trim.""" # Inductor capture diff --git a/vllm/models/deepseek_v41/decoder_replay_layers.py b/vllm/models/deepseek_v41/decoder_replay_layers.py index 8d929162ebe2..98e2e5872d29 100644 --- a/vllm/models/deepseek_v41/decoder_replay_layers.py +++ b/vllm/models/deepseek_v41/decoder_replay_layers.py @@ -43,7 +43,8 @@ class DecoderReplayLayers: ``run_layers`` takes a batch's layer inputs and returns its per-row outputs. ``row_buffers`` hold per-row results the source layer's indexer publishes for the layers after it; they are compacted to the replay rows - in place. + in place. ``metadata_prefixes`` are the attention metadata keys the layers + read, so the replay batch builds only those. """ def __init__( @@ -51,13 +52,15 @@ def __init__( window: int, run_layers: Callable[..., tuple[torch.Tensor, ...]], row_buffers: list[torch.Tensor], + metadata_prefixes: set[str], ) -> None: self.window = window self.run_layers = run_layers self.row_buffers = row_buffers + self.metadata_prefixes = metadata_prefixes # Set by the model state every step; None runs the layers on the batch. self.replay_batch: ReplayBatch | None = None - # PIECEWISE model graphs of more tokens break out of the graph to run the + # PIECEWISE model graphs of at least this many tokens break out to run the # layers on the replay batch; set with the replay graphs. self.trim_threshold: int | None = None self._graph_outputs: list[torch.Tensor] | None = None @@ -68,7 +71,7 @@ def __call__( if self.trim_threshold is not None and in_piecewise_cudagraph(): batch_descriptor = get_forward_context().batch_descriptor assert batch_descriptor is not None - if batch_descriptor.num_tokens > self.trim_threshold: + if batch_descriptor.num_tokens >= self.trim_threshold: return self._run_in_graph_break(hidden_states, *states) return self._run(hidden_states, *states) diff --git a/vllm/models/deepseek_v41/nvidia/model.py b/vllm/models/deepseek_v41/nvidia/model.py index f7647ec81d8b..c41d4917d745 100644 --- a/vllm/models/deepseek_v41/nvidia/model.py +++ b/vllm/models/deepseek_v41/nvidia/model.py @@ -754,6 +754,15 @@ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): and self.layers[cut].attn.swa_cache_layer.bounded_replay ): self.decoder_replay_start = cut + 1 + # The attention metadata keys the replay layers read. + metadata_prefixes = set() + for layer in islice(self.layers, cut + 1, self.end_layer): + attn = typing.cast(DeepseekV4DecoderLayer, layer).attn + metadata_prefixes.add(attn.swa_cache_layer.prefix) + if attn.compressed_cache_prefix is not None: + metadata_prefixes.add(attn.compressed_cache_prefix) + if attn.indexer is not None: + metadata_prefixes.add(attn.indexer.k_cache.prefix) self.decoder_replay_layers = DecoderReplayLayers( config.sliding_window, self._run_replay_layers, @@ -762,6 +771,7 @@ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): for buf in (self.topk_indices_buffer, self.candidate_block_buffer) if buf is not None ], + metadata_prefixes, ) logger.info_once( "Decoder SWA bounded replay: in eager prefill steps, layers " diff --git a/vllm/models/deepseek_v41/nvidia/model_state.py b/vllm/models/deepseek_v41/nvidia/model_state.py index 10cf1533c6ab..f0387d70e7cb 100644 --- a/vllm/models/deepseek_v41/nvidia/model_state.py +++ b/vllm/models/deepseek_v41/nvidia/model_state.py @@ -439,20 +439,20 @@ def _prepare_replay_batch( capture_desc: BatchExecutionDescriptor | None = None, ) -> None: """Set the replay layers' batch for this forward, or none when nothing - trims. Eager steps trim, and so do PIECEWISE graph steps above the trim - threshold, whose graph breaks out to the replay layers; FULL graphs and - smaller PIECEWISE ones keep the layers on the whole batch. Ranks share - the graph mode and size; under data parallelism every rank replays if - any does.""" + trims. Eager steps trim, and so do PIECEWISE graph steps at the trim + threshold or above, whose graph breaks out to the replay layers; FULL + graphs and smaller PIECEWISE ones keep the layers on the whole batch. + Ranks share the graph mode and size; under data parallelism every rank + replays if any does.""" layers = self.decoder_replay_layers assert layers is not None layers.replay_batch = None - # Captures and PIECEWISE graphs above the trim threshold (which break out - # to the replay layers) run the layers on a replay batch, trimmed or not. + # Captures and PIECEWISE graphs at the trim threshold or above (which break + # out to the replay layers) run them on a replay batch, trimmed or not. replays = capture_desc is not None or ( cudagraph_mode == CUDAGraphMode.PIECEWISE and layers.trim_threshold is not None - and input_batch.num_tokens_after_padding > layers.trim_threshold + and input_batch.num_tokens_after_padding >= layers.trim_threshold ) if cudagraph_mode != CUDAGraphMode.NONE and not replays: return @@ -520,7 +520,7 @@ def _prepare_replay_batch( run_graph, ) - def capture_cudagraphs( + def capture_inner_cudagraphs( self, input_buffers: InputBuffers, block_tables: BlockTables, @@ -627,12 +627,16 @@ def _replay_groups( ) -> list[list[AttentionGroup]]: """The attention groups with metadata builders of their own, like each microbatch's: a builder keeps the metadata it built, and the runner's - hold the batch's.""" + hold the batch's. Only the groups the replay layers read are built.""" if self._replay_attn_groups is None: + assert self.decoder_replay_layers is not None + prefixes = self.decoder_replay_layers.metadata_prefixes self._replay_attn_groups = [] for groups in attn_groups: replay_groups = [] for group in groups: + if prefixes.isdisjoint(group.layer_names): + continue replay_group = replace(group, metadata_builders=[]) replay_group.create_metadata_builders( self.vllm_config, diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index b201358aea0b..3b11c0fe9f0e 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -1065,7 +1065,7 @@ def capture_model(self, *, profile_only: bool = False) -> int: self.lora_config, self ), ) - self.model_state.capture_cudagraphs( + self.model_state.capture_inner_cudagraphs( input_buffers, self.block_tables, self.attn_groups, diff --git a/vllm/v1/worker/gpu/model_states/interface.py b/vllm/v1/worker/gpu/model_states/interface.py index 9ac7706de880..f03caf4dba83 100644 --- a/vllm/v1/worker/gpu/model_states/interface.py +++ b/vllm/v1/worker/gpu/model_states/interface.py @@ -127,7 +127,7 @@ def remove_request(self, req_id: str) -> None: def apply_staged_writes(self) -> None: return None - def capture_cudagraphs( + def capture_inner_cudagraphs( self, input_buffers: InputBuffers, block_tables: BlockTables, From f21ea3c94787abc842e8f7799ca3ff9efd2f040b Mon Sep 17 00:00:00 2001 From: Juntian Liu Date: Thu, 1 Oct 2026 23:05:33 +0000 Subject: [PATCH 4/8] [Perf][DSv4.1] Default replay graph sizes to 8 per power-of-two interval Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Juntian Liu --- vllm/config/compilation.py | 3 ++- vllm/models/deepseek_v41/nvidia/decoder_replay_cudagraph.py | 6 +++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/vllm/config/compilation.py b/vllm/config/compilation.py index 48c3686e06cb..8417496ff320 100644 --- a/vllm/config/compilation.py +++ b/vllm/config/compilation.py @@ -572,7 +572,8 @@ class CompilationConfig: decoder_replay_cudagraph_capture_sizes: list[int] = field(default_factory=list) """Decoder replay CUDA graph sizes for YOCO models (e.g. DeepSeek-V4.1). - If empty, use the model's capture sizes up to + If empty, capture 8 sizes per power-of-two interval (e.g. step 128 in + [1024, 2048), step 256 in [2048, 4096)) up to min(max_cudagraph_capture_size, max_num_seqs * sliding_window), including that upper bound. Explicit sizes override this inference and must not exceed max_num_batched_tokens. Larger replay batches run eagerly.""" diff --git a/vllm/models/deepseek_v41/nvidia/decoder_replay_cudagraph.py b/vllm/models/deepseek_v41/nvidia/decoder_replay_cudagraph.py index 5daff505e303..0aab916fa8d0 100644 --- a/vllm/models/deepseek_v41/nvidia/decoder_replay_cudagraph.py +++ b/vllm/models/deepseek_v41/nvidia/decoder_replay_cudagraph.py @@ -30,7 +30,11 @@ def __init__(self, vllm_config: VllmConfig, device, layers: DecoderReplayLayers) compilation.max_cudagraph_capture_size, scheduler.max_num_seqs * layers.window, ) - sizes = [min(s, bound) for s in compilation.cudagraph_capture_sizes] + # 8 sizes per power-of-two interval, e.g. step 256 in [2048, 4096). + sizes, s = [bound], 1 + while s < bound: + sizes.append(s) + s += 1 << max(s.bit_length() - 4, 0) sizes = sorted(set(sizes)) if sizes and max(sizes) > scheduler.max_num_batched_tokens: raise ValueError( From 9b122bd59e36dc289bf59261c3d2f4777f069315 Mon Sep 17 00:00:00 2001 From: Juntian Liu Date: Fri, 2 Oct 2026 22:19:45 +0000 Subject: [PATCH 5/8] [Perf][DSv4.1] Add cudagraph_decoder_replay switch and trim threshold bound; clarify replay batch naming Add compilation_config.cudagraph_decoder_replay (default on) to turn the replay graphs and PIECEWISE trimming off, and reject a decoder_replay_trim_threshold at or below the replay window, where no graph has anything to trim. Keep the threshold in the model state only: the replay layers break out of a PIECEWISE graph whenever a replay batch is set, which the model state does at the threshold or above. Build the replay batch's DP metadata once, after its padding is known. Rename replays/trims to force_replay_batch/needs_replay_batch, replay_graphs to replay_cudagraphs, and tighten the comments. Trim the tests to the regression checks. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Juntian Liu --- ...test_deepseek_v41_decoder_replay_layers.py | 30 ++---- .../models/test_deepseek_v41_replay_batch.py | 14 ++- vllm/config/compilation.py | 11 ++- .../deepseek_v41/decoder_replay_layers.py | 37 +++----- .../nvidia/decoder_replay_cudagraph.py | 28 +++--- .../models/deepseek_v41/nvidia/model_state.py | 93 ++++++++++--------- 6 files changed, 105 insertions(+), 108 deletions(-) diff --git a/tests/models/test_deepseek_v41_decoder_replay_layers.py b/tests/models/test_deepseek_v41_decoder_replay_layers.py index b6bb99afcea8..9ba2df5c8f4c 100644 --- a/tests/models/test_deepseek_v41_decoder_replay_layers.py +++ b/tests/models/test_deepseek_v41_decoder_replay_layers.py @@ -125,22 +125,12 @@ def _graph_context(num_tokens): ) -def test_graph_break_writes_fixed_buffers(): - layers = DecoderReplayLayers(WINDOW, lambda hidden, *rest: (hidden * 2,), [], set()) - layers.trim_threshold = NUM_TOKENS - 1 - states = _states(NUM_TOKENS) - with override_forward_context(_graph_context(NUM_TOKENS)): - (first,) = layers(*states) - (second,) = layers(states[0] * 3, *states[1:]) - assert first.data_ptr() == second.data_ptr() - assert torch.equal(second, states[0] * 6) - - def test_replay_graph_matches_eager(monkeypatch): + """The replay graph of the next captured size pads and runs the rows.""" monkeypatch.setattr(cudagraph_utils, "get_pp_group", MagicMock) capture = torch.cuda.stream(torch.cuda.Stream()) monkeypatch.setattr(cudagraph_utils, "graph_capture", lambda device: capture) - compilation = CompilationConfig(decoder_replay_cudagraph_capture_sizes=[4, 8]) + compilation = CompilationConfig(decoder_replay_cudagraph_capture_sizes=[4]) cfg = MagicMock(compilation_config=compilation, speculative_config=None) cfg.scheduler_config = MagicMock(max_num_seqs=2, max_num_batched_tokens=64) cfg.cache_config.use_kda_recoverssm = False @@ -159,12 +149,12 @@ def prepare(desc): manager.capture_replay_graphs(prepare) states = [ - torch.randn(12, *b.shape[1:], device=DEVICE).to(b.dtype) for b in manager.inputs + torch.randn(12, *b.shape[1:], device=DEVICE).to(b.dtype) + for b in manager.input_buffers ] - for row_ids in ([11, 2, 7], [1, 2, 3, 4, 5, 6, 7, 8]): - rows = torch.tensor(row_ids, device=DEVICE) - desc = manager.dispatch(2, len(row_ids), None, 0) - with override_forward_context(_graph_context(desc.num_tokens)): - actual = manager.run(rows, states) - for a, e in zip(actual, run_layers(*(t[rows] for t in states))): - torch.testing.assert_close(a, e, rtol=0, atol=0) + rows = torch.tensor([11, 2, 7], device=DEVICE) + desc = manager.dispatch(2, 3, None, 0) + with override_forward_context(_graph_context(desc.num_tokens)): + actual = manager.run(rows, states) + for a, e in zip(actual, run_layers(*(t[rows] for t in states))): + torch.testing.assert_close(a, e, rtol=0, atol=0) diff --git a/tests/models/test_deepseek_v41_replay_batch.py b/tests/models/test_deepseek_v41_replay_batch.py index 07459208aaec..31d286431ec5 100644 --- a/tests/models/test_deepseek_v41_replay_batch.py +++ b/tests/models/test_deepseek_v41_replay_batch.py @@ -59,9 +59,7 @@ def state(monkeypatch): cfg.parallel_config.data_parallel_size = 1 cfg.compilation_config.fast_moe_cold_start = False cfg.compilation_config.cudagraph_mode = CUDAGraphMode.NONE - layers = SimpleNamespace( - window=WINDOW, replay_batch=None, trim_threshold=None, metadata_prefixes=set() - ) + layers = SimpleNamespace(window=WINDOW, replay_batch=None, metadata_prefixes=set()) model = SimpleNamespace(token_lookback_depth=0, decoder_replay_layers=layers) builds: list = [] @@ -190,7 +188,7 @@ def test_replay_batch_keeps_adaptive_verification_query_bound(state): assert replay is not None and build.batch.max_query_len == 200 -def test_only_eager_steps_trim(state): +def test_graph_steps_trim_only_piecewise_at_threshold(state): """A CUDA graph keeps the layers on its whole batch, unless it is a PIECEWISE one at the trim threshold or above; an eager step trims a batch to its rows.""" batch = _input_batch(QUERY_LENS, SEQ_LENS, PREFILLING, num_tokens_after_padding=512) @@ -201,7 +199,7 @@ def test_only_eager_steps_trim(state): assert replay.rows.tolist() == REPLAY_ROWS assert build.cg_mode == CUDAGraphMode.NONE assert build.batch.num_tokens_after_padding == len(REPLAY_ROWS) - state.decoder_replay_layers.trim_threshold = 256 + state._trim_threshold = 256 replay, _ = _prepare(state, batch, CUDAGraphMode.PIECEWISE) assert replay is not None and replay.rows.tolist() == REPLAY_ROWS assert _prepare(state, batch, CUDAGraphMode.FULL) == (None, None) @@ -264,14 +262,14 @@ def test_idle_dp_rank_dummy_trims_with_its_peers(dp_state): def test_dp_ranks_pad_to_one_replay_graph(dp_state): """Every rank pads its replay batch to the graph fitting the largest rank's.""" - dp_state.replay_graphs = MagicMock() - dp_state.replay_graphs.dispatch.return_value = BatchExecutionDescriptor( + dp_state.replay_cudagraphs = MagicMock() + dp_state.replay_cudagraphs.dispatch.return_value = BatchExecutionDescriptor( CUDAGraphMode.PIECEWISE, 512, None ) dp_state.other = (True, 300) batch = _input_batch(QUERY_LENS, SEQ_LENS, PREFILLING) replay, _ = _prepare(dp_state, batch, CUDAGraphMode.NONE) - assert dp_state.replay_graphs.dispatch.call_args.args[:2] == (3, 300) + assert dp_state.replay_cudagraphs.dispatch.call_args.args[:2] == (3, 300) assert replay.forward_context.batch_descriptor.num_tokens == 512 dp_metadata = replay.forward_context.dp_metadata assert dp_metadata.num_tokens_across_dp_cpu.tolist() == [512, 512] diff --git a/vllm/config/compilation.py b/vllm/config/compilation.py index 8417496ff320..4649c8900232 100644 --- a/vllm/config/compilation.py +++ b/vllm/config/compilation.py @@ -570,6 +570,12 @@ class CompilationConfig: If we limit the video count per prompt to `0`, it will also be set to `0` (i.e., fall back to image-only mode).""" + cudagraph_decoder_replay: bool = True + """Run the decoder replay layers of YOCO models (e.g. DeepSeek-V4.1) in CUDA + graphs of their own, and trim them in PIECEWISE graph steps. Requires + breakable PIECEWISE graphs; off under LoRA, prompt embeddings and non-first + PP ranks.""" + decoder_replay_cudagraph_capture_sizes: list[int] = field(default_factory=list) """Decoder replay CUDA graph sizes for YOCO models (e.g. DeepSeek-V4.1). If empty, capture 8 sizes per power-of-two interval (e.g. step 128 in @@ -580,8 +586,9 @@ class CompilationConfig: decoder_replay_trim_threshold: int = Field(default=768, ge=0) """For YOCO models (e.g. DeepSeek-V4.1), PIECEWISE graphs with at least this - many padded tokens trim the decoder replay batch. Set to 0 to trim at every size. - Independent of the replay graph capture sizes; eager steps still trim.""" + many padded tokens trim the decoder replay batch; must exceed the replay + window (128), below which nothing trims. Independent of the replay graph + capture sizes; eager steps still trim.""" # Inductor capture compile_sizes: list[int | str] | None = None diff --git a/vllm/models/deepseek_v41/decoder_replay_layers.py b/vllm/models/deepseek_v41/decoder_replay_layers.py index 98e2e5872d29..d2d7ffd15364 100644 --- a/vllm/models/deepseek_v41/decoder_replay_layers.py +++ b/vllm/models/deepseek_v41/decoder_replay_layers.py @@ -7,9 +7,10 @@ ``DeepseekV41ModelState`` prepares those rows as a sub-batch with attention metadata and a forward context of its own, like a microbatch; ``DecoderReplayLayers`` gathers the layer inputs by its rows, runs the layers -under that context and scatters the outputs back to batch rows. Steps that run -in a CUDA graph keep the layers on the whole batch, inside the graph; an eager -step's replay batch may run in a graph of its own (decoder_replay_cudagraph.py). +under that context and scatters the outputs back to batch rows. FULL graphs and +small PIECEWISE ones keep the layers on the whole batch; larger PIECEWISE graphs +break out to the replay batch, which may run in a graph of its own +(decoder_replay_cudagraph.py). """ from collections.abc import Callable @@ -20,7 +21,6 @@ from vllm.compilation.breakable_cudagraph import eager_break_during_capture from vllm.forward_context import ( ForwardContext, - get_forward_context, in_piecewise_cudagraph, override_forward_context, ) @@ -29,8 +29,7 @@ @dataclass class ReplayBatch: - """The replay layers' batch of one forward: its rows of the batch, its - forward context and, when it runs in a CUDA graph, the graph's runner.""" + """One forward's replay batch; ``run_graph`` is set when it runs in a graph.""" rows: torch.Tensor forward_context: ForwardContext @@ -60,34 +59,27 @@ def __init__( self.metadata_prefixes = metadata_prefixes # Set by the model state every step; None runs the layers on the batch. self.replay_batch: ReplayBatch | None = None - # PIECEWISE model graphs of at least this many tokens break out to run the - # layers on the replay batch; set with the replay graphs. - self.trim_threshold: int | None = None - self._graph_outputs: list[torch.Tensor] | None = None + self._break_outputs: list[torch.Tensor] | None = None def __call__( self, hidden_states: torch.Tensor | MoEOutput, *states: torch.Tensor | None ) -> tuple[torch.Tensor, ...]: - if self.trim_threshold is not None and in_piecewise_cudagraph(): - batch_descriptor = get_forward_context().batch_descriptor - assert batch_descriptor is not None - if batch_descriptor.num_tokens >= self.trim_threshold: - return self._run_in_graph_break(hidden_states, *states) + if self.replay_batch is not None and in_piecewise_cudagraph(): + return self._run_in_graph_break(hidden_states, *states) return self._run(hidden_states, *states) @eager_break_during_capture def _run_in_graph_break( self, hidden_states: torch.Tensor | MoEOutput, *states: torch.Tensor | None ) -> tuple[torch.Tensor, ...]: - """An eager break of the model's graph: the outputs land in fixed buffers, - which the graph's next segment reads.""" + """A graph break: the outputs land in fixed buffers the next segment reads.""" outputs = self._run(hidden_states, *states) - if self._graph_outputs is None: - # The largest model graph is captured first. - self._graph_outputs = [torch.empty_like(out) for out in outputs] + if self._break_outputs is None: + # Sized by the first call: the largest model graph is captured first. + self._break_outputs = [torch.empty_like(out) for out in outputs] return tuple( buf[: out.shape[0]].copy_(out) - for buf, out in zip(self._graph_outputs, outputs) + for buf, out in zip(self._break_outputs, outputs) ) def _run( @@ -96,8 +88,7 @@ def _run( replay_batch = self.replay_batch if replay_batch is None: return self.run_layers(hidden_states, *states) - # A trimming step holds a prefill longer than the window, more tokens - # than any step whose MoE leaves its finalize to the next layer. + # Replay batch steps hold more than `window` tokens: no deferred MoE finalize. assert isinstance(hidden_states, torch.Tensor) rows = replay_batch.rows num_rows = rows.shape[0] diff --git a/vllm/models/deepseek_v41/nvidia/decoder_replay_cudagraph.py b/vllm/models/deepseek_v41/nvidia/decoder_replay_cudagraph.py index 0aab916fa8d0..d501d40fd365 100644 --- a/vllm/models/deepseek_v41/nvidia/decoder_replay_cudagraph.py +++ b/vllm/models/deepseek_v41/nvidia/decoder_replay_cudagraph.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""CUDA graphs of the decoder replay layers on a trimming step's replay batch.""" +"""CUDA graphs of the decoder replay layers on a step's replay batch.""" from collections.abc import Callable from copy import copy @@ -13,8 +13,10 @@ from vllm.config.kernel import MEGA_MOE_BACKENDS from vllm.forward_context import get_forward_context, override_forward_context from vllm.models.deepseek_v41.decoder_replay_layers import DecoderReplayLayers -from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor as Desc -from vllm.v1.worker.gpu.cudagraph_utils import CudaGraphManager +from vllm.v1.worker.gpu.cudagraph_utils import ( + BatchExecutionDescriptor, + CudaGraphManager, +) class DecoderReplayCudaGraphManager(CudaGraphManager): @@ -36,13 +38,13 @@ def __init__(self, vllm_config: VllmConfig, device, layers: DecoderReplayLayers) sizes.append(s) s += 1 << max(s.bit_length() - 4, 0) sizes = sorted(set(sizes)) - if sizes and max(sizes) > scheduler.max_num_batched_tokens: + if max(sizes) > scheduler.max_num_batched_tokens: raise ValueError( "decoder_replay_cudagraph_capture_sizes must not exceed " "max_num_batched_tokens" ) compilation.cudagraph_capture_sizes = sizes - compilation.max_cudagraph_capture_size = max(sizes, default=0) + compilation.max_cudagraph_capture_size = max(sizes) replay_config = copy(vllm_config) replay_config.compilation_config = compilation super().__init__(replay_config, device, CUDAGraphMode.PIECEWISE, 1) @@ -59,18 +61,20 @@ def __init__(self, vllm_config: VllmConfig, device, layers: DecoderReplayLayers) shapes = [(h,), (), (), (hc,), (hc, 1), (hc, hc), (hc, h)] dtypes = [act, torch.int64, ids, f32, f32, f32, act] n = compilation.max_cudagraph_capture_size - self.inputs = [ + self.input_buffers = [ torch.zeros(n, *s, dtype=d, device=device) for s, d in zip(shapes, dtypes) ] - def capture_replay_graphs(self, prepare: Callable[[Desc], None]) -> None: + def capture_replay_graphs( + self, prepare: Callable[[BatchExecutionDescriptor], None] + ) -> None: """Capture each size on the replay batch ``prepare`` sets on the layers.""" - def create_forward_fn(desc: Desc, warmup: bool): + def create_forward_fn(desc: BatchExecutionDescriptor, warmup: bool): prepare(desc) assert self.layers.replay_batch is not None context = self.layers.replay_batch.forward_context - inputs = [buf[: desc.num_tokens] for buf in self.inputs] + inputs = [buf[: desc.num_tokens] for buf in self.input_buffers] def forward_fn(cg_mode: CUDAGraphMode) -> None: with override_forward_context( @@ -86,10 +90,12 @@ def forward_fn(cg_mode: CUDAGraphMode) -> None: def run(self, rows: torch.Tensor, inputs: tuple) -> tuple[torch.Tensor, ...]: """Replay the forward context's graph on ``rows`` of ``inputs``.""" num_rows = rows.shape[0] - for buf, t in zip(self.inputs, inputs): + for buf, t in zip(self.input_buffers, inputs): torch.index_select(t, 0, rows, out=buf[:num_rows]) batch_descriptor = get_forward_context().batch_descriptor assert batch_descriptor is not None num_tokens = batch_descriptor.num_tokens - outputs = self.breakable_cg_runner(*(buf[:num_tokens] for buf in self.inputs)) + outputs = self.breakable_cg_runner( + *(buf[:num_tokens] for buf in self.input_buffers) + ) return tuple(out[:num_rows] for out in outputs) diff --git a/vllm/models/deepseek_v41/nvidia/model_state.py b/vllm/models/deepseek_v41/nvidia/model_state.py index f0387d70e7cb..e9c0b5d136ca 100644 --- a/vllm/models/deepseek_v41/nvidia/model_state.py +++ b/vllm/models/deepseek_v41/nvidia/model_state.py @@ -240,23 +240,30 @@ def __init__( ) self._replay_attn_groups: list[list[AttentionGroup]] | None = None layers = self.decoder_replay_layers - self.replay_graphs: DecoderReplayCudaGraphManager | None = None + compilation_config = vllm_config.compilation_config + self.replay_cudagraphs: DecoderReplayCudaGraphManager | None = None + # PIECEWISE graphs of this many tokens or more break out to the replay batch. + self._trim_threshold: int | None = None if ( layers is not None - and vllm_config.compilation_config.cudagraph_mode.has_piecewise_cudagraphs() + and compilation_config.cudagraph_decoder_replay + and compilation_config.cudagraph_mode.has_piecewise_cudagraphs() and is_breakable_cudagraph_enabled() and get_pp_group().is_first_rank - # Neither the trimmed rows' LoRA mapping nor optional input ids are - # captured. + # The graphs' fixed inputs hold no LoRA mapping and no None input ids. and vllm_config.lora_config is None and not vllm_config.model_config.enable_prompt_embeds ): - self.replay_graphs = DecoderReplayCudaGraphManager( + threshold = compilation_config.decoder_replay_trim_threshold + if threshold <= layers.window: + raise ValueError( + "decoder_replay_trim_threshold must exceed the replay window " + f"of {layers.window} tokens: smaller graphs have nothing to trim" + ) + self.replay_cudagraphs = DecoderReplayCudaGraphManager( vllm_config, device, layers ) - layers.trim_threshold = ( - vllm_config.compilation_config.decoder_replay_trim_threshold - ) + self._trim_threshold = threshold def add_request(self, req_index: int, new_req_data: NewRequestData) -> None: super().add_request(req_index, new_req_data) @@ -438,23 +445,19 @@ def _prepare_replay_batch( replay_start: torch.Tensor, capture_desc: BatchExecutionDescriptor | None = None, ) -> None: - """Set the replay layers' batch for this forward, or none when nothing - trims. Eager steps trim, and so do PIECEWISE graph steps at the trim - threshold or above, whose graph breaks out to the replay layers; FULL - graphs and smaller PIECEWISE ones keep the layers on the whole batch. - Ranks share the graph mode and size; under data parallelism every rank - replays if any does.""" + """Set the replay layers' batch for this forward, or none. Eager steps trim; + captures and PIECEWISE graphs at the trim threshold or above break out to a + replay batch, trimmed or not; FULL and smaller PIECEWISE graphs keep the + whole batch. Under DP every rank replays if any does.""" layers = self.decoder_replay_layers assert layers is not None layers.replay_batch = None - # Captures and PIECEWISE graphs at the trim threshold or above (which break - # out to the replay layers) run them on a replay batch, trimmed or not. - replays = capture_desc is not None or ( + force_replay_batch = capture_desc is not None or ( cudagraph_mode == CUDAGraphMode.PIECEWISE - and layers.trim_threshold is not None - and input_batch.num_tokens_after_padding >= layers.trim_threshold + and self._trim_threshold is not None + and input_batch.num_tokens_after_padding >= self._trim_threshold ) - if cudagraph_mode != CUDAGraphMode.NONE and not replays: + if cudagraph_mode != CUDAGraphMode.NONE and not force_replay_batch: return num_reqs = input_batch.num_reqs @@ -466,31 +469,34 @@ def _prepare_replay_batch( ) # Dummy batches (captures, an idle DP rank's) are not prefills: they # keep their rows unless a rank trims. - trims = replays or bool( + needs_replay_batch = force_replay_batch or bool( (kept_lens < query_lens)[input_batch.is_prefilling_np[:num_reqs]].any() ) num_tokens = int(kept_lens.sum()) - dp_metadata = None + dp_tokens = None if self.vllm_config.parallel_config.data_parallel_size > 1: - trims, dp_metadata = self._agree_across_dp(trims, num_tokens) - if not trims: + needs_replay_batch, dp_tokens = self._agree_across_dp( + needs_replay_batch, num_tokens + ) + if not needs_replay_batch: return - graphs, run_graph = self.replay_graphs, None + cudagraphs, run_graph = self.replay_cudagraphs, None num_padded, cg_mode = num_tokens, CUDAGraphMode.NONE - dp_tokens = dp_metadata.num_tokens_across_dp_cpu if dp_metadata else None - if graphs is not None: + if cudagraphs is not None: # Every rank runs the graph that fits the largest rank's batch. - max_tokens = num_tokens if dp_tokens is None else int(dp_tokens.max()) - desc = capture_desc or graphs.dispatch(num_reqs, max_tokens, None, 0) + dp_max_tokens = num_tokens if dp_tokens is None else int(dp_tokens.max()) + desc = capture_desc or cudagraphs.dispatch(num_reqs, dp_max_tokens, None, 0) if desc.cg_mode == CUDAGraphMode.PIECEWISE: num_padded, cg_mode = desc.num_tokens, desc.cg_mode - run_graph = graphs.run + run_graph = cudagraphs.run if dp_tokens is not None: dp_tokens.fill_(num_padded) - dp_metadata = DPMetadata.make( - self.vllm_config.parallel_config, num_padded, dp_tokens - ) + dp_metadata = None + if dp_tokens is not None: + dp_metadata = DPMetadata.make( + self.vllm_config.parallel_config, num_padded, dp_tokens + ) kept_batch, kept_slot_mappings = self._kept_input_batch( input_batch, slot_mappings, replay_start, kept_lens, num_padded ) @@ -528,8 +534,8 @@ def capture_inner_cudagraphs( kv_cache_config: KVCacheConfig, ) -> None: def prepare(desc: BatchExecutionDescriptor) -> None: - # The runtime path on a dummy batch: the graph reads the buffers a - # real step's replay batch fills. + # Capture reuses the runtime path on a dummy batch, so the graph reads + # the buffers a real step fills. num_reqs = min(desc.num_tokens, self.max_num_reqs) self._prepare_replay_batch( InputBatch.make_dummy(num_reqs, desc.num_tokens, input_buffers), @@ -542,10 +548,10 @@ def prepare(desc: BatchExecutionDescriptor) -> None: capture_desc=desc, ) - if self.replay_graphs is not None: - # Profiling captures run on throwaway groups: build their own builders. + if self.replay_cudagraphs is not None: + # profile_only captures use pre-KV-cache groups: drop the cache. self._replay_attn_groups = None - self.replay_graphs.capture_replay_graphs(prepare) + self.replay_cudagraphs.capture_replay_graphs(prepare) def _kept_input_batch( self, @@ -607,20 +613,19 @@ def _kept_input_batch( return kept_batch, kept_slot_mappings def _agree_across_dp( - self, trims: bool, num_tokens: int - ) -> tuple[bool, DPMetadata | None]: - """Whether any rank trims and, then, the replay layers' DP metadata from - every rank's replay token count.""" + self, needs_replay_batch: bool, num_tokens: int + ) -> tuple[bool, torch.Tensor | None]: + """Whether any rank needs a replay batch and, if so, each rank's token count.""" parallel_config = self.vllm_config.parallel_config agreed = torch.zeros(2, parallel_config.data_parallel_size, dtype=torch.int32) agreed[:, parallel_config.data_parallel_rank] = torch.tensor( - [trims, num_tokens] + [needs_replay_batch, num_tokens] ) if not should_skip_dp_coordination(): dist.all_reduce(agreed, group=get_dp_group().cpu_group) if not agreed[0].any(): return False, None - return True, DPMetadata.make(parallel_config, num_tokens, agreed[1]) + return True, agreed[1] def _replay_groups( self, attn_groups: list[list[AttentionGroup]] From f61deaf3d29b29acbf75e250c5e5e64e360dcb54 Mon Sep 17 00:00:00 2001 From: Juntian Liu Date: Sat, 3 Oct 2026 01:31:43 +0000 Subject: [PATCH 6/8] [Perf][DSv4.1] Keep one replay row for an idle DP rank's dummy batch When one DP rank prefills and its peer is idle, the peer's dummy batch is padded to the prefill's token count and joined the replay layers' batch untrimmed: make_dummy's fake requests index stale request slots whose _keeps_rows_np is still set by the sampler warmup's prompt_logprobs requests. The prefilling rank, already trimmed to its window, then waited inside every replay-layer MegaMoE for the peer's full-size layer. A batch of context-free non-prefill rows can only be a dummy, and nothing reads it, so keep one row: the rank still runs every replay layer and its MoE exchange. DEP2 DeepSeek-V4.1-Flash on GB200 (128K cached prefix + 8K new tokens on rank 0, rank 1 idle): the prefill step drops from 213.7 to 126.9 ms; with rank 1 decoding, 127.1 vs 127.2 ms. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Juntian Liu --- tests/models/test_deepseek_v41_replay_batch.py | 7 ++++--- vllm/models/deepseek_v41/nvidia/model_state.py | 9 +++++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/tests/models/test_deepseek_v41_replay_batch.py b/tests/models/test_deepseek_v41_replay_batch.py index 31d286431ec5..810a1dbd9a02 100644 --- a/tests/models/test_deepseek_v41_replay_batch.py +++ b/tests/models/test_deepseek_v41_replay_batch.py @@ -252,12 +252,13 @@ def test_dp_ranks_replay_together(dp_state): def test_idle_dp_rank_dummy_trims_with_its_peers(dp_state): - dummy = _input_batch([512], [512], [False]) + """An idle rank's dummy keeps one row, so peers do not wait on its replay.""" + dummy = _input_batch([256, 256], [256, 256], [False, False]) dp_state.other = (True, 129) replay, _ = _prepare(dp_state, dummy, CUDAGraphMode.NONE) - assert replay is not None and replay.rows.shape[0] == WINDOW + assert replay is not None and replay.rows.shape[0] == 1 dp_metadata = replay.forward_context.dp_metadata - assert dp_metadata.num_tokens_across_dp_cpu.tolist() == [WINDOW, 129] + assert dp_metadata.num_tokens_across_dp_cpu.tolist() == [1, 129] def test_dp_ranks_pad_to_one_replay_graph(dp_state): diff --git a/vllm/models/deepseek_v41/nvidia/model_state.py b/vllm/models/deepseek_v41/nvidia/model_state.py index e9c0b5d136ca..c09e4582ea34 100644 --- a/vllm/models/deepseek_v41/nvidia/model_state.py +++ b/vllm/models/deepseek_v41/nvidia/model_state.py @@ -467,6 +467,15 @@ def _prepare_replay_batch( query_lens, np.minimum(query_lens, layers.window), ) + # A batch of context-free non-prefill rows is a dummy (an idle DP rank's): + # nothing reads it and its idx_mapping hits stale slots, so keep one row. + seq_lens = input_batch.seq_lens_cpu_upper_bound[:num_reqs].numpy() + is_dummy = not input_batch.is_prefilling_np[:num_reqs].any() and bool( + (seq_lens == query_lens).all() + ) + if is_dummy and capture_desc is None: + kept_lens = np.zeros_like(kept_lens) + kept_lens[0] = 1 # Dummy batches (captures, an idle DP rank's) are not prefills: they # keep their rows unless a rank trims. needs_replay_batch = force_replay_batch or bool( From 7f1df1b197d2911ec65e3ecdcae293591d3afa8d Mon Sep 17 00:00:00 2001 From: Juntian Liu Date: Mon, 5 Oct 2026 23:50:41 +0000 Subject: [PATCH 7/8] [Perf][DSv4.1] Fake capture_inner_cudagraphs in the capture_model tests The capture_model tests fake the model state with a SimpleNamespace, which lacks the new capture_inner_cudagraphs hook the runner now calls. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Juntian Liu --- tests/v1/worker/test_gpu_model_runner_v2.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/v1/worker/test_gpu_model_runner_v2.py b/tests/v1/worker/test_gpu_model_runner_v2.py index 7586245a3543..0caa132d7e45 100644 --- a/tests/v1/worker/test_gpu_model_runner_v2.py +++ b/tests/v1/worker/test_gpu_model_runner_v2.py @@ -259,7 +259,9 @@ def _make_capture_runner(captured: bool) -> GPUModelRunner: """Minimal V2 runner for capture_model: fakes everything except the cudagraph_manager's needs_capture decision.""" runner = GPUModelRunner.__new__(GPUModelRunner) - runner.model_state = SimpleNamespace(supports_mm_inputs=False) + runner.model_state = SimpleNamespace( + supports_mm_inputs=False, capture_inner_cudagraphs=lambda *args: None + ) runner.cudagraph_manager = SimpleNamespace( needs_capture=lambda: captured, capture=lambda *args, **kwargs: None, From 95ea212a4a78672048677afc70af0655964d90ee Mon Sep 17 00:00:00 2001 From: Juntian Liu Date: Mon, 5 Oct 2026 23:50:53 +0000 Subject: [PATCH 8/8] [Perf][DSv4.1] Start default replay graph sizes at the sliding window A replay batch that dispatches a replay graph holds at least `window` rows: eager steps replay only when a prefill trims to its last `window` rows, and PIECEWISE steps at the trim threshold hold more than `window` tokens. Sizes below the window were never replayed, so start there and step by the window, widening to 1/8 of the power-of-two interval past 16 * window. With the GB200 default bound of 1024 this captures 8 replay graphs instead of 64. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Juntian Liu --- vllm/config/compilation.py | 10 ++++------ .../deepseek_v41/nvidia/decoder_replay_cudagraph.py | 7 ++++--- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/vllm/config/compilation.py b/vllm/config/compilation.py index 4649c8900232..afe576daa9d3 100644 --- a/vllm/config/compilation.py +++ b/vllm/config/compilation.py @@ -577,12 +577,10 @@ class CompilationConfig: PP ranks.""" decoder_replay_cudagraph_capture_sizes: list[int] = field(default_factory=list) - """Decoder replay CUDA graph sizes for YOCO models (e.g. DeepSeek-V4.1). - If empty, capture 8 sizes per power-of-two interval (e.g. step 128 in - [1024, 2048), step 256 in [2048, 4096)) up to - min(max_cudagraph_capture_size, max_num_seqs * sliding_window), including - that upper bound. Explicit sizes override this inference and must not exceed - max_num_batched_tokens. Larger replay batches run eagerly.""" + """Decoder replay CUDA graph sizes for YOCO models (e.g. DeepSeek-V4.1), at + most max_num_batched_tokens. If empty: multiples of sliding_window up to + min(max_cudagraph_capture_size, max_num_seqs * sliding_window), coarser past + 16 * sliding_window. Larger replay batches run eagerly.""" decoder_replay_trim_threshold: int = Field(default=768, ge=0) """For YOCO models (e.g. DeepSeek-V4.1), PIECEWISE graphs with at least this diff --git a/vllm/models/deepseek_v41/nvidia/decoder_replay_cudagraph.py b/vllm/models/deepseek_v41/nvidia/decoder_replay_cudagraph.py index d501d40fd365..871f535584c6 100644 --- a/vllm/models/deepseek_v41/nvidia/decoder_replay_cudagraph.py +++ b/vllm/models/deepseek_v41/nvidia/decoder_replay_cudagraph.py @@ -32,11 +32,12 @@ def __init__(self, vllm_config: VllmConfig, device, layers: DecoderReplayLayers) compilation.max_cudagraph_capture_size, scheduler.max_num_seqs * layers.window, ) - # 8 sizes per power-of-two interval, e.g. step 256 in [2048, 4096). - sizes, s = [bound], 1 + # Replay batches hold at least `window` rows: step by window, or by 1/8 + # of the power-of-two interval once wider (256 in [2048, 4096)). + sizes, s = [bound], layers.window while s < bound: sizes.append(s) - s += 1 << max(s.bit_length() - 4, 0) + s += max(layers.window, 1 << max(s.bit_length() - 4, 0)) sizes = sorted(set(sizes)) if max(sizes) > scheduler.max_num_batched_tokens: raise ValueError(