Skip to content
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 76 additions & 0 deletions tests/v1/spec_decode/test_dflash_buffers.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
# SPDX-License-Identifier: Apache-2.0
Comment thread
MatthewBonanni marked this conversation as resolved.
Resolved
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

import pytest
import torch

from vllm.config import (
CacheConfig,
ModelConfig,
ParallelConfig,
SchedulerConfig,
SpeculativeConfig,
VllmConfig,
)
from vllm.v1.spec_decode.dflash import DFlashProposer

DFLASH_TARGET_DIR = "Qwen/Qwen3-8B"
DFLASH_DRAFT_DIR = "z-lab/Qwen3-8B-DFlash-b16"

NUM_SPECULATIVE_TOKENS = 3


def _dflash_vllm_config(max_num_seqs: int) -> VllmConfig:
model_config = ModelConfig(
model=DFLASH_TARGET_DIR,
runner="generate",
max_model_len=8192,
trust_remote_code=True,
)
speculative_config = SpeculativeConfig(
target_model_config=model_config,
target_parallel_config=ParallelConfig(),
model=DFLASH_DRAFT_DIR,
method="dflash",
num_speculative_tokens=NUM_SPECULATIVE_TOKENS,
)
return VllmConfig(
model_config=model_config,
scheduler_config=SchedulerConfig(
max_num_seqs=max_num_seqs,
max_num_batched_tokens=8192,
max_model_len=model_config.max_model_len,
is_encoder_decoder=model_config.is_encoder_decoder,
),
cache_config=CacheConfig(
block_size=16,
gpu_memory_utilization=0.9,
cache_dtype="auto",
),
parallel_config=ParallelConfig(),
speculative_config=speculative_config,
)


@pytest.mark.parametrize("max_num_seqs", [33, 32])
def test_dflash_query_buffers_cover_cudagraph_padding(max_num_seqs: int):
"""The cudagraph dispatcher pads drafter batches up to the next capture
size, which exceeds max_query_tokens whenever it is not itself a capture
size (any odd max_num_seqs: e.g. 33 * 4 = 132 pads to 136 with the default
ladder). The persistent query buffers must cover the padded size, or
slicing them at that size silently yields short tensors and the drafter
crashes during cudagraph warmup.
"""
vllm_config = _dflash_vllm_config(max_num_seqs)
proposer = DFlashProposer(vllm_config, torch.device("cuda"))

max_query_tokens = max_num_seqs * (1 + NUM_SPECULATIVE_TOKENS)
capture_sizes = vllm_config.compilation_config.cudagraph_capture_sizes or []
padded_max_query = min(
(size for size in capture_sizes if size >= max_query_tokens),
default=max_query_tokens,
)

assert proposer.positions.shape[0] >= padded_max_query
assert proposer._slot_mapping_buffer.shape[0] >= padded_max_query
assert proposer.max_positions >= proposer.max_num_tokens + padded_max_query
19 changes: 16 additions & 3 deletions vllm/v1/spec_decode/dflash.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,21 @@ def __init__(

# Only next_token_ids and mask tokens are query tokens, all other context is K/V
self.max_query_tokens = self.max_batch_size * (1 + self.num_speculative_tokens)
# The cudagraph dispatcher pads query batches up to the next capture
# size, which exceeds max_query_tokens whenever max_query_tokens is not
# itself a capture size (e.g. max_num_seqs=33, k=3 gives 33 * 4 = 132
# query tokens, padded up to the capture size 136). Slicing
# exactly-sized buffers at the padded size would silently return short
# tensors. Capture sizes may still be adjusted after this point
# (resolve_cudagraph_mode_and_sizes), but never above
# max_cudagraph_capture_size and dispatch never pads batches larger
# than that, so it is a safe upper bound for the buffer capacity.
Comment thread
MatthewBonanni marked this conversation as resolved.
Resolved
self.max_padded_query_tokens = max(
self.max_query_tokens,
vllm_config.compilation_config.max_cudagraph_capture_size or 0,
)
# Positions covers both context states + query states
self.max_positions = self.max_num_tokens + self.max_query_tokens
self.max_positions = self.max_num_tokens + self.max_padded_query_tokens

# Separate context buffers to keep query buffer addresses stable for CUDA graphs
self._context_slot_mapping_buffer = torch.zeros(
Expand All @@ -48,7 +61,7 @@ def __init__(
device=device,
)
self._slot_mapping_buffer = torch.zeros(
self.max_query_tokens,
self.max_padded_query_tokens,
dtype=torch.int64,
device=device,
)
Expand All @@ -58,7 +71,7 @@ def __init__(
device=device,
)
self.positions = torch.zeros(
self.max_query_tokens,
self.max_padded_query_tokens,
dtype=torch.int64,
device=device,
)
Expand Down
Loading