Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 17 additions & 1 deletion tests/v1/spec_decode/test_dflash2.py
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,16 @@ def test_selector_asks_for_fp32_proposal_logits():


@pytest.mark.skip_global_cleanup
def test_dflash2_model_decoder_layer_cls(monkeypatch):
@pytest.mark.parametrize(
("num_dflash_query_tokens", "expected_conv_block_size"),
[
pytest.param(5, 5, id="bonus-anchor"),
pytest.param(4, 4, id="sample-anchor"),
],
)
def test_dflash2_model_decoder_layer_cls(
monkeypatch, num_dflash_query_tokens, expected_conv_block_size
):
from types import SimpleNamespace

from vllm.config import set_current_vllm_config
Expand Down Expand Up @@ -207,6 +216,7 @@ class MockGroup:
quantization=None,
),
num_speculative_tokens=4,
num_dflash_query_tokens=num_dflash_query_tokens,
enable_adaptive_verification=False,
),
model_config=SimpleNamespace(
Expand All @@ -228,3 +238,9 @@ class MockGroup:
# 4. Assert that the layers are DFlash2Qwen3DecoderLayer (the subclass)
assert len(model.layers) == 2
assert isinstance(model.layers[0], DFlash2Qwen3DecoderLayer)

# 5. The convolutions span one draft block, whose width is the query layout
# the checkpoint asks for, not a fixed 1 + num_speculative_tokens.
for layer in model.layers:
assert layer.attention_conv.block_size == expected_conv_block_size
assert layer.mlp_conv.block_size == expected_conv_block_size
21 changes: 18 additions & 3 deletions vllm/config/speculative.py
Original file line number Diff line number Diff line change
Expand Up @@ -1744,7 +1744,8 @@ def max_num_new_slots_for_drafting(self) -> int:
==================== ============= ======== ================
EAGLE3 eagle3 No 0
P-EAGLE eagle3 Yes K - 1
DFlash dflash Yes K
DFlash (bonus) dflash Yes K
DFlash (anchor) dflash Yes K - 1
DSpark dspark Yes K - 1
MTP mtp No 0
N-gram ngram No 0
Expand All @@ -1755,8 +1756,8 @@ def max_num_new_slots_for_drafting(self) -> int:
num_draft_tokens = self.num_speculative_tokens

if self.use_dflash():
# DFlash uses one bonus query followed by K mask queries.
return num_draft_tokens
# The scheduler already budgets one query slot per request.
return self.num_dflash_query_tokens - 1

if self.parallel_drafting:
if self.uses_draft_model():
Expand Down Expand Up @@ -1798,6 +1799,20 @@ def use_eagle(self) -> bool:
def use_dflash(self) -> bool:
return self.method == "dflash"

def dflash_samples_from_anchor(self) -> bool:
if not self.use_dflash() or self.draft_model_config is None:
return False
hf_config = self.draft_model_config.hf_config
dflash_config = getattr(hf_config, "dflash_config", None) or {}
if isinstance(dflash_config, Mapping):
return bool(dflash_config.get("sample_from_anchor", False))
return bool(getattr(dflash_config, "sample_from_anchor", False))

@property
def num_dflash_query_tokens(self) -> int:
"""Number of DFlash query positions that produce one draft block."""
return self.num_speculative_tokens + int(not self.dflash_samples_from_anchor())

def use_dspark(self) -> bool:
return self.method == "dspark"

Expand Down
12 changes: 8 additions & 4 deletions vllm/config/vllm.py
Original file line number Diff line number Diff line change
Expand Up @@ -604,10 +604,7 @@ def num_lookahead_tokens(self) -> int:
if speculative_config is None:
return 0
if speculative_config.use_dflash():
# DFlash requires an extra lookahead slot since it uses in-fill-style
# decoding instead of standard next-token sampling, so it has a query
# for the last sampled token plus queries for each draft token.
return self.num_speculative_tokens + 1
return speculative_config.num_dflash_query_tokens
if speculative_config.use_eagle() or speculative_config.uses_draft_model():
# DSpark (covered by use_eagle) drafts a block of num_speculative_tokens
# query tokens in which the anchor itself is the first prediction
Expand Down Expand Up @@ -2479,6 +2476,13 @@ def _get_v1_model_runner_unsupported_features(self) -> list[str]:
if self.speculative_config:
if self.speculative_config.method == "dspark":
unsupported.append("dspark speculative decoding")
# The V1 runner hardcodes the 1 + K DFlash query layout, so an
# anchor-sampling checkpoint would silently draft one token too many.
if (
self.speculative_config.use_dflash()
and self.speculative_config.dflash_samples_from_anchor()
):
unsupported.append("dflash sample_from_anchor")
if self.speculative_config.enable_adaptive_verification:
unsupported.append("adaptive draft verification")

Expand Down
3 changes: 1 addition & 2 deletions vllm/model_executor/models/qwen3_dflash2.py
Original file line number Diff line number Diff line change
Expand Up @@ -127,8 +127,7 @@ def __init__(
hidden_size=config.hidden_size,
taps=int(draft_config["conv_kernel_size"]),
group_size=int(draft_config["conv_group_size"]),
# Query tokens per request: the bonus token plus the mask tokens.
block_size=1 + speculative_config.num_speculative_tokens,
block_size=speculative_config.num_dflash_query_tokens,
params_dtype=vllm_config.model_config.dtype,
)
self.attention_conv = DFlashGroupedConv(
Expand Down
23 changes: 5 additions & 18 deletions vllm/v1/worker/gpu/spec_decode/dflash/speculator.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,8 @@ def __init__(self, vllm_config: VllmConfig, device: torch.device):
# Multimodal inputs not currently supported.
self.supports_mm_inputs = False

# Each request emits exactly (bonus + N mask) query tokens per step.
self.num_query_per_req = 1 + self.num_speculative_steps
self.num_query_per_req = self.speculative_config.num_dflash_query_tokens
self.sample_from_anchor = self.speculative_config.dflash_samples_from_anchor()

self.parallel_drafting_token_id = get_parallel_drafting_token_id(
self.draft_model_config.hf_config
Expand All @@ -57,20 +57,6 @@ def __init__(self, vllm_config: VllmConfig, device: torch.device):
self.draft_model_config.hf_config
)

# Whether the anchor query position is itself a prediction. DFlash default uses
# the anchor as the bonus token (only mask tokens predict); DSpark samples from
# the anchor and the N-1 mask token positions. See _prepare_dflash_inputs_kernel
dflash_config = (
getattr(self.draft_model_config.hf_config, "dflash_config", None) or {}
)
if dflash_config.get("sample_from_anchor", False):
raise ValueError(
"sample_from_anchor=True is not supported for DFlash. "
"DFlash uses a fixed 1+N query layout where the anchor "
"is the bonus token."
)
self.sample_from_anchor = False

# Context positions for the K/V precompute. Populated by
# prepare_dflash_inputs, and processed by the model's
# precompute_and_store_context_kv method. NOT captured by CUDA graphs.
Expand Down Expand Up @@ -631,8 +617,9 @@ def _prepare_dflash_inputs_kernel(
tl.store(out_query_slot_mapping_ptr + query_idx, q_slot, mask=is_query)

# --- Sample indices / positions / idx_mapping ---
# When SAMPLE_FROM_ANCHOR (DSpark), so we sample at EVERY query position
# and each position k predicts the NEXT token (sampled position = query_pos + 1).
# When SAMPLE_FROM_ANCHOR (DSpark, or DFlash with sample_from_anchor=True),
# we sample at EVERY query position and each position k predicts the NEXT
# token (sampled position = query_pos + 1).
# Otherwise (DFlash default) the anchor is the bonus token and only the mask tokens
# at offsets > 0 are sampled from, each AT its own position.
sample_off = 0 if SAMPLE_FROM_ANCHOR else 1
Expand Down
Loading