From 0d35edb2861f91b47e18b3247bad5a7705bf7d8d Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Tue, 28 Jul 2026 13:09:51 +0000 Subject: [PATCH 01/40] [Spec Decode] DSpark adaptive verification Adaptively size the DSpark draft-verification budget from per-request confidence instead of always verifying every drafted token. A Triton kernel ranks draft slots by survival probability and admits the prefix that maximises estimated accepted tokens per millisecond, using cost curves profiled from real dummy steps at startup. Key pieces: - Varlen decode cudagraphs: decode descriptors are captured on the token grid directly, at min(num_tokens, max_num_seqs) requests and bounded by max_query_len, so trimmed batches still match a FULL graph instead of falling back to PIECEWISE. Every captured slot holds at least one token; capturing with empty slots corrupts replay. - Cost model: a step function inside the cudagraph-captured region (execution pads to the next captured size) and interpolated above it, where nothing pads. Draft costs are keyed on execution mode so eager samples never price graph replays, with a median filter guarding against outliers. - Confidence EMA defaults to 0.8, the point where the estimator is unbiased across batch sizes: smoothing trades noise for lag, and lag does not average out across a batch the way noise does. Limitations: - Output logprobs are rejected when confidence-based verification is enabled: verification compacts logits after the forward pass, while compute_topk_scores uses the scheduled layout. - The FULL varlen decode graphs require AttentionCGSupport.ALWAYS, which the DSV4 backends only report on SM100. Elsewhere decode falls back to PIECEWISE and the throughput win does not apply. Evaluated on DeepSeek-V4-Flash-DSpark, TP=4, SM100, at this commit's tree: - GSM8K 0.945 (c=16) / 0.946 (c=64), 0 invalid, pass bar 0.84 - MTBench 80/80 completed, no degenerate output - Throughput vs fixed 7-token verification: parity (+/-3%) at c=1-64, 1.71x at c=128, 2.19x at c=256 (11280 and 14347 tok/s) - vs no speculation: adaptive is 1.5-2.9x ahead at every concurrency, while fixed 7-token drops 33% below no-spec at c=256 AI assistance was used for this change. Co-authored-by: OpenAI Codex Co-authored-by: Claude Opus 5 (1M context) Signed-off-by: Lucas Wilkinson --- ...epSeek-V4-Flash-DSpark-confidence-TP4.yaml | 22 + .../spec_decode/test_adaptive_verification.py | 120 +++++ tests/v1/spec_decode/test_dynamic_sd_cug.py | 74 +++ vllm/config/compilation.py | 21 + vllm/config/speculative.py | 19 + .../layers/sparse_attn_indexer.py | 1 + vllm/model_executor/models/qwen3_dspark.py | 61 ++- vllm/models/deepseek_v4/nvidia/dspark.py | 28 +- .../deepseek_v4/nvidia/flashinfer_sparse.py | 15 +- vllm/models/deepseek_v4/sparse_mla.py | 13 +- vllm/sampling_params.py | 11 + vllm/utils/deep_gemm.py | 15 +- vllm/v1/attention/backend.py | 21 +- vllm/v1/attention/backends/mla/indexer.py | 89 +++- vllm/v1/attention/backends/mla/sparse_swa.py | 45 +- vllm/v1/worker/gpu/async_utils.py | 84 ++++ vllm/v1/worker/gpu/block_table.py | 25 + vllm/v1/worker/gpu/cudagraph_utils.py | 46 +- vllm/v1/worker/gpu/dp_utils.py | 5 + vllm/v1/worker/gpu/input_batch.py | 30 +- vllm/v1/worker/gpu/model_runner.py | 152 +++++- vllm/v1/worker/gpu/model_states/default.py | 4 +- .../gpu/spec_decode/adaptive_verification.py | 475 ++++++++++++++++++ .../gpu/spec_decode/dflash/speculator.py | 21 +- .../gpu/spec_decode/dspark/speculator.py | 19 + vllm/v1/worker/gpu/structured_outputs.py | 31 +- 26 files changed, 1353 insertions(+), 94 deletions(-) create mode 100644 tests/evals/gsm8k/configs/DeepSeek-V4-Flash-DSpark-confidence-TP4.yaml create mode 100644 tests/v1/spec_decode/test_adaptive_verification.py create mode 100644 vllm/v1/worker/gpu/spec_decode/adaptive_verification.py diff --git a/tests/evals/gsm8k/configs/DeepSeek-V4-Flash-DSpark-confidence-TP4.yaml b/tests/evals/gsm8k/configs/DeepSeek-V4-Flash-DSpark-confidence-TP4.yaml new file mode 100644 index 000000000000..4e5be4623c6a --- /dev/null +++ b/tests/evals/gsm8k/configs/DeepSeek-V4-Flash-DSpark-confidence-TP4.yaml @@ -0,0 +1,22 @@ +model_name: "deepseek-ai/DeepSeek-V4-Flash-DSpark" +accuracy_threshold: 0.92 +num_questions: 1319 +num_fewshot: 5 +startup_max_wait_seconds: 1800 +server_args: >- + --tokenizer-mode deepseek_v4 + --trust-remote-code + --dtype bfloat16 + --max-model-len 8192 + --tensor-parallel-size 4 + --enable-expert-parallel + --block-size 256 + --gpu-memory-utilization 0.5 + --kv-cache-dtype fp8 + --max-num-batched-tokens 16384 + --max-num-seqs 32 + --speculative-config '{"method":"dspark", + "model":"deepseek-ai/DeepSeek-V4-Flash-DSpark", + "attention_backend":"FLASH_ATTN","num_speculative_tokens":7, + "draft_sample_method":"probabilistic", + "dspark_enable_confidence_based_verification":true}' diff --git a/tests/v1/spec_decode/test_adaptive_verification.py b/tests/v1/spec_decode/test_adaptive_verification.py new file mode 100644 index 000000000000..f0a4e8c009a0 --- /dev/null +++ b/tests/v1/spec_decode/test_adaptive_verification.py @@ -0,0 +1,120 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from types import SimpleNamespace + +import numpy as np + +from vllm.v1.worker.gpu.async_utils import StepTimingSample +from vllm.v1.worker.gpu.spec_decode.adaptive_verification import ( + AdaptiveVerificationManager, + build_cost_tables_from_curves, +) + + +def make_manager( + confidences: np.ndarray, verify_cost_ms: np.ndarray +) -> AdaptiveVerificationManager: + num_reqs, num_steps = confidences.shape + manager = AdaptiveVerificationManager.__new__(AdaptiveVerificationManager) + manager.num_speculative_steps = num_steps + manager._stale_idx = 0 + manager._stale_confidences = [SimpleNamespace(np=confidences)] + manager.req_states = SimpleNamespace( + req_id_to_index={"low": 0, "high": 1}, + num_computed_tokens_np=np.ones(num_reqs, dtype=np.int32), + prefill_len=SimpleNamespace(np=np.ones(num_reqs, dtype=np.int32)), + ) + manager.cost_tables = (np.zeros(num_reqs + 1), verify_cost_ms) + return manager + + +def test_budget_stops_where_marginal_drafts_stop_paying_for_themselves(): + # Verification is cheap up to two extra tokens, then jumps 100x; only the + # highest-confidence draft is worth the cheap slot. + manager = make_manager( + np.array([[0.1, 0.1], [0.9, 0.9]], dtype=np.float32), + np.array([1.0, 1.0, 1.0, 1.0, 100.0, 100.0, 100.0]), + ) + + manager.get_num_tokens( + {"low": 3, "high": 3}, + {"low": [1, 2], "high": [3, 4]}, + has_structured_output=False, + ) + valid_drafts, num_non_draft_tokens, draft_budget = manager._batch_budget + + assert draft_budget == 1 + assert valid_drafts == {"low": 2, "high": 2} + assert num_non_draft_tokens == {"low": 1, "high": 1} + + +def test_structured_output_placeholders_do_not_consume_budget(): + manager = make_manager( + np.array([[0.9, 0.9], [0.9, 0.9]], dtype=np.float32), + np.ones(7), + ) + + manager.get_num_tokens( + {"low": 3, "high": 3}, + {"low": [1, -1], "high": [3, 4]}, + has_structured_output=True, + ) + valid_drafts, _, draft_budget = manager._batch_budget + + assert valid_drafts == {"low": 1, "high": 2} + assert draft_budget == 3 + + +def test_step_timing_cost_tables_are_monotonic_with_fixed_overhead(): + draft, verify = build_cost_tables_from_curves( + [(1, 2.0), (2, 3.0)], + [(1, 4.0), (2, 3.0)], + max_num_reqs=2, + max_batch_tokens=2, + ) + + assert draft.tolist() == [2.0, 2.0, 3.0] + assert verify.tolist() == [5.0, 5.0, 5.0] + + +def test_profiled_batches_seed_cost_curves_via_consumer(): + manager = AdaptiveVerificationManager.__new__(AdaptiveVerificationManager) + manager.req_states = SimpleNamespace(max_num_batched_tokens=4096, max_num_reqs=64) + manager.num_speculative_steps = 7 + manager.num_bonus_tokens = 1 + manager._profile_samples = [] + curves: dict[str, list[tuple[int, float]]] = {} + manager.set_cost_curves = lambda draft, verify: curves.update( + draft=draft, verify=verify + ) + + for batch in manager.batches_to_profile([8, 1024]): + assert batch["timing_enabled"] + num_tokens = batch["num_tokens"] + manager.consume_step_timing( + StepTimingSample( + forward_ms=float(num_tokens), + drafter_ms=1.0, + num_target_tokens=num_tokens, + num_reqs=num_tokens // 8, + # Only the captured sizes replay a graph; the tail sizes + # run eager. + full_cudagraph=num_tokens <= 1024, + ) + ) + manager.set_initial_cost_curves() + + # Tail beyond the last capture size: 1.5x then doubling to the max. + assert curves["verify"] == [ + (8, 8.0), + (1024, 1024.0), + (1536, 1536.0), + (2048, 2048.0), + (4096, 4096.0), + ] + # Eager batches must not contribute to the draft curve: keyed by request + # count they would land inside the captured range and, once made monotonic, + # smear that eager cost across every larger request count. + assert curves["draft"] == [(1, 1.0), (128, 1.0)] + assert manager._profile_samples == [] diff --git a/tests/v1/spec_decode/test_dynamic_sd_cug.py b/tests/v1/spec_decode/test_dynamic_sd_cug.py index d75495ea606a..fa2ebb77171c 100644 --- a/tests/v1/spec_decode/test_dynamic_sd_cug.py +++ b/tests/v1/spec_decode/test_dynamic_sd_cug.py @@ -326,3 +326,77 @@ def test_dynamic_sd_only_captures_scheduled_query_lengths(monkeypatch): assert desc.num_tokens == num_tokens assert desc.num_reqs is None assert desc.num_active_loras == 0 + + +def test_adaptive_verification_captures_non_uniform_full_graph(monkeypatch): + monkeypatch.setattr( + gpu_cudagraph_utils, + "get_pp_group", + lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True), + ) + + max_spec_tokens = 7 + vllm_config = _create_vllm_config_for_dsd( + max_num_seqs=64, + max_spec_tokens=max_spec_tokens, + use_dynamic_sd=False, + ) + manager = gpu_cudagraph_utils.CudaGraphManager( + vllm_config=vllm_config, + device=torch.device("cpu"), + cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE, + decode_query_len=max_spec_tokens + 1, + varlen_decode=True, + ) + manager._graphs_captured = True + + desc = manager.dispatch( + num_reqs=16, + num_tokens=80, + uniform_token_count=None, + num_active_loras=0, + max_query_len=8, + ) + + assert desc.cg_mode == CUDAGraphMode.FULL + assert desc.num_tokens == 80 + assert desc.num_reqs == 64 + assert desc.max_query_len == 8 + + for num_tokens in range(1, 8): + desc = manager.dispatch( + num_reqs=num_tokens, + num_tokens=num_tokens, + uniform_token_count=None, + num_active_loras=0, + max_query_len=1, + ) + + assert desc.cg_mode == CUDAGraphMode.FULL + assert desc.num_tokens >= num_tokens + assert desc.num_reqs >= num_tokens + assert desc.max_query_len == 8 + + desc = manager.dispatch( + num_reqs=1, + num_tokens=8, + uniform_token_count=None, + num_active_loras=0, + max_query_len=8, + ) + + assert desc.cg_mode == CUDAGraphMode.FULL + assert desc.num_reqs == 8 + + desc = manager.dispatch( + num_reqs=8, + num_tokens=8, + uniform_token_count=None, + num_active_loras=0, + max_query_len=1, + ) + + assert desc.cg_mode == CUDAGraphMode.FULL + assert desc.num_tokens == 8 + assert desc.num_reqs == 8 + assert desc.max_query_len == 8 diff --git a/vllm/config/compilation.py b/vllm/config/compilation.py index 4eb66613a81b..93190c48e67d 100644 --- a/vllm/config/compilation.py +++ b/vllm/config/compilation.py @@ -1375,6 +1375,7 @@ def resolve_cudagraph_mode_and_sizes( kv_cache_config: "KVCacheConfig | None" = None, max_num_reqs: int | None = None, is_profiling: bool = False, + varlen_decode: bool = False, ) -> CUDAGraphMode: from vllm.v1.attention.backend import AttentionCGSupport @@ -1457,6 +1458,26 @@ def resolve_cudagraph_mode_and_sizes( cudagraph_mode = CUDAGraphMode.NONE logger.warning(msg) + # Adaptive verification trims the draft budget per request, so decode graphs + # are varlen and need a backend that handles arbitrary query lengths. + if ( + cudagraph_mode.decode_mode() == CUDAGraphMode.FULL + and varlen_decode + and min_cg_support != AttentionCGSupport.ALWAYS + ): + msg = ( + f"CUDAGraphMode.{cudagraph_mode.name} is not supported with " + f"varlen decode for attention backend {min_cg_attn_backend} " + f"(support: {min_cg_support})" + ) + if self.splitting_ops_contain_attention(): + msg += "; setting cudagraph_mode=PIECEWISE" + cudagraph_mode = CUDAGraphMode.PIECEWISE + else: + msg += "; setting cudagraph_mode=NONE" + cudagraph_mode = CUDAGraphMode.NONE + logger.warning(msg) + # double check that we can support full cudagraph if they are requested # even after automatic downgrades if ( diff --git a/vllm/config/speculative.py b/vllm/config/speculative.py index e61909796842..a08dadff98d4 100644 --- a/vllm/config/speculative.py +++ b/vllm/config/speculative.py @@ -236,6 +236,22 @@ class SpeculativeConfig: synthetic_acceptance_rates. Only valid when rejection_sample_method is 'synthetic'. Mutually exclusive with synthetic_acceptance_rates.""" + dspark_enable_confidence_based_verification: bool = True + """Whether to enable DSpark confidence-based verification.""" + + dspark_confidence_ema_alpha: float = 0.8 + """Weight of the latest DSpark confidence observation in the per-request + exponential moving average. Set to 1 to disable smoothing. Smoothing trades + less noise for more lag; lag does not average out across a batch, so + heavier smoothing helps small batches but hurts large ones. 0.8 is the + point where the estimator is unbiased at every batch size measured.""" + + @property + def use_confidence_based_verification(self) -> bool: + return ( + self.method == "dspark" and self.dspark_enable_confidence_based_verification + ) + @staticmethod def _acceptance_length_to_rates(length: float, n: int) -> list[float]: """Mean acceptance length to unconditional per-position rates, using @@ -1332,6 +1348,9 @@ def _verify_args(self) -> Self: f"than zero ({self.num_speculative_tokens})." ) + if not 0 < self.dspark_confidence_ema_alpha <= 1: + raise ValueError("dspark_confidence_ema_alpha must be in (0, 1].") + if self.rejection_sample_method == "synthetic": # Consolidate to per-position rates self.synthetic_acceptance_rates = self._resolve_synthetic_acceptance_rates( diff --git a/vllm/model_executor/layers/sparse_attn_indexer.py b/vllm/model_executor/layers/sparse_attn_indexer.py index 4047d1a548d3..cb6363f9c59d 100644 --- a/vllm/model_executor/layers/sparse_attn_indexer.py +++ b/vllm/model_executor/layers/sparse_attn_indexer.py @@ -609,6 +609,7 @@ def sparse_attn_indexer( decode_metadata.schedule_metadata, max_model_len=max_model_len, clean_logits=False, + indices=decode_metadata.indices, ) num_rows = logits.shape[0] topk_indices = topk_indices_buffer[:num_padded_tokens, :topk_tokens] diff --git a/vllm/model_executor/models/qwen3_dspark.py b/vllm/model_executor/models/qwen3_dspark.py index 03ad01e1d91a..2cadf4d01e20 100644 --- a/vllm/model_executor/models/qwen3_dspark.py +++ b/vllm/model_executor/models/qwen3_dspark.py @@ -10,8 +10,9 @@ DFlash Qwen3 draft (see qwen3_dflash.py). DSpark adds: * ``markov_head``: low-rank V x r / r x V transition bias added to the base logits, sampled left-to-right by the speculator (the sequential stage). + * ``confidence_head``: per-position acceptance-probability estimate. -DSparkMarkovHead is shared with the DSV4-style DSpark model. +DSparkMarkovHead and DSparkConfidenceHead are shared with the DSV4-style DSpark model. """ from collections.abc import Iterable @@ -21,6 +22,7 @@ from vllm.config import VllmConfig from vllm.logger import init_logger +from vllm.model_executor.layers.linear import ReplicatedLinear from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.vocab_parallel_embedding import ( @@ -102,8 +104,36 @@ def apply_bias_gathered( return logits.scatter_(1, index, corrected.squeeze(-1)) +class DSparkConfidenceHead(nn.Module): + """DSpark acceptance-confidence head.""" + + def __init__( + self, + input_dim: int, + prefix: str, + bias: bool = False, + with_markov: bool = True, + ) -> None: + super().__init__() + self.with_markov = with_markov + self.proj = ReplicatedLinear( + input_dim, + 1, + bias=bias, + return_bias=False, + params_dtype=torch.float32, + prefix=maybe_prefix(prefix, "proj"), + ) + + def forward(self, hidden: torch.Tensor, markov_embed: torch.Tensor) -> torch.Tensor: + x = ( + torch.cat([hidden, markov_embed], dim=-1) if self.with_markov else hidden + ).float() + return self.proj(x).squeeze(-1) + + class Qwen3DSparkModel(DFlashQwen3Model): - """DFlash Qwen3 backbone + DSpark Markov head.""" + """DFlash Qwen3 backbone + DSpark Markov / confidence heads.""" def __init__( self, @@ -126,6 +156,18 @@ def __init__( prefix=maybe_prefix(prefix, "markov_head"), quant_config=self.quant_config, ) + self.confidence_head: DSparkConfidenceHead | None = None + if getattr(config, "enable_confidence_head", False): + with_markov = getattr(config, "confidence_head_with_markov", False) + input_dim = config.hidden_size + if with_markov: + input_dim += config.markov_rank + self.confidence_head = DSparkConfidenceHead( + input_dim, + prefix=maybe_prefix(prefix, "confidence_head"), + bias=True, + with_markov=with_markov, + ) class Qwen3DSparkForCausalLM(DFlashQwen3ForCausalLM): @@ -197,11 +239,18 @@ def apply_markov_bias_gathered( self.logits_processor.scale, ) + def compute_confidence( + self, head_hidden: torch.Tensor, markov_embed: torch.Tensor + ) -> torch.Tensor: + assert self.model.confidence_head is not None + return self.model.confidence_head(head_hidden, markov_embed) + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]): model_weights = {} includes_embed_tokens = False includes_lm_head = False includes_draft_id_mapping = False + includes_confidence_head = False for name, loaded_weight in weights: # t2d is training-only; the draft remaps via d2t at sampling time. if "t2d" in name: @@ -215,22 +264,26 @@ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]): includes_embed_tokens = True if "lm_head" in name: includes_lm_head = True + if "confidence_head" in name: + includes_confidence_head = True model_weights[name] = loaded_weight # Sets has_own_embed_tokens / has_own_lm_head so load_dspark_model # knows whether to keep these or alias the target's. process_eagle_weight(self, name) # mask_embedding is an unused placeholder param; DSpark masks via the vocab row. - # confidence_head is not wired into inference yet; skip its weights. # embed_tokens / lm_head are optional; when omitted they are shared from # the target by load_dspark_model, so skip the unloaded params here. - skip_substrs = ["mask_embedding", "confidence_head"] + skip_substrs = ["mask_embedding"] if not includes_embed_tokens: skip_substrs.append("embed_tokens") if not includes_lm_head: skip_substrs.append("lm_head") if not includes_draft_id_mapping: skip_substrs.append("draft_id_to_target_id") + if self.model.confidence_head is None or not includes_confidence_head: + self.model.confidence_head = None + skip_substrs.append("confidence_head") loader = AutoWeightsLoader(self, skip_substrs=skip_substrs) loader.load_weights(model_weights.items()) self.model._build_fused_kv_buffers() diff --git a/vllm/models/deepseek_v4/nvidia/dspark.py b/vllm/models/deepseek_v4/nvidia/dspark.py index 5478c423a223..373696d5982d 100644 --- a/vllm/models/deepseek_v4/nvidia/dspark.py +++ b/vllm/models/deepseek_v4/nvidia/dspark.py @@ -39,6 +39,7 @@ ) from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.model_executor.models.qwen3_dspark import ( + DSparkConfidenceHead, DSparkMarkovHead, ) from vllm.model_executor.models.utils import maybe_prefix @@ -113,7 +114,7 @@ def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: ] ) - # Heads: final norm + hc_head, and the Markov head + # Heads: final norm + hc_head, and the Markov + confidence heads # Loaded from the "final" MTP layer weights (mtp.*) in the target checkpoint self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) hc_dim = self.hc_mult * config.hidden_size @@ -136,6 +137,12 @@ def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: config.dspark_markov_rank, prefix=maybe_prefix(prefix, "markov_head"), ) + self.confidence_head: DSparkConfidenceHead | None = None + if getattr(config, "enable_confidence_head", True): + self.confidence_head = DSparkConfidenceHead( + config.hidden_size + config.dspark_markov_rank, + prefix=maybe_prefix(prefix, "confidence_head"), + ) def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: return self.embed_tokens(input_ids) @@ -369,6 +376,12 @@ def markov_embed(self, token_ids: torch.Tensor) -> torch.Tensor: def markov_bias(self, markov_embed: torch.Tensor) -> torch.Tensor: return self.model.markov_head.bias(markov_embed, self.logits_processor) + def compute_confidence( + self, head_hidden: torch.Tensor, markov_embed: torch.Tensor + ) -> torch.Tensor: + assert self.model.confidence_head is not None + return self.model.confidence_head(head_hidden, markov_embed) + # --- Weight loading ---------------------------------------------------- def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: @@ -407,6 +420,7 @@ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: params_dict = dict(self.named_parameters()) loaded_params: set[str] = set() + loaded_confidence_head = False tp_size = get_tensor_model_parallel_world_size() tp_rank = get_tensor_model_parallel_rank() @@ -419,6 +433,8 @@ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: if mapped is None: continue name = mapped + if "confidence_head." in name: + loaded_confidence_head = True # ``.scale`` -> per-method scale suffix. if name.endswith(".scale"): @@ -461,8 +477,8 @@ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: continue # Stacked rules only apply to decoder-layer weights. Head-stack params - # (main_proj/norm/hc_head/markov_head) load directly — otherwise e.g. - # "markov_w1" would collide with the "w1" shard rule. + # (main_proj/norm/hc_head/markov_head/confidence_head) load directly — + # otherwise e.g. "markov_w1" would collide with the "w1" shard rule. is_layer_param = name.startswith("model.layers.") for param_name, weight_name, stacked_shard_id in stacked_params_mapping: if not is_layer_param or weight_name not in name: @@ -488,6 +504,8 @@ def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: loaded_params.add(name) self._finalize_moe() + if self.model.confidence_head is not None and not loaded_confidence_head: + self.model.confidence_head = None logger.info_once("DSpark draft model loaded: %d params", len(loaded_params)) return loaded_params @@ -505,8 +523,7 @@ def _remap_dspark_name(self, name: str) -> str | None: return None stage = int(m.group(1)) rest = m.group(2) - # The confidence head is not wired into inference yet; drop its weights. - if rest.startswith("confidence_head."): + if rest.startswith("confidence_head.") and self.model.confidence_head is None: return None # Head-stack params live at model level (mtp.last), context combiner at # model level (mtp.0); everything else is a per-layer decoder block. @@ -516,6 +533,7 @@ def _remap_dspark_name(self, name: str) -> str | None: "hc_head_base", "hc_head_scale", "markov_head.", + "confidence_head.", ) if rest.startswith(("main_proj.", "main_norm.")) or rest.startswith( head_prefixes diff --git a/vllm/models/deepseek_v4/nvidia/flashinfer_sparse.py b/vllm/models/deepseek_v4/nvidia/flashinfer_sparse.py index d8742e610342..b31329bb59f3 100644 --- a/vllm/models/deepseek_v4/nvidia/flashinfer_sparse.py +++ b/vllm/models/deepseek_v4/nvidia/flashinfer_sparse.py @@ -309,20 +309,19 @@ def _build_sparse_index_metadata( assert swa_metadata.decode_swa_indices is not None assert swa_metadata.block_table is not None + decode_swa_width = swa_metadata.decode_swa_indices.shape[-1] + assert num_prefill_tokens == 0 or decode_swa_width == self.window_size decode_swa_indices = swa_metadata.decode_swa_indices.reshape( - num_decode_tokens, self.window_size + num_decode_tokens, decode_swa_width ) decode_compressed_topk_lens = None decode_compressed_indices_are_local = False decode_is_valid_token = None if swa_only: - assert self.topk_indices_buffer is not None compressed_kv_cache = swa_k_cache decode_compressed_indices = None - prefill_topk_indices = self.topk_indices_buffer[ - num_decode_tokens:num_tokens, :0 - ] + prefill_topk_indices = decode_swa_indices.new_empty(num_prefill_tokens, 0) compressed_block_table = None compressed_block_size = swa_metadata.block_size top_k = 0 @@ -405,7 +404,7 @@ def _build_sparse_index_metadata( swa_metadata.block_size, compressed_block_table, compressed_block_size, - self.window_size, + decode_swa_width, self.compress_ratio, top_k, decode_compressed_indices_are_local=decode_compressed_indices_are_local, @@ -489,8 +488,6 @@ def _forward( # uniform-q batches, and this avoids flattening mixed batches into one call. if num_decode_tokens > 0: decode_cu = query_start_loc[: num_decodes + 1] - decode_cu_cpu = query_start_loc_cpu[: num_decodes + 1] - decode_lens_cpu = decode_cu_cpu[1:] - decode_cu_cpu[:-1] flashinfer_trtllm_batch_decode_sparse_mla_dsv4( query=query[:num_decode_tokens], swa_kv_cache=swa_k_cache, @@ -504,7 +501,7 @@ def _forward( bmm2_scale=bmm2_scale, sinks=self.attn_sink, cum_seq_lens_q=decode_cu, - max_q_len=int(decode_lens_cpu.max().item()), + max_q_len=swa_metadata.max_decode_query_len, ) if num_prefill_tokens > 0: diff --git a/vllm/models/deepseek_v4/sparse_mla.py b/vllm/models/deepseek_v4/sparse_mla.py index 4ac27bd59a8d..a9d78aca1a8e 100644 --- a/vllm/models/deepseek_v4/sparse_mla.py +++ b/vllm/models/deepseek_v4/sparse_mla.py @@ -9,6 +9,7 @@ from vllm.config import VllmConfig from vllm.config.cache import CacheDType +from vllm.platforms import current_platform from vllm.platforms.interface import DeviceCapability from vllm.triton_utils import tl, triton from vllm.utils.math_utils import cdiv @@ -134,7 +135,17 @@ class DeepseekV4FlashMLAMetadata(AttentionMetadata): class DeepseekV4FlashMLAMetadataBuilder( AttentionMetadataBuilder[DeepseekV4FlashMLAMetadata] ): - _cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH + @classmethod + def get_cudagraph_support( + cls, + vllm_config: VllmConfig, + kv_cache_spec: AttentionSpec, + ) -> AttentionCGSupport: + if current_platform.is_cuda() and current_platform.is_device_capability_family( + 100 + ): + return AttentionCGSupport.ALWAYS + return AttentionCGSupport.UNIFORM_BATCH def __init__( self, diff --git a/vllm/sampling_params.py b/vllm/sampling_params.py index 5dedbde372e4..c837fa474f17 100644 --- a/vllm/sampling_params.py +++ b/vllm/sampling_params.py @@ -891,6 +891,17 @@ def _validate_spec_decode( if speculative_config is None: return + # Adaptive verification compacts logits after the forward pass, while + # compute_topk_scores uses the scheduled layout in cu_num_logits_np. + if ( + speculative_config.use_confidence_based_verification + and self.num_logprobs is not None + ): + raise ValueError( + "Output logprobs are not supported with DSpark confidence-based " + "verification." + ) + # Some sampling parameters are not yet compatible with spec decoding. if self.min_p > _SAMPLING_EPS or self.logit_bias: raise VLLMValidationError( diff --git a/vllm/utils/deep_gemm.py b/vllm/utils/deep_gemm.py index 0a1644bcc5c6..b9480ce5b22c 100644 --- a/vllm/utils/deep_gemm.py +++ b/vllm/utils/deep_gemm.py @@ -542,7 +542,10 @@ def fp8_fp4_mqa_logits( def get_paged_mqa_logits_metadata( - context_lens: torch.Tensor, block_size: int, num_sms: int + context_lens: torch.Tensor, + block_size: int, + num_sms: int, + indices: torch.Tensor | None = None, ) -> torch.Tensor: """Build scheduling metadata for paged MQA logits. @@ -551,6 +554,7 @@ def get_paged_mqa_logits_metadata( per batch element. block_size: KV-cache block size in tokens (e.g., 64). num_sms: Number of SMs available. 132 for Hopper + indices: Optional request index for each varlen row. Returns: Backend-specific tensor consumed by `fp8_fp4_paged_mqa_logits` to @@ -559,7 +563,10 @@ def get_paged_mqa_logits_metadata( _lazy_init() if _get_paged_mqa_logits_metadata_impl is None: return _missing() - return _get_paged_mqa_logits_metadata_impl(context_lens, block_size, num_sms) + kwargs = {} if indices is None else {"indices": indices} + return _get_paged_mqa_logits_metadata_impl( + context_lens, block_size, num_sms, **kwargs + ) def fp8_fp4_paged_mqa_logits( @@ -571,6 +578,7 @@ def fp8_fp4_paged_mqa_logits( schedule_metadata: torch.Tensor, max_model_len: int, clean_logits: bool, + indices: torch.Tensor | None = None, ) -> torch.Tensor: """Compute MQA logits using a paged KV-cache. @@ -595,6 +603,7 @@ def fp8_fp4_paged_mqa_logits( used to distribute work across SMs. max_model_len: Maximum sequence length used to size the logits output. clean_logits: Whether to clean the unfilled logits into `-inf`. + indices: Optional request index for each varlen row. Returns: Logits tensor of shape [B * next_n, max_model_len], dtype @@ -603,6 +612,7 @@ def fp8_fp4_paged_mqa_logits( _lazy_init() if _fp8_fp4_paged_mqa_logits_impl is None: return _missing() + kwargs = {} if indices is None else {"indices": indices} return _fp8_fp4_paged_mqa_logits_impl( q, kv_cache, @@ -612,6 +622,7 @@ def fp8_fp4_paged_mqa_logits( schedule_metadata, max_model_len, clean_logits=clean_logits, + **kwargs, ) diff --git a/vllm/v1/attention/backend.py b/vllm/v1/attention/backend.py index 97bfd419f8c1..fae085c21ea7 100644 --- a/vllm/v1/attention/backend.py +++ b/vllm/v1/attention/backend.py @@ -16,7 +16,6 @@ kFp8StaticTensorSym, kNvfp4Dynamic, ) -from vllm.utils.torch_utils import np_to_pinned_tensor if TYPE_CHECKING: from vllm.config import VllmConfig @@ -548,17 +547,19 @@ def token_to_req_indices(self, buffer: torch.Tensor) -> torch.Tensor: assert self._token_to_req_indices_cache.shape[0] >= num_tokens return self._token_to_req_indices_cache[:num_tokens] - starts = np.asarray(self.query_start_loc_cpu, dtype=np.int32) - query_lens = np.diff(starts) - token_to_req_indices = np.repeat( - np.arange(query_lens.shape[0], dtype=np.int32), query_lens - ) - num_mapped_tokens = token_to_req_indices.shape[0] + # Built from the device query_start_loc: adaptive verification decides the + # per-request draft split on device, so the CPU copy carries the right total + # but not the right per-request boundaries. Padding requests have a query + # length of zero and drop out of the repeat. + num_mapped_tokens = int(self.query_start_loc_cpu[-1]) + query_lens = self.query_start_loc[1:] - self.query_start_loc[:-1] assert buffer.shape[0] >= max(num_mapped_tokens, num_tokens) - # copy from CPU to GPU - buffer[:num_mapped_tokens].copy_( - np_to_pinned_tensor(token_to_req_indices), non_blocking=True + token_to_req_indices = torch.repeat_interleave( + torch.arange(query_lens.shape[0], dtype=torch.int32, device=buffer.device), + query_lens, + output_size=num_mapped_tokens, ) + buffer[:num_mapped_tokens].copy_(token_to_req_indices) if num_mapped_tokens < num_tokens: buffer[num_mapped_tokens:num_tokens].zero_() self._token_to_req_indices_cache = buffer[: max(num_mapped_tokens, num_tokens)] diff --git a/vllm/v1/attention/backends/mla/indexer.py b/vllm/v1/attention/backends/mla/indexer.py index 7051023a7c7f..7cb68465c1f0 100644 --- a/vllm/v1/attention/backends/mla/indexer.py +++ b/vllm/v1/attention/backends/mla/indexer.py @@ -418,6 +418,7 @@ class DeepSeekV32IndexerDecodeMetadata: requires_padding: bool schedule_metadata: torch.Tensor global_seq_lens: torch.Tensor | None = None + indices: torch.Tensor | None = None @dataclass @@ -452,6 +453,14 @@ def get_max_prefill_buffer_size(vllm_config: VllmConfig): return max_model_len * 40 +def _supports_varlen_paged_mqa_logits() -> bool: + return ( + current_platform.is_cuda() + and current_platform.is_device_capability_family(100) + and has_deep_gemm() + ) + + class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder): # The indexer opts out of the shared reorder-threshold vote (see __init__), # so this is None; its own split uses self.decode_threshold. @@ -464,6 +473,8 @@ def get_cudagraph_support( vllm_config: VllmConfig, kv_cache_spec: KVCacheSpec, ) -> AttentionCGSupport: + if _supports_varlen_paged_mqa_logits(): + return AttentionCGSupport.ALWAYS return AttentionCGSupport.UNIFORM_BATCH def __init__(self, *args, block_table_width: int, **kwargs) -> None: @@ -515,10 +526,12 @@ def __init__(self, *args, block_table_width: int, **kwargs) -> None: self.use_flattening = not current_platform.is_device_capability_family( 100 ) and next_n not in (1, 2) + self.supports_varlen = _supports_varlen_paged_mqa_logits() logger.info_once( - "DSA indexer decode path: use_flattening=%s " + "DSA indexer decode path: use_flattening=%s supports_varlen=%s " "(next_n=%d, use_fp4_indexer_cache=%s)", self.use_flattening, + self.supports_varlen, next_n, self.use_fp4_indexer_cache, ) @@ -547,6 +560,11 @@ def __init__(self, *args, block_table_width: int, **kwargs) -> None: dtype=torch.int32, device=self.device, ) + self.decode_indices_buffer = torch.zeros( + (scheduler_config.max_num_batched_tokens,), + dtype=torch.int32, + device=self.device, + ) self.arange_buffer = torch.arange( max( scheduler_config.max_num_seqs * next_n, @@ -626,25 +644,16 @@ def _prepare_decode_tensors( next_n: int, max_decode_len: int, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, int, bool]: - """Expand seq_lens/block_table/decode_lens for the decode kernels. - - Flatten path (not use_native, max_decode_len > 1): - Each multi-token decode request is expanded into individual - single-token entries so the kernel always sees next_n=1. - - Native path (use_native or max_decode_len == 1): - Plain decode or spec-decode with 2D per-token context lengths. - - Returns (seq_lens, block_table, decode_lens, batch_size, requires_padding). - seq_lens is 1D (batch_size,) for flatten/plain, 2D (B, max_decode_len) - for native MTP. - """ + """Prepare native or per-token flattened decode tensors.""" min_decode_len = int(decode_lens_cpu.min().item()) - if not use_native and max_decode_len > 1: + if not use_native: assert self.decode_seq_lens_buffer.dim() == 1 - if min_decode_len == max_decode_len: - # Uniform decode lengths. - num_decode_tokens = num_decodes * max_decode_len + if ( + not self.supports_varlen + and min_decode_len == max_decode_len + and num_decodes * max_decode_len == num_decode_tokens + ): + # Uniform decode lengths with no cudagraph token padding. _prepare_uniform_decode_kernel[(num_decode_tokens,)]( seq_lens, self.decode_seq_lens_buffer, @@ -761,6 +770,28 @@ def _prepare_global_decode_seq_lens( self.global_decode_seq_lens_buffer[actual_expanded:num_decode_tokens] = 0 return self.global_decode_seq_lens_buffer[:num_decode_tokens] + def _build_varlen_decode_indices( + self, + decode_lens: torch.Tensor, + decode_lens_cpu: torch.Tensor, + num_decode_tokens: int, + ) -> torch.Tensor: + """Build request ids for flattened SM100 varlen rows.""" + indices = self.decode_indices_buffer[:num_decode_tokens] + actual_expanded = int(decode_lens_cpu.sum().item()) + num_decodes = decode_lens.shape[0] + indices[:actual_expanded] = torch.repeat_interleave( + self.arange_buffer[:num_decodes], + decode_lens, + output_size=actual_expanded, + ) + if actual_expanded < num_decode_tokens: + pad = num_decode_tokens - actual_expanded + indices[actual_expanded:num_decode_tokens] = ( + num_decodes + self.arange_buffer[:pad] + ) + return indices + def build( self, common_prefix_len: int, @@ -775,12 +806,11 @@ def build( slot_mapping = common_attn_metadata.slot_mapping block_table = common_attn_metadata.block_table_tensor dcp_local_seq_lens = common_attn_metadata.dcp_local_seq_lens - num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens = ( split_decodes_and_prefills( common_attn_metadata, decode_threshold=self.decode_threshold, - require_uniform=not self.use_flattening, + require_uniform=not (self.use_flattening or self.supports_varlen), treat_short_extends_as_decodes=not self.use_pcp, ) ) @@ -888,7 +918,10 @@ def build( max_decode_len = int(decode_lens_cpu.max().item()) next_n = 1 + self.num_speculative_tokens - use_native = not self.use_flattening and max_decode_len <= next_n + use_native = ( + not (self.use_flattening or self.supports_varlen) + and max_decode_len <= next_n + ) global_seq_lens_for_decode = self._prepare_global_decode_seq_lens( global_seq_lens=global_seq_lens_for_decode, @@ -900,6 +933,14 @@ def build( max_decode_len=max_decode_len, ) + decode_indices = None + if self.supports_varlen: + decode_indices = self._build_varlen_decode_indices( + decode_lens=decode_lens, + decode_lens_cpu=decode_lens_cpu, + num_decode_tokens=num_decode_tokens, + ) + seq_lens, block_table, decode_lens, batch_size, requires_padding = ( self._prepare_decode_tensors( seq_lens=seq_lens, @@ -915,9 +956,7 @@ def build( ) ) - seq_lens_is_buffer_view = (use_native and next_n > 1) or ( - not use_native and max_decode_len > 1 - ) + seq_lens_is_buffer_view = not use_native or next_n > 1 # DCP: localize the now-expanded per-token global bounds to this # rank's owned KV. Done here (after expansion) so each token's global @@ -952,6 +991,7 @@ def build( seq_lens, self.kv_cache_spec.storage_block_size, self.num_sms, + indices=decode_indices, ) decode_metadata = DeepSeekV32IndexerDecodeMetadata( @@ -960,6 +1000,7 @@ def build( decode_lens=decode_lens, requires_padding=requires_padding, schedule_metadata=self.scheduler_metadata_buffer, + indices=decode_indices, global_seq_lens=global_seq_lens_for_decode, ) diff --git a/vllm/v1/attention/backends/mla/sparse_swa.py b/vllm/v1/attention/backends/mla/sparse_swa.py index c823f82be29b..5f29642304a4 100644 --- a/vllm/v1/attention/backends/mla/sparse_swa.py +++ b/vllm/v1/attention/backends/mla/sparse_swa.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from dataclasses import dataclass, field -from typing import ClassVar, cast +from typing import cast import torch @@ -25,6 +25,7 @@ from vllm.v1.attention.backends.utils import split_decodes_and_prefills from vllm.v1.attention.ops.flashmla import FlashMLASchedMeta, get_mla_metadata from vllm.v1.kv_cache_interface import ( + AttentionSpec, KVCacheSpec, MLAAttentionSpec, SlidingWindowMLASpec, @@ -183,6 +184,7 @@ class DeepseekSparseSWAMetadata: num_prefills: int = 0 num_decode_tokens: int = 0 num_prefill_tokens: int = 0 + max_decode_query_len: int = 1 # Pre-computed prefill metadata shared across all DeepseekV4 attention layers. prefill_seq_lens: torch.Tensor | None = None @@ -399,7 +401,18 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder): """ reorder_batch_threshold: int | None = None - _cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH + + @classmethod + def get_cudagraph_support( + cls, + vllm_config: VllmConfig, + kv_cache_spec: AttentionSpec, + ) -> AttentionCGSupport: + if current_platform.is_cuda() and current_platform.is_device_capability_family( + 100 + ): + return AttentionCGSupport.ALWAYS + return AttentionCGSupport.UNIFORM_BATCH def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -426,6 +439,7 @@ def __init__(self, *args, **kwargs): 2 if (spec_config is not None and spec_config.parallel_drafting) else 1 ) self.decode_threshold = 1 + spec_mult * self.num_speculative_tokens + self.max_decode_query_len = 1 + self.num_speculative_tokens self.reorder_batch_threshold = None hf_config = self.vllm_config.model_config.hf_config @@ -519,6 +533,12 @@ def build( common_attn_metadata, decode_threshold=self.decode_threshold ) ) + if num_prefills > 0: + # split_decodes_and_prefills counts trailing cudagraph request padding as + # prefills. Those rows have no tokens, so drop them here rather than let + # them size prefill metadata and the chunk plan. + query_lens_cpu = query_start_loc_cpu[1:] - query_start_loc_cpu[:-1] + num_prefills = int(torch.count_nonzero(query_lens_cpu[num_decodes:])) # NOTE: Ensure all metadata tensors maintain fixed memory addresses # for CUDA graph compatibility. @@ -643,6 +663,7 @@ def build( num_prefills=num_prefills, num_decode_tokens=num_decode_tokens, num_prefill_tokens=num_prefill_tokens, + max_decode_query_len=self.max_decode_query_len, tile_sched_swaonly=tile_sched[_LAYER_TYPE_SWAONLY], tile_sched_c4a=tile_sched[_LAYER_TYPE_C4A], tile_sched_c128a=tile_sched[_LAYER_TYPE_C128A], @@ -718,7 +739,9 @@ def _build_deepseek_v4_metadata( ) result["prefill_seq_lens"] = seq_lens[num_decodes:] - result["prefill_seq_lens_cpu"] = seq_lens_cpu[num_decodes:] + result["prefill_seq_lens_cpu"] = seq_lens_cpu[ + num_decodes : num_decodes + num_prefills + ] result["prefill_gather_lens"] = pfx_gather_lens result["prefill_query_lens_cpu"] = ( query_start_loc_cpu[num_decodes + 1 : num_decodes + num_prefills + 1] @@ -752,6 +775,14 @@ def _compute_swa_indices_and_lens_kernel( is_valid = tl.load(is_valid_token_ptr + token_idx) if not is_valid: tl.store(swa_lens_ptr + pid, 0) + # Clear the row so a padded token cannot gather through stale indices. + for i in range(0, window_size, TRITON_BLOCK_SIZE): + offset = i + tl.arange(0, TRITON_BLOCK_SIZE) + tl.store( + swa_indices_ptr + pid * swa_indices_stride + offset, + -1, + mask=offset < window_size, + ) return req_idx = tl.load(token_to_req_indices_ptr + token_idx) @@ -818,6 +849,14 @@ def _compute_dspark_noncausal_swa_indices_kernel( is_valid = tl.load(is_valid_token_ptr + token_idx) if not is_valid: tl.store(swa_lens_ptr + pid, 0) + # Clear the row so a padded token cannot gather through stale indices. + for i in range(0, index_width, TRITON_BLOCK_SIZE): + offset = i + tl.arange(0, TRITON_BLOCK_SIZE) + tl.store( + swa_indices_ptr + pid * swa_indices_stride + offset, + -1, + mask=offset < index_width, + ) return req_idx = tl.load(token_to_req_indices_ptr + token_idx) diff --git a/vllm/v1/worker/gpu/async_utils.py b/vllm/v1/worker/gpu/async_utils.py index ab91d9dded1d..817385e25b80 100644 --- a/vllm/v1/worker/gpu/async_utils.py +++ b/vllm/v1/worker/gpu/async_utils.py @@ -1,6 +1,9 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import contextlib +from collections.abc import Callable +from dataclasses import dataclass +from typing import TYPE_CHECKING, NamedTuple import numpy as np import torch @@ -15,6 +18,87 @@ ) from vllm.v1.worker.gpu.sample.output import SamplerOutput +if TYPE_CHECKING: + from vllm.v1.worker.gpu.input_batch import InputBatch + + +@dataclass(frozen=True) +class StepTimingSample: + forward_ms: float + drafter_ms: float + num_target_tokens: int + num_reqs: int + full_cudagraph: bool + + +class StepTimingEvents(NamedTuple): + forward_start: torch.cuda.Event + forward_end: torch.cuda.Event + drafter_start: torch.cuda.Event + drafter_end: torch.cuda.Event + + +class StepTimingCollector: + """Times startup profiling steps, which run strictly one at a time. + + Only enabled alongside a drafter, so every step it times has one. Record + calls no-op unless ``start`` armed the collector, so callers need no guards. + """ + + def __init__(self, enabled: bool): + self._events = ( + StepTimingEvents( + *( + torch.cuda.Event(enable_timing=True) + for _ in StepTimingEvents._fields + ) + ) + if enabled + else None + ) + self._armed: StepTimingEvents | None = None + self._batch = (False, 0, 0) + self.consumer: Callable[[StepTimingSample], None] | None = None + + def start(self, enabled: bool) -> None: + self._armed = self._events if enabled else None + + def record_batch(self, input_batch: "InputBatch", full_cudagraph: bool) -> None: + """Costs from different execution modes must not share a cost curve.""" + self._batch = (full_cudagraph, input_batch.num_tokens, input_batch.num_reqs) + + def forward_start(self) -> None: + if self._armed is not None: + self._armed.forward_start.record() + + def forward_end(self) -> None: + if self._armed is not None: + self._armed.forward_end.record() + + def drafter_start(self) -> None: + if self._armed is not None: + self._armed.drafter_start.record() + + def drafter_end(self) -> None: + if self._armed is not None: + self._armed.drafter_end.record() + + def finish(self) -> None: + events, self._armed = self._armed, None + if events is None: + return + events.drafter_end.synchronize() + full_cudagraph, num_target_tokens, num_reqs = self._batch + sample = StepTimingSample( + events.forward_start.elapsed_time(events.forward_end), + events.drafter_start.elapsed_time(events.drafter_end), + num_target_tokens, + num_reqs, + full_cudagraph, + ) + if self.consumer is not None: + self.consumer(sample) + class AsyncOutput(AsyncModelRunnerOutput): def __init__( diff --git a/vllm/v1/worker/gpu/block_table.py b/vllm/v1/worker/gpu/block_table.py index 90fc104dc29f..dc37b12f2aad 100644 --- a/vllm/v1/worker/gpu/block_table.py +++ b/vllm/v1/worker/gpu/block_table.py @@ -5,6 +5,7 @@ import torch from vllm.triton_utils import tl, triton +from vllm.utils.math_utils import cdiv from vllm.v1.attention.backends.utils import PAD_SLOT_ID from vllm.v1.worker.gpu.buffer_utils import ( FusedStagedWriter, @@ -175,6 +176,30 @@ def get_dummy_block_tables(self, num_reqs: int) -> tuple[torch.Tensor, ...]: block_table[:num_reqs].zero_() for block_table in self.input_block_tables ) + def set_dummy_context( + self, num_reqs: int, query_len: int, context_len: int, num_kv_blocks: int + ) -> int: + """Give each dummy request a disjoint span of KV blocks. + + Used when profiling step cost, so the profiled step reads a realistic + amount of cache instead of attending over nothing. + + Returns: + The context length actually used, clamped to what the cache holds. + """ + for bpk, block_size in zip(self.blocks_per_kv_block, self.kernel_block_sizes): + capacity = (num_kv_blocks * bpk // num_reqs) * block_size + context_len = min(context_len, capacity - query_len) + context_len = max(context_len, 0) + for block_table, block_size in zip( + self.input_block_tables, self.kernel_block_sizes + ): + num_blocks = cdiv(context_len + query_len, block_size) + block_table[:num_reqs, :num_blocks] = torch.arange( + num_reqs * num_blocks, dtype=block_table.dtype, device=self.device + ).view(num_reqs, num_blocks) + return context_len + def compute_slot_mappings( self, idx_mapping: torch.Tensor, diff --git a/vllm/v1/worker/gpu/cudagraph_utils.py b/vllm/v1/worker/gpu/cudagraph_utils.py index 3a174cba80d7..d6042cfb082b 100644 --- a/vllm/v1/worker/gpu/cudagraph_utils.py +++ b/vllm/v1/worker/gpu/cudagraph_utils.py @@ -54,6 +54,9 @@ class BatchExecutionDescriptor: num_tokens: int num_reqs: int | None # None means no request padding is needed (PIECEWISE graphs) uniform_token_count: int | None = None + # Upper bound on per-request query length. Varlen decode graphs leave + # uniform_token_count unset, so this is what keeps a prefill batch out of one. + max_query_len: int | None = None num_active_loras: int = 0 @@ -75,14 +78,21 @@ def _is_compatible( num_tokens: int, uniform_token_count: int | None, num_active_loras: int, + max_query_len: int | None, ) -> bool: # desc.uniform_token_count=None (PIECEWISE) can handle any uniform_token_count # desc.num_reqs=None means no request padding needed (PIECEWISE) + # desc.max_query_len=None means the graph does not constrain query length; a + # caller that does not track max_query_len must not match one that does return ( ( desc.uniform_token_count is None or desc.uniform_token_count == uniform_token_count ) + and ( + desc.max_query_len is None + or (max_query_len is not None and desc.max_query_len >= max_query_len) + ) and (desc.num_reqs is None or desc.num_reqs >= num_reqs) and desc.num_tokens >= num_tokens and desc.num_active_loras == num_active_loras @@ -113,6 +123,7 @@ def __init__( cudagraph_mode: CUDAGraphMode, decode_query_len: int, lora_capture_cases: list[int] | None = None, + varlen_decode: bool = False, ): self.vllm_config = vllm_config self.device = device @@ -121,6 +132,7 @@ def __init__( assert self.compilation_config is not None self.cudagraph_mode = cudagraph_mode self.decode_query_len = decode_query_len + self.varlen_decode = varlen_decode self.dp_size = vllm_config.parallel_config.data_parallel_size self.tp_size = vllm_config.parallel_config.tensor_parallel_size @@ -220,12 +232,28 @@ def _init_candidates(self) -> None: else: decode_query_lens = [self.decode_query_len] + capture_varlen_decode = ( + separate_decode_routine and bool(decode_mode) and self.varlen_decode + ) for num_tokens, num_active_loras in product( capture_sizes, self.lora_capture_cases ): + # Varlen decode graphs take any mix of 1..decode_query_len tokens per + # request, so capture on the token grid instead of rounding up to a + # uniform query length. + if capture_varlen_decode and num_tokens <= max_decode_tokens: + desc = BatchExecutionDescriptor( + cg_mode=decode_mode, + num_tokens=num_tokens, + num_reqs=min(num_tokens, self.max_num_reqs), + max_query_len=self.decode_query_len, + num_active_loras=num_active_loras, + ) + descs_by_mode[decode_mode].append(desc) + descs_by_token_lora[(num_tokens, num_active_loras)].append(desc) # Capture uniform decode specfifc graphs if required # (i.e. separate decode routine) - if separate_decode_routine and decode_mode: + elif separate_decode_routine and decode_mode and not self.varlen_decode: for decode_query_len in decode_query_lens: rounded_num_tokens = round_up(num_tokens, decode_query_len) rounded_num_reqs = rounded_num_tokens // decode_query_len @@ -362,12 +390,19 @@ def capture( compilation_counter.num_cudagraph_captured += 1 self._graphs_captured = True + def captured_token_counts(self) -> list[int]: + """Sorted token counts with a captured graph, ignoring LoRA variants.""" + return sorted( + {desc.num_tokens for desc in self.graphs if desc.num_active_loras == 0} + ) + def dispatch( self, num_reqs: int, num_tokens: int, uniform_token_count: int | None, num_active_loras: int, + max_query_len: int | None = None, ) -> BatchExecutionDescriptor: """Find matching cudagraph descriptor from priority-ordered candidates.""" @@ -381,6 +416,7 @@ def dispatch( num_tokens, uniform_token_count, effective_loras, + max_query_len, ): return desc return BatchExecutionDescriptor( @@ -429,6 +465,7 @@ def __init__( cudagraph_mode: CUDAGraphMode, decode_query_len: int, lora_capture_cases: list[int] | None = None, + varlen_decode: bool = False, ): super().__init__( vllm_config, @@ -436,6 +473,7 @@ def __init__( cudagraph_mode, decode_query_len, lora_capture_cases=lora_capture_cases, + varlen_decode=varlen_decode, ) self.hidden_states: torch.Tensor | None = None self.aux_hidden_states: list[torch.Tensor] = [] @@ -499,6 +537,7 @@ def create_forward_fn( attn_groups, kv_cache_config, full_cudagraph=desc.cg_mode == CUDAGraphMode.FULL, + max_query_len=desc.max_query_len, ) # Capture with dummy rows marked as padding. @@ -590,8 +629,11 @@ def prepare_inputs_to_capture( attn_groups: list[list[AttentionGroup]], kv_cache_config: KVCacheConfig, full_cudagraph: bool, + max_query_len: int | None = None, ) -> AttentionState: - input_batch = InputBatch.make_dummy(num_reqs, num_tokens, input_buffers) + input_batch = InputBatch.make_dummy( + num_reqs, num_tokens, input_buffers, max_query_len=max_query_len + ) input_block_tables = block_tables.get_dummy_block_tables(num_reqs) slot_mappings = block_tables.get_dummy_slot_mappings(num_tokens) slot_mappings_by_layer = build_slot_mappings_by_layer( diff --git a/vllm/v1/worker/gpu/dp_utils.py b/vllm/v1/worker/gpu/dp_utils.py index ee9b924ba13a..71601b4ae91c 100644 --- a/vllm/v1/worker/gpu/dp_utils.py +++ b/vllm/v1/worker/gpu/dp_utils.py @@ -21,6 +21,7 @@ def sync_cudagraph_and_dp_padding( uniform_token_count: int | None, dp_size: int, dp_rank: int, + max_query_len: int | None = None, num_active_loras: int = 0, ) -> tuple[BatchExecutionDescriptor, torch.Tensor | None]: """ @@ -77,6 +78,7 @@ def sync_cudagraph_and_dp_padding( synced_num_tokens, synced_uniform_token_count, num_active_loras=num_active_loras, + max_query_len=max_query_len, ) # Update num_tokens_across_dp to reflect padded size. @@ -92,6 +94,7 @@ def dispatch_cg_and_sync_dp( uniform_token_count: int | None, dp_size: int, dp_rank: int, + max_query_len: int | None = None, need_eager: bool = False, num_active_loras: int = 0, ) -> tuple[BatchExecutionDescriptor, torch.Tensor | None]: @@ -112,6 +115,7 @@ def dispatch_cg_and_sync_dp( num_tokens, uniform_token_count, num_active_loras=num_active_loras, + max_query_len=max_query_len, ) if dp_size == 1: @@ -125,5 +129,6 @@ def dispatch_cg_and_sync_dp( uniform_token_count, dp_size, dp_rank, + max_query_len=max_query_len, num_active_loras=num_active_loras, ) diff --git a/vllm/v1/worker/gpu/input_batch.py b/vllm/v1/worker/gpu/input_batch.py index a9ee523b0ac5..c1c7d4960668 100644 --- a/vllm/v1/worker/gpu/input_batch.py +++ b/vllm/v1/worker/gpu/input_batch.py @@ -99,12 +99,19 @@ class InputBatch: # [num_reqs] per-request prompt length, only populated for R-SWA. prompt_lens: torch.Tensor | None + # Longest query the batch may contain. Set when a cudagraph descriptor promises + # a query length this batch's own split does not reach, so attention metadata + # stays valid for every replay the graph serves. + max_query_len: int | None = None + @classmethod def make_dummy( cls, num_reqs: int, num_tokens: int, input_buffers: InputBuffers, + context_len: int = 0, + max_query_len: int | None = None, ) -> "InputBatch": assert 0 < num_reqs <= num_tokens device = input_buffers.device @@ -116,9 +123,12 @@ def make_dummy( expanded_local_pos = torch.zeros(num_reqs, dtype=torch.int32, device=device) # Distribute the remainder evenly so that no dummy request exceeds - # ceil(num_tokens / num_reqs) <= max_model_len tokens. + # ceil(num_tokens / num_reqs) <= max_model_len tokens. Varlen graphs + # accept any split with non-empty slots, so this shape works for them + # too; attention metadata is built from the promised max_query_len. base_tokens = num_tokens // num_reqs num_extra = num_tokens % num_reqs + assert max_query_len is None or base_tokens + (num_extra > 0) <= max_query_len num_scheduled_tokens = np.full(num_reqs, base_tokens, dtype=np.int32) if num_extra > 0: num_scheduled_tokens[-num_extra:] += 1 @@ -144,6 +154,14 @@ def make_dummy( input_ids = input_buffers.input_ids[:num_tokens].zero_() positions = input_buffers.positions[:num_tokens].zero_() + if context_len: + # Decode-like shape: each request continues after context_len + # already-computed tokens. + seq_lens += context_len + local_pos = np.arange(num_tokens, dtype=np.int64) - np.repeat( + query_start_loc_np[:-1], num_scheduled_tokens + ) + positions.copy_(torch.from_numpy(local_pos + context_len)) input_buffers.is_padding[:num_tokens].fill_(True) is_padding = input_buffers.is_padding[:num_tokens] @@ -151,8 +169,7 @@ def make_dummy( logits_indices = query_start_loc[1:] - 1 cu_num_logits = torch.arange(num_reqs + 1, device=device, dtype=torch.int32) cu_num_logits_np = np.arange(num_reqs + 1, dtype=np.int32) - # Dummy: seq_len == query_len (fresh-prefill shape). - seq_lens_cpu_upper_bound = torch.from_numpy(num_scheduled_tokens.copy()) + seq_lens_cpu_upper_bound = torch.from_numpy(num_scheduled_tokens + context_len) return cls( req_ids=req_ids, num_reqs=num_reqs, @@ -171,9 +188,11 @@ def make_dummy( seq_lens=seq_lens, seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound, dcp_local_seq_lens=None, - num_computed_tokens_np=np.zeros(num_reqs, dtype=np.int32), + num_computed_tokens_np=np.full(num_reqs, context_len, dtype=np.int32), prefill_len_np=np.zeros(num_reqs, dtype=np.int32), - num_computed_prefill_tokens_np=np.zeros(num_reqs, dtype=np.int32), + num_computed_prefill_tokens_np=np.full( + num_reqs, context_len, dtype=np.int32 + ), is_prefilling_np=np.zeros(num_reqs, dtype=np.bool_), max_seq_len_np=None, input_ids=input_ids, @@ -184,6 +203,7 @@ def make_dummy( cu_num_logits_np=cu_num_logits_np, has_structured_output_reqs=False, prompt_lens=None, + max_query_len=max_query_len, ) diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 0a1b726c0a0b..f4d76d010eb8 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -68,7 +68,11 @@ from vllm.v1.worker.block_table import get_block_table_width from vllm.v1.worker.cp_utils import check_attention_cp_compatibility from vllm.v1.worker.gpu import pcp_manager as pcp -from vllm.v1.worker.gpu.async_utils import AsyncOutput, AsyncPoolingOutput +from vllm.v1.worker.gpu.async_utils import ( + AsyncOutput, + AsyncPoolingOutput, + StepTimingCollector, +) from vllm.v1.worker.gpu.attn_utils import ( build_slot_mappings_by_layer, get_kv_cache_spec, @@ -120,6 +124,9 @@ from vllm.v1.worker.gpu.sample.sampler import Sampler from vllm.v1.worker.gpu.shutdown import free_before_shutdown from vllm.v1.worker.gpu.spec_decode import init_speculator +from vllm.v1.worker.gpu.spec_decode.adaptive_verification import ( + AdaptiveVerificationManager, +) from vllm.v1.worker.gpu.spec_decode.eagle.eagle3_utils import ( set_eagle3_aux_hidden_state_layers, ) @@ -245,6 +252,14 @@ def __init__(self, vllm_config: VllmConfig, device: torch.device): and self.speculative_config.use_multi_module_mtp() else 1 ) + + self.step_timing = StepTimingCollector( + self.is_last_pp_rank + and not self.is_pooling_model + and self.speculative_config is not None + and self.speculative_config.use_confidence_based_verification + ) + # General request states. self.req_states = RequestState( max_num_reqs=self.max_num_reqs, @@ -255,6 +270,7 @@ def __init__(self, vllm_config: VllmConfig, device: torch.device): device=self.device, num_prefill_lookahead=num_prefill_lookahead, ) + self.adaptive_verification: AdaptiveVerificationManager | None = None self.input_buffers = InputBuffers( max_num_reqs=self.max_num_reqs, max_num_tokens=self.max_num_tokens, @@ -398,6 +414,7 @@ def load_model(self, load_dummy_weights: bool = False, *args, **kwargs) -> None: max_num_logits=self.max_num_reqs * self.decode_query_len, vocab_size=self.vocab_size, device=self.device, + mask_stride=self.decode_query_len, ) if self.is_pooling_model and self.is_last_pp_rank: @@ -499,6 +516,17 @@ def initialize_kv_cache(self, kv_cache_config: KVCacheConfig) -> None: attn_cg_support = attn_cg_support.narrow( *self.model_state.get_additional_cg_support() ) + if ( + self.speculative_config is not None + and self.speculative_config.use_confidence_based_verification + ): + self.adaptive_verification = AdaptiveVerificationManager( + self.req_states, + self.input_buffers.query_start_loc, + self.model_state.num_new_sampled_tokens_per_step, + self.speculative_config.dspark_confidence_ema_alpha, + ) + self.step_timing.consumer = self.adaptive_verification.consume_step_timing self.block_tables = BlockTables( block_sizes=block_sizes, max_num_reqs=self.max_num_reqs, @@ -520,6 +548,8 @@ def initialize_kv_cache(self, kv_cache_config: KVCacheConfig) -> None: initialize_mamba_ssu_backend( self.vllm_config.mamba_config, self.kv_cache_config ) + if self.adaptive_verification is not None: + self.compilation_config.cudagraph_mode = CUDAGraphMode.FULL_AND_PIECEWISE cudagraph_mode = self.compilation_config.resolve_cudagraph_mode_and_sizes( attn_cg_support.min_cg_support, attn_cg_support.min_cg_attn_backend, @@ -528,6 +558,7 @@ def initialize_kv_cache(self, kv_cache_config: KVCacheConfig) -> None: tensor_parallel_size=self.parallel_config.tensor_parallel_size, kv_cache_config=self.kv_cache_config, max_num_reqs=self.max_num_reqs, + varlen_decode=self.adaptive_verification is not None, ) self.cudagraph_manager = ModelCudaGraphManager( self.vllm_config, @@ -535,6 +566,7 @@ def initialize_kv_cache(self, kv_cache_config: KVCacheConfig) -> None: cudagraph_mode, decode_query_len=self.decode_query_len, lora_capture_cases=self.lora_capture_cases, + varlen_decode=self.adaptive_verification is not None, ) check_attention_cp_compatibility(self.vllm_config) if isinstance(self.speculator, DraftModelSpeculator): @@ -582,8 +614,10 @@ def _dummy_run( *args, skip_attn: bool = False, uniform_decode: bool = False, + context_len: int = 0, skip_eplb: bool = False, is_profile: bool = False, + timing_enabled: bool = False, **kwargs, ) -> tuple[torch.Tensor | None, torch.Tensor | None]: if self.is_encoder_only: @@ -643,6 +677,8 @@ def _dummy_run( dummy_run=True, skip_attn_for_dummy_run=skip_attn, is_profile=is_profile, + context_len=context_len, + timing_enabled=timing_enabled, ) self.kv_connector.set_disabled(False) @@ -658,9 +694,12 @@ def _dummy_run( aux_hidden_states = self.execute_model_state.aux_hidden_states self.execute_model_state = None + self.step_timing.forward_end() + # dummy run the eagle speculator's propose to ensure DP/EP sync. if self.speculator is not None: assert self.sampler is not None + self.step_timing.drafter_start() mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None if self.speculator.supports_mm_inputs: mm_inputs = ( @@ -701,6 +740,9 @@ def _dummy_run( mm_inputs=mm_inputs, is_profile=is_profile, ) + self.step_timing.drafter_end() + + self.step_timing.finish() assert hidden_states is not None # Last PP rank always has hidden_states sample_hidden_states = hidden_states[input_batch.logits_indices] @@ -818,6 +860,12 @@ def capture_model(self) -> int: ) if self.speculator is not None: self.speculator.capture() + if self.adaptive_verification is not None: + for batch in self.adaptive_verification.batches_to_profile( + self.cudagraph_manager.captured_token_counts() + ): + self._dummy_run(**batch) + self.adaptive_verification.set_initial_cost_curves() end_time = time.perf_counter() end_free_gpu_memory = torch.accelerator.get_memory_info()[0] @@ -894,6 +942,8 @@ def add_requests(self, scheduler_output: SchedulerOutput) -> None: max_tokens=sampling_params.max_tokens if sampling_params else 1, # type: ignore[arg-type] ) req_index = self.req_states.req_id_to_index[req_id] + if self.adaptive_verification is not None: + self.adaptive_verification.add_request(req_index) if self.pooling_runner is not None: assert new_req_data.pooling_params is not None @@ -966,9 +1016,13 @@ def update_requests(self, scheduler_output: SchedulerOutput) -> None: ) def prepare_inputs( - self, scheduler_output: SchedulerOutput, batch_desc: BatchExecutionDescriptor + self, + scheduler_output: SchedulerOutput, + batch_desc: BatchExecutionDescriptor, + # May be less than scheduler_output.total_num_scheduled_tokens: + # adaptive verification trims the draft budget before running. + num_tokens: int, ) -> InputBatch: - num_tokens = scheduler_output.total_num_scheduled_tokens num_tokens_after_padding = batch_desc.num_tokens assert num_tokens > 0 if envs.VLLM_MOE_SKIP_PADDING: @@ -982,7 +1036,10 @@ def prepare_inputs( num_reqs = len(num_tokens_per_req) # batch_idx -> req_id - req_ids = sort_batch_req_ids(num_tokens_per_req, self.decode_query_len) + draft_tokens = scheduler_output.scheduled_spec_decode_tokens + req_ids = sort_batch_req_ids( + num_tokens_per_req, draft_tokens, self.decode_query_len + ) numtoks_iter = map(num_tokens_per_req.get, req_ids) num_scheduled_tokens = np.fromiter(numtoks_iter, dtype=np.int32, count=num_reqs) @@ -991,7 +1048,6 @@ def prepare_inputs( idx_mapping = async_copy_to_gpu(idx_mapping_np, device=self.device) # Get the number of draft tokens for each request. - draft_tokens = scheduler_output.scheduled_spec_decode_tokens num_draft_tokens_per_req = None if not draft_tokens: # No draft token scheduled (common case). @@ -1020,9 +1076,13 @@ def prepare_inputs( np.cumsum(num_logits, out=cu_num_logits_np[1:]) cu_num_logits = async_copy_to_gpu(cu_num_logits_np, device=self.device) - max_expand_len = self.decode_query_len - expanded_idx_mapping, expanded_local_pos = expand_idx_mapping( - idx_mapping, total_num_logits, cu_num_logits, max_expand_len + adaptive_verification = ( + self.adaptive_verification if num_draft_tokens_per_req is not None else None + ) + if adaptive_verification is not None: + num_scheduled_tokens = adaptive_verification.compact_batch( + num_draft_tokens_per_req, + num_scheduled_tokens, ) # Get query_start_loc. @@ -1034,9 +1094,22 @@ def prepare_inputs( # Pad for full CUDA graph mode. # Some attention backends like FA3 require query_start_loc to be non-decreasing. query_start_loc_np[num_reqs + 1 :] = num_tokens - async_copy_to_gpu(query_start_loc_np, out=self.input_buffers.query_start_loc) + query_start_loc = self.input_buffers.query_start_loc + async_copy_to_gpu(query_start_loc_np, out=query_start_loc) + if adaptive_verification is not None: + cu_num_logits, query_start_loc, total_num_draft_tokens = ( + adaptive_verification.reallocate_drafts(req_ids, idx_mapping) + ) + total_num_logits = num_reqs * num_bonus_tokens + total_num_draft_tokens + if draft_tokens: + expanded_idx_mapping, expanded_local_pos = expand_idx_mapping( + idx_mapping, + total_num_logits, + cu_num_logits, + self.decode_query_len, + ) query_start_loc_np = query_start_loc_np[: num_reqs_padded + 1] - query_start_loc = self.input_buffers.query_start_loc[: num_reqs_padded + 1] + query_start_loc = query_start_loc[: num_reqs_padded + 1] prefill_len_np = self.req_states.prefill_len.np[idx_mapping_np] computed_prefill_tokens_np = self.req_states.num_computed_prefill_tokens num_computed_prefill_tokens_np = computed_prefill_tokens_np[idx_mapping_np] @@ -1249,6 +1322,8 @@ def execute_model( dummy_run: bool = False, skip_attn_for_dummy_run: bool = False, is_profile: bool = False, + context_len: int = 0, + timing_enabled: bool = False, ) -> ModelRunnerOutput | IntermediateTensors | None: if not dummy_run: # Update the request states. @@ -1283,6 +1358,19 @@ def execute_model( # cross-attention cache with dynamic encoder outputs. skip_compiled = True + if ( + self.adaptive_verification is not None + and scheduler_output.scheduled_spec_decode_tokens + and not dummy_run + ): + num_toks = self.adaptive_verification.get_num_tokens( + scheduler_output.num_scheduled_tokens, + scheduler_output.scheduled_spec_decode_tokens, + scheduler_output.has_structured_output_requests, + ) + # Trimming drafts makes the batch non-uniform. + uniform_tok_count = None + batch_desc, num_tokens_across_dp = dispatch_cg_and_sync_dp( self.cudagraph_manager, num_reqs, @@ -1290,6 +1378,7 @@ def execute_model( uniform_tok_count, self.dp_size, self.dp_rank, + max_query_len=max_query_len, need_eager=is_profile or skip_compiled, num_active_loras=num_active_loras, ) @@ -1299,10 +1388,12 @@ def execute_model( empty_output = self.kv_connector.no_forward(scheduler_output) return empty_output + self.step_timing.start(timing_enabled) + if not dummy_run: # Common case. # Prepare all the inputs and copy to the input buffers. - input_batch = self.prepare_inputs(scheduler_output, batch_desc) + input_batch = self.prepare_inputs(scheduler_output, batch_desc, num_toks) block_tables, slot_mappings = self.prepare_attn(input_batch) # Mamba "align" pre-copy: migrate recurrent state across block # boundaries before the forward. Runs only on real batches, and @@ -1325,10 +1416,23 @@ def execute_model( self._set_active_loras(*lora_inputs) else: # No actual tokens to run. A dummy run for DP or memory profiling. + dummy_num_reqs = batch_desc.num_reqs or num_reqs + if context_len and self.kernel_block_sizes: + query_len = batch_desc.max_query_len or cdiv( + batch_desc.num_tokens, dummy_num_reqs + ) + context_len = self.block_tables.set_dummy_context( + dummy_num_reqs, + query_len, + min(context_len, self.max_model_len - query_len), + self.kv_cache_config.num_blocks, + ) input_batch = InputBatch.make_dummy( - batch_desc.num_reqs or num_reqs, + dummy_num_reqs, batch_desc.num_tokens, self.input_buffers, + context_len=context_len, + max_query_len=batch_desc.max_query_len, ) if not skip_attn_for_dummy_run: block_tables, slot_mappings = self.prepare_dummy_attn(input_batch) @@ -1427,6 +1531,11 @@ def execute_model( # Update the EPLB meta. self.eplb.prepare_forward(self.model_config, input_batch.num_tokens) + self.step_timing.record_batch( + input_batch, batch_desc.cg_mode == CUDAGraphMode.FULL + ) + self.step_timing.forward_start() + # Run model. if batch_desc.cg_mode == CUDAGraphMode.FULL: # Use explicit cudagraph replay for FULL mode. @@ -1637,6 +1746,11 @@ def sample_tokens( mm_inputs=mm_inputs, ) self.req_states.draft_tokens[input_batch.idx_mapping] = draft_tokens + if self.adaptive_verification is not None: + self.adaptive_verification.record_confidences( + self.speculator.draft_token_confidence_probs, + input_batch, + ) if self.num_speculative_steps > 0: # Spec-decode and diffusion LLMs both use draft tokens but the latter does @@ -1769,9 +1883,15 @@ class ExecuteModelState(NamedTuple): def sort_batch_req_ids( - num_tokens_per_req: dict[str, int], decode_query_len: int + num_tokens_per_req: dict[str, int], + draft_tokens: dict[str, list[int]], + decode_query_len: int, ) -> list[str]: - # Order decode -> short_extend -> prefill; split_decodes_and_prefills - # relies on uniform decodes (query_len == decode_query_len) leading. - key = lambda r: ((num := num_tokens_per_req[r]) != decode_query_len, num) + # Order verification/decode -> short_extend -> prefill; + # split_decodes_and_prefills relies on decode-like requests leading. + key = lambda r: ( + not draft_tokens.get(r), + (num := num_tokens_per_req[r]) != decode_query_len, + num, + ) return sorted(num_tokens_per_req, key=key) diff --git a/vllm/v1/worker/gpu/model_states/default.py b/vllm/v1/worker/gpu/model_states/default.py index c4c1d6969db5..b4e965e5b756 100644 --- a/vllm/v1/worker/gpu/model_states/default.py +++ b/vllm/v1/worker/gpu/model_states/default.py @@ -151,7 +151,9 @@ def prepare_attn( input_batch.query_start_loc_np[: num_reqs + 1] ) query_start_loc_gpu = input_batch.query_start_loc[: num_reqs + 1] - max_query_len = input_batch.num_scheduled_tokens.max().item() + max_query_len = input_batch.max_query_len + if max_query_len is None: + max_query_len = input_batch.num_scheduled_tokens.max().item() seq_lens_cpu_upper_bound = input_batch.seq_lens_cpu_upper_bound if for_capture: # Capture with worst-case max_seq_len so the graph is valid at any replay. diff --git a/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py b/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py new file mode 100644 index 000000000000..759ccaa83a36 --- /dev/null +++ b/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py @@ -0,0 +1,475 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Adaptive verification for DSpark speculative decoding.""" + +from collections import defaultdict +from collections.abc import Iterable, Iterator +from typing import TYPE_CHECKING + +import numpy as np +import torch + +from vllm.distributed.parallel_state import get_tp_group +from vllm.logger import init_logger +from vllm.triton_utils import tl, triton +from vllm.utils.gpu_sync_debug import gpu_sync_allowed +from vllm.v1.utils import CpuGpuBuffer +from vllm.v1.worker.gpu.async_utils import StepTimingSample, stream +from vllm.v1.worker.gpu.buffer_utils import async_copy_to_gpu + +logger = init_logger(__name__) +_FIXED_OVERHEAD_MS = 1.0 +_PROFILE_REPLAYS = 5 +# KV context each profiling request pretends to carry, so the profiled step reads +# a realistic amount of cache rather than attending over nothing. +_PROFILE_CONTEXT_LEN = 8192 +# Median-filter width for cost curves; 5 also survives two adjacent outliers. +_OUTLIER_FILTER_WIDTH = 5 + +if TYPE_CHECKING: + from vllm.v1.worker.gpu.input_batch import InputBatch + from vllm.v1.worker.gpu.states import RequestState + + +@triton.jit +def _keyed_product(left_value, left_key, right_value, right_key): + value = tl.where(left_key == right_key, left_value * right_value, right_value) + return value, right_key + + +@triton.jit(do_not_specialize=["num_reqs", "draft_token_budget"]) +def _assign_draft_token_budget_kernel( + confidence_probs_ptr, + confidence_probs_stride, + idx_mapping_ptr, + capacities_ptr, + num_reqs, + draft_token_budget, + NUM_STEPS: tl.constexpr, + BLOCK_SIZE: tl.constexpr, +): + candidate_idx = tl.arange(0, BLOCK_SIZE) + req_idx = candidate_idx // NUM_STEPS + step = candidate_idx % NUM_STEPS + valid_candidate = req_idx < num_reqs + req_state_idx = tl.load(idx_mapping_ptr + req_idx, mask=valid_candidate, other=0) + probability = tl.load( + confidence_probs_ptr + req_state_idx * confidence_probs_stride + step, + mask=valid_candidate, + other=0.0, + ) + survival, _ = tl.associative_scan( + (probability, req_idx), + axis=0, + combine_fn=_keyed_product, + ) + num_valid = tl.load( + capacities_ptr + req_idx, + mask=valid_candidate, + other=0, + ) + survival = tl.where(valid_candidate & (step < num_valid), survival, -float("inf")) + + min_int32: tl.constexpr = -2147483648 + score_bits = survival.to(tl.int32, bitcast=True) + sort_key = tl.where( + score_bits >> 31 == 0, + score_bits ^ -1, + score_bits ^ min_int32, + ) + packed = ((sort_key.to(tl.int64) & 0xFFFFFFFF) << 32) | candidate_idx.to(tl.int64) + admitted_idx = (tl.sort(packed) & 0xFFFFFFFF).to(tl.int32) + + tl.store(capacities_ptr + candidate_idx, 0, mask=candidate_idx < num_reqs) + tl.debug_barrier() + tl.atomic_add( + capacities_ptr + admitted_idx // NUM_STEPS, + 1, + mask=candidate_idx < draft_token_budget, + ) + + +def build_cost_tables_from_curves( + draft_curve: list[tuple[int, float]], + verify_curve: list[tuple[int, float]], + max_num_reqs: int, + max_batch_tokens: int, + cudagraph_limit: int = 0, +) -> tuple[np.ndarray, np.ndarray]: + """Build cost tables: graph-padded below the capture limit, smooth above. + + Args: + cudagraph_limit: Largest cudagraph-captured size. At or below it, + execution pads up to the next captured size, so cost is a step + function. Above it there is no padding, so cost is continuous. + """ + + def build_table(limit: int, curve: list[tuple[int, float]]) -> np.ndarray: + xs, ys = np.asarray(curve, dtype=np.float64).T + # Drop isolated outliers before enforcing monotonicity: the cumulative + # max propagates a single bad point across every larger size, so an + # unfiltered spike corrupts the whole tail. A median filter is exact on + # monotone data, so genuine steps survive untouched. + if len(ys) >= _OUTLIER_FILTER_WIDTH: + k = _OUTLIER_FILTER_WIDTH // 2 + padded = np.pad(ys, k, mode="edge") + ys = np.array( + [ + np.median(padded[i : i + _OUTLIER_FILTER_WIDTH]) + for i in range(len(ys)) + ] + ) + ys = np.maximum.accumulate(ys) + values = np.arange(limit + 1) + # Execution pads to the next captured size, so cost is a step + # function of the padded size: smooth interpolation would invent + # marginal per-token costs that don't exist within a pad bucket. + idx = np.searchsorted(xs, values, side="left") + result = ys[np.minimum(idx, len(xs) - 1)] + # Past the capture limit nothing pads, so cost really is continuous in + # size; snapping to the next profiled point overestimates badly when + # the profiled points are far apart. Interpolate only between points + # that are themselves past the limit: crossing the limit loses + # cudagraphs entirely, which is a genuine discontinuity, so the first + # point above it must not be blended with the last one below. + if cudagraph_limit: + smooth = values > cudagraph_limit + above = xs > cudagraph_limit + if smooth.any() and above.any(): + result[smooth] = np.interp(values[smooth], xs[above], ys[above]) + if len(xs) > 1: + after = values > xs[-1] + slope = (ys[-1] - ys[-2]) / (xs[-1] - xs[-2]) + result[after] = ys[-1] + slope * (values[after] - xs[-1]) + return result + + draft_table = np.maximum(build_table(max_num_reqs, draft_curve), 0.0) + verify_table = np.maximum(build_table(max_batch_tokens, verify_curve), 1e-6) + verify_table += _FIXED_OVERHEAD_MS + return draft_table, verify_table + + +class AdaptiveVerificationManager: + def __init__( + self, + req_states: "RequestState", + query_start_loc: torch.Tensor, + num_bonus_tokens: int, + confidence_ema_alpha: float, + ): + self.req_states = req_states + self.num_speculative_steps = req_states.num_speculative_steps + device = req_states.device + self._copy_stream = torch.cuda.Stream(device) + + self.num_bonus_tokens = num_bonus_tokens + self.confidence_ema_alpha = confidence_ema_alpha + self.query_start_loc = query_start_loc + self.cost_tables: tuple[np.ndarray, np.ndarray] | None = None + # Largest cudagraph-captured token count; above it nothing pads. + self._cudagraph_limit = 0 + self._profile_samples: list[StepTimingSample] = [] + self._batch_budget: tuple[dict[str, int], dict[str, int], int] | None = None + max_num_reqs = req_states.max_num_reqs + self._confidence_ema = torch.empty( + (max_num_reqs, self.num_speculative_steps), + dtype=torch.float32, + device=device, + ) + self._batch_draft_capacity = torch.empty( + max_num_reqs, dtype=torch.int32, device=device + ) + self._num_non_draft_tokens = torch.empty_like(query_start_loc[:-1]) + self._cu_num_logits = torch.empty_like(query_start_loc) + + # Two D2H slots preserve stale inputs for budget selection. + self._stale_confidences = [ + CpuGpuBuffer( + max_num_reqs, + self.num_speculative_steps, + dtype=torch.float32, + device=device, + ) + for _ in range(2) + ] + self._copy_events = [torch.cuda.Event(blocking=True) for _ in range(2)] + self._pending_resets: list[int] = [] + self._stale_idx = 0 + for slot in self._stale_confidences: + slot.np.fill(1.0) + + def consume_step_timing(self, sample: StepTimingSample) -> None: + """Collect startup profiling timings; only profiling steps are timed.""" + self._profile_samples.append(sample) + + def add_request(self, req_idx: int) -> None: + self._stale_confidences[self._stale_idx].np[req_idx].fill(1.0) + self._pending_resets.append(req_idx) + self._confidence_ema[req_idx].fill_(1.0) + + def batches_to_profile(self, capture_sizes: list[int]) -> Iterator[dict[str, int]]: + """Dummy-run kwargs whose step timings seed the cost tables. + + Timings flow back through consume_step_timing; call + set_initial_cost_curves once every batch has run.""" + max_num_tokens = self.req_states.max_num_batched_tokens + size = self._cudagraph_limit = capture_sizes[-1] if capture_sizes else 0 + # Also profile beyond the capture limit: real steps run there + # (piecewise/eager) and linear extrapolation badly underestimates + # them. These runs double as JIT warmup for the piecewise shapes. + tail_sizes: set[int] = set() + if size: + tail_sizes.add(min(size + size // 2, max_num_tokens)) + while size < max_num_tokens: + size = min(size * 2, max_num_tokens) + tail_sizes.add(size) + tail_sizes -= set(capture_sizes) + for num_tokens in capture_sizes + sorted(tail_sizes): + for _ in range(_PROFILE_REPLAYS): + yield { + "num_tokens": num_tokens, + "context_len": _PROFILE_CONTEXT_LEN, + "timing_enabled": True, + } + + def set_initial_cost_curves(self) -> None: + samples, self._profile_samples = self._profile_samples, [] + + def median_curve( + points: Iterable[tuple[int, float]], + ) -> list[tuple[int, float]]: + grouped: defaultdict[int, list[float]] = defaultdict(list) + for key, value in points: + grouped[key].append(value) + return [(k, float(np.median(v))) for k, v in sorted(grouped.items())] + + # Draft curve: eager-target steps inflate drafter timings (the CPU is + # still launching kernels, opening gaps between the drafter's events), + # and request counts — unlike token counts — collide across execution + # modes, so only graph-replay samples may price the draft curve. + draft_curve = median_curve( + (s.num_reqs, s.drafter_ms) for s in samples if s.full_cudagraph + ) + verify_curve = median_curve( + (s.num_target_tokens, s.forward_ms) for s in samples + ) + self.set_cost_curves(draft_curve, verify_curve) + + def set_cost_curves( + self, + draft_curve: list[tuple[int, float]], + verify_curve: list[tuple[int, float]], + ) -> None: + draft_curve, verify_curve = get_tp_group().broadcast_object( + (draft_curve, verify_curve), src=0 + ) + if not draft_curve or not verify_curve: + logger.warning_once("DSpark could not profile step costs.") + return + self.cost_tables = build_cost_tables_from_curves( + draft_curve, + verify_curve, + self.req_states.max_num_reqs, + self.req_states.max_num_batched_tokens, + self._cudagraph_limit, + ) + logger.debug("DSpark cost tables: %s", self.cost_tables) + + def record_confidences( + self, + confidence_probs: torch.Tensor, + input_batch: "InputBatch", + ) -> None: + """Fold this step's confidences into the EMA and start copying them to + the CPU, where a later step's budget reads them as stale inputs.""" + num_reqs = input_batch.num_reqs + ready_idx = self._stale_idx ^ 1 + with gpu_sync_allowed(): + self._copy_events[ready_idx].synchronize() + if self._pending_resets: + self._stale_confidences[ready_idx].np[self._pending_resets] = 1.0 + self._pending_resets.clear() + # Last step's copy has landed: budgets read it, this step overwrites the + # slot they were reading before. + self._stale_idx, write_idx = ready_idx, self._stale_idx + + # Fold this step's scores into the per-request EMA in place. New + # requests blend with the optimistic 1.0 seed, which washes out of the + # average within a couple of steps. + idx_mapping = input_batch.idx_mapping + ema = self._confidence_ema[idx_mapping] + ema.lerp_(confidence_probs[:num_reqs], self.confidence_ema_alpha) + self._confidence_ema[idx_mapping] = ema + write_slot = self._stale_confidences[write_idx] + write_slot.gpu.copy_(self._confidence_ema) + + current_stream = torch.cuda.current_stream(self.req_states.device) + self._copy_stream.wait_stream(current_stream) + with stream(self._copy_stream, current_stream): + write_slot.copy_to_cpu() + self._copy_events[write_idx].record() + + def get_num_tokens( + self, + num_tokens_per_req: dict[str, int], + draft_tokens: dict[str, list[int]], + has_structured_output: bool = False, + ) -> int: + """Token count once the draft budget is trimmed to fit. + + Stashes the chosen budget in ``_batch_budget`` for the compaction and + reallocation that follow in the same step. + """ + assert self.cost_tables is not None + req_ids = list(num_tokens_per_req) + num_reqs = len(req_ids) + scheduled_tokens = np.fromiter( + num_tokens_per_req.values(), dtype=np.int32, count=num_reqs + ) + scheduled_drafts = np.fromiter( + (len(draft_tokens.get(req_id, ())) for req_id in req_ids), + dtype=np.int32, + count=num_reqs, + ) + valid_drafts = scheduled_drafts + if has_structured_output: + valid_drafts = np.fromiter( + ( + sum(token >= 0 for token in draft_tokens.get(req_id, ())) + for req_id in req_ids + ), + dtype=np.int32, + count=len(req_ids), + ) + num_non_draft_tokens = scheduled_tokens - scheduled_drafts + slots = np.fromiter( + (self.req_states.req_id_to_index[req_id] for req_id in req_ids), + dtype=np.int32, + count=len(req_ids), + ) + stale_confidences = self._stale_confidences[self._stale_idx].np[slots] + survival_probability = np.cumprod(stale_confidences.astype(np.float64), axis=1) + steps = np.arange(self.num_speculative_steps) + valid = steps[None, :] < valid_drafts[:, None] + scores = np.sort(survival_probability[valid])[::-1] + num_non_draft_tokens_total = int(num_non_draft_tokens.sum()) + max_draft_budget = int(valid_drafts.sum()) + draft_cost_ms, verify_cost_ms = self.cost_tables + num_sampling_requests = np.count_nonzero( + self.req_states.num_computed_tokens_np[slots] + num_non_draft_tokens + >= self.req_states.prefill_len.np[slots] + ) + num_tokens_to_estimated_accepted_tokens = np.concatenate( + ([num_sampling_requests], num_sampling_requests + np.cumsum(scores)) + ) + costs = ( + draft_cost_ms[len(req_ids)] + + verify_cost_ms[ + num_non_draft_tokens_total : num_non_draft_tokens_total + + max_draft_budget + + 1 + ] + ) + valid_drafts_per_req = { + req_id: int(num_valid_drafts) + for req_id, num_valid_drafts in zip(req_ids, valid_drafts, strict=True) + } + num_non_draft_tokens_per_req = { + req_id: int(num_tokens) + for req_id, num_tokens in zip(req_ids, num_non_draft_tokens, strict=True) + } + draft_budget = int(np.argmax(num_tokens_to_estimated_accepted_tokens / costs)) + self._batch_budget = ( + valid_drafts_per_req, + num_non_draft_tokens_per_req, + draft_budget, + ) + return sum(num_non_draft_tokens_per_req.values()) + draft_budget + + def compact_batch( + self, + num_draft_tokens_per_req: np.ndarray, + num_scheduled_tokens: np.ndarray, + ) -> np.ndarray: + batch_budget = self._batch_budget + assert batch_budget is not None + _, _, draft_budget = batch_budget + num_drafts = int(num_draft_tokens_per_req.sum()) + if draft_budget == num_drafts: + return num_scheduled_tokens + + num_non_draft_tokens = num_scheduled_tokens - num_draft_tokens_per_req + is_verification_request = num_draft_tokens_per_req > 0 + num_verification_reqs = int(is_verification_request.sum()) + # sort_batch_req_ids keeps verification requests at the front. + assert np.all(is_verification_request[:num_verification_reqs]) + # for the CPU side buffer we distribute draft tokens evenly + draft_lens_cpu = np.zeros_like(num_non_draft_tokens) + draft_lens_cpu[:num_verification_reqs] = draft_budget // num_verification_reqs + draft_lens_cpu[: draft_budget % num_verification_reqs] += 1 + return num_non_draft_tokens + draft_lens_cpu + + def reallocate_drafts( + self, req_ids: list[str], idx_mapping: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor, int]: + batch_budget, self._batch_budget = self._batch_budget, None + assert batch_budget is not None + valid_drafts_per_req, num_non_draft_tokens_per_req, draft_budget = batch_budget + num_reqs = idx_mapping.shape[0] + valid_drafts = np.fromiter( + (valid_drafts_per_req[req_id] for req_id in req_ids), + dtype=np.int32, + count=num_reqs, + ) + num_non_draft_tokens = np.fromiter( + (num_non_draft_tokens_per_req[req_id] for req_id in req_ids), + dtype=np.int32, + count=num_reqs, + ) + num_tokens = int(num_non_draft_tokens.sum()) + draft_budget + + # Rank draft slots by survival probability and admit the best prefix. + capacities = self._batch_draft_capacity[:num_reqs] + if draft_budget == 0: + capacities.zero_() + else: + async_copy_to_gpu(valid_drafts, out=capacities) + if draft_budget != int(valid_drafts.sum()): + block_size = triton.next_power_of_2( + num_reqs * self.num_speculative_steps + ) + _assign_draft_token_budget_kernel[(1,)]( + self._confidence_ema, + self._confidence_ema.stride(0), + idx_mapping, + capacities, + num_reqs, + draft_budget, + NUM_STEPS=self.num_speculative_steps, + BLOCK_SIZE=block_size, + num_warps=4 if block_size <= 256 else 8, + ) + + num_non_draft_tokens_gpu = self._num_non_draft_tokens[:num_reqs] + async_copy_to_gpu( + num_non_draft_tokens, + out=num_non_draft_tokens_gpu, + ) + self._cu_num_logits[:1].zero_() + torch.cumsum( + capacities + self.num_bonus_tokens, + dim=0, + out=self._cu_num_logits[1 : num_reqs + 1], + ) + self.query_start_loc[:1].zero_() + torch.cumsum( + capacities + num_non_draft_tokens_gpu, + dim=0, + out=self.query_start_loc[1 : num_reqs + 1], + ) + self.query_start_loc[num_reqs + 1 :].fill_(num_tokens) + return ( + self._cu_num_logits[: num_reqs + 1], + self.query_start_loc, + draft_budget, + ) diff --git a/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py b/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py index 333648637bd7..2c160a75a37a 100644 --- a/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py @@ -135,11 +135,10 @@ def init_cudagraph_manager(self, cudagraph_mode: CUDAGraphMode) -> None: def capture(self) -> None: logger.info("Capturing model for %s speculator...", self._speculator_name) - # Reset sampling indices to zero to prevent stale values from prior - # dummy runs from being baked into the captured graph. + # Padded sample rows must not scatter into a live request during capture. self.sample_indices.zero_() self.sample_pos.zero_() - self.sample_idx_mapping.zero_() + self.sample_idx_mapping.fill_(-1) assert self.query_cudagraph_manager is not None self.query_cudagraph_manager.capture( self._generate_draft, @@ -521,6 +520,7 @@ def _prepare_dflash_inputs_kernel( num_rejected = tl.load(num_rejected_ptr + req_idx) valid_ctx_end = ctx_end - num_rejected + num_valid_ctx = valid_ctx_end - ctx_start num_sampled = tl.load(num_sampled_ptr + req_idx) if num_sampled > 0: @@ -534,22 +534,27 @@ def _prepare_dflash_inputs_kernel( j = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) is_ctx = j < num_ctx - is_query = (j >= num_ctx) & (j < num_ctx + num_query_per_req) - query_off = j - num_ctx + is_valid_ctx = j < num_valid_ctx + is_query = (j >= num_valid_ctx) & (j < num_valid_ctx + num_query_per_req) + query_off = j - num_valid_ctx # --- Context positions / slots --- ctx_pos_idx = ctx_start + tl.where(is_ctx, j, 0) - ctx_pos = tl.load(target_positions_ptr + ctx_pos_idx, mask=is_ctx, other=0) + ctx_pos = tl.load(target_positions_ptr + ctx_pos_idx, mask=is_valid_ctx, other=0) ctx_block_num = ctx_pos // block_size ctx_block_num = tl.minimum(ctx_block_num, block_table_stride - 1) ctx_block_id = tl.load( block_table_ptr + req_idx * block_table_stride + ctx_block_num, - mask=is_ctx, + mask=is_valid_ctx, other=0, ).to(tl.int64) ctx_slot = ctx_block_id * block_size + (ctx_pos % block_size) tl.store(out_context_positions_ptr + ctx_start + j, ctx_pos, mask=is_ctx) - tl.store(out_context_slot_mapping_ptr + ctx_start + j, ctx_slot, mask=is_ctx) + tl.store( + out_context_slot_mapping_ptr + ctx_start + j, + tl.where(is_valid_ctx, ctx_slot, PAD_SLOT_ID), + mask=is_ctx, + ) # --- Query positions / input_ids / slots --- query_pos = last_valid_pos + 1 + query_off diff --git a/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py b/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py index a3207cd3dad5..a23b43f25ccc 100644 --- a/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py @@ -76,6 +76,13 @@ def __init__(self, vllm_config: VllmConfig, device: torch.device): self.draft_model_config.hf_config, "dspark_draft_topk", None ) + self.draft_token_confidence_probs = torch.empty_like( + self.draft_tokens, dtype=torch.float32 + ) + self.use_confidence_based_verification = ( + self.speculative_config.use_confidence_based_verification + ) + def load_draft_model( self, target_model: torch.nn.Module, @@ -149,6 +156,7 @@ def _sample_sequential(self, num_reqs: int, head_hidden: torch.Tensor) -> None: idx_map = self.sample_idx_mapping[:num_sample].view(num_reqs, n_spec) sample_pos = self.sample_pos[:num_sample].view(num_reqs, n_spec) + confidence_markov_embeds = [] # Anchor (bonus) token per request = the input id at query offset 0, # read via the precomputed persistent index (fixed buffer for capture). @@ -157,6 +165,8 @@ def _sample_sequential(self, num_reqs: int, head_hidden: torch.Tensor) -> None: for i in range(n_spec): # Sequential stage: Markov bias from the previously sampled token. markov_embed = self.model.markov_embed(prev) + if self.use_confidence_based_verification: + confidence_markov_embeds.append(markov_embed) bias = self.model.markov_bias(markov_embed) logits_i = base_logits[:, i] + bias draft_sampled_i = self._sample_logits( @@ -202,6 +212,15 @@ def _sample_sequential_topk(self, num_reqs: int, head_hidden: torch.Tensor) -> N self.draft_tokens[:num_reqs, i] = draft_sampled_i prev = draft_sampled_i + if self.use_confidence_based_verification: + torch.sigmoid( + self.model.compute_confidence( + sample_hidden, + torch.stack(confidence_markov_embeds, dim=1).flatten(0, 1), + ).view(num_reqs, n_spec), + out=self.draft_token_confidence_probs[:num_reqs], + ) + def _generate_draft( self, num_reqs: int, diff --git a/vllm/v1/worker/gpu/structured_outputs.py b/vllm/v1/worker/gpu/structured_outputs.py index 34f00086d2a2..6b55b65c44e4 100644 --- a/vllm/v1/worker/gpu/structured_outputs.py +++ b/vllm/v1/worker/gpu/structured_outputs.py @@ -10,7 +10,13 @@ class StructuredOutputsWorker: - def __init__(self, max_num_logits: int, vocab_size: int, device: torch.device): + def __init__( + self, + max_num_logits: int, + vocab_size: int, + device: torch.device, + mask_stride: int, + ): self.logits_indices = torch.zeros( max_num_logits, dtype=torch.int32, device=device ) @@ -19,6 +25,7 @@ def __init__(self, max_num_logits: int, vocab_size: int, device: torch.device): ) self.device = device self.copy_stream = torch.cuda.Stream() + self.mask_stride = mask_stride def apply_grammar_bitmask( self, @@ -45,7 +52,13 @@ def apply_grammar_bitmask( req_idx = req_id_to_idx[grammar_req_id] logits_start_idx = cu_num_logits[req_idx] logits_end_idx = cu_num_logits[req_idx + 1] - mapping.extend(range(logits_start_idx, logits_end_idx)) + # Key by (request, position) rather than absolute logit index: + # adaptive verification finalizes per-request logit offsets on + # device, so the kernel resolves them from the GPU cu_num_logits. + mapping.extend( + req_idx * self.mask_stride + position + for position in range(logits_end_idx - logits_start_idx) + ) # Asynchronously copy the mapping to GPU. with torch.cuda.stream(self.copy_stream): @@ -69,9 +82,11 @@ def apply_grammar_bitmask( logits, logits.stride(0), logits_indices, + input_batch.cu_num_logits, bitmask, bitmask.stride(0), vocab_size, + MASK_STRIDE=self.mask_stride, BLOCK_SIZE=BLOCK_SIZE, ) @@ -87,13 +102,21 @@ def _apply_grammar_bitmask_kernel( logits_ptr, logits_stride, logits_indices_ptr, + cu_num_logits_ptr, bitmask_ptr, bitmask_stride, vocab_size, + MASK_STRIDE: tl.constexpr, BLOCK_SIZE: tl.constexpr, ): bitmask_idx = tl.program_id(0) - logits_idx = tl.load(logits_indices_ptr + bitmask_idx) + mapping_idx = tl.load(logits_indices_ptr + bitmask_idx) + req_idx = mapping_idx // MASK_STRIDE + position_idx = mapping_idx % MASK_STRIDE + logits_idx = tl.load(cu_num_logits_ptr + req_idx) + num_req_logits = tl.load(cu_num_logits_ptr + req_idx + 1) - logits_idx + logits_idx += position_idx + position_is_active = position_idx < num_req_logits # Load the bitmask. block_id = tl.program_id(1) @@ -111,5 +134,5 @@ def _apply_grammar_bitmask_kernel( tl.store( logits_ptr + logits_idx * logits_stride + block_offset, -float("inf"), - mask=bitmask & (block_offset < vocab_size), + mask=position_is_active & bitmask & (block_offset < vocab_size), ) From a772e95f3ae7c7f928aa8221ff56cd1c68725464 Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Thu, 30 Jul 2026 19:19:15 +0000 Subject: [PATCH 02/40] [Spec Decode] Keep non-adaptive request ordering identical to main Drafts-first ordering is only required by adaptive verification (compact_batch assumes verification requests lead); every other speculative method keeps main's ordering byte-for-byte. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_011Vq5NzWmCNi3F2WX3Vr7op Signed-off-by: Lucas Wilkinson --- vllm/v1/worker/gpu/model_runner.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index f4d76d010eb8..4c8096467df0 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -1037,8 +1037,13 @@ def prepare_inputs( # batch_idx -> req_id draft_tokens = scheduler_output.scheduled_spec_decode_tokens + # Drafts-first ordering is only required by adaptive verification + # (compact_batch assumes verification requests lead); keep main's + # ordering untouched otherwise. req_ids = sort_batch_req_ids( - num_tokens_per_req, draft_tokens, self.decode_query_len + num_tokens_per_req, + draft_tokens if self.adaptive_verification is not None else {}, + self.decode_query_len, ) numtoks_iter = map(num_tokens_per_req.get, req_ids) num_scheduled_tokens = np.fromiter(numtoks_iter, dtype=np.int32, count=num_reqs) From c4c8eec0d833098c296a0bb17a180df01a96a736 Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Thu, 30 Jul 2026 19:19:20 +0000 Subject: [PATCH 03/40] [Spec Decode] Source decode max_q_len from attention metadata The static hint bounded verification rows (1 + k) but not short extends, which the decode split admits up to decode_threshold (1 + 2k); rows past the hint were silently uncovered by the TRTLLM kernel schedule. Bound it with min(common max_query_len, decode_threshold) instead, and build the CPU upper bounds (seq_lens_cpu_upper_bound, promised max_query_len) from pre-trim scheduled counts, which the GPU allocation can never exceed. Adds an invariant test for compact_batch totals and per-row bounds. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_011Vq5NzWmCNi3F2WX3Vr7op Signed-off-by: Lucas Wilkinson --- .../spec_decode/test_adaptive_verification.py | 25 +++++++++++++++++++ vllm/v1/attention/backends/mla/sparse_swa.py | 9 +++++-- vllm/v1/worker/gpu/model_runner.py | 11 +++++++- 3 files changed, 42 insertions(+), 3 deletions(-) diff --git a/tests/v1/spec_decode/test_adaptive_verification.py b/tests/v1/spec_decode/test_adaptive_verification.py index f0a4e8c009a0..d24b2273f3bb 100644 --- a/tests/v1/spec_decode/test_adaptive_verification.py +++ b/tests/v1/spec_decode/test_adaptive_verification.py @@ -118,3 +118,28 @@ def test_profiled_batches_seed_cost_curves_via_consumer(): # smear that eager cost across every larger request count. assert curves["draft"] == [(1, 1.0), (128, 1.0)] assert manager._profile_samples == [] + + +def test_compact_batch_preserves_totals_and_bounds(): + # The CPU placeholder layout must keep the batch total equal to the GPU + # total and every verification row within decode_query_len, or downstream + # CPU metadata desyncs from the reallocated GPU boundaries. + manager = make_manager( + np.array([[0.9, 0.9], [0.9, 0.9], [1.0, 1.0]], dtype=np.float32), + np.array([1.0] * 44 + [100.0] * 3), + ) + manager.req_states.req_id_to_index["prefill"] = 2 + manager.req_states.num_computed_tokens_np = np.zeros(3, dtype=np.int32) + manager.req_states.prefill_len.np = np.array([0, 0, 60], dtype=np.int32) + num_tokens = manager.get_num_tokens( + {"low": 3, "high": 3, "prefill": 40}, + {"low": [1, 2], "high": [3, 4]}, + ) + scheduled = np.array([3, 3, 40], dtype=np.int32) + drafts = np.array([2, 2, 0], dtype=np.int32) + compacted = manager.compact_batch(drafts, scheduled) + + assert int(compacted.sum()) == num_tokens + num_steps = manager.num_speculative_steps + assert (compacted[:2] <= 1 + num_steps).all() + assert compacted[2] == 40 diff --git a/vllm/v1/attention/backends/mla/sparse_swa.py b/vllm/v1/attention/backends/mla/sparse_swa.py index 5f29642304a4..6a07256b4245 100644 --- a/vllm/v1/attention/backends/mla/sparse_swa.py +++ b/vllm/v1/attention/backends/mla/sparse_swa.py @@ -439,7 +439,6 @@ def __init__(self, *args, **kwargs): 2 if (spec_config is not None and spec_config.parallel_drafting) else 1 ) self.decode_threshold = 1 + spec_mult * self.num_speculative_tokens - self.max_decode_query_len = 1 + self.num_speculative_tokens self.reorder_batch_threshold = None hf_config = self.vllm_config.model_config.hf_config @@ -663,7 +662,13 @@ def build( num_prefills=num_prefills, num_decode_tokens=num_decode_tokens, num_prefill_tokens=num_prefill_tokens, - max_decode_query_len=self.max_decode_query_len, + # Upper bound on decode-split rows for the kernel's max_q_len + # hint. common max_query_len bounds every row (scheduled max under + # adaptive verification), clamped to what the split can admit so a + # mixed batch's prefill max does not inflate decode scheduling. + max_decode_query_len=min( + common_attn_metadata.max_query_len, self.decode_threshold + ), tile_sched_swaonly=tile_sched[_LAYER_TYPE_SWAONLY], tile_sched_c4a=tile_sched[_LAYER_TYPE_C4A], tile_sched_c128a=tile_sched[_LAYER_TYPE_C128A], diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 4c8096467df0..e0088f153356 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -1084,7 +1084,11 @@ def prepare_inputs( adaptive_verification = ( self.adaptive_verification if num_draft_tokens_per_req is not None else None ) + scheduled_num_tokens = num_scheduled_tokens if adaptive_verification is not None: + # The GPU may regrant trimmed drafts unevenly, but never beyond a + # request's scheduled count: the pre-trim counts remain the + # per-request upper bound for CPU-side metadata. num_scheduled_tokens = adaptive_verification.compact_batch( num_draft_tokens_per_req, num_scheduled_tokens, @@ -1175,7 +1179,7 @@ def prepare_inputs( seq_lens_cpu_upper_bound_np = np.zeros(num_reqs_padded, dtype=np.int32) np.add( num_computed_tokens_np, - num_scheduled_tokens, + scheduled_num_tokens, out=seq_lens_cpu_upper_bound_np[:num_reqs], ) seq_lens_cpu_upper_bound = torch.from_numpy(seq_lens_cpu_upper_bound_np) @@ -1221,6 +1225,11 @@ def prepare_inputs( cu_num_logits_np=cu_num_logits_np, has_structured_output_reqs=scheduler_output.has_structured_output_requests, prompt_lens=prompt_lens, + max_query_len=( + int(scheduled_num_tokens.max()) + if adaptive_verification is not None + else None + ), ) return pcp.maybe_partition_pcp_batch(self.pcp_manager, input_batch) From ae28ecf56f6aadd7db2a10f52af3eff26f4b0f06 Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Thu, 30 Jul 2026 19:19:22 +0000 Subject: [PATCH 04/40] [Spec Decode] Prune implementation-focused unit tests Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_011Vq5NzWmCNi3F2WX3Vr7op Signed-off-by: Lucas Wilkinson --- .../spec_decode/test_adaptive_verification.py | 13 ---- tests/v1/spec_decode/test_dynamic_sd_cug.py | 74 ------------------- 2 files changed, 87 deletions(-) diff --git a/tests/v1/spec_decode/test_adaptive_verification.py b/tests/v1/spec_decode/test_adaptive_verification.py index d24b2273f3bb..4d76c78cea47 100644 --- a/tests/v1/spec_decode/test_adaptive_verification.py +++ b/tests/v1/spec_decode/test_adaptive_verification.py @@ -8,7 +8,6 @@ from vllm.v1.worker.gpu.async_utils import StepTimingSample from vllm.v1.worker.gpu.spec_decode.adaptive_verification import ( AdaptiveVerificationManager, - build_cost_tables_from_curves, ) @@ -66,18 +65,6 @@ def test_structured_output_placeholders_do_not_consume_budget(): assert draft_budget == 3 -def test_step_timing_cost_tables_are_monotonic_with_fixed_overhead(): - draft, verify = build_cost_tables_from_curves( - [(1, 2.0), (2, 3.0)], - [(1, 4.0), (2, 3.0)], - max_num_reqs=2, - max_batch_tokens=2, - ) - - assert draft.tolist() == [2.0, 2.0, 3.0] - assert verify.tolist() == [5.0, 5.0, 5.0] - - def test_profiled_batches_seed_cost_curves_via_consumer(): manager = AdaptiveVerificationManager.__new__(AdaptiveVerificationManager) manager.req_states = SimpleNamespace(max_num_batched_tokens=4096, max_num_reqs=64) diff --git a/tests/v1/spec_decode/test_dynamic_sd_cug.py b/tests/v1/spec_decode/test_dynamic_sd_cug.py index fa2ebb77171c..d75495ea606a 100644 --- a/tests/v1/spec_decode/test_dynamic_sd_cug.py +++ b/tests/v1/spec_decode/test_dynamic_sd_cug.py @@ -326,77 +326,3 @@ def test_dynamic_sd_only_captures_scheduled_query_lengths(monkeypatch): assert desc.num_tokens == num_tokens assert desc.num_reqs is None assert desc.num_active_loras == 0 - - -def test_adaptive_verification_captures_non_uniform_full_graph(monkeypatch): - monkeypatch.setattr( - gpu_cudagraph_utils, - "get_pp_group", - lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True), - ) - - max_spec_tokens = 7 - vllm_config = _create_vllm_config_for_dsd( - max_num_seqs=64, - max_spec_tokens=max_spec_tokens, - use_dynamic_sd=False, - ) - manager = gpu_cudagraph_utils.CudaGraphManager( - vllm_config=vllm_config, - device=torch.device("cpu"), - cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE, - decode_query_len=max_spec_tokens + 1, - varlen_decode=True, - ) - manager._graphs_captured = True - - desc = manager.dispatch( - num_reqs=16, - num_tokens=80, - uniform_token_count=None, - num_active_loras=0, - max_query_len=8, - ) - - assert desc.cg_mode == CUDAGraphMode.FULL - assert desc.num_tokens == 80 - assert desc.num_reqs == 64 - assert desc.max_query_len == 8 - - for num_tokens in range(1, 8): - desc = manager.dispatch( - num_reqs=num_tokens, - num_tokens=num_tokens, - uniform_token_count=None, - num_active_loras=0, - max_query_len=1, - ) - - assert desc.cg_mode == CUDAGraphMode.FULL - assert desc.num_tokens >= num_tokens - assert desc.num_reqs >= num_tokens - assert desc.max_query_len == 8 - - desc = manager.dispatch( - num_reqs=1, - num_tokens=8, - uniform_token_count=None, - num_active_loras=0, - max_query_len=8, - ) - - assert desc.cg_mode == CUDAGraphMode.FULL - assert desc.num_reqs == 8 - - desc = manager.dispatch( - num_reqs=8, - num_tokens=8, - uniform_token_count=None, - num_active_loras=0, - max_query_len=1, - ) - - assert desc.cg_mode == CUDAGraphMode.FULL - assert desc.num_tokens == 8 - assert desc.num_reqs == 8 - assert desc.max_query_len == 8 From f8c57028abdf1a9309d3ed5e3b233b08f7fe8bf6 Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Thu, 30 Jul 2026 19:19:23 +0000 Subject: [PATCH 05/40] [Spec Decode] Rename adaptive verification config knobs dspark_enable_confidence_based_verification -> enable_adaptive_verification dspark_confidence_ema_alpha -> adaptive_verification_ema_alpha Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_011Vq5NzWmCNi3F2WX3Vr7op Signed-off-by: Lucas Wilkinson --- ...DeepSeek-V4-Flash-DSpark-confidence-TP4.yaml | 2 +- vllm/config/speculative.py | 17 ++++++++--------- vllm/sampling_params.py | 2 +- vllm/v1/worker/gpu/model_runner.py | 6 +++--- .../worker/gpu/spec_decode/dspark/speculator.py | 8 ++++---- 5 files changed, 17 insertions(+), 18 deletions(-) diff --git a/tests/evals/gsm8k/configs/DeepSeek-V4-Flash-DSpark-confidence-TP4.yaml b/tests/evals/gsm8k/configs/DeepSeek-V4-Flash-DSpark-confidence-TP4.yaml index 4e5be4623c6a..850c153e9e62 100644 --- a/tests/evals/gsm8k/configs/DeepSeek-V4-Flash-DSpark-confidence-TP4.yaml +++ b/tests/evals/gsm8k/configs/DeepSeek-V4-Flash-DSpark-confidence-TP4.yaml @@ -19,4 +19,4 @@ server_args: >- "model":"deepseek-ai/DeepSeek-V4-Flash-DSpark", "attention_backend":"FLASH_ATTN","num_speculative_tokens":7, "draft_sample_method":"probabilistic", - "dspark_enable_confidence_based_verification":true}' + "enable_adaptive_verification":true}' diff --git a/vllm/config/speculative.py b/vllm/config/speculative.py index a08dadff98d4..c52b8a06e076 100644 --- a/vllm/config/speculative.py +++ b/vllm/config/speculative.py @@ -236,10 +236,11 @@ class SpeculativeConfig: synthetic_acceptance_rates. Only valid when rejection_sample_method is 'synthetic'. Mutually exclusive with synthetic_acceptance_rates.""" - dspark_enable_confidence_based_verification: bool = True - """Whether to enable DSpark confidence-based verification.""" + enable_adaptive_verification: bool = True + """Whether to adaptively size the draft-verification budget from + per-request confidence. Only applies to method="dspark".""" - dspark_confidence_ema_alpha: float = 0.8 + adaptive_verification_ema_alpha: float = 0.8 """Weight of the latest DSpark confidence observation in the per-request exponential moving average. Set to 1 to disable smoothing. Smoothing trades less noise for more lag; lag does not average out across a batch, so @@ -247,10 +248,8 @@ class SpeculativeConfig: point where the estimator is unbiased at every batch size measured.""" @property - def use_confidence_based_verification(self) -> bool: - return ( - self.method == "dspark" and self.dspark_enable_confidence_based_verification - ) + def use_adaptive_verification(self) -> bool: + return self.method == "dspark" and self.enable_adaptive_verification @staticmethod def _acceptance_length_to_rates(length: float, n: int) -> list[float]: @@ -1348,8 +1347,8 @@ def _verify_args(self) -> Self: f"than zero ({self.num_speculative_tokens})." ) - if not 0 < self.dspark_confidence_ema_alpha <= 1: - raise ValueError("dspark_confidence_ema_alpha must be in (0, 1].") + if not 0 < self.adaptive_verification_ema_alpha <= 1: + raise ValueError("adaptive_verification_ema_alpha must be in (0, 1].") if self.rejection_sample_method == "synthetic": # Consolidate to per-position rates diff --git a/vllm/sampling_params.py b/vllm/sampling_params.py index c837fa474f17..f26dc8deec23 100644 --- a/vllm/sampling_params.py +++ b/vllm/sampling_params.py @@ -894,7 +894,7 @@ def _validate_spec_decode( # Adaptive verification compacts logits after the forward pass, while # compute_topk_scores uses the scheduled layout in cu_num_logits_np. if ( - speculative_config.use_confidence_based_verification + speculative_config.use_adaptive_verification and self.num_logprobs is not None ): raise ValueError( diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index e0088f153356..68c20673ec12 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -257,7 +257,7 @@ def __init__(self, vllm_config: VllmConfig, device: torch.device): self.is_last_pp_rank and not self.is_pooling_model and self.speculative_config is not None - and self.speculative_config.use_confidence_based_verification + and self.speculative_config.use_adaptive_verification ) # General request states. @@ -518,13 +518,13 @@ def initialize_kv_cache(self, kv_cache_config: KVCacheConfig) -> None: ) if ( self.speculative_config is not None - and self.speculative_config.use_confidence_based_verification + and self.speculative_config.use_adaptive_verification ): self.adaptive_verification = AdaptiveVerificationManager( self.req_states, self.input_buffers.query_start_loc, self.model_state.num_new_sampled_tokens_per_step, - self.speculative_config.dspark_confidence_ema_alpha, + self.speculative_config.adaptive_verification_ema_alpha, ) self.step_timing.consumer = self.adaptive_verification.consume_step_timing self.block_tables = BlockTables( diff --git a/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py b/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py index a23b43f25ccc..6c65b73f9e16 100644 --- a/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py @@ -79,8 +79,8 @@ def __init__(self, vllm_config: VllmConfig, device: torch.device): self.draft_token_confidence_probs = torch.empty_like( self.draft_tokens, dtype=torch.float32 ) - self.use_confidence_based_verification = ( - self.speculative_config.use_confidence_based_verification + self.use_adaptive_verification = ( + self.speculative_config.use_adaptive_verification ) def load_draft_model( @@ -165,7 +165,7 @@ def _sample_sequential(self, num_reqs: int, head_hidden: torch.Tensor) -> None: for i in range(n_spec): # Sequential stage: Markov bias from the previously sampled token. markov_embed = self.model.markov_embed(prev) - if self.use_confidence_based_verification: + if self.use_adaptive_verification: confidence_markov_embeds.append(markov_embed) bias = self.model.markov_bias(markov_embed) logits_i = base_logits[:, i] + bias @@ -212,7 +212,7 @@ def _sample_sequential_topk(self, num_reqs: int, head_hidden: torch.Tensor) -> N self.draft_tokens[:num_reqs, i] = draft_sampled_i prev = draft_sampled_i - if self.use_confidence_based_verification: + if self.use_adaptive_verification: torch.sigmoid( self.model.compute_confidence( sample_hidden, From 9c708552f44cfdfc42f0ead9ecafe0e2918cd460 Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Thu, 30 Jul 2026 19:19:29 +0000 Subject: [PATCH 06/40] [Spec Decode] Fall back to fixed-length verification without cost profiles If no FULL varlen graphs were captured (enforce_eager, or the attention backend lacks varlen support) step costs cannot be profiled; disable adaptive verification with a warning instead of asserting on the first spec step. Reject adaptive verification with pipeline parallelism at config time: cost curves and confidences only exist on the last rank. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_011Vq5NzWmCNi3F2WX3Vr7op Signed-off-by: Lucas Wilkinson --- vllm/config/vllm.py | 11 +++++++++++ vllm/v1/worker/gpu/model_runner.py | 26 +++++++++++++++++++------- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index 4cef46020bed..ec4e0ffda148 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -2240,6 +2240,17 @@ def _get_v2_model_runner_unsupported_features(self) -> list[str]: ): unsupported.append("EAGLE3 with pipeline parallelism") + if ( + speculative_config.use_adaptive_verification + and self.parallel_config.pipeline_parallel_size > 1 + ): + # Cost curves and confidences only exist on the last PP rank; + # earlier ranks would diverge on the trimmed batch shape. + # Set enable_adaptive_verification=false to use dspark with PP. + unsupported.append( + "DSpark adaptive verification with pipeline parallelism" + ) + if self.parallel_config.enable_dbo: unsupported.append("dual batch overlap") diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 68c20673ec12..fa13ddee892f 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -1377,13 +1377,25 @@ def execute_model( and scheduler_output.scheduled_spec_decode_tokens and not dummy_run ): - num_toks = self.adaptive_verification.get_num_tokens( - scheduler_output.num_scheduled_tokens, - scheduler_output.scheduled_spec_decode_tokens, - scheduler_output.has_structured_output_requests, - ) - # Trimming drafts makes the batch non-uniform. - uniform_tok_count = None + if self.adaptive_verification.cost_tables is None: + # No FULL graphs were captured (enforce_eager, or the attention + # backend lacks varlen support), so step costs could not be + # profiled. Consistent across ranks: TP shares curves via + # broadcast and PP is rejected at config time. + logger.warning_once( + "DSpark adaptive verification disabled: no cudagraph cost " + "profile is available. Falling back to fixed-length " + "verification." + ) + self.adaptive_verification = None + else: + num_toks = self.adaptive_verification.get_num_tokens( + scheduler_output.num_scheduled_tokens, + scheduler_output.scheduled_spec_decode_tokens, + scheduler_output.has_structured_output_requests, + ) + # Trimming drafts makes the batch non-uniform. + uniform_tok_count = None batch_desc, num_tokens_across_dp = dispatch_cg_and_sync_dp( self.cudagraph_manager, From cb068aacf046c6b6cd59ec083babb1e5438cc178 Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Thu, 30 Jul 2026 19:19:34 +0000 Subject: [PATCH 07/40] [Spec Decode] Cap draft budget to one rejection-sampler chunk The chunked verification path indexes logits by scheduled offsets and cannot address the compacted layout, so the budget must keep total logits within a single chunk. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_011Vq5NzWmCNi3F2WX3Vr7op Signed-off-by: Lucas Wilkinson --- .../spec_decode/test_adaptive_verification.py | 18 ++++++++++++++++++ vllm/v1/worker/gpu/model_runner.py | 6 +++++- .../gpu/spec_decode/adaptive_verification.py | 11 ++++++++++- .../gpu/spec_decode/rejection_sampler.py | 9 +++++++-- 4 files changed, 40 insertions(+), 4 deletions(-) diff --git a/tests/v1/spec_decode/test_adaptive_verification.py b/tests/v1/spec_decode/test_adaptive_verification.py index 4d76c78cea47..ff6b9b79291d 100644 --- a/tests/v1/spec_decode/test_adaptive_verification.py +++ b/tests/v1/spec_decode/test_adaptive_verification.py @@ -25,6 +25,8 @@ def make_manager( prefill_len=SimpleNamespace(np=np.ones(num_reqs, dtype=np.int32)), ) manager.cost_tables = (np.zeros(num_reqs + 1), verify_cost_ms) + manager._max_total_logits = 1 << 30 + manager.num_bonus_tokens = 1 return manager @@ -130,3 +132,19 @@ def test_compact_batch_preserves_totals_and_bounds(): num_steps = manager.num_speculative_steps assert (compacted[:2] <= 1 + num_steps).all() assert compacted[2] == 40 + + +def test_budget_caps_at_one_rejection_sampler_chunk(): + # The chunked verification path cannot address the compacted logits + # layout, so the budget must keep total logits within a single chunk. + manager = make_manager( + np.array([[0.9, 0.9], [0.9, 0.9]], dtype=np.float32), + np.ones(7), + ) + manager._max_total_logits = 3 # 2 bonus logits + at most 1 draft + manager.get_num_tokens( + {"low": 3, "high": 3}, + {"low": [1, 2], "high": [3, 4]}, + ) + _, _, draft_budget = manager._batch_budget + assert draft_budget <= 1 diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index fa13ddee892f..327c975a38e3 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -130,7 +130,10 @@ from vllm.v1.worker.gpu.spec_decode.eagle.eagle3_utils import ( set_eagle3_aux_hidden_state_layers, ) -from vllm.v1.worker.gpu.spec_decode.rejection_sampler import RejectionSampler +from vllm.v1.worker.gpu.spec_decode.rejection_sampler import ( + RejectionSampler, + max_chunk_logits, +) from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator from vllm.v1.worker.gpu.spec_decode.utils import DraftTokensHandler from vllm.v1.worker.gpu.states import RequestState @@ -525,6 +528,7 @@ def initialize_kv_cache(self, kv_cache_config: KVCacheConfig) -> None: self.input_buffers.query_start_loc, self.model_state.num_new_sampled_tokens_per_step, self.speculative_config.adaptive_verification_ema_alpha, + max_total_logits=max_chunk_logits(self.vocab_size), ) self.step_timing.consumer = self.adaptive_verification.consume_step_timing self.block_tables = BlockTables( diff --git a/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py b/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py index 759ccaa83a36..0679c0055f33 100644 --- a/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py +++ b/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py @@ -156,6 +156,7 @@ def __init__( query_start_loc: torch.Tensor, num_bonus_tokens: int, confidence_ema_alpha: float, + max_total_logits: int, ): self.req_states = req_states self.num_speculative_steps = req_states.num_speculative_steps @@ -164,6 +165,10 @@ def __init__( self.num_bonus_tokens = num_bonus_tokens self.confidence_ema_alpha = confidence_ema_alpha + # Rejection sampling verifies logits in one contiguous chunk; the + # chunked path indexes by scheduled (untrimmed) offsets and cannot + # address the compacted layout, so the budget must fit one chunk. + self._max_total_logits = max_total_logits self.query_start_loc = query_start_loc self.cost_tables: tuple[np.ndarray, np.ndarray] | None = None # Largest cudagraph-captured token count; above it nothing pads. @@ -353,7 +358,11 @@ def get_num_tokens( valid = steps[None, :] < valid_drafts[:, None] scores = np.sort(survival_probability[valid])[::-1] num_non_draft_tokens_total = int(num_non_draft_tokens.sum()) - max_draft_budget = int(valid_drafts.sum()) + max_draft_budget = min( + int(valid_drafts.sum()), + max(0, self._max_total_logits - num_reqs * self.num_bonus_tokens), + ) + scores = scores[:max_draft_budget] draft_cost_ms, verify_cost_ms = self.cost_tables num_sampling_requests = np.count_nonzero( self.req_states.num_computed_tokens_np[slots] + num_non_draft_tokens diff --git a/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py b/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py index 63d479efa0b2..a139ac7d25d5 100644 --- a/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py +++ b/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py @@ -32,6 +32,11 @@ _FP32_BYTES = 4 +def max_chunk_logits(vocab_size: int) -> int: + """Largest number of logits rows one verification chunk may hold.""" + return max(1, MAX_CHUNK_BYTES // (vocab_size * _FP32_BYTES)) + + def _iter_request_chunks( cu_num_logits: np.ndarray, max_chunk_logits: int ) -> Iterator[tuple[int, int]]: @@ -245,14 +250,14 @@ def __call__( max_num_logprobs = self.sampler.sampling_states.max_num_logprobs( input_batch.idx_mapping_np ) - max_chunk_logits = max(1, MAX_CHUNK_BYTES // (logits.shape[1] * _FP32_BYTES)) + chunk_logit_limit = max_chunk_logits(logits.shape[1]) sampled, num_sampled, logprobs_tensors = self._verify_in_chunks( logits, input_batch, draft_logits, draft_sampled, pos, - max_chunk_logits, + chunk_logit_limit, max_num_logprobs, ) From 4cbe277ae6b208085f541b73706fcf43ab10d0d5 Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Thu, 30 Jul 2026 19:19:36 +0000 Subject: [PATCH 08/40] [Spec Decode] Do not treat async draft placeholders as grammar-rejected Under async scheduling every scheduled draft id is a -1 placeholder at budget time; only the sync scheduler's -1 marks grammar rejection. Filtering placeholders zeroed the entire batch's speculation whenever a structured-output request was present. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_011Vq5NzWmCNi3F2WX3Vr7op Signed-off-by: Lucas Wilkinson --- .../spec_decode/test_adaptive_verification.py | 19 +++++++++++++++++++ vllm/v1/worker/gpu/model_runner.py | 1 + .../gpu/spec_decode/adaptive_verification.py | 8 +++++++- 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/tests/v1/spec_decode/test_adaptive_verification.py b/tests/v1/spec_decode/test_adaptive_verification.py index ff6b9b79291d..5489212eee41 100644 --- a/tests/v1/spec_decode/test_adaptive_verification.py +++ b/tests/v1/spec_decode/test_adaptive_verification.py @@ -27,6 +27,7 @@ def make_manager( manager.cost_tables = (np.zeros(num_reqs + 1), verify_cost_ms) manager._max_total_logits = 1 << 30 manager.num_bonus_tokens = 1 + manager._count_rejected_drafts = True return manager @@ -148,3 +149,21 @@ def test_budget_caps_at_one_rejection_sampler_chunk(): ) _, _, draft_budget = manager._batch_budget assert draft_budget <= 1 + + +def test_async_placeholder_drafts_are_not_treated_as_grammar_rejected(): + # Under async scheduling every draft id is a -1 placeholder at budget + # time; the grammar-validity filter must not zero the budget batch-wide. + manager = make_manager( + np.array([[0.9, 0.9], [0.9, 0.9]], dtype=np.float32), + np.ones(7), + ) + manager._count_rejected_drafts = False + manager.get_num_tokens( + {"low": 3, "high": 3}, + {"low": [-1, -1], "high": [-1, -1]}, + has_structured_output=True, + ) + valid_drafts, _, draft_budget = manager._batch_budget + assert valid_drafts == {"low": 2, "high": 2} + assert draft_budget > 0 diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 327c975a38e3..be1f55466e71 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -529,6 +529,7 @@ def initialize_kv_cache(self, kv_cache_config: KVCacheConfig) -> None: self.model_state.num_new_sampled_tokens_per_step, self.speculative_config.adaptive_verification_ema_alpha, max_total_logits=max_chunk_logits(self.vocab_size), + count_rejected_drafts=not self.scheduler_config.async_scheduling, ) self.step_timing.consumer = self.adaptive_verification.consume_step_timing self.block_tables = BlockTables( diff --git a/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py b/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py index 0679c0055f33..0543cc908015 100644 --- a/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py +++ b/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py @@ -157,6 +157,7 @@ def __init__( num_bonus_tokens: int, confidence_ema_alpha: float, max_total_logits: int, + count_rejected_drafts: bool, ): self.req_states = req_states self.num_speculative_steps = req_states.num_speculative_steps @@ -169,6 +170,11 @@ def __init__( # chunked path indexes by scheduled (untrimmed) offsets and cannot # address the compacted layout, so the budget must fit one chunk. self._max_total_logits = max_total_logits + # The sync scheduler overwrites grammar-rejected drafts with -1, so + # they can be excluded from the budget. Under async scheduling every + # draft is a -1 placeholder at budget time, so the convention does not + # hold and the filter must not run. + self._count_rejected_drafts = count_rejected_drafts self.query_start_loc = query_start_loc self.cost_tables: tuple[np.ndarray, np.ndarray] | None = None # Largest cudagraph-captured token count; above it nothing pads. @@ -337,7 +343,7 @@ def get_num_tokens( count=num_reqs, ) valid_drafts = scheduled_drafts - if has_structured_output: + if has_structured_output and self._count_rejected_drafts: valid_drafts = np.fromiter( ( sum(token >= 0 for token in draft_tokens.get(req_id, ())) From 75b1522637bab27d615b0eb2ede777a7e5fc3889 Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Thu, 30 Jul 2026 19:19:38 +0000 Subject: [PATCH 09/40] [Spec Decode] Gate adaptive verification on LoRA, confidence head, model state LoRA + adaptive verification is rejected at config time (the per-token LoRA mapping cannot follow the GPU-side draft reallocation). Checkpoints without a confidence head and non-default model states fall back to fixed-length verification with a warning. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_011Vq5NzWmCNi3F2WX3Vr7op Signed-off-by: Lucas Wilkinson --- vllm/config/vllm.py | 10 ++++++ vllm/v1/worker/gpu/model_runner.py | 33 ++++++++++++++----- .../gpu/spec_decode/dspark/speculator.py | 9 +++++ 3 files changed, 43 insertions(+), 9 deletions(-) diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index ec4e0ffda148..f454375d078c 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -2240,6 +2240,16 @@ def _get_v2_model_runner_unsupported_features(self) -> list[str]: ): unsupported.append("EAGLE3 with pipeline parallelism") + if ( + speculative_config.use_adaptive_verification + and self.lora_config is not None + ): + # The per-token LoRA mapping is built from CPU placeholder + # boundaries, while the trimmed batch's true boundaries are + # decided on the GPU. Set enable_adaptive_verification=false + # to combine dspark with LoRA. + unsupported.append("DSpark adaptive verification with LoRA") + if ( speculative_config.use_adaptive_verification and self.parallel_config.pipeline_parallel_size > 1 diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index be1f55466e71..528c266800b9 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -117,6 +117,7 @@ from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache from vllm.v1.worker.gpu.mm.lora import set_active_mm_loras from vllm.v1.worker.gpu.model_states import init_model_state +from vllm.v1.worker.gpu.model_states.default import DefaultModelState from vllm.v1.worker.gpu.pool.pooling_runner import PoolingRunner from vllm.v1.worker.gpu.pp_utils import PPHandler from vllm.v1.worker.gpu.sample.output import SamplerOutput @@ -522,16 +523,30 @@ def initialize_kv_cache(self, kv_cache_config: KVCacheConfig) -> None: if ( self.speculative_config is not None and self.speculative_config.use_adaptive_verification + # The checkpoint may lack a confidence head (the speculator clears + # its flag at load time). + and getattr(self.speculator, "use_adaptive_verification", False) ): - self.adaptive_verification = AdaptiveVerificationManager( - self.req_states, - self.input_buffers.query_start_loc, - self.model_state.num_new_sampled_tokens_per_step, - self.speculative_config.adaptive_verification_ema_alpha, - max_total_logits=max_chunk_logits(self.vocab_size), - count_rejected_drafts=not self.scheduler_config.async_scheduling, - ) - self.step_timing.consumer = self.adaptive_verification.consume_step_timing + if isinstance(self.model_state, DefaultModelState): + self.adaptive_verification = AdaptiveVerificationManager( + self.req_states, + self.input_buffers.query_start_loc, + self.model_state.num_new_sampled_tokens_per_step, + self.speculative_config.adaptive_verification_ema_alpha, + max_total_logits=max_chunk_logits(self.vocab_size), + count_rejected_drafts=not self.scheduler_config.async_scheduling, + ) + self.step_timing.consumer = ( + self.adaptive_verification.consume_step_timing + ) + else: + # Hybrid model states derive per-request structure from the + # scheduled counts, which are placeholders under trimming. + logger.warning_once( + "DSpark adaptive verification is not supported with %s; " + "using fixed-length verification.", + type(self.model_state).__name__, + ) self.block_tables = BlockTables( block_sizes=block_sizes, max_num_reqs=self.max_num_reqs, diff --git a/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py b/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py index 6c65b73f9e16..c20996379959 100644 --- a/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py @@ -29,10 +29,13 @@ from vllm.config import VllmConfig from vllm.config.compilation import CUDAGraphMode +from vllm.logger import init_logger from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample from vllm.v1.worker.gpu.spec_decode.dflash.speculator import DFlashSpeculator from vllm.v1.worker.gpu.spec_decode.dspark.utils import load_dspark_model +logger = init_logger(__name__) + class DSparkSpeculator(DFlashSpeculator): _speculator_name = "DSpark" @@ -105,6 +108,12 @@ def load_draft_model( dtype=self.draft_logits.dtype, device=self.device, ) + if self.use_adaptive_verification and model.model.confidence_head is None: + logger.warning_once( + "DSpark checkpoint has no confidence head; disabling adaptive " + "verification and using fixed-length verification." + ) + self.use_adaptive_verification = False return model def _sample_logits( From 48daecce26ac303d850fd6d85ad50d9024bb380e Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Thu, 30 Jul 2026 21:58:46 +0000 Subject: [PATCH 10/40] cleanup Signed-off-by: Lucas Wilkinson --- .../spec_decode/test_adaptive_verification.py | 37 ------------ vllm/config/vllm.py | 17 +++--- .../deepseek_v4/nvidia/flashinfer_sparse.py | 11 ++-- vllm/v1/worker/gpu/model_runner.py | 2 - .../gpu/spec_decode/adaptive_verification.py | 56 ++++--------------- 5 files changed, 24 insertions(+), 99 deletions(-) diff --git a/tests/v1/spec_decode/test_adaptive_verification.py b/tests/v1/spec_decode/test_adaptive_verification.py index 5489212eee41..d71936990331 100644 --- a/tests/v1/spec_decode/test_adaptive_verification.py +++ b/tests/v1/spec_decode/test_adaptive_verification.py @@ -27,7 +27,6 @@ def make_manager( manager.cost_tables = (np.zeros(num_reqs + 1), verify_cost_ms) manager._max_total_logits = 1 << 30 manager.num_bonus_tokens = 1 - manager._count_rejected_drafts = True return manager @@ -42,7 +41,6 @@ def test_budget_stops_where_marginal_drafts_stop_paying_for_themselves(): manager.get_num_tokens( {"low": 3, "high": 3}, {"low": [1, 2], "high": [3, 4]}, - has_structured_output=False, ) valid_drafts, num_non_draft_tokens, draft_budget = manager._batch_budget @@ -51,23 +49,6 @@ def test_budget_stops_where_marginal_drafts_stop_paying_for_themselves(): assert num_non_draft_tokens == {"low": 1, "high": 1} -def test_structured_output_placeholders_do_not_consume_budget(): - manager = make_manager( - np.array([[0.9, 0.9], [0.9, 0.9]], dtype=np.float32), - np.ones(7), - ) - - manager.get_num_tokens( - {"low": 3, "high": 3}, - {"low": [1, -1], "high": [3, 4]}, - has_structured_output=True, - ) - valid_drafts, _, draft_budget = manager._batch_budget - - assert valid_drafts == {"low": 1, "high": 2} - assert draft_budget == 3 - - def test_profiled_batches_seed_cost_curves_via_consumer(): manager = AdaptiveVerificationManager.__new__(AdaptiveVerificationManager) manager.req_states = SimpleNamespace(max_num_batched_tokens=4096, max_num_reqs=64) @@ -149,21 +130,3 @@ def test_budget_caps_at_one_rejection_sampler_chunk(): ) _, _, draft_budget = manager._batch_budget assert draft_budget <= 1 - - -def test_async_placeholder_drafts_are_not_treated_as_grammar_rejected(): - # Under async scheduling every draft id is a -1 placeholder at budget - # time; the grammar-validity filter must not zero the budget batch-wide. - manager = make_manager( - np.array([[0.9, 0.9], [0.9, 0.9]], dtype=np.float32), - np.ones(7), - ) - manager._count_rejected_drafts = False - manager.get_num_tokens( - {"low": 3, "high": 3}, - {"low": [-1, -1], "high": [-1, -1]}, - has_structured_output=True, - ) - valid_drafts, _, draft_budget = manager._batch_budget - assert valid_drafts == {"low": 2, "high": 2} - assert draft_budget > 0 diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index f454375d078c..2f91b2d8d3ba 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -2244,22 +2244,19 @@ def _get_v2_model_runner_unsupported_features(self) -> list[str]: speculative_config.use_adaptive_verification and self.lora_config is not None ): - # The per-token LoRA mapping is built from CPU placeholder - # boundaries, while the trimmed batch's true boundaries are - # decided on the GPU. Set enable_adaptive_verification=false - # to combine dspark with LoRA. - unsupported.append("DSpark adaptive verification with LoRA") + # The per-token LoRA mapping is built from CPU placeholder boundaries, + # while the trimmed batch's true boundaries are decided on the GPU. + unsupported.append("adaptive verification with LoRA") if ( speculative_config.use_adaptive_verification and self.parallel_config.pipeline_parallel_size > 1 ): - # Cost curves and confidences only exist on the last PP rank; + # Cost curves and confidences currently only exist on the last PP rank; # earlier ranks would diverge on the trimmed batch shape. - # Set enable_adaptive_verification=false to use dspark with PP. - unsupported.append( - "DSpark adaptive verification with pipeline parallelism" - ) + # TODO: we should be able to support adaptive verification with PP by + # broadcasting the cost curves and confidences to all ranks. + unsupported.append("adaptive verification with pipeline parallelism") if self.parallel_config.enable_dbo: unsupported.append("dual batch overlap") diff --git a/vllm/models/deepseek_v4/nvidia/flashinfer_sparse.py b/vllm/models/deepseek_v4/nvidia/flashinfer_sparse.py index b31329bb59f3..0de5c4479c03 100644 --- a/vllm/models/deepseek_v4/nvidia/flashinfer_sparse.py +++ b/vllm/models/deepseek_v4/nvidia/flashinfer_sparse.py @@ -309,19 +309,20 @@ def _build_sparse_index_metadata( assert swa_metadata.decode_swa_indices is not None assert swa_metadata.block_table is not None - decode_swa_width = swa_metadata.decode_swa_indices.shape[-1] - assert num_prefill_tokens == 0 or decode_swa_width == self.window_size decode_swa_indices = swa_metadata.decode_swa_indices.reshape( - num_decode_tokens, decode_swa_width + num_decode_tokens, self.window_size ) decode_compressed_topk_lens = None decode_compressed_indices_are_local = False decode_is_valid_token = None if swa_only: + assert self.topk_indices_buffer is not None compressed_kv_cache = swa_k_cache decode_compressed_indices = None - prefill_topk_indices = decode_swa_indices.new_empty(num_prefill_tokens, 0) + prefill_topk_indices = self.topk_indices_buffer[ + num_decode_tokens:num_tokens, :0 + ] compressed_block_table = None compressed_block_size = swa_metadata.block_size top_k = 0 @@ -404,7 +405,7 @@ def _build_sparse_index_metadata( swa_metadata.block_size, compressed_block_table, compressed_block_size, - decode_swa_width, + self.window_size, self.compress_ratio, top_k, decode_compressed_indices_are_local=decode_compressed_indices_are_local, diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 528c266800b9..7605fe232783 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -534,7 +534,6 @@ def initialize_kv_cache(self, kv_cache_config: KVCacheConfig) -> None: self.model_state.num_new_sampled_tokens_per_step, self.speculative_config.adaptive_verification_ema_alpha, max_total_logits=max_chunk_logits(self.vocab_size), - count_rejected_drafts=not self.scheduler_config.async_scheduling, ) self.step_timing.consumer = ( self.adaptive_verification.consume_step_timing @@ -1412,7 +1411,6 @@ def execute_model( num_toks = self.adaptive_verification.get_num_tokens( scheduler_output.num_scheduled_tokens, scheduler_output.scheduled_spec_decode_tokens, - scheduler_output.has_structured_output_requests, ) # Trimming drafts makes the batch non-uniform. uniform_tok_count = None diff --git a/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py b/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py index 0543cc908015..cd5848e6a18f 100644 --- a/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py +++ b/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py @@ -18,13 +18,10 @@ from vllm.v1.worker.gpu.buffer_utils import async_copy_to_gpu logger = init_logger(__name__) -_FIXED_OVERHEAD_MS = 1.0 _PROFILE_REPLAYS = 5 # KV context each profiling request pretends to carry, so the profiled step reads # a realistic amount of cache rather than attending over nothing. _PROFILE_CONTEXT_LEN = 8192 -# Median-filter width for cost curves; 5 also survives two adjacent outliers. -_OUTLIER_FILTER_WIDTH = 5 if TYPE_CHECKING: from vllm.v1.worker.gpu.input_batch import InputBatch @@ -106,19 +103,6 @@ def build_cost_tables_from_curves( def build_table(limit: int, curve: list[tuple[int, float]]) -> np.ndarray: xs, ys = np.asarray(curve, dtype=np.float64).T - # Drop isolated outliers before enforcing monotonicity: the cumulative - # max propagates a single bad point across every larger size, so an - # unfiltered spike corrupts the whole tail. A median filter is exact on - # monotone data, so genuine steps survive untouched. - if len(ys) >= _OUTLIER_FILTER_WIDTH: - k = _OUTLIER_FILTER_WIDTH // 2 - padded = np.pad(ys, k, mode="edge") - ys = np.array( - [ - np.median(padded[i : i + _OUTLIER_FILTER_WIDTH]) - for i in range(len(ys)) - ] - ) ys = np.maximum.accumulate(ys) values = np.arange(limit + 1) # Execution pads to the next captured size, so cost is a step @@ -145,7 +129,6 @@ def build_table(limit: int, curve: list[tuple[int, float]]) -> np.ndarray: draft_table = np.maximum(build_table(max_num_reqs, draft_curve), 0.0) verify_table = np.maximum(build_table(max_batch_tokens, verify_curve), 1e-6) - verify_table += _FIXED_OVERHEAD_MS return draft_table, verify_table @@ -157,7 +140,6 @@ def __init__( num_bonus_tokens: int, confidence_ema_alpha: float, max_total_logits: int, - count_rejected_drafts: bool, ): self.req_states = req_states self.num_speculative_steps = req_states.num_speculative_steps @@ -170,11 +152,6 @@ def __init__( # chunked path indexes by scheduled (untrimmed) offsets and cannot # address the compacted layout, so the budget must fit one chunk. self._max_total_logits = max_total_logits - # The sync scheduler overwrites grammar-rejected drafts with -1, so - # they can be excluded from the budget. Under async scheduling every - # draft is a -1 placeholder at budget time, so the convention does not - # hold and the filter must not run. - self._count_rejected_drafts = count_rejected_drafts self.query_start_loc = query_start_loc self.cost_tables: tuple[np.ndarray, np.ndarray] | None = None # Largest cudagraph-captured token count; above it nothing pads. @@ -324,7 +301,6 @@ def get_num_tokens( self, num_tokens_per_req: dict[str, int], draft_tokens: dict[str, list[int]], - has_structured_output: bool = False, ) -> int: """Token count once the draft budget is trimmed to fit. @@ -342,16 +318,6 @@ def get_num_tokens( dtype=np.int32, count=num_reqs, ) - valid_drafts = scheduled_drafts - if has_structured_output and self._count_rejected_drafts: - valid_drafts = np.fromiter( - ( - sum(token >= 0 for token in draft_tokens.get(req_id, ())) - for req_id in req_ids - ), - dtype=np.int32, - count=len(req_ids), - ) num_non_draft_tokens = scheduled_tokens - scheduled_drafts slots = np.fromiter( (self.req_states.req_id_to_index[req_id] for req_id in req_ids), @@ -361,11 +327,11 @@ def get_num_tokens( stale_confidences = self._stale_confidences[self._stale_idx].np[slots] survival_probability = np.cumprod(stale_confidences.astype(np.float64), axis=1) steps = np.arange(self.num_speculative_steps) - valid = steps[None, :] < valid_drafts[:, None] + valid = steps[None, :] < scheduled_drafts[:, None] scores = np.sort(survival_probability[valid])[::-1] num_non_draft_tokens_total = int(num_non_draft_tokens.sum()) max_draft_budget = min( - int(valid_drafts.sum()), + int(scheduled_drafts.sum()), max(0, self._max_total_logits - num_reqs * self.num_bonus_tokens), ) scores = scores[:max_draft_budget] @@ -385,9 +351,9 @@ def get_num_tokens( + 1 ] ) - valid_drafts_per_req = { - req_id: int(num_valid_drafts) - for req_id, num_valid_drafts in zip(req_ids, valid_drafts, strict=True) + num_drafts_per_req = { + req_id: int(num_drafts) + for req_id, num_drafts in zip(req_ids, scheduled_drafts, strict=True) } num_non_draft_tokens_per_req = { req_id: int(num_tokens) @@ -395,7 +361,7 @@ def get_num_tokens( } draft_budget = int(np.argmax(num_tokens_to_estimated_accepted_tokens / costs)) self._batch_budget = ( - valid_drafts_per_req, + num_drafts_per_req, num_non_draft_tokens_per_req, draft_budget, ) @@ -429,10 +395,10 @@ def reallocate_drafts( ) -> tuple[torch.Tensor, torch.Tensor, int]: batch_budget, self._batch_budget = self._batch_budget, None assert batch_budget is not None - valid_drafts_per_req, num_non_draft_tokens_per_req, draft_budget = batch_budget + num_drafts_per_req, num_non_draft_tokens_per_req, draft_budget = batch_budget num_reqs = idx_mapping.shape[0] - valid_drafts = np.fromiter( - (valid_drafts_per_req[req_id] for req_id in req_ids), + scheduled_drafts = np.fromiter( + (num_drafts_per_req[req_id] for req_id in req_ids), dtype=np.int32, count=num_reqs, ) @@ -448,8 +414,8 @@ def reallocate_drafts( if draft_budget == 0: capacities.zero_() else: - async_copy_to_gpu(valid_drafts, out=capacities) - if draft_budget != int(valid_drafts.sum()): + async_copy_to_gpu(scheduled_drafts, out=capacities) + if draft_budget != int(scheduled_drafts.sum()): block_size = triton.next_power_of_2( num_reqs * self.num_speculative_steps ) From 5105a5718fff66fa5780ebcb56abe35c926a9034 Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Thu, 30 Jul 2026 22:06:10 +0000 Subject: [PATCH 11/40] [Spec Decode] Drop cudagraph prefill-padding trim from SWA builder Trailing request padding only reaches the DSV4 SWA builder alongside real prefills under a mixed FULL cudagraph, which adaptive verification never selects (it pins FULL_AND_PIECEWISE). Decode graphs pad requests but split_decodes_and_prefills reports zero prefills for them, so the trim never fired. Restore main's behavior. Signed-off-by: Lucas Wilkinson Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016Wv66G1jbqBSxLFcfVxLA5 Signed-off-by: Lucas Wilkinson --- vllm/v1/attention/backends/mla/sparse_swa.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/vllm/v1/attention/backends/mla/sparse_swa.py b/vllm/v1/attention/backends/mla/sparse_swa.py index 6a07256b4245..2a691abe1641 100644 --- a/vllm/v1/attention/backends/mla/sparse_swa.py +++ b/vllm/v1/attention/backends/mla/sparse_swa.py @@ -532,12 +532,6 @@ def build( common_attn_metadata, decode_threshold=self.decode_threshold ) ) - if num_prefills > 0: - # split_decodes_and_prefills counts trailing cudagraph request padding as - # prefills. Those rows have no tokens, so drop them here rather than let - # them size prefill metadata and the chunk plan. - query_lens_cpu = query_start_loc_cpu[1:] - query_start_loc_cpu[:-1] - num_prefills = int(torch.count_nonzero(query_lens_cpu[num_decodes:])) # NOTE: Ensure all metadata tensors maintain fixed memory addresses # for CUDA graph compatibility. @@ -744,9 +738,7 @@ def _build_deepseek_v4_metadata( ) result["prefill_seq_lens"] = seq_lens[num_decodes:] - result["prefill_seq_lens_cpu"] = seq_lens_cpu[ - num_decodes : num_decodes + num_prefills - ] + result["prefill_seq_lens_cpu"] = seq_lens_cpu[num_decodes:] result["prefill_gather_lens"] = pfx_gather_lens result["prefill_query_lens_cpu"] = ( query_start_loc_cpu[num_decodes + 1 : num_decodes + num_prefills + 1] From 5b6ce974235f458cdd0c067ed3e8d6ff6caa40b2 Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Thu, 30 Jul 2026 22:17:23 +0000 Subject: [PATCH 12/40] [Spec Decode] Resolve profiling step timings without a per-step sync Give each timed step its own events and publish the samples in one pass after the profiling runs finish, so the profiler no longer blocks on drafter_end every step. Waiting on the last step's event is enough: timed steps are recorded in issue order on a single stream. Signed-off-by: Lucas Wilkinson Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016Wv66G1jbqBSxLFcfVxLA5 Signed-off-by: Lucas Wilkinson --- vllm/v1/worker/gpu/async_utils.py | 53 +++++++++++++++++++----------- vllm/v1/worker/gpu/model_runner.py | 1 + 2 files changed, 34 insertions(+), 20 deletions(-) diff --git a/vllm/v1/worker/gpu/async_utils.py b/vllm/v1/worker/gpu/async_utils.py index 817385e25b80..f9d2c8c59321 100644 --- a/vllm/v1/worker/gpu/async_utils.py +++ b/vllm/v1/worker/gpu/async_utils.py @@ -43,25 +43,28 @@ class StepTimingCollector: Only enabled alongside a drafter, so every step it times has one. Record calls no-op unless ``start`` armed the collector, so callers need no guards. + Each step gets its own events, so the timed steps queue back-to-back and + ``flush`` resolves them all behind a single sync. """ def __init__(self, enabled: bool): - self._events = ( + self._enabled = enabled + self._armed: StepTimingEvents | None = None + self._batch = (False, 0, 0) + self._pending: list[tuple[StepTimingEvents, tuple[bool, int, int]]] = [] + self.consumer: Callable[[StepTimingSample], None] | None = None + + def start(self, enabled: bool) -> None: + self._armed = ( StepTimingEvents( *( torch.cuda.Event(enable_timing=True) for _ in StepTimingEvents._fields ) ) - if enabled + if enabled and self._enabled else None ) - self._armed: StepTimingEvents | None = None - self._batch = (False, 0, 0) - self.consumer: Callable[[StepTimingSample], None] | None = None - - def start(self, enabled: bool) -> None: - self._armed = self._events if enabled else None def record_batch(self, input_batch: "InputBatch", full_cudagraph: bool) -> None: """Costs from different execution modes must not share a cost curve.""" @@ -85,19 +88,29 @@ def drafter_end(self) -> None: def finish(self) -> None: events, self._armed = self._armed, None - if events is None: + if events is not None: + self._pending.append((events, self._batch)) + + def flush(self) -> None: + """Hand every timed step to the consumer. + + Timed steps are recorded in issue order on one stream, so waiting on + the last drafter event leaves all the earlier ones ready to read. + """ + pending, self._pending = self._pending, [] + if not pending or self.consumer is None: return - events.drafter_end.synchronize() - full_cudagraph, num_target_tokens, num_reqs = self._batch - sample = StepTimingSample( - events.forward_start.elapsed_time(events.forward_end), - events.drafter_start.elapsed_time(events.drafter_end), - num_target_tokens, - num_reqs, - full_cudagraph, - ) - if self.consumer is not None: - self.consumer(sample) + pending[-1][0].drafter_end.synchronize() + for events, (full_cudagraph, num_target_tokens, num_reqs) in pending: + self.consumer( + StepTimingSample( + events.forward_start.elapsed_time(events.forward_end), + events.drafter_start.elapsed_time(events.drafter_end), + num_target_tokens, + num_reqs, + full_cudagraph, + ) + ) class AsyncOutput(AsyncModelRunnerOutput): diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 7605fe232783..9d52c1f9f434 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -884,6 +884,7 @@ def capture_model(self) -> int: self.cudagraph_manager.captured_token_counts() ): self._dummy_run(**batch) + self.step_timing.flush() self.adaptive_verification.set_initial_cost_curves() end_time = time.perf_counter() From 1ea4f64ffa9567f2dc431e176608fd09a2318654 Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Thu, 30 Jul 2026 22:18:01 +0000 Subject: [PATCH 13/40] cleanup Signed-off-by: Lucas Wilkinson --- .../spec_decode/test_adaptive_verification.py | 3 +- .../gpu/spec_decode/adaptive_verification.py | 40 ++++++++++++------- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/tests/v1/spec_decode/test_adaptive_verification.py b/tests/v1/spec_decode/test_adaptive_verification.py index d71936990331..1f9985a71b45 100644 --- a/tests/v1/spec_decode/test_adaptive_verification.py +++ b/tests/v1/spec_decode/test_adaptive_verification.py @@ -17,8 +17,7 @@ def make_manager( num_reqs, num_steps = confidences.shape manager = AdaptiveVerificationManager.__new__(AdaptiveVerificationManager) manager.num_speculative_steps = num_steps - manager._stale_idx = 0 - manager._stale_confidences = [SimpleNamespace(np=confidences)] + manager._confidence_ema = confidences manager.req_states = SimpleNamespace( req_id_to_index={"low": 0, "high": 1}, num_computed_tokens_np=np.ones(num_reqs, dtype=np.int32), diff --git a/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py b/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py index cd5848e6a18f..300c53af5344 100644 --- a/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py +++ b/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py @@ -159,7 +159,8 @@ def __init__( self._profile_samples: list[StepTimingSample] = [] self._batch_budget: tuple[dict[str, int], dict[str, int], int] | None = None max_num_reqs = req_states.max_num_reqs - self._confidence_ema = torch.empty( + # Current per-slot confidences + self._confidence_probs = torch.empty( (max_num_reqs, self.num_speculative_steps), dtype=torch.float32, device=device, @@ -185,6 +186,11 @@ def __init__( self._stale_idx = 0 for slot in self._stale_confidences: slot.np.fill(1.0) + # Smoothed copy, CPU-only: batch-level sizing averages over requests, so + # it benefits from less noise and tolerates the lag. + self._confidence_ema = np.ones( + (max_num_reqs, self.num_speculative_steps), dtype=np.float32 + ) def consume_step_timing(self, sample: StepTimingSample) -> None: """Collect startup profiling timings; only profiling steps are timed.""" @@ -193,7 +199,8 @@ def consume_step_timing(self, sample: StepTimingSample) -> None: def add_request(self, req_idx: int) -> None: self._stale_confidences[self._stale_idx].np[req_idx].fill(1.0) self._pending_resets.append(req_idx) - self._confidence_ema[req_idx].fill_(1.0) + self._confidence_ema[req_idx] = 1.0 + self._confidence_probs[req_idx].fill_(1.0) def batches_to_profile(self, capture_sizes: list[int]) -> Iterator[dict[str, int]]: """Dummy-run kwargs whose step timings seed the cost tables. @@ -268,8 +275,8 @@ def record_confidences( confidence_probs: torch.Tensor, input_batch: "InputBatch", ) -> None: - """Fold this step's confidences into the EMA and start copying them to - the CPU, where a later step's budget reads them as stale inputs.""" + """Publish this step's raw confidences for the ranking kernel and start + copying them to the CPU, where a later step's budget smooths them.""" num_reqs = input_batch.num_reqs ready_idx = self._stale_idx ^ 1 with gpu_sync_allowed(): @@ -280,16 +287,19 @@ def record_confidences( # Last step's copy has landed: budgets read it, this step overwrites the # slot they were reading before. self._stale_idx, write_idx = ready_idx, self._stale_idx + # Fold the freshly-readable raw scores into the CPU-side EMA. New + # requests blend with the optimistic 1.0 seed, which washes out within + # a couple of steps. + alpha = self.confidence_ema_alpha + np.add( + alpha * self._stale_confidences[ready_idx].np, + (1.0 - alpha) * self._confidence_ema, + out=self._confidence_ema, + ) - # Fold this step's scores into the per-request EMA in place. New - # requests blend with the optimistic 1.0 seed, which washes out of the - # average within a couple of steps. - idx_mapping = input_batch.idx_mapping - ema = self._confidence_ema[idx_mapping] - ema.lerp_(confidence_probs[:num_reqs], self.confidence_ema_alpha) - self._confidence_ema[idx_mapping] = ema + self._confidence_probs[input_batch.idx_mapping] = confidence_probs[:num_reqs] write_slot = self._stale_confidences[write_idx] - write_slot.gpu.copy_(self._confidence_ema) + write_slot.gpu.copy_(self._confidence_probs) current_stream = torch.cuda.current_stream(self.req_states.device) self._copy_stream.wait_stream(current_stream) @@ -324,7 +334,7 @@ def get_num_tokens( dtype=np.int32, count=len(req_ids), ) - stale_confidences = self._stale_confidences[self._stale_idx].np[slots] + stale_confidences = self._confidence_ema[slots] survival_probability = np.cumprod(stale_confidences.astype(np.float64), axis=1) steps = np.arange(self.num_speculative_steps) valid = steps[None, :] < scheduled_drafts[:, None] @@ -420,8 +430,8 @@ def reallocate_drafts( num_reqs * self.num_speculative_steps ) _assign_draft_token_budget_kernel[(1,)]( - self._confidence_ema, - self._confidence_ema.stride(0), + self._confidence_probs, + self._confidence_probs.stride(0), idx_mapping, capacities, num_reqs, From 4db5223b5c6beabf31a7030a1bc5ac5eeb9f6bca Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Fri, 31 Jul 2026 02:00:43 +0000 Subject: [PATCH 14/40] cleanup Signed-off-by: Lucas Wilkinson --- .../spec_decode/test_adaptive_verification.py | 27 ++-- vllm/v1/worker/gpu/async_utils.py | 121 +++++++++--------- vllm/v1/worker/gpu/model_runner.py | 29 +---- .../gpu/spec_decode/adaptive_verification.py | 117 ++++++----------- .../gpu/spec_decode/dflash/speculator.py | 5 +- 5 files changed, 122 insertions(+), 177 deletions(-) diff --git a/tests/v1/spec_decode/test_adaptive_verification.py b/tests/v1/spec_decode/test_adaptive_verification.py index 1f9985a71b45..8a29933c21a2 100644 --- a/tests/v1/spec_decode/test_adaptive_verification.py +++ b/tests/v1/spec_decode/test_adaptive_verification.py @@ -53,27 +53,23 @@ def test_profiled_batches_seed_cost_curves_via_consumer(): manager.req_states = SimpleNamespace(max_num_batched_tokens=4096, max_num_reqs=64) manager.num_speculative_steps = 7 manager.num_bonus_tokens = 1 - manager._profile_samples = [] curves: dict[str, list[tuple[int, float]]] = {} manager.set_cost_curves = lambda draft, verify: curves.update( draft=draft, verify=verify ) - for batch in manager.batches_to_profile([8, 1024]): - assert batch["timing_enabled"] - num_tokens = batch["num_tokens"] - manager.consume_step_timing( - StepTimingSample( - forward_ms=float(num_tokens), - drafter_ms=1.0, - num_target_tokens=num_tokens, - num_reqs=num_tokens // 8, - # Only the captured sizes replay a graph; the tail sizes - # run eager. - full_cudagraph=num_tokens <= 1024, - ) + timings = [ + StepTimingSample( + forward_ms=float(batch["num_tokens"]), + drafter_ms=1.0, + num_target_tokens=batch["num_tokens"], + num_reqs=batch["num_tokens"] // 8, + # Only the captured sizes replay a graph; the tail sizes run eager. + full_cudagraph=batch["num_tokens"] <= 1024, ) - manager.set_initial_cost_curves() + for batch in manager.batches_to_profile([8, 1024]) + ] + manager.set_initial_cost_curves(timings) # Tail beyond the last capture size: 1.5x then doubling to the max. assert curves["verify"] == [ @@ -87,7 +83,6 @@ def test_profiled_batches_seed_cost_curves_via_consumer(): # count they would land inside the captured range and, once made monotonic, # smear that eager cost across every larger request count. assert curves["draft"] == [(1, 1.0), (128, 1.0)] - assert manager._profile_samples == [] def test_compact_batch_preserves_totals_and_bounds(): diff --git a/vllm/v1/worker/gpu/async_utils.py b/vllm/v1/worker/gpu/async_utils.py index f9d2c8c59321..ef47b50a8309 100644 --- a/vllm/v1/worker/gpu/async_utils.py +++ b/vllm/v1/worker/gpu/async_utils.py @@ -1,9 +1,9 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import contextlib -from collections.abc import Callable -from dataclasses import dataclass -from typing import TYPE_CHECKING, NamedTuple +from collections.abc import Iterator +from dataclasses import dataclass, field +from typing import TYPE_CHECKING import numpy as np import torch @@ -31,39 +31,58 @@ class StepTimingSample: full_cudagraph: bool -class StepTimingEvents(NamedTuple): - forward_start: torch.cuda.Event - forward_end: torch.cuda.Event - drafter_start: torch.cuda.Event - drafter_end: torch.cuda.Event +def _timing_event() -> torch.cuda.Event: + return torch.cuda.Event(enable_timing=True) + + +@dataclass +class StepTimingEvents: + forward_start: torch.cuda.Event = field(default_factory=_timing_event) + forward_end: torch.cuda.Event = field(default_factory=_timing_event) + drafter_start: torch.cuda.Event = field(default_factory=_timing_event) + drafter_end: torch.cuda.Event = field(default_factory=_timing_event) class StepTimingCollector: - """Times startup profiling steps, which run strictly one at a time. + """Times the steps run inside ``collect``; record calls no-op outside it. - Only enabled alongside a drafter, so every step it times has one. Record - calls no-op unless ``start`` armed the collector, so callers need no guards. - Each step gets its own events, so the timed steps queue back-to-back and - ``flush`` resolves them all behind a single sync. + Every step gets its own events, so the steps queue back-to-back and the + block resolves them all behind one sync on the way out. """ - def __init__(self, enabled: bool): - self._enabled = enabled - self._armed: StepTimingEvents | None = None + def __init__(self): + self._collecting = False + self._step: StepTimingEvents | None = None self._batch = (False, 0, 0) - self._pending: list[tuple[StepTimingEvents, tuple[bool, int, int]]] = [] - self.consumer: Callable[[StepTimingSample], None] | None = None - - def start(self, enabled: bool) -> None: - self._armed = ( - StepTimingEvents( - *( - torch.cuda.Event(enable_timing=True) - for _ in StepTimingEvents._fields - ) + self._timed: list[tuple[StepTimingEvents, tuple[bool, int, int]]] = [] + + @contextlib.contextmanager + def collect(self) -> Iterator[list[StepTimingSample]]: + """Time every step run in this block. + + The yielded list holds one sample per timed step once the block exits; + it stays empty inside the block, where the timings are still on device. + """ + samples: list[StepTimingSample] = [] + self._collecting = True + try: + yield samples + finally: + self._collecting = False + timed, self._timed, self._step = self._timed, [], None + if not timed: + return + # Same stream, issue order: once the last step is done, so are the rest. + timed[-1][0].drafter_end.synchronize() + samples.extend( + StepTimingSample( + events.forward_start.elapsed_time(events.forward_end), + events.drafter_start.elapsed_time(events.drafter_end), + num_target_tokens, + num_reqs, + full_cudagraph, ) - if enabled and self._enabled - else None + for events, (full_cudagraph, num_target_tokens, num_reqs) in timed ) def record_batch(self, input_batch: "InputBatch", full_cudagraph: bool) -> None: @@ -71,46 +90,24 @@ def record_batch(self, input_batch: "InputBatch", full_cudagraph: bool) -> None: self._batch = (full_cudagraph, input_batch.num_tokens, input_batch.num_reqs) def forward_start(self) -> None: - if self._armed is not None: - self._armed.forward_start.record() + if self._collecting: + self._step = StepTimingEvents() + self._step.forward_start.record() def forward_end(self) -> None: - if self._armed is not None: - self._armed.forward_end.record() + if self._step is not None: + self._step.forward_end.record() def drafter_start(self) -> None: - if self._armed is not None: - self._armed.drafter_start.record() + if self._step is not None: + self._step.drafter_start.record() def drafter_end(self) -> None: - if self._armed is not None: - self._armed.drafter_end.record() - - def finish(self) -> None: - events, self._armed = self._armed, None - if events is not None: - self._pending.append((events, self._batch)) - - def flush(self) -> None: - """Hand every timed step to the consumer. - - Timed steps are recorded in issue order on one stream, so waiting on - the last drafter event leaves all the earlier ones ready to read. - """ - pending, self._pending = self._pending, [] - if not pending or self.consumer is None: - return - pending[-1][0].drafter_end.synchronize() - for events, (full_cudagraph, num_target_tokens, num_reqs) in pending: - self.consumer( - StepTimingSample( - events.forward_start.elapsed_time(events.forward_end), - events.drafter_start.elapsed_time(events.drafter_end), - num_target_tokens, - num_reqs, - full_cudagraph, - ) - ) + """Ends the step: only steps that reach here have a draft cost.""" + if self._step is not None: + self._step.drafter_end.record() + self._timed.append((self._step, self._batch)) + self._step = None class AsyncOutput(AsyncModelRunnerOutput): diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 9d52c1f9f434..165caa3b8b69 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -257,12 +257,7 @@ def __init__(self, vllm_config: VllmConfig, device: torch.device): else 1 ) - self.step_timing = StepTimingCollector( - self.is_last_pp_rank - and not self.is_pooling_model - and self.speculative_config is not None - and self.speculative_config.use_adaptive_verification - ) + self.step_timing = StepTimingCollector() # General request states. self.req_states = RequestState( @@ -535,9 +530,6 @@ def initialize_kv_cache(self, kv_cache_config: KVCacheConfig) -> None: self.speculative_config.adaptive_verification_ema_alpha, max_total_logits=max_chunk_logits(self.vocab_size), ) - self.step_timing.consumer = ( - self.adaptive_verification.consume_step_timing - ) else: # Hybrid model states derive per-request structure from the # scheduled counts, which are placeholders under trimming. @@ -636,7 +628,6 @@ def _dummy_run( context_len: int = 0, skip_eplb: bool = False, is_profile: bool = False, - timing_enabled: bool = False, **kwargs, ) -> tuple[torch.Tensor | None, torch.Tensor | None]: if self.is_encoder_only: @@ -697,7 +688,6 @@ def _dummy_run( skip_attn_for_dummy_run=skip_attn, is_profile=is_profile, context_len=context_len, - timing_enabled=timing_enabled, ) self.kv_connector.set_disabled(False) @@ -761,8 +751,6 @@ def _dummy_run( ) self.step_timing.drafter_end() - self.step_timing.finish() - assert hidden_states is not None # Last PP rank always has hidden_states sample_hidden_states = hidden_states[input_batch.logits_indices] return hidden_states, sample_hidden_states @@ -880,12 +868,12 @@ def capture_model(self) -> int: if self.speculator is not None: self.speculator.capture() if self.adaptive_verification is not None: - for batch in self.adaptive_verification.batches_to_profile( - self.cudagraph_manager.captured_token_counts() - ): - self._dummy_run(**batch) - self.step_timing.flush() - self.adaptive_verification.set_initial_cost_curves() + with self.step_timing.collect() as timings: + for batch in self.adaptive_verification.batches_to_profile( + self.cudagraph_manager.captured_token_counts() + ): + self._dummy_run(**batch) + self.adaptive_verification.set_initial_cost_curves(timings) end_time = time.perf_counter() end_free_gpu_memory = torch.accelerator.get_memory_info()[0] @@ -1357,7 +1345,6 @@ def execute_model( skip_attn_for_dummy_run: bool = False, is_profile: bool = False, context_len: int = 0, - timing_enabled: bool = False, ) -> ModelRunnerOutput | IntermediateTensors | None: if not dummy_run: # Update the request states. @@ -1433,8 +1420,6 @@ def execute_model( empty_output = self.kv_connector.no_forward(scheduler_output) return empty_output - self.step_timing.start(timing_enabled) - if not dummy_run: # Common case. # Prepare all the inputs and copy to the input buffers. diff --git a/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py b/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py index 300c53af5344..79140882350e 100644 --- a/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py +++ b/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py @@ -11,7 +11,6 @@ from vllm.distributed.parallel_state import get_tp_group from vllm.logger import init_logger -from vllm.triton_utils import tl, triton from vllm.utils.gpu_sync_debug import gpu_sync_allowed from vllm.v1.utils import CpuGpuBuffer from vllm.v1.worker.gpu.async_utils import StepTimingSample, stream @@ -28,64 +27,43 @@ from vllm.v1.worker.gpu.states import RequestState -@triton.jit -def _keyed_product(left_value, left_key, right_value, right_key): - value = tl.where(left_key == right_key, left_value * right_value, right_value) - return value, right_key - - -@triton.jit(do_not_specialize=["num_reqs", "draft_token_budget"]) -def _assign_draft_token_budget_kernel( - confidence_probs_ptr, - confidence_probs_stride, - idx_mapping_ptr, - capacities_ptr, - num_reqs, - draft_token_budget, - NUM_STEPS: tl.constexpr, - BLOCK_SIZE: tl.constexpr, -): - candidate_idx = tl.arange(0, BLOCK_SIZE) - req_idx = candidate_idx // NUM_STEPS - step = candidate_idx % NUM_STEPS - valid_candidate = req_idx < num_reqs - req_state_idx = tl.load(idx_mapping_ptr + req_idx, mask=valid_candidate, other=0) - probability = tl.load( - confidence_probs_ptr + req_state_idx * confidence_probs_stride + step, - mask=valid_candidate, - other=0.0, - ) - survival, _ = tl.associative_scan( - (probability, req_idx), - axis=0, - combine_fn=_keyed_product, - ) - num_valid = tl.load( - capacities_ptr + req_idx, - mask=valid_candidate, - other=0, - ) - survival = tl.where(valid_candidate & (step < num_valid), survival, -float("inf")) - - min_int32: tl.constexpr = -2147483648 - score_bits = survival.to(tl.int32, bitcast=True) - sort_key = tl.where( - score_bits >> 31 == 0, - score_bits ^ -1, - score_bits ^ min_int32, - ) - packed = ((sort_key.to(tl.int64) & 0xFFFFFFFF) << 32) | candidate_idx.to(tl.int64) - admitted_idx = (tl.sort(packed) & 0xFFFFFFFF).to(tl.int32) - - tl.store(capacities_ptr + candidate_idx, 0, mask=candidate_idx < num_reqs) - tl.debug_barrier() - tl.atomic_add( - capacities_ptr + admitted_idx // NUM_STEPS, - 1, - mask=candidate_idx < draft_token_budget, +def _assign_draft_token_budget( + confidence_probs: torch.Tensor, + idx_mapping: torch.Tensor, + capacities: torch.Tensor, + draft_budget: int, + num_steps: int, +) -> None: + """Admit the globally best `draft_budget` draft slots, in place. + + Every (request, step) slot is scored by its survival probability, the running + product of that request's per-position confidences, and the highest scores win. + Survival only decreases along a request, so a global top-k always admits + continuously along steps with a request. + + `capacities` enters holding each request's scheduled draft count (which bounds its + eligible slots) and leaves holding the admitted count. + """ + survival = confidence_probs[idx_mapping].cumprod(dim=1) + steps = torch.arange(num_steps, device=survival.device) + valid = steps[None, :] < capacities[:, None] + survival = survival.masked_fill(~valid, -float("inf")) + order = survival.flatten().argsort(descending=True, stable=True) + # Invalid slots score -inf and sort last, so the valid ones occupy exactly the first + # valid.sum() ranks; everything between the budget and there is a real draft that + # missed the cut. Decrementing keeps this sync-free. + ranks = torch.arange(order.shape[0], device=survival.device) + rejected = (ranks >= draft_budget) & (ranks < valid.sum()) + capacities.scatter_add_( + 0, order // num_steps, torch.where(rejected, -1, 0).to(capacities.dtype) ) +_assign_draft_token_budget_compiled = torch.compile( + _assign_draft_token_budget, dynamic=True +) + + def build_cost_tables_from_curves( draft_curve: list[tuple[int, float]], verify_curve: list[tuple[int, float]], @@ -156,7 +134,6 @@ def __init__( self.cost_tables: tuple[np.ndarray, np.ndarray] | None = None # Largest cudagraph-captured token count; above it nothing pads. self._cudagraph_limit = 0 - self._profile_samples: list[StepTimingSample] = [] self._batch_budget: tuple[dict[str, int], dict[str, int], int] | None = None max_num_reqs = req_states.max_num_reqs # Current per-slot confidences @@ -192,10 +169,6 @@ def __init__( (max_num_reqs, self.num_speculative_steps), dtype=np.float32 ) - def consume_step_timing(self, sample: StepTimingSample) -> None: - """Collect startup profiling timings; only profiling steps are timed.""" - self._profile_samples.append(sample) - def add_request(self, req_idx: int) -> None: self._stale_confidences[self._stale_idx].np[req_idx].fill(1.0) self._pending_resets.append(req_idx) @@ -205,8 +178,8 @@ def add_request(self, req_idx: int) -> None: def batches_to_profile(self, capture_sizes: list[int]) -> Iterator[dict[str, int]]: """Dummy-run kwargs whose step timings seed the cost tables. - Timings flow back through consume_step_timing; call - set_initial_cost_curves once every batch has run.""" + Run these inside StepTimingCollector.collect(), then hand the block's + timings to set_initial_cost_curves.""" max_num_tokens = self.req_states.max_num_batched_tokens size = self._cudagraph_limit = capture_sizes[-1] if capture_sizes else 0 # Also profile beyond the capture limit: real steps run there @@ -224,12 +197,9 @@ def batches_to_profile(self, capture_sizes: list[int]) -> Iterator[dict[str, int yield { "num_tokens": num_tokens, "context_len": _PROFILE_CONTEXT_LEN, - "timing_enabled": True, } - def set_initial_cost_curves(self) -> None: - samples, self._profile_samples = self._profile_samples, [] - + def set_initial_cost_curves(self, samples: list[StepTimingSample]) -> None: def median_curve( points: Iterable[tuple[int, float]], ) -> list[tuple[int, float]]: @@ -420,25 +390,20 @@ def reallocate_drafts( num_tokens = int(num_non_draft_tokens.sum()) + draft_budget # Rank draft slots by survival probability and admit the best prefix. + # capacities enters holding each request's valid draft count (the kernel + # uses it to bound eligible slots) and leaves holding the admitted count. capacities = self._batch_draft_capacity[:num_reqs] if draft_budget == 0: capacities.zero_() else: async_copy_to_gpu(scheduled_drafts, out=capacities) if draft_budget != int(scheduled_drafts.sum()): - block_size = triton.next_power_of_2( - num_reqs * self.num_speculative_steps - ) - _assign_draft_token_budget_kernel[(1,)]( + _assign_draft_token_budget_compiled( self._confidence_probs, - self._confidence_probs.stride(0), idx_mapping, capacities, - num_reqs, draft_budget, - NUM_STEPS=self.num_speculative_steps, - BLOCK_SIZE=block_size, - num_warps=4 if block_size <= 256 else 8, + self.num_speculative_steps, ) num_non_draft_tokens_gpu = self._num_non_draft_tokens[:num_reqs] diff --git a/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py b/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py index 2c160a75a37a..e477f5d98312 100644 --- a/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py @@ -594,7 +594,10 @@ def _prepare_dflash_inputs_kernel( # seq_lens is the absolute sequence length the draft attention # reads up to (context + query), not just the count of accepted # tokens this step. - tl.store(out_seq_lens_ptr + req_idx, last_valid_pos + 1 + num_query_per_req) + tl.store( + out_seq_lens_ptr + req_idx, + tl.minimum(last_valid_pos + 1 + num_query_per_req, max_model_len), + ) # Copy sampling state. tl.store( out_temperature_ptr + req_state_idx, From aad37e15aad1ea6a490ec3c51b258ea384f10ae7 Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Fri, 31 Jul 2026 14:15:37 +0000 Subject: [PATCH 15/40] fix precommit Signed-off-by: Lucas Wilkinson --- vllm/models/deepseek_v4/sparse_mla.py | 4 ++-- vllm/v1/attention/backends/mla/sparse_swa.py | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/vllm/models/deepseek_v4/sparse_mla.py b/vllm/models/deepseek_v4/sparse_mla.py index a9d78aca1a8e..76aeffc085f6 100644 --- a/vllm/models/deepseek_v4/sparse_mla.py +++ b/vllm/models/deepseek_v4/sparse_mla.py @@ -23,7 +23,7 @@ ) from vllm.v1.attention.backends.mla.compressor_utils import get_compressed_slot_mapping from vllm.v1.attention.backends.utils import split_decodes_and_prefills -from vllm.v1.kv_cache_interface import AttentionSpec +from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheSpec # Pad C128A topk width to this alignment. 128 covers both h_q=64 (B_TOPK=64) and # h_q=128 (B_TOPK=128). FlashMLA decode asserts extra_topk % B_TOPK == 0; @@ -139,7 +139,7 @@ class DeepseekV4FlashMLAMetadataBuilder( def get_cudagraph_support( cls, vllm_config: VllmConfig, - kv_cache_spec: AttentionSpec, + kv_cache_spec: KVCacheSpec, ) -> AttentionCGSupport: if current_platform.is_cuda() and current_platform.is_device_capability_family( 100 diff --git a/vllm/v1/attention/backends/mla/sparse_swa.py b/vllm/v1/attention/backends/mla/sparse_swa.py index 2a691abe1641..cf1e8013ce11 100644 --- a/vllm/v1/attention/backends/mla/sparse_swa.py +++ b/vllm/v1/attention/backends/mla/sparse_swa.py @@ -25,7 +25,6 @@ from vllm.v1.attention.backends.utils import split_decodes_and_prefills from vllm.v1.attention.ops.flashmla import FlashMLASchedMeta, get_mla_metadata from vllm.v1.kv_cache_interface import ( - AttentionSpec, KVCacheSpec, MLAAttentionSpec, SlidingWindowMLASpec, @@ -406,7 +405,7 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder): def get_cudagraph_support( cls, vllm_config: VllmConfig, - kv_cache_spec: AttentionSpec, + kv_cache_spec: KVCacheSpec, ) -> AttentionCGSupport: if current_platform.is_cuda() and current_platform.is_device_capability_family( 100 From ea368ee36761c08762f67616ce8afd8bc721a5f7 Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Fri, 31 Jul 2026 14:19:02 +0000 Subject: [PATCH 16/40] [Spec Decode] Only require a topk buffer on compressed DSV4 layers The warmup workspace reservation asserted topk_indices_buffer on every layer, but reads it only when compress_ratio > 1. DSpark draft layers are SWA-only and are built without one, so memory profiling died on the assert before the engine could start. Scope it to the branch that uses it. Signed-off-by: Lucas Wilkinson Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016Wv66G1jbqBSxLFcfVxLA5 Signed-off-by: Lucas Wilkinson --- vllm/models/deepseek_v4/nvidia/flashmla.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/vllm/models/deepseek_v4/nvidia/flashmla.py b/vllm/models/deepseek_v4/nvidia/flashmla.py index 29ec2d6f2abf..31eb4f90002e 100644 --- a/vllm/models/deepseek_v4/nvidia/flashmla.py +++ b/vllm/models/deepseek_v4/nvidia/flashmla.py @@ -96,8 +96,12 @@ def forward_mqa( // self.compress_ratio ) M = N + self.window_size + self.max_num_batched_tokens - assert self.topk_indices_buffer is not None - top_k = 0 if swa_only else self.topk_indices_buffer.shape[-1] + # SWA-only layers (e.g. the DSpark draft) never allocate a topk + # buffer and never read top_k here, so only require it when set. + top_k = 0 + if not swa_only: + assert self.topk_indices_buffer is not None + top_k = self.topk_indices_buffer.shape[-1] combined_topk = round_up(top_k + self.window_size, 128) current_workspace_manager().get_simultaneous( ((self.PREFILL_CHUNK_SIZE, M, q.shape[-1]), torch.bfloat16), From 45a820121f494823e1bed6f38e525fa884899c60 Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Fri, 31 Jul 2026 14:38:04 +0000 Subject: [PATCH 17/40] Revert "[Spec Decode] Only require a topk buffer on compressed DSV4 layers" This reverts commit 84b033d18ea5899f1d34f8978e780e4694d30d5c. Signed-off-by: Lucas Wilkinson --- vllm/models/deepseek_v4/nvidia/flashmla.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/vllm/models/deepseek_v4/nvidia/flashmla.py b/vllm/models/deepseek_v4/nvidia/flashmla.py index 31eb4f90002e..29ec2d6f2abf 100644 --- a/vllm/models/deepseek_v4/nvidia/flashmla.py +++ b/vllm/models/deepseek_v4/nvidia/flashmla.py @@ -96,12 +96,8 @@ def forward_mqa( // self.compress_ratio ) M = N + self.window_size + self.max_num_batched_tokens - # SWA-only layers (e.g. the DSpark draft) never allocate a topk - # buffer and never read top_k here, so only require it when set. - top_k = 0 - if not swa_only: - assert self.topk_indices_buffer is not None - top_k = self.topk_indices_buffer.shape[-1] + assert self.topk_indices_buffer is not None + top_k = 0 if swa_only else self.topk_indices_buffer.shape[-1] combined_topk = round_up(top_k + self.window_size, 128) current_workspace_manager().get_simultaneous( ((self.PREFILL_CHUNK_SIZE, M, q.shape[-1]), torch.bfloat16), From e5655838222bf065ed0ac535279b067424ea8b54 Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Fri, 31 Jul 2026 14:51:33 +0000 Subject: [PATCH 18/40] [Spec Decode] Require a topk buffer only where it is read The warmup workspace reservation (#50298) asserts topk_indices_buffer on every layer but dereferences it only when compress_ratio > 1. SWA-only layers, including the DSpark draft, are built without one, so memory profiling died on the assert before the engine could start. Signed-off-by: Lucas Wilkinson Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016Wv66G1jbqBSxLFcfVxLA5 Signed-off-by: Lucas Wilkinson --- vllm/models/deepseek_v4/nvidia/flashmla.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm/models/deepseek_v4/nvidia/flashmla.py b/vllm/models/deepseek_v4/nvidia/flashmla.py index 29ec2d6f2abf..ff15ef3b4534 100644 --- a/vllm/models/deepseek_v4/nvidia/flashmla.py +++ b/vllm/models/deepseek_v4/nvidia/flashmla.py @@ -96,7 +96,7 @@ def forward_mqa( // self.compress_ratio ) M = N + self.window_size + self.max_num_batched_tokens - assert self.topk_indices_buffer is not None + assert swa_only or self.topk_indices_buffer is not None top_k = 0 if swa_only else self.topk_indices_buffer.shape[-1] combined_topk = round_up(top_k + self.window_size, 128) current_workspace_manager().get_simultaneous( From e6e8bea24e7a2c4486688307f3c3155765ea409a Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Fri, 31 Jul 2026 18:58:54 +0000 Subject: [PATCH 19/40] minor cleanup Signed-off-by: Lucas Wilkinson --- vllm/v1/worker/gpu/block_table.py | 10 +++------- vllm/v1/worker/gpu/model_runner.py | 25 ++++++++----------------- 2 files changed, 11 insertions(+), 24 deletions(-) diff --git a/vllm/v1/worker/gpu/block_table.py b/vllm/v1/worker/gpu/block_table.py index dc37b12f2aad..9c4eccc0f9c6 100644 --- a/vllm/v1/worker/gpu/block_table.py +++ b/vllm/v1/worker/gpu/block_table.py @@ -179,13 +179,9 @@ def get_dummy_block_tables(self, num_reqs: int) -> tuple[torch.Tensor, ...]: def set_dummy_context( self, num_reqs: int, query_len: int, context_len: int, num_kv_blocks: int ) -> int: - """Give each dummy request a disjoint span of KV blocks. - - Used when profiling step cost, so the profiled step reads a realistic - amount of cache instead of attending over nothing. - - Returns: - The context length actually used, clamped to what the cache holds. + """ + Give each dummy request a disjoint span of KV blocks. + Used when profiling step cost. """ for bpk, block_size in zip(self.blocks_per_kv_block, self.kernel_block_sizes): capacity = (num_kv_blocks * bpk // num_reqs) * block_size diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 165caa3b8b69..c00874162923 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -117,7 +117,6 @@ from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache from vllm.v1.worker.gpu.mm.lora import set_active_mm_loras from vllm.v1.worker.gpu.model_states import init_model_state -from vllm.v1.worker.gpu.model_states.default import DefaultModelState from vllm.v1.worker.gpu.pool.pooling_runner import PoolingRunner from vllm.v1.worker.gpu.pp_utils import PPHandler from vllm.v1.worker.gpu.sample.output import SamplerOutput @@ -522,22 +521,14 @@ def initialize_kv_cache(self, kv_cache_config: KVCacheConfig) -> None: # its flag at load time). and getattr(self.speculator, "use_adaptive_verification", False) ): - if isinstance(self.model_state, DefaultModelState): - self.adaptive_verification = AdaptiveVerificationManager( - self.req_states, - self.input_buffers.query_start_loc, - self.model_state.num_new_sampled_tokens_per_step, - self.speculative_config.adaptive_verification_ema_alpha, - max_total_logits=max_chunk_logits(self.vocab_size), - ) - else: - # Hybrid model states derive per-request structure from the - # scheduled counts, which are placeholders under trimming. - logger.warning_once( - "DSpark adaptive verification is not supported with %s; " - "using fixed-length verification.", - type(self.model_state).__name__, - ) + self.adaptive_verification = AdaptiveVerificationManager( + self.req_states, + self.input_buffers.query_start_loc, + self.model_state.num_new_sampled_tokens_per_step, + self.speculative_config.adaptive_verification_ema_alpha, + max_total_logits=max_chunk_logits(self.vocab_size), + ) + self.block_tables = BlockTables( block_sizes=block_sizes, max_num_reqs=self.max_num_reqs, From da62b61ded8b90df7749faaec3885e3960c0503f Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Fri, 31 Jul 2026 19:00:56 +0000 Subject: [PATCH 20/40] cleanup Signed-off-by: Lucas Wilkinson --- vllm/v1/worker/gpu/cudagraph_utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/vllm/v1/worker/gpu/cudagraph_utils.py b/vllm/v1/worker/gpu/cudagraph_utils.py index d6042cfb082b..78d4f705310b 100644 --- a/vllm/v1/worker/gpu/cudagraph_utils.py +++ b/vllm/v1/worker/gpu/cudagraph_utils.py @@ -239,8 +239,7 @@ def _init_candidates(self) -> None: capture_sizes, self.lora_capture_cases ): # Varlen decode graphs take any mix of 1..decode_query_len tokens per - # request, so capture on the token grid instead of rounding up to a - # uniform query length. + # request, worst case 1 token per request (or max_num_reqs) if capture_varlen_decode and num_tokens <= max_decode_tokens: desc = BatchExecutionDescriptor( cg_mode=decode_mode, From af4751b594090c428dd368ae7fce43cafff73a23 Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Tue, 4 Aug 2026 17:19:05 +0000 Subject: [PATCH 21/40] wip: review comments Signed-off-by: Lucas Wilkinson --- .../spec_decode/test_adaptive_verification.py | 3 +- vllm/config/speculative.py | 10 ----- vllm/models/deepseek_v4/amd/rocm.py | 8 ++-- vllm/models/deepseek_v4/attention.py | 3 ++ .../deepseek_v4/nvidia/flashinfer_sparse.py | 9 ++-- vllm/models/deepseek_v4/nvidia/flashmla.py | 20 ++++++++- vllm/models/deepseek_v4/sparse_mla.py | 45 +++++++++---------- vllm/models/deepseek_v4/xpu/xpu_sparse.py | 4 +- vllm/sampling_params.py | 3 ++ vllm/v1/attention/backends/mla/sparse_swa.py | 19 +++----- vllm/v1/worker/gpu/model_runner.py | 1 - .../gpu/spec_decode/adaptive_verification.py | 21 +-------- .../gpu/spec_decode/dflash/speculator.py | 16 +++---- 13 files changed, 71 insertions(+), 91 deletions(-) diff --git a/tests/v1/spec_decode/test_adaptive_verification.py b/tests/v1/spec_decode/test_adaptive_verification.py index 8a29933c21a2..bc2bf35f8073 100644 --- a/tests/v1/spec_decode/test_adaptive_verification.py +++ b/tests/v1/spec_decode/test_adaptive_verification.py @@ -17,7 +17,8 @@ def make_manager( num_reqs, num_steps = confidences.shape manager = AdaptiveVerificationManager.__new__(AdaptiveVerificationManager) manager.num_speculative_steps = num_steps - manager._confidence_ema = confidences + manager._stale_confidences = [SimpleNamespace(np=confidences)] + manager._stale_idx = 0 manager.req_states = SimpleNamespace( req_id_to_index={"low": 0, "high": 1}, num_computed_tokens_np=np.ones(num_reqs, dtype=np.int32), diff --git a/vllm/config/speculative.py b/vllm/config/speculative.py index c52b8a06e076..714bb99b95d2 100644 --- a/vllm/config/speculative.py +++ b/vllm/config/speculative.py @@ -240,13 +240,6 @@ class SpeculativeConfig: """Whether to adaptively size the draft-verification budget from per-request confidence. Only applies to method="dspark".""" - adaptive_verification_ema_alpha: float = 0.8 - """Weight of the latest DSpark confidence observation in the per-request - exponential moving average. Set to 1 to disable smoothing. Smoothing trades - less noise for more lag; lag does not average out across a batch, so - heavier smoothing helps small batches but hurts large ones. 0.8 is the - point where the estimator is unbiased at every batch size measured.""" - @property def use_adaptive_verification(self) -> bool: return self.method == "dspark" and self.enable_adaptive_verification @@ -1347,9 +1340,6 @@ def _verify_args(self) -> Self: f"than zero ({self.num_speculative_tokens})." ) - if not 0 < self.adaptive_verification_ema_alpha <= 1: - raise ValueError("adaptive_verification_ema_alpha must be in (0, 1].") - if self.rejection_sample_method == "synthetic": # Consolidate to per-position rates self.synthetic_acceptance_rates = self._resolve_synthetic_acceptance_rates( diff --git a/vllm/models/deepseek_v4/amd/rocm.py b/vllm/models/deepseek_v4/amd/rocm.py index 7c232fe9e528..33479c3c889f 100644 --- a/vllm/models/deepseek_v4/amd/rocm.py +++ b/vllm/models/deepseek_v4/amd/rocm.py @@ -14,9 +14,9 @@ from vllm.models.deepseek_v4.attention import DeepseekV4Attention from vllm.models.deepseek_v4.common.ops import dequantize_and_gather_k_cache from vllm.models.deepseek_v4.sparse_mla import ( - DeepseekV4FlashMLABackend, DeepseekV4FlashMLAMetadata, - DeepseekV4FlashMLAMetadataBuilder, + DeepseekV4SparseMLABackend, + DeepseekV4SparseMLAMetadataBuilder, ) from vllm.platforms import current_platform from vllm.triton_utils import tl, triton @@ -314,7 +314,7 @@ class DeepseekV4ROCMAiterSparseSWAMetadata(DeepseekSparseSWAMetadata): decode_swa_ragged_indptr: torch.Tensor | None = None -class DeepseekV4ROCMAiterMLASparseMetadataBuilder(DeepseekV4FlashMLAMetadataBuilder): +class DeepseekV4ROCMAiterMLASparseMetadataBuilder(DeepseekV4SparseMLAMetadataBuilder): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.c128a_decode_topk_ragged_indices_buffer: torch.Tensor | None = None @@ -431,7 +431,7 @@ def build( ) -class DeepseekV4ROCMAiterMLASparseBackend(DeepseekV4FlashMLABackend): +class DeepseekV4ROCMAiterMLASparseBackend(DeepseekV4SparseMLABackend): @staticmethod def get_name() -> str: return "ROCM_FLASHMLA_SPARSE_DSV4" diff --git a/vllm/models/deepseek_v4/attention.py b/vllm/models/deepseek_v4/attention.py index c56a36bcd2ff..79034e9ed08b 100644 --- a/vllm/models/deepseek_v4/attention.py +++ b/vllm/models/deepseek_v4/attention.py @@ -134,6 +134,8 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): # Provided by the platform subclass. backend_cls: ClassVar[type[AttentionBackend]] + # Backend for the SWA cache layer; None uses the default SWA backend. + swa_backend_cls: ClassVar[type[AttentionBackend] | None] = None # KV-cache per-token block format (both layouts are paged). True (default) # = fp8_ds_mla (UE8M0 block-scaled fp8 packed as uint8); False = plain # bf16 / per-tensor fp8 KV row. Backends can override the instance hook when @@ -322,6 +324,7 @@ def __init__( dtype=self.kv_cache_torch_dtype, prefix=f"{prefix}.swa_cache", cache_config=cache_config, + backend_cls=self.swa_backend_cls, ) # Register with compilation context for metadata lookup. diff --git a/vllm/models/deepseek_v4/nvidia/flashinfer_sparse.py b/vllm/models/deepseek_v4/nvidia/flashinfer_sparse.py index 0de5c4479c03..27031e02c577 100644 --- a/vllm/models/deepseek_v4/nvidia/flashinfer_sparse.py +++ b/vllm/models/deepseek_v4/nvidia/flashinfer_sparse.py @@ -18,8 +18,8 @@ deep_gemm_fp8_o_proj, ) from vllm.models.deepseek_v4.sparse_mla import ( - DeepseekV4FlashMLABackend, DeepseekV4FlashMLAMetadata, + DeepseekV4SparseMLABackend, ) from vllm.platforms import current_platform from vllm.platforms.interface import DeviceCapability @@ -74,11 +74,10 @@ def _pad_to_supported_q_heads(num_heads: int) -> int: ) -class DeepseekV4FlashInferMLASparseBackend(DeepseekV4FlashMLABackend): +class DeepseekV4FlashInferMLASparseBackend(DeepseekV4SparseMLABackend): """FlashInfer backend using the DSv4 sparse metadata/cache layout. - Inheriting from the FlashMLA V4 backend reuses its - ``DeepseekV4FlashMLAMetadata`` builder. + Inherits the base and backend reuses its``DeepseekV4SparseMLAMetadataBuilder`` """ supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16] @@ -159,7 +158,7 @@ def get_kv_cache_shape( ) -> tuple[int, ...]: device_capability = current_platform.get_device_capability() if device_capability is not None and device_capability.major == 12: - return DeepseekV4FlashMLABackend.get_kv_cache_shape( + return DeepseekV4SparseMLABackend.get_kv_cache_shape( num_blocks, block_size, num_kv_heads, diff --git a/vllm/models/deepseek_v4/nvidia/flashmla.py b/vllm/models/deepseek_v4/nvidia/flashmla.py index ff15ef3b4534..e08ed67334c1 100644 --- a/vllm/models/deepseek_v4/nvidia/flashmla.py +++ b/vllm/models/deepseek_v4/nvidia/flashmla.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from typing import TYPE_CHECKING, cast +from typing import TYPE_CHECKING, ClassVar, cast import torch @@ -21,6 +21,11 @@ DeepseekV4FlashMLAMetadata, ) from vllm.utils.math_utils import round_up +from vllm.v1.attention.backend import AttentionCGSupport +from vllm.v1.attention.backends.mla.sparse_swa import ( + DeepseekSparseSWABackend, + DeepseekSparseSWAMetadataBuilder, +) from vllm.v1.attention.ops.flashmla import ( flash_mla_sparse_fwd, flash_mla_with_kvcache, @@ -31,10 +36,23 @@ from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata +class DeepseekSparseSWAFlashMLAMetadataBuilder(DeepseekSparseSWAMetadataBuilder): + """SWA metadata for the FlashMLA decode path, which allows varlen decode.""" + + _cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.ALWAYS + + +class DeepseekSparseSWAFlashMLABackend(DeepseekSparseSWABackend): + @staticmethod + def get_builder_cls() -> type[DeepseekSparseSWAFlashMLAMetadataBuilder]: + return DeepseekSparseSWAFlashMLAMetadataBuilder + + class DeepseekV4FlashMLAAttention(DeepseekV4Attention): """FlashMLA sparse MLA attention layer for DeepSeek V4 (CUDA).""" backend_cls = DeepseekV4FlashMLABackend + swa_backend_cls = DeepseekSparseSWAFlashMLABackend def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) diff --git a/vllm/models/deepseek_v4/sparse_mla.py b/vllm/models/deepseek_v4/sparse_mla.py index 76aeffc085f6..db8ab96e90ff 100644 --- a/vllm/models/deepseek_v4/sparse_mla.py +++ b/vllm/models/deepseek_v4/sparse_mla.py @@ -9,7 +9,6 @@ from vllm.config import VllmConfig from vllm.config.cache import CacheDType -from vllm.platforms import current_platform from vllm.platforms.interface import DeviceCapability from vllm.triton_utils import tl, triton from vllm.utils.math_utils import cdiv @@ -23,7 +22,7 @@ ) from vllm.v1.attention.backends.mla.compressor_utils import get_compressed_slot_mapping from vllm.v1.attention.backends.utils import split_decodes_and_prefills -from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheSpec +from vllm.v1.kv_cache_interface import AttentionSpec # Pad C128A topk width to this alignment. 128 covers both h_q=64 (B_TOPK=64) and # h_q=128 (B_TOPK=128). FlashMLA decode asserts extra_topk % B_TOPK == 0; @@ -33,8 +32,8 @@ _C128A_TOPK_ALIGNMENT = 128 -class DeepseekV4FlashMLABackend(AttentionBackend): - """DeepSeek-V4 sparse-MLA backend. +class DeepseekV4SparseMLABackend(AttentionBackend): + """DeepSeek-V4 sparse-MLA backend base. Subclasses ``AttentionBackend`` directly (not the V3.2 ``FlashMLASparseBackend``): DeepSeek-V4 runs its own attention layer @@ -55,12 +54,8 @@ def get_supported_kernel_block_sizes() -> list[int | MultipleOf]: return [256] @staticmethod - def get_name() -> str: - return "FLASHMLA_SPARSE_DSV4" - - @staticmethod - def get_builder_cls() -> type["DeepseekV4FlashMLAMetadataBuilder"]: - return DeepseekV4FlashMLAMetadataBuilder + def get_builder_cls() -> type["DeepseekV4SparseMLAMetadataBuilder"]: + return DeepseekV4SparseMLAMetadataBuilder @staticmethod def get_impl_cls() -> type[Any]: @@ -68,7 +63,7 @@ def get_impl_cls() -> type[Any]: # not the generic ``Attention``/``MLAAttention`` layer, so the backend's # impl class is never instantiated. raise NotImplementedError( - "DeepseekV4FlashMLABackend has no separate impl class; DeepSeek-V4 " + "DeepseekV4SparseMLABackend has no separate impl class; DeepSeek-V4 " "attention runs through DeepseekV4Attention." ) @@ -132,20 +127,10 @@ class DeepseekV4FlashMLAMetadata(AttentionMetadata): c128a_prefill_topk_indices: torch.Tensor | None = None -class DeepseekV4FlashMLAMetadataBuilder( +class DeepseekV4SparseMLAMetadataBuilder( AttentionMetadataBuilder[DeepseekV4FlashMLAMetadata] ): - @classmethod - def get_cudagraph_support( - cls, - vllm_config: VllmConfig, - kv_cache_spec: KVCacheSpec, - ) -> AttentionCGSupport: - if current_platform.is_cuda() and current_platform.is_device_capability_family( - 100 - ): - return AttentionCGSupport.ALWAYS - return AttentionCGSupport.UNIFORM_BATCH + _cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH def __init__( self, @@ -305,6 +290,20 @@ def _build_c128a_metadata( return result +class DeepseekV4FlashMLAMetadataBuilder(DeepseekV4SparseMLAMetadataBuilder): + _cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.ALWAYS + + +class DeepseekV4FlashMLABackend(DeepseekV4SparseMLABackend): + @staticmethod + def get_name() -> str: + return "FLASHMLA_SPARSE_DSV4" + + @staticmethod + def get_builder_cls() -> type[DeepseekV4FlashMLAMetadataBuilder]: + return DeepseekV4FlashMLAMetadataBuilder + + def build_c128a_topk_metadata( positions: torch.Tensor, compress_ratio: int, diff --git a/vllm/models/deepseek_v4/xpu/xpu_sparse.py b/vllm/models/deepseek_v4/xpu/xpu_sparse.py index 73439d86461d..9b39c716c725 100644 --- a/vllm/models/deepseek_v4/xpu/xpu_sparse.py +++ b/vllm/models/deepseek_v4/xpu/xpu_sparse.py @@ -19,8 +19,8 @@ dequantize_and_gather_k_cache, ) from vllm.models.deepseek_v4.sparse_mla import ( - DeepseekV4FlashMLABackend, DeepseekV4FlashMLAMetadata, + DeepseekV4SparseMLABackend, ) from vllm.models.deepseek_v4.xpu.xpu_sparse_decode_fp8 import ( xpu_sparse_decode_fp8, @@ -32,7 +32,7 @@ from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata -class DeepseekV4XPUSparseBackend(DeepseekV4FlashMLABackend): +class DeepseekV4XPUSparseBackend(DeepseekV4SparseMLABackend): @staticmethod def get_name() -> str: return "XPU_V4_MLA_SPARSE" diff --git a/vllm/sampling_params.py b/vllm/sampling_params.py index f26dc8deec23..23997e9a3972 100644 --- a/vllm/sampling_params.py +++ b/vllm/sampling_params.py @@ -893,6 +893,9 @@ def _validate_spec_decode( # Adaptive verification compacts logits after the forward pass, while # compute_topk_scores uses the scheduled layout in cu_num_logits_np. + # TODO(lucas): lift this restriction. The true boundaries exist on device as + # cu_num_logits; cu_num_generated_tokens is only read host-side after + # the logprobs D2H, so it could ride along on that copy. if ( speculative_config.use_adaptive_verification and self.num_logprobs is not None diff --git a/vllm/v1/attention/backends/mla/sparse_swa.py b/vllm/v1/attention/backends/mla/sparse_swa.py index cf1e8013ce11..65857d2ff36d 100644 --- a/vllm/v1/attention/backends/mla/sparse_swa.py +++ b/vllm/v1/attention/backends/mla/sparse_swa.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from dataclasses import dataclass, field -from typing import cast +from typing import ClassVar, cast import torch @@ -61,8 +61,10 @@ def __init__( dtype: torch.dtype, prefix: str, cache_config: CacheConfig, + backend_cls: "type[AttentionBackend] | None" = None, ): super().__init__() + self.backend_cls = backend_cls or DeepseekSparseSWABackend self.kv_cache = torch.tensor([]) self.head_dim = head_dim self.window_size = window_size @@ -104,7 +106,7 @@ def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec: def forward(self): ... def get_attn_backend(self) -> type[AttentionBackend]: - return DeepseekSparseSWABackend + return self.backend_cls class DeepseekSparseSWABackend(AttentionBackend): @@ -400,18 +402,7 @@ class DeepseekSparseSWAMetadataBuilder(AttentionMetadataBuilder): """ reorder_batch_threshold: int | None = None - - @classmethod - def get_cudagraph_support( - cls, - vllm_config: VllmConfig, - kv_cache_spec: KVCacheSpec, - ) -> AttentionCGSupport: - if current_platform.is_cuda() and current_platform.is_device_capability_family( - 100 - ): - return AttentionCGSupport.ALWAYS - return AttentionCGSupport.UNIFORM_BATCH + _cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index c00874162923..4755bedf1dae 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -525,7 +525,6 @@ def initialize_kv_cache(self, kv_cache_config: KVCacheConfig) -> None: self.req_states, self.input_buffers.query_start_loc, self.model_state.num_new_sampled_tokens_per_step, - self.speculative_config.adaptive_verification_ema_alpha, max_total_logits=max_chunk_logits(self.vocab_size), ) diff --git a/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py b/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py index 79140882350e..939477f67330 100644 --- a/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py +++ b/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py @@ -116,7 +116,6 @@ def __init__( req_states: "RequestState", query_start_loc: torch.Tensor, num_bonus_tokens: int, - confidence_ema_alpha: float, max_total_logits: int, ): self.req_states = req_states @@ -125,7 +124,6 @@ def __init__( self._copy_stream = torch.cuda.Stream(device) self.num_bonus_tokens = num_bonus_tokens - self.confidence_ema_alpha = confidence_ema_alpha # Rejection sampling verifies logits in one contiguous chunk; the # chunked path indexes by scheduled (untrimmed) offsets and cannot # address the compacted layout, so the budget must fit one chunk. @@ -163,16 +161,10 @@ def __init__( self._stale_idx = 0 for slot in self._stale_confidences: slot.np.fill(1.0) - # Smoothed copy, CPU-only: batch-level sizing averages over requests, so - # it benefits from less noise and tolerates the lag. - self._confidence_ema = np.ones( - (max_num_reqs, self.num_speculative_steps), dtype=np.float32 - ) def add_request(self, req_idx: int) -> None: self._stale_confidences[self._stale_idx].np[req_idx].fill(1.0) self._pending_resets.append(req_idx) - self._confidence_ema[req_idx] = 1.0 self._confidence_probs[req_idx].fill_(1.0) def batches_to_profile(self, capture_sizes: list[int]) -> Iterator[dict[str, int]]: @@ -246,7 +238,7 @@ def record_confidences( input_batch: "InputBatch", ) -> None: """Publish this step's raw confidences for the ranking kernel and start - copying them to the CPU, where a later step's budget smooths them.""" + copying them to the CPU, where a later step's budget reads them.""" num_reqs = input_batch.num_reqs ready_idx = self._stale_idx ^ 1 with gpu_sync_allowed(): @@ -257,15 +249,6 @@ def record_confidences( # Last step's copy has landed: budgets read it, this step overwrites the # slot they were reading before. self._stale_idx, write_idx = ready_idx, self._stale_idx - # Fold the freshly-readable raw scores into the CPU-side EMA. New - # requests blend with the optimistic 1.0 seed, which washes out within - # a couple of steps. - alpha = self.confidence_ema_alpha - np.add( - alpha * self._stale_confidences[ready_idx].np, - (1.0 - alpha) * self._confidence_ema, - out=self._confidence_ema, - ) self._confidence_probs[input_batch.idx_mapping] = confidence_probs[:num_reqs] write_slot = self._stale_confidences[write_idx] @@ -304,7 +287,7 @@ def get_num_tokens( dtype=np.int32, count=len(req_ids), ) - stale_confidences = self._confidence_ema[slots] + stale_confidences = self._stale_confidences[self._stale_idx].np[slots] survival_probability = np.cumprod(stale_confidences.astype(np.float64), axis=1) steps = np.arange(self.num_speculative_steps) valid = steps[None, :] < scheduled_drafts[:, None] diff --git a/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py b/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py index e477f5d98312..97c284c03d4a 100644 --- a/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/dflash/speculator.py @@ -520,7 +520,6 @@ def _prepare_dflash_inputs_kernel( num_rejected = tl.load(num_rejected_ptr + req_idx) valid_ctx_end = ctx_end - num_rejected - num_valid_ctx = valid_ctx_end - ctx_start num_sampled = tl.load(num_sampled_ptr + req_idx) if num_sampled > 0: @@ -534,27 +533,22 @@ def _prepare_dflash_inputs_kernel( j = block_idx * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) is_ctx = j < num_ctx - is_valid_ctx = j < num_valid_ctx - is_query = (j >= num_valid_ctx) & (j < num_valid_ctx + num_query_per_req) - query_off = j - num_valid_ctx + is_query = (j >= num_ctx) & (j < num_ctx + num_query_per_req) + query_off = j - num_ctx # --- Context positions / slots --- ctx_pos_idx = ctx_start + tl.where(is_ctx, j, 0) - ctx_pos = tl.load(target_positions_ptr + ctx_pos_idx, mask=is_valid_ctx, other=0) + ctx_pos = tl.load(target_positions_ptr + ctx_pos_idx, mask=is_ctx, other=0) ctx_block_num = ctx_pos // block_size ctx_block_num = tl.minimum(ctx_block_num, block_table_stride - 1) ctx_block_id = tl.load( block_table_ptr + req_idx * block_table_stride + ctx_block_num, - mask=is_valid_ctx, + mask=is_ctx, other=0, ).to(tl.int64) ctx_slot = ctx_block_id * block_size + (ctx_pos % block_size) tl.store(out_context_positions_ptr + ctx_start + j, ctx_pos, mask=is_ctx) - tl.store( - out_context_slot_mapping_ptr + ctx_start + j, - tl.where(is_valid_ctx, ctx_slot, PAD_SLOT_ID), - mask=is_ctx, - ) + tl.store(out_context_slot_mapping_ptr + ctx_start + j, ctx_slot, mask=is_ctx) # --- Query positions / input_ids / slots --- query_pos = last_valid_pos + 1 + query_off From 70596c8243b208b2971164c0eb05f7f2bee6cbb4 Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Tue, 4 Aug 2026 21:23:45 +0000 Subject: [PATCH 22/40] [Spec Decode] Inherit target quantization for the DSV4 DSpark draft Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_016Wv66G1jbqBSxLFcfVxLA5 Signed-off-by: Lucas Wilkinson --- vllm/config/speculative.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/vllm/config/speculative.py b/vllm/config/speculative.py index 714bb99b95d2..bf7755261805 100644 --- a/vllm/config/speculative.py +++ b/vllm/config/speculative.py @@ -974,6 +974,9 @@ def __post_init__(self): self.draft_model_config.hf_config.architectures = [ "DSparkDraftModel" ] + self.draft_model_config.quantization = ( + self.target_model_config.quantization + ) self.update_arch_() elif ( self.method == "dspark" From eed2eefe18e2c5460352e6c84ac26d104486d4b5 Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Tue, 4 Aug 2026 21:23:50 +0000 Subject: [PATCH 23/40] [Spec Decode] Compute draft confidences in the topk sampling path too Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_016Wv66G1jbqBSxLFcfVxLA5 Signed-off-by: Lucas Wilkinson --- vllm/v1/worker/gpu/spec_decode/dspark/speculator.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py b/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py index c20996379959..2425c82def89 100644 --- a/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py @@ -184,6 +184,15 @@ def _sample_sequential(self, num_reqs: int, head_hidden: torch.Tensor) -> None: self.draft_tokens[:num_reqs, i] = draft_sampled_i prev = draft_sampled_i + if self.use_adaptive_verification: + torch.sigmoid( + self.model.compute_confidence( + sample_hidden, + torch.stack(confidence_markov_embeds, dim=1).flatten(0, 1), + ).view(num_reqs, n_spec), + out=self.draft_token_confidence_probs[:num_reqs], + ) + def _sample_sequential_topk(self, num_reqs: int, head_hidden: torch.Tensor) -> None: """Apply the sequential Markov head only to top-k base-logit candidates. @@ -205,10 +214,13 @@ def _sample_sequential_topk(self, num_reqs: int, head_hidden: torch.Tensor) -> N base_logits.fill_(float("-inf")) idx_map = self.sample_idx_mapping[:num_sample].view(num_reqs, n_spec) sample_pos = self.sample_pos[:num_sample].view(num_reqs, n_spec) + confidence_markov_embeds = [] prev = self.input_buffers.input_ids[self._anchor_idx[:num_reqs]] for i in range(n_spec): markov_embed = self.model.markov_embed(prev) + if self.use_adaptive_verification: + confidence_markov_embeds.append(markov_embed) logits_i = self.model.apply_markov_bias_gathered( markov_embed, base_logits[:, i], From 3c59d1e240f3a82b3564087a175007af4a7f9e97 Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Wed, 5 Aug 2026 04:30:36 +0000 Subject: [PATCH 24/40] [Spec Decode] Rename max_chunk_logits helper to avoid shadowing The module-level helper shared a name with the max_chunk_logits parameter of _iter_request_chunks and _verify_in_chunks, which is why the caller had to introduce a chunk_logit_limit local. Rename the helper to follow the surrounding get_* convention; the parameter name predates it. Signed-off-by: Lucas Wilkinson Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_011Vq5NzWmCNi3F2WX3Vr7op --- vllm/v1/worker/gpu/model_runner.py | 4 ++-- vllm/v1/worker/gpu/spec_decode/rejection_sampler.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 4755bedf1dae..67d08daa959b 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -132,7 +132,7 @@ ) from vllm.v1.worker.gpu.spec_decode.rejection_sampler import ( RejectionSampler, - max_chunk_logits, + get_max_chunk_logits, ) from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator from vllm.v1.worker.gpu.spec_decode.utils import DraftTokensHandler @@ -525,7 +525,7 @@ def initialize_kv_cache(self, kv_cache_config: KVCacheConfig) -> None: self.req_states, self.input_buffers.query_start_loc, self.model_state.num_new_sampled_tokens_per_step, - max_total_logits=max_chunk_logits(self.vocab_size), + max_total_logits=get_max_chunk_logits(self.vocab_size), ) self.block_tables = BlockTables( diff --git a/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py b/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py index a139ac7d25d5..888daf22f20b 100644 --- a/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py +++ b/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py @@ -32,7 +32,7 @@ _FP32_BYTES = 4 -def max_chunk_logits(vocab_size: int) -> int: +def get_max_chunk_logits(vocab_size: int) -> int: """Largest number of logits rows one verification chunk may hold.""" return max(1, MAX_CHUNK_BYTES // (vocab_size * _FP32_BYTES)) @@ -250,7 +250,7 @@ def __call__( max_num_logprobs = self.sampler.sampling_states.max_num_logprobs( input_batch.idx_mapping_np ) - chunk_logit_limit = max_chunk_logits(logits.shape[1]) + chunk_logit_limit = get_max_chunk_logits(logits.shape[1]) sampled, num_sampled, logprobs_tensors = self._verify_in_chunks( logits, input_batch, From 7ea349d61598af5611c94018dfa8c4f6a2050ca0 Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Wed, 5 Aug 2026 04:37:54 +0000 Subject: [PATCH 25/40] [Spec Decode] Align DSpark GSM8K eval config with the vLLM recipe Add --attention_config.use_fp4_indexer_cache=True, which both the published DeepSeek-V4-Flash recipe and the in-repo DSV4 config (configs/moe-refactor/DeepSeek-V4-Flash-deepgemm-mega-moe.yaml) set. It defaults to False, so the eval was exercising a different DSV4 sparse attention indexer path than deployments use -- the same area this series touches in sparse_swa.py. Drop the redundant --dtype bfloat16; neither reference config sets it. GSM8K 1319q: 0.9409, invalid 0.000, mean acceptance length 4.05. Log confirms the path changed: DSA indexer decode path: use_flattening=False supports_varlen=True (next_n=8, use_fp4_indexer_cache=True) Signed-off-by: Lucas Wilkinson Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_011Vq5NzWmCNi3F2WX3Vr7op --- .../gsm8k/configs/DeepSeek-V4-Flash-DSpark-confidence-TP4.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/evals/gsm8k/configs/DeepSeek-V4-Flash-DSpark-confidence-TP4.yaml b/tests/evals/gsm8k/configs/DeepSeek-V4-Flash-DSpark-confidence-TP4.yaml index 850c153e9e62..2ea04d39d3b9 100644 --- a/tests/evals/gsm8k/configs/DeepSeek-V4-Flash-DSpark-confidence-TP4.yaml +++ b/tests/evals/gsm8k/configs/DeepSeek-V4-Flash-DSpark-confidence-TP4.yaml @@ -6,13 +6,13 @@ startup_max_wait_seconds: 1800 server_args: >- --tokenizer-mode deepseek_v4 --trust-remote-code - --dtype bfloat16 --max-model-len 8192 --tensor-parallel-size 4 --enable-expert-parallel --block-size 256 --gpu-memory-utilization 0.5 --kv-cache-dtype fp8 + --attention_config.use_fp4_indexer_cache=True --max-num-batched-tokens 16384 --max-num-seqs 32 --speculative-config '{"method":"dspark", From 387b08409590859f98fd8cb79f0a13a1224eb68b Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Thu, 6 Aug 2026 18:36:52 +0000 Subject: [PATCH 26/40] review comments Signed-off-by: Lucas Wilkinson --- vllm/config/compilation.py | 21 ---------- vllm/config/speculative.py | 14 ++++--- vllm/config/vllm.py | 4 +- vllm/model_executor/models/qwen3_dspark.py | 3 +- vllm/models/deepseek_v4/nvidia/dspark.py | 3 +- vllm/sampling_params.py | 2 +- vllm/v1/attention/backend.py | 17 ++++++++ vllm/v1/attention/backends/flashinfer.py | 3 ++ vllm/v1/attention/backends/mla/indexer.py | 10 +++++ vllm/v1/worker/gpu/attn_utils.py | 20 ++++++++++ vllm/v1/worker/gpu/dp_utils.py | 12 +++++- vllm/v1/worker/gpu/model_runner.py | 22 ++++++---- .../gpu/spec_decode/dspark/speculator.py | 40 +++++++++---------- 13 files changed, 109 insertions(+), 62 deletions(-) diff --git a/vllm/config/compilation.py b/vllm/config/compilation.py index 93190c48e67d..4eb66613a81b 100644 --- a/vllm/config/compilation.py +++ b/vllm/config/compilation.py @@ -1375,7 +1375,6 @@ def resolve_cudagraph_mode_and_sizes( kv_cache_config: "KVCacheConfig | None" = None, max_num_reqs: int | None = None, is_profiling: bool = False, - varlen_decode: bool = False, ) -> CUDAGraphMode: from vllm.v1.attention.backend import AttentionCGSupport @@ -1458,26 +1457,6 @@ def resolve_cudagraph_mode_and_sizes( cudagraph_mode = CUDAGraphMode.NONE logger.warning(msg) - # Adaptive verification trims the draft budget per request, so decode graphs - # are varlen and need a backend that handles arbitrary query lengths. - if ( - cudagraph_mode.decode_mode() == CUDAGraphMode.FULL - and varlen_decode - and min_cg_support != AttentionCGSupport.ALWAYS - ): - msg = ( - f"CUDAGraphMode.{cudagraph_mode.name} is not supported with " - f"varlen decode for attention backend {min_cg_attn_backend} " - f"(support: {min_cg_support})" - ) - if self.splitting_ops_contain_attention(): - msg += "; setting cudagraph_mode=PIECEWISE" - cudagraph_mode = CUDAGraphMode.PIECEWISE - else: - msg += "; setting cudagraph_mode=NONE" - cudagraph_mode = CUDAGraphMode.NONE - logger.warning(msg) - # double check that we can support full cudagraph if they are requested # even after automatic downgrades if ( diff --git a/vllm/config/speculative.py b/vllm/config/speculative.py index bf7755261805..790610b28fa6 100644 --- a/vllm/config/speculative.py +++ b/vllm/config/speculative.py @@ -237,12 +237,8 @@ class SpeculativeConfig: Mutually exclusive with synthetic_acceptance_rates.""" enable_adaptive_verification: bool = True - """Whether to adaptively size the draft-verification budget from - per-request confidence. Only applies to method="dspark".""" - - @property - def use_adaptive_verification(self) -> bool: - return self.method == "dspark" and self.enable_adaptive_verification + """Whether to adaptively size the draft-verification budget from per-request + confidence. Currently only supported for method="dspark".""" @staticmethod def _acceptance_length_to_rates(length: float, n: int) -> list[float]: @@ -1122,6 +1118,12 @@ def __post_init__(self): self.target_parallel_config, self.draft_tensor_parallel_size ) ) + + # Only DSpark produces the per-request confidences the adaptive budget + # needs, so resolve the flag once here instead of re-deriving it. + if self.method != "dspark": + self.enable_adaptive_verification = False + return self def _validate_suffix_decoding(self): diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index 2f91b2d8d3ba..4a29c235be75 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -2241,7 +2241,7 @@ def _get_v2_model_runner_unsupported_features(self) -> list[str]: unsupported.append("EAGLE3 with pipeline parallelism") if ( - speculative_config.use_adaptive_verification + speculative_config.enable_adaptive_verification and self.lora_config is not None ): # The per-token LoRA mapping is built from CPU placeholder boundaries, @@ -2249,7 +2249,7 @@ def _get_v2_model_runner_unsupported_features(self) -> list[str]: unsupported.append("adaptive verification with LoRA") if ( - speculative_config.use_adaptive_verification + speculative_config.enable_adaptive_verification and self.parallel_config.pipeline_parallel_size > 1 ): # Cost curves and confidences currently only exist on the last PP rank; diff --git a/vllm/model_executor/models/qwen3_dspark.py b/vllm/model_executor/models/qwen3_dspark.py index 2cadf4d01e20..d15aa45461cf 100644 --- a/vllm/model_executor/models/qwen3_dspark.py +++ b/vllm/model_executor/models/qwen3_dspark.py @@ -242,8 +242,9 @@ def apply_markov_bias_gathered( def compute_confidence( self, head_hidden: torch.Tensor, markov_embed: torch.Tensor ) -> torch.Tensor: + """Per-position acceptance probability for each drafted token.""" assert self.model.confidence_head is not None - return self.model.confidence_head(head_hidden, markov_embed) + return torch.sigmoid(self.model.confidence_head(head_hidden, markov_embed)) def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]): model_weights = {} diff --git a/vllm/models/deepseek_v4/nvidia/dspark.py b/vllm/models/deepseek_v4/nvidia/dspark.py index 373696d5982d..b702d7ec9cbd 100644 --- a/vllm/models/deepseek_v4/nvidia/dspark.py +++ b/vllm/models/deepseek_v4/nvidia/dspark.py @@ -379,8 +379,9 @@ def markov_bias(self, markov_embed: torch.Tensor) -> torch.Tensor: def compute_confidence( self, head_hidden: torch.Tensor, markov_embed: torch.Tensor ) -> torch.Tensor: + """Per-position acceptance probability for each drafted token.""" assert self.model.confidence_head is not None - return self.model.confidence_head(head_hidden, markov_embed) + return torch.sigmoid(self.model.confidence_head(head_hidden, markov_embed)) # --- Weight loading ---------------------------------------------------- diff --git a/vllm/sampling_params.py b/vllm/sampling_params.py index 23997e9a3972..2f9628b8e56a 100644 --- a/vllm/sampling_params.py +++ b/vllm/sampling_params.py @@ -897,7 +897,7 @@ def _validate_spec_decode( # cu_num_logits; cu_num_generated_tokens is only read host-side after # the logprobs D2H, so it could ride along on that copy. if ( - speculative_config.use_adaptive_verification + speculative_config.enable_adaptive_verification and self.num_logprobs is not None ): raise ValueError( diff --git a/vllm/v1/attention/backend.py b/vllm/v1/attention/backend.py index fae085c21ea7..06f18cb60919 100644 --- a/vllm/v1/attention/backend.py +++ b/vllm/v1/attention/backend.py @@ -632,6 +632,8 @@ class AttentionMetadataBuilder(ABC, Generic[M]): # Does this backend/builder support updating the block table in existing # metadata supports_update_block_table: bool = False + # Do not access directly. See supports_device_cpu_query_lens_mismatch(). + _supports_device_cpu_query_lens_mismatch: ClassVar[bool] = True # Whether the builder constructor requires the block-table width. requires_block_table_width: ClassVar[bool] = False @@ -657,6 +659,21 @@ def get_cudagraph_support( """Get the cudagraph support level of this builder class.""" return cls._cudagraph_support + @classmethod + def supports_device_cpu_query_lens_mismatch( + cls: type["AttentionMetadataBuilder"], + vllm_config: "VllmConfig", + kv_cache_spec: "KVCacheSpec", + ) -> bool: + """Whether this backend can run a batch whose device query_start_loc disagrees + with the CPU one; backends that plan off the CPU query lengths must opt out. + + Currently only verification requests are affected: adaptive verification trims + their drafts on device, so the CPU lengths bound them from above. The + decode/prefill split point and the CPU prefill query lengths both stay correct. + """ + return cls._supports_device_cpu_query_lens_mismatch + def _init_reorder_batch_threshold( self, reorder_batch_threshold: int | None = 1, diff --git a/vllm/v1/attention/backends/flashinfer.py b/vllm/v1/attention/backends/flashinfer.py index 5251e25e5e4c..526463719285 100755 --- a/vllm/v1/attention/backends/flashinfer.py +++ b/vllm/v1/attention/backends/flashinfer.py @@ -620,6 +620,9 @@ class FlashInferMetadata: class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): kv_cache_spec: AttentionSpec reorder_batch_threshold: int = 1 + # The wrappers are planned from qo_indptr_cpu, so the CPU query offsets have to be + # the ones the kernel runs on. + _supports_device_cpu_query_lens_mismatch: ClassVar[bool] = False def __init__( self, diff --git a/vllm/v1/attention/backends/mla/indexer.py b/vllm/v1/attention/backends/mla/indexer.py index 7cb68465c1f0..3a4abd7debf9 100644 --- a/vllm/v1/attention/backends/mla/indexer.py +++ b/vllm/v1/attention/backends/mla/indexer.py @@ -477,6 +477,16 @@ def get_cudagraph_support( return AttentionCGSupport.ALWAYS return AttentionCGSupport.UNIFORM_BATCH + @classmethod + def supports_device_cpu_query_lens_mismatch( + cls, + vllm_config: VllmConfig, + kv_cache_spec: KVCacheSpec, + ) -> bool: + # Only the varlen paged MQA logits kernel takes per-request query lengths from + # device tensors; otherwise the indexer needs uniform ones. + return _supports_varlen_paged_mqa_logits() + def __init__(self, *args, block_table_width: int, **kwargs) -> None: super().__init__(*args, **kwargs) scheduler_config = self.vllm_config.scheduler_config diff --git a/vllm/v1/worker/gpu/attn_utils.py b/vllm/v1/worker/gpu/attn_utils.py index 598bddf47ab0..429ac87d0424 100644 --- a/vllm/v1/worker/gpu/attn_utils.py +++ b/vllm/v1/worker/gpu/attn_utils.py @@ -180,6 +180,26 @@ def init_attn_backend( return attn_groups, attn_cg_support_info, kernel_block_sizes +def get_device_cpu_query_lens_mismatch_support( + attn_groups: list[list[AttentionGroup]], + vllm_config: VllmConfig, +) -> tuple[bool, str | None]: + """Whether every backend tolerates a device/CPU query length mismatch. + See AttentionMetadataBuilder.supports_device_cpu_query_lens_mismatch() for details. + + Returns the aggregate answer and, when it is False, the first backend that does not. + """ + for groups in attn_groups: + for group in groups: + builder = group.get_metadata_builder(0) + if not builder.supports_device_cpu_query_lens_mismatch( + vllm_config, + group.kv_cache_spec, + ): + return False, group.backend.__name__ + return True, None + + def _allocate_kv_cache( kv_cache_config: KVCacheConfig, shared_layers: dict[str, str], device: torch.device ): diff --git a/vllm/v1/worker/gpu/dp_utils.py b/vllm/v1/worker/gpu/dp_utils.py index 71601b4ae91c..fcdb4e28cff7 100644 --- a/vllm/v1/worker/gpu/dp_utils.py +++ b/vllm/v1/worker/gpu/dp_utils.py @@ -31,15 +31,17 @@ def sync_cudagraph_and_dp_padding( """ assert dp_size > 1, "DP size must be greater than 1" group = get_dp_group().cpu_group - tensor = torch.zeros(3, dp_size, dtype=torch.int32, device="cpu") + tensor = torch.zeros(4, dp_size, dtype=torch.int32, device="cpu") tensor[0][dp_rank] = num_tokens tensor[1][dp_rank] = desired_batch_desc.cg_mode.value tensor[2][dp_rank] = uniform_token_count or 0 # (0 means None) + tensor[3][dp_rank] = max_query_len or -1 # (-1 means None) dist.all_reduce(tensor, group=group) num_tokens_across_dp = tensor[0] cg_mode_across_dp = tensor[1] uniform_token_counts_across_dp = tensor[2] + max_query_lens_across_dp = tensor[3] if torch.all(num_tokens_across_dp == 0).item(): synced_desc = BatchExecutionDescriptor( @@ -70,6 +72,12 @@ def sync_cudagraph_and_dp_padding( ): synced_uniform_token_count = None + # Varlen decode graphs are selected by the query-length bound, so ranks must agree + # on it or they pad to different token counts below. + synced_max_query_len: int | None = None + if bool(torch.all(max_query_lens_across_dp != -1).item()): + synced_max_query_len = int(max_query_lens_across_dp.max().item()) + # Dispatch for the final synced values, use num_reqs instead of synced_num_reqs # so we don't perform request padding for PIECEWISE graphs. # num_active_loras is per-rank and doesn't need cross-rank agreement. @@ -78,7 +86,7 @@ def sync_cudagraph_and_dp_padding( synced_num_tokens, synced_uniform_token_count, num_active_loras=num_active_loras, - max_query_len=max_query_len, + max_query_len=synced_max_query_len, ) # Update num_tokens_across_dp to reflect padded size. diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 67d08daa959b..227409274eb7 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -75,6 +75,7 @@ ) from vllm.v1.worker.gpu.attn_utils import ( build_slot_mappings_by_layer, + get_device_cpu_query_lens_mismatch_support, get_kv_cache_spec, init_attn_backend, init_kv_cache, @@ -514,13 +515,19 @@ def initialize_kv_cache(self, kv_cache_config: KVCacheConfig) -> None: attn_cg_support = attn_cg_support.narrow( *self.model_state.get_additional_cg_support() ) - if ( - self.speculative_config is not None - and self.speculative_config.use_adaptive_verification - # The checkpoint may lack a confidence head (the speculator clears - # its flag at load time). - and getattr(self.speculator, "use_adaptive_verification", False) - ): + # The speculator clears the flag at load time when the checkpoint has + # no confidence head, so it holds the effective value. + if getattr(self.speculator, "enable_adaptive_verification", False): + supported, backend = get_device_cpu_query_lens_mismatch_support( + self.attn_groups, self.vllm_config + ) + if not supported: + raise ValueError( + "Adaptive verification trims verification requests on device, which" + f" the {backend} attention backend does not support. support. Pass " + "enable_adaptive_verification=false in the speculative config, or " + "use a backend that does." + ) self.adaptive_verification = AdaptiveVerificationManager( self.req_states, self.input_buffers.query_start_loc, @@ -559,7 +566,6 @@ def initialize_kv_cache(self, kv_cache_config: KVCacheConfig) -> None: tensor_parallel_size=self.parallel_config.tensor_parallel_size, kv_cache_config=self.kv_cache_config, max_num_reqs=self.max_num_reqs, - varlen_decode=self.adaptive_verification is not None, ) self.cudagraph_manager = ModelCudaGraphManager( self.vllm_config, diff --git a/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py b/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py index 2425c82def89..b91e9a26461b 100644 --- a/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py @@ -82,8 +82,8 @@ def __init__(self, vllm_config: VllmConfig, device: torch.device): self.draft_token_confidence_probs = torch.empty_like( self.draft_tokens, dtype=torch.float32 ) - self.use_adaptive_verification = ( - self.speculative_config.use_adaptive_verification + self.enable_adaptive_verification = ( + self.speculative_config.enable_adaptive_verification ) def load_draft_model( @@ -108,12 +108,12 @@ def load_draft_model( dtype=self.draft_logits.dtype, device=self.device, ) - if self.use_adaptive_verification and model.model.confidence_head is None: + if self.enable_adaptive_verification and model.model.confidence_head is None: logger.warning_once( "DSpark checkpoint has no confidence head; disabling adaptive " "verification and using fixed-length verification." ) - self.use_adaptive_verification = False + self.enable_adaptive_verification = False return model def _sample_logits( @@ -174,7 +174,7 @@ def _sample_sequential(self, num_reqs: int, head_hidden: torch.Tensor) -> None: for i in range(n_spec): # Sequential stage: Markov bias from the previously sampled token. markov_embed = self.model.markov_embed(prev) - if self.use_adaptive_verification: + if self.enable_adaptive_verification: confidence_markov_embeds.append(markov_embed) bias = self.model.markov_bias(markov_embed) logits_i = base_logits[:, i] + bias @@ -184,13 +184,13 @@ def _sample_sequential(self, num_reqs: int, head_hidden: torch.Tensor) -> None: self.draft_tokens[:num_reqs, i] = draft_sampled_i prev = draft_sampled_i - if self.use_adaptive_verification: - torch.sigmoid( - self.model.compute_confidence( - sample_hidden, - torch.stack(confidence_markov_embeds, dim=1).flatten(0, 1), - ).view(num_reqs, n_spec), - out=self.draft_token_confidence_probs[:num_reqs], + if self.enable_adaptive_verification: + confidence = self.model.compute_confidence( + sample_hidden, + torch.stack(confidence_markov_embeds, dim=1).flatten(0, 1), + ) + self.draft_token_confidence_probs[:num_reqs] = confidence.view( + num_reqs, n_spec ) def _sample_sequential_topk(self, num_reqs: int, head_hidden: torch.Tensor) -> None: @@ -219,7 +219,7 @@ def _sample_sequential_topk(self, num_reqs: int, head_hidden: torch.Tensor) -> N for i in range(n_spec): markov_embed = self.model.markov_embed(prev) - if self.use_adaptive_verification: + if self.enable_adaptive_verification: confidence_markov_embeds.append(markov_embed) logits_i = self.model.apply_markov_bias_gathered( markov_embed, @@ -233,13 +233,13 @@ def _sample_sequential_topk(self, num_reqs: int, head_hidden: torch.Tensor) -> N self.draft_tokens[:num_reqs, i] = draft_sampled_i prev = draft_sampled_i - if self.use_adaptive_verification: - torch.sigmoid( - self.model.compute_confidence( - sample_hidden, - torch.stack(confidence_markov_embeds, dim=1).flatten(0, 1), - ).view(num_reqs, n_spec), - out=self.draft_token_confidence_probs[:num_reqs], + if self.enable_adaptive_verification: + confidence = self.model.compute_confidence( + sample_hidden, + torch.stack(confidence_markov_embeds, dim=1).flatten(0, 1), + ) + self.draft_token_confidence_probs[:num_reqs] = confidence.view( + num_reqs, n_spec ) def _generate_draft( From e82c2b47ed336d2212b504cf58206ef6419fed23 Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Thu, 6 Aug 2026 19:22:25 +0000 Subject: [PATCH 27/40] cleanup Signed-off-by: Lucas Wilkinson --- vllm/envs.py | 9 ++++++ .../gpu/spec_decode/adaptive_verification.py | 29 ++++++++----------- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/vllm/envs.py b/vllm/envs.py index 0b16f78fcd6f..aad1cc215277 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -57,6 +57,7 @@ VLLM_XLA_CACHE_PATH: str = os.path.join(VLLM_CACHE_ROOT, "xla_cache") VLLM_XLA_CHECK_RECOMPILATION: bool = False VLLM_SPARSE_INDEXER_MAX_LOGITS_MB: int = 512 + VLLM_ADAPTIVE_VERIFICATION_PROFILE_CONTEXT_LEN: int = 8192 VLLM_USE_RAY_COMPILED_DAG_CHANNEL_TYPE: Literal["auto", "nccl", "shm"] = "auto" VLLM_USE_RAY_COMPILED_DAG_OVERLAP_COMM: bool = False VLLM_USE_RAY_WRAPPED_PP_COMM: bool = True @@ -1050,6 +1051,14 @@ def _resolve_rust_cli_path() -> str | None: "VLLM_SPARSE_INDEXER_MAX_LOGITS_MB": lambda: int( os.getenv("VLLM_SPARSE_INDEXER_MAX_LOGITS_MB", "512") ), + # KV context length each adaptive-verification profiling request pretends to + # carry, so the profiled step reads a realistic amount of cache. + # Raise it for long-context deployments, where step cost is dominated by + # attention over a much larger KV cache than the default assumes. + # Clamped to max_model_len - query_len. Default: 8192 tokens + "VLLM_ADAPTIVE_VERIFICATION_PROFILE_CONTEXT_LEN": lambda: int( + os.getenv("VLLM_ADAPTIVE_VERIFICATION_PROFILE_CONTEXT_LEN", "8192") + ), # If set, the OpenAI API server will stay alive even after the underlying # AsyncLLMEngine errors and stops serving requests "VLLM_KEEP_ALIVE_ON_ENGINE_DEATH": lambda: bool( diff --git a/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py b/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py index 939477f67330..e12e86bbd18c 100644 --- a/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py +++ b/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py @@ -9,6 +9,7 @@ import numpy as np import torch +import vllm.envs as envs from vllm.distributed.parallel_state import get_tp_group from vllm.logger import init_logger from vllm.utils.gpu_sync_debug import gpu_sync_allowed @@ -18,9 +19,6 @@ logger = init_logger(__name__) _PROFILE_REPLAYS = 5 -# KV context each profiling request pretends to carry, so the profiled step reads -# a realistic amount of cache rather than attending over nothing. -_PROFILE_CONTEXT_LEN = 8192 if TYPE_CHECKING: from vllm.v1.worker.gpu.input_batch import InputBatch @@ -42,21 +40,18 @@ def _assign_draft_token_budget( continuously along steps with a request. `capacities` enters holding each request's scheduled draft count (which bounds its - eligible slots) and leaves holding the admitted count. + eligible slots) and leaves holding the admitted count. The caller only calls this + when `draft_budget < capacities.sum()`, so every winner is a real draft slot. """ survival = confidence_probs[idx_mapping].cumprod(dim=1) steps = torch.arange(num_steps, device=survival.device) - valid = steps[None, :] < capacities[:, None] - survival = survival.masked_fill(~valid, -float("inf")) - order = survival.flatten().argsort(descending=True, stable=True) - # Invalid slots score -inf and sort last, so the valid ones occupy exactly the first - # valid.sum() ranks; everything between the budget and there is a real draft that - # missed the cut. Decrementing keeps this sync-free. - ranks = torch.arange(order.shape[0], device=survival.device) - rejected = (ranks >= draft_budget) & (ranks < valid.sum()) - capacities.scatter_add_( - 0, order // num_steps, torch.where(rejected, -1, 0).to(capacities.dtype) - ) + # Out-of-range slots score -inf so they never outrank a real draft. + out_of_range = steps[None, :] >= capacities[:, None] + survival = survival.masked_fill(out_of_range, -float("inf")) + flat = survival.flatten() + winners = flat.topk(draft_budget).indices + admitted = torch.zeros_like(flat, dtype=torch.bool).index_fill_(0, winners, True) + torch.sum(admitted.view_as(survival), dim=1, dtype=capacities.dtype, out=capacities) _assign_draft_token_budget_compiled = torch.compile( @@ -188,7 +183,7 @@ def batches_to_profile(self, capture_sizes: list[int]) -> Iterator[dict[str, int for _ in range(_PROFILE_REPLAYS): yield { "num_tokens": num_tokens, - "context_len": _PROFILE_CONTEXT_LEN, + "context_len": envs.VLLM_ADAPTIVE_VERIFICATION_PROFILE_CONTEXT_LEN, } def set_initial_cost_curves(self, samples: list[StepTimingSample]) -> None: @@ -380,7 +375,7 @@ def reallocate_drafts( capacities.zero_() else: async_copy_to_gpu(scheduled_drafts, out=capacities) - if draft_budget != int(scheduled_drafts.sum()): + if draft_budget < int(scheduled_drafts.sum()): _assign_draft_token_budget_compiled( self._confidence_probs, idx_mapping, From 49f9571bfcfae7fad4147da16664de7b039e099d Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Thu, 6 Aug 2026 20:06:05 +0000 Subject: [PATCH 28/40] review comments Signed-off-by: Lucas Wilkinson --- vllm/v1/worker/gpu/block_table.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/vllm/v1/worker/gpu/block_table.py b/vllm/v1/worker/gpu/block_table.py index 9c4eccc0f9c6..d3b871e7c034 100644 --- a/vllm/v1/worker/gpu/block_table.py +++ b/vllm/v1/worker/gpu/block_table.py @@ -180,20 +180,21 @@ def set_dummy_context( self, num_reqs: int, query_len: int, context_len: int, num_kv_blocks: int ) -> int: """ - Give each dummy request a disjoint span of KV blocks. + Give each dummy request its own span of KV blocks. Used when profiling step cost. """ - for bpk, block_size in zip(self.blocks_per_kv_block, self.kernel_block_sizes): - capacity = (num_kv_blocks * bpk // num_reqs) * block_size - context_len = min(context_len, capacity - query_len) - context_len = max(context_len, 0) - for block_table, block_size in zip( - self.input_block_tables, self.kernel_block_sizes + for block_table, block_size, bpk in zip( + self.input_block_tables, self.kernel_block_sizes, self.blocks_per_kv_block ): - num_blocks = cdiv(context_len + query_len, block_size) - block_table[:num_reqs, :num_blocks] = torch.arange( + num_blocks = min( + cdiv(context_len + query_len, block_size), block_table.shape[1] + ) + # Spans are disjoint until the pool runs out, then they wrap and share + # blocks: profiling only needs the reads to be realistic, not distinct. + block_ids = torch.arange( num_reqs * num_blocks, dtype=block_table.dtype, device=self.device - ).view(num_reqs, num_blocks) + ) % (num_kv_blocks * bpk) + block_table[:num_reqs, :num_blocks] = block_ids.view(num_reqs, num_blocks) return context_len def compute_slot_mappings( From e5933514e8194e3e57158a6d8fae88e73c10210e Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Thu, 6 Aug 2026 22:25:09 -0400 Subject: [PATCH 29/40] Update vllm/config/speculative.py Co-authored-by: Benjamin Chislett Signed-off-by: Lucas Wilkinson --- vllm/config/speculative.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/vllm/config/speculative.py b/vllm/config/speculative.py index 790610b28fa6..b3ac35084b77 100644 --- a/vllm/config/speculative.py +++ b/vllm/config/speculative.py @@ -1119,10 +1119,8 @@ def __post_init__(self): ) ) - # Only DSpark produces the per-request confidences the adaptive budget - # needs, so resolve the flag once here instead of re-deriving it. - if self.method != "dspark": - self.enable_adaptive_verification = False + if self.method != "dspark" and self.enable_adaptive_verification: + raise ValueError("Adaptive verification only supported with DSpark") return self From 01db35acfce525a06736c91c3e6f3a6fa2723810 Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Fri, 7 Aug 2026 04:00:00 +0000 Subject: [PATCH 30/40] review comments Signed-off-by: Lucas Wilkinson --- .buildkite/test_areas/lm_eval.yaml | 20 ++++++++ docs/features/speculative_decoding/README.md | 2 + .../adaptive_verification.md | 49 +++++++++++++++++++ ...epSeek-V4-Flash-DSpark-confidence-TP4.yaml | 5 +- .../gsm8k/configs/models-spec-decode.txt | 1 + tests/evals/gsm8k/test_gsm8k_correctness.py | 35 +++++++++++++ vllm/config/speculative.py | 2 +- vllm/config/vllm.py | 10 ++++ vllm/v1/attention/backend.py | 37 +++++++------- vllm/v1/attention/backends/flashinfer.py | 9 ++-- vllm/v1/attention/backends/mla/indexer.py | 16 +++--- vllm/v1/attention/selector.py | 15 ++++++ vllm/v1/worker/gpu/attn_utils.py | 22 ++++----- vllm/v1/worker/gpu/model_runner.py | 44 ++++++----------- .../gpu/spec_decode/dspark/speculator.py | 9 ++-- 15 files changed, 198 insertions(+), 78 deletions(-) create mode 100644 docs/features/speculative_decoding/adaptive_verification.md create mode 100644 tests/evals/gsm8k/configs/models-spec-decode.txt diff --git a/.buildkite/test_areas/lm_eval.yaml b/.buildkite/test_areas/lm_eval.yaml index 5507fa6023c1..84ebf0d13aff 100644 --- a/.buildkite/test_areas/lm_eval.yaml +++ b/.buildkite/test_areas/lm_eval.yaml @@ -101,6 +101,26 @@ steps: commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-pcp.txt +- label: LM Eval Spec Decode (4xB200) + key: lm-eval-spec-decode-4xb200 + timeout_in_minutes: 120 + device: b200-k8s + num_devices: 4 + optional: true + source_file_dependencies: + - tests/evals/gsm8k/configs/DeepSeek-V4-Flash-DSpark-confidence-TP4.yaml + - tests/evals/gsm8k/configs/models-spec-decode.txt + - vllm/models/deepseek_v4/ + - vllm/model_executor/models/qwen3_dspark.py + - vllm/v1/spec_decode/metrics.py + - vllm/v1/worker/gpu/spec_decode/adaptive_verification.py + - vllm/v1/worker/gpu/spec_decode/dflash/ + - vllm/v1/worker/gpu/spec_decode/dspark/ + - vllm/v1/worker/gpu/spec_decode/rejection_sampler.py + - vllm/v1/worker/gpu/spec_decode/speculator.py + commands: + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-spec-decode.txt + - label: LM Eval Large Models EP (2xB200) key: lm-eval-large-models-ep-2xb200 timeout_in_minutes: 60 diff --git a/docs/features/speculative_decoding/README.md b/docs/features/speculative_decoding/README.md index ceb25dbfd02f..6af7e3421da0 100644 --- a/docs/features/speculative_decoding/README.md +++ b/docs/features/speculative_decoding/README.md @@ -18,6 +18,7 @@ vLLM supports a variety of methods of speculative decoding. Model-based methods - [Hidden State Extraction](extract_hidden_states.md) - [Custom Proposer Backend (Experimental)](#custom-proposer-backend-experimental) - [Dynamic Speculative Decoding](dynamic_speculative_decoding.md) +- [Adaptive Verification](adaptive_verification.md) ## Method Selection at a Glance @@ -35,6 +36,7 @@ depend on your model family, traffic pattern, hardware, and sampling settings. | Suffix decoding | Low to medium gain | Medium gain | No extra draft model; dynamic speculation depth. | | Custom Proposer | Varies | Varies | Bring your own proposer class (experimental). | | Dynamic Speculative Decoding | High gain | Higher than base SD method | Useful for RL or workload with fluctuating QPS | +| Adaptive Verification | High gain | Higher than base SD method | Sizes verification per request from drafter confidence; currently DSpark only. | For reproducible measurements in your environment, use [`examples/features/speculative_decoding/spec_decode_offline.py`](../../../examples/features/speculative_decoding/spec_decode_offline.py) diff --git a/docs/features/speculative_decoding/adaptive_verification.md b/docs/features/speculative_decoding/adaptive_verification.md new file mode 100644 index 000000000000..5f48feb98144 --- /dev/null +++ b/docs/features/speculative_decoding/adaptive_verification.md @@ -0,0 +1,49 @@ +# Adaptive Verification + +Speculative decoding buys fewer decode steps with more compute. At batch size 1 that is a good trade: the GPU is memory-bound with spare compute, so the extra draft tokens are close to free. At batch size 256 it is a much more delicate one. Draft tokens now compete with real tokens for the same compute, and every rejected token is compute wasted; with enough of them, throughput drops. + +That matters because per-position acceptance decays fast. While the GPU is memory-bound that slot is effectively free and worth the gamble; once it saturates the gamble has a real throughput cost. The crossover moves with load and with workload-dependent acceptance rates, so no static `num_speculative_tokens` is right across concurrencies. + +Adaptive verification decides per step how much of the draft to verify instead. Every (request, position) draft slot is scored by its *survival probability*, the running product of that request's per-position confidences, and the highest-scoring slots are admitted until a global budget is spent. Slots compete across requests: position 5 of a confident request can outrank position 1 of a doubtful one, so one request keeps its full block while another could be trimmed after a token or two. + +The budget itself comes from a cost model profiled at startup. vLLM measures what a step costs at each shape, then picks the token count that maximizes expected accepted tokens per second. + +The practical effect is that one configuration holds up across the whole load range, which removes most of the need to tune `num_speculative_tokens` per deployment. + +## Support + +Adaptive verification needs per-position acceptance estimates, so today its currently only supported for DSpark with a **confidence head**. + +## Usage + +It is enabled by default for DSpark: + +```bash +vllm serve deepseek-ai/DeepSeek-V4-Flash-DSpark \ + --tokenizer-mode deepseek_v4 --trust-remote-code \ + --speculative-config '{ + "method": "dspark", + "model": "deepseek-ai/DeepSeek-V4-Flash-DSpark", + "num_speculative_tokens": 7, + "draft_sample_method": "probabilistic", + "enable_adaptive_verification": true + }' +``` + +Set `enable_adaptive_verification: false` to verify the full block for every request. + +`num_speculative_tokens` must be at least the checkpoint's block size (`dspark_block_size`, e.g. 7 for DeepSeek-V4-Flash); smaller values feed the block machinery a layout it was not trained for and are rejected at startup. + +## Requirements and limitations + +- The attention backend must tolerate device-decided query lengths, since the CPU lengths only bound them from above. Backends that plan off the CPU lengths are excluded by the attention selector, and rejected at startup for models that hard-wire their backend. +- Full cudagraphs are required: step costs are profiled from captured graphs, so `--enforce-eager` is rejected at startup. +- Not supported with LoRA (the per-token LoRA mapping is built from CPU-side boundaries), pipeline parallelism (cost curves and confidences exist only on the last rank), or output logprobs (to be fixed). + +## Tuning the cost profile + +Step costs are profiled against a synthetic KV context, 8192 tokens by default. Deployments serving much longer contexts may want to raise it so the profiled step reads a more realistic amount of cache (this matters a bit less for sparse attention models like DeepSeek-v4 since the cheap indexer is the main cost that scales with context length). + +```bash +export VLLM_ADAPTIVE_VERIFICATION_PROFILE_CONTEXT_LEN=131072 +``` diff --git a/tests/evals/gsm8k/configs/DeepSeek-V4-Flash-DSpark-confidence-TP4.yaml b/tests/evals/gsm8k/configs/DeepSeek-V4-Flash-DSpark-confidence-TP4.yaml index 2ea04d39d3b9..61fbaf33e29a 100644 --- a/tests/evals/gsm8k/configs/DeepSeek-V4-Flash-DSpark-confidence-TP4.yaml +++ b/tests/evals/gsm8k/configs/DeepSeek-V4-Flash-DSpark-confidence-TP4.yaml @@ -1,5 +1,8 @@ model_name: "deepseek-ai/DeepSeek-V4-Flash-DSpark" accuracy_threshold: 0.92 +# lower acceptance length since trimming drafts at higher concurrency reduces +# acceptance length, this is very loose lower bound for smoke testing +min_acceptance_length: 1.2 num_questions: 1319 num_fewshot: 5 startup_max_wait_seconds: 1800 @@ -14,7 +17,7 @@ server_args: >- --kv-cache-dtype fp8 --attention_config.use_fp4_indexer_cache=True --max-num-batched-tokens 16384 - --max-num-seqs 32 + --max-num-seqs 128 --speculative-config '{"method":"dspark", "model":"deepseek-ai/DeepSeek-V4-Flash-DSpark", "attention_backend":"FLASH_ATTN","num_speculative_tokens":7, diff --git a/tests/evals/gsm8k/configs/models-spec-decode.txt b/tests/evals/gsm8k/configs/models-spec-decode.txt new file mode 100644 index 000000000000..05f620435c37 --- /dev/null +++ b/tests/evals/gsm8k/configs/models-spec-decode.txt @@ -0,0 +1 @@ +DeepSeek-V4-Flash-DSpark-confidence-TP4.yaml diff --git a/tests/evals/gsm8k/test_gsm8k_correctness.py b/tests/evals/gsm8k/test_gsm8k_correctness.py index e1ae6ec54272..5a237cf94764 100644 --- a/tests/evals/gsm8k/test_gsm8k_correctness.py +++ b/tests/evals/gsm8k/test_gsm8k_correctness.py @@ -12,6 +12,7 @@ import shlex import pytest +import requests import yaml from tests.utils import RemoteOpenAIServer @@ -65,6 +66,28 @@ def run_gsm8k_eval(eval_config: dict, server_url: str) -> dict: return results +def get_acceptance_length(server_url: str) -> float: + """Mean tokens emitted per verification step, from the server's counters. + + 1.0 means every draft was rejected (speculation bought nothing); the + theoretical maximum is 1 + num_speculative_tokens. + """ + response = requests.get(f"{server_url.rstrip('/').removesuffix('/v1')}/metrics") + response.raise_for_status() + counters: dict[str, float] = {} + for line in response.text.splitlines(): + if line.startswith("vllm:spec_decode_num_"): + name, _, value = line.partition(" ") + counters[name.split("{")[0]] = float(value) + + num_drafts = counters.get("vllm:spec_decode_num_drafts_total", 0.0) + num_accepted = counters.get("vllm:spec_decode_num_accepted_tokens_total", 0.0) + assert num_drafts > 0, ( + "no drafts recorded; speculative decoding did not run for this config" + ) + return 1.0 + num_accepted / num_drafts + + def test_gsm8k_correctness(config_filename): """Test GSM8K correctness for a given model configuration.""" eval_config = yaml.safe_load(config_filename.read_text(encoding="utf-8")) @@ -168,4 +191,16 @@ def test_gsm8k_correctness(config_filename): f"{expected_metric:.4f} - {tol:.4f} = {expected_metric - tol:.4f}" ) + # Speculative configs additionally assert that drafts are actually + # landing: accuracy alone passes even when every draft is rejected. + min_acceptance_length = eval_config.get("min_acceptance_length") + if min_acceptance_length is not None: + acceptance_length = get_acceptance_length(server_url) + print(f" Mean acceptance length: {acceptance_length:.3f}") + print(f" Minimum acceptance length: {min_acceptance_length:.3f}") + assert acceptance_length >= min_acceptance_length, ( + f"Acceptance length too low: {acceptance_length:.3f} < " + f"{min_acceptance_length:.3f}" + ) + print(f"✅ GSM8K test passed for {eval_config['model_name']}") diff --git a/vllm/config/speculative.py b/vllm/config/speculative.py index 0e0ee9005dd1..362980f4fd84 100644 --- a/vllm/config/speculative.py +++ b/vllm/config/speculative.py @@ -237,7 +237,7 @@ class SpeculativeConfig: synthetic_acceptance_rates. Only valid when rejection_sample_method is 'synthetic'. Mutually exclusive with synthetic_acceptance_rates.""" - enable_adaptive_verification: bool = True + enable_adaptive_verification: bool = False """Whether to adaptively size the draft-verification budget from per-request confidence. Currently only supported for method="dspark".""" diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index 10ee378503a4..7fabc90a4d91 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -2348,6 +2348,16 @@ def _get_v2_model_runner_unsupported_features(self) -> list[str]: # while the trimmed batch's true boundaries are decided on the GPU. unsupported.append("adaptive verification with LoRA") + if ( + speculative_config.enable_adaptive_verification + and self.compilation_config.cudagraph_mode == CUDAGraphMode.NONE + ): + # The draft budget divides by step costs profiled from captured + # cudagraphs; eager execution captures none. + unsupported.append( + "adaptive verification with enforce_eager/cudagraph_mode=none" + ) + if ( speculative_config.enable_adaptive_verification and self.parallel_config.pipeline_parallel_size > 1 diff --git a/vllm/v1/attention/backend.py b/vllm/v1/attention/backend.py index 2975b690d854..3ec3307fd462 100644 --- a/vllm/v1/attention/backend.py +++ b/vllm/v1/attention/backend.py @@ -280,6 +280,17 @@ def supports_batch_invariance(cls) -> bool: def supports_kv_connector(cls) -> bool: return True + @classmethod + def supports_device_cpu_query_lens_mismatch(cls) -> bool: + """Whether this backend can run a batch whose device query_start_loc disagrees + with the CPU one; backends that plan off the CPU query lengths must opt out. + + Currently only verification requests are affected: adaptive verification trims + their drafts on device, so the CPU lengths bound them from above. The + decode/prefill split point and the CPU prefill query lengths both stay correct. + """ + return True + @classmethod def supports_pcp(cls) -> bool: try: @@ -334,6 +345,7 @@ def validate_configuration( use_batch_invariant: bool = False, use_kv_connector: bool = False, use_pcp: bool = False, + use_adaptive_verification: bool = False, ) -> list[str]: invalid_reasons = [] if not cls.supports_head_size(head_size): @@ -376,6 +388,14 @@ def validate_configuration( invalid_reasons.append("KV connector not supported") if use_pcp and not cls.supports_pcp(): invalid_reasons.append("PCP not supported") + if ( + use_adaptive_verification + and not cls.supports_device_cpu_query_lens_mismatch() + ): + invalid_reasons.append( + "device-cpu query lens mismatch not supported, " + "this is needed for adaptive verification" + ) combination_reason = cls.supports_combination( head_size, dtype, @@ -632,8 +652,6 @@ class AttentionMetadataBuilder(ABC, Generic[M]): # Does this backend/builder support updating the block table in existing # metadata supports_update_block_table: bool = False - # Do not access directly. See supports_device_cpu_query_lens_mismatch(). - _supports_device_cpu_query_lens_mismatch: ClassVar[bool] = True # Whether the builder constructor requires the block-table width. requires_block_table_width: ClassVar[bool] = False @@ -659,21 +677,6 @@ def get_cudagraph_support( """Get the cudagraph support level of this builder class.""" return cls._cudagraph_support - @classmethod - def supports_device_cpu_query_lens_mismatch( - cls: type["AttentionMetadataBuilder"], - vllm_config: "VllmConfig", - kv_cache_spec: "KVCacheSpec", - ) -> bool: - """Whether this backend can run a batch whose device query_start_loc disagrees - with the CPU one; backends that plan off the CPU query lengths must opt out. - - Currently only verification requests are affected: adaptive verification trims - their drafts on device, so the CPU lengths bound them from above. The - decode/prefill split point and the CPU prefill query lengths both stay correct. - """ - return cls._supports_device_cpu_query_lens_mismatch - def _init_reorder_batch_threshold( self, reorder_batch_threshold: int | None = 1, diff --git a/vllm/v1/attention/backends/flashinfer.py b/vllm/v1/attention/backends/flashinfer.py index 526463719285..00260d6121e9 100755 --- a/vllm/v1/attention/backends/flashinfer.py +++ b/vllm/v1/attention/backends/flashinfer.py @@ -382,6 +382,12 @@ def get_name() -> str: def supports_non_causal(cls) -> bool: return True + @classmethod + def supports_device_cpu_query_lens_mismatch(cls) -> bool: + # The wrappers are planned from qo_indptr_cpu, so the CPU query offsets + # have to be the ones the kernel runs on. + return False + @classmethod def supports_sliding_window(cls) -> bool: return True @@ -620,9 +626,6 @@ class FlashInferMetadata: class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): kv_cache_spec: AttentionSpec reorder_batch_threshold: int = 1 - # The wrappers are planned from qo_indptr_cpu, so the CPU query offsets have to be - # the ones the kernel runs on. - _supports_device_cpu_query_lens_mismatch: ClassVar[bool] = False def __init__( self, diff --git a/vllm/v1/attention/backends/mla/indexer.py b/vllm/v1/attention/backends/mla/indexer.py index 3a4abd7debf9..30951ca4c12c 100644 --- a/vllm/v1/attention/backends/mla/indexer.py +++ b/vllm/v1/attention/backends/mla/indexer.py @@ -128,6 +128,12 @@ class DeepseekV32IndexerBackend(AttentionBackend): def supports_pcp(cls) -> bool: return True + @classmethod + def supports_device_cpu_query_lens_mismatch(cls) -> bool: + # Only the varlen paged MQA logits kernel takes per-request query + # lengths from device tensors; otherwise the indexer needs uniform ones. + return _supports_varlen_paged_mqa_logits() + @staticmethod def get_name() -> str: return "DEEPSEEK_V32_INDEXER" @@ -477,16 +483,6 @@ def get_cudagraph_support( return AttentionCGSupport.ALWAYS return AttentionCGSupport.UNIFORM_BATCH - @classmethod - def supports_device_cpu_query_lens_mismatch( - cls, - vllm_config: VllmConfig, - kv_cache_spec: KVCacheSpec, - ) -> bool: - # Only the varlen paged MQA logits kernel takes per-request query lengths from - # device tensors; otherwise the indexer needs uniform ones. - return _supports_varlen_paged_mqa_logits() - def __init__(self, *args, block_table_width: int, **kwargs) -> None: super().__init__(*args, **kwargs) scheduler_config = self.vllm_config.scheduler_config diff --git a/vllm/v1/attention/selector.py b/vllm/v1/attention/selector.py index bff26e026676..00f2aed5c506 100644 --- a/vllm/v1/attention/selector.py +++ b/vllm/v1/attention/selector.py @@ -37,6 +37,7 @@ class AttentionSelectorConfig(NamedTuple): use_batch_invariant: bool = False use_kv_connector: bool = False use_pcp: bool = False + use_adaptive_verification: bool = False def __repr__(self): return ( @@ -54,6 +55,7 @@ def __repr__(self): f"use_non_causal={self.use_non_causal}, " f"use_batch_invariant={self.use_batch_invariant}, " f"use_kv_connector={self.use_kv_connector}, " + f"use_adaptive_verification={self.use_adaptive_verification}, " f"use_pcp={self.use_pcp})" ) @@ -136,6 +138,18 @@ def get_attn_backend( kv_transfer_config is not None and kv_transfer_config.is_kv_transfer_instance ) + speculative_config = vllm_config.speculative_config + use_adaptive_verification = ( + speculative_config is not None + and speculative_config.enable_adaptive_verification + ) + if use_adaptive_verification: + from vllm.compilation.backends import model_tag + + # The drafter always runs full-length blocks; only the verifier sees + # the trimmed, device-decided query lengths. + use_adaptive_verification = model_tag != "dspark_head" + attn_type = attn_type or AttentionType.DECODER attn_selector_config = AttentionSelectorConfig( head_size=head_size, @@ -153,6 +167,7 @@ def get_attn_backend( use_batch_invariant=envs.VLLM_BATCH_INVARIANT, use_kv_connector=use_kv_connector, use_pcp=vllm_config.parallel_config.prefill_context_parallel_size > 1, + use_adaptive_verification=use_adaptive_verification, ) # A per-KV-group override (keyed by KVCacheSpecKind) takes precedence over diff --git a/vllm/v1/worker/gpu/attn_utils.py b/vllm/v1/worker/gpu/attn_utils.py index 429ac87d0424..e088eee814d7 100644 --- a/vllm/v1/worker/gpu/attn_utils.py +++ b/vllm/v1/worker/gpu/attn_utils.py @@ -180,24 +180,20 @@ def init_attn_backend( return attn_groups, attn_cg_support_info, kernel_block_sizes -def get_device_cpu_query_lens_mismatch_support( +def get_query_lens_mismatch_unsupported_backend( attn_groups: list[list[AttentionGroup]], - vllm_config: VllmConfig, -) -> tuple[bool, str | None]: - """Whether every backend tolerates a device/CPU query length mismatch. - See AttentionMetadataBuilder.supports_device_cpu_query_lens_mismatch() for details. +) -> str | None: + """Name the first backend needing the CPU query lengths to be exact, if any. - Returns the aggregate answer and, when it is False, the first backend that does not. + The attention selector already excludes these when adaptive verification is + enabled, but models that hard-wire their backend never consult it. See + AttentionBackend.supports_device_cpu_query_lens_mismatch(). """ for groups in attn_groups: for group in groups: - builder = group.get_metadata_builder(0) - if not builder.supports_device_cpu_query_lens_mismatch( - vllm_config, - group.kv_cache_spec, - ): - return False, group.backend.__name__ - return True, None + if not group.backend.supports_device_cpu_query_lens_mismatch(): + return group.backend.__name__ + return None def _allocate_kv_cache( diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 1d86f7aab525..bf8b1e1f65fd 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -74,8 +74,8 @@ ) from vllm.v1.worker.gpu.attn_utils import ( build_slot_mappings_by_layer, - get_device_cpu_query_lens_mismatch_support, get_kv_cache_spec, + get_query_lens_mismatch_unsupported_backend, init_attn_backend, init_kv_cache, ) @@ -510,10 +510,10 @@ def initialize_kv_cache(self, kv_cache_config: KVCacheConfig) -> None: # The speculator clears the flag at load time when the checkpoint has # no confidence head, so it holds the effective value. if getattr(self.speculator, "enable_adaptive_verification", False): - supported, backend = get_device_cpu_query_lens_mismatch_support( - self.attn_groups, self.vllm_config - ) - if not supported: + # The selector rejects unsupported backends, but models that + # hard-wire theirs (e.g. DeepSeek-V4) never go through it. + backend = get_query_lens_mismatch_unsupported_backend(self.attn_groups) + if backend is not None: raise ValueError( "Adaptive verification trims verification requests on device, which" f" the {backend} attention backend does not support. support. Pass " @@ -1034,9 +1034,7 @@ def prepare_inputs( # batch_idx -> req_id draft_tokens = scheduler_output.scheduled_spec_decode_tokens - # Drafts-first ordering is only required by adaptive verification - # (compact_batch assumes verification requests lead); keep main's - # ordering untouched otherwise. + # Drafts-first ordering is required by adaptive verification req_ids = sort_batch_req_ids( num_tokens_per_req, draft_tokens if self.adaptive_verification is not None else {}, @@ -1083,9 +1081,9 @@ def prepare_inputs( ) scheduled_num_tokens = num_scheduled_tokens if adaptive_verification is not None: - # The GPU may regrant trimmed drafts unevenly, but never beyond a - # request's scheduled count: the pre-trim counts remain the - # per-request upper bound for CPU-side metadata. + # num_scheduled_tokens represents the draft budget evenly distributed across + # all verification requests, `reallocate_drafts` will unevenly assign the + # draft budget to requests on the GPU side only. num_scheduled_tokens = adaptive_verification.compact_batch( num_draft_tokens_per_req, num_scheduled_tokens, @@ -1373,24 +1371,12 @@ def execute_model( and scheduler_output.scheduled_spec_decode_tokens and not dummy_run ): - if self.adaptive_verification.cost_tables is None: - # No FULL graphs were captured (enforce_eager, or the attention - # backend lacks varlen support), so step costs could not be - # profiled. Consistent across ranks: TP shares curves via - # broadcast and PP is rejected at config time. - logger.warning_once( - "DSpark adaptive verification disabled: no cudagraph cost " - "profile is available. Falling back to fixed-length " - "verification." - ) - self.adaptive_verification = None - else: - num_toks = self.adaptive_verification.get_num_tokens( - scheduler_output.num_scheduled_tokens, - scheduler_output.scheduled_spec_decode_tokens, - ) - # Trimming drafts makes the batch non-uniform. - uniform_tok_count = None + num_toks = self.adaptive_verification.get_num_tokens( + scheduler_output.num_scheduled_tokens, + scheduler_output.scheduled_spec_decode_tokens, + ) + # Trimming drafts makes the batch non-uniform. + uniform_tok_count = None batch_desc, num_tokens_across_dp = dispatch_cg_and_sync_dp( self.cudagraph_manager, diff --git a/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py b/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py index 3267f38c9cff..19fc45201846 100644 --- a/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py @@ -109,11 +109,12 @@ def load_draft_model( device=self.device, ) if self.enable_adaptive_verification and model.model.confidence_head is None: - logger.warning_once( - "DSpark checkpoint has no confidence head; disabling adaptive " - "verification and using fixed-length verification." + raise ValueError( + "Adaptive verification needs a DSpark checkpoint with a confidence " + "head, and this one has none. Pass " + "enable_adaptive_verification=false in the speculative config to verify" + " a fixed number of drafts instead." ) - self.enable_adaptive_verification = False return model def _sample_logits( From 3becde2bcea7011f2edea40af4eff254065f7912 Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Fri, 7 Aug 2026 22:00:38 +0000 Subject: [PATCH 31/40] review comments Signed-off-by: Lucas Wilkinson --- .../test_rejection_sampler_chunking.py | 116 ++++++++++++++++++ tests/v1/worker/test_gpu_batch_ordering.py | 6 +- vllm/v1/worker/gpu/block_table.py | 22 ---- vllm/v1/worker/gpu/input_batch.py | 67 +++++++--- vllm/v1/worker/gpu/model_runner.py | 20 ++- .../gpu/spec_decode/rejection_sampler.py | 15 ++- 6 files changed, 194 insertions(+), 52 deletions(-) create mode 100644 tests/v1/spec_decode/test_rejection_sampler_chunking.py diff --git a/tests/v1/spec_decode/test_rejection_sampler_chunking.py b/tests/v1/spec_decode/test_rejection_sampler_chunking.py new file mode 100644 index 000000000000..333170a732a2 --- /dev/null +++ b/tests/v1/spec_decode/test_rejection_sampler_chunking.py @@ -0,0 +1,116 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Chunk selection in the rejection sampler's verification loop. + +Adaptive verification compacts the logits on device but leaves +``cu_num_logits_np`` describing the pre-trim layout, so the CPU prefix sums must +not be used to slice the compacted tensor. +""" + +import numpy as np +import pytest +import torch + +from vllm.v1.worker.gpu.spec_decode.rejection_sampler import ( + RejectionSampler, + _iter_request_chunks, +) + + +class _FakeInputBatch: + def __init__(self, cu_num_logits_np: np.ndarray, num_logits: int): + self.num_reqs = cu_num_logits_np.size - 1 + # Pre-trim layout, as prepare_inputs leaves it. + self.cu_num_logits_np = cu_num_logits_np + # Compacted device layout, as reallocate_drafts produces it. + step = num_logits // self.num_reqs + device_cu = np.arange(self.num_reqs + 1, dtype=np.int32) * step + device_cu[-1] = num_logits + self.cu_num_logits = torch.from_numpy(device_cu) + self.idx_mapping = torch.arange(self.num_reqs) + self.idx_mapping_np = np.arange(self.num_reqs) + self.expanded_idx_mapping = torch.zeros(num_logits, dtype=torch.int32) + self.expanded_local_pos = torch.zeros(num_logits, dtype=torch.int32) + + +def _run(sampler, logits, input_batch, max_chunk_logits): + return RejectionSampler._verify_in_chunks( + sampler, + logits, + input_batch, + None, + torch.zeros(logits.shape[0], dtype=torch.int64), + torch.zeros(logits.shape[0], dtype=torch.int32), + max_chunk_logits, + -1, # NO_LOGPROBS + ) + + +@pytest.fixture +def sampler(): + """A RejectionSampler stub that records the ranges _verify is asked for.""" + obj = object.__new__(RejectionSampler) + obj.sampler = type("S", (), {"logprobs_mode": "raw_logprobs"})() + obj.calls = [] + + def fake_verify(logits, draft_logits, draft_sampled, pos, cu_num_logits, *args): + obj.calls.append((logits.shape[0], cu_num_logits.tolist())) + n = cu_num_logits.shape[0] - 1 + return ( + None, + torch.zeros(n, dtype=torch.int64), + torch.zeros(n, dtype=torch.int32), + ) + + def fake_logprobs(*args, **kwargs): + return None + + obj._verify = fake_verify + obj._get_logprobs_tensors = fake_logprobs + return obj + + +def test_compacted_logits_do_not_use_stale_cpu_offsets(sampler): + """pre_trim_total > chunk_limit >= compacted_total must stay one chunk. + + Chunking off the stale CPU prefix sums would slice the compacted logits with + pre-trim offsets, silently reading the wrong rows for every request past the + first chunk boundary. + """ + num_reqs, chunk_limit = 8, 40 + # Scheduler handed out 8 logits per request; the budget trimmed it to 4. + pre_trim = np.arange(num_reqs + 1, dtype=np.int32) * 8 # total 64 > 40 + compacted_total = 32 # <= 40 + assert pre_trim[-1] > chunk_limit >= compacted_total + + input_batch = _FakeInputBatch(pre_trim, compacted_total) + logits = torch.zeros(compacted_total, 16) + _run(sampler, logits, input_batch, chunk_limit) + + assert len(sampler.calls) == 1, "compacted batch must verify in a single chunk" + num_rows, cu_num_logits = sampler.calls[0] + assert num_rows == compacted_total + assert cu_num_logits == input_batch.cu_num_logits.tolist(), ( + "must verify against the device offsets, not the pre-trim CPU ones" + ) + + +def test_oversized_batch_still_chunks(sampler): + """Without compaction, a batch past the limit is still split per request.""" + num_reqs, chunk_limit = 8, 24 + cu = np.arange(num_reqs + 1, dtype=np.int32) * 8 # total 64 > 24 + input_batch = _FakeInputBatch(cu, int(cu[-1])) + logits = torch.zeros(int(cu[-1]), 16) + _run(sampler, logits, input_batch, chunk_limit) + + assert len(sampler.calls) > 1 + assert sum(rows for rows, _ in sampler.calls) == int(cu[-1]) + assert all(rows <= chunk_limit for rows, _ in sampler.calls) + + +def test_iter_request_chunks_never_splits_a_request(): + cu = np.array([0, 3, 6, 9, 12], dtype=np.int32) + chunks = list(_iter_request_chunks(cu, 7)) + assert chunks[0][0] == 0 and chunks[-1][1] == 4 + for start, end in chunks: + assert end > start diff --git a/tests/v1/worker/test_gpu_batch_ordering.py b/tests/v1/worker/test_gpu_batch_ordering.py index 2c2115dc81eb..ca1e67744aba 100644 --- a/tests/v1/worker/test_gpu_batch_ordering.py +++ b/tests/v1/worker/test_gpu_batch_ordering.py @@ -40,14 +40,14 @@ def _make_common_attn_metadata(query_lens: list[int]) -> CommonAttentionMetadata def test_sort_batch_req_ids_no_spec(): # decode_query_len == 1: plain ascending order (decodes first). num_tokens_per_req = {"p1": 100, "d1": 1, "p2": 7, "d2": 1} - assert sort_batch_req_ids(num_tokens_per_req, 1) == ["d1", "d2", "p2", "p1"] + assert sort_batch_req_ids(num_tokens_per_req, {}, 1) == ["d1", "d2", "p2", "p1"] def test_sort_batch_req_ids_spec_decode(): # decode_query_len == 2 (MTP k=1): uniform decodes lead, then the 1-token # chunked-prefill tail, then longer prefills. num_tokens_per_req = {"tail": 1, "d1": 2, "p1": 100, "d2": 2} - assert sort_batch_req_ids(num_tokens_per_req, 2) == ["d1", "d2", "tail", "p1"] + assert sort_batch_req_ids(num_tokens_per_req, {}, 2) == ["d1", "d2", "tail", "p1"] def test_spec_decodes_lead_short_prefill_tail(): @@ -55,7 +55,7 @@ def test_spec_decodes_lead_short_prefill_tail(): # uniform 2-token decodes as decodes even when a 1-token prefill tail is # in the batch (indexer-style: require_uniform, threshold=1+k). num_tokens_per_req = {"tail": 1, **{f"d{i}": 2 for i in range(8)}} - req_ids = sort_batch_req_ids(num_tokens_per_req, 2) + req_ids = sort_batch_req_ids(num_tokens_per_req, {}, 2) query_lens = [num_tokens_per_req[r] for r in req_ids] assert query_lens == [2] * 8 + [1] diff --git a/vllm/v1/worker/gpu/block_table.py b/vllm/v1/worker/gpu/block_table.py index d3b871e7c034..90fc104dc29f 100644 --- a/vllm/v1/worker/gpu/block_table.py +++ b/vllm/v1/worker/gpu/block_table.py @@ -5,7 +5,6 @@ import torch from vllm.triton_utils import tl, triton -from vllm.utils.math_utils import cdiv from vllm.v1.attention.backends.utils import PAD_SLOT_ID from vllm.v1.worker.gpu.buffer_utils import ( FusedStagedWriter, @@ -176,27 +175,6 @@ def get_dummy_block_tables(self, num_reqs: int) -> tuple[torch.Tensor, ...]: block_table[:num_reqs].zero_() for block_table in self.input_block_tables ) - def set_dummy_context( - self, num_reqs: int, query_len: int, context_len: int, num_kv_blocks: int - ) -> int: - """ - Give each dummy request its own span of KV blocks. - Used when profiling step cost. - """ - for block_table, block_size, bpk in zip( - self.input_block_tables, self.kernel_block_sizes, self.blocks_per_kv_block - ): - num_blocks = min( - cdiv(context_len + query_len, block_size), block_table.shape[1] - ) - # Spans are disjoint until the pool runs out, then they wrap and share - # blocks: profiling only needs the reads to be realistic, not distinct. - block_ids = torch.arange( - num_reqs * num_blocks, dtype=block_table.dtype, device=self.device - ) % (num_kv_blocks * bpk) - block_table[:num_reqs, :num_blocks] = block_ids.view(num_reqs, num_blocks) - return context_len - def compute_slot_mappings( self, idx_mapping: torch.Tensor, diff --git a/vllm/v1/worker/gpu/input_batch.py b/vllm/v1/worker/gpu/input_batch.py index 5edc13bbb31e..6c9c979401e9 100644 --- a/vllm/v1/worker/gpu/input_batch.py +++ b/vllm/v1/worker/gpu/input_batch.py @@ -1,12 +1,17 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from dataclasses import dataclass +from typing import TYPE_CHECKING import numpy as np import torch from vllm.triton_utils import tl, triton from vllm.utils import random_uuid +from vllm.utils.math_utils import cdiv + +if TYPE_CHECKING: + from vllm.v1.worker.gpu.block_table import BlockTables class InputBuffers: @@ -110,7 +115,6 @@ def make_dummy( num_reqs: int, num_tokens: int, input_buffers: InputBuffers, - context_len: int = 0, max_query_len: int | None = None, ) -> "InputBatch": assert 0 < num_reqs <= num_tokens @@ -154,14 +158,6 @@ def make_dummy( input_ids = input_buffers.input_ids[:num_tokens].zero_() positions = input_buffers.positions[:num_tokens].zero_() - if context_len: - # Decode-like shape: each request continues after context_len - # already-computed tokens. - seq_lens += context_len - local_pos = np.arange(num_tokens, dtype=np.int64) - np.repeat( - query_start_loc_np[:-1], num_scheduled_tokens - ) - positions.copy_(torch.from_numpy(local_pos + context_len)) input_buffers.is_padding[:num_tokens].fill_(True) is_padding = input_buffers.is_padding[:num_tokens] @@ -169,7 +165,9 @@ def make_dummy( logits_indices = query_start_loc[1:] - 1 cu_num_logits = torch.arange(num_reqs + 1, device=device, dtype=torch.int32) cu_num_logits_np = np.arange(num_reqs + 1, dtype=np.int32) - seq_lens_cpu_upper_bound = torch.from_numpy(num_scheduled_tokens + context_len) + # Copy so set_dummy_context can add context in place without touching + # num_scheduled_tokens. + seq_lens_cpu_upper_bound = torch.from_numpy(num_scheduled_tokens.copy()) return cls( req_ids=req_ids, num_reqs=num_reqs, @@ -188,11 +186,9 @@ def make_dummy( seq_lens=seq_lens, seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound, dcp_local_seq_lens=None, - num_computed_tokens_np=np.full(num_reqs, context_len, dtype=np.int32), + num_computed_tokens_np=np.zeros(num_reqs, dtype=np.int32), prefill_len_np=np.zeros(num_reqs, dtype=np.int32), - num_computed_prefill_tokens_np=np.full( - num_reqs, context_len, dtype=np.int32 - ), + num_computed_prefill_tokens_np=np.zeros(num_reqs, dtype=np.int32), is_prefilling_np=np.zeros(num_reqs, dtype=np.bool_), max_seq_len_np=None, input_ids=input_ids, @@ -207,6 +203,49 @@ def make_dummy( ) +def set_dummy_context( + input_batch: InputBatch, + block_tables: "BlockTables", + context_len: int, + num_kv_blocks: int, + max_model_len: int, +) -> None: + """Give each dummy request context_len of context, used when profiling step cost.""" + if not block_tables.input_block_tables: + # Attention-free models have no KV context to fabricate. + return + num_reqs = input_batch.num_reqs + query_len = input_batch.max_query_len or int(input_batch.num_scheduled_tokens.max()) + context_len = max(min(context_len, max_model_len - query_len), 0) + if not context_len: + return + + # Decode-like shape: each request continues after context_len + # already-computed tokens. + input_batch.seq_lens += context_len + input_batch.seq_lens_cpu_upper_bound += context_len + input_batch.num_computed_tokens_np.fill(context_len) + input_batch.num_computed_prefill_tokens_np.fill(context_len) + local_pos = np.arange(input_batch.num_tokens, dtype=np.int64) - np.repeat( + input_batch.query_start_loc_np[:-1], input_batch.num_scheduled_tokens + ) + input_batch.positions.copy_(torch.from_numpy(local_pos + context_len)) + + seq_len = context_len + query_len + for block_table, block_size, bpk in zip( + block_tables.input_block_tables, + block_tables.kernel_block_sizes, + block_tables.blocks_per_kv_block, + ): + num_blocks = min(cdiv(seq_len, block_size), block_table.shape[1]) + # Spans are disjoint until the pool runs out, then they wrap and share + # blocks: profiling only needs the reads to be realistic, not distinct. + block_ids = torch.arange( + num_reqs * num_blocks, dtype=block_table.dtype, device=block_table.device + ) % (num_kv_blocks * bpk) + block_table[:num_reqs, :num_blocks] = block_ids.view(num_reqs, num_blocks) + + @triton.jit def _prepare_prefill_inputs_kernel( input_ids_ptr, diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index bf8b1e1f65fd..90715a7eb9f8 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -102,6 +102,7 @@ post_update_num_computed_tokens, prepare_pos_seq_lens, prepare_prefill_inputs, + set_dummy_context, ) from vllm.v1.worker.gpu.kv_connector import ( NO_OP_KV_CONNECTOR, @@ -1422,25 +1423,22 @@ def execute_model( else: # No actual tokens to run. A dummy run for DP or memory profiling. dummy_num_reqs = batch_desc.num_reqs or num_reqs - if context_len and self.kernel_block_sizes: - query_len = batch_desc.max_query_len or cdiv( - batch_desc.num_tokens, dummy_num_reqs - ) - context_len = self.block_tables.set_dummy_context( - dummy_num_reqs, - query_len, - min(context_len, self.max_model_len - query_len), - self.kv_cache_config.num_blocks, - ) input_batch = InputBatch.make_dummy( dummy_num_reqs, batch_desc.num_tokens, self.input_buffers, - context_len=context_len, max_query_len=batch_desc.max_query_len, ) if not skip_attn_for_dummy_run: block_tables, slot_mappings = self.prepare_dummy_attn(input_batch) + if context_len: + set_dummy_context( + input_batch, + self.block_tables, + context_len, + self.kv_cache_config.num_blocks, + self.max_model_len, + ) else: assert batch_desc.cg_mode != CUDAGraphMode.FULL, ( "Attention metadata must be prepared for dummy runs when using " diff --git a/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py b/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py index 888daf22f20b..9f4841a2835c 100644 --- a/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py +++ b/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from collections.abc import Iterator +from collections.abc import Iterable, Iterator import numpy as np import torch @@ -181,11 +181,22 @@ def _verify_in_chunks( ) -> tuple[torch.Tensor, torch.Tensor, LogprobsTensors | None]: cu_num_logits_np = input_batch.cu_num_logits_np use_processed_logits = self.sampler.logprobs_mode in PROCESSED_LOGPROBS_MODES + num_reqs = input_batch.num_reqs + + if logits.shape[0] <= max_chunk_logits: + # One chunk covers the batch. Adaptive verification compacts the logits + # without updating cu_num_logits_np (it keeps the pre-compacted layout), + # so the stale sums must not pick chunk boundaries; its budget cap + # guarantees the compacted batch always lands here. + request_chunks: Iterable[tuple[int, int]] = ((0, num_reqs),) + else: + request_chunks = _iter_request_chunks(cu_num_logits_np, max_chunk_logits) + sampled_chunks: list[torch.Tensor] = [] num_sampled_chunks: list[torch.Tensor] = [] logprobs_chunks: list[LogprobsTensors] = [] - for start, end in _iter_request_chunks(cu_num_logits_np, max_chunk_logits): + for start, end in request_chunks: lo = int(cu_num_logits_np[start]) hi = int(cu_num_logits_np[end]) chunk_cu_num_logits_np = cu_num_logits_np[start : end + 1] - lo From d80ff5b6a3d384a87548ffe70fd8819a1bb27612 Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Fri, 7 Aug 2026 22:05:27 +0000 Subject: [PATCH 32/40] remove slop tests Signed-off-by: Lucas Wilkinson --- .../test_rejection_sampler_chunking.py | 116 ------------------ 1 file changed, 116 deletions(-) delete mode 100644 tests/v1/spec_decode/test_rejection_sampler_chunking.py diff --git a/tests/v1/spec_decode/test_rejection_sampler_chunking.py b/tests/v1/spec_decode/test_rejection_sampler_chunking.py deleted file mode 100644 index 333170a732a2..000000000000 --- a/tests/v1/spec_decode/test_rejection_sampler_chunking.py +++ /dev/null @@ -1,116 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Chunk selection in the rejection sampler's verification loop. - -Adaptive verification compacts the logits on device but leaves -``cu_num_logits_np`` describing the pre-trim layout, so the CPU prefix sums must -not be used to slice the compacted tensor. -""" - -import numpy as np -import pytest -import torch - -from vllm.v1.worker.gpu.spec_decode.rejection_sampler import ( - RejectionSampler, - _iter_request_chunks, -) - - -class _FakeInputBatch: - def __init__(self, cu_num_logits_np: np.ndarray, num_logits: int): - self.num_reqs = cu_num_logits_np.size - 1 - # Pre-trim layout, as prepare_inputs leaves it. - self.cu_num_logits_np = cu_num_logits_np - # Compacted device layout, as reallocate_drafts produces it. - step = num_logits // self.num_reqs - device_cu = np.arange(self.num_reqs + 1, dtype=np.int32) * step - device_cu[-1] = num_logits - self.cu_num_logits = torch.from_numpy(device_cu) - self.idx_mapping = torch.arange(self.num_reqs) - self.idx_mapping_np = np.arange(self.num_reqs) - self.expanded_idx_mapping = torch.zeros(num_logits, dtype=torch.int32) - self.expanded_local_pos = torch.zeros(num_logits, dtype=torch.int32) - - -def _run(sampler, logits, input_batch, max_chunk_logits): - return RejectionSampler._verify_in_chunks( - sampler, - logits, - input_batch, - None, - torch.zeros(logits.shape[0], dtype=torch.int64), - torch.zeros(logits.shape[0], dtype=torch.int32), - max_chunk_logits, - -1, # NO_LOGPROBS - ) - - -@pytest.fixture -def sampler(): - """A RejectionSampler stub that records the ranges _verify is asked for.""" - obj = object.__new__(RejectionSampler) - obj.sampler = type("S", (), {"logprobs_mode": "raw_logprobs"})() - obj.calls = [] - - def fake_verify(logits, draft_logits, draft_sampled, pos, cu_num_logits, *args): - obj.calls.append((logits.shape[0], cu_num_logits.tolist())) - n = cu_num_logits.shape[0] - 1 - return ( - None, - torch.zeros(n, dtype=torch.int64), - torch.zeros(n, dtype=torch.int32), - ) - - def fake_logprobs(*args, **kwargs): - return None - - obj._verify = fake_verify - obj._get_logprobs_tensors = fake_logprobs - return obj - - -def test_compacted_logits_do_not_use_stale_cpu_offsets(sampler): - """pre_trim_total > chunk_limit >= compacted_total must stay one chunk. - - Chunking off the stale CPU prefix sums would slice the compacted logits with - pre-trim offsets, silently reading the wrong rows for every request past the - first chunk boundary. - """ - num_reqs, chunk_limit = 8, 40 - # Scheduler handed out 8 logits per request; the budget trimmed it to 4. - pre_trim = np.arange(num_reqs + 1, dtype=np.int32) * 8 # total 64 > 40 - compacted_total = 32 # <= 40 - assert pre_trim[-1] > chunk_limit >= compacted_total - - input_batch = _FakeInputBatch(pre_trim, compacted_total) - logits = torch.zeros(compacted_total, 16) - _run(sampler, logits, input_batch, chunk_limit) - - assert len(sampler.calls) == 1, "compacted batch must verify in a single chunk" - num_rows, cu_num_logits = sampler.calls[0] - assert num_rows == compacted_total - assert cu_num_logits == input_batch.cu_num_logits.tolist(), ( - "must verify against the device offsets, not the pre-trim CPU ones" - ) - - -def test_oversized_batch_still_chunks(sampler): - """Without compaction, a batch past the limit is still split per request.""" - num_reqs, chunk_limit = 8, 24 - cu = np.arange(num_reqs + 1, dtype=np.int32) * 8 # total 64 > 24 - input_batch = _FakeInputBatch(cu, int(cu[-1])) - logits = torch.zeros(int(cu[-1]), 16) - _run(sampler, logits, input_batch, chunk_limit) - - assert len(sampler.calls) > 1 - assert sum(rows for rows, _ in sampler.calls) == int(cu[-1]) - assert all(rows <= chunk_limit for rows, _ in sampler.calls) - - -def test_iter_request_chunks_never_splits_a_request(): - cu = np.array([0, 3, 6, 9, 12], dtype=np.int32) - chunks = list(_iter_request_chunks(cu, 7)) - assert chunks[0][0] == 0 and chunks[-1][1] == 4 - for start, end in chunks: - assert end > start From 6636dfed5c9b3ea33e63cc8e667ba48bfa007404 Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Sun, 9 Aug 2026 04:16:34 +0000 Subject: [PATCH 33/40] CI fixes Signed-off-by: Lucas Wilkinson --- tests/v1/streaming_input/test_gpu_model_runner_v2_streaming.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/v1/streaming_input/test_gpu_model_runner_v2_streaming.py b/tests/v1/streaming_input/test_gpu_model_runner_v2_streaming.py index 489284d6677b..31c1a6f4846a 100644 --- a/tests/v1/streaming_input/test_gpu_model_runner_v2_streaming.py +++ b/tests/v1/streaming_input/test_gpu_model_runner_v2_streaming.py @@ -42,6 +42,7 @@ def mock_model_runner_with_req_states(): runner.prompt_logprobs_worker = None runner.is_last_pp_rank = False runner.pooling_runner = None + runner.adaptive_verification = None # Mock staged writes — they use Triton kernels that require GPU runner.req_states.apply_staged_writes = Mock() From 1160bcebf64bdcc50942092b8f307e4990b5ba9a Mon Sep 17 00:00:00 2001 From: Benjamin Chislett Date: Mon, 10 Aug 2026 10:59:54 -0400 Subject: [PATCH 34/40] Apply suggestion from @benchislett Signed-off-by: Benjamin Chislett --- docs/features/speculative_decoding/adaptive_verification.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/features/speculative_decoding/adaptive_verification.md b/docs/features/speculative_decoding/adaptive_verification.md index 5f48feb98144..20f427d5318e 100644 --- a/docs/features/speculative_decoding/adaptive_verification.md +++ b/docs/features/speculative_decoding/adaptive_verification.md @@ -32,8 +32,6 @@ vllm serve deepseek-ai/DeepSeek-V4-Flash-DSpark \ Set `enable_adaptive_verification: false` to verify the full block for every request. -`num_speculative_tokens` must be at least the checkpoint's block size (`dspark_block_size`, e.g. 7 for DeepSeek-V4-Flash); smaller values feed the block machinery a layout it was not trained for and are rejected at startup. - ## Requirements and limitations - The attention backend must tolerate device-decided query lengths, since the CPU lengths only bound them from above. Backends that plan off the CPU lengths are excluded by the attention selector, and rejected at startup for models that hard-wire their backend. From 1854fdfb6c4f382ed9aaab8199a7be5a5fdead51 Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Mon, 10 Aug 2026 21:33:57 +0000 Subject: [PATCH 35/40] [Doc] Fix adaptive verification default in docs `enable_adaptive_verification` defaults to false, but the doc said it was enabled by default for DSpark. Also fix a grammar slip in the support note and a doubled word in the unsupported-backend error message. Co-Authored-By: Claude Opus 5 Signed-off-by: Lucas Wilkinson --- docs/features/speculative_decoding/adaptive_verification.md | 4 ++-- vllm/v1/worker/gpu/model_runner.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/features/speculative_decoding/adaptive_verification.md b/docs/features/speculative_decoding/adaptive_verification.md index 20f427d5318e..58c9d57cee71 100644 --- a/docs/features/speculative_decoding/adaptive_verification.md +++ b/docs/features/speculative_decoding/adaptive_verification.md @@ -12,11 +12,11 @@ The practical effect is that one configuration holds up across the whole load ra ## Support -Adaptive verification needs per-position acceptance estimates, so today its currently only supported for DSpark with a **confidence head**. +Adaptive verification needs per-position acceptance estimates, so today it is only supported for DSpark with a **confidence head**. ## Usage -It is enabled by default for DSpark: +It is off by default. Enable it in the speculative config: ```bash vllm serve deepseek-ai/DeepSeek-V4-Flash-DSpark \ diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index d59841b27924..6f8719c9a674 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -527,7 +527,7 @@ def initialize_kv_cache(self, kv_cache_config: KVCacheConfig) -> None: if backend is not None: raise ValueError( "Adaptive verification trims verification requests on device, which" - f" the {backend} attention backend does not support. support. Pass " + f" the {backend} attention backend does not support. Pass " "enable_adaptive_verification=false in the speculative config, or " "use a backend that does." ) From a6ef5acc5ef7ff62221eac609ba29b2876e4ac33 Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Tue, 11 Aug 2026 02:44:07 +0000 Subject: [PATCH 36/40] review comments Signed-off-by: Lucas Wilkinson --- vllm/v1/attention/backend.py | 10 +++++++--- vllm/v1/worker/gpu/input_batch.py | 5 +++-- vllm/v1/worker/gpu/model_runner.py | 19 +++++++++++++++---- .../gpu/spec_decode/adaptive_verification.py | 7 +++++-- 4 files changed, 30 insertions(+), 11 deletions(-) diff --git a/vllm/v1/attention/backend.py b/vllm/v1/attention/backend.py index 3ec3307fd462..e984e77f77ef 100644 --- a/vllm/v1/attention/backend.py +++ b/vllm/v1/attention/backend.py @@ -286,10 +286,14 @@ def supports_device_cpu_query_lens_mismatch(cls) -> bool: with the CPU one; backends that plan off the CPU query lengths must opt out. Currently only verification requests are affected: adaptive verification trims - their drafts on device, so the CPU lengths bound them from above. The - decode/prefill split point and the CPU prefill query lengths both stay correct. + their drafts on device. On the CPU the draft budget is evenly distributed across + requests, so the total draft budget, the decode/prefill split point and the CPU + prefill query lengths all stay correct. + + SSM backends opt out: their recurrent-state planning is built from the CPU + per-request boundaries, which the trimmed batch no longer matches. """ - return True + return not cls.is_ssm() @classmethod def supports_pcp(cls) -> bool: diff --git a/vllm/v1/worker/gpu/input_batch.py b/vllm/v1/worker/gpu/input_batch.py index 6c9c979401e9..5060ddf27b0b 100644 --- a/vllm/v1/worker/gpu/input_batch.py +++ b/vllm/v1/worker/gpu/input_batch.py @@ -54,9 +54,10 @@ class InputBatch: expanded_local_pos: torch.Tensor # [num_reqs] - # batch_idx -> num_scheduled_tokens + # batch_idx -> num_scheduled_tokens, (upper bound when using adaptive verification) num_scheduled_tokens: np.ndarray - # sum(num_scheduled_tokens) + # number of tokens in the batch, + # may be < sum(num_scheduled_tokens) when using adaptive verification num_tokens: int num_tokens_after_padding: int # Sum of draft tokens scheduled across requests. diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 6f8719c9a674..33fd81a9719a 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -61,6 +61,7 @@ from vllm.utils.math_utils import cdiv from vllm.utils.mem_utils import DeviceMemoryProfiler, format_gib from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE +from vllm.v1.attention.backend import AttentionCGSupport from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec from vllm.v1.outputs import ( @@ -531,6 +532,16 @@ def initialize_kv_cache(self, kv_cache_config: KVCacheConfig) -> None: "enable_adaptive_verification=false in the speculative config, or " "use a backend that does." ) + # Decode graphs become varlen + if attn_cg_support.min_cg_support != AttentionCGSupport.ALWAYS: + raise ValueError( + "Adaptive verification captures varlen decode cudagraphs, so every" + " attention builder must report AttentionCGSupport.ALWAYS, but " + f"{attn_cg_support.min_cg_attn_backend} reports " + f"{attn_cg_support.min_cg_support}. Pass " + "enable_adaptive_verification=false in the speculative config, or " + "use a backend that does." + ) self.adaptive_verification = AdaptiveVerificationManager( self.req_states, self.input_buffers.query_start_loc, @@ -1090,7 +1101,7 @@ def prepare_inputs( adaptive_verification = ( self.adaptive_verification if num_draft_tokens_per_req is not None else None ) - scheduled_num_tokens = num_scheduled_tokens + num_scheduled_tokens_upper_bound = num_scheduled_tokens if adaptive_verification is not None: # num_scheduled_tokens represents the draft budget evenly distributed across # all verification requests, `reallocate_drafts` will unevenly assign the @@ -1185,7 +1196,7 @@ def prepare_inputs( seq_lens_cpu_upper_bound_np = np.zeros(num_reqs_padded, dtype=np.int32) np.add( num_computed_tokens_np, - scheduled_num_tokens, + num_scheduled_tokens_upper_bound, out=seq_lens_cpu_upper_bound_np[:num_reqs], ) seq_lens_cpu_upper_bound = torch.from_numpy(seq_lens_cpu_upper_bound_np) @@ -1208,7 +1219,7 @@ def prepare_inputs( idx_mapping_np=idx_mapping_np, expanded_idx_mapping=expanded_idx_mapping, expanded_local_pos=expanded_local_pos, - num_scheduled_tokens=num_scheduled_tokens, + num_scheduled_tokens=num_scheduled_tokens_upper_bound, num_tokens=num_tokens, num_tokens_after_padding=num_tokens_after_padding, num_draft_tokens=total_num_draft_tokens, @@ -1232,7 +1243,7 @@ def prepare_inputs( has_structured_output_reqs=scheduler_output.has_structured_output_requests, prompt_lens=prompt_lens, max_query_len=( - int(scheduled_num_tokens.max()) + int(num_scheduled_tokens_upper_bound.max()) if adaptive_verification is not None else None ), diff --git a/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py b/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py index e12e86bbd18c..e59f53bd27de 100644 --- a/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py +++ b/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py @@ -216,8 +216,11 @@ def set_cost_curves( (draft_curve, verify_curve), src=0 ) if not draft_curve or not verify_curve: - logger.warning_once("DSpark could not profile step costs.") - return + raise RuntimeError( + "Adaptive verification could not profile step costs. Pass " + "`enable_adaptive_verification=false` in the speculative config to " + "verify a fixed number of drafts instead." + ) self.cost_tables = build_cost_tables_from_curves( draft_curve, verify_curve, From b67f23194566d86065cfd1a808c20cd6191f1662 Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Tue, 11 Aug 2026 03:30:23 +0000 Subject: [PATCH 37/40] review comments Signed-off-by: Lucas Wilkinson --- .../spec_decode/test_adaptive_verification.py | 45 ++++++++++++++++++- vllm/v1/worker/gpu/model_runner.py | 9 ++-- .../gpu/spec_decode/adaptive_verification.py | 23 ++++++++-- 3 files changed, 70 insertions(+), 7 deletions(-) diff --git a/tests/v1/spec_decode/test_adaptive_verification.py b/tests/v1/spec_decode/test_adaptive_verification.py index bc2bf35f8073..5fc88de9984a 100644 --- a/tests/v1/spec_decode/test_adaptive_verification.py +++ b/tests/v1/spec_decode/test_adaptive_verification.py @@ -103,7 +103,8 @@ def test_compact_batch_preserves_totals_and_bounds(): ) scheduled = np.array([3, 3, 40], dtype=np.int32) drafts = np.array([2, 2, 0], dtype=np.int32) - compacted = manager.compact_batch(drafts, scheduled) + cu_num_logits_np = np.array([0, 3, 6, 7], dtype=np.int32) + compacted, _ = manager.compact_batch(drafts, scheduled, cu_num_logits_np) assert int(compacted.sum()) == num_tokens num_steps = manager.num_speculative_steps @@ -125,3 +126,45 @@ def test_budget_caps_at_one_rejection_sampler_chunk(): ) _, _, draft_budget = manager._batch_budget assert draft_budget <= 1 + + +def test_zero_budget_rebuilds_cpu_cu_num_logits(): + # When one bonus row per request already overflows a verification chunk, the + # budget clamps to zero but the batch still needs chunking. Every capacity is + # zeroed on device, so the CPU can name that layout exactly -- and must, since + # _iter_request_chunks slices the compacted logits with these offsets. + # + # The third request is a chunked prefill (no drafts, still mid-prompt). The + # runner gives *every* request num_bonus_tokens logits rows regardless + # (num_logits = num_draft_tokens_per_req + num_bonus_tokens), so the rebuilt + # offsets stay uniform rather than skipping non-verification rows. + manager = make_manager( + np.array([[0.9, 0.9], [0.9, 0.9], [1.0, 1.0]], dtype=np.float32), + np.ones(64), + ) + manager.req_states.req_id_to_index["prefill"] = 2 + manager.req_states.num_computed_tokens_np = np.zeros(3, dtype=np.int32) + manager.req_states.prefill_len.np = np.array([0, 0, 60], dtype=np.int32) + manager._max_total_logits = 2 # < 3 requests * 1 bonus token + + manager.get_num_tokens( + {"low": 3, "high": 3, "prefill": 40}, + {"low": [1, 2], "high": [3, 4]}, + ) + _, _, draft_budget = manager._batch_budget + assert draft_budget == 0 + + scheduled = np.array([3, 3, 40], dtype=np.int32) + drafts = np.array([2, 2, 0], dtype=np.int32) + scheduled_cu_num_logits = np.array([0, 3, 6, 7], dtype=np.int32) + compacted, cu_num_logits_np = manager.compact_batch( + drafts, scheduled, scheduled_cu_num_logits + ) + + # One bonus row per request, matching cumsum(capacities + num_bonus_tokens) + # with every capacity zeroed -- the prefill row included. + expected = np.arange(4, dtype=np.int32) * manager.num_bonus_tokens + assert np.array_equal(cu_num_logits_np, expected) + assert cu_num_logits_np.dtype == scheduled_cu_num_logits.dtype + # The prefill keeps its scheduled tokens; only drafts are dropped. + assert np.array_equal(compacted, np.array([1, 1, 40], dtype=np.int32)) diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 33fd81a9719a..a6b1d06f0791 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -1106,9 +1106,12 @@ def prepare_inputs( # num_scheduled_tokens represents the draft budget evenly distributed across # all verification requests, `reallocate_drafts` will unevenly assign the # draft budget to requests on the GPU side only. - num_scheduled_tokens = adaptive_verification.compact_batch( - num_draft_tokens_per_req, - num_scheduled_tokens, + num_scheduled_tokens, cu_num_logits_np = ( + adaptive_verification.compact_batch( + num_draft_tokens_per_req, + num_scheduled_tokens, + cu_num_logits_np, + ) ) # Get query_start_loc. diff --git a/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py b/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py index e59f53bd27de..4a594e30fb42 100644 --- a/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py +++ b/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py @@ -332,15 +332,32 @@ def compact_batch( self, num_draft_tokens_per_req: np.ndarray, num_scheduled_tokens: np.ndarray, - ) -> np.ndarray: + cu_num_logits_np: np.ndarray, + ) -> tuple[np.ndarray, np.ndarray]: + """Compact the CPU batch to the chosen draft budget. + + Returns the compacted per-request token counts and the CPU cu_num_logits_np. + If the draft budget is 0, we can know cu_num_logits_np exactly, otherwise + its unchanged/an-upper-bound. + """ batch_budget = self._batch_budget assert batch_budget is not None _, _, draft_budget = batch_budget num_drafts = int(num_draft_tokens_per_req.sum()) if draft_budget == num_drafts: - return num_scheduled_tokens + return num_scheduled_tokens, cu_num_logits_np num_non_draft_tokens = num_scheduled_tokens - num_draft_tokens_per_req + if draft_budget == 0: + # The draft budget is 0, so we can know cu_num_logits_np exactly. This helps + # when we would exceed the sampler logit chunk size. + num_reqs = num_scheduled_tokens.shape[0] + cu_num_logits_np = ( + np.arange(num_reqs + 1, dtype=cu_num_logits_np.dtype) + * self.num_bonus_tokens + ) + return num_non_draft_tokens, cu_num_logits_np + is_verification_request = num_draft_tokens_per_req > 0 num_verification_reqs = int(is_verification_request.sum()) # sort_batch_req_ids keeps verification requests at the front. @@ -349,7 +366,7 @@ def compact_batch( draft_lens_cpu = np.zeros_like(num_non_draft_tokens) draft_lens_cpu[:num_verification_reqs] = draft_budget // num_verification_reqs draft_lens_cpu[: draft_budget % num_verification_reqs] += 1 - return num_non_draft_tokens + draft_lens_cpu + return num_non_draft_tokens + draft_lens_cpu, cu_num_logits_np def reallocate_drafts( self, req_ids: list[str], idx_mapping: torch.Tensor From 7b032b5dcbea272ae8d8021aacb87b3ac72e7853 Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Wed, 12 Aug 2026 02:15:26 +0000 Subject: [PATCH 38/40] fux test Signed-off-by: Lucas Wilkinson --- tests/v1/worker/test_gpu_batch_ordering.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/v1/worker/test_gpu_batch_ordering.py b/tests/v1/worker/test_gpu_batch_ordering.py index fa79a9255ac1..4ed69d74b55b 100644 --- a/tests/v1/worker/test_gpu_batch_ordering.py +++ b/tests/v1/worker/test_gpu_batch_ordering.py @@ -55,6 +55,7 @@ def _make_runner( num_computed = np.array([s[0] for s in req_states.values()], dtype=np.int32) runner: Any = GPUModelRunner.__new__(GPUModelRunner) runner.decode_query_len = decode_query_len + runner.adaptive_verification = None runner.req_states = SimpleNamespace( req_id_to_index={req_id: i for i, req_id in enumerate(req_states)}, # The runner keeps this as min(num_computed_tokens, prefill_len). @@ -73,6 +74,7 @@ def _uniform_token_count( scheduler_output = SimpleNamespace( num_scheduled_tokens={req_id: query_len for req_id in req_states}, total_num_scheduled_tokens=query_len * len(req_states), + scheduled_spec_decode_tokens={}, ) runner = _make_runner(req_states, decode_query_len=query_len) _, uniform_tok_count = runner.gather_batch_req_state(scheduler_output, dummy_run) @@ -155,6 +157,7 @@ def test_uniform_decode_uses_state_index_not_batch_position(): """ runner: Any = GPUModelRunner.__new__(GPUModelRunner) runner.decode_query_len = 8 + runner.adaptive_verification = None # State arrays in state-index order: a prefilling request, then two decodes. runner.req_states = SimpleNamespace( req_id_to_index={"prefilling": 0, "decode_a": 1, "decode_b": 2}, @@ -164,6 +167,7 @@ def test_uniform_decode_uses_state_index_not_batch_position(): scheduler_output = SimpleNamespace( num_scheduled_tokens={"decode_a": 8, "decode_b": 8}, total_num_scheduled_tokens=16, + scheduled_spec_decode_tokens={}, ) state, uniform_tok_count = runner.gather_batch_req_state(scheduler_output, False) From 5cd5b8a0c826d7dd121fb234f27a5219c4a4e5c2 Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Wed, 12 Aug 2026 02:45:44 +0000 Subject: [PATCH 39/40] review refactors Signed-off-by: Lucas Wilkinson --- vllm/v1/worker/gpu/model_runner.py | 54 +++++++------------ .../gpu/spec_decode/adaptive_verification.py | 48 +++++++++++++++++ 2 files changed, 67 insertions(+), 35 deletions(-) diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 4731c0124bfc..7038daaf32d1 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -61,7 +61,6 @@ from vllm.utils.math_utils import cdiv from vllm.utils.mem_utils import DeviceMemoryProfiler, format_gib from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE -from vllm.v1.attention.backend import AttentionCGSupport from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec from vllm.v1.outputs import ( @@ -81,7 +80,6 @@ from vllm.v1.worker.gpu.attn_utils import ( build_slot_mappings_by_layer, get_kv_cache_spec, - get_query_lens_mismatch_unsupported_backend, init_attn_backend, init_kv_cache, ) @@ -133,6 +131,7 @@ from vllm.v1.worker.gpu.spec_decode import init_speculator from vllm.v1.worker.gpu.spec_decode.adaptive_verification import ( AdaptiveVerificationManager, + maybe_create_adaptive_verification_manager, ) from vllm.v1.worker.gpu.spec_decode.eagle.eagle3_utils import ( set_eagle3_aux_hidden_state_layers, @@ -531,33 +530,17 @@ def initialize_kv_cache(self, kv_cache_config: KVCacheConfig) -> None: ) # The speculator clears the flag at load time when the checkpoint has # no confidence head, so it holds the effective value. - if getattr(self.speculator, "enable_adaptive_verification", False): - # The selector rejects unsupported backends, but models that - # hard-wire theirs (e.g. DeepSeek-V4) never go through it. - backend = get_query_lens_mismatch_unsupported_backend(self.attn_groups) - if backend is not None: - raise ValueError( - "Adaptive verification trims verification requests on device, which" - f" the {backend} attention backend does not support. Pass " - "enable_adaptive_verification=false in the speculative config, or " - "use a backend that does." - ) - # Decode graphs become varlen - if attn_cg_support.min_cg_support != AttentionCGSupport.ALWAYS: - raise ValueError( - "Adaptive verification captures varlen decode cudagraphs, so every" - " attention builder must report AttentionCGSupport.ALWAYS, but " - f"{attn_cg_support.min_cg_attn_backend} reports " - f"{attn_cg_support.min_cg_support}. Pass " - "enable_adaptive_verification=false in the speculative config, or " - "use a backend that does." - ) - self.adaptive_verification = AdaptiveVerificationManager( - self.req_states, - self.input_buffers.query_start_loc, - self.model_state.num_new_sampled_tokens_per_step, - max_total_logits=get_max_chunk_logits(self.vocab_size), - ) + self.adaptive_verification = maybe_create_adaptive_verification_manager( + enable_adaptive_verification=getattr( + self.speculator, "enable_adaptive_verification", False + ), + attn_groups=self.attn_groups, + attn_cg_support=attn_cg_support, + req_states=self.req_states, + query_start_loc=self.input_buffers.query_start_loc, + num_bonus_tokens=self.model_state.num_new_sampled_tokens_per_step, + max_total_logits=get_max_chunk_logits(self.vocab_size), + ) self.block_tables = BlockTables( block_sizes=block_sizes, @@ -1060,15 +1043,11 @@ def gather_batch_req_state( return None, get_uniform_token_count(num_reqs, num_toks, max_query_len) draft_tokens = scheduler_output.scheduled_spec_decode_tokens - adaptive_draft_tokens = draft_tokens if self.adaptive_verification else {} - if adaptive_draft_tokens: - num_toks = self.adaptive_verification.get_num_tokens( # type: ignore[union-attr] - num_tokens_per_req, adaptive_draft_tokens - ) # batch_idx -> req_id req_ids = sort_batch_req_ids( - num_tokens_per_req, adaptive_draft_tokens, self.decode_query_len + num_tokens_per_req, draft_tokens, self.decode_query_len ) + numtoks_iter = map(num_tokens_per_req.__getitem__, req_ids) num_scheduled_tokens = np.fromiter(numtoks_iter, dtype=np.int32, count=num_reqs) @@ -1080,6 +1059,11 @@ def gather_batch_req_state( ] is_prefilling_np = num_computed_prefill_tokens_np < prefill_len_np + if self.adaptive_verification is not None: + num_toks = self.adaptive_verification.get_num_tokens( + num_tokens_per_req, draft_tokens + ) + batch_state = BatchReqState( req_ids=req_ids, num_scheduled_tokens=num_scheduled_tokens, diff --git a/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py b/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py index 4a594e30fb42..12dc8c7c656e 100644 --- a/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py +++ b/vllm/v1/worker/gpu/spec_decode/adaptive_verification.py @@ -13,16 +13,22 @@ from vllm.distributed.parallel_state import get_tp_group from vllm.logger import init_logger from vllm.utils.gpu_sync_debug import gpu_sync_allowed +from vllm.v1.attention.backend import AttentionCGSupport from vllm.v1.utils import CpuGpuBuffer from vllm.v1.worker.gpu.async_utils import StepTimingSample, stream +from vllm.v1.worker.gpu.attn_utils import ( + get_query_lens_mismatch_unsupported_backend, +) from vllm.v1.worker.gpu.buffer_utils import async_copy_to_gpu logger = init_logger(__name__) _PROFILE_REPLAYS = 5 if TYPE_CHECKING: + from vllm.v1.worker.gpu.attn_utils import AttentionCGSupportInfo from vllm.v1.worker.gpu.input_batch import InputBatch from vllm.v1.worker.gpu.states import RequestState + from vllm.v1.worker.utils import AttentionGroup def _assign_draft_token_budget( @@ -427,3 +433,45 @@ def reallocate_drafts( self.query_start_loc, draft_budget, ) + + +def maybe_create_adaptive_verification_manager( + *, + enable_adaptive_verification: bool, + attn_groups: list[list["AttentionGroup"]], + attn_cg_support: "AttentionCGSupportInfo", + req_states: "RequestState", + query_start_loc: torch.Tensor, + num_bonus_tokens: int, + max_total_logits: int, +) -> AdaptiveVerificationManager | None: + if not enable_adaptive_verification: + return None + + # The selector rejects unsupported backends, but models that + # hard-wire theirs (e.g. DeepSeek-V4) never go through it. + backend = get_query_lens_mismatch_unsupported_backend(attn_groups) + if backend is not None: + raise ValueError( + "Adaptive verification trims verification requests on device, which" + f" the {backend} attention backend does not support. Pass " + "enable_adaptive_verification=false in the speculative config, or " + "use a backend that does." + ) + + if attn_cg_support.min_cg_support != AttentionCGSupport.ALWAYS: + raise ValueError( + "Adaptive verification captures varlen decode cudagraphs, so every" + " attention builder must report AttentionCGSupport.ALWAYS, but " + f"{attn_cg_support.min_cg_attn_backend} reports " + f"{attn_cg_support.min_cg_support}. Pass " + "enable_adaptive_verification=false in the speculative config, or " + "use a backend that does." + ) + + return AdaptiveVerificationManager( + req_states, + query_start_loc, + num_bonus_tokens, + max_total_logits=max_total_logits, + ) From e2e335334669d1c94c7351937474c0104dcbfdfb Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Wed, 12 Aug 2026 03:10:14 +0000 Subject: [PATCH 40/40] [Bugfix] Skip adaptive sizing for no-draft batches Co-authored-by: OpenAI Codex Signed-off-by: Lucas Wilkinson --- tests/v1/worker/test_gpu_batch_ordering.py | 27 ++++++++++++++++++++++ vllm/v1/worker/gpu/model_runner.py | 2 +- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/tests/v1/worker/test_gpu_batch_ordering.py b/tests/v1/worker/test_gpu_batch_ordering.py index 4ed69d74b55b..5037dcc7a96b 100644 --- a/tests/v1/worker/test_gpu_batch_ordering.py +++ b/tests/v1/worker/test_gpu_batch_ordering.py @@ -87,6 +87,33 @@ def test_spec_decode_batch_is_uniform_decode(): assert _uniform_token_count(decodes, 8) == 8 +def test_adaptive_verification_sizes_only_batches_with_drafts(): + decodes = {"d0": (16, 16), "d1": (16, 16)} + runner = _make_runner(decodes, decode_query_len=8) + manager = SimpleNamespace( + get_num_tokens=lambda _num_tokens_per_req, _draft_tokens: 12 + ) + runner.adaptive_verification = manager + scheduler_output = SimpleNamespace( + num_scheduled_tokens={req_id: 8 for req_id in decodes}, + total_num_scheduled_tokens=16, + scheduled_spec_decode_tokens={}, + ) + + state, uniform_tok_count = runner.gather_batch_req_state(scheduler_output, False) + assert state is not None + assert state.num_tokens == 16 + assert uniform_tok_count == 8 + + scheduler_output.scheduled_spec_decode_tokens = { + req_id: [1, 2] for req_id in decodes + } + state, uniform_tok_count = runner.gather_batch_req_state(scheduler_output, False) + assert state is not None + assert state.num_tokens == 12 + assert uniform_tok_count is None + + def test_prompt_chunk_of_decode_query_len_is_not_uniform_decode(): # Two prompt chunks whose query length coincides with the K+1 spec-decode # query length must reject the batch (issue #49918). diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 7038daaf32d1..b7b5fb00b221 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -1059,7 +1059,7 @@ def gather_batch_req_state( ] is_prefilling_np = num_computed_prefill_tokens_np < prefill_len_np - if self.adaptive_verification is not None: + if self.adaptive_verification is not None and draft_tokens: num_toks = self.adaptive_verification.get_num_tokens( num_tokens_per_req, draft_tokens )