Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .buildkite/hardware_tests/cpu.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ steps:
- |
bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 30m "
pytest -x -v -s \
tests/v1/worker/test_gpu_autoregressive_speculator.py \
tests/v1/worker/test_gpu_target_dependent_ar_speculator.py \
tests/v1/worker/test_gpu_extract_hidden_states_speculator.py \
tests/v1/worker/test_gpu_batch_ordering.py \
tests/v1/worker/test_gpu_rejection_sampler_chunking.py \
Expand Down
4 changes: 2 additions & 2 deletions tests/v1/cudagraph/test_cudagraph_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,10 @@
from vllm.v1.worker.gpu.dp_utils import dispatch_cg_and_sync_dp
from vllm.v1.worker.gpu.input_batch import InputBuffers
from vllm.v1.worker.gpu.pcp_manager import PCPManager
from vllm.v1.worker.gpu.spec_decode.autoregressive import (
from vllm.v1.worker.gpu.spec_decode.target_dependent_ar import (
cudagraph_utils as spec_cudagraph_utils,
)
from vllm.v1.worker.gpu.spec_decode.autoregressive.cudagraph_utils import (
from vllm.v1.worker.gpu.spec_decode.target_dependent_ar.cudagraph_utils import (
SpeculatorCudaGraphManager,
)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,17 +22,17 @@
from vllm.v1.attention.backends.utils import split_decodes_and_prefills
from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor
from vllm.v1.worker.gpu.spec_decode import speculator as base_spec_module
from vllm.v1.worker.gpu.spec_decode.autoregressive import speculator as spec_module
from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import (
AutoRegressiveSpeculator,
)
from vllm.v1.worker.gpu.spec_decode.multi_module_mtp.speculator import (
MultiModuleMTPSpeculator,
)
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
from vllm.v1.worker.gpu.spec_decode.target_dependent_ar import speculator as spec_module
from vllm.v1.worker.gpu.spec_decode.target_dependent_ar.speculator import (
TargetDependentARSpeculator,
)


class _TestSpeculator(AutoRegressiveSpeculator):
class _TestSpeculator(TargetDependentARSpeculator):
def load_draft_model(self, target_model, target_attn_layer_names):
return self.test_draft_model

Expand Down
2 changes: 1 addition & 1 deletion vllm/v1/worker/gpu/model_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -820,7 +820,7 @@ def _dummy_run(
"skip_attn must only be True for initial memory profiling."
)

# Create a dummy scheduler output. Plain draft-model speculation adds
# Create a dummy scheduler output. Standalone AR speculation adds
# one correction slot per request during prefill. The scheduler
# accounts for these slots, while dummy runs bypass the scheduler.
# Adjust the dummy token count only when the expanded draft batch
Expand Down
6 changes: 3 additions & 3 deletions vllm/v1/worker/gpu/spec_decode/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -71,11 +71,11 @@ def init_speculator(

return EagleSpeculator(vllm_config, device)
elif speculative_config.uses_draft_model():
from vllm.v1.worker.gpu.spec_decode.draft_model.speculator import (
PlainDraftModelSpeculator,
from vllm.v1.worker.gpu.spec_decode.standalone_ar.speculator import (
StandaloneARSpeculator,
)

return PlainDraftModelSpeculator(vllm_config, device)
return StandaloneARSpeculator(vllm_config, device)
elif speculative_config.use_ngram():
from vllm.v1.worker.gpu.spec_decode.ngram.speculator import (
NgramGPUSpeculator,
Expand Down
8 changes: 4 additions & 4 deletions vllm/v1/worker/gpu/spec_decode/eagle/speculator.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,13 @@

import torch.nn as nn

from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import (
AutoRegressiveSpeculator,
)
from vllm.v1.worker.gpu.spec_decode.eagle.utils import load_eagle_model
from vllm.v1.worker.gpu.spec_decode.target_dependent_ar.speculator import (
TargetDependentARSpeculator,
)


class EagleSpeculator(AutoRegressiveSpeculator):
class EagleSpeculator(TargetDependentARSpeculator):
def load_draft_model(
self,
target_model: nn.Module,
Expand Down
6 changes: 3 additions & 3 deletions vllm/v1/worker/gpu/spec_decode/gemma4/speculator.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@
from vllm.logger import init_logger
from vllm.model_executor.model_loader import get_model
from vllm.model_executor.model_loader.utils import get_draft_load_config
from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import (
AutoRegressiveSpeculator,
from vllm.v1.worker.gpu.spec_decode.target_dependent_ar.speculator import (
TargetDependentARSpeculator,
)

logger = init_logger(__name__)
Expand All @@ -45,7 +45,7 @@ def _copy_target_kv_scales(attn: nn.Module, target_attn: nn.Module) -> None:
getattr(attn, scale_name).copy_(getattr(target_attn, scale_name))


class Gemma4Speculator(AutoRegressiveSpeculator):
class Gemma4Speculator(TargetDependentARSpeculator):
@property
def advance_draft_positions(self) -> bool:
# Gemma4 MTP is Q-only and reads K/V from the target's existing cache.
Expand Down
8 changes: 4 additions & 4 deletions vllm/v1/worker/gpu/spec_decode/mtp/speculator.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,13 @@

import torch.nn as nn

from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import (
AutoRegressiveSpeculator,
)
from vllm.v1.worker.gpu.spec_decode.eagle.utils import load_eagle_model
from vllm.v1.worker.gpu.spec_decode.target_dependent_ar.speculator import (
TargetDependentARSpeculator,
)


class MTPSpeculator(AutoRegressiveSpeculator):
class MTPSpeculator(TargetDependentARSpeculator):
share_mtp_topk_indices: bool = False

def load_draft_model(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,11 @@
from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer
from vllm.v1.worker.gpu.dp_utils import DPSyncState, dispatch_cg_and_sync_dp
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
from vllm.v1.worker.gpu.spec_decode.autoregressive.cudagraph_utils import (
SpeculatorCudaGraphManager,
)
from vllm.v1.worker.gpu.spec_decode.eagle.utils import load_eagle_model
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
from vllm.v1.worker.gpu.spec_decode.target_dependent_ar.cudagraph_utils import (
SpeculatorCudaGraphManager,
)
from vllm.v1.worker.utils import get_uniform_decode_token_count

logger = init_logger(__name__)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,22 +26,22 @@
logger = init_logger(__name__)


class PlainDraftModelSpeculator(DraftModelSpeculator):
"""Speculative decoding using a separate smaller draft LM.
class StandaloneARSpeculator(DraftModelSpeculator):
"""Standalone autoregressive (AR) drafting with a separate smaller LM.

Unlike Eagle, the draft model runs fully independently of the target model.
Step 0 builds an expanded buffer (accepted + correction token + rejected
slots) via a Triton kernel; steps 1..k-1 are single-token decode steps.
"""

# Plain draft model needs one extra slot per request for correction.
# Standalone AR drafting needs one extra slot per request for correction.
num_extra_query_per_req = 1

def __init__(self, vllm_config: VllmConfig, device: torch.device):
super().__init__(vllm_config, device)

# draft_max_seq_len is read by the parent's attention metadata builder.
# Plain draft model doesn't do per-batch adjustment; cap at max.
# Standalone AR drafting doesn't do per-batch adjustment; cap at max.
self.draft_max_seq_len = self.max_model_len

self.last_token_indices = torch.zeros(
Expand All @@ -65,10 +65,10 @@ def __init__(self, vllm_config: VllmConfig, device: torch.device):
self.supports_mm_inputs = False

def init_cudagraph_manager(self, cudagraph_mode: CUDAGraphMode) -> None:
pass # CUDA graph not yet supported for plain draft model speculator.
pass # CUDA graph not yet supported for standalone AR speculator.

def capture(self) -> None:
pass # CUDA graph not yet supported for plain draft model speculator.
pass # CUDA graph not yet supported for standalone AR speculator.

def set_attn(
self,
Expand Down Expand Up @@ -415,7 +415,7 @@ def _prepare_prefill_inputs_kernel(
max_model_len,
BLOCK_SIZE: tl.constexpr,
):
"""Per-request step-0 input preparation for the plain draft-model speculator.
"""Per-request step-0 input preparation for the standalone AR speculator.

Output layout for request i (out_start = query_start_loc[i] + i):
[out_start, out_start + num_valid) accepted tokens
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,22 @@
from vllm.v1.worker.gpu.dp_utils import DPSyncState, dispatch_cg_and_sync_dp
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
from vllm.v1.worker.gpu.model_states.interface import ModelState
from vllm.v1.worker.gpu.spec_decode.autoregressive.cudagraph_utils import (
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
from vllm.v1.worker.gpu.spec_decode.target_dependent_ar.cudagraph_utils import (
SpeculatorCudaGraphManager,
)
from vllm.v1.worker.gpu.spec_decode.speculator import DraftModelSpeculator
from vllm.v1.worker.utils import AttentionGroup, get_uniform_decode_token_count

logger = init_logger(__name__)


class AutoRegressiveSpeculator(DraftModelSpeculator):
class TargetDependentARSpeculator(DraftModelSpeculator):
"""Autoregressive (AR) drafting that depends on target model state.

Base for Eagle, MTP, and Gemma4 drafters, which consume the target's
hidden states (and, for Gemma4, its KV cache) at each draft step.
"""

def __init__(self, vllm_config: VllmConfig, device: torch.device):
super().__init__(vllm_config, device)

Expand Down
Loading