[Bugfix][MRV2][Spec Decode] Honor dynamic K in autoregressive speculators, 1.2~1.3x kernel perf improvement by yewentao256 · Pull Request #57053 · vllm-project/vllm

yewentao256 · 2026-09-15T18:30:01Z

Purpose

MRV2 did not propagate SchedulerOutput.num_spec_tokens_to_schedule to autoregressive speculators, causing unnecessary draft forwards and stale draft tokens.

This PR fixes the issue

Test

Acc covered in current unit test

Perf can be seen in this AI generated script

from functools import partial
from types import SimpleNamespace

import torch
import triton

from vllm.v1.spec_decode.dynamic.utils import build_dynamic_sd_schedule_lookup
from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers
from vllm.v1.worker.gpu.spec_decode.autoregressive.speculator import (
    AutoRegressiveSpeculator,
)

MAX_K = 5
MAX_BATCH_SIZE = 48
HIDDEN_SIZE = 4096
NUM_REQS = (1, 2, 4, 8, 16, 32, 48)
DYNAMIC_K_SCHEDULE = [(1, MAX_BATCH_SIZE, 3)]


class SyntheticSpeculator(AutoRegressiveSpeculator):
    """MRV2 proposer whose model forward is replaced by a matmul."""

    def load_draft_model(self, target_model, target_attn_layer_names):
        raise NotImplementedError("This benchmark does not load a model")

    def __init__(self, device: torch.device) -> None:
        # Deliberately skip the real constructor: it allocates model-related state.
        self.num_speculative_steps = MAX_K
        self.max_num_reqs = MAX_BATCH_SIZE
        self.max_model_len = 8192
        self.dp_size = 1
        self.dp_rank = 0
        self.pcp_manager = None
        self.eplb_state = None
        self.use_fused_multi_step_decode = True
        self.prefill_cudagraph_manager = None
        self.decode_cudagraph_manager = None
        self.speculative_config = SimpleNamespace(
            uses_dynamic_speculative_decoding=lambda: True,
        )

        self.input_buffers = InputBuffers(MAX_BATCH_SIZE, MAX_BATCH_SIZE, device)
        self.hidden_states = torch.zeros(
            MAX_BATCH_SIZE, HIDDEN_SIZE, dtype=torch.bfloat16, device=device
        )
        self.last_token_indices = torch.zeros(
            MAX_BATCH_SIZE, dtype=torch.int64, device=device
        )
        self.current_draft_step = torch.zeros((), dtype=torch.int64, device=device)
        self.sample_src_positions = torch.zeros(
            MAX_BATCH_SIZE, dtype=torch.int64, device=device
        )
        self.idx_mapping = torch.zeros(MAX_BATCH_SIZE, dtype=torch.int32, device=device)
        self.temperature = torch.zeros(MAX_BATCH_SIZE, device=device)
        self.seeds = torch.zeros(MAX_BATCH_SIZE, dtype=torch.int64, device=device)
        self.draft_tokens = torch.zeros(
            MAX_BATCH_SIZE, MAX_K, dtype=torch.int64, device=device
        )

        self.weight = torch.randn(
            HIDDEN_SIZE, HIDDEN_SIZE, dtype=torch.bfloat16, device=device
        )
        self.src = torch.randn(
            MAX_BATCH_SIZE, HIDDEN_SIZE, dtype=torch.bfloat16, device=device
        )
        self.dst = torch.empty_like(self.src)
        self.count_steps = False
        self.steps = 0

    def _synthetic_forward(self, num_tokens: int) -> None:
        torch.mm(self.src[:num_tokens], self.weight, out=self.dst[:num_tokens])
        self.src, self.dst = self.dst, self.src
        if self.count_steps:
            self.steps += 1

    def _prefill(self, num_reqs: int, num_tokens: int, *args, **kwargs) -> None:
        self._synthetic_forward(num_tokens)
        self.draft_tokens[:num_reqs, 0].zero_()

    def _generate_draft(
        self, num_reqs: int, num_tokens_padded: int, *args, **kwargs
    ) -> None:
        self._synthetic_forward(num_tokens_padded)


def count_forwards(speculator: SyntheticSpeculator, run) -> int:
    speculator.steps = 0
    speculator.count_steps = True
    run()
    speculator.count_steps = False
    return speculator.steps


def main() -> None:
    if not torch.cuda.is_available():
        raise RuntimeError("CUDA is required")

    device = torch.device("cuda")
    speculator = SyntheticSpeculator(device)
    target_buffers = InputBuffers(MAX_BATCH_SIZE, MAX_BATCH_SIZE, device)
    target_hidden_states = torch.randn_like(speculator.hidden_states)
    temperature = torch.zeros(MAX_BATCH_SIZE, device=device)
    seeds = torch.zeros(MAX_BATCH_SIZE, dtype=torch.int64, device=device)
    last_sampled = torch.zeros(MAX_BATCH_SIZE, dtype=torch.int64, device=device)
    dense_schedule = build_dynamic_sd_schedule_lookup(
        DYNAMIC_K_SCHEDULE,
        vllm_max_batch_size=MAX_BATCH_SIZE,
        vllm_num_speculative_tokens=MAX_K,
    )

    print("batch  sched_k  buggy_ms  fixed_ms  speedup  forwards(buggy->fixed)")
    for num_reqs in NUM_REQS:
        batch = InputBatch.make_dummy(num_reqs, num_reqs, target_buffers)
        common_args = dict(
            input_batch=batch,
            attn_metadata={},
            slot_mappings={},
            last_hidden_states=target_hidden_states[:num_reqs],
            aux_hidden_states=None,
            num_sampled=torch.ones(num_reqs, dtype=torch.int32, device=device),
            num_rejected=torch.zeros(num_reqs, dtype=torch.int32, device=device),
            last_sampled=last_sampled,
            next_prefill_tokens=target_buffers.input_ids,
            temperature=temperature,
            seeds=seeds,
            dummy_run=True,
            skip_attn_for_dummy_run=True,
            is_profile=True,
        )
        dynamic_k = dense_schedule[num_reqs]

        # Buggy: model_runner omitted runtime K, so propose() used max K=5.
        buggy = partial(speculator.propose, **common_args)
        # Fixed: model_runner passes the scheduler-selected runtime K=3.
        fixed = partial(
            speculator.propose,
            **common_args,
            num_speculative_tokens=dynamic_k,
        )

        buggy_steps = count_forwards(speculator, buggy)
        fixed_steps = count_forwards(speculator, fixed)
        assert dynamic_k == 3
        assert buggy_steps == MAX_K
        assert fixed_steps == dynamic_k

        buggy_ms = triton.testing.do_bench(buggy)
        fixed_ms = triton.testing.do_bench(fixed)
        print(
            f"{num_reqs:>5}  {dynamic_k:>7}  {buggy_ms:>8.3f}  "
            f"{fixed_ms:>8.3f}  {buggy_ms / fixed_ms:>7.3f}x  "
            f"{buggy_steps}->{fixed_steps}"
        )


if __name__ == "__main__":
    main()

And we can get

batch  sched_k  buggy_ms  fixed_ms  speedup  forwards(buggy->fixed)
    1        3     0.151     0.115    1.306x  5->3
    2        3     0.148     0.115    1.286x  5->3
    4        3     0.149     0.115    1.298x  5->3
    8        3     0.148     0.115    1.282x  5->3
   16        3     0.151     0.115    1.315x  5->3
   32        3     0.150     0.114    1.306x  5->3
   48        3     0.149     0.115    1.287x  5->3

Signed-off-by: yewentao256 <zhyanwentao@126.com>

claude

Claude Code Review

This repository is configured for manual code reviews. Comment @claude review for a one-time review, or @claude review always to subscribe this PR to a review on every future push.

_{Tip: disable this comment in your organization's Code Review settings.}

yewentao256 · 2026-09-15T18:30:12Z

/ci run

github-actions · 2026-09-15T18:31:57Z

✅ Triggered Buildkite CI #89149 for commit fe7cf037e38d.

yewentao256 · 2026-09-16T18:14:04Z

/ci run

github-actions · 2026-09-16T18:16:17Z

✅ Triggered Buildkite CI #89399 for commit 2471a1c1430e.

mergify · 2026-09-16T18:23:16Z

Hi @yewentao256, the pre-commit checks have failed. Please run:

uv pip install pre-commit>=4.5.1
pre-commit install
pre-commit run --all-files

Then, commit the changes and push to your branch.

For future commits, pre-commit will run automatically on changed files before each commit.

Signed-off-by: yewentao256 <zhyanwentao@126.com>

yewentao256 · 2026-09-16T18:43:17Z

/ci run

github-actions · 2026-09-16T18:44:01Z

❌ This PR is 1 commit behind upstream main. Your branch must contain every commit currently on upstream main. No new CI build was started. Merge or rebase onto the latest main, then rerun /ci run. To test this branch at your own risk, use /ci run --allow-stale.

yewentao256 · 2026-09-16T19:03:51Z

/ci run --allow-stale

github-actions · 2026-09-16T19:04:11Z

✅ Triggered Buildkite CI #89416 for commit ed2ec72bacef.

⚠️ This PR is 4 commits behind upstream main. Running CI at your own risk because --allow-stale was requested; outdated CI configuration may cause failures. Before merging, merge or rebase onto the latest main, then rerun /ci run on the latest PR commit.

yewentao256 · 2026-09-18T20:15:56Z

/ci run

github-actions · 2026-09-18T20:17:39Z

✅ Triggered Buildkite CI #89905 for commit 00cee63e04bf.

yewentao256 · 2026-09-21T19:51:55Z

/ci run

github-actions · 2026-09-21T19:52:16Z

✅ Triggered Buildkite CI #90287 for commit 79f458273224.

github-actions · 2026-09-26T13:17:57Z

✅ Triggered Buildkite CI #91398 for commit 22e5ac983299.

mergify · 2026-09-29T04:36:29Z

This pull request has merge conflicts that must be resolved before it can be
merged. Please rebase the PR, @yewentao256.

https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/syncing-a-fork

LucasWilkinson

LGTM to me, i do think as a follow-up once MRV1 is deleted we should fold this into the adaptive verification code

would be good for @benchislett and @TheEpicDolphin to review also

Signed-off-by: yewentao256 <zhyanwentao@126.com>

yewentao256 · 2026-09-30T20:36:20Z

/ci run

github-actions · 2026-09-30T20:36:40Z

❌ This PR is 1 commit behind upstream main. Your branch must contain every commit currently on upstream main. No new CI build was started. Merge or rebase onto the latest main, then rerun /ci run. To test this branch at your own risk, use /ci run --allow-stale.

yewentao256 · 2026-09-30T20:37:10Z

/ci run --allow-stale

github-actions · 2026-09-30T20:37:57Z

✅ Triggered Buildkite CI #92245 for commit 80a5c341071f.

⚠️ This PR is 1 commit behind upstream main. Running CI at your own risk because --allow-stale was requested; outdated CI configuration may cause failures. Before merging, merge or rebase onto the latest main, then rerun /ci run on the latest PR commit.

yewentao256 · 2026-10-02T13:03:48Z

/ci run

github-actions · 2026-10-02T13:04:59Z

✅ Triggered Buildkite CI #92624 for commit ac75935610ad.

vllm-agent · 2026-10-02T13:28:21Z

CI selector (shadow): 127 test steps (165 jobs) instead of 78 (97 jobs)

Shadow mode: this changes nothing about what CI runs. It shows what the evidence-based selector would pick for this PR, next to today's rules. How it works.

Feedback welcome: reply here if it would skip a step this change needs, or runs something unrelated.

steps (jobs)	Today's rules	Selector	Would skip	Would add
NVIDIA, CPU and others	78 (97)	127 (165)	13 (19)	62 (87)
AMD mirrors	69 (85)	112 (144)	9 (12)	52 (71)

Selector would run (127)

amd-kernels-mi355
amd-lm-eval-small-models-harness
async-engine-inputs-utils-worker
basic-correctness ×2
basic-correctness-cpu-offload
basic-correctness-cumem
basic-correctness-prefetch-offload
basic-correctness-sleep-mode
basic-models-tests-extra-initialization ×14
basic-models-tests-other
batch-invariance-b200
batch-invariance-h100
benchmarks-cli-test
cpu-language-generation-and-pooling-model-tests ×3
cpu-multimodal-config
cpu-spec-decode-tests
crosslayer-kv-layout-distributed-nixlconnector-pd-accuracy-tests-4-gpus
cudagraph
distributed-compile-comm-4-gpus
distributed-compile-rpc-tests-2-gpus
distributed-dp-tests-2-gpus
distributed-dp-tests-4-gpus
distributed-flashinfer-nixlconnector-pd-accuracy-4-gpus
distributed-model-tests-2-gpus ×3
distributed-mooncakeconnector-pd-accuracy-4-gpus
distributed-nixlconnector-pd-accuracy-4-gpus
distributed-tests-8xh100
distributed-torchrun-examples-4-gpus
distributed-torchrun-shutdown-tests-2-gpus
dp-ep-distributed-nixlconnector-pd-accuracy-tests-4-gpus
e2e-core-1-gpu
e2e-core-large-memory
e2e-scheduling-1-gpu
e2e-scheduling-accuracy-1-gpu
elastic-ep-scaling-test
engine
engine-1-gpu
entrypoints-integration-api-server ×4
entrypoints-integration-api-server-generate
entrypoints-integration-api-server-openai-chat_completion
entrypoints-integration-api-server-openai-completion
entrypoints-integration-llm
entrypoints-integration-multimodal
entrypoints-integration-pooling
entrypoints-integration-responses-api
entrypoints-integration-speech_to_text
entrypoints-unit-tests
examples
extract-hidden-states-integration
extract-hidden-states-integration-2-gpus
fault-tolerance-e2e-2xh100
fusion-e2e-config-sweep-h100
fusion-e2e-quick-h100
fusion-e2e-tp2-ar-rms-config-sweep-h100
fusion-e2e-tp2-b200
fusion-e2e-tp2-quick-h100
hybrid-ssm-nixlconnector-pd-accuracy-tests-4-gpus
hybrid-ssm-nixlconnector-pd-prefix-cache-2-gpus
kernels-b200 ×3
kernels-deepgemm-test-h100
kernels-root-misc-test-b200
kimi-k3-prefix-cache-4xb200 ×2
kv-offload-large
kv-offload-medium
kv-offload-small
language-models-tests-extra-standard ×2
language-models-tests-granite-l4-compatibility
language-models-tests-hybrid ×2
language-models-tests-standard
lm-eval-dspark-watermark-2xh100
lm-eval-small-models
lm-eval-turboquant-k3v4nc
lm-eval-turboquant-k8v4
lm-eval-turboquant-t3nc
lm-eval-turboquant-t4nc
lm-eval-watermarking
lora ×4
lora-tp-distributed ×4
metrics-tracing-2-gpus
model-executor
mooncake-ec-tcp-e2e-2-gpus
mrcr-eval-small-models
multi-modal-accuracy-eval-small-models
multi-modal-models-standard-1-qwen2
multi-modal-models-standard-2-qwen3-gemma
multi-modal-models-standard-3-llava-qwen2-vl
multi-modal-models-standard-4-other-whisper
multiconnector-nixl-offloading-pd-accuracy-2-gpus
multiconnector-nixl-offloading-pd-edge-cases-2-gpus
nixlconnector-pd-edge-cases-2-gpus
openai-api-correctness
pipeline-context-parallelism-4-gpus
plugin-tests-2-gpus
push-nixlconnector-pp-prefill-pd-accuracy-4-gpus
pytorch-compilation-dynamic-shapes
pytorch-compilation-unit-tests
pytorch-compilation-unit-tests-h100
pytorch-fullgraph-cudagraph-l4-compatibility
pytorch-fullgraph-test
quantized-moe-test-b200
rayexecutorv2-4-gpus
regression
replayssm-e2e
rust-frontend-core-correctness
rust-frontend-distributed
rust-frontend-openai-coverage
rust-frontend-serve-admin-coverage
rust-frontend-tool-use
samplers-multimodal-beam-search
samplers-test
scale-out-ec-e2e-2-gpus
sharded-rdt-weight-transfer
spec-decode-draft-model ×4
spec-decode-eagle-1-deepseek-qwen
spec-decode-eagle-2-llama3-qwen-vl-other
spec-decode-mtp-deepseek-mimo
spec-decode-mtp-gemma4
spec-decode-mtp-qwen3-5
spec-decode-ngram-suffix
spec-decode-speculators
v1-core
v1-executor-worker
v1-kv-connectors ×4
v1-metrics-lmeval
v1-others-cpu
v1-sample
v1-spec-decode

Would skip (today's rules run them) (13)

ascend-npu-test
basic-models-test-other-cpu
basic-models-tests-initialization
cpu-params-env-tokenizers-parser
cpu-reasoning-renderers
cpu-tool-parsers
kernels-fla-ops-test-b200
kernels-mhc-test-b200
multi-modal-processor ×4
multi-modal-processor-cpu ×4
pytorch-compilation-passes-unit-tests
v1-kv-offload
v1-logits-oracle

Would add (today's rules do not run them) (62)

amd-kernels-mi355 (Python record)
amd-lm-eval-small-models-harness (Python record)
basic-models-tests-extra-initialization ×14 (Python record)
batch-invariance-b200 (Python record)
batch-invariance-h100 (Python record)
crosslayer-kv-layout-distributed-nixlconnector-pd-accuracy-tests-4-gpus (Python record)
distributed-compile-comm-4-gpus (Python record)
distributed-dp-tests-4-gpus (Python record)
distributed-flashinfer-nixlconnector-pd-accuracy-4-gpus (Python record)
distributed-model-tests-2-gpus ×3 (code map)
distributed-mooncakeconnector-pd-accuracy-4-gpus (Python record)
distributed-nixlconnector-pd-accuracy-4-gpus (Python record)
distributed-tests-8xh100 (Python record)
distributed-torchrun-examples-4-gpus (Python record)
dp-ep-distributed-nixlconnector-pd-accuracy-tests-4-gpus (Python record)
elastic-ep-scaling-test (Python record)
engine (code map)
engine-1-gpu (Python record)
entrypoints-unit-tests (Python record)
examples (code map)
extract-hidden-states-integration (code map)
extract-hidden-states-integration-2-gpus (code map)
fusion-e2e-config-sweep-h100 (Python record)
fusion-e2e-quick-h100 (Python record)
fusion-e2e-tp2-ar-rms-config-sweep-h100 (Python record)
fusion-e2e-tp2-b200 (Python record)
fusion-e2e-tp2-quick-h100 (Python record)
hybrid-ssm-nixlconnector-pd-accuracy-tests-4-gpus (Python record)
hybrid-ssm-nixlconnector-pd-prefix-cache-2-gpus (Python record)
kernels-b200 ×3 (Python record)
kernels-deepgemm-test-h100 (Python record)
kimi-k3-prefix-cache-4xb200 ×2 (code map)
kv-offload-large (Python record)
kv-offload-medium (Python record)
kv-offload-small (Python record)
language-models-tests-extra-standard ×2 (code map)
lm-eval-small-models (Python record)
lm-eval-turboquant-k3v4nc (Python record)
lm-eval-turboquant-k8v4 (Python record)
lm-eval-turboquant-t3nc (Python record)
lm-eval-turboquant-t4nc (Python record)
lora ×4 (Python record)
lora-tp-distributed ×4 (Python record)
model-executor (code map)
mrcr-eval-small-models (Python record)
multi-modal-accuracy-eval-small-models (Python record)
multiconnector-nixl-offloading-pd-accuracy-2-gpus (Python record)
multiconnector-nixl-offloading-pd-edge-cases-2-gpus (Python record)
nixlconnector-pd-edge-cases-2-gpus (Python record)
openai-api-correctness (Python record)
plugin-tests-2-gpus (Python record)
push-nixlconnector-pp-prefill-pd-accuracy-4-gpus (Python record)
quantized-moe-test-b200 (Python record)
rayexecutorv2-4-gpus (Python record)
rust-frontend-core-correctness (Python record)
rust-frontend-openai-coverage (Python record)
rust-frontend-serve-admin-coverage (Python record)
rust-frontend-tool-use (Python record)
samplers-multimodal-beam-search (Python record)
samplers-test (Python record)
scale-out-ec-e2e-2-gpus (Python record)
sharded-rdt-weight-transfer (Python record)

AMD mirrors: would skip (9)

basic-models-tests-initialization
fault-tolerance-e2e-2xh100
kernels-fla-ops-test-b200
kernels-mhc-test-b200
multi-modal-processor ×4
platform-tests
pytorch-compilation-passes-unit-tests
v1-kv-offload
v1-logits-oracle

AMD mirrors: would add (52)

crosslayer-kv-layout-distributed-nixlconnector-pd-accuracy-tests-4-gpus (Python record)
distributed-flashinfer-nixlconnector-pd-accuracy-4-gpus (Python record)
distributed-mooncakeconnector-pd-accuracy-4-gpus (Python record)
distributed-nixlconnector-pd-accuracy-4-gpus (Python record)
distributed-tests-8xh100 (Python record)
distributed-torchrun-examples-4-gpus (Python record)
dp-ep-distributed-nixlconnector-pd-accuracy-tests-4-gpus (Python record)
engine (code map)
engine-1-gpu (Python record)
entrypoints-unit-tests (Python record)
examples (code map)
extract-hidden-states-integration (code map)
extract-hidden-states-integration-2-gpus (code map)
fusion-e2e-config-sweep-h100 (Python record)
fusion-e2e-quick-h100 (Python record)
fusion-e2e-tp2-ar-rms-config-sweep-h100 (Python record)
fusion-e2e-tp2-b200 (Python record)
fusion-e2e-tp2-quick-h100 (Python record)
hybrid-ssm-nixlconnector-pd-accuracy-tests-4-gpus (Python record)
hybrid-ssm-nixlconnector-pd-prefix-cache-2-gpus (Python record)
kernels-moe-test ×5 (Python record)
kernels-quantization-test ×6 (Python record)
kv-offload-medium (Python record)
kv-offload-small (Python record)
language-models-tests-extra-standard ×2 (code map)
lm-eval-small-models (Python record)
lm-eval-turboquant-k3v4nc (Python record)
lm-eval-turboquant-k8v4 (Python record)
lm-eval-turboquant-t3nc (Python record)
lm-eval-turboquant-t4nc (Python record)
lora ×4 (Python record)
lora-tp-distributed ×4 (Python record)
model-executor (code map)
mrcr-eval-small-models (Python record)
multi-modal-accuracy-eval-small-models (Python record)
multiconnector-nixl-offloading-pd-accuracy-2-gpus (Python record)
multiconnector-nixl-offloading-pd-edge-cases-2-gpus (Python record)
nixlconnector-pd-edge-cases-2-gpus (Python record)
nixlconnector-pd-spec-decode-acceptance-2-gpus (code map)
openai-api-correctness (Python record)
plugin-tests-2-gpus (Python record)
push-nixlconnector-pp-prefill-pd-accuracy-4-gpus (Python record)
quantization ×4 (Python record)
quantized-models-test (Python record)
rust-frontend-core-correctness (Python record)
rust-frontend-openai-coverage (Python record)
rust-frontend-serve-admin-coverage (Python record)
rust-frontend-tool-use (Python record)
samplers-multimodal-beam-search (Python record)
samplers-test (Python record)
scale-out-ec-e2e-2-gpus (Python record)
sharded-rdt-weight-transfer (Python record)

_{15 changed files · base 111f71a6db · head 2e34632e67 · Python record: build 92561 at 4056c8ac1f · kernel record: table 4056c8a (build 92561), map 4056c8a · not counted: 10 build steps, 5 A100 steps the generator no longer emits, 107 optional steps the selector would also run}

Signed-off-by: yewentao256 <zhyanwentao@126.com>

yewentao256 · 2026-10-02T20:13:08Z

/ci run

github-actions · 2026-10-02T20:13:28Z

✅ Triggered Buildkite CI #92695 for commit 2e34632e67f8.

LucasWilkinson

LGTM but we should decide if we want to land this as a temporary fix or unify with adaptive verification now

(so maybe hold off landing)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

… in config/vllm.py and the V2 model runner - vllm/config/vllm.py: num_lookahead_tokens keeps Uno's branch first (K lookahead slots: one seed and K-1 noisy rows), then upstream's fill-in rule (vllm-project#59105): DFlash and fill-in DSpark reserve K + 1, anchor-sampling DSpark, EAGLE and draft models reserve K. - vllm/v1/worker/gpu/model_runner.py: ExecuteModelState carries upstream's num_spec_tokens_to_schedule (vllm-project#57053) beside Uno's skip_speculator_proposal and zero_next_draft_req_ids. The proposal stays in _run_speculator_proposal, after the AsyncOutput D2H handoff and behind the proposal-skip check; it keeps the watermark preparation, the draft workspace lane and adaptive-verification confidences, and now passes num_speculative_tokens to propose() and blanks draft rows past the scheduled count to -1, as upstream does. _publish_draft_tokens publishes only the first num_spec_tokens_to_schedule columns, for every speculator and for diffusion drafts. Dynamic K and Uno: the scheduler lowers num_spec_tokens_to_schedule below K only when num_speculative_tokens_per_batch_size is set, and Uno's config rejects that option ("Uno requires a fixed num_speculative_tokens"). Uno therefore always runs at its configured K and its proposals and acceptance are unchanged. The generic blanking and column slice still apply to Uno, so if the scheduler ever asked for fewer tokens, the extra drafts would be dropped before verification and the output would stay lossless. Signed-off-by: MJ <121681560+mjbrn@users.noreply.github.com>

The V2 runner now passes the scheduled spec-token count to every speculator. Uno requires a fixed K and drafts all K rows in one launch, so it accepts the argument like DFlash; the runner blanks any rows past the scheduled count. The sample_tokens ordering test carries the new ExecuteModelState field and expects the new keyword. Signed-off-by: MJ <121681560+mjbrn@users.noreply.github.com>

honor dynamic K in autoregressive speculators

fe7cf03

Signed-off-by: yewentao256 <zhyanwentao@126.com>

yewentao256 requested review from WoosukKwon and njhill as code owners September 15, 2026 18:30

claude Bot reviewed Sep 15, 2026

View reviewed changes

yewentao256 added the ready ONLY add when PR is ready to merge/full CI is needed label Sep 15, 2026

mergify Bot added nvidia speculative-decoding dflash mrv2 Model Runner V2 specific labels Sep 15, 2026

github-project-automation Bot added this to NVIDIA and Sprint - DFlash Sep 15, 2026

mergify Bot added the bug Something isn't working label Sep 15, 2026

github-project-automation Bot moved this to Backlog in Sprint - DFlash Sep 15, 2026

Merge branch 'main' into wentao-fix-spec-decode-param

2471a1c

fix precommit issue

ed2ec72

Signed-off-by: yewentao256 <zhyanwentao@126.com>

Merge branch 'main' into wentao-fix-spec-decode-param

00cee63

Merge branch 'main' into wentao-fix-spec-decode-param

79f4582

mergify Bot added the needs-rebase label Sep 29, 2026

LucasWilkinson reviewed Sep 30, 2026

View reviewed changes

Merge branch 'main' into wentao-fix-spec-decode-param

80a5c34

Signed-off-by: yewentao256 <zhyanwentao@126.com>

mergify Bot removed the needs-rebase label Sep 30, 2026

Merge branch 'main' into wentao-fix-spec-decode-param

ac75935

yewentao256 and others added 2 commits October 2, 2026 20:12

fix ci

2b666eb

Signed-off-by: yewentao256 <zhyanwentao@126.com>

Merge branch 'main' into wentao-fix-spec-decode-param

2e34632

yewentao256 enabled auto-merge (squash) October 3, 2026 00:16

LucasWilkinson approved these changes Oct 5, 2026

View reviewed changes

yewentao256 merged commit 74c5cbc into main Oct 5, 2026
184 checks passed

yewentao256 deleted the wentao-fix-spec-decode-param branch October 5, 2026 18:54

MatthewBonanni mentioned this pull request Oct 5, 2026

[CI] Fix test_mixed_warmup_gate after #57053 #60116

Merged

robertgshaw2-redhat pushed a commit that referenced this pull request Oct 5, 2026

[CI] Fix test_mixed_warmup_gate after #57053 (#60116)

8542c3e

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

mjbrn mentioned this pull request Oct 5, 2026

[Feature][Spec Decode] Add Uno shared-model drafting with a draft-only LoRA adapter (Model Runner V2) #55947

Open

cheese-cakee mentioned this pull request Oct 6, 2026

[Bugfix][MRV2][Spec Decode] Accept num_speculative_tokens in NgramGPUSpeculator.propose #60300

Merged

6 tasks done

konfan12345678 mentioned this pull request Oct 8, 2026

[BugFix][MRV2] Honor runtime speculative draft length on Ascend vllm-project/vllm-ascend#17980

Draft

gf239 mentioned this pull request Oct 8, 2026

[Perf][Attention] FlashInfer: skip the seq_lens copy with speculative decoding #57352

Open

3 of 4 tasks

Conversation

yewentao256 commented Sep 15, 2026

Purpose

Test

Uh oh!

claude Bot left a comment

Choose a reason for hiding this comment

Claude Code Review

Uh oh!

yewentao256 commented Sep 15, 2026

Uh oh!

github-actions Bot commented Sep 15, 2026

Uh oh!

yewentao256 commented Sep 16, 2026

Uh oh!

github-actions Bot commented Sep 16, 2026

Uh oh!

mergify Bot commented Sep 16, 2026

Uh oh!

yewentao256 commented Sep 16, 2026

Uh oh!

github-actions Bot commented Sep 16, 2026

Uh oh!

yewentao256 commented Sep 16, 2026

Uh oh!

github-actions Bot commented Sep 16, 2026

Uh oh!

yewentao256 commented Sep 18, 2026

Uh oh!

github-actions Bot commented Sep 18, 2026

Uh oh!

yewentao256 commented Sep 21, 2026

Uh oh!

github-actions Bot commented Sep 21, 2026

Uh oh!

github-actions Bot commented Sep 26, 2026

Uh oh!

mergify Bot commented Sep 29, 2026

Uh oh!

LucasWilkinson left a comment

Choose a reason for hiding this comment

Uh oh!

yewentao256 commented Sep 30, 2026

Uh oh!

github-actions Bot commented Sep 30, 2026

Uh oh!

yewentao256 commented Sep 30, 2026

Uh oh!

github-actions Bot commented Sep 30, 2026

Uh oh!

yewentao256 commented Oct 2, 2026

Uh oh!

github-actions Bot commented Oct 2, 2026

Uh oh!

vllm-agent commented Oct 2, 2026 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

CI selector (shadow): 127 test steps (165 jobs) instead of 78 (97 jobs)

Uh oh!

yewentao256 commented Oct 2, 2026

Uh oh!

github-actions Bot commented Oct 2, 2026

Uh oh!

LucasWilkinson left a comment • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Choose a reason for hiding this comment

Uh oh!

Uh oh!

Reviewers

Assignees

Labels

Projects

Milestone

Development

Uh oh!

3 participants

vllm-agent commented Oct 2, 2026 •

edited

Loading

LucasWilkinson left a comment •

edited

Loading