diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index 208b5a679e72..c2b41594d6a2 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -1160,6 +1160,15 @@ def schedule(self, throttle_prefills: bool = False) -> SchedulerOutput: self.dynamic_sd_lookup[len(num_scheduled_tokens)], ) + # Skip spec decode when all scheduled requests need ≤1 output token. + # Speculative decoding is useless for 1-token outputs and adds ~40ms + # of draft generation + verification overhead. + if num_spec_tokens_to_schedule > 0: + if (scheduled_new_reqs + and not scheduled_running_reqs + and all(req.max_tokens <= 1 for req in scheduled_new_reqs)): + num_spec_tokens_to_schedule = 0 + scheduled_encoder_input_stats = None if ( self.log_stats