From 2e114ec2c23b02e0621f8f67660f429f1cb0353d Mon Sep 17 00:00:00 2001 From: Simon Veitner Date: Mon, 12 Oct 2026 09:46:16 +0000 Subject: [PATCH 1/7] [Frontend] Tokenize the decision state once per request /v1/systemone rendered the chat template and tokenized the whole state once per question, in two renderer pool jobs each. A 64-question decision on a 27k-token state spent about 2 s of CPU there and queued every other request behind it on the single-worker pool. For more than one question, NextTokenStrategy now renders a probe with a marker after the last "\n\n" that every question's user turn shares, which ends the state. It tokenizes the text before the marker once and builds each prompt as those ids plus the ids of the rest of its prompt. That takes three render pool jobs per decision instead of two per question: the probe render, the prefix tokenization, and one job that renders every question to check it and tokenizes the short question windows. The two template jobs run under the chat template render timeout; the prefix tokenization is not timed, as in preprocess_chat. A single question keeps the per-question render, which is cheaper there. The prefix goes through the renderer's tokenize path with the request's tokenize params, so an oversized state fails the character check before tokenization, as before. The joined prompts then go through preprocess_cmpl for the token length check and cache_salt. When the prefix fits but the whole prompt does not, the token length error reports the exact count instead of "at least". The read falls back to the per-question render when the request has images, the renderer is not HfRenderer, the user turns share no "\n\n", the marker is not found exactly once, the text before it does not end with the "\n\n" separator, the template changes any question's text, or the last 64 characters of the prefix tokenize differently with a question after them. That window check is a heuristic backstop for tokenizers whose split depends on the next characters, such as GPT-2. OnlineRenderer.build_chat_params is shared with preprocess_chat, and HfRenderer.run_chat_template_job_async runs one template job on the render pool. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Simon Veitner --- .../structured_decisions/test_strategies.py | 268 ++++++++++++++---- .../openai/decisions/strategies.py | 137 ++++++++- vllm/renderers/hf.py | 12 +- vllm/renderers/online_renderer.py | 50 ++-- 4 files changed, 385 insertions(+), 82 deletions(-) diff --git a/tests/entrypoints/generate/structured_decisions/test_strategies.py b/tests/entrypoints/generate/structured_decisions/test_strategies.py index 6ddfa5944443..63946c243620 100644 --- a/tests/entrypoints/generate/structured_decisions/test_strategies.py +++ b/tests/entrypoints/generate/structured_decisions/test_strategies.py @@ -8,7 +8,7 @@ from io import BytesIO from types import SimpleNamespace from typing import Any -from unittest.mock import MagicMock +from unittest.mock import AsyncMock, MagicMock, patch import numpy as np import pybase64 as base64 @@ -19,7 +19,7 @@ from PIL import Image from starlette.datastructures import State -from vllm.config import ModelConfig +from vllm.config import ModelConfig, VllmConfig from vllm.engine.protocol import EngineClient from vllm.entrypoints.generate.structured_decisions.api_router import ( register_structured_decisions_api_router, @@ -41,6 +41,8 @@ from vllm.entrypoints.openai.decisions.serving import OpenAIServingDecisions from vllm.entrypoints.openai.decisions.state import init_decisions_state from vllm.entrypoints.openai.decisions.strategies import ( + _MARKER, + READ_STRATEGIES, ClefFlashStrategy, D3Strategy, DiffusionGemmaCanvasStrategy, @@ -48,6 +50,7 @@ LiquidStrategy, NextTokenStrategy, ReadContext, + _messages, cap_pixels, reply_label_ids, select_read_strategy, @@ -60,6 +63,7 @@ init_exception_handler, ) from vllm.entrypoints.serve.utils.request_logger import RequestLogger +from vllm.exceptions import VLLMValidationError from vllm.inputs import mm_input, tokens_input from vllm.logprobs import Logprob from vllm.multimodal.media import MediaConnector, MediaWithBytes @@ -224,31 +228,62 @@ def test_labels_start_the_reply(qwen): reply_label_ids(tokenizer, tokenizer.encode("team:")) -@pytest.fixture -def engine_and_renderer(qwen): - tokenizer, _ = qwen - model_config = ModelConfig(model="Qwen/Qwen3-0.6B", dtype="float32") - renderer = HfRenderer( - SimpleNamespace( - model_config=model_config, - parallel_config=SimpleNamespace(_api_process_rank=0), - ), - tokenizer, - ) - engine = MagicMock(spec=EngineClient) - engine.errored = False - engine.model_config = model_config - engine.renderer = renderer - engine.input_processor = MagicMock() - online_renderer = OnlineRenderer( - model_config, - renderer, - request_logger=None, - chat_template=None, - chat_template_content_format="auto", +async def generate(prompt, params, request_id, **kwargs): + label_ids = params.logprob_token_ids + yield RequestOutput( + request_id=request_id, + prompt=None, + prompt_token_ids=prompt["prompt_token_ids"], + prompt_logprobs=None, + outputs=[ + CompletionOutput( + index=0, + text="A", + token_ids=[label_ids[0]], + cumulative_logprob=None, + logprobs=[{token_id: Logprob(-1.0) for token_id in label_ids}], + finish_reason="length", + ) + ], + finished=True, ) - yield engine, online_renderer - renderer.shutdown() + + +@pytest.fixture +def make_context(): + from vllm.tokenizers import get_tokenizer + + renderers = [] + + def make(model="Qwen/Qwen3-0.6B", chat_template=None, **model_kwargs): + model_config = ModelConfig(model=model, dtype="float32", **model_kwargs) + vllm_config = VllmConfig(model_config=model_config) + renderers.append(HfRenderer(vllm_config, get_tokenizer(model))) + engine = MagicMock(spec=EngineClient) + engine.errored = False + engine.model_config = model_config + engine.vllm_config = vllm_config + engine.renderer = renderers[-1] + engine.input_processor = MagicMock() + engine.generate.side_effect = generate + online_renderer = OnlineRenderer( + model_config, + renderers[-1], + request_logger=None, + chat_template=None, + chat_template_content_format="auto", + ) + return ReadContext(engine, online_renderer, chat_template, "auto", {}) + + yield make + for renderer in renderers: + renderer.shutdown() + + +@pytest.fixture +def engine_and_renderer(make_context): + context = make_context() + return context.engine_client, context.online_renderer def serving(engine, strategy) -> ServingStructuredDecisions: @@ -260,28 +295,6 @@ def serving(engine, strategy) -> ServingStructuredDecisions: @pytest.fixture def serving_and_engine(engine_and_renderer): engine, online_renderer = engine_and_renderer - - async def generate(prompt, params, request_id, **kwargs): - label_ids = params.logprob_token_ids - yield RequestOutput( - request_id=request_id, - prompt=None, - prompt_token_ids=prompt["prompt_token_ids"], - prompt_logprobs=None, - outputs=[ - CompletionOutput( - index=0, - text="A", - token_ids=[label_ids[0]], - cumulative_logprob=None, - logprobs=[{token_id: Logprob(-1.0) for token_id in label_ids}], - finish_reason="length", - ) - ], - finished=True, - ) - - engine.generate.side_effect = generate strategy = NextTokenStrategy(ReadContext(engine, online_renderer, None, "auto", {})) return serving(engine, strategy), engine @@ -339,6 +352,165 @@ async def test_request_metadata_reaches_every_question( ) +# The first question has no "Question:" line, so EDITING_TEMPLATE keeps it as is. +QUESTIONS = [ + build_question("refund", "noul", "", {"true": "a refund is due"}, 26), + build_question( + "team", "choice", "Which team?", {"billing": "pay", "ship": None}, 26 + ), + build_question("urgency", "score", "How urgent?", ["low", "medium", "high"], 26), +] + +SPLICE_STATES = { + "empty": "", + "spaces": " padded ", + "crlf": "line one\r\nline two\r\n", + "nbsp": "trailing nbsp\u00a0", + "combining": "cafe\u0301 x\u0301\u0301", + "im_end": "user wrote <|im_end|> and <|im_start|>assistant", + "json": '{"ticket": 58213, "tags": ["billing", "urgent"]}', + "paragraphs": "First.\n\nSecond.\n\nThird.", +} + +SPLICE_MODELS = { + "Qwen3ForCausalLM": "Qwen/Qwen3-0.6B", + "Qwen3_5ForConditionalGeneration": "Qwen/Qwen3.5-0.8B", + "Qwen3_5MoeForConditionalGeneration": "Qwen/Qwen3.5-35B-A3B", + "DiffusionGemmaForBlockDiffusion": "google/diffusiongemma-26B-A4B-it", + "Lfm2VlForConditionalGeneration": "LiquidAI/d1-3B", +} + +CHATML_TEMPLATE = ( + "{% for m in messages %}<|im_start|>{{ m.role }}\n" + "{{ m.content }}<|im_end|>\n{% endfor %}<|im_start|>assistant\n" +) +EDITING_TEMPLATE = CHATML_TEMPLATE.replace( + "{{ m.content }}", "{{ m.content | replace('Question:', 'Q:') }}" +) +SEPARATOR_TEMPLATE = CHATML_TEMPLATE.replace( + "{{ m.content }}", "{{ m.content | replace('\\n\\n', ' ') }}" +) + + +async def spliced_and_full(strategy, state, instructions=None): + read_request = strategy._read_request(None, "tenant") + contents = [strategy._content(state, q) for q in QUESTIONS] + spliced = await strategy._render_spliced(read_request, contents, instructions) + full = [ + await strategy._render(read_request, _messages(content, instructions)) + for content in contents + ] + for engine_input in [*(spliced or []), *full]: + engine_input.pop("arrival_time") + return spliced, full + + +# Clef-Omni encodes its own prompt, so it never splices. +SPLICE_ARCHITECTURES = [ + a for a, s in READ_STRATEGIES.items() if issubclass(s, NextTokenStrategy) +] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("architecture", SPLICE_ARCHITECTURES) +async def test_spliced_prompts_match_full_render(make_context, architecture): + context = make_context(SPLICE_MODELS[architecture]) + model_config = context.engine_client.model_config + assert model_config.architecture == architecture + strategy = select_read_strategy(model_config, ("generate",))(context) + for name, state in SPLICE_STATES.items(): + for instructions in (None, " You triage tickets.\n"): + spliced, full = await spliced_and_full(strategy, state, instructions) + # Qwen3.5 trims the empty state's separator off, so that one falls back. + assert spliced == full or (spliced is None and not state), name + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "chat_template,state", + [ + (EDITING_TEMPLATE, "state"), + (None, f"a state with the split marker {_MARKER}"), + ], + ids=["template_edits_question", "marker_in_state"], +) +async def test_splice_falls_back_to_full_render(make_context, chat_template, state): + strategy = NextTokenStrategy(make_context(chat_template=chat_template)) + spliced, _ = await spliced_and_full(strategy, state) + assert spliced is None + with patch.object(strategy, "_render", wraps=strategy._render) as render: + reads = await strategy.read( + QUESTIONS, + None, + state, + request_id="r", + chat_template_kwargs=None, + lora_request=None, + priority=0, + ) + assert render.call_count == len(QUESTIONS) == len(reads) + + +@pytest.mark.asyncio +async def test_images_skip_the_splice(make_context, monkeypatch): + strategy = NextTokenStrategy(make_context()) + image = {"type": "image_pil", "image_pil": Image.new("RGB", (8, 8))} + monkeypatch.setattr(strategy, "_image_parts", AsyncMock(return_value=[image])) + render_spliced = AsyncMock() + monkeypatch.setattr(strategy, "_render_spliced", render_spliced) + render = strategy._render + + async def text_render(read_request, messages): + assert messages[-1]["content"][0] is image + text = messages[-1]["content"][-1]["text"] + return await render(read_request, _messages(text, None)) + + monkeypatch.setattr(strategy, "_render", text_render) + reads = await strategy.read( + QUESTIONS, + None, + "state", + request_id="r", + chat_template_kwargs=None, + lora_request=None, + priority=0, + media={"image": ["https://example.org/a.png"]}, + ) + assert len(reads) == len(QUESTIONS) + render_spliced.assert_not_called() + + +@pytest.mark.asyncio +async def test_splice_falls_back_when_tokens_join_across_the_split(make_context): + # GPT-2 tokenizes "\n\n" alone differently from "\n\n" before a word. + strategy = NextTokenStrategy.__new__(NextTokenStrategy) + strategy.context = make_context("openai-community/gpt2", CHATML_TEMPLATE) + spliced, _ = await spliced_and_full(strategy, "state") + assert spliced is None + + +@pytest.mark.asyncio +async def test_splice_falls_back_when_the_template_edits_the_separator(make_context): + # Tabs then " 0" tokenize alike in the window, so only the separator check sees it. + strategy = NextTokenStrategy(make_context(chat_template=SEPARATOR_TEMPLATE)) + score = build_question("level", "score", "", ["low", "high"], 26) + contents = [strategy._content("x" + "\t" * 71, score)] * 2 + read_request = strategy._read_request(None) + assert await strategy._render_spliced(read_request, contents, None) is None + + +@pytest.mark.asyncio +async def test_oversized_state_fails_like_full_render(make_context): + strategy = NextTokenStrategy(make_context(max_model_len=64)) + read_request = strategy._read_request(None) + contents = [strategy._content("ticket " * 2000, q) for q in QUESTIONS] + with pytest.raises(VLLMValidationError) as spliced: + await strategy._render_spliced(read_request, contents, None) + with pytest.raises(VLLMValidationError) as full: + await strategy._render(read_request, _messages(contents[0], None)) + assert spliced.value.parameter == full.value.parameter == "input_text" + + def test_canvas_read(): strategy = DiffusionGemmaCanvasStrategy.__new__(DiffusionGemmaCanvasStrategy) strategy.thought, strategy.end, strategy.pad = [10, 11, 12, 13], 106, 0 diff --git a/vllm/entrypoints/openai/decisions/strategies.py b/vllm/entrypoints/openai/decisions/strategies.py index fd241ff34fe6..66dbc302661b 100644 --- a/vllm/entrypoints/openai/decisions/strategies.py +++ b/vllm/entrypoints/openai/decisions/strategies.py @@ -9,13 +9,14 @@ import hashlib import json import math +import os import random import time from abc import ABC, abstractmethod from collections.abc import Mapping, Sequence from dataclasses import dataclass from functools import cached_property -from typing import Any, ClassVar, TypeVar +from typing import Any, ClassVar, TypeVar, cast import numpy as np from PIL import Image @@ -25,7 +26,13 @@ from vllm.engine.protocol import EngineClient from vllm.entrypoints.generate.label_reads import next_token_label_reads from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel -from vllm.inputs import EngineInput, MultiModalInput, TokensPrompt, tokens_input +from vllm.inputs import ( + EngineInput, + MultiModalInput, + TextPrompt, + TokensPrompt, + tokens_input, +) from vllm.logger import init_logger from vllm.lora.request import LoRARequest from vllm.multimodal.media import ( @@ -38,6 +45,8 @@ from vllm.pooling_params import PoolingParams from vllm.renderers import ChatParams, TokenizeParams, merge_kwargs from vllm.renderers.chat_utils import ChatTemplateContentFormatOption +from vllm.renderers.hf import HfRenderer +from vllm.renderers.inputs import TokPrompt from vllm.renderers.inputs.preprocess import extract_prompt_components from vllm.renderers.online_renderer import OnlineRenderer from vllm.sampling_params import SamplingParams @@ -218,6 +227,17 @@ def cap_pixels(image: Image.Image, max_pixels: int) -> Image.Image: return image.resize(size, Image.Resampling.BICUBIC) +# Marks where a question goes in the rendered prompt. +_MARKER = "\ue000" + + +def _messages(content: Any, instructions: str | None) -> list[dict[str, Any]]: + messages = [{"role": "user", "content": content}] + if instructions: + messages.insert(0, {"role": "system", "content": instructions}) + return messages + + class NextTokenStrategy(ReadStrategy): """Autoregressive models. Each question is one request: the state, then the question with its labeled options, and the logprobs of the label tokens as @@ -306,7 +326,7 @@ def _read_request( async def _render( self, read_request: ReadPromptRequest, messages: list[dict[str, Any]] - ) -> tuple[EngineInput, list[int]]: + ) -> EngineInput: ctx = self.context _, (engine_input,) = await ctx.online_renderer.preprocess_chat( read_request, @@ -318,10 +338,85 @@ async def _render( # sender cache believing the engine has the images. skip_mm_cache=True, ) - prompt_ids = extract_prompt_components( - ctx.engine_client.model_config, engine_input - ).token_ids - return engine_input, list(prompt_ids or []) + return engine_input + + async def _render_spliced( + self, + read_request: ReadPromptRequest, + contents: list[str], + instructions: str | None, + ) -> list[EngineInput] | None: + """Every prompt with the state tokenized once, or None to fall back.""" + ctx = self.context + online_renderer = ctx.online_renderer + renderer = online_renderer.renderer + if not isinstance(renderer, HfRenderer): + return None + chat_params = online_renderer.build_chat_params( + read_request, + ctx.chat_template, + ctx.chat_template_content_format, + ctx.default_chat_template_kwargs, + ) + tok_params = read_request.build_tok_params(online_renderer.model_config) + + # Split after the last "\n\n" the contents share, which ends the state. + cut = os.path.commonprefix(contents).rfind("\n\n") + 2 + if cut < 2: + return None + suffixes = [c[cut:] for c in contents] + + def render(content: str) -> str | None: + messages: list[Any] = _messages(content, instructions) + _, prompt = renderer.render_messages(messages, chat_params) + return cast(TextPrompt, prompt).get("prompt") + + def ids(prompts: Sequence[TokPrompt]) -> list[list[int]]: + return [list(cast(TokensPrompt, p)["prompt_token_ids"]) for p in prompts] + + arrival_time = time.time() + probe = await renderer.run_chat_template_job_async( + lambda: render(contents[0][:cut] + _MARKER) + ) + if not isinstance(probe, str) or probe.count(_MARKER) != 1: + return None + prefix, tail = probe.split(_MARKER) + if not prefix.endswith("\n\n"): + return None + (prefix_ids,) = ids( + await renderer.tokenize_prompts_async( + [TextPrompt(prompt=prefix)], tok_params + ) + ) + head = prefix[-64:] + prompts = [prefix + q + tail for q in suffixes] + + def window_ids() -> list[list[int]] | None: + if any(render(c) != p for c, p in zip(contents, prompts)): + return None + texts = [head, *(head + q + tail for q in suffixes)] + return ids( + renderer.tokenize_prompts( + [TextPrompt(prompt=t) for t in texts], tok_params + ) + ) + + tokenized = await renderer.run_chat_template_job_async(window_ids) + if tokenized is None: + return None + head_ids, *windows = tokenized + if any(w[: len(head_ids)] != head_ids for w in windows): + return None + engine_inputs = await online_renderer.preprocess_cmpl( + read_request, + [ + TokensPrompt(prompt=p, prompt_token_ids=prefix_ids + w[len(head_ids) :]) + for p, w in zip(prompts, windows) + ], + ) + for engine_input in engine_inputs: + engine_input["arrival_time"] = arrival_time + return engine_inputs async def read( self, @@ -341,15 +436,27 @@ async def read( read_request = self._read_request(chat_template_kwargs, cache_salt) image_parts = await self._image_parts((media or {}).get("image", ())) + contents = [self._content(state, q) for q in questions] + rendered = None + if len(contents) > 1 and not image_parts: + rendered = await self._render_spliced(read_request, contents, instructions) + if rendered is None: + rendered = [] + for text in contents: + content: Any = text + if image_parts: + content = [*image_parts, {"type": "text", "text": text}] + messages = _messages(content, instructions) + rendered.append(await self._render(read_request, messages)) + slots, engine_inputs, params = [], [], [] - for q in questions: - content: Any = self._content(state, q) - if image_parts: - content = [*image_parts, {"type": "text", "text": content}] - messages = [{"role": "user", "content": content}] - if instructions: - messages.insert(0, {"role": "system", "content": instructions}) - engine_input, prompt_ids = await self._render(read_request, messages) + for q, engine_input in zip(questions, rendered): + prompt_ids = list( + extract_prompt_components( + ctx.engine_client.model_config, engine_input + ).token_ids + or [] + ) if prompt_ids[len(prompt_ids) - len(self.tail) :] != self.tail: raise StructuredDecisionError( "these chat options end the prompt differently, so the " diff --git a/vllm/renderers/hf.py b/vllm/renderers/hf.py index 48af41a7956c..b935de214e18 100644 --- a/vllm/renderers/hf.py +++ b/vllm/renderers/hf.py @@ -8,9 +8,9 @@ import itertools import weakref from collections import defaultdict, deque -from collections.abc import Sequence +from collections.abc import Callable, Sequence from functools import lru_cache -from typing import TYPE_CHECKING, Any, Final, Literal, cast, overload +from typing import TYPE_CHECKING, Any, Final, Literal, TypeVar, cast, overload import jinja2 import jinja2.ext @@ -76,6 +76,8 @@ logger = init_logger(__name__) +_R = TypeVar("_R") + # Cache of `tokenizer -> prompt_embeds placeholder token ID`. Keyed by the # tokenizer object (not `id(tokenizer)`) so a fresh tokenizer landing at a @@ -1008,6 +1010,12 @@ async def _render_with_timeout(self, coro): f"VLLM_CHAT_TEMPLATE_RENDER_TIMEOUT to override." ) from None + async def run_chat_template_job_async(self, fn: Callable[[], _R]) -> _R: + """Run ``fn``, which renders chat templates, as one render pool job under + the chat template render timeout.""" + loop = asyncio.get_running_loop() + return await self._render_with_timeout(loop.run_in_executor(self._executor, fn)) + def _can_produce_offsets(self) -> bool: # Only fast tokenizers expose offset_mapping. return self.tokenizer is not None and self.tokenizer.is_fast diff --git a/vllm/renderers/online_renderer.py b/vllm/renderers/online_renderer.py index babd0ff6e5fc..dccfd0f6102f 100644 --- a/vllm/renderers/online_renderer.py +++ b/vllm/renderers/online_renderer.py @@ -749,6 +749,34 @@ async def preprocess_cmpl( skip_mm_cache=skip_mm_cache, ) + def build_chat_params( + self, + request: Any, + default_template: str | None, + default_template_content_format: ChatTemplateContentFormatOption, + default_template_kwargs: dict[str, Any] | None, + tool_dicts: list[dict[str, Any]] | None = None, + ) -> ChatParams: + """The chat params that preprocess_chat renders ``request`` with.""" + mm_config = self.model_config.multimodal_config + default_template_kwargs = merge_kwargs( + default_template_kwargs, + dict( + tools=tool_dicts, + tokenize=( + is_mistral_tokenizer(self.renderer.tokenizer) + or self.model_config.enable_prompt_embeds + ), + ), + ) + return request.build_chat_params( + default_template, default_template_content_format + ).with_defaults( + default_template_kwargs, + default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None), + default_mm_processor_kwargs=getattr(request, "mm_processor_kwargs", None), + ) + async def preprocess_chat( self, request: Any, @@ -768,26 +796,14 @@ async def preprocess_chat( media_io_kwargs=getattr(request, "media_io_kwargs", None), trust_request_mm_kwargs=self.trust_request_mm_kwargs, ) - mm_config = self.model_config.multimodal_config - - default_template_kwargs = merge_kwargs( - default_template_kwargs, - dict( - tools=tool_dicts, - tokenize=( - is_mistral_tokenizer(renderer.tokenizer) - or self.model_config.enable_prompt_embeds - ), - ), - ) tok_params = request.build_tok_params(self.model_config) - chat_params = request.build_chat_params( - default_template, default_template_content_format - ).with_defaults( + chat_params = self.build_chat_params( + request, + default_template, + default_template_content_format, default_template_kwargs, - default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None), - default_mm_processor_kwargs=getattr(request, "mm_processor_kwargs", None), + tool_dicts, ) reuse_ids = _reused_prompt_token_ids(request, renderer, messages) From 7be54ac43ee2144289a4d59a000a9dccf480c571 Mon Sep 17 00:00:00 2001 From: Simon Veitner Date: Tue, 13 Oct 2026 08:27:27 +0000 Subject: [PATCH 2/7] [Frontend] Let big decisions take turns in the engine With the state tokenized once, every decision in a burst sends all its reads to the engine within a second or two. The engine admits them all and serves them in arrival order, so a 1-question decision that arrives during a burst of long decisions waits for the whole burst. A decision with more than one question whose prompts hold more tokens than max_num_batched_tokens, counting the shared state once as the prefix cache does, needs more than one engine step of prefill. Such decisions now take turns: at most two have reads in the engine at a time, and the others wait in FIFO order. Each admitted decision still sends all its reads at once, so its state stays in the prefix cache. Shorter decisions and 1-question decisions, which are a single engine request, skip the wait. Under --scheduling-policy priority the gate is off, since the engine orders requests by priority itself. Qwen3-4B on one H200, 64 clients, each with its own state, 2 bursts per cell (128 decisions), a 1-question bystander sent every second. Parent vs this change, run side by side: | workload | bystander p50 / p95 | decision p50 | questions/s | |-----------|------------------------------|----------------|--------------| | 64q x 27k | 34.7 / 65.6 -> 2.0 / 5.4 s | 42.2 -> 33.1 s | 58 -> 65 | | 8q x 27k | 21.3 / 40.8 -> 1.1 / 1.5 s | 20.3 -> 19.5 s | 12 -> 12 | | 16q x 175 | n=2, burst under 1 s | 0.43 -> 0.41 s | 1448 -> 1516 | | 1q x 27k | 20.0 / 37.2 -> 19.3 / 37.0 s | 17.0 -> 17.6 s | 2 -> 2 | Throughput holds for these workloads. 16q x 175 stays under the budget, and so does 64q x 175 (14.7k prompt tokens, 3.3k with the state counted once): about 1,850-1,900 questions/s either way, run side by side. The rows above were measured with the state counted per question, which gates each of them the same way; one burst each of 64q x 27k and 8q x 27k on this revision gave a bystander 2.1 / 6.2 s and 1.1 / 1.6 s. 1q x 27k is not gated either, and a bystander there waits as it would behind 64 long completions. With --scheduling-policy priority and the bystander sent at priority -1 (gate off), 64q x 27k gives a bystander 1.9 / 16.5 s and 60 questions/s. Limits: - The gate is per API server process, so N API servers admit up to 2N long decisions; api_server_count defaults to the data-parallel size. - --max-num-queued-reqs/tokens do not count decisions waiting at the gate, so a waiting decision can still get a 503 when its reads are added. - DiffusionGemma inherits read() and is gated the same way, on its prompt tokens. Clef-Omni reads a whole decision as one pooling request, so it is not gated. Generative scoring (generative_scoring/serving.py) fans out its reads the same way; that is left for a separate change. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Simon Veitner --- .../structured_decisions/test_strategies.py | 43 +++++++++++++++++++ .../openai/decisions/strategies.py | 38 +++++++++++----- 2 files changed, 71 insertions(+), 10 deletions(-) diff --git a/tests/entrypoints/generate/structured_decisions/test_strategies.py b/tests/entrypoints/generate/structured_decisions/test_strategies.py index 63946c243620..6cced5bb5bd3 100644 --- a/tests/entrypoints/generate/structured_decisions/test_strategies.py +++ b/tests/entrypoints/generate/structured_decisions/test_strategies.py @@ -480,6 +480,49 @@ async def text_render(read_request, messages): render_spliced.assert_not_called() +@pytest.mark.asyncio +@pytest.mark.parametrize( + "policy,num_questions,budget_fraction,gated", + [ + ("fcfs", 2, 1.5, True), + ("fcfs", 2, 0.75, False), + ("fcfs", 1, 1.5, False), + ("priority", 2, 1.5, False), + ], +) +async def test_big_decisions_take_the_gate( + make_context, policy, num_questions, budget_fraction, gated +): + context = make_context() + scheduler_config = context.engine_client.vllm_config.scheduler_config + scheduler_config.policy = policy + strategy = NextTokenStrategy(context) + gate = strategy._big_decision_gate = asyncio.Semaphore(1) + gate_full = [] + + async def record_gate(prompt, params, request_id, **kwargs): + gate_full.append(gate.locked()) + async for output in generate(prompt, params, request_id): + yield output + + context.engine_client.generate.side_effect = record_gate + # The engine prefills a shared state once, so only a state over budget is big. + state = "state " * int(scheduler_config.max_num_batched_tokens * budget_fraction) + read = strategy.read( + QUESTIONS[:num_questions], + None, + state, + request_id="r", + chat_template_kwargs=None, + lora_request=None, + priority=0, + ) + reads = await asyncio.wait_for(read, 30) + assert len(reads) == num_questions + assert gate_full == [gated] * num_questions + assert not gate.locked() + + @pytest.mark.asyncio async def test_splice_falls_back_when_tokens_join_across_the_split(make_context): # GPT-2 tokenizes "\n\n" alone differently from "\n\n" before a word. diff --git a/vllm/entrypoints/openai/decisions/strategies.py b/vllm/entrypoints/openai/decisions/strategies.py index 66dbc302661b..79b40fb97993 100644 --- a/vllm/entrypoints/openai/decisions/strategies.py +++ b/vllm/entrypoints/openai/decisions/strategies.py @@ -6,6 +6,7 @@ """ import asyncio +import contextlib import hashlib import json import math @@ -65,6 +66,8 @@ logger = init_logger(__name__) +_BIG_DECISIONS_IN_ENGINE = 2 + @dataclass class ReadContext: @@ -136,6 +139,14 @@ class ReadStrategy(ABC): def __init__(self, context: ReadContext): self.context = context + scheduler_config = context.engine_client.vllm_config.scheduler_config + # The priority policy orders requests in the engine instead. + self._big_decision_tokens = ( + math.inf + if scheduler_config.policy == "priority" + else scheduler_config.max_num_batched_tokens + ) + self._big_decision_gate = asyncio.Semaphore(_BIG_DECISIONS_IN_ENGINE) @cached_property def _connector(self) -> MediaConnector: @@ -449,7 +460,7 @@ async def read( messages = _messages(content, instructions) rendered.append(await self._render(read_request, messages)) - slots, engine_inputs, params = [], [], [] + slots, prompts, engine_inputs, params = [], [], [], [] for q, engine_input in zip(questions, rendered): prompt_ids = list( extract_prompt_components( @@ -462,21 +473,28 @@ async def read( "these chat options end the prompt differently, so the " "labels' tokens are unknown" ) + prompts.append(prompt_ids) slots.append(self._label_groups(q)) engine_inputs.append(self._read_input(engine_input, prompt_ids)) params.append( self._sampling_params([i for ids in slots[-1] for i in ids], prompt_ids) ) - label_reads = await next_token_label_reads( - ctx.engine_client, - engine_inputs, - params, - request_id, - lora_request=lora_request, - trace_headers=trace_headers, - priority=priority, - ) + # Decisions over one engine step take turns so short ones need not wait. + shared = len(os.path.commonprefix(prompts)) + num_tokens = shared + sum(len(p) - shared for p in prompts) + del prompts # Do not hold the id copies while waiting at the gate. + big = len(questions) > 1 and num_tokens > self._big_decision_tokens + async with self._big_decision_gate if big else contextlib.nullcontext(): + label_reads = await next_token_label_reads( + ctx.engine_client, + engine_inputs, + params, + request_id, + lora_request=lora_request, + trace_headers=trace_headers, + priority=priority, + ) reads = [] for groups, label_read in zip(slots, label_reads): From 9bd8134abb8decd4f1644f725d225793a756f345 Mon Sep 17 00:00:00 2001 From: Simon Veitner Date: Tue, 13 Oct 2026 08:04:31 +0000 Subject: [PATCH 3/7] [Frontend] Fall back when whitespace follows the splice point A question whose text after the split point (the last blank line the questions share) begins with whitespace can extend a newline run that starts before the 64-character head window. The head check then passes while the run tokenizes differently in the full prompt, so the spliced ids differ from the per-question render. This showed up with byte-level BPE tokenizers. Render such decisions per question instead. Signed-off-by: Simon Veitner Co-Authored-By: Claude Opus 5.5 (1M context) --- .../structured_decisions/test_strategies.py | 16 ++++++++++++++-- vllm/entrypoints/openai/decisions/strategies.py | 3 +++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/tests/entrypoints/generate/structured_decisions/test_strategies.py b/tests/entrypoints/generate/structured_decisions/test_strategies.py index 6cced5bb5bd3..88322ff6d363 100644 --- a/tests/entrypoints/generate/structured_decisions/test_strategies.py +++ b/tests/entrypoints/generate/structured_decisions/test_strategies.py @@ -392,9 +392,9 @@ async def test_request_metadata_reaches_every_question( ) -async def spliced_and_full(strategy, state, instructions=None): +async def spliced_and_full(strategy, state, instructions=None, questions=QUESTIONS): read_request = strategy._read_request(None, "tenant") - contents = [strategy._content(state, q) for q in QUESTIONS] + contents = [strategy._content(state, q) for q in questions] spliced = await strategy._render_spliced(read_request, contents, instructions) full = [ await strategy._render(read_request, _messages(content, instructions)) @@ -532,6 +532,18 @@ async def test_splice_falls_back_when_tokens_join_across_the_split(make_context) assert spliced is None +@pytest.mark.asyncio +async def test_splice_matches_full_render_when_newlines_cross_the_split(make_context): + # The newline run is longer than the head window. + strategy = NextTokenStrategy(make_context()) + questions = [ + build_question(qid, "noul", "Note:" + "\n" * n + "Is it urgent?", None, 26) + for qid, n in (("a", 200), ("b", 201)) + ] + spliced, full = await spliced_and_full(strategy, "state", questions=questions) + assert spliced in (None, full) + + @pytest.mark.asyncio async def test_splice_falls_back_when_the_template_edits_the_separator(make_context): # Tabs then " 0" tokenize alike in the window, so only the separator check sees it. diff --git a/vllm/entrypoints/openai/decisions/strategies.py b/vllm/entrypoints/openai/decisions/strategies.py index 79b40fb97993..ca2a9c4cc46b 100644 --- a/vllm/entrypoints/openai/decisions/strategies.py +++ b/vllm/entrypoints/openai/decisions/strategies.py @@ -376,6 +376,9 @@ async def _render_spliced( if cut < 2: return None suffixes = [c[cut:] for c in contents] + # Whitespace after the cut can join a run longer than the head window. + if any(q[:1].isspace() for q in suffixes): + return None def render(content: str) -> str | None: messages: list[Any] = _messages(content, instructions) From 9dbb2ff5edb7120a418b787cbc7f3e2614c41a7a Mon Sep 17 00:00:00 2001 From: Simon Veitner Date: Tue, 13 Oct 2026 10:06:48 +0000 Subject: [PATCH 4/7] [Frontend] Splice decision prompts from their full renders Render each question's prompt with render_messages_async, one render pool job at a time as the per-question path does, and split the rendered texts after the last blank line they all share. This drops the marker probe, the checks of the probe against the full renders, the HfRenderer check and the render pool helper in hf.py. A template that edits the question no longer forces a fallback. The splice gives up at the first render that returns token ids instead of text. The tests of the removed guards go too. Signed-off-by: Simon Veitner Co-Authored-By: Claude Opus 5.5 (1M context) --- .../structured_decisions/test_strategies.py | 50 ++--------- .../openai/decisions/strategies.py | 84 ++++++------------- vllm/renderers/hf.py | 12 +-- 3 files changed, 35 insertions(+), 111 deletions(-) diff --git a/tests/entrypoints/generate/structured_decisions/test_strategies.py b/tests/entrypoints/generate/structured_decisions/test_strategies.py index 88322ff6d363..0d7e48521ad4 100644 --- a/tests/entrypoints/generate/structured_decisions/test_strategies.py +++ b/tests/entrypoints/generate/structured_decisions/test_strategies.py @@ -8,7 +8,7 @@ from io import BytesIO from types import SimpleNamespace from typing import Any -from unittest.mock import AsyncMock, MagicMock, patch +from unittest.mock import AsyncMock, MagicMock import numpy as np import pybase64 as base64 @@ -41,7 +41,6 @@ from vllm.entrypoints.openai.decisions.serving import OpenAIServingDecisions from vllm.entrypoints.openai.decisions.state import init_decisions_state from vllm.entrypoints.openai.decisions.strategies import ( - _MARKER, READ_STRATEGIES, ClefFlashStrategy, D3Strategy, @@ -352,7 +351,6 @@ async def test_request_metadata_reaches_every_question( ) -# The first question has no "Question:" line, so EDITING_TEMPLATE keeps it as is. QUESTIONS = [ build_question("refund", "noul", "", {"true": "a refund is due"}, 26), build_question( @@ -384,12 +382,6 @@ async def test_request_metadata_reaches_every_question( "{% for m in messages %}<|im_start|>{{ m.role }}\n" "{{ m.content }}<|im_end|>\n{% endfor %}<|im_start|>assistant\n" ) -EDITING_TEMPLATE = CHATML_TEMPLATE.replace( - "{{ m.content }}", "{{ m.content | replace('Question:', 'Q:') }}" -) -SEPARATOR_TEMPLATE = CHATML_TEMPLATE.replace( - "{{ m.content }}", "{{ m.content | replace('\\n\\n', ' ') }}" -) async def spliced_and_full(strategy, state, instructions=None, questions=QUESTIONS): @@ -421,34 +413,16 @@ async def test_spliced_prompts_match_full_render(make_context, architecture): for name, state in SPLICE_STATES.items(): for instructions in (None, " You triage tickets.\n"): spliced, full = await spliced_and_full(strategy, state, instructions) - # Qwen3.5 trims the empty state's separator off, so that one falls back. + # An empty state may leave no shared blank line, so it may fall back. assert spliced == full or (spliced is None and not state), name @pytest.mark.asyncio -@pytest.mark.parametrize( - "chat_template,state", - [ - (EDITING_TEMPLATE, "state"), - (None, f"a state with the split marker {_MARKER}"), - ], - ids=["template_edits_question", "marker_in_state"], -) -async def test_splice_falls_back_to_full_render(make_context, chat_template, state): - strategy = NextTokenStrategy(make_context(chat_template=chat_template)) - spliced, _ = await spliced_and_full(strategy, state) - assert spliced is None - with patch.object(strategy, "_render", wraps=strategy._render) as render: - reads = await strategy.read( - QUESTIONS, - None, - state, - request_id="r", - chat_template_kwargs=None, - lora_request=None, - priority=0, - ) - assert render.call_count == len(QUESTIONS) == len(reads) +async def test_token_id_renders_skip_the_splice(make_context): + strategy = NextTokenStrategy(make_context()) + read_request = strategy._read_request({"tokenize": True}) + contents = [strategy._content("state", q) for q in QUESTIONS] + assert await strategy._render_spliced(read_request, contents, None) is None @pytest.mark.asyncio @@ -544,16 +518,6 @@ async def test_splice_matches_full_render_when_newlines_cross_the_split(make_con assert spliced in (None, full) -@pytest.mark.asyncio -async def test_splice_falls_back_when_the_template_edits_the_separator(make_context): - # Tabs then " 0" tokenize alike in the window, so only the separator check sees it. - strategy = NextTokenStrategy(make_context(chat_template=SEPARATOR_TEMPLATE)) - score = build_question("level", "score", "", ["low", "high"], 26) - contents = [strategy._content("x" + "\t" * 71, score)] * 2 - read_request = strategy._read_request(None) - assert await strategy._render_spliced(read_request, contents, None) is None - - @pytest.mark.asyncio async def test_oversized_state_fails_like_full_render(make_context): strategy = NextTokenStrategy(make_context(max_model_len=64)) diff --git a/vllm/entrypoints/openai/decisions/strategies.py b/vllm/entrypoints/openai/decisions/strategies.py index ca2a9c4cc46b..13a8f219b771 100644 --- a/vllm/entrypoints/openai/decisions/strategies.py +++ b/vllm/entrypoints/openai/decisions/strategies.py @@ -46,8 +46,6 @@ from vllm.pooling_params import PoolingParams from vllm.renderers import ChatParams, TokenizeParams, merge_kwargs from vllm.renderers.chat_utils import ChatTemplateContentFormatOption -from vllm.renderers.hf import HfRenderer -from vllm.renderers.inputs import TokPrompt from vllm.renderers.inputs.preprocess import extract_prompt_components from vllm.renderers.online_renderer import OnlineRenderer from vllm.sampling_params import SamplingParams @@ -238,11 +236,7 @@ def cap_pixels(image: Image.Image, max_pixels: int) -> Image.Image: return image.resize(size, Image.Resampling.BICUBIC) -# Marks where a question goes in the rendered prompt. -_MARKER = "\ue000" - - -def _messages(content: Any, instructions: str | None) -> list[dict[str, Any]]: +def _messages(content: Any, instructions: str | None) -> list[Any]: messages = [{"role": "user", "content": content}] if instructions: messages.insert(0, {"role": "system", "content": instructions}) @@ -357,75 +351,49 @@ async def _render_spliced( contents: list[str], instructions: str | None, ) -> list[EngineInput] | None: - """Every prompt with the state tokenized once, or None to fall back.""" + """Every prompt with the shared start tokenized once, or None to fall back.""" ctx = self.context online_renderer = ctx.online_renderer renderer = online_renderer.renderer - if not isinstance(renderer, HfRenderer): - return None chat_params = online_renderer.build_chat_params( read_request, ctx.chat_template, ctx.chat_template_content_format, ctx.default_chat_template_kwargs, ) - tok_params = read_request.build_tok_params(online_renderer.model_config) - - # Split after the last "\n\n" the contents share, which ends the state. - cut = os.path.commonprefix(contents).rfind("\n\n") + 2 - if cut < 2: - return None - suffixes = [c[cut:] for c in contents] - # Whitespace after the cut can join a run longer than the head window. - if any(q[:1].isspace() for q in suffixes): - return None - - def render(content: str) -> str | None: - messages: list[Any] = _messages(content, instructions) - _, prompt = renderer.render_messages(messages, chat_params) - return cast(TextPrompt, prompt).get("prompt") - - def ids(prompts: Sequence[TokPrompt]) -> list[list[int]]: - return [list(cast(TokensPrompt, p)["prompt_token_ids"]) for p in prompts] - arrival_time = time.time() - probe = await renderer.run_chat_template_job_async( - lambda: render(contents[0][:cut] + _MARKER) - ) - if not isinstance(probe, str) or probe.count(_MARKER) != 1: - return None - prefix, tail = probe.split(_MARKER) - if not prefix.endswith("\n\n"): - return None - (prefix_ids,) = ids( - await renderer.tokenize_prompts_async( - [TextPrompt(prompt=prefix)], tok_params + texts = [] + for content in contents: + _, prompt = await renderer.render_messages_async( + _messages(content, instructions), chat_params ) - ) - head = prefix[-64:] - prompts = [prefix + q + tail for q in suffixes] - - def window_ids() -> list[list[int]] | None: - if any(render(c) != p for c, p in zip(contents, prompts)): + if not isinstance(text := prompt.get("prompt"), str): return None - texts = [head, *(head + q + tail for q in suffixes)] - return ids( - renderer.tokenize_prompts( - [TextPrompt(prompt=t) for t in texts], tok_params - ) - ) - - tokenized = await renderer.run_chat_template_job_async(window_ids) - if tokenized is None: + texts.append(text) + # Split after the last blank line all prompts share. + cut = os.path.commonprefix(texts).rfind("\n\n") + 2 + # Whitespace after the cut can join a run longer than the head window. + if cut < 2 or any(t[cut : cut + 1].isspace() for t in texts): return None - head_ids, *windows = tokenized + # Tokens of the head must not change when a question follows it. + head = texts[0][max(0, cut - 64) : cut] + tokenized = await renderer.tokenize_prompts_async( + [ + TextPrompt(prompt=t) + for t in (texts[0][:cut], head, *(head + text[cut:] for text in texts)) + ], + read_request.build_tok_params(online_renderer.model_config), + ) + prefix_ids, head_ids, *windows = ( + cast(TokensPrompt, p)["prompt_token_ids"] for p in tokenized + ) if any(w[: len(head_ids)] != head_ids for w in windows): return None engine_inputs = await online_renderer.preprocess_cmpl( read_request, [ - TokensPrompt(prompt=p, prompt_token_ids=prefix_ids + w[len(head_ids) :]) - for p, w in zip(prompts, windows) + TokensPrompt(prompt=t, prompt_token_ids=prefix_ids + w[len(head_ids) :]) + for t, w in zip(texts, windows) ], ) for engine_input in engine_inputs: diff --git a/vllm/renderers/hf.py b/vllm/renderers/hf.py index b935de214e18..48af41a7956c 100644 --- a/vllm/renderers/hf.py +++ b/vllm/renderers/hf.py @@ -8,9 +8,9 @@ import itertools import weakref from collections import defaultdict, deque -from collections.abc import Callable, Sequence +from collections.abc import Sequence from functools import lru_cache -from typing import TYPE_CHECKING, Any, Final, Literal, TypeVar, cast, overload +from typing import TYPE_CHECKING, Any, Final, Literal, cast, overload import jinja2 import jinja2.ext @@ -76,8 +76,6 @@ logger = init_logger(__name__) -_R = TypeVar("_R") - # Cache of `tokenizer -> prompt_embeds placeholder token ID`. Keyed by the # tokenizer object (not `id(tokenizer)`) so a fresh tokenizer landing at a @@ -1010,12 +1008,6 @@ async def _render_with_timeout(self, coro): f"VLLM_CHAT_TEMPLATE_RENDER_TIMEOUT to override." ) from None - async def run_chat_template_job_async(self, fn: Callable[[], _R]) -> _R: - """Run ``fn``, which renders chat templates, as one render pool job under - the chat template render timeout.""" - loop = asyncio.get_running_loop() - return await self._render_with_timeout(loop.run_in_executor(self._executor, fn)) - def _can_produce_offsets(self) -> bool: # Only fast tokenizers expose offset_mapping. return self.tokenizer is not None and self.tokenizer.is_fast From bd3ed958b010ae0def8f7a03978e24a5d19a8108 Mon Sep 17 00:00:00 2001 From: Simon Veitner Date: Thu, 15 Oct 2026 09:49:50 +0000 Subject: [PATCH 5/7] [Frontend] Run the decision server tests on the real renderer The decision server fixture and the state init test now build their engines with a vllm_config, which every strategy reads for the big decision gate. The server fixture renders through the real Qwen3 OnlineRenderer instead of a stub, so multi-question reads on both /v1/systemone and /v1/decisions take the spliced path. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Simon Veitner --- .../structured_decisions/test_strategies.py | 29 +++++++------------ 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/tests/entrypoints/generate/structured_decisions/test_strategies.py b/tests/entrypoints/generate/structured_decisions/test_strategies.py index 0d7e48521ad4..22efb80e42b9 100644 --- a/tests/entrypoints/generate/structured_decisions/test_strategies.py +++ b/tests/entrypoints/generate/structured_decisions/test_strategies.py @@ -63,7 +63,7 @@ ) from vllm.entrypoints.serve.utils.request_logger import RequestLogger from vllm.exceptions import VLLMValidationError -from vllm.inputs import mm_input, tokens_input +from vllm.inputs import mm_input from vllm.logprobs import Logprob from vllm.multimodal.media import MediaConnector, MediaWithBytes from vllm.outputs import ( @@ -179,7 +179,10 @@ def qwen(): def test_decisions_state_initializes(qwen): tokenizer, _ = qwen engine = SimpleNamespace( - model_config=model("Qwen3ForCausalLM"), renderer=None, input_processor=None + model_config=model("Qwen3ForCausalLM"), + vllm_config=VllmConfig(), + renderer=None, + input_processor=None, ) state = State() state.online_renderer = SimpleNamespace( @@ -1025,11 +1028,13 @@ def test_liquid_caps_images_like_the_checkpoint(): @pytest.fixture -def decision_server(qwen): +def decision_server(qwen, make_context): tokenizer, _ = qwen + context = make_context() class Engine: - model_config = SimpleNamespace(is_encoder_decoder=False) + model_config = context.engine_client.model_config + vllm_config = context.engine_client.vllm_config renderer = input_processor = None errored = False failure = None @@ -1070,21 +1075,7 @@ async def generate(self, prompt, params, request_id, **kwargs): finished=True, ) - class Renderer: - renderer = SimpleNamespace(get_tokenizer=lambda: tokenizer) - - async def preprocess_chat(self, request, messages, **kwargs): - await asyncio.sleep(0) - ids = tokenizer.apply_chat_template( - messages, - add_generation_prompt=True, - enable_thinking=False, - tokenize=True, - return_dict=False, - ) - return [], [tokens_input(ids)] - - engine, renderer = Engine(), Renderer() + engine, renderer = Engine(), context.online_renderer strategy = NextTokenStrategy(ReadContext(engine, renderer, None, "auto", {})) models = OpenAIServingModels( engine, [BaseModelPath(name="test-model", model_path="test-model")] From 1add18834ec3bb438b4a1fec3f749dd25042ec1e Mon Sep 17 00:00:00 2001 From: Simon Veitner Date: Thu, 15 Oct 2026 10:11:29 +0000 Subject: [PATCH 6/7] [Frontend] Gate and splice D3 decisions like next-token ones D3 reads now wait at the same big-decision gate as next-token reads, and tokenize the start their prompts share once. The gate counts a shared state once only when the prefix cache is on, since D3 runs without it and prefills the state for every question. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Simon Veitner --- .../structured_decisions/test_strategies.py | 84 ++++++++++++++++++- .../openai/decisions/strategies.py | 62 ++++++++++---- 2 files changed, 127 insertions(+), 19 deletions(-) diff --git a/tests/entrypoints/generate/structured_decisions/test_strategies.py b/tests/entrypoints/generate/structured_decisions/test_strategies.py index 22efb80e42b9..b5b2a8c90f7d 100644 --- a/tests/entrypoints/generate/structured_decisions/test_strategies.py +++ b/tests/entrypoints/generate/structured_decisions/test_strategies.py @@ -883,18 +883,22 @@ async def process_for_engine_async(prompt, arrival_time): } -@pytest.fixture -def d3(engine_and_renderer): - engine, online_renderer = engine_and_renderer +def use_d3_config(model_config): letters = [chr(c) for c in range(ord("A"), ord("Z") + 1)] codes = [*letters, *(a + b for a in letters for b in letters)][:255] - engine.model_config.hf_config.decision_config = { + model_config.hf_config.decision_config = { "format_version": 1, "prompt": "d3", "codes": codes, "token_ids": list(range(255)), "temperature": D3_TEMPERATURE, } + + +@pytest.fixture +def d3(engine_and_renderer): + engine, online_renderer = engine_and_renderer + use_d3_config(engine.model_config) tokenizer = online_renderer.renderer.get_tokenizer() async def encode(prompt, params, request_id, **kwargs): @@ -969,6 +973,78 @@ async def test_d3_takes_more_options_than_letters(d3): assert len(response.answers["many"]["probabilities"]) == 40 +async def d3_read(strategy, state, questions=QUESTIONS): + sent, gate_full = [], [] + + async def encode(prompt, params, request_id, **kwargs): + sent.append(prompt["prompt_token_ids"]) + gate_full.append(strategy._big_decision_gate.locked()) + yield PoolingRequestOutput( + request_id, PoolingOutput(torch.zeros(255)), sent[-1], 0, True + ) + + strategy.context.engine_client.encode.side_effect = encode + read = strategy.read( + questions, + None, + strategy.render_state(state), + request_id="r", + chat_template_kwargs=None, + lora_request=None, + priority=0, + ) + reads = await asyncio.wait_for(read, 30) + assert len(reads) == len(questions) + return sent, gate_full + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model", ["Qwen/Qwen3-0.6B", "Qwen/Qwen3.5-0.8B"]) +async def test_d3_spliced_prompts_match_full_encode(make_context, model): + context = make_context(model) + use_d3_config(context.engine_client.model_config) + strategy = D3Strategy(context) + long_state = "\n\n".join(f"Paragraph {i}: charged twice." for i in range(50)) + for name, state in {**SPLICE_STATES, "long": long_state}.items(): + sent, _ = await d3_read(strategy, state) + text = strategy.render_state(state) + full = [strategy._tokens(strategy._prompt(text, q, [])) for q in QUESTIONS] + assert sent == full, name + + tokenized = [] + tokens = strategy._tokens + strategy._tokens = lambda text: tokenized.append(text) or tokens(text) + await d3_read(strategy, long_state) + assert sum("Paragraph 0:" in text for text in tokenized) == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "prefix_caching,num_questions,budget_fraction,gated", + [ + # Without the prefix cache every read prefills the whole state. + (False, 3, 0.5, True), + (True, 3, 0.5, False), + (True, 3, 1.5, True), + (False, 1, 1.5, False), + ], +) +async def test_big_d3_decisions_take_the_gate( + make_context, prefix_caching, num_questions, budget_fraction, gated +): + context = make_context() + use_d3_config(context.engine_client.model_config) + vllm_config = context.engine_client.vllm_config + vllm_config.cache_config.enable_prefix_caching = prefix_caching + budget = vllm_config.scheduler_config.max_num_batched_tokens + strategy = D3Strategy(context) + strategy._big_decision_gate = asyncio.Semaphore(1) + state = "state " * int(budget * budget_fraction) + _, gate_full = await d3_read(strategy, state, QUESTIONS[:num_questions]) + assert gate_full == [gated] * num_questions + assert not strategy._big_decision_gate.locked() + + @pytest.mark.asyncio async def test_d3_rejects_instructions(d3): serving, engine, _ = d3 diff --git a/vllm/entrypoints/openai/decisions/strategies.py b/vllm/entrypoints/openai/decisions/strategies.py index 13a8f219b771..d72739da5777 100644 --- a/vllm/entrypoints/openai/decisions/strategies.py +++ b/vllm/entrypoints/openai/decisions/strategies.py @@ -65,6 +65,8 @@ logger = init_logger(__name__) _BIG_DECISIONS_IN_ENGINE = 2 +#: Characters before the splice point tokenized again with each question. +_SPLICE_HEAD_CHARS = 64 @dataclass @@ -145,6 +147,18 @@ def __init__(self, context: ReadContext): else scheduler_config.max_num_batched_tokens ) self._big_decision_gate = asyncio.Semaphore(_BIG_DECISIONS_IN_ENGINE) + cache_config = context.engine_client.vllm_config.cache_config + self._prefix_caching = cache_config.enable_prefix_caching + + def _turn( + self, prompts: list[list[int]] + ) -> contextlib.AbstractAsyncContextManager[Any]: + """The gate a decision waits at so big decisions take turns.""" + # With the prefix cache the engine computes the shared start once. + shared = len(os.path.commonprefix(prompts)) if self._prefix_caching else 0 + num_tokens = shared + sum(len(p) - shared for p in prompts) + big = len(prompts) > 1 and num_tokens > self._big_decision_tokens + return self._big_decision_gate if big else contextlib.nullcontext() @cached_property def _connector(self) -> MediaConnector: @@ -236,6 +250,16 @@ def cap_pixels(image: Image.Image, max_pixels: int) -> Image.Image: return image.resize(size, Image.Resampling.BICUBIC) +def _splice_point(texts: list[str]) -> int | None: + """Where the texts split into the start they share and their own ends.""" + # Split after the last blank line all texts share. + cut = os.path.commonprefix(texts).rfind("\n\n") + 2 + # Whitespace after the cut can join a run longer than the head window. + if cut < 2 or any(t[cut : cut + 1].isspace() for t in texts): + return None + return cut + + def _messages(content: Any, instructions: str | None) -> list[Any]: messages = [{"role": "user", "content": content}] if instructions: @@ -370,13 +394,10 @@ async def _render_spliced( if not isinstance(text := prompt.get("prompt"), str): return None texts.append(text) - # Split after the last blank line all prompts share. - cut = os.path.commonprefix(texts).rfind("\n\n") + 2 - # Whitespace after the cut can join a run longer than the head window. - if cut < 2 or any(t[cut : cut + 1].isspace() for t in texts): + if (cut := _splice_point(texts)) is None: return None # Tokens of the head must not change when a question follows it. - head = texts[0][max(0, cut - 64) : cut] + head = texts[0][max(0, cut - _SPLICE_HEAD_CHARS) : cut] tokenized = await renderer.tokenize_prompts_async( [ TextPrompt(prompt=t) @@ -452,11 +473,9 @@ async def read( ) # Decisions over one engine step take turns so short ones need not wait. - shared = len(os.path.commonprefix(prompts)) - num_tokens = shared + sum(len(p) - shared for p in prompts) + turn = self._turn(prompts) del prompts # Do not hold the id copies while waiting at the gate. - big = len(questions) > 1 and num_tokens > self._big_decision_tokens - async with self._big_decision_gate if big else contextlib.nullcontext(): + async with turn: label_reads = await next_token_label_reads( ctx.engine_client, engine_inputs, @@ -1111,6 +1130,18 @@ def _prompt(self, state: str, q: Question, media: Sequence[str]) -> str: def _tokens(self, text: str) -> list[int]: return self.tokenizer.encode(text, add_special_tokens=False) + def _tokens_spliced(self, texts: list[str]) -> list[list[int]]: + """Each text's tokens, with the start all texts share tokenized once.""" + if len(texts) > 1 and (cut := _splice_point(texts)) is not None: + # Tokens of the head must not change when a question follows it. + head = texts[0][max(0, cut - _SPLICE_HEAD_CHARS) : cut] + head_ids = self._tokens(head) + windows = [self._tokens(head + t[cut:]) for t in texts] + if all(w[: len(head_ids)] == head_ids for w in windows): + prefix_ids = self._tokens(texts[0][:cut]) + return [prefix_ids + w[len(head_ids) :] for w in windows] + return [self._tokens(t) for t in texts] + async def _render_media( self, prefix: str, media: Mapping[str, Sequence[str]], cache_salt: str | None ) -> MultiModalInput: @@ -1197,12 +1228,12 @@ async def read( "the questions' prompts do not share their media" ) media_input = await self._render_media(prefix, media or {}, cache_salt) - for p in prompts: - ids = media_input["prompt_token_ids"] + self._tokens(p[split:]) + for text_ids in self._tokens_spliced([p[split:] for p in prompts]): + ids = media_input["prompt_token_ids"] + text_ids engine_inputs.append({**media_input, "prompt_token_ids": ids}) prompt_ids.append(ids) else: - prompt_ids = [self._tokens(p) for p in prompts] + prompt_ids = self._tokens_spliced(prompts) engine_inputs = [ tokens_input(ids, cache_salt=cache_salt) for ids in prompt_ids ] @@ -1232,9 +1263,10 @@ async def read_one(n: int, engine_input: EngineInput) -> list[float]: ) return result.outputs.data.tolist() - logits = await asyncio.gather( - *(read_one(n, inputs) for n, inputs in enumerate(engine_inputs)) - ) + async with self._turn(prompt_ids): + logits = await asyncio.gather( + *(read_one(n, inputs) for n, inputs in enumerate(engine_inputs)) + ) reads = [] for q, ids, code_logits in zip(questions, prompt_ids, logits): From 811d101e7319c6da5a231464c1e2cee936d69fec Mon Sep 17 00:00:00 2001 From: Simon Veitner Date: Thu, 15 Oct 2026 10:19:52 +0000 Subject: [PATCH 7/7] [Frontend] Read the decision gate's config when a decision arrives The strategy constructor no longer reads the engine client's config, so a strategy can be built from a context that has no engine yet. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Simon Veitner --- .../openai/decisions/strategies.py | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/vllm/entrypoints/openai/decisions/strategies.py b/vllm/entrypoints/openai/decisions/strategies.py index d72739da5777..2f275b1554b4 100644 --- a/vllm/entrypoints/openai/decisions/strategies.py +++ b/vllm/entrypoints/openai/decisions/strategies.py @@ -139,26 +139,27 @@ class ReadStrategy(ABC): def __init__(self, context: ReadContext): self.context = context - scheduler_config = context.engine_client.vllm_config.scheduler_config - # The priority policy orders requests in the engine instead. - self._big_decision_tokens = ( - math.inf - if scheduler_config.policy == "priority" - else scheduler_config.max_num_batched_tokens - ) self._big_decision_gate = asyncio.Semaphore(_BIG_DECISIONS_IN_ENGINE) - cache_config = context.engine_client.vllm_config.cache_config - self._prefix_caching = cache_config.enable_prefix_caching def _turn( self, prompts: list[list[int]] ) -> contextlib.AbstractAsyncContextManager[Any]: """The gate a decision waits at so big decisions take turns.""" + vllm_config = self.context.engine_client.vllm_config + scheduler_config = vllm_config.scheduler_config + # The priority policy orders requests in the engine instead. + if len(prompts) < 2 or scheduler_config.policy == "priority": + return contextlib.nullcontext() # With the prefix cache the engine computes the shared start once. - shared = len(os.path.commonprefix(prompts)) if self._prefix_caching else 0 + shared = ( + len(os.path.commonprefix(prompts)) + if vllm_config.cache_config.enable_prefix_caching + else 0 + ) num_tokens = shared + sum(len(p) - shared for p in prompts) - big = len(prompts) > 1 and num_tokens > self._big_decision_tokens - return self._big_decision_gate if big else contextlib.nullcontext() + if num_tokens > scheduler_config.max_num_batched_tokens: + return self._big_decision_gate + return contextlib.nullcontext() @cached_property def _connector(self) -> MediaConnector: