[Bugfix] Treat max_tokens as output upper bound, not input-space reservation

tests/entrypoints/openai/chat_completion/test_serving_chat.py

@@ -1684,7 +1684,14 @@ def test_serving_chat_starts_without_tokenizer():
 @pytest.mark.asyncio
 async def test_serving_chat_mistral_token_ids_prompt_is_validated():
     """Regression test: when the Mistral tokenizer path returns token IDs
-    directly, we must still apply input length + max_tokens validation.
+    directly, we must still apply input length validation and clamp the
+    generation budget to the context the prompt leaves over.
+
+    `max_tokens` is an upper bound on generation rather than a reservation off
+    the context window, so a 95-token prompt against `max_model_len=100` is
+    accepted even though the client asked for 10 output tokens. Genuine
+    overflow is covered by
+    test_serving_chat_mistral_token_ids_prompt_too_long_is_rejected.
     """
     mock_engine = MagicMock(spec=AsyncLLM)
     mock_engine.errored = False
@@ -1696,9 +1703,8 @@ async def test_serving_chat_mistral_token_ids_prompt_is_validated():
         MockVllmConfig(mock_engine.model_config, parallel_config=MockParallelConfig()),
         tokenizer=mock_tokenizer,
     )
-    # Force the Mistral chat template renderer to return token IDs.
-    # Choose a prompt length that is < max_model_len, but large enough that
-    # adding max_tokens should exceed the model context window.
+    # Force the Mistral chat template renderer to return token IDs. The prompt
+    # fits the context on its own; only the naive prompt+output sum overflows.
     mock_renderer.render_messages_async = AsyncMock(
         return_value=(
             [],
@@ -1715,8 +1721,12 @@ async def test_serving_chat_mistral_token_ids_prompt_is_validated():
         max_tokens=10,
     )
 
-    with pytest.raises(VLLMValidationError):
-        await serving_chat.create_chat_completion(req)
+    # Must not raise: the prompt alone fits within max_model_len.
+    await serving_chat.create_chat_completion(req)
+
+    # get_max_tokens() clamps the sampling budget to the 5 tokens left over.
+    sampling_params = mock_engine.generate.call_args.args[1]
+    assert 1 <= sampling_params.max_tokens <= 5
 
 
 @pytest.mark.asyncio

tests/entrypoints/openai/responses/test_serving_responses.py

@@ -515,7 +515,11 @@ async def test_online_renderer_applies_responses_token_budget_to_harmony_prompt(
     result = await renderer.render_responses(request)
 
     assert not isinstance(result, ErrorResponse)
-    assert result.engine_input["prompt_token_ids"] == [4, 5, 6]
+    # `max_tokens` is an output upper bound rather than a reservation off the
+    # context window, so the prompt budget is the full max_model_len minus the
+    # one token held back for output: 6 tokens truncate to 4, not to
+    # max_model_len - max_output_tokens.
+    assert result.engine_input["prompt_token_ids"] == [3, 4, 5, 6]
 
 
 @pytest.mark.asyncio

tests/entrypoints/pooling/embed/test_protocol.py

@@ -4,12 +4,15 @@
 underlying packing helpers, plus Cohere-specific serving helpers."""
 
 import struct
+from unittest.mock import Mock
 
 import numpy as np
 import pybase64 as base64
 import pytest
 
+from vllm.config import ModelConfig, PoolerConfig
 from vllm.entrypoints.pooling.embed.protocol import (
+    EmbeddingCompletionRequest,
     build_typed_embeddings,
 )
 
@@ -162,3 +165,59 @@ def test_subset_types(self, sample_embeddings: list[list[float]]):
     def test_unknown_type_ignored(self, sample_embeddings: list[list[float]]):
         result = build_typed_embeddings(sample_embeddings, ["float", "unknown_type"])
         assert result.float is not None
+
+
+def _embedding_model_config(
+    *,
+    max_model_len: int = 128,
+    max_embed_len: int | None = None,
+    enable_chunked_processing: bool = False,
+) -> Mock:
+    model_config = Mock(spec=ModelConfig)
+    model_config.max_model_len = max_model_len
+    model_config.encoder_config = None
+    model_config.pooler_config = PoolerConfig(
+        seq_pooling_type="CLS",
+        max_embed_len=max_embed_len,
+        enable_chunked_processing=enable_chunked_processing,
+    )
+    return model_config
+
+
+class TestMaxEmbedLenCapsInput:
+    """``build_tok_params`` used to enforce ``max_embed_len`` indirectly, by
+    setting ``max_output_tokens = max_model_len - max_embed_len`` and relying on
+    ``max_input_tokens == max_total_tokens - max_output_tokens``. The cap is now
+    expressed directly as ``max_total_tokens``."""
+
+    def test_non_chunked_max_embed_len_caps_input(self):
+        cfg = _embedding_model_config(max_model_len=128, max_embed_len=64)
+
+        tok_params = EmbeddingCompletionRequest(model="m", input="hi").build_tok_params(
+            cfg
+        )
+
+        assert tok_params.max_input_tokens == 64
+        assert tok_params.max_output_tokens == 0
+
+    def test_chunked_max_embed_len_caps_input(self):
+        cfg = _embedding_model_config(
+            max_model_len=128, max_embed_len=64, enable_chunked_processing=True
+        )
+
+        tok_params = EmbeddingCompletionRequest(model="m", input="hi").build_tok_params(
+            cfg
+        )
+
+        assert tok_params.max_input_tokens == 64
+        assert tok_params.max_output_tokens == 0
+
+    def test_no_max_embed_len_falls_back_to_model_len(self):
+        cfg = _embedding_model_config(max_model_len=128)
+
+        tok_params = EmbeddingCompletionRequest(model="m", input="hi").build_tok_params(
+            cfg
+        )
+
+        assert tok_params.max_input_tokens == 128
+        assert tok_params.max_output_tokens == 0

tests/models/language/pooling/test_truncation_control.py

@@ -55,6 +55,8 @@ def test_max_truncation_size(vllm_model):
 
     prompt_tokens = vllm_output[0].prompt_token_ids
 
+    # Pooling requests emit an embedding rather than generated tokens, so -1
+    # truncates to the full context length.
     assert len(prompt_tokens) == max_model_len
 
 

tests/renderers/test_completions.py

@@ -247,6 +247,8 @@ def test_neg_truncation(self):
         )
 
         assert len(results) == 1
+        # max_output_tokens defaults to 0, so there is no output budget to
+        # reserve and -1 truncates to the full max_total_tokens.
         assert len(results[0]["prompt_token_ids"]) == 100  # max_total_tokens
 
     def test_truncation_left(self):
@@ -505,6 +507,122 @@ def test_explicit_side_text_pretokenization_guard(self):
         assert renderer.tokenizer._captured_text_len <= 100
 
 
+class TestMaxTokensNotReservation:
+    """Regression tests for GitHub issue #42474.
+
+    ``max_tokens`` must be treated as an upper bound on generation length, not
+    as a hard reservation that is subtracted from the available input space.
+    """
+
+    def test_large_max_tokens_does_not_eat_prompt_budget(self):
+        """Prompt fits in context even when max_tokens is large."""
+        renderer = _build_renderer(MockModelConfig())
+
+        # 50-token prompt with a 100-token context window and max_tokens=60.
+        # Before the fix: max_input_tokens = 100 - 60 = 40 → 50 > 40 → error.
+        # After the fix:  max_input_tokens = 100            → 50 ≤ 100 → OK.
+        prompts = renderer.render_prompts(
+            _preprocess_prompt(renderer.model_config, "x" * 50)
+        )
+        results = renderer.tokenize_prompts(
+            prompts,
+            TokenizeParams(max_total_tokens=100, max_output_tokens=60),
+        )
+        assert len(results) == 1
+        assert len(results[0]["prompt_token_ids"]) == 50
+
+    def test_max_tokens_zero_still_rejects_oversized_prompt(self):
+        """Genuine prompt overflow is still caught regardless of max_tokens."""
+        renderer = _build_renderer(MockModelConfig())
+
+        # 150-token prompt with 100-token context → should still fail.
+        prompts = renderer.render_prompts(
+            _preprocess_prompt(renderer.model_config, "x" * 150)
+        )
+        with pytest.raises(VLLMValidationError, match="maximum context length"):
+            renderer.tokenize_prompts(
+                prompts,
+                TokenizeParams(max_total_tokens=100, max_output_tokens=0),
+            )
+
+    def test_negative_truncation_uses_max_total(self):
+        """truncate_prompt_tokens=-1 reserves one output token."""
+        renderer = _build_renderer(MockModelConfig())
+
+        prompts = renderer.render_prompts(
+            _preprocess_prompt(renderer.model_config, "x" * 200)
+        )
+        results = renderer.tokenize_prompts(
+            prompts,
+            TokenizeParams(
+                max_total_tokens=100,
+                max_output_tokens=40,
+                truncate_prompt_tokens=-1,
+            ),
+        )
+        # -1 maps to max_total_tokens (100) rather than the old
+        # max_total - max_output (60), minus the one token held back so
+        # truncation cannot leave a zero-token output budget.
+        assert len(results[0]["prompt_token_ids"]) == 99
+
+    def test_negative_pad_uses_max_total(self):
+        """pad_prompt_tokens=-1 pads to max_total_tokens, not max_input."""
+        renderer = _build_renderer(MockModelConfig())
+        pad_id = renderer.tokenizer.pad_token_id
+
+        prompts = renderer.render_prompts(
+            _preprocess_prompt(renderer.model_config, "x" * 30)
+        )
+        results = renderer.tokenize_prompts(
+            prompts,
+            TokenizeParams(
+                max_total_tokens=100,
+                max_output_tokens=40,
+                pad_prompt_tokens=-1,
+            ),
+        )
+        # -1 should map to max_total_tokens (100), not max_total - max_output (60).
+        assert results[0]["prompt_token_ids"] == list(range(30)) + [pad_id] * 70
+
+    def test_max_tokens_exceeds_context_is_still_clamped_by_api(self):
+        """Renderer allows the prompt; API layer clamps max_tokens later."""
+        renderer = _build_renderer(MockModelConfig())
+
+        # Simulates the reported scenario: max_tokens > context - prompt_length.
+        # Renderer should accept the 40-token prompt; get_max_tokens() clamps
+        # sampling max_tokens to 60 at the entrypoint layer.
+        prompts = renderer.render_prompts(
+            _preprocess_prompt(renderer.model_config, "x" * 40)
+        )
+        results = renderer.tokenize_prompts(
+            prompts,
+            TokenizeParams(max_total_tokens=100, max_output_tokens=80),
+        )
+        assert len(results) == 1
+        assert len(results[0]["prompt_token_ids"]) == 40
+
+    def test_with_kwargs_preserves_output_budget(self):
+        """with_kwargs without an explicit max_length keeps the output budget.
+
+        Under the old semantics `with_kwargs()` recomputed
+        `max_output_tokens = max_total - max_input`, which (after this fix)
+        would have silently reset a real output budget to zero.
+        """
+        tok_params = TokenizeParams(max_total_tokens=100, max_output_tokens=60)
+        chained = tok_params.with_kwargs(truncation_side="left")
+
+        assert chained.max_output_tokens == 60
+        assert chained.max_input_tokens == 100
+
+    def test_with_kwargs_explicit_max_length_sets_budget(self):
+        """An explicit max_length still re-derives the output budget."""
+        tok_params = TokenizeParams(max_total_tokens=100, max_output_tokens=60)
+        chained = tok_params.with_kwargs(max_length=80)
+
+        assert chained.max_output_tokens == 20
+        assert chained.max_input_tokens == 100
+
+
 class TestRenderEmbedPrompt:
     def _create_test_embed_bytes(self, tensor: torch.Tensor) -> bytes:
         """Helper to create base64-encoded tensor bytes."""

vllm/entrypoints/pooling/base/protocol.py

@@ -183,15 +183,15 @@ def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
             if pooler_config.enable_chunked_processing:
                 max_total_tokens = pooler_config.max_embed_len
             else:
-                max_embed_len = pooler_config.max_embed_len or default_max_total_tokens
-                max_output_tokens = default_max_total_tokens - max_embed_len
+                max_total_tokens = (
+                    pooler_config.max_embed_len or default_max_total_tokens
+                )
 
         tok_params = self._build_pooling_tok_params(
             model_config,
             add_special_tokens=self.add_special_tokens,
             max_total_tokens=max_total_tokens,
             max_output_tokens=max_output_tokens,
-            max_output_tokens_param="max_model_len - max_embed_len",
         )
 
         if (