[Bugfix][Core] Validate prompt token ids when skip_tokenizer_init is set

tests/entrypoints/llm/test_prompt_validation.py

@@ -23,6 +23,20 @@ def test_out_of_vocab_token(vllm_runner):
         runner.llm.generate({"prompt_token_ids": [999999]})
 
 
+def test_out_of_vocab_token_skip_tokenizer_init(vllm_runner):
+    # Without a tokenizer the model vocab size is the only bound; an
+    # out-of-range id must not reach the embedding lookup.
+    with (
+        vllm_runner(
+            "openai-community/gpt2",
+            enforce_eager=True,
+            skip_tokenizer_init=True,
+        ) as runner,
+        pytest.raises(VLLMValidationError, match="out of vocabulary"),
+    ):
+        runner.llm.generate({"prompt_token_ids": [999999]})
+
+
 def test_require_mm_embeds(vllm_runner):
     with (
         vllm_runner(

vllm/renderers/base.py

@@ -455,10 +455,10 @@ def validate_token_ids(
     ) -> None:
         """Raise VLLMValidationError if a token id is out of vocabulary.
 
-        Skipped when the tokenizer is not initialized.
+        Without a tokenizer, the model vocab size is the only bound.
         """
         tokenizer = self.tokenizer
-        if not token_ids or tokenizer is None:
+        if not token_ids:
             return
 
         max_input_id = max(token_ids)
@@ -474,15 +474,17 @@ def validate_token_ids(
 
         # Here we take the max of the two to determine if a token id is
         # truly out-of-vocabulary.
-        model_vocab_size = self.model_config.get_vocab_size()
+        max_token_id = self.model_config.get_vocab_size() - 1
+        if tokenizer is not None:
+            max_token_id = max(tokenizer.max_token_id, max_token_id)
         # A negative id is out of vocabulary just like an over-large one,
         # but is not caught by the upper-bound check below. Reject it here
         # so it is not used as an embedding index downstream.
         if min_input_id < 0:
             raise VLLMValidationError(
                 f"Token id {min_input_id} is out of vocabulary", parameter=parameter
             )
-        if max_input_id > max(tokenizer.max_token_id, model_vocab_size - 1):
+        if max_input_id > max_token_id:
             raise VLLMValidationError(
                 f"Token id {max_input_id} is out of vocabulary", parameter=parameter
             )