[Bugfix][Structured Output] Stop XGrammar token batches at termination

tests/v1/spec_decode/test_mtp_structured_output.py

@@ -262,6 +262,54 @@ def test_validate_tokens_then_bitmask_round_trip(backend):
     assert not grammar.is_terminated()
 
 
+def test_xgrammar_accept_tokens_stops_at_termination(capfd):
+    """Tokens after a terminating EOS do not reach the matcher."""
+    tokenizer, _, request, prompt = _make_manager_and_request("xgrammar")
+    grammar = request.structured_output_request.grammar
+
+    assert grammar.accept_tokens(request.request_id, prompt)
+
+    eos = tokenizer.eos_token_id
+    trailing = tokenizer.encode("\n")[0]
+    processed_before = grammar.num_processed_tokens
+
+    assert grammar.accept_tokens(request.request_id, [eos, trailing])
+    assert grammar.is_terminated()
+    assert grammar.num_processed_tokens == processed_before + 1
+    assert "trying to accept new token" not in capfd.readouterr().err
+
+    processed_after_eos = grammar.num_processed_tokens
+    assert grammar.accept_tokens(request.request_id, [trailing])
+    assert grammar.num_processed_tokens == processed_after_eos
+    assert "trying to accept new token" not in capfd.readouterr().err
+
+    grammar.reset()
+    assert not grammar.is_terminated()
+    assert grammar.num_processed_tokens == 0
+
+
+def test_xgrammar_validate_tokens_stops_at_termination(capfd):
+    """Validation rolls back after reaching a terminating EOS."""
+    tokenizer, _, request, prompt = _make_manager_and_request("xgrammar")
+    grammar = request.structured_output_request.grammar
+
+    assert grammar.accept_tokens(request.request_id, prompt)
+
+    eos = tokenizer.eos_token_id
+    trailing = tokenizer.encode("\n")[0]
+
+    assert grammar.validate_tokens([eos, trailing]) == [eos]
+    assert "trying to accept new token" not in capfd.readouterr().err
+    # Check matcher state directly to verify validation rolled it back.
+    assert not grammar.matcher.is_terminated()
+
+    assert grammar.accept_tokens(request.request_id, [eos])
+    assert grammar.is_terminated()
+
+    assert grammar.validate_tokens([trailing]) == []
+    assert "trying to accept new token" not in capfd.readouterr().err
+
+
 class _MarkerReasoner:
     """Stub reasoner whose reasoning-end marker is a single fixed token."""
 

vllm/v1/structured_output/backend_xgrammar.py

@@ -157,11 +157,12 @@ class XgrammarGrammar(StructuredOutputGrammar):
     def accept_tokens(self, request_id: str, tokens: list[int]) -> bool:
         """Accepts a list of tokens and advances the FSM.
 
-        Returns True if the FSM was advanced successfully.
-        Returns False if the FSM failed to advance.
+        Returns True if all grammar-constrained tokens were accepted.
+        Tokens after termination are ignored. Returns False if the FSM
+        failed to advance.
         """
         if self._is_terminated:
-            return False
+            return True
         for token in tokens:
             if not self.matcher.accept_token(token):
                 logger.error(
@@ -172,7 +173,9 @@ def accept_tokens(self, request_id: str, tokens: list[int]) -> bool:
                 )
                 return False
             self.num_processed_tokens += 1
-        self._is_terminated = self.matcher.is_terminated()
+            self._is_terminated = self.matcher.is_terminated()
+            if self._is_terminated:
+                break
         return True
 
     def validate_tokens(self, tokens: list[int]) -> list[int]:
@@ -181,10 +184,15 @@ def validate_tokens(self, tokens: list[int]) -> list[int]:
 
         Returns the prefix list of tokens that are accepted by the FSM.
         """
+        if self._is_terminated:
+            return []
+
         accepted_tokens = []
         for token in tokens:
             if self.matcher.accept_token(token):
                 accepted_tokens.append(token)
+                if self.matcher.is_terminated():
+                    break
             else:
                 break
         if len(accepted_tokens) > 0:
@@ -204,8 +212,9 @@ def is_terminated(self) -> bool:
         return self._is_terminated
 
     def reset(self):
-        self.num_processed_tokens = 0
         self.matcher.reset()
+        self.num_processed_tokens = 0
+        self._is_terminated = False
 
 
 # cf https://github.com/mlc-ai/xgrammar/blob/a32ac892676d2eedc0327416105b9b06edfb94b2/cpp/json_schema_converter.cc