[Structured Outputs] Make auto pick one backend per server instead of per request

docs/features/structured_outputs.md

@@ -34,10 +34,12 @@ You can see the complete list of supported parameters on the [OpenAI-Compatible
 Structured outputs are supported by default in the OpenAI-Compatible Server. You
 may choose to specify the backend to use by setting the
 `--structured-outputs-config.backend` flag to `vllm serve`. The default backend is `auto`,
-which will try to choose an appropriate backend based on the details of the
-request. You may also choose a specific backend, along with
-some options. A full set of options is available in the `vllm serve --help`
-text.
+which selects one backend for the whole server from its tokenizer: `guidance` for
+Tekken-based Mistral tokenizers and `xgrammar` otherwise. It does not fall back to
+another backend per request, so a constraint the selected backend cannot enforce
+is rejected; choose a backend explicitly to use features only it supports. You may
+also choose a specific backend, along with some options. A full set of options is
+available in the `vllm serve --help` text.
 
 Now let's see an example for each of the cases, starting with the `choice`, as it's the easiest one:
 

tests/entrypoints/llm/test_struct_output_generate.py

@@ -21,6 +21,7 @@
     SamplingParams,
     StructuredOutputsParams,
 )
+from vllm.utils.mistral import is_mistral_tokenizer
 
 SAMPLE_REGEX = (
     r"((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.){3}"
@@ -835,8 +836,14 @@ def test_structured_output_auto_mode(
             "that fits this schema: "
             f"{unsupported_json_schema}. Make the response as short as possible."
         )
-        # This would fail with the default of "xgrammar", but in "auto"
-        # we will handle fallback automatically.
+        tokenizer = runner.llm.get_tokenizer()
+        if not (is_mistral_tokenizer(tokenizer) and tokenizer.is_tekken):
+            # `auto` selects xgrammar from the tokenizer and does not fall back
+            # to another backend for a schema xgrammar cannot enforce.
+            with pytest.raises(VLLMValidationError, match="not supported by xgrammar"):
+                runner.llm.generate(prompts, sampling_params=sampling_params)
+            return
+        # Tekken Mistral tokenizers select guidance, which enforces the schema.
         outputs = runner.llm.generate(
             prompts, sampling_params=sampling_params, use_tqdm=True
         )

tests/v1/structured_output/test_validation.py

@@ -311,54 +311,41 @@ def test_unhandled_spec_is_a_validation_error(validate):
 
 
 @pytest.mark.parametrize(
-    "schema, expected_backend",
+    "schema",
     [
         # multipleOf is unsupported by xgrammar.
-        (
-            {
-                "type": "object",
-                "properties": {"n": {"type": "integer", "multipleOf": 2}},
-            },
-            "guidance",
-        ),
-        (
-            {
-                "type": "object",
-                "properties": {"n": {"type": ["number", "null"], "multipleOf": 3}},
-            },
-            "guidance",
-        ),
-        (
-            {
-                "type": ["string", "null"],
-                "pattern": "^a+$",
-                "maxLength": 2,
-            },
-            "guidance",
-        ),
-        # patternProperties + properties is also unsupported by guidance.
-        (
-            {
-                "type": "object",
-                "properties": {"a": {"type": "string"}},
-                "patternProperties": {"^a$": {"type": "string"}},
-            },
-            "outlines",
-        ),
+        {
+            "type": "object",
+            "properties": {"n": {"type": "integer", "multipleOf": 2}},
+        },
+        # patternProperties + properties is unsupported by xgrammar and guidance.
+        {
+            "type": "object",
+            "properties": {"a": {"type": "string"}},
+            "patternProperties": {"^a$": {"type": "string"}},
+        },
     ],
 )
-def test_auto_backend_falls_back_on_unsupported_schema(schema, expected_backend):
-    """`auto` falls back on rejection, so it must catch what the validators raise."""
-    params = SamplingParams(structured_outputs=StructuredOutputsParams(json=schema))
-    # trick to create tokenizer that guidance backend accepts
+def test_auto_backend_rejects_schema_without_fallback(schema):
+    """`auto` resolves from the tokenizer, not the request: EngineCore compiles
+    every request with one backend, so a per-request fallback would make the
+    outcome depend on which request reached the engine first."""
     tokenizer = object.__new__(TokenizersBackend)
+    assert _validate_auto(JSON_SCHEMA, tokenizer)._backend == "xgrammar"
+    with pytest.raises(VLLMValidationError, match="not supported by xgrammar"):
+        _validate_auto(schema, tokenizer)
+
+
+def _validate_auto(schema, tokenizer) -> StructuredOutputsParams:
+    params = SamplingParams(structured_outputs=StructuredOutputsParams(json=schema))
     params._validate_structured_outputs(
         _StubModelConfig(is_diffusion=False),
         StructuredOutputsConfig(backend="auto"),
         tokenizer=tokenizer,
     )
     assert params.structured_outputs is not None
-    assert params.structured_outputs._backend == expected_backend
+    assert params.structured_outputs._backend_was_auto
+    return params.structured_outputs
 
 
 # ================================================
@@ -450,6 +437,18 @@ def test_supported_tokenizer(self, monkeypatch, tokenizer_id, patch_mistral):
             params = self._validate_guidance(self._make_tokenizer(tokenizer_id))
             assert params.structured_outputs._backend == "guidance"
 
+        def test_auto_selects_guidance_for_tekken_mistral(self, monkeypatch):
+            monkeypatch.setattr(
+                mistral_tokenizers, "MistralTokenizer", self._StubVllmMistralTokenizer
+            )
+            monkeypatch.setattr(
+                backend_guidance,
+                "validate_guidance_grammar",
+                lambda *_args, **_kwargs: None,
+            )
+            tokenizer = self._make_tokenizer("vllm-mistral-tekken")
+            assert _validate_auto(JSON_SCHEMA, tokenizer)._backend == "guidance"
+
     class TestPR49625Regression:
         def test_manager_rejects_mixed_structured_output_backends(self, monkeypatch):
             xgrammar_backend = Mock()

vllm/config/structured_outputs.py

@@ -19,9 +19,10 @@ class StructuredOutputsConfig:
 
     backend: StructuredOutputsBackend = "auto"
     """Which engine will be used for structured outputs (e.g. JSON schema,
-    regex, etc) by default. With "auto", we will make opinionated choices
-    based on request contents and what the backend libraries currently support,
-    so the behavior is subject to change in each release."""
+    regex, etc) by default. With "auto", one backend is chosen for every request
+    from the tokenizer (guidance for Tekken-based Mistral tokenizers, xgrammar
+    otherwise), without falling back per request, and the choice is subject to
+    change in each release."""
     disable_any_whitespace: bool = False
     """If `True`, json output will always be compact without any whitespace.
     If `False`, the model may generate whitespace between JSON fields,

vllm/sampling_params.py

@@ -17,6 +17,7 @@
 
 import vllm.envs as envs
 from vllm.config import ModelConfig, SpeculativeConfig, StructuredOutputsConfig
+from vllm.config.structured_outputs import StructuredOutputsBackend
 from vllm.exceptions import VLLMValidationError
 from vllm.logger import init_logger
 from vllm.tokenizers import TokenizerLike
@@ -268,6 +269,21 @@ def _get_llg_tokenizer(tokenizer: TokenizerLike) -> Any:
     return tokenizer.llg_tokenizer if is_mistral_tokenizer(tokenizer) else None
 
 
+def _auto_structured_outputs_backend(
+    tokenizer: TokenizerLike,
+) -> StructuredOutputsBackend:
+    """Resolve `auto` from the tokenizer alone, so every request on a server
+    selects the same backend: EngineCore compiles all requests with one."""
+    # The Mistral tool parser generates Lark grammars for Tekken Mistral
+    # tokenizers. llguidance takes the tokenizer from mistral-common and never
+    # lets a regex match a special token. The xgrammar backend declares no
+    # special tokens for Mistral tokenizers, so a regex like `.` can match
+    # `[TOOL_CALLS]` or `[INST]` as plain text.
+    if is_mistral_tokenizer(tokenizer) and tokenizer.is_tekken:
+        return "guidance"
+    return "xgrammar"
+
+
 class SamplingParams(
     PydanticMsgspecMixin,
     msgspec.Struct,
@@ -1321,7 +1337,6 @@ def _validate_structured_outputs(
 
         from vllm.v1.structured_output.backend_guidance import (
             GuidanceBackend,
-            has_guidance_unsupported_json_features,
             validate_guidance_grammar,
         )
         from vllm.v1.structured_output.backend_lm_format_enforcer import (
@@ -1331,7 +1346,12 @@ def _validate_structured_outputs(
             validate_structured_output_request_outlines,
         )
         from vllm.v1.structured_output.backend_xgrammar import validate_xgrammar_grammar
-        from vllm.v1.structured_output.utils import grammar_is_likely_lark
+
+        if backend == "auto":
+            backend = _auto_structured_outputs_backend(tokenizer)
+            self.structured_outputs._backend = backend
+            # Remember that this backend was set automatically
+            self.structured_outputs._backend_was_auto = True
 
         if backend.startswith("xgrammar"):
             # xgrammar with no fallback
@@ -1371,75 +1391,6 @@ def _validate_structured_outputs(
                     "backends or tokenizer_mode='hf' instead."
                 )
             validate_structured_output_request_lm_format_enforcer(self)
-        elif (
-            backend == "auto"
-            and is_mistral_tokenizer(tokenizer)
-            and tokenizer.is_tekken
-            and self.structured_outputs.grammar
-            and grammar_is_likely_lark(self.structured_outputs.grammar)
-        ):
-            # Lark grammars for Tekken Mistral tokenizers, including the ones
-            # the Mistral tool parser generates, go to guidance. llguidance
-            # takes the tokenizer from mistral-common and never lets a regex
-            # match a special token. The xgrammar backend declares no special
-            # tokens for Mistral tokenizers, so a regex like `.` can match
-            # `[TOOL_CALLS]` or `[INST]` as plain text.
-            validate_guidance_grammar(self, tokenizer=_get_llg_tokenizer(tokenizer))
-            self.structured_outputs._backend = "guidance"
-            self.structured_outputs._backend_was_auto = True
-        else:
-            # NOTE: backend must be "auto" here, because we have
-            # checked supported_backends above.
-            # In this mode, we set opinionated defaults based on what we think
-            # will satisfy the most use cases without having to worry about
-            # this setting. We include fallback behavior here, but not with any
-            # other setting where a specific backend was specified.
-            try:
-                validate_xgrammar_grammar(self)
-                self.structured_outputs._backend = "xgrammar"
-            except VLLMValidationError:
-                # The request either failed validation
-                # or includes some jsonschema feature(s) that
-                # are not supported in xgrammar.
-
-                skip_guidance = _is_non_tekken_mistral(tokenizer)
-
-                # Check if schema has features unsupported by guidance
-                so_params = self.structured_outputs
-                if not skip_guidance and so_params.json:
-                    if isinstance(so_params.json, str):
-                        try:
-                            schema = json_mod.loads(so_params.json)
-                        except json_mod.JSONDecodeError as e:
-                            raise VLLMValidationError(
-                                "Invalid JSON grammar specification."
-                            ) from e
-                    else:
-                        schema = so_params.json
-                    skip_guidance = has_guidance_unsupported_json_features(schema)
-
-                if skip_guidance:
-                    # Fall back to outlines for non-Tekken Mistral tokenizers or
-                    # schemas containing features unsupported by guidance.
-                    validate_structured_output_request_outlines(self)
-                    self.structured_outputs._backend = "outlines"
-                else:
-                    # Fall back to guidance by default.
-                    if not GuidanceBackend.is_tokenizer_supported(tokenizer):
-                        raise VLLMValidationError(
-                            "No compatible structured output backend was found for "
-                            "this request. XGrammar rejected the constraint, and "
-                            "Guidance does not support this tokenizer. Configure a "
-                            "fast tokenizer or select a compatible backend explicitly."
-                        ) from None
-                    validate_guidance_grammar(
-                        self,
-                        tokenizer=_get_llg_tokenizer(tokenizer),
-                    )
-                    self.structured_outputs._backend = "guidance"
-            # Remember that this backend was set automatically
-            self.structured_outputs._backend_was_auto = True
-
         # Run post-init validation. This is also important to ensure subsequent
         # roundtrip serialization/deserialization won't fail.
         self.structured_outputs.__post_init__()

vllm/v1/structured_output/backend_guidance.py

@@ -90,32 +90,6 @@ def _walk_json_for_additional_properties(data: object):
                 _walk_json_for_additional_properties(item)
 
 
-def has_guidance_unsupported_json_features(schema: dict[str, Any]) -> bool:
-    """Check if JSON schema contains features unsupported by guidance/llguidance."""
-
-    def check_object(obj: dict[str, Any]) -> bool:
-        if not isinstance(obj, dict):
-            return False
-
-        # patternProperties is not supported by llguidance
-        if "patternProperties" in obj:
-            return True
-
-        # Recursively check all nested objects and arrays
-        for value in obj.values():
-            if isinstance(value, dict):
-                if check_object(value):
-                    return True
-            elif isinstance(value, list):
-                for item in value:
-                    if isinstance(item, dict) and check_object(item):
-                        return True
-
-        return False
-
-    return check_object(schema)
-
-
 def process_for_additional_properties(
     guide_json: str | dict[str, Any],
 ) -> dict[str, Any]: