From c866150ddc508c4f11f7b0a20b9e694be137e10a Mon Sep 17 00:00:00 2001 From: Bowen Date: Fri, 11 Sep 2026 21:05:50 -0700 Subject: [PATCH] [Bugfix] Make xgrammar an import-time optional dependency vllm/parser/harmony.py and vllm/tool_parsers/structural_tag_registry.py imported xgrammar unconditionally and are pulled in by `from vllm import LLM` and by the OpenAI server, so vLLM could not start on platforms without an xgrammar wheel (e.g. s390x). Bind the xgrammar names to PlaceholderModule when the import fails so structural tag builders raise on first use instead, defer the xgrammar annotations in backend_xgrammar.py, and guard the import in tests/standalone_tests/lazy_imports.py plus a subprocess regression test. Fixes #56559 Signed-off-by: Bowen --- tests/standalone_tests/lazy_imports.py | 4 +- .../test_structural_tag_registry.py | 23 +++++ vllm/parser/harmony.py | 64 +++++++++----- vllm/tool_parsers/structural_tag_registry.py | 88 ++++++++++++------- vllm/v1/structured_output/backend_xgrammar.py | 2 + 5 files changed, 129 insertions(+), 52 deletions(-) diff --git a/tests/standalone_tests/lazy_imports.py b/tests/standalone_tests/lazy_imports.py index fff5c54f276d..0bdb9065d5d0 100644 --- a/tests/standalone_tests/lazy_imports.py +++ b/tests/standalone_tests/lazy_imports.py @@ -11,7 +11,9 @@ # too many processes before we set the number of compiler threads. # Lazy import `cv2` to avoid bothering users who only use text models. # `cv2` can easily mess up the environment. -module_names = ["torch._inductor.async_compile", "cv2"] +# Lazy import `xgrammar` because it has no wheels for some platforms +# (e.g. s390x) and is only needed once structured output is requested. +module_names = ["torch._inductor.async_compile", "cv2", "xgrammar"] # set all modules in `module_names` to be None. # if we import any modules during `import vllm`, there would be a diff --git a/tests/tool_parsers/test_structural_tag_registry.py b/tests/tool_parsers/test_structural_tag_registry.py index 3221d1fb2c41..466d1a9df388 100644 --- a/tests/tool_parsers/test_structural_tag_registry.py +++ b/tests/tool_parsers/test_structural_tag_registry.py @@ -2,6 +2,8 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import json +import subprocess +import sys from types import SimpleNamespace from unittest.mock import MagicMock @@ -1030,3 +1032,24 @@ def test_tool_strict_level_from_name(): ToolStrictLevel.from_name("strict") with pytest.raises(ValueError, match="expected one of auto, function, parameter"): ToolStrictLevel.from_name("off") + + +def test_import_without_xgrammar(): + """Importing vLLM must not require xgrammar (it has no wheels on some + platforms); structural tag builders fail only when actually used.""" + code = """ +import sys + +sys.modules["xgrammar"] = None + +from vllm import LLM # noqa: F401 +from vllm.tool_parsers.structural_tag_registry import get_hermes_structural_tag + +try: + get_hermes_structural_tag([], [], "auto", False) +except ImportError as error: + assert "xgrammar" in str(error), error +else: + raise AssertionError("expected ImportError when xgrammar is unavailable") +""" + subprocess.run([sys.executable, "-c", code], check=True) diff --git a/vllm/parser/harmony.py b/vllm/parser/harmony.py index 8c944733e32b..9020757e4ca3 100644 --- a/vllm/parser/harmony.py +++ b/vllm/parser/harmony.py @@ -10,21 +10,6 @@ from typing import TYPE_CHECKING, NamedTuple from openai_harmony import HarmonyError, Message, Role -from xgrammar import StructuralTag -from xgrammar.openai_tool_call_schema import BuiltinToolParam, FunctionToolParam -from xgrammar.structural_tag import ( - AnyTextFormat, - ConstStringFormat, - Format, - GrammarFormat, - JSONSchemaFormat, - OptionalFormat, - OrFormat, - RegexFormat, - SequenceFormat, - TagFormat, - TriggeredTagsFormat, -) from vllm.entrypoints.chat_utils import make_tool_call_id from vllm.entrypoints.generate.base.protocol import ( @@ -52,10 +37,49 @@ get_function_parameters, register_vllm_structural_tag, ) +from vllm.utils.import_utils import PlaceholderModule if TYPE_CHECKING: from openai_harmony import Message, StreamableParser +try: + from xgrammar import StructuralTag + from xgrammar.openai_tool_call_schema import BuiltinToolParam, FunctionToolParam + from xgrammar.structural_tag import ( + AnyTextFormat, + ConstStringFormat, + Format, + GrammarFormat, + JSONSchemaFormat, + OptionalFormat, + OrFormat, + RegexFormat, + SequenceFormat, + TagFormat, + TriggeredTagsFormat, + ) +except ImportError: + # xgrammar has no wheels for some platforms (e.g. s390x). Only the + # structural tag builders below need it, so fail on first use instead + # of at import time. + _xgrammar = PlaceholderModule("xgrammar") + _xgr_tool_schema = _xgrammar.placeholder_attr("openai_tool_call_schema") + _xgr_structural_tag = _xgrammar.placeholder_attr("structural_tag") + StructuralTag = _xgrammar.placeholder_attr("StructuralTag") + BuiltinToolParam = _xgr_tool_schema.placeholder_attr("BuiltinToolParam") + FunctionToolParam = _xgr_tool_schema.placeholder_attr("FunctionToolParam") + AnyTextFormat = _xgr_structural_tag.placeholder_attr("AnyTextFormat") + ConstStringFormat = _xgr_structural_tag.placeholder_attr("ConstStringFormat") + Format = _xgr_structural_tag.placeholder_attr("Format") + GrammarFormat = _xgr_structural_tag.placeholder_attr("GrammarFormat") + JSONSchemaFormat = _xgr_structural_tag.placeholder_attr("JSONSchemaFormat") + OptionalFormat = _xgr_structural_tag.placeholder_attr("OptionalFormat") + OrFormat = _xgr_structural_tag.placeholder_attr("OrFormat") + RegexFormat = _xgr_structural_tag.placeholder_attr("RegexFormat") + SequenceFormat = _xgr_structural_tag.placeholder_attr("SequenceFormat") + TagFormat = _xgr_structural_tag.placeholder_attr("TagFormat") + TriggeredTagsFormat = _xgr_structural_tag.placeholder_attr("TriggeredTagsFormat") + logger = init_logger(__name__) @@ -402,8 +426,6 @@ def _normalize_recipient(recipient: str | None) -> str | None: " to=functions.{name}{channel}{constrain}<|message|>", "{channel} to=functions.{name}{constrain}<|message|>", ] -_JSON_CONTENT = JSONSchemaFormat(json_schema={"type": "object"}) -_ANY_CONTENT = AnyTextFormat() def _assemble_tag( @@ -416,7 +438,7 @@ def _assemble_tag( elements=[ TagFormat( begin="<|channel|>analysis<|message|>", - content=_ANY_CONTENT, + content=AnyTextFormat(), end="<|end|>", ), ConstStringFormat(value="<|start|>assistant"), @@ -431,7 +453,7 @@ def _assemble_tag( elements=[ TagFormat( begin="<|channel|>commentary<|message|>", - content=_ANY_CONTENT, + content=AnyTextFormat(), end="<|end|>", ), ConstStringFormat(value="<|start|>assistant"), @@ -494,7 +516,7 @@ def get_harmony_structural_tag( tags.extend( TagFormat( begin=_FINAL_BEGIN.format(constrain=constrain), - content=_ANY_CONTENT, + content=AnyTextFormat(), end=_END_TAG, ) for constrain in _JSON_CONSTRAINS + [""] @@ -508,7 +530,7 @@ def get_harmony_structural_tag( def _params_to_final_content(params: StructuredOutputsParams) -> Format | None: """Map StructuredOutputsParams in a XGrammar Format.""" if params.json_object: - return _JSON_CONTENT + return JSONSchemaFormat(json_schema={"type": "object"}) if params.json is not None: schema = params.json if isinstance(schema, str): diff --git a/vllm/tool_parsers/structural_tag_registry.py b/vllm/tool_parsers/structural_tag_registry.py index ba86395b7a38..c22b5d01958e 100644 --- a/vllm/tool_parsers/structural_tag_registry.py +++ b/vllm/tool_parsers/structural_tag_registry.py @@ -1,6 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from __future__ import annotations + from collections.abc import Callable, Sequence from typing import Any, Literal, TypeAlias @@ -9,32 +11,63 @@ from openai.types.responses.tool import Tool as ResponsesTool from openai.types.responses.tool_choice_allowed import ToolChoiceAllowed from openai.types.responses.tool_choice_function import ToolChoiceFunction -from xgrammar import StructuralTag, normalize_tool_choice -from xgrammar import get_model_structural_tag as get_xgrammar_model_structural_tag -from xgrammar.openai_tool_call_schema import ( - BuiltinToolParam, - FunctionToolParam, -) -from xgrammar.structural_tag import ( - AnyTextFormat, - ConstStringFormat, - JSONSchemaFormat, - OptionalFormat, - OrFormat, - PlusFormat, - RegexFormat, - SequenceFormat, - StarFormat, - TagFormat, - TagsWithSeparatorFormat, - TriggeredTagsFormat, -) from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionNamedToolChoiceParam, ChatCompletionToolsParam, ) from vllm.tool_parsers.tool_strict_level import ToolStrictLevel +from vllm.utils.import_utils import PlaceholderModule + +try: + from xgrammar import StructuralTag, normalize_tool_choice + from xgrammar import get_model_structural_tag as get_xgrammar_model_structural_tag + from xgrammar.openai_tool_call_schema import ( + BuiltinToolParam, + FunctionToolParam, + ) + from xgrammar.structural_tag import ( + AnyTextFormat, + ConstStringFormat, + JSONSchemaFormat, + OptionalFormat, + OrFormat, + PlusFormat, + RegexFormat, + SequenceFormat, + StarFormat, + TagFormat, + TagsWithSeparatorFormat, + TriggeredTagsFormat, + ) +except ImportError: + # xgrammar has no wheels for some platforms (e.g. s390x). Only the + # structural tag builders below need it, so fail on first use instead + # of at import time. + _xgrammar = PlaceholderModule("xgrammar") + _xgr_tool_schema = _xgrammar.placeholder_attr("openai_tool_call_schema") + _xgr_structural_tag = _xgrammar.placeholder_attr("structural_tag") + StructuralTag = _xgrammar.placeholder_attr("StructuralTag") + normalize_tool_choice = _xgrammar.placeholder_attr("normalize_tool_choice") + get_xgrammar_model_structural_tag = _xgrammar.placeholder_attr( + "get_model_structural_tag" + ) + BuiltinToolParam = _xgr_tool_schema.placeholder_attr("BuiltinToolParam") + FunctionToolParam = _xgr_tool_schema.placeholder_attr("FunctionToolParam") + AnyTextFormat = _xgr_structural_tag.placeholder_attr("AnyTextFormat") + ConstStringFormat = _xgr_structural_tag.placeholder_attr("ConstStringFormat") + JSONSchemaFormat = _xgr_structural_tag.placeholder_attr("JSONSchemaFormat") + OptionalFormat = _xgr_structural_tag.placeholder_attr("OptionalFormat") + OrFormat = _xgr_structural_tag.placeholder_attr("OrFormat") + PlusFormat = _xgr_structural_tag.placeholder_attr("PlusFormat") + RegexFormat = _xgr_structural_tag.placeholder_attr("RegexFormat") + SequenceFormat = _xgr_structural_tag.placeholder_attr("SequenceFormat") + StarFormat = _xgr_structural_tag.placeholder_attr("StarFormat") + TagFormat = _xgr_structural_tag.placeholder_attr("TagFormat") + TagsWithSeparatorFormat = _xgr_structural_tag.placeholder_attr( + "TagsWithSeparatorFormat" + ) + TriggeredTagsFormat = _xgr_structural_tag.placeholder_attr("TriggeredTagsFormat") ToolChoice: TypeAlias = ( Literal["none", "auto", "required"] @@ -44,16 +77,11 @@ ) AllowedToolRef: TypeAlias = dict[str, object] SimplifiedToolChoice: TypeAlias = Literal["auto", "required", "forced"] -StructuralTagBuilder: TypeAlias = Callable[ - [ - list[FunctionToolParam], - list[BuiltinToolParam], - SimplifiedToolChoice, - bool, - str, - ], - StructuralTag, -] +# Quoted so the placeholders above are never evaluated at import time. +StructuralTagBuilder: TypeAlias = ( + "Callable[[list[FunctionToolParam], list[BuiltinToolParam], " + "SimplifiedToolChoice, bool, str], StructuralTag]" +) # Keep this list in sync with xgrammar.builtin_structural_tag. It is used for # vLLM-side validation and for documenting the xgrammar builtin surface that diff --git a/vllm/v1/structured_output/backend_xgrammar.py b/vllm/v1/structured_output/backend_xgrammar.py index 1ba7e2734d9e..e51e28d52c89 100644 --- a/vllm/v1/structured_output/backend_xgrammar.py +++ b/vllm/v1/structured_output/backend_xgrammar.py @@ -1,6 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from __future__ import annotations + import json from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any