diff --git a/python/sglang/srt/parser/chat_parsing/content_parsers.py b/python/sglang/srt/parser/chat_parsing/content_parsers.py
index d857b61d228b..f00a87b6e2c1 100644
--- a/python/sglang/srt/parser/chat_parsing/content_parsers.py
+++ b/python/sglang/srt/parser/chat_parsing/content_parsers.py
@@ -79,7 +79,7 @@ def _capture(m: Any) -> str:
try:
return json.loads(working)
- except json.JSONDecodeError as e:
+ except (json.JSONDecodeError, RecursionError) as e:
if args.get("allow_non_json"):
return _text(text, args)
if working == text:
@@ -149,12 +149,12 @@ def _kv_lines(text: str, args: dict) -> dict:
"kv-lines": _kv_lines,
}
-# Parsers whose output is the verbatim body text (modulo whitespace) — chunks
-# from these fields stream with `dirty=False` because each chunk is part of
-# the final value. Structured parsers (`json`, `xml-inline`, `kv-lines`) only
-# produce a meaningful value on close, so their chunks stream raw bytes
-# flagged `dirty=True` while the parsed value is delivered in `region_close`.
-STREAMABLE_PARSERS = frozenset({"text", "int", "float", "bool"})
+# Parsers whose output is the verbatim body text, apart from outer whitespace that
+# `strip` trims at close. Their chunks stream with `dirty=False` unless a `transform`
+# reshapes the value. Every other parser only produces its value on close, so its
+# chunks stream raw bytes flagged `dirty=True` while the parsed value is delivered
+# in `region_close`.
+STREAMABLE_PARSERS = frozenset({"text"})
def parse_content(text: str, name: str, args: dict) -> Any:
diff --git a/python/sglang/srt/parser/chat_parsing/response_parser.py b/python/sglang/srt/parser/chat_parsing/response_parser.py
index 24fdc19f7e19..721ab934903b 100644
--- a/python/sglang/srt/parser/chat_parsing/response_parser.py
+++ b/python/sglang/srt/parser/chat_parsing/response_parser.py
@@ -62,7 +62,7 @@ def _coerce(raw: str, types: tuple[str, ...]) -> Any:
decoded = json.loads(raw)
if isinstance(decoded, dict if type_name == "object" else list):
return decoded
- except ValueError:
+ except (ValueError, RecursionError):
continue
return raw # `string` params, unknown types and failed casts all keep the original text
@@ -83,7 +83,13 @@ def parse_response(
response_template = load_response_template(response_template)
stream = ResponseParser(response_template, prefix=prefix, tools=tools)
stream.feed(text)
- message, _ = stream.finalize()
+ try:
+ message, _ = stream.finalize()
+ except ValueError:
+ if not stream._malformed:
+ raise
+ if stream._malformed:
+ raise next(iter(stream._malformed.values()))
return message
@@ -107,7 +113,14 @@ class ResponseParser:
Pass OpenAI-style `tools=` dictionaries to cast tool-call arguments
using the calling tool's JSON schema as each region closes.
- Events can be either "region_open", "region_chunk", or "region_close".
+ Events can be "region_open", "region_chunk", "region_close", or "region_malformed".
+ Open, close and malformed events carry `start` and `end`, the span of `input_text` the
+ event consumed: its opening or closing delimiter, empty for implicit boundaries.
+ Explicit opens also carry the opener's named `captures`. A region whose value fails to
+ parse ends with "region_malformed" instead of "region_close", carrying the `error`, and
+ parsing continues after it. Only a plain "text" field without a `transform` streams
+ `dirty=False` chunks: verbatim pieces of the region's value, whose outer whitespace
+ `strip` may still trim at close. Other chunks are raw input parsed on close.
ResponseParser requires the chat `prefix` (i.e. the chat history, the prefill before the current generation).
This is because chat templates or assistant prefills can sometimes write part of the message, and if we
@@ -152,10 +165,18 @@ def __init__(
self._body: str = ""
self._opened: bool = False
self._finalized: bool = False
+ self._prefix_len: int = 0
+ self._malformed: dict[str, Exception] = {}
self.initial_events: list[dict] = []
if prefix:
self._consume_prefix(prefix)
+ @property
+ def input_text(self) -> str:
+ """Raw parser input after start-anchor truncation. Before the first `feed()`, this is
+ exactly the truncated prefix, so its length marks where generated text begins."""
+ return self._buffer
+
def _consume_prefix(self, prefix: str) -> None:
"""Loads the prefix (the chat prefill sent to the model), right-truncates it to the start of the
assistant message (as determined by start_anchor) and then runs the remainder through the parser.
@@ -169,6 +190,7 @@ def _consume_prefix(self, prefix: str) -> None:
return
self._buffer = truncated
self._process(self.initial_events, eos=False)
+ self._prefix_len = len(truncated)
def feed(self, text: str) -> list[dict]:
"""Feeds more text/tokens from the model output into the tokenizer, and returns any events that result
@@ -194,7 +216,11 @@ def _is_empty(v: Any) -> bool:
raise RuntimeError("ResponseParser already finalized")
events: list[dict] = []
self._process(events, eos=True)
- missing = [n for n, f in self._spec.fields.items() if not f.optional and n not in self._output]
+ missing = [
+ n
+ for n, f in self._spec.fields.items()
+ if not f.optional and n not in self._output and n not in self._malformed
+ ]
if missing:
raise ValueError(f"Required response_template fields missing from parsed output: {missing}")
defaults = self._spec.defaults
@@ -213,12 +239,12 @@ def _process(self, events: list[dict], eos: bool) -> None:
self._accumulate(events, self._buffer[self._pos : m.start()])
self._pos = m.end()
if kind == "open":
- self._close_current(events)
+ self._close_current(events, m.start(), m.start())
self._open_explicit(events, field, m)
else: # "close" (always the implicit region's close here,
# since explicit regions only expose their own close)
had_content = self._opened
- self._close_current(events)
+ self._close_current(events, m.start(), m.end())
# Zero-width close on an already-empty region would just
# re-fire next iteration -- bail out to make progress.
if not had_content and m.start() == m.end():
@@ -230,7 +256,7 @@ def _process(self, events: list[dict], eos: bool) -> None:
if self._pos < len(self._buffer):
self._accumulate(events, self._buffer[self._pos :])
self._pos = len(self._buffer)
- self._close_current(events)
+ self._close_current(events, self._pos, self._pos)
break
# Stream everything up to the earliest still-pending delimiter. When
# nothing is pending `hold_start == len(self._buffer)`, so this flushes
@@ -331,27 +357,31 @@ def _accumulate(self, events: list[dict], text: str) -> None:
"""Route `text` into the currently active region. When the current
region is the null sink (no implicit declared, no explicit open), we
silently discard. Every routed chunk emits a `region_chunk` event so
- consumers can render live; `dirty=True` flags chunks from structured
- parsers (json, xml-inline, kv-lines) whose raw bytes will only be
- parsed into the final value on close."""
+ consumers can render live; `dirty=True` flags chunks whose raw bytes
+ only become the final value once parsed or transformed on close."""
if not text or self._current is None:
return
field = self._spec.fields[self._current]
if not self._opened:
- events.append({"type": "region_open", "field": self._current})
+ events.append({"type": "region_open", "field": self._current, "start": self._pos, "end": self._pos})
self._opened = True
self._body += text
- dirty = field.content not in STREAMABLE_PARSERS
- events.append({"type": "region_chunk", "field": self._current, "text": text, "dirty": dirty})
+ # Prefix bytes held back as a possible delimiter belong to the prompt, not the chunk.
+ text = text[max(0, self._prefix_len - self._pos) :]
+ if text:
+ dirty = field.content not in STREAMABLE_PARSERS or field.transform is not None
+ events.append({"type": "region_chunk", "field": self._current, "text": text, "dirty": dirty})
def _open_explicit(self, events: list[dict], field: ResponseTemplateField, m: Any) -> None:
self._current = field.name
self._captures = {k: v for k, v in m.groupdict().items() if v is not None}
self._body = ""
self._opened = True
- events.append({"type": "region_open", "field": field.name})
+ events.append(
+ {"type": "region_open", "field": field.name, "start": m.start(), "end": m.end(), "captures": dict(self._captures)}
+ )
- def _close_current(self, events: list[dict]) -> None:
+ def _close_current(self, events: list[dict], close_start: int, close_end: int) -> None:
"""Close the current region and reset to the implicit/null region.
Skipped (aside from the reset) when the current region never opened --
avoids vacuous open/close pairs at every explicit boundary."""
@@ -359,22 +389,30 @@ def _close_current(self, events: list[dict]) -> None:
self._reset_to_implicit()
return
field = self._spec.fields[self._current]
- value = process_field(self._body, field, self._captures)
- if self._tool_params:
- value = self._coerce_tool_calls(value)
- if field.join is not None:
- if not isinstance(value, str):
+ try:
+ value = process_field(self._body, field, self._captures)
+ if self._tool_params:
+ value = self._coerce_tool_calls(value)
+ if field.join is not None and not isinstance(value, str):
raise ValueError(
f"Field '{field.name}': 'join' requires each match to parse to a string, "
f"got {type(value).__name__}."
)
+ except (KeyError, TypeError, ValueError) as error:
+ self._malformed.setdefault(self._current, error)
+ events.append(
+ {"type": "region_malformed", "field": self._current, "start": close_start, "end": close_end, "error": error}
+ )
+ self._reset_to_implicit()
+ return
+ if field.join is not None:
previous = self._output.get(self._current)
self._output[self._current] = value if previous is None else previous + field.join + value
elif field.repeats:
self._output.setdefault(self._current, []).append(value)
else:
self._output[self._current] = value
- events.append({"type": "region_close", "field": self._current, "value": value})
+ events.append({"type": "region_close", "field": self._current, "start": close_start, "end": close_end, "value": value})
self._reset_to_implicit()
def _coerce_tool_calls(self, value: Any) -> Any:
diff --git a/test/registered/unit/parser/test_chat_parsing.py b/test/registered/unit/parser/test_chat_parsing.py
index d3d6e6d505e6..bc7f2cd4d531 100644
--- a/test/registered/unit/parser/test_chat_parsing.py
+++ b/test/registered/unit/parser/test_chat_parsing.py
@@ -27,6 +27,7 @@
from sglang.srt.parser.chat_parsing.response_parser import _coerce, _schema_types
from sglang.srt.parser.chat_parsing.response_templates import load_response_template
from sglang.test.ci.ci_register import register_cpu_ci
+from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=8, suite="base-a-test-cpu")
@@ -953,7 +954,7 @@ def _chunk_random(text: str, rng: random.Random):
yield text[prev:]
-class ResponseEventStreamTest(unittest.TestCase):
+class ResponseEventStreamTest(CustomTestCase):
def test_stream_matches_whole_string_all_templates_fixed_chunking(self):
"""For every fixed chunking step we try, the streamed finalize()
output must equal the whole-string parse. Regression coverage for
@@ -1066,16 +1067,17 @@ def test_region_chunks_reconstruct_text_regions(self):
self.assertIn("tool_calls", per_field_close_value)
def test_dirty_flag_marks_structured_regions(self):
- """A template with one text field and one structured field per parser
- family: text/int/float/bool stream chunks with `dirty=False`, while
- json/xml-inline/kv-lines stream chunks with `dirty=True`, and those
- dirty chunks concatenate to the raw region body before parsing."""
+ """Only plain text without a `transform` streams `dirty=False` chunks; int/float/bool,
+ structured parsers and transforms stream dirty chunks of the raw region body."""
spec = {
"defaults": {"role": "assistant"},
"start_anchor": "<|assistant|>",
"fields": {
"thinking": {"open": "", "close": "", "content": "text"},
"score": {"open": "", "close": "", "content": "int"},
+ "ratio": {"open": "", "close": "", "content": "float"},
+ "flag": {"open": "", "close": "", "content": "bool"},
+ "labeled": {"open": "", "close": "", "transform": {"label": "{content}"}},
"json_call": {"open": "", "close": "", "content": "json"},
"xml_call": {
"open": "",
@@ -1090,7 +1092,10 @@ def test_dirty_flag_marks_structured_regions(self):
},
},
}
- text = 'hello world42{"a": 1, "b": 2}k1: v1\nk2: v2'
+ text = (
+ " hello world 420.5truetag"
+ '{"a": 1, "b": 2}k1: v1\nk2: v2'
+ )
# Drive byte-by-byte to maximise chunk count.
streamer = ResponseParser(spec, prefix="")
events: list[dict] = []
@@ -1101,22 +1106,24 @@ def test_dirty_flag_marks_structured_regions(self):
per_field_chunks: dict[str, list[str]] = {}
per_field_dirty: dict[str, set[bool]] = {}
+ per_field_close_value: dict[str, object] = {}
for ev in events:
if ev["type"] == "region_chunk":
per_field_chunks.setdefault(ev["field"], []).append(ev["text"])
per_field_dirty.setdefault(ev["field"], set()).add(ev["dirty"])
+ elif ev["type"] == "region_close":
+ per_field_close_value[ev["field"]] = ev["value"]
- # Clean (streamable) regions.
- for field in ("thinking", "score"):
- self.assertEqual(per_field_dirty[field], {False}, f"{field} should be clean")
- # Dirty (structured) regions.
- for field in ("json_call", "xml_call", "kv_call"):
+ self.assertEqual(per_field_dirty["thinking"], {False}, "thinking should be clean")
+ for field in ("score", "ratio", "flag", "labeled", "json_call", "xml_call", "kv_call"):
self.assertEqual(per_field_dirty[field], {True}, f"{field} should be dirty")
# Dirty chunks reconstruct the raw region body (un-parsed). Clean
# chunks reconstruct the verbatim body too: stripping happens at close.
- self.assertEqual("".join(per_field_chunks["thinking"]), "hello world")
+ self.assertEqual("".join(per_field_chunks["thinking"]), " hello world ")
+ self.assertEqual(per_field_close_value["thinking"], "hello world")
self.assertEqual("".join(per_field_chunks["score"]), "42")
+ self.assertEqual(per_field_close_value["labeled"], {"label": "tag"})
self.assertEqual("".join(per_field_chunks["json_call"]), '{"a": 1, "b": 2}')
self.assertEqual("".join(per_field_chunks["xml_call"]), "")
self.assertEqual("".join(per_field_chunks["kv_call"]), "k1: v1\nk2: v2")
@@ -1159,6 +1166,137 @@ def test_empty_input_streams_cleanly(self):
self.assertEqual(result, {"role": "assistant"})
self.assertEqual(final_events, [])
+ def test_region_events_expose_delimiter_offsets(self):
+ spec = {
+ "start_anchor": "",
+ "fields": {
+ "content": {"content_args": {"strip": False}},
+ "tag": {"open_pattern": r"", "close": ""},
+ },
+ }
+ text = "plainbodytail"
+ expected = [
+ {"type": "region_open", "field": "content", "start": 0, "end": 0},
+ {"type": "region_close", "field": "content", "start": 5, "end": 5, "value": "plain"},
+ {"type": "region_open", "field": "tag", "start": 5, "end": 15, "captures": {}},
+ {"type": "region_close", "field": "tag", "start": 19, "end": 25, "value": "body"},
+ {"type": "region_open", "field": "content", "start": 25, "end": 25},
+ {"type": "region_close", "field": "content", "start": 29, "end": 29, "value": "tail"},
+ ]
+ for prefix_len in (0, len("plain\w+)>",
+ "close": "",
+ "content": "json",
+ "transform": {"name": "{name}", "arguments": "{content}"},
+ }
+ },
+ }
+ parser = ResponseParser(spec, prefix="history[BEGIN]{"city":"Paris"}')[0]
+
+ self.assertEqual(prefix_end, len("")))
+ self.assertEqual(opening["captures"], {"name": "get_weather"})
+ opening["captures"]["name"] = "edited"
+ self.assertEqual(parser.feed("")[-1]["value"]["name"], "get_weather")
+
+ def test_held_prefix_bytes_are_not_rechunked(self):
+ spec = {"start_anchor": "[BEGIN]", "fields": {"content": {}, "tag": {"open": "", "close": ""}}}
+ parser = ResponseParser(spec, prefix="[BEGIN]Existing <")
+ chunks = [event["text"] for event in parser.feed("b> tail") if event["type"] == "region_chunk"]
+ message, _ = parser.finalize()
+
+ self.assertEqual(chunks, ["b> tail"])
+ self.assertEqual(message["content"], "Existing tail")
+
+ def test_malformed_region_events_recover_and_continue(self):
+ spec = {
+ "start_anchor": "[BEGIN]",
+ "fields": {
+ "tool_calls": {
+ "open_pattern": r"\w+)>",
+ "close": "",
+ "content": "json",
+ "repeats": True,
+ "transform": {
+ "type": "function",
+ "function": {"name": "{name}", "arguments": "{content}"},
+ },
+ }
+ },
+ }
+ parser = ResponseParser(spec, prefix="")
+ events = parser.feed('{"x":{"x":1}')
+ message, _ = parser.finalize()
+ opening, malformed = events[0], events[2]
+
+ self.assertEqual(malformed["type"], "region_malformed")
+ self.assertIsInstance(malformed["error"], ValueError)
+ self.assertEqual(parser.input_text[opening["end"] : malformed["start"]], '{"x":')
+ self.assertEqual(parser.input_text[malformed["start"] : malformed["end"]], "")
+ self.assertEqual(
+ message["tool_calls"],
+ [{"type": "function", "function": {"name": "good", "arguments": {"x": 1}}}],
+ )
+
+ parser = ResponseParser(spec, prefix="")
+ parser.feed('{"x":')
+ _, events = parser.finalize()
+ self.assertEqual(events[-1]["type"], "region_malformed")
+ self.assertEqual(events[-1]["start"], len(parser.input_text))
+
+ deep = "[" * 20000
+ tools = [{"type": "function", "function": {"name": "good", "parameters": {"properties": {"x": {"type": "array"}}}}}]
+ parser = ResponseParser(spec, prefix="", tools=tools)
+ events = parser.feed(f'{deep}{{"x": "{deep}"}}')
+ message, _ = parser.finalize()
+ self.assertEqual(events[2]["type"], "region_malformed")
+ self.assertIsInstance(events[2]["error"], ValueError)
+ self.assertEqual(message["tool_calls"], [{"type": "function", "function": {"name": "good", "arguments": {"x": deep}}}])
+
+ def test_parse_response_raises_for_malformed_region(self):
+ spec = {
+ "start_anchor": "<|assistant|>",
+ "fields": {"x": {"open": "", "close": "", "content": "json"}},
+ }
+ with self.assertRaisesRegex(ValueError, "could not parse region as JSON"):
+ parse_response('{"name":', spec, prefix="")
+ with self.assertRaisesRegex(ValueError, "could not parse region as JSON"):
+ parse_response("", spec, prefix='<|assistant|>{"name":')
+
+ required = {
+ "start_anchor": "<|assistant|>",
+ "fields": {
+ "x": {"open": "", "close": "", "content": "json", "optional": False},
+ "y": {"open": "", "close": "", "optional": False},
+ "n": {"open": "", "close": "", "content": "int"},
+ },
+ }
+ for prefix, text in [("", "{"), ("<|assistant|>{", ""), ("", "{"), ("", "{a")]:
+ with self.subTest(prefix=prefix, text=text), self.assertRaisesRegex(ValueError, "could not parse region as JSON"):
+ parse_response(text, required, prefix=prefix)
+ parser = ResponseParser(required, prefix="")
+ parser.feed("{")
+ with self.assertRaisesRegex(ValueError, r"missing from parsed output: \['y'\]"):
+ parser.finalize()
+
class PrefixAndTruncationTest(unittest.TestCase):
def test_prefix_lands_inside_explicit_region(self):
@@ -1323,7 +1461,7 @@ def test_prefix_partial_pattern_at_boundary(self):
"""Post-truncation prefix ends mid-delimiter. The first feed completes
the match; initial_events is empty (no region opened within the
prefix yet) and the open fires from `feed()`."""
- prefix = "<|im_start|>assistant\n`
+ prefix = "<|im_start|>assistant\n` # codespell:ignore
stream = ResponseParser(qwen3_template, prefix=prefix)
self.assertEqual(stream.initial_events, [])
events = stream.feed("nk>real body")