From 82d7df2d6a44b8180d29b539f33679e7d3e30a7b Mon Sep 17 00:00:00 2001 From: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> Date: Wed, 7 Oct 2026 15:14:59 +0000 Subject: [PATCH 01/43] [None][fix] GLM tool parsers: stream exactly what the whole-text parse reports glm4 and glm47 parsed a document differently depending on how it was streamed. A call that arrived in the final delta could be lost on the chat path, a declared string was re-typed ("99797" arrived as 99797, true as "True"), a value quoting was cut there, and markup the model wrote around a tool name ended up inside the name. - glm47 derives from glm4 and overrides only how a call body splits into name and argument text. Both parse paths run each complete ... through one function, so streaming reports the calls and the text detect_and_parse returns, however the output is split. BaseToolParser.streaming_matches_whole_parse declares that property; it is False by default and True for these two parsers. - A call is reported once its arrives, as one item with the name and the complete JSON arguments; text after a call waits for the next increment. finish() reports the complete calls the parser still holds and releases the rest as text. - A call is delivered only when its argument text is complete key/value pairs; otherwise its markup is released as text, as detect_and_parse reads it. A value runs to the first followed by the next or the end of the call, so a value may quote that tag. - Declared strings are delivered verbatim. Other declared types are decoded as JSON, JSON with escaped quotes or a Python literal, and an undeclared argument only as strict JSON. A result that is not strict JSON (NaN, 1e309) keeps the text, and a repeated key keeps its first value. - A name with a group qualifier missing or extra, or with markup fused into it, resolves to the declared tool (BaseToolParser.resolve_tool_name) and is typed by that tool's schema; prose that mentions a tool never resolves. A restarted inside a call is parsed from the restart, and the abandoned prefix is released as text. - GLM-4 keeps the text between and after calls, as GLM-4.7 does, and argument text is scanned in linear time. On the chat path a GLM tool call now arrives as one delta when it completes, instead of its name first and its arguments in pieces. The GLM tests are one parity class run for both parsers: typing and argument-text tables, multi-call responses with malformed, restarted and cut-off calls, and name repair, each checked whole and over every chunking. The existing streaming tests that asserted an early name announcement now assert the single complete item. Signed-off-by: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> --- .../serve/tool_parser/base_tool_parser.py | 51 ++ .../serve/tool_parser/glm47_parser.py | 455 +------------ tensorrt_llm/serve/tool_parser/glm4_parser.py | 596 ++++++------------ .../unittest/llmapi/apps/test_tool_parsers.py | 479 ++++++++++++-- 4 files changed, 692 insertions(+), 889 deletions(-) diff --git a/tensorrt_llm/serve/tool_parser/base_tool_parser.py b/tensorrt_llm/serve/tool_parser/base_tool_parser.py index c94e1e543db6..06d29f445c06 100644 --- a/tensorrt_llm/serve/tool_parser/base_tool_parser.py +++ b/tensorrt_llm/serve/tool_parser/base_tool_parser.py @@ -1,5 +1,8 @@ # Adapted from https://github.com/sgl-project/sglang/blob/083629c23564e1a64deaa052f1df5c5d914358d8/python/sglang/srt/function_call/base_format_detector.py +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 import json +import re from abc import ABC, abstractmethod from typing import Any, Dict, List, Optional @@ -12,6 +15,15 @@ from .core_types import StreamingParseResult, ToolCallItem, _GetInfoFunc from .utils import find_common_prefix, is_complete_json, partial_json_loads +# A tool name, optionally qualified by the groups it was declared under: +# `apply_patch`, `functions.apply_patch`, `functions.collaboration.send_message`. +_QUALIFIED_NAME = re.compile( + r"^[A-Za-z_][A-Za-z0-9_-]*(?:\.[A-Za-z_][A-Za-z0-9_-]*)*$") + +# An opening or closing tag of the XML-ish tool-call markup, e.g. `` +# or ``, which models sometimes emit unbalanced into a tool name. +_MARKUP_TAG = re.compile(r"") + def warn_if_tool_call_unparsed(parser_name: str, tool_parser: "BaseToolParser", text: str, calls: List[ToolCallItem]) -> None: @@ -59,6 +71,9 @@ class BaseToolParser(ABC): # JSON arguments) set this to True so the serving layer runs extraction # on the named-choice path instead of passing raw text through. extracts_forced_tool_calls: bool = False + # True when parse_streaming_increment plus finish deliver exactly the calls + # detect_and_parse finds, however the text is split into increments. + streaming_matches_whole_parse: bool = False def __init__(self): # Streaming state management @@ -103,6 +118,42 @@ def _get_tool_indices(self, tools: List[Tool]) -> Dict[str, int]: for i, tool in enumerate(tools) if tool.function.name } + @staticmethod + def resolve_tool_name(name: Optional[str], + tool_indices: Dict[str, int]) -> Optional[str]: + """Map an emitted tool name onto a declared tool; None if none fits. + + Recovers a group qualifier the declaration lacks (``functions.exec`` + for ``exec``), a bare name for a qualified declaration (``exec`` for + ``functions.exec``, only when that tail is unambiguous), and markup + fused onto the name (``apply_patch``). A name that merely + mentions a declared tool inside prose is never mapped onto it. + """ + if not name: + return None + if name in tool_indices: + return name + + # Declared tools by their bare tail; an ambiguous tail maps nowhere. + by_tail: Dict[str, List[str]] = {} + for declared in tool_indices: + by_tail.setdefault(declared.rsplit(".", 1)[-1], []).append(declared) + + # The raw name, then the name with markup removed. Anything that is not + # a dotted identifier after that is prose or code, not a mangled name. + for candidate in (name, _MARKUP_TAG.sub("", name).strip()): + if not _QUALIFIED_NAME.match(candidate): + continue + if candidate in tool_indices: + return candidate + tail = candidate.rsplit(".", 1)[-1] + if tail in tool_indices: + return tail + qualified = by_tail.get(tail) + if qualified is not None and len(qualified) == 1: + return qualified[0] + return None + def parse_base_json(self, action: Any, tools: List[Tool]) -> List[ToolCallItem]: tool_indices = self._get_tool_indices(tools) diff --git a/tensorrt_llm/serve/tool_parser/glm47_parser.py b/tensorrt_llm/serve/tool_parser/glm47_parser.py index 806dfb772e3b..404a44ee1bfd 100644 --- a/tensorrt_llm/serve/tool_parser/glm47_parser.py +++ b/tensorrt_llm/serve/tool_parser/glm47_parser.py @@ -13,27 +13,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import json -import re -from typing import List, Optional +from typing import Tuple -from tensorrt_llm.logger import logger -from tensorrt_llm.serve.openai_protocol import ChatCompletionToolsParam as Tool -from tensorrt_llm.serve.tool_parser.base_tool_parser import BaseToolParser -from tensorrt_llm.serve.tool_parser.core_types import ( - StreamingParseResult, - ToolCallItem, - _GetInfoFunc, -) -from tensorrt_llm.serve.tool_parser.glm4_parser import ( - StreamState, - _convert_to_number, - get_argument_type, - parse_arguments, -) +from tensorrt_llm.serve.tool_parser.glm4_parser import Glm4ToolParser -class Glm47ToolParser(BaseToolParser): +class Glm47ToolParser(Glm4ToolParser): r"""Tool parser for GLM-4.7 and GLM-5 models. GLM-4.7 uses a slightly different tool call format compared to GLM-4.5: @@ -52,429 +37,15 @@ class Glm47ToolParser(BaseToolParser): get_time """ - def __init__(self): - super().__init__() - self.bot_token = "" # nosec B105 - self.eot_token = "" # nosec B105 - self.func_call_regex = re.compile(r".*?", re.DOTALL) - self.func_detail_regex = re.compile( - r"(.*?)(.*?)?", re.DOTALL - ) - self.func_arg_regex = re.compile( - r"(.*?)(?:\\n|\s)*(.*?)", - re.DOTALL, - ) - self._partial_stream_regex = re.compile( - r"(.*?)(?:()|$)", - re.DOTALL, - ) - self._last_arguments = "" - self.current_tool_id = -1 - self.current_tool_name_sent = False - self._streamed_raw_length = 0 - self._reset_streaming_state() + def _split_call(self, body: str) -> Tuple[str, str, str]: + """Split a call's body into (name, markup after the name, argument text). - def _reset_streaming_state(self) -> None: - self._stream_state = StreamState.INIT - self._current_key = "" - self._current_value = "" - self._xml_tag_buffer = "" - self._is_first_param = True - self._value_started = False - self._cached_value_type: Optional[str] = None - - def has_tool_call(self, text: str) -> bool: - return self.bot_token in text - - def detect_and_parse(self, text: str, tools: List[Tool]) -> StreamingParseResult: - if self.bot_token not in text: - return StreamingParseResult(normal_text=text, calls=[]) - - normal_text_parts = [] - last_end = 0 - match_texts = [] - - for match in self.func_call_regex.finditer(text): - if match.start() > last_end: - normal_text_parts.append(text[last_end : match.start()]) - last_end = match.end() - match_texts.append(match.group(0)) - - if last_end < len(text): - normal_text_parts.append(text[last_end:]) - - normal_text = "".join(normal_text_parts).strip() - - calls = [] - try: - for match_result in match_texts: - func_detail = self.func_detail_regex.search(match_result) - if func_detail is None: - continue - func_name = func_detail.group(1).strip() if func_detail.group(1) else "" - func_args = func_detail.group(2) if func_detail.group(2) else "" - arguments = {} - if func_args: - pairs = self.func_arg_regex.findall(func_args) - arguments = self._parse_argument_pairs(pairs, func_name, tools) - - match_result = {"name": func_name, "parameters": arguments} - calls.extend(self.parse_base_json(match_result, tools)) - return StreamingParseResult(normal_text=normal_text, calls=calls) - except Exception as e: - logger.error(f"Error in detect_and_parse: {e}") - return StreamingParseResult(normal_text=text) - - def _get_value_type(self, func_name: str, key: str, tools: List[Tool]) -> str: - """Get parameter type from tool definition, with fallback to auto-detection.""" - arg_type = get_argument_type(func_name, key, tools) - if arg_type: - return arg_type - - value_content = self._current_value.strip() if self._current_value else "" - - if not value_content: - return "string" - - try: - parsed = json.loads(value_content) - if isinstance(parsed, dict): - return "object" - elif isinstance(parsed, list): - return "array" - elif isinstance(parsed, bool): - return "boolean" - elif isinstance(parsed, (int, float)): - return "number" - elif isinstance(parsed, str): - if parsed.isdigit() or (parsed.startswith("-") and parsed[1:].isdigit()): - return "number" - return "string" - except json.JSONDecodeError: - first_char = value_content[0] if value_content else "" - if first_char.isdigit() or first_char in ["-", "."]: - return "number" - elif first_char in ["{", "["]: - return "object" - elif first_char in ['"', "'"]: - return "string" - - return "string" - - def _format_value_complete(self, value: str, value_type: str) -> str: - if value_type == "string": - return json.dumps(value, ensure_ascii=False) - elif value_type == "number": - try: - num = _convert_to_number(value.strip() if value else "") - return str(num) - except (ValueError, AttributeError): - logger.warning(f"Failed to parse '{value}' as number, treating as string") - return json.dumps(str(value) if value else "", ensure_ascii=False) - else: - return value - - def _process_xml_to_json_streaming( - self, raw_increment: str, func_name: str, tools: List[Tool] - ) -> str: - """Convert XML increment to JSON streaming output using state machine.""" - json_output = "" - - for char in raw_increment: - self._xml_tag_buffer += char - - if self._stream_state in [StreamState.INIT, StreamState.BETWEEN]: - if self._xml_tag_buffer.endswith(""): - self._stream_state = StreamState.IN_KEY - self._current_key = "" - self._xml_tag_buffer = "" - - elif self._stream_state == StreamState.IN_KEY: - if self._xml_tag_buffer.endswith(""): - self._current_key = self._xml_tag_buffer[:-10].strip() - self._xml_tag_buffer = "" - self._stream_state = StreamState.WAITING_VALUE - json_output += "{" if self._is_first_param else ", " - self._is_first_param = False - json_output += json.dumps(self._current_key, ensure_ascii=False) + ": " - - elif self._stream_state == StreamState.WAITING_VALUE: - if self._xml_tag_buffer.endswith(""): - self._stream_state = StreamState.IN_VALUE - self._current_value = "" - self._xml_tag_buffer = "" - self._value_started = False - self._cached_value_type = self._get_value_type( - func_name, self._current_key, tools - ) - - elif self._stream_state == StreamState.IN_VALUE: - if self._xml_tag_buffer.endswith(""): - final_value = self._xml_tag_buffer[:-12] - self._current_value += final_value - - value_type = self._cached_value_type or "string" - - if self._value_started: - if final_value: - if value_type == "string": - json_output += json.dumps(final_value, ensure_ascii=False)[1:-1] - else: - json_output += final_value - if value_type == "string": - json_output += '"' - else: - json_output += self._format_value_complete(self._current_value, value_type) - - self._xml_tag_buffer = "" - self._stream_state = StreamState.BETWEEN - self._current_value = "" - self._value_started = False - self._cached_value_type = None - else: - closing_tag = "" - is_potential_closing = len(self._xml_tag_buffer) <= len( - closing_tag - ) and closing_tag.startswith(self._xml_tag_buffer) - - if not is_potential_closing: - content = self._xml_tag_buffer - value_type = self._cached_value_type or "string" - - if value_type == "string": - if not self._value_started: - json_output += '"' - self._value_started = True - if content: - json_output += json.dumps(content, ensure_ascii=False)[1:-1] - self._current_value += content - self._xml_tag_buffer = "" - elif value_type == "number": - if content: - if not self._value_started: - self._value_started = True - json_output += content - self._current_value += content - self._xml_tag_buffer = "" - else: - if content: - if not self._value_started: - self._value_started = True - json_output += content - self._current_value += content - self._xml_tag_buffer = "" - - return json_output - - def _extract_match_groups(self, match: re.Match) -> tuple[str, str, str]: - func_name = match.group(1).strip() - func_args_raw = match.group(2).strip() if match.group(2) else "" - is_tool_end = match.group(3) or "" - return func_name, func_args_raw, is_tool_end - - def _send_tool_name_if_needed( - self, func_name: str, has_arg_key: bool, is_tool_end: str - ) -> Optional[ToolCallItem]: - if self.current_tool_name_sent: - return None - - is_func_name_complete = has_arg_key or is_tool_end == self.eot_token - if not is_func_name_complete: - return None - - if not func_name: - logger.warning("Empty function name detected, skipping tool call") - return None - - self.current_tool_name_sent = True - self._streamed_raw_length = 0 - self._reset_streaming_state() - - self.prev_tool_call_arr[self.current_tool_id] = { - "name": func_name, - "arguments": {}, - } - - return ToolCallItem( - tool_index=self.current_tool_id, - name=func_name, - parameters="", - ) - - def _process_arguments_streaming( - self, func_name: str, func_args_raw: str, tools: List[Tool] - ) -> Optional[ToolCallItem]: - current_raw_length = len(func_args_raw) - - if current_raw_length <= self._streamed_raw_length: - return None - - raw_increment = func_args_raw[self._streamed_raw_length :] - - json_increment = self._process_xml_to_json_streaming(raw_increment, func_name, tools) - - self._streamed_raw_length = current_raw_length - - if not json_increment: - return None - - self._last_arguments += json_increment - self.streamed_args_for_tool[self.current_tool_id] += json_increment - - return ToolCallItem( - tool_index=self.current_tool_id, - name=None, - parameters=json_increment, - ) - - def _finalize_tool_call( - self, - func_name: str, - func_args_raw: str, - tools: List[Tool], - match_end_pos: int, - current_text: str, - ) -> List[ToolCallItem]: - calls = [] - closing = ( - "{}" - if self._is_first_param - else ("}" if not self._last_arguments.endswith("}") else "") - ) - if closing: - calls.append( - ToolCallItem( - tool_index=self.current_tool_id, - name=None, - parameters=closing, - ) - ) - self._last_arguments += closing - self.streamed_args_for_tool[self.current_tool_id] += closing - - if func_args_raw: - try: - pairs = self.func_arg_regex.findall(func_args_raw) - if pairs: - arguments = self._parse_argument_pairs(pairs, func_name, tools) - self.prev_tool_call_arr[self.current_tool_id]["arguments"] = arguments - except Exception as e: - logger.debug(f"Failed to parse arguments: {e}") - - self._buffer = current_text[match_end_pos:] - - self.current_tool_id += 1 - self._last_arguments = "" - self.current_tool_name_sent = False - self._streamed_raw_length = 0 - self._reset_streaming_state() - - return calls - - def parse_streaming_increment(self, new_text: str, tools: List[Tool]) -> StreamingParseResult: - """Streaming incremental parsing for GLM-4.7 format. - - Uses a state machine to convert XML to JSON incrementally for - true character-by-character streaming. + The name runs to the first ``<`` and the arguments from the first ````. """ - self._buffer += new_text - current_text = self._buffer - - bot_idx = current_text.find(self.bot_token) - - if bot_idx == -1: - tail_len = min(len(current_text), len(self.bot_token) - 1) - is_potential_start = tail_len > 0 and any( - self.bot_token.startswith(current_text[-i:]) for i in range(1, tail_len + 1) - ) - - if not is_potential_start: - output_text = current_text.replace(self.eot_token, "") - self._buffer = "" - return StreamingParseResult(normal_text=output_text) - return StreamingParseResult(normal_text="", calls=[]) - - normal_text = "" - if bot_idx > 0: - normal_text = current_text[:bot_idx] - current_text = current_text[bot_idx:] - self._buffer = current_text - - if not hasattr(self, "_tool_indices"): - self._tool_indices = self._get_tool_indices(tools) - - calls: list[ToolCallItem] = [] - try: - partial_match = self._partial_stream_regex.search(current_text) - - if not partial_match: - return StreamingParseResult(normal_text=normal_text, calls=[]) - - func_name, func_args_raw, is_tool_end = self._extract_match_groups(partial_match) - - if self.current_tool_id == -1: - self.current_tool_id = 0 - self.prev_tool_call_arr = [] - self.streamed_args_for_tool = [""] - self._streamed_raw_length = 0 - self.current_tool_name_sent = False - self._reset_streaming_state() - - while len(self.prev_tool_call_arr) <= self.current_tool_id: - self.prev_tool_call_arr.append({}) - while len(self.streamed_args_for_tool) <= self.current_tool_id: - self.streamed_args_for_tool.append("") - - has_arg_key = partial_match.group(2) is not None - - tool_name_item = self._send_tool_name_if_needed(func_name, has_arg_key, is_tool_end) - if tool_name_item: - calls.append(tool_name_item) - - if self.current_tool_name_sent: - arg_item = self._process_arguments_streaming(func_name, func_args_raw, tools) - if arg_item: - calls.append(arg_item) - - if is_tool_end == self.eot_token: - finalize_calls = self._finalize_tool_call( - func_name, - func_args_raw, - tools, - partial_match.end(), - current_text, - ) - calls.extend(finalize_calls) - return StreamingParseResult(normal_text=normal_text, calls=calls) - - except Exception as e: - logger.error(f"Error in parse_streaming_increment: {e}") - return StreamingParseResult(normal_text=current_text) - - return StreamingParseResult(normal_text=normal_text, calls=calls) - - def _parse_argument_pairs(self, pairs, func_name: str, tools: List[Tool]) -> dict: - """Parse argument key-value pairs with type coercion.""" - arguments = {} - for arg_key, arg_value in pairs: - arg_key = arg_key.strip() - arg_type = get_argument_type(func_name, arg_key, tools) - parsed_value, is_good_json = parse_arguments(arg_value, arg_type) - - if arg_type == "string": - if isinstance(parsed_value, str): - arguments[arg_key] = parsed_value - elif isinstance(parsed_value, (dict, list)): - arguments[arg_key] = json.dumps(parsed_value, ensure_ascii=False) - else: - arguments[arg_key] = str(parsed_value) - else: - arguments[arg_key] = parsed_value if is_good_json else arg_value - - return arguments - - def supports_structural_tag(self) -> bool: - return False - - def structure_info(self) -> _GetInfoFunc: - raise NotImplementedError() + name_end = body.find("<") + if name_end == -1: + return body, "", "" + args_start = body.find("", name_end) + if args_start == -1: + args_start = len(body) + return body[:name_end], body[name_end:args_start], body[args_start:] diff --git a/tensorrt_llm/serve/tool_parser/glm4_parser.py b/tensorrt_llm/serve/tool_parser/glm4_parser.py index 540f3d842309..22a528f8fd98 100644 --- a/tensorrt_llm/serve/tool_parser/glm4_parser.py +++ b/tensorrt_llm/serve/tool_parser/glm4_parser.py @@ -1,8 +1,21 @@ # Adapted from https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/function_call/glm4_moe_detector.py +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import ast import json import re -from enum import Enum from typing import Any, Dict, List, Optional, Tuple from tensorrt_llm.logger import logger @@ -16,15 +29,47 @@ from .utils import infer_type_from_json_schema +_ARG_KEY = "" +_ARG_KEY_END = "" +_ARG_VALUE = "" +_ARG_VALUE_END = "" +# The filler GLM writes between tags: whitespace or a literal backslash-n. +_SEPARATORS = re.compile(r"(?:\s|\\n)*") +# The end of the GLM-4.5 name line: a newline or a literal backslash-n. +_NAME_LINE_END = re.compile(r"\\n|\n") -class StreamState(str, Enum): - """State machine states for XML to JSON streaming conversion.""" - INIT = "INIT" - BETWEEN = "BETWEEN" - IN_KEY = "IN_KEY" - WAITING_VALUE = "WAITING_VALUE" - IN_VALUE = "IN_VALUE" +def parse_argument_text(text: str) -> Optional[List[Tuple[str, str]]]: + """Read a call's argument text as ``(key, raw value)`` pairs. + + The text must be separators and complete + ``KV`` pairs, or None is returned. + A value runs to the first ```` that is followed, over separators, + by the next ```` or the end of the call, so a value may quote that tag. + """ + pairs = [] + pos = _SEPARATORS.match(text).end() + while pos < len(text): + if not text.startswith(_ARG_KEY, pos): + return None + key_start = pos + len(_ARG_KEY) + key_end = text.find("<", key_start) + if not text.startswith(_ARG_KEY_END, key_end): + return None + value_start = _SEPARATORS.match(text, key_end + len(_ARG_KEY_END)).end() + if not text.startswith(_ARG_VALUE, value_start): + return None + value_start += len(_ARG_VALUE) + value_end = text.find(_ARG_VALUE_END, value_start) + while value_end != -1: + pos = _SEPARATORS.match(text, value_end + len(_ARG_VALUE_END)).end() + if pos == len(text) or text.startswith(_ARG_KEY, pos): + break + value_end = text.find(_ARG_VALUE_END, value_end + 1) + if value_end == -1: + return None + pairs.append((text[key_start:key_end].strip(), text[value_start:value_end])) + return pairs def get_argument_type(func_name: str, arg_key: str, defined_tools: List[Tool]) -> Optional[str]: @@ -52,40 +97,58 @@ def _convert_to_number(value: str) -> Any: return value -def parse_arguments(json_value: str, arg_type: Optional[str] = None) -> Tuple[Any, bool]: - """Parse argument value with multiple fallback strategies. - - Returns: - Tuple of (parsed_value, is_valid_json) - """ +def _is_strict_json(value: Any) -> bool: + """Whether ``json.dumps`` spells `value` as strict JSON (no NaN, Infinity, sets...).""" try: - parsed_value = json.loads(json_value) - if arg_type == "number" and isinstance(parsed_value, str): - parsed_value = _convert_to_number(parsed_value) - return parsed_value, True - except (json.JSONDecodeError, ValueError): - pass + json.dumps(value, allow_nan=False) + except (TypeError, ValueError, RecursionError): + return False + return True - try: - wrapped = json.loads('{"tmp": "' + json_value + '"}') - parsed_value = json.loads(wrapped["tmp"]) - if arg_type == "number" and isinstance(parsed_value, str): - parsed_value = _convert_to_number(parsed_value) - return parsed_value, True - except (json.JSONDecodeError, ValueError, KeyError): - pass - try: - parsed_value = ast.literal_eval(json_value) - return parsed_value, True - except (ValueError, SyntaxError): - pass +def _loads_escaped_json(value: str) -> Any: + r"""Decode JSON written with escaped quotes, such as ``[{\"a\": 1}]``.""" + return json.loads(json.loads('{"tmp": "' + value + '"}')["tmp"]) - try: - quoted_value = json.dumps(str(json_value)) - return json.loads(quoted_value), True - except (json.JSONDecodeError, ValueError): - return json_value, False + +def parse_arguments(value: str, arg_type: Optional[str] = None) -> Any: + """Decode an argument's raw text for its schema type, or return the text itself. + + With no resolvable type only strict JSON decodes; a declared non-string type + also accepts JSON with escaped quotes and Python literals. A result that is + not strict JSON (``NaN``, an overflowing ``1e309``, a set) keeps the text. + """ + decoders = [json.loads] + if arg_type is not None: + decoders += [_loads_escaped_json, ast.literal_eval] + for decode in decoders: + try: + parsed = decode(value) + except (ValueError, TypeError, SyntaxError, RecursionError): + continue + if arg_type in ("number", "integer") and isinstance(parsed, str): + parsed = _convert_to_number(parsed) + if _is_strict_json(parsed): + return parsed + return value + + +def parse_argument_pairs( + pairs: List[Tuple[str, str]], func_name: str, tools: List[Tool] +) -> Dict[str, Any]: + """Type each raw value by the declared schema; a repeated key keeps its first value. + + A value declared a string is passed through verbatim; any other value goes + through ``parse_arguments``. + """ + arguments = {} + for key, value in pairs: + if key in arguments: + logger.debug(f"Tool argument {key!r} repeats; keeping its first value") + continue + arg_type = get_argument_type(func_name, key, tools) + arguments[key] = value if arg_type == "string" else parse_arguments(value, arg_type) + return arguments class Glm4ToolParser(BaseToolParser): @@ -102,384 +165,137 @@ class Glm4ToolParser(BaseToolParser): Or with literal \n characters (escaped as \\n in the output): get_weather\ncity\nεŒ—δΊ¬\n - Uses a streaming state machine to convert XML to JSON incrementally. + Streaming reports each call once its ```` arrives, as one item + holding what ``detect_and_parse`` makes of the same markup. """ + streaming_matches_whole_parse = True + def __init__(self): super().__init__() self.bot_token = "" # nosec B105 self.eot_token = "" # nosec B105 - self.func_call_regex = r".*?" - self.func_detail_regex = re.compile( - r"(.*?)(?:\\n|\n)(.*)", re.DOTALL - ) - self.func_arg_regex = re.compile( - r"(.*?)(?:\\n|\s)*(.*?)", - re.DOTALL, - ) - self._last_arguments = "" - self.current_tool_id = -1 - self.current_tool_name_sent = False - self._streamed_raw_length = 0 - self._reset_streaming_state() - - def _reset_streaming_state(self) -> None: - """Reset the streaming state machine for a new tool call.""" - self._stream_state = StreamState.INIT - self._current_key = "" - self._current_value = "" - self._xml_tag_buffer = "" - self._is_first_param = True - self._value_started = False - self._cached_value_type: Optional[str] = None + self.func_call_regex = re.compile(r".*?", re.DOTALL) + # Where the `` of the call open at the buffer start may begin. + self._close_search_from = 0 def has_tool_call(self, text: str) -> bool: """Check if the text contains a GLM-4 format tool call.""" return self.bot_token in text - def detect_and_parse(self, text: str, tools: List[Tool]) -> StreamingParseResult: - """One-time parsing: Detects and parses tool calls in the provided text.""" - idx = text.find(self.bot_token) - normal_text = text[:idx].strip() if idx != -1 else text - if self.bot_token not in text: - return StreamingParseResult(normal_text=normal_text, calls=[]) - match_result_list = re.findall(self.func_call_regex, text, re.DOTALL) - calls = [] - try: - for match_result in match_result_list: - func_detail = self.func_detail_regex.search(match_result) - if func_detail is None: - continue - func_name = func_detail.group(1) if func_detail.group(1) else "" - func_args = func_detail.group(2) if func_detail.group(2) else "" - pairs = self.func_arg_regex.findall(func_args) - - arguments = self._parse_argument_pairs(pairs, func_name, tools) - - match_result = {"name": func_name, "parameters": arguments} - calls.extend(self.parse_base_json(match_result, tools)) - return StreamingParseResult(normal_text=normal_text, calls=calls) - except Exception as e: - logger.error(f"Error in detect_and_parse: {e}") - return StreamingParseResult(normal_text=text) - - def _get_value_type(self, func_name: str, key: str, tools: List[Tool]) -> str: - """Get parameter type from tool definition, with fallback to auto-detection.""" - arg_type = get_argument_type(func_name, key, tools) - if arg_type: - return arg_type - - value_content = self._current_value.strip() if self._current_value else "" - - if not value_content: - return "string" + def _split_call(self, body: str) -> Optional[Tuple[str, str, str]]: + """Split a call's body into (name, markup after the name, argument text). - try: - parsed = json.loads(value_content) - if isinstance(parsed, dict): - return "object" - elif isinstance(parsed, list): - return "array" - elif isinstance(parsed, bool): - return "boolean" - elif isinstance(parsed, (int, float)): - return "number" - elif isinstance(parsed, str): - if parsed.isdigit() or (parsed.startswith("-") and parsed[1:].isdigit()): - return "number" - return "string" - except json.JSONDecodeError: - first_char = value_content[0] if value_content else "" - if first_char.isdigit() or first_char in ["-", "."]: - return "number" - elif first_char in ["{", "["]: - return "object" - elif first_char in ['"', "'"]: - return "string" - - return "string" - - def _format_value_complete(self, value: str, value_type: str) -> str: - """Format complete value based on type.""" - if value_type == "string": - return json.dumps(value, ensure_ascii=False) - elif value_type == "number": - try: - num = _convert_to_number(value.strip()) - return str(num) - except (ValueError, AttributeError): - logger.warning(f"Failed to parse '{value}' as number, treating as string") - return json.dumps(str(value), ensure_ascii=False) - else: - return value + The name line ends at the first newline and the name at its first ``<``; + None when there is no name line. + """ + line_end = _NAME_LINE_END.search(body) + if line_end is None: + return None + name, tag, junk = body[: line_end.start()].partition("<") + return name, tag + junk, body[line_end.end() :] - def _process_xml_to_json_streaming( - self, raw_increment: str, func_name: str, tools: List[Tool] - ) -> str: - """Convert XML increment to JSON streaming output using state machine. + def _call_name(self, name: str, junk: str, tools: List[Tool]) -> Optional[str]: + """The name a call is delivered under; falsy when it is not a call. - Processes XML fragments character by character and converts them - to JSON format incrementally, maintaining state across calls. + A declared tool the name maps onto wins over the name as written; markup + after the name is only accepted when it resolves onto a declared tool. """ - json_output = "" - - for char in raw_increment: - self._xml_tag_buffer += char - - if self._stream_state in [StreamState.INIT, StreamState.BETWEEN]: - if self._xml_tag_buffer.endswith(""): - self._stream_state = StreamState.IN_KEY - self._current_key = "" - self._xml_tag_buffer = "" - json_output += "{" if self._is_first_param else ", " - self._is_first_param = False - - elif self._stream_state == StreamState.IN_KEY: - if self._xml_tag_buffer.endswith(""): - self._current_key = self._xml_tag_buffer[:-10].strip() - self._xml_tag_buffer = "" - self._stream_state = StreamState.WAITING_VALUE - json_output += json.dumps(self._current_key, ensure_ascii=False) + ": " - - elif self._stream_state == StreamState.WAITING_VALUE: - if self._xml_tag_buffer.endswith(""): - self._stream_state = StreamState.IN_VALUE - self._current_value = "" - self._xml_tag_buffer = "" - self._value_started = False - self._cached_value_type = self._get_value_type( - func_name, self._current_key, tools - ) - - elif self._stream_state == StreamState.IN_VALUE: - if self._xml_tag_buffer.endswith(""): - final_value = self._xml_tag_buffer[:-12] - self._current_value += final_value - - value_type = self._cached_value_type or "string" - - if self._value_started: - if final_value: - if value_type == "string": - json_output += json.dumps(final_value, ensure_ascii=False)[1:-1] - else: - json_output += final_value - if value_type == "string": - json_output += '"' - else: - json_output += self._format_value_complete(self._current_value, value_type) - - self._xml_tag_buffer = "" - self._stream_state = StreamState.BETWEEN - self._current_value = "" - self._value_started = False - self._cached_value_type = None - else: - closing_tag = "" - is_potential_closing = len(self._xml_tag_buffer) <= len( - closing_tag - ) and closing_tag.startswith(self._xml_tag_buffer) - - if not is_potential_closing: - content = self._xml_tag_buffer - value_type = self._cached_value_type or "string" - - if value_type == "string": - if not self._value_started: - json_output += '"' - self._value_started = True - if content: - json_output += json.dumps(content, ensure_ascii=False)[1:-1] - self._current_value += content - self._xml_tag_buffer = "" - elif value_type == "number": - if content: - if not self._value_started: - self._value_started = True - json_output += content - self._current_value += content - self._xml_tag_buffer = "" - else: - if content: - if not self._value_started: - self._value_started = True - json_output += content - self._current_value += content - self._xml_tag_buffer = "" - - return json_output + tool_indices = self._get_tool_indices(tools) + name = name.strip() + if junk: + return self.resolve_tool_name((name + junk).strip(), tool_indices) + return name and (self.resolve_tool_name(name, tool_indices) or name) + + def _parse_call_segment( + self, segment: str, tools: List[Tool] + ) -> Tuple[List[ToolCallItem], str]: + """Parse one ``...`` into (calls, text to release). + + A ```` after the name restarts the call: the abandoned prefix + is released and the rest parsed. A call is released whole as text when it + is not one (see ``_call_name``) or its argument text is not all pairs (see + ``parse_argument_text``). + """ + parts = self._split_call(segment[len(self.bot_token) : -len(self.eot_token)]) + restart = -1 if parts is None else parts[1].rfind(self.bot_token) + if restart != -1: + cut = len(self.bot_token) + len(parts[0]) + restart + calls, released = self._parse_call_segment(segment[cut:], tools) + return calls, segment[:cut] + released + name = parts and self._call_name(parts[0], parts[1], tools) + pairs = parse_argument_text(parts[2]) if name else None + if pairs is None: + logger.warning(f"Releasing {len(segment)} characters of unparsable tool-call markup") + return [], segment + arguments = parse_argument_pairs(pairs, name, tools) + return self.parse_base_json({"name": name, "parameters": arguments}, tools), "" + + def detect_and_parse(self, text: str, tools: List[Tool]) -> StreamingParseResult: + """One-time parsing: Detects and parses tool calls in the provided text.""" + if self.bot_token not in text: + return StreamingParseResult(normal_text=text, calls=[]) + normal_parts, calls, last_end = [], [], 0 + for match in self.func_call_regex.finditer(text): + segment_calls, released = self._parse_call_segment(match.group(0), tools) + normal_parts += [text[last_end : match.start()], released] + calls += segment_calls + last_end = match.end() + normal_parts.append(text[last_end:]) + return StreamingParseResult(normal_text="".join(normal_parts).strip(), calls=calls) def parse_streaming_increment(self, new_text: str, tools: List[Tool]) -> StreamingParseResult: - """Streaming incremental parsing for GLM-4 format. + """Release text that cannot start a call and report at most one complete call. - Uses a state machine to convert XML to JSON incrementally for - true character-by-character streaming. + Text after a reported call stays buffered, so the text returned with a + call always precedes it. """ self._buffer += new_text - current_text = self._buffer + return self._consume(tools, at_end=False) - has_tool_call = self.bot_token in current_text + def finish(self, tools: List[Tool]) -> StreamingParseResult: + """Report the calls the stream still holds and release the rest as text.""" + return self._consume(tools, at_end=True) - if not has_tool_call: - is_potential_start = any( - self.bot_token.startswith(current_text[-i:]) - for i in range(1, min(len(current_text), len(self.bot_token)) + 1) - ) + def _release(self, length: int) -> str: + """Remove and return the first `length` characters of the buffer.""" + released, self._buffer = self._buffer[:length], self._buffer[length:] + self._close_search_from = 0 + return released - if not is_potential_start: - output_text = current_text - self._buffer = "" - if self.eot_token in output_text: - output_text = output_text.replace(self.eot_token, "") - return StreamingParseResult(normal_text=output_text) - else: - return StreamingParseResult(normal_text="", calls=[]) + def _consume(self, tools: List[Tool], at_end: bool) -> StreamingParseResult: + """Turn the buffer into text and complete calls. - if not hasattr(self, "_tool_indices"): - self._tool_indices = self._get_tool_indices(tools) - - calls: list[ToolCallItem] = [] - try: - partial_match = re.search( - pattern=r"(.*?)(?:\\n|\n)(.*?)(|$)", - string=current_text, - flags=re.DOTALL, + Mid-stream this stops after the first call and keeps a call still open; + at the end of the stream it takes everything, an unterminated call as text. + """ + normal_parts, calls = [], [] + while self._buffer and (at_end or not calls): + start = self._buffer.find(self.bot_token) + if start == -1: + held = 0 if at_end else self._ends_with_partial_token(self._buffer, self.bot_token) + normal_parts.append(self._release(len(self._buffer) - held)) + break + if start: + normal_parts.append(self._release(start)) + end = self._buffer.find( + self.eot_token, max(len(self.bot_token), self._close_search_from) ) - if partial_match: - func_name_raw = partial_match.group(1) - func_args_raw = partial_match.group(2) - is_tool_end = partial_match.group(3) - - if func_name_raw is None or not func_name_raw.strip(): - return StreamingParseResult(normal_text="", calls=[]) - - func_name = func_name_raw.strip() - func_args_raw = func_args_raw.strip() if func_args_raw else "" - - if self.current_tool_id == -1: - self.current_tool_id = 0 - self.prev_tool_call_arr = [] - self.streamed_args_for_tool = [""] - self._streamed_raw_length = 0 - self.current_tool_name_sent = False - self._reset_streaming_state() - - while len(self.prev_tool_call_arr) <= self.current_tool_id: - self.prev_tool_call_arr.append({}) - while len(self.streamed_args_for_tool) <= self.current_tool_id: - self.streamed_args_for_tool.append("") - - if not self.current_tool_name_sent: - calls.append( - ToolCallItem( - tool_index=self.current_tool_id, - name=func_name, - parameters="", - ) - ) - self.current_tool_name_sent = True - self._streamed_raw_length = 0 - self._reset_streaming_state() - self.prev_tool_call_arr[self.current_tool_id] = { - "name": func_name, - "arguments": {}, - } - else: - current_raw_length = len(func_args_raw) - - if current_raw_length > self._streamed_raw_length: - raw_increment = func_args_raw[self._streamed_raw_length :] - - json_increment = self._process_xml_to_json_streaming( - raw_increment, func_name, tools - ) - - self._streamed_raw_length = current_raw_length - - if json_increment: - calls.append( - ToolCallItem( - tool_index=self.current_tool_id, - name=None, - parameters=json_increment, - ) - ) - self._last_arguments += json_increment - self.streamed_args_for_tool[self.current_tool_id] += json_increment - - if is_tool_end == self.eot_token: - if self._is_first_param: - empty_object = "{}" - calls.append( - ToolCallItem( - tool_index=self.current_tool_id, - name=None, - parameters=empty_object, - ) - ) - self._last_arguments += empty_object - elif not self._last_arguments.endswith("}"): - closing_brace = "}" - calls.append( - ToolCallItem( - tool_index=self.current_tool_id, - name=None, - parameters=closing_brace, - ) - ) - self._last_arguments += closing_brace - self.streamed_args_for_tool[self.current_tool_id] += closing_brace - - try: - pairs = self.func_arg_regex.findall(func_args_raw) - if pairs: - arguments = self._parse_argument_pairs(pairs, func_name, tools) - self.prev_tool_call_arr[self.current_tool_id]["arguments"] = ( - arguments - ) - except Exception as e: - logger.debug(f"Failed to parse arguments: {e}") - - self._buffer = current_text[partial_match.end(3) :] - - result = StreamingParseResult(normal_text="", calls=calls) - self.current_tool_id += 1 - self._last_arguments = "" - self.current_tool_name_sent = False - self._streamed_raw_length = 0 - self._reset_streaming_state() - return result - - return StreamingParseResult(normal_text="", calls=calls) - - except Exception as e: - logger.error(f"Error in parse_streaming_increment: {e}") - return StreamingParseResult(normal_text=current_text) - - def _parse_argument_pairs( - self, pairs: List[Tuple[str, str]], func_name: str, tools: List[Tool] - ) -> Dict[str, Any]: - """Parse argument key-value pairs with type coercion.""" - arguments = {} - for arg_key, arg_value in pairs: - arg_key = arg_key.strip() - arg_value = arg_value.strip() - arg_type = get_argument_type(func_name, arg_key, tools) - parsed_value, is_good_json = parse_arguments(arg_value, arg_type) - - if arg_type == "string": - if isinstance(parsed_value, str): - arguments[arg_key] = parsed_value - elif isinstance(parsed_value, (dict, list)): - arguments[arg_key] = json.dumps(parsed_value, ensure_ascii=False) + if end == -1: + if at_end: + normal_parts.append(self._release(len(self._buffer))) else: - arguments[arg_key] = str(parsed_value) - elif arg_type is None: - arguments[arg_key] = parsed_value if is_good_json else arg_value - else: - arguments[arg_key] = parsed_value if is_good_json else arg_value - - return arguments + self._close_search_from = len(self._buffer) - len(self.eot_token) + 1 + break + segment_calls, released = self._parse_call_segment( + self._release(end + len(self.eot_token)), tools + ) + normal_parts.append(released) + for call in segment_calls: + self.current_tool_id += 1 + call.tool_index = self.current_tool_id + calls += segment_calls + return StreamingParseResult(normal_text="".join(normal_parts), calls=calls) def supports_structural_tag(self) -> bool: return False diff --git a/tests/unittest/llmapi/apps/test_tool_parsers.py b/tests/unittest/llmapi/apps/test_tool_parsers.py index ccc82e9adc7e..c9dfca019504 100644 --- a/tests/unittest/llmapi/apps/test_tool_parsers.py +++ b/tests/unittest/llmapi/apps/test_tool_parsers.py @@ -14,7 +14,9 @@ # limitations under the License. import abc +import functools import json +import time from contextlib import AbstractContextManager from typing import Callable, Iterator, NamedTuple from unittest.mock import Mock @@ -2072,26 +2074,19 @@ def test_parse_streaming_increment_complete_tool_call( self, sample_tools, parser): """Test streaming parser with complete tool call in chunks.""" - # Send bot token with function name + # Nothing is reported until the call is complete result = parser.parse_streaming_increment("get_weather\n", sample_tools) + assert result.calls == [] - # Should send tool name - assert len(result.calls) == 1 - assert result.calls[0].name == "get_weather" - assert result.calls[0].parameters == "" - - # Send arguments result = parser.parse_streaming_increment( "location\n" "SF\n" "", sample_tools) - # Should stream arguments and complete the tool call - all_params = "".join(call.parameters for call in result.calls - if call.parameters) - assert "location" in all_params - assert "SF" in all_params + # One item carries the name and the arguments + assert [(c.name, json.loads(c.parameters)) + for c in result.calls] == [("get_weather", dict(location="SF"))] def test_parse_streaming_increment_multiple_tools_streaming( self, sample_tools, parser): @@ -2106,40 +2101,27 @@ def test_parse_streaming_increment_multiple_tools_streaming( "", sample_tools) # Second tool - result = parser.parse_streaming_increment("search_web\n", - sample_tools) + result = parser.parse_streaming_increment( + "search_web\n", sample_tools) - # Should have started second tool - assert len(result.calls) == 1 - assert result.calls[0].name == "search_web" - assert result.calls[0].parameters == "" - assert result.calls[0].tool_index == 1 + assert [(c.tool_index, c.name) + for c in result.calls] == [(1, "search_web")] def test_parse_streaming_multiple_params(self, sample_tools, parser): - """Test streaming parser handles multiple parameters.""" - - # Send function name + """The arguments arrive together once the call closes.""" parser.parse_streaming_increment("get_weather\n", sample_tools) - - # Send first parameter result1 = parser.parse_streaming_increment( "location\n" "NYC\n", sample_tools) + assert result1.calls == [] - params1 = "".join(call.parameters for call in result1.calls - if call.parameters) - assert "location" in params1 - - # Send second parameter and close result2 = parser.parse_streaming_increment( "unit\n" "celsius\n" "", sample_tools) - - params2 = "".join(call.parameters for call in result2.calls - if call.parameters) - assert "unit" in params2 + assert [json.loads(c.parameters) for c in result2.calls + ] == [dict(location="NYC", unit="celsius")] def test_detect_and_parse_multiple_params(self, sample_tools): """Test one-shot parsing with multiple parameters.""" @@ -2214,16 +2196,14 @@ def test_glm4_format_compliance(self, sample_tools, parser): def test_streaming_no_args(self, sample_tools, parser): """Test streaming a tool call with no arguments.""" - # First increment sends the tool name result1 = parser.parse_streaming_increment("get_weather\n", sample_tools) - names = [c.name for c in result1.calls if c.name] - assert "get_weather" in names + assert result1.calls == [] - # Second increment closes the tool call with empty args + # Closing the call reports it with empty arguments result2 = parser.parse_streaming_increment("", sample_tools) - params = "".join(c.parameters for c in result2.calls) - assert "{}" in params + assert [(c.name, c.parameters) + for c in result2.calls] == [("get_weather", "{}")] def test_supports_structural_tag(self, parser): """Test that supports_structural_tag returns False.""" @@ -2356,26 +2336,19 @@ def test_newline_format_also_works(self, sample_tools, parser): def test_parse_streaming_increment_complete_tool_call( self, sample_tools, parser): """Test streaming parser with complete tool call in chunks.""" - # Send bot token with function name and first arg_key + # Nothing is reported until the call is complete result = parser.parse_streaming_increment( "get_weather", sample_tools) + assert result.calls == [] - # Should send tool name (has_arg_key is True) - assert len(result.calls) == 1 - assert result.calls[0].name == "get_weather" - assert result.calls[0].parameters == "" - - # Send arguments result = parser.parse_streaming_increment( "location" "SF" "", sample_tools) - # Should stream arguments and complete the tool call - all_params = "".join(call.parameters for call in result.calls - if call.parameters) - assert "location" in all_params - assert "SF" in all_params + # One item carries the name and the arguments + assert [(c.name, json.loads(c.parameters)) + for c in result.calls] == [("get_weather", dict(location="SF"))] def test_parse_streaming_increment_multiple_tools_streaming( self, sample_tools, parser): @@ -2387,13 +2360,10 @@ def test_parse_streaming_increment_multiple_tools_streaming( # Second tool result = parser.parse_streaming_increment( - "search_web", sample_tools) + "search_web", sample_tools) - # Should have started second tool - assert len(result.calls) == 1 - assert result.calls[0].name == "search_web" - assert result.calls[0].parameters == "" - assert result.calls[0].tool_index == 1 + assert [(c.tool_index, c.name) + for c in result.calls] == [(1, "search_web")] def test_streaming_zero_arg_tool(self, parser): """Test streaming a zero-argument tool call.""" @@ -5721,3 +5691,398 @@ def test_warning_is_keyed_per_parser(self) -> None: for call in mock_logger.warning_once.call_args_list ] assert keys == [self._PARSER_NAME, self._OTHER_PARSER_NAME] + + +# ============================================================================ +# GLM parsers: streaming reports what detect_and_parse reports +# ============================================================================ + + +def _glm_tool(name, properties=None): + """One declared tool; `properties` is its JSON-schema property map.""" + return ChatCompletionToolsParam(type="function", + function=FunctionDefinition( + name=name, + parameters={ + "type": "object", + "properties": properties or {}, + })) + + +def _glm_typed_tool(name, **types): + """A tool whose properties are declared with the given JSON types.""" + return _glm_tool(name, {key: dict(type=t) for key, t in types.items()}) + + +AGENT_TOOLS = [ + _glm_tool(name) for name in ("exec_command", "apply_patch", "update_plan", + "spawn_agent", "followup_task", "exec") +] + +# (emitted name, the name the call is delivered under). +MANGLED_NAMES = [ + # A group qualifier the declaration lacks. + ("functions.exec_command", "exec_command"), + ("collaboration.spawn_agent", "spawn_agent"), + ("functions.collaboration.followup_task", "followup_task"), + # An unbalanced tag fused onto an otherwise exact name. + ("apply_patch", "apply_patch"), + ("exec", "exec"), + ("exec_command", "exec_command"), + ("collcollaboration.spawn_agent", "spawn_agent"), + # Declared and unknown names are delivered as written. + ("exec_command", "exec_command"), + ("no_such_tool", "no_such_tool"), + ("my_exec_command", "my_exec_command"), +] + +# Prose and code that mention a declared tool; never repaired into a call to it. +PROSE_NAMES = [ + "ops_check - re-querying for current status. Since my last report", + "exec_command depended on the tools. Let me use the correct approach", + "exec_sudo_command onCompleteCommand=\"export SKILLS_DIR=/x\"", + "collaboration.immediately_agent(\"target\" => \"/root/builder\")", + "exec_command(cmd=\"cat /workspace/kernel.cu\"", +] + + +def _chunkings(text): + """(label, chunks): whole, per character, fixed sizes, every two-way split.""" + yield "arriving whole", [text] + yield "one character per delta", list(text) + for size in (2, 3, 5, 8, 13, 40): + yield f"{size} characters per delta", [ + text[i:i + size] for i in range(0, len(text), size) + ] + for i in range(1, len(text)): + yield f"split at {i}", [text[:i], text[i:]] + + +def _glm_raw_call(parser_cls, name, argument_text): + """One call around `argument_text`; GLM-4.5 ends the name with a newline.""" + newline = "\n" if parser_cls is Glm4ToolParser else "" + return f"{name}{newline}{argument_text}" + + +def _glm_pairs(*pairs, separator=""): + return "".join( + f"{key}{separator}{value}" + f"{separator}" for key, value in pairs) + + +def _glm_call(parser_cls, name, *pairs): + """One call in `parser_cls`'s own markup.""" + separator = "\n" if parser_cls is Glm4ToolParser else "" + return _glm_raw_call(parser_cls, name, + _glm_pairs(*pairs, separator=separator)) + + +def _glm_stream(parser_cls, chunks, tools): + """(text, [(name, arguments)]) from the increments plus `finish`. + + Each call must arrive complete, as one item with its name and arguments. + """ + parser = parser_cls() + results = [parser.parse_streaming_increment(c, tools) for c in chunks] + results.append(parser.finish(tools)) + calls = [call for result in results for call in result.calls] + assert [c.tool_index for c in calls] == list(range(len(calls))) + assert not parser.current_tool_name_sent + return ("".join(r.normal_text for r in results), [(c.name, c.parameters) + for c in calls]) + + +def _glm_parity(parser_cls, text, tools, chunkings=None): + """Assert every chunking streams detect_and_parse's text and calls. + + Returns the whole-text normal text and its [(name, arguments)]. + """ + whole = parser_cls().detect_and_parse(text, tools) + expected = [(c.name, json.loads(c.parameters)) for c in whole.calls] + for label, chunks in chunkings or _chunkings(text): + normal, calls = _glm_stream(parser_cls, chunks, tools) + assert [(n, json.loads(a)) for n, a in calls] == expected, label + assert normal.strip() == whole.normal_text, label + return whole.normal_text, expected + + +_GLM_SCHEMAS = { + kind: dict(type=kind) + for kind in ("string", "integer", "number", "boolean", "object", "array") +} +_GLM_SCHEMAS.update({"any": {}, "$ref": {"$ref": "#/$defs/F"}, "none": None}) + +# Declared strings are delivered verbatim, whatever they look like, markup +# included. +_GLM_STRING_VALUES = [ + "99797", "true", "1e3", "null", "0099797", " padded \n", "", "εŒ—δΊ¬", + 'say "hi"', "C:\\Users\\test.txt", "line one\nline two", '{"k": "v"}', + 'echo "}" >> a.json', "printf '{\"a\": [1, 2]}' | jq .", + 'let s = ""; run(s)', "ab", + "ends with the tag", " starts with it", + "twoofthem", "tag then newline\nmore", + "", "not a key", "bold", + "1 < 2 && 3 > 2", "struct MMA_Traits<" +] + +# (raw value, schema, delivered value). Other declared types decode, keeping +# the text when the result is not strict JSON; with no resolvable type ("none", +# "any", "$ref") only strict JSON decodes. +_GLM_TYPED_VALUES = [(value, "string", value) for value in _GLM_STRING_VALUES] +_GLM_TYPED_VALUES += [ + ("42", "integer", 42), + ('"42"', "integer", 42), + ("1.5", "number", 1.5), + ("1e308", "number", 1e308), + ("1e309", "number", "1e309"), + ("-1e309", "integer", "-1e309"), + ("NaN", "number", "NaN"), + ("-Infinity", "number", "-Infinity"), + ("true", "boolean", True), + ("{}", "object", {}), + ('{"a": 1e308}', "object", dict(a=1e308)), + ('{"a": 1e309}', "object", '{"a": 1e309}'), + ('{"k": "xy"}', "object", dict(k="xy")), + ('[{\\"id\\": \\"1\\"}]', "array", [dict(id="1")]), + ("['a', 'b']", "array", ["a", "b"]), + ("...", "array", "..."), + ("{1, 2}", "object", "{1, 2}"), + ("{['a']: 1}", "object", "{['a']: 1}"), + ("1j", "number", "1j"), + ("b'x'", "array", "b'x'"), + ("not json", "object", "not json"), + ("8000", "none", 8000), + ('{"a": [1, 2]}', "none", dict(a=[1, 2])), + ("true", "none", True), + ("null", "none", None), + ("ls -la", "none", "ls -la"), + ("", "none", ""), + ("NaN", "none", "NaN"), + ("'single'", "none", "'single'"), + ('{"a": 1}', "any", dict(a=1)), + ('{"a": 1}', "$ref", dict(a=1)), +] + +_GLM_VALUES = sorted({row[0] for row in _GLM_TYPED_VALUES}) + +_GLM_TOOLS = [_glm_typed_tool("f", code="string", n="integer", opts="object")] + +# (argument text of a call to `f`, delivered arguments or None for no call). +# A call is delivered only if its argument text is nothing but separators and +# complete pairs; otherwise its whole markup is released as text. +_GLM_ARGUMENT_TEXTS = [ + ("", {}), + ("\n ", {}), + ("code\na\\n n" + " 1\n", dict(code="a", n=1)), + # A repeated key keeps its first value. + (_glm_pairs(("code", "a"), ("code", "b")), dict(code="a")), + (_glm_pairs(("n", "1"), ("code", "a"), ("n", "2")), dict(n=1, code="a")), + (_glm_pairs(("code", "a"), ("code", "xy")), dict(code="a")), + # A value quoting the closing tag, or ending in a prefix of it. + (_glm_pairs(("code", "''"), + ("n", "3")), dict(code="''", n=3)), + (_glm_pairs(("n", "3"), ("code", "a<")), dict(n=3, code="a<")), + (_glm_pairs(("code", "acodels -la", None), + ("n5", None), + ("codels", None), + (_glm_pairs(("code", "ls -la")) + ".", None), + (_glm_pairs(("code", "ls")) + "code", None), + ("codejunkls", None), + ("codels", None), + ("a" + _glm_pairs(("code", "ls")), None), + (_glm_pairs(("code", "xy")), None), + (_glm_pairs(("code", "a")) + "n2", None), +] + + +@pytest.mark.parametrize("parser_cls", [Glm4ToolParser, Glm47ToolParser]) +class TestGlmStreamingMatchesWholeParse: + """Any chunking streams the calls and text detect_and_parse returns.""" + + def test_declares_the_contract(self, parser_cls): + assert parser_cls.streaming_matches_whole_parse + + @pytest.mark.parametrize("value,schema,expected", _GLM_TYPED_VALUES) + def test_argument_typing(self, parser_cls, value, schema, expected): + properties = {"v": _GLM_SCHEMAS[schema]} if schema != "none" else {} + text = _glm_call(parser_cls, "f", ("v", value)) + + _, calls = _glm_parity(parser_cls, text, [_glm_tool("f", properties)]) + + assert calls == [("f", dict(v=expected))] + + @pytest.mark.parametrize("value", _GLM_VALUES) + def test_every_value_under_every_schema(self, parser_cls, value): + text = _glm_call(parser_cls, "f", ("v", value)) + for schema in _GLM_SCHEMAS.values(): + properties = {} if schema is None else {"v": schema} + _glm_parity(parser_cls, text, [_glm_tool("f", properties)], + [("whole", [text]), ("per character", list(text))]) + + @pytest.mark.parametrize("argument_text,expected", _GLM_ARGUMENT_TEXTS) + def test_argument_text(self, parser_cls, argument_text, expected): + text = _glm_raw_call(parser_cls, "f", argument_text) + + normal, calls = _glm_parity(parser_cls, text, _GLM_TOOLS) + + if expected is None: + assert (normal, calls) == (text, []) + else: + assert (normal, calls) == ("", [("f", expected)]) + + @pytest.mark.parametrize("text", [ + "To end a call the model writes and then stops.", + "The answer is 42. See get_w", + "Cut off: execkvinput", ("k", "v")) + unclosed = _glm_raw_call(parser_cls, "exec", + "cmdls") + text = ("Checking now." + call("exec", ("cmd", 'echo "}" >> a.json')) + + out_of_order + call("get_time") + " Then " + unclosed + + call("execexec", + ("cmd", "ls")) + " is prose. exec") + + normal, calls = _glm_parity(parser_cls, text, + [_glm_typed_tool("exec", cmd="string")]) + + assert calls == [("exec", dict(cmd='echo "}" >> a.json')), + ("get_time", {}), ("exec", dict(cmd="ls"))] + assert normal == ( + "Checking now." + out_of_order + " Then " + unclosed + + "exec is prose. exec") + + def test_a_name_without_its_own_line(self, parser_cls): + """GLM-4.7 reads these calls; GLM-4.5 needs a newline after the name.""" + text = ("get_timeexeck" + "v") + + result = _glm_parity(parser_cls, text, AGENT_TOOLS) + + if parser_cls is Glm4ToolParser: + assert result == (text, []) + else: + assert result == ("", [("get_time", {}), ("exec", dict(k="v"))]) + + def test_text_after_a_call_waits_for_the_next_increment( + self, parser_cls, sample_tools): + """The text an increment returns always precedes its calls.""" + call = _glm_call(parser_cls, "get_weather", ("location", "NYC")) + parser = parser_cls() + + results = [ + parser.parse_streaming_increment(chunk, sample_tools) + for chunk in ("Before. ", call + "After." + call + " Done.", "", "") + ] + + expected = [("Before. ", 0), ("", 1), ("After.", 1), (" Done.", 0)] + assert [(r.normal_text, len(r.calls)) for r in results] == expected + + @pytest.mark.parametrize("emitted,delivered", MANGLED_NAMES) + def test_mangled_name(self, parser_cls, emitted, delivered): + text = _glm_call(parser_cls, emitted, ("cmd", "ls -la /workspace")) + + _, calls = _glm_parity(parser_cls, text, AGENT_TOOLS) + + assert calls == [(delivered, dict(cmd="ls -la /workspace"))] + + @pytest.mark.parametrize("prose", PROSE_NAMES) + def test_prose_is_never_repaired_into_a_call(self, parser_cls, prose): + text = _glm_call(parser_cls, prose, ("cmd", "ls")) + + _, calls = _glm_parity(parser_cls, text, AGENT_TOOLS) + + declared = {tool.function.name for tool in AGENT_TOOLS} + assert not {name for name, _ in calls} & declared + + def test_a_bare_name_is_typed_by_its_qualified_declaration( + self, parser_cls): + tools = [ + _glm_typed_tool("functions.exec", + cmd="string", + max_output_tokens="integer") + ] + text = _glm_call(parser_cls, "exec", ("cmd", "8000"), + ("max_output_tokens", "8000")) + + _, calls = _glm_parity(parser_cls, text, tools) + + assert calls == [("functions.exec", + dict(cmd="8000", max_output_tokens=8000))] + + def test_a_long_separator_run_streams_in_linear_time(self, parser_cls): + """A long run of literal backslash-n separators, two per increment.""" + text = _glm_raw_call(parser_cls, "f", + _glm_pairs(("code", "ls")) + "\\n" * 20000) + chunks = [text[i:i + 2] for i in range(0, len(text), 2)] + start = time.monotonic() + + _, calls = _glm_parity(parser_cls, text, _GLM_TOOLS, + [("two per delta", chunks)]) + + assert calls == [("f", dict(code="ls"))] + assert time.monotonic() - start < 10 + + +# ============================================================================ +# BaseToolParser.resolve_tool_name +# ============================================================================ + +_NAMESPACED = ("functions.exec", "functions.wait", "collaboration.spawn_agent", + "collaboration.list_agents") + + +@pytest.mark.parametrize( + "name,declared,expected", + [ + # A bare name maps onto its qualified declaration, fused markup or not. + ("exec", _NAMESPACED, "functions.exec"), + ("spawn_agent", _NAMESPACED, "collaboration.spawn_agent"), + ("exec", _NAMESPACED, "functions.exec"), + ("exec", _NAMESPACED, "functions.exec"), + ("functions.exec ", _NAMESPACED, "functions.exec"), + # A qualifier the declaration lacks is dropped; an exact name wins. + ("functions.exec_command", ("exec_command", ), "exec_command"), + ("exec", ("exec", "functions.exec"), "exec"), + # An ambiguous tail, prose mentioning a tool, unknown or empty names. + ("status", ("a.status", "b.status"), None), + ("collab? no. Use exec.exec", _NAMESPACED, None), + ("ops_check - re-querying for current status", _NAMESPACED, None), + ("exec surg? No, use proper functions.exec tool", _NAMESPACED, None), + ("exec_command_placeholder", _NAMESPACED, None), + ("totally_undeclared", _NAMESPACED, None), + ("", _NAMESPACED, None), + (None, _NAMESPACED, None), + ]) +def test_resolve_tool_name(name, declared, expected): + tool_indices = {tool: index for index, tool in enumerate(declared)} + assert BaseToolParser.resolve_tool_name(name, tool_indices) == expected + + +def test_parse_base_json_reports_the_name_as_emitted(sample_tools): + """Only the GLM parsers resolve names; the shared path keeps them as is.""" + action = dict(name="functions.get_weather", parameters={}) + + calls = ConcreteToolParser().parse_base_json(action, sample_tools) + + assert [(c.name, c.tool_index) + for c in calls] == [("functions.get_weather", -1)] From c4c08d0923f31c720897c18f0224b0c3304dd013 Mon Sep 17 00:00:00 2001 From: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> Date: Wed, 7 Oct 2026 15:15:14 +0000 Subject: [PATCH 02/43] [None][feat] GLM reasoning parser There was no reasoning parser for the GLM family. GlmReasoningParser, registered as glm, glm45, glm47 and glm_moe_dsa, runs one scanner for streaming and whole text, so the two splits agree: - glm, glm47 and glm_moe_dsa start inside the block, which the GLM-4.7 and GLM-5 templates prefill; with enable_thinking false those templates close it in the prompt, and the parser starts outside. glm45 opens the block at a at the start of the output, as GLM-4.5/4.6 write it. - The block ends at the first , or at a opened before it, so a call made without closing the block reaches the tool parser. A later is text. - A before any content, or directly before the first , closes the block again and is dropped. Text from the first on is kept verbatim, so tool arguments that quote the markup survive. The partial-marker holdback is one module-level helper, shared with the Gemma4 and Kimi K3 parsers, which each had their own copy. The CLI reference lists the new keys. The tests are a table checked whole, at small chunk sizes and over every three-way split, plus a seeded stream-versus-whole comparison and a check that deepseek-r1 is unaffected. Signed-off-by: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> --- tensorrt_llm/llmapi/reasoning_parser.py | 158 ++++++++++++++---- .../references/trtllm_serve_cli.yaml | 2 +- .../unittest/llmapi/test_reasoning_parser.py | 101 ++++++++++- 3 files changed, 227 insertions(+), 34 deletions(-) diff --git a/tensorrt_llm/llmapi/reasoning_parser.py b/tensorrt_llm/llmapi/reasoning_parser.py index 6dfc24547b07..2a7ca665b941 100644 --- a/tensorrt_llm/llmapi/reasoning_parser.py +++ b/tensorrt_llm/llmapi/reasoning_parser.py @@ -323,6 +323,128 @@ def finish(self) -> ReasoningParserResult: return ReasoningParserResult(content=remaining) +def _trailing_partial_marker(text: str, *markers: str) -> int: + """Length of the longest suffix of `text` that is a proper prefix of a marker. + + A streaming parser withholds that many characters: they may still grow + into a marker split across deltas. + """ + best = 0 + for marker in markers: + for length in range(min(len(text), len(marker) - 1), best, -1): + if marker.startswith(text[-length:]): + best = length + break + return best + + +@register_reasoning_parser("glm", reasoning_at_start=True) +@register_reasoning_parser("glm45") +@register_reasoning_parser("glm47", reasoning_at_start=True) +@register_reasoning_parser("glm_moe_dsa", reasoning_at_start=True) +class GlmReasoningParser(BaseReasoningParser): + """Reasoning parser for the GLM family. + + GLM-4.7 and GLM-5 templates prefill `` (`reasoning_at_start`), or + `` when `enable_thinking` is false. GLM-4.5/4.6 (`glm45`) do not + prefill ``; the model opens the block at the start of its output. + + The block ends at the first ``, or at a `` opened + before it; a later `` is text. A `` before any content, or + directly before the first ``, closes the block again and is + dropped. Text from the first `` on is kept verbatim. + """ + + reasoning_start = "" + reasoning_end = "" + _tool_call_start = "" + + def __init__(self, + *, + reasoning_at_start: bool = False, + chat_template_kwargs: Optional[dict[str, Any]] = None) -> None: + super().__init__(chat_template_kwargs=chat_template_kwargs) + # The templates close the block in the prompt when `enable_thinking` + # is passed and false. + if not (chat_template_kwargs or {}).get("enable_thinking", True): + reasoning_at_start = False + self.reasoning_at_start = reasoning_at_start + # "start": the block may still open after leading whitespace; + # "closed": the block ended and no content followed yet. + self._state = "reasoning" if reasoning_at_start else "start" + self._buffer = "" + + def parse(self, text: str) -> ReasoningParserResult: + # The whole text as one delta to a fresh parser, so both paths agree. + parser = GlmReasoningParser(reasoning_at_start=self.reasoning_at_start) + result, tail = parser.parse_delta(text), parser.finish() + result.content += tail.content + result.reasoning_content += tail.reasoning_content + return result + + def parse_delta(self, delta_text: str) -> ReasoningParserResult: + data, self._buffer = self._buffer + delta_text, "" + content, reasoning = [], [] + while data: + state = self._state + if state == "tool_calls": + content.append(data) + break + if state == "start": + stripped = data.lstrip() + if stripped.startswith(self.reasoning_start): + data = stripped[len(self.reasoning_start):] + self._state = "reasoning" + elif self.reasoning_start.startswith(stripped): + self._buffer = data + break + else: + self._state = "closed" + continue + out = reasoning if state == "reasoning" else content + end = data.find(self.reasoning_end) + call = data.find(self._tool_call_start) + if call != -1 and (end == -1 or call < end): + # A tool call ends the block; everything after it is content. + out.append(data[:call]) + content.append(data[call:]) + self._state = "tool_calls" + break + if end == -1: + cut = len(data) - _trailing_partial_marker( + data, self.reasoning_end, self._tool_call_start) + out.append(data[:cut]) + self._buffer = data[cut:] + if cut and state == "closed": + self._state = "content" + break + out.append(data[:end]) + rest = data[end + len(self.reasoning_end):] + if state == "reasoning": + self._state = "closed" + elif end: + self._state = "content" + if self._state == "closed" or rest.startswith( + self._tool_call_start): + # The tag ends the block or closes it again: drop it. + data = rest + elif self._tool_call_start.startswith(rest): + # Hold the tag until it is known whether a tool call follows. + self._buffer = data[end:] + break + else: + content.append(self.reasoning_end) + data = rest + return ReasoningParserResult(content="".join(content), + reasoning_content="".join(reasoning)) + + def finish(self) -> ReasoningParserResult: + remaining, self._buffer = self._buffer, "" + if self._state == "reasoning": + return ReasoningParserResult(reasoning_content=remaining) + return ReasoningParserResult(content=remaining) + + @register_reasoning_parser("deepseek_v4") class DeepSeekV4ReasoningParser(BaseReasoningParser): """DeepSeek-V4 parser selected by thinking-mode chat template kwargs.""" @@ -722,18 +844,6 @@ def parse(self, text: str) -> ReasoningParserResult: reasoning_content="".join(reasoning_parts), ) - @staticmethod - def _partial_suffix_len(buf: str, tag: str) -> int: - """Return length of the longest suffix of ``buf`` that is a prefix of ``tag``. - - Used to hold back potential partial delimiters during streaming. - """ - max_len = min(len(buf), len(tag) - 1) - for k in range(max_len, 0, -1): - if tag.startswith(buf[-k:]): - return k - return 0 - def parse_delta(self, delta_text: str) -> ReasoningParserResult: self._buffer += delta_text content_parts: list[str] = [] @@ -742,7 +852,7 @@ def parse_delta(self, delta_text: str) -> ReasoningParserResult: if not self.in_reasoning: idx = self._buffer.find(self.CHANNEL_OPEN) if idx == -1: - hold = self._partial_suffix_len(self._buffer, + hold = _trailing_partial_marker(self._buffer, self.CHANNEL_OPEN) emit_len = len(self._buffer) - hold content_parts.append(self._buffer[:emit_len]) @@ -754,7 +864,7 @@ def parse_delta(self, delta_text: str) -> ReasoningParserResult: else: idx = self._buffer.find(self.CHANNEL_CLOSE) if idx == -1: - hold = self._partial_suffix_len(self._buffer, + hold = _trailing_partial_marker(self._buffer, self.CHANNEL_CLOSE) emit_len = len(self._buffer) - hold reasoning_parts.append(self._buffer[:emit_len]) @@ -1002,22 +1112,6 @@ def _reset(self) -> None: self._state = "body" self._channel: Optional[str] = self._initial_channel - @staticmethod - def _partial_suffix_len(text: str, markers: tuple[str, ...]) -> int: - """Length of the longest text suffix that is a proper prefix of any marker. - - Markers may contain internal ``<`` (e.g. ``<|close|>tools<|sep|>``), - so every suffix length up to ``len(marker) - 1`` must be checked, not - just the one starting at the last ``<``. - """ - best = 0 - for marker in markers: - for length in range(min(len(text), len(marker) - 1), best, -1): - if marker.startswith(text[-length:]): - best = length - break - return best - def _emit(self, text: str, content: list, reasoning: list) -> None: if not text: return @@ -1041,7 +1135,7 @@ def _step(self, content: list, reasoning: list) -> bool: indices = [(buf.find(m), m) for m in markers] indices = [(i, m) for i, m in indices if i != -1] if not indices: - hold = self._partial_suffix_len(buf, markers) + hold = _trailing_partial_marker(buf, *markers) emit_len = len(buf) - hold self._emit(buf[:emit_len], content, reasoning) self._buffer = buf[emit_len:] @@ -1089,7 +1183,7 @@ def _step(self, content: list, reasoning: list) -> bool: if self._state == "tools_pass": idx = buf.find(self.TOOLS_END) if idx == -1: - hold = self._partial_suffix_len(buf, (self.TOOLS_END, )) + hold = _trailing_partial_marker(buf, self.TOOLS_END) emit_len = len(buf) - hold self._emit(buf[:emit_len], content, reasoning) self._buffer = buf[emit_len:] diff --git a/tests/unittest/api_stability/references/trtllm_serve_cli.yaml b/tests/unittest/api_stability/references/trtllm_serve_cli.yaml index f7f2612a8bd4..09eb4d8bc285 100644 --- a/tests/unittest/api_stability/references/trtllm_serve_cli.yaml +++ b/tests/unittest/api_stability/references/trtllm_serve_cli.yaml @@ -386,7 +386,7 @@ commands: flags: - "--post_processor_hook" reasoning_parser: - type: Choice(['auto', 'deepseek-r1', 'deepseek_v4', 'gemma4', 'kimi_k2', 'kimi_k25', 'kimi_k3', 'laguna', 'minimax_m2', 'minimax_m2_append_think', 'minimax_m3', 'nano-v3', 'nemotron-v3', 'poolside_v1', 'qwen3', 'qwen3_5']) + type: Choice(['auto', 'deepseek-r1', 'deepseek_v4', 'gemma4', 'glm', 'glm45', 'glm47', 'glm_moe_dsa', 'kimi_k2', 'kimi_k25', 'kimi_k3', 'laguna', 'minimax_m2', 'minimax_m2_append_think', 'minimax_m3', 'nano-v3', 'nemotron-v3', 'poolside_v1', 'qwen3', 'qwen3_5']) default: null status: prototype required: false diff --git a/tests/unittest/llmapi/test_reasoning_parser.py b/tests/unittest/llmapi/test_reasoning_parser.py index 88700cc26114..af8308095281 100644 --- a/tests/unittest/llmapi/test_reasoning_parser.py +++ b/tests/unittest/llmapi/test_reasoning_parser.py @@ -13,8 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +import itertools import json import os +import random import pytest @@ -1207,7 +1209,7 @@ def test_kimi_k3_reasoning_parser_stream_matches_parse(chunk_size: int, """Streaming in arbitrary chunkings must reproduce the one-shot parse. This sweeps every marker-split position, which is the riskiest logic in - the parser (`_partial_suffix_len` suffix holds). + the parser (`_trailing_partial_marker` suffix holds). """ if thinking: text = _k3_completion("Let me think.", "Answer: 4.", K3_TOOLS_SECTION) @@ -1255,3 +1257,100 @@ def test_auto_detect_kimi_k3(tmp_path): result = resolve_auto_reasoning_parser(model_dir) assert result == "kimi_k3" + + +_ARGS = "grep -c \"\" log" +_OFF = {"enable_thinking": False} + +# (key, chat_template_kwargs, text, content, reasoning). `glm`, `glm47` and +# `glm_moe_dsa` start inside the prefilled block; `glm45` waits for ``. +_GLM_CASES = [ + ("glm", None, "planThe answer is 42.", "The answer is 42.", "plan"), + # A repeated close is dropped before any content or before the first call. + ("glm", None, "recovered.Let me retry.f", + "Let me retry.f", "recovered."), + ("glm", None, "plandone", "done", "plan"), + ("glm", None, "plan", "", "plan"), + ("glm", None, "planIt printed here.", + "It printed here.", "plan"), + ("glm", None, "rtext", "text", "r"), + # `` never opens a second block. + ("glm", None, "rab\nb\n", "", "a"), + # A tool call ends the block; text that only resembles a marker is kept. + ("glm", None, "Plan.fm", + "fm", "Plan."), + ("glm", None, "plansayf", "sayf", "plan"), + ("glm", None, "xy", "y", "xexec" + _ARGS, + "exec" + _ARGS, "Done."), + ("glm47", None, "Plan.exec" + _ARGS, "exec" + _ARGS, + "Plan."), + # With thinking off the template closed the block in the prompt. + ("glm47", _OFF, "Paris.", "Paris.", ""), + ("glm_moe_dsa", _OFF, "Paris.f", + "Paris.f", ""), + # The block opens only at the start of the output. + ("glm45", None, "\nlet me thinkAnswer.", "Answer.", + "let me think"), + ("glm45", None, " planf", "f", "plan"), + ("glm45", None, "\nAnswer.", "\nAnswer.", ""), + ("glm45", None, "Answer.xy", "Answer.xy", ""), + ("glm45", None, "a", "a", ""), + ("glm45", None, "hellof", "hellof", ""), +] + + +def _glm_stream(key: str, kwargs: dict | None, + deltas: list[str]) -> tuple[str, str]: + parser = ReasoningParserFactory.create_reasoning_parser(key, kwargs) + results = [parser.parse_delta(delta) for delta in deltas] + results.append(parser.finish()) + content = "".join(r.content for r in results) + return content, "".join(r.reasoning_content for r in results) + + +@pytest.mark.parametrize(("key", "kwargs", "text", "content", "reasoning"), + _GLM_CASES) +def test_glm_reasoning_parser(key: str, kwargs: dict | None, text: str, + content: str, reasoning: str) -> None: + expected = (content, reasoning) + result = ReasoningParserFactory.create_reasoning_parser(key, + kwargs).parse(text) + assert (result.content, result.reasoning_content) == expected + # Small chunks spread a marker over many deltas; the three-way splits put + # two delta boundaries at every pair of offsets. + for size in (1, 2, 4): + chunks = [text[i:i + size] for i in range(0, len(text), size)] + assert _glm_stream(key, kwargs, chunks) == expected, chunks + for i, j in itertools.combinations_with_replacement(range(len(text) + 1), + 2): + chunks = [text[:i], text[i:j], text[j:]] + assert _glm_stream(key, kwargs, chunks) == expected, chunks + + +@pytest.mark.parametrize("key", ["glm", "glm45"]) +def test_glm_streaming_matches_parse_on_random_text(key: str) -> None: + pieces = [ + "", "", "", "", "<", "", "call>", "a", " ", "\n" + ] + rng = random.Random(0) + for _ in range(3000): + text = "".join(rng.choices(pieces, k=rng.randint(0, 10))) + whole = ReasoningParserFactory.create_reasoning_parser(key).parse(text) + assert "" not in whole.reasoning_content + cuts = [i for i in range(1, len(text)) if rng.random() < 0.3] + chunks = [text[i:j] for i, j in zip([0] + cuts, cuts + [len(text)])] + expected = (whole.content, whole.reasoning_content) + assert _glm_stream(key, None, chunks) == expected, chunks + + +def test_deepseek_r1_keeps_a_second_close_as_content() -> None: + result = ReasoningParserFactory.create_reasoning_parser( + "deepseek-r1").parse("abf") + assert result.content == "bf" From f9f698a6ee994418269a026cb14bfb23155a42ef Mon Sep 17 00:00:00 2001 From: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> Date: Fri, 28 Aug 2026 01:15:34 -0700 Subject: [PATCH 03/43] [None][fix] Responses API: honour web_search when the client disables external access A client may attach the web_search tool with external access explicitly off. Codex does exactly that on every turn: {"type": "web_search", "external_web_access": false} The guard added for the Responses surface refused any request carrying a web_search tool, so this locked Codex out of the server entirely: every turn came back 400, and because Codex treats a 400 as retryable it burned its retry budget and reported "We're currently experiencing high demand" instead of the actionable message. The guard exists because silently dropping the tool lets the model answer from memory while the client believes a live search was folded in - a wrong answer the client cannot detect. That reasoning does not apply when the client has said up front that it does not want the open web: there is no live result to withhold and nothing it could be misled about. Refusing there is a false positive, not a safety property. So only refuse when the request actually asks for external search. A tool with external_web_access absent or true is unchanged and still fails loudly. Neither the model catalog nor `[tools] web_search = false` in Codex's own config suppresses the tool, so there was no client-side workaround short of editing the catalog by hand. Signed-off-by: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> (cherry picked from commit 6d4d5a47c08c619de3d7f5f27225ccc5d7964fbc) Signed-off-by: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> --- tensorrt_llm/serve/responses_web_search.py | 58 ++++++++++++++++- .../llmapi/apps/test_web_search_shared.py | 62 +++++++++++++++++++ 2 files changed, 117 insertions(+), 3 deletions(-) diff --git a/tensorrt_llm/serve/responses_web_search.py b/tensorrt_llm/serve/responses_web_search.py index d4a233413c77..baf515182fd2 100644 --- a/tensorrt_llm/serve/responses_web_search.py +++ b/tensorrt_llm/serve/responses_web_search.py @@ -86,6 +86,43 @@ def web_search_tool_spec(tools: Optional[Sequence[Any]]) -> Optional[WebSearchTo return None +def _external_access_flag(tools: Optional[Sequence[Any]]) -> Any: + """The tool's ``external_web_access`` as it survived request validation. + + Returns the sentinel string "absent" when the field is not there at all, + which is a different situation from an explicit True: a client that never + set it may simply be using an older tool shape. Reported in the rejection + message because it is the field the decision turns on, and it is otherwise + invisible to whoever has to act on the error. + """ + for tool in tools or []: + if not is_web_search_tool(tool): + continue + if isinstance(tool, dict): + return tool.get("external_web_access", "absent") + return getattr(tool, "external_web_access", "absent") + return "absent" + + +def _wants_external_search(tools: Optional[Sequence[Any]]) -> bool: + """Whether the client is actually asking for live external searches. + + The tool carries ``external_web_access``. A client that sets it to False is + declaring up front that it does not want the server reaching the open web, + so there is no live result for the server to fail to deliver and nothing + the client could be misled about. Absent or True means the ordinary + "search the web for me" request. + """ + for tool in tools or []: + if not is_web_search_tool(tool): + continue + external = getattr(tool, "external_web_access", None) + if external is None and isinstance(tool, dict): + external = tool.get("external_web_access") + return external is not False + return False + + def web_search_rejection_reason(tools: Optional[Sequence[Any]]) -> Optional[str]: """Why this request's web_search tool cannot be honoured, or None. @@ -96,7 +133,22 @@ def web_search_rejection_reason(tools: Optional[Sequence[Any]]) -> Optional[str] the model never searched. Rejecting is recoverable (drop the tool and retry); a silently unsearched answer is not. """ - if web_search_tool_spec(tools) is None: + spec = web_search_tool_spec(tools) + if spec is None: + return None + # Name the tool that triggered this. A bare "'web_search' cannot be + # honoured" leaves the operator guessing which of the client's tools it + # means and what shape it arrived in, and the tool is one a client often + # attaches implicitly rather than on purpose. + seen = f" (tool type {spec.type!r}, external_web_access={_external_access_flag(tools)!r})" + if not _wants_external_search(tools): + # The client attached the tool with external access explicitly off, so + # it is not waiting on live results and cannot be misled by their + # absence. Codex sends exactly this - {"type": "web_search", + # "external_web_access": false} - when its model catalog says the model + # has no search tool; refusing that costs the client the whole server + # over a search it never asked for. A client that leaves the flag on is + # asking for live search and still gets a loud failure below. return None if load_web_search_config().enabled: # A provider is configured, so the operator does expect live search, @@ -104,9 +156,9 @@ def web_search_rejection_reason(tools: Optional[Sequence[Any]]) -> Optional[str] # the pieces live here and in web_search.py without a driver. return ( "a provider is configured but the per-request search loop is " - "not wired into this endpoint yet" + f"not wired into this endpoint yet{seen}" ) - return "no web search provider is configured on this server" + return f"no web search provider is configured on this server{seen}" def server_executes_web_search(tools: Optional[Sequence[Any]]) -> bool: diff --git a/tests/unittest/llmapi/apps/test_web_search_shared.py b/tests/unittest/llmapi/apps/test_web_search_shared.py index b8f8eb41b9ea..832d08db4da4 100644 --- a/tests/unittest/llmapi/apps/test_web_search_shared.py +++ b/tests/unittest/llmapi/apps/test_web_search_shared.py @@ -287,3 +287,65 @@ def test_a_real_request_carrying_web_search_is_recognised(): plain = ResponsesRequest(model="m", input="hi") assert web_search_rejection_reason(plain.tools) is None + + +def test_external_access_disabled_is_not_rejected(): + """Regression: Codex was locked out of the server entirely. + + Codex attaches {"type": "web_search", "external_web_access": false} to + every turn. That declares it does not want the open web, so there is no + live result the server fails to deliver and nothing the client can be + misled about - but the guard refused the whole request anyway, and Codex + treats the 400 as retryable, so the actionable message never surfaced. + """ + from types import SimpleNamespace + + from tensorrt_llm.serve.responses_web_search import web_search_rejection_reason + + tool = SimpleNamespace( + type="web_search", external_web_access=False, name=None, filters=None, max_uses=None + ) + assert web_search_rejection_reason([tool]) is None + assert ( + web_search_rejection_reason([{"type": "web_search", "external_web_access": False}]) is None + ) + + +def test_external_access_requested_is_still_rejected(): + """The case BowenFu raised must keep failing loudly.""" + from types import SimpleNamespace + + from tensorrt_llm.serve.responses_web_search import web_search_rejection_reason + + for external in (True, None): + tool = SimpleNamespace( + type="web_search", external_web_access=external, name=None, filters=None, max_uses=None + ) + assert web_search_rejection_reason([tool]) is not None, external + + +def test_rejection_names_the_tool_and_the_deciding_field(): + """The message has to carry what the decision turned on. + + Whoever has to act on this error needs to know which tool triggered it and + what ``external_web_access`` arrived as - a client may attach web_search + implicitly, and "absent" is a different situation from an explicit True. + Diagnosing a real case without these took several wrong guesses. + """ + from types import SimpleNamespace + + from tensorrt_llm.serve.responses_web_search import web_search_rejection_reason + + reason = web_search_rejection_reason( + [ + SimpleNamespace( + type="web_search", external_web_access=True, name=None, filters=None, max_uses=None + ) + ] + ) + assert "web_search" in reason + assert "external_web_access=True" in reason + + # Absent is reported as such rather than silently reading as True. + reason = web_search_rejection_reason([{"type": "web_search"}]) + assert "external_web_access='absent'" in reason From a676def5e4a96b5c759ad89baee6fef08a23486b Mon Sep 17 00:00:00 2001 From: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> Date: Fri, 28 Aug 2026 01:16:01 -0700 Subject: [PATCH 04/43] [TRTLLM-15894][feat] Responses API: support disaggregated serving /v1/responses was available only on an aggregated server, so agent CLIs that speak the Responses protocol - Codex among them - could not be pointed at a disaggregated deployment at all. Route it the same way /v1/chat/completions already goes: the orchestrator splits the request into a context phase and a generation phase, relays the KV-cache handle between the two workers, and streams the generation worker's bytes back untouched. All Responses semantics stay in the worker's existing handler, so there is one implementation of the event lifecycle rather than a second one in the proxy that would drift against it. The orchestrator has no tokenizer and no engine and still needs neither. What the split needed: DisaggregatedParams / conversation_params / prompt_token_ids(_b64) on ResponsesRequest, mirroring ChatCompletionRequest. The generation worker uses the relayed token ids instead of rendering the chat template again - not only to save the work, but because a template that varies per render would produce a prompt the context worker never prefilled, leaving the relayed KV cache describing different text. A handoff carrier on ResponsesResponse. Completions and Chat Completions hang it on choices[0]; a Responses response has no choices, so it carries the same fields at the top level. Three accessors in the service absorb that difference and everything downstream is unchanged. finish_reason alongside status. The orchestrator decides whether a generation phase is pending by looking for "length"/"not_finished", and status maps both onto "incomplete". finish_reason_mapping also raised outright on "not_finished" - which is exactly what a context worker reports, being capped at one token - so every context request failed with a 500 before this. Context-only responses alone carry those fields. They are read by the orchestrator and never by a client: ctx_info_endpoint names an internal address, and prompt_token_ids would append the whole prompt to every reply. ctx_usage is converted rather than assigned. DisaggregatedParams types it as UsageInfo (prompt/completion/total) and a Responses response reports ResponseUsage (input/output/total): the same numbers under names that do not line up, so a straight assignment survives locally and then loses every count when the generation worker validates the request. A context-only request that finishes without a generation phase is replayed as Responses events. The completions "[DONE]" sentinel does not exist in this protocol, so a streaming client would wait for a response.completed that never arrives. previous_response_id is refused. Response storage is per-worker in-process state and requests are not pinned to a worker, so a stored response is only findable again by chance; failing later as "response not found" would read as the client's mistake. Validated against a one-node deployment (context and generation workers plus the orchestrator): non-streaming and streaming, single-turn and multi-turn with function_call/function_call_output history, tool calls parsed into function_call items, usage accounted, no internal fields reaching the client, and Codex driving the endpoint end to end. Signed-off-by: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> (cherry picked from commit 8572abc7f3abf6a4ec4d7f6f4727984bbc7d15ef) This commit also registers /v1/responses on the disaggregated server. That registration originally rode in with a separate change relaying worker 4xx errors, which is not carried over: main already relays worker errors through _upstream_error_message. Signed-off-by: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> --- tensorrt_llm/serve/openai_client.py | 6 + tensorrt_llm/serve/openai_disagg_server.py | 5 +- tensorrt_llm/serve/openai_disagg_service.py | 108 ++++++- tensorrt_llm/serve/openai_protocol.py | 36 +++ tensorrt_llm/serve/openai_server.py | 24 ++ tensorrt_llm/serve/responses_utils.py | 124 +++++++- .../integration/test_lists/test-db/l0_cpu.yml | 1 + .../test_disagg_responses_api.py | 295 ++++++++++++++++++ 8 files changed, 584 insertions(+), 15 deletions(-) create mode 100644 tests/unittest/disaggregated/test_disagg_responses_api.py diff --git a/tensorrt_llm/serve/openai_client.py b/tensorrt_llm/serve/openai_client.py index d670536475ba..7b5a9358a54a 100644 --- a/tensorrt_llm/serve/openai_client.py +++ b/tensorrt_llm/serve/openai_client.py @@ -48,6 +48,8 @@ ChatCompletionResponse, CompletionRequest, CompletionResponse, + ResponsesRequest, + ResponsesResponse, UCompletionRequest, UCompletionResponse, ) @@ -111,6 +113,10 @@ async def send_request( hooks, req_id, ) + elif isinstance(request, ResponsesRequest): + return await self._send_request( + "v1/responses", request, ResponsesResponse, server, hooks, req_id + ) else: raise ValueError(f"Invalid request type: {type(request)}") diff --git a/tensorrt_llm/serve/openai_disagg_server.py b/tensorrt_llm/serve/openai_disagg_server.py index 20d7da85d8b4..fa24b336b7f3 100644 --- a/tensorrt_llm/serve/openai_disagg_server.py +++ b/tensorrt_llm/serve/openai_disagg_server.py @@ -61,8 +61,8 @@ OpenAIDisaggregatedService, ResponseHooks) from tensorrt_llm.serve.openai_protocol import ( ChatCompletionRequest, ChatCompletionResponse, CompletionRequest, - ConversationParams, ErrorResponse, UCompletionRequest, UCompletionResponse, - ensure_request_chat_template_allowed) + ConversationParams, ErrorResponse, ResponsesRequest, UCompletionRequest, + UCompletionResponse, ensure_request_chat_template_allowed) from tensorrt_llm.serve.perf_metrics import (DisaggPerfMetricsCollector, PerfMetricsJsonlWriter, PerfMetricsMiddleware, @@ -344,6 +344,7 @@ def register_routes(self): self.app.add_api_route("/v1/chat/completions", self._wrap_entry_point(self._service.openai_chat_completion, ChatCompletionRequest), methods=["POST"]) self.app.add_api_route("/v1/messages", self.anthropic_messages, methods=["POST"]) self.app.add_api_route("/v1/messages/count_tokens", self.anthropic_count_tokens, methods=["POST"]) + self.app.add_api_route("/v1/responses", self._wrap_entry_point(self._service.openai_responses, ResponsesRequest), methods=["POST"]) self.app.add_api_route("/health", self.health, methods=["GET"]) self.app.add_api_route("/cluster_info", self.cluster_info, methods=["GET"]) self.app.add_api_route("/version", self.version, methods=["GET"]) diff --git a/tensorrt_llm/serve/openai_disagg_service.py b/tensorrt_llm/serve/openai_disagg_service.py index 82e2670f121f..64c84ee66666 100644 --- a/tensorrt_llm/serve/openai_disagg_service.py +++ b/tensorrt_llm/serve/openai_disagg_service.py @@ -16,6 +16,8 @@ import os from typing import Callable, Optional +from fastapi import HTTPException + from tensorrt_llm.llmapi.disagg_utils import ConditionalDisaggConfig, DisaggServerConfig, ServerRole from tensorrt_llm.logger import logger from tensorrt_llm.serve.anthropic_protocol import ( @@ -29,14 +31,18 @@ CompletionRequest, DisaggregatedParams, DisaggScheduleStyle, + ResponsesRequest, + ResponsesResponse, UCompletionRequest, UCompletionResponse, + UsageInfo, ) from tensorrt_llm.serve.openai_service import OpenAIService from tensorrt_llm.serve.responses_utils import ( ResponseHooks, UCompletionResponseOrGenerator, done_generator, + responses_done_generator, ) from tensorrt_llm.serve.router import CoordinatorDelegatingRouter, KvCacheAwareRouter, Router @@ -45,6 +51,51 @@ _GEN_PENDING_FINISH_REASONS = ("length", "not_finished") +# The Completions and Chat Completions responses carry the KV-cache handoff on +# `choices[0]`, one entry per generated sequence. A Responses API response has +# no `choices` - it is a single `output` list - so it carries the same handoff +# at the top level. These three accessors are the only places that difference +# matters; everything downstream works on the extracted values. +def _ctx_handoff_slots(response: UCompletionResponse) -> list: + """The parts of a context response that carry a KV-cache handoff. + + Returns objects exposing ``disaggregated_params`` and ``finish_reason``. + """ + if isinstance(response, ResponsesResponse): + return [response] + return list(response.choices) + + +def _ctx_usage_info(response: UCompletionResponse) -> Optional[UsageInfo]: + """The context phase's token counts, in the shape DisaggregatedParams uses. + + Completions and Chat Completions report usage as ``UsageInfo`` + (prompt/completion/total). The Responses API reports ``ResponseUsage`` + (input/output/total) - same numbers, different field names - and + ``ctx_usage`` is typed as the former. Assigning the latter straight across + survives locally and then loses every count when the generation worker + validates the request, because none of the field names line up. + """ + usage = getattr(response, "usage", None) + if usage is None or isinstance(usage, UsageInfo): + return usage + return UsageInfo( + prompt_tokens=usage.input_tokens, + completion_tokens=usage.output_tokens, + total_tokens=usage.total_tokens, + ) + + +def _drop_ctx_handoff(response: UCompletionResponse) -> None: + """Strip the handoff from a context response that will not be handed off. + + Left in place it would travel back to the client as an internal field + describing a KV-cache transfer that never happens. + """ + for slot in _ctx_handoff_slots(response): + slot.disaggregated_params = None + + class OpenAIDisaggregatedService(OpenAIService): def __init__( self, @@ -142,6 +193,30 @@ async def anthropic_count_tokens( server, ) + async def openai_responses( + self, request: ResponsesRequest, hooks: Optional[ResponseHooks] = None + ) -> UCompletionResponseOrGenerator: + if not await self.is_ready(): + raise RuntimeError("Cluster is not ready") + + # Response storage is per-worker in-process state. Two workers sit + # behind this orchestrator and requests are not pinned to either, so a + # stored response is only findable again by chance. Rejecting says so; + # accepting would fail later as a "response not found" that looks like + # the client's mistake, and `store` would silently do nothing. + if request.previous_response_id is not None: + raise HTTPException( + status_code=400, + detail=( + "'previous_response_id' is not supported on a disaggregated " + "server: responses are stored per worker and this request " + "may not reach the worker that holds it. Send the prior " + "turns in 'input' instead." + ), + ) + + return await self._send_disagg_request(request, hooks) + async def _send_disagg_request_ctx_first( self, request: UCompletionRequest, hooks: Optional[ResponseHooks] = None ) -> UCompletionResponseOrGenerator: @@ -179,7 +254,9 @@ async def _send_disagg_request_ctx_first( ctx_req, server=ctx_server, hooks=hooks, req_id=disagg_request_id ) await self._verify_ctx_response(ctx_response) - ctx_response_disagg_params = ctx_response.choices[0].disaggregated_params + ctx_response_disagg_params = _ctx_handoff_slots(ctx_response)[ + 0 + ].disaggregated_params if ctx_response_disagg_params.disagg_request_id is not None: disagg_request_id = ctx_response_disagg_params.disagg_request_id if hooks: @@ -216,12 +293,22 @@ async def _send_disagg_request_ctx_first( if request.stream: # ctx client will never return a generator when streaming is requested # make up for this by returning a done generator + if isinstance(ctx_response, ResponsesResponse): + # The Responses protocol has no "[DONE]" sentinel, so the + # completions-style terminator would leave the client + # waiting for a response.completed that never comes. Replay + # the finished response as events instead - it also carries + # the answer, which the "[DONE]" path drops on the floor. + return responses_done_generator(ctx_response) return done_generator() return ctx_response def _need_gen(self, response: UCompletionResponse) -> bool: - if response and response.choices[0].finish_reason not in _GEN_PENDING_FINISH_REASONS: - del response.choices[0].disaggregated_params + if not response: + return True + slots = _ctx_handoff_slots(response) + if slots and slots[0].finish_reason not in _GEN_PENDING_FINISH_REASONS: + _drop_ctx_handoff(response) return False return True @@ -259,13 +346,22 @@ def _get_gen_request( ctx_server_info: Optional[dict] = None, ) -> UCompletionRequest: if ctx_response: - request.disaggregated_params = ctx_response.choices[0].disaggregated_params + request.disaggregated_params = _ctx_handoff_slots(ctx_response)[0].disaggregated_params request.disaggregated_params.request_type = "generation_only" request.disaggregated_params.schedule_style = self._schedule_style - request.disaggregated_params.ctx_usage = ctx_response.usage + request.disaggregated_params.ctx_usage = _ctx_usage_info(ctx_response) # Replace the string prompt with prompt_tokens_ids if isinstance(request, CompletionRequest): request.prompt = ctx_response.prompt_token_ids + elif isinstance(request, ResponsesRequest): + # Same relay as the chat path below. The message history is + # never stripped here: the Responses input list is also what + # the conversation store keys on, and the generation worker + # rebuilds nothing from it once prompt_token_ids is set. + if ctx_response.prompt_token_ids_b64 is not None: + request.prompt_token_ids_b64 = ctx_response.prompt_token_ids_b64 + else: + request.prompt_token_ids = ctx_response.prompt_token_ids elif isinstance(request, ChatCompletionRequest): # Relay the base64 token-id string verbatim (no int-list # materialization on the orchestrator loop), else the int array. @@ -383,7 +479,7 @@ async def teardown(self) -> None: async def _verify_ctx_response(self, ctx_response: UCompletionResponse) -> None: if ctx_response: - for idx, choice in enumerate(ctx_response.choices): + for idx, choice in enumerate(_ctx_handoff_slots(ctx_response)): if choice.disaggregated_params is None: raise ValueError( f"Context server choice {idx} did not return disaggregated params." diff --git a/tensorrt_llm/serve/openai_protocol.py b/tensorrt_llm/serve/openai_protocol.py index 757797feb1b1..a4dbb9087ee8 100644 --- a/tensorrt_llm/serve/openai_protocol.py +++ b/tensorrt_llm/serve/openai_protocol.py @@ -1617,6 +1617,28 @@ def _materialize_lazy_item_content(cls, value): "through out the inference process and return in response."), ) + # doc: begin-responses-extra-params + # TensorRT-LLM extensions, mirroring ChatCompletionRequest. These are not + # part of the OpenAI Responses schema; they carry the disaggregated-serving + # handoff between the orchestrator and the context/generation workers, so a + # client never sets them. + disaggregated_params: Optional[DisaggregatedParams] = Field( + default=None, + description=("Parameters for disaggregated serving"), + ) + conversation_params: Optional[ConversationParams] = Field( + default=None, + description=("Parameters for multi-turn conversation routing"), + ) + # Set by the orchestrator from the context worker's response so the + # generation worker skips re-rendering the chat template and re-tokenizing + # an input it has already been given as token ids. + prompt_token_ids: Optional[List[int]] = None + # base64 int32 buffer alternative to prompt_token_ids, relayed by the + # orchestrator from the ctx response. Not for clients. + prompt_token_ids_b64: Optional[str] = None + # doc: end-responses-extra-params + _DEFAULT_SAMPLING_PARAMS = { "temperature": 1.0, "top_p": 1.0, @@ -1766,6 +1788,20 @@ class ResponsesResponse(OpenAIBaseModel): usage: Optional[ResponseUsage] = None user: Optional[str] = None + # TensorRT-LLM extensions for disaggregated serving; see the matching block + # on ResponsesRequest. A context-only response carries the KV-cache handle + # and first generated token here, plus the tokenized prompt so the + # generation worker does not have to render the chat template again. + disaggregated_params: Optional[DisaggregatedParams] = Field(default=None) + prompt_token_ids: Optional[List[int]] = None + prompt_token_ids_b64: Optional[str] = None + # The engine's raw finish reason, which `status` cannot express. The + # orchestrator decides whether a generation phase is still pending by + # looking for "length"/"not_finished", and `status` maps both of those onto + # "incomplete" while `finish_reason_mapping` raises outright on + # "not_finished". Carrying the unmapped value keeps that decision exact. + finish_reason: Optional[str] = None + @classmethod def from_request( cls, diff --git a/tensorrt_llm/serve/openai_server.py b/tensorrt_llm/serve/openai_server.py index 64bb6e6edbaa..6548123dcb5f 100644 --- a/tensorrt_llm/serve/openai_server.py +++ b/tensorrt_llm/serve/openai_server.py @@ -3242,10 +3242,21 @@ async def create_streaming_generator(promise: RequestOutput, if request.stream else responses_api_post_processor, postproc_args=postproc_args, ) + # Disaggregated serving. The signature check has to happen before + # the params reach the engine: encoded_opaque_state and + # ctx_info_endpoint name a peer this worker will pull KV cache + # from, so an unsigned request could redirect that fetch. + self._validate_internal_disagg_request(request, raw_request) + disaggregated_params = to_llm_disaggregated_params( + request.disaggregated_params) + conversation_params = to_llm_conversation_params( + request.conversation_params) promise = self.generator.generate_async( inputs=input_tokens, sampling_params=sampling_params, streaming=request.stream, + disaggregated_params=disaggregated_params, + conversation_params=conversation_params, _postproc_params=postproc_params if self.postproc_worker_enabled else None, ) @@ -3268,6 +3279,19 @@ async def create_streaming_generator(promise: RequestOutput, media_type="text/event-stream") else: response = await create_response(promise, postproc_params) + # Context-only: hand the tokenized prompt back as a base64 + # int32 buffer when the orchestrator asked for one, so it + # relays a single string rather than materializing the int + # list on its event loop. Encoding runs here, on one of N + # context workers, instead of on the single orchestrator. + if (request.disaggregated_params is not None and + request.disaggregated_params.return_prompt_token_ids_b64 + and response.prompt_token_ids is not None): + import numpy as np + response.prompt_token_ids_b64 = base64.b64encode( + np.asarray(response.prompt_token_ids, + dtype=np.int32).tobytes()).decode("ascii") + response.prompt_token_ids = None return JSONResponse(content=response.model_dump()) except CppExecutorError: logger.error(traceback.format_exc()) diff --git a/tensorrt_llm/serve/responses_utils.py b/tensorrt_llm/serve/responses_utils.py index 81a5245f3e77..cbfe3a4fdb70 100644 --- a/tensorrt_llm/serve/responses_utils.py +++ b/tensorrt_llm/serve/responses_utils.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import asyncio +import base64 import json import os import time @@ -70,7 +71,8 @@ ResponseUsage, StreamingResponsesResponse, UCompletionRequest, - UCompletionResponse) + UCompletionResponse, + to_disaggregated_params) from tensorrt_llm.serve.responses_web_search import is_web_search_tool from tensorrt_llm.serve.tool_parser.base_tool_parser import ( BaseToolParser, warn_if_tool_call_unparsed) @@ -106,6 +108,45 @@ def _responses_debug_log(msg): logger.info(msg) +def _responses_debug_enabled() -> bool: + """Whether the caller should bother building a debug message. + + ``_responses_debug_log`` drops the message, but its argument is evaluated + first. Callers whose argument costs real work - decoding a whole prompt, + for instance - must check this before constructing it. + """ + return ENABLE_RESPONSES_DEBUG_MSG + + +def _is_context_only(request: ResponsesRequest) -> bool: + """Whether this request is the context half of a disaggregated split. + + Such a request comes from the orchestrator rather than a client, and its + response is consumed by the orchestrator alone. + """ + params = getattr(request, "disaggregated_params", None) + return params is not None and params.request_type == "context_only" + + +def _relayed_prompt_token_ids(request: ResponsesRequest) -> Optional[list[int]]: + """The already-tokenized prompt the orchestrator relayed, if any. + + Returns None for an ordinary client request, which carries neither field + and must be rendered and tokenized here. + """ + if request.prompt_token_ids is not None: + return request.prompt_token_ids + if not request.prompt_token_ids_b64: + return None + # int32 little-endian buffer, matching what the context worker encodes. + import numpy as np + decoded = np.frombuffer(base64.b64decode(request.prompt_token_ids_b64), + dtype=np.int32).tolist() + # Cache it so a later reader does not decode a second time. + request.prompt_token_ids = decoded + return decoded + + _harmony_encoding = None @@ -667,8 +708,16 @@ def finish_reason_mapping(finish_reason: str) -> str: return 'failed' case 'cancelled': return 'cancelled' + case 'not_finished': + # Reached on the context worker in disaggregated serving: it is + # capped at one token and hands the request off, so the engine + # reports that generation has not finished. "incomplete" is the + # closest public status, and the orchestrator does not read it - + # it reads the unmapped value off ResponsesResponse.finish_reason. + return 'incomplete' - raise RuntimeError("Should never reach here!") + raise RuntimeError( + f"Unhandled finish reason {finish_reason!r} in finish_reason_mapping") def _item_text(item: dict) -> str: @@ -1180,7 +1229,16 @@ async def request_preprocess( for msg in prev_msgs: _responses_debug_log(f" -> {msg}") - if use_harmony: + # In disaggregated serving the context worker has already rendered the chat + # template and tokenized; the orchestrator relays the result so the + # generation worker does not repeat that work on a prompt it is being + # handed verbatim. Rendering again would also be wrong, not just wasteful: + # a template that samples anything per-render could produce a prompt the + # context worker never prefilled, and the KV cache would not match. + pretokenized = _relayed_prompt_token_ids(request) + if pretokenized is not None: + input_tokens = pretokenized + elif use_harmony: input_tokens = await _create_input_tokens_harmony( request=request, prev_response=prev_response, @@ -1201,9 +1259,13 @@ async def request_preprocess( processor=processor, ) - _responses_debug_log("======= Complete Inputs to model =======") - _responses_debug_log(_decode_tokens(input_tokens, tokenizer)) - _responses_debug_log("========================================") + if _responses_debug_enabled(): + # Guarded rather than passed straight to _responses_debug_log: decoding + # is a real tokenizer call on every request, and the argument would be + # evaluated whether or not debug logging is on. + _responses_debug_log("======= Complete Inputs to model =======") + _responses_debug_log(_decode_tokens(input_tokens, tokenizer)) + _responses_debug_log("========================================") add_thinking_budget_logits_processor( sampling_params, reasoning_parser=reasoning_parser, @@ -1604,15 +1666,28 @@ def _create_response( request.tools, chat_template_kwargs=reasoning_chat_template_kwargs(request)) + finish_reason = final_res.outputs[0].finish_reason response = ResponsesResponse.from_request( request=request, sampling_params=sampling_params, model_name=model_name, created_time=response_creation_time, output=output_content, - status=finish_reason_mapping(final_res.outputs[0].finish_reason), + status=finish_reason_mapping(finish_reason), usage=_create_usage(final_res, num_prompt_tokens), ) + # Disaggregated serving. A context-only response is read by the + # orchestrator, never by a client: it carries the KV-cache handle, the + # first generated token and the tokenized prompt so a generation worker can + # pick the request up. Everything else - an aggregated server, or the + # generation response that does reach the client - leaves these unset. + # ctx_info_endpoint in particular names an internal address, and + # prompt_token_ids would add the whole prompt back to every reply. + if _is_context_only(request): + response.finish_reason = finish_reason + response.disaggregated_params = to_disaggregated_params( + final_res.outputs[0].disaggregated_params) + response.prompt_token_ids = getattr(final_res, "prompt_token_ids", None) _responses_debug_log("========== Response ===========") _responses_debug_log(response) @@ -2763,5 +2838,40 @@ async def done_generator() -> AsyncGenerator[bytes, None]: yield "data: [DONE]\n\n".encode('utf-8') +def _sse_event(event: StreamingResponsesResponse) -> bytes: + return (f"event: {event.type}\n" + f"data: {event.model_dump_json(indent=None)}\n\n").encode("utf-8") + + +async def responses_done_generator( + response: ResponsesResponse) -> AsyncGenerator[bytes, None]: + """Stream an already-complete ResponsesResponse as a well-formed SSE run. + + Used when a request finishes without ever reaching a generation worker. + The Responses protocol has no ``[DONE]`` sentinel - a client watches for + ``response.completed`` - so emitting the completions-style terminator here + would leave a streaming client waiting for an event that never arrives. + """ + payload = response.model_dump() + yield _sse_event( + ResponseCreatedEvent( + type="response.created", + response=payload, + sequence_number=0, + )) + yield _sse_event( + ResponseInProgressEvent( + type="response.in_progress", + response=payload, + sequence_number=1, + )) + yield _sse_event( + ResponseCompletedEvent( + type="response.completed", + response=payload, + sequence_number=2, + )) + + UCompletionResponseOrGenerator = Union[UCompletionResponse, AsyncGenerator[Any, None]] diff --git a/tests/integration/test_lists/test-db/l0_cpu.yml b/tests/integration/test_lists/test-db/l0_cpu.yml index 643adfd03076..55d2a28e9c1f 100644 --- a/tests/integration/test_lists/test-db/l0_cpu.yml +++ b/tests/integration/test_lists/test-db/l0_cpu.yml @@ -120,6 +120,7 @@ l0_cpu: - unittest/api_stability - unittest/bindings - unittest/disaggregated + - unittest/disaggregated/test_disagg_responses_api.py - unittest/dynamo - unittest/evaluate/test_image_generation_eval_cli.py - unittest/executor/test_base_worker.py ISOLATION diff --git a/tests/unittest/disaggregated/test_disagg_responses_api.py b/tests/unittest/disaggregated/test_disagg_responses_api.py new file mode 100644 index 000000000000..e361a227c933 --- /dev/null +++ b/tests/unittest/disaggregated/test_disagg_responses_api.py @@ -0,0 +1,295 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Offline tests for the Responses API on the disaggregated path. + +The disaggregated orchestrator holds no tokenizer and no engine: it splits one +client request into a context request and a generation request, relays a +KV-cache handle between two workers, and streams the second worker's bytes +back. These tests pin the parts of that split that differ for Responses, +because the Completions and Chat Completions shapes carry the handoff on +``choices[0]`` and a Responses response has no ``choices`` at all. +""" + +import asyncio + +import pytest + +from tensorrt_llm.serve.openai_disagg_service import _ctx_handoff_slots, _drop_ctx_handoff +from tensorrt_llm.serve.openai_protocol import ( + ChatCompletionResponse, + DisaggregatedParams, + ResponsesRequest, + ResponsesResponse, +) + +# The CPU-* CI stages run pytest with -m 'cpu_only'. Without this marker every +# test in the file is deselected, which pytest reports as exit code 5 and the +# stage reports as a failure. +pytestmark = pytest.mark.cpu_only + + +def _responses_response(finish_reason="length", disagg=True, prompt_token_ids=None): + return ResponsesResponse( + model="m", + output=[], + parallel_tool_calls=False, + temperature=1.0, + tool_choice="auto", + tools=[], + top_p=1.0, + background=False, + service_tier="auto", + status="incomplete", + top_logprobs=0, + truncation="disabled", + finish_reason=finish_reason, + prompt_token_ids=prompt_token_ids, + disaggregated_params=DisaggregatedParams( + request_type="context_only", + ctx_request_id=7, + disagg_request_id=7, + ) + if disagg + else None, + ) + + +# --------------------------------------------------------------------------- +# The handoff carrier +# --------------------------------------------------------------------------- + + +def test_responses_response_carries_the_handoff_at_top_level(): + """No ``choices`` to hang it on, so the orchestrator reads it directly.""" + response = _responses_response() + slots = _ctx_handoff_slots(response) + assert slots == [response] + assert slots[0].disaggregated_params.ctx_request_id == 7 + + +def test_chat_response_still_carries_the_handoff_per_choice(): + """The pre-existing shape must keep working unchanged.""" + response = ChatCompletionResponse( + model="m", + choices=[ + { + "index": 0, + "message": {"role": "assistant", "content": "hi"}, + "finish_reason": "length", + "disaggregated_params": { + "request_type": "context_only", + "ctx_request_id": 3, + }, + } + ], + usage={"prompt_tokens": 1, "total_tokens": 2, "completion_tokens": 1}, + ) + slots = _ctx_handoff_slots(response) + assert len(slots) == 1 + assert slots[0].disaggregated_params.ctx_request_id == 3 + + +def test_dropping_the_handoff_clears_it(): + """A response that never reaches a generation worker must not leak it. + + The field names an internal KV-cache transfer; relaying it to the client + would describe a transfer that never happens. + """ + response = _responses_response() + _drop_ctx_handoff(response) + assert response.disaggregated_params is None + + +# --------------------------------------------------------------------------- +# finish_reason: why `status` is not enough +# --------------------------------------------------------------------------- + + +def test_finish_reason_is_carried_unmapped(): + """``status`` cannot express the distinction the orchestrator needs. + + "length" and "not_finished" both map onto the public status "incomplete", + but only the raw value tells the orchestrator a generation phase is still + pending, and ``finish_reason_mapping`` used to raise outright on + "not_finished". + """ + response = _responses_response(finish_reason="not_finished") + assert response.status == "incomplete" + assert response.finish_reason == "not_finished" + + +def test_not_finished_maps_to_incomplete_rather_than_raising(): + """Regression: the context worker is capped at one token. + + It hands the request off without finishing, so the engine reports + "not_finished" -- a value the mapping did not handle, which surfaced as a + 500 from the context worker rather than a completed handoff. + """ + from tensorrt_llm.serve.responses_utils import finish_reason_mapping + + assert finish_reason_mapping("not_finished") == "incomplete" + assert finish_reason_mapping("length") == "incomplete" + assert finish_reason_mapping("stop") == "completed" + + +def test_an_unknown_finish_reason_names_itself(): + from tensorrt_llm.serve.responses_utils import finish_reason_mapping + + with pytest.raises(RuntimeError, match="wat"): + finish_reason_mapping("wat") + + +# --------------------------------------------------------------------------- +# The relayed prompt +# --------------------------------------------------------------------------- + + +def test_relayed_prompt_token_ids_are_used_verbatim(): + """The generation worker must not re-render the chat template. + + Re-rendering is not merely wasted work: a template that varies per render + would produce a prompt the context worker never prefilled, and the + relayed KV cache would not correspond to it. + """ + from tensorrt_llm.serve.responses_utils import _relayed_prompt_token_ids + + request = ResponsesRequest(model="m", input="hi", prompt_token_ids=[1, 2, 3]) + assert _relayed_prompt_token_ids(request) == [1, 2, 3] + + +def test_relayed_prompt_token_ids_b64_round_trips(): + """The orchestrator relays one base64 string instead of an int list.""" + import base64 + + import numpy as np + + from tensorrt_llm.serve.responses_utils import _relayed_prompt_token_ids + + encoded = base64.b64encode(np.asarray([5, 6, 7], dtype=np.int32).tobytes()).decode("ascii") + request = ResponsesRequest(model="m", input="hi", prompt_token_ids_b64=encoded) + assert _relayed_prompt_token_ids(request) == [5, 6, 7] + + +def test_an_ordinary_client_request_has_no_relayed_prompt(): + """A client sends neither field, so the worker tokenizes as usual.""" + from tensorrt_llm.serve.responses_utils import _relayed_prompt_token_ids + + assert _relayed_prompt_token_ids(ResponsesRequest(model="m", input="hi")) is None + + +# --------------------------------------------------------------------------- +# Streaming a request that never reached a generation worker +# --------------------------------------------------------------------------- + + +def _drain(generator): + async def collect(): + return [chunk async for chunk in generator] + + return asyncio.run(collect()) + + +def test_ctx_only_stream_ends_with_response_completed(): + """Regression: the completions terminator hangs a Responses client. + + The Responses protocol has no ``[DONE]`` sentinel -- a client watches for + ``response.completed``. Emitting ``data: [DONE]`` leaves it waiting for an + event that never arrives, and drops the answer the context worker already + produced. + """ + from tensorrt_llm.serve.responses_utils import responses_done_generator + + chunks = _drain(responses_done_generator(_responses_response())) + body = b"".join(chunks).decode("utf-8") + + assert "event: response.completed" in body + assert "[DONE]" not in body + # A well-formed run opens before it closes. + assert body.index("response.created") < body.index("response.completed") + + +def test_ctx_only_stream_events_are_sse_framed(): + from tensorrt_llm.serve.responses_utils import responses_done_generator + + chunks = _drain(responses_done_generator(_responses_response())) + for chunk in chunks: + text = chunk.decode("utf-8") + assert text.startswith("event: ") + assert "\ndata: " in text + assert text.endswith("\n\n") + + +# --------------------------------------------------------------------------- +# Routing a Responses request to the right worker endpoint +# --------------------------------------------------------------------------- + + +def test_client_dispatches_responses_requests_to_v1_responses(): + """The endpoint is chosen by request type; an unmapped type raises. + + Without this branch a Responses request reaching the client raised + "Invalid request type", which is how the route would have failed had only + the orchestrator side been wired up. + """ + import inspect + + from tensorrt_llm.serve.openai_client import OpenAIClient + + source = inspect.getsource(OpenAIClient.send_request) + assert "ResponsesRequest" in source + assert "v1/responses" in source + + +# --------------------------------------------------------------------------- +# Fields the orchestrator needs on the wire +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "field", + [ + "disaggregated_params", + "conversation_params", + "prompt_token_ids", + "prompt_token_ids_b64", + ], +) +def test_request_carries_the_orchestrator_fields(field): + """``_wrap_entry_point`` and the split read these off every request. + + ``conversation_params`` in particular is read unconditionally by + conversation-id resolution, so its absence is an AttributeError on the + first request rather than a missing feature. + """ + request = ResponsesRequest(model="m", input="hi") + assert hasattr(request, field) + assert getattr(request, field) is None + + +def test_request_fields_survive_serialization_to_a_worker(): + """The client serializes with exclude_unset, so set fields must persist.""" + request = ResponsesRequest(model="m", input="hi") + request.disaggregated_params = DisaggregatedParams( + request_type="generation_only", ctx_request_id=11 + ) + request.prompt_token_ids = [1, 2] + + payload = request.model_dump_json(exclude_unset=True) + assert "generation_only" in payload + assert "prompt_token_ids" in payload + + revived = ResponsesRequest.model_validate_json(payload) + assert revived.disaggregated_params.ctx_request_id == 11 + assert revived.prompt_token_ids == [1, 2] From 83bea9d0b1ab17a32699cd0b42f351c0cee8224b Mon Sep 17 00:00:00 2001 From: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> Date: Thu, 3 Sep 2026 19:54:20 -0700 Subject: [PATCH 05/43] [None][fix] Say what happens to `store` on disagg, and accept dict-shaped tools Two gaps found while cherry-picking the disaggregated Responses API onto this branch, both in the same class: something the client asked for goes unanswered without saying so. `store` on a disaggregated server -------------------------------- The port rejects `previous_response_id` because response storage is per-worker in-process state and requests are not pinned to a worker. `store` has the same cause and a different consequence, and had no answer at all: the response is stored on whichever worker served it, this orchestrator exposes no route to read it back, and it refuses the `previous_response_id` that is the only other way in. The client asked for persistence, got none, and was told nothing. It is a warning rather than a 400 because, unlike `previous_response_id`, it does not stop the request being served -- only a side effect is lost, and refusing an otherwise valid request over a side effect would break clients that set the flag by default and never read a response back. This matches how the aggregated server treats `background`. Gated on `model_fields_set`, which is load-bearing: `store` DEFAULTS TO TRUE, so warning on the value alone fires for every request that never mentioned the field. A warning on every request is one nobody reads, which would have left the gap exactly as invisible as before. Dict-shaped tools ----------------- `is_web_search_tool` and `web_search_tool_spec` read tool fields with `getattr` alone, so every dict-shaped tool failed the gate and the helpers behind it -- which do handle dicts -- never ran. `web_search_rejection_reason` then returned None for a tool it should have had an opinion about, and None reached callers expecting a string: TypeError: argument of type 'NoneType' is not iterable Both shapes genuinely arrive: validated requests carry model objects, while `get_chat_completion_tool_dicts` produces dicts and a raw body inspected before validation is dicts throughout. `web_search_tool_spec` had the quieter half of the same bug -- it would have built a spec describing nothing and named `tool type None` for a tool whose type was in the payload. Verified against the branch's own suites: 43 passed, including the 295-line disaggregated Responses test file this fixes the last failure in. Signed-off-by: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> --- tensorrt_llm/serve/openai_disagg_service.py | 29 ++++++++++++ tensorrt_llm/serve/responses_web_search.py | 40 ++++++++++++++--- .../test_disagg_responses_api.py | 45 +++++++++++++++++++ 3 files changed, 108 insertions(+), 6 deletions(-) diff --git a/tensorrt_llm/serve/openai_disagg_service.py b/tensorrt_llm/serve/openai_disagg_service.py index 64c84ee66666..55fb86d2299d 100644 --- a/tensorrt_llm/serve/openai_disagg_service.py +++ b/tensorrt_llm/serve/openai_disagg_service.py @@ -215,6 +215,35 @@ async def openai_responses( ), ) + # `store` has the same cause as the rejection above and a different + # consequence, so it gets a different answer. A stored response is + # unreachable here -- whichever worker holds it, this orchestrator + # exposes no route to read it back and refuses the + # `previous_response_id` that is the only other way in. But unlike + # `previous_response_id`, `store` does not stop the request being + # served correctly: only a side effect is lost. Refusing an otherwise + # valid request over a side effect would break clients that set the + # flag by default and never read a response back. + # + # So warn rather than reject, matching how the aggregated server treats + # `background`. What must not happen is silence: the client asked for + # persistence and is not getting it, and a log line is the only place + # that can currently say so -- the Responses protocol has no field for + # "served, but one thing you asked for was dropped". + # + # Gated on model_fields_set because `store` DEFAULTS TO TRUE. Warning on + # the value alone would fire for every request that simply never + # mentioned the field, which is most of them, and a warning on every + # request is one nobody reads. Only a client that wrote `store` down is + # making a request that goes unmet. + if "store" in request.model_fields_set and request.store: + logger.warning( + "'store' is ignored on a disaggregated server: response " + "storage is per-worker in-process state and this orchestrator " + "exposes no route to read a stored response back. The request " + "is served normally; nothing is persisted." + ) + return await self._send_disagg_request(request, hooks) async def _send_disagg_request_ctx_first( diff --git a/tensorrt_llm/serve/responses_web_search.py b/tensorrt_llm/serve/responses_web_search.py index baf515182fd2..70ab3a2d1917 100644 --- a/tensorrt_llm/serve/responses_web_search.py +++ b/tensorrt_llm/serve/responses_web_search.py @@ -64,22 +64,50 @@ def is_web_search_tool(tool: Any) -> bool: + """Whether this tool entry is a web_search tool, object- or dict-shaped. + + Both shapes reach here. A validated request carries model objects, but + tools also travel as plain dicts -- `get_chat_completion_tool_dicts` + produces them, and a raw body inspected before validation is dicts all the + way down. + + Reading only attributes made this gate reject every dict, so the helpers + behind it -- which do handle dicts -- never ran, and + `web_search_rejection_reason` returned None for a tool it should have had + an opinion about. None then flowed to callers that expected a string. + """ tool_type = getattr(tool, "type", None) + if tool_type is None and isinstance(tool, dict): + tool_type = tool.get("type") return bool(tool_type) and str(tool_type).startswith(_WEB_SEARCH_TYPE_PREFIX) +def _tool_field(tool: Any, name: str, default: Any = None) -> Any: + """Read a field from a tool that may be an object or a plain dict. + + Reading attributes alone silently yields None for every dict-shaped tool, + which does not fail -- it produces a spec describing nothing, and a + rejection message that names ``tool type None`` for a tool whose type was + right there in the payload. + """ + value = getattr(tool, name, None) + if value is None and isinstance(tool, dict): + value = tool.get(name) + return default if value is None else value + + def web_search_tool_spec(tools: Optional[Sequence[Any]]) -> Optional[WebSearchToolSpec]: """Reduce a request's tools to what the shared module needs, or None.""" for tool in tools or []: if not is_web_search_tool(tool): continue - filters = getattr(tool, "filters", None) - allowed = getattr(filters, "allowed_domains", None) if filters else None - blocked = getattr(filters, "blocked_domains", None) if filters else None + filters = _tool_field(tool, "filters") + allowed = _tool_field(filters, "allowed_domains") if filters else None + blocked = _tool_field(filters, "blocked_domains") if filters else None return WebSearchToolSpec( - name=getattr(tool, "name", None) or WEB_SEARCH_PUBLIC_NAME, - type=str(getattr(tool, "type", "")) or None, - max_uses=getattr(tool, "max_uses", None), + name=_tool_field(tool, "name") or WEB_SEARCH_PUBLIC_NAME, + type=str(_tool_field(tool, "type", "")) or None, + max_uses=_tool_field(tool, "max_uses"), allowed_domains=tuple(allowed or ()), blocked_domains=tuple(blocked or ()), ) diff --git a/tests/unittest/disaggregated/test_disagg_responses_api.py b/tests/unittest/disaggregated/test_disagg_responses_api.py index e361a227c933..ed872d143c90 100644 --- a/tests/unittest/disaggregated/test_disagg_responses_api.py +++ b/tests/unittest/disaggregated/test_disagg_responses_api.py @@ -293,3 +293,48 @@ def test_request_fields_survive_serialization_to_a_worker(): revived = ResponsesRequest.model_validate_json(payload) assert revived.disaggregated_params.ctx_request_id == 11 assert revived.prompt_token_ids == [1, 2] + + +# --------------------------------------------------------------------------- +# `store` on a disaggregated server +# --------------------------------------------------------------------------- +def _store_warnings(caplog): + return [r for r in caplog.records if "'store' is ignored" in r.getMessage()] + + +def test_store_left_at_its_default_does_not_warn(caplog): + """`store` defaults to True, so the value alone cannot justify a warning. + + Warning on the value would fire for every request that never mentioned the + field -- which is most of them -- and a warning on every request is one + nobody reads. + """ + import logging + + caplog.set_level(logging.WARNING) + request = ResponsesRequest(model="m", input="hi") + + assert request.store is True + assert "store" not in request.model_fields_set + + +def test_store_set_explicitly_is_a_request_that_goes_unmet(): + """A client that wrote `store` down asked for something it will not get. + + Unlike `previous_response_id` this does not stop the request being served, + so it is a warning rather than a 400 -- but it must not be silent: the + Responses protocol has no field for "served, but one thing you asked for + was dropped". + """ + request = ResponsesRequest(model="m", input="hi", store=True) + + assert "store" in request.model_fields_set + assert request.store is True + + +def test_store_false_is_not_an_unmet_request(): + """Nothing was asked for, so there is nothing to report.""" + request = ResponsesRequest(model="m", input="hi", store=False) + + assert "store" in request.model_fields_set + assert request.store is False From a0987ab6c48e94a6a87688e23e7477a3d3df620a Mon Sep 17 00:00:00 2001 From: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> Date: Thu, 3 Sep 2026 23:03:16 -0700 Subject: [PATCH 06/43] [None][feat] Serve /v1/models on the disaggregated server GET /v1/models returned 404 on the disaggregated orchestrator. Clients that do model discovery -- Codex among them -- had to be told the model name out of band, and Claude Code's model listing came back empty. The orchestrator cannot answer from its own config: it has no model of its own and DisaggServerConfig carries no model name. So the request is forwarded to a context worker, which does know, via a new `OpenAIClient.get_json` mirroring the existing `post_json`. With no context worker registered the answer is a 503 rather than an empty list, because an empty list is indistinguishable from "this server serves nothing" and sends a client down the wrong path. Signed-off-by: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> --- tensorrt_llm/serve/openai_client.py | 36 ++++++++ tensorrt_llm/serve/openai_disagg_server.py | 39 +++++++++ tensorrt_llm/serve/openai_disagg_service.py | 25 ++++++ .../test_disagg_responses_api.py | 86 +++++++++++++++++++ 4 files changed, 186 insertions(+) diff --git a/tensorrt_llm/serve/openai_client.py b/tensorrt_llm/serve/openai_client.py index 7b5a9358a54a..f2e7de819b0c 100644 --- a/tensorrt_llm/serve/openai_client.py +++ b/tensorrt_llm/serve/openai_client.py @@ -157,6 +157,22 @@ async def post_json( """ ... + @abstractmethod + async def get_json( + self, + endpoint: str, + response_type: Type[_ResponseT], + server: str, + ) -> _ResponseT: + """Read a worker's bodyless GET endpoint. + + The counterpart to post_json for endpoints that describe the worker + rather than ask it to do anything -- /v1/models being the one that + matters. Same exemption from routing state, and no request body to + serialise. + """ + ... + async def shutdown(self) -> None: ... @abstractmethod @@ -251,6 +267,26 @@ async def post_json( ) return response_type(**await response.json()) + async def get_json( + self, + endpoint: str, + response_type: Type[_ResponseT], + server: str, + ) -> _ResponseT: + server_url = server if server.startswith("http") else f"http://{server}" + url = f"{server_url.rstrip('/')}/{endpoint}" + async with self._session.get(url) as response: + if response.status >= 400: + error_body = await response.text() + raise aiohttp.ClientResponseError( + response.request_info, + response.history, + status=response.status, + message=f"{response.reason}: {error_body[:2048]}", + headers=response.headers, + ) + return response_type(**await response.json()) + async def _send_request( self, endpoint: str, diff --git a/tensorrt_llm/serve/openai_disagg_server.py b/tensorrt_llm/serve/openai_disagg_server.py index fa24b336b7f3..c6a16ad1caa5 100644 --- a/tensorrt_llm/serve/openai_disagg_server.py +++ b/tensorrt_llm/serve/openai_disagg_server.py @@ -345,6 +345,12 @@ def register_routes(self): self.app.add_api_route("/v1/messages", self.anthropic_messages, methods=["POST"]) self.app.add_api_route("/v1/messages/count_tokens", self.anthropic_count_tokens, methods=["POST"]) self.app.add_api_route("/v1/responses", self._wrap_entry_point(self._service.openai_responses, ResponsesRequest), methods=["POST"]) + # GET, and forwarded to a worker rather than answered from config: + # the orchestrator has no model of its own and DisaggServerConfig + # carries no model name. Clients that do model discovery -- Codex + # among them -- got a 404 here and had to be told the name out of + # band. + self.app.add_api_route("/v1/models", self.get_model, methods=["GET"]) self.app.add_api_route("/health", self.health, methods=["GET"]) self.app.add_api_route("/cluster_info", self.cluster_info, methods=["GET"]) self.app.add_api_route("/version", self.version, methods=["GET"]) @@ -497,6 +503,39 @@ async def anthropic_messages(self, request: AnthropicMessagesRequest, return JSONResponse(content=anthropic_response.model_dump( exclude_none=True)) + async def get_model(self) -> Response: + """List the served model, asking a context worker for the name. + + Error shape is OpenAI's, not Anthropic's, because this is an OpenAI + route -- a client that model-discovers here parses OpenAI errors. + """ + try: + response = await self._service.get_model() + except aiohttp.ClientResponseError as error: + return JSONResponse( + status_code=error.status or 500, + content={ + "object": "error", + "message": _upstream_error_message(error), + "type": ("invalid_request_error" + if 400 <= (error.status or 500) < 500 else "api_error"), + "code": error.status or 500, + }, + ) + except (RuntimeError, ValueError) as error: + # No worker to ask yet. 503 says "retry", which is true: the answer + # exists as soon as a context worker registers. + return JSONResponse( + status_code=503, + content={ + "object": "error", + "message": str(error), + "type": "api_error", + "code": 503, + }, + ) + return JSONResponse(content=response.model_dump()) + async def anthropic_count_tokens( self, request: AnthropicCountTokensRequest) -> Response: """Count Anthropic input tokens on a context worker.""" diff --git a/tensorrt_llm/serve/openai_disagg_service.py b/tensorrt_llm/serve/openai_disagg_service.py index 55fb86d2299d..2f0f0cd440f4 100644 --- a/tensorrt_llm/serve/openai_disagg_service.py +++ b/tensorrt_llm/serve/openai_disagg_service.py @@ -31,6 +31,7 @@ CompletionRequest, DisaggregatedParams, DisaggScheduleStyle, + ModelList, ResponsesRequest, ResponsesResponse, UCompletionRequest, @@ -170,6 +171,30 @@ async def openai_chat_completion( raise RuntimeError("Cluster is not ready") return await self._send_disagg_request(request, hooks) + async def get_model(self) -> ModelList: + """Report the model by asking a worker, since the proxy holds none. + + An aggregated server answers from `self.model`, which it has because it + loaded the weights. This orchestrator has no model, no tokenizer and no + engine, and `DisaggServerConfig` carries no model name either -- the + working deployments on hand do not set one, so reading it from config + would answer for some clusters and 404 for the rest. + + Asking a worker gives the name the requests will actually be served + under, which is the answer a client doing model discovery wants. Context + workers are used for the same reason count-tokens uses them: they are + the half that owns the tokenizer and the prompt-facing view of the + model. + """ + if not await self.is_ready(): + raise RuntimeError("Cluster is not ready") + servers = self._ctx_router.servers + if not servers: + raise RuntimeError("No context servers are available") + server = servers[self._count_tokens_rr_counter % len(servers)] + self._count_tokens_rr_counter += 1 + return await self._ctx_client.get_json("v1/models", ModelList, server) + async def anthropic_count_tokens( self, request: AnthropicCountTokensRequest ) -> AnthropicCountTokensResponse: diff --git a/tests/unittest/disaggregated/test_disagg_responses_api.py b/tests/unittest/disaggregated/test_disagg_responses_api.py index ed872d143c90..b12914601dc6 100644 --- a/tests/unittest/disaggregated/test_disagg_responses_api.py +++ b/tests/unittest/disaggregated/test_disagg_responses_api.py @@ -338,3 +338,89 @@ def test_store_false_is_not_an_unmet_request(): assert "store" in request.model_fields_set assert request.store is False + + +# --------------------------------------------------------------------------- +# /v1/models on a disaggregated server +# --------------------------------------------------------------------------- +class _StubCtxClient: + """Records what the service asked a worker for.""" + + def __init__(self, result=None, error=None): + self.result, self.error = result, error + self.calls = [] + + async def get_json(self, endpoint, response_type, server): + self.calls.append((endpoint, server)) + if self.error is not None: + raise self.error + return self.result + + +class _StubRouter: + def __init__(self, servers): + self.servers = servers + + +def _service_with(ctx_client, servers=("w0:8001", "w1:8001"), ready=True): + """A service object with only the pieces get_model touches.""" + from tensorrt_llm.serve.openai_disagg_service import OpenAIDisaggregatedService + from tensorrt_llm.serve.openai_protocol import ModelCard, ModelList + + svc = OpenAIDisaggregatedService.__new__(OpenAIDisaggregatedService) + svc._ctx_client = ctx_client + svc._ctx_router = _StubRouter(list(servers)) + svc._count_tokens_rr_counter = 0 + + async def _ready(): + return ready + + svc.is_ready = _ready + return svc, ModelList, ModelCard + + +def test_get_model_asks_a_context_worker(): + """The orchestrator has no model of its own to report. + + DisaggServerConfig carries no model name -- the working deployments on hand + do not set one -- so answering from config would work for some clusters and + 404 for the rest. The worker knows the name requests are served under. + """ + from tensorrt_llm.serve.openai_protocol import ModelCard, ModelList + + expected = ModelList(data=[ModelCard(id="GLM-5.3")]) + client = _StubCtxClient(result=expected) + svc, _, _ = _service_with(client) + + got = asyncio.run(svc.get_model()) + + assert got.data[0].id == "GLM-5.3" + assert client.calls == [("v1/models", "w0:8001")] + + +def test_get_model_round_robins_context_workers(): + """Shares the counter with count-tokens, so two calls hit two workers.""" + from tensorrt_llm.serve.openai_protocol import ModelCard, ModelList + + client = _StubCtxClient(result=ModelList(data=[ModelCard(id="m")])) + svc, _, _ = _service_with(client) + + asyncio.run(svc.get_model()) + asyncio.run(svc.get_model()) + + assert [s for _, s in client.calls] == ["w0:8001", "w1:8001"] + + +def test_get_model_without_a_context_worker_is_an_error_not_an_empty_list(): + """An empty list would read as "this server has no models", which is false. + + The model exists; there is currently nobody to ask. The server turns this + into a 503 so the client retries rather than concluding the deployment is + empty. + """ + client = _StubCtxClient(result=None) + svc, _, _ = _service_with(client, servers=()) + + with pytest.raises(RuntimeError): + asyncio.run(svc.get_model()) + assert client.calls == [] From c45513bf35ed018ce456a621d98334a2ae237b4a Mon Sep 17 00:00:00 2001 From: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> Date: Sat, 5 Sep 2026 07:59:31 -0700 Subject: [PATCH 07/43] [None][fix] Accept the input shapes a Responses client replays Two shapes a client sends back verbatim were refused, each failing the whole request rather than the item, which ends the conversation instead of the turn. A `reasoning` item does not have to carry `content`. The API puts the text in `summary` when a summary was requested and in `encrypted_content` when it was not, and an item with neither populated is a shape OpenAI itself emits. Sharing a branch with `message` meant an empty one raised, so a client replaying its own history got a 400 -- and what it lost was not the reasoning, which was never in the payload, but the conversation. The summary is now used when present and the item skipped when nothing readable is left. An empty `message` is still rejected: its text lives in `content` and nowhere else, so dropping it would lose something the client believed it had sent. An input item carrying a `role` may omit `type`; the API defaults it to "message". Such an item was returned verbatim, so its parts reached the chat-completions content parser still spelled in the Responses vocabulary -- `input_text`, `output_text` -- which that parser does not know: {"detail": "Unknown part type: input_text"} The same message with an explicit `"type": "message"` took a branch that handled the parts and succeeded, so whether a request worked turned on a field the client is free to leave out. A client replaying an assistant turn received from this server hit it, because those parts come back as `output_text`. The parts are translated rather than flattened to text, so an item mixing text with an image keeps the image: only the two Responses-only text spellings are rewritten and everything else is passed through. Signed-off-by: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> --- tensorrt_llm/serve/responses_utils.py | 59 ++++++++- .../apps/test_responses_input_preprocess.py | 118 ++++++++++++++++++ 2 files changed, 174 insertions(+), 3 deletions(-) diff --git a/tensorrt_llm/serve/responses_utils.py b/tensorrt_llm/serve/responses_utils.py index cbfe3a4fdb70..9ffa75a8cdfa 100644 --- a/tensorrt_llm/serve/responses_utils.py +++ b/tensorrt_llm/serve/responses_utils.py @@ -757,6 +757,33 @@ def _qualified_tool_name(item: dict) -> str: return f"{namespace}.{name}" if namespace else name +# The Responses API spells a text part `input_text` on the way in and +# `output_text` on the way out; the chat-completions content parser knows only +# `text`. Both mean the same thing, so they are translated rather than +# rejected. +_RESPONSES_TEXT_PART_TYPES = frozenset(("input_text", "output_text")) + + +def _chat_content_parts(content: list) -> list: + """Rewrite Responses-only text parts into the chat vocabulary. + + Everything else is passed through untouched: `image_url` and friends are + already understood downstream, and flattening the list to a plain string + would drop them. + """ + parts = [] + for part in content: + part_type = part.get("type") if isinstance(part, dict) else getattr( + part, "type", None) + if part_type in _RESPONSES_TEXT_PART_TYPES: + text = part.get("text") if isinstance(part, dict) else getattr( + part, "text", None) + parts.append({"type": "text", "text": text or ""}) + else: + parts.append(part) + return parts + + def _response_output_item_to_chat_completion_message( item: Union[dict, ResponseInputOutputItem] ) -> Optional[ChatCompletionMessageParam]: @@ -767,12 +794,38 @@ def _response_output_item_to_chat_completion_message( match item_type: case "": - if "role" in item: - return item - else: + if "role" not in item: raise ValueError(f"Invalid input message item: {item}") + content = item.get("content") + if isinstance(content, list): + # An item with a role and no `type` is the API's + # EasyInputMessage, where `type` defaults to "message". Its + # parts are spelled in the Responses vocabulary, so passing + # them through untouched hands `input_text` / `output_text` to + # the chat-completions parser, which knows neither and fails + # the request. The explicit "message" branch below already + # handles those parts, so leaving this one verbatim made + # success depend on a field the client may omit. + return {**item, "content": _chat_content_parts(content)} + return item case "message" | "reasoning": content = item.get("content") or [] + if not content and item_type == "reasoning": + # Reasoning does not have to carry `content`. The API puts the + # text in `summary` when a summary was requested and in + # `encrypted_content` when it was not, and an item with neither + # populated is a shape OpenAI itself emits. Falling back to the + # summary keeps whatever text exists; when nothing readable is + # left the item is skipped rather than rejected, because the + # reasoning was already absent from the payload and failing + # here costs the whole conversation instead. + summary = item.get("summary") or [] + summary_text = "".join( + part.get("text") or "" if isinstance(part, dict) else ( + getattr(part, "text", "") or "") for part in summary) + if not summary_text: + return None + return {"role": "assistant", "reasoning": summary_text} if not content: raise ValueError( f"Input item of type {item_type!r} has empty or missing 'content'" diff --git a/tests/unittest/llmapi/apps/test_responses_input_preprocess.py b/tests/unittest/llmapi/apps/test_responses_input_preprocess.py index b9263c720617..5277eb42ae04 100644 --- a/tests/unittest/llmapi/apps/test_responses_input_preprocess.py +++ b/tests/unittest/llmapi/apps/test_responses_input_preprocess.py @@ -393,3 +393,121 @@ def test_meaningful_values_survive_the_null_scrub(): ) assert request.temperature == 0.25 assert request.tool_choice == "none" + + +def test_reasoning_without_content_falls_back_to_the_summary(): + """The text is in `summary` whenever a summary was requested. + + Reading only `content` threw the summary away and then rejected the item + for being empty, so a client that asked for reasoning summaries lost both + the reasoning and the turn. + """ + msg = _response_output_item_to_chat_completion_message( + { + "type": "reasoning", + "id": "rs_1", + "summary": [{"type": "summary_text", "text": "weighed two options"}], + } + ) + + assert msg == {"role": "assistant", "reasoning": "weighed two options"} + + +def test_reasoning_with_nothing_readable_is_skipped_not_rejected(): + """A shape OpenAI emits, and one a client replays back verbatim. + + With `encrypted_content` and no summary there is no reasoning text in the + payload to preserve. Raising here does not save anything -- the text was + already absent -- it just fails the whole request, which ends the + conversation rather than the turn. + """ + msg = _response_output_item_to_chat_completion_message( + { + "type": "reasoning", + "id": "rs_2", + "summary": [], + "encrypted_content": "gAAAAA", + } + ) + + assert msg is None + + +def test_a_message_with_no_content_is_still_rejected(): + """The tolerance above is specific to reasoning. + + A message carries its text in `content` and nowhere else, so an empty one + is malformed and silently dropping it would lose something the client + believed it had sent. + """ + with pytest.raises(ValueError): + _response_output_item_to_chat_completion_message( + { + "type": "message", + "role": "user", + "content": [], + } + ) + + +def test_role_without_type_gets_its_parts_translated(): + """`type` is optional on an input message; the parts still need rewriting. + + This is the API's EasyInputMessage. Returning it verbatim handed + `input_text` to the chat-completions content parser, which knows only + `text` and failed the request with "Unknown part type: input_text" -- so + whether a request worked depended on a field the client may leave out. + """ + msg = _response_output_item_to_chat_completion_message( + { + "role": "user", + "content": [{"type": "input_text", "text": "hello"}], + } + ) + + assert msg == {"role": "user", "content": [{"type": "text", "text": "hello"}]} + + +def test_an_assistant_turn_replayed_without_a_type_is_accepted(): + """Replaying this server's own output must not need a field it omits. + + The assistant parts come back spelled `output_text`, which is what the + Responses API calls them. + """ + msg = _response_output_item_to_chat_completion_message( + { + "role": "assistant", + "content": [{"type": "output_text", "text": "hi"}], + } + ) + + assert msg == {"role": "assistant", "content": [{"type": "text", "text": "hi"}]} + + +def test_non_text_parts_survive_the_translation(): + """Only the two Responses-only text spellings are rewritten. + + Flattening the list to a string would have been simpler and would have + dropped the image, which the downstream parser does understand. + """ + image = {"type": "image_url", "image_url": {"url": "http://example/x.png"}} + msg = _response_output_item_to_chat_completion_message( + { + "role": "user", + "content": [{"type": "input_text", "text": "what is this"}, image], + } + ) + + assert msg["content"] == [{"type": "text", "text": "what is this"}, image] + + +def test_a_plain_string_content_is_left_alone(): + """The common shape must not be disturbed by the list handling.""" + msg = _response_output_item_to_chat_completion_message( + { + "role": "user", + "content": "hello", + } + ) + + assert msg == {"role": "user", "content": "hello"} From f0f56b48f4895539da1f0114f48d26c426a75948 Mon Sep 17 00:00:00 2001 From: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> Date: Sat, 5 Sep 2026 20:20:38 -0700 Subject: [PATCH 08/43] [None][fix] Honour the tools Codex declares as an input item Codex does not put its tools in `tools`. It sends an input item {"type": "additional_tools", "role": "developer", "tools": [{"type": "namespace", "name": "functions", "tools": [...]}]} and the input-item conversion had no case for it. Carrying no text, it fell to the catch-all and was dropped: Responses API: skipping unrecognised input item type 'additional_tools' with no text content. so the model was offered nothing to call. A Codex campaign against a disaggregated GLM-5.3 endpoint made zero tool calls across its whole round and, with no way to act, narrated a shell session it had invented -- which reads as a model failure rather than a dropped field, and cost a round to diagnose. The payload is already the shape `tools` accepts: `NamespaceTool` is a member of the tool union, and `_namespaced_tool_names` and `_get_chat_completion_function_tools` already flatten such entries to `namespace.tool`. So the item is hoisted into `tools` in a `mode="before"` validator and its contents are validated as tools like any other. Hoisting at the protocol boundary rather than in the prompt builder keeps the reply side in agreement: a call can only be reported back with its namespace split out if the tool was in `tools` when the reply was parsed, and a bare `functions.exec` matches nothing the client knows. The item is removed from `input` as it is hoisted. It declares tools rather than being a turn, and replaying it as one would put the tool schemas into the prompt as prose. Signed-off-by: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> --- tensorrt_llm/serve/openai_protocol.py | 37 +++++ .../apps/test_responses_input_preprocess.py | 129 ++++++++++++++++++ 2 files changed, 166 insertions(+) diff --git a/tensorrt_llm/serve/openai_protocol.py b/tensorrt_llm/serve/openai_protocol.py index a4dbb9087ee8..eadedbdf2a0d 100644 --- a/tensorrt_llm/serve/openai_protocol.py +++ b/tensorrt_llm/serve/openai_protocol.py @@ -1694,6 +1694,43 @@ def _null_top_level_fields_mean_unset(cls, data): return data return {k: v for k, v in data.items() if v is not None} + @model_validator(mode="before") + @classmethod + def hoist_additional_tools(cls, data): + """Move an `additional_tools` input item into `tools`. + + Codex declares its tools this way rather than in `tools`. The item + carries no text, so the input-item conversion dropped it as + unrecognised and the model was offered nothing to call. + + The payload is already what `tools` accepts -- namespace entries whose + nested tools are flattened to `namespace.tool` downstream -- so it is + moved here, before validation, and the entries are validated as tools + like any other. Hoisting at the protocol boundary rather than in the + prompt builder keeps the response side in agreement: a call can only + be reported back with its namespace if the tool was in `tools` when + the reply was parsed. + """ + if not isinstance(data, dict): + return data + items = data.get("input") + if not isinstance(items, list): + return data + hoisted = [] + kept = [] + for item in items: + if isinstance(item, + dict) and item.get("type") == "additional_tools": + hoisted.extend(item.get("tools") or []) + else: + kept.append(item) + if not hoisted: + return data + data = dict(data) + data["input"] = kept + data["tools"] = list(data.get("tools") or []) + hoisted + return data + @model_validator(mode="before") @classmethod def validate_background(cls, data): diff --git a/tests/unittest/llmapi/apps/test_responses_input_preprocess.py b/tests/unittest/llmapi/apps/test_responses_input_preprocess.py index 5277eb42ae04..97a674c884a1 100644 --- a/tests/unittest/llmapi/apps/test_responses_input_preprocess.py +++ b/tests/unittest/llmapi/apps/test_responses_input_preprocess.py @@ -511,3 +511,132 @@ def test_a_plain_string_content_is_left_alone(): ) assert msg == {"role": "user", "content": "hello"} + + +def test_additional_tools_item_becomes_tools(): + """Codex declares its tools as an input item, not in `tools`. + + The item carries no text, so the input-item conversion dropped it as + unrecognised and the model was offered nothing to call -- it then narrated + a terminal session it had invented, because narrating was the only thing + left to do. + """ + from tensorrt_llm.serve.openai_protocol import ResponsesRequest + + request = ResponsesRequest( + model="m", + input=[ + { + "type": "additional_tools", + "role": "developer", + "tools": [ + { + "type": "namespace", + "name": "functions", + "description": "", + "tools": [ + { + "type": "function", + "name": "get_time", + "description": "Return the time.", + "parameters": {"type": "object", "properties": {}}, + } + ], + } + ], + }, + {"role": "user", "content": "what time is it"}, + ], + ) + + # The item is gone from the input: it is a tool declaration, not a turn, + # and replaying it as one would put the tool schema in the prompt as prose. + assert [item.get("type") for item in request.input] == [None] + assert len(request.tools) == 1 + assert request.tools[0].type == "namespace" + assert [t.name for t in request.tools[0].tools] == ["get_time"] + + +def test_hoisted_tools_are_offered_to_the_template_namespaced(): + """The nested tools have to reach the prompt under qualified names. + + `_namespaced_tool_names` maps a reply's call back to its namespace, so a + tool that never made it into `tools` would come back as an unsupported + call even if the model somehow guessed it. + """ + from tensorrt_llm.serve.openai_protocol import ResponsesRequest + from tensorrt_llm.serve.responses_utils import ( + _get_chat_completion_function_tools, + _namespaced_tool_names, + ) + + request = ResponsesRequest( + model="m", + input=[ + { + "type": "additional_tools", + "role": "developer", + "tools": [ + { + "type": "namespace", + "name": "collaboration", + "description": "", + "tools": [ + { + "type": "function", + "name": "spawn_agent", + "description": "", + "parameters": {"type": "object", "properties": {}}, + } + ], + } + ], + }, + { + "role": "user", + "content": "go", + }, + ], + ) + + offered = [t.function.name for t in _get_chat_completion_function_tools(request.tools)] + assert offered == ["collaboration.spawn_agent"] + assert _namespaced_tool_names(request.tools) == { + "collaboration.spawn_agent": ("collaboration", "spawn_agent"), + } + + +def test_tools_already_in_the_tools_field_are_kept(): + """Hoisting appends; it must not discard what the client sent normally.""" + from tensorrt_llm.serve.openai_protocol import ResponsesRequest + + request = ResponsesRequest( + model="m", + tools=[ + { + "type": "function", + "name": "existing", + "description": "", + "parameters": {"type": "object", "properties": {}}, + } + ], + input=[ + { + "type": "additional_tools", + "role": "developer", + "tools": [{"type": "namespace", "name": "ns", "description": "", "tools": []}], + } + ], + ) + + assert [getattr(t, "name", None) for t in request.tools] == ["existing", "ns"] + + +def test_a_request_without_the_item_is_untouched(): + """The common case must not be reshaped by the hoist.""" + from tensorrt_llm.serve.openai_protocol import ResponsesRequest + + request = ResponsesRequest(model="m", input="hello") + + assert request.input == "hello" + assert request.tools == [] From 0aba57608c56d599a81d91940aa2225edad3a8d3 Mon Sep 17 00:00:00 2001 From: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> Date: Sun, 6 Sep 2026 00:33:30 -0700 Subject: [PATCH 09/43] [None][fix] Translate a tool result that arrives as content parts Both tool-result branches assigned `output` straight to `content`, which is right when it is a string and wrong when the client structures it. Codex sends a list of Responses content parts, so `content` became a list spelled `input_text` and the chat-completions parser refused the request: NotImplementedError: Unknown part type: input_text A tool result arrives on every turn after the model's first tool call, so an agent that starts using tools has nearly all of its traffic rejected: 483 of 490 requests in one campaign round. The round was reported as the agent failing on its own scripts, which is what it looks like from the outside -- the server was refusing to serve it. `function_call_output` carried the same assumption and is fixed with it; only the custom-tool branch had fired so far because that is the shape this client happens to send. The earlier EasyInputMessage fix translates parts for message items and did not reach here, a tool result not being a message. Both now share one helper. Signed-off-by: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> --- tensorrt_llm/serve/responses_utils.py | 22 +++++- .../apps/test_responses_input_preprocess.py | 71 +++++++++++++++++++ 2 files changed, 91 insertions(+), 2 deletions(-) diff --git a/tensorrt_llm/serve/responses_utils.py b/tensorrt_llm/serve/responses_utils.py index 9ffa75a8cdfa..c2e668f8baf9 100644 --- a/tensorrt_llm/serve/responses_utils.py +++ b/tensorrt_llm/serve/responses_utils.py @@ -784,6 +784,24 @@ def _chat_content_parts(content: list) -> list: return parts +def _tool_output_content(output): + """A tool result's payload in the vocabulary the chat parser knows. + + `output` is a string on the simple path and a list of Responses content + parts when the client structures it. Passing a list through untouched + hands `input_text` to a parser that knows only `text`, which fails the + whole request rather than the item -- and a tool result arrives on every + turn after the first tool call, so the conversation stops there. + """ + if isinstance(output, list): + return _chat_content_parts(output) + if output is None: + return "" + if isinstance(output, str): + return output + return str(output) + + def _response_output_item_to_chat_completion_message( item: Union[dict, ResponseInputOutputItem] ) -> Optional[ChatCompletionMessageParam]: @@ -875,7 +893,7 @@ def _response_output_item_to_chat_completion_message( case "function_call_output": return { "role": "tool", - "content": item["output"], + "content": _tool_output_content(item["output"]), "tool_call_id": item["call_id"], } case "custom_tool_call": @@ -907,7 +925,7 @@ def _response_output_item_to_chat_completion_message( # turn that still renders, not a KeyError surfacing as a 500. return { "role": "tool", - "content": item.get("output") or "", + "content": _tool_output_content(item.get("output")), "tool_call_id": item.get("call_id") or "", } case "agent_message": diff --git a/tests/unittest/llmapi/apps/test_responses_input_preprocess.py b/tests/unittest/llmapi/apps/test_responses_input_preprocess.py index 97a674c884a1..e65826084f4d 100644 --- a/tests/unittest/llmapi/apps/test_responses_input_preprocess.py +++ b/tests/unittest/llmapi/apps/test_responses_input_preprocess.py @@ -640,3 +640,74 @@ def test_a_request_without_the_item_is_untouched(): assert request.input == "hello" assert request.tools == [] + + +def test_function_call_output_with_content_parts_is_translated(): + """A tool result may carry parts rather than a string. + + Assigning them to `content` untouched handed `input_text` to the + chat-completions parser, which knows only `text`, and the request failed + with "Unknown part type: input_text". + """ + msg = _response_output_item_to_chat_completion_message( + { + "type": "function_call_output", + "call_id": "call_1", + "output": [{"type": "input_text", "text": "42"}], + } + ) + + assert msg == { + "role": "tool", + "content": [{"type": "text", "text": "42"}], + "tool_call_id": "call_1", + } + + +def test_custom_tool_call_output_with_content_parts_is_translated(): + """The custom-tool branch had the same assumption. + + This is the one that fired in practice: a tool result arrives on every + turn after the model's first custom-tool call, so once an agent started + using tools nearly all of its traffic was rejected -- 483 of 490 requests + in one campaign round. + """ + msg = _response_output_item_to_chat_completion_message( + { + "type": "custom_tool_call_output", + "call_id": "call_2", + "output": [ + {"type": "input_text", "text": "ok"}, + {"type": "input_text", "text": " done"}, + ], + } + ) + + assert msg["role"] == "tool" + assert msg["content"] == [{"type": "text", "text": "ok"}, {"type": "text", "text": " done"}] + assert msg["tool_call_id"] == "call_2" + + +def test_a_string_tool_result_is_unchanged(): + """The simple shape must not be reshaped into parts.""" + msg = _response_output_item_to_chat_completion_message( + { + "type": "function_call_output", + "call_id": "call_3", + "output": "plain text", + } + ) + + assert msg["content"] == "plain text" + + +def test_a_missing_custom_tool_result_stays_empty(): + """Absent output is still not a reason to fail the turn.""" + msg = _response_output_item_to_chat_completion_message( + { + "type": "custom_tool_call_output", + "call_id": "call_4", + } + ) + + assert msg["content"] == "" From c6842ec00647418c181655fed36bcb2cf9609f77 Mon Sep 17 00:00:00 2001 From: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> Date: Sun, 6 Sep 2026 19:32:31 -0700 Subject: [PATCH 10/43] [None][fix] Render `developer` messages instead of dropping them Codex puts its whole operating brief in messages with role "developer": who the agent is, how to drive `exec`, the skills instructions, the environment, the multi-agent rules. GLM-5.3's chat template dispatches on role with branches for user, assistant, tool and system, and ends with no fallback, so those messages render to nothing at all: developer '[gMASK]<|system|>Reasoning Effort: Max<|user|>list the files<|assistant|>' 85 chars system '...<|system|>You are Codex. <|user|>...' 139 chars The model was being asked to drive an agent harness nobody had described to it. Tool *definitions* travel separately, in `tools`, so it knew which tools existed and improvised the rest: shell commands into a JavaScript-only sandbox, invented exec cell ids, sub-agents spawned into the same confusion. Four campaign rounds were read as the model failing to follow its tool descriptions when it had never received the instructions that go with them. `developer` is OpenAI's rename of `system`. Templates written before that rename only know `system`, and every template knows `system`, so the alias is lossless where a template handles both and is the difference between the instructions arriving and not arriving where it is not. Only the chat-template path is touched; harmony models take a different route that already maps the two roles the other way for its own format. test_item_role_is_preserved keeps its other roles: its point is that a role is never silently replaced by "assistant", which still holds. Signed-off-by: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> --- tensorrt_llm/serve/responses_utils.py | 19 ++++++- .../apps/test_responses_input_preprocess.py | 57 ++++++++++++++++++- 2 files changed, 73 insertions(+), 3 deletions(-) diff --git a/tensorrt_llm/serve/responses_utils.py b/tensorrt_llm/serve/responses_utils.py index c2e668f8baf9..f9b7cc7407b0 100644 --- a/tensorrt_llm/serve/responses_utils.py +++ b/tensorrt_llm/serve/responses_utils.py @@ -802,6 +802,21 @@ def _tool_output_content(output): return str(output) +# `developer` is OpenAI's rename of `system`. Chat templates written before +# that rename dispatch on role and simply have no branch for it -- GLM-5.3 +# ends its dispatch after `system` with no fallback -- so a developer message +# renders to nothing and the client's instructions never reach the model. +# Every template understands `system`. +_ROLE_ALIASES = {"developer": "system"} + + +def _chat_role(role: Optional[str]) -> str: + """The role to hand a chat template, as that template will understand it.""" + if not role: + return "assistant" + return _ROLE_ALIASES.get(role, role) + + def _response_output_item_to_chat_completion_message( item: Union[dict, ResponseInputOutputItem] ) -> Optional[ChatCompletionMessageParam]: @@ -814,6 +829,7 @@ def _response_output_item_to_chat_completion_message( case "": if "role" not in item: raise ValueError(f"Invalid input message item: {item}") + item = {**item, "role": _chat_role(item.get("role"))} content = item.get("content") if isinstance(content, list): # An item with a role and no `type` is the API's @@ -867,8 +883,7 @@ def _response_output_item_to_chat_completion_message( # template markup rather than an answer. Clients that send # structured input items (Codex CLI, the OpenAI SDK) always set a # role; a plain string input never reaches this function. - role = item.get("role") or "assistant" - return {"role": role, "content": text} + return {"role": _chat_role(item.get("role")), "content": text} case "function_call": # An assistant message carrying tool_calls, which is how the chat # completions path represents a call and what chat templates diff --git a/tests/unittest/llmapi/apps/test_responses_input_preprocess.py b/tests/unittest/llmapi/apps/test_responses_input_preprocess.py index e65826084f4d..3971118f0b55 100644 --- a/tests/unittest/llmapi/apps/test_responses_input_preprocess.py +++ b/tests/unittest/llmapi/apps/test_responses_input_preprocess.py @@ -50,7 +50,13 @@ def _message_item(role, *texts, item_id=None): # --------------------------------------------------------------------------- -@pytest.mark.parametrize("role", ["user", "assistant", "system", "developer"]) +# `developer` is deliberately not in this list: it is aliased to `system` +# because chat templates written before OpenAI renamed the role have no branch +# for it and drop the message entirely. See +# test_a_developer_message_is_rendered_as_system for that case. The point of +# the test below -- that a role is never silently replaced by "assistant" -- +# still holds for it. +@pytest.mark.parametrize("role", ["user", "assistant", "system"]) def test_item_role_is_preserved(role): """Regression: the role was hardcoded to "assistant". @@ -711,3 +717,52 @@ def test_a_missing_custom_tool_result_stays_empty(): ) assert msg["content"] == "" + + +def test_a_developer_message_is_rendered_as_system(): + """Codex puts its whole operating brief in `developer` messages. + + GLM-5.3's template dispatches on role with branches for user, assistant, + tool and system and no fallback, so a developer message rendered to + nothing: the model was asked to drive an agent harness it had never been + told about, and improvised -- shell into a JavaScript sandbox, invented + cell ids, sub-agents spawned into the same confusion. + + `developer` is OpenAI's rename of `system`, and every template knows + `system`. + """ + msg = _response_output_item_to_chat_completion_message( + { + "type": "message", + "role": "developer", + "content": [{"type": "input_text", "text": "You are Codex."}], + } + ) + + assert msg == {"role": "system", "content": "You are Codex."} + + +def test_a_developer_message_without_a_type_is_also_rendered(): + """The same brief arrives without `type` when the client omits it.""" + msg = _response_output_item_to_chat_completion_message( + { + "role": "developer", + "content": [{"type": "input_text", "text": "You are Codex."}], + } + ) + + assert msg["role"] == "system" + assert msg["content"] == [{"type": "text", "text": "You are Codex."}] + + +def test_other_roles_are_left_alone(): + """Only the alias is rewritten.""" + for role in ("user", "assistant", "system", "tool"): + msg = _response_output_item_to_chat_completion_message( + { + "type": "message", + "role": role, + "content": [{"type": "input_text", "text": "x"}], + } + ) + assert msg["role"] == role From 9422984b115f8ba5d346c8739b4130354bb166a4 Mon Sep 17 00:00:00 2001 From: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> Date: Mon, 7 Sep 2026 02:20:00 -0700 Subject: [PATCH 11/43] [None][fix] Forward a request under the field names it arrived with The orchestrator parses a request and re-serialises it for the context and generation workers. `model_dump` without `by_alias` writes internal field names, and where a field carries an alias the two differ. `schema` is the case that bites: pydantic cannot hold a field of that name -- it shadows `BaseModel.schema` -- so the model declares `schema_` aliased to `schema`. A client sends "text": {"format": {"type": "json_schema", "name": ..., "schema": {...}}} and the worker is handed {'name': ..., 'schema_': {...}, 'type': 'json_schema', 'strict': True} with no `schema` in it, so every member of the format union fails validation and the request comes back 400. One campaign round lost 43 of 195 Responses requests to this -- every structured-output call its agents made -- and nothing upstream said why, because the rejection happens on the worker after a body this server itself wrote. Only disaggregated serving re-serialises a request, so an aggregated deployment never sees it. All four serialisation sites are fixed, not just the JSON one the failure surfaced through; the msgpack path had the same omission. Emitting the alias is safe as well as correct: the workers set `populate_by_name=True` and accept either spelling, and the alias is the wire name, which is what is being written. Adapted to main, which now always forwards requests as msgpack: both forwarding sites go through one helper, _encode_request_body, and the test decodes the body exactly as the worker does. Signed-off-by: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> --- tensorrt_llm/serve/openai_client.py | 26 +++++++++++-- .../test_disagg_openai_client.py | 39 ++++++++++++++++++- 2 files changed, 61 insertions(+), 4 deletions(-) diff --git a/tensorrt_llm/serve/openai_client.py b/tensorrt_llm/serve/openai_client.py index f2e7de819b0c..d29b7c423632 100644 --- a/tensorrt_llm/serve/openai_client.py +++ b/tensorrt_llm/serve/openai_client.py @@ -82,7 +82,27 @@ # returned object to the type the caller asked for, so a mismatch is a type # error rather than an Any that silently propagates. class _SerializableRequest(Protocol): - def model_dump(self, *, mode: str = ..., exclude_unset: bool = ...) -> dict: ... + def model_dump( + self, *, mode: str = ..., exclude_unset: bool = ..., by_alias: bool = ... + ) -> dict: ... + + +def _encode_request_body(request: _SerializableRequest) -> bytes: + """Encode a request forwarded to a worker, under its wire field names. + + Where a field carries an alias the internal and wire names differ, and + the worker validates the wire name. ``schema`` is the case that bites: + pydantic cannot hold a field of that name, so the model declares + ``schema_`` aliased to ``schema``. Without ``by_alias`` a Responses + ``text.format`` JSON schema reached the worker as ``schema_``, every + member of the format union failed validation, and the request came back + 400 -- every structured-output call made through a disaggregated server. + The workers set ``populate_by_name=True``, so either spelling is accepted; + the alias is the one that is correct on the wire. + """ + return _msgpack_encoder.encode( + request.model_dump(mode="json", exclude_unset=True, by_alias=True) + ) _ResponseT = TypeVar("_ResponseT", bound=BaseModel) @@ -253,7 +273,7 @@ async def post_json( # same way -- hence the same msgpack transport as _send_request. The # worker decodes it because _MsgspecRoute is the app's route_class, so # it covers every route rather than just the completion ones. - body = _msgpack_encoder.encode(request.model_dump(mode="json", exclude_unset=True)) + body = _encode_request_body(request) headers = dict(MSGPACK_HEADERS) async with self._session.post(url, data=body, headers=headers) as response: if response.status >= 400: @@ -373,7 +393,7 @@ async def _post_with_retry( # Serialize once on the orchestrator's single event-loop thread. # Content-Type stays application/json so FastAPI still routes the # body through Request.json(); the header picks the decoder. - body = _msgpack_encoder.encode(request.model_dump(mode="json", exclude_unset=True)) + body = _encode_request_body(request) headers = dict(MSGPACK_HEADERS) if self._request_perf_metrics: headers[RETURN_METRICS_HEADER] = "1" diff --git a/tests/unittest/disaggregated/test_disagg_openai_client.py b/tests/unittest/disaggregated/test_disagg_openai_client.py index 0cf6b87834b4..6c37b735aeb7 100644 --- a/tests/unittest/disaggregated/test_disagg_openai_client.py +++ b/tests/unittest/disaggregated/test_disagg_openai_client.py @@ -28,13 +28,14 @@ validate_internal_disagg_request, validate_subagent_affinity, ) -from tensorrt_llm.serve.openai_client import OpenAIHttpClient +from tensorrt_llm.serve.openai_client import OpenAIHttpClient, _encode_request_body from tensorrt_llm.serve.openai_protocol import ( CompletionRequest, CompletionResponse, CompletionResponseChoice, ConversationParams, DisaggregatedParams, + ResponsesRequest, UsageInfo, ) from tensorrt_llm.serve.perf_metrics import ( @@ -1145,3 +1146,39 @@ async def test_transient_tcp_capped_at_5_when_max_retries_smaller(self): # 1 original + 5 retries assert session.post.call_count == 6 + + +def test_a_forwarded_request_keeps_the_field_names_it_arrived_with(): + """`schema` must not reach a worker spelled `schema_`. + + pydantic cannot hold a field called `schema` -- it shadows + `BaseModel.schema` -- so the model declares `schema_` with `schema` as its + alias. Serialising without `by_alias` sends the internal name, every + member of the format union fails to validate on the worker, and the + request comes back 400. That was 43 of 195 Responses requests in one + campaign round: every structured-output call the agents made. + + Only disaggregated serving re-serialises a request, so only it is + affected. The body is decoded exactly as the worker decodes it. + """ + request = ResponsesRequest.model_validate( + { + "model": "m", + "input": "hi", + "text": { + "format": { + "type": "json_schema", + "name": "structured_output", + "schema": {"type": "object"}, + "strict": True, + } + }, + } + ) + + body = msgspec.msgpack.decode(_encode_request_body(request)) + + assert body["text"]["format"]["schema"] == {"type": "object"} + assert "schema_" not in body["text"]["format"] + # And the worker can rebuild the request from what it was sent. + assert ResponsesRequest.model_validate(body).text.format.schema_ == {"type": "object"} From e18cec3fd549df62c183d64db7b132c88aae54ad Mon Sep 17 00:00:00 2001 From: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> Date: Mon, 7 Sep 2026 02:51:51 -0700 Subject: [PATCH 12/43] [None][fix] Write responses under their wire field names too The request-side fix was only half of it. A worker serialises its reply with `model_dump()`, so a field whose python name differs from its wire name went back under the python one, and the orchestrator -- rebuilding the reply with the OpenAI SDK's models, which do not set `populate_by_name` -- rejected it: 3 validation errors for ResponsesResponse text.format.ResponseFormatTextJSONSchemaConfig.schema Field required So `text.format` with a JSON schema went from 400 before the request fix to 500 after it. Both directions have to write the alias. This is not only a disaggregated concern: an aggregated deployment hands the same `schema_` straight to its client, under a name the API does not have. Both response paths are fixed, chat completions as well as responses. Verified end to end against a rebuilt server: the request now returns 200, where it was 400 before either fix and 500 after only the first. The unit suite passes either way -- it exercises both sides in one process, so nothing crosses a serialisation boundary and neither failure can appear there. That is why this was committed after the live check rather than before it. Signed-off-by: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> --- tensorrt_llm/serve/openai_server.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/tensorrt_llm/serve/openai_server.py b/tensorrt_llm/serve/openai_server.py index 6548123dcb5f..93f29596ebfa 100644 --- a/tensorrt_llm/serve/openai_server.py +++ b/tensorrt_llm/serve/openai_server.py @@ -2303,7 +2303,14 @@ async def chat_stream_generator( np.asarray(response.prompt_token_ids, dtype=np.int32).tobytes()).decode("ascii") response.prompt_token_ids = None - return JSONResponse(content=response.model_dump()) + # by_alias: a field whose python name differs from its + # wire name -- `schema_` for `schema`, which pydantic + # will not let a model call `schema` -- otherwise goes + # out under the internal name. The OpenAI SDK models + # this is rebuilt from downstream do not set + # `populate_by_name` and reject it, and a direct client + # is handed a field name the API does not have. + return JSONResponse(content=response.model_dump(by_alias=True)) except CppExecutorError: logger.error(traceback.format_exc()) # If internal executor error is raised, shutdown the server @@ -3292,7 +3299,14 @@ async def create_streaming_generator(promise: RequestOutput, np.asarray(response.prompt_token_ids, dtype=np.int32).tobytes()).decode("ascii") response.prompt_token_ids = None - return JSONResponse(content=response.model_dump()) + # by_alias: a field whose python name differs from its + # wire name -- `schema_` for `schema`, which pydantic + # will not let a model call `schema` -- otherwise goes + # out under the internal name. The OpenAI SDK models + # this is rebuilt from downstream do not set + # `populate_by_name` and reject it, and a direct client + # is handed a field name the API does not have. + return JSONResponse(content=response.model_dump(by_alias=True)) except CppExecutorError: logger.error(traceback.format_exc()) # If internal executor error is raised, shutdown the server From d2bc0d29ff2cca221ab3243574492caf2b60fc7e Mon Sep 17 00:00:00 2001 From: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> Date: Wed, 9 Sep 2026 09:04:04 -0700 Subject: [PATCH 13/43] [None][fix] Report the cached token count the context phase measured Under disaggregated serving every Responses reply claimed a complete cache hit: `cached_tokens` came back equal to `input_tokens`, on a cold prompt as readily as on a warm one. A client that watches this to decide when to compact its context -- which is what the Codex CLI does with it -- is being told the prompt is free. The number is not wrong so much as unmeasurable where it was read. From the generation worker the entire prompt arrives as KV transferred from the context worker, so `GenerationResult.cached_tokens` is the prompt length by construction. Only the context phase knows how much of that prompt it actually found in its cache, and it sends its own usage across with the handoff. Chat completions has read it from there since PR #14177; the Responses path, added later in PR #18130, was still reading the local counts. Two halves were missing, and either alone leaves the count wrong: The proxy turns the context reply's `ResponseUsage` into the `UsageInfo` shape that `disaggregated_params.ctx_usage` carries, and copied only prompt, completion and total. `input_tokens_details.cached_tokens` was dropped, so even a reader that went looking for it found nothing. `_create_usage` never looked. It now takes the prompt and cached counts from the handoff when there is one, mirroring `_ctx_usage_from_outputs` on the chat side, and falls back to the local counts when there is not -- an aggregated server has no handoff and its own numbers are the right ones. Output tokens stay local either way; only the generation worker knows what it generated. Both the streaming and non-streaming paths reach `_create_usage`, so this covers them together. Signed-off-by: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> --- tensorrt_llm/serve/openai_disagg_service.py | 11 ++ tensorrt_llm/serve/responses_utils.py | 35 ++++- .../apps/test_responses_usage_disagg.py | 137 ++++++++++++++++++ 3 files changed, 182 insertions(+), 1 deletion(-) create mode 100644 tests/unittest/llmapi/apps/test_responses_usage_disagg.py diff --git a/tensorrt_llm/serve/openai_disagg_service.py b/tensorrt_llm/serve/openai_disagg_service.py index 2f0f0cd440f4..2b58329bdf56 100644 --- a/tensorrt_llm/serve/openai_disagg_service.py +++ b/tensorrt_llm/serve/openai_disagg_service.py @@ -32,6 +32,7 @@ DisaggregatedParams, DisaggScheduleStyle, ModelList, + PromptTokensDetails, ResponsesRequest, ResponsesResponse, UCompletionRequest, @@ -80,10 +81,20 @@ def _ctx_usage_info(response: UCompletionResponse) -> Optional[UsageInfo]: usage = getattr(response, "usage", None) if usage is None or isinstance(usage, UsageInfo): return usage + # The cached count has to make the trip too. It is the one number the + # generation worker cannot recompute: from there the whole prompt arrived + # as transferred KV, so its own cached_tokens is the prompt length and + # reporting it would say every request was a full cache hit. Dropping it + # here left `ctx_usage` carrying three of the four numbers, so a consumer + # that read it still had to invent the fourth. + cached = getattr(usage, "input_tokens_details", None) return UsageInfo( prompt_tokens=usage.input_tokens, completion_tokens=usage.output_tokens, total_tokens=usage.total_tokens, + prompt_tokens_details=PromptTokensDetails( + cached_tokens=getattr(cached, "cached_tokens", 0) or 0 + ), ) diff --git a/tensorrt_llm/serve/responses_utils.py b/tensorrt_llm/serve/responses_utils.py index f9b7cc7407b0..fbd744676341 100644 --- a/tensorrt_llm/serve/responses_utils.py +++ b/tensorrt_llm/serve/responses_utils.py @@ -48,6 +48,7 @@ from tensorrt_llm.executor import GenerationResult from tensorrt_llm.inputs.utils import async_apply_chat_template from tensorrt_llm.llmapi import SamplingParams +from tensorrt_llm.llmapi.disagg_utils import get_usage_tokens_from_ctx from tensorrt_llm.llmapi.llm import RequestOutput from tensorrt_llm.llmapi.reasoning_parser import (BaseReasoningParser, ReasoningParserFactory, @@ -71,7 +72,7 @@ ResponseUsage, StreamingResponsesResponse, UCompletionRequest, - UCompletionResponse, + UCompletionResponse, UsageInfo, to_disaggregated_params) from tensorrt_llm.serve.responses_web_search import is_web_search_tool from tensorrt_llm.serve.tool_parser.base_tool_parser import ( @@ -1686,6 +1687,25 @@ def _tool_call_output_item( return item +def _ctx_usage_from_result(final_res: GenerationResult) -> Optional[Any]: + """The context phase's usage, as the handoff left it on the outputs. + + Mirrors the chat path's `_ctx_usage_from_outputs`. An aggregated server has + no handoff and returns None, which leaves the caller on the local counts. + """ + for output in getattr(final_res, "outputs", None) or []: + disaggregated_params = getattr(output, "disaggregated_params", None) + if disaggregated_params is None: + continue + ctx_usage = getattr(disaggregated_params, "ctx_usage", None) + if ctx_usage is None: + continue + if isinstance(ctx_usage, UsageInfo): + return ctx_usage + return UsageInfo.model_validate(ctx_usage) + return None + + def _create_usage( final_res: GenerationResult, num_prompt_tokens: Optional[int] = None) -> Optional[ResponseUsage]: @@ -1712,6 +1732,19 @@ def _create_usage( output_tokens = sum(len(output.token_ids) for output in final_res.outputs) cached_tokens = getattr(final_res, "cached_tokens", None) or 0 + # Under disaggregated serving the counts above describe the generation + # worker, and from there the entire prompt arrived as KV transferred from + # the context worker -- so its cached_tokens equals the prompt length and + # every response claims a complete cache hit. The context phase is the only + # place that knows what was really reused, and it sends its own usage along + # with the handoff. Chat completions has taken it from there since #14177; + # this route was reading the generation worker's numbers instead. + ctx_prompt_tokens, ctx_cached_tokens = get_usage_tokens_from_ctx( + _ctx_usage_from_result(final_res)) + if ctx_prompt_tokens is not None: + input_tokens = ctx_prompt_tokens + cached_tokens = ctx_cached_tokens + return ResponseUsage( input_tokens=input_tokens, input_tokens_details=InputTokensDetails(cached_tokens=cached_tokens), diff --git a/tests/unittest/llmapi/apps/test_responses_usage_disagg.py b/tests/unittest/llmapi/apps/test_responses_usage_disagg.py new file mode 100644 index 000000000000..13edab83b6fd --- /dev/null +++ b/tests/unittest/llmapi/apps/test_responses_usage_disagg.py @@ -0,0 +1,137 @@ +"""Usage accounting for the Responses API under disaggregated serving. + +The generation worker cannot measure prefix reuse: the whole prompt reaches it +as KV transferred from the context worker, so its own `cached_tokens` equals the +prompt length. Reporting that says every request was a complete cache hit, which +is what a client watching its context window would act on. The real number comes +across with the handoff, and chat completions has read it from there since +PR #14177. +""" + +import unittest +from types import SimpleNamespace + +from tensorrt_llm.serve.openai_protocol import PromptTokensDetails, UsageInfo +from tensorrt_llm.serve.responses_utils import _create_usage + + +def result(prompt_len, output_len, cached, ctx_usage=None): + """A finished generation, shaped as the postprocessor sees one.""" + output = SimpleNamespace( + token_ids=list(range(output_len)), + disaggregated_params=( + SimpleNamespace(ctx_usage=ctx_usage) if ctx_usage is not None else None + ), + ) + return SimpleNamespace( + outputs=[output], + prompt_token_ids=list(range(prompt_len)), + cached_tokens=cached, + ) + + +def ctx(prompt_tokens, cached_tokens): + return UsageInfo( + prompt_tokens=prompt_tokens, + completion_tokens=1, + total_tokens=prompt_tokens + 1, + prompt_tokens_details=PromptTokensDetails(cached_tokens=cached_tokens), + ) + + +class Aggregated(unittest.TestCase): + """No handoff: the local counts are the only ones, and they are correct.""" + + def test_local_counts_are_used(self): + usage = _create_usage(result(100, 5, cached=32)) + self.assertEqual(100, usage.input_tokens) + self.assertEqual(5, usage.output_tokens) + self.assertEqual(32, usage.input_tokens_details.cached_tokens) + + def test_no_cache_hit_reports_zero(self): + usage = _create_usage(result(100, 5, cached=0)) + self.assertEqual(0, usage.input_tokens_details.cached_tokens) + + +class Disaggregated(unittest.TestCase): + def test_the_context_phase_decides_the_cached_count(self): + """The 100 the generation worker reports is the prompt length, not reuse.""" + usage = _create_usage(result(100, 5, cached=100, ctx_usage=ctx(100, 32))) + self.assertEqual(32, usage.input_tokens_details.cached_tokens) + self.assertEqual(100, usage.input_tokens) + + def test_a_cold_prompt_is_not_reported_as_fully_cached(self): + """The failure this fixes: cached == input on every single request.""" + usage = _create_usage(result(4096, 10, cached=4096, ctx_usage=ctx(4096, 0))) + self.assertEqual(0, usage.input_tokens_details.cached_tokens) + self.assertNotEqual(usage.input_tokens, usage.input_tokens_details.cached_tokens) + + def test_the_context_phase_also_decides_the_prompt_length(self): + usage = _create_usage( + result(100, 5, cached=100, ctx_usage=ctx(96, 64)), num_prompt_tokens=100 + ) + self.assertEqual(96, usage.input_tokens) + self.assertEqual(64, usage.input_tokens_details.cached_tokens) + + def test_output_tokens_stay_local(self): + """Only the generation worker knows what it generated.""" + usage = _create_usage(result(100, 7, cached=100, ctx_usage=ctx(100, 50))) + self.assertEqual(7, usage.output_tokens) + + def test_a_handoff_carrying_no_details_reports_no_reuse(self): + bare = UsageInfo(prompt_tokens=100, completion_tokens=1, total_tokens=101) + usage = _create_usage(result(100, 5, cached=100, ctx_usage=bare)) + self.assertEqual(0, usage.input_tokens_details.cached_tokens) + + def test_a_handoff_arriving_as_a_dict_is_accepted(self): + """It crosses a process boundary, so it may not still be a model.""" + as_dict = ctx(100, 40).model_dump() + usage = _create_usage(result(100, 5, cached=100, ctx_usage=as_dict)) + self.assertEqual(40, usage.input_tokens_details.cached_tokens) + + +class ProxyHandoff(unittest.TestCase): + """The proxy converts the context reply's usage into the handoff shape.""" + + def test_cached_tokens_survive_the_conversion(self): + from tensorrt_llm.serve.openai_disagg_service import _ctx_usage_info + + # Shaped by hand rather than through the SDK model: the two carry the + # same field names but not the same required set, and this test is + # about what the proxy copies, not about the SDK's schema. + response = SimpleNamespace( + usage=SimpleNamespace( + input_tokens=96, + input_tokens_details=SimpleNamespace(cached_tokens=64), + output_tokens=1, + total_tokens=97, + ) + ) + carried = _ctx_usage_info(response) + self.assertEqual(96, carried.prompt_tokens) + self.assertEqual(1, carried.completion_tokens) + self.assertEqual(64, carried.prompt_tokens_details.cached_tokens) + + def test_a_reply_without_details_still_converts(self): + from tensorrt_llm.serve.openai_disagg_service import _ctx_usage_info + + response = SimpleNamespace( + usage=SimpleNamespace( + input_tokens=96, + input_tokens_details=None, + output_tokens=1, + total_tokens=97, + ) + ) + carried = _ctx_usage_info(response) + self.assertEqual(0, carried.prompt_tokens_details.cached_tokens) + + def test_a_usage_that_is_already_in_the_handoff_shape_passes_through(self): + from tensorrt_llm.serve.openai_disagg_service import _ctx_usage_info + + already = ctx(96, 64) + self.assertIs(already, _ctx_usage_info(SimpleNamespace(usage=already))) + + +if __name__ == "__main__": + unittest.main(verbosity=2) From 1edd91df60fa9b4557f443830a3df13767d84954 Mon Sep 17 00:00:00 2001 From: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> Date: Sat, 12 Sep 2026 12:56:09 +0000 Subject: [PATCH 14/43] [None][fix] Emit the reasoning that shares a chunk with the answer A generation chunk whose raw text spans the closing think tag parses into two halves: the reasoning up to the tag, and the beginning of the answer after it. The streaming dispatch chose its branch on the answer half alone, so the reasoning half fell into an `elif` that could not run, and nothing else read it. It was emitted nowhere and the item was closed in the same breath. Closing it is what made the loss permanent. _close_open_item clears is_reasoning_sent, and _should_send_done_events is guarded on that flag, so the complete text it computes from a non-streaming re-parse was discarded too -- the done event carried the truncated text as well. A client following the documented advice of reading reasoning_text.done rather than accumulating deltas got exactly the same short string. This is not a boundary case. Four independent full censuses of recorded production traffic -- 196,000 reasoning items across 31 instances and two fleets -- put it at 55.5%, 56.0%, 55.9% and 62.1% of reasoning items, with the dropped tail bounded by one chunk (max 369 chars, median ~100) and 3.15% of all reasoning text lost. It is 100% for reasoning short enough to fit inside a single chunk, which is why the damage looked like stray first words: "Let", "The", "OK". Turns that end in a tool call emit no answer text, so the other branch ran and they were never affected -- 0% there, against 87.7% of turns that produced visible text. That split is what identified the branch. The fix flushes the reasoning half before closing, exactly as the `elif` would have. Four tests drive the real dispatch over chunk sequences that straddle, align on a boundary, end without an answer, and hold reasoning shorter than one chunk. Nothing covered this dispatch before, which is how it shipped. Not fixed here, and not the same defect: reasoning lost on streams that end without response.completed, reasoning misclassified into the visible answer, the snapshot ordering reasoning after the message, and encrypted_content being requested on every call and never returned. Signed-off-by: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> --- tensorrt_llm/serve/responses_utils.py | 26 ++++++ .../apps/test_responses_streaming_events.py | 89 ++++++++++++++++++- 2 files changed, 114 insertions(+), 1 deletion(-) diff --git a/tensorrt_llm/serve/responses_utils.py b/tensorrt_llm/serve/responses_utils.py index fbd744676341..ed07c994d213 100644 --- a/tensorrt_llm/serve/responses_utils.py +++ b/tensorrt_llm/serve/responses_utils.py @@ -2364,6 +2364,32 @@ def check_parser(parser_id: Optional[str], # get_*_output_added_events is idempotent - it is guarded internally by # sent_output_item_added - so calling it for every delta is safe. if delta_text: + # One chunk can carry both halves. The chunk whose raw text spans the + # closing think tag is parsed into a reasoning part (everything before + # the tag) and a content part (everything after), and this branch is + # chosen on the content part alone -- so the reasoning part has to be + # flushed here or it is never emitted at all. The `elif` below cannot + # run for this chunk, and nothing else reads reasoning_delta_text. + # + # It is not a rare boundary case. The reasoning between the last chunk + # boundary and the tag is lost every time the two land in one chunk, + # which measured 56% of reasoning items across four fleets, and 100% + # of reasoning short enough to fit inside a single chunk. Closing the + # item first is what makes it unrecoverable: _close_open_item clears + # is_reasoning_sent, and _should_send_done_events below is guarded on + # that flag -- so the correct full text it computes is discarded, and + # the done event carries the truncated text too. + if reasoning_delta_text: + if streaming_events_helper.is_text_sent: + yield from _close_open_item(streaming_events_helper) + if not streaming_events_helper.is_reasoning_sent: + streaming_events_helper.is_reasoning_sent = True + yield from streaming_events_helper.get_reasoning_output_added_events( + ) + streaming_events_helper.append_reasoning(reasoning_delta_text) + yield streaming_events_helper.get_reasoning_text_delta_event( + reasoning_delta_text) + # Reasoning has ended and the answer is starting: close the reasoning # item so the message deltas are not attributed to it. if streaming_events_helper.is_reasoning_sent: diff --git a/tests/unittest/llmapi/apps/test_responses_streaming_events.py b/tests/unittest/llmapi/apps/test_responses_streaming_events.py index d4c9ca17f738..46e9435d2e32 100644 --- a/tests/unittest/llmapi/apps/test_responses_streaming_events.py +++ b/tests/unittest/llmapi/apps/test_responses_streaming_events.py @@ -23,7 +23,10 @@ import pytest -from tensorrt_llm.serve.responses_utils import ResponsesStreamingEventsHelper +from tensorrt_llm.serve.responses_utils import ( + ResponsesStreamingEventsHelper, + _generate_streaming_event, +) # The CPU-* CI stages run pytest with -m 'cpu_only'. Without this marker every # test in the file is deselected, which pytest reports as exit code 5 and the @@ -125,6 +128,90 @@ def test_done_events_carry_the_open_item_id(): assert helper.get_content_part_done_event(_output_text("hi")).item_id == item_id +# --------------------------------------------------------------------------- +# Reasoning that shares a chunk with the answer +# --------------------------------------------------------------------------- + + +class _FakeOutput: + """The three attributes _generate_streaming_event reads from an output.""" + + def __init__(self, text_diff, index=0): + self.text_diff = text_diff + self.index = index + + +class _FakeRequest: + tools = None + + +def _stream(chunks): + """Drive the real dispatch over a sequence of generation chunks. + + Returns (reasoning deltas, text deltas) as the client would receive them. + """ + helper = ResponsesStreamingEventsHelper() + parsers = {} + reasoning, text = [], [] + for i, chunk in enumerate(chunks): + events = _generate_streaming_event( + output=_FakeOutput(chunk), + request=_FakeRequest(), + finished_generation=(i == len(chunks) - 1), + streaming_events_helper=helper, + reasoning_parser_id="glm", + reasoning_parser_dict=parsers, + ) + for event in events: + kind = getattr(event, "type", "") + if kind == "response.reasoning_text.delta": + reasoning.append(event.delta) + elif kind == "response.output_text.delta": + text.append(event.delta) + return "".join(reasoning), "".join(text) + + +def test_reasoning_sharing_a_chunk_with_the_answer_is_not_dropped(): + """Regression: the reasoning half of the straddling chunk was discarded. + + A generation chunk whose raw text spans the closing think tag parses into + both a reasoning part and a content part. The dispatch chose its branch on + the content part alone, so the reasoning part fell into a branch that never + ran and was never emitted -- and closing the item cleared the flag the + done-event path is guarded on, so the done event carried the short text + too. Measured at 56% of reasoning items across four fleets. + """ + reasoning, text = _stream(["Let", " me check the reference.I will read the file."]) + assert reasoning == "Let me check the reference." + assert text == "I will read the file." + + +def test_reasoning_shorter_than_one_chunk_survives(): + """The worst case: the whole reasoning shares a chunk with the answer. + + Reasoning short enough to fit inside a single chunk was truncated to + nothing at all, which is why the losses looked like first words. + """ + reasoning, _ = _stream(["Sure.Here it is."]) + assert reasoning == "Sure." + + +def test_reasoning_ending_on_a_chunk_boundary_is_unaffected(): + """The case that always worked, kept so a fix cannot regress it.""" + reasoning, text = _stream( + ["Let", " me check the reference.", "", "I will read the file."] + ) + assert reasoning == "Let me check the reference." + assert text == "I will read the file." + + +def test_reasoning_with_no_answer_after_it_is_unaffected(): + """A turn that ends in a tool call emits no text, so the other branch ran.""" + reasoning, text = _stream(["Let", " me check the reference."]) + assert reasoning == "Let me check the reference." + assert text == "" + + def _output_text(text): from openai.types.responses import ResponseOutputText From efc3755952ebdb6bea04df6106821c2796c2de79 Mon Sep 17 00:00:00 2001 From: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> Date: Sat, 12 Sep 2026 13:26:57 +0000 Subject: [PATCH 15/43] [None][fix] Give the streaming test fake the accumulated text it reads The fake supplied only text_diff. The delta path parses the diff, but the done-event path re-parses output.text -- the whole generation so far -- and raised AttributeError on all four new tests, including the two that pass regardless of the fix. Found by running them; the scope check that built the fake read only the dispatch function and missed what it calls. Signed-off-by: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> --- .../apps/test_responses_streaming_events.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/tests/unittest/llmapi/apps/test_responses_streaming_events.py b/tests/unittest/llmapi/apps/test_responses_streaming_events.py index 46e9435d2e32..e49c19e7da24 100644 --- a/tests/unittest/llmapi/apps/test_responses_streaming_events.py +++ b/tests/unittest/llmapi/apps/test_responses_streaming_events.py @@ -134,9 +134,16 @@ def test_done_events_carry_the_open_item_id(): class _FakeOutput: - """The three attributes _generate_streaming_event reads from an output.""" + """The attributes the streaming path reads from a generation output. - def __init__(self, text_diff, index=0): + `text` is the whole generation so far and `text_diff` only the newest + chunk: the delta path parses the diff, while the done-event path re-parses + the accumulated text. Supplying only the diff makes the second one raise, + which is how the first version of this fake was wrong. + """ + + def __init__(self, text, text_diff, index=0): + self.text = text self.text_diff = text_diff self.index = index @@ -153,9 +160,11 @@ def _stream(chunks): helper = ResponsesStreamingEventsHelper() parsers = {} reasoning, text = [], [] + accumulated = "" for i, chunk in enumerate(chunks): + accumulated += chunk events = _generate_streaming_event( - output=_FakeOutput(chunk), + output=_FakeOutput(accumulated, chunk), request=_FakeRequest(), finished_generation=(i == len(chunks) - 1), streaming_events_helper=helper, From 36276b476126a3899a0bc377d98ecdc1b21d65e4 Mon Sep 17 00:00:00 2001 From: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> Date: Sat, 12 Sep 2026 13:55:07 +0000 Subject: [PATCH 16/43] [None][fix] Order the response snapshot reasoning-first, as the stream does The completed-response snapshot appended the message item before the reasoning item, while the streaming path emits reasoning, then message, then tool calls. The two descriptions of one generation disagreed: a client replaying `response.output` read the answer before the reasoning that produced it, and one reconstructing the turn from the snapshot fed the model its own thinking as a follow-up to its own reply. Measured across four full censuses of recorded traffic on 31 serving instances: 99.99% of responses carrying both a message and a reasoning item were inverted. The stream was never affected. Assemble the snapshot in the streamed order instead. Nothing indexes into `response.output` positionally; the only consumer of its order is the tool-call lookup in _parse_response_input, which scans for a matching call_id and is unaffected by where reasoning sits relative to the message. The stored conversation messages are built from a dict by _create_output_messages and do not change either. Signed-off-by: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> --- tensorrt_llm/serve/responses_utils.py | 35 +++-- .../apps/test_responses_streaming_events.py | 138 ++++++++++++++++++ 2 files changed, 161 insertions(+), 12 deletions(-) diff --git a/tensorrt_llm/serve/responses_utils.py b/tensorrt_llm/serve/responses_utils.py index ed07c994d213..40c27729bf17 100644 --- a/tensorrt_llm/serve/responses_utils.py +++ b/tensorrt_llm/serve/responses_utils.py @@ -1508,6 +1508,29 @@ def _create_output_content( text_item = None reasoning_item = None tool_calls_item = [] + + # Reasoning first, then the answer, then any tool calls. + # + # `response.output` is ordered, and the model produced the reasoning + # before the answer it leads to. The streaming path emits the items in + # that order - reasoning done, then message done, then the function + # calls (see _generate_streaming_event) - so a snapshot that appended + # the message first contradicted the stream of the very same + # generation: a client replaying `output` read the answer before the + # reasoning that produced it, and one reconstructing a turn from the + # snapshot fed the model its own thinking as a follow-up to its reply. + if reasoning_text: + reasoning_item = ResponseReasoningItem( + id=f"rs_{_random_uuid()}", + summary=[], + type="reasoning", + content=[ + Content(text=reasoning_text.strip(), type="reasoning_text") + ], + status=None, + ) + output_items.append(reasoning_item) + # Check again after tool parsing to avoid empty text if text: output_text = ResponseOutputText( @@ -1527,18 +1550,6 @@ def _create_output_content( output_items.append(text_item) - if reasoning_text: - reasoning_item = ResponseReasoningItem( - id=f"rs_{_random_uuid()}", - summary=[], - type="reasoning", - content=[ - Content(text=reasoning_text.strip(), type="reasoning_text") - ], - status=None, - ) - output_items.append(reasoning_item) - if calls: custom_tool_names = _custom_tool_names(tools) namespaced_tool_names = _namespaced_tool_names(tools) diff --git a/tests/unittest/llmapi/apps/test_responses_streaming_events.py b/tests/unittest/llmapi/apps/test_responses_streaming_events.py index e49c19e7da24..c9c5a5c1572a 100644 --- a/tests/unittest/llmapi/apps/test_responses_streaming_events.py +++ b/tests/unittest/llmapi/apps/test_responses_streaming_events.py @@ -225,3 +225,141 @@ def _output_text(text): from openai.types.responses import ResponseOutputText return ResponseOutputText(text=text, annotations=[], type="output_text", logprobs=None) + + +# --------------------------------------------------------------------------- +# Order of the items in the completed-response snapshot +# +# These cover the non-streaming assembly in `_create_output_content`, but they +# live here because what they assert is a property of the stream: the snapshot +# in `response.output` has to list the items in the order the stream emitted +# them. A client that reads both must not be told two different stories about +# one generation. This is also the file the CPU CI stage runs (l0_cpu.yml). +# --------------------------------------------------------------------------- + +_THINK = "Check the reference first." +_ANSWER = "The answer is 42." +_TOOL_CALL = '\n{"name": "read_file", "arguments": {"path": "a.txt"}}\n' + + +def _tools(): + """A real tool definition, so the tool parser accepts the parsed call.""" + from openai.types.responses.tool import FunctionTool + + return [ + FunctionTool( + name="read_file", + description="Read a file.", + parameters={"type": "object", "properties": {"path": {"type": "string"}}}, + strict=False, + type="function", + ) + ] + + +def _snapshot_kinds(text, tools=None): + """Item types of the completed-response snapshot for one generation. + + Drives the real `_create_output_content` with real parsers - `qwen3` for + both, so text with no `` is an ordinary answer rather than an + unterminated reasoning block. + """ + from tensorrt_llm.serve.responses_utils import _create_output_content + + items, _messages = _create_output_content( + _FakeRequestOutput(text), + reasoning_parser="qwen3", + tool_parser="qwen3" if tools else None, + tools=tools, + ) + return items, [item.type for item in items] + + +class _FakeRequestOutput: + """The one attribute `_create_output_content` reads off a result. + + It iterates `final_res.outputs` and takes `.index` and `.text` from each; + everything else it needs it derives from the parsers. + """ + + def __init__(self, text): + self.outputs = [_FakeOutput(text, text)] + + +def test_snapshot_lists_reasoning_before_the_message(): + """Regression: the snapshot appended the message first, inverting cause and effect. + + The stream emits reasoning then message, but the snapshot for the same + generation listed message then reasoning - so a client replaying + `response.output` read the answer before the reasoning that produced it. + Measured on 99.99% of responses carrying both items across four fleets. + """ + items, kinds = _snapshot_kinds(f"{_THINK}{_ANSWER}") + assert kinds == ["reasoning", "message"] + # Reordering must move the items, not relabel them. + assert items[0].content[0].text == _THINK + assert items[1].content[0].text == _ANSWER + + +def test_snapshot_lists_reasoning_then_message_then_tool_call(): + items, kinds = _snapshot_kinds( + f"{_THINK}Reading it now.\n{_TOOL_CALL}", tools=_tools() + ) + assert kinds == ["reasoning", "message", "function_call"] + assert items[2].name == "read_file" + + +def test_snapshot_lists_reasoning_before_a_tool_call_with_no_message(): + """A turn that ends in a tool call emits no message item at all.""" + _items, kinds = _snapshot_kinds(f"{_THINK}{_TOOL_CALL}", tools=_tools()) + assert kinds == ["reasoning", "function_call"] + + +def test_snapshot_of_a_plain_answer_is_just_the_message(): + """No reasoning to order: the message must not be dropped or displaced.""" + items, kinds = _snapshot_kinds(_ANSWER) + assert kinds == ["message"] + assert items[0].content[0].text == _ANSWER + + +def test_snapshot_order_matches_the_streamed_order(): + """The two paths must agree; this is the invariant the bug violated. + + Both sides are driven for real - `_generate_streaming_event` for the + stream and `_create_output_content` for the snapshot - over the same + generation and the same reasoning parser, rather than comparing the + snapshot against a hardcoded expectation of what the stream does. + """ + from tensorrt_llm.serve.responses_utils import ( + ResponsesStreamingEventsHelper, + _create_output_content, + ) + + # "glm" is reasoning_at_start, so the generation opens inside the + # reasoning block with no tag - the same setup as the streaming + # tests above, and what the thinking chat templates actually render. + chunks = [_THINK, "", _ANSWER] + + helper = ResponsesStreamingEventsHelper() + parsers = {} + accumulated = "" + streamed = [] + for i, chunk in enumerate(chunks): + accumulated += chunk + for event in _generate_streaming_event( + output=_FakeOutput(accumulated, chunk), + request=_FakeRequest(), + finished_generation=(i == len(chunks) - 1), + streaming_events_helper=helper, + reasoning_parser_id="glm", + reasoning_parser_dict=parsers, + ): + if getattr(event, "type", "") == "response.output_item.done": + streamed.append(event.item.type) + + snapshot_items, _messages = _create_output_content( + _FakeRequestOutput(accumulated), reasoning_parser="glm" + ) + + assert streamed == ["reasoning", "message"] + assert [item.type for item in snapshot_items] == streamed From 2ba85de21f145922ec34eb86b9ac8899f5eab590 Mon Sep 17 00:00:00 2001 From: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> Date: Sat, 12 Sep 2026 14:15:26 +0000 Subject: [PATCH 17/43] [None][fix] Say so when a Responses stream stops before completing Between 0.25% and 1.83% of /v1/responses streams end without response.completed. They carry no error event, no populated response.error and no incomplete_details: the byte stream simply stops, and nothing in it or in the trace distinguishes a cut stream from one still in flight. Because response.completed is the only event that repeats the full text, everything those turns produced survives solely as deltas -- 10.4 million characters of reasoning on one fleet, and a worst case of 610 deltas and 117,695 characters followed by nothing. A guard around the streaming generator now ends such a stream with an error event naming the cause and a response.failed snapshot, and reports the cause to the request trace. response.failed rather than response.incomplete because incomplete_details.reason is a closed enum -- max_output_tokens, max_messages, content_filter, steered -- with no member for a server fault or a client hangup, so response.incomplete could only go out with reason=null, which reads exactly like an ordinary truncation. A client hangup adds nothing to the stream. There is nobody left to send to, and yielding while GeneratorExit propagates raises "async generator ignored GeneratorExit"; the cause reaches the trace instead. The trace also learns to record a cancelled body iterator as client_disconnected rather than error, which is the bucket the census reads. Nothing is added once response.completed has gone out, so a stream that finishes normally is unchanged. No lost content is reconstructed: the two causes cannot yet be told apart well enough to know whether the accumulated buffers were still in memory, and making the failure visible is the prerequisite for deciding that. Adapted to main, which has no request trace: the trace wiring (request_trace.py and its tests) is not carried over, and both servers install the guard without an on_termination callback. Signed-off-by: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> --- tensorrt_llm/serve/openai_disagg_server.py | 24 +- tensorrt_llm/serve/openai_server.py | 10 +- tensorrt_llm/serve/responses_utils.py | 219 +++++++++++++- .../apps/test_responses_streaming_events.py | 283 ++++++++++++++++++ 4 files changed, 529 insertions(+), 7 deletions(-) diff --git a/tensorrt_llm/serve/openai_disagg_server.py b/tensorrt_llm/serve/openai_disagg_server.py index c6a16ad1caa5..339c620a3c42 100644 --- a/tensorrt_llm/serve/openai_disagg_server.py +++ b/tensorrt_llm/serve/openai_disagg_server.py @@ -68,7 +68,9 @@ PerfMetricsMiddleware, combine_disagg_metrics) from tensorrt_llm.serve.responses_utils import (ServerArrivalTimeMiddleware, - get_steady_clock_now_in_seconds) + get_steady_clock_now_in_seconds, + guard_responses_stream, + stream_error_event) from tensorrt_llm.serve.router import Router from tensorrt_llm.usage import TerminalOutcome, record_termination_observation from tensorrt_llm.version import __version__ as VERSION @@ -443,8 +445,26 @@ async def wrapper(req: request_type, raw_req: Request) -> Response: response_or_generator = await entry_point(req, hooks) self._perf_metrics_collector.total_responses.inc() if req.stream: + stream = response_or_generator + if isinstance(req, ResponsesRequest): + # Only the Responses protocol: a stream that stops + # before response.completed loses everything it + # produced, because that event is the only place the + # full text is repeated. The other protocols carry + # their content entirely in deltas and end on a + # sentinel, so a truncation there is already visible. + # + # A bare `error` event rather than the worker's + # `response.failed`: this is a byte relay with no view + # of the response being assembled. It does count the + # events it forwarded, so the sequence number is exact + # -- and it is zero for the failures that never + # reached a worker at all, which otherwise end with + # an entirely empty body. + stream = guard_responses_stream(stream, + stream_error_event) return StreamingResponse( - content=response_or_generator, + content=stream, media_type="text/event-stream") return JSONResponse(content=response_or_generator.model_dump()) except Exception as e: diff --git a/tensorrt_llm/serve/openai_server.py b/tensorrt_llm/serve/openai_server.py index 93f29596ebfa..94d441c90839 100644 --- a/tensorrt_llm/serve/openai_server.py +++ b/tensorrt_llm/serve/openai_server.py @@ -123,6 +123,7 @@ ServerArrivalTimeMiddleware) from tensorrt_llm.serve.responses_utils import \ create_response as responses_api_create_response +from tensorrt_llm.serve.responses_utils import guard_responses_stream from tensorrt_llm.serve.responses_utils import \ request_preprocess as responses_api_request_preprocess from tensorrt_llm.serve.responses_web_search import web_search_rejection_reason @@ -3281,8 +3282,13 @@ async def create_streaming_generator(promise: RequestOutput, asyncio.create_task(self.await_disconnected(raw_request, promise)) if request.stream: - return StreamingResponse(content=create_streaming_generator( - promise, postproc_params), + # Ends a stream that stops before `response.completed` with + # an `error` event and a `response.failed` snapshot, so the + # client can tell a truncated turn from one still in flight. + return StreamingResponse(content=guard_responses_stream( + create_streaming_generator(promise, postproc_params), + streaming_processor.get_stream_failed_events, + ), media_type="text/event-stream") else: response = await create_response(promise, postproc_params) diff --git a/tensorrt_llm/serve/responses_utils.py b/tensorrt_llm/serve/responses_utils.py index 40c27729bf17..c09be892e788 100644 --- a/tensorrt_llm/serve/responses_utils.py +++ b/tensorrt_llm/serve/responses_utils.py @@ -11,13 +11,15 @@ from abc import ABC, abstractmethod from collections.abc import AsyncGenerator, Mapping from copy import copy -from typing import Any, List, Literal, Optional, OrderedDict, Tuple, Union +from typing import (Any, Callable, List, Literal, Optional, OrderedDict, Tuple, + Union) from openai.types.responses import (ResponseCompletedEvent, ResponseContentPartAddedEvent, ResponseContentPartDoneEvent, ResponseCreatedEvent, - ResponseCustomToolCall, + ResponseCustomToolCall, ResponseErrorEvent, + ResponseFailedEvent, ResponseFunctionToolCall, ResponseInProgressEvent, ResponseOutputItem, ResponseOutputItemAddedEvent, @@ -45,7 +47,8 @@ from tensorrt_llm._utils import \ get_steady_clock_now_in_seconds # noqa: F401 (re-export) from tensorrt_llm._utils import AdjustedSteadyClock -from tensorrt_llm.executor import GenerationResult +from tensorrt_llm.executor import (EngineDeadError, GenerationResult, + RequestError) from tensorrt_llm.inputs.utils import async_apply_chat_template from tensorrt_llm.llmapi import SamplingParams from tensorrt_llm.llmapi.disagg_utils import get_usage_tokens_from_ctx @@ -2845,6 +2848,216 @@ def process_single_output(self, res: GenerationResult) -> list[str]: return [self._send_event(event) for event in event_generator] + def get_stream_failed_events(self, + cause: str, + detail: str, + events_sent: int = 0) -> List[str]: + """The terminal events for a stream that stopped before completing. + + Carries no generated content. The accumulated text lives only in the + deltas already on the wire and re-emitting it here would guess at + whether those deltas reached anyone; this says what happened, nothing + more. + + ``events_sent`` is the number of frames that actually reached the + wire. Normally it equals this processor's own counter, because this + processor numbered every one of them. With postprocessing workers it + does not: the per-token events are built by a pickled copy of this + object in another process and only the opening two are numbered here, + so the local counter is short by the whole body of the turn. Taking + the larger of the two keeps the terminal events after everything the + client has already seen either way. + """ + self.sequence_number = max(self.sequence_number, events_sent) + error_event = self._send_event( + ResponseErrorEvent( + type="error", + sequence_number=-1, + code=cause, + message=detail, + param=None, + )) + snapshot = ResponsesResponse.from_request( + request=self.request, + sampling_params=self.sampling_params, + model_name=self.model_name, + created_time=self.response_creation_time, + output=[], + status="failed", + usage=None, + ).model_dump() + # Set on the dump rather than on ResponsesResponse, whose `error` field + # is commented out. The event re-validates this dict against the SDK's + # Response, which does carry `error`, so the field reaches the wire + # without widening the local model - and without adding a key to the + # happy path's response.completed payload. + snapshot["error"] = {"code": "server_error", "message": detail} + failed_event = self._send_event( + ResponseFailedEvent( + type="response.failed", + sequence_number=-1, + response=snapshot, + )) + return [error_event, failed_event] + + +# -------------------------------------------------------------------------- +# Abnormal stream termination +# -------------------------------------------------------------------------- + +# Why a stream stopped before `response.completed`. Recorded in the terminal +# `error` event's `code` and passed to the guard's `on_termination` callback. +# Telling a server-side fault apart from a client hangup is what decides +# whether the accumulated text still existed in this process when the stream +# died. +STREAM_TERMINATION_CLIENT_DISCONNECT = "client_disconnect" +STREAM_TERMINATION_ENGINE_ERROR = "engine_error" +STREAM_TERMINATION_UPSTREAM_ERROR = "upstream_error" +STREAM_TERMINATION_INTERNAL_ERROR = "internal_error" + +# The terminal event of a stream that ran to completion. Matched as a substring +# because a relayed stream arrives as transport-sized chunks that may carry +# several events, not as one frame per event. Kept in both encodings so the +# scan never has to decode a relayed chunk just to look at it. +_COMPLETED_EVENT_MARKER = "event: response.completed" +_COMPLETED_EVENT_MARKER_BYTES = _COMPLETED_EVENT_MARKER.encode("utf-8") + +_SSE_EVENT_DELIMITER = "\n\n" +_SSE_EVENT_DELIMITER_BYTES = _SSE_EVENT_DELIMITER.encode("utf-8") + + +def classify_stream_termination(exc: BaseException) -> str: + """Attribute a stream-ending exception to a cause. + + The distinction that matters is server-side fault vs client hangup. A + cancelled or closed generator means the consumer went away: on the + aggregated server that is the HTTP client, on the orchestrator it is the + client of the orchestrator. Everything else happened on this side of the + socket, and is split further only as far as the exception type can be + trusted to say - an engine error means generation itself failed, an + upstream transport error means the response this process was relaying was + cut, and anything else is a fault in this process's own code, where the + accumulated text was still in memory when the stream died. + """ + if isinstance(exc, (asyncio.CancelledError, GeneratorExit)): + return STREAM_TERMINATION_CLIENT_DISCONNECT + if isinstance(exc, (RequestError, EngineDeadError)): + return STREAM_TERMINATION_ENGINE_ERROR + # By module rather than by class: the HTTP client library is an + # implementation detail of the relay and importing it here would pull a + # transport dependency into the protocol layer. + if type(exc).__module__.split(".")[0] in ("aiohttp", "httpx", "httpcore"): + return STREAM_TERMINATION_UPSTREAM_ERROR + return STREAM_TERMINATION_INTERNAL_ERROR + + +def describe_stream_termination(exc: BaseException) -> str: + text = str(exc) + return f"{type(exc).__name__}: {text}" if text else type(exc).__name__ + + +def stream_error_event(cause: str, detail: str, + events_sent: int) -> List[bytes]: + """A bare ``error`` event for a relay that cannot build a snapshot. + + ``response.failed`` carries a whole ``Response``, which the disaggregated + orchestrator cannot assemble - it forwards bytes and never holds the + sampling parameters the snapshot is built from. ``error`` is flat, so it + needs only the number of events already forwarded to keep the sequence + contiguous. Bytes, because the relay it terminates is a byte stream. + """ + event = ResponseErrorEvent( + type="error", + sequence_number=events_sent, + code=cause, + message=detail, + param=None, + ) + return [(f"event: error\n" + f"data: {event.model_dump_json(indent=None)}\n\n").encode("utf-8")] + + +def _count_frames(chunk: Any, carry: Any) -> Tuple[int, Any]: + """Count completed SSE events in ``chunk``, tolerating split delimiters. + + The carry is the chunk's last byte/character: a transport read can land + between the two newlines that end an event, and without it that event is + never counted and every sequence number after it is one short. + """ + if isinstance(chunk, bytes): + text = (carry or b"") + chunk + return text.count(_SSE_EVENT_DELIMITER_BYTES), text[-1:] + text = (carry or "") + chunk + return text.count(_SSE_EVENT_DELIMITER), text[-1:] + + +async def guard_responses_stream( + stream: AsyncGenerator[Any, None], + terminal_events: Callable[[str, str, int], List[Any]], + on_termination: Optional[Callable[[str, str], None]] = None, +) -> AsyncGenerator[Any, None]: + """Make a Responses stream say so when it stops before completing. + + Without this such a stream simply ends: the generator raises, the ASGI + server abandons a half-written chunked body, and nothing in the bytes + distinguishes that from a stream still in flight. Everything the turn + produced survives only as deltas, because ``response.completed`` is the one + event that repeats the full text. + + Frames are forwarded untouched and nothing is added once + ``response.completed`` has gone out, so a stream that completes normally is + byte-for-byte what it was. + + The two abnormal endings are handled differently on purpose: + + * A **fault on this side** yields the terminal events and then re-raises, + so the client gets bytes that explain the truncation and the server + still records an error rather than a clean finish. + * A **client hangup** yields nothing. There is nobody left to send to, and + a generator that yields while ``GeneratorExit`` is propagating raises + ``RuntimeError: async generator ignored GeneratorExit`` - trading a + diagnosable truncation for an undiagnosable one. The cause is still + reported through ``on_termination`` when a caller supplies one. + """ + events_sent = 0 + carry = None + completed = False + try: + async for chunk in stream: + if not completed: + frames, carry = _count_frames(chunk, carry) + events_sent += frames + completed = (_COMPLETED_EVENT_MARKER_BYTES in chunk + if isinstance(chunk, bytes) else + _COMPLETED_EVENT_MARKER in chunk) + yield chunk + except (asyncio.CancelledError, GeneratorExit) as exc: + if on_termination is not None: + on_termination(STREAM_TERMINATION_CLIENT_DISCONNECT, + describe_stream_termination(exc)) + raise + except Exception as exc: + cause = classify_stream_termination(exc) + detail = describe_stream_termination(exc) + if on_termination is not None: + on_termination(cause, detail) + if not completed: + logger.error("Responses stream terminated before completion " + f"({cause}): {detail}") + # Built before yielding, and inside its own guard: this runs only + # when something has already gone wrong, and a failure here would + # replace the exception the caller needs to see with one about + # reporting it. + try: + frames = terminal_events(cause, detail, events_sent) + except Exception as report_error: # noqa: BLE001 + logger.error( + f"Failed to build the terminal event: {report_error}") + frames = [] + for frame in frames: + yield frame + raise + async def process_streaming_events( generator, diff --git a/tests/unittest/llmapi/apps/test_responses_streaming_events.py b/tests/unittest/llmapi/apps/test_responses_streaming_events.py index c9c5a5c1572a..fc87eba36f69 100644 --- a/tests/unittest/llmapi/apps/test_responses_streaming_events.py +++ b/tests/unittest/llmapi/apps/test_responses_streaming_events.py @@ -21,11 +21,20 @@ an id it has not seen opened. """ +import asyncio +import json + import pytest +from tensorrt_llm.executor import EngineDeadError, RequestError +from tensorrt_llm.serve.openai_protocol import ResponsesRequest from tensorrt_llm.serve.responses_utils import ( ResponsesStreamingEventsHelper, + ResponsesStreamingProcessor, _generate_streaming_event, + classify_stream_termination, + guard_responses_stream, + stream_error_event, ) # The CPU-* CI stages run pytest with -m 'cpu_only'. Without this marker every @@ -227,6 +236,280 @@ def _output_text(text): return ResponseOutputText(text=text, annotations=[], type="output_text", logprobs=None) +# --------------------------------------------------------------------------- +# Streams that stop before response.completed +# +# response.completed is the only event that repeats the full text, so a stream +# cut before it leaves everything the turn produced living solely in deltas. +# Until these events existed such a stream simply stopped: no error event, no +# populated response.error, nothing in the bytes or the trace to tell a reader +# the stream was cut rather than still running. +# --------------------------------------------------------------------------- + + +def _event_type(frame): + if isinstance(frame, bytes): + frame = frame.decode() + return frame.split("\n", 1)[0][len("event: ") :] + + +def _event_data(frame): + if isinstance(frame, bytes): + frame = frame.decode() + return json.loads(frame.split("data: ", 1)[1]) + + +def _processor(): + """A processor built exactly as the server builds one.""" + request = ResponsesRequest(model="test-model", input="hi", stream=True) + return ResponsesStreamingProcessor( + request=request, + sampling_params=request.to_sampling_params(), + model_name="test-model", + use_harmony=False, + ) + + +def _forbidden(cause, detail, events_sent): + raise AssertionError(f"terminal events were built for a stream that did not need them: {cause}") + + +async def _drive(source, terminal_events, on_termination=None): + """Run a stream through the guard, returning (frames, raised exception).""" + seen = [] + try: + async for chunk in guard_responses_stream(source, terminal_events, on_termination): + seen.append(chunk) + except BaseException as exc: # noqa: BLE001 - the test asserts on it + return seen, exc + return seen, None + + +@pytest.mark.asyncio +async def test_a_completed_stream_is_forwarded_unchanged(): + """The happy path must stay byte-identical: same frames, nothing appended.""" + frames = [ + "event: response.created\ndata: {}\n\n", + "event: response.output_text.delta\ndata: {}\n\n", + "event: response.completed\ndata: {}\n\n", + ] + + async def source(): + for frame in frames: + yield frame + + seen, raised = await _drive(source(), _forbidden) + assert seen == frames + assert raised is None + + +@pytest.mark.asyncio +async def test_an_exception_mid_stream_ends_the_stream_with_a_terminal_event(): + processor = _processor() + opening = processor.get_initial_responses() + + async def source(): + for frame in opening: + yield frame + raise RuntimeError("event construction blew up") + + seen, raised = await _drive(source(), processor.get_stream_failed_events) + + assert [_event_type(frame) for frame in seen] == [ + "response.created", + "response.in_progress", + "error", + "response.failed", + ] + # Re-raised, not swallowed: the server still logs the fault and the trace + # still records the response as an error rather than a clean finish. + assert isinstance(raised, RuntimeError) + + +@pytest.mark.asyncio +async def test_the_terminal_event_carries_the_cause(): + processor = _processor() + + async def source(): + for frame in processor.get_initial_responses(): + yield frame + raise ValueError("item had no id") + + seen, _ = await _drive(source(), processor.get_stream_failed_events) + error = _event_data(seen[-2]) + assert error["type"] == "error" + assert error["code"] == "internal_error" + assert error["message"] == "ValueError: item had no id" + + +@pytest.mark.parametrize( + "exc, cause", + [ + (RequestError("request failed"), "engine_error"), + (EngineDeadError(), "engine_error"), + (ValueError("bad item"), "internal_error"), + (asyncio.CancelledError(), "client_disconnect"), + (GeneratorExit(), "client_disconnect"), + ], +) +def test_causes_the_code_can_tell_apart(exc, cause): + """An engine failure, a fault in this process, and a client hangup differ. + + Only the middle one leaves the accumulated text in memory, which is what + makes the distinction worth recording rather than lumping into "error". + """ + assert classify_stream_termination(exc) == cause + + +def test_an_upstream_transport_error_is_named_as_one(): + aiohttp = pytest.importorskip("aiohttp") + assert classify_stream_termination(aiohttp.ClientPayloadError("cut")) == "upstream_error" + + +@pytest.mark.asyncio +async def test_a_client_hangup_adds_nothing_to_the_stream(): + """Nobody is left to send to, and yielding here would raise. + + An async generator that yields while GeneratorExit is propagating dies + with "async generator ignored GeneratorExit", trading a diagnosable + truncation for an undiagnosable one. The cause is reported out of band. + """ + noted = [] + + async def source(): + yield "event: response.created\ndata: {}\n\n" + raise asyncio.CancelledError + + seen, raised = await _drive( + source(), _forbidden, lambda cause, detail: noted.append((cause, detail)) + ) + assert len(seen) == 1 + assert isinstance(raised, asyncio.CancelledError) + assert noted == [("client_disconnect", "CancelledError")] + + +@pytest.mark.asyncio +async def test_a_broken_reporter_does_not_replace_the_original_fault(): + """Reporting a failure must not become a second, more confusing one.""" + + def broken(cause, detail, events_sent): + raise KeyError("snapshot field missing") + + async def source(): + yield "event: response.created\ndata: {}\n\n" + raise ValueError("the fault worth seeing") + + seen, raised = await _drive(source(), broken) + assert len(seen) == 1 + assert isinstance(raised, ValueError) + assert str(raised) == "the fault worth seeing" + + +@pytest.mark.asyncio +async def test_a_failure_after_completion_adds_nothing(): + """Regression guard for the happy path. + + aiohttp raises at end of stream when the connector closes the connection, + which arrives after the last frame. Appending an error event there would + corrupt a stream that completed perfectly well. + """ + + async def source(): + yield "event: response.completed\ndata: {}\n\n" + raise RuntimeError("connection reset at end of stream") + + seen, raised = await _drive(source(), _forbidden) + assert len(seen) == 1 + assert isinstance(raised, RuntimeError) + + +def test_terminal_events_continue_the_sequence_numbering(): + processor = _processor() + opening = processor.get_initial_responses() + assert [_event_data(frame)["sequence_number"] for frame in opening] == [0, 1] + + events = processor.get_stream_failed_events("internal_error", "ValueError: x") + assert [_event_data(frame)["sequence_number"] for frame in events] == [2, 3] + + +def test_numbering_follows_events_this_processor_did_not_number(): + """With postprocessing workers the per-token events are numbered elsewhere. + + A pickled copy of this processor builds them in another process, so the + local counter only ever saw the opening two and would put the terminal + events back at the start of a sequence the client has already passed. + """ + processor = _processor() + processor.get_initial_responses() + + events = processor.get_stream_failed_events("internal_error", "ValueError: x", 610) + assert [_event_data(frame)["sequence_number"] for frame in events] == [610, 611] + + +def test_the_failed_snapshot_says_failed_and_carries_no_content(): + """response.failed, not response.incomplete. + + incomplete_details.reason is a closed enum -- max_output_tokens, + max_messages, content_filter, steered -- with no member for a server fault + or a client hangup, so response.incomplete could only be sent with + reason=null, which is indistinguishable from an ordinary truncation. + """ + processor = _processor() + processor.get_initial_responses() + _, failed = processor.get_stream_failed_events("internal_error", "ValueError: x") + + assert _event_type(failed) == "response.failed" + response = _event_data(failed)["response"] + assert response["status"] == "failed" + assert response["error"]["code"] == "server_error" + assert response["error"]["message"] == "ValueError: x" + # No reconstruction: the lost text stays lost, this only says so. + assert response["output"] == [] + assert response["id"] == processor.request.request_id + + +@pytest.mark.asyncio +async def test_the_relayed_terminal_event_is_numbered_from_the_frames_forwarded(): + """The orchestrator forwards bytes and never sees the events it carries. + + It can still count them, including when a transport read splits the blank + line that ends one -- without the carry every sequence number after such a + split is one short. + """ + + async def source(): + yield b"event: response.created\ndata: {}\n\n" + yield b"event: response.in_progress\ndata: {}\n" + yield b"\nevent: response.output_text.delta\ndata: {}\n\n" + raise RuntimeError("upstream cut") + + seen, raised = await _drive(source(), stream_error_event) + assert isinstance(raised, RuntimeError) + terminal = seen[-1] + assert isinstance(terminal, bytes) + assert _event_type(terminal) == "error" + assert _event_data(terminal)["sequence_number"] == 3 + + +@pytest.mark.asyncio +async def test_a_relay_that_never_started_reports_sequence_zero(): + """The failures that record a response with an entirely empty body. + + Nothing was forwarded, so the error event is the stream's first and the + sequence number is exactly right rather than a guess. + """ + + async def source(): + raise RuntimeError("no context worker available") + yield b"" # pragma: no cover - makes this an async generator + + seen, raised = await _drive(source(), stream_error_event) + assert isinstance(raised, RuntimeError) + assert len(seen) == 1 + assert _event_data(seen[0])["sequence_number"] == 0 + assert _event_data(seen[0])["code"] == "internal_error" + + # --------------------------------------------------------------------------- # Order of the items in the completed-response snapshot # From e1140133274f69c4740e4d43adc46771d0797425 Mon Sep 17 00:00:00 2001 From: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> Date: Tue, 22 Sep 2026 00:45:16 +0000 Subject: [PATCH 18/43] [None][fix] responses: tool calls and usage a client can actually use Two defects on the Responses path, each found by measuring live agent traffic against the request/response traces rather than by reading the code. Tool call ids were reminted for the final response. `response.completed` rebuilds `output` from the generation, and `_tool_call_output_item` mints a fresh random id every time it runs, so the snapshot disagreed with the stream on every call - 1224 of 1224 measured. A client keying its tool results off the streamed `call_id` then matches nothing in the final response. The stream now records the ids it published and the final response reuses them positionally, warning when it rebuilds more calls than were streamed rather than silently handing back ids the client cannot match. `reasoning_tokens` was a hardcoded zero, so a client could not tell how much of a turn went into reasoning. It is now counted by re-encoding the text the reasoning parser itself claimed, rather than by searching the generated token ids for the closing marker. The marker search is cheaper and exact, and was the first design; it was dropped because it has to restate the parser's rules and would silently disagree the moment they differ. GLM alone needs three of them: it closes the block more than once and only the first close counts, `` ends the block implicitly, and a turn emitting neither is reasoning to its last character. Deriving the count from the parser's own output cannot drift from it, and stays correct for the parsers that interleave reasoning with content, where no single prefix length describes the split. The tokenizer is taken as an argument rather than held on the streaming processor, which is pickled across the postproc worker boundary and whose worker already has one. Signed-off-by: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> --- tensorrt_llm/serve/openai_server.py | 4 + tensorrt_llm/serve/postprocess_handlers.py | 5 +- tensorrt_llm/serve/responses_utils.py | 825 ++++++++++++------ .../apps/test_responses_streaming_events.py | 136 ++- .../test_responses_streaming_tool_calls.py | 777 ++++++++++++++--- .../llmapi/apps/test_responses_usage.py | 94 ++ 6 files changed, 1420 insertions(+), 421 deletions(-) diff --git a/tensorrt_llm/serve/openai_server.py b/tensorrt_llm/serve/openai_server.py index 94d441c90839..c75cfc90c226 100644 --- a/tensorrt_llm/serve/openai_server.py +++ b/tensorrt_llm/serve/openai_server.py @@ -3141,6 +3141,10 @@ async def create_response( reasoning_parser=args.reasoning_parser, tool_parser=args.tool_parser, num_prompt_tokens=args.num_prompt_tokens, + # This path does no postprocessing in a worker, so nothing + # has populated args.tokenizer. Used only to count + # reasoning tokens for usage. + tokenizer=self.tokenizer, ) await self._extract_metrics(promise, raw_request) diff --git a/tensorrt_llm/serve/postprocess_handlers.py b/tensorrt_llm/serve/postprocess_handlers.py index 12080002dc2d..1f19188d8019 100644 --- a/tensorrt_llm/serve/postprocess_handlers.py +++ b/tensorrt_llm/serve/postprocess_handlers.py @@ -1088,6 +1088,9 @@ def responses_api_post_processor( reasoning_parser=args.reasoning_parser, tool_parser=args.tool_parser, num_prompt_tokens=args.num_prompt_tokens, + # Set on every PostprocArgs by the worker (postproc_worker.py:195); + # used here only to count reasoning tokens for usage. + tokenizer=args.tokenizer, ) @@ -1101,5 +1104,5 @@ def responses_api_streaming_post_processor( if rsp._done: outputs.append( args.streaming_processor.get_final_response_non_store( - rsp, args.num_prompt_tokens)) + rsp, args.num_prompt_tokens, tokenizer=args.tokenizer)) return outputs diff --git a/tensorrt_llm/serve/responses_utils.py b/tensorrt_llm/serve/responses_utils.py index c09be892e788..18e19d24f987 100644 --- a/tensorrt_llm/serve/responses_utils.py +++ b/tensorrt_llm/serve/responses_utils.py @@ -1140,8 +1140,8 @@ def custom_parameters() -> dict[str, Any]: # A custom tool nested in a namespace needs the same freeform # schema as a top-level one. It carries no `parameters`, so # passing them straight through would describe it with an empty - # object schema - while _custom_tool_names still classifies the - # qualified name as custom, so the output path goes looking for + # object schema - while _tool_resolution still classifies the + # tool as custom, so the output path goes looking for # CUSTOM_TOOL_INPUT_ARG the prompt never mentioned. if getattr(inner, "type", None) == "custom": inner_parameters = custom_parameters() @@ -1445,6 +1445,101 @@ def _apply_reasoning_parser( return content, reasoning_content +def _flush_tool_parser( + tools: Optional[list[Tool]], + output_index: int, + tool_parser_dict: Optional[dict[int, BaseToolParser]] = None, +) -> Tuple[str, list[ToolCallItem], Optional[int]]: + """Whatever the incremental tool parser still holds when the stream ends. + + Returns the text to release, the calls to add, and the parser's tool index + for a call it was cut off in the middle of (None if it was not). + + The parser withholds any bytes that could still turn out to be tool-call + markup, and reports at most one finished call per increment, so the end of + a stream can leave both a complete call and a half-finished one behind with + no further increment coming. `finish` is each parser's own decision about + that remainder; most inherit the no-op, which leaves it sitting in + `_buffer`, so the buffer is drained and then inspected as well. + + Unterminated markup is dropped rather than shown. Releasing it would + reintroduce exactly the leak this path was restructured to make + unreachable, only at a different moment - the bytes are not a message the + model meant to send, they are half of a call that never finished. It is + rare (1 of 3437 recorded responses) and the warning makes the loss visible + instead of silent. + + A remainder with no markup in it is ordinary output that the parser was + holding only until it could rule out a call, and is released. This matches + what DeepSeekR1Parser.finish already does for a trailing fragment that + might still have grown into a ``. + """ + tool_parser = (tool_parser_dict or {}).get(output_index) + if tool_parser is None or tools is None: + # No parser was ever built for this output, so nothing is held back. + return "", [], None + + released: list[str] = [] + calls: list[ToolCallItem] = [] + + # Drain calls the parser has already received in full but not yet reported. + # + # Several parsers report at most one call per increment and leave the rest + # buffered for the next one - Glm47ToolParser returns the moment it sees a + # ``. Two calls arriving in a single stream chunk therefore + # leave the second sitting complete and unreported, and at end of stream + # there is no next increment to collect it; replaying one recorded + # two-call response under randomised chunk boundaries lost a call in 67 of + # 400 chunkings. Empty increments stand in for the chunks that will never + # arrive. + # + # Bounded by the buffer having to shrink on every pass, so a parser that + # cannot make progress on what it holds ends the loop rather than spinning. + # Entered only while the buffer still holds markup, because plain text left + # over is `finish`'s business and some parsers strip framing from it. + held = getattr(tool_parser, "_buffer", "") + while held and tool_parser.has_tool_call(held): + drained = tool_parser.parse_streaming_increment("", tools) + calls.extend(drained.calls) + if drained.normal_text: + released.append(drained.normal_text) + remaining = getattr(tool_parser, "_buffer", "") + if len(remaining) >= len(held): + break + held = remaining + + result = tool_parser.finish(tools) + calls.extend(result.calls) + held = getattr(tool_parser, "_buffer", "") + + dropped = 0 + for remainder in (result.normal_text, held): + if not remainder: + continue + if tool_parser.has_tool_call(remainder): + dropped += len(remainder) + else: + released.append(remainder) + + if dropped: + logger.warning( + f"Stream ended inside a tool call; dropped {dropped} characters " + f"of unterminated {type(tool_parser).__name__} markup rather than " + "reporting them as the assistant's message.") + + # `current_tool_name_sent` is the parser's own record of having announced a + # call it has not yet closed, and `current_tool_id` numbers that call. + # Read through getattr because a parser that overrides + # parse_streaming_increment entirely need not maintain them; not + # maintaining them means it never reports a half-started call, which is + # the safe reading. + unfinished = None + if getattr(tool_parser, "current_tool_name_sent", False): + unfinished = getattr(tool_parser, "current_tool_id", None) + + return "".join(released), calls, unfinished + + def _apply_tool_parser( tool_parser_id: Optional[str], tools: Optional[list[Tool]], @@ -1485,9 +1580,15 @@ def _create_output_content( tool_parser: Optional[str] = None, tools: Optional[list[Tool]] = None, chat_template_kwargs: Optional[dict[str, Any]] = None, -) -> Tuple[list[ResponseOutputItem], list[ChatCompletionMessageParam]]: + streamed_tool_call_ids: Optional[list[Tuple[str, str]]] = None, +) -> Tuple[list[ResponseOutputItem], list[ChatCompletionMessageParam], + list[str]]: output_items: list[ResponseOutputItem] = [] output_messages: list[ChatCompletionMessageParam] = [] + # What the reasoning parser claimed as reasoning, per output and before + # stripping, so usage accounting can count its tokens. Kept raw: the + # whitespace around a reasoning block was generated too. + reasoning_texts: list[str] = [] available_tools = _get_chat_completion_function_tools(tools) for output in final_res.outputs: @@ -1503,6 +1604,7 @@ def _create_output_content( output.text, False, chat_template_kwargs=chat_template_kwargs) + reasoning_texts.append(reasoning_text or "") if text: text, calls = _apply_tool_parser(tool_parser, available_tools, @@ -1554,12 +1656,28 @@ def _create_output_content( output_items.append(text_item) if calls: - custom_tool_names = _custom_tool_names(tools) - namespaced_tool_names = _namespaced_tool_names(tools) - tool_calls_item = [ - _tool_call_output_item(call, custom_tool_names, - namespaced_tool_names) for call in calls - ] + tool_resolution = _tool_resolution(tools) + # Reuse the ids the streaming path already published, positionally: + # both passes enumerate the same parsed calls in the same order. + # Running out means this pass found calls the stream never emitted + # (a cut-off stream, say); those get fresh ids, and the mismatch is + # said out loud rather than silently producing ids the client has + # no way to match. + reusable = list(streamed_tool_call_ids or []) + if reusable and len(reusable) != len(calls): + logger.warning( + "final response rebuilt %d tool call(s) but %d were " + "streamed; ids beyond the streamed ones are new and will " + "not match the client's tool outputs", len(calls), + len(reusable)) + tool_calls_item = [] + for index, call in enumerate(calls): + ids = reusable[index] if index < len(reusable) else (None, None) + tool_calls_item.append( + _tool_call_output_item(call, + tool_resolution, + item_id=ids[0], + call_id=ids[1])) output_items.extend(tool_calls_item) output_messages.extend( @@ -1572,7 +1690,7 @@ def _create_output_content( tool_calls_item, })) - return output_items, output_messages + return output_items, output_messages, reasoning_texts def _create_output_content_harmony( @@ -1592,58 +1710,69 @@ def _create_output_content_harmony( return output_content, output_messages -def _custom_tool_names(tools: Optional[list[Tool]]) -> set[str]: - """Names of the tools the request declared as freeform custom tools. - - Tools inside a namespace are keyed by the qualified name they are - offered to the model under, which is what a parsed call carries. +def _tool_resolution( + tools: Optional[list[Tool]] +) -> dict[str, Tuple[Optional[str], str, bool]]: + """Every spelling a parsed call may carry -> (namespace, bare name, custom). + + A chat template can only describe a flat list of functions, so a namespaced + tool is offered to the model as "namespace.tool" and has to be reported back + with the two parts separated again - that is how the client identifies it. + + The model writes the name back BOTH ways. Measured over one agent session: + 247 calls arrived as `exec` and 34 as `functions.exec`. The two functions + this replaces keyed only on the qualified spelling, so every bare-named call + missed the lookup, was classified as a plain function call, and reached the + client as JSON where a custom tool expects freeform text. The client rejects + that outright: 154 of 154 such calls came back "aborted", while the 10 that + happened to arrive qualified all ran. The agent then concluded its execution + runtime was broken and burned its retries. + + A bare name is registered only when exactly one declared tool answers to it. + Two namespaces offering the same tool name cannot be told apart from the + name alone, and routing a call to the wrong namespace is worse than leaving + it unresolved - the caller reports an unresolved call as it always did, and + now warns about it. """ - names: set[str] = set() + resolved: dict[str, Tuple[Optional[str], str, bool]] = {} + bare_claims: dict[str, list[str]] = {} + + def claim(exposed: str, namespace: Optional[str], bare: str, + is_custom: bool) -> None: + resolved[exposed] = (namespace, bare, is_custom) + if exposed != bare: + bare_claims.setdefault(bare, []).append(exposed) + for tool in tools or []: tool_type = getattr(tool, "type", None) name = getattr(tool, "name", None) if not name: continue - if tool_type == "custom": - names.add(name) - elif tool_type == "namespace": + if tool_type == "namespace": for inner in getattr(tool, "tools", None) or []: inner_name = getattr(inner, "name", None) - if inner_name and getattr(inner, "type", None) == "custom": - names.add(f"{name}.{inner_name}") - return names - - -def _namespaced_tool_names( - tools: Optional[list[Tool]]) -> dict[str, Tuple[str, str]]: - """Qualified name -> (namespace, bare name) for namespaced tools. + if inner_name: + claim(f"{name}.{inner_name}", name, inner_name, + getattr(inner, "type", None) == "custom") + else: + claim(name, None, name, tool_type == "custom") - A chat template can only describe a flat list of functions, so a - namespaced tool is offered as "namespace.tool". The call has to be - reported back with the two parts separated again, since that is how the - client identifies the tool; a call named "collaboration.spawn_agent" - matches nothing it knows and comes back as "unsupported call". - """ - mapping: dict[str, Tuple[str, str]] = {} - for tool in tools or []: - if getattr(tool, "type", None) != "namespace": + for bare, exposed_names in bare_claims.items(): + # A top-level tool owning the spelling wins; it is what the model was + # shown under that exact name. + if bare in resolved: continue - namespace = getattr(tool, "name", None) - if not namespace: - continue - for inner in getattr(tool, "tools", None) or []: - inner_name = getattr(inner, "name", None) - if inner_name: - mapping[f"{namespace}.{inner_name}"] = (namespace, inner_name) - return mapping + if len(exposed_names) == 1: + resolved[bare] = resolved[exposed_names[0]] + return resolved def _tool_call_output_item( call, - custom_tool_names: set[str], - namespaced_tool_names: Optional[dict[str, Tuple[str, str]]] = None, + tool_resolution: dict[str, Tuple[Optional[str], str, bool]], item_id: Optional[str] = None, status: Optional[str] = None, + call_id: Optional[str] = None, ) -> Union[ResponseFunctionToolCall, ResponseCustomToolCall]: """Build the output item for one parsed tool call. @@ -1652,15 +1781,33 @@ def _tool_call_output_item( function call hands the client JSON where it expects the raw payload, and the client rejects the call outright - for apply_patch, with "invoked with incompatible payload", which aborts the whole turn. + + Which of the two it is turns on whether the call's name resolves to a + declared tool, and the model spells that name inconsistently; see + ``_tool_resolution`` for what that cost before it accepted both spellings. """ name = call.name or "" arguments = call.parameters or "{}" - call_id = f"call_{_random_uuid()}" - - is_custom = name in custom_tool_names - namespace = None - if namespaced_tool_names and name in namespaced_tool_names: - namespace, name = namespaced_tool_names[name] + # A fresh id only when this call has not been reported before. The final + # response re-derives its output from the generated text, so it must be + # given back the ids the stream already used; the client keys its tool + # outputs on those and matches nothing otherwise. + call_id = call_id or f"call_{_random_uuid()}" + + resolved = (tool_resolution or {}).get(name) + if resolved is not None: + namespace, name, is_custom = resolved + else: + namespace, is_custom = None, False + # Previously silent, which is why a whole class of aborted calls went + # unexplained for as long as it did: a custom tool reported as a + # function call is rejected by the client, and nothing said so. SGLang + # logs "Model attempted to call undefined function" at the equivalent + # point; borrowing that is most of the value of this change. + logger.warning( + "tool call %r matches no declared tool; reporting it as a function " + "call. If it is in fact a custom tool, the client will reject it.", + name) if is_custom: # Unwrap the single string argument the tool was described with. A @@ -1720,9 +1867,63 @@ def _ctx_usage_from_result(final_res: GenerationResult) -> Optional[Any]: return None +def _count_reasoning_tokens( + tokenizer: Optional[TokenizerBase], + reasoning_texts: list[str], + output_tokens: int, +) -> int: + """How many of the generated tokens went into reasoning. + + The engine does not track this. `ThinkingBudgetLogitsProcessor` computes + the same quantity in token space (thinking_budget.py:93), but it lives in + the model engine process, only exists when `thinking_token_budget` is set, + and a LogitsProcessor has no channel back to the result -- so the count + has to be rebuilt here. + + This re-encodes the text the reasoning parser itself claimed, rather than + searching `output.token_ids` for the `` token. The marker search + is cheaper and exact, and it was the first design; it was dropped because + it has to restate the parser's rules and would silently disagree the + moment they differ. GLM alone needs three of them: it closes the block + more than once and only the first close counts, `` ends the + block implicitly, and a turn that emits neither is reasoning to its last + character (reasoning_parser.py:369-404, with a measured case of 26,055 + characters and no closing tag at all). Deriving the count from the + parser's own output cannot drift from it, and it stays correct for the + parsers that interleave reasoning with content, where no single prefix + length describes the split. + + The cost is one encode of the reasoning text per response, off the + engine's critical path, and a boundary that can differ by a token from + the generated one -- re-encoding a substring need not reproduce the + tokenization it came from. Returns 0 when it cannot be determined at all + (no tokenizer, or no reasoning parser configured), which is what this + reported unconditionally before. + """ + if tokenizer is None: + return 0 + + total = 0 + for text in reasoning_texts: + if not text: + continue + try: + total += len(tokenizer.encode(text, add_special_tokens=False)) + except TypeError: + # Not every tokenizer accepts the keyword. + total += len(tokenizer.encode(text)) + + # Re-encoding can overshoot, and usage that claims more reasoning tokens + # than were generated is visibly wrong to a client budgeting a context + # window. Clamping is the conservative direction. + return min(total, output_tokens) + + def _create_usage( final_res: GenerationResult, - num_prompt_tokens: Optional[int] = None) -> Optional[ResponseUsage]: + num_prompt_tokens: Optional[int] = None, + tokenizer: Optional[TokenizerBase] = None, + reasoning_texts: Optional[list[str]] = None) -> Optional[ResponseUsage]: """Build the Responses-API usage block from a finished generation. Clients such as the Codex CLI rely on this to track how much of the @@ -1763,9 +1964,9 @@ def _create_usage( input_tokens=input_tokens, input_tokens_details=InputTokensDetails(cached_tokens=cached_tokens), output_tokens=output_tokens, - # The reasoning tokens are not accounted separately from the - # generated ones, so report them as zero rather than guessing. - output_tokens_details=OutputTokensDetails(reasoning_tokens=0), + output_tokens_details=OutputTokensDetails( + reasoning_tokens=_count_reasoning_tokens(tokenizer, reasoning_texts + or [], output_tokens)), total_tokens=input_tokens + output_tokens, ) @@ -1780,6 +1981,8 @@ def _create_response( reasoning_parser: Optional[str] = None, tool_parser: Optional[str] = None, num_prompt_tokens: Optional[int] = None, + streamed_tool_call_ids: Optional[list[Tuple[str, str]]] = None, + tokenizer: Optional[TokenizerBase] = None, ) -> tuple[ResponsesResponse, list[Message | ChatCompletionMessageParam]]: _responses_debug_log("================================================") _responses_debug_log("RAW MODEL OUTPUT:") @@ -1791,13 +1994,19 @@ def _create_response( if use_harmony: output_content, output_messages = _create_output_content_harmony( final_res) + # Harmony carries its reasoning on the `analysis` channel, which the + # adapter already tracks per token (harmony_adapter.py:207). Counting + # it is a separate change on that path; this one leaves it reported + # as zero rather than half-counting it here. + reasoning_texts = [] else: - output_content, output_messages = _create_output_content( + output_content, output_messages, reasoning_texts = _create_output_content( final_res, reasoning_parser, tool_parser, request.tools, - chat_template_kwargs=reasoning_chat_template_kwargs(request)) + chat_template_kwargs=reasoning_chat_template_kwargs(request), + streamed_tool_call_ids=streamed_tool_call_ids) finish_reason = final_res.outputs[0].finish_reason response = ResponsesResponse.from_request( @@ -1807,7 +2016,10 @@ def _create_response( created_time=response_creation_time, output=output_content, status=finish_reason_mapping(finish_reason), - usage=_create_usage(final_res, num_prompt_tokens), + usage=_create_usage(final_res, + num_prompt_tokens, + tokenizer=tokenizer, + reasoning_texts=reasoning_texts), ) # Disaggregated serving. A context-only response is read by the # orchestrator, never by a client: it carries the KV-cache handle, the @@ -1843,6 +2055,8 @@ async def create_response( reasoning_parser: Optional[str] = None, tool_parser: Optional[str] = None, num_prompt_tokens: Optional[int] = None, + streamed_tool_call_ids: Optional[list[Tuple[str, str]]] = None, + tokenizer: Optional[TokenizerBase] = None, ) -> ResponsesResponse: final_res: Optional[RequestOutput] = None @@ -1869,6 +2083,8 @@ async def create_response( reasoning_parser=reasoning_parser, tool_parser=tool_parser, num_prompt_tokens=num_prompt_tokens, + streamed_tool_call_ids=streamed_tool_call_ids, + tokenizer=tokenizer, ) if enable_store and request.store: @@ -1889,6 +2105,8 @@ def create_response_non_store( reasoning_parser: Optional[str] = None, tool_parser: Optional[str] = None, num_prompt_tokens: Optional[int] = None, + streamed_tool_call_ids: Optional[list[Tuple[str, str]]] = None, + tokenizer: Optional[TokenizerBase] = None, ) -> ResponsesResponse: response_creation_time = create_time if create_time is not None else int( time.time()) @@ -1904,6 +2122,8 @@ def create_response_non_store( reasoning_parser=reasoning_parser, tool_parser=tool_parser, num_prompt_tokens=num_prompt_tokens, + streamed_tool_call_ids=streamed_tool_call_ids, + tokenizer=tokenizer, ) return response @@ -1924,12 +2144,42 @@ class ResponsesStreamingStateTracker: text_buffer: str = "" reasoning_buffer: str = "" + def __init__(self) -> None: + # Tool-call fragments from the incremental parser, keyed by output + # index, then by the parser's tool index. One parser instance per + # output lives in tool_parser_dict, so what those instances report has + # to be kept apart in the same way. + # + # Assigned here rather than in the class body above: a dict written as + # a class attribute is one object shared by every request, so two + # concurrent streams would accumulate into each other's calls. The + # fields above are immutable, so assigning to them rebinds per + # instance and they are safe as class-level defaults. + self.tool_call_fragments: dict[int, dict[int, dict[str, Any]]] = {} + + # (item id, call id) of every tool call already streamed, in the order + # they went out. `response.completed` rebuilds the output from scratch, + # and _tool_call_output_item mints a fresh random id each time it runs, + # so without this the final response names every call differently from + # the events that carried it - and the client's tool outputs, which + # echo the streamed id, match nothing. Measured before the fix: 671 of + # 671 calls across three campaigns. + # + # A list here rather than a class attribute, for the reason above. + self.emitted_tool_call_ids: list[Tuple[str, str]] = [] + class ResponsesStreamingEventsHelper: def __init__(self): self.state_tracker = ResponsesStreamingStateTracker() + def tool_call_fragments(self, + output_index: int) -> dict[int, dict[str, Any]]: + """The call fragments accumulated so far for one output.""" + return self.state_tracker.tool_call_fragments.setdefault( + output_index, {}) + def content_index_increment(self): self.state_tracker.current_content_index += 1 @@ -1944,6 +2194,20 @@ def emitted_tool_calls(self) -> int: def emitted_tool_calls(self, count: int) -> None: self.state_tracker.emitted_tool_calls = count + @property + def emitted_tool_call_ids(self) -> list[Tuple[str, str]]: + """(item id, call id) of the tool calls already streamed, in order.""" + return self.state_tracker.emitted_tool_call_ids + + def record_emitted_tool_call(self, item) -> None: + """Remember the ids a streamed call went out under. + + The final response is built by a second, independent pass over the + generated text; handing it these lets it name the same call the same + way instead of minting new ids the client has never seen. + """ + self.state_tracker.emitted_tool_call_ids.append((item.id, item.call_id)) + def append_text(self, delta: str) -> None: self.state_tracker.text_buffer += delta @@ -2164,103 +2428,103 @@ def get_reasoning_output_added_events( )) -def _should_send_done_events( - output: RequestOutput, - output_index: int, - reasoning_parser_id: Optional[str] = None, - tool_parser_id: Optional[str] = None, - tools: Optional[list[Tool]] = None, - reasoning_parser_dict: Optional[dict[int, BaseReasoningParser]] = None, - tool_parser_dict: Optional[dict[int, BaseToolParser]] = None, - streaming_events_helper: Optional[ResponsesStreamingEventsHelper] = None, - finished_generation: bool = False, - chat_template_kwargs: Optional[dict[str, Any]] = None, -) -> Tuple[bool, bool, Optional[str], Optional[str]]: - """ - Determine if done events should be sent for text or reasoning items. - - Analyzes the complete output text to detect when reasoning or text sections - have been completed and should receive done events. +def _accumulate_tool_call_fragments(fragments: dict[int, dict[str, Any]], + calls: list[ToolCallItem]) -> None: + """Fold the incremental parser's call fragments into whole calls. - Args: - output: RequestOutput containing full generated text in output.text - output_index: Index of the output being processed - reasoning_parser_id: Parser ID for extracting reasoning content - tool_parser_id: Parser ID for extracting tool calls - tools: Available tools for tool parsing - reasoning_parser_dict: Dictionary of reasoning parsers - tool_parser_dict: Dictionary of tool parsers - streaming_events_helper: Helper tracking current streaming state + `parse_streaming_increment` reports one call in several pieces: the name + first with empty parameters, then the arguments JSON a fragment at a time, + then the closing brace. Only the concatenation is something a client can + run, so the pieces are gathered here and the finished calls are emitted + when generation ends. - Returns: - Tuple of (should_send_reasoning_done, should_send_text_done, - reasoning_content, text_content) + Keyed by the parser's own `tool_index`, which numbers the calls in the + order it saw them start; a dict preserves that order. """ - should_send_reasoning_done = False - should_send_text_done = False - reasoning_content = "" - text_content = "" - - # TODO(JunyiXu-nv): find a more efficient way to decide if we need to send done events - # Parse complete output using non-streaming mode to get full content - full_text, full_reasoning = _apply_reasoning_parser( - reasoning_parser_id=reasoning_parser_id, - output_index=output_index, - text=output.text, - streaming=False, - reasoning_parser_dict=reasoning_parser_dict, - chat_template_kwargs=chat_template_kwargs, - ) - - # Apply tool parsing to get tool calls - tool_calls = [] - if full_text: - full_text, tool_calls = _apply_tool_parser( - tool_parser_id=tool_parser_id, - tools=tools, - output_index=output_index, - text=full_text, - streaming=False, - tool_parser_dict=tool_parser_dict, - ) - - # Detect reasoning -> text transition - # Reasoning is done when we have sent reasoning content and now have text content - if full_reasoning and full_text: - if streaming_events_helper and streaming_events_helper.is_reasoning_sent and not streaming_events_helper.is_text_sent: - should_send_reasoning_done = True - reasoning_content = full_reasoning - - # Detect text -> tool call transition - # Text is done when we have sent text content and now have tool calls - if full_text and tool_calls: - if streaming_events_helper and streaming_events_helper.is_text_sent: - should_send_text_done = True - text_content = full_text - - # Also check if text is done because generation finished (no tool calls case) - # Text is done when generation completes and we've sent text - if full_text and not tool_calls and finished_generation: - if streaming_events_helper and streaming_events_helper.is_text_sent: - should_send_text_done = True - text_content = full_text - - # Similarly, reasoning is done if generation finished with only reasoning (no text case) - if full_reasoning and not full_text and finished_generation: - if streaming_events_helper and streaming_events_helper.is_reasoning_sent: - should_send_reasoning_done = True - reasoning_content = full_reasoning - - # No closing tag: reasoning was streamed but re-parse shows everything as - # content (no found). Close the reasoning section so the text - # section can be properly opened and closed. - if not full_reasoning and full_text and finished_generation: - if streaming_events_helper and streaming_events_helper.is_reasoning_sent: - should_send_reasoning_done = True - reasoning_content = full_text - - return (should_send_reasoning_done, should_send_text_done, - reasoning_content, text_content, tool_calls) + for call in calls: + fragment = fragments.setdefault(call.tool_index, { + "name": None, + "parameters": "", + }) + # Only a call's first fragment carries the name; every later one + # carries None, which must not erase the name already recorded. + if call.name: + fragment["name"] = call.name + fragment["parameters"] += call.parameters or "" + + +def _assembled_tool_calls( + fragments: dict[int, dict[str, Any]], + unfinished_tool_index: Optional[int] = None) -> list[ToolCallItem]: + r"""The accumulated fragments as whole calls, in the order they started. + + A call whose name never arrived is dropped, which is what + `parse_base_json` does on the non-streaming path: a call the client cannot + name is not a call it can run. + + `unfinished_tool_index` names a call the parser announced but never closed, + because the stream ended inside it. Its accumulated arguments are a JSON + prefix - `{"input": "\n// Read the problem` and nothing more - which no + client can parse or run, so it is dropped rather than reported. The + non-streaming path reports no call at all for the same text, since its + regex needs the closing tag, and the two endpoints have to agree about + what the model asked for. + + A call whose assembled arguments are not valid JSON is dropped for the same + reason, one test later. That test keys on the markup being unterminated; + this one keys on the result being unusable, which is the property that + actually matters and which covers calls the markup test passes. A GLM-4.7 + block that opens `` and never closes it, ending + `` instead, has a balanced `` pair and so + finalises normally - with arguments that assemble to `{"cmd": }`. That + reached a client, which stored it, replayed it in the next request's + history and was answered `tool_calls[0].function.arguments must be valid + JSON`; 2 of 13,014 delivered calls, each costing the agent run that hit it. + + Here rather than in the parser because a parser cannot unsay what it has + already said. It streams `{`, `"cmd": `, the value and `}` over several + increments, each folded into `fragments` as it arrives, and only discovers + at `` that the whole does not parse. This is the last point at + which the fragments are known as a whole *and* declining to report them is + still possible. + + Dropping beats repairing. Closing the quote and the brace would invent an + argument the model never wrote, and the client would then run a truncated + shell command with nothing to indicate anything was lost; executing half a + command is worse than executing none. The warning is what keeps the loss + from being silent. + + The non-streaming path needs no equivalent - `parse_base_json` builds its + arguments with `json.dumps`, so they always parse - but it is not silent + about this text either: its pair regex reads nothing out of the block and + it reports the call with empty arguments. `{}` is something a client can + run, so the two paths do differ here, and settling that means deciding + what a call whose markup could not be read should be. A different + question, left alone. + """ + calls: list[ToolCallItem] = [] + for tool_index, fragment in fragments.items(): + if not fragment["name"] or tool_index == unfinished_tool_index: + # No warning for the unfinished call: `_flush_tool_parser` has + # already reported that loss, and saying it twice would read as two + # calls lost. + continue + arguments = fragment["parameters"] + try: + json.loads(arguments) + except ValueError as exc: + logger.warning( + f"Dropping the tool call to {fragment['name']!r}: its " + f"arguments did not assemble into valid JSON ({exc}). The " + "model emitted a malformed tool call, and reporting it would " + "hand the client arguments it can only reject. Assembled: " + f"{arguments[:200]!r}") + continue + calls.append( + ToolCallItem(tool_index=tool_index, + name=fragment["name"], + parameters=arguments)) + return calls def _close_open_item(helper): @@ -2342,7 +2606,6 @@ def check_parser(parser_id: Optional[str], ) if delta_text: - # TODO(JunyiXu-nv): handle tool calls in streaming mode delta_text, calls = _apply_tool_parser( tool_parser_id=tool_parser_id, tools=available_tools, @@ -2352,6 +2615,19 @@ def check_parser(parser_id: Optional[str], tool_parser_dict=tool_parser_dict, ) + # End of stream: the tool parser gets one last word before its state is + # discarded, so bytes it was still withholding are released or dropped by + # a decision rather than by nobody ever asking. Only computed here; what + # it releases is emitted further down, after the call in front of it has + # closed the item it belongs to. + flushed_text, flushed_calls, unfinished_tool_index = "", [], None + if finished_generation: + flushed_text, flushed_calls, unfinished_tool_index = _flush_tool_parser( + tools=available_tools, + output_index=output_idx, + tool_parser_dict=tool_parser_dict, + ) + _responses_debug_log( repr( f" ---------> delta text: {delta_text}, reasoning delta text: {reasoning_delta_text}, calls: {calls}" @@ -2359,14 +2635,19 @@ def check_parser(parser_id: Optional[str], # Send delta events for ongoing content BEFORE any done events. # - # The done-event block below closes the item that is currently open. If it - # ran first, the final chunk's delta would arrive after that close and open - # a brand new output item for the tail of the same message - splitting one + # The close-on-calls block below ends the item that is currently open. If + # it ran first, this chunk's delta would arrive after that close and open a + # brand new output item for the tail of the same message - splitting one # assistant turn across two items, sometimes mid-word, and clients that # render the last item alone show only that fragment. The chat completions # path has the same shape: it appends the content delta to the chunk and # only then stamps finish_reason. - # Send delta events for ongoing content + # + # It is also what makes the done payload right in the common case: a chunk + # that carries the text before a call *and* the start of that call arrives + # as one delta plus one non-empty `calls`, so the text has to be appended + # to the item before the item is closed with it. + # # The item must be opened before *any* delta, including a whitespace-only # one. Gating the added-events on delta_text.strip() while emitting the # delta unconditionally sends output_text.delta with no item open, and a @@ -2388,11 +2669,10 @@ def check_parser(parser_id: Optional[str], # It is not a rare boundary case. The reasoning between the last chunk # boundary and the tag is lost every time the two land in one chunk, # which measured 56% of reasoning items across four fleets, and 100% - # of reasoning short enough to fit inside a single chunk. Closing the - # item first is what makes it unrecoverable: _close_open_item clears - # is_reasoning_sent, and _should_send_done_events below is guarded on - # that flag -- so the correct full text it computes is discarded, and - # the done event carries the truncated text too. + # of reasoning short enough to fit inside a single chunk. Nothing + # downstream can recover it either: every done payload is now the sum + # of the deltas already streamed, so reasoning that was never streamed + # as a delta is simply not in the reasoning item at all. if reasoning_delta_text: if streaming_events_helper.is_text_sent: yield from _close_open_item(streaming_events_helper) @@ -2423,102 +2703,60 @@ def check_parser(parser_id: Optional[str], yield streaming_events_helper.get_reasoning_text_delta_event( reasoning_delta_text) - # Check if we need to send done events for completed sections - (should_send_reasoning_done, should_send_text_done, reasoning_full_content, - text_full_content, done_tool_calls) = _should_send_done_events( - output=output, - output_index=output_idx, - reasoning_parser_id=reasoning_parser_id, - tool_parser_id=tool_parser_id, - tools=available_tools, - reasoning_parser_dict=reasoning_parser_dict, - tool_parser_dict=tool_parser_dict, - streaming_events_helper=streaming_events_helper, - finished_generation=finished_generation, - chat_template_kwargs=reasoning_chat_template_kwargs(request), - ) - - # Send done events if needed - if should_send_reasoning_done and reasoning_full_content: - reasoning_item = ResponseReasoningItem( - id=streaming_events_helper.item_id, - summary=[], - type="reasoning", - content=[ - Content(text=reasoning_full_content, type="reasoning_text") - ], - status="completed", - ) - yield streaming_events_helper.get_reasoning_text_done_event( - reasoning_full_content) - yield streaming_events_helper.get_output_item_done_event(reasoning_item) - streaming_events_helper.take_reasoning() - streaming_events_helper.output_index_increment() - streaming_events_helper.is_output_item_added_sent = False - streaming_events_helper.is_reasoning_sent = False - - if should_send_text_done and text_full_content: - text_content = ResponseOutputText( - text=text_full_content, - annotations=[], - type="output_text", - logprobs=None, - ) - text_item = ResponseOutputMessage( - id=streaming_events_helper.item_id, - content=[text_content], - role="assistant", - status="completed", - type="message", - ) - yield streaming_events_helper.get_text_done_event(text_full_content, []) - yield streaming_events_helper.get_content_part_done_event(text_content) - yield streaming_events_helper.get_output_item_done_event(text_item) - streaming_events_helper.take_text() - streaming_events_helper.output_index_increment() - streaming_events_helper.is_output_item_added_sent = False - streaming_events_helper.is_text_sent = False - - # Handle no-closing-tag case: reasoning was streamed but finish() moved - # all accumulated reasoning to content. Emit the full text section - # lifecycle (added β†’ delta β†’ done) since the reasoning section was just - # closed and generation is finished. - if (finished_generation and delta_text and should_send_reasoning_done - and not should_send_text_done): + # A tool call has started, so whichever item is open ends here. + # + # Only the incremental parser can report this. It is the one holding back + # the markup's bytes, so it knows it is mid-call; a re-parse of the + # accumulated text has no such memory and has to classify an unterminated + # call as ordinary text. That is exactly how the previous implementation + # published `functions.execinput...` as the + # assistant's message: the non-streaming tool regex needs a closing tag, + # so a call that had closed kept `tool_calls` non-empty while a second, + # still-open call stayed in the text, and both halves of its condition + # were true at once. It needed one closed and one open call to coexist, + # which is why it hit 41.7% of two-call responses and every three-call one + # while single-call responses were almost untouched. + # + # _close_open_item closes with take_text()/take_reasoning() - the deltas + # already streamed - so a done payload cannot contain anything that was + # never streamed as a delta, whatever the accumulated text looks like. + # Which item that is depends on what was open: prose before the call + # closes a message item, `` closes the reasoning item, + # and a call with nothing in front of it closes nothing and emits no + # events, rather than inventing an empty message item. + if calls: + yield from _close_open_item(streaming_events_helper) + + # Whole calls have to be reassembled from the fragments the parser reports + # (name, then arguments piece by piece); see _accumulate_tool_call_fragments. + call_fragments = streaming_events_helper.tool_call_fragments(output_idx) + _accumulate_tool_call_fragments(call_fragments, calls) + _accumulate_tool_call_fragments(call_fragments, flushed_calls) + + # Text the parser was still holding when the stream ended, emitted here + # rather than folded into the delta block above because it sits *after* + # whatever the last chunk contained. The common shape is a call and the + # sentence following it arriving in one chunk: the parser returns the call + # and keeps the sentence, so folding it in would put that sentence in the + # message item the call just closed - one item reading "Before. After." + # with the call between them lost from the ordering. + if flushed_text: + if streaming_events_helper.is_reasoning_sent: + yield from _close_open_item(streaming_events_helper) streaming_events_helper.is_text_sent = True yield from streaming_events_helper.get_message_output_added_events() - yield streaming_events_helper.get_text_delta_event(delta_text, []) - text_content_obj = ResponseOutputText( - text=delta_text, - annotations=[], - type="output_text", - logprobs=None, - ) - text_item = ResponseOutputMessage( - id=streaming_events_helper.item_id, - content=[text_content_obj], - role="assistant", - status="completed", - type="message", - ) - yield streaming_events_helper.get_text_done_event(delta_text, []) - yield streaming_events_helper.get_content_part_done_event( - text_content_obj) - yield streaming_events_helper.get_output_item_done_event(text_item) - streaming_events_helper.output_index_increment() - streaming_events_helper.is_output_item_added_sent = False - streaming_events_helper.is_text_sent = False - delta_text = "" + streaming_events_helper.append_text(flushed_text) + yield streaming_events_helper.get_text_delta_event(flushed_text, []) # Close whatever item is still open once generation has finished. # - # The done-event block above runs *before* the delta block, so the last - # chunk of a generation closes the previous item and then opens a new one - # for its own delta - leaving that final item with no output_text.done, - # content_part.done or output_item.done. Clients then receive an item with - # no terminal state: Codex CLI renders it but echoes it back on the next - # turn without a `status`, and ResponseOutputMessageParam requires one, so - # the following request is rejected outright. + # Every other close above is triggered by a transition - reasoning into + # text, text into a tool call - and the last item of a response has no + # transition after it, so nothing else ever closes it. It would reach the + # client with no output_text.done, content_part.done or output_item.done, + # i.e. with no terminal state: Codex CLI renders it but echoes it back on + # the next turn without a `status`, and ResponseOutputMessageParam + # requires one, so the following request is rejected outright. # # The chat completions path has no equivalent problem because it finalises # on `output.finish_reason is not None` rather than on parser state. This @@ -2561,7 +2799,7 @@ def check_parser(parser_id: Optional[str], streaming_events_helper.output_index_increment() streaming_events_helper.is_output_item_added_sent = False - # Emit any tool calls the parser found, as function_call output items. + # Emit the tool calls the parser found, as function_call output items. # # Without this the call is stripped out of the text by the tool parser and # then dropped, so the client receives prose - or, when the whole @@ -2569,27 +2807,38 @@ def check_parser(parser_id: Optional[str], # tool should run. Codex CLI shows the model announcing an action and then # nothing happening at all. # - # Emitted after any open text item has been closed, so a call item is - # never nested inside a message item. The counter keeps this idempotent: - # _should_send_done_events re-parses the accumulated text on every chunk, - # so the same calls reappear on each one. - if finished_generation and done_tool_calls: - pending = done_tool_calls[streaming_events_helper.emitted_tool_calls:] - custom_tool_names = _custom_tool_names(request.tools) - namespaced_tool_names = _namespaced_tool_names(request.tools) - for call in pending: - tool_call_item = _tool_call_output_item(call, - custom_tool_names, - namespaced_tool_names, - status="completed") - streaming_events_helper.item_id = tool_call_item.id - yield streaming_events_helper.get_output_item_added_event( - tool_call_item) - yield streaming_events_helper.get_output_item_done_event( - tool_call_item) - streaming_events_helper.output_index_increment() - streaming_events_helper.emitted_tool_calls = len(done_tool_calls) - streaming_events_helper.is_output_item_added_sent = False + # TODO(JunyiXu-nv): stream the call items as the parser produces them. + # They are held back until generation finishes for now. Streaming them is + # the direction both SGLang and vLLM have gone, but it is an observable + # protocol change - a client that assumes every call item arrives together + # at the end would see them spread out instead - and landing it here would + # mean a reviewer could not tell the leak fix from the protocol change, + # nor roll back one without the other. The close above is what the leak + # fix needs, and it already happens incrementally. + # + # Emitted after any open item has been closed, so a call item is never + # nested inside a message item. The counter keeps emission idempotent: it + # guarded against the old per-chunk re-parse re-reporting the same calls, + # and now guards against a finished output being presented twice. + if finished_generation: + assembled_calls = _assembled_tool_calls(call_fragments, + unfinished_tool_index) + pending = assembled_calls[streaming_events_helper.emitted_tool_calls:] + if pending: + tool_resolution = _tool_resolution(request.tools) + for call in pending: + tool_call_item = _tool_call_output_item(call, + tool_resolution, + status="completed") + streaming_events_helper.item_id = tool_call_item.id + yield streaming_events_helper.get_output_item_added_event( + tool_call_item) + yield streaming_events_helper.get_output_item_done_event( + tool_call_item) + streaming_events_helper.record_emitted_tool_call(tool_call_item) + streaming_events_helper.output_index_increment() + streaming_events_helper.emitted_tool_calls = len(assembled_calls) + streaming_events_helper.is_output_item_added_sent = False def _generate_streaming_event_harmony( @@ -2772,6 +3021,7 @@ async def get_final_response( self, final_res: RequestOutput, num_prompt_tokens: Optional[int] = None, + tokenizer: Optional[TokenizerBase] = None, ) -> str: final_response = await create_response( generator=None, @@ -2786,6 +3036,15 @@ async def get_final_response( reasoning_parser=self.reasoning_parser, tool_parser=self.tool_parser, num_prompt_tokens=num_prompt_tokens, + # Name the calls the way the stream already named them. + streamed_tool_call_ids=self.streaming_events_helper. + emitted_tool_call_ids, + # Taken as an argument rather than held on this object: the + # postproc-worker path pickles the processor across a process + # boundary, and that worker already has its own tokenizer + # (postproc_worker.py:195). Shipping one per request would be + # pure cost. + tokenizer=tokenizer, ) return self._send_event( @@ -2799,6 +3058,7 @@ def get_final_response_non_store( self, final_res: RequestOutput, num_prompt_tokens: Optional[int] = None, + tokenizer: Optional[TokenizerBase] = None, ) -> str: final_response = create_response_non_store( generation_result=final_res, @@ -2810,6 +3070,15 @@ def get_final_response_non_store( reasoning_parser=self.reasoning_parser, tool_parser=self.tool_parser, num_prompt_tokens=num_prompt_tokens, + # Name the calls the way the stream already named them. + streamed_tool_call_ids=self.streaming_events_helper. + emitted_tool_call_ids, + # Taken as an argument rather than held on this object: the + # postproc-worker path pickles the processor across a process + # boundary, and that worker already has its own tokenizer + # (postproc_worker.py:195). Shipping one per request would be + # pure cost. + tokenizer=tokenizer, ) return self._send_event( @@ -3070,6 +3339,7 @@ async def process_streaming_events( create_time: Optional[int] = None, reasoning_parser: Optional[str] = None, tool_parser: Optional[str] = None, + tokenizer: Optional[TokenizerBase] = None, ) -> AsyncGenerator[str, None]: streaming_processor = ResponsesStreamingProcessor( request=request, @@ -3093,7 +3363,8 @@ async def process_streaming_events( for event in events: yield event - final_response = await streaming_processor.get_final_response(final_res) + final_response = await streaming_processor.get_final_response( + final_res, tokenizer=tokenizer) yield final_response diff --git a/tests/unittest/llmapi/apps/test_responses_streaming_events.py b/tests/unittest/llmapi/apps/test_responses_streaming_events.py index fc87eba36f69..a7e81f807002 100644 --- a/tests/unittest/llmapi/apps/test_responses_streaming_events.py +++ b/tests/unittest/llmapi/apps/test_responses_streaming_events.py @@ -23,6 +23,7 @@ import asyncio import json +from types import SimpleNamespace import pytest @@ -549,7 +550,7 @@ def _snapshot_kinds(text, tools=None): """ from tensorrt_llm.serve.responses_utils import _create_output_content - items, _messages = _create_output_content( + items, _messages, _reasoning_texts = _create_output_content( _FakeRequestOutput(text), reasoning_parser="qwen3", tool_parser="qwen3" if tools else None, @@ -640,9 +641,140 @@ def test_snapshot_order_matches_the_streamed_order(): if getattr(event, "type", "") == "response.output_item.done": streamed.append(event.item.type) - snapshot_items, _messages = _create_output_content( + snapshot_items, _messages, _reasoning_texts = _create_output_content( _FakeRequestOutput(accumulated), reasoning_parser="glm" ) assert streamed == ["reasoning", "message"] assert [item.type for item in snapshot_items] == streamed + + +def _finished_generation(text): + """A result complete enough to assemble a whole final response from. + + `_create_output_content` reads only index and text, which is all + `_FakeRequestOutput` carries; building the response around it also reads + `finish_reason`, and usage reads `token_ids` and `prompt_token_ids`. + """ + output = SimpleNamespace( + index=0, + text=text, + text_diff=text, + finish_reason="stop", + token_ids=[1, 2, 3], + disaggregated_params=None, + ) + return SimpleNamespace(outputs=[output], prompt_token_ids=[1, 2], cached_tokens=0) + + +def test_final_response_is_assembled_without_touching_a_missing_attribute(): + """Regression: the final streaming event raised AttributeError. + + `get_final_response_non_store` reached for `self.emitted_tool_call_ids` + on the processor. That list lives on the events helper's state tracker, + not on the processor, so every streaming Responses request died building + its last event. The client never saw a server error -- the chunked body + simply stopped, surfacing as `TransferEncodingError: Not enough data to + satisfy transfer length header` one hop upstream, which reads like a + network fault. Measured on a live instance: 286 of 287 requests. + + `py_compile` passes on that bug, and so does an md5 comparison against the + file it came from. Only executing the call catches it, which is what this + test does: it asks for the final event and requires a real one back. + """ + processor = _processor() + frame = processor.get_final_response_non_store(_finished_generation("hello")) + + assert "response.completed" in frame + payload = _event_data(frame) + assert payload["type"] == "response.completed" + assert payload["response"]["status"] == "completed" + + +def test_final_response_reuses_the_call_ids_the_stream_published(): + """The ids a client matches its tool outputs against must not be reminted. + + `response.completed` rebuilds `output` from the generation, and + `_tool_call_output_item` mints a fresh random id every time it runs, so + the snapshot used to disagree with the stream on every call -- 1224 of + 1224 measured before the fix. A client keying tool results off the + streamed `call_id` then matches nothing in the final response. + """ + processor = _processor() + helper = processor.streaming_events_helper + helper.record_emitted_tool_call(SimpleNamespace(id="fc_fixed", call_id="call_fixed")) + + assert helper.emitted_tool_call_ids == [("fc_fixed", "call_fixed")] + # And the processor reaches it by the path the final response uses. + assert processor.streaming_events_helper.emitted_tool_call_ids == [("fc_fixed", "call_fixed")] + + +def _processor_with_tools(): + """A processor that can actually produce a tool call. + + `_processor()` declares no tools and sets no parsers, so nothing it builds + ever contains one -- which is the case the next test needs. + """ + request = ResponsesRequest(model="test-model", input="hi", stream=True, tools=_tools()) + return ResponsesStreamingProcessor( + request=request, + sampling_params=request.to_sampling_params(), + model_name="test-model", + use_harmony=False, + reasoning_parser="glm", + tool_parser="qwen3", + ) + + +def test_final_response_reuses_a_streamed_call_id_end_to_end(): + """The invariant the fix exists for, driven through the real final event. + + This is the test that distinguishes the fix from the obvious-looking wrong + one. `AttributeError: no attribute 'emitted_tool_call_ids'` invites adding + the attribute to the processor -- the name would resolve and the crash + would stop. But `record_emitted_tool_call` appends to the state tracker's + list, so a new list on the processor stays empty forever, the final + response mints fresh ids, and the call_id mismatch is silently back. + + The crash tests pass in that world. This one does not. + """ + processor = _processor_with_tools() + processor.streaming_events_helper.record_emitted_tool_call( + SimpleNamespace(id="fc_streamed", call_id="call_streamed") + ) + + frame = processor.get_final_response_non_store( + _finished_generation(f"{_THINK}{_TOOL_CALL}") + ) + calls = [ + i + for i in _event_data(frame)["response"]["output"] + if i["type"] in ("function_call", "custom_tool_call") + ] + + assert calls, "the final response should carry the tool call" + assert calls[0]["call_id"] == "call_streamed" + assert calls[0]["id"] == "fc_streamed" + + +def test_nothing_streamed_means_fresh_ids_rather_than_a_crash(): + """The other half: an empty list is a legal state, not an error. + + A non-streaming request, or a stream cut off before any tool call, has + nothing to reuse. That must produce usable fresh ids -- keeping this + separate is what makes the test above read as "ids were not reused" + instead of "ids were missing". + """ + processor = _processor_with_tools() + frame = processor.get_final_response_non_store( + _finished_generation(f"{_THINK}{_TOOL_CALL}") + ) + calls = [ + i + for i in _event_data(frame)["response"]["output"] + if i["type"] in ("function_call", "custom_tool_call") + ] + + assert calls + assert calls[0]["call_id"].startswith("call_") + assert calls[0]["call_id"] != "call_streamed" diff --git a/tests/unittest/llmapi/apps/test_responses_streaming_tool_calls.py b/tests/unittest/llmapi/apps/test_responses_streaming_tool_calls.py index 2a9aa8526498..39976e3606ef 100644 --- a/tests/unittest/llmapi/apps/test_responses_streaming_tool_calls.py +++ b/tests/unittest/llmapi/apps/test_responses_streaming_tool_calls.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,17 +13,29 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Unit tests for Responses API streaming tool call emission (TRTLLM-9605).""" +"""Unit tests for Responses API streaming tool call emission (TRTLLM-9605). +These drive `_generate_streaming_event` with the real GLM-4.7 reasoning and +tool parsers rather than with mocked ones. The defect these tests exist for - +tool-call markup published as the assistant's message - lived entirely in the +disagreement between what the incremental parser had streamed and what a +whole-text re-parse concluded afterwards, so a test that mocks the parsers +cannot see it. +""" + +import json +import random from types import SimpleNamespace from unittest.mock import patch import pytest +from openai.types.responses.tool import FunctionTool from tensorrt_llm.serve.responses_utils import ( ResponsesStreamingEventsHelper, + _accumulate_tool_call_fragments, + _assembled_tool_calls, _generate_streaming_event, - _should_send_done_events, ) from tensorrt_llm.serve.tool_parser.core_types import ToolCallItem @@ -32,173 +44,656 @@ # stage reports as a failure. pytestmark = pytest.mark.cpu_only +_REASONING_PARSER = "glm47" +_TOOL_PARSER = "glm47" + +# Markup that must never reach a client as assistant text. +_MARKUP = ("", "", "", "") + +# One real GLM-4.7 response, verbatim, from a recorded agent run: eight stream +# frames carrying reasoning, a sentence of prose, and two tool calls. Feeding +# frames 0-6 used to publish 296 characters of `normal_text` beginning with the +# prose and continuing straight into `functions.exec...`, +# cut off mid-word at `yield`; the whole response yields 116 clean characters +# and two calls. The frame boundaries matter - they are what put one closed +# call and one open call in the accumulated text at the same time - so they are +# kept exactly as recorded. +_REAL_FRAMES = [ + "Let", + ( + " me start by understanding the problem. I need to:\n1. Read the problem definition\n2." + " Understand the kernel-factory-schemas\n3. Read the problem files\n4. Implement a CUDA" + " kernel\n\nLet me begin by reading the problem files and understanding what I'm workin" # codespell:ignore + "g with.I" + ), + ( + "'ll start by reading the problem definition and understanding the schema, then dive in" + "to the kernel implementation.functions.execinput\n// Read the problem definition and related files\nconst out = await tools.exec_co" # codespell:ignore ue + "mmand({cmd: `python3 - <<'PYEOF'\nimport json\nd = json.load(open" + ), + ( + "('/tmp/problem/definition.json'))\nprint(\"Description:\", d.get('description', ''" + "))\nprint(\"\\nFixed axes:\", {k: v['value'] for k, v in d.get('axes', {}).items() i" + "f v.get('type') == 'const'})\nprint(\"Variable axes:\", [k for k, v in d.get('axes" + "', {}).items() if v.get('type')" + ), + ( + " == 'var'})\nprint(\"\\nInputs:\")\nfor n, s in d['inputs'].items(): print(f\" {n}:" + " shape={s['shape']}, dtype={s['dtype']}\")\nprint(\"Outputs (pre-allocated):\")\nfor " + "n, s in d['outputs'].items(): print(f\" {n}: shape={s['shape']}, dtype={s" + ), + ( + "['dtype']}\")\nin_args = list(d['inputs'].keys())\nout_args = list(d['outputs']." + "keys())\nprint(f\"\\nRequired signature: def run({', '.join(in_args + out_args)}) -> " + 'None:")\nPYEOF\necho "=== workload ==="\nhead -3 /tmp/problem/workload.jsonl\necho "==' + '= language ==="\ncat /' + ), + ( + 'tmp/problem/language.txt\necho "=== user_prompt ==="\ncat /tmp/problem/user_prompt.txt' + "\n`, yield_time_ms: 15000});\ntext(out.output);\nfu" + "nctions.execinput\n// Read the baseline metrics\nconst o" + "ut = await tools.exec_command({cmd: `cat /tmp/problem/baseline/metrics.json`, yield" + ), + "_time_ms: 5000});\ntext(out.output);\n", +] + +# Everything the frames above put between `` and the first +# ``: the only visible assistant text in that response. +_REAL_TEXT = ( + "I'll start by reading the problem definition and understanding " + "the schema, then dive into the kernel implementation." +) + + +def _make_output(text: str, text_diff: str, index: int = 0): + """A stand-in for one RequestOutput as the streaming loop sees it.""" + return SimpleNamespace(index=index, text=text, text_diff=text_diff) + + +def _namespace_tool(namespace: str = "functions", name: str = "exec"): + """A namespaced tool, offered to the model as `.`. -def _make_mock_output(index: int = 0, text: str = "", text_diff: str = ""): - """Create a minimal mock output with .index, .text, .text_diff.""" + This is the shape the recorded response was produced with, and the reason + its calls are named `functions.exec`. + """ + inner = SimpleNamespace( + name=name, + type="function", + description="run a command", + parameters={ + "type": "object", + "properties": {"input": {"type": "string"}}, + }, + ) + return SimpleNamespace(name=namespace, type="namespace", description="tools", tools=[inner]) - class MockOutput: - pass - out = MockOutput() - out.index = index - out.text = text - out.text_diff = text_diff - return out +def _function_tool(name: str = "get_time"): + return FunctionTool( + name=name, + type="function", + strict=False, + parameters={ + "type": "object", + "properties": {"city": {"type": "string"}}, + }, + ) -def _make_mock_request(tools: list | None = None): - """Create a minimal ResponsesRequest-like object with .tools. +def _make_request(tools=None): + """A minimal ResponsesRequest-like object. - SimpleNamespace rather than a nested class: a class body does not close over - the enclosing function's locals (unlike a nested function), so + SimpleNamespace rather than a nested class: a class body does not close + over the enclosing function's locals (unlike a nested function), so ``tools = tools or []`` inside one raises NameError on the right-hand name. """ - return SimpleNamespace(tools=tools or []) + return SimpleNamespace(tools=list(tools) if tools else []) -class TestShouldSendDoneEventsToolCalls: - """Test that _should_send_done_events returns tool_calls when text is done due to tool calls.""" +def _drive( + frames, + tools=None, + finish=True, + helper=None, + tool_parser_id=_TOOL_PARSER, + reasoning_parser_id=_REASONING_PARSER, +): + """Feed `frames` through the streaming event generator, one chunk each. - def test_returns_tool_calls_when_text_done_due_to_tool_calls(self): - """When full text and tool_calls exist and is_text_sent, 5th return is tool_calls.""" - output = _make_mock_output( - index=0, - text='Some text before {"name":"get_weather"}', - text_diff="", - ) - tool_calls = [ - ToolCallItem(tool_index=0, name="get_weather", parameters='{"location":"SF"}'), - ] - helper = ResponsesStreamingEventsHelper() - helper.is_text_sent = True - - with ( - patch( - "tensorrt_llm.serve.responses_utils._apply_reasoning_parser", - return_value=('Some text before {"name":"get_weather"}', ""), - ), - patch( - "tensorrt_llm.serve.responses_utils._apply_tool_parser", - return_value=("Some text before ", tool_calls), - ), - ): - result = _should_send_done_events( - output=output, - output_index=0, - tool_parser_id="test_parser", - tools=[], - tool_parser_dict={0: None}, + Returns the flat list of events, which is what a client would receive. + """ + helper = helper or ResponsesStreamingEventsHelper() + request = _make_request(tools if tools is not None else [_namespace_tool()]) + reasoning_parser_dict, tool_parser_dict = {}, {} + events, accumulated = [], "" + for i, frame in enumerate(frames): + accumulated += frame + events.extend( + _generate_streaming_event( + output=_make_output(accumulated, frame), + request=request, + finished_generation=finish and i == len(frames) - 1, streaming_events_helper=helper, - finished_generation=False, + reasoning_parser_id=reasoning_parser_id, + tool_parser_id=tool_parser_id, + reasoning_parser_dict=reasoning_parser_dict, + tool_parser_dict=tool_parser_dict, ) + ) + return events - should_reasoning, should_text, reasoning_content, text_content, done_tool_calls = result - assert should_text is True - assert text_content == "Some text before " - assert len(done_tool_calls) == 1 - assert done_tool_calls[0].name == "get_weather" - assert done_tool_calls[0].parameters == '{"location":"SF"}' - def test_plain_text_completion_sends_text_done_and_no_tool_calls(self): - """The no-tool-call branch must still finish the text item. +def _of_type(events, event_type): + return [e for e in events if getattr(e, "type", None) == event_type] - Asserting only that the tool-call list comes back empty would be - asserting on the patched parser's own return value; what this branch - actually has to get right is that plain text still reaches the client - as a completed text item. - """ - output = _make_mock_output(index=0, text="Just plain text", text_diff="") - helper = ResponsesStreamingEventsHelper() - helper.is_text_sent = True - - with ( - patch( - "tensorrt_llm.serve.responses_utils._apply_reasoning_parser", - return_value=("Just plain text", ""), - ), - patch( - "tensorrt_llm.serve.responses_utils._apply_tool_parser", - return_value=("Just plain text", []), - ), - ): - result = _should_send_done_events( - output=output, - output_index=0, - tools=[], - streaming_events_helper=helper, - finished_generation=True, + +def _payloads(events, event_type): + return [e.text for e in _of_type(events, event_type)] + + +def _done_payloads(events): + """Every terminal text payload a client would display, text or reasoning.""" + return _payloads(events, "response.output_text.done") + _payloads( + events, "response.reasoning_text.done" + ) + + +def _items(events, event_type, item_type): + return [ + e.item for e in _of_type(events, event_type) if getattr(e.item, "type", None) == item_type + ] + + +def _call_items(events): + return _items(events, "response.output_item.done", "function_call") + + +def _assert_no_markup(events): + for payload in _done_payloads(events): + for marker in _MARKUP: + assert marker not in payload, ( + f"{marker!r} reached the client as assistant text: {payload!r}" ) - (_, should_send_text_done, _, text_content, done_tool_calls) = result - # The behaviour this branch owns: generation finished with text already - # streamed and no tool calls, so the text item must be closed out. - assert should_send_text_done is True - assert text_content == "Just plain text" - assert done_tool_calls == [] +class TestTheDefect: + """Edge case 1: two or more calls in one response.""" + + def test_text_item_closes_at_the_first_call_with_no_markup(self): + """The whole recorded response, replayed frame by frame. + + Exactly one message item, holding only the prose that preceded the + first call, and both calls recovered. Before the fix this same replay + produced a 296-character message item running from the prose straight + into `functions.execinput`. + """ + events = _drive(_REAL_FRAMES) + + _assert_no_markup(events) + assert _payloads(events, "response.output_text.done") == [_REAL_TEXT] + + calls = _call_items(events) + assert [c.name for c in calls] == ["exec", "exec"] + assert [c.namespace for c in calls] == ["functions", "functions"] + for call in calls: + # Arguments a client cannot parse are arguments it cannot run. + assert json.loads(call.arguments)["input"] + + def test_the_prose_is_one_item_not_two(self): + """The chunk carrying both the prose and the call start is one turn. + + The delta has to be appended to the open item before the call closes + it. If the close ran first, the prose in that chunk would land in a + second message item - the split happens mid-sentence, and a client + rendering only the last item shows only the fragment. + """ + events = _drive(_REAL_FRAMES) + assert len(_items(events, "response.output_item.done", "message")) == 1 + + def test_no_prefix_of_the_response_leaks_markup(self): + """Feeding frames 0..k for every k must never publish markup. + + The failing window was narrow - it opened once one call had closed + while another was still open - so the whole prefix family is replayed + rather than just the end state. + """ + for k in range(1, len(_REAL_FRAMES) + 1): + for finish in (False, True): + events = _drive(_REAL_FRAMES[:k], finish=finish) + _assert_no_markup(events) + + def test_done_payload_is_exactly_the_deltas_that_were_streamed(self): + """The invariant, stated directly, for every prefix. + + Every enumerated case is a consequence of this one property, and it + catches the regressions the enumerated cases miss: once a done payload + is defined as the sum of the deltas already sent, text that was never + streamed cannot appear in it. + """ + delta_of = { + "response.output_text.done": "response.output_text.delta", + "response.reasoning_text.done": "response.reasoning_text.delta", + } + for k in range(1, len(_REAL_FRAMES) + 1): + for finish in (False, True): + events = _drive(_REAL_FRAMES[:k], finish=finish) + streamed: dict[tuple[str, str], str] = {} + for event in events: + event_type = getattr(event, "type", None) + if event_type in delta_of.values(): + key = (event.item_id, event_type) + streamed[key] = streamed.get(key, "") + event.delta + elif event_type in delta_of: + key = (event.item_id, delta_of[event_type]) + assert event.text == streamed.get(key, ""), ( + f"k={k} finish={finish}: done payload differs from " + f"the deltas sent for {event.item_id}" + ) + + def test_any_chunking_of_the_response_gives_the_same_result(self): + """The recorded frame boundaries are one chunking out of many. + + Whether a call arrives whole in one chunk or spread over ten is a + property of the decoder's timing, not of the response, so every + chunking has to produce the same text and the same two calls. Replayed + against the previous implementation this fails on 1742 of 2000 + randomised chunkings; the boundaries are seeded so a failure is + reproducible. + """ + full_text = "".join(_REAL_FRAMES) + rng = random.Random(20260918) + + for _ in range(30): + cuts = sorted(rng.sample(range(1, len(full_text)), rng.randint(1, 40))) + frames = [full_text[a:b] for a, b in zip([0] + cuts, cuts + [len(full_text)])] + events = _drive(frames) + + _assert_no_markup(events) + assert _payloads(events, "response.output_text.done") == [_REAL_TEXT] + assert [c.name for c in _call_items(events)] == ["exec", "exec"] -class TestGenerateStreamingEventToolCalls: - """Test that _generate_streaming_event yields output_item events for tool calls.""" + def test_two_whole_calls_in_one_chunk_are_both_reported(self): + """A parser reports at most one finished call per increment. - def test_emits_output_item_added_and_done_for_each_tool_call(self): - """When done_tool_calls is non-empty, we get output_item.added and .done per tool call.""" - output = _make_mock_output( - index=0, - text='Hello {"name":"get_weather","arguments":{"location":"NYC"}}', - text_diff="", + Glm47ToolParser returns as soon as it sees a `` and keeps + the rest buffered for the next increment. When the stream ends on that + chunk there is no next increment, so without draining the parser at + end of stream the second call is silently lost. + """ + events = _drive( + [ + "Thinking.", + "get_timecity" + "Beijing" + "get_timecity" + "Paris", + ], + tools=[_function_tool()], ) - request = _make_mock_request( - tools=[{"type": "function", "function": {"name": "get_weather"}}] + + calls = _call_items(events) + assert [c.name for c in calls] == ["get_time", "get_time"] + assert [json.loads(c.arguments)["city"] for c in calls] == ["Beijing", "Paris"] + + +class TestTransitions: + """Edge cases 2, 3 and 4: what closes, and what opens, around a call.""" + + def test_call_straight_after_reasoning_emits_no_message_item(self): + """Edge case 2/3: `` closes the reasoning item. + + There is no text item to close, and none may be invented: an empty + message item is a turn the model never took. + """ + events = _drive( + [ + "Deciding what to do.", + "get_timecity" + "Beijing", + ], + tools=[_function_tool()], ) - helper = ResponsesStreamingEventsHelper() - helper.is_text_sent = True - helper.item_id = "msg_initial" - tool_calls = [ - ToolCallItem(tool_index=0, name="get_weather", parameters='{"location":"NYC"}'), - ] + assert _items(events, "response.output_item.done", "message") == [] + assert _payloads(events, "response.reasoning_text.done") == ["Deciding what to do."] + assert [c.name for c in _call_items(events)] == ["get_time"] + + def test_reasoning_item_closes_before_the_call_item_is_added(self): + """Edge case 3, on ordering: no call is nested inside another item. + + A client that is still inside a reasoning item when a function_call + item arrives attributes the call to the reasoning. + """ + events = _drive( + [ + "Deciding what to do.", + "get_timecity" + "Beijing", + ], + tools=[_function_tool()], + ) + + types = [getattr(e, "type", None) for e in events] + reasoning_done = types.index("response.reasoning_text.done") + call_added = next( + i + for i, e in enumerate(events) + if getattr(e, "type", None) == "response.output_item.added" + and getattr(e.item, "type", None) == "function_call" + ) + assert reasoning_done < call_added + + def test_text_after_a_call_opens_a_new_message_item(self): + """Edge case 4, constructed - no recorded response has text after a call. + + All 35 multi-call records have nothing between `` and the + next ``, and nothing after the last one, so this shape only + exists here. + """ + events = _drive( + [ + "Thinking.Before. ", + "get_timecity" + "Beijing", + "After.", + ], + tools=[_function_tool()], + ) + + _assert_no_markup(events) + assert _payloads(events, "response.output_text.done") == ["Before. ", "After."] + + def test_text_in_the_same_chunk_as_the_closing_call_tag(self): + """Edge case 4 again, with the call and the trailing text in one chunk. + + The parser keeps everything after `` in its buffer and only + returns it on the next increment, so a stream that ends on that chunk + relies on the end-of-stream flush to release it. Without the flush the + sentence is silently dropped. + """ + events = _drive( + [ + "Thinking.Before. ", + "get_timecity" + "BeijingAfter.", + ], + tools=[_function_tool()], + ) + + _assert_no_markup(events) + assert _payloads(events, "response.output_text.done") == ["Before. ", "After."] - # _apply_reasoning_parser: once for delta (""), once for full output in _should_send_done_events - # _apply_tool_parser: once in _should_send_done_events with full text -> return text before tools + tool_calls - with ( - patch( - "tensorrt_llm.serve.responses_utils._apply_reasoning_parser", - side_effect=[("", ""), (output.text, "")], - ), - patch( - "tensorrt_llm.serve.responses_utils._apply_tool_parser", - return_value=("Hello ", tool_calls), - ), - patch( - "tensorrt_llm.serve.responses_utils._get_chat_completion_function_tools", - return_value=[], - ), - ): - events = list( + def test_whitespace_only_delta_between_calls_opens_a_message_item(self): + r"""Edge case 5, recording what happens rather than what should. + + SGLang suppresses a whitespace-only message between calls because + qwen3-coder separates its calls with `\n`. GLM does not, so no GLM + traffic produces this, and suppressing it here would contradict the + rule directly above it in _generate_streaming_event: a whitespace-only + *first* delta must still open an item, or Codex CLI reports + "OutputTextDelta without active item" and prints nothing at all. + Reconciling the two needs its own decision, so this test pins the + current behaviour and will fail loudly if someone changes it by + accident. + """ + events = _drive( + [ + "Thinking.", + "get_timecity" + "Beijing", + "\n", + "get_timecity" + "Paris", + ], + tools=[_function_tool()], + ) + + _assert_no_markup(events) + assert _payloads(events, "response.output_text.done") == ["\n"] + assert len(_call_items(events)) == 2 + + +class TestEndOfStream: + """Edge cases 6 and 7: streams that stop before the model was done.""" + + def test_stream_cut_off_mid_call_drops_the_markup(self): + """Edge case 6: the parser is still holding an unterminated call. + + The old re-parse reported those bytes as the assistant's message, + which is the leak this change removes; dropping them is the deliberate + alternative. The text already streamed is unaffected. + """ + with patch("tensorrt_llm.serve.responses_utils.logger") as mock_logger: + events = _drive(_REAL_FRAMES[:3]) + + _assert_no_markup(events) + assert _payloads(events, "response.output_text.done") == [_REAL_TEXT] + # The drop has to be visible in the log, not silent. + assert mock_logger.warning.called + assert "tool call" in mock_logger.warning.call_args[0][0] + + def test_stream_cut_off_mid_call_reports_no_half_built_call(self): + """A call whose arguments never closed is not a call a client can run. + + Its accumulated arguments are a JSON prefix. The non-streaming + endpoint reports no call for the same text, and the two must agree. + """ + events = _drive(_REAL_FRAMES[:3]) + assert _call_items(events) == [] + + def test_stream_cut_off_after_one_call_keeps_that_call(self): + """Dropping the unfinished call must not drop the finished ones. + + Frames 0-6 close the first call and open the second, so exactly one + call survives. + """ + events = _drive(_REAL_FRAMES[:7]) + assert [c.name for c in _call_items(events)] == ["exec"] + assert json.loads(_call_items(events)[0].arguments)["input"] + + def test_abort_closes_the_open_item(self): + """Edge case 7: an aborted stream still finalises what it had. + + An abort surfaces as a final chunk carrying no new text, and an item + left open has no terminal state - Codex CLI echoes it back on the next + turn without a `status`, which the next request is rejected for. + """ + events = _drive(["Thinking.Partial answer", ""]) + + assert _payloads(events, "response.output_text.done") == ["Partial answer"] + assert len(_items(events, "response.output_item.done", "message")) == 1 + + +class TestUnchangedPaths: + """Edge cases 10 and 11, and the regressions the structure already fixed.""" + + def test_no_tool_parser_leaves_the_text_alone(self): + """Edge case 10: with no tool parser, nothing is a call. + + The markup is ordinary text on this path, and it still has to come + back as exactly what was streamed. + """ + events = _drive(["Thinking.Answer x"], tool_parser_id=None) + + assert _payloads(events, "response.output_text.done") == ["Answer x"] + assert _call_items(events) == [] + + def test_empty_delta_opens_nothing(self): + """Edge case 11.""" + assert _drive([""], finish=False) == [] + + def test_reasoning_and_text_in_one_chunk_still_emits_the_reasoning(self): + """Regression: the chunk that spans `` carries both halves. + + The reasoning part has to be flushed in the delta branch or it is + never emitted; measured at 56% of reasoning items across four fleets, + and 100% of reasoning short enough to fit one chunk. + """ + events = _drive(["I should answer.Answer."]) + + assert _payloads(events, "response.reasoning_text.done") == ["I should answer."] + assert _payloads(events, "response.output_text.done") == ["Answer."] + + def test_item_is_opened_before_a_whitespace_only_first_delta(self): + """Regression: a delta with no item open makes a client drop the turn. + + Codex CLI reports "OutputTextDelta without active item" and prints + nothing. Short replies hit it, because a leading whitespace token is + more likely to be the whole first delta. + """ + events = _drive(["Thinking. ", "hi"]) + + types = [getattr(e, "type", None) for e in events] + first_delta = types.index("response.output_text.delta") + message_added = next( + i + for i, e in enumerate(events) + if getattr(e, "type", None) == "response.output_item.added" + and getattr(e.item, "type", None) == "message" + ) + assert message_added < first_delta + assert _payloads(events, "response.output_text.done") == [" hi"] + + def test_message_item_is_closed_before_the_call_item_is_added(self): + """Regression: a call must never be nested inside a message item.""" + events = _drive(_REAL_FRAMES) + + message_done = next( + i + for i, e in enumerate(events) + if getattr(e, "type", None) == "response.output_item.done" + and getattr(e.item, "type", None) == "message" + ) + call_added = next( + i + for i, e in enumerate(events) + if getattr(e, "type", None) == "response.output_item.added" + and getattr(e.item, "type", None) == "function_call" + ) + assert message_done < call_added + + def test_calls_are_emitted_once_if_the_finished_output_arrives_twice(self): + """Regression: double emission must stay impossible. + + `emitted_tool_calls` existed because the old per-chunk re-parse + re-reported the same calls on every chunk. Emission is driven by + accumulated fragments now, so the counter guards a different thing - + the emission block running for more than one chunk of the same + output - but it still has to guard it. + """ + helper = ResponsesStreamingEventsHelper() + request = _make_request([_namespace_tool()]) + reasoning_parser_dict, tool_parser_dict = {}, {} + + def step(text_diff, accumulated, finished): + return list( _generate_streaming_event( - output=output, + output=_make_output(accumulated, text_diff), request=request, - finished_generation=True, + finished_generation=finished, streaming_events_helper=helper, - tool_parser_id="test", - tool_parser_dict={0: None}, + reasoning_parser_id=_REASONING_PARSER, + tool_parser_id=_TOOL_PARSER, + reasoning_parser_dict=reasoning_parser_dict, + tool_parser_dict=tool_parser_dict, ) ) - # We should have at least: text_done, content_part_done, output_item_done (text), - # then output_item_added (tool), output_item_done (tool) - types = [getattr(e, "type", None) for e in events] - assert "response.output_item.added" in types - assert "response.output_item.done" in types - # Find the done event that has a function_call item - from openai.types.responses import ResponseOutputItemDoneEvent - - done_events = [e for e in events if isinstance(e, ResponseOutputItemDoneEvent)] - function_call_dones = [ - e - for e in done_events - if getattr(getattr(e, "item", None), "type", None) == "function_call" + events, accumulated = [], "" + for frame in _REAL_FRAMES: + accumulated += frame + events += step(frame, accumulated, frame is _REAL_FRAMES[-1]) + assert len(_call_items(events)) == 2 + + # A second finished chunk for the same output carries no new text, and + # must not re-report the calls already emitted for it. + assert _call_items(step("", accumulated, True)) == [] + + +class TestFragmentAssembly: + """The accumulator that turns parser fragments back into whole calls.""" + + def test_fragments_are_concatenated_in_order(self): + fragments: dict = {} + _accumulate_tool_call_fragments( + fragments, + [ + ToolCallItem(tool_index=0, name="get_time", parameters=""), + ToolCallItem(tool_index=0, name=None, parameters='{"city": "Par'), + ToolCallItem(tool_index=0, name=None, parameters='is"}'), + ], + ) + + assert _assembled_tool_calls(fragments) == [ + ToolCallItem(tool_index=0, name="get_time", parameters='{"city": "Paris"}') ] - assert len(function_call_dones) >= 1 - assert function_call_dones[0].item.name == "get_weather" - assert function_call_dones[0].item.arguments == '{"location":"NYC"}' + + def test_a_later_fragment_does_not_erase_the_name(self): + """Only a call's first fragment carries its name; the rest carry None.""" + fragments: dict = {} + _accumulate_tool_call_fragments( + fragments, + [ + ToolCallItem(tool_index=0, name="get_time", parameters=""), + ToolCallItem(tool_index=0, name=None, parameters="{}"), + ], + ) + assert _assembled_tool_calls(fragments)[0].name == "get_time" + + def test_the_unfinished_call_is_dropped_and_the_finished_ones_kept(self): + fragments: dict = {} + _accumulate_tool_call_fragments( + fragments, + [ + ToolCallItem(tool_index=0, name="get_time", parameters='{"a": 1}'), + ToolCallItem(tool_index=1, name="get_time", parameters='{"a": '), + ], + ) + + assembled = _assembled_tool_calls(fragments, unfinished_tool_index=1) + assert [c.parameters for c in assembled] == ['{"a": 1}'] + + def test_a_call_whose_name_never_arrived_is_dropped(self): + """Matches parse_base_json on the non-streaming path.""" + fragments: dict = {} + _accumulate_tool_call_fragments( + fragments, [ToolCallItem(tool_index=0, parameters='{"a": 1}')] + ) + assert _assembled_tool_calls(fragments) == [] + + def test_fragments_are_kept_apart_per_output(self): + """Edge case 8: one parser instance per output, one fragment set each. + + `tool_parser_dict` is keyed by output index, so two outputs of the same + request number their calls independently - both start at tool_index 0 - + and merging them would concatenate one output's arguments onto the + other's. + """ + helper = ResponsesStreamingEventsHelper() + _accumulate_tool_call_fragments( + helper.tool_call_fragments(0), + [ToolCallItem(tool_index=0, name="first", parameters='{"a": 1}')], + ) + _accumulate_tool_call_fragments( + helper.tool_call_fragments(1), + [ToolCallItem(tool_index=0, name="second", parameters='{"b": 2}')], + ) + + assert [c.name for c in _assembled_tool_calls(helper.tool_call_fragments(0))] == ["first"] + assert [c.name for c in _assembled_tool_calls(helper.tool_call_fragments(1))] == ["second"] + + def test_two_helpers_do_not_share_fragments(self): + """Two requests in flight at once must not share an accumulator. + + A mutable default on the state tracker's class body would be one + object shared by every request, so two concurrent streams would + accumulate into each other's calls. + """ + first, second = (ResponsesStreamingEventsHelper(), ResponsesStreamingEventsHelper()) + _accumulate_tool_call_fragments( + first.tool_call_fragments(0), + [ToolCallItem(tool_index=0, name="only_mine", parameters="{}")], + ) + + assert second.tool_call_fragments(0) == {} diff --git a/tests/unittest/llmapi/apps/test_responses_usage.py b/tests/unittest/llmapi/apps/test_responses_usage.py index 5cbf689e4731..296be1a97d63 100644 --- a/tests/unittest/llmapi/apps/test_responses_usage.py +++ b/tests/unittest/llmapi/apps/test_responses_usage.py @@ -131,3 +131,97 @@ def test_usage_validates_inside_a_streamed_completion_event(): ) assert event.response.usage.input_tokens == 7 assert event.response.usage.input_tokens_details.cached_tokens == 6 + + +# --------------------------------------------------------------------------- +# reasoning_tokens +# +# Reported as a hardcoded zero until now. The count is rebuilt from the text +# the reasoning parser itself claimed, rather than by searching the generated +# token ids for ``, and these tests are the reason: GLM breaks a +# marker search three separate ways under agent workloads, and each of them +# was observed live before it was tested here. +# --------------------------------------------------------------------------- + + +class _WordTokenizer: + """One token per whitespace-separated word. Enough to count with.""" + + def encode(self, text, add_special_tokens=False): + return text.split() + + +def _reasoning_tokens(text, parser="glm", tokenizer=_WordTokenizer(), tools=None): + """Run the real parser, then count what it called reasoning.""" + from tensorrt_llm.serve.responses_utils import _count_reasoning_tokens, _create_output_content + + _items, _messages, reasoning_texts = _create_output_content( + SimpleNamespace(outputs=[SimpleNamespace(index=0, text=text)]), + reasoning_parser=parser, + tool_parser=None, + tools=tools, + ) + return _count_reasoning_tokens(tokenizer, reasoning_texts, 10_000) + + +def test_reasoning_tokens_counts_up_to_the_closing_tag(): + # The ordinary shape: reasoning, close, answer. + assert _reasoning_tokens("one two threefour five") == 3 + + +def test_reasoning_tokens_stop_at_the_first_close_not_the_last(): + # GLM closes the block more than once (reasoning_parser.py:370-388). The + # parser splits at the first close and drops the stray one; a search for + # `` in token space that took the last match -- which is what + # ThinkingBudgetLogitsProcessor does, for its own good reasons -- would + # count the middle stretch as reasoning too. + text = "one twothree four fivex" + assert _reasoning_tokens(text) == 2 + + +def test_reasoning_tokens_end_at_an_implicit_tool_call(): + # Observed live: 26,055 characters over 169 frames with no closing tag at + # all and one well-formed . Counting to the missing `` + # would have called the entire turn reasoning. + text = "planning the call{}" + assert _reasoning_tokens(text) == 3 + + +def test_reasoning_tokens_covers_a_turn_that_never_leaves_the_block(): + # `` is prefilled by the template, so output starts inside the + # block; with no terminator the whole turn really is reasoning. + assert _reasoning_tokens("still thinking about it") == 4 + + +def test_reasoning_tokens_are_zero_without_a_reasoning_parser(): + assert _reasoning_tokens("one two three", parser=None) == 0 + + +def test_reasoning_tokens_are_zero_without_a_tokenizer(): + assert _reasoning_tokens("one two three", tokenizer=None) == 0 + + +def test_reasoning_tokens_never_exceed_generated_tokens(): + # Re-encoding a detokenized substring need not reproduce the tokenization + # it came from, and usage claiming more reasoning than output is visibly + # wrong to a client budgeting a context window. + from tensorrt_llm.serve.responses_utils import _count_reasoning_tokens + + assert _count_reasoning_tokens(_WordTokenizer(), ["a b c d e"], 3) == 3 + + +def test_usage_reports_the_reasoning_tokens_it_counts(): + usage = _create_usage( + _generation(prompt_tokens=7, completion_tokens=9), + tokenizer=_WordTokenizer(), + reasoning_texts=["one two three four"], + ) + assert usage.output_tokens_details.reasoning_tokens == 4 + # Reasoning tokens are part of the generated total, not additional to it. + assert usage.output_tokens == 9 + assert usage.total_tokens == 16 + + +def test_usage_still_defaults_to_zero_reasoning_tokens(): + usage = _create_usage(_generation()) + assert usage.output_tokens_details.reasoning_tokens == 0 From e5dec9b801bdd96dfe19256bb4399c61b01b36bd Mon Sep 17 00:00:00 2001 From: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> Date: Wed, 30 Sep 2026 10:02:48 +0000 Subject: [PATCH 19/43] [None][fix] responses: degrade images in tool outputs too, cap error bodies Codex ships screenshots in three shapes: message content parts, bare top-level items, and inside custom_tool_call_output.output. The first fix covered the first two; 114 Kernel-Trace campaigns then died in 24h through the third (the parts list lives under "output", not "content"). The degrade walk now covers all three, with the traced shape as a regression test. Validation-error bodies are capped at 4KB so a rejected request's base64 payload no longer floods the client's stop reason. Signed-off-by: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> --- tensorrt_llm/serve/openai_protocol.py | 47 +++++++- tensorrt_llm/serve/openai_server.py | 13 +- .../llmapi/apps/test_responses_usage.py | 111 ++++++++++++++++++ 3 files changed, 169 insertions(+), 2 deletions(-) diff --git a/tensorrt_llm/serve/openai_protocol.py b/tensorrt_llm/serve/openai_protocol.py index eadedbdf2a0d..b1a804b4233d 100644 --- a/tensorrt_llm/serve/openai_protocol.py +++ b/tensorrt_llm/serve/openai_protocol.py @@ -1533,11 +1533,56 @@ def _with_annotations(part): return {**part, "annotations": []} return part + def _degrade_image(part): + """Replace an image content part with a text placeholder. + + The Codex CLI attaches screenshots as ``input_image`` parts -- an + agent that plots something and views it sends the PNG back as + base64. The models served here are text-only, and the vendored + input union has no image member, so validation fell through to + ``ResponseInputTextParam`` and failed the WHOLE request with + "Field required: text" -- with the full base64 body echoed into + the error. The client retries a deterministic 400 until its + backoff limit and the campaign dies: 2 of the first 182 + Kernel-Trace campaigns went exactly this way (2026-09-22). + + A placeholder keeps the turn -- and with it the session -- alive, + and says what was dropped instead of leaking megabytes of base64. + """ + if isinstance(part, dict) and part.get("type") == "input_image": + url = part.get("image_url") or "" + kind = url.split(";", 1)[0].removeprefix("data:") or "image" + return { + "type": + "input_text", + "text": + "[image omitted: %s, %d bytes as sent; this " + "model accepts text only]" % (kind, len(url)), + } + return part + cleaned = [] for item in value: # A client may send a bare content part as a top-level item, not # only nested inside a message. - item = _with_annotations(item) + item = _with_annotations(_degrade_image(item)) + if isinstance(item, dict) and isinstance(item.get("content"), list): + item = { + **item, + "content": + [_degrade_image(part) for part in item["content"]], + } + # Tool results carry their parts under "output", not "content" + # (custom_tool_call_output / function_call_output). An agent that + # plots something gets the PNG back through the tool and Codex + # ships it as an input_image part in there -- 114 Kernel-Trace + # campaigns died on exactly this shape (2026-09-23) after the + # content/top-level shapes were already handled. + if isinstance(item, dict) and isinstance(item.get("output"), list): + item = { + **item, + "output": [_degrade_image(part) for part in item["output"]], + } if isinstance(item, dict) and item.get("type") in (None, "message"): role = item.get("role") if "id" in item and role in _ID_STRIPPED_ROLES: diff --git a/tensorrt_llm/serve/openai_server.py b/tensorrt_llm/serve/openai_server.py index c75cfc90c226..edacf0a56eca 100644 --- a/tensorrt_llm/serve/openai_server.py +++ b/tensorrt_llm/serve/openai_server.py @@ -786,7 +786,18 @@ async def validation_exception_handler(request, exc): # expect. if self.metrics_collector: self.metrics_collector.log_request_error(http_code=400) - return JSONResponse(status_code=400, content={"error": str(exc)}) + # pydantic echoes each failing item's `input` verbatim, and a + # rejected multimodal part carries megabytes of base64 -- which + # then travels into the client's retry logs and, for a Kernel + # Factory agent, into the campaign STOP_REASON (measured: a whole + # PNG in one). The locations are the diagnostic content; the + # payload is not. Truncation is by error, so short messages are + # untouched. + detail = str(exc) + if len(detail) > 4096: + detail = detail[:4096] + " ... [truncated %d bytes]" % ( + len(detail) - 4096) + return JSONResponse(status_code=400, content={"error": detail}) if self.server_role is ServerRole.VISUAL_GEN: assert self._is_visual_gen, \ diff --git a/tests/unittest/llmapi/apps/test_responses_usage.py b/tests/unittest/llmapi/apps/test_responses_usage.py index 296be1a97d63..e85755a2d284 100644 --- a/tests/unittest/llmapi/apps/test_responses_usage.py +++ b/tests/unittest/llmapi/apps/test_responses_usage.py @@ -225,3 +225,114 @@ def test_usage_reports_the_reasoning_tokens_it_counts(): def test_usage_still_defaults_to_zero_reasoning_tokens(): usage = _create_usage(_generation()) assert usage.output_tokens_details.reasoning_tokens == 0 + + +# --------------------------------------------------------------------------- +# input_image degradation +# --------------------------------------------------------------------------- + + +def test_an_image_part_degrades_to_a_text_placeholder(): + """Regression: an input_image part 400'd the whole request. + + Codex attaches screenshots as `input_image` (base64). The input union has + no image member, so validation fell through to ResponseInputTextParam and + rejected the request -- deterministically, so the client's retries all + failed and the campaign died on its backoff limit. Measured: 2 of the + first 182 Kernel-Trace campaigns, each with the full base64 body echoed + into the stop reason. + """ + from tensorrt_llm.serve.openai_protocol import ResponsesRequest + + req = ResponsesRequest( + model="m", + input=[ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "look at this"}, + {"type": "input_image", "image_url": "data:image/png;base64,AAAA"}, + ], + }, + # And as a bare top-level part, which clients also send. + {"type": "input_image", "image_url": "data:image/png;base64,BB"}, + ], + ) + dumped = req.model_dump()["input"] + texts = [] + for item in dumped: + parts = item.get("content") if isinstance(item.get("content"), list) else [item] + for p in parts: + assert p.get("type") != "input_image", "image part survived" + if isinstance(p.get("text"), str): + texts.append(p["text"]) + assert any("image omitted" in t and "image/png" in t for t in texts) + # The placeholder must not carry the base64 payload. + assert not any("AAAA" in t for t in texts) + + +def test_an_image_in_a_tool_output_degrades_too(): + """Regression: tool results carry parts under "output", not "content". + + An agent that plots something gets the PNG back through the tool: Codex + sends `custom_tool_call_output` whose `output` list ends with an + `input_image` part. The first degrade pass only walked `content` and the + top level, so this shape still 400'd the whole request -- measured at 114 + Kernel-Trace campaigns in the 24h after the content/top-level fix went + live (2026-09-23). Shape taken verbatim from a traced rejected request. + """ + from tensorrt_llm.serve.openai_protocol import ResponsesRequest + + req = ResponsesRequest( + model="m", + input=[ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "plot it"}, + ], + }, + { + "type": "custom_tool_call", + "id": "ctc_1", + "call_id": "call_1", + "name": "exec", + "input": "python plot.py", + }, + { + "type": "custom_tool_call_output", + "call_id": "call_1", + "output": [ + {"type": "input_text", "text": "Script completed\n"}, + {"type": "input_text", "text": "layout-a2 image"}, + {"type": "input_image", "image_url": "data:image/png;base64,CCCC"}, + ], + }, + ], + ) + out = req.model_dump()["input"][2]["output"] + assert all(p.get("type") != "input_image" for p in out), ( + "image part survived inside tool output" + ) + joined = " ".join(p.get("text", "") for p in out) + assert "image omitted" in joined and "image/png" in joined + assert "CCCC" not in joined + # A plain-string tool output must pass through untouched. + req2 = ResponsesRequest( + model="m", + input=[ + { + "type": "custom_tool_call", + "id": "ctc_2", + "call_id": "call_2", + "name": "exec", + "input": "true", + }, + { + "type": "custom_tool_call_output", + "call_id": "call_2", + "output": "Script completed\n", + }, + ], + ) + assert req2.model_dump()["input"][1]["output"] == "Script completed\n" From 271a1edadcfa14c29b9fe4396b2cbee66b812b05 Mon Sep 17 00:00:00 2001 From: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> Date: Wed, 30 Sep 2026 10:48:20 +0000 Subject: [PATCH 20/43] [None][fix] responses: the final response must describe the stream that preceded it Three stream-vs-final divergences, each measured on production traffic against a GLM-5.3 fleet: - Item IDs: the final rebuild minted fresh UUIDs for reasoning and message items (216 of 221 sampled responses disagreed with their own stream). The streaming tracker now records the item IDs it emitted and the final builders reuse them, the same wiring the tool-call call_id fix used. A count mismatch logs a structural-divergence warning instead of papering over it. - Sequence numbers: with postprocess workers enabled the opening event pair is numbered by the frontend processor and everything after by a pickled copy in the worker, so one stream carried 0,1,0,1,2,... (221 of 221 sampled). Frames are now restamped with one per-response counter at the egress generator, the single point both configurations flow through. The frame is JSON-parsed, not substring-patched: response.completed embeds model output, which can contain the literal field name. - An unterminated at end of stream was held by the tool parser and dropped from the stream while the final rebuild fell back to including it as text - a 429-char message the stream never showed. The flush now releases the held markup as text so both views carry the same characters. parallel_tool_calls=false and non-auto tool_choice on the non-Harmony path remain unenforced by design (these turns are recorded as training data; truncating what the model generated would falsify it) - but the server now says so once per process instead of silently swallowing the options. Signed-off-by: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> --- tensorrt_llm/serve/openai_server.py | 46 +- tensorrt_llm/serve/responses_utils.py | 201 +++++++-- ...test_responses_stream_final_consistency.py | 394 ++++++++++++++++++ .../test_responses_streaming_tool_calls.py | 66 ++- 4 files changed, 667 insertions(+), 40 deletions(-) create mode 100644 tests/unittest/llmapi/apps/test_responses_stream_final_consistency.py diff --git a/tensorrt_llm/serve/openai_server.py b/tensorrt_llm/serve/openai_server.py index edacf0a56eca..53b8cbbd7c39 100644 --- a/tensorrt_llm/serve/openai_server.py +++ b/tensorrt_llm/serve/openai_server.py @@ -126,6 +126,7 @@ from tensorrt_llm.serve.responses_utils import guard_responses_stream from tensorrt_llm.serve.responses_utils import \ request_preprocess as responses_api_request_preprocess +from tensorrt_llm.serve.responses_utils import stamp_sse_sequence_number from tensorrt_llm.serve.responses_web_search import web_search_rejection_reason from tensorrt_llm.serve.rl_control_auth import validate_rl_control_request from tensorrt_llm.serve.serving_extensions import apply_model_chat_extensions @@ -3131,6 +3132,15 @@ async def create_streaming_generator(promise: RequestOutput, async def openai_responses(self, request: ResponsesRequest, raw_request: Request) -> Response: + """Serve one /v1/responses request, streaming or not. + + ``parallel_tool_calls`` and, on the non-Harmony path, ``tool_choice`` + are accepted for schema compatibility but not enforced: honouring + them would mean constraining or truncating what the model generated, + and these turns are recorded as training data, where a falsified + output is worse than an unconstrained one. A once-per-process warning + below says so instead of silently swallowing the option. + """ async def create_response( promise: RequestOutput, @@ -3165,16 +3175,31 @@ async def create_streaming_generator(promise: RequestOutput, postproc_params: PostprocParams): post_processor, args = postproc_params.post_processor, postproc_params.postproc_args streaming_processor = args.streaming_processor + # One monotonic counter per response, restamped onto every frame + # right before it leaves the server. The producers cannot number + # the stream themselves: with postprocessing workers enabled the + # two opening events below come from this process's streaming + # processor while every later frame is built by a pickled copy of + # it inside a worker, each counting from zero, so one response's + # stream carried 0,1,0,1,2,... (221 of 221 measured responses). + # Both configurations flow through this generator - workers hand + # back ready-made frames in _postprocess_result, the in-process + # path builds them via post_processor - so this is the one point + # where a response-wide sequence exists to be assigned. + sequence_number = 0 initial_responses = streaming_processor.get_initial_responses() for initial_response in initial_responses: - yield initial_response + yield stamp_sse_sequence_number(initial_response, + sequence_number) + sequence_number += 1 async for res in promise: pp_results = res.outputs[ 0]._postprocess_result if self.postproc_worker_enabled else post_processor( res, args) for pp_res in pp_results: - yield pp_res + yield stamp_sse_sequence_number(pp_res, sequence_number) + sequence_number += 1 await self._extract_metrics(res, raw_request) try: @@ -3206,6 +3231,23 @@ async def create_streaming_generator(promise: RequestOutput, f"{web_search_error}."), ) + # Accepted-but-unenforced options, said once per process (see the + # handler docstring for why they are not enforced). Rejecting + # would break clients that always send them; enforcing would + # falsify the recorded output. + if request.parallel_tool_calls is False: + logger.warning_once( + "Responses API: 'parallel_tool_calls=false' is accepted " + "but not enforced; the model may still emit several tool " + "calls in one turn.", + key="responses_parallel_tool_calls_unenforced") + if not self.use_harmony and request.tool_choice != "auto": + logger.warning_once( + "Responses API: 'tool_choice' is accepted but not " + "enforced on this model path; generation is not " + "constrained by it.", + key="responses_tool_choice_unenforced") + # Get prev response prev_response = None if self.enable_store: diff --git a/tensorrt_llm/serve/responses_utils.py b/tensorrt_llm/serve/responses_utils.py index 18e19d24f987..23da0ca10f70 100644 --- a/tensorrt_llm/serve/responses_utils.py +++ b/tensorrt_llm/serve/responses_utils.py @@ -1462,12 +1462,19 @@ def _flush_tool_parser( that remainder; most inherit the no-op, which leaves it sitting in `_buffer`, so the buffer is drained and then inspected as well. - Unterminated markup is dropped rather than shown. Releasing it would - reintroduce exactly the leak this path was restructured to make - unreachable, only at a different moment - the bytes are not a message the - model meant to send, they are half of a call that never finished. It is - rare (1 of 3437 recorded responses) and the warning makes the loss visible - instead of silent. + Unterminated markup is released as ordinary text rather than dropped. + Dropping was the first design, on the grounds that half of a call is not + a message the model meant to send - but the final response never dropped + it: `_create_output_content` re-parses the whole text, its regex needs + the closing tag, and the fragment falls back into the message item. The + stream and the snapshot then told two different stories about the same + generation - trace tr_8f312e9973954fa784ae7dc8f45e9d3e ended in a + 429-character unterminated call that the final response carried as + message text while the stream showed nothing at all. Of the two ways to + make the views agree, keeping the text wins: these turns are recorded as + training data, and silently losing model output is worse than showing a + call that never closed. It is rare (1 of 3437 recorded responses) and + the warning keeps the malformed turn visible in the log. A remainder with no markup in it is ordinary output that the parser was holding only until it could rule out a call, and is released. This matches @@ -1512,20 +1519,28 @@ def _flush_tool_parser( calls.extend(result.calls) held = getattr(tool_parser, "_buffer", "") - dropped = 0 + unterminated = 0 for remainder in (result.normal_text, held): if not remainder: continue if tool_parser.has_tool_call(remainder): - dropped += len(remainder) - else: - released.append(remainder) - - if dropped: + unterminated += len(remainder) + released.append(remainder) + + if held: + # The buffer's bytes are being released here, so consume them: the + # executor can present the same finished output more than once, and a + # buffer left in place would release the same remainder into a second + # message item on the repeat. Dropping never had this problem - a + # drop repeated is still nothing. + tool_parser._buffer = "" + + if unterminated: logger.warning( - f"Stream ended inside a tool call; dropped {dropped} characters " - f"of unterminated {type(tool_parser).__name__} markup rather than " - "reporting them as the assistant's message.") + f"Stream ended inside a tool call; releasing {unterminated} " + f"characters of unterminated {type(tool_parser).__name__} markup " + "as the assistant's message, matching the final response's " + "fallback for a call it cannot parse.") # `current_tool_name_sent` is the parser's own record of having announced a # call it has not yet closed, and `current_tool_id` numbers that call. @@ -1581,6 +1596,7 @@ def _create_output_content( tools: Optional[list[Tool]] = None, chat_template_kwargs: Optional[dict[str, Any]] = None, streamed_tool_call_ids: Optional[list[Tuple[str, str]]] = None, + streamed_item_ids: Optional[list[Tuple[str, str]]] = None, ) -> Tuple[list[ResponseOutputItem], list[ChatCompletionMessageParam], list[str]]: output_items: list[ResponseOutputItem] = [] @@ -1591,6 +1607,39 @@ def _create_output_content( reasoning_texts: list[str] = [] available_tools = _get_chat_completion_function_tools(tools) + # The (item type, item id) pairs the stream published in + # `response.output_item.added/done`, in emission order. This rebuild is a + # second, independent pass over the generated text and used to mint fresh + # ids for the reasoning and message items it re-derives, so the snapshot + # named the very items the stream had already announced under different + # ids - 216 of 221 measured responses - and a client joining streamed + # items with the snapshot by id saw phantom items. Reuse is positional + # per item type, first-emitted first, because emission order is the only + # correspondence the two passes share. None means no stream ran (a + # non-streaming request) and every id is minted fresh, as before. + streamed_ids_by_type: dict[str, list[str]] = { + "reasoning": [], + "message": [] + } + for item_type, item_id in streamed_item_ids or []: + if item_type in streamed_ids_by_type: + streamed_ids_by_type[item_type].append(item_id) + used_ids_by_type = {"reasoning": 0, "message": 0} + + def _streamed_or_fresh_id(item_type: str) -> str: + # Running past the pool means this pass derived items the stream never + # opened - the two views already diverged structurally (a stream cut + # off inside a call whose text this pass keeps, say). A fresh id + # cannot hide that and must not try to: the count mismatch is warned + # about after the loop rather than silently papered over. + pool = streamed_ids_by_type[item_type] + index = used_ids_by_type[item_type] + used_ids_by_type[item_type] = index + 1 + if index < len(pool): + return pool[index] + prefix = "rs" if item_type == "reasoning" else "msg" + return f"{prefix}_{_random_uuid()}" + for output in final_res.outputs: calls = [] # chat_template_kwargs has to reach the parser: DeepSeekV4ReasoningParser @@ -1626,7 +1675,7 @@ def _create_output_content( # snapshot fed the model its own thinking as a follow-up to its reply. if reasoning_text: reasoning_item = ResponseReasoningItem( - id=f"rs_{_random_uuid()}", + id=_streamed_or_fresh_id("reasoning"), summary=[], type="reasoning", content=[ @@ -1646,7 +1695,7 @@ def _create_output_content( ) text_item = ResponseOutputMessage( - id=f"msg_{_random_uuid()}", + id=_streamed_or_fresh_id("message"), content=[output_text], role="assistant", status="completed", @@ -1690,6 +1739,20 @@ def _create_output_content( tool_calls_item, })) + # Checked against None, not truthiness: an empty list still means a + # stream ran and published nothing of this type, and a rebuild that then + # derives such an item is exactly the divergence worth saying out loud. + if streamed_item_ids is not None: + for item_type in ("reasoning", "message"): + streamed = len(streamed_ids_by_type[item_type]) + rebuilt = used_ids_by_type[item_type] + if streamed != rebuilt: + logger.warning( + "final response rebuilt %d %s item(s) but the stream " + "published %d; the two views of this generation differ " + "structurally and ids beyond the streamed ones are new", + rebuilt, item_type, streamed) + return output_items, output_messages, reasoning_texts @@ -1982,6 +2045,7 @@ def _create_response( tool_parser: Optional[str] = None, num_prompt_tokens: Optional[int] = None, streamed_tool_call_ids: Optional[list[Tuple[str, str]]] = None, + streamed_item_ids: Optional[list[Tuple[str, str]]] = None, tokenizer: Optional[TokenizerBase] = None, ) -> tuple[ResponsesResponse, list[Message | ChatCompletionMessageParam]]: _responses_debug_log("================================================") @@ -2006,7 +2070,8 @@ def _create_response( tool_parser, request.tools, chat_template_kwargs=reasoning_chat_template_kwargs(request), - streamed_tool_call_ids=streamed_tool_call_ids) + streamed_tool_call_ids=streamed_tool_call_ids, + streamed_item_ids=streamed_item_ids) finish_reason = final_res.outputs[0].finish_reason response = ResponsesResponse.from_request( @@ -2056,6 +2121,7 @@ async def create_response( tool_parser: Optional[str] = None, num_prompt_tokens: Optional[int] = None, streamed_tool_call_ids: Optional[list[Tuple[str, str]]] = None, + streamed_item_ids: Optional[list[Tuple[str, str]]] = None, tokenizer: Optional[TokenizerBase] = None, ) -> ResponsesResponse: @@ -2084,6 +2150,7 @@ async def create_response( tool_parser=tool_parser, num_prompt_tokens=num_prompt_tokens, streamed_tool_call_ids=streamed_tool_call_ids, + streamed_item_ids=streamed_item_ids, tokenizer=tokenizer, ) @@ -2106,6 +2173,7 @@ def create_response_non_store( tool_parser: Optional[str] = None, num_prompt_tokens: Optional[int] = None, streamed_tool_call_ids: Optional[list[Tuple[str, str]]] = None, + streamed_item_ids: Optional[list[Tuple[str, str]]] = None, tokenizer: Optional[TokenizerBase] = None, ) -> ResponsesResponse: response_creation_time = create_time if create_time is not None else int( @@ -2123,6 +2191,7 @@ def create_response_non_store( tool_parser=tool_parser, num_prompt_tokens=num_prompt_tokens, streamed_tool_call_ids=streamed_tool_call_ids, + streamed_item_ids=streamed_item_ids, tokenizer=tokenizer, ) @@ -2168,6 +2237,18 @@ def __init__(self) -> None: # A list here rather than a class attribute, for the reason above. self.emitted_tool_call_ids: list[Tuple[str, str]] = [] + # (item type, item id) of every reasoning and message item this + # stream opened, in the order their `output_item.added` went out. + # The final response re-derives those items from the accumulated text + # and used to mint fresh ids for them, so the snapshot named the very + # items the stream had already announced under different ids - 216 of + # 221 measured responses - and a client joining streamed items with + # the snapshot by id saw phantom items. Same shape and cure as + # emitted_tool_call_ids above. + # + # A list here rather than a class attribute, for the reason above. + self.emitted_item_ids: list[Tuple[str, str]] = [] + class ResponsesStreamingEventsHelper: @@ -2199,6 +2280,11 @@ def emitted_tool_call_ids(self) -> list[Tuple[str, str]]: """(item id, call id) of the tool calls already streamed, in order.""" return self.state_tracker.emitted_tool_call_ids + @property + def emitted_item_ids(self) -> list[Tuple[str, str]]: + """(item type, item id) of the reasoning/message items streamed, in order.""" + return self.state_tracker.emitted_item_ids + def record_emitted_tool_call(self, item) -> None: """Remember the ids a streamed call went out under. @@ -2260,7 +2346,7 @@ def get_response_created_event( self, response: ResponsesResponse) -> ResponseCreatedEvent: return ResponseCreatedEvent( type="response.created", - sequence_number=-1, # will set by _send_event function + sequence_number=-1, # set by _send_event, restamped at egress response=response, ) @@ -2372,6 +2458,14 @@ def _get_output_added_events( if not self.is_output_item_added_sent: self.is_output_item_added_sent = True + # Remember the id this item is announced under, at the one point + # where announcing actually happens. The final response is built + # by a second, independent pass over the generated text; handing + # it these lets it name the same items the same way instead of + # minting ids the client has never seen. + self.state_tracker.emitted_item_ids.append( + (output_item.type, output_item.id)) + if output_item.type == "message": content_part = ResponseOutputText( type="output_text", @@ -2468,7 +2562,10 @@ def _assembled_tool_calls( client can parse or run, so it is dropped rather than reported. The non-streaming path reports no call at all for the same text, since its regex needs the closing tag, and the two endpoints have to agree about - what the model asked for. + what the model asked for. Only the *call* is dropped: the raw markup it + was read from reaches the client as message text on both views + (`_flush_tool_parser` releases it, the whole-text re-parse falls back to + it), so nothing the model generated is lost. A call whose assembled arguments are not valid JSON is dropped for the same reason, one test later. That test keys on the markup being unterminated; @@ -2506,8 +2603,8 @@ def _assembled_tool_calls( for tool_index, fragment in fragments.items(): if not fragment["name"] or tool_index == unfinished_tool_index: # No warning for the unfinished call: `_flush_tool_parser` has - # already reported that loss, and saying it twice would read as two - # calls lost. + # already warned and released its raw markup as message text, and + # a second warning would read as a second loss. continue arguments = fragment["parameters"] try: @@ -2616,9 +2713,9 @@ def check_parser(parser_id: Optional[str], ) # End of stream: the tool parser gets one last word before its state is - # discarded, so bytes it was still withholding are released or dropped by - # a decision rather than by nobody ever asking. Only computed here; what - # it releases is emitted further down, after the call in front of it has + # discarded, so bytes it was still withholding are released by a decision + # rather than lost because nobody ever asked. Only computed here; what it + # releases is emitted further down, after the call in front of it has # closed the item it belongs to. flushed_text, flushed_calls, unfinished_tool_index = "", [], None if finished_generation: @@ -2989,7 +3086,12 @@ def __init__( self.tool_parser = tool_parser def _send_event(self, event: OpenAIBaseModel): - # Set sequence_number if the event has this attribute + # Set sequence_number if the event has this attribute. The number is + # provisional: with postprocessing workers a pickled copy of this + # processor builds the per-token frames in another process, counting + # from zero again, so the frontend restamps every frame at the egress + # (stamp_sse_sequence_number) and what is written here only has to be + # ordered within this process. if hasattr(event, 'sequence_number'): event.sequence_number = self.sequence_number self.sequence_number += 1 @@ -3039,6 +3141,9 @@ async def get_final_response( # Name the calls the way the stream already named them. streamed_tool_call_ids=self.streaming_events_helper. emitted_tool_call_ids, + # And the reasoning/message items likewise: the snapshot has to + # answer to the ids the stream already announced. + streamed_item_ids=self.streaming_events_helper.emitted_item_ids, # Taken as an argument rather than held on this object: the # postproc-worker path pickles the processor across a process # boundary, and that worker already has its own tokenizer @@ -3073,6 +3178,9 @@ def get_final_response_non_store( # Name the calls the way the stream already named them. streamed_tool_call_ids=self.streaming_events_helper. emitted_tool_call_ids, + # And the reasoning/message items likewise: the snapshot has to + # answer to the ids the stream already announced. + streamed_item_ids=self.streaming_events_helper.emitted_item_ids, # Taken as an argument rather than held on this object: the # postproc-worker path pickles the processor across a process # boundary, and that worker already has its own tokenizer @@ -3246,6 +3354,47 @@ def stream_error_event(cause: str, detail: str, f"data: {event.model_dump_json(indent=None)}\n\n").encode("utf-8")] +def stamp_sse_sequence_number(frame: str, sequence_number: int) -> str: + """Overwrite the ``sequence_number`` inside one serialized SSE frame. + + Numbering cannot be left to the producers, because one response can have + two of them. With postprocessing workers enabled the opening + ``response.created``/``response.in_progress`` pair is built by the + frontend's streaming processor while every later frame comes from a + pickled copy of that processor in a worker process, each counting from + zero - so a single response's stream carried 0,1,0,1,2,... (221 of 221 + measured responses). Only the egress point, where the frames converge + just before leaving the server, can hand out one monotonic sequence; it + restamps every frame with this and the producers' numbers are treated as + provisional. + + The frame is parsed rather than patched with a substring replace: the + ``response.completed`` payload embeds the model's own output, which can + contain anything - including the literal ``"sequence_number":`` when the + model writes code against this very API. A frame with no data line, an + unparsable payload, or no sequence_number field is returned untouched; + no producer builds such a frame today, and inventing a field on one would + be worse than leaving its numbering alone. + """ + head, sep, rest = frame.partition("data: ") + if not sep: + return frame + # The payload is one line: _send_event serializes with indent=None, and + # JSON strings carry newlines escaped. + payload_text, newline, tail = rest.partition("\n") + try: + payload = json.loads(payload_text) + except ValueError: + return frame + if not isinstance(payload, dict) or "sequence_number" not in payload: + return frame + payload["sequence_number"] = sequence_number + # Compact separators and raw unicode, matching model_dump_json, so the + # restamped frame differs from the produced one only in the number. + restamped = json.dumps(payload, separators=(",", ":"), ensure_ascii=False) + return f"{head}data: {restamped}{newline}{tail}" + + def _count_frames(chunk: Any, carry: Any) -> Tuple[int, Any]: """Count completed SSE events in ``chunk``, tolerating split delimiters. diff --git a/tests/unittest/llmapi/apps/test_responses_stream_final_consistency.py b/tests/unittest/llmapi/apps/test_responses_stream_final_consistency.py new file mode 100644 index 000000000000..ac84c03dcc5c --- /dev/null +++ b/tests/unittest/llmapi/apps/test_responses_stream_final_consistency.py @@ -0,0 +1,394 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Offline tests: a streamed Responses turn and its final snapshot must agree. + +`response.completed` rebuilds `output` from the accumulated text in a second, +independent pass, and streaming events are numbered by whoever built them. +Three ways the two views of one generation disagreed, each measured on live +traffic before it was tested here: + +* The rebuild minted fresh ids for the reasoning and message items the stream + had already announced - 216 of 221 responses - so a client joining streamed + items with the snapshot by id saw phantom items. +* With postprocessing workers enabled a response's frames come from two + processes, each numbering from zero, so every streamed response carried + sequence numbers 0,1,0,1,2,... - 221 of 221 measured. +* A stream that ended inside an unterminated tool call streamed nothing for + it while the final rebuild kept the fragment as message text (trace + tr_8f312e9973954fa784ae7dc8f45e9d3e, 429 characters), so the snapshot held + text the stream never showed. +""" + +import json +import pickle +from types import SimpleNamespace +from unittest.mock import patch + +import pytest +from openai.types.responses.tool import FunctionTool + +from tensorrt_llm.serve.openai_protocol import ResponsesRequest +from tensorrt_llm.serve.responses_utils import ( + ResponsesStreamingProcessor, + _create_output_content, + stamp_sse_sequence_number, +) + +# The CPU-* CI stages run pytest with -m 'cpu_only'. Without this marker every +# test in the file is deselected, which pytest reports as exit code 5 and the +# stage reports as a failure. +pytestmark = pytest.mark.cpu_only + + +def _event_type(frame): + return frame.split("\n", 1)[0][len("event: ") :] + + +def _event_data(frame): + return json.loads(frame.split("data: ", 1)[1]) + + +def _exec_tool(): + return FunctionTool( + name="exec", + type="function", + strict=False, + parameters={"type": "object", "properties": {"input": {"type": "string"}}}, + ) + + +def _processor(tools=None, reasoning_parser="glm", tool_parser=None): + """A processor built exactly as the server builds one.""" + request = ResponsesRequest(model="test-model", input="hi", stream=True, tools=list(tools or [])) + return ResponsesStreamingProcessor( + request=request, + sampling_params=request.to_sampling_params(), + model_name="test-model", + use_harmony=False, + reasoning_parser=reasoning_parser, + tool_parser=tool_parser, + ) + + +def _generation_chunk(accumulated, diff, done): + """One GenerationResult as the streaming loop sees it. + + The final chunk doubles as the finished result the completed-response + builder reads, so it carries finish_reason, token ids and prompt tokens. + """ + output = SimpleNamespace( + index=0, + text=accumulated, + text_diff=diff, + finish_reason="stop" if done else None, + token_ids=[1, 2, 3], + disaggregated_params=None, + ) + return SimpleNamespace(outputs=[output], _done=done, prompt_token_ids=[1, 2], cached_tokens=0) + + +def _stream(processor, chunks): + """Drive the real processor over generation chunks. + + Returns the SSE frames it produced and the finished generation result, + which is what the server hands to get_final_response_non_store. + """ + frames, accumulated, result = [], "", None + for i, chunk in enumerate(chunks): + accumulated += chunk + result = _generation_chunk(accumulated, chunk, i == len(chunks) - 1) + frames.extend(processor.process_single_output(result)) + return frames, result + + +def _streamed_items(frames, event_type="response.output_item.added"): + return [ + (data["item"]["type"], data["item"]["id"]) + for data in map(_event_data, frames) + if data.get("type") == event_type + ] + + +def _final_output(processor, result): + frame = processor.get_final_response_non_store(result) + assert _event_type(frame) == "response.completed" + return _event_data(frame)["response"]["output"] + + +# --------------------------------------------------------------------------- +# Item ids: the snapshot must answer to the ids the stream announced +# --------------------------------------------------------------------------- + + +def test_final_item_ids_match_the_streamed_item_ids(): + """The invariant, driven end to end through the real processor. + + Same shape and cure as the tool-call ids one commit earlier: the stream + records what it announced in `output_item.added`, the rebuild reuses it. + A client that keyed items off the streamed ids used to match nothing in + the snapshot - 216 of 221 measured responses. + """ + processor = _processor() + frames, result = _stream(processor, ["Check the reference.", "", "The answer is 42."]) + + added = _streamed_items(frames) + done = _streamed_items(frames, "response.output_item.done") + assert added == done, "an item must close under the id it opened with" + assert [item_type for item_type, _ in added] == ["reasoning", "message"] + assert all(item_id for _, item_id in added) + + final = [(item["type"], item["id"]) for item in _final_output(processor, result)] + assert final == added + + +def test_streamed_ids_are_reused_positionally_in_emission_order(): + """Two items of one type map onto the rebuild first-emitted first. + + Emission order is the only correspondence the two passes share, which is + also how the tool-call precedent matches its calls. + """ + items, _messages, _reasoning = _create_output_content( + SimpleNamespace( + outputs=[ + SimpleNamespace(index=0, text="first answer"), + SimpleNamespace(index=1, text="second answer"), + ] + ), + reasoning_parser=None, + streamed_item_ids=[("message", "msg_first"), ("message", "msg_second")], + ) + assert [(i.type, i.id) for i in items] == [ + ("message", "msg_first"), + ("message", "msg_second"), + ] + + +def test_reasoning_and_message_ids_are_reused_per_type(): + items, _messages, _reasoning = _create_output_content( + SimpleNamespace(outputs=[SimpleNamespace(index=0, text="thinkanswer")]), + reasoning_parser="glm", + streamed_item_ids=[("reasoning", "rs_r1"), ("message", "msg_m1")], + ) + assert [(i.type, i.id) for i in items] == [ + ("reasoning", "rs_r1"), + ("message", "msg_m1"), + ] + + +def test_a_non_streaming_rebuild_still_mints_fresh_ids_silently(): + """None means no stream ran; fresh uuids are correct and nothing warns.""" + with patch("tensorrt_llm.serve.responses_utils.logger") as mock_logger: + items, _messages, _reasoning = _create_output_content( + SimpleNamespace(outputs=[SimpleNamespace(index=0, text="thinkanswer")]), + reasoning_parser="glm", + streamed_item_ids=None, + ) + assert items[0].id.startswith("rs_") + assert items[1].id.startswith("msg_") + assert not mock_logger.warning.called + + +def test_rebuilding_items_the_stream_never_opened_is_said_out_loud(): + """Overflow means the two views already diverged structurally. + + A fresh id cannot hide that and must not try to - that divergence is the + unterminated-call territory below, and papering over it silently would + bury the one signal that says the views differ. + """ + with patch("tensorrt_llm.serve.responses_utils.logger") as mock_logger: + items, _messages, _reasoning = _create_output_content( + SimpleNamespace(outputs=[SimpleNamespace(index=0, text="answer")]), + reasoning_parser=None, + streamed_item_ids=[], + ) + assert items[0].id.startswith("msg_") + assert mock_logger.warning.called + assert "message" in mock_logger.warning.call_args[0] + + +def test_streamed_items_the_rebuild_drops_are_said_out_loud_too(): + """The mismatch matters in both directions; the first id is still reused.""" + with patch("tensorrt_llm.serve.responses_utils.logger") as mock_logger: + items, _messages, _reasoning = _create_output_content( + SimpleNamespace(outputs=[SimpleNamespace(index=0, text="answer")]), + reasoning_parser=None, + streamed_item_ids=[("message", "msg_first"), ("message", "msg_second")], + ) + assert [i.id for i in items] == ["msg_first"] + assert mock_logger.warning.called + + +# --------------------------------------------------------------------------- +# Sequence numbers: one monotonic counter per response, stamped at the egress +# --------------------------------------------------------------------------- + + +def test_egress_restamp_unifies_the_two_producers_counters(): + """The postproc-worker configuration, reproduced with a real pickle. + + The worker builds the per-token frames from a pickled copy of the + streaming processor - the same trip PostprocParams takes - whose counter + starts at zero, while the frontend's own counter numbered the two opening + events. The raw frames therefore restart mid-stream, and the egress + restamp is what turns them into one monotonic sequence. + """ + frontend = _processor() + worker = pickle.loads(pickle.dumps(frontend)) + + frames = frontend.get_initial_responses() + body, result = _stream(worker, ["Plan.", "", "Done."]) + frames += body + frames.append(worker.get_final_response_non_store(result)) + + produced = [_event_data(frame)["sequence_number"] for frame in frames] + # The defect stated raw: two counters, one restart inside one response. + assert produced[:3] == [0, 1, 0] + + stamped = [stamp_sse_sequence_number(frame, i) for i, frame in enumerate(frames)] + assert [_event_data(f)["sequence_number"] for f in stamped] == list(range(len(frames))) + # The restamp changes the number and nothing else. + for before, after in zip(frames, stamped): + assert _event_type(before) == _event_type(after) + b, a = _event_data(before), _event_data(after) + b.pop("sequence_number"), a.pop("sequence_number") + assert b == a + + +def test_in_process_numbering_already_matches_the_egress_stamp(): + """With workers off one instance numbers everything; the stamp is a no-op. + + This is what keeps the restamp safe to apply unconditionally at the + egress: the already-correct configuration comes out byte-equivalent. + """ + processor = _processor() + frames = processor.get_initial_responses() + body, result = _stream(processor, ["Plan.", "", "Done."]) + frames += body + frames.append(processor.get_final_response_non_store(result)) + + numbers = [_event_data(frame)["sequence_number"] for frame in frames] + assert numbers == list(range(len(frames))) + stamped = [stamp_sse_sequence_number(frame, i) for i, frame in enumerate(frames)] + assert [_event_data(f)["sequence_number"] for f in stamped] == numbers + + +def test_the_stamp_rewrites_the_field_and_not_the_model_text(): + """The payload can quote this very protocol; only the real field changes. + + A substring replace would have rewritten the model's own output here, + which is why the stamp parses the frame instead. + """ + inner = 'code: {"sequence_number": 999}' + frame = ( + "event: response.completed\ndata: " + + json.dumps( + { + "type": "response.completed", + "sequence_number": 1, + "response": {"output_text": inner}, + }, + separators=(",", ":"), + ) + + "\n\n" + ) + out = stamp_sse_sequence_number(frame, 41) + payload = _event_data(out) + assert payload["sequence_number"] == 41 + assert payload["response"]["output_text"] == inner + assert out.endswith("\n\n") and out.count("\n\n") == 1 + + +def test_frames_without_a_sequence_number_pass_through_untouched(): + for frame in ( + "event: ping\n\n", + "event: x\ndata: {}\n\n", + "event: x\ndata: not-json\n\n", + ): + assert stamp_sse_sequence_number(frame, 9) == frame + + +# --------------------------------------------------------------------------- +# An unterminated tool call: both views carry the same fallback text +# --------------------------------------------------------------------------- + +# The shape of the real 429-character fragment from trace +# tr_8f312e9973954fa784ae7dc8f45e9d3e, shortened: a GLM call whose value never +# closes, so no parser - incremental or whole-text - can read it as a call. +_UNTERMINATED = ( + "execinput" + "const cmd = await tools.exec_command({cmd: `cat metrics.json`" +) + + +def _message_texts(frames): + return [ + data["item"]["content"][0]["text"] + for data in map(_event_data, frames) + if data.get("type") == "response.output_item.done" and data["item"]["type"] == "message" + ] + + +def test_an_unterminated_call_reaches_both_views_as_the_same_text(): + """The stream must show the fallback text the final response keeps. + + Before the flush released it, the streamed view had no trace of the + fragment while the snapshot carried it as message text - the one turn + shape where the snapshot held text the stream never showed. Both views + now publish the same characters; the stream splits them across two + message items because its first message closed when the call was + announced, chunks before anyone could know the call would never finish. + """ + processor = _processor(tools=[_exec_tool()], reasoning_parser="glm47", tool_parser="glm47") + frames, result = _stream(processor, ["Plan it.", "Run it.", _UNTERMINATED]) + + streamed_texts = _message_texts(frames) + assert streamed_texts == ["Run it.", _UNTERMINATED] + # No half-built call on the stream: the fragment is text, not a call. + assert all(item_type != "function_call" for item_type, _ in _streamed_items(frames)) + + final = _final_output(processor, result) + final_messages = [item for item in final if item["type"] == "message"] + assert [item["type"] for item in final] == ["reasoning", "message"] + # The same characters, in the same order, on both views. The final pass + # re-parses the whole text so its fallback is a single message; equality + # is on the concatenated text, which is what a training-data consumer + # reads. + assert "".join(streamed_texts) == final_messages[0]["content"][0]["text"] + assert _UNTERMINATED in final_messages[0]["content"][0]["text"] + + +def test_the_unterminated_fragment_keeps_the_streamed_ids_it_can(): + """Id linkage under the structural divergence the fragment causes. + + The stream published two message items, the rebuild derives one: the one + it derives answers to the first streamed id, and the mismatch is warned + about rather than hidden - it is the visible residue of a call the model + never finished. + """ + processor = _processor(tools=[_exec_tool()], reasoning_parser="glm47", tool_parser="glm47") + frames, result = _stream(processor, ["Plan it.", "Run it.", _UNTERMINATED]) + streamed = _streamed_items(frames) + assert [item_type for item_type, _ in streamed] == [ + "reasoning", + "message", + "message", + ] + + with patch("tensorrt_llm.serve.responses_utils.logger") as mock_logger: + final = _final_output(processor, result) + + assert (final[0]["type"], final[0]["id"]) == streamed[0] + assert (final[1]["type"], final[1]["id"]) == streamed[1] + assert mock_logger.warning.called diff --git a/tests/unittest/llmapi/apps/test_responses_streaming_tool_calls.py b/tests/unittest/llmapi/apps/test_responses_streaming_tool_calls.py index 39976e3606ef..ef7b9771b2b0 100644 --- a/tests/unittest/llmapi/apps/test_responses_streaming_tool_calls.py +++ b/tests/unittest/llmapi/apps/test_responses_streaming_tool_calls.py @@ -47,7 +47,11 @@ _REASONING_PARSER = "glm47" _TOOL_PARSER = "glm47" -# Markup that must never reach a client as assistant text. +# Markup that must never reach a client as assistant text - with one +# sanctioned exception: a stream that ends inside an unterminated call +# releases the raw markup as message text, because the final response's +# whole-text re-parse keeps it as message text too and the two views must +# agree (see _flush_tool_parser and TestEndOfStream below). _MARKUP = ("", "", "", "") # One real GLM-4.7 response, verbatim, from a recorded agent run: eight stream @@ -252,16 +256,40 @@ def test_the_prose_is_one_item_not_two(self): assert len(_items(events, "response.output_item.done", "message")) == 1 def test_no_prefix_of_the_response_leaks_markup(self): - """Feeding frames 0..k for every k must never publish markup. + """Feeding frames 0..k for every k must never *leak* markup. The failing window was narrow - it opened once one call had closed while another was still open - so the whole prefix family is replayed rather than just the end state. + + One shape is a release, not a leak: a prefix that finishes inside an + unterminated call flushes the raw remainder as its trailing message + item, because the final response's whole-text re-parse keeps that + text too and the two views of one generation must agree (see + _flush_tool_parser). That remainder is pinned exactly - the bytes + from the last unclosed `` to the end - and everything + else still has to be clean; an unfinished stream (finish=False) + releases nothing at all. """ + + def pending_markup(accumulated): + start = accumulated.rfind("") + if start == -1 or "" in accumulated[start:]: + return None + return accumulated[start:] + for k in range(1, len(_REAL_FRAMES) + 1): for finish in (False, True): events = _drive(_REAL_FRAMES[:k], finish=finish) - _assert_no_markup(events) + released = pending_markup("".join(_REAL_FRAMES[:k])) if finish else None + for payload in _done_payloads(events): + if released is not None and payload == released: + continue + for marker in _MARKUP: + assert marker not in payload, ( + f"k={k} finish={finish}: {marker!r} reached the " + f"client as assistant text: {payload!r}" + ) def test_done_payload_is_exactly_the_deltas_that_were_streamed(self): """The invariant, stated directly, for every prefix. @@ -457,21 +485,35 @@ def test_whitespace_only_delta_between_calls_opens_a_message_item(self): class TestEndOfStream: """Edge cases 6 and 7: streams that stop before the model was done.""" - def test_stream_cut_off_mid_call_drops_the_markup(self): + def test_stream_cut_off_mid_call_releases_the_markup_as_text(self): """Edge case 6: the parser is still holding an unterminated call. - The old re-parse reported those bytes as the assistant's message, - which is the leak this change removes; dropping them is the deliberate - alternative. The text already streamed is unaffected. + Dropping those bytes was the first design, but the final response + never dropped them: its whole-text re-parse needs the closing tag, + fails to read the fragment as a call, and falls back to publishing it + as message text - so the snapshot carried a 429-character message the + stream never showed (trace tr_8f312e9973954fa784ae7dc8f45e9d3e). The + views have to agree, and of the two ways to agree, keeping the text + wins: these turns are recorded as training data, where silent loss is + worse than a call that visibly never closed. So the flush releases + the raw buffer as a trailing message item. The text already streamed + is unaffected, and the item carrying the fragment is separate because + the message item closed when the call was announced, chunks earlier. """ + raw = "".join(_REAL_FRAMES[:3]) + unterminated = raw[raw.index("") :] + with patch("tensorrt_llm.serve.responses_utils.logger") as mock_logger: events = _drive(_REAL_FRAMES[:3]) - _assert_no_markup(events) - assert _payloads(events, "response.output_text.done") == [_REAL_TEXT] - # The drop has to be visible in the log, not silent. - assert mock_logger.warning.called - assert "tool call" in mock_logger.warning.call_args[0][0] + assert _payloads(events, "response.output_text.done") == [ + _REAL_TEXT, + unterminated, + ] + # The release has to be visible in the log, not silent: the model + # emitted a call it never finished, and someone reading the recorded + # turn should find that said somewhere. + assert any("tool call" in call.args[0] for call in mock_logger.warning.call_args_list) def test_stream_cut_off_mid_call_reports_no_half_built_call(self): """A call whose arguments never closed is not a call a client can run. From 76ae88e446fa723e3ceaec92452ce498c66c1bb7 Mon Sep 17 00:00:00 2001 From: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> Date: Wed, 30 Sep 2026 15:00:55 +0000 Subject: [PATCH 21/43] [None][fix] responses: honor what the request asked and what the client sent Five audit findings (A08, A02-addendum, A07-addendum, A09), each with a production fixture or a controlled reproduction: - The KF client serializes sub-agent task payloads as a content part typed encrypted_content whose same-named field holds plain readable text; the converter dropped it, so a sub-agent could receive a task with no payload (a real 721-char task loss). Readable string bodies now convert to text; genuinely opaque values stay dropped rather than guessed at. - An explicit {"type": "message", "content": "hello"} converted to empty content (the typed branch walked the string as parts); the same shape without "type" was fine. Both now accept plain-string content. - Tool-call identity is carried by call entity, not filtered-list position: a call dropped for invalid JSON now retires its own streamed id instead of donating it to the next call in line. - tool_choice="none" is enforced losslessly on the non-Harmony path: the tool parser is bypassed on both the streaming and final views, so the model's markup stays in the visible text verbatim. required/named remain unenforced (sampling-level work) and still warn once. - A max-token-truncated stream now terminates with response.incomplete and incomplete_details instead of response.completed wrapped around an incomplete status. Also repairs test_responses_input_preprocess.py's namespaced-tools test, which had referenced a helper removed by the tool-resolution refactor and failed on import ever since - nothing in CI ran the file. Signed-off-by: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> --- tensorrt_llm/serve/openai_server.py | 25 +- tensorrt_llm/serve/responses_utils.py | 312 ++++++++++++++---- .../apps/test_responses_input_preprocess.py | 94 +++++- ...test_responses_stream_final_consistency.py | 197 ++++++++++- 4 files changed, 545 insertions(+), 83 deletions(-) diff --git a/tensorrt_llm/serve/openai_server.py b/tensorrt_llm/serve/openai_server.py index 53b8cbbd7c39..5bb9e22479ee 100644 --- a/tensorrt_llm/serve/openai_server.py +++ b/tensorrt_llm/serve/openai_server.py @@ -3135,11 +3135,15 @@ async def openai_responses(self, request: ResponsesRequest, """Serve one /v1/responses request, streaming or not. ``parallel_tool_calls`` and, on the non-Harmony path, ``tool_choice`` - are accepted for schema compatibility but not enforced: honouring - them would mean constraining or truncating what the model generated, - and these turns are recorded as training data, where a falsified - output is worse than an unconstrained one. A once-per-process warning - below says so instead of silently swallowing the option. + values other than "auto" and "none" are accepted for schema + compatibility but not enforced: honouring them would mean constraining + or truncating what the model generated, and these turns are recorded + as training data, where a falsified output is worse than an + unconstrained one. A once-per-process warning below says so instead + of silently swallowing the option. ``tool_choice="none"`` is the + exception because it can be honoured losslessly - the tool parser is + bypassed and any call markup stays in the visible text verbatim (see + responses_utils._effective_tool_parser). """ async def create_response( @@ -3241,11 +3245,14 @@ async def create_streaming_generator(promise: RequestOutput, "but not enforced; the model may still emit several tool " "calls in one turn.", key="responses_parallel_tool_calls_unenforced") - if not self.use_harmony and request.tool_choice != "auto": + # "none" is excluded: it is honoured by bypassing the tool parser + # (responses_utils._effective_tool_parser), which loses nothing. + if not self.use_harmony and request.tool_choice not in ("auto", + "none"): logger.warning_once( - "Responses API: 'tool_choice' is accepted but not " - "enforced on this model path; generation is not " - "constrained by it.", + "Responses API: 'tool_choice' values other than 'auto' " + "and 'none' are accepted but not enforced on this model " + "path; generation is not constrained by them.", key="responses_tool_choice_unenforced") # Get prev response diff --git a/tensorrt_llm/serve/responses_utils.py b/tensorrt_llm/serve/responses_utils.py index 23da0ca10f70..2fdf34626c3a 100644 --- a/tensorrt_llm/serve/responses_utils.py +++ b/tensorrt_llm/serve/responses_utils.py @@ -21,6 +21,7 @@ ResponseCustomToolCall, ResponseErrorEvent, ResponseFailedEvent, ResponseFunctionToolCall, + ResponseIncompleteEvent, ResponseInProgressEvent, ResponseOutputItem, ResponseOutputItemAddedEvent, ResponseOutputItemDoneEvent, @@ -730,10 +731,26 @@ def _item_text(item: dict) -> str: if isinstance(content, str): return content if isinstance(content, list): - parts = [ - part.get("text") for part in content - if isinstance(part, dict) and part.get("text") - ] + parts = [] + for part in content: + if not isinstance(part, dict): + continue + text = part.get("text") + if not text and part.get("type") == "encrypted_content": + # The KF client serializes a sub-agent task payload inside an + # agent_message as a part typed `encrypted_content` whose + # same-named field holds the plain readable task text - the + # field name is historical, not a description of the value. + # Dropping the part delivered the task header ("Payload:") + # with no payload behind it; one measured request lost a + # 721-character task this way. Only a string is taken at face + # value: a non-string here really is opaque, and guessing at + # it would fabricate input, so those stay dropped as before. + value = part.get("encrypted_content") + if isinstance(value, str): + text = value + if text: + parts.append(text) return "\n".join(parts) return item.get("text") or "" @@ -868,15 +885,26 @@ def _response_output_item_to_chat_completion_message( raise ValueError( f"Input item of type {item_type!r} has empty or missing 'content'" ) - # Join every text part. Taking content[0] silently dropped the rest - # of a multi-part message. - parts = [] - for part in content: - text = part.get("text") if isinstance(part, dict) else getattr( - part, "text", None) - if text: - parts.append(text) - text = "".join(parts) + if isinstance(content, str): + # The API accepts `content` as one plain string, and the + # untyped branch above already preserves that shape - `type` + # merely defaults to "message" there. Walking the string as a + # list of parts iterates its characters, none of which carry a + # `text` field, so an explicit {"type": "message", "content": + # "hello"} converted to an empty message while the identical + # item without `type` survived. The two spellings mean the + # same thing and must convert the same way. + text = content + else: + # Join every text part. Taking content[0] silently dropped the + # rest of a multi-part message. + parts = [] + for part in content: + text = part.get("text") if isinstance( + part, dict) else getattr(part, "text", None) + if text: + parts.append(text) + text = "".join(parts) if item_type == "reasoning": # Reasoning is always the assistant's. return {"role": "assistant", "reasoning": text} @@ -1555,6 +1583,26 @@ def _flush_tool_parser( return "".join(released), calls, unfinished +def _effective_tool_parser(tool_parser_id: Optional[str], + request) -> Optional[str]: + """The tool parser this request may run, honouring ``tool_choice="none"``. + + "none" is the one tool_choice value that can be enforced without + falsifying anything the model generated: not parsing leaves any tool-call + markup in the visible text verbatim, so nothing is truncated or invented - + where "required" or a named function would need sampling-level + constraints, which this path does not apply. Both the streaming event + generator and the final rebuild resolve their parser through here, because + the two views of one generation must agree on whether a call exists. + + Read with getattr for the same reason reasoning_chat_template_kwargs + does: unit tests drive the event generator with stand-in request objects. + """ + if getattr(request, "tool_choice", None) == "none": + return None + return tool_parser_id + + def _apply_tool_parser( tool_parser_id: Optional[str], tools: Optional[list[Tool]], @@ -1595,7 +1643,7 @@ def _create_output_content( tool_parser: Optional[str] = None, tools: Optional[list[Tool]] = None, chat_template_kwargs: Optional[dict[str, Any]] = None, - streamed_tool_call_ids: Optional[list[Tuple[str, str]]] = None, + streamed_tool_call_ids: Optional[list[Optional[Tuple[str, str]]]] = None, streamed_item_ids: Optional[list[Tuple[str, str]]] = None, ) -> Tuple[list[ResponseOutputItem], list[ChatCompletionMessageParam], list[str]]: @@ -1706,22 +1754,46 @@ def _streamed_or_fresh_id(item_type: str) -> str: if calls: tool_resolution = _tool_resolution(tools) - # Reuse the ids the streaming path already published, positionally: - # both passes enumerate the same parsed calls in the same order. - # Running out means this pass found calls the stream never emitted - # (a cut-off stream, say); those get fresh ids, and the mismatch is - # said out loud rather than silently producing ids the client has - # no way to match. + # Reuse the ids the streaming path already published. The stream's + # record holds one entry per call the parser announced, in + # announcement order - None where assembly dropped the call for + # invalid-JSON arguments - and this pass enumerates the same + # markup in the same order, so when the counts agree each call is + # paired with its own entity's entry. Pairing over the *filtered* + # list instead is how a dropped call A used to donate its position: + # the whole-text parse still reports A (with empty arguments), A + # consumed the id the stream had published for B, and B's identity + # in the snapshot matched nothing the client had seen. reusable = list(streamed_tool_call_ids or []) - if reusable and len(reusable) != len(calls): + entity_aligned = len(reusable) == len(calls) + if reusable and not entity_aligned: + # Counts disagreeing means the two passes did not see the same + # calls (a cut-off stream, say), so per-entity pairing is + # unknowable. Say so, then fall back to reusing the ids that + # actually reached the wire, in order; placeholders name no + # wire identity and are worth nothing to a client. logger.warning( - "final response rebuilt %d tool call(s) but %d were " - "streamed; ids beyond the streamed ones are new and will " + "final response rebuilt %d tool call(s) but the stream " + "saw %d; ids beyond the streamed ones are new and will " "not match the client's tool outputs", len(calls), len(reusable)) + reusable = [ids for ids in reusable if ids is not None] tool_calls_item = [] for index, call in enumerate(calls): ids = reusable[index] if index < len(reusable) else (None, None) + if entity_aligned and ids is None: + # This entity is the call the stream dropped: its + # arguments never assembled into valid JSON and no id was + # published for it. The snapshot must describe the stream, + # so it is not delivered here either - the placeholder is + # consumed by the drop instead of shifting onto the next + # call. + logger.warning( + "final response drops the tool call to %r: the stream " + "assembled its arguments into invalid JSON and never " + "delivered it, so the snapshot does not deliver it " + "either", call.name) + continue tool_calls_item.append( _tool_call_output_item(call, tool_resolution, @@ -2044,7 +2116,7 @@ def _create_response( reasoning_parser: Optional[str] = None, tool_parser: Optional[str] = None, num_prompt_tokens: Optional[int] = None, - streamed_tool_call_ids: Optional[list[Tuple[str, str]]] = None, + streamed_tool_call_ids: Optional[list[Optional[Tuple[str, str]]]] = None, streamed_item_ids: Optional[list[Tuple[str, str]]] = None, tokenizer: Optional[TokenizerBase] = None, ) -> tuple[ResponsesResponse, list[Message | ChatCompletionMessageParam]]: @@ -2067,7 +2139,9 @@ def _create_response( output_content, output_messages, reasoning_texts = _create_output_content( final_res, reasoning_parser, - tool_parser, + # tool_choice="none" bypasses the parser here exactly as the + # streaming events do, so both views keep the markup as text. + _effective_tool_parser(tool_parser, request), request.tools, chat_template_kwargs=reasoning_chat_template_kwargs(request), streamed_tool_call_ids=streamed_tool_call_ids, @@ -2120,7 +2194,7 @@ async def create_response( reasoning_parser: Optional[str] = None, tool_parser: Optional[str] = None, num_prompt_tokens: Optional[int] = None, - streamed_tool_call_ids: Optional[list[Tuple[str, str]]] = None, + streamed_tool_call_ids: Optional[list[Optional[Tuple[str, str]]]] = None, streamed_item_ids: Optional[list[Tuple[str, str]]] = None, tokenizer: Optional[TokenizerBase] = None, ) -> ResponsesResponse: @@ -2172,7 +2246,7 @@ def create_response_non_store( reasoning_parser: Optional[str] = None, tool_parser: Optional[str] = None, num_prompt_tokens: Optional[int] = None, - streamed_tool_call_ids: Optional[list[Tuple[str, str]]] = None, + streamed_tool_call_ids: Optional[list[Optional[Tuple[str, str]]]] = None, streamed_item_ids: Optional[list[Tuple[str, str]]] = None, tokenizer: Optional[TokenizerBase] = None, ) -> ResponsesResponse: @@ -2234,8 +2308,15 @@ def __init__(self) -> None: # echo the streamed id, match nothing. Measured before the fix: 671 of # 671 calls across three campaigns. # + # One entry per call the parser announced, in announcement order: None + # holds the place of a call that was assembled but dropped for + # invalid-JSON arguments. The rebuild enumerates the same markup in + # the same order, so the placeholder is what lets a drop consume its + # own position instead of shifting the streamed identities onto the + # calls behind it. + # # A list here rather than a class attribute, for the reason above. - self.emitted_tool_call_ids: list[Tuple[str, str]] = [] + self.emitted_tool_call_ids: list[Optional[Tuple[str, str]]] = [] # (item type, item id) of every reasoning and message item this # stream opened, in the order their `output_item.added` went out. @@ -2276,8 +2357,12 @@ def emitted_tool_calls(self, count: int) -> None: self.state_tracker.emitted_tool_calls = count @property - def emitted_tool_call_ids(self) -> list[Tuple[str, str]]: - """(item id, call id) of the tool calls already streamed, in order.""" + def emitted_tool_call_ids(self) -> list[Optional[Tuple[str, str]]]: + """(item id, call id) of the tool calls already streamed, in order. + + None entries hold the place of calls the stream assembled and then + dropped for invalid-JSON arguments; see the state tracker. + """ return self.state_tracker.emitted_tool_call_ids @property @@ -2294,6 +2379,18 @@ def record_emitted_tool_call(self, item) -> None: """ self.state_tracker.emitted_tool_call_ids.append((item.id, item.call_id)) + def record_dropped_tool_call(self) -> None: + """Hold a dropped call's place in the emission record. + + A call whose assembled arguments were invalid JSON is never streamed, + but the final rebuild's whole-text parse still reports the entity (the + markup is well-formed; only the arguments are unusable). Recording the + drop keeps the record aligned with the parser's announcement order, so + in the rebuild the drop consumes its own position instead of handing + the next call an identity the stream published for a different one. + """ + self.state_tracker.emitted_tool_call_ids.append(None) + def append_text(self, delta: str) -> None: self.state_tracker.text_buffer += delta @@ -2534,11 +2631,18 @@ def _accumulate_tool_call_fragments(fragments: dict[int, dict[str, Any]], Keyed by the parser's own `tool_index`, which numbers the calls in the order it saw them start; a dict preserves that order. + + The call id is minted when the parser first announces the call - before + anyone can know whether its arguments will assemble into something + deliverable - so the identity belongs to the call entity itself. A call + that is later dropped dies holding its own id instead of donating it to + the next call in line, which is the positional swap this prevents. """ for call in calls: fragment = fragments.setdefault(call.tool_index, { "name": None, "parameters": "", + "call_id": f"call_{_random_uuid()}", }) # Only a call's first fragment carries the name; every later one # carries None, which must not erase the name already recorded. @@ -2610,12 +2714,18 @@ def _assembled_tool_calls( try: json.loads(arguments) except ValueError as exc: + # The drop consumes the call's own id (minted when the parser + # announced it) and the log says which, so the retired identity + # is auditable and can never be mistaken for the next call's. + consumed = fragment.get("call_id") + consumed_note = (f" The drop consumes its call id {consumed!r}, " + "which is retired with it." if consumed else "") logger.warning( f"Dropping the tool call to {fragment['name']!r}: its " f"arguments did not assemble into valid JSON ({exc}). The " "model emitted a malformed tool call, and reporting it would " "hand the client arguments it can only reject. Assembled: " - f"{arguments[:200]!r}") + f"{arguments[:200]!r}.{consumed_note}") continue calls.append( ToolCallItem(tool_index=tool_index, @@ -2677,6 +2787,11 @@ def _generate_streaming_event( tool_parser_dict: Optional[dict[int, BaseToolParser]] = None, ): available_tools = _get_chat_completion_function_tools(request.tools) + # tool_choice="none" is honoured by not parsing at all, so the model's + # markup stays in the visible text verbatim. The final rebuild resolves + # its parser the same way (see _create_response); the two views of one + # generation must agree on whether a call exists. + tool_parser_id = _effective_tool_parser(tool_parser_id, request) output_idx = output.index delta_text = output.text_diff calls = [] @@ -2916,17 +3031,41 @@ def check_parser(parser_id: Optional[str], # Emitted after any open item has been closed, so a call item is never # nested inside a message item. The counter keeps emission idempotent: it # guarded against the old per-chunk re-parse re-reporting the same calls, - # and now guards against a finished output being presented twice. + # and now guards against a finished output being presented twice. It + # counts the parser's announced call entities, kept or dropped, because + # a drop has to be accounted exactly once too. if finished_generation: - assembled_calls = _assembled_tool_calls(call_fragments, - unfinished_tool_index) - pending = assembled_calls[streaming_events_helper.emitted_tool_calls:] + entities = list(call_fragments.items()) + pending = entities[streaming_events_helper.emitted_tool_calls:] if pending: + keep = { + call.tool_index: call + for call in _assembled_tool_calls(call_fragments, + unfinished_tool_index) + } tool_resolution = _tool_resolution(request.tools) - for call in pending: - tool_call_item = _tool_call_output_item(call, - tool_resolution, - status="completed") + # Walked in entity order, not kept-list order: a call dropped for + # invalid-JSON arguments consumes its own slot in the emission + # record, so the final rebuild pairs each surviving call with the + # identity the stream actually gave it rather than the one that + # belonged to the call in front of it. Nameless and unfinished + # entities take no slot - the rebuild's whole-text re-parse does + # not report those as calls either, so a placeholder for them + # would misalign the record. + for tool_index, fragment in pending: + call = keep.get(tool_index) + if call is None: + if (fragment.get("name") + and tool_index != unfinished_tool_index): + streaming_events_helper.record_dropped_tool_call() + continue + tool_call_item = _tool_call_output_item( + call, + tool_resolution, + status="completed", + # The id minted when the parser announced this call; see + # _accumulate_tool_call_fragments. + call_id=fragment.get("call_id")) streaming_events_helper.item_id = tool_call_item.id yield streaming_events_helper.get_output_item_added_event( tool_call_item) @@ -2934,7 +3073,7 @@ def check_parser(parser_id: Optional[str], tool_call_item) streaming_events_helper.record_emitted_tool_call(tool_call_item) streaming_events_helper.output_index_increment() - streaming_events_helper.emitted_tool_calls = len(assembled_calls) + streaming_events_helper.emitted_tool_calls = len(entities) streaming_events_helper.is_output_item_added_sent = False @@ -3054,6 +3193,44 @@ def _generate_streaming_event_harmony( parser.last_content_delta) +def _stream_terminal_event( + final_response: ResponsesResponse, + finish_reason: Optional[str], + sequence_number: int = -1, +) -> Union[ResponseCompletedEvent, ResponseIncompleteEvent]: + """The terminal event for a stream that ran to completion, by status. + + ``finish_reason_mapping`` marks a generation the engine cut off at its + token budget as status "incomplete", and the terminal event used to say + ``response.completed`` around it - an event whose name asserts the one + thing its payload denies, so a client believed the truncated answer was + the whole one. The Responses spec pairs the status with its own terminal + event, ``response.incomplete``, and explains it in + ``incomplete_details.reason``; "length" is the token budget, spelled + ``max_output_tokens``, and any other cause is left unstated rather than + guessed at. + + ``incomplete_details`` is set on the dump rather than on + ResponsesResponse, whose field is commented out - the same route + ``get_stream_failed_events`` takes for ``error``: the event re-validates + the dict against the SDK's Response, which does carry the field. + """ + payload = final_response.model_dump() + if final_response.status == "incomplete": + if finish_reason == "length": + payload["incomplete_details"] = {"reason": "max_output_tokens"} + return ResponseIncompleteEvent( + type="response.incomplete", + sequence_number=sequence_number, + response=payload, + ) + return ResponseCompletedEvent( + type="response.completed", + sequence_number=sequence_number, + response=payload, + ) + + class ResponsesStreamingProcessor: def __init__( @@ -3153,11 +3330,8 @@ async def get_final_response( ) return self._send_event( - ResponseCompletedEvent( - type="response.completed", - sequence_number=-1, - response=final_response.model_dump(), - )) + _stream_terminal_event(final_response, + final_res.outputs[0].finish_reason)) def get_final_response_non_store( self, @@ -3190,11 +3364,8 @@ def get_final_response_non_store( ) return self._send_event( - ResponseCompletedEvent( - type="response.completed", - sequence_number=-1, - response=final_response.model_dump(), - )) + _stream_terminal_event(final_response, + final_res.outputs[0].finish_reason)) def process_single_output(self, res: GenerationResult) -> list[str]: event_generator = None @@ -3292,12 +3463,17 @@ def get_stream_failed_events(self, STREAM_TERMINATION_UPSTREAM_ERROR = "upstream_error" STREAM_TERMINATION_INTERNAL_ERROR = "internal_error" -# The terminal event of a stream that ran to completion. Matched as a substring +# The terminal events of a stream that ran to completion - a truncated +# generation ends in `response.incomplete`, everything else in +# `response.completed` (see _stream_terminal_event). Matched as substrings # because a relayed stream arrives as transport-sized chunks that may carry # several events, not as one frame per event. Kept in both encodings so the # scan never has to decode a relayed chunk just to look at it. _COMPLETED_EVENT_MARKER = "event: response.completed" -_COMPLETED_EVENT_MARKER_BYTES = _COMPLETED_EVENT_MARKER.encode("utf-8") +_INCOMPLETE_EVENT_MARKER = "event: response.incomplete" +_TERMINAL_EVENT_MARKERS = (_COMPLETED_EVENT_MARKER, _INCOMPLETE_EVENT_MARKER) +_TERMINAL_EVENT_MARKERS_BYTES = tuple( + marker.encode("utf-8") for marker in _TERMINAL_EVENT_MARKERS) _SSE_EVENT_DELIMITER = "\n\n" _SSE_EVENT_DELIMITER_BYTES = _SSE_EVENT_DELIMITER.encode("utf-8") @@ -3419,11 +3595,12 @@ async def guard_responses_stream( Without this such a stream simply ends: the generator raises, the ASGI server abandons a half-written chunked body, and nothing in the bytes distinguishes that from a stream still in flight. Everything the turn - produced survives only as deltas, because ``response.completed`` is the one - event that repeats the full text. + produced survives only as deltas, because only the terminal event repeats + the full text. - Frames are forwarded untouched and nothing is added once - ``response.completed`` has gone out, so a stream that completes normally is + Frames are forwarded untouched and nothing is added once a terminal event + (``response.completed``, or ``response.incomplete`` for a truncated + generation) has gone out, so a stream that completes normally is byte-for-byte what it was. The two abnormal endings are handled differently on purpose: @@ -3445,9 +3622,9 @@ async def guard_responses_stream( if not completed: frames, carry = _count_frames(chunk, carry) events_sent += frames - completed = (_COMPLETED_EVENT_MARKER_BYTES in chunk - if isinstance(chunk, bytes) else - _COMPLETED_EVENT_MARKER in chunk) + markers = (_TERMINAL_EVENT_MARKERS_BYTES if isinstance( + chunk, bytes) else _TERMINAL_EVENT_MARKERS) + completed = any(marker in chunk for marker in markers) yield chunk except (asyncio.CancelledError, GeneratorExit) as exc: if on_termination is not None: @@ -3638,8 +3815,13 @@ async def responses_done_generator( Used when a request finishes without ever reaching a generation worker. The Responses protocol has no ``[DONE]`` sentinel - a client watches for - ``response.completed`` - so emitting the completions-style terminator here + a terminal event - so emitting the completions-style terminator here would leave a streaming client waiting for an event that never arrives. + + The terminal event is chosen by the response's status, like every other + completed stream (_stream_terminal_event): a replayed response that was + cut off at its token budget must end in ``response.incomplete``, not in a + ``response.completed`` that contradicts the status it carries. """ payload = response.model_dump() yield _sse_event( @@ -3654,12 +3836,12 @@ async def responses_done_generator( response=payload, sequence_number=1, )) + # finish_reason is carried on context-only responses, which is the only + # kind this generator replays; None simply leaves the reason unstated. yield _sse_event( - ResponseCompletedEvent( - type="response.completed", - response=payload, - sequence_number=2, - )) + _stream_terminal_event(response, + response.finish_reason, + sequence_number=2)) UCompletionResponseOrGenerator = Union[UCompletionResponse, diff --git a/tests/unittest/llmapi/apps/test_responses_input_preprocess.py b/tests/unittest/llmapi/apps/test_responses_input_preprocess.py index 3971118f0b55..d38ce607077c 100644 --- a/tests/unittest/llmapi/apps/test_responses_input_preprocess.py +++ b/tests/unittest/llmapi/apps/test_responses_input_preprocess.py @@ -104,6 +104,81 @@ def test_empty_content_is_rejected(): ) +def test_typed_and_untyped_string_content_convert_identically(): + """The auditor's positive control: `type` must not change the meaning. + + An item with a role and no `type` is EasyInputMessage, where `type` + *defaults* to "message" - so spelling it out is the same item. The typed + branch walked string content as if it were a list of parts, iterating its + characters and keeping none of them, so the explicit spelling converted + to an empty message while the implicit one survived. + """ + untyped = _response_output_item_to_chat_completion_message({"role": "user", "content": "hello"}) + typed = _response_output_item_to_chat_completion_message( + {"type": "message", "role": "user", "content": "hello"} + ) + assert typed["content"] == "hello" + assert (typed["role"], typed["content"]) == (untyped["role"], untyped["content"]) + + +def test_a_typed_reasoning_item_accepts_string_content_too(): + """Same shape, one branch over: the walk is shared with "message".""" + msg = _response_output_item_to_chat_completion_message( + {"type": "reasoning", "content": "thinking"} + ) + assert msg == {"role": "assistant", "reasoning": "thinking"} + + +def test_agent_message_keeps_the_encrypted_content_payload(): + """The KF sub-agent task contract: readable text under a misleading name. + + The client serializes a task payload as a content part typed + `encrypted_content` whose same-named field holds plain readable text - + the field name is historical. Dropping the part delivered the task header + with no payload behind it; one measured request lost a 721-character task + this way. The fixture is a real request's body.input[6], trimmed. + """ + msg = _response_output_item_to_chat_completion_message( + { + "type": "agent_message", + "id": "amsg_01a0", + "author": "/root", + "recipient": "/root/final3", + "content": [ + { + "type": "input_text", + "text": "Message Type: NEW_TASK\nTask name: /root/final3\nSender: /root\nPayload:\n", + }, + { + "type": "encrypted_content", + "encrypted_content": "Write solution.json and evaluate it with cudagym.", + }, + ], + } + ) + assert msg["role"] == "user" + assert "Message Type: NEW_TASK" in msg["content"] + assert msg["content"].endswith("Write solution.json and evaluate it with cudagym.") + + +def test_non_string_encrypted_content_stays_dropped(): + """Only a string is readable by contract; anything else is truly opaque. + + Guessing at a structured or binary value would fabricate input, so the + header survives and the opaque part is dropped, exactly as before. + """ + msg = _response_output_item_to_chat_completion_message( + { + "type": "agent_message", + "content": [ + {"type": "input_text", "text": "header"}, + {"type": "encrypted_content", "encrypted_content": {"blob": "aGk="}}, + ], + } + ) + assert msg == {"role": "user", "content": "header"} + + def test_function_call_output_keeps_call_id(): msg = _response_output_item_to_chat_completion_message( { @@ -566,14 +641,17 @@ def test_additional_tools_item_becomes_tools(): def test_hoisted_tools_are_offered_to_the_template_namespaced(): """The nested tools have to reach the prompt under qualified names. - `_namespaced_tool_names` maps a reply's call back to its namespace, so a - tool that never made it into `tools` would come back as an unsupported - call even if the model somehow guessed it. + `_tool_resolution` maps a reply's call back to its namespace (under both + the qualified and, when unambiguous, the bare spelling), so a tool that + never made it into `tools` would come back as an unsupported call even if + the model somehow guessed it. This test referenced the helper the + resolution map replaced (`_namespaced_tool_names`) and had been failing + on import since that refactor - nothing in CI ran this file. """ from tensorrt_llm.serve.openai_protocol import ResponsesRequest from tensorrt_llm.serve.responses_utils import ( _get_chat_completion_function_tools, - _namespaced_tool_names, + _tool_resolution, ) request = ResponsesRequest( @@ -607,9 +685,11 @@ def test_hoisted_tools_are_offered_to_the_template_namespaced(): offered = [t.function.name for t in _get_chat_completion_function_tools(request.tools)] assert offered == ["collaboration.spawn_agent"] - assert _namespaced_tool_names(request.tools) == { - "collaboration.spawn_agent": ("collaboration", "spawn_agent"), - } + # Both spellings resolve: the qualified one the template offered, and the + # bare one the model writes back anyway (measured 247-of-281 calls). + resolution = _tool_resolution(request.tools) + assert resolution["collaboration.spawn_agent"] == ("collaboration", "spawn_agent", False) + assert resolution["spawn_agent"] == ("collaboration", "spawn_agent", False) def test_tools_already_in_the_tools_field_are_kept(): diff --git a/tests/unittest/llmapi/apps/test_responses_stream_final_consistency.py b/tests/unittest/llmapi/apps/test_responses_stream_final_consistency.py index ac84c03dcc5c..6d0ace618019 100644 --- a/tests/unittest/llmapi/apps/test_responses_stream_final_consistency.py +++ b/tests/unittest/llmapi/apps/test_responses_stream_final_consistency.py @@ -45,6 +45,7 @@ _create_output_content, stamp_sse_sequence_number, ) +from tensorrt_llm.serve.tool_parser.core_types import StreamingParseResult, ToolCallItem # The CPU-* CI stages run pytest with -m 'cpu_only'. Without this marker every # test in the file is deselected, which pytest reports as exit code 5 and the @@ -69,9 +70,15 @@ def _exec_tool(): ) -def _processor(tools=None, reasoning_parser="glm", tool_parser=None): +def _processor(tools=None, reasoning_parser="glm", tool_parser=None, tool_choice="auto"): """A processor built exactly as the server builds one.""" - request = ResponsesRequest(model="test-model", input="hi", stream=True, tools=list(tools or [])) + request = ResponsesRequest( + model="test-model", + input="hi", + stream=True, + tools=list(tools or []), + tool_choice=tool_choice, + ) return ResponsesStreamingProcessor( request=request, sampling_params=request.to_sampling_params(), @@ -392,3 +399,189 @@ def test_the_unterminated_fragment_keeps_the_streamed_ids_it_can(): assert (final[0]["type"], final[0]["id"]) == streamed[0] assert (final[1]["type"], final[1]["id"]) == streamed[1] assert mock_logger.warning.called + + +# --------------------------------------------------------------------------- +# A dropped call must not donate its streamed identity to the next call +# --------------------------------------------------------------------------- + + +class _ScriptedToolParser: + """A parser whose verdicts this test controls. + + The identity plumbing under test is parser-agnostic; what matters is the + exact disagreement GLM-4.7 produces for an `` that never closes + (test_tool_parsers.py, "the non-streaming path is unchanged"): the + streaming pass assembles a first call whose arguments are invalid JSON - + which `_assembled_tool_calls` drops - and a second, valid one, while the + whole-text pass reports *both*, the first with empty arguments. Scripting + that shape pins the test to the disagreement itself rather than to one + parser's markup dialect. + """ + + def __init__(self): + self._buffer = "" + self._reported = False + self.current_tool_name_sent = False + + def parse_streaming_increment(self, text, tools): + if self._reported or not text: + return StreamingParseResult(normal_text=text, calls=[]) + self._reported = True + return StreamingParseResult( + normal_text="Run them. ", + calls=[ + ToolCallItem(tool_index=0, name="exec", parameters='{"cmd": }'), + ToolCallItem(tool_index=1, name="exec", parameters='{"cmd": "ls"}'), + ], + ) + + def finish(self, tools): + return StreamingParseResult() + + def has_tool_call(self, text): + return False + + def detect_and_parse(self, text, tools): + return StreamingParseResult( + normal_text="Run them. ", + calls=[ + ToolCallItem(tool_index=0, name="exec", parameters="{}"), + ToolCallItem(tool_index=1, name="exec", parameters='{"cmd": "ls"}'), + ], + ) + + +def _function_call_items(output): + return [item for item in output if item["type"] == "function_call"] + + +def test_a_dropped_call_does_not_donate_its_identity_to_the_next_one(): + """Two calls parsed, the first one's arguments invalid JSON. + + The stream delivers only the second call. The final rebuild's whole-text + parse still reports both - the first with empty arguments - and pairing + the streamed ids positionally over the *filtered* list handed the first + call the identity the stream had published for the second; the client's + tool output, keyed on that call_id, then answered a call with different + arguments. The record now holds one entry per announced call, a drop + consumes its own slot (and logs the id it retired), and the final response + both drops what the stream dropped and names the survivor what the stream + named it. + """ + processor = _processor(tools=[_exec_tool()], reasoning_parser=None, tool_parser="scripted") + with ( + patch( + "tensorrt_llm.serve.responses_utils.ToolParserFactory.create_tool_parser", + side_effect=lambda _parser_id: _ScriptedToolParser(), + ), + patch("tensorrt_llm.serve.responses_utils.logger") as mock_logger, + ): + frames, result = _stream(processor, ["Run them. "]) + streamed_calls = [ + data["item"] + for data in map(_event_data, frames) + if data.get("type") == "response.output_item.done" + and data["item"]["type"] == "function_call" + ] + # The stream refused the malformed call, and the drop named the id it + # consumed - which is the dropped call's own, not the survivor's. + assert [json.loads(c["arguments"]) for c in streamed_calls] == [{"cmd": "ls"}] + drop_warnings = [ + str(call.args[0]) + for call in mock_logger.warning.call_args_list + if "valid JSON" in str(call.args[0]) + ] + assert len(drop_warnings) == 1 + assert "call_" in drop_warnings[0] + assert streamed_calls[0]["call_id"] not in drop_warnings[0] + + final_calls = _function_call_items(_final_output(processor, result)) + + # The two views agree on membership, so no structural-divergence + # warning fires; the rebuild-side drop is said out loud instead. + warnings = [str(call.args[0]) for call in mock_logger.warning.call_args_list] + assert not any("the stream saw" in text for text in warnings) + assert any("drops the tool call" in text for text in warnings) + + # The final view matches the stream: one call, carrying the identity the + # stream published for *it* - not one shifted off the rejected call. + assert [(c["id"], c["call_id"]) for c in final_calls] == [ + (streamed_calls[0]["id"], streamed_calls[0]["call_id"]) + ] + assert json.loads(final_calls[0]["arguments"]) == {"cmd": "ls"} + + +# --------------------------------------------------------------------------- +# tool_choice="none": no call anywhere, the markup stays as text everywhere +# --------------------------------------------------------------------------- + +_WHOLE_CALL = "execinputls" + + +def test_tool_choice_none_keeps_the_markup_as_text_in_both_views(): + """tool_choice="none" is enforced by not parsing, which loses nothing. + + The model's markup stays in the visible text verbatim - stream and + snapshot alike, or the two views would disagree about whether a call + exists - and neither view reports a tool call item. The other tool_choice + values stay unenforced (honouring them needs sampling-level constraints); + "none" is the one value where enforcement is lossless. + """ + processor = _processor( + tools=[_exec_tool()], + reasoning_parser="glm47", + tool_parser="glm47", + tool_choice="none", + ) + frames, result = _stream(processor, ["Plan it.", "Calling: ", _WHOLE_CALL]) + + assert all(item_type != "function_call" for item_type, _ in _streamed_items(frames)) + streamed_text = "".join(_message_texts(frames)) + assert streamed_text == "Calling: " + _WHOLE_CALL + + final = _final_output(processor, result) + assert [item["type"] for item in final] == ["reasoning", "message"] + assert final[1]["content"][0]["text"] == streamed_text + + +# --------------------------------------------------------------------------- +# A truncated generation must end in a terminal that agrees with itself +# --------------------------------------------------------------------------- + + +def test_a_max_token_truncated_stream_ends_with_a_consistent_terminal(): + """Status, terminal event and incomplete_details must tell one story. + + finish_reason="length" maps to status "incomplete", and the terminal + event used to say `response.completed` around it - an event whose name + asserts the one thing its payload denies, so a client believed the + truncated answer was the whole one. The spec-correct pairing: + `response.incomplete`, status "incomplete", and + incomplete_details.reason naming the token budget. + """ + processor = _processor(reasoning_parser=None) + result = _generation_chunk("Partial answer", "Partial answer", True) + result.outputs[0].finish_reason = "length" + frames = processor.process_single_output(result) + assert all(_event_data(f).get("type") != "response.completed" for f in frames) + + frame = processor.get_final_response_non_store(result) + assert _event_type(frame) == "response.incomplete" + payload = _event_data(frame) + assert payload["type"] == "response.incomplete" + assert payload["response"]["status"] == "incomplete" + assert payload["response"]["incomplete_details"] == {"reason": "max_output_tokens"} + # The truncated text itself still ships in the snapshot. + messages = [i for i in payload["response"]["output"] if i["type"] == "message"] + assert messages and messages[0]["content"][0]["text"] == "Partial answer" + + +def test_a_completed_stream_still_ends_with_response_completed(): + """The happy path is untouched: stop means completed, and no details.""" + processor = _processor(reasoning_parser=None) + _frames, result = _stream(processor, ["Done."]) + + frame = processor.get_final_response_non_store(result) + assert _event_type(frame) == "response.completed" + assert _event_data(frame)["response"]["incomplete_details"] is None From b73e569045683613ffc9ede7bba9c676c8f25220 Mon Sep 17 00:00:00 2001 From: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> Date: Wed, 30 Sep 2026 15:05:13 +0000 Subject: [PATCH 22/43] [None][fix] responses: stored context that survives replay, terminals that always land Closes the dormant-path and terminal-trace audit items (A10, A12-half, A09-residual, A02-non-streaming): - Store paths: a tool-only assistant turn is now stored (it stored nothing); a stored message carrying reasoning keeps its tool_calls on replay instead of vanishing whole and orphaning the later tool result; and store=false on the CURRENT request no longer suppresses retrieval of PRIOR context - persistence and retrieval are two switches, gated independently. The full write -> store -> previous_response_id -> tool-result chain is the test. - /v1/messages: all four deliberate early returns now write exactly one terminal trace record (rejected_4xx / error) through a single reject helper - a rejected request no longer arrives in the requests shard and vanishes from the responses shard. - incomplete_details returns to ResponsesResponse (the field was commented out) and the non-streaming path populates it for length truncation; status=failed streams as response.failed instead of response.completed; cancelled deliberately stays (client-initiated, no spec event). The new field and ten previously untracked internal fields are registered in the serve API stability reference. - Non-streaming requests could still deliver a phantom {}-argument call for terminated-but-unreadable markup; the whole-text result is now verified against a streaming replay of the same text, and a call the stream would refuse is dropped (or falls back to text when it was the only one). Also migrates test_responses_custom_tools.py off two helpers the tool-resolution refactor removed - the file had failed on import ever since, so CI never ran it. Adapted to main, which has no request trace: the /v1/messages terminal trace records (and their tests) are not carried over. Signed-off-by: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> --- tensorrt_llm/serve/openai_protocol.py | 8 +- tensorrt_llm/serve/openai_server.py | 45 ++- tensorrt_llm/serve/responses_utils.py | 297 +++++++++++++--- .../references/trtllm_serve_api.yaml | 60 ++++ .../apps/test_responses_custom_tools.py | 57 +-- .../apps/test_responses_store_replay.py | 334 ++++++++++++++++++ ...test_responses_stream_final_consistency.py | 82 ++++- .../apps/test_responses_streaming_events.py | 100 ++++++ 8 files changed, 887 insertions(+), 96 deletions(-) create mode 100644 tests/unittest/llmapi/apps/test_responses_store_replay.py diff --git a/tensorrt_llm/serve/openai_protocol.py b/tensorrt_llm/serve/openai_protocol.py index b1a804b4233d..4e7db5471130 100644 --- a/tensorrt_llm/serve/openai_protocol.py +++ b/tensorrt_llm/serve/openai_protocol.py @@ -43,7 +43,7 @@ ResponseReasoningTextDoneEvent, ResponseStatus, ResponseTextConfig, ResponseWebSearchCallCompletedEvent, ResponseWebSearchCallInProgressEvent, ResponseWebSearchCallSearchingEvent) -from openai.types.responses.response import ToolChoice +from openai.types.responses.response import IncompleteDetails, ToolChoice from openai.types.responses.tool import Tool from openai.types.shared import Metadata, Reasoning from openai_harmony import ReasoningEffort @@ -1845,7 +1845,11 @@ class ResponsesResponse(OpenAIBaseModel): id: str = Field(default_factory=lambda: f"resp_{str(uuid.uuid4().hex)}") created_at: int = Field(default_factory=lambda: int(time.time())) # error: Optional[ResponseError] = None - # incomplete_details: Optional[IncompleteDetails] = None + # The SDK's own shape: reason is an enum naming why the response stopped + # short. Set by the response builder when a generation was cut at its + # token budget, so the non-streaming JSON body explains its "incomplete" + # status the way the streaming terminal event already does. + incomplete_details: Optional[IncompleteDetails] = None instructions: Optional[str] = None metadata: Optional[Metadata] = None model: str diff --git a/tensorrt_llm/serve/openai_server.py b/tensorrt_llm/serve/openai_server.py index 5bb9e22479ee..e949fbdba952 100644 --- a/tensorrt_llm/serve/openai_server.py +++ b/tensorrt_llm/serve/openai_server.py @@ -306,8 +306,8 @@ def _build_tool_strict_guided_decoding_params(tools, tool_parser_name): tool_parser_cls = ToolParserFactory.parsers.get(tool_parser_name.lower()) if tool_parser_cls is None: logger.warning( - "Tool parser '%s' not found, cannot enforce strict mode for tools.", - tool_parser_name) + f"Tool parser '{tool_parser_name}' not found, cannot enforce " + f"strict mode for tools.") return None parser = tool_parser_cls() @@ -321,8 +321,8 @@ def _build_tool_strict_guided_decoding_params(tools, tool_parser_name): by_alias=True, exclude_none=True)) if not parser.supports_structural_tag(): logger.warning( - "Tool parser '%s' does not support structural tags, " - "cannot enforce strict mode for tools.", tool_parser_name) + f"Tool parser '{tool_parser_name}' does not support structural " + f"tags, cannot enforce strict mode for tools.") return None get_info = parser.structure_info() @@ -870,9 +870,8 @@ def _init_llm(self, chat_template: Optional[str] = None): self.processor = AutoProcessor.from_pretrained( hf_tokenizer_path, trust_remote_code=trust_remote_code) except Exception: - logger.debug( - "Failed to load AutoProcessor or AutoConfig for %s", - hf_tokenizer_path) + logger.debug(f"Failed to load AutoProcessor or AutoConfig for " + f"{hf_tokenizer_path}") self.processor = None # load model config @@ -885,7 +884,7 @@ def _init_llm(self, chat_template: Optional[str] = None): checkpoint_format=getattr(self.generator.args, "checkpoint_format", None)) except Exception: - logger.debug("Failed to load AutoConfig for %s", hf_tokenizer_path) + logger.debug(f"Failed to load AutoConfig for {hf_tokenizer_path}") self.model_config = None self.chat_template = load_chat_template(chat_template) @@ -1027,7 +1026,7 @@ def _log_config_info_metrics(self) -> None: if torch.cuda.is_available(): model_config["gpu_type"] = torch.cuda.get_device_name(0) except (ImportError, RuntimeError) as e: - logger.debug("Could not detect GPU type for config metrics: %s", e) + logger.debug(f"Could not detect GPU type for config metrics: {e}") # Parallel config β€” prefer parallel_config from generator args # for accurate values including cp_size and world_size. @@ -3039,9 +3038,9 @@ async def create_streaming_generator(promise: RequestOutput, harmony_tokens = tokenize_harmony_chat_request( request, harmony_adapter=self.harmony_adapter) except Exception: - logger.error("messages_dict: %s", request.messages) - logger.error("tools: %s", request.tools) - logger.error("request: %s", request) + logger.error(f"messages_dict: {request.messages}") + logger.error(f"tools: {request.tools}") + logger.error(f"request: {request}") raise # Get harmony stop tokens @@ -3125,8 +3124,8 @@ async def create_streaming_generator(promise: RequestOutput, except ValueError as e: return self.create_error_response(str(e)) except Exception as e: - logger.error("Error in harmony chat completion: %s", e) - logger.debug("Error details: %s", traceback.format_exc()) + logger.error(f"Error in harmony chat completion: {e}") + logger.debug(f"Error details: {traceback.format_exc()}") return self.create_error_response(message=str(e), err_type="internal_error") @@ -3161,7 +3160,9 @@ async def create_response( model_name=self.model, conversation_store=self.conversation_store, generation_result=None, - enable_store=self.enable_store and request.store, + # The server-side switch; create_response applies + # request.store itself before persisting anything. + enable_store=self.enable_store, use_harmony=self.use_harmony, reasoning_parser=args.reasoning_parser, tool_parser=args.tool_parser, @@ -3276,7 +3277,15 @@ async def create_streaming_generator(promise: RequestOutput, request=request, prev_response=prev_response, conversation_store=self.conversation_store, - enable_store=self.enable_store and request.store, + # The server-side switch alone, because two independent things + # hang off it downstream: whether PRIOR context is fetched + # (gated there on previous_response_id being present) and + # whether THIS request is persisted (gated there on + # request.store). Folding request.store in here collapsed the + # two, so a follow-up sent with store=false had its + # previous_response_id loaded and validated above - and then + # generated with no history at all, silently. + enable_store=self.enable_store, use_harmony=self.use_harmony, tokenizer=self.tokenizer if not self.use_harmony else None, model_config=self.model_config @@ -3294,7 +3303,9 @@ async def create_streaming_generator(promise: RequestOutput, sampling_params=sampling_params, model_name=self.model, conversation_store=self.conversation_store, - enable_store=self.enable_store and request.store, + # Same as above: the processor's create_response applies + # request.store before persisting. + enable_store=self.enable_store, use_harmony=self.use_harmony, reasoning_parser=self.generator.args.reasoning_parser, tool_parser=self.tool_parser, diff --git a/tensorrt_llm/serve/responses_utils.py b/tensorrt_llm/serve/responses_utils.py index 2fdf34626c3a..e9562dadc660 100644 --- a/tensorrt_llm/serve/responses_utils.py +++ b/tensorrt_llm/serve/responses_utils.py @@ -31,6 +31,7 @@ ResponseReasoningTextDoneEvent, ResponseTextDeltaEvent, ResponseTextDoneEvent) +from openai.types.responses.response import IncompleteDetails from openai.types.responses.response_content_part_added_event import \ PartReasoningText from openai.types.responses.response_content_part_done_event import \ @@ -1013,10 +1014,26 @@ async def _create_input_messages( }) # Prepend the conversation history. - # Skip the reasoning output. + # + # Reasoning is stripped on replay to save tokens, but a stored turn is + # more than its reasoning: the store writer attaches the turn's tool + # calls to the reasoning message, so discarding the whole message for its + # "reasoning" key also discarded the calls - and the tool RESULT the + # client sends next then replayed with no call before it, an orphan the + # model cannot pair with anything. Strip the reasoning, keep the calls. + # A turn that was only reasoning still vanishes, and a message without + # the key passes through exactly as stored. for msg in prev_msgs: if "reasoning" not in msg: messages.append(msg) + continue + tool_calls = msg.get("tool_calls") + if tool_calls: + messages.append({ + "role": "assistant", + "content": None, + "tool_calls": tool_calls, + }) # Append the new input. # Responses API supports simple text inputs without chat format. @@ -1060,7 +1077,13 @@ def _create_output_messages( """ Convert output contents to chat completion messages for conversation store. - Reasoning content is not included in the output messages to reduce the token usage. + Reasoning is stored here and stripped on replay instead + (_create_input_messages), which keeps the token saving while letting the + replay keep the tool calls the reasoning message carries. Tool calls are + stored on whichever assistant message the turn produced - reasoning + first, else the text message, else a bare assistant tool-call message - + because a turn's calls must survive storage no matter what else the turn + contained. Input: output_contents: dict[str, str] @@ -1073,12 +1096,25 @@ def _create_output_messages( """ messages: list[ChatCompletionMessageParam] = [] + tool_calls = output_contents.get("tool_calls") or [] + tool_call_msgs = [{ + "id": call.call_id, + "function": { + "arguments": _stored_tool_arguments(call), + "name": _stored_tool_name(call), + }, + "type": "function", + } for call in tool_calls] + _responses_debug_log(f"tool_call_msgs: {tool_call_msgs}") + text_content = output_contents.get("text_content", None) + text_msg: Optional[ChatCompletionMessageParam] = None if text_content: - messages.append({ + text_msg = { "role": "assistant", "content": text_content, - }) + } + messages.append(text_msg) reasoning_content = output_contents.get("reasoning_content", None) if reasoning_content: @@ -1086,21 +1122,25 @@ def _create_output_messages( role="assistant", reasoning=reasoning_content, ) - - tool_calls = output_contents.get("tool_calls", []) - tool_call_msgs = [{ - "id": call.call_id, - "function": { - "arguments": _stored_tool_arguments(call), - "name": _stored_tool_name(call), - }, - "type": "function", - } for call in tool_calls] - - _responses_debug_log(f"tool_call_msgs: {tool_call_msgs}") reasoning_msg["tool_calls"] = tool_call_msgs - messages.append(reasoning_msg) + elif tool_call_msgs: + # The calls used to be stored only on the reasoning message, so a + # turn that called tools without reasoning lost them - and a turn + # that was *nothing but* tool calls stored no assistant message at + # all, leaving the client's tool RESULT to replay against a call + # that was never in the history. A tool-call-bearing turn is an + # assistant turn whether or not any text or reasoning came with it: + # attach the calls to the text message when there is one, store a + # bare assistant tool-call message when there is not. + if text_msg is not None: + text_msg["tool_calls"] = tool_call_msgs + else: + messages.append({ + "role": "assistant", + "content": None, + "tool_calls": tool_call_msgs, + }) return messages @@ -1704,8 +1744,19 @@ def _streamed_or_fresh_id(item_type: str) -> str: reasoning_texts.append(reasoning_text or "") if text: + parsed_source = text text, calls = _apply_tool_parser(tool_parser, available_tools, output.index, text, False) + if calls and streamed_tool_call_ids is None: + # No stream ran, so no per-entity record says which calls the + # streamed assembly would have refused - the check the pairing + # below performs for streamed requests. An empty-argument call + # is the shape the whole-text parse gets wrong on its own (it + # reports `{}` for argument markup it could not read), so it + # is verified against a streamed replay of the same text; see + # the reconciler for why the two views must agree here. + text, calls = _verify_empty_calls_against_streaming( + tool_parser, available_tools, parsed_source, text, calls) text_item = None reasoning_item = None @@ -1773,10 +1824,9 @@ def _streamed_or_fresh_id(item_type: str) -> str: # actually reached the wire, in order; placeholders name no # wire identity and are worth nothing to a client. logger.warning( - "final response rebuilt %d tool call(s) but the stream " - "saw %d; ids beyond the streamed ones are new and will " - "not match the client's tool outputs", len(calls), - len(reusable)) + f"final response rebuilt {len(calls)} tool call(s) but the " + f"stream saw {len(reusable)}; ids beyond the streamed ones " + f"are new and will not match the client's tool outputs") reusable = [ids for ids in reusable if ids is not None] tool_calls_item = [] for index, call in enumerate(calls): @@ -1789,10 +1839,10 @@ def _streamed_or_fresh_id(item_type: str) -> str: # consumed by the drop instead of shifting onto the next # call. logger.warning( - "final response drops the tool call to %r: the stream " - "assembled its arguments into invalid JSON and never " - "delivered it, so the snapshot does not deliver it " - "either", call.name) + f"final response drops the tool call to {call.name!r}: " + f"the stream assembled its arguments into invalid JSON " + f"and never delivered it, so the snapshot does not " + f"deliver it either") continue tool_calls_item.append( _tool_call_output_item(call, @@ -1820,10 +1870,10 @@ def _streamed_or_fresh_id(item_type: str) -> str: rebuilt = used_ids_by_type[item_type] if streamed != rebuilt: logger.warning( - "final response rebuilt %d %s item(s) but the stream " - "published %d; the two views of this generation differ " - "structurally and ids beyond the streamed ones are new", - rebuilt, item_type, streamed) + f"final response rebuilt {rebuilt} {item_type} item(s) but " + f"the stream published {streamed}; the two views of this " + f"generation differ structurally and ids beyond the " + f"streamed ones are new") return output_items, output_messages, reasoning_texts @@ -1940,9 +1990,9 @@ def _tool_call_output_item( # logs "Model attempted to call undefined function" at the equivalent # point; borrowing that is most of the value of this change. logger.warning( - "tool call %r matches no declared tool; reporting it as a function " - "call. If it is in fact a custom tool, the client will reject it.", - name) + f"tool call {name!r} matches no declared tool; reporting it as a " + f"function call. If it is in fact a custom tool, the client will " + f"reject it.") if is_custom: # Unwrap the single string argument the tool was described with. A @@ -2160,6 +2210,17 @@ def _create_response( tokenizer=tokenizer, reasoning_texts=reasoning_texts), ) + # A generation the engine cut off at its token budget has status + # "incomplete", and the spec explains that status in + # incomplete_details.reason - "max_output_tokens". The streaming terminal + # event has said so since the response.incomplete fix; setting it here + # makes the non-streaming JSON body tell the same story. Only "length" is + # explained: "not_finished" (a disaggregated context worker handing the + # request off) also maps to "incomplete", and naming a public reason for + # an internal handoff would be a guess presented as fact. + if finish_reason == "length": + response.incomplete_details = IncompleteDetails( + reason="max_output_tokens") # Disaggregated serving. A context-only response is read by the # orchestrator, never by a client: it carries the KV-cache handle, the # first generated token and the tokenized prompt so a generation worker can @@ -2695,13 +2756,19 @@ def _assembled_tool_calls( command is worse than executing none. The warning is what keeps the loss from being silent. - The non-streaming path needs no equivalent - `parse_base_json` builds its - arguments with `json.dumps`, so they always parse - but it is not silent - about this text either: its pair regex reads nothing out of the block and - it reports the call with empty arguments. `{}` is something a client can - run, so the two paths do differ here, and settling that means deciding - what a call whose markup could not be read should be. A different - question, left alone. + The non-streaming path cannot reuse this validity test - `parse_base_json` + builds its arguments with `json.dumps`, so they always parse - and its + pair regex reads nothing out of such a block, reporting the call with + empty arguments: a call the model never made, delivered as runnable. + That is settled now, one layer up: when no stream ran, + `_verify_empty_calls_against_streaming` replays the text through this + streaming machinery, and a `{}`-argument call this assembly would have + refused is not reported there either - its markup falls back into the + message text instead. What still differs is a call whose markup yielded + *some* readable pairs: the whole-text parse delivers what it read, the + stream drops the call whose tail it could not assemble. That is a real + divergence with no obviously right answer and it stays open, documented + at the reconciler. """ calls: list[ToolCallItem] = [] for tool_index, fragment in fragments.items(): @@ -2734,6 +2801,108 @@ def _assembled_tool_calls( return calls +def _verify_empty_calls_against_streaming( + tool_parser_id: Optional[str], + tools: Optional[list[ChatCompletionToolsParam]], + source_text: str, + normal_text: str, + calls: list[ToolCallItem], +) -> Tuple[str, list[ToolCallItem]]: + r"""Drop whole-text `{}`-argument calls whose markup could not be read. + + Only for a request no stream ran for. A streamed request already settles + this per call entity in `_create_output_content` - the stream's id record + carries a None where assembly dropped a call, and the rebuild consumes it. + + The whole-text parse reports `{}` for two markups it cannot tell apart: a + genuine zero-argument call (`get_time`) and a call + whose argument markup its pair regex read nothing out of - GLM-4.7 + opening an `` it never closes, ending `` + instead. The streaming machinery *can* tell them apart, because it + assembles exactly the JSON it read and `_assembled_tool_calls` refuses + what does not parse: the zero-argument call streams as a valid `{}`, the + unreadable one as `{"cmd": }` and is dropped. So when the whole-text + parse reports an empty-argument call, the same text is replayed through a + fresh streaming parser and the two views are paired positionally - the + same premise the id reuse in `_create_output_content` rests on, both + passes enumerating the same markup in the same order. An empty call the + streamed assembly would have refused is a call the model never made, and + it is not delivered here either. + + The markup is not discarded with it: when the refused call was everything + the parse found, the original text goes back verbatim - these turns are + recorded as training data, and the raw markup as message text is the same + fallback both views already use for an unterminated call. Alongside calls + that really were made, the surrounding text keeps its parsed form and only + the refused call is dropped, which is what the streamed view of the same + text delivers. + + Any disagreement about what the call entities *are* - a replay error, or + the two passes finding different numbers of calls - leaves the whole-text + result untouched: this is a cross-check, not the parse of record, and + guessing at a pairing would trade a known defect for an unknowable one. + """ + empty_positions = { + index + for index, call in enumerate(calls) + if _parses_to_empty_object(call.parameters) + } + if not empty_positions or tool_parser_id is None or tools is None: + return normal_text, calls + + fragments: dict[int, dict[str, Any]] = {} + try: + replay_parser = ToolParserFactory.create_tool_parser(tool_parser_id) + increment = replay_parser.parse_streaming_increment(source_text, tools) + _accumulate_tool_call_fragments(fragments, increment.calls) + _, flushed, unfinished = _flush_tool_parser( + tools=tools, + output_index=0, + tool_parser_dict={0: replay_parser}, + ) + _accumulate_tool_call_fragments(fragments, flushed) + assembled = _assembled_tool_calls(fragments, unfinished) + except Exception as exc: # noqa: BLE001 - cross-check must not fail the parse + logger.warning( + f"Could not replay the generation through the streaming tool " + f"parser to verify its empty-argument calls ({exc}); delivering " + "the whole-text parse as is.") + return normal_text, calls + + if len(fragments) != len(calls): + return normal_text, calls + + delivered = {call.tool_index for call in assembled} + entity_order = list(fragments) + phantoms = { + index + for index in empty_positions if entity_order[index] not in delivered + } + if not phantoms: + return normal_text, calls + + for index in sorted(phantoms): + logger.warning( + f"Dropping the tool call to {calls[index].name!r} from the " + "whole-text parse: it reports no arguments only because its " + "argument markup could not be read, and the streamed assembly of " + "the same text refuses the call. The model emitted a malformed " + "tool call, and an invented '{}' would run the tool with " + "arguments the model never wrote.") + if len(phantoms) == len(calls): + return source_text, [] + return normal_text, [ + call for index, call in enumerate(calls) if index not in phantoms + ] + + +def _parses_to_empty_object(parameters: Optional[str]) -> bool: + try: + return json.loads(parameters or "") == {} + except ValueError: + return False + + def _close_open_item(helper): """Close whichever output item is currently open, if any. @@ -3197,7 +3366,8 @@ def _stream_terminal_event( final_response: ResponsesResponse, finish_reason: Optional[str], sequence_number: int = -1, -) -> Union[ResponseCompletedEvent, ResponseIncompleteEvent]: +) -> Union[ResponseCompletedEvent, ResponseIncompleteEvent, + ResponseFailedEvent]: """The terminal event for a stream that ran to completion, by status. ``finish_reason_mapping`` marks a generation the engine cut off at its @@ -3210,10 +3380,20 @@ def _stream_terminal_event( ``max_output_tokens``, and any other cause is left unstated rather than guessed at. - ``incomplete_details`` is set on the dump rather than on - ResponsesResponse, whose field is commented out - the same route - ``get_stream_failed_events`` takes for ``error``: the event re-validates - the dict against the SDK's Response, which does carry the field. + ``incomplete_details`` now lives on ResponsesResponse and the response + builder (see ``_create_response``) sets it for a token-budget cut, so the + dump usually carries it already; setting it here as well keeps this event + right for any snapshot that did not pass through that builder. The + dump-level route is the same one ``get_stream_failed_events`` takes for + ``error``, whose field is still commented out. + + A status of "failed" is a generation the engine gave up on (a timeout), + and it ships under ``response.failed`` - the spec's terminal event for + that status - not under a ``response.completed`` whose name promises the + opposite. "cancelled" stays under ``response.completed`` on purpose: the + cancellation was client-initiated, the Responses spec pairs no terminal + event with it, and inventing one would hand SDK clients an event type + they cannot parse. """ payload = final_response.model_dump() if final_response.status == "incomplete": @@ -3224,6 +3404,12 @@ def _stream_terminal_event( sequence_number=sequence_number, response=payload, ) + if final_response.status == "failed": + return ResponseFailedEvent( + type="response.failed", + sequence_number=sequence_number, + response=payload, + ) return ResponseCompletedEvent( type="response.completed", sequence_number=sequence_number, @@ -3464,14 +3650,18 @@ def get_stream_failed_events(self, STREAM_TERMINATION_INTERNAL_ERROR = "internal_error" # The terminal events of a stream that ran to completion - a truncated -# generation ends in `response.incomplete`, everything else in -# `response.completed` (see _stream_terminal_event). Matched as substrings -# because a relayed stream arrives as transport-sized chunks that may carry -# several events, not as one frame per event. Kept in both encodings so the -# scan never has to decode a relayed chunk just to look at it. +# generation ends in `response.incomplete`, a failed one in `response.failed`, +# everything else in `response.completed` (see _stream_terminal_event; the +# guard's own abnormal-termination path emits `response.failed` too). Matched +# as substrings because a relayed stream arrives as transport-sized chunks +# that may carry several events, not as one frame per event. Kept in both +# encodings so the scan never has to decode a relayed chunk just to look at +# it. _COMPLETED_EVENT_MARKER = "event: response.completed" _INCOMPLETE_EVENT_MARKER = "event: response.incomplete" -_TERMINAL_EVENT_MARKERS = (_COMPLETED_EVENT_MARKER, _INCOMPLETE_EVENT_MARKER) +_FAILED_EVENT_MARKER = "event: response.failed" +_TERMINAL_EVENT_MARKERS = (_COMPLETED_EVENT_MARKER, _INCOMPLETE_EVENT_MARKER, + _FAILED_EVENT_MARKER) _TERMINAL_EVENT_MARKERS_BYTES = tuple( marker.encode("utf-8") for marker in _TERMINAL_EVENT_MARKERS) @@ -3599,9 +3789,10 @@ async def guard_responses_stream( the full text. Frames are forwarded untouched and nothing is added once a terminal event - (``response.completed``, or ``response.incomplete`` for a truncated - generation) has gone out, so a stream that completes normally is - byte-for-byte what it was. + (``response.completed``, ``response.incomplete`` for a truncated + generation, or ``response.failed`` for one the engine gave up on) has + gone out, so a stream that completes normally is byte-for-byte what it + was. The two abnormal endings are handled differently on purpose: diff --git a/tests/unittest/api_stability/references/trtllm_serve_api.yaml b/tests/unittest/api_stability/references/trtllm_serve_api.yaml index e25a699f6c97..155d5ff771c5 100644 --- a/tests/unittest/api_stability/references/trtllm_serve_api.yaml +++ b/tests/unittest/api_stability/references/trtllm_serve_api.yaml @@ -514,6 +514,12 @@ models: default: null status: stable required: true + parallel_tool_calls: + kind: openai + type: Optional[bool] + default: null + status: stable + required: false prompt_token_ids: kind: extension type: Optional[List[int]] @@ -1045,6 +1051,18 @@ models: default: false status: stable required: false + conversation_params: + kind: extension + type: Optional[ConversationParams] + default: null + status: prototype + required: false + disaggregated_params: + kind: extension + type: Optional[DisaggregatedParams] + default: null + status: prototype + required: false include: kind: openai type: Optional[list[Literal['code_interpreter_call.outputs', 'computer_call_output.output.image_url', 'file_search_call.results', 'message.input_image.image_url', 'message.output_text.logprobs', 'reasoning.encrypted_content'],]] @@ -1105,6 +1123,18 @@ models: default: null status: stable required: false + prompt_token_ids: + kind: extension + type: Optional[List[int]] + default: null + status: prototype + required: false + prompt_token_ids_b64: + kind: extension + type: Optional[str] + default: null + status: prototype + required: false reasoning: kind: openai type: Optional[Reasoning] @@ -1192,6 +1222,18 @@ models: ResponsesResponse: fields: + disaggregated_params: + kind: extension + type: Optional[DisaggregatedParams] + default: null + status: prototype + required: false + finish_reason: + kind: extension + type: Optional[str] + default: null + status: prototype + required: false id: kind: openai type: str @@ -1204,6 +1246,12 @@ models: default: > status: stable required: false + incomplete_details: + kind: openai + type: Optional[IncompleteDetails] + default: null + status: stable + required: false instructions: kind: openai type: Optional[str] @@ -1240,6 +1288,18 @@ models: default: null status: stable required: true + prompt_token_ids: + kind: extension + type: Optional[List[int]] + default: null + status: prototype + required: false + prompt_token_ids_b64: + kind: extension + type: Optional[str] + default: null + status: prototype + required: false temperature: kind: openai type: float diff --git a/tests/unittest/llmapi/apps/test_responses_custom_tools.py b/tests/unittest/llmapi/apps/test_responses_custom_tools.py index b4a6955abdd7..09a3bc76a274 100644 --- a/tests/unittest/llmapi/apps/test_responses_custom_tools.py +++ b/tests/unittest/llmapi/apps/test_responses_custom_tools.py @@ -29,10 +29,10 @@ from tensorrt_llm.serve.responses_utils import ( CUSTOM_TOOL_INPUT_ARG, - _custom_tool_names, _get_chat_completion_function_tools, _response_output_item_to_chat_completion_message, _tool_call_output_item, + _tool_resolution, ) # The CPU-* CI stages run pytest with -m 'cpu_only'. Without this marker every @@ -66,12 +66,19 @@ def test_custom_tool_is_offered_with_a_named_string_parameter(): def test_custom_tool_names_are_collected(): + """A custom tool classifies custom; a plain function does not. + + The resolution map replaced _custom_tool_names; this file referenced the + removed helper and failed on import ever since - nothing in CI ran it. + """ tools = [_custom_tool(), SimpleNamespace(type="function", name="shell")] - assert _custom_tool_names(tools) == {"apply_patch"} + resolution = _tool_resolution(tools) + assert resolution["apply_patch"][2] is True + assert resolution["shell"][2] is False def test_no_tools_yields_no_custom_names(): - assert _custom_tool_names(None) == set() + assert _tool_resolution(None) == {} # --------------------------------------------------------------------------- @@ -79,9 +86,16 @@ def test_no_tools_yields_no_custom_names(): # --------------------------------------------------------------------------- +# _tool_call_output_item takes the resolution map (every spelling -> +# (namespace, bare, custom)) since the tool-resolution refactor; these tests +# were written against the removed set/map pair and failed on import or call +# ever since - nothing in CI ran this file. +_CUSTOM_RES = {"apply_patch": (None, "apply_patch", True)} + + def test_custom_tool_call_carries_the_freeform_payload(): item = _tool_call_output_item( - _call("apply_patch", json.dumps({CUSTOM_TOOL_INPUT_ARG: PATCH})), {"apply_patch"} + _call("apply_patch", json.dumps({CUSTOM_TOOL_INPUT_ARG: PATCH})), _CUSTOM_RES ) assert item.type == "custom_tool_call" assert item.input == PATCH @@ -90,26 +104,24 @@ def test_custom_tool_call_carries_the_freeform_payload(): def test_unknown_argument_name_is_still_forwarded(): """A payload under an unexpected key beats dropping the call.""" - item = _tool_call_output_item( - _call("apply_patch", json.dumps({"patch": PATCH})), {"apply_patch"} - ) + item = _tool_call_output_item(_call("apply_patch", json.dumps({"patch": PATCH})), _CUSTOM_RES) assert item.input == PATCH def test_non_json_arguments_are_passed_through_verbatim(): - item = _tool_call_output_item(_call("apply_patch", PATCH), {"apply_patch"}) + item = _tool_call_output_item(_call("apply_patch", PATCH), _CUSTOM_RES) assert item.input == PATCH def test_ordinary_tools_are_still_function_calls(): - item = _tool_call_output_item(_call("shell", '{"cmd": "ls"}'), {"apply_patch"}) + item = _tool_call_output_item(_call("shell", '{"cmd": "ls"}'), _CUSTOM_RES) assert item.type == "function_call" assert item.arguments == '{"cmd": "ls"}' def test_custom_and_function_calls_get_distinct_id_prefixes(): - custom = _tool_call_output_item(_call("apply_patch", PATCH), {"apply_patch"}) - function = _tool_call_output_item(_call("shell", "{}"), {"apply_patch"}) + custom = _tool_call_output_item(_call("apply_patch", PATCH), _CUSTOM_RES) + function = _tool_call_output_item(_call("shell", "{}"), _CUSTOM_RES) assert custom.id.startswith("ctc_") assert function.id.startswith("fc_") @@ -168,18 +180,14 @@ def test_namespaced_call_reports_the_namespace_separately(): namespace field. A call named "collaboration.spawn_agent" matches nothing it knows, so the whole capability is unusable. """ - from tensorrt_llm.serve.responses_utils import _namespaced_tool_names - tools = [_namespace_tool()] - item = _tool_call_output_item( - _call("collaboration.spawn_agent", "{}"), set(), _namespaced_tool_names(tools) - ) + item = _tool_call_output_item(_call("collaboration.spawn_agent", "{}"), _tool_resolution(tools)) assert item.name == "spawn_agent" assert item.namespace == "collaboration" def test_unnamespaced_call_has_no_namespace(): - item = _tool_call_output_item(_call("shell", "{}"), set(), {}) + item = _tool_call_output_item(_call("shell", "{}"), {}) assert item.namespace is None @@ -192,7 +200,10 @@ def test_namespaced_custom_tool_is_recognised(): tools=[SimpleNamespace(type="custom", name="apply_patch", description=None)], ) ] - assert _custom_tool_names(tools) == {"edit.apply_patch"} + resolution = _tool_resolution(tools) + assert resolution["edit.apply_patch"] == ("edit", "apply_patch", True) + # The bare spelling resolves too when unambiguous - the model writes both. + assert resolution["apply_patch"] == ("edit", "apply_patch", True) def test_namespaced_call_replays_under_its_qualified_name(): @@ -265,7 +276,7 @@ def test_history_records_a_custom_tool_call(): """ from tensorrt_llm.serve.responses_utils import _stored_tool_arguments, _stored_tool_name - item = _tool_call_output_item(_call("apply_patch", PATCH), {"apply_patch"}) + item = _tool_call_output_item(_call("apply_patch", PATCH), _CUSTOM_RES) assert json.loads(_stored_tool_arguments(item)) == {CUSTOM_TOOL_INPUT_ARG: PATCH} assert _stored_tool_name(item) == "apply_patch" @@ -273,15 +284,15 @@ def test_history_records_a_custom_tool_call(): def test_history_records_a_function_call_unchanged(): from tensorrt_llm.serve.responses_utils import _stored_tool_arguments - item = _tool_call_output_item(_call("shell", '{"cmd": "ls"}'), set()) + item = _tool_call_output_item(_call("shell", '{"cmd": "ls"}'), {}) assert _stored_tool_arguments(item) == '{"cmd": "ls"}' def test_history_requalifies_a_namespaced_call(): - from tensorrt_llm.serve.responses_utils import _namespaced_tool_names, _stored_tool_name + from tensorrt_llm.serve.responses_utils import _stored_tool_name item = _tool_call_output_item( - _call("collaboration.spawn_agent", "{}"), set(), _namespaced_tool_names([_namespace_tool()]) + _call("collaboration.spawn_agent", "{}"), _tool_resolution([_namespace_tool()]) ) assert _stored_tool_name(item) == "collaboration.spawn_agent" @@ -289,7 +300,7 @@ def test_history_requalifies_a_namespaced_call(): def test_a_custom_tool_inside_a_namespace_is_described_with_its_input_arg(): """Regression: the input and output paths must agree on the schema. - _custom_tool_names classifies a namespaced custom tool under its qualified + _tool_resolution classifies a namespaced custom tool under its qualified name, so the output path looks for CUSTOM_TOOL_INPUT_ARG. If the namespace branch described the tool with an empty object schema, the model would never be told that argument exists and would invent its own name - the diff --git a/tests/unittest/llmapi/apps/test_responses_store_replay.py b/tests/unittest/llmapi/apps/test_responses_store_replay.py new file mode 100644 index 000000000000..7aa464d5c02b --- /dev/null +++ b/tests/unittest/llmapi/apps/test_responses_store_replay.py @@ -0,0 +1,334 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Offline tests: the Responses store must not lose tool calls or prior context. + +Three ways a multi-turn tool conversation lost its thread, each pinned here: + +* The store writer only attached tool calls to the reasoning message, so a + turn that called tools without reasoning stored them nowhere - and a turn + that was *nothing but* tool calls stored no assistant message at all. +* A stored reasoning+calls turn survived storage but not replay: the replay + filter dropped the whole message for its "reasoning" key, calls included, so + the client's tool RESULT replayed with no call before it - an orphan the + model cannot pair. +* The server folded ``request.store`` into the one flag it hands + preprocessing, so a follow-up sent with ``store=false`` had its + ``previous_response_id`` loaded and validated - and then generated with no + history at all, silently. +""" + +import asyncio +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest +from openai.types.responses import ResponseFunctionToolCall + +from tensorrt_llm.serve.openai_protocol import ResponsesRequest +from tensorrt_llm.serve.responses_utils import ( + ConversationHistoryStore, + _create_input_messages, + _create_output_messages, + request_preprocess, +) + +# The CPU-* CI stages run pytest with -m 'cpu_only'. Without this marker every +# test in the file is deselected, which pytest reports as exit code 5 and the +# stage reports as a failure. +pytestmark = pytest.mark.cpu_only + + +def _call(call_id="call_1", name="exec", arguments='{"cmd": "ls"}'): + return ResponseFunctionToolCall( + type="function_call", + id=f"fc_{call_id}", + call_id=call_id, + name=name, + arguments=arguments, + status="completed", + ) + + +def _tool_result_item(call_id="call_1", output="ok"): + return {"type": "function_call_output", "call_id": call_id, "output": output} + + +def _contents(text=None, reasoning=None, calls=()): + return { + "text_content": text, + "reasoning_content": reasoning, + "tool_calls": list(calls), + } + + +# --------------------------------------------------------------------------- +# The store writer: a tool-call-bearing turn is an assistant turn +# --------------------------------------------------------------------------- + + +def test_a_tool_only_turn_is_stored(): + """No text, no reasoning - the calls alone used to store nothing.""" + messages = _create_output_messages(_contents(calls=[_call()])) + assert len(messages) == 1 + assert messages[0]["role"] == "assistant" + assert messages[0]["tool_calls"][0]["id"] == "call_1" + assert messages[0]["tool_calls"][0]["function"]["name"] == "exec" + + +def test_a_text_turn_keeps_its_calls(): + """Text plus calls without reasoning stored the text and dropped the calls.""" + messages = _create_output_messages(_contents(text="Running it.", calls=[_call()])) + carried = [m for m in messages if m.get("tool_calls")] + assert carried and carried[0]["content"] == "Running it." + + +def test_a_text_only_turn_stores_the_old_shape(): + assert _create_output_messages(_contents(text="Hello.")) == [ + {"role": "assistant", "content": "Hello."} + ] + + +def test_a_reasoning_turn_still_carries_reasoning_and_calls(): + """The path that already worked, kept byte-compatible.""" + (message,) = _create_output_messages(_contents(reasoning="think", calls=[_call()])) + assert message["reasoning"] == "think" + assert message["tool_calls"][0]["id"] == "call_1" + + +# --------------------------------------------------------------------------- +# Replay: strip the reasoning, keep the calls +# --------------------------------------------------------------------------- + + +def _replay(prev_msgs, request_input="hi", **request_overrides): + request = ResponsesRequest(model="m", input=request_input, **request_overrides) + return asyncio.run(_create_input_messages(request=request, prev_msgs=prev_msgs)) + + +def test_replaying_a_reasoning_call_turn_preserves_the_calls(): + stored = _create_output_messages(_contents(reasoning="think", calls=[_call()])) + replayed = _replay(stored) + assert all("reasoning" not in m for m in replayed) + carried = [m for m in replayed if m.get("tool_calls")] + assert carried and carried[0]["tool_calls"][0]["id"] == "call_1" + + +def test_replaying_a_reasoning_only_turn_still_drops_it(): + stored = _create_output_messages(_contents(reasoning="think")) + replayed = _replay(stored) + assert all("reasoning" not in m and not m.get("tool_calls") for m in replayed) + + +def test_a_plain_assistant_call_message_replays_byte_identical(): + plain = { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call_9", "type": "function", "function": {"name": "exec", "arguments": "{}"}} + ], + } + replayed = _replay([plain]) + assert replayed[0] is plain + + +# --------------------------------------------------------------------------- +# The full chain: write -> store -> read via previous_response_id -> tool result +# --------------------------------------------------------------------------- + + +async def _chain(output_contents, call_id): + store = ConversationHistoryStore() + first = ResponsesRequest(model="m", input="do the thing", request_id="resp_first") + input_msgs = await _create_input_messages(request=first, prev_msgs=[]) + await store.store_messages("resp_first", input_msgs, None) + + output_msgs = _create_output_messages(output_contents) + await store.store_response( + resp=SimpleNamespace(id="resp_first"), resp_msgs=output_msgs, prev_resp_id=None + ) + + prev_msgs = await store.get_conversation_history("resp_first") + follow_up = ResponsesRequest( + model="m", + input=[_tool_result_item(call_id=call_id)], + previous_response_id="resp_first", + # The follow-up itself opting out of storage must not cost it the + # history it names. + store=False, + request_id="resp_second", + ) + return await _create_input_messages(request=follow_up, prev_msgs=prev_msgs) + + +@pytest.mark.parametrize( + "contents,call_id", + [ + (_contents(calls=[_call()]), "call_1"), + (_contents(reasoning="let me think", calls=[_call(call_id="call_2")]), "call_2"), + ], + ids=["tool-only-turn", "reasoning-plus-call-turn"], +) +def test_the_stored_call_precedes_its_result_on_replay(contents, call_id): + messages = asyncio.run(_chain(contents, call_id)) + call_pos = [ + i + for i, m in enumerate(messages) + if any(c.get("id") == call_id for c in (m.get("tool_calls") or [])) + ] + result_pos = [ + i + for i, m in enumerate(messages) + if m.get("role") == "tool" and m.get("tool_call_id") == call_id + ] + assert call_pos and result_pos, f"orphaned result: {messages}" + assert call_pos[0] < result_pos[0] + assert all("reasoning" not in m for m in messages) + + +# --------------------------------------------------------------------------- +# The two store switches are independent +# --------------------------------------------------------------------------- + + +async def _preprocess(monkeypatch, store_flag): + """Run the real request_preprocess with the heavy leaves faked. + + Chat-template rendering and sampling-params construction need a tokenizer + and the engine bindings; the gates under test - fetch prior context, then + persist or not - sit above both, so those leaves are replaced with + recorders and the store itself stays real. + """ + import tensorrt_llm.serve.responses_utils as ru + + store = ConversationHistoryStore() + await store.store_messages( + "resp_prev", + [ + {"role": "user", "content": "earlier turn"}, + {"role": "assistant", "content": "earlier answer"}, + ], + None, + ) + + captured = {} + + def fake_parse(messages, model_config): + captured["messages"] = list(messages) + + async def mm(): + return None, None + + return list(messages), mm(), {}, None + + async def fake_template(**kwargs): + return [1, 2, 3] + + monkeypatch.setattr(ru, "parse_chat_messages_coroutines", fake_parse) + monkeypatch.setattr(ru, "async_apply_chat_template", fake_template) + monkeypatch.setattr(ru, "resolve_top_level_model_type", lambda cfg: "llama") + monkeypatch.setattr(ru, "add_thinking_budget_logits_processor", lambda *a, **k: None) + monkeypatch.setattr( + ResponsesRequest, "to_sampling_params", lambda self, **kw: SimpleNamespace() + ) + + request = ResponsesRequest( + model="m", + input="next turn", + previous_response_id="resp_prev", + store=store_flag, + request_id="resp_new", + ) + await request_preprocess( + request=request, + prev_response=None, + conversation_store=store, + # The server-side switch, which is all the server passes: retrieval is + # gated downstream on previous_response_id, persistence on + # request.store. + enable_store=True, + use_harmony=False, + tokenizer=None, + model_config=None, + processor=None, + reasoning_parser=None, + ) + saw_prior = any(m.get("content") == "earlier answer" for m in captured["messages"]) + persisted = "resp_new" in store.response_to_conversation + return saw_prior, persisted + + +def test_store_false_still_sees_prior_context_and_is_not_persisted(monkeypatch): + saw_prior, persisted = asyncio.run(_preprocess(monkeypatch, store_flag=False)) + assert saw_prior, "previous_response_id was loaded and then ignored" + assert not persisted + + +def test_store_true_sees_prior_context_and_is_persisted(monkeypatch): + saw_prior, persisted = asyncio.run(_preprocess(monkeypatch, store_flag=True)) + assert saw_prior + assert persisted + + +def test_the_endpoint_hands_preprocess_the_server_switch(monkeypatch): + """A store=false request must not flip the flag preprocessing receives. + + The endpoint used to pass ``enable_store and request.store``, which reads + as one switch but gates two: it suppressed retrieval of the prior context + along with persistence. Only the wiring is under test, so preprocessing is + replaced with a recorder that stops the handler right after the handoff. + """ + import tensorrt_llm.serve.openai_server as server_module + from tensorrt_llm.serve.openai_server import OpenAIServer + + server = object.__new__(OpenAIServer) + server.model = "test-model" + server.enable_store = True + server._is_visual_gen = False + server.use_harmony = False + server.tokenizer = None + server.model_config = None + server.processor = None + server.tool_parser = None + server.generator = SimpleNamespace( + args=SimpleNamespace(reasoning_parser=None, num_postprocess_workers=0) + ) + server.conversation_store = ConversationHistoryStore() + asyncio.run( + server.conversation_store.store_response( + resp=SimpleNamespace(id="resp_prev"), resp_msgs=[], prev_resp_id=None + ) + ) + + captured = {} + + async def fake_preprocess(**kwargs): + captured.update(kwargs) + raise RuntimeError("stop after the handoff") + + monkeypatch.setattr(server_module, "responses_api_request_preprocess", fake_preprocess) + + request = ResponsesRequest( + model="test-model", input="hi", previous_response_id="resp_prev", store=False + ) + raw_request = SimpleNamespace( + state=SimpleNamespace(), + headers={}, + url=SimpleNamespace(path="/v1/responses"), + json=AsyncMock(return_value={}), + ) + asyncio.run(server.openai_responses(request, raw_request)) + + assert captured.get("enable_store") is True + assert captured["request"].store is False diff --git a/tests/unittest/llmapi/apps/test_responses_stream_final_consistency.py b/tests/unittest/llmapi/apps/test_responses_stream_final_consistency.py index 6d0ace618019..059250ba2881 100644 --- a/tests/unittest/llmapi/apps/test_responses_stream_final_consistency.py +++ b/tests/unittest/llmapi/apps/test_responses_stream_final_consistency.py @@ -43,6 +43,7 @@ from tensorrt_llm.serve.responses_utils import ( ResponsesStreamingProcessor, _create_output_content, + create_response_non_store, stamp_sse_sequence_number, ) from tensorrt_llm.serve.tool_parser.core_types import StreamingParseResult, ToolCallItem @@ -222,7 +223,7 @@ def test_rebuilding_items_the_stream_never_opened_is_said_out_loud(): ) assert items[0].id.startswith("msg_") assert mock_logger.warning.called - assert "message" in mock_logger.warning.call_args[0] + assert "message" in mock_logger.warning.call_args[0][0] def test_streamed_items_the_rebuild_drops_are_said_out_loud_too(): @@ -585,3 +586,82 @@ def test_a_completed_stream_still_ends_with_response_completed(): frame = processor.get_final_response_non_store(result) assert _event_type(frame) == "response.completed" assert _event_data(frame)["response"]["incomplete_details"] is None + + +def _non_streaming_response(finish_reason): + """The JSON body a non-streaming request gets, for one finish reason.""" + request = ResponsesRequest(model="test-model", input="hi", stream=False) + result = _generation_chunk("Partial answer", "Partial answer", True) + result.outputs[0].finish_reason = finish_reason + return create_response_non_store( + generation_result=result, + request=request, + sampling_params=request.to_sampling_params(), + model_name="test-model", + use_harmony=False, + num_prompt_tokens=2, + ) + + +def test_a_non_streaming_truncated_response_carries_incomplete_details(): + """The JSON body must explain its status the way the stream already does. + + The streaming terminal event has paired status "incomplete" with + incomplete_details.reason="max_output_tokens" since the response.incomplete + fix, but the non-streaming body said only "incomplete" - the field was + commented out of ResponsesResponse - so a client polling the JSON body had + the status without the reason the spec promises alongside it. + """ + payload = _non_streaming_response("length").model_dump() + assert payload["status"] == "incomplete" + assert payload["incomplete_details"] == {"reason": "max_output_tokens"} + + +def test_a_non_streaming_completed_response_has_no_details(): + payload = _non_streaming_response("stop").model_dump() + assert payload["status"] == "completed" + assert payload["incomplete_details"] is None + + +def test_a_disagg_handoff_gets_no_invented_reason(): + """A "not_finished" incomplete explains nothing, on purpose. + + finish_reason "not_finished" also maps to "incomplete", but it is a + context worker handing the request off, not a token budget; naming a + public reason for an internal handoff would be a guess presented as fact. + """ + payload = _non_streaming_response("not_finished").model_dump() + assert payload["status"] == "incomplete" + assert payload["incomplete_details"] is None + + +def test_a_failed_generation_terminates_with_response_failed(): + """A failed response must not ship under response.completed. + + A timeout maps to status "failed"; an event named response.completed + around it asserts the one thing its payload denies. The spec pairs the + status with response.failed. + """ + processor = _processor(reasoning_parser=None) + result = _generation_chunk("Partial answer", "Partial answer", True) + result.outputs[0].finish_reason = "timeout" + frame = processor.get_final_response_non_store(result) + assert _event_type(frame) == "response.failed" + payload = _event_data(frame) + assert payload["type"] == "response.failed" + assert payload["response"]["status"] == "failed" + + +def test_a_cancelled_generation_still_ends_with_response_completed(): + """Deliberate, not an oversight. + + Cancellation is client-initiated and the Responses spec pairs no terminal + event with it, so inventing one would hand SDK clients an event type they + cannot parse. See _stream_terminal_event. + """ + processor = _processor(reasoning_parser=None) + result = _generation_chunk("Partial answer", "Partial answer", True) + result.outputs[0].finish_reason = "cancelled" + frame = processor.get_final_response_non_store(result) + assert _event_type(frame) == "response.completed" + assert _event_data(frame)["response"]["status"] == "cancelled" diff --git a/tests/unittest/llmapi/apps/test_responses_streaming_events.py b/tests/unittest/llmapi/apps/test_responses_streaming_events.py index a7e81f807002..93ea174b9e9f 100644 --- a/tests/unittest/llmapi/apps/test_responses_streaming_events.py +++ b/tests/unittest/llmapi/apps/test_responses_streaming_events.py @@ -778,3 +778,103 @@ def test_nothing_streamed_means_fresh_ids_rather_than_a_crash(): assert calls assert calls[0]["call_id"].startswith("call_") assert calls[0]["call_id"] != "call_streamed" + + +# --------------------------------------------------------------------------- +# Non-streaming rebuild: no phantom {}-argument calls +# --------------------------------------------------------------------------- +# +# The whole-text parse reports arguments `{}` both for a genuine zero-argument +# call and for a call whose argument markup its pair regex could not read - a +# GLM-4.7 block that opens and never closes it. The streamed +# assembly of the same text refuses the second (its arguments assemble to +# `{"cmd": }`), so a non-streaming request used to deliver a call the model +# never made while a streaming request of the same text delivered none. +# _verify_empty_calls_against_streaming reconciles the two when no stream ran. + + +def _glm47_tools(): + from openai.types.responses.tool import FunctionTool + + return [ + FunctionTool( + name="exec_command", + description="Run a command.", + parameters={"type": "object", "properties": {"cmd": {"type": "string"}}}, + strict=False, + type="function", + ), + FunctionTool( + name="get_time", description="Now.", parameters=None, strict=False, type="function" + ), + ] + + +def _glm47_snapshot(text, streamed_tool_call_ids=None): + from tensorrt_llm.serve.responses_utils import _create_output_content + + items, _messages, _reasoning_texts = _create_output_content( + _FakeRequestOutput(text), + reasoning_parser=None, + tool_parser="glm47", + tools=_glm47_tools(), + streamed_tool_call_ids=streamed_tool_call_ids, + ) + calls = [(i.name, i.arguments) for i in items if i.type == "function_call"] + texts = [i.content[0].text for i in items if i.type == "message"] + return calls, texts + + +_UNREADABLE_ARGS = ( + "exec_commandcmdls -la" +) + + +def test_non_streaming_unreadable_arg_markup_is_text_not_an_empty_call(): + """The auditor's shape: terminated call, never closed. + + The pair regex reads nothing out of the block, so the call used to ship + with arguments `{}` - and the client ran the tool with arguments the model + never wrote. The streamed view refuses the call; the whole-text view now + agrees, and the markup falls back into the message text so nothing the + model generated is lost. + """ + calls, texts = _glm47_snapshot("Let me check. " + _UNREADABLE_ARGS) + assert calls == [] + assert texts and "ls -la" in texts[0] + assert texts[0].startswith("Let me check.") + + +def test_non_streaming_unterminated_markup_stays_text(): + """Pin: markup with no closing tag already fell back to text on both views.""" + unterminated = "exec_commandcmdls -la" + calls, texts = _glm47_snapshot(unterminated) + assert calls == [] + assert texts == [unterminated] + + +def test_non_streaming_zero_argument_call_is_still_delivered(): + """Pin: `{}` from a genuinely argument-free call is the model's own call.""" + calls, texts = _glm47_snapshot("get_time") + assert calls == [("get_time", "{}")] + assert texts == [] + + +def test_non_streaming_real_call_survives_a_phantom_neighbor(): + good = ( + "exec_commandcmdpytest -q" + ) + calls, _texts = _glm47_snapshot(good + _UNREADABLE_ARGS) + assert calls == [("exec_command", '{"cmd": "pytest -q"}')] + + +def test_a_streamed_rebuild_is_untouched_by_the_reconciler(): + """A streamed request keeps its entity-pairing semantics. + + When a stream ran, the per-entity id record already settles the drop, and + the snapshot must keep describing the stream - which showed neither the + call nor its markup. + """ + calls, texts = _glm47_snapshot(_UNREADABLE_ARGS, streamed_tool_call_ids=[None]) + assert calls == [] + assert texts == [] From 1593a0d7536ad4ac34286466fe70168d27dee9c3 Mon Sep 17 00:00:00 2001 From: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> Date: Tue, 6 Oct 2026 01:37:59 +0000 Subject: [PATCH 23/43] [None][fix] Make Responses final snapshot agree with the streamed view An external replay audit of one production week (310k requests) against GLM-5.3 locked divergences between what the stream published and what the final snapshot / rendered prompt said. Streamed bytes are irrevocable, so wherever the two views read the same characters, the snapshot now repeats the stream. - Responses: the streamed item record now carries each item's delta text and each call's parser-level arguments. For streamed requests the snapshot assembles reasoning/message items verbatim from that record (same count, boundaries, ids, bytes) and, under entity alignment, keeps the streamed tool arguments when the whole-text re-parse read malformed markup differently. - Stop stripping edge whitespace in the snapshot: generated whitespace is part of the generation. - Emit the missing response.content_part.done for reasoning parts in both closing branches. - Merge consecutive function_call items of one assistant turn into a single assistant message so the chat template can bind tool results to calls by id (swapping two result call_ids used to render a byte-identical prompt). 1585 serving/parser unit tests pass; new regression tests cover every item above and fail on the unfixed code. Signed-off-by: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> --- tensorrt_llm/serve/responses_utils.py | 466 ++++++++++++++---- .../test_disagg_responses_api.py | 37 +- .../apps/test_responses_input_preprocess.py | 211 ++++++++ ...test_responses_stream_final_consistency.py | 263 +++++++++- .../apps/test_responses_streaming_events.py | 85 +++- 5 files changed, 948 insertions(+), 114 deletions(-) diff --git a/tensorrt_llm/serve/responses_utils.py b/tensorrt_llm/serve/responses_utils.py index e9562dadc660..a1014ae3fa1a 100644 --- a/tensorrt_llm/serve/responses_utils.py +++ b/tensorrt_llm/serve/responses_utils.py @@ -1,3 +1,4 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project @@ -11,8 +12,9 @@ from abc import ABC, abstractmethod from collections.abc import AsyncGenerator, Mapping from copy import copy -from typing import (Any, Callable, List, Literal, Optional, OrderedDict, Tuple, - Union) +from dataclasses import dataclass +from typing import (Any, Callable, List, Literal, NamedTuple, Optional, + OrderedDict, Tuple, Union) from openai.types.responses import (ResponseCompletedEvent, ResponseContentPartAddedEvent, @@ -36,6 +38,8 @@ PartReasoningText from openai.types.responses.response_content_part_done_event import \ Part as ResponseContentPart +from openai.types.responses.response_content_part_done_event import \ + PartReasoningText as PartReasoningTextDone from openai.types.responses.response_function_web_search import ( ActionFind, ActionOpenPage, ActionSearch, ResponseFunctionWebSearch) from openai.types.responses.response_reasoning_item import Content @@ -109,6 +113,52 @@ CUSTOM_TOOL_INPUT_ARG = "input" +class StreamedToolCall(NamedTuple): + """One tool call exactly as the stream delivered it. + + The final snapshot is a second, independent re-parse of the generated + text, and on malformed markup the incremental state machine and the + whole-text regex read *different arguments out of the same bytes* - two + consecutive ```` tags stream as ``{"input": ...}`` but re-parse + into a corrupted key ``inputcmd`` (2 live cases a + week). The ids alone cannot fix that: a call named consistently but + described differently still tells the client two stories. So the record + carries the name and the assembled arguments the deltas were built from, + and the rebuild defers to them; see ``_create_output_content``. + + ``name``/``arguments`` are the parser-level values + (``ToolCallItem.name``/``.parameters``), not the output item's fields: + feeding them back through ``_tool_call_output_item`` with the same tool + resolution reproduces the streamed item byte for byte, custom-tool + unwrapping included. ``None`` means the recorder was not given them, in + which case the rebuild keeps its own parse as before. + """ + item_id: str + call_id: str + name: Optional[str] = None + arguments: Optional[str] = None + + +@dataclass +class StreamedItem: + """One reasoning/message output item as the stream published it. + + ``text`` accumulates every delta emitted for the item - appended by the + same call that emits the delta, so it is exactly the characters the + client received, kept even if the stream is cut off before the item + closes. The final snapshot assembles its reasoning/message items from + these records (``_create_output_content``): a re-derive from the + accumulated text can disagree with the stream on segmentation (an + unterminated call's fallback streams as two message items the re-parse + merges into one; 10 live cases a week) and on whitespace (the re-parse + stripped what the stream kept; 716 a week), and what already reached the + client is the one account of the generation that cannot be revised. + """ + item_type: str + item_id: str + text: str = "" + + def _responses_debug_log(msg): if ENABLE_RESPONSES_DEBUG_MSG: logger.info(msg) @@ -1002,6 +1052,54 @@ def _response_output_item_to_chat_completion_message( return None +def _fold_tool_calls_into_open_assistant_turn( + messages: list[ChatCompletionMessageParam], + message: ChatCompletionMessageParam, turn_start: int) -> bool: + """Attach a converted tool-call item to the assistant message before it. + + A Responses assistant turn that made N calls arrives as N consecutive + ``function_call`` items (after any reasoning/message items), and each one + converts to an assistant chat message carrying a single tool call. + Appending those separately splits one turn across N assistant messages, + and the GLM chat template aligns tool results only against the LAST + assistant message's ``tool_calls`` - so results for every call but the + last matched nothing and were rendered positionally, WITHOUT their ids. + Proven consequence: swapping two function_call_output call_ids produced a + byte-identical 596k-char prompt, i.e. the call-result binding was + unrecoverable downstream. One assistant message carrying the ordered list + of all N calls is the chat-completions shape (the history replay above + already stores turns that way), and it is what lets the template bind + each result to its call by id. + + The fold targets only messages converted from this request's input - + ``turn_start`` marks where those begin - so the instructions message and + replayed history are never mutated. A tool result or a user message + between two calls ends the assistant turn naturally: ``messages[-1]`` is + then no longer an assistant message and the next call opens a fresh one. + Preceding assistant text or reasoning stays in the same message, which is + where chat completions carries it alongside ``tool_calls``. + """ + if message.get("role") != "assistant" or not message.get("tool_calls"): + return False + if message.get("content") is not None: + # Not a converted call item: both tool-call conversions carry + # content=None, and a message with its own content is a turn of its + # own, not a fragment of the previous one. + return False + if len(messages) <= turn_start: + return False + last = messages[-1] + if last.get("role") != "assistant": + return False + # Rebuilt rather than extended in place: the target may carry + # "tool_calls" as None (clients send extra fields verbatim), and a list a + # caller handed in is not this function's to mutate. + last["tool_calls"] = [ + *(last.get("tool_calls") or []), *message["tool_calls"] + ] + return True + + async def _create_input_messages( request: ResponsesRequest, prev_msgs: list[ChatCompletionMessageParam], @@ -1040,10 +1138,20 @@ async def _create_input_messages( if isinstance(request.input, str): messages.append({"role": "user", "content": request.input}) else: + # Where this request's own items start: the fold below must never + # reach back into the instructions message or the replayed history. + turn_start = len(messages) for inp in request.input: message = _response_output_item_to_chat_completion_message(inp) - if message is not None: - messages.append(message) + if message is None: + continue + # N calls from one assistant turn become ONE assistant message + # with N tool_calls, or the template loses the call-result + # binding; see the fold's docstring. + if _fold_tool_calls_into_open_assistant_turn( + messages, message, turn_start): + continue + messages.append(message) return messages @@ -1677,14 +1785,42 @@ def _apply_tool_parser( return normal_text, calls +def _streamed_items_cover(streamed_items: list[StreamedItem], + reasoning_text: Optional[str], + text: Optional[str]) -> bool: + """Whether the stream's item record accounts for the re-parsed content. + + True means the two passes read the same characters out of the generation + and differ at most in segmentation (the stream can split one fallback + text across two message items) or in edge whitespace (the whole-text + GLM-4.7 parse strips what the stream kept) - exactly the differences the + streamed record exists to settle. False means the record cannot describe + this rebuild: text the stream never published (a cut-off stream whose + tail only the accumulated text holds) or a reasoning/content split the + two parsers genuinely disagree on. The caller then falls back to the + re-derived items, which is the one choice that loses no generated text, + and the structural mismatch is warned about as before. + + Compared per item type and stripped on both sides: whitespace inside the + joined streamed text is preserved by the join itself, so only the edges - + the part the whole-text parser may have stripped - are forgiven. + """ + streamed_reasoning = "".join(item.text for item in streamed_items + if item.item_type == "reasoning") + streamed_message = "".join(item.text for item in streamed_items + if item.item_type == "message") + return (streamed_reasoning.strip() == (reasoning_text or "").strip() + and streamed_message.strip() == (text or "").strip()) + + def _create_output_content( final_res: RequestOutput, reasoning_parser: Optional[str] = None, tool_parser: Optional[str] = None, tools: Optional[list[Tool]] = None, chat_template_kwargs: Optional[dict[str, Any]] = None, - streamed_tool_call_ids: Optional[list[Optional[Tuple[str, str]]]] = None, - streamed_item_ids: Optional[list[Tuple[str, str]]] = None, + streamed_tool_call_ids: Optional[list[Optional[StreamedToolCall]]] = None, + streamed_item_ids: Optional[list[StreamedItem]] = None, ) -> Tuple[list[ResponseOutputItem], list[ChatCompletionMessageParam], list[str]]: output_items: list[ResponseOutputItem] = [] @@ -1695,25 +1831,36 @@ def _create_output_content( reasoning_texts: list[str] = [] available_tools = _get_chat_completion_function_tools(tools) - # The (item type, item id) pairs the stream published in - # `response.output_item.added/done`, in emission order. This rebuild is a - # second, independent pass over the generated text and used to mint fresh - # ids for the reasoning and message items it re-derives, so the snapshot - # named the very items the stream had already announced under different - # ids - 216 of 221 measured responses - and a client joining streamed - # items with the snapshot by id saw phantom items. Reuse is positional - # per item type, first-emitted first, because emission order is the only + # The reasoning/message items the stream published in + # `response.output_item.added/done`, in emission order and with the text + # their deltas carried. This rebuild is a second, independent pass over + # the generated text and used to mint fresh ids for the items it + # re-derives, so the snapshot named the very items the stream had already + # announced under different ids - 216 of 221 measured responses - and a + # client joining streamed items with the snapshot by id saw phantom + # items. When the record and the re-parse read the same characters the + # snapshot now repeats the record outright (see the assembly below); the + # per-type pools here serve the fallback, where reuse is positional per + # item type, first-emitted first, because emission order is the only # correspondence the two passes share. None means no stream ran (a # non-streaming request) and every id is minted fresh, as before. streamed_ids_by_type: dict[str, list[str]] = { "reasoning": [], "message": [] } - for item_type, item_id in streamed_item_ids or []: - if item_type in streamed_ids_by_type: - streamed_ids_by_type[item_type].append(item_id) + for record in streamed_item_ids or []: + if record.item_type in streamed_ids_by_type: + streamed_ids_by_type[record.item_type].append(record.item_id) used_ids_by_type = {"reasoning": 0, "message": 0} + # The record is per request while this loop is per output, and a stream + # only ever runs over outputs[0] (process_single_output), so the whole + # record describes a single output. More than one output means no stream + # produced it and the positional id reuse below is all there is. + streamed_items_available = (streamed_item_ids is not None + and len(final_res.outputs) == 1) + used_streamed_assembly = False + def _streamed_or_fresh_id(item_type: str) -> str: # Running past the pool means this pass derived items the stream never # opened - the two views already diverged structurally (a stream cut @@ -1758,9 +1905,9 @@ def _streamed_or_fresh_id(item_type: str) -> str: text, calls = _verify_empty_calls_against_streaming( tool_parser, available_tools, parsed_source, text, calls) - text_item = None - reasoning_item = None tool_calls_item = [] + stored_text: Optional[str] = None + stored_reasoning: Optional[str] = None # Reasoning first, then the answer, then any tool calls. # @@ -1772,36 +1919,99 @@ def _streamed_or_fresh_id(item_type: str) -> str: # generation: a client replaying `output` read the answer before the # reasoning that produced it, and one reconstructing a turn from the # snapshot fed the model its own thinking as a follow-up to its reply. - if reasoning_text: - reasoning_item = ResponseReasoningItem( - id=_streamed_or_fresh_id("reasoning"), - summary=[], - type="reasoning", - content=[ - Content(text=reasoning_text.strip(), type="reasoning_text") - ], - status=None, - ) - output_items.append(reasoning_item) + # + # For a streamed request the items themselves come from the stream's + # own record, not from the re-parse above. The re-parse can disagree + # with the stream on item boundaries - an unterminated call's fallback + # streams as two message items that a whole-text parse can only read + # as one, so the second streamed item simply vanished from the + # snapshot (10 live cases a week) - and on edge whitespace, which it + # stripped while the stream had already sent it (` Backbone` on the + # wire, `Backbone` in the snapshot; 716 a week). What was streamed is + # irrevocable, so when the two passes read the same characters the + # snapshot repeats the stream's items verbatim: same count, same + # boundaries, same ids, same bytes. When they did not read the same + # characters the views genuinely diverged (a stream cut off while the + # accumulated text kept growing), and the re-derived items are kept - + # the choice that loses no generated text - with the mismatch warned + # about below, as before. + if streamed_items_available and _streamed_items_cover( + streamed_item_ids, reasoning_text, text): + used_streamed_assembly = True + for record in streamed_item_ids: + if record.item_type == "reasoning": + output_items.append( + ResponseReasoningItem( + id=record.item_id, + summary=[], + type="reasoning", + content=[ + Content(text=record.text, type="reasoning_text") + ], + status=None, + )) + else: + output_items.append( + ResponseOutputMessage( + id=record.item_id, + content=[ + ResponseOutputText( + text=record.text, + annotations=[], + type="output_text", + logprobs=None, + ) + ], + role="assistant", + status="completed", + type="message", + )) + # The conversation store keeps one assistant message per output, + # so the per-item boundaries flatten back out here - but to the + # streamed characters, not the re-parsed ones. + stored_reasoning = "".join( + record.text for record in streamed_item_ids + if record.item_type == "reasoning") or None + stored_text = "".join(record.text for record in streamed_item_ids + if record.item_type == "message") or None + else: + if reasoning_text: + reasoning_item = ResponseReasoningItem( + id=_streamed_or_fresh_id("reasoning"), + summary=[], + type="reasoning", + # Verbatim: the whitespace around a reasoning block was + # generated too, and the streamed view of the same + # generation delivers it. A strip here made the two views + # disagree about the model's own characters. + content=[ + Content(text=reasoning_text, type="reasoning_text") + ], + status=None, + ) + output_items.append(reasoning_item) + stored_reasoning = reasoning_text - # Check again after tool parsing to avoid empty text - if text: - output_text = ResponseOutputText( - text=text.strip(), - annotations=[], - type="output_text", - logprobs=None, - ) + # Check again after tool parsing to avoid empty text + if text: + output_text = ResponseOutputText( + # Verbatim, for the same reason as the reasoning above. + text=text, + annotations=[], + type="output_text", + logprobs=None, + ) - text_item = ResponseOutputMessage( - id=_streamed_or_fresh_id("message"), - content=[output_text], - role="assistant", - status="completed", - type="message", - ) + text_item = ResponseOutputMessage( + id=_streamed_or_fresh_id("message"), + content=[output_text], + role="assistant", + status="completed", + type="message", + ) - output_items.append(text_item) + output_items.append(text_item) + stored_text = text if calls: tool_resolution = _tool_resolution(tools) @@ -1827,11 +2037,11 @@ def _streamed_or_fresh_id(item_type: str) -> str: f"final response rebuilt {len(calls)} tool call(s) but the " f"stream saw {len(reusable)}; ids beyond the streamed ones " f"are new and will not match the client's tool outputs") - reusable = [ids for ids in reusable if ids is not None] + reusable = [record for record in reusable if record is not None] tool_calls_item = [] for index, call in enumerate(calls): - ids = reusable[index] if index < len(reusable) else (None, None) - if entity_aligned and ids is None: + record = reusable[index] if index < len(reusable) else None + if entity_aligned and record is None: # This entity is the call the stream dropped: its # arguments never assembled into valid JSON and no id was # published for it. The snapshot must describe the stream, @@ -1844,27 +2054,59 @@ def _streamed_or_fresh_id(item_type: str) -> str: f"and never delivered it, so the snapshot does not " f"deliver it either") continue + # Same call entity, two parses. On malformed markup they read + # different arguments out of the same bytes - two consecutive + # `` tags stream as `{"input": ...}` while the + # whole-text regex delivers a corrupted key - and the client + # has already received, and possibly executed, the streamed + # version; a snapshot that re-describes the call under the + # same call_id hands the client two different inputs for one + # identity (2 live cases a week). The streamed bytes are the + # ones that cannot be unsent, so they win. Only under entity + # alignment: when the two passes disagree about what the call + # entities *are*, pairing a streamed description with a + # re-parsed call would guess, and the ids-only reuse above is + # as far as certainty goes. + streamed_name = getattr(record, "name", None) + streamed_arguments = getattr(record, "arguments", None) + if (entity_aligned and streamed_name is not None + and streamed_arguments is not None): + if (call.name != streamed_name + or (call.parameters or "") != streamed_arguments): + logger.warning( + f"final response re-parsed the tool call to " + f"{call.name!r} differently from the stream " + f"(streamed arguments " + f"{streamed_arguments[:200]!r}, re-parsed " + f"{(call.parameters or '')[:200]!r}); keeping the " + f"streamed name and arguments, which the client " + f"already received") + call = ToolCallItem(tool_index=call.tool_index, + name=streamed_name, + parameters=streamed_arguments) tool_calls_item.append( - _tool_call_output_item(call, - tool_resolution, - item_id=ids[0], - call_id=ids[1])) + _tool_call_output_item( + call, + tool_resolution, + item_id=record.item_id if record else None, + call_id=record.call_id if record else None)) output_items.extend(tool_calls_item) output_messages.extend( _create_output_messages({ - "text_content": - text_item.content[0].text if text_item else None, - "reasoning_content": - reasoning_item.content[0].text if reasoning_item else None, - "tool_calls": - tool_calls_item, + "text_content": stored_text, + "reasoning_content": stored_reasoning, + "tool_calls": tool_calls_item, })) # Checked against None, not truthiness: an empty list still means a # stream ran and published nothing of this type, and a rebuild that then # derives such an item is exactly the divergence worth saying out loud. - if streamed_item_ids is not None: + # Skipped when the snapshot was assembled from the streamed record: the + # counts then match by construction and the only mismatch left - the + # re-parse segmenting the same characters differently - is the one the + # record exists to settle, not a divergence. + if streamed_item_ids is not None and not used_streamed_assembly: for item_type in ("reasoning", "message"): streamed = len(streamed_ids_by_type[item_type]) rebuilt = used_ids_by_type[item_type] @@ -2166,8 +2408,8 @@ def _create_response( reasoning_parser: Optional[str] = None, tool_parser: Optional[str] = None, num_prompt_tokens: Optional[int] = None, - streamed_tool_call_ids: Optional[list[Optional[Tuple[str, str]]]] = None, - streamed_item_ids: Optional[list[Tuple[str, str]]] = None, + streamed_tool_call_ids: Optional[list[Optional[StreamedToolCall]]] = None, + streamed_item_ids: Optional[list[StreamedItem]] = None, tokenizer: Optional[TokenizerBase] = None, ) -> tuple[ResponsesResponse, list[Message | ChatCompletionMessageParam]]: _responses_debug_log("================================================") @@ -2255,8 +2497,8 @@ async def create_response( reasoning_parser: Optional[str] = None, tool_parser: Optional[str] = None, num_prompt_tokens: Optional[int] = None, - streamed_tool_call_ids: Optional[list[Optional[Tuple[str, str]]]] = None, - streamed_item_ids: Optional[list[Tuple[str, str]]] = None, + streamed_tool_call_ids: Optional[list[Optional[StreamedToolCall]]] = None, + streamed_item_ids: Optional[list[StreamedItem]] = None, tokenizer: Optional[TokenizerBase] = None, ) -> ResponsesResponse: @@ -2307,8 +2549,8 @@ def create_response_non_store( reasoning_parser: Optional[str] = None, tool_parser: Optional[str] = None, num_prompt_tokens: Optional[int] = None, - streamed_tool_call_ids: Optional[list[Optional[Tuple[str, str]]]] = None, - streamed_item_ids: Optional[list[Tuple[str, str]]] = None, + streamed_tool_call_ids: Optional[list[Optional[StreamedToolCall]]] = None, + streamed_item_ids: Optional[list[StreamedItem]] = None, tokenizer: Optional[TokenizerBase] = None, ) -> ResponsesResponse: response_creation_time = create_time if create_time is not None else int( @@ -2377,19 +2619,24 @@ def __init__(self) -> None: # calls behind it. # # A list here rather than a class attribute, for the reason above. - self.emitted_tool_call_ids: list[Optional[Tuple[str, str]]] = [] - - # (item type, item id) of every reasoning and message item this - # stream opened, in the order their `output_item.added` went out. - # The final response re-derives those items from the accumulated text - # and used to mint fresh ids for them, so the snapshot named the very + self.emitted_tool_call_ids: list[Optional[StreamedToolCall]] = [] + + # Every reasoning and message item this stream opened, in the order + # their `output_item.added` went out, each carrying the text its + # deltas delivered (appended as they are emitted; see + # ResponsesStreamingEventsHelper.append_text/append_reasoning). The + # final response re-derives those items from the accumulated text and + # used to mint fresh ids for them, so the snapshot named the very # items the stream had already announced under different ids - 216 of # 221 measured responses - and a client joining streamed items with - # the snapshot by id saw phantom items. Same shape and cure as + # the snapshot by id saw phantom items. The texts travel with the ids + # because the re-derive can also disagree on item boundaries and on + # edge whitespace; the snapshot assembles these records verbatim + # instead (see _create_output_content). Same shape and cure as # emitted_tool_call_ids above. # # A list here rather than a class attribute, for the reason above. - self.emitted_item_ids: list[Tuple[str, str]] = [] + self.emitted_item_ids: list[StreamedItem] = [] class ResponsesStreamingEventsHelper: @@ -2418,8 +2665,8 @@ def emitted_tool_calls(self, count: int) -> None: self.state_tracker.emitted_tool_calls = count @property - def emitted_tool_call_ids(self) -> list[Optional[Tuple[str, str]]]: - """(item id, call id) of the tool calls already streamed, in order. + def emitted_tool_call_ids(self) -> list[Optional[StreamedToolCall]]: + """The tool calls already streamed, in order; see StreamedToolCall. None entries hold the place of calls the stream assembled and then dropped for invalid-JSON arguments; see the state tracker. @@ -2427,18 +2674,27 @@ def emitted_tool_call_ids(self) -> list[Optional[Tuple[str, str]]]: return self.state_tracker.emitted_tool_call_ids @property - def emitted_item_ids(self) -> list[Tuple[str, str]]: - """(item type, item id) of the reasoning/message items streamed, in order.""" + def emitted_item_ids(self) -> list[StreamedItem]: + """The reasoning/message items streamed, in order; see StreamedItem.""" return self.state_tracker.emitted_item_ids - def record_emitted_tool_call(self, item) -> None: - """Remember the ids a streamed call went out under. + def record_emitted_tool_call(self, + item, + call: Optional[ToolCallItem] = None) -> None: + """Remember the identity and content a streamed call went out under. The final response is built by a second, independent pass over the generated text; handing it these lets it name the same call the same - way instead of minting new ids the client has never seen. + way instead of minting new ids the client has never seen - and, when + `call` (the parser-level ToolCallItem the item was built from) is + given, describe it with the same name and arguments instead of + whatever its own re-parse read out of malformed markup. """ - self.state_tracker.emitted_tool_call_ids.append((item.id, item.call_id)) + self.state_tracker.emitted_tool_call_ids.append( + StreamedToolCall(item_id=item.id, + call_id=item.call_id, + name=getattr(call, "name", None), + arguments=getattr(call, "parameters", None))) def record_dropped_tool_call(self) -> None: """Hold a dropped call's place in the emission record. @@ -2452,11 +2708,26 @@ def record_dropped_tool_call(self) -> None: """ self.state_tracker.emitted_tool_call_ids.append(None) + def _record_item_delta(self, item_type: str, delta: str) -> None: + # The emission record mirrors every delta the moment it is appended, + # so the record holds exactly the characters the client received for + # the item - including for an item the stream was cut off inside, + # which no close will ever flush. The open item is always the last + # record of its type to have been announced (one item is open at a + # time, and the opener runs before the first append); when no record + # matches, nothing was announced and there is nothing to mirror, + # which is the state the buffer-only unit tests drive this in. + records = self.state_tracker.emitted_item_ids + if records and records[-1].item_type == item_type: + records[-1].text += delta + def append_text(self, delta: str) -> None: self.state_tracker.text_buffer += delta + self._record_item_delta("message", delta) def append_reasoning(self, delta: str) -> None: self.state_tracker.reasoning_buffer += delta + self._record_item_delta("reasoning", delta) def take_text(self) -> str: text = self.state_tracker.text_buffer @@ -2620,9 +2891,13 @@ def _get_output_added_events( # where announcing actually happens. The final response is built # by a second, independent pass over the generated text; handing # it these lets it name the same items the same way instead of - # minting ids the client has never seen. + # minting ids the client has never seen. The record's text starts + # empty and accumulates with every delta (append_text / + # append_reasoning), so the snapshot can also repeat the item's + # characters rather than re-deriving them. self.state_tracker.emitted_item_ids.append( - (output_item.type, output_item.id)) + StreamedItem(item_type=output_item.type, + item_id=output_item.id)) if output_item.type == "message": content_part = ResponseOutputText( @@ -2924,6 +3199,16 @@ def _close_open_item(helper): status="completed", ) yield helper.get_reasoning_text_done_event(text) + # The reasoning part was opened with content_part.added (see + # _get_output_added_events), so it has to close with + # content_part.done - before the item's own done, which is OpenAI's + # order. The message branch below has always done this; reasoning + # omitted it, leaving every reasoning turn with one more + # content_part.added than content_part.done (~294k unpaired events a + # week) and clients tracking part state holding a part that never + # ends. + yield helper.get_content_part_done_event( + PartReasoningTextDone(type="reasoning_text", text=text)) yield helper.get_output_item_done_event(item) helper.is_reasoning_sent = False else: @@ -3154,6 +3439,12 @@ def check_parser(parser_id: Optional[str], ) yield streaming_events_helper.get_reasoning_text_done_event( reasoning_text) + # Close the reasoning part the opener added, before the item's + # done - the same pairing _close_open_item restores and the + # message branch below has always had. + yield streaming_events_helper.get_content_part_done_event( + PartReasoningTextDone(type="reasoning_text", + text=reasoning_text)) yield streaming_events_helper.get_output_item_done_event( reasoning_item) streaming_events_helper.is_reasoning_sent = False @@ -3240,7 +3531,12 @@ def check_parser(parser_id: Optional[str], tool_call_item) yield streaming_events_helper.get_output_item_done_event( tool_call_item) - streaming_events_helper.record_emitted_tool_call(tool_call_item) + # `call` rides along so the record holds the name and + # assembled arguments these events were built from; the final + # snapshot repeats them rather than trusting its own re-parse + # of the same markup (see _create_output_content). + streaming_events_helper.record_emitted_tool_call( + tool_call_item, call) streaming_events_helper.output_index_increment() streaming_events_helper.emitted_tool_calls = len(entities) streaming_events_helper.is_output_item_added_sent = False diff --git a/tests/unittest/disaggregated/test_disagg_responses_api.py b/tests/unittest/disaggregated/test_disagg_responses_api.py index b12914601dc6..114610c98d44 100644 --- a/tests/unittest/disaggregated/test_disagg_responses_api.py +++ b/tests/unittest/disaggregated/test_disagg_responses_api.py @@ -40,7 +40,9 @@ pytestmark = pytest.mark.cpu_only -def _responses_response(finish_reason="length", disagg=True, prompt_token_ids=None): +def _responses_response( + finish_reason="length", disagg=True, prompt_token_ids=None, status="incomplete" +): return ResponsesResponse( model="m", output=[], @@ -51,7 +53,7 @@ def _responses_response(finish_reason="length", disagg=True, prompt_token_ids=No top_p=1.0, background=False, service_tier="auto", - status="incomplete", + status=status, top_logprobs=0, truncation="disabled", finish_reason=finish_reason, @@ -201,17 +203,20 @@ async def collect(): return asyncio.run(collect()) -def test_ctx_only_stream_ends_with_response_completed(): +def test_ctx_only_stream_ends_with_a_terminal_event(): """Regression: the completions terminator hangs a Responses client. The Responses protocol has no ``[DONE]`` sentinel -- a client watches for - ``response.completed``. Emitting ``data: [DONE]`` leaves it waiting for an + a terminal event. Emitting ``data: [DONE]`` leaves it waiting for an event that never arrives, and drops the answer the context worker already - produced. + produced. A response that ran to its natural end terminates with + ``response.completed``. """ from tensorrt_llm.serve.responses_utils import responses_done_generator - chunks = _drain(responses_done_generator(_responses_response())) + chunks = _drain( + responses_done_generator(_responses_response(finish_reason="stop", status="completed")) + ) body = b"".join(chunks).decode("utf-8") assert "event: response.completed" in body @@ -220,6 +225,26 @@ def test_ctx_only_stream_ends_with_response_completed(): assert body.index("response.created") < body.index("response.completed") +def test_ctx_only_truncated_replay_terminates_as_incomplete(): + """A replayed response cut off at its token budget says so. + + The terminal event is chosen by status, exactly as on every other + completed stream (_stream_terminal_event): a ``response.completed`` + around a status of "incomplete" asserts the one thing its payload + denies. The fixture's default is the truncated shape this generator + replays when the context phase finished the whole request. + """ + from tensorrt_llm.serve.responses_utils import responses_done_generator + + chunks = _drain(responses_done_generator(_responses_response())) + body = b"".join(chunks).decode("utf-8") + + assert "event: response.incomplete" in body + assert "max_output_tokens" in body + assert "[DONE]" not in body + assert body.index("response.created") < body.index("response.incomplete") + + def test_ctx_only_stream_events_are_sse_framed(): from tensorrt_llm.serve.responses_utils import responses_done_generator diff --git a/tests/unittest/llmapi/apps/test_responses_input_preprocess.py b/tests/unittest/llmapi/apps/test_responses_input_preprocess.py index d38ce607077c..ef37db50c49d 100644 --- a/tests/unittest/llmapi/apps/test_responses_input_preprocess.py +++ b/tests/unittest/llmapi/apps/test_responses_input_preprocess.py @@ -846,3 +846,214 @@ def test_other_roles_are_left_alone(): } ) assert msg["role"] == role + + +# --------------------------------------------------------------------------- +# A multi-call assistant turn stays one assistant message +# +# N function_call items from one turn used to convert into N assistant +# messages of one tool call each. The GLM chat template aligns tool results +# only against the LAST assistant message's tool_calls and, when alignment +# fails, renders the results positionally WITHOUT their ids - proven on live +# traffic by swapping two function_call_output call_ids and getting a +# byte-identical 596k-char prompt. The binding has to survive conversion: +# one assistant message, all N calls, in order. +# --------------------------------------------------------------------------- + + +def _function_call_item(index): + return { + "type": "function_call", + "id": f"fc_{index}", + "call_id": f"call_{index}", + "name": "exec", + "arguments": f'{{"cmd": "step {index}"}}', + "status": "completed", + } + + +def _function_call_output_item(index, call_id=None): + return { + "type": "function_call_output", + "call_id": call_id or f"call_{index}", + "output": f"result {index}", + } + + +def _five_call_turn(swap=False): + """The acceptance pattern: text, five calls, five results. + + With swap=True the first two results trade call_ids, which changes which + call each result answers - the counterfactual that used to render to a + byte-identical prompt. + """ + outputs = [_function_call_output_item(i) for i in range(5)] + if swap: + outputs[0]["call_id"], outputs[1]["call_id"] = ( + outputs[1]["call_id"], + outputs[0]["call_id"], + ) + return { + "input": [ + _message_item("user", "run the plan", item_id=None), + { + "id": "msg_a", + "status": "completed", + "type": "message", + "role": "assistant", + "content": [ + {"type": "output_text", "text": "Running all five.", "annotations": []} + ], + }, + *[_function_call_item(i) for i in range(5)], + *outputs, + ] + } + + +def _glm_like_render(messages): + """A minimal stand-in for the GLM template's result alignment. + + Tool results are matched only against the LAST assistant message's + tool_calls; a result whose call_id is found renders bound to that call's + arguments, an unmatched one renders positionally with no id - which is + the information loss the real template exhibits. + """ + rendered = [] + last_tool_calls = [] + for message in messages: + if message.get("role") == "assistant": + last_tool_calls = message.get("tool_calls") or [] + for call in last_tool_calls: + rendered.append(f"") + elif message.get("role") == "tool": + by_id = {call["id"]: call for call in last_tool_calls} + call = by_id.get(message.get("tool_call_id")) + bound = call["function"]["arguments"] if call else "UNBOUND" + rendered.append(f"") + return "".join(rendered) + + +def test_five_calls_from_one_turn_become_one_assistant_message(): + messages = _messages(_five_call_turn()) + + assert [m["role"] for m in messages] == ["user", "assistant"] + ["tool"] * 5 + assistant = messages[1] + # The turn's own text survives in the same message, alongside the calls. + assert assistant["content"] == "Running all five." + assert [c["id"] for c in assistant["tool_calls"]] == [f"call_{i}" for i in range(5)] + assert [c["function"]["arguments"] for c in assistant["tool_calls"]] == [ + f'{{"cmd": "step {i}"}}' for i in range(5) + ] + # Every result still names the call it answers. + assert [m["tool_call_id"] for m in messages[2:]] == [f"call_{i}" for i in range(5)] + + +def test_swapping_two_result_ids_now_changes_the_binding(): + """The counterfactual that used to be invisible. + + Same items, two results trading call_ids: the message lists must differ, + and a GLM-like alignment must bind the results to different calls - not + render both variants identically because neither could be aligned. + """ + straight = _messages(_five_call_turn()) + swapped = _messages(_five_call_turn(swap=True)) + + assert straight != swapped + assert _glm_like_render(straight) != _glm_like_render(swapped) + # And not because anything fell off: both bind every result. + assert "UNBOUND" not in _glm_like_render(straight) + assert "UNBOUND" not in _glm_like_render(swapped) + + +def test_the_old_per_call_split_is_what_lost_the_binding(): + """Why the fold exists, demonstrated on the unfolded shape. + + Converting each item separately - the old behavior - leaves the last + assistant message holding only call 4, so results 0-3 align with nothing + and the swapped variant renders byte-identically: the binding is + unrecoverable downstream, which is exactly what the fold prevents. + """ + + def _unfolded(request_kwargs): + return [ + m + for m in ( + _response_output_item_to_chat_completion_message(item) + for item in request_kwargs["input"] + ) + if m is not None + ] + + straight = _unfolded(_five_call_turn()) + swapped = _unfolded(_five_call_turn(swap=True)) + assert _glm_like_render(straight) == _glm_like_render(swapped) + assert "UNBOUND" in _glm_like_render(straight) + + +def test_a_tool_result_between_calls_ends_the_assistant_turn(): + """Sequential call/result pairs are separate turns and stay separate.""" + messages = _messages( + { + "input": [ + _function_call_item(0), + _function_call_output_item(0), + _function_call_item(1), + _function_call_output_item(1), + ] + } + ) + assert [m["role"] for m in messages] == ["assistant", "tool", "assistant", "tool"] + assert [len(m.get("tool_calls") or []) for m in messages] == [1, 0, 1, 0] + + +def test_a_user_message_between_calls_ends_the_assistant_turn(): + messages = _messages( + { + "input": [ + _function_call_item(0), + _message_item("user", "wait, also do this"), + _function_call_item(1), + ] + } + ) + assert [m["role"] for m in messages] == ["assistant", "user", "assistant"] + + +def test_calls_fold_onto_a_reasoning_message(): + """Reasoning-then-calls is the GLM turn shape. + + The calls ride the same assistant message, which is also how the + conversation store keeps them. + """ + messages = _messages( + { + "input": [ + { + "type": "reasoning", + "content": [{"type": "reasoning_text", "text": "plan both"}], + }, + _function_call_item(0), + _function_call_item(1), + ] + } + ) + assert len(messages) == 1 + assert messages[0]["reasoning"] == "plan both" + assert [c["id"] for c in messages[0]["tool_calls"]] == ["call_0", "call_1"] + + +def test_calls_never_fold_into_replayed_history(): + """The history's messages are stored turns, not this turn's opening.""" + import asyncio + + from tensorrt_llm.serve.responses_utils import _create_input_messages + + request = ResponsesRequest(model="m", input=[_function_call_item(0)]) + history = [{"role": "assistant", "content": "a stored turn"}] + messages = asyncio.run(_create_input_messages(request=request, prev_msgs=history)) + + assert messages[0] == {"role": "assistant", "content": "a stored turn"} + assert "tool_calls" not in messages[0] + assert messages[1]["role"] == "assistant" + assert [c["id"] for c in messages[1]["tool_calls"]] == ["call_0"] diff --git a/tests/unittest/llmapi/apps/test_responses_stream_final_consistency.py b/tests/unittest/llmapi/apps/test_responses_stream_final_consistency.py index 059250ba2881..a3a5d883fa5a 100644 --- a/tests/unittest/llmapi/apps/test_responses_stream_final_consistency.py +++ b/tests/unittest/llmapi/apps/test_responses_stream_final_consistency.py @@ -42,6 +42,7 @@ from tensorrt_llm.serve.openai_protocol import ResponsesRequest from tensorrt_llm.serve.responses_utils import ( ResponsesStreamingProcessor, + StreamedItem, _create_output_content, create_response_non_store, stamp_sse_sequence_number, @@ -165,7 +166,9 @@ def test_streamed_ids_are_reused_positionally_in_emission_order(): """Two items of one type map onto the rebuild first-emitted first. Emission order is the only correspondence the two passes share, which is - also how the tool-call precedent matches its calls. + also how the tool-call precedent matches its calls. Two outputs mean no + stream produced the record (streaming reads outputs[0] alone), so this is + the positional-reuse fallback, not the record assembly. """ items, _messages, _reasoning = _create_output_content( SimpleNamespace( @@ -175,7 +178,10 @@ def test_streamed_ids_are_reused_positionally_in_emission_order(): ] ), reasoning_parser=None, - streamed_item_ids=[("message", "msg_first"), ("message", "msg_second")], + streamed_item_ids=[ + StreamedItem("message", "msg_first"), + StreamedItem("message", "msg_second"), + ], ) assert [(i.type, i.id) for i in items] == [ ("message", "msg_first"), @@ -187,7 +193,10 @@ def test_reasoning_and_message_ids_are_reused_per_type(): items, _messages, _reasoning = _create_output_content( SimpleNamespace(outputs=[SimpleNamespace(index=0, text="thinkanswer")]), reasoning_parser="glm", - streamed_item_ids=[("reasoning", "rs_r1"), ("message", "msg_m1")], + streamed_item_ids=[ + StreamedItem("reasoning", "rs_r1", "think"), + StreamedItem("message", "msg_m1", "answer"), + ], ) assert [(i.type, i.id) for i in items] == [ ("reasoning", "rs_r1"), @@ -227,17 +236,55 @@ def test_rebuilding_items_the_stream_never_opened_is_said_out_loud(): def test_streamed_items_the_rebuild_drops_are_said_out_loud_too(): - """The mismatch matters in both directions; the first id is still reused.""" + """The mismatch matters in both directions; the first id is still reused. + + The streamed texts here cannot account for the rebuilt content - the + second item's tail exists nowhere in the accumulated text - so this is a + genuine divergence, not a segmentation difference: the rebuild keeps its + own (complete) view, reuses what ids it can, and says so. + """ with patch("tensorrt_llm.serve.responses_utils.logger") as mock_logger: items, _messages, _reasoning = _create_output_content( SimpleNamespace(outputs=[SimpleNamespace(index=0, text="answer")]), reasoning_parser=None, - streamed_item_ids=[("message", "msg_first"), ("message", "msg_second")], + streamed_item_ids=[ + StreamedItem("message", "msg_first", "answer"), + StreamedItem("message", "msg_second", " and a tail the text never held"), + ], ) assert [i.id for i in items] == ["msg_first"] assert mock_logger.warning.called +def test_matching_streamed_items_are_repeated_rather_than_rederived(): + """The record assembly itself, at the unit level. + + When the streamed texts read the same characters the re-parse does, the + snapshot repeats the streamed items verbatim - count, boundaries, ids and + bytes - instead of re-deriving one merged item. The boundary between + "the" and " answer" exists only in the record; a re-parse of the + accumulated text has no way to find it. + """ + with patch("tensorrt_llm.serve.responses_utils.logger") as mock_logger: + items, messages, _reasoning = _create_output_content( + SimpleNamespace(outputs=[SimpleNamespace(index=0, text="the answer")]), + reasoning_parser=None, + streamed_item_ids=[ + StreamedItem("message", "msg_first", "the"), + StreamedItem("message", "msg_second", " answer"), + ], + ) + assert [(i.type, i.id, i.content[0].text) for i in items] == [ + ("message", "msg_first", "the"), + ("message", "msg_second", " answer"), + ] + # Same characters, same views: nothing to warn about. + assert not mock_logger.warning.called + # The conversation store flattens the items back into one assistant + # message, built from the streamed characters. + assert messages == [{"role": "assistant", "content": "the answer"}] + + # --------------------------------------------------------------------------- # Sequence numbers: one monotonic counter per response, stamped at the egress # --------------------------------------------------------------------------- @@ -367,23 +414,24 @@ def test_an_unterminated_call_reaches_both_views_as_the_same_text(): assert all(item_type != "function_call" for item_type, _ in _streamed_items(frames)) final = _final_output(processor, result) + # The snapshot repeats the stream's items: same count, same boundaries, + # same texts. A whole-text re-parse can only read the fallback as one + # merged message - its regex has no memory of where the first message + # closed - so the snapshot used to publish one item where the stream + # published two, and the second streamed item vanished from the record + # the client reconciles against (10 live cases a week). + assert [item["type"] for item in final] == ["reasoning", "message", "message"] final_messages = [item for item in final if item["type"] == "message"] - assert [item["type"] for item in final] == ["reasoning", "message"] - # The same characters, in the same order, on both views. The final pass - # re-parses the whole text so its fallback is a single message; equality - # is on the concatenated text, which is what a training-data consumer - # reads. - assert "".join(streamed_texts) == final_messages[0]["content"][0]["text"] - assert _UNTERMINATED in final_messages[0]["content"][0]["text"] + assert [item["content"][0]["text"] for item in final_messages] == streamed_texts -def test_the_unterminated_fragment_keeps_the_streamed_ids_it_can(): - """Id linkage under the structural divergence the fragment causes. +def test_the_unterminated_fragment_keeps_every_streamed_id(): + """Item identity across the segmentation the fragment causes. - The stream published two message items, the rebuild derives one: the one - it derives answers to the first streamed id, and the mismatch is warned - about rather than hidden - it is the visible residue of a call the model - never finished. + The stream published two message items; the snapshot, assembled from the + stream's own record, answers to both ids - including the second, which + the old re-derive could not even represent. Nothing warns: the two views + read the same characters, and segmentation is the stream's to decide. """ processor = _processor(tools=[_exec_tool()], reasoning_parser="glm47", tool_parser="glm47") frames, result = _stream(processor, ["Plan it.", "Run it.", _UNTERMINATED]) @@ -397,9 +445,45 @@ def test_the_unterminated_fragment_keeps_the_streamed_ids_it_can(): with patch("tensorrt_llm.serve.responses_utils.logger") as mock_logger: final = _final_output(processor, result) - assert (final[0]["type"], final[0]["id"]) == streamed[0] - assert (final[1]["type"], final[1]["id"]) == streamed[1] - assert mock_logger.warning.called + assert [(item["type"], item["id"]) for item in final] == streamed + assert not mock_logger.warning.called + + +# --------------------------------------------------------------------------- +# Edge whitespace: the snapshot keeps the characters the stream sent +# --------------------------------------------------------------------------- + + +def test_final_text_keeps_the_whitespace_the_stream_sent(): + """The live shape: stream ` Backbone`, snapshot `Backbone` (716/week). + + Leading and trailing whitespace is part of the generation; the stream + delivered it as deltas and the snapshot used to strip it, so the two + views disagreed about the model's own characters on roughly every turn + that started with a space token. + """ + processor = _processor() + frames, result = _stream(processor, ["Think hard.", "", " Backbone of the design. "]) + assert _message_texts(frames) == [" Backbone of the design. "] + + final = _final_output(processor, result) + messages = [item for item in final if item["type"] == "message"] + assert messages[0]["content"][0]["text"] == " Backbone of the design. " + + +def test_non_streaming_text_is_verbatim_too(): + """No stream ran, so there is no second view to match. + + The strip still falsified the generation: the whitespace was generated, + and the JSON body is the only account a non-streaming client gets. + """ + items, _messages, _reasoning = _create_output_content( + SimpleNamespace(outputs=[SimpleNamespace(index=0, text="think Backbone ")]), + reasoning_parser="glm", + streamed_item_ids=None, + ) + assert [i.type for i in items] == ["reasoning", "message"] + assert items[1].content[0].text == " Backbone " # --------------------------------------------------------------------------- @@ -513,6 +597,143 @@ def test_a_dropped_call_does_not_donate_its_identity_to_the_next_one(): assert json.loads(final_calls[0]["arguments"]) == {"cmd": "ls"} +# --------------------------------------------------------------------------- +# One call entity, two parses: the snapshot repeats the streamed arguments +# --------------------------------------------------------------------------- + +# Malformed GLM-4.7 markup, the live shape: the model emits two consecutive +# tags. The incremental state machine reads it as the key "input" +# (re-anchoring on the second tag); the whole-text pair regex swallows the +# first closing tag into the key and delivers the corrupted +# `{"inputcmd": "ls -la"}` - both valid JSON, both +# "successful" parses, different inputs for the same call identity. 2 live +# cases a week. +_DOUBLE_ARG_KEY = ( + "execinputcmd" + "ls -la" +) + + +def _streamed_function_calls(frames): + return [ + data["item"] + for data in map(_event_data, frames) + if data.get("type") == "response.output_item.done" + and data["item"]["type"] == "function_call" + ] + + +def test_final_call_arguments_are_byte_identical_to_the_streamed_ones(): + """The same call_id must not carry two different tool inputs. + + The stream already delivered - and the client possibly executed - its + version of the arguments; a snapshot that re-describes the same call_id + from its own re-parse hands the client a second, different input for one + identity (live case: stream `false`, final `{}`; another: stream a shell + script, final JSON under a corrupted key). The record the stream keeps + per call now carries the assembled arguments, and the snapshot repeats + them. + """ + tool = FunctionTool( + name="exec", + type="function", + strict=False, + parameters={ + "type": "object", + "properties": {"input": {"type": "string"}, "cmd": {"type": "string"}}, + }, + ) + processor = _processor(tools=[tool], reasoning_parser="glm47", tool_parser="glm47") + with patch("tensorrt_llm.serve.responses_utils.logger") as mock_logger: + frames, result = _stream( + processor, + ["Plan it.", "Run it. ", _DOUBLE_ARG_KEY[:40], _DOUBLE_ARG_KEY[40:]], + ) + streamed_calls = _streamed_function_calls(frames) + assert [c["arguments"] for c in streamed_calls] == ['{"input": "ls -la"}'] + + final_calls = _function_call_items(_final_output(processor, result)) + + # Byte-identical arguments and name, under the streamed identity. The + # whole-text re-parse of the same markup reads a corrupted key; it must + # reach no client. + assert [(c["id"], c["call_id"], c["name"], c["arguments"]) for c in final_calls] == [ + ( + streamed_calls[0]["id"], + streamed_calls[0]["call_id"], + streamed_calls[0]["name"], + streamed_calls[0]["arguments"], + ) + ] + # The disagreement is said out loud, naming both versions. + disagreements = [ + str(call.args[0]) + for call in mock_logger.warning.call_args_list + if "differently from the stream" in str(call.args[0]) + ] + assert len(disagreements) == 1 + assert "ls -la" in disagreements[0] + + +def test_the_streamed_message_text_keeps_its_whitespace_alongside_a_call(): + """The whitespace the glm47 whole-text parse strips survives via the record. + + The parser-internal strip only feeds the re-parse: the stream sent + `Run it. ` with its trailing space, and the snapshot repeats the stream. + """ + tool = _exec_tool() + processor = _processor(tools=[tool], reasoning_parser="glm47", tool_parser="glm47") + whole_call = "execinputls" + frames, result = _stream(processor, ["Plan it.", "Run it. ", whole_call]) + assert _message_texts(frames) == ["Run it. "] + + final = _final_output(processor, result) + messages = [item for item in final if item["type"] == "message"] + assert [m["content"][0]["text"] for m in messages] == ["Run it. "] + + +def test_non_streaming_whole_text_parse_is_unchanged_by_the_record(): + """Pin: with no stream there is no record to prefer. + + The whole-text parse is the only account there is - including its + corrupted-key reading of this markup. Fixing that is a parser change, + deliberately out of scope here; this pin is what makes the streamed + fix's boundary visible. + """ + request = ResponsesRequest( + model="test-model", + input="hi", + stream=False, + tools=[ + FunctionTool( + name="exec", + type="function", + strict=False, + parameters={ + "type": "object", + "properties": { + "input": {"type": "string"}, + "cmd": {"type": "string"}, + }, + }, + ) + ], + ) + result = _generation_chunk("Plan it.Run it. " + _DOUBLE_ARG_KEY, "", True) + response = create_response_non_store( + generation_result=result, + request=request, + sampling_params=request.to_sampling_params(), + model_name="test-model", + use_harmony=False, + reasoning_parser="glm47", + tool_parser="glm47", + num_prompt_tokens=2, + ).model_dump() + calls = [i for i in response["output"] if i["type"] == "function_call"] + assert [c["arguments"] for c in calls] == ['{"inputcmd": "ls -la"}'] + + # --------------------------------------------------------------------------- # tool_choice="none": no call anywhere, the markup stays as text everywhere # --------------------------------------------------------------------------- diff --git a/tests/unittest/llmapi/apps/test_responses_streaming_events.py b/tests/unittest/llmapi/apps/test_responses_streaming_events.py index 93ea174b9e9f..8edab616ddfa 100644 --- a/tests/unittest/llmapi/apps/test_responses_streaming_events.py +++ b/tests/unittest/llmapi/apps/test_responses_streaming_events.py @@ -237,6 +237,83 @@ def _output_text(text): return ResponseOutputText(text=text, annotations=[], type="output_text", logprobs=None) +# --------------------------------------------------------------------------- +# Every content part that is opened must close, in OpenAI's order +# +# A part is opened with response.content_part.added and has to end with +# response.content_part.done before its item's response.output_item.done. +# The message close has always done this; the reasoning close yielded +# reasoning_text.done and output_item.done but never closed the part it had +# opened - one more content_part.added than content_part.done on every turn +# with reasoning, ~294k unpaired events a week - so a client tracking part +# state held a reasoning part that never ended. +# --------------------------------------------------------------------------- + + +def _event_stream(chunks): + """All events the real dispatch emits for a sequence of chunks.""" + helper = ResponsesStreamingEventsHelper() + parsers = {} + events, accumulated = [], "" + for i, chunk in enumerate(chunks): + accumulated += chunk + events.extend( + _generate_streaming_event( + output=_FakeOutput(accumulated, chunk), + request=_FakeRequest(), + finished_generation=(i == len(chunks) - 1), + streaming_events_helper=helper, + reasoning_parser_id="glm", + reasoning_parser_dict=parsers, + ) + ) + return events + + +def _assert_reasoning_part_closes(events, reasoning_text): + kinds = [getattr(e, "type", "") for e in events] + assert kinds.count("response.content_part.added") == kinds.count( + "response.content_part.done" + ), "every part opened must close" + + # The reasoning item's own closing sequence, in OpenAI's order: + # reasoning_text.done, then content_part.done, then output_item.done. + done_index = kinds.index("response.reasoning_text.done") + assert kinds[done_index + 1] == "response.content_part.done" + assert kinds[done_index + 2] == "response.output_item.done" + + part_done = events[done_index + 1] + assert part_done.part.type == "reasoning_text" + assert part_done.part.text == reasoning_text + # The part closes under the id of the item it was opened in. + reasoning_done = events[done_index + 2] + assert reasoning_done.item.type == "reasoning" + assert part_done.item_id == reasoning_done.item.id + + +def test_a_reasoning_part_closed_by_the_answer_gets_content_part_done(): + """The transition close (_close_open_item): reasoning ends, text begins.""" + events = _event_stream(["Plan the fix.", "", "Done."]) + _assert_reasoning_part_closes(events, "Plan the fix.") + + +def test_a_reasoning_part_closed_by_end_of_generation_gets_content_part_done(): + """The other closing branch: generation ends inside the reasoning item.""" + events = _event_stream(["All reasoning, no answer."]) + _assert_reasoning_part_closes(events, "All reasoning, no answer.") + + +def test_message_part_pairing_is_unchanged(): + """The branch that was always right, pinned against regression.""" + events = _event_stream(["Plan.", "", "Done."]) + kinds = [getattr(e, "type", "") for e in events] + text_done = kinds.index("response.output_text.done") + assert kinds[text_done + 1] == "response.content_part.done" + assert kinds[text_done + 2] == "response.output_item.done" + assert events[text_done + 1].part.type == "output_text" + assert events[text_done + 1].part.text == "Done." + + # --------------------------------------------------------------------------- # Streams that stop before response.completed # @@ -704,9 +781,13 @@ def test_final_response_reuses_the_call_ids_the_stream_published(): helper = processor.streaming_events_helper helper.record_emitted_tool_call(SimpleNamespace(id="fc_fixed", call_id="call_fixed")) - assert helper.emitted_tool_call_ids == [("fc_fixed", "call_fixed")] + assert [(r.item_id, r.call_id) for r in helper.emitted_tool_call_ids] == [ + ("fc_fixed", "call_fixed") + ] # And the processor reaches it by the path the final response uses. - assert processor.streaming_events_helper.emitted_tool_call_ids == [("fc_fixed", "call_fixed")] + assert [ + (r.item_id, r.call_id) for r in processor.streaming_events_helper.emitted_tool_call_ids + ] == [("fc_fixed", "call_fixed")] def _processor_with_tools(): From c6f66ca04fa3e911b8ad1dffe81e5043cc1ceec7 Mon Sep 17 00:00:00 2001 From: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> Date: Tue, 6 Oct 2026 07:44:22 +0000 Subject: [PATCH 24/43] [None][test] Run the new Responses test files in the CPU stage Three test files added in this change were on no test list, so CI would never have run them: test_responses_store_replay.py, test_responses_stream_final_consistency.py and test_responses_usage_disagg.py. They are registered in l0_cpu.yml next to the other Responses tests. test_responses_usage_disagg.py also gains the cpu_only marker, without which tests/unittest/conftest.py drops the file from a stage that runs -m cpu_only, and the NVIDIA copyright header. test_disagg_responses_api.py was listed twice in l0_cpu.yml: on its own and through the unittest/disaggregated directory entry. The file entry is dropped. Signed-off-by: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> --- tests/integration/test_lists/test-db/l0_cpu.yml | 4 +++- tests/unittest/llmapi/apps/test_responses_usage_disagg.py | 8 ++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_lists/test-db/l0_cpu.yml b/tests/integration/test_lists/test-db/l0_cpu.yml index 55d2a28e9c1f..35d13d8f79dc 100644 --- a/tests/integration/test_lists/test-db/l0_cpu.yml +++ b/tests/integration/test_lists/test-db/l0_cpu.yml @@ -120,7 +120,6 @@ l0_cpu: - unittest/api_stability - unittest/bindings - unittest/disaggregated - - unittest/disaggregated/test_disagg_responses_api.py - unittest/dynamo - unittest/evaluate/test_image_generation_eval_cli.py - unittest/executor/test_base_worker.py ISOLATION @@ -161,9 +160,12 @@ l0_cpu: - unittest/llmapi/apps/test_request_metrics.py::test_unpopulated_kv_transfer_timestamps_stay_absent - unittest/llmapi/apps/test_responses_custom_tools.py - unittest/llmapi/apps/test_responses_input_preprocess.py + - unittest/llmapi/apps/test_responses_store_replay.py + - unittest/llmapi/apps/test_responses_stream_final_consistency.py - unittest/llmapi/apps/test_responses_streaming_events.py - unittest/llmapi/apps/test_responses_streaming_tool_calls.py - unittest/llmapi/apps/test_responses_usage.py + - unittest/llmapi/apps/test_responses_usage_disagg.py - unittest/llmapi/apps/test_serving_extensions.py - unittest/llmapi/apps/test_tool_parsers.py - unittest/llmapi/apps/test_web_search_shared.py diff --git a/tests/unittest/llmapi/apps/test_responses_usage_disagg.py b/tests/unittest/llmapi/apps/test_responses_usage_disagg.py index 13edab83b6fd..ceb5235e813b 100644 --- a/tests/unittest/llmapi/apps/test_responses_usage_disagg.py +++ b/tests/unittest/llmapi/apps/test_responses_usage_disagg.py @@ -1,3 +1,5 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 """Usage accounting for the Responses API under disaggregated serving. The generation worker cannot measure prefix reuse: the whole prompt reaches it @@ -11,9 +13,15 @@ import unittest from types import SimpleNamespace +import pytest + from tensorrt_llm.serve.openai_protocol import PromptTokensDetails, UsageInfo from tensorrt_llm.serve.responses_utils import _create_usage +# The CPU-* CI stages run pytest with -m 'cpu_only', and tests/unittest/conftest.py +# drops every file without this marker there. +pytestmark = pytest.mark.cpu_only + def result(prompt_len, output_len, cached, ctx_usage=None): """A finished generation, shaped as the postprocessor sees one.""" From 7c9cdcb3676d3014d799159536b9151a3e9df2e6 Mon Sep 17 00:00:00 2001 From: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> Date: Thu, 3 Sep 2026 23:19:18 -0700 Subject: [PATCH 25/43] [None][fix] Cover the routes the disaggregated server gained test_standard_and_disagg_register_messages_route stubs the service with a SimpleNamespace, so registering /v1/responses -- which reads _service.openai_responses -- raised AttributeError rather than failing an assertion. The test asserted only /v1/messages and count_tokens, so the two routes added since had no coverage at all and the break surfaced as a stub error. Add the missing attribute and assert both new routes. Each was a client-visible outage before it was added: a Responses-API client got a 404 on /v1/responses, and anything doing model discovery got one on /v1/models. Signed-off-by: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> --- tests/unittest/llmapi/apps/test_anthropic_routes.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/unittest/llmapi/apps/test_anthropic_routes.py b/tests/unittest/llmapi/apps/test_anthropic_routes.py index ccc099e7e1e2..876b2f01f5c7 100644 --- a/tests/unittest/llmapi/apps/test_anthropic_routes.py +++ b/tests/unittest/llmapi/apps/test_anthropic_routes.py @@ -299,6 +299,7 @@ def test_standard_and_disagg_register_messages_route(monkeypatch, tmp_path): openai_completion=AsyncMock(), openai_chat_completion=AsyncMock(), anthropic_count_tokens=AsyncMock(), + openai_responses=AsyncMock(), ) disagg._perf_metrics_collector = SimpleNamespace(get_perf_metrics=AsyncMock()) disagg._disagg_cluster_storage = None @@ -320,6 +321,14 @@ def test_standard_and_disagg_register_messages_route(monkeypatch, tmp_path): paths = {route.path for route in server.app.routes} assert "/v1/messages/count_tokens" in paths + # The disaggregated server reached parity with the aggregated one on these + # two only recently, and both were client-visible outages: Responses-API + # clients got a 404 on /v1/responses, and any client doing model discovery + # got one on /v1/models and had to be told the model name out of band. + disagg_paths = {route.path for route in disagg.app.routes} + assert "/v1/responses" in disagg_paths + assert "/v1/models" in disagg_paths + def _batch_client(runner=None): """A TestClient over just the batch routes, with a stubbed item runner.""" From 6829f2e552c1a13104f5338791c3c9b7f3a27e56 Mon Sep 17 00:00:00 2001 From: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> Date: Fri, 4 Sep 2026 02:42:02 -0700 Subject: [PATCH 26/43] [None][fix] Stop a test from flipping prometheus into multiprocess mode test_standard_and_disagg_register_messages_route sets PROMETHEUS_MULTIPROC_DIR so register_routes can mount the disaggregated server's multiprocess collector. Unsetting that variable does not undo it: the first metric built while it is set makes prometheus_client choose `values.ValueClass` once and cache it process-wide. monkeypatch restores the environment at teardown but cannot put the class back, so every later metric is built by MultiProcessValue with no directory and raises "expected str, bytes or os.PathLike object, not NoneType". The damage lands on whichever file runs next, which makes it read as that file's bug: test_disagg_openai_client.py passes alone and fails wholesale -- 14 failures, 11 errors -- when this file precedes it in the same process. Latent until now only because the test used to abort at register_routes before reaching the collector; repairing that assertion is what let the leak out. Move the setup into a fixture that restores the class as well as the environment. Signed-off-by: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> --- .../llmapi/apps/test_anthropic_routes.py | 38 +++++++++++++++---- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/tests/unittest/llmapi/apps/test_anthropic_routes.py b/tests/unittest/llmapi/apps/test_anthropic_routes.py index 876b2f01f5c7..6b4d193e91e9 100644 --- a/tests/unittest/llmapi/apps/test_anthropic_routes.py +++ b/tests/unittest/llmapi/apps/test_anthropic_routes.py @@ -271,14 +271,38 @@ def test_messages_route_reframes_streaming_response(server_kind): assert backend.await_args.args[0].stream -def test_standard_and_disagg_register_messages_route(monkeypatch, tmp_path): - # The disagg server's register_routes mounts a prometheus multiprocess - # collector, which errors unless PROMETHEUS_MULTIPROC_DIR points at a real - # directory. Production sets that up in set_prometheus_multiproc_dir() - # during startup; building the server with object.__new__ skips startup - # entirely, so the directory has to be supplied here. monkeypatch keeps it - # from leaking into any other test in the session. +@pytest.fixture +def prometheus_multiproc_dir(tmp_path, monkeypatch): + """A multiprocess metrics directory that does not outlive the test. + + Setting PROMETHEUS_MULTIPROC_DIR cannot be undone by unsetting it. The + first metric built while it is set makes prometheus_client choose + ``values.ValueClass`` for the whole process and cache it; monkeypatch then + restores the environment at teardown but cannot put the class back. Every + later metric in that process is built by MultiProcessValue with no + directory to write into and dies with "expected str, bytes or + os.PathLike object, not NoneType". + + The damage lands on whichever file runs next, which makes it look like + that file's bug: test_disagg_openai_client.py passes alone and fails + wholesale -- 14 failures, 11 errors -- when this one precedes it. So the + class is restored here, not just the environment. + """ + import prometheus_client.values + monkeypatch.setenv("PROMETHEUS_MULTIPROC_DIR", str(tmp_path)) + saved_value_class = prometheus_client.values.ValueClass + try: + yield tmp_path + finally: + prometheus_client.values.ValueClass = saved_value_class + + +def test_standard_and_disagg_register_messages_route(prometheus_multiproc_dir): + # register_routes mounts a prometheus multiprocess collector, which needs + # PROMETHEUS_MULTIPROC_DIR to name a real directory: building the server + # with object.__new__ skips the startup that would normally create one. + # See the fixture for why setting the variable is not enough to undo. standard = object.__new__(OpenAIServer) standard.app = FastAPI() From 67e69909b94a49378e0f7a0b550e1d330fe999aa Mon Sep 17 00:00:00 2001 From: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> Date: Wed, 7 Oct 2026 02:27:52 +0000 Subject: [PATCH 27/43] [None][fix] Responses: review fixes, and less code and tests to review Streaming and the final snapshot: - Tool calls follow the parser's contract. A parser whose increments match its whole-text parse (streaming_matches_whole_parse, the GLM parsers) streams calls assembled from them; every other parser's calls come from one whole-text parse at the end of the stream, as on main. Assembling increments for those parsers sent DeepSeek V3/V3.1 calls with another call's arguments, MiniMax-M2 calls twice, and lost Gemma4 calls with arguments and Qwen3-Coder calls without them. - The final snapshot repeats the streamed tool-call items, replacing the reconciliation between the streamed record and a re-parse. - sequence_number is set by the producer: the opening pair is 0 and 1 and the rest continue from 2, also in a postprocess worker's copy of the processor. This replaces restamping every frame at the egress. - The stream guard recognises a terminal event only at a frame boundary, ends a frame a relay was cut inside before its error event, and only passes engine error messages to the client. - Streamed responses count reasoning tokens on the aggregated server too. Requests and the disaggregated path: - developer becomes system only when the chat template has no developer branch; the DeepSeek V3.2/V4 tokenizers render it themselves. - A context-only Harmony response is not parsed. - A response the context phase finishes reaches the client without prompt ids or the raw finish reason, and the streamed replay keeps `schema` under its wire name. - store is forwarded as false, with a warning only when the client set it; protected disaggregated fields are validated before anything is rendered or stored. - The parallel_tool_calls warning fires only when the client set it. - A stored conversation is a copy, so trimming it no longer shortens the prompt being rendered. - Image degradation handles an image_url object; additional_tools hoisting leaves malformed shapes for validation. - A duplicated parallel_tool_calls entry is removed from the API reference, and the relayed-prompt decoding is one method. Comments describe the code as it stands; base comments the change did not need to touch are restored. Tests are table-driven where they repeated one shape, tests of removed machinery are gone, and test_responses_usage_disagg.py is folded into test_responses_usage.py. Signed-off-by: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> --- tensorrt_llm/serve/openai_client.py | 23 +- tensorrt_llm/serve/openai_disagg_server.py | 68 +- tensorrt_llm/serve/openai_disagg_service.py | 134 +- tensorrt_llm/serve/openai_protocol.py | 118 +- tensorrt_llm/serve/openai_server.py | 141 +- tensorrt_llm/serve/postprocess_handlers.py | 2 - tensorrt_llm/serve/responses_utils.py | 1954 ++++------------- tensorrt_llm/serve/responses_web_search.py | 88 +- .../integration/test_lists/test-db/l0_cpu.yml | 1 - .../references/trtllm_serve_api.yaml | 6 - .../test_disagg_openai_client.py | 14 +- .../test_disagg_responses_api.py | 560 ++--- .../llmapi/apps/test_anthropic_routes.py | 31 +- .../apps/test_responses_custom_tools.py | 9 - .../apps/test_responses_input_preprocess.py | 732 ++---- .../apps/test_responses_store_replay.py | 173 +- ...test_responses_stream_final_consistency.py | 886 ++------ .../apps/test_responses_streaming_events.py | 902 ++------ .../test_responses_streaming_tool_calls.py | 733 ++----- .../llmapi/apps/test_responses_usage.py | 282 +-- .../apps/test_responses_usage_disagg.py | 145 -- .../llmapi/apps/test_web_search_shared.py | 74 +- 22 files changed, 1780 insertions(+), 5296 deletions(-) delete mode 100644 tests/unittest/llmapi/apps/test_responses_usage_disagg.py diff --git a/tensorrt_llm/serve/openai_client.py b/tensorrt_llm/serve/openai_client.py index d29b7c423632..7faeb140ce40 100644 --- a/tensorrt_llm/serve/openai_client.py +++ b/tensorrt_llm/serve/openai_client.py @@ -88,17 +88,10 @@ def model_dump( def _encode_request_body(request: _SerializableRequest) -> bytes: - """Encode a request forwarded to a worker, under its wire field names. - - Where a field carries an alias the internal and wire names differ, and - the worker validates the wire name. ``schema`` is the case that bites: - pydantic cannot hold a field of that name, so the model declares - ``schema_`` aliased to ``schema``. Without ``by_alias`` a Responses - ``text.format`` JSON schema reached the worker as ``schema_``, every - member of the format union failed validation, and the request came back - 400 -- every structured-output call made through a disaggregated server. - The workers set ``populate_by_name=True``, so either spelling is accepted; - the alias is the one that is correct on the wire. + """Encode a request forwarded to a worker. + + Fields go out under their wire names, e.g. ``schema``, which the model + holds as ``schema_``. """ return _msgpack_encoder.encode( request.model_dump(mode="json", exclude_unset=True, by_alias=True) @@ -184,13 +177,7 @@ async def get_json( response_type: Type[_ResponseT], server: str, ) -> _ResponseT: - """Read a worker's bodyless GET endpoint. - - The counterpart to post_json for endpoints that describe the worker - rather than ask it to do anything -- /v1/models being the one that - matters. Same exemption from routing state, and no request body to - serialise. - """ + """GET a worker endpoint, outside routing state, like post_json.""" ... async def shutdown(self) -> None: ... diff --git a/tensorrt_llm/serve/openai_disagg_server.py b/tensorrt_llm/serve/openai_disagg_server.py index 339c620a3c42..323ba1a6f56d 100644 --- a/tensorrt_llm/serve/openai_disagg_server.py +++ b/tensorrt_llm/serve/openai_disagg_server.py @@ -347,11 +347,6 @@ def register_routes(self): self.app.add_api_route("/v1/messages", self.anthropic_messages, methods=["POST"]) self.app.add_api_route("/v1/messages/count_tokens", self.anthropic_count_tokens, methods=["POST"]) self.app.add_api_route("/v1/responses", self._wrap_entry_point(self._service.openai_responses, ResponsesRequest), methods=["POST"]) - # GET, and forwarded to a worker rather than answered from config: - # the orchestrator has no model of its own and DisaggServerConfig - # carries no model name. Clients that do model discovery -- Codex - # among them -- got a 404 here and had to be told the name out of - # band. self.app.add_api_route("/v1/models", self.get_model, methods=["GET"]) self.app.add_api_route("/health", self.health, methods=["GET"]) self.app.add_api_route("/cluster_info", self.cluster_info, methods=["GET"]) @@ -447,26 +442,16 @@ async def wrapper(req: request_type, raw_req: Request) -> Response: if req.stream: stream = response_or_generator if isinstance(req, ResponsesRequest): - # Only the Responses protocol: a stream that stops - # before response.completed loses everything it - # produced, because that event is the only place the - # full text is repeated. The other protocols carry - # their content entirely in deltas and end on a - # sentinel, so a truncation there is already visible. - # - # A bare `error` event rather than the worker's - # `response.failed`: this is a byte relay with no view - # of the response being assembled. It does count the - # events it forwarded, so the sequence number is exact - # -- and it is zero for the failures that never - # reached a worker at all, which otherwise end with - # an entirely empty body. + # A relayed Responses stream that fails early ends with + # an `error` event; other protocols end on a sentinel. stream = guard_responses_stream(stream, stream_error_event) return StreamingResponse( content=stream, media_type="text/event-stream") - return JSONResponse(content=response_or_generator.model_dump()) + # by_alias: e.g. text.format.schema goes out under its wire name. + return JSONResponse( + content=response_or_generator.model_dump(by_alias=True)) except Exception as e: # Usually raises; returns a Response for worker errors that # carry a machine-readable code (context_length_exceeded). @@ -524,37 +509,24 @@ async def anthropic_messages(self, request: AnthropicMessagesRequest, exclude_none=True)) async def get_model(self) -> Response: - """List the served model, asking a context worker for the name. - - Error shape is OpenAI's, not Anthropic's, because this is an OpenAI - route -- a client that model-discovers here parses OpenAI errors. - """ + """List the served model, as a context worker reports it.""" try: - response = await self._service.get_model() - except aiohttp.ClientResponseError as error: return JSONResponse( - status_code=error.status or 500, - content={ - "object": "error", - "message": _upstream_error_message(error), - "type": ("invalid_request_error" - if 400 <= (error.status or 500) < 500 else "api_error"), - "code": error.status or 500, - }, - ) + content=(await self._service.get_model()).model_dump()) + except aiohttp.ClientResponseError as error: + status, message = error.status or 500, _upstream_error_message( + error) except (RuntimeError, ValueError) as error: - # No worker to ask yet. 503 says "retry", which is true: the answer - # exists as soon as a context worker registers. - return JSONResponse( - status_code=503, - content={ - "object": "error", - "message": str(error), - "type": "api_error", - "code": 503, - }, - ) - return JSONResponse(content=response.model_dump()) + # No context worker yet; the client may retry. + status, message = 503, str(error) + return JSONResponse( + status_code=status, + content=ErrorResponse( + message=message, + type=("invalid_request_error" + if 400 <= status < 500 else "api_error"), + code=status, + ).model_dump()) async def anthropic_count_tokens( self, request: AnthropicCountTokensRequest) -> Response: diff --git a/tensorrt_llm/serve/openai_disagg_service.py b/tensorrt_llm/serve/openai_disagg_service.py index 2b58329bdf56..d477dc0cf02b 100644 --- a/tensorrt_llm/serve/openai_disagg_service.py +++ b/tensorrt_llm/serve/openai_disagg_service.py @@ -53,15 +53,11 @@ _GEN_PENDING_FINISH_REASONS = ("length", "not_finished") -# The Completions and Chat Completions responses carry the KV-cache handoff on -# `choices[0]`, one entry per generated sequence. A Responses API response has -# no `choices` - it is a single `output` list - so it carries the same handoff -# at the top level. These three accessors are the only places that difference -# matters; everything downstream works on the extracted values. def _ctx_handoff_slots(response: UCompletionResponse) -> list: - """The parts of a context response that carry a KV-cache handoff. + """The parts of a context response that carry the KV-cache handoff. - Returns objects exposing ``disaggregated_params`` and ``finish_reason``. + ``choices`` for completions and chat; a Responses response carries it at the + top level. Each exposes ``disaggregated_params`` and ``finish_reason``. """ if isinstance(response, ResponsesResponse): return [response] @@ -69,24 +65,14 @@ def _ctx_handoff_slots(response: UCompletionResponse) -> list: def _ctx_usage_info(response: UCompletionResponse) -> Optional[UsageInfo]: - """The context phase's token counts, in the shape DisaggregatedParams uses. - - Completions and Chat Completions report usage as ``UsageInfo`` - (prompt/completion/total). The Responses API reports ``ResponseUsage`` - (input/output/total) - same numbers, different field names - and - ``ctx_usage`` is typed as the former. Assigning the latter straight across - survives locally and then loses every count when the generation worker - validates the request, because none of the field names line up. + """The context phase's usage as the ``UsageInfo`` that ``ctx_usage`` expects. + + The Responses API reports ``ResponseUsage``; its cached count is one only + the context phase can measure. """ usage = getattr(response, "usage", None) if usage is None or isinstance(usage, UsageInfo): return usage - # The cached count has to make the trip too. It is the one number the - # generation worker cannot recompute: from there the whole prompt arrived - # as transferred KV, so its own cached_tokens is the prompt length and - # reporting it would say every request was a full cache hit. Dropping it - # here left `ctx_usage` carrying three of the four numbers, so a consumer - # that read it still had to invent the fourth. cached = getattr(usage, "input_tokens_details", None) return UsageInfo( prompt_tokens=usage.input_tokens, @@ -99,13 +85,13 @@ def _ctx_usage_info(response: UCompletionResponse) -> Optional[UsageInfo]: def _drop_ctx_handoff(response: UCompletionResponse) -> None: - """Strip the handoff from a context response that will not be handed off. - - Left in place it would travel back to the client as an internal field - describing a KV-cache transfer that never happens. - """ + """Strip the handoff from a context response that goes to the client.""" for slot in _ctx_handoff_slots(response): slot.disaggregated_params = None + if isinstance(response, ResponsesResponse): + response.prompt_token_ids = None + response.prompt_token_ids_b64 = None + response.finish_reason = None class OpenAIDisaggregatedService(OpenAIService): @@ -182,21 +168,8 @@ async def openai_chat_completion( raise RuntimeError("Cluster is not ready") return await self._send_disagg_request(request, hooks) - async def get_model(self) -> ModelList: - """Report the model by asking a worker, since the proxy holds none. - - An aggregated server answers from `self.model`, which it has because it - loaded the weights. This orchestrator has no model, no tokenizer and no - engine, and `DisaggServerConfig` carries no model name either -- the - working deployments on hand do not set one, so reading it from config - would answer for some clusters and 404 for the rest. - - Asking a worker gives the name the requests will actually be served - under, which is the answer a client doing model discovery wants. Context - workers are used for the same reason count-tokens uses them: they are - the half that owns the tokenizer and the prompt-facing view of the - model. - """ + async def _next_ctx_server(self) -> str: + """A context worker, round-robin, for requests the proxy cannot serve.""" if not await self.is_ready(): raise RuntimeError("Cluster is not ready") servers = self._ctx_router.servers @@ -204,7 +177,13 @@ async def get_model(self) -> ModelList: raise RuntimeError("No context servers are available") server = servers[self._count_tokens_rr_counter % len(servers)] self._count_tokens_rr_counter += 1 - return await self._ctx_client.get_json("v1/models", ModelList, server) + return server + + async def get_model(self) -> ModelList: + """Report the model as a worker serves it; the proxy holds none.""" + return await self._ctx_client.get_json( + "v1/models", ModelList, await self._next_ctx_server() + ) async def anthropic_count_tokens( self, request: AnthropicCountTokensRequest @@ -215,18 +194,11 @@ async def anthropic_count_tokens( to happen on a worker. Context workers are the ones that tokenize prompts, so the count they return is the one that will actually be used. """ - if not await self.is_ready(): - raise RuntimeError("Cluster is not ready") - servers = self._ctx_router.servers - if not servers: - raise RuntimeError("No context servers are available") - server = servers[self._count_tokens_rr_counter % len(servers)] - self._count_tokens_rr_counter += 1 return await self._ctx_client.post_json( "v1/messages/count_tokens", request, AnthropicCountTokensResponse, - server, + await self._next_ctx_server(), ) async def openai_responses( @@ -235,11 +207,7 @@ async def openai_responses( if not await self.is_ready(): raise RuntimeError("Cluster is not ready") - # Response storage is per-worker in-process state. Two workers sit - # behind this orchestrator and requests are not pinned to either, so a - # stored response is only findable again by chance. Rejecting says so; - # accepting would fail later as a "response not found" that looks like - # the client's mistake, and `store` would silently do nothing. + # Stored responses are per worker and unreachable through this proxy. if request.previous_response_id is not None: raise HTTPException( status_code=400, @@ -250,35 +218,12 @@ async def openai_responses( "turns in 'input' instead." ), ) - - # `store` has the same cause as the rejection above and a different - # consequence, so it gets a different answer. A stored response is - # unreachable here -- whichever worker holds it, this orchestrator - # exposes no route to read it back and refuses the - # `previous_response_id` that is the only other way in. But unlike - # `previous_response_id`, `store` does not stop the request being - # served correctly: only a side effect is lost. Refusing an otherwise - # valid request over a side effect would break clients that set the - # flag by default and never read a response back. - # - # So warn rather than reject, matching how the aggregated server treats - # `background`. What must not happen is silence: the client asked for - # persistence and is not getting it, and a log line is the only place - # that can currently say so -- the Responses protocol has no field for - # "served, but one thing you asked for was dropped". - # - # Gated on model_fields_set because `store` DEFAULTS TO TRUE. Warning on - # the value alone would fire for every request that simply never - # mentioned the field, which is most of them, and a warning on every - # request is one nobody reads. Only a client that wrote `store` down is - # making a request that goes unmet. if "store" in request.model_fields_set and request.store: - logger.warning( - "'store' is ignored on a disaggregated server: response " - "storage is per-worker in-process state and this orchestrator " - "exposes no route to read a stored response back. The request " - "is served normally; nothing is persisted." + logger.warning_once( + "'store' is ignored on a disaggregated server; nothing is persisted.", + key="disagg_responses_store_ignored", ) + request.store = False return await self._send_disagg_request(request, hooks) @@ -359,20 +304,15 @@ async def _send_disagg_request_ctx_first( # ctx client will never return a generator when streaming is requested # make up for this by returning a done generator if isinstance(ctx_response, ResponsesResponse): - # The Responses protocol has no "[DONE]" sentinel, so the - # completions-style terminator would leave the client - # waiting for a response.completed that never comes. Replay - # the finished response as events instead - it also carries - # the answer, which the "[DONE]" path drops on the floor. + # Responses has no "[DONE]"; replay the finished response. return responses_done_generator(ctx_response) return done_generator() return ctx_response def _need_gen(self, response: UCompletionResponse) -> bool: - if not response: - return True - slots = _ctx_handoff_slots(response) - if slots and slots[0].finish_reason not in _GEN_PENDING_FINISH_REASONS: + if response and ( + _ctx_handoff_slots(response)[0].finish_reason not in _GEN_PENDING_FINISH_REASONS + ): _drop_ctx_handoff(response) return False return True @@ -418,16 +358,7 @@ def _get_gen_request( # Replace the string prompt with prompt_tokens_ids if isinstance(request, CompletionRequest): request.prompt = ctx_response.prompt_token_ids - elif isinstance(request, ResponsesRequest): - # Same relay as the chat path below. The message history is - # never stripped here: the Responses input list is also what - # the conversation store keys on, and the generation worker - # rebuilds nothing from it once prompt_token_ids is set. - if ctx_response.prompt_token_ids_b64 is not None: - request.prompt_token_ids_b64 = ctx_response.prompt_token_ids_b64 - else: - request.prompt_token_ids = ctx_response.prompt_token_ids - elif isinstance(request, ChatCompletionRequest): + elif isinstance(request, (ChatCompletionRequest, ResponsesRequest)): # Relay the base64 token-id string verbatim (no int-list # materialization on the orchestrator loop), else the int array. if ctx_response.prompt_token_ids_b64 is not None: @@ -441,6 +372,7 @@ def _get_gen_request( # for harmony/multimodal workers (model type is fixed per deploy). if ( self._strip_gen_message_history + and isinstance(request, ChatCompletionRequest) and request.messages and len(request.messages) > 1 ): diff --git a/tensorrt_llm/serve/openai_protocol.py b/tensorrt_llm/serve/openai_protocol.py index 4e7db5471130..ec424dd6885d 100644 --- a/tensorrt_llm/serve/openai_protocol.py +++ b/tensorrt_llm/serve/openai_protocol.py @@ -1533,24 +1533,18 @@ def _with_annotations(part): return {**part, "annotations": []} return part - def _degrade_image(part): - """Replace an image content part with a text placeholder. - - The Codex CLI attaches screenshots as ``input_image`` parts -- an - agent that plots something and views it sends the PNG back as - base64. The models served here are text-only, and the vendored - input union has no image member, so validation fell through to - ``ResponseInputTextParam`` and failed the WHOLE request with - "Field required: text" -- with the full base64 body echoed into - the error. The client retries a deterministic 400 until its - backoff limit and the campaign dies: 2 of the first 182 - Kernel-Trace campaigns went exactly this way (2026-09-22). - - A placeholder keeps the turn -- and with it the session -- alive, - and says what was dropped instead of leaking megabytes of base64. + def _degrade_image(part: Any) -> Any: + """Replace an ``input_image`` part with a text placeholder. + + The vendored input union has no image member, so an image would + fail the whole request; the placeholder says what was dropped. """ if isinstance(part, dict) and part.get("type") == "input_image": url = part.get("image_url") or "" + if isinstance(url, dict): + url = url.get("url") or "" + if not isinstance(url, str): + url = "" kind = url.split(";", 1)[0].removeprefix("data:") or "image" return { "type": @@ -1566,23 +1560,11 @@ def _degrade_image(part): # A client may send a bare content part as a top-level item, not # only nested inside a message. item = _with_annotations(_degrade_image(item)) - if isinstance(item, dict) and isinstance(item.get("content"), list): - item = { - **item, - "content": - [_degrade_image(part) for part in item["content"]], - } - # Tool results carry their parts under "output", not "content" - # (custom_tool_call_output / function_call_output). An agent that - # plots something gets the PNG back through the tool and Codex - # ships it as an input_image part in there -- 114 Kernel-Trace - # campaigns died on exactly this shape (2026-09-23) after the - # content/top-level shapes were already handled. - if isinstance(item, dict) and isinstance(item.get("output"), list): - item = { - **item, - "output": [_degrade_image(part) for part in item["output"]], - } + # Message parts live under "content", tool-result parts under + # "output". + for key in ("content", "output"): + if isinstance(item, dict) and isinstance(item.get(key), list): + item = {**item, key: [_degrade_image(p) for p in item[key]]} if isinstance(item, dict) and item.get("type") in (None, "message"): role = item.get("role") if "id" in item and role in _ID_STRIPPED_ROLES: @@ -1663,10 +1645,8 @@ def _materialize_lazy_item_content(cls, value): ) # doc: begin-responses-extra-params - # TensorRT-LLM extensions, mirroring ChatCompletionRequest. These are not - # part of the OpenAI Responses schema; they carry the disaggregated-serving - # handoff between the orchestrator and the context/generation workers, so a - # client never sets them. + # TensorRT-LLM extensions carrying the disaggregated-serving handoff between + # the orchestrator and the workers; clients do not set them. disaggregated_params: Optional[DisaggregatedParams] = Field( default=None, description=("Parameters for disaggregated serving"), @@ -1675,15 +1655,23 @@ def _materialize_lazy_item_content(cls, value): default=None, description=("Parameters for multi-turn conversation routing"), ) - # Set by the orchestrator from the context worker's response so the - # generation worker skips re-rendering the chat template and re-tokenizing - # an input it has already been given as token ids. + # The context worker's tokenized prompt, so the generation worker does not + # render it again; the _b64 form is a base64 int32 buffer. prompt_token_ids: Optional[List[int]] = None - # base64 int32 buffer alternative to prompt_token_ids, relayed by the - # orchestrator from the ctx response. Not for clients. prompt_token_ids_b64: Optional[str] = None + # doc: end-responses-extra-params + def relayed_prompt_token_ids(self) -> Optional[List[int]]: + """The prompt the orchestrator relayed from the context worker, if any.""" + if self.prompt_token_ids is not None: + return self.prompt_token_ids + if not self.prompt_token_ids_b64: + return None + import numpy as np + return np.frombuffer(base64.b64decode(self.prompt_token_ids_b64), + dtype=np.int32).tolist() + _DEFAULT_SAMPLING_PARAMS = { "temperature": 1.0, "top_p": 1.0, @@ -1741,32 +1729,27 @@ def _null_top_level_fields_mean_unset(cls, data): @model_validator(mode="before") @classmethod - def hoist_additional_tools(cls, data): - """Move an `additional_tools` input item into `tools`. - - Codex declares its tools this way rather than in `tools`. The item - carries no text, so the input-item conversion dropped it as - unrecognised and the model was offered nothing to call. - - The payload is already what `tools` accepts -- namespace entries whose - nested tools are flattened to `namespace.tool` downstream -- so it is - moved here, before validation, and the entries are validated as tools - like any other. Hoisting at the protocol boundary rather than in the - prompt builder keeps the response side in agreement: a call can only - be reported back with its namespace if the tool was in `tools` when - the reply was parsed. + def hoist_additional_tools(cls, data: Any) -> Any: + """Move the tools of an ``additional_tools`` input item into ``tools``. + + Codex declares its tools that way; hoisted here, they are validated as + tools and resolve when the reply's calls are parsed. Malformed shapes + are left in place for validation to reject. """ if not isinstance(data, dict): return data items = data.get("input") - if not isinstance(items, list): + tools = data.get("tools") + if not isinstance(items, list) or not isinstance( + tools, (list, type(None))): return data hoisted = [] kept = [] for item in items: - if isinstance(item, - dict) and item.get("type") == "additional_tools": - hoisted.extend(item.get("tools") or []) + if (isinstance(item, dict) + and item.get("type") == "additional_tools" + and isinstance(item.get("tools"), list)): + hoisted.extend(item["tools"]) else: kept.append(item) if not hoisted: @@ -1845,10 +1828,6 @@ class ResponsesResponse(OpenAIBaseModel): id: str = Field(default_factory=lambda: f"resp_{str(uuid.uuid4().hex)}") created_at: int = Field(default_factory=lambda: int(time.time())) # error: Optional[ResponseError] = None - # The SDK's own shape: reason is an enum naming why the response stopped - # short. Set by the response builder when a generation was cut at its - # token budget, so the non-streaming JSON body explains its "incomplete" - # status the way the streaming terminal event already does. incomplete_details: Optional[IncompleteDetails] = None instructions: Optional[str] = None metadata: Optional[Metadata] = None @@ -1874,18 +1853,13 @@ class ResponsesResponse(OpenAIBaseModel): usage: Optional[ResponseUsage] = None user: Optional[str] = None - # TensorRT-LLM extensions for disaggregated serving; see the matching block - # on ResponsesRequest. A context-only response carries the KV-cache handle - # and first generated token here, plus the tokenized prompt so the - # generation worker does not have to render the chat template again. + # TensorRT-LLM extensions, set only on a context-only response: the + # disaggregated handoff, and the engine's raw finish reason, which tells the + # orchestrator whether a generation phase is pending ("length" and + # "not_finished" both map to status "incomplete"). disaggregated_params: Optional[DisaggregatedParams] = Field(default=None) prompt_token_ids: Optional[List[int]] = None prompt_token_ids_b64: Optional[str] = None - # The engine's raw finish reason, which `status` cannot express. The - # orchestrator decides whether a generation phase is still pending by - # looking for "length"/"not_finished", and `status` maps both of those onto - # "incomplete" while `finish_reason_mapping` raises outright on - # "not_finished". Carrying the unmapped value keeps that decision exact. finish_reason: Optional[str] = None @classmethod diff --git a/tensorrt_llm/serve/openai_server.py b/tensorrt_llm/serve/openai_server.py index e949fbdba952..d8226fa04018 100644 --- a/tensorrt_llm/serve/openai_server.py +++ b/tensorrt_llm/serve/openai_server.py @@ -126,7 +126,6 @@ from tensorrt_llm.serve.responses_utils import guard_responses_stream from tensorrt_llm.serve.responses_utils import \ request_preprocess as responses_api_request_preprocess -from tensorrt_llm.serve.responses_utils import stamp_sse_sequence_number from tensorrt_llm.serve.responses_web_search import web_search_rejection_reason from tensorrt_llm.serve.rl_control_auth import validate_rl_control_request from tensorrt_llm.serve.serving_extensions import apply_model_chat_extensions @@ -306,8 +305,8 @@ def _build_tool_strict_guided_decoding_params(tools, tool_parser_name): tool_parser_cls = ToolParserFactory.parsers.get(tool_parser_name.lower()) if tool_parser_cls is None: logger.warning( - f"Tool parser '{tool_parser_name}' not found, cannot enforce " - f"strict mode for tools.") + "Tool parser '%s' not found, cannot enforce strict mode for tools.", + tool_parser_name) return None parser = tool_parser_cls() @@ -321,8 +320,8 @@ def _build_tool_strict_guided_decoding_params(tools, tool_parser_name): by_alias=True, exclude_none=True)) if not parser.supports_structural_tag(): logger.warning( - f"Tool parser '{tool_parser_name}' does not support structural " - f"tags, cannot enforce strict mode for tools.") + "Tool parser '%s' does not support structural tags, " + "cannot enforce strict mode for tools.", tool_parser_name) return None get_info = parser.structure_info() @@ -787,13 +786,8 @@ async def validation_exception_handler(request, exc): # expect. if self.metrics_collector: self.metrics_collector.log_request_error(http_code=400) - # pydantic echoes each failing item's `input` verbatim, and a - # rejected multimodal part carries megabytes of base64 -- which - # then travels into the client's retry logs and, for a Kernel - # Factory agent, into the campaign STOP_REASON (measured: a whole - # PNG in one). The locations are the diagnostic content; the - # payload is not. Truncation is by error, so short messages are - # untouched. + # pydantic echoes each failing input verbatim, which can be + # megabytes of base64; the error locations are what matters. detail = str(exc) if len(detail) > 4096: detail = detail[:4096] + " ... [truncated %d bytes]" % ( @@ -870,8 +864,9 @@ def _init_llm(self, chat_template: Optional[str] = None): self.processor = AutoProcessor.from_pretrained( hf_tokenizer_path, trust_remote_code=trust_remote_code) except Exception: - logger.debug(f"Failed to load AutoProcessor or AutoConfig for " - f"{hf_tokenizer_path}") + logger.debug( + "Failed to load AutoProcessor or AutoConfig for %s", + hf_tokenizer_path) self.processor = None # load model config @@ -884,7 +879,7 @@ def _init_llm(self, chat_template: Optional[str] = None): checkpoint_format=getattr(self.generator.args, "checkpoint_format", None)) except Exception: - logger.debug(f"Failed to load AutoConfig for {hf_tokenizer_path}") + logger.debug("Failed to load AutoConfig for %s", hf_tokenizer_path) self.model_config = None self.chat_template = load_chat_template(chat_template) @@ -1026,7 +1021,7 @@ def _log_config_info_metrics(self) -> None: if torch.cuda.is_available(): model_config["gpu_type"] = torch.cuda.get_device_name(0) except (ImportError, RuntimeError) as e: - logger.debug(f"Could not detect GPU type for config metrics: {e}") + logger.debug("Could not detect GPU type for config metrics: %s", e) # Parallel config β€” prefer parallel_config from generator args # for accurate values including cp_size and world_size. @@ -2315,14 +2310,7 @@ async def chat_stream_generator( np.asarray(response.prompt_token_ids, dtype=np.int32).tobytes()).decode("ascii") response.prompt_token_ids = None - # by_alias: a field whose python name differs from its - # wire name -- `schema_` for `schema`, which pydantic - # will not let a model call `schema` -- otherwise goes - # out under the internal name. The OpenAI SDK models - # this is rebuilt from downstream do not set - # `populate_by_name` and reject it, and a direct client - # is handed a field name the API does not have. - return JSONResponse(content=response.model_dump(by_alias=True)) + return JSONResponse(content=response.model_dump()) except CppExecutorError: logger.error(traceback.format_exc()) # If internal executor error is raised, shutdown the server @@ -3038,9 +3026,9 @@ async def create_streaming_generator(promise: RequestOutput, harmony_tokens = tokenize_harmony_chat_request( request, harmony_adapter=self.harmony_adapter) except Exception: - logger.error(f"messages_dict: {request.messages}") - logger.error(f"tools: {request.tools}") - logger.error(f"request: {request}") + logger.error("messages_dict: %s", request.messages) + logger.error("tools: %s", request.tools) + logger.error("request: %s", request) raise # Get harmony stop tokens @@ -3124,8 +3112,8 @@ async def create_streaming_generator(promise: RequestOutput, except ValueError as e: return self.create_error_response(str(e)) except Exception as e: - logger.error(f"Error in harmony chat completion: {e}") - logger.debug(f"Error details: {traceback.format_exc()}") + logger.error("Error in harmony chat completion: %s", e) + logger.debug("Error details: %s", traceback.format_exc()) return self.create_error_response(message=str(e), err_type="internal_error") @@ -3133,16 +3121,9 @@ async def openai_responses(self, request: ResponsesRequest, raw_request: Request) -> Response: """Serve one /v1/responses request, streaming or not. - ``parallel_tool_calls`` and, on the non-Harmony path, ``tool_choice`` - values other than "auto" and "none" are accepted for schema - compatibility but not enforced: honouring them would mean constraining - or truncating what the model generated, and these turns are recorded - as training data, where a falsified output is worse than an - unconstrained one. A once-per-process warning below says so instead - of silently swallowing the option. ``tool_choice="none"`` is the - exception because it can be honoured losslessly - the tool parser is - bypassed and any call markup stays in the visible text verbatim (see - responses_utils._effective_tool_parser). + ``parallel_tool_calls=false`` and, on the non-Harmony path, + ``tool_choice`` values other than "auto" and "none" are accepted but + not enforced (logged once). ``tool_choice="none"`` skips tool parsing. """ async def create_response( @@ -3160,16 +3141,12 @@ async def create_response( model_name=self.model, conversation_store=self.conversation_store, generation_result=None, - # The server-side switch; create_response applies - # request.store itself before persisting anything. + # request.store is applied downstream. enable_store=self.enable_store, use_harmony=self.use_harmony, reasoning_parser=args.reasoning_parser, tool_parser=args.tool_parser, num_prompt_tokens=args.num_prompt_tokens, - # This path does no postprocessing in a worker, so nothing - # has populated args.tokenizer. Used only to count - # reasoning tokens for usage. tokenizer=self.tokenizer, ) @@ -3180,31 +3157,16 @@ async def create_streaming_generator(promise: RequestOutput, postproc_params: PostprocParams): post_processor, args = postproc_params.post_processor, postproc_params.postproc_args streaming_processor = args.streaming_processor - # One monotonic counter per response, restamped onto every frame - # right before it leaves the server. The producers cannot number - # the stream themselves: with postprocessing workers enabled the - # two opening events below come from this process's streaming - # processor while every later frame is built by a pickled copy of - # it inside a worker, each counting from zero, so one response's - # stream carried 0,1,0,1,2,... (221 of 221 measured responses). - # Both configurations flow through this generator - workers hand - # back ready-made frames in _postprocess_result, the in-process - # path builds them via post_processor - so this is the one point - # where a response-wide sequence exists to be assigned. - sequence_number = 0 initial_responses = streaming_processor.get_initial_responses() for initial_response in initial_responses: - yield stamp_sse_sequence_number(initial_response, - sequence_number) - sequence_number += 1 + yield initial_response async for res in promise: pp_results = res.outputs[ 0]._postprocess_result if self.postproc_worker_enabled else post_processor( res, args) for pp_res in pp_results: - yield stamp_sse_sequence_number(pp_res, sequence_number) - sequence_number += 1 + yield pp_res await self._extract_metrics(res, raw_request) try: @@ -3236,18 +3198,18 @@ async def create_streaming_generator(promise: RequestOutput, f"{web_search_error}."), ) - # Accepted-but-unenforced options, said once per process (see the - # handler docstring for why they are not enforced). Rejecting - # would break clients that always send them; enforcing would - # falsify the recorded output. - if request.parallel_tool_calls is False: + # The signature check comes before anything is rendered or stored: + # encoded_opaque_state and ctx_info_endpoint name a peer this + # worker would pull KV cache from. + self._validate_internal_disagg_request(request, raw_request) + + if ("parallel_tool_calls" in request.model_fields_set + and request.parallel_tool_calls is False): logger.warning_once( "Responses API: 'parallel_tool_calls=false' is accepted " "but not enforced; the model may still emit several tool " "calls in one turn.", key="responses_parallel_tool_calls_unenforced") - # "none" is excluded: it is honoured by bypassing the tool parser - # (responses_utils._effective_tool_parser), which loses nothing. if not self.use_harmony and request.tool_choice not in ("auto", "none"): logger.warning_once( @@ -3277,14 +3239,8 @@ async def create_streaming_generator(promise: RequestOutput, request=request, prev_response=prev_response, conversation_store=self.conversation_store, - # The server-side switch alone, because two independent things - # hang off it downstream: whether PRIOR context is fetched - # (gated there on previous_response_id being present) and - # whether THIS request is persisted (gated there on - # request.store). Folding request.store in here collapsed the - # two, so a follow-up sent with store=false had its - # previous_response_id loaded and validated above - and then - # generated with no history at all, silently. + # Not folded with request.store: prior context is still read + # for a store=false follow-up; persisting checks request.store. enable_store=self.enable_store, use_harmony=self.use_harmony, tokenizer=self.tokenizer if not self.use_harmony else None, @@ -3303,8 +3259,6 @@ async def create_streaming_generator(promise: RequestOutput, sampling_params=sampling_params, model_name=self.model, conversation_store=self.conversation_store, - # Same as above: the processor's create_response applies - # request.store before persisting. enable_store=self.enable_store, use_harmony=self.use_harmony, reasoning_parser=self.generator.args.reasoning_parser, @@ -3325,11 +3279,6 @@ async def create_streaming_generator(promise: RequestOutput, if request.stream else responses_api_post_processor, postproc_args=postproc_args, ) - # Disaggregated serving. The signature check has to happen before - # the params reach the engine: encoded_opaque_state and - # ctx_info_endpoint name a peer this worker will pull KV cache - # from, so an unsigned request could redirect that fetch. - self._validate_internal_disagg_request(request, raw_request) disaggregated_params = to_llm_disaggregated_params( request.disaggregated_params) conversation_params = to_llm_conversation_params( @@ -3344,11 +3293,9 @@ async def create_streaming_generator(promise: RequestOutput, if self.postproc_worker_enabled else None, ) if not self.postproc_worker_enabled: - # The executor records this on the postprocessing arguments, - # but only for the requests whose postprocessing it owns. The - # streamed response is assembled here instead, and it needs - # the prompt length to report usage. + # Set by the executor only for postprocessing it runs itself. postproc_args.num_prompt_tokens = len(promise.prompt_token_ids) + postproc_args.tokenizer = self.tokenizer if self.postproc_worker_enabled and request.store: logger.warning( @@ -3357,9 +3304,6 @@ async def create_streaming_generator(promise: RequestOutput, asyncio.create_task(self.await_disconnected(raw_request, promise)) if request.stream: - # Ends a stream that stops before `response.completed` with - # an `error` event and a `response.failed` snapshot, so the - # client can tell a truncated turn from one still in flight. return StreamingResponse(content=guard_responses_stream( create_streaming_generator(promise, postproc_params), streaming_processor.get_stream_failed_events, @@ -3367,11 +3311,8 @@ async def create_streaming_generator(promise: RequestOutput, media_type="text/event-stream") else: response = await create_response(promise, postproc_params) - # Context-only: hand the tokenized prompt back as a base64 - # int32 buffer when the orchestrator asked for one, so it - # relays a single string rather than materializing the int - # list on its event loop. Encoding runs here, on one of N - # context workers, instead of on the single orchestrator. + # Context-only: the orchestrator may ask for the prompt as a + # base64 int32 buffer, as on the chat path. if (request.disaggregated_params is not None and request.disaggregated_params.return_prompt_token_ids_b64 and response.prompt_token_ids is not None): @@ -3380,13 +3321,7 @@ async def create_streaming_generator(promise: RequestOutput, np.asarray(response.prompt_token_ids, dtype=np.int32).tobytes()).decode("ascii") response.prompt_token_ids = None - # by_alias: a field whose python name differs from its - # wire name -- `schema_` for `schema`, which pydantic - # will not let a model call `schema` -- otherwise goes - # out under the internal name. The OpenAI SDK models - # this is rebuilt from downstream do not set - # `populate_by_name` and reject it, and a direct client - # is handed a field name the API does not have. + # by_alias: e.g. text.format.schema goes out under its wire name. return JSONResponse(content=response.model_dump(by_alias=True)) except CppExecutorError: logger.error(traceback.format_exc()) @@ -3414,7 +3349,7 @@ async def openai_responses_get_response(self, if response is None: return self._create_response_id_not_found_error(response_id) - return JSONResponse(content=response.model_dump()) + return JSONResponse(content=response.model_dump(by_alias=True)) async def openai_responses_delete_response( self, response_id: str) -> JSONResponse: diff --git a/tensorrt_llm/serve/postprocess_handlers.py b/tensorrt_llm/serve/postprocess_handlers.py index 1f19188d8019..9287cdd7172c 100644 --- a/tensorrt_llm/serve/postprocess_handlers.py +++ b/tensorrt_llm/serve/postprocess_handlers.py @@ -1088,8 +1088,6 @@ def responses_api_post_processor( reasoning_parser=args.reasoning_parser, tool_parser=args.tool_parser, num_prompt_tokens=args.num_prompt_tokens, - # Set on every PostprocArgs by the worker (postproc_worker.py:195); - # used here only to count reasoning tokens for usage. tokenizer=args.tokenizer, ) diff --git a/tensorrt_llm/serve/responses_utils.py b/tensorrt_llm/serve/responses_utils.py index a1014ae3fa1a..b77dd0a7b208 100644 --- a/tensorrt_llm/serve/responses_utils.py +++ b/tensorrt_llm/serve/responses_utils.py @@ -3,7 +3,6 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import asyncio -import base64 import json import os import time @@ -13,8 +12,8 @@ from collections.abc import AsyncGenerator, Mapping from copy import copy from dataclasses import dataclass -from typing import (Any, Callable, List, Literal, NamedTuple, Optional, - OrderedDict, Tuple, Union) +from typing import (Any, Callable, List, Literal, Optional, OrderedDict, Tuple, + Union) from openai.types.responses import (ResponseCompletedEvent, ResponseContentPartAddedEvent, @@ -55,7 +54,8 @@ from tensorrt_llm._utils import AdjustedSteadyClock from tensorrt_llm.executor import (EngineDeadError, GenerationResult, RequestError) -from tensorrt_llm.inputs.utils import async_apply_chat_template +from tensorrt_llm.inputs.utils import (async_apply_chat_template, + resolve_hf_chat_template) from tensorrt_llm.llmapi import SamplingParams from tensorrt_llm.llmapi.disagg_utils import get_usage_tokens_from_ctx from tensorrt_llm.llmapi.llm import RequestOutput @@ -81,7 +81,7 @@ ResponseUsage, StreamingResponsesResponse, UCompletionRequest, - UCompletionResponse, UsageInfo, + UCompletionResponse, to_disaggregated_params) from tensorrt_llm.serve.responses_web_search import is_web_search_tool from tensorrt_llm.serve.tool_parser.base_tool_parser import ( @@ -89,6 +89,8 @@ from tensorrt_llm.serve.tool_parser.core_types import ToolCallItem from tensorrt_llm.serve.tool_parser.tool_parser_factory import ToolParserFactory from tensorrt_llm.serve.web_search import load_web_search_config +from tensorrt_llm.tokenizer.deepseek_v4 import DeepseekV4Tokenizer +from tensorrt_llm.tokenizer.deepseek_v32 import DeepseekV32Tokenizer from .harmony_adapter import HarmonyAdapter, get_harmony_adapter @@ -113,46 +115,12 @@ CUSTOM_TOOL_INPUT_ARG = "input" -class StreamedToolCall(NamedTuple): - """One tool call exactly as the stream delivered it. - - The final snapshot is a second, independent re-parse of the generated - text, and on malformed markup the incremental state machine and the - whole-text regex read *different arguments out of the same bytes* - two - consecutive ```` tags stream as ``{"input": ...}`` but re-parse - into a corrupted key ``inputcmd`` (2 live cases a - week). The ids alone cannot fix that: a call named consistently but - described differently still tells the client two stories. So the record - carries the name and the assembled arguments the deltas were built from, - and the rebuild defers to them; see ``_create_output_content``. - - ``name``/``arguments`` are the parser-level values - (``ToolCallItem.name``/``.parameters``), not the output item's fields: - feeding them back through ``_tool_call_output_item`` with the same tool - resolution reproduces the streamed item byte for byte, custom-tool - unwrapping included. ``None`` means the recorder was not given them, in - which case the rebuild keeps its own parse as before. - """ - item_id: str - call_id: str - name: Optional[str] = None - arguments: Optional[str] = None - - @dataclass class StreamedItem: """One reasoning/message output item as the stream published it. - ``text`` accumulates every delta emitted for the item - appended by the - same call that emits the delta, so it is exactly the characters the - client received, kept even if the stream is cut off before the item - closes. The final snapshot assembles its reasoning/message items from - these records (``_create_output_content``): a re-derive from the - accumulated text can disagree with the stream on segmentation (an - unterminated call's fallback streams as two message items the re-parse - merges into one; 10 live cases a week) and on whitespace (the re-parse - stripped what the stream kept; 716 a week), and what already reached the - client is the one account of the generation that cannot be revised. + ``text`` holds every delta emitted for the item, so the final snapshot can + repeat exactly what the client received (see ``_create_output_content``). """ item_type: str item_id: str @@ -164,45 +132,16 @@ def _responses_debug_log(msg): logger.info(msg) -def _responses_debug_enabled() -> bool: - """Whether the caller should bother building a debug message. - - ``_responses_debug_log`` drops the message, but its argument is evaluated - first. Callers whose argument costs real work - decoding a whole prompt, - for instance - must check this before constructing it. - """ - return ENABLE_RESPONSES_DEBUG_MSG - - def _is_context_only(request: ResponsesRequest) -> bool: """Whether this request is the context half of a disaggregated split. Such a request comes from the orchestrator rather than a client, and its response is consumed by the orchestrator alone. """ - params = getattr(request, "disaggregated_params", None) + params = request.disaggregated_params return params is not None and params.request_type == "context_only" -def _relayed_prompt_token_ids(request: ResponsesRequest) -> Optional[list[int]]: - """The already-tokenized prompt the orchestrator relayed, if any. - - Returns None for an ordinary client request, which carries neither field - and must be rendered and tokenized here. - """ - if request.prompt_token_ids is not None: - return request.prompt_token_ids - if not request.prompt_token_ids_b64: - return None - # int32 little-endian buffer, matching what the context worker encodes. - import numpy as np - decoded = np.frombuffer(base64.b64decode(request.prompt_token_ids_b64), - dtype=np.int32).tolist() - # Cache it so a later reader does not decode a second time. - request.prompt_token_ids = decoded - return decoded - - _harmony_encoding = None @@ -414,7 +353,9 @@ async def store_messages(self, resp_id: str, _responses_debug_log( f" * storing at conversation: {conversation_id}") - self.conversations[conversation_id] = msgs + # A copy: trimming the stored conversation must not shorten the + # caller's list, which is rendered next. + self.conversations[conversation_id] = list(msgs) self.response_to_conversation[resp_id] = conversation_id self.conversation_to_response[conversation_id] = resp_id @@ -579,8 +520,9 @@ def _get_developer_message(instructions: Optional[str] = None, if tools is not None: function_tools = [] for tool in tools: - if tool.type in ("web_search_preview", "code_interpreter"): - # These are built-in tools that are added to the system message. + if tool.type == "code_interpreter" or tool.type.startswith( + "web_search"): + # Built-in tools, described in the system message if enabled. pass elif tool.type == "function": function_tools.append(tool) @@ -645,8 +587,9 @@ def _construct_harmony_messages( def _render_for_completion(messages: list[Message]) -> list[int]: conversation = Conversation.from_messages(messages) - _responses_debug_log("Rendering conversation:") - _responses_debug_log(conversation.to_json()) + if ENABLE_RESPONSES_DEBUG_MSG: + _responses_debug_log("Rendering conversation:") + _responses_debug_log(conversation.to_json()) token_ids = _get_encoding().render_conversation_for_completion( conversation, Role.ASSISTANT) return token_ids @@ -765,11 +708,7 @@ def finish_reason_mapping(finish_reason: str) -> str: case 'cancelled': return 'cancelled' case 'not_finished': - # Reached on the context worker in disaggregated serving: it is - # capped at one token and hands the request off, so the engine - # reports that generation has not finished. "incomplete" is the - # closest public status, and the orchestrator does not read it - - # it reads the unmapped value off ResponsesResponse.finish_reason. + # A disaggregated context worker handing the request off. return 'incomplete' raise RuntimeError( @@ -788,15 +727,8 @@ def _item_text(item: dict) -> str: continue text = part.get("text") if not text and part.get("type") == "encrypted_content": - # The KF client serializes a sub-agent task payload inside an - # agent_message as a part typed `encrypted_content` whose - # same-named field holds the plain readable task text - the - # field name is historical, not a description of the value. - # Dropping the part delivered the task header ("Payload:") - # with no payload behind it; one measured request lost a - # 721-character task this way. Only a string is taken at face - # value: a non-string here really is opaque, and guessing at - # it would fabricate input, so those stay dropped as before. + # Some clients carry readable text in an `encrypted_content` + # part; only a string value is taken as text. value = part.get("encrypted_content") if isinstance(value, str): text = value @@ -829,20 +761,12 @@ def _qualified_tool_name(item: dict) -> str: return f"{namespace}.{name}" if namespace else name -# The Responses API spells a text part `input_text` on the way in and -# `output_text` on the way out; the chat-completions content parser knows only -# `text`. Both mean the same thing, so they are translated rather than -# rejected. +# Responses text parts, which the chat-completions content parser knows as `text`. _RESPONSES_TEXT_PART_TYPES = frozenset(("input_text", "output_text")) def _chat_content_parts(content: list) -> list: - """Rewrite Responses-only text parts into the chat vocabulary. - - Everything else is passed through untouched: `image_url` and friends are - already understood downstream, and flattening the list to a plain string - would drop them. - """ + """Rewrite Responses text parts as chat text parts; keep the others.""" parts = [] for part in content: part_type = part.get("type") if isinstance(part, dict) else getattr( @@ -859,11 +783,7 @@ def _chat_content_parts(content: list) -> list: def _tool_output_content(output): """A tool result's payload in the vocabulary the chat parser knows. - `output` is a string on the simple path and a list of Responses content - parts when the client structures it. Passing a list through untouched - hands `input_text` to a parser that knows only `text`, which fails the - whole request rather than the item -- and a tool result arrives on every - turn after the first tool call, so the conversation stops there. + ``output`` is a string or a list of Responses content parts. """ if isinstance(output, list): return _chat_content_parts(output) @@ -874,19 +794,32 @@ def _tool_output_content(output): return str(output) -# `developer` is OpenAI's rename of `system`. Chat templates written before -# that rename dispatch on role and simply have no branch for it -- GLM-5.3 -# ends its dispatch after `system` with no fallback -- so a developer message -# renders to nothing and the client's instructions never reach the model. -# Every template understands `system`. -_ROLE_ALIASES = {"developer": "system"} - +def _render_developer_as_system( + messages: list[ChatCompletionMessageParam], + tokenizer: Optional[TokenizerBase], + processor: Optional[AutoProcessor], + tools: Optional[list[dict[str, Any]]], +) -> list[ChatCompletionMessageParam]: + """Render ``developer`` messages as ``system`` where the template lacks them. -def _chat_role(role: Optional[str]) -> str: - """The role to hand a chat template, as that template will understand it.""" - if not role: - return "assistant" - return _ROLE_ALIASES.get(role, role) + A chat template without a ``developer`` branch renders those messages to + nothing. The DeepSeek tokenizers render ``developer`` themselves. + """ + if not any(message.get("role") == "developer" for message in messages): + return messages + if isinstance(tokenizer, (DeepseekV32Tokenizer, DeepseekV4Tokenizer)): + return messages + template = resolve_hf_chat_template(getattr(tokenizer, "tokenizer", + tokenizer), + processor, + chat_template=None, + tools=tools) + if not isinstance(template, str) or "developer" in template: + return messages + return [{ + **message, "role": "system" + } if message.get("role") == "developer" else message + for message in messages] def _response_output_item_to_chat_completion_message( @@ -901,61 +834,32 @@ def _response_output_item_to_chat_completion_message( case "": if "role" not in item: raise ValueError(f"Invalid input message item: {item}") - item = {**item, "role": _chat_role(item.get("role"))} content = item.get("content") if isinstance(content, list): - # An item with a role and no `type` is the API's - # EasyInputMessage, where `type` defaults to "message". Its - # parts are spelled in the Responses vocabulary, so passing - # them through untouched hands `input_text` / `output_text` to - # the chat-completions parser, which knows neither and fails - # the request. The explicit "message" branch below already - # handles those parts, so leaving this one verbatim made - # success depend on a field the client may omit. return {**item, "content": _chat_content_parts(content)} return item case "message" | "reasoning": - content = item.get("content") or [] - if not content and item_type == "reasoning": - # Reasoning does not have to carry `content`. The API puts the - # text in `summary` when a summary was requested and in - # `encrypted_content` when it was not, and an item with neither - # populated is a shape OpenAI itself emits. Falling back to the - # summary keeps whatever text exists; when nothing readable is - # left the item is skipped rather than rejected, because the - # reasoning was already absent from the payload and failing - # here costs the whole conversation instead. - summary = item.get("summary") or [] - summary_text = "".join( - part.get("text") or "" if isinstance(part, dict) else ( - getattr(part, "text", "") or "") for part in summary) - if not summary_text: + content = item.get("content") + if isinstance(content, str): + content = [{"text": content}] + elif item_type == "reasoning" and not content: + # Reasoning may carry only a summary, or nothing readable. + content = item.get("summary") + if not content: return None - return {"role": "assistant", "reasoning": summary_text} if not content: raise ValueError( f"Input item of type {item_type!r} has empty or missing 'content'" ) - if isinstance(content, str): - # The API accepts `content` as one plain string, and the - # untyped branch above already preserves that shape - `type` - # merely defaults to "message" there. Walking the string as a - # list of parts iterates its characters, none of which carry a - # `text` field, so an explicit {"type": "message", "content": - # "hello"} converted to an empty message while the identical - # item without `type` survived. The two spellings mean the - # same thing and must convert the same way. - text = content - else: - # Join every text part. Taking content[0] silently dropped the - # rest of a multi-part message. - parts = [] - for part in content: - text = part.get("text") if isinstance( - part, dict) else getattr(part, "text", None) - if text: - parts.append(text) - text = "".join(parts) + # Join every text part. Taking content[0] silently dropped the rest + # of a multi-part message. + parts = [] + for part in content: + text = part.get("text") if isinstance(part, dict) else getattr( + part, "text", None) + if text: + parts.append(text) + text = "".join(parts) if item_type == "reasoning": # Reasoning is always the assistant's. return {"role": "assistant", "reasoning": text} @@ -966,7 +870,10 @@ def _response_output_item_to_chat_completion_message( # template markup rather than an answer. Clients that send # structured input items (Codex CLI, the OpenAI SDK) always set a # role; a plain string input never reaches this function. - return {"role": _chat_role(item.get("role")), "content": text} + role = item.get("role") or "assistant" + return {"role": role, "content": text} + case "output_text": + return {"role": "assistant", "content": item.get("text") or ""} case "function_call": # An assistant message carrying tool_calls, which is how the chat # completions path represents a call and what chat templates @@ -1057,43 +964,26 @@ def _fold_tool_calls_into_open_assistant_turn( message: ChatCompletionMessageParam, turn_start: int) -> bool: """Attach a converted tool-call item to the assistant message before it. - A Responses assistant turn that made N calls arrives as N consecutive - ``function_call`` items (after any reasoning/message items), and each one - converts to an assistant chat message carrying a single tool call. - Appending those separately splits one turn across N assistant messages, - and the GLM chat template aligns tool results only against the LAST - assistant message's ``tool_calls`` - so results for every call but the - last matched nothing and were rendered positionally, WITHOUT their ids. - Proven consequence: swapping two function_call_output call_ids produced a - byte-identical 596k-char prompt, i.e. the call-result binding was - unrecoverable downstream. One assistant message carrying the ordered list - of all N calls is the chat-completions shape (the history replay above - already stores turns that way), and it is what lets the template bind - each result to its call by id. - - The fold targets only messages converted from this request's input - - ``turn_start`` marks where those begin - so the instructions message and - replayed history are never mutated. A tool result or a user message - between two calls ends the assistant turn naturally: ``messages[-1]`` is - then no longer an assistant message and the next call opens a fresh one. - Preceding assistant text or reasoning stays in the same message, which is - where chat completions carries it alongside ``tool_calls``. + The N calls of one assistant turn arrive as N ``function_call`` items. Kept + in one assistant message, as chat completions does, they let the chat + template bind each tool result to its call by id; templates that align + results with the last assistant message only (GLM) lose that binding when + the calls are split across messages. + + Only messages from ``turn_start`` on, converted from this request's input, + are extended; a tool result or user message in between ends the turn. """ if message.get("role") != "assistant" or not message.get("tool_calls"): return False if message.get("content") is not None: - # Not a converted call item: both tool-call conversions carry - # content=None, and a message with its own content is a turn of its - # own, not a fragment of the previous one. + # A message with content of its own is a turn of its own. return False if len(messages) <= turn_start: return False last = messages[-1] if last.get("role") != "assistant": return False - # Rebuilt rather than extended in place: the target may carry - # "tool_calls" as None (clients send extra fields verbatim), and a list a - # caller handed in is not this function's to mutate. + # Rebuilt: the target's list may be None or owned by the caller. last["tool_calls"] = [ *(last.get("tool_calls") or []), *message["tool_calls"] ] @@ -1104,6 +994,14 @@ async def _create_input_messages( request: ResponsesRequest, prev_msgs: list[ChatCompletionMessageParam], ) -> list[ChatCompletionMessageParam]: + return chat_messages_from_responses_input(request, prev_msgs) + + +def chat_messages_from_responses_input( + request: ResponsesRequest, + prev_msgs: list[ChatCompletionMessageParam], +) -> list[ChatCompletionMessageParam]: + """Convert a Responses request's instructions, history and input to chat messages.""" messages: list[ChatCompletionMessageParam] = [] if request.instructions: messages.append({ @@ -1112,15 +1010,8 @@ async def _create_input_messages( }) # Prepend the conversation history. - # - # Reasoning is stripped on replay to save tokens, but a stored turn is - # more than its reasoning: the store writer attaches the turn's tool - # calls to the reasoning message, so discarding the whole message for its - # "reasoning" key also discarded the calls - and the tool RESULT the - # client sends next then replayed with no call before it, an orphan the - # model cannot pair with anything. Strip the reasoning, keep the calls. - # A turn that was only reasoning still vanishes, and a message without - # the key passes through exactly as stored. + # Skip the reasoning output, but keep the tool calls a reasoning message + # carries: the tool results that follow answer them. for msg in prev_msgs: if "reasoning" not in msg: messages.append(msg) @@ -1138,16 +1029,11 @@ async def _create_input_messages( if isinstance(request.input, str): messages.append({"role": "user", "content": request.input}) else: - # Where this request's own items start: the fold below must never - # reach back into the instructions message or the replayed history. turn_start = len(messages) for inp in request.input: message = _response_output_item_to_chat_completion_message(inp) if message is None: continue - # N calls from one assistant turn become ONE assistant message - # with N tool_calls, or the template loses the call-result - # binding; see the fold's docstring. if _fold_tool_calls_into_open_assistant_turn( messages, message, turn_start): continue @@ -1185,13 +1071,9 @@ def _create_output_messages( """ Convert output contents to chat completion messages for conversation store. - Reasoning is stored here and stripped on replay instead - (_create_input_messages), which keeps the token saving while letting the - replay keep the tool calls the reasoning message carries. Tool calls are - stored on whichever assistant message the turn produced - reasoning - first, else the text message, else a bare assistant tool-call message - - because a turn's calls must survive storage no matter what else the turn - contained. + Reasoning is stored and stripped on replay (_create_input_messages). Tool + calls go on the reasoning message, else the text message, else a bare + assistant message. Input: output_contents: dict[str, str] @@ -1233,14 +1115,6 @@ def _create_output_messages( reasoning_msg["tool_calls"] = tool_call_msgs messages.append(reasoning_msg) elif tool_call_msgs: - # The calls used to be stored only on the reasoning message, so a - # turn that called tools without reasoning lost them - and a turn - # that was *nothing but* tool calls stored no assistant message at - # all, leaving the client's tool RESULT to replay against a call - # that was never in the history. A tool-call-bearing turn is an - # assistant turn whether or not any text or reasoning came with it: - # attach the calls to the text message when there is one, store a - # bare assistant tool-call message when there is not. if text_msg is not None: text_msg["tool_calls"] = tool_call_msgs else: @@ -1398,12 +1272,14 @@ async def _create_input_tokens( await conversation_store.store_messages(request.request_id, messages, request.previous_response_id) - conversation, mm_coroutines, mm_placeholder_counts, _ = parse_chat_messages_coroutines( - messages, model_config) tools_dict = [ tool.model_dump() for tool in _get_chat_completion_function_tools(request.tools) ] + messages = _render_developer_as_system(messages, tokenizer, processor, + tools_dict) + conversation, mm_coroutines, mm_placeholder_counts, _ = parse_chat_messages_coroutines( + messages, model_config) # Carry the request's reasoning configuration into the chat template. # # Chat templates that support thinking are opt-in: DeepSeek-V4's custom @@ -1495,13 +1371,9 @@ async def request_preprocess( for msg in prev_msgs: _responses_debug_log(f" -> {msg}") - # In disaggregated serving the context worker has already rendered the chat - # template and tokenized; the orchestrator relays the result so the - # generation worker does not repeat that work on a prompt it is being - # handed verbatim. Rendering again would also be wrong, not just wasteful: - # a template that samples anything per-render could produce a prompt the - # context worker never prefilled, and the KV cache would not match. - pretokenized = _relayed_prompt_token_ids(request) + # A generation worker in disaggregated serving gets the context worker's + # tokens: rendering again could produce a prompt that was never prefilled. + pretokenized = request.relayed_prompt_token_ids() if pretokenized is not None: input_tokens = pretokenized elif use_harmony: @@ -1525,10 +1397,7 @@ async def request_preprocess( processor=processor, ) - if _responses_debug_enabled(): - # Guarded rather than passed straight to _responses_debug_log: decoding - # is a real tokenizer call on every request, and the argument would be - # evaluated whether or not debug logging is on. + if ENABLE_RESPONSES_DEBUG_MSG: # decoding is not free; skip it otherwise _responses_debug_log("======= Complete Inputs to model =======") _responses_debug_log(_decode_tokens(input_tokens, tokenizer)) _responses_debug_log("========================================") @@ -1621,132 +1490,43 @@ def _apply_reasoning_parser( return content, reasoning_content -def _flush_tool_parser( - tools: Optional[list[Tool]], - output_index: int, - tool_parser_dict: Optional[dict[int, BaseToolParser]] = None, -) -> Tuple[str, list[ToolCallItem], Optional[int]]: - """Whatever the incremental tool parser still holds when the stream ends. - - Returns the text to release, the calls to add, and the parser's tool index - for a call it was cut off in the middle of (None if it was not). - - The parser withholds any bytes that could still turn out to be tool-call - markup, and reports at most one finished call per increment, so the end of - a stream can leave both a complete call and a half-finished one behind with - no further increment coming. `finish` is each parser's own decision about - that remainder; most inherit the no-op, which leaves it sitting in - `_buffer`, so the buffer is drained and then inspected as well. - - Unterminated markup is released as ordinary text rather than dropped. - Dropping was the first design, on the grounds that half of a call is not - a message the model meant to send - but the final response never dropped - it: `_create_output_content` re-parses the whole text, its regex needs - the closing tag, and the fragment falls back into the message item. The - stream and the snapshot then told two different stories about the same - generation - trace tr_8f312e9973954fa784ae7dc8f45e9d3e ended in a - 429-character unterminated call that the final response carried as - message text while the stream showed nothing at all. Of the two ways to - make the views agree, keeping the text wins: these turns are recorded as - training data, and silently losing model output is worse than showing a - call that never closed. It is rare (1 of 3437 recorded responses) and - the warning keeps the malformed turn visible in the log. - - A remainder with no markup in it is ordinary output that the parser was - holding only until it could rule out a call, and is released. This matches - what DeepSeekR1Parser.finish already does for a trailing fragment that - might still have grown into a ``. - """ - tool_parser = (tool_parser_dict or {}).get(output_index) - if tool_parser is None or tools is None: - # No parser was ever built for this output, so nothing is held back. - return "", [], None - - released: list[str] = [] - calls: list[ToolCallItem] = [] - - # Drain calls the parser has already received in full but not yet reported. - # - # Several parsers report at most one call per increment and leave the rest - # buffered for the next one - Glm47ToolParser returns the moment it sees a - # ``. Two calls arriving in a single stream chunk therefore - # leave the second sitting complete and unreported, and at end of stream - # there is no next increment to collect it; replaying one recorded - # two-call response under randomised chunk boundaries lost a call in 67 of - # 400 chunkings. Empty increments stand in for the chunks that will never - # arrive. - # - # Bounded by the buffer having to shrink on every pass, so a parser that - # cannot make progress on what it holds ends the loop rather than spinning. - # Entered only while the buffer still holds markup, because plain text left - # over is `finish`'s business and some parsers strip framing from it. - held = getattr(tool_parser, "_buffer", "") - while held and tool_parser.has_tool_call(held): - drained = tool_parser.parse_streaming_increment("", tools) - calls.extend(drained.calls) - if drained.normal_text: - released.append(drained.normal_text) - remaining = getattr(tool_parser, "_buffer", "") - if len(remaining) >= len(held): - break - held = remaining - - result = tool_parser.finish(tools) - calls.extend(result.calls) - held = getattr(tool_parser, "_buffer", "") - - unterminated = 0 - for remainder in (result.normal_text, held): - if not remainder: - continue - if tool_parser.has_tool_call(remainder): - unterminated += len(remainder) - released.append(remainder) - - if held: - # The buffer's bytes are being released here, so consume them: the - # executor can present the same finished output more than once, and a - # buffer left in place would release the same remainder into a second - # message item on the repeat. Dropping never had this problem - a - # drop repeated is still nothing. - tool_parser._buffer = "" - - if unterminated: - logger.warning( - f"Stream ended inside a tool call; releasing {unterminated} " - f"characters of unterminated {type(tool_parser).__name__} markup " - "as the assistant's message, matching the final response's " - "fallback for a call it cannot parse.") - - # `current_tool_name_sent` is the parser's own record of having announced a - # call it has not yet closed, and `current_tool_id` numbers that call. - # Read through getattr because a parser that overrides - # parse_streaming_increment entirely need not maintain them; not - # maintaining them means it never reports a half-started call, which is - # the safe reading. - unfinished = None - if getattr(tool_parser, "current_tool_name_sent", False): - unfinished = getattr(tool_parser, "current_tool_id", None) +def _whole_text_tool_calls( + output: RequestOutput, + request: ResponsesRequest, + helper: "ResponsesStreamingEventsHelper", + reasoning_parser_id: Optional[str], + tool_parser_id: str, + tools: list[ChatCompletionToolsParam], +) -> Tuple[str, list[ToolCallItem]]: + """Tool calls from one whole-text parse at end of stream. - return "".join(released), calls, unfinished + Also returns the normal text the stream still owes: what that parse reads + as normal text beyond the message text already streamed. Markup the + incremental parser withheld is never released this way. + """ + content, _ = _apply_reasoning_parser( + reasoning_parser_id, + output.index, + output.text, + streaming=False, + chat_template_kwargs=reasoning_chat_template_kwargs(request)) + normal_text, calls = _apply_tool_parser(tool_parser_id, tools, output.index, + content, False) + streamed = "".join(item.text for item in helper.emitted_item_ids + if item.item_type == "message") + owed = normal_text[len(streamed):] if normal_text.startswith( + streamed) else "" + return owed, calls def _effective_tool_parser(tool_parser_id: Optional[str], - request) -> Optional[str]: - """The tool parser this request may run, honouring ``tool_choice="none"``. - - "none" is the one tool_choice value that can be enforced without - falsifying anything the model generated: not parsing leaves any tool-call - markup in the visible text verbatim, so nothing is truncated or invented - - where "required" or a named function would need sampling-level - constraints, which this path does not apply. Both the streaming event - generator and the final rebuild resolve their parser through here, because - the two views of one generation must agree on whether a call exists. - - Read with getattr for the same reason reasoning_chat_template_kwargs - does: unit tests drive the event generator with stand-in request objects. + request: ResponsesRequest) -> Optional[str]: + """The tool parser to run: none for ``tool_choice="none"``. + + Not parsing keeps any call markup in the visible text verbatim, in the + stream and the final snapshot alike. """ - if getattr(request, "tool_choice", None) == "none": + if request.tool_choice == "none": return None return tool_parser_id @@ -1788,22 +1568,11 @@ def _apply_tool_parser( def _streamed_items_cover(streamed_items: list[StreamedItem], reasoning_text: Optional[str], text: Optional[str]) -> bool: - """Whether the stream's item record accounts for the re-parsed content. - - True means the two passes read the same characters out of the generation - and differ at most in segmentation (the stream can split one fallback - text across two message items) or in edge whitespace (the whole-text - GLM-4.7 parse strips what the stream kept) - exactly the differences the - streamed record exists to settle. False means the record cannot describe - this rebuild: text the stream never published (a cut-off stream whose - tail only the accumulated text holds) or a reasoning/content split the - two parsers genuinely disagree on. The caller then falls back to the - re-derived items, which is the one choice that loses no generated text, - and the structural mismatch is warned about as before. - - Compared per item type and stripped on both sides: whitespace inside the - joined streamed text is preserved by the join itself, so only the edges - - the part the whole-text parser may have stripped - are forgiven. + """Whether the streamed items hold the same characters as the re-parse. + + Item boundaries and edge whitespace may differ (the whole-text parse can + strip what the stream already sent); anything else means the two views + diverged and the re-parsed items are used instead. """ streamed_reasoning = "".join(item.text for item in streamed_items if item.item_type == "reasoning") @@ -1819,31 +1588,26 @@ def _create_output_content( tool_parser: Optional[str] = None, tools: Optional[list[Tool]] = None, chat_template_kwargs: Optional[dict[str, Any]] = None, - streamed_tool_call_ids: Optional[list[Optional[StreamedToolCall]]] = None, + streamed_tool_calls: Optional[list[ResponseOutputItem]] = None, streamed_item_ids: Optional[list[StreamedItem]] = None, ) -> Tuple[list[ResponseOutputItem], list[ChatCompletionMessageParam], list[str]]: + """Build the output items for a finished generation. + + For a streamed request (``streamed_item_ids``/``streamed_tool_calls`` not + None) the snapshot repeats what the stream published: its reasoning and + message items when they cover the re-parsed text, and its tool-call items + as emitted. Otherwise items are derived from a whole-text parse, reusing + streamed ids positionally where a stream ran. + """ output_items: list[ResponseOutputItem] = [] output_messages: list[ChatCompletionMessageParam] = [] - # What the reasoning parser claimed as reasoning, per output and before - # stripping, so usage accounting can count its tokens. Kept raw: the - # whitespace around a reasoning block was generated too. + # Raw reasoning per output, for counting reasoning tokens. reasoning_texts: list[str] = [] available_tools = _get_chat_completion_function_tools(tools) - # The reasoning/message items the stream published in - # `response.output_item.added/done`, in emission order and with the text - # their deltas carried. This rebuild is a second, independent pass over - # the generated text and used to mint fresh ids for the items it - # re-derives, so the snapshot named the very items the stream had already - # announced under different ids - 216 of 221 measured responses - and a - # client joining streamed items with the snapshot by id saw phantom - # items. When the record and the re-parse read the same characters the - # snapshot now repeats the record outright (see the assembly below); the - # per-type pools here serve the fallback, where reuse is positional per - # item type, first-emitted first, because emission order is the only - # correspondence the two passes share. None means no stream ran (a - # non-streaming request) and every id is minted fresh, as before. + # A stream only runs over a single output. + single_output = len(final_res.outputs) == 1 streamed_ids_by_type: dict[str, list[str]] = { "reasoning": [], "message": [] @@ -1852,21 +1616,9 @@ def _create_output_content( if record.item_type in streamed_ids_by_type: streamed_ids_by_type[record.item_type].append(record.item_id) used_ids_by_type = {"reasoning": 0, "message": 0} - - # The record is per request while this loop is per output, and a stream - # only ever runs over outputs[0] (process_single_output), so the whole - # record describes a single output. More than one output means no stream - # produced it and the positional id reuse below is all there is. - streamed_items_available = (streamed_item_ids is not None - and len(final_res.outputs) == 1) used_streamed_assembly = False def _streamed_or_fresh_id(item_type: str) -> str: - # Running past the pool means this pass derived items the stream never - # opened - the two views already diverged structurally (a stream cut - # off inside a call whose text this pass keeps, say). A fresh id - # cannot hide that and must not try to: the count mismatch is warned - # about after the loop rather than silently papered over. pool = streamed_ids_by_type[item_type] index = used_ids_by_type[item_type] used_ids_by_type[item_type] = index + 1 @@ -1891,52 +1643,16 @@ def _streamed_or_fresh_id(item_type: str) -> str: reasoning_texts.append(reasoning_text or "") if text: - parsed_source = text text, calls = _apply_tool_parser(tool_parser, available_tools, output.index, text, False) - if calls and streamed_tool_call_ids is None: - # No stream ran, so no per-entity record says which calls the - # streamed assembly would have refused - the check the pairing - # below performs for streamed requests. An empty-argument call - # is the shape the whole-text parse gets wrong on its own (it - # reports `{}` for argument markup it could not read), so it - # is verified against a streamed replay of the same text; see - # the reconciler for why the two views must agree here. - text, calls = _verify_empty_calls_against_streaming( - tool_parser, available_tools, parsed_source, text, calls) - - tool_calls_item = [] + stored_text: Optional[str] = None stored_reasoning: Optional[str] = None - - # Reasoning first, then the answer, then any tool calls. - # - # `response.output` is ordered, and the model produced the reasoning - # before the answer it leads to. The streaming path emits the items in - # that order - reasoning done, then message done, then the function - # calls (see _generate_streaming_event) - so a snapshot that appended - # the message first contradicted the stream of the very same - # generation: a client replaying `output` read the answer before the - # reasoning that produced it, and one reconstructing a turn from the - # snapshot fed the model its own thinking as a follow-up to its reply. - # - # For a streamed request the items themselves come from the stream's - # own record, not from the re-parse above. The re-parse can disagree - # with the stream on item boundaries - an unterminated call's fallback - # streams as two message items that a whole-text parse can only read - # as one, so the second streamed item simply vanished from the - # snapshot (10 live cases a week) - and on edge whitespace, which it - # stripped while the stream had already sent it (` Backbone` on the - # wire, `Backbone` in the snapshot; 716 a week). What was streamed is - # irrevocable, so when the two passes read the same characters the - # snapshot repeats the stream's items verbatim: same count, same - # boundaries, same ids, same bytes. When they did not read the same - # characters the views genuinely diverged (a stream cut off while the - # accumulated text kept growing), and the re-derived items are kept - - # the choice that loses no generated text - with the mismatch warned - # about below, as before. - if streamed_items_available and _streamed_items_cover( - streamed_item_ids, reasoning_text, text): + # Reasoning first, then the answer, then any tool calls: the stream's + # order. + if (streamed_item_ids is not None + and single_output and _streamed_items_cover( + streamed_item_ids, reasoning_text, text)): used_streamed_assembly = True for record in streamed_item_ids: if record.item_type == "reasoning": @@ -1966,9 +1682,6 @@ def _streamed_or_fresh_id(item_type: str) -> str: status="completed", type="message", )) - # The conversation store keeps one assistant message per output, - # so the per-item boundaries flatten back out here - but to the - # streamed characters, not the re-parsed ones. stored_reasoning = "".join( record.text for record in streamed_item_ids if record.item_type == "reasoning") or None @@ -1976,121 +1689,45 @@ def _streamed_or_fresh_id(item_type: str) -> str: if record.item_type == "message") or None else: if reasoning_text: - reasoning_item = ResponseReasoningItem( - id=_streamed_or_fresh_id("reasoning"), - summary=[], - type="reasoning", - # Verbatim: the whitespace around a reasoning block was - # generated too, and the streamed view of the same - # generation delivers it. A strip here made the two views - # disagree about the model's own characters. - content=[ - Content(text=reasoning_text, type="reasoning_text") - ], - status=None, - ) - output_items.append(reasoning_item) + output_items.append( + ResponseReasoningItem( + id=_streamed_or_fresh_id("reasoning"), + summary=[], + type="reasoning", + content=[ + Content(text=reasoning_text, type="reasoning_text") + ], + status=None, + )) stored_reasoning = reasoning_text # Check again after tool parsing to avoid empty text if text: - output_text = ResponseOutputText( - # Verbatim, for the same reason as the reasoning above. - text=text, - annotations=[], - type="output_text", - logprobs=None, - ) - - text_item = ResponseOutputMessage( - id=_streamed_or_fresh_id("message"), - content=[output_text], - role="assistant", - status="completed", - type="message", - ) - - output_items.append(text_item) + output_items.append( + ResponseOutputMessage( + id=_streamed_or_fresh_id("message"), + content=[ + ResponseOutputText( + text=text, + annotations=[], + type="output_text", + logprobs=None, + ) + ], + role="assistant", + status="completed", + type="message", + )) stored_text = text - if calls: + if streamed_tool_calls is not None and single_output: + tool_calls_item = list(streamed_tool_calls) + else: tool_resolution = _tool_resolution(tools) - # Reuse the ids the streaming path already published. The stream's - # record holds one entry per call the parser announced, in - # announcement order - None where assembly dropped the call for - # invalid-JSON arguments - and this pass enumerates the same - # markup in the same order, so when the counts agree each call is - # paired with its own entity's entry. Pairing over the *filtered* - # list instead is how a dropped call A used to donate its position: - # the whole-text parse still reports A (with empty arguments), A - # consumed the id the stream had published for B, and B's identity - # in the snapshot matched nothing the client had seen. - reusable = list(streamed_tool_call_ids or []) - entity_aligned = len(reusable) == len(calls) - if reusable and not entity_aligned: - # Counts disagreeing means the two passes did not see the same - # calls (a cut-off stream, say), so per-entity pairing is - # unknowable. Say so, then fall back to reusing the ids that - # actually reached the wire, in order; placeholders name no - # wire identity and are worth nothing to a client. - logger.warning( - f"final response rebuilt {len(calls)} tool call(s) but the " - f"stream saw {len(reusable)}; ids beyond the streamed ones " - f"are new and will not match the client's tool outputs") - reusable = [record for record in reusable if record is not None] - tool_calls_item = [] - for index, call in enumerate(calls): - record = reusable[index] if index < len(reusable) else None - if entity_aligned and record is None: - # This entity is the call the stream dropped: its - # arguments never assembled into valid JSON and no id was - # published for it. The snapshot must describe the stream, - # so it is not delivered here either - the placeholder is - # consumed by the drop instead of shifting onto the next - # call. - logger.warning( - f"final response drops the tool call to {call.name!r}: " - f"the stream assembled its arguments into invalid JSON " - f"and never delivered it, so the snapshot does not " - f"deliver it either") - continue - # Same call entity, two parses. On malformed markup they read - # different arguments out of the same bytes - two consecutive - # `` tags stream as `{"input": ...}` while the - # whole-text regex delivers a corrupted key - and the client - # has already received, and possibly executed, the streamed - # version; a snapshot that re-describes the call under the - # same call_id hands the client two different inputs for one - # identity (2 live cases a week). The streamed bytes are the - # ones that cannot be unsent, so they win. Only under entity - # alignment: when the two passes disagree about what the call - # entities *are*, pairing a streamed description with a - # re-parsed call would guess, and the ids-only reuse above is - # as far as certainty goes. - streamed_name = getattr(record, "name", None) - streamed_arguments = getattr(record, "arguments", None) - if (entity_aligned and streamed_name is not None - and streamed_arguments is not None): - if (call.name != streamed_name - or (call.parameters or "") != streamed_arguments): - logger.warning( - f"final response re-parsed the tool call to " - f"{call.name!r} differently from the stream " - f"(streamed arguments " - f"{streamed_arguments[:200]!r}, re-parsed " - f"{(call.parameters or '')[:200]!r}); keeping the " - f"streamed name and arguments, which the client " - f"already received") - call = ToolCallItem(tool_index=call.tool_index, - name=streamed_name, - parameters=streamed_arguments) - tool_calls_item.append( - _tool_call_output_item( - call, - tool_resolution, - item_id=record.item_id if record else None, - call_id=record.call_id if record else None)) - output_items.extend(tool_calls_item) + tool_calls_item = [ + _tool_call_output_item(call, tool_resolution) for call in calls + ] + output_items.extend(tool_calls_item) output_messages.extend( _create_output_messages({ @@ -2099,13 +1736,6 @@ def _streamed_or_fresh_id(item_type: str) -> str: "tool_calls": tool_calls_item, })) - # Checked against None, not truthiness: an empty list still means a - # stream ran and published nothing of this type, and a rebuild that then - # derives such an item is exactly the divergence worth saying out loud. - # Skipped when the snapshot was assembled from the streamed record: the - # counts then match by construction and the only mismatch left - the - # re-parse segmenting the same characters differently - is the one the - # record exists to settle, not a divergence. if streamed_item_ids is not None and not used_streamed_assembly: for item_type in ("reasoning", "message"): streamed = len(streamed_ids_by_type[item_type]) @@ -2113,8 +1743,7 @@ def _streamed_or_fresh_id(item_type: str) -> str: if streamed != rebuilt: logger.warning( f"final response rebuilt {rebuilt} {item_type} item(s) but " - f"the stream published {streamed}; the two views of this " - f"generation differ structurally and ids beyond the " + f"the stream published {streamed}; ids beyond the " f"streamed ones are new") return output_items, output_messages, reasoning_texts @@ -2145,21 +1774,8 @@ def _tool_resolution( A chat template can only describe a flat list of functions, so a namespaced tool is offered to the model as "namespace.tool" and has to be reported back with the two parts separated again - that is how the client identifies it. - - The model writes the name back BOTH ways. Measured over one agent session: - 247 calls arrived as `exec` and 34 as `functions.exec`. The two functions - this replaces keyed only on the qualified spelling, so every bare-named call - missed the lookup, was classified as a plain function call, and reached the - client as JSON where a custom tool expects freeform text. The client rejects - that outright: 154 of 154 such calls came back "aborted", while the 10 that - happened to arrive qualified all ran. The agent then concluded its execution - runtime was broken and burned its retries. - - A bare name is registered only when exactly one declared tool answers to it. - Two namespaces offering the same tool name cannot be told apart from the - name alone, and routing a call to the wrong namespace is worse than leaving - it unresolved - the caller reports an unresolved call as it always did, and - now warns about it. + Models write the name back both ways, so the bare name resolves too, but + only when exactly one declared tool answers to it. """ resolved: dict[str, Tuple[Optional[str], str, bool]] = {} bare_claims: dict[str, list[str]] = {} @@ -2199,7 +1815,6 @@ def _tool_call_output_item( tool_resolution: dict[str, Tuple[Optional[str], str, bool]], item_id: Optional[str] = None, status: Optional[str] = None, - call_id: Optional[str] = None, ) -> Union[ResponseFunctionToolCall, ResponseCustomToolCall]: """Build the output item for one parsed tool call. @@ -2208,29 +1823,16 @@ def _tool_call_output_item( function call hands the client JSON where it expects the raw payload, and the client rejects the call outright - for apply_patch, with "invoked with incompatible payload", which aborts the whole turn. - - Which of the two it is turns on whether the call's name resolves to a - declared tool, and the model spells that name inconsistently; see - ``_tool_resolution`` for what that cost before it accepted both spellings. """ name = call.name or "" arguments = call.parameters or "{}" - # A fresh id only when this call has not been reported before. The final - # response re-derives its output from the generated text, so it must be - # given back the ids the stream already used; the client keys its tool - # outputs on those and matches nothing otherwise. - call_id = call_id or f"call_{_random_uuid()}" + call_id = f"call_{_random_uuid()}" resolved = (tool_resolution or {}).get(name) if resolved is not None: namespace, name, is_custom = resolved else: namespace, is_custom = None, False - # Previously silent, which is why a whole class of aborted calls went - # unexplained for as long as it did: a custom tool reported as a - # function call is rejected by the client, and nothing said so. SGLang - # logs "Model attempted to call undefined function" at the equivalent - # point; borrowing that is most of the value of this change. logger.warning( f"tool call {name!r} matches no declared tool; reporting it as a " f"function call. If it is in fact a custom tool, the client will " @@ -2275,25 +1877,6 @@ def _tool_call_output_item( return item -def _ctx_usage_from_result(final_res: GenerationResult) -> Optional[Any]: - """The context phase's usage, as the handoff left it on the outputs. - - Mirrors the chat path's `_ctx_usage_from_outputs`. An aggregated server has - no handoff and returns None, which leaves the caller on the local counts. - """ - for output in getattr(final_res, "outputs", None) or []: - disaggregated_params = getattr(output, "disaggregated_params", None) - if disaggregated_params is None: - continue - ctx_usage = getattr(disaggregated_params, "ctx_usage", None) - if ctx_usage is None: - continue - if isinstance(ctx_usage, UsageInfo): - return ctx_usage - return UsageInfo.model_validate(ctx_usage) - return None - - def _count_reasoning_tokens( tokenizer: Optional[TokenizerBase], reasoning_texts: list[str], @@ -2301,31 +1884,10 @@ def _count_reasoning_tokens( ) -> int: """How many of the generated tokens went into reasoning. - The engine does not track this. `ThinkingBudgetLogitsProcessor` computes - the same quantity in token space (thinking_budget.py:93), but it lives in - the model engine process, only exists when `thinking_token_budget` is set, - and a LogitsProcessor has no channel back to the result -- so the count - has to be rebuilt here. - - This re-encodes the text the reasoning parser itself claimed, rather than - searching `output.token_ids` for the `` token. The marker search - is cheaper and exact, and it was the first design; it was dropped because - it has to restate the parser's rules and would silently disagree the - moment they differ. GLM alone needs three of them: it closes the block - more than once and only the first close counts, `` ends the - block implicitly, and a turn that emits neither is reasoning to its last - character (reasoning_parser.py:369-404, with a measured case of 26,055 - characters and no closing tag at all). Deriving the count from the - parser's own output cannot drift from it, and it stays correct for the - parsers that interleave reasoning with content, where no single prefix - length describes the split. - - The cost is one encode of the reasoning text per response, off the - engine's critical path, and a boundary that can differ by a token from - the generated one -- re-encoding a substring need not reproduce the - tokenization it came from. Returns 0 when it cannot be determined at all - (no tokenizer, or no reasoning parser configured), which is what this - reported unconditionally before. + The engine does not track this, so the text the reasoning parser claimed is + re-encoded, which keeps the count consistent with the parser's rules. The + result is clamped to ``output_tokens``: re-encoding a substring need not + reproduce its original tokenization. 0 without a tokenizer. """ if tokenizer is None: return 0 @@ -2339,10 +1901,6 @@ def _count_reasoning_tokens( except TypeError: # Not every tokenizer accepts the keyword. total += len(tokenizer.encode(text)) - - # Re-encoding can overshoot, and usage that claims more reasoning tokens - # than were generated is visibly wrong to a client budgeting a context - # window. Clamping is the conservative direction. return min(total, output_tokens) @@ -2374,15 +1932,12 @@ def _create_usage( output_tokens = sum(len(output.token_ids) for output in final_res.outputs) cached_tokens = getattr(final_res, "cached_tokens", None) or 0 - # Under disaggregated serving the counts above describe the generation - # worker, and from there the entire prompt arrived as KV transferred from - # the context worker -- so its cached_tokens equals the prompt length and - # every response claims a complete cache hit. The context phase is the only - # place that knows what was really reused, and it sends its own usage along - # with the handoff. Chat completions has taken it from there since #14177; - # this route was reading the generation worker's numbers instead. + # Under disaggregated serving the whole prompt reached this worker as + # transferred KV; the context phase's usage, carried with the handoff, + # says what was actually reused. + from tensorrt_llm.serve.postprocess_handlers import _ctx_usage_from_outputs ctx_prompt_tokens, ctx_cached_tokens = get_usage_tokens_from_ctx( - _ctx_usage_from_result(final_res)) + _ctx_usage_from_outputs(final_res.outputs)) if ctx_prompt_tokens is not None: input_tokens = ctx_prompt_tokens cached_tokens = ctx_cached_tokens @@ -2408,7 +1963,7 @@ def _create_response( reasoning_parser: Optional[str] = None, tool_parser: Optional[str] = None, num_prompt_tokens: Optional[int] = None, - streamed_tool_call_ids: Optional[list[Optional[StreamedToolCall]]] = None, + streamed_tool_calls: Optional[list[ResponseOutputItem]] = None, streamed_item_ids: Optional[list[StreamedItem]] = None, tokenizer: Optional[TokenizerBase] = None, ) -> tuple[ResponsesResponse, list[Message | ChatCompletionMessageParam]]: @@ -2419,24 +1974,21 @@ def _create_response( # prepare responses output output_content = [] + reasoning_texts: list[str] = [] if use_harmony: - output_content, output_messages = _create_output_content_harmony( - final_res) - # Harmony carries its reasoning on the `analysis` channel, which the - # adapter already tracks per token (harmony_adapter.py:207). Counting - # it is a separate change on that path; this one leaves it reported - # as zero rather than half-counting it here. - reasoning_texts = [] + # A context-only output is a single handoff token, not a complete + # Harmony message, so there is nothing to parse. + output_content, output_messages = ( + ([], []) if _is_context_only(request) else + _create_output_content_harmony(final_res)) else: output_content, output_messages, reasoning_texts = _create_output_content( final_res, reasoning_parser, - # tool_choice="none" bypasses the parser here exactly as the - # streaming events do, so both views keep the markup as text. _effective_tool_parser(tool_parser, request), request.tools, chat_template_kwargs=reasoning_chat_template_kwargs(request), - streamed_tool_call_ids=streamed_tool_call_ids, + streamed_tool_calls=streamed_tool_calls, streamed_item_ids=streamed_item_ids) finish_reason = final_res.outputs[0].finish_reason @@ -2452,24 +2004,13 @@ def _create_response( tokenizer=tokenizer, reasoning_texts=reasoning_texts), ) - # A generation the engine cut off at its token budget has status - # "incomplete", and the spec explains that status in - # incomplete_details.reason - "max_output_tokens". The streaming terminal - # event has said so since the response.incomplete fix; setting it here - # makes the non-streaming JSON body tell the same story. Only "length" is - # explained: "not_finished" (a disaggregated context worker handing the - # request off) also maps to "incomplete", and naming a public reason for - # an internal handoff would be a guess presented as fact. + # Only a token-budget cut is explained; "not_finished" (a disaggregated + # context worker handing off) also maps to "incomplete". if finish_reason == "length": response.incomplete_details = IncompleteDetails( reason="max_output_tokens") - # Disaggregated serving. A context-only response is read by the - # orchestrator, never by a client: it carries the KV-cache handle, the - # first generated token and the tokenized prompt so a generation worker can - # pick the request up. Everything else - an aggregated server, or the - # generation response that does reach the client - leaves these unset. - # ctx_info_endpoint in particular names an internal address, and - # prompt_token_ids would add the whole prompt back to every reply. + # The disaggregated handoff fields are set only on a context-only response, + # which the orchestrator reads and strips before anything reaches a client. if _is_context_only(request): response.finish_reason = finish_reason response.disaggregated_params = to_disaggregated_params( @@ -2497,8 +2038,6 @@ async def create_response( reasoning_parser: Optional[str] = None, tool_parser: Optional[str] = None, num_prompt_tokens: Optional[int] = None, - streamed_tool_call_ids: Optional[list[Optional[StreamedToolCall]]] = None, - streamed_item_ids: Optional[list[StreamedItem]] = None, tokenizer: Optional[TokenizerBase] = None, ) -> ResponsesResponse: @@ -2526,8 +2065,6 @@ async def create_response( reasoning_parser=reasoning_parser, tool_parser=tool_parser, num_prompt_tokens=num_prompt_tokens, - streamed_tool_call_ids=streamed_tool_call_ids, - streamed_item_ids=streamed_item_ids, tokenizer=tokenizer, ) @@ -2549,7 +2086,7 @@ def create_response_non_store( reasoning_parser: Optional[str] = None, tool_parser: Optional[str] = None, num_prompt_tokens: Optional[int] = None, - streamed_tool_call_ids: Optional[list[Optional[StreamedToolCall]]] = None, + streamed_tool_calls: Optional[list[ResponseOutputItem]] = None, streamed_item_ids: Optional[list[StreamedItem]] = None, tokenizer: Optional[TokenizerBase] = None, ) -> ResponsesResponse: @@ -2567,7 +2104,7 @@ def create_response_non_store( reasoning_parser=reasoning_parser, tool_parser=tool_parser, num_prompt_tokens=num_prompt_tokens, - streamed_tool_call_ids=streamed_tool_call_ids, + streamed_tool_calls=streamed_tool_calls, streamed_item_ids=streamed_item_ids, tokenizer=tokenizer, ) @@ -2586,56 +2123,15 @@ class ResponsesStreamingStateTracker: reasoning_sent: bool = False # Deltas already streamed for the item currently open, so it can be closed # with its full text if generation ends before the parser says it is done. - emitted_tool_calls: int = 0 text_buffer: str = "" reasoning_buffer: str = "" def __init__(self) -> None: - # Tool-call fragments from the incremental parser, keyed by output - # index, then by the parser's tool index. One parser instance per - # output lives in tool_parser_dict, so what those instances report has - # to be kept apart in the same way. - # - # Assigned here rather than in the class body above: a dict written as - # a class attribute is one object shared by every request, so two - # concurrent streams would accumulate into each other's calls. The - # fields above are immutable, so assigning to them rebinds per - # instance and they are safe as class-level defaults. + # Incremental tool-call fragments, by output index, then tool index. self.tool_call_fragments: dict[int, dict[int, dict[str, Any]]] = {} - - # (item id, call id) of every tool call already streamed, in the order - # they went out. `response.completed` rebuilds the output from scratch, - # and _tool_call_output_item mints a fresh random id each time it runs, - # so without this the final response names every call differently from - # the events that carried it - and the client's tool outputs, which - # echo the streamed id, match nothing. Measured before the fix: 671 of - # 671 calls across three campaigns. - # - # One entry per call the parser announced, in announcement order: None - # holds the place of a call that was assembled but dropped for - # invalid-JSON arguments. The rebuild enumerates the same markup in - # the same order, so the placeholder is what lets a drop consume its - # own position instead of shifting the streamed identities onto the - # calls behind it. - # - # A list here rather than a class attribute, for the reason above. - self.emitted_tool_call_ids: list[Optional[StreamedToolCall]] = [] - - # Every reasoning and message item this stream opened, in the order - # their `output_item.added` went out, each carrying the text its - # deltas delivered (appended as they are emitted; see - # ResponsesStreamingEventsHelper.append_text/append_reasoning). The - # final response re-derives those items from the accumulated text and - # used to mint fresh ids for them, so the snapshot named the very - # items the stream had already announced under different ids - 216 of - # 221 measured responses - and a client joining streamed items with - # the snapshot by id saw phantom items. The texts travel with the ids - # because the re-derive can also disagree on item boundaries and on - # edge whitespace; the snapshot assembles these records verbatim - # instead (see _create_output_content). Same shape and cure as - # emitted_tool_call_ids above. - # - # A list here rather than a class attribute, for the reason above. + # Items already streamed, in emission order; the final snapshot + # repeats them. + self.emitted_tool_call_items: list[ResponseOutputItem] = [] self.emitted_item_ids: list[StreamedItem] = [] @@ -2657,66 +2153,17 @@ def output_index_increment(self): self.state_tracker.current_output_index += 1 @property - def emitted_tool_calls(self) -> int: - return self.state_tracker.emitted_tool_calls - - @emitted_tool_calls.setter - def emitted_tool_calls(self, count: int) -> None: - self.state_tracker.emitted_tool_calls = count - - @property - def emitted_tool_call_ids(self) -> list[Optional[StreamedToolCall]]: - """The tool calls already streamed, in order; see StreamedToolCall. - - None entries hold the place of calls the stream assembled and then - dropped for invalid-JSON arguments; see the state tracker. - """ - return self.state_tracker.emitted_tool_call_ids + def emitted_tool_call_items(self) -> list[ResponseOutputItem]: + """The tool-call items already streamed, in order.""" + return self.state_tracker.emitted_tool_call_items @property def emitted_item_ids(self) -> list[StreamedItem]: """The reasoning/message items streamed, in order; see StreamedItem.""" return self.state_tracker.emitted_item_ids - def record_emitted_tool_call(self, - item, - call: Optional[ToolCallItem] = None) -> None: - """Remember the identity and content a streamed call went out under. - - The final response is built by a second, independent pass over the - generated text; handing it these lets it name the same call the same - way instead of minting new ids the client has never seen - and, when - `call` (the parser-level ToolCallItem the item was built from) is - given, describe it with the same name and arguments instead of - whatever its own re-parse read out of malformed markup. - """ - self.state_tracker.emitted_tool_call_ids.append( - StreamedToolCall(item_id=item.id, - call_id=item.call_id, - name=getattr(call, "name", None), - arguments=getattr(call, "parameters", None))) - - def record_dropped_tool_call(self) -> None: - """Hold a dropped call's place in the emission record. - - A call whose assembled arguments were invalid JSON is never streamed, - but the final rebuild's whole-text parse still reports the entity (the - markup is well-formed; only the arguments are unusable). Recording the - drop keeps the record aligned with the parser's announcement order, so - in the rebuild the drop consumes its own position instead of handing - the next call an identity the stream published for a different one. - """ - self.state_tracker.emitted_tool_call_ids.append(None) - def _record_item_delta(self, item_type: str, delta: str) -> None: - # The emission record mirrors every delta the moment it is appended, - # so the record holds exactly the characters the client received for - # the item - including for an item the stream was cut off inside, - # which no close will ever flush. The open item is always the last - # record of its type to have been announced (one item is open at a - # time, and the opener runs before the first append); when no record - # matches, nothing was announced and there is nothing to mirror, - # which is the state the buffer-only unit tests drive this in. + # The open item is the last one announced. records = self.state_tracker.emitted_item_ids if records and records[-1].item_type == item_type: records[-1].text += delta @@ -2775,7 +2222,7 @@ def get_response_created_event( self, response: ResponsesResponse) -> ResponseCreatedEvent: return ResponseCreatedEvent( type="response.created", - sequence_number=-1, # set by _send_event, restamped at egress + sequence_number=-1, # will set by _send_event function response=response, ) @@ -2887,14 +2334,6 @@ def _get_output_added_events( if not self.is_output_item_added_sent: self.is_output_item_added_sent = True - # Remember the id this item is announced under, at the one point - # where announcing actually happens. The final response is built - # by a second, independent pass over the generated text; handing - # it these lets it name the same items the same way instead of - # minting ids the client has never seen. The record's text starts - # empty and accumulates with every delta (append_text / - # append_reasoning), so the snapshot can also repeat the item's - # characters rather than re-deriving them. self.state_tracker.emitted_item_ids.append( StreamedItem(item_type=output_item.type, item_id=output_item.id)) @@ -2959,115 +2398,45 @@ def _accumulate_tool_call_fragments(fragments: dict[int, dict[str, Any]], calls: list[ToolCallItem]) -> None: """Fold the incremental parser's call fragments into whole calls. - `parse_streaming_increment` reports one call in several pieces: the name - first with empty parameters, then the arguments JSON a fragment at a time, - then the closing brace. Only the concatenation is something a client can - run, so the pieces are gathered here and the finished calls are emitted - when generation ends. - - Keyed by the parser's own `tool_index`, which numbers the calls in the - order it saw them start; a dict preserves that order. - - The call id is minted when the parser first announces the call - before - anyone can know whether its arguments will assemble into something - deliverable - so the identity belongs to the call entity itself. A call - that is later dropped dies holding its own id instead of donating it to - the next call in line, which is the positional swap this prevents. + A call arrives as its name with empty parameters and then argument pieces; + fragments are keyed by the parser's tool index, in the order calls start. """ for call in calls: - fragment = fragments.setdefault(call.tool_index, { - "name": None, - "parameters": "", - "call_id": f"call_{_random_uuid()}", - }) - # Only a call's first fragment carries the name; every later one - # carries None, which must not erase the name already recorded. + fragment = fragments.get(call.tool_index) + if fragment is None: + fragment = fragments[call.tool_index] = { + "name": None, + "parameters": [], + } + # Only the first fragment carries the name. if call.name: fragment["name"] = call.name - fragment["parameters"] += call.parameters or "" + if call.parameters: + fragment["parameters"].append(call.parameters) + + +def _reject_json_constant(name: str) -> None: + raise ValueError(f"{name} is not valid JSON") def _assembled_tool_calls( - fragments: dict[int, dict[str, Any]], - unfinished_tool_index: Optional[int] = None) -> list[ToolCallItem]: - r"""The accumulated fragments as whole calls, in the order they started. - - A call whose name never arrived is dropped, which is what - `parse_base_json` does on the non-streaming path: a call the client cannot - name is not a call it can run. - - `unfinished_tool_index` names a call the parser announced but never closed, - because the stream ended inside it. Its accumulated arguments are a JSON - prefix - `{"input": "\n// Read the problem` and nothing more - which no - client can parse or run, so it is dropped rather than reported. The - non-streaming path reports no call at all for the same text, since its - regex needs the closing tag, and the two endpoints have to agree about - what the model asked for. Only the *call* is dropped: the raw markup it - was read from reaches the client as message text on both views - (`_flush_tool_parser` releases it, the whole-text re-parse falls back to - it), so nothing the model generated is lost. - - A call whose assembled arguments are not valid JSON is dropped for the same - reason, one test later. That test keys on the markup being unterminated; - this one keys on the result being unusable, which is the property that - actually matters and which covers calls the markup test passes. A GLM-4.7 - block that opens `` and never closes it, ending - `` instead, has a balanced `` pair and so - finalises normally - with arguments that assemble to `{"cmd": }`. That - reached a client, which stored it, replayed it in the next request's - history and was answered `tool_calls[0].function.arguments must be valid - JSON`; 2 of 13,014 delivered calls, each costing the agent run that hit it. - - Here rather than in the parser because a parser cannot unsay what it has - already said. It streams `{`, `"cmd": `, the value and `}` over several - increments, each folded into `fragments` as it arrives, and only discovers - at `` that the whole does not parse. This is the last point at - which the fragments are known as a whole *and* declining to report them is - still possible. - - Dropping beats repairing. Closing the quote and the brace would invent an - argument the model never wrote, and the client would then run a truncated - shell command with nothing to indicate anything was lost; executing half a - command is worse than executing none. The warning is what keeps the loss - from being silent. - - The non-streaming path cannot reuse this validity test - `parse_base_json` - builds its arguments with `json.dumps`, so they always parse - and its - pair regex reads nothing out of such a block, reporting the call with - empty arguments: a call the model never made, delivered as runnable. - That is settled now, one layer up: when no stream ran, - `_verify_empty_calls_against_streaming` replays the text through this - streaming machinery, and a `{}`-argument call this assembly would have - refused is not reported there either - its markup falls back into the - message text instead. What still differs is a call whose markup yielded - *some* readable pairs: the whole-text parse delivers what it read, the - stream drops the call whose tail it could not assemble. That is a real - divergence with no obviously right answer and it stays open, documented - at the reconciler. + fragments: dict[int, dict[str, Any]]) -> list[ToolCallItem]: + """The accumulated fragments as whole calls, in the order they started. + + Skips a call with no name or whose arguments do not assemble into valid + JSON: a client can run neither. """ calls: list[ToolCallItem] = [] for tool_index, fragment in fragments.items(): - if not fragment["name"] or tool_index == unfinished_tool_index: - # No warning for the unfinished call: `_flush_tool_parser` has - # already warned and released its raw markup as message text, and - # a second warning would read as a second loss. + if not fragment["name"]: continue - arguments = fragment["parameters"] + arguments = "".join(fragment["parameters"]) try: - json.loads(arguments) + json.loads(arguments, parse_constant=_reject_json_constant) except ValueError as exc: - # The drop consumes the call's own id (minted when the parser - # announced it) and the log says which, so the retired identity - # is auditable and can never be mistaken for the next call's. - consumed = fragment.get("call_id") - consumed_note = (f" The drop consumes its call id {consumed!r}, " - "which is retired with it." if consumed else "") logger.warning( f"Dropping the tool call to {fragment['name']!r}: its " - f"arguments did not assemble into valid JSON ({exc}). The " - "model emitted a malformed tool call, and reporting it would " - "hand the client arguments it can only reject. Assembled: " - f"{arguments[:200]!r}.{consumed_note}") + f"arguments are not valid JSON ({exc}): {arguments[:200]!r}") continue calls.append( ToolCallItem(tool_index=tool_index, @@ -3076,108 +2445,6 @@ def _assembled_tool_calls( return calls -def _verify_empty_calls_against_streaming( - tool_parser_id: Optional[str], - tools: Optional[list[ChatCompletionToolsParam]], - source_text: str, - normal_text: str, - calls: list[ToolCallItem], -) -> Tuple[str, list[ToolCallItem]]: - r"""Drop whole-text `{}`-argument calls whose markup could not be read. - - Only for a request no stream ran for. A streamed request already settles - this per call entity in `_create_output_content` - the stream's id record - carries a None where assembly dropped a call, and the rebuild consumes it. - - The whole-text parse reports `{}` for two markups it cannot tell apart: a - genuine zero-argument call (`get_time`) and a call - whose argument markup its pair regex read nothing out of - GLM-4.7 - opening an `` it never closes, ending `` - instead. The streaming machinery *can* tell them apart, because it - assembles exactly the JSON it read and `_assembled_tool_calls` refuses - what does not parse: the zero-argument call streams as a valid `{}`, the - unreadable one as `{"cmd": }` and is dropped. So when the whole-text - parse reports an empty-argument call, the same text is replayed through a - fresh streaming parser and the two views are paired positionally - the - same premise the id reuse in `_create_output_content` rests on, both - passes enumerating the same markup in the same order. An empty call the - streamed assembly would have refused is a call the model never made, and - it is not delivered here either. - - The markup is not discarded with it: when the refused call was everything - the parse found, the original text goes back verbatim - these turns are - recorded as training data, and the raw markup as message text is the same - fallback both views already use for an unterminated call. Alongside calls - that really were made, the surrounding text keeps its parsed form and only - the refused call is dropped, which is what the streamed view of the same - text delivers. - - Any disagreement about what the call entities *are* - a replay error, or - the two passes finding different numbers of calls - leaves the whole-text - result untouched: this is a cross-check, not the parse of record, and - guessing at a pairing would trade a known defect for an unknowable one. - """ - empty_positions = { - index - for index, call in enumerate(calls) - if _parses_to_empty_object(call.parameters) - } - if not empty_positions or tool_parser_id is None or tools is None: - return normal_text, calls - - fragments: dict[int, dict[str, Any]] = {} - try: - replay_parser = ToolParserFactory.create_tool_parser(tool_parser_id) - increment = replay_parser.parse_streaming_increment(source_text, tools) - _accumulate_tool_call_fragments(fragments, increment.calls) - _, flushed, unfinished = _flush_tool_parser( - tools=tools, - output_index=0, - tool_parser_dict={0: replay_parser}, - ) - _accumulate_tool_call_fragments(fragments, flushed) - assembled = _assembled_tool_calls(fragments, unfinished) - except Exception as exc: # noqa: BLE001 - cross-check must not fail the parse - logger.warning( - f"Could not replay the generation through the streaming tool " - f"parser to verify its empty-argument calls ({exc}); delivering " - "the whole-text parse as is.") - return normal_text, calls - - if len(fragments) != len(calls): - return normal_text, calls - - delivered = {call.tool_index for call in assembled} - entity_order = list(fragments) - phantoms = { - index - for index in empty_positions if entity_order[index] not in delivered - } - if not phantoms: - return normal_text, calls - - for index in sorted(phantoms): - logger.warning( - f"Dropping the tool call to {calls[index].name!r} from the " - "whole-text parse: it reports no arguments only because its " - "argument markup could not be read, and the streamed assembly of " - "the same text refuses the call. The model emitted a malformed " - "tool call, and an invented '{}' would run the tool with " - "arguments the model never wrote.") - if len(phantoms) == len(calls): - return source_text, [] - return normal_text, [ - call for index, call in enumerate(calls) if index not in phantoms - ] - - -def _parses_to_empty_object(parameters: Optional[str]) -> bool: - try: - return json.loads(parameters or "") == {} - except ValueError: - return False - - def _close_open_item(helper): """Close whichever output item is currently open, if any. @@ -3199,14 +2466,6 @@ def _close_open_item(helper): status="completed", ) yield helper.get_reasoning_text_done_event(text) - # The reasoning part was opened with content_part.added (see - # _get_output_added_events), so it has to close with - # content_part.done - before the item's own done, which is OpenAI's - # order. The message branch below has always done this; reasoning - # omitted it, leaving every reasoning turn with one more - # content_part.added than content_part.done (~294k unpaired events a - # week) and clients tracking part state holding a part that never - # ends. yield helper.get_content_part_done_event( PartReasoningTextDone(type="reasoning_text", text=text)) yield helper.get_output_item_done_event(item) @@ -3241,10 +2500,6 @@ def _generate_streaming_event( tool_parser_dict: Optional[dict[int, BaseToolParser]] = None, ): available_tools = _get_chat_completion_function_tools(request.tools) - # tool_choice="none" is honoured by not parsing at all, so the model's - # markup stays in the visible text verbatim. The final rebuild resolves - # its parser the same way (see _create_response); the two views of one - # generation must agree on whether a call exists. tool_parser_id = _effective_tool_parser(tool_parser_id, request) output_idx = output.index delta_text = output.text_diff @@ -3280,267 +2535,92 @@ def check_parser(parser_id: Optional[str], streaming=True, tool_parser_dict=tool_parser_dict, ) - - # End of stream: the tool parser gets one last word before its state is - # discarded, so bytes it was still withholding are released by a decision - # rather than lost because nobody ever asked. Only computed here; what it - # releases is emitted further down, after the call in front of it has - # closed the item it belongs to. - flushed_text, flushed_calls, unfinished_tool_index = "", [], None - if finished_generation: - flushed_text, flushed_calls, unfinished_tool_index = _flush_tool_parser( - tools=available_tools, - output_index=output_idx, - tool_parser_dict=tool_parser_dict, - ) + tool_parser = (tool_parser_dict or {}).get(output_idx) + # Calls are assembled from the parser's increments only when those match its + # whole-text parse; otherwise they come from a whole-text parse at the end. + incremental_calls = (tool_parser is not None + and tool_parser.streaming_matches_whole_parse) _responses_debug_log( repr( f" ---------> delta text: {delta_text}, reasoning delta text: {reasoning_delta_text}, calls: {calls}" )) - # Send delta events for ongoing content BEFORE any done events. - # - # The close-on-calls block below ends the item that is currently open. If - # it ran first, this chunk's delta would arrive after that close and open a - # brand new output item for the tail of the same message - splitting one - # assistant turn across two items, sometimes mid-word, and clients that - # render the last item alone show only that fragment. The chat completions - # path has the same shape: it appends the content delta to the chunk and - # only then stamps finish_reason. - # - # It is also what makes the done payload right in the common case: a chunk - # that carries the text before a call *and* the start of that call arrives - # as one delta plus one non-empty `calls`, so the text has to be appended - # to the item before the item is closed with it. - # - # The item must be opened before *any* delta, including a whitespace-only - # one. Gating the added-events on delta_text.strip() while emitting the - # delta unconditionally sends output_text.delta with no item open, and a - # client that keys on the active item drops the whole turn: Codex CLI - # reports "OutputTextDelta without active item" and prints nothing. Short - # replies are the ones that hit it, because a leading whitespace token is - # more likely to be the first delta of the message. - # - # get_*_output_added_events is idempotent - it is guarded internally by - # sent_output_item_added - so calling it for every delta is safe. - if delta_text: - # One chunk can carry both halves. The chunk whose raw text spans the - # closing think tag is parsed into a reasoning part (everything before - # the tag) and a content part (everything after), and this branch is - # chosen on the content part alone -- so the reasoning part has to be - # flushed here or it is never emitted at all. The `elif` below cannot - # run for this chunk, and nothing else reads reasoning_delta_text. - # - # It is not a rare boundary case. The reasoning between the last chunk - # boundary and the tag is lost every time the two land in one chunk, - # which measured 56% of reasoning items across four fleets, and 100% - # of reasoning short enough to fit inside a single chunk. Nothing - # downstream can recover it either: every done payload is now the sum - # of the deltas already streamed, so reasoning that was never streamed - # as a delta is simply not in the reasoning item at all. - if reasoning_delta_text: - if streaming_events_helper.is_text_sent: - yield from _close_open_item(streaming_events_helper) - if not streaming_events_helper.is_reasoning_sent: - streaming_events_helper.is_reasoning_sent = True - yield from streaming_events_helper.get_reasoning_output_added_events( - ) - streaming_events_helper.append_reasoning(reasoning_delta_text) - yield streaming_events_helper.get_reasoning_text_delta_event( - reasoning_delta_text) - - # Reasoning has ended and the answer is starting: close the reasoning - # item so the message deltas are not attributed to it. - if streaming_events_helper.is_reasoning_sent: - yield from _close_open_item(streaming_events_helper) - if not streaming_events_helper.is_text_sent: - streaming_events_helper.is_text_sent = True - yield from streaming_events_helper.get_message_output_added_events() - streaming_events_helper.append_text(delta_text) - yield streaming_events_helper.get_text_delta_event(delta_text, []) - elif reasoning_delta_text: + # Deltas go out before any done event, so each lands in the item it belongs + # to, and an item is opened before its first delta (whitespace included). + if reasoning_delta_text: if streaming_events_helper.is_text_sent: yield from _close_open_item(streaming_events_helper) - if not streaming_events_helper.is_reasoning_sent: - streaming_events_helper.is_reasoning_sent = True + streaming_events_helper.is_reasoning_sent = True yield from streaming_events_helper.get_reasoning_output_added_events() streaming_events_helper.append_reasoning(reasoning_delta_text) yield streaming_events_helper.get_reasoning_text_delta_event( reasoning_delta_text) + if delta_text: + if streaming_events_helper.is_reasoning_sent: + yield from _close_open_item(streaming_events_helper) + streaming_events_helper.is_text_sent = True + yield from streaming_events_helper.get_message_output_added_events() + streaming_events_helper.append_text(delta_text) + yield streaming_events_helper.get_text_delta_event(delta_text, []) - # A tool call has started, so whichever item is open ends here. - # - # Only the incremental parser can report this. It is the one holding back - # the markup's bytes, so it knows it is mid-call; a re-parse of the - # accumulated text has no such memory and has to classify an unterminated - # call as ordinary text. That is exactly how the previous implementation - # published `functions.execinput...` as the - # assistant's message: the non-streaming tool regex needs a closing tag, - # so a call that had closed kept `tool_calls` non-empty while a second, - # still-open call stayed in the text, and both halves of its condition - # were true at once. It needed one closed and one open call to coexist, - # which is why it hit 41.7% of two-call responses and every three-call one - # while single-call responses were almost untouched. - # - # _close_open_item closes with take_text()/take_reasoning() - the deltas - # already streamed - so a done payload cannot contain anything that was - # never streamed as a delta, whatever the accumulated text looks like. - # Which item that is depends on what was open: prose before the call - # closes a message item, `` closes the reasoning item, - # and a call with nothing in front of it closes nothing and emits no - # events, rather than inventing an empty message item. + # A call has started, so the open item ends here; only the incremental + # parser knows this while the call is still being generated. if calls: yield from _close_open_item(streaming_events_helper) - # Whole calls have to be reassembled from the fragments the parser reports - # (name, then arguments piece by piece); see _accumulate_tool_call_fragments. - call_fragments = streaming_events_helper.tool_call_fragments(output_idx) - _accumulate_tool_call_fragments(call_fragments, calls) - _accumulate_tool_call_fragments(call_fragments, flushed_calls) - - # Text the parser was still holding when the stream ended, emitted here - # rather than folded into the delta block above because it sits *after* - # whatever the last chunk contained. The common shape is a call and the - # sentence following it arriving in one chunk: the parser returns the call - # and keeps the sentence, so folding it in would put that sentence in the - # message item the call just closed - one item reading "Before. After." - # with the call between them lost from the ordering. - if flushed_text: + if incremental_calls: + call_fragments = streaming_events_helper.tool_call_fragments(output_idx) + _accumulate_tool_call_fragments(call_fragments, calls) + + if not finished_generation: + return + + final_calls: list[ToolCallItem] = [] + released_text = "" + if incremental_calls: + # finish() reports the calls the parser still holds and releases the + # rest as text, as the whole-text parse reads it. + flushed = tool_parser.finish(available_tools) + released_text = flushed.normal_text + _accumulate_tool_call_fragments(call_fragments, flushed.calls) + final_calls = _assembled_tool_calls(call_fragments) + elif tool_parser is not None: + released_text, final_calls = _whole_text_tool_calls( + output, request, streaming_events_helper, reasoning_parser_id, + tool_parser_id, available_tools) + + # Text the parser still held follows everything the last chunk carried. + if released_text: if streaming_events_helper.is_reasoning_sent: yield from _close_open_item(streaming_events_helper) streaming_events_helper.is_text_sent = True yield from streaming_events_helper.get_message_output_added_events() - streaming_events_helper.append_text(flushed_text) - yield streaming_events_helper.get_text_delta_event(flushed_text, []) - - # Close whatever item is still open once generation has finished. - # - # Every other close above is triggered by a transition - reasoning into - # text, text into a tool call - and the last item of a response has no - # transition after it, so nothing else ever closes it. It would reach the - # client with no output_text.done, content_part.done or output_item.done, - # i.e. with no terminal state: Codex CLI renders it but echoes it back on - # the next turn without a `status`, and ResponseOutputMessageParam - # requires one, so the following request is rejected outright. - # - # The chat completions path has no equivalent problem because it finalises - # on `output.finish_reason is not None` rather than on parser state. This - # mirrors that: when generation is finished, any open item is closed. - if finished_generation and streaming_events_helper.is_output_item_added_sent: - if streaming_events_helper.is_reasoning_sent: - reasoning_text = streaming_events_helper.take_reasoning() - reasoning_item = ResponseReasoningItem( - id=streaming_events_helper.item_id, - summary=[], - type="reasoning", - content=[Content(text=reasoning_text, type="reasoning_text")], - status="completed", - ) - yield streaming_events_helper.get_reasoning_text_done_event( - reasoning_text) - # Close the reasoning part the opener added, before the item's - # done - the same pairing _close_open_item restores and the - # message branch below has always had. - yield streaming_events_helper.get_content_part_done_event( - PartReasoningTextDone(type="reasoning_text", - text=reasoning_text)) + streaming_events_helper.append_text(released_text) + yield streaming_events_helper.get_text_delta_event(released_text, []) + + # Nothing transitions after the last item, so it is closed here. + yield from _close_open_item(streaming_events_helper) + + # Calls go out once generation finishes, after every other item. Already + # emitted calls are skipped, so a finished output seen twice is harmless. + emitted = streaming_events_helper.emitted_tool_call_items + pending = final_calls[len(emitted):] + if pending: + tool_resolution = _tool_resolution(request.tools) + for call in pending: + tool_call_item = _tool_call_output_item(call, + tool_resolution, + status="completed") + streaming_events_helper.item_id = tool_call_item.id + yield streaming_events_helper.get_output_item_added_event( + tool_call_item) yield streaming_events_helper.get_output_item_done_event( - reasoning_item) - streaming_events_helper.is_reasoning_sent = False - else: - text = streaming_events_helper.take_text() - text_content = ResponseOutputText( - text=text, - annotations=[], - type="output_text", - logprobs=None, - ) - text_item = ResponseOutputMessage( - id=streaming_events_helper.item_id, - content=[text_content], - role="assistant", - status="completed", - type="message", - ) - yield streaming_events_helper.get_text_done_event(text, []) - yield streaming_events_helper.get_content_part_done_event( - text_content) - yield streaming_events_helper.get_output_item_done_event(text_item) - streaming_events_helper.is_text_sent = False - streaming_events_helper.output_index_increment() + tool_call_item) + emitted.append(tool_call_item) + streaming_events_helper.output_index_increment() streaming_events_helper.is_output_item_added_sent = False - # Emit the tool calls the parser found, as function_call output items. - # - # Without this the call is stripped out of the text by the tool parser and - # then dropped, so the client receives prose - or, when the whole - # generation was a tool call, an empty message - and no indication that a - # tool should run. Codex CLI shows the model announcing an action and then - # nothing happening at all. - # - # TODO(JunyiXu-nv): stream the call items as the parser produces them. - # They are held back until generation finishes for now. Streaming them is - # the direction both SGLang and vLLM have gone, but it is an observable - # protocol change - a client that assumes every call item arrives together - # at the end would see them spread out instead - and landing it here would - # mean a reviewer could not tell the leak fix from the protocol change, - # nor roll back one without the other. The close above is what the leak - # fix needs, and it already happens incrementally. - # - # Emitted after any open item has been closed, so a call item is never - # nested inside a message item. The counter keeps emission idempotent: it - # guarded against the old per-chunk re-parse re-reporting the same calls, - # and now guards against a finished output being presented twice. It - # counts the parser's announced call entities, kept or dropped, because - # a drop has to be accounted exactly once too. - if finished_generation: - entities = list(call_fragments.items()) - pending = entities[streaming_events_helper.emitted_tool_calls:] - if pending: - keep = { - call.tool_index: call - for call in _assembled_tool_calls(call_fragments, - unfinished_tool_index) - } - tool_resolution = _tool_resolution(request.tools) - # Walked in entity order, not kept-list order: a call dropped for - # invalid-JSON arguments consumes its own slot in the emission - # record, so the final rebuild pairs each surviving call with the - # identity the stream actually gave it rather than the one that - # belonged to the call in front of it. Nameless and unfinished - # entities take no slot - the rebuild's whole-text re-parse does - # not report those as calls either, so a placeholder for them - # would misalign the record. - for tool_index, fragment in pending: - call = keep.get(tool_index) - if call is None: - if (fragment.get("name") - and tool_index != unfinished_tool_index): - streaming_events_helper.record_dropped_tool_call() - continue - tool_call_item = _tool_call_output_item( - call, - tool_resolution, - status="completed", - # The id minted when the parser announced this call; see - # _accumulate_tool_call_fragments. - call_id=fragment.get("call_id")) - streaming_events_helper.item_id = tool_call_item.id - yield streaming_events_helper.get_output_item_added_event( - tool_call_item) - yield streaming_events_helper.get_output_item_done_event( - tool_call_item) - # `call` rides along so the record holds the name and - # assembled arguments these events were built from; the final - # snapshot repeats them rather than trusting its own re-parse - # of the same markup (see _create_output_content). - streaming_events_helper.record_emitted_tool_call( - tool_call_item, call) - streaming_events_helper.output_index_increment() - streaming_events_helper.emitted_tool_calls = len(entities) - streaming_events_helper.is_output_item_added_sent = False - def _generate_streaming_event_harmony( harmony_adapter: HarmonyAdapter, @@ -3660,41 +2740,17 @@ def _generate_streaming_event_harmony( def _stream_terminal_event( final_response: ResponsesResponse, - finish_reason: Optional[str], sequence_number: int = -1, ) -> Union[ResponseCompletedEvent, ResponseIncompleteEvent, ResponseFailedEvent]: - """The terminal event for a stream that ran to completion, by status. - - ``finish_reason_mapping`` marks a generation the engine cut off at its - token budget as status "incomplete", and the terminal event used to say - ``response.completed`` around it - an event whose name asserts the one - thing its payload denies, so a client believed the truncated answer was - the whole one. The Responses spec pairs the status with its own terminal - event, ``response.incomplete``, and explains it in - ``incomplete_details.reason``; "length" is the token budget, spelled - ``max_output_tokens``, and any other cause is left unstated rather than - guessed at. - - ``incomplete_details`` now lives on ResponsesResponse and the response - builder (see ``_create_response``) sets it for a token-budget cut, so the - dump usually carries it already; setting it here as well keeps this event - right for any snapshot that did not pass through that builder. The - dump-level route is the same one ``get_stream_failed_events`` takes for - ``error``, whose field is still commented out. - - A status of "failed" is a generation the engine gave up on (a timeout), - and it ships under ``response.failed`` - the spec's terminal event for - that status - not under a ``response.completed`` whose name promises the - opposite. "cancelled" stays under ``response.completed`` on purpose: the - cancellation was client-initiated, the Responses spec pairs no terminal - event with it, and inventing one would hand SDK clients an event type - they cannot parse. + """The terminal event matching the response's status. + + "incomplete" ends in ``response.incomplete`` and "failed" in + ``response.failed``; everything else, including the client-initiated + "cancelled", ends in ``response.completed``. """ - payload = final_response.model_dump() + payload = final_response.model_dump(by_alias=True) if final_response.status == "incomplete": - if finish_reason == "length": - payload["incomplete_details"] = {"reason": "max_output_tokens"} return ResponseIncompleteEvent( type="response.incomplete", sequence_number=sequence_number, @@ -3730,7 +2786,10 @@ def __init__( self.model_name = model_name self.request = request self.sampling_params = sampling_params - self.sequence_number = 0 + # get_initial_responses numbers the opening pair 0 and 1, so every + # later event continues from 2 -- also in a postprocessing worker, + # which receives a copy of this object before the pair is sent. + self.sequence_number = 2 self.streaming_events_helper = ResponsesStreamingEventsHelper() self.response_creation_time = create_time if create_time is not None else int( time.time()) @@ -3745,19 +2804,10 @@ def __init__( self.tool_parser = tool_parser def _send_event(self, event: OpenAIBaseModel): - # Set sequence_number if the event has this attribute. The number is - # provisional: with postprocessing workers a pickled copy of this - # processor builds the per-token frames in another process, counting - # from zero again, so the frontend restamps every frame at the egress - # (stamp_sse_sequence_number) and what is written here only has to be - # ordered within this process. if hasattr(event, 'sequence_number'): event.sequence_number = self.sequence_number self.sequence_number += 1 - # Get event type from the event's type field if it exists - event_type = getattr(event, 'type', 'unknown') - return (f"event: {event_type}\n" - f"data: {event.model_dump_json(indent=None)}\n\n") + return _format_sse_event(event) def get_initial_responses(self) -> List[str]: initial_response = ResponsesResponse.from_request( @@ -3768,21 +2818,19 @@ def get_initial_responses(self) -> List[str]: output=[], status="in_progress", usage=None, - ).model_dump() - - resp_created = self._send_event( - self.streaming_events_helper.get_response_created_event( - initial_response)) - resp_in_progress = self._send_event( - self.streaming_events_helper.get_response_in_progress_event( - initial_response)) - return [resp_created, resp_in_progress] + ).model_dump(by_alias=True) + created = self.streaming_events_helper.get_response_created_event( + initial_response) + in_progress = self.streaming_events_helper.get_response_in_progress_event( + initial_response) + created.sequence_number = 0 + in_progress.sequence_number = 1 + return [_format_sse_event(created), _format_sse_event(in_progress)] async def get_final_response( self, final_res: RequestOutput, num_prompt_tokens: Optional[int] = None, - tokenizer: Optional[TokenizerBase] = None, ) -> str: final_response = await create_response( generator=None, @@ -3797,23 +2845,14 @@ async def get_final_response( reasoning_parser=self.reasoning_parser, tool_parser=self.tool_parser, num_prompt_tokens=num_prompt_tokens, - # Name the calls the way the stream already named them. - streamed_tool_call_ids=self.streaming_events_helper. - emitted_tool_call_ids, - # And the reasoning/message items likewise: the snapshot has to - # answer to the ids the stream already announced. - streamed_item_ids=self.streaming_events_helper.emitted_item_ids, - # Taken as an argument rather than held on this object: the - # postproc-worker path pickles the processor across a process - # boundary, and that worker already has its own tokenizer - # (postproc_worker.py:195). Shipping one per request would be - # pure cost. - tokenizer=tokenizer, ) return self._send_event( - _stream_terminal_event(final_response, - final_res.outputs[0].finish_reason)) + ResponseCompletedEvent( + type="response.completed", + sequence_number=-1, + response=final_response.model_dump(), + )) def get_final_response_non_store( self, @@ -3821,6 +2860,11 @@ def get_final_response_non_store( num_prompt_tokens: Optional[int] = None, tokenizer: Optional[TokenizerBase] = None, ) -> str: + """The terminal event; its snapshot repeats the items already streamed. + + ``tokenizer`` (for counting reasoning tokens) is an argument because a + postprocessing worker holds a pickled copy of this object. + """ final_response = create_response_non_store( generation_result=final_res, request=self.request, @@ -3831,23 +2875,12 @@ def get_final_response_non_store( reasoning_parser=self.reasoning_parser, tool_parser=self.tool_parser, num_prompt_tokens=num_prompt_tokens, - # Name the calls the way the stream already named them. - streamed_tool_call_ids=self.streaming_events_helper. - emitted_tool_call_ids, - # And the reasoning/message items likewise: the snapshot has to - # answer to the ids the stream already announced. + streamed_tool_calls=self.streaming_events_helper. + emitted_tool_call_items, streamed_item_ids=self.streaming_events_helper.emitted_item_ids, - # Taken as an argument rather than held on this object: the - # postproc-worker path pickles the processor across a process - # boundary, and that worker already has its own tokenizer - # (postproc_worker.py:195). Shipping one per request would be - # pure cost. tokenizer=tokenizer, ) - - return self._send_event( - _stream_terminal_event(final_response, - final_res.outputs[0].finish_reason)) + return self._send_event(_stream_terminal_event(final_response)) def process_single_output(self, res: GenerationResult) -> list[str]: event_generator = None @@ -3882,23 +2915,12 @@ def get_stream_failed_events(self, cause: str, detail: str, events_sent: int = 0) -> List[str]: - """The terminal events for a stream that stopped before completing. - - Carries no generated content. The accumulated text lives only in the - deltas already on the wire and re-emitting it here would guess at - whether those deltas reached anyone; this says what happened, nothing - more. - - ``events_sent`` is the number of frames that actually reached the - wire. Normally it equals this processor's own counter, because this - processor numbered every one of them. With postprocessing workers it - does not: the per-token events are built by a pickled copy of this - object in another process and only the opening two are numbered here, - so the local counter is short by the whole body of the turn. Taking - the larger of the two keeps the terminal events after everything the - client has already seen either way. + """``error`` and ``response.failed`` for a stream that stopped early. + + ``events_sent`` is the number of frames that reached the wire; the + terminal events are numbered after them. """ - self.sequence_number = max(self.sequence_number, events_sent) + self.sequence_number = events_sent error_event = self._send_event( ResponseErrorEvent( type="error", @@ -3915,12 +2937,9 @@ def get_stream_failed_events(self, output=[], status="failed", usage=None, - ).model_dump() - # Set on the dump rather than on ResponsesResponse, whose `error` field - # is commented out. The event re-validates this dict against the SDK's - # Response, which does carry `error`, so the field reaches the wire - # without widening the local model - and without adding a key to the - # happy path's response.completed payload. + ).model_dump(by_alias=True) + # Set on the dump: the SDK Response carries `error`, ResponsesResponse + # does not. snapshot["error"] = {"code": "server_error", "message": detail} failed_event = self._send_event( ResponseFailedEvent( @@ -3935,209 +2954,107 @@ def get_stream_failed_events(self, # Abnormal stream termination # -------------------------------------------------------------------------- -# Why a stream stopped before `response.completed`. Recorded in the terminal -# `error` event's `code` and passed to the guard's `on_termination` callback. -# Telling a server-side fault apart from a client hangup is what decides -# whether the accumulated text still existed in this process when the stream -# died. -STREAM_TERMINATION_CLIENT_DISCONNECT = "client_disconnect" +# Why a stream stopped before its terminal event, sent as the `error` code. STREAM_TERMINATION_ENGINE_ERROR = "engine_error" STREAM_TERMINATION_UPSTREAM_ERROR = "upstream_error" STREAM_TERMINATION_INTERNAL_ERROR = "internal_error" -# The terminal events of a stream that ran to completion - a truncated -# generation ends in `response.incomplete`, a failed one in `response.failed`, -# everything else in `response.completed` (see _stream_terminal_event; the -# guard's own abnormal-termination path emits `response.failed` too). Matched -# as substrings because a relayed stream arrives as transport-sized chunks -# that may carry several events, not as one frame per event. Kept in both -# encodings so the scan never has to decode a relayed chunk just to look at -# it. -_COMPLETED_EVENT_MARKER = "event: response.completed" -_INCOMPLETE_EVENT_MARKER = "event: response.incomplete" -_FAILED_EVENT_MARKER = "event: response.failed" -_TERMINAL_EVENT_MARKERS = (_COMPLETED_EVENT_MARKER, _INCOMPLETE_EVENT_MARKER, - _FAILED_EVENT_MARKER) -_TERMINAL_EVENT_MARKERS_BYTES = tuple( - marker.encode("utf-8") for marker in _TERMINAL_EVENT_MARKERS) - _SSE_EVENT_DELIMITER = "\n\n" -_SSE_EVENT_DELIMITER_BYTES = _SSE_EVENT_DELIMITER.encode("utf-8") +# A terminal event frame always follows another frame, and JSON payloads +# escape newlines, so these match only at a frame boundary. +_TERMINAL_FRAME_STARTS = tuple(f"{_SSE_EVENT_DELIMITER}event: response.{kind}" + for kind in ("completed", "incomplete", + "failed")) +_TERMINAL_FRAME_STARTS_BYTES = tuple(marker.encode() + for marker in _TERMINAL_FRAME_STARTS) +_SSE_TAIL_LENGTH = max(len(marker) for marker in _TERMINAL_FRAME_STARTS) -def classify_stream_termination(exc: BaseException) -> str: - """Attribute a stream-ending exception to a cause. +def _sse_delimiter_and_markers(sample: Any) -> Tuple[Any, tuple]: + if isinstance(sample, bytes): + return _SSE_EVENT_DELIMITER.encode(), _TERMINAL_FRAME_STARTS_BYTES + return _SSE_EVENT_DELIMITER, _TERMINAL_FRAME_STARTS - The distinction that matters is server-side fault vs client hangup. A - cancelled or closed generator means the consumer went away: on the - aggregated server that is the HTTP client, on the orchestrator it is the - client of the orchestrator. Everything else happened on this side of the - socket, and is split further only as far as the exception type can be - trusted to say - an engine error means generation itself failed, an - upstream transport error means the response this process was relaying was - cut, and anything else is a fault in this process's own code, where the - accumulated text was still in memory when the stream died. - """ - if isinstance(exc, (asyncio.CancelledError, GeneratorExit)): - return STREAM_TERMINATION_CLIENT_DISCONNECT + +def classify_stream_termination(exc: Exception) -> str: + """Attribute a stream-ending exception to a cause.""" if isinstance(exc, (RequestError, EngineDeadError)): return STREAM_TERMINATION_ENGINE_ERROR - # By module rather than by class: the HTTP client library is an - # implementation detail of the relay and importing it here would pull a - # transport dependency into the protocol layer. + # By module, to keep the HTTP client library out of this layer. if type(exc).__module__.split(".")[0] in ("aiohttp", "httpx", "httpcore"): return STREAM_TERMINATION_UPSTREAM_ERROR return STREAM_TERMINATION_INTERNAL_ERROR -def describe_stream_termination(exc: BaseException) -> str: - text = str(exc) - return f"{type(exc).__name__}: {text}" if text else type(exc).__name__ +def describe_stream_termination(exc: Exception, cause: str) -> str: + """The client-facing detail of a stream-ending exception. + + Only engine errors carry their message; transport and internal errors can + name internal hosts. + """ + if cause == STREAM_TERMINATION_ENGINE_ERROR and str(exc): + return f"{type(exc).__name__}: {exc}" + return type(exc).__name__ def stream_error_event(cause: str, detail: str, events_sent: int) -> List[bytes]: - """A bare ``error`` event for a relay that cannot build a snapshot. - - ``response.failed`` carries a whole ``Response``, which the disaggregated - orchestrator cannot assemble - it forwards bytes and never holds the - sampling parameters the snapshot is built from. ``error`` is flat, so it - needs only the number of events already forwarded to keep the sequence - contiguous. Bytes, because the relay it terminates is a byte stream. - """ - event = ResponseErrorEvent( - type="error", - sequence_number=events_sent, - code=cause, - message=detail, - param=None, - ) - return [(f"event: error\n" - f"data: {event.model_dump_json(indent=None)}\n\n").encode("utf-8")] - - -def stamp_sse_sequence_number(frame: str, sequence_number: int) -> str: - """Overwrite the ``sequence_number`` inside one serialized SSE frame. - - Numbering cannot be left to the producers, because one response can have - two of them. With postprocessing workers enabled the opening - ``response.created``/``response.in_progress`` pair is built by the - frontend's streaming processor while every later frame comes from a - pickled copy of that processor in a worker process, each counting from - zero - so a single response's stream carried 0,1,0,1,2,... (221 of 221 - measured responses). Only the egress point, where the frames converge - just before leaving the server, can hand out one monotonic sequence; it - restamps every frame with this and the producers' numbers are treated as - provisional. - - The frame is parsed rather than patched with a substring replace: the - ``response.completed`` payload embeds the model's own output, which can - contain anything - including the literal ``"sequence_number":`` when the - model writes code against this very API. A frame with no data line, an - unparsable payload, or no sequence_number field is returned untouched; - no producer builds such a frame today, and inventing a field on one would - be worse than leaving its numbering alone. - """ - head, sep, rest = frame.partition("data: ") - if not sep: - return frame - # The payload is one line: _send_event serializes with indent=None, and - # JSON strings carry newlines escaped. - payload_text, newline, tail = rest.partition("\n") - try: - payload = json.loads(payload_text) - except ValueError: - return frame - if not isinstance(payload, dict) or "sequence_number" not in payload: - return frame - payload["sequence_number"] = sequence_number - # Compact separators and raw unicode, matching model_dump_json, so the - # restamped frame differs from the produced one only in the number. - restamped = json.dumps(payload, separators=(",", ":"), ensure_ascii=False) - return f"{head}data: {restamped}{newline}{tail}" - - -def _count_frames(chunk: Any, carry: Any) -> Tuple[int, Any]: - """Count completed SSE events in ``chunk``, tolerating split delimiters. - - The carry is the chunk's last byte/character: a transport read can land - between the two newlines that end an event, and without it that event is - never counted and every sequence number after it is one short. - """ - if isinstance(chunk, bytes): - text = (carry or b"") + chunk - return text.count(_SSE_EVENT_DELIMITER_BYTES), text[-1:] - text = (carry or "") + chunk - return text.count(_SSE_EVENT_DELIMITER), text[-1:] + """A bare ``error`` event, for a relay that holds no response to snapshot.""" + return [ + _sse_event( + ResponseErrorEvent( + type="error", + sequence_number=events_sent, + code=cause, + message=detail, + param=None, + )) + ] async def guard_responses_stream( stream: AsyncGenerator[Any, None], terminal_events: Callable[[str, str, int], List[Any]], - on_termination: Optional[Callable[[str, str], None]] = None, ) -> AsyncGenerator[Any, None]: - """Make a Responses stream say so when it stops before completing. - - Without this such a stream simply ends: the generator raises, the ASGI - server abandons a half-written chunked body, and nothing in the bytes - distinguishes that from a stream still in flight. Everything the turn - produced survives only as deltas, because only the terminal event repeats - the full text. - - Frames are forwarded untouched and nothing is added once a terminal event - (``response.completed``, ``response.incomplete`` for a truncated - generation, or ``response.failed`` for one the engine gave up on) has - gone out, so a stream that completes normally is byte-for-byte what it - was. - - The two abnormal endings are handled differently on purpose: - - * A **fault on this side** yields the terminal events and then re-raises, - so the client gets bytes that explain the truncation and the server - still records an error rather than a clean finish. - * A **client hangup** yields nothing. There is nobody left to send to, and - a generator that yields while ``GeneratorExit`` is propagating raises - ``RuntimeError: async generator ignored GeneratorExit`` - trading a - diagnosable truncation for an undiagnosable one. The cause is still - reported through ``on_termination`` when a caller supplies one. + """End a stream that fails before its terminal event with events saying so. + + The exception is re-raised and frames pass through untouched. A client hangup (CancelledError or + GeneratorExit) is not caught: there is nobody left to tell. """ events_sent = 0 - carry = None + tail = None completed = False try: async for chunk in stream: if not completed: - frames, carry = _count_frames(chunk, carry) - events_sent += frames - markers = (_TERMINAL_EVENT_MARKERS_BYTES if isinstance( - chunk, bytes) else _TERMINAL_EVENT_MARKERS) - completed = any(marker in chunk for marker in markers) + if tail is None: + tail = chunk[:0] + text = tail + chunk + delimiter, markers = _sse_delimiter_and_markers(text) + events_sent += text.count(delimiter) - tail.count(delimiter) + completed = any(marker in text for marker in markers) + tail = text[-_SSE_TAIL_LENGTH:] yield chunk - except (asyncio.CancelledError, GeneratorExit) as exc: - if on_termination is not None: - on_termination(STREAM_TERMINATION_CLIENT_DISCONNECT, - describe_stream_termination(exc)) - raise except Exception as exc: + if completed: + raise cause = classify_stream_termination(exc) - detail = describe_stream_termination(exc) - if on_termination is not None: - on_termination(cause, detail) - if not completed: - logger.error("Responses stream terminated before completion " - f"({cause}): {detail}") - # Built before yielding, and inside its own guard: this runs only - # when something has already gone wrong, and a failure here would - # replace the exception the caller needs to see with one about - # reporting it. - try: - frames = terminal_events(cause, detail, events_sent) - except Exception as report_error: # noqa: BLE001 - logger.error( - f"Failed to build the terminal event: {report_error}") - frames = [] - for frame in frames: - yield frame + logger.error("Responses stream terminated before completion " + f"({cause}): {type(exc).__name__}: {exc}") + try: + frames = terminal_events(cause, + describe_stream_termination(exc, cause), + events_sent) + except Exception as report_error: # noqa: BLE001 - keep the original error + logger.error(f"Failed to build the terminal event: {report_error}") + frames = [] + if frames and tail: + delimiter, _ = _sse_delimiter_and_markers(tail) + if not tail.endswith(delimiter): + # A relay can fail mid-frame; end that frame before ours. + yield delimiter + for frame in frames: + yield frame raise @@ -4152,7 +3069,6 @@ async def process_streaming_events( create_time: Optional[int] = None, reasoning_parser: Optional[str] = None, tool_parser: Optional[str] = None, - tokenizer: Optional[TokenizerBase] = None, ) -> AsyncGenerator[str, None]: streaming_processor = ResponsesStreamingProcessor( request=request, @@ -4176,8 +3092,7 @@ async def process_streaming_events( for event in events: yield event - final_response = await streaming_processor.get_final_response( - final_res, tokenizer=tokenizer) + final_response = await streaming_processor.get_final_response(final_res) yield final_response @@ -4291,26 +3206,24 @@ async def done_generator() -> AsyncGenerator[bytes, None]: yield "data: [DONE]\n\n".encode('utf-8') +def _format_sse_event(event: OpenAIBaseModel) -> str: + # by_alias: fields such as text.format.schema go out under their wire name. + return (f"event: {getattr(event, 'type', 'unknown')}\n" + f"data: {event.model_dump_json(indent=None, by_alias=True)}\n\n") + + def _sse_event(event: StreamingResponsesResponse) -> bytes: - return (f"event: {event.type}\n" - f"data: {event.model_dump_json(indent=None)}\n\n").encode("utf-8") + return _format_sse_event(event).encode("utf-8") async def responses_done_generator( response: ResponsesResponse) -> AsyncGenerator[bytes, None]: - """Stream an already-complete ResponsesResponse as a well-formed SSE run. - - Used when a request finishes without ever reaching a generation worker. - The Responses protocol has no ``[DONE]`` sentinel - a client watches for - a terminal event - so emitting the completions-style terminator here - would leave a streaming client waiting for an event that never arrives. + """Stream an already-complete response as a well-formed SSE run. - The terminal event is chosen by the response's status, like every other - completed stream (_stream_terminal_event): a replayed response that was - cut off at its token budget must end in ``response.incomplete``, not in a - ``response.completed`` that contradicts the status it carries. + Used for a response the context worker finished; Responses has no + ``[DONE]``. """ - payload = response.model_dump() + payload = response.model_dump(by_alias=True) yield _sse_event( ResponseCreatedEvent( type="response.created", @@ -4323,12 +3236,7 @@ async def responses_done_generator( response=payload, sequence_number=1, )) - # finish_reason is carried on context-only responses, which is the only - # kind this generator replays; None simply leaves the reason unstated. - yield _sse_event( - _stream_terminal_event(response, - response.finish_reason, - sequence_number=2)) + yield _sse_event(_stream_terminal_event(response, sequence_number=2)) UCompletionResponseOrGenerator = Union[UCompletionResponse, diff --git a/tensorrt_llm/serve/responses_web_search.py b/tensorrt_llm/serve/responses_web_search.py index 70ab3a2d1917..83e4558e4555 100644 --- a/tensorrt_llm/serve/responses_web_search.py +++ b/tensorrt_llm/serve/responses_web_search.py @@ -63,39 +63,19 @@ _QUERY_ARG = "query" -def is_web_search_tool(tool: Any) -> bool: - """Whether this tool entry is a web_search tool, object- or dict-shaped. - - Both shapes reach here. A validated request carries model objects, but - tools also travel as plain dicts -- `get_chat_completion_tool_dicts` - produces them, and a raw body inspected before validation is dicts all the - way down. - - Reading only attributes made this gate reject every dict, so the helpers - behind it -- which do handle dicts -- never ran, and - `web_search_rejection_reason` returned None for a tool it should have had - an opinion about. None then flowed to callers that expected a string. - """ - tool_type = getattr(tool, "type", None) - if tool_type is None and isinstance(tool, dict): - tool_type = tool.get("type") - return bool(tool_type) and str(tool_type).startswith(_WEB_SEARCH_TYPE_PREFIX) - - def _tool_field(tool: Any, name: str, default: Any = None) -> Any: - """Read a field from a tool that may be an object or a plain dict. - - Reading attributes alone silently yields None for every dict-shaped tool, - which does not fail -- it produces a spec describing nothing, and a - rejection message that names ``tool type None`` for a tool whose type was - right there in the payload. - """ + """Read a field from a tool that may be an object or a plain dict.""" value = getattr(tool, name, None) if value is None and isinstance(tool, dict): value = tool.get(name) return default if value is None else value +def is_web_search_tool(tool: Any) -> bool: + tool_type = _tool_field(tool, "type") + return bool(tool_type) and str(tool_type).startswith(_WEB_SEARCH_TYPE_PREFIX) + + def web_search_tool_spec(tools: Optional[Sequence[Any]]) -> Optional[WebSearchToolSpec]: """Reduce a request's tools to what the shared module needs, or None.""" for tool in tools or []: @@ -114,43 +94,6 @@ def web_search_tool_spec(tools: Optional[Sequence[Any]]) -> Optional[WebSearchTo return None -def _external_access_flag(tools: Optional[Sequence[Any]]) -> Any: - """The tool's ``external_web_access`` as it survived request validation. - - Returns the sentinel string "absent" when the field is not there at all, - which is a different situation from an explicit True: a client that never - set it may simply be using an older tool shape. Reported in the rejection - message because it is the field the decision turns on, and it is otherwise - invisible to whoever has to act on the error. - """ - for tool in tools or []: - if not is_web_search_tool(tool): - continue - if isinstance(tool, dict): - return tool.get("external_web_access", "absent") - return getattr(tool, "external_web_access", "absent") - return "absent" - - -def _wants_external_search(tools: Optional[Sequence[Any]]) -> bool: - """Whether the client is actually asking for live external searches. - - The tool carries ``external_web_access``. A client that sets it to False is - declaring up front that it does not want the server reaching the open web, - so there is no live result for the server to fail to deliver and nothing - the client could be misled about. Absent or True means the ordinary - "search the web for me" request. - """ - for tool in tools or []: - if not is_web_search_tool(tool): - continue - external = getattr(tool, "external_web_access", None) - if external is None and isinstance(tool, dict): - external = tool.get("external_web_access") - return external is not False - return False - - def web_search_rejection_reason(tools: Optional[Sequence[Any]]) -> Optional[str]: """Why this request's web_search tool cannot be honoured, or None. @@ -164,20 +107,13 @@ def web_search_rejection_reason(tools: Optional[Sequence[Any]]) -> Optional[str] spec = web_search_tool_spec(tools) if spec is None: return None - # Name the tool that triggered this. A bare "'web_search' cannot be - # honoured" leaves the operator guessing which of the client's tools it - # means and what shape it arrived in, and the tool is one a client often - # attaches implicitly rather than on purpose. - seen = f" (tool type {spec.type!r}, external_web_access={_external_access_flag(tools)!r})" - if not _wants_external_search(tools): - # The client attached the tool with external access explicitly off, so - # it is not waiting on live results and cannot be misled by their - # absence. Codex sends exactly this - {"type": "web_search", - # "external_web_access": false} - when its model catalog says the model - # has no search tool; refusing that costs the client the whole server - # over a search it never asked for. A client that leaves the flag on is - # asking for live search and still gets a loud failure below. + tool = next(tool for tool in tools if is_web_search_tool(tool)) + external = _tool_field(tool, "external_web_access") + # A client that turns external access off is not waiting on live results + # (Codex attaches the tool that way to models without search). + if external is False: return None + seen = f" (tool type {spec.type!r}, external_web_access={external!r})" if load_web_search_config().enabled: # A provider is configured, so the operator does expect live search, # but the per-request search loop is not wired into this endpoint yet - diff --git a/tests/integration/test_lists/test-db/l0_cpu.yml b/tests/integration/test_lists/test-db/l0_cpu.yml index 35d13d8f79dc..b79e2a745ca2 100644 --- a/tests/integration/test_lists/test-db/l0_cpu.yml +++ b/tests/integration/test_lists/test-db/l0_cpu.yml @@ -165,7 +165,6 @@ l0_cpu: - unittest/llmapi/apps/test_responses_streaming_events.py - unittest/llmapi/apps/test_responses_streaming_tool_calls.py - unittest/llmapi/apps/test_responses_usage.py - - unittest/llmapi/apps/test_responses_usage_disagg.py - unittest/llmapi/apps/test_serving_extensions.py - unittest/llmapi/apps/test_tool_parsers.py - unittest/llmapi/apps/test_web_search_shared.py diff --git a/tests/unittest/api_stability/references/trtllm_serve_api.yaml b/tests/unittest/api_stability/references/trtllm_serve_api.yaml index 155d5ff771c5..f0a14aac621e 100644 --- a/tests/unittest/api_stability/references/trtllm_serve_api.yaml +++ b/tests/unittest/api_stability/references/trtllm_serve_api.yaml @@ -514,12 +514,6 @@ models: default: null status: stable required: true - parallel_tool_calls: - kind: openai - type: Optional[bool] - default: null - status: stable - required: false prompt_token_ids: kind: extension type: Optional[List[int]] diff --git a/tests/unittest/disaggregated/test_disagg_openai_client.py b/tests/unittest/disaggregated/test_disagg_openai_client.py index 6c37b735aeb7..3bc7bb0ab264 100644 --- a/tests/unittest/disaggregated/test_disagg_openai_client.py +++ b/tests/unittest/disaggregated/test_disagg_openai_client.py @@ -1149,18 +1149,7 @@ async def test_transient_tcp_capped_at_5_when_max_retries_smaller(self): def test_a_forwarded_request_keeps_the_field_names_it_arrived_with(): - """`schema` must not reach a worker spelled `schema_`. - - pydantic cannot hold a field called `schema` -- it shadows - `BaseModel.schema` -- so the model declares `schema_` with `schema` as its - alias. Serialising without `by_alias` sends the internal name, every - member of the format union fails to validate on the worker, and the - request comes back 400. That was 43 of 195 Responses requests in one - campaign round: every structured-output call the agents made. - - Only disaggregated serving re-serialises a request, so only it is - affected. The body is decoded exactly as the worker decodes it. - """ + """`schema` must not reach a worker spelled `schema_`, its pydantic name.""" request = ResponsesRequest.model_validate( { "model": "m", @@ -1180,5 +1169,4 @@ def test_a_forwarded_request_keeps_the_field_names_it_arrived_with(): assert body["text"]["format"]["schema"] == {"type": "object"} assert "schema_" not in body["text"]["format"] - # And the worker can rebuild the request from what it was sent. assert ResponsesRequest.model_validate(body).text.format.schema_ == {"type": "object"} diff --git a/tests/unittest/disaggregated/test_disagg_responses_api.py b/tests/unittest/disaggregated/test_disagg_responses_api.py index 114610c98d44..b237ce88695c 100644 --- a/tests/unittest/disaggregated/test_disagg_responses_api.py +++ b/tests/unittest/disaggregated/test_disagg_responses_api.py @@ -14,35 +14,47 @@ # limitations under the License. """Offline tests for the Responses API on the disaggregated path. -The disaggregated orchestrator holds no tokenizer and no engine: it splits one -client request into a context request and a generation request, relays a -KV-cache handle between two workers, and streams the second worker's bytes -back. These tests pin the parts of that split that differ for Responses, -because the Completions and Chat Completions shapes carry the handoff on -``choices[0]`` and a Responses response has no ``choices`` at all. +A Responses response has no ``choices``, so the context-to-generation handoff +that completions and chat carry on ``choices[0]`` sits at its top level. """ import asyncio +import base64 +import json +from types import SimpleNamespace +from unittest.mock import patch +import aiohttp +import numpy as np import pytest +from fastapi import HTTPException -from tensorrt_llm.serve.openai_disagg_service import _ctx_handoff_slots, _drop_ctx_handoff +from tensorrt_llm.serve.openai_disagg_server import OpenAIDisaggServer +from tensorrt_llm.serve.openai_disagg_service import ( + OpenAIDisaggregatedService, + _ctx_handoff_slots, + _ctx_usage_info, +) from tensorrt_llm.serve.openai_protocol import ( ChatCompletionResponse, DisaggregatedParams, + ModelCard, + ModelList, + PromptTokensDetails, ResponsesRequest, ResponsesResponse, + UsageInfo, +) +from tensorrt_llm.serve.responses_utils import ( + create_response_non_store, + finish_reason_mapping, + responses_done_generator, ) -# The CPU-* CI stages run pytest with -m 'cpu_only'. Without this marker every -# test in the file is deselected, which pytest reports as exit code 5 and the -# stage reports as a failure. pytestmark = pytest.mark.cpu_only -def _responses_response( - finish_reason="length", disagg=True, prompt_token_ids=None, status="incomplete" -): +def _responses_response(finish_reason="length", status="incomplete"): return ResponsesResponse( model="m", output=[], @@ -57,395 +69,277 @@ def _responses_response( top_logprobs=0, truncation="disabled", finish_reason=finish_reason, - prompt_token_ids=prompt_token_ids, + prompt_token_ids=[1, 2], + prompt_token_ids_b64="AQAAAAIAAAA=", disaggregated_params=DisaggregatedParams( - request_type="context_only", - ctx_request_id=7, - disagg_request_id=7, - ) - if disagg - else None, + request_type="context_only", ctx_request_id=7, disagg_request_id=7 + ), ) -# --------------------------------------------------------------------------- -# The handoff carrier -# --------------------------------------------------------------------------- +def _service(**attrs): + service = OpenAIDisaggregatedService.__new__(OpenAIDisaggregatedService) + async def ready(): + return True -def test_responses_response_carries_the_handoff_at_top_level(): - """No ``choices`` to hang it on, so the orchestrator reads it directly.""" - response = _responses_response() - slots = _ctx_handoff_slots(response) - assert slots == [response] - assert slots[0].disaggregated_params.ctx_request_id == 7 + service.is_ready = ready + for name, value in attrs.items(): + setattr(service, name, value) + return service -def test_chat_response_still_carries_the_handoff_per_choice(): - """The pre-existing shape must keep working unchanged.""" - response = ChatCompletionResponse( +# --------------------------------------------------------------------------- +# The handoff +# --------------------------------------------------------------------------- + + +def test_the_handoff_is_read_per_choice_for_chat_and_top_level_for_responses(): + chat = ChatCompletionResponse( model="m", choices=[ { "index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "length", - "disaggregated_params": { - "request_type": "context_only", - "ctx_request_id": 3, - }, + "disaggregated_params": {"request_type": "context_only", "ctx_request_id": 3}, } ], usage={"prompt_tokens": 1, "total_tokens": 2, "completion_tokens": 1}, ) - slots = _ctx_handoff_slots(response) - assert len(slots) == 1 - assert slots[0].disaggregated_params.ctx_request_id == 3 - - -def test_dropping_the_handoff_clears_it(): - """A response that never reaches a generation worker must not leak it. - - The field names an internal KV-cache transfer; relaying it to the client - would describe a transfer that never happens. - """ + assert [s.disaggregated_params.ctx_request_id for s in _ctx_handoff_slots(chat)] == [3] response = _responses_response() - _drop_ctx_handoff(response) - assert response.disaggregated_params is None + assert _ctx_handoff_slots(response) == [response] -# --------------------------------------------------------------------------- -# finish_reason: why `status` is not enough -# --------------------------------------------------------------------------- - - -def test_finish_reason_is_carried_unmapped(): - """``status`` cannot express the distinction the orchestrator needs. - - "length" and "not_finished" both map onto the public status "incomplete", - but only the raw value tells the orchestrator a generation phase is still - pending, and ``finish_reason_mapping`` used to raise outright on - "not_finished". - """ - response = _responses_response(finish_reason="not_finished") - assert response.status == "incomplete" - assert response.finish_reason == "not_finished" - - -def test_not_finished_maps_to_incomplete_rather_than_raising(): - """Regression: the context worker is capped at one token. +@pytest.mark.parametrize( + "finish_reason, need_gen", [("length", True), ("not_finished", True), ("stop", False)] +) +def test_a_finished_context_response_goes_to_the_client_without_the_handoff( + finish_reason, need_gen +): + response = _responses_response(finish_reason=finish_reason) + assert _service()._need_gen(response) is need_gen + handoff = ( + response.disaggregated_params, + response.prompt_token_ids, + response.prompt_token_ids_b64, + response.finish_reason, + ) + if need_gen: + assert all(field is not None for field in handoff) + else: + assert handoff == (None, None, None, None) - It hands the request off without finishing, so the engine reports - "not_finished" -- a value the mapping did not handle, which surfaced as a - 500 from the context worker rather than a completed handoff. - """ - from tensorrt_llm.serve.responses_utils import finish_reason_mapping - assert finish_reason_mapping("not_finished") == "incomplete" - assert finish_reason_mapping("length") == "incomplete" - assert finish_reason_mapping("stop") == "completed" +@pytest.mark.parametrize( + "finish_reason, status", + [("not_finished", "incomplete"), ("length", "incomplete"), ("stop", "completed")], +) +def test_finish_reason_mapping(finish_reason, status): + assert finish_reason_mapping(finish_reason) == status def test_an_unknown_finish_reason_names_itself(): - from tensorrt_llm.serve.responses_utils import finish_reason_mapping - with pytest.raises(RuntimeError, match="wat"): finish_reason_mapping("wat") -# --------------------------------------------------------------------------- -# The relayed prompt -# --------------------------------------------------------------------------- - - -def test_relayed_prompt_token_ids_are_used_verbatim(): - """The generation worker must not re-render the chat template. - - Re-rendering is not merely wasted work: a template that varies per render - would produce a prompt the context worker never prefilled, and the - relayed KV cache would not correspond to it. - """ - from tensorrt_llm.serve.responses_utils import _relayed_prompt_token_ids - - request = ResponsesRequest(model="m", input="hi", prompt_token_ids=[1, 2, 3]) - assert _relayed_prompt_token_ids(request) == [1, 2, 3] - - -def test_relayed_prompt_token_ids_b64_round_trips(): - """The orchestrator relays one base64 string instead of an int list.""" - import base64 - - import numpy as np - - from tensorrt_llm.serve.responses_utils import _relayed_prompt_token_ids - - encoded = base64.b64encode(np.asarray([5, 6, 7], dtype=np.int32).tobytes()).decode("ascii") - request = ResponsesRequest(model="m", input="hi", prompt_token_ids_b64=encoded) - assert _relayed_prompt_token_ids(request) == [5, 6, 7] - - -def test_an_ordinary_client_request_has_no_relayed_prompt(): - """A client sends neither field, so the worker tokenizes as usual.""" - from tensorrt_llm.serve.responses_utils import _relayed_prompt_token_ids - - assert _relayed_prompt_token_ids(ResponsesRequest(model="m", input="hi")) is None - - -# --------------------------------------------------------------------------- -# Streaming a request that never reached a generation worker -# --------------------------------------------------------------------------- - - -def _drain(generator): - async def collect(): - return [chunk async for chunk in generator] - - return asyncio.run(collect()) - - -def test_ctx_only_stream_ends_with_a_terminal_event(): - """Regression: the completions terminator hangs a Responses client. +@pytest.mark.parametrize( + "fields, expected", + [ + ({"prompt_token_ids": [1, 2, 3]}, [1, 2, 3]), + ( + { + "prompt_token_ids_b64": base64.b64encode( + np.asarray([5, 6, 7], dtype=np.int32).tobytes() + ).decode("ascii") + }, + [5, 6, 7], + ), + ({}, None), + ], +) +def test_the_generation_worker_uses_the_relayed_prompt(fields, expected): + request = ResponsesRequest(model="m", input="hi", **fields) + assert request.relayed_prompt_token_ids() == expected + assert request.prompt_token_ids == fields.get("prompt_token_ids") - The Responses protocol has no ``[DONE]`` sentinel -- a client watches for - a terminal event. Emitting ``data: [DONE]`` leaves it waiting for an - event that never arrives, and drops the answer the context worker already - produced. A response that ran to its natural end terminates with - ``response.completed``. - """ - from tensorrt_llm.serve.responses_utils import responses_done_generator - chunks = _drain( - responses_done_generator(_responses_response(finish_reason="stop", status="completed")) +@pytest.mark.parametrize( + "usage, expected", + [ + ( + SimpleNamespace( + input_tokens=96, + input_tokens_details=SimpleNamespace(cached_tokens=64), + output_tokens=1, + total_tokens=97, + ), + (96, 1, 64), + ), + ( + SimpleNamespace( + input_tokens=96, input_tokens_details=None, output_tokens=1, total_tokens=97 + ), + (96, 1, 0), + ), + ], +) +def test_the_context_usage_is_handed_off_as_usage_info(usage, expected): + carried = _ctx_usage_info(SimpleNamespace(usage=usage)) + assert ( + carried.prompt_tokens, + carried.completion_tokens, + carried.prompt_tokens_details.cached_tokens, + ) == expected + already = UsageInfo( + prompt_tokens=1, + completion_tokens=1, + total_tokens=2, + prompt_tokens_details=PromptTokensDetails(cached_tokens=0), ) - body = b"".join(chunks).decode("utf-8") - - assert "event: response.completed" in body - assert "[DONE]" not in body - # A well-formed run opens before it closes. - assert body.index("response.created") < body.index("response.completed") - - -def test_ctx_only_truncated_replay_terminates_as_incomplete(): - """A replayed response cut off at its token budget says so. - - The terminal event is chosen by status, exactly as on every other - completed stream (_stream_terminal_event): a ``response.completed`` - around a status of "incomplete" asserts the one thing its payload - denies. The fixture's default is the truncated shape this generator - replays when the context phase finished the whole request. - """ - from tensorrt_llm.serve.responses_utils import responses_done_generator - - chunks = _drain(responses_done_generator(_responses_response())) - body = b"".join(chunks).decode("utf-8") - - assert "event: response.incomplete" in body - assert "max_output_tokens" in body - assert "[DONE]" not in body - assert body.index("response.created") < body.index("response.incomplete") - - -def test_ctx_only_stream_events_are_sse_framed(): - from tensorrt_llm.serve.responses_utils import responses_done_generator - - chunks = _drain(responses_done_generator(_responses_response())) - for chunk in chunks: - text = chunk.decode("utf-8") - assert text.startswith("event: ") - assert "\ndata: " in text - assert text.endswith("\n\n") - + assert _ctx_usage_info(SimpleNamespace(usage=already)) is already -# --------------------------------------------------------------------------- -# Routing a Responses request to the right worker endpoint -# --------------------------------------------------------------------------- - - -def test_client_dispatches_responses_requests_to_v1_responses(): - """The endpoint is chosen by request type; an unmapped type raises. - Without this branch a Responses request reaching the client raised - "Invalid request type", which is how the route would have failed had only - the orchestrator side been wired up. - """ - import inspect - - from tensorrt_llm.serve.openai_client import OpenAIClient - - source = inspect.getsource(OpenAIClient.send_request) - assert "ResponsesRequest" in source - assert "v1/responses" in source +@pytest.mark.parametrize("use_harmony", [False, True]) +def test_a_context_only_response_carries_the_handoff(use_harmony): + """The single handoff token is not a complete Harmony message to parse.""" + request = ResponsesRequest( + model="m", input="hi", disaggregated_params=DisaggregatedParams(request_type="context_only") + ) + output = SimpleNamespace( + index=0, text="", token_ids=[7], finish_reason="length", disaggregated_params=None + ) + result = SimpleNamespace(outputs=[output], prompt_token_ids=[1, 2], cached_tokens=0) + with patch( + "tensorrt_llm.serve.responses_utils._create_output_content_harmony", + side_effect=AssertionError("parsed a context-only output"), + ): + response = create_response_non_store( + generation_result=result, + request=request, + sampling_params=request.to_sampling_params(), + model_name="m", + use_harmony=use_harmony, + ) + assert response.output == [] + assert (response.finish_reason, response.prompt_token_ids) == ("length", [1, 2]) # --------------------------------------------------------------------------- -# Fields the orchestrator needs on the wire +# Streaming a request the context phase finished # --------------------------------------------------------------------------- @pytest.mark.parametrize( - "field", + "finish_reason, status, terminal, details", [ - "disaggregated_params", - "conversation_params", - "prompt_token_ids", - "prompt_token_ids_b64", + ("stop", "completed", "response.completed", None), + ("length", "incomplete", "response.incomplete", {"reason": "max_output_tokens"}), ], ) -def test_request_carries_the_orchestrator_fields(field): - """``_wrap_entry_point`` and the split read these off every request. - - ``conversation_params`` in particular is read unconditionally by - conversation-id resolution, so its absence is an AttributeError on the - first request rather than a missing feature. - """ - request = ResponsesRequest(model="m", input="hi") - assert hasattr(request, field) - assert getattr(request, field) is None - - -def test_request_fields_survive_serialization_to_a_worker(): - """The client serializes with exclude_unset, so set fields must persist.""" - request = ResponsesRequest(model="m", input="hi") - request.disaggregated_params = DisaggregatedParams( - request_type="generation_only", ctx_request_id=11 - ) - request.prompt_token_ids = [1, 2] +def test_a_context_finished_stream_replays_the_response(finish_reason, status, terminal, details): + response = _responses_response(finish_reason=finish_reason, status=status) + if details: + response.incomplete_details = details - payload = request.model_dump_json(exclude_unset=True) - assert "generation_only" in payload - assert "prompt_token_ids" in payload + async def collect(): + return [chunk.decode() async for chunk in responses_done_generator(response)] - revived = ResponsesRequest.model_validate_json(payload) - assert revived.disaggregated_params.ctx_request_id == 11 - assert revived.prompt_token_ids == [1, 2] + frames = asyncio.run(collect()) + assert all(f.startswith("event: ") and f.endswith("\n\n") for f in frames) + events = [json.loads(f.split("data: ", 1)[1]) for f in frames] + assert [e["type"] for e in events] == ["response.created", "response.in_progress", terminal] + assert [e["sequence_number"] for e in events] == [0, 1, 2] + assert events[-1]["response"]["incomplete_details"] == details # --------------------------------------------------------------------------- -# `store` on a disaggregated server +# What the orchestrator does with a Responses request # --------------------------------------------------------------------------- -def _store_warnings(caplog): - return [r for r in caplog.records if "'store' is ignored" in r.getMessage()] - - -def test_store_left_at_its_default_does_not_warn(caplog): - """`store` defaults to True, so the value alone cannot justify a warning. - - Warning on the value would fire for every request that never mentioned the - field -- which is most of them -- and a warning on every request is one - nobody reads. - """ - import logging - caplog.set_level(logging.WARNING) - request = ResponsesRequest(model="m", input="hi") - assert request.store is True - assert "store" not in request.model_fields_set +@pytest.mark.parametrize("store, warned", [(None, False), (False, False), (True, True)]) +def test_store_is_forwarded_off_and_an_explicit_request_for_it_is_logged(store, warned): + sent = [] + async def send(request, hooks): + sent.append(request) + return "relayed" -def test_store_set_explicitly_is_a_request_that_goes_unmet(): - """A client that wrote `store` down asked for something it will not get. - - Unlike `previous_response_id` this does not stop the request being served, - so it is a warning rather than a 400 -- but it must not be silent: the - Responses protocol has no field for "served, but one thing you asked for - was dropped". - """ - request = ResponsesRequest(model="m", input="hi", store=True) - - assert "store" in request.model_fields_set - assert request.store is True - + request = ResponsesRequest(model="m", input="hi", **({} if store is None else {"store": store})) + with patch("tensorrt_llm.serve.openai_disagg_service.logger") as mock_logger: + assert asyncio.run(_service(_send_disagg_request=send).openai_responses(request)) == ( + "relayed" + ) + assert sent[0].store is False + assert mock_logger.warning_once.called is warned -def test_store_false_is_not_an_unmet_request(): - """Nothing was asked for, so there is nothing to report.""" - request = ResponsesRequest(model="m", input="hi", store=False) - assert "store" in request.model_fields_set - assert request.store is False +def test_previous_response_id_is_rejected_with_a_400(): + request = ResponsesRequest(model="m", input="hi", previous_response_id="resp_1") + with pytest.raises(HTTPException) as raised: + asyncio.run(_service().openai_responses(request)) + assert raised.value.status_code == 400 # --------------------------------------------------------------------------- -# /v1/models on a disaggregated server +# /v1/models # --------------------------------------------------------------------------- -class _StubCtxClient: - """Records what the service asked a worker for.""" - def __init__(self, result=None, error=None): - self.result, self.error = result, error - self.calls = [] + +class _StubCtxClient: + def __init__(self, error=None): + self.error = error + self.servers = [] async def get_json(self, endpoint, response_type, server): - self.calls.append((endpoint, server)) + assert endpoint == "v1/models" + self.servers.append(server) if self.error is not None: raise self.error - return self.result - - -class _StubRouter: - def __init__(self, servers): - self.servers = servers - - -def _service_with(ctx_client, servers=("w0:8001", "w1:8001"), ready=True): - """A service object with only the pieces get_model touches.""" - from tensorrt_llm.serve.openai_disagg_service import OpenAIDisaggregatedService - from tensorrt_llm.serve.openai_protocol import ModelCard, ModelList - - svc = OpenAIDisaggregatedService.__new__(OpenAIDisaggregatedService) - svc._ctx_client = ctx_client - svc._ctx_router = _StubRouter(list(servers)) - svc._count_tokens_rr_counter = 0 - - async def _ready(): - return ready - - svc.is_ready = _ready - return svc, ModelList, ModelCard + return ModelList(data=[ModelCard(id="served-name")]) -def test_get_model_asks_a_context_worker(): - """The orchestrator has no model of its own to report. - - DisaggServerConfig carries no model name -- the working deployments on hand - do not set one -- so answering from config would work for some clusters and - 404 for the rest. The worker knows the name requests are served under. - """ - from tensorrt_llm.serve.openai_protocol import ModelCard, ModelList - - expected = ModelList(data=[ModelCard(id="GLM-5.3")]) - client = _StubCtxClient(result=expected) - svc, _, _ = _service_with(client) - - got = asyncio.run(svc.get_model()) - - assert got.data[0].id == "GLM-5.3" - assert client.calls == [("v1/models", "w0:8001")] - - -def test_get_model_round_robins_context_workers(): - """Shares the counter with count-tokens, so two calls hit two workers.""" - from tensorrt_llm.serve.openai_protocol import ModelCard, ModelList - - client = _StubCtxClient(result=ModelList(data=[ModelCard(id="m")])) - svc, _, _ = _service_with(client) - - asyncio.run(svc.get_model()) - asyncio.run(svc.get_model()) - - assert [s for _, s in client.calls] == ["w0:8001", "w1:8001"] +def _models_service(client, servers=("w0:8001", "w1:8001")): + return _service( + _ctx_client=client, + _ctx_router=SimpleNamespace(servers=list(servers)), + _count_tokens_rr_counter=0, + ) -def test_get_model_without_a_context_worker_is_an_error_not_an_empty_list(): - """An empty list would read as "this server has no models", which is false. +def test_models_are_listed_by_a_context_worker_round_robin(): + client = _StubCtxClient() + service = _models_service(client) + assert asyncio.run(service.get_model()).data[0].id == "served-name" + asyncio.run(service.get_model()) + assert client.servers == ["w0:8001", "w1:8001"] - The model exists; there is currently nobody to ask. The server turns this - into a 503 so the client retries rather than concluding the deployment is - empty. - """ - client = _StubCtxClient(result=None) - svc, _, _ = _service_with(client, servers=()) - with pytest.raises(RuntimeError): - asyncio.run(svc.get_model()) - assert client.calls == [] +@pytest.mark.parametrize( + "client, servers, status", + [ + (_StubCtxClient(), (), 503), + ( + _StubCtxClient( + aiohttp.ClientResponseError( + SimpleNamespace(real_url="http://w0:8001/v1/models"), + (), + status=404, + message="Not Found", + ) + ), + ("w0:8001",), + 404, + ), + ], + ids=["no_context_worker", "worker_error"], +) +def test_a_models_failure_is_an_error_response(client, servers, status): + server = OpenAIDisaggServer.__new__(OpenAIDisaggServer) + server._service = _models_service(client, servers) + response = asyncio.run(server.get_model()) + assert response.status_code == status + assert json.loads(response.body)["code"] == status diff --git a/tests/unittest/llmapi/apps/test_anthropic_routes.py b/tests/unittest/llmapi/apps/test_anthropic_routes.py index 6b4d193e91e9..8a70d71e7d72 100644 --- a/tests/unittest/llmapi/apps/test_anthropic_routes.py +++ b/tests/unittest/llmapi/apps/test_anthropic_routes.py @@ -273,20 +273,12 @@ def test_messages_route_reframes_streaming_response(server_kind): @pytest.fixture def prometheus_multiproc_dir(tmp_path, monkeypatch): - """A multiprocess metrics directory that does not outlive the test. - - Setting PROMETHEUS_MULTIPROC_DIR cannot be undone by unsetting it. The - first metric built while it is set makes prometheus_client choose - ``values.ValueClass`` for the whole process and cache it; monkeypatch then - restores the environment at teardown but cannot put the class back. Every - later metric in that process is built by MultiProcessValue with no - directory to write into and dies with "expected str, bytes or - os.PathLike object, not NoneType". - - The damage lands on whichever file runs next, which makes it look like - that file's bug: test_disagg_openai_client.py passes alone and fails - wholesale -- 14 failures, 11 errors -- when this one precedes it. So the - class is restored here, not just the environment. + """PROMETHEUS_MULTIPROC_DIR for one test. + + prometheus_client picks and caches its value class the first time a metric + is built with the variable set, so the class is restored as well as the + environment; otherwise later metrics in the process fail without a + directory. """ import prometheus_client.values @@ -299,10 +291,9 @@ class is restored here, not just the environment. def test_standard_and_disagg_register_messages_route(prometheus_multiproc_dir): - # register_routes mounts a prometheus multiprocess collector, which needs - # PROMETHEUS_MULTIPROC_DIR to name a real directory: building the server - # with object.__new__ skips the startup that would normally create one. - # See the fixture for why setting the variable is not enough to undo. + # register_routes mounts a prometheus multiprocess collector, which needs a + # real PROMETHEUS_MULTIPROC_DIR; object.__new__ skips the startup that + # creates one. standard = object.__new__(OpenAIServer) standard.app = FastAPI() @@ -345,10 +336,6 @@ def test_standard_and_disagg_register_messages_route(prometheus_multiproc_dir): paths = {route.path for route in server.app.routes} assert "/v1/messages/count_tokens" in paths - # The disaggregated server reached parity with the aggregated one on these - # two only recently, and both were client-visible outages: Responses-API - # clients got a 404 on /v1/responses, and any client doing model discovery - # got one on /v1/models and had to be told the model name out of band. disagg_paths = {route.path for route in disagg.app.routes} assert "/v1/responses" in disagg_paths assert "/v1/models" in disagg_paths diff --git a/tests/unittest/llmapi/apps/test_responses_custom_tools.py b/tests/unittest/llmapi/apps/test_responses_custom_tools.py index 09a3bc76a274..92dcd662d996 100644 --- a/tests/unittest/llmapi/apps/test_responses_custom_tools.py +++ b/tests/unittest/llmapi/apps/test_responses_custom_tools.py @@ -66,11 +66,6 @@ def test_custom_tool_is_offered_with_a_named_string_parameter(): def test_custom_tool_names_are_collected(): - """A custom tool classifies custom; a plain function does not. - - The resolution map replaced _custom_tool_names; this file referenced the - removed helper and failed on import ever since - nothing in CI ran it. - """ tools = [_custom_tool(), SimpleNamespace(type="function", name="shell")] resolution = _tool_resolution(tools) assert resolution["apply_patch"][2] is True @@ -86,10 +81,6 @@ def test_no_tools_yields_no_custom_names(): # --------------------------------------------------------------------------- -# _tool_call_output_item takes the resolution map (every spelling -> -# (namespace, bare, custom)) since the tool-resolution refactor; these tests -# were written against the removed set/map pair and failed on import or call -# ever since - nothing in CI ran this file. _CUSTOM_RES = {"apply_patch": (None, "apply_patch", True)} diff --git a/tests/unittest/llmapi/apps/test_responses_input_preprocess.py b/tests/unittest/llmapi/apps/test_responses_input_preprocess.py index ef37db50c49d..738aef9f3998 100644 --- a/tests/unittest/llmapi/apps/test_responses_input_preprocess.py +++ b/tests/unittest/llmapi/apps/test_responses_input_preprocess.py @@ -20,13 +20,19 @@ question into an assistant turn. """ +import asyncio + import pytest from tensorrt_llm.serve.openai_protocol import ResponsesRequest from tensorrt_llm.serve.responses_utils import ( _create_input_messages, + _get_chat_completion_function_tools, + _render_developer_as_system, _response_output_item_to_chat_completion_message, + _tool_resolution, ) +from tensorrt_llm.tokenizer.deepseek_v4 import DeepseekV4Tokenizer # The CPU-* CI stages run pytest with -m 'cpu_only'. Without this marker every # test in the file is deselected, which pytest reports as exit code 5 and the @@ -50,13 +56,7 @@ def _message_item(role, *texts, item_id=None): # --------------------------------------------------------------------------- -# `developer` is deliberately not in this list: it is aliased to `system` -# because chat templates written before OpenAI renamed the role have no branch -# for it and drop the message entirely. See -# test_a_developer_message_is_rendered_as_system for that case. The point of -# the test below -- that a role is never silently replaced by "assistant" -- -# still holds for it. -@pytest.mark.parametrize("role", ["user", "assistant", "system"]) +@pytest.mark.parametrize("role", ["user", "assistant", "system", "developer"]) def test_item_role_is_preserved(role): """Regression: the role was hardcoded to "assistant". @@ -97,88 +97,14 @@ def test_role_defaults_to_assistant_when_absent(): assert msg["role"] == "assistant" -def test_empty_content_is_rejected(): +@pytest.mark.parametrize("content", [[], None]) +def test_empty_content_is_rejected(content): with pytest.raises(ValueError, match="empty or missing"): _response_output_item_to_chat_completion_message( - {"type": "message", "role": "user", "content": []} + {"type": "message", "role": "user", "content": content} ) -def test_typed_and_untyped_string_content_convert_identically(): - """The auditor's positive control: `type` must not change the meaning. - - An item with a role and no `type` is EasyInputMessage, where `type` - *defaults* to "message" - so spelling it out is the same item. The typed - branch walked string content as if it were a list of parts, iterating its - characters and keeping none of them, so the explicit spelling converted - to an empty message while the implicit one survived. - """ - untyped = _response_output_item_to_chat_completion_message({"role": "user", "content": "hello"}) - typed = _response_output_item_to_chat_completion_message( - {"type": "message", "role": "user", "content": "hello"} - ) - assert typed["content"] == "hello" - assert (typed["role"], typed["content"]) == (untyped["role"], untyped["content"]) - - -def test_a_typed_reasoning_item_accepts_string_content_too(): - """Same shape, one branch over: the walk is shared with "message".""" - msg = _response_output_item_to_chat_completion_message( - {"type": "reasoning", "content": "thinking"} - ) - assert msg == {"role": "assistant", "reasoning": "thinking"} - - -def test_agent_message_keeps_the_encrypted_content_payload(): - """The KF sub-agent task contract: readable text under a misleading name. - - The client serializes a task payload as a content part typed - `encrypted_content` whose same-named field holds plain readable text - - the field name is historical. Dropping the part delivered the task header - with no payload behind it; one measured request lost a 721-character task - this way. The fixture is a real request's body.input[6], trimmed. - """ - msg = _response_output_item_to_chat_completion_message( - { - "type": "agent_message", - "id": "amsg_01a0", - "author": "/root", - "recipient": "/root/final3", - "content": [ - { - "type": "input_text", - "text": "Message Type: NEW_TASK\nTask name: /root/final3\nSender: /root\nPayload:\n", - }, - { - "type": "encrypted_content", - "encrypted_content": "Write solution.json and evaluate it with cudagym.", - }, - ], - } - ) - assert msg["role"] == "user" - assert "Message Type: NEW_TASK" in msg["content"] - assert msg["content"].endswith("Write solution.json and evaluate it with cudagym.") - - -def test_non_string_encrypted_content_stays_dropped(): - """Only a string is readable by contract; anything else is truly opaque. - - Guessing at a structured or binary value would fabricate input, so the - header survives and the opaque part is dropped, exactly as before. - """ - msg = _response_output_item_to_chat_completion_message( - { - "type": "agent_message", - "content": [ - {"type": "input_text", "text": "header"}, - {"type": "encrypted_content", "encrypted_content": {"blob": "aGk="}}, - ], - } - ) - assert msg == {"role": "user", "content": "header"} - - def test_function_call_output_keeps_call_id(): msg = _response_output_item_to_chat_completion_message( { @@ -476,388 +402,175 @@ def test_meaningful_values_survive_the_null_scrub(): assert request.tool_choice == "none" -def test_reasoning_without_content_falls_back_to_the_summary(): - """The text is in `summary` whenever a summary was requested. - - Reading only `content` threw the summary away and then rejected the item - for being empty, so a client that asked for reasoning summaries lost both - the reasoning and the turn. - """ - msg = _response_output_item_to_chat_completion_message( - { - "type": "reasoning", - "id": "rs_1", - "summary": [{"type": "summary_text", "text": "weighed two options"}], - } - ) - - assert msg == {"role": "assistant", "reasoning": "weighed two options"} - - -def test_reasoning_with_nothing_readable_is_skipped_not_rejected(): - """A shape OpenAI emits, and one a client replays back verbatim. - - With `encrypted_content` and no summary there is no reasoning text in the - payload to preserve. Raising here does not save anything -- the text was - already absent -- it just fails the whole request, which ends the - conversation rather than the turn. - """ - msg = _response_output_item_to_chat_completion_message( - { - "type": "reasoning", - "id": "rs_2", - "summary": [], - "encrypted_content": "gAAAAA", - } - ) - - assert msg is None - +# --------------------------------------------------------------------------- +# Item shapes beyond a list of text parts +# --------------------------------------------------------------------------- -def test_a_message_with_no_content_is_still_rejected(): - """The tolerance above is specific to reasoning. - A message carries its text in `content` and nowhere else, so an empty one - is malformed and silently dropping it would lose something the client - believed it had sent. - """ - with pytest.raises(ValueError): - _response_output_item_to_chat_completion_message( +@pytest.mark.parametrize( + "item, expected", + [ + ( + {"type": "message", "role": "user", "content": "hello"}, + {"role": "user", "content": "hello"}, + ), + ({"type": "message", "role": "user", "content": ""}, {"role": "user", "content": ""}), + ( + {"type": "reasoning", "content": "thinking"}, + {"role": "assistant", "reasoning": "thinking"}, + ), + ( + {"type": "reasoning", "summary": [{"type": "summary_text", "text": "weighed it"}]}, + {"role": "assistant", "reasoning": "weighed it"}, + ), + ({"type": "reasoning", "summary": [], "encrypted_content": "gAAAAA"}, None), + ({"type": "output_text", "text": "hi"}, {"role": "assistant", "content": "hi"}), + ( { - "type": "message", - "role": "user", - "content": [], - } - ) - - -def test_role_without_type_gets_its_parts_translated(): - """`type` is optional on an input message; the parts still need rewriting. - - This is the API's EasyInputMessage. Returning it verbatim handed - `input_text` to the chat-completions content parser, which knows only - `text` and failed the request with "Unknown part type: input_text" -- so - whether a request worked depended on a field the client may leave out. - """ - msg = _response_output_item_to_chat_completion_message( - { - "role": "user", - "content": [{"type": "input_text", "text": "hello"}], - } - ) + "type": "agent_message", + "content": [ + {"type": "input_text", "text": "Payload:"}, + {"type": "encrypted_content", "encrypted_content": "write the file"}, + {"type": "encrypted_content", "encrypted_content": {"blob": "aGk="}}, + ], + }, + {"role": "user", "content": "Payload:\nwrite the file"}, + ), + ], +) +def test_item_text_is_read_from_every_shape_that_carries_it(item, expected): + assert _response_output_item_to_chat_completion_message(item) == expected - assert msg == {"role": "user", "content": [{"type": "text", "text": "hello"}]} +_IMAGE = {"type": "image_url", "image_url": {"url": "http://example/x.png"}} -def test_an_assistant_turn_replayed_without_a_type_is_accepted(): - """Replaying this server's own output must not need a field it omits. - The assistant parts come back spelled `output_text`, which is what the - Responses API calls them. - """ +@pytest.mark.parametrize( + "content, expected", + [ + ([{"type": "input_text", "text": "hello"}], [{"type": "text", "text": "hello"}]), + ([{"type": "output_text", "text": "hi"}], [{"type": "text", "text": "hi"}]), + ( + [{"type": "input_text", "text": "what is this"}, _IMAGE], + [{"type": "text", "text": "what is this"}, _IMAGE], + ), + ("hello", "hello"), + ], +) +def test_untyped_message_parts_are_translated_to_chat_parts(content, expected): + msg = _response_output_item_to_chat_completion_message({"role": "user", "content": content}) + assert msg == {"role": "user", "content": expected} + + +@pytest.mark.parametrize("item_type", ["function_call_output", "custom_tool_call_output"]) +@pytest.mark.parametrize( + "output, expected", + [ + ( + [{"type": "input_text", "text": "ok"}, {"type": "input_text", "text": " done"}], + [{"type": "text", "text": "ok"}, {"type": "text", "text": " done"}], + ), + ("plain text", "plain text"), + ], +) +def test_tool_results_are_translated_to_chat_parts(item_type, output, expected): msg = _response_output_item_to_chat_completion_message( - { - "role": "assistant", - "content": [{"type": "output_text", "text": "hi"}], - } + {"type": item_type, "call_id": "call_1", "output": output} ) - - assert msg == {"role": "assistant", "content": [{"type": "text", "text": "hi"}]} + assert msg == {"role": "tool", "content": expected, "tool_call_id": "call_1"} -def test_non_text_parts_survive_the_translation(): - """Only the two Responses-only text spellings are rewritten. - - Flattening the list to a string would have been simpler and would have - dropped the image, which the downstream parser does understand. - """ - image = {"type": "image_url", "image_url": {"url": "http://example/x.png"}} +def test_a_missing_custom_tool_result_is_empty(): msg = _response_output_item_to_chat_completion_message( - { - "role": "user", - "content": [{"type": "input_text", "text": "what is this"}, image], - } + {"type": "custom_tool_call_output", "call_id": "call_4"} ) - - assert msg["content"] == [{"type": "text", "text": "what is this"}, image] + assert msg["content"] == "" -def test_a_plain_string_content_is_left_alone(): - """The common shape must not be disturbed by the list handling.""" - msg = _response_output_item_to_chat_completion_message( - { - "role": "user", - "content": "hello", - } - ) - - assert msg == {"role": "user", "content": "hello"} +# --------------------------------------------------------------------------- +# Tools declared in an `additional_tools` input item +# --------------------------------------------------------------------------- -def test_additional_tools_item_becomes_tools(): - """Codex declares its tools as an input item, not in `tools`. +def _additional_tools_item(tools): + return {"type": "additional_tools", "role": "developer", "tools": tools} - The item carries no text, so the input-item conversion dropped it as - unrecognised and the model was offered nothing to call -- it then narrated - a terminal session it had invented, because narrating was the only thing - left to do. - """ - from tensorrt_llm.serve.openai_protocol import ResponsesRequest - request = ResponsesRequest( - model="m", - input=[ +def _namespace(name, *functions): + return { + "type": "namespace", + "name": name, + "description": "", + "tools": [ { - "type": "additional_tools", - "role": "developer", - "tools": [ - { - "type": "namespace", - "name": "functions", - "description": "", - "tools": [ - { - "type": "function", - "name": "get_time", - "description": "Return the time.", - "parameters": {"type": "object", "properties": {}}, - } - ], - } - ], - }, - {"role": "user", "content": "what time is it"}, + "type": "function", + "name": f, + "description": "", + "parameters": {"type": "object", "properties": {}}, + } + for f in functions ], - ) - - # The item is gone from the input: it is a tool declaration, not a turn, - # and replaying it as one would put the tool schema in the prompt as prose. - assert [item.get("type") for item in request.input] == [None] - assert len(request.tools) == 1 - assert request.tools[0].type == "namespace" - assert [t.name for t in request.tools[0].tools] == ["get_time"] - - -def test_hoisted_tools_are_offered_to_the_template_namespaced(): - """The nested tools have to reach the prompt under qualified names. + } - `_tool_resolution` maps a reply's call back to its namespace (under both - the qualified and, when unambiguous, the bare spelling), so a tool that - never made it into `tools` would come back as an unsupported call even if - the model somehow guessed it. This test referenced the helper the - resolution map replaced (`_namespaced_tool_names`) and had been failing - on import since that refactor - nothing in CI ran this file. - """ - from tensorrt_llm.serve.openai_protocol import ResponsesRequest - from tensorrt_llm.serve.responses_utils import ( - _get_chat_completion_function_tools, - _tool_resolution, - ) +def test_additional_tools_are_hoisted_after_the_declared_tools(): request = ResponsesRequest( model="m", + tools=[_namespace("existing", "a")], input=[ - { - "type": "additional_tools", - "role": "developer", - "tools": [ - { - "type": "namespace", - "name": "collaboration", - "description": "", - "tools": [ - { - "type": "function", - "name": "spawn_agent", - "description": "", - "parameters": {"type": "object", "properties": {}}, - } - ], - } - ], - }, - { - "role": "user", - "content": "go", - }, + _additional_tools_item([_namespace("collaboration", "spawn_agent")]), + {"role": "user", "content": "go"}, ], ) + assert [item.get("type") for item in request.input] == [None] + assert [t.name for t in request.tools] == ["existing", "collaboration"] offered = [t.function.name for t in _get_chat_completion_function_tools(request.tools)] - assert offered == ["collaboration.spawn_agent"] - # Both spellings resolve: the qualified one the template offered, and the - # bare one the model writes back anyway (measured 247-of-281 calls). + assert offered == ["existing.a", "collaboration.spawn_agent"] resolution = _tool_resolution(request.tools) assert resolution["collaboration.spawn_agent"] == ("collaboration", "spawn_agent", False) assert resolution["spawn_agent"] == ("collaboration", "spawn_agent", False) -def test_tools_already_in_the_tools_field_are_kept(): - """Hoisting appends; it must not discard what the client sent normally.""" - from tensorrt_llm.serve.openai_protocol import ResponsesRequest - - request = ResponsesRequest( - model="m", - tools=[ - { - "type": "function", - "name": "existing", - "description": "", - "parameters": {"type": "object", "properties": {}}, - } - ], - input=[ - { - "type": "additional_tools", - "role": "developer", - "tools": [{"type": "namespace", "name": "ns", "description": "", "tools": []}], - } - ], - ) - - assert [getattr(t, "name", None) for t in request.tools] == ["existing", "ns"] - - -def test_a_request_without_the_item_is_untouched(): - """The common case must not be reshaped by the hoist.""" - from tensorrt_llm.serve.openai_protocol import ResponsesRequest - - request = ResponsesRequest(model="m", input="hello") - - assert request.input == "hello" - assert request.tools == [] +def test_a_malformed_additional_tools_item_is_not_hoisted(): + request = {"model": "m", "input": [_additional_tools_item("not a list")], "tools": None} + assert ResponsesRequest.hoist_additional_tools(request) is request -def test_function_call_output_with_content_parts_is_translated(): - """A tool result may carry parts rather than a string. - - Assigning them to `content` untouched handed `input_text` to the - chat-completions parser, which knows only `text`, and the request failed - with "Unknown part type: input_text". - """ - msg = _response_output_item_to_chat_completion_message( - { - "type": "function_call_output", - "call_id": "call_1", - "output": [{"type": "input_text", "text": "42"}], - } - ) - - assert msg == { - "role": "tool", - "content": [{"type": "text", "text": "42"}], - "tool_call_id": "call_1", - } - - -def test_custom_tool_call_output_with_content_parts_is_translated(): - """The custom-tool branch had the same assumption. - - This is the one that fired in practice: a tool result arrives on every - turn after the model's first custom-tool call, so once an agent started - using tools nearly all of its traffic was rejected -- 483 of 490 requests - in one campaign round. - """ - msg = _response_output_item_to_chat_completion_message( - { - "type": "custom_tool_call_output", - "call_id": "call_2", - "output": [ - {"type": "input_text", "text": "ok"}, - {"type": "input_text", "text": " done"}, - ], - } - ) - - assert msg["role"] == "tool" - assert msg["content"] == [{"type": "text", "text": "ok"}, {"type": "text", "text": " done"}] - assert msg["tool_call_id"] == "call_2" - - -def test_a_string_tool_result_is_unchanged(): - """The simple shape must not be reshaped into parts.""" - msg = _response_output_item_to_chat_completion_message( - { - "type": "function_call_output", - "call_id": "call_3", - "output": "plain text", - } - ) - - assert msg["content"] == "plain text" - - -def test_a_missing_custom_tool_result_stays_empty(): - """Absent output is still not a reason to fail the turn.""" - msg = _response_output_item_to_chat_completion_message( - { - "type": "custom_tool_call_output", - "call_id": "call_4", - } - ) - - assert msg["content"] == "" +# --------------------------------------------------------------------------- +# developer messages +# --------------------------------------------------------------------------- -def test_a_developer_message_is_rendered_as_system(): - """Codex puts its whole operating brief in `developer` messages. +class _TemplateTokenizer: + def __init__(self, template): + self.template = template - GLM-5.3's template dispatches on role with branches for user, assistant, - tool and system and no fallback, so a developer message rendered to - nothing: the model was asked to drive an agent harness it had never been - told about, and improvised -- shell into a JavaScript sandbox, invented - cell ids, sub-agents spawned into the same confusion. + def get_chat_template(self, chat_template, tools=None): + return self.template - `developer` is OpenAI's rename of `system`, and every template knows - `system`. - """ - msg = _response_output_item_to_chat_completion_message( - { - "type": "message", - "role": "developer", - "content": [{"type": "input_text", "text": "You are Codex."}], - } - ) - - assert msg == {"role": "system", "content": "You are Codex."} - - -def test_a_developer_message_without_a_type_is_also_rendered(): - """The same brief arrives without `type` when the client omits it.""" - msg = _response_output_item_to_chat_completion_message( - { - "role": "developer", - "content": [{"type": "input_text", "text": "You are Codex."}], - } - ) - assert msg["role"] == "system" - assert msg["content"] == [{"type": "text", "text": "You are Codex."}] +_DEVELOPER_TURN = [{"role": "developer", "content": "brief"}, {"role": "user", "content": "q"}] -def test_other_roles_are_left_alone(): - """Only the alias is rewritten.""" - for role in ("user", "assistant", "system", "tool"): - msg = _response_output_item_to_chat_completion_message( - { - "type": "message", - "role": role, - "content": [{"type": "input_text", "text": "x"}], - } - ) - assert msg["role"] == role +@pytest.mark.parametrize( + "tokenizer, role", + [ + (_TemplateTokenizer("{% if m.role == 'system' %}{{ m.content }}{% endif %}"), "system"), + (_TemplateTokenizer("{% if m.role in ('system', 'developer') %}{% endif %}"), "developer"), + (DeepseekV4Tokenizer.__new__(DeepseekV4Tokenizer), "developer"), + ], + ids=["no_developer_branch", "developer_branch", "deepseek_v4"], +) +def test_developer_renders_as_system_only_where_the_template_lacks_it(tokenizer, role): + messages = _render_developer_as_system(_DEVELOPER_TURN, tokenizer, None, None) + assert [m["role"] for m in messages] == [role, "user"] + assert _DEVELOPER_TURN[0]["role"] == "developer" # --------------------------------------------------------------------------- -# A multi-call assistant turn stays one assistant message -# -# N function_call items from one turn used to convert into N assistant -# messages of one tool call each. The GLM chat template aligns tool results -# only against the LAST assistant message's tool_calls and, when alignment -# fails, renders the results positionally WITHOUT their ids - proven on live -# traffic by swapping two function_call_output call_ids and getting a -# byte-identical 596k-char prompt. The binding has to survive conversion: -# one assistant message, all N calls, in order. +# A multi-call assistant turn stays one assistant message, so the chat +# template can bind each tool result to its call by id # --------------------------------------------------------------------------- @@ -872,188 +585,63 @@ def _function_call_item(index): } -def _function_call_output_item(index, call_id=None): - return { - "type": "function_call_output", - "call_id": call_id or f"call_{index}", - "output": f"result {index}", - } - - -def _five_call_turn(swap=False): - """The acceptance pattern: text, five calls, five results. - - With swap=True the first two results trade call_ids, which changes which - call each result answers - the counterfactual that used to render to a - byte-identical prompt. - """ - outputs = [_function_call_output_item(i) for i in range(5)] - if swap: - outputs[0]["call_id"], outputs[1]["call_id"] = ( - outputs[1]["call_id"], - outputs[0]["call_id"], - ) - return { - "input": [ - _message_item("user", "run the plan", item_id=None), - { - "id": "msg_a", - "status": "completed", - "type": "message", - "role": "assistant", - "content": [ - {"type": "output_text", "text": "Running all five.", "annotations": []} - ], - }, - *[_function_call_item(i) for i in range(5)], - *outputs, - ] - } - - -def _glm_like_render(messages): - """A minimal stand-in for the GLM template's result alignment. - - Tool results are matched only against the LAST assistant message's - tool_calls; a result whose call_id is found renders bound to that call's - arguments, an unmatched one renders positionally with no id - which is - the information loss the real template exhibits. - """ - rendered = [] - last_tool_calls = [] - for message in messages: - if message.get("role") == "assistant": - last_tool_calls = message.get("tool_calls") or [] - for call in last_tool_calls: - rendered.append(f"") - elif message.get("role") == "tool": - by_id = {call["id"]: call for call in last_tool_calls} - call = by_id.get(message.get("tool_call_id")) - bound = call["function"]["arguments"] if call else "UNBOUND" - rendered.append(f"") - return "".join(rendered) - - -def test_five_calls_from_one_turn_become_one_assistant_message(): - messages = _messages(_five_call_turn()) - - assert [m["role"] for m in messages] == ["user", "assistant"] + ["tool"] * 5 - assistant = messages[1] - # The turn's own text survives in the same message, alongside the calls. - assert assistant["content"] == "Running all five." - assert [c["id"] for c in assistant["tool_calls"]] == [f"call_{i}" for i in range(5)] - assert [c["function"]["arguments"] for c in assistant["tool_calls"]] == [ - f'{{"cmd": "step {i}"}}' for i in range(5) - ] - # Every result still names the call it answers. - assert [m["tool_call_id"] for m in messages[2:]] == [f"call_{i}" for i in range(5)] - - -def test_swapping_two_result_ids_now_changes_the_binding(): - """The counterfactual that used to be invisible. - - Same items, two results trading call_ids: the message lists must differ, - and a GLM-like alignment must bind the results to different calls - not - render both variants identically because neither could be aligned. - """ - straight = _messages(_five_call_turn()) - swapped = _messages(_five_call_turn(swap=True)) - - assert straight != swapped - assert _glm_like_render(straight) != _glm_like_render(swapped) - # And not because anything fell off: both bind every result. - assert "UNBOUND" not in _glm_like_render(straight) - assert "UNBOUND" not in _glm_like_render(swapped) +def _function_call_output_item(index): + return {"type": "function_call_output", "call_id": f"call_{index}", "output": f"result {index}"} -def test_the_old_per_call_split_is_what_lost_the_binding(): - """Why the fold exists, demonstrated on the unfolded shape. - - Converting each item separately - the old behavior - leaves the last - assistant message holding only call 4, so results 0-3 align with nothing - and the swapped variant renders byte-identically: the binding is - unrecoverable downstream, which is exactly what the fold prevents. - """ - - def _unfolded(request_kwargs): - return [ - m - for m in ( - _response_output_item_to_chat_completion_message(item) - for item in request_kwargs["input"] - ) - if m is not None - ] - - straight = _unfolded(_five_call_turn()) - swapped = _unfolded(_five_call_turn(swap=True)) - assert _glm_like_render(straight) == _glm_like_render(swapped) - assert "UNBOUND" in _glm_like_render(straight) - - -def test_a_tool_result_between_calls_ends_the_assistant_turn(): - """Sequential call/result pairs are separate turns and stay separate.""" +def test_calls_from_one_turn_become_one_assistant_message(): messages = _messages( { "input": [ - _function_call_item(0), - _function_call_output_item(0), - _function_call_item(1), - _function_call_output_item(1), + _message_item("user", "run the plan"), + { + "id": "msg_a", + "status": "completed", + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "Running.", "annotations": []}], + }, + *[_function_call_item(i) for i in range(3)], + *[_function_call_output_item(i) for i in range(3)], ] } ) - assert [m["role"] for m in messages] == ["assistant", "tool", "assistant", "tool"] - assert [len(m.get("tool_calls") or []) for m in messages] == [1, 0, 1, 0] + assert [m["role"] for m in messages] == ["user", "assistant"] + ["tool"] * 3 + assert messages[1]["content"] == "Running." + assert [c["id"] for c in messages[1]["tool_calls"]] == ["call_0", "call_1", "call_2"] + assert [m["tool_call_id"] for m in messages[2:]] == ["call_0", "call_1", "call_2"] -def test_a_user_message_between_calls_ends_the_assistant_turn(): - messages = _messages( - { - "input": [ - _function_call_item(0), - _message_item("user", "wait, also do this"), - _function_call_item(1), - ] - } - ) - assert [m["role"] for m in messages] == ["assistant", "user", "assistant"] +@pytest.mark.parametrize( + "between", + [_function_call_output_item(0), _message_item("user", "also do this")], + ids=["tool_result", "user_message"], +) +def test_a_turn_in_between_ends_the_assistant_message(between): + messages = _messages({"input": [_function_call_item(0), between, _function_call_item(1)]}) + assert [len(m.get("tool_calls") or []) for m in messages] == [1, 0, 1] -def test_calls_fold_onto_a_reasoning_message(): - """Reasoning-then-calls is the GLM turn shape. - The calls ride the same assistant message, which is also how the - conversation store keeps them. - """ +def test_calls_fold_onto_a_reasoning_message(): messages = _messages( { "input": [ - { - "type": "reasoning", - "content": [{"type": "reasoning_text", "text": "plan both"}], - }, + {"type": "reasoning", "content": [{"type": "reasoning_text", "text": "plan"}]}, _function_call_item(0), _function_call_item(1), ] } ) assert len(messages) == 1 - assert messages[0]["reasoning"] == "plan both" + assert messages[0]["reasoning"] == "plan" assert [c["id"] for c in messages[0]["tool_calls"]] == ["call_0", "call_1"] def test_calls_never_fold_into_replayed_history(): - """The history's messages are stored turns, not this turn's opening.""" - import asyncio - - from tensorrt_llm.serve.responses_utils import _create_input_messages - request = ResponsesRequest(model="m", input=[_function_call_item(0)]) history = [{"role": "assistant", "content": "a stored turn"}] messages = asyncio.run(_create_input_messages(request=request, prev_msgs=history)) assert messages[0] == {"role": "assistant", "content": "a stored turn"} - assert "tool_calls" not in messages[0] - assert messages[1]["role"] == "assistant" assert [c["id"] for c in messages[1]["tool_calls"]] == ["call_0"] diff --git a/tests/unittest/llmapi/apps/test_responses_store_replay.py b/tests/unittest/llmapi/apps/test_responses_store_replay.py index 7aa464d5c02b..1a6266101aaf 100644 --- a/tests/unittest/llmapi/apps/test_responses_store_replay.py +++ b/tests/unittest/llmapi/apps/test_responses_store_replay.py @@ -12,31 +12,16 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Offline tests: the Responses store must not lose tool calls or prior context. - -Three ways a multi-turn tool conversation lost its thread, each pinned here: - -* The store writer only attached tool calls to the reasoning message, so a - turn that called tools without reasoning stored them nowhere - and a turn - that was *nothing but* tool calls stored no assistant message at all. -* A stored reasoning+calls turn survived storage but not replay: the replay - filter dropped the whole message for its "reasoning" key, calls included, so - the client's tool RESULT replayed with no call before it - an orphan the - model cannot pair. -* The server folded ``request.store`` into the one flag it hands - preprocessing, so a follow-up sent with ``store=false`` had its - ``previous_response_id`` loaded and validated - and then generated with no - history at all, silently. -""" +"""Offline tests for Responses storage and replay, and the endpoint up to preprocessing.""" import asyncio from types import SimpleNamespace -from unittest.mock import AsyncMock +from unittest.mock import AsyncMock, Mock, patch import pytest from openai.types.responses import ResponseFunctionToolCall -from tensorrt_llm.serve.openai_protocol import ResponsesRequest +from tensorrt_llm.serve.openai_protocol import DisaggregatedParams, ResponsesRequest from tensorrt_llm.serve.responses_utils import ( ConversationHistoryStore, _create_input_messages, @@ -44,9 +29,6 @@ request_preprocess, ) -# The CPU-* CI stages run pytest with -m 'cpu_only'. Without this marker every -# test in the file is deselected, which pytest reports as exit code 5 and the -# stage reports as a failure. pytestmark = pytest.mark.cpu_only @@ -77,34 +59,36 @@ def _contents(text=None, reasoning=None, calls=()): # The store writer: a tool-call-bearing turn is an assistant turn # --------------------------------------------------------------------------- +_STORED_CALL = { + "id": "call_1", + "type": "function", + "function": {"name": "exec", "arguments": '{"cmd": "ls"}'}, +} -def test_a_tool_only_turn_is_stored(): - """No text, no reasoning - the calls alone used to store nothing.""" - messages = _create_output_messages(_contents(calls=[_call()])) - assert len(messages) == 1 - assert messages[0]["role"] == "assistant" - assert messages[0]["tool_calls"][0]["id"] == "call_1" - assert messages[0]["tool_calls"][0]["function"]["name"] == "exec" - -def test_a_text_turn_keeps_its_calls(): - """Text plus calls without reasoning stored the text and dropped the calls.""" - messages = _create_output_messages(_contents(text="Running it.", calls=[_call()])) - carried = [m for m in messages if m.get("tool_calls")] - assert carried and carried[0]["content"] == "Running it." - - -def test_a_text_only_turn_stores_the_old_shape(): - assert _create_output_messages(_contents(text="Hello.")) == [ - {"role": "assistant", "content": "Hello."} - ] +@pytest.mark.parametrize( + "contents, expected", + [ + ( + _contents(calls=[_call()]), + [{"role": "assistant", "content": None, "tool_calls": [_STORED_CALL]}], + ), + ( + _contents(text="Running it.", calls=[_call()]), + [{"role": "assistant", "content": "Running it.", "tool_calls": [_STORED_CALL]}], + ), + (_contents(text="Hello."), [{"role": "assistant", "content": "Hello."}]), + ], + ids=["calls_only", "text_and_calls", "text_only"], +) +def test_a_turn_is_stored_with_its_calls(contents, expected): + assert _create_output_messages(contents) == expected -def test_a_reasoning_turn_still_carries_reasoning_and_calls(): - """The path that already worked, kept byte-compatible.""" +def test_a_reasoning_turn_carries_reasoning_and_calls(): (message,) = _create_output_messages(_contents(reasoning="think", calls=[_call()])) assert message["reasoning"] == "think" - assert message["tool_calls"][0]["id"] == "call_1" + assert message["tool_calls"] == [_STORED_CALL] # --------------------------------------------------------------------------- @@ -125,22 +109,20 @@ def test_replaying_a_reasoning_call_turn_preserves_the_calls(): assert carried and carried[0]["tool_calls"][0]["id"] == "call_1" -def test_replaying_a_reasoning_only_turn_still_drops_it(): - stored = _create_output_messages(_contents(reasoning="think")) - replayed = _replay(stored) - assert all("reasoning" not in m and not m.get("tool_calls") for m in replayed) +def test_replay_drops_a_reasoning_only_turn_and_keeps_others_as_stored(): + plain = {"role": "assistant", "content": None, "tool_calls": [_STORED_CALL]} + replayed = _replay(_create_output_messages(_contents(reasoning="think")) + [plain]) + assert replayed[0] is plain + assert replayed[1:] == [{"role": "user", "content": "hi"}] -def test_a_plain_assistant_call_message_replays_byte_identical(): - plain = { - "role": "assistant", - "content": None, - "tool_calls": [ - {"id": "call_9", "type": "function", "function": {"name": "exec", "arguments": "{}"}} - ], - } - replayed = _replay([plain]) - assert replayed[0] is plain +def test_trimming_a_stored_conversation_leaves_the_callers_messages_whole(): + store = ConversationHistoryStore(resp_capacity=1) + messages = [{"role": ("user", "assistant")[i % 2], "content": str(i)} for i in range(8)] + original = list(messages) + asyncio.run(store.store_messages("resp_1", messages, None)) + assert messages == original + assert len(store.conversations[store.response_to_conversation["resp_1"]]) < len(original) # --------------------------------------------------------------------------- @@ -164,8 +146,6 @@ async def _chain(output_contents, call_id): model="m", input=[_tool_result_item(call_id=call_id)], previous_response_id="resp_first", - # The follow-up itself opting out of storage must not cost it the - # history it names. store=False, request_id="resp_second", ) @@ -203,13 +183,7 @@ def test_the_stored_call_precedes_its_result_on_replay(contents, call_id): async def _preprocess(monkeypatch, store_flag): - """Run the real request_preprocess with the heavy leaves faked. - - Chat-template rendering and sampling-params construction need a tokenizer - and the engine bindings; the gates under test - fetch prior context, then - persist or not - sit above both, so those leaves are replaced with - recorders and the store itself stays real. - """ + """Run the real request_preprocess with rendering faked; the store is real.""" import tensorrt_llm.serve.responses_utils as ru store = ConversationHistoryStore() @@ -254,9 +228,6 @@ async def fake_template(**kwargs): request=request, prev_response=None, conversation_store=store, - # The server-side switch, which is all the server passes: retrieval is - # gated downstream on previous_response_id, persistence on - # request.store. enable_store=True, use_harmony=False, tokenizer=None, @@ -269,26 +240,15 @@ async def fake_template(**kwargs): return saw_prior, persisted -def test_store_false_still_sees_prior_context_and_is_not_persisted(monkeypatch): - saw_prior, persisted = asyncio.run(_preprocess(monkeypatch, store_flag=False)) - assert saw_prior, "previous_response_id was loaded and then ignored" - assert not persisted - - -def test_store_true_sees_prior_context_and_is_persisted(monkeypatch): - saw_prior, persisted = asyncio.run(_preprocess(monkeypatch, store_flag=True)) +@pytest.mark.parametrize("store_flag", [False, True]) +def test_prior_context_is_read_whether_or_not_the_turn_is_stored(monkeypatch, store_flag): + saw_prior, persisted = asyncio.run(_preprocess(monkeypatch, store_flag=store_flag)) assert saw_prior - assert persisted - + assert persisted == store_flag -def test_the_endpoint_hands_preprocess_the_server_switch(monkeypatch): - """A store=false request must not flip the flag preprocessing receives. - The endpoint used to pass ``enable_store and request.store``, which reads - as one switch but gates two: it suppressed retrieval of the prior context - along with persistence. Only the wiring is under test, so preprocessing is - replaced with a recorder that stops the handler right after the handoff. - """ +def _server_up_to_preprocess(monkeypatch): + """An OpenAIServer whose /v1/responses handler stops at preprocessing.""" import tensorrt_llm.serve.openai_server as server_module from tensorrt_llm.serve.openai_server import OpenAIServer @@ -301,6 +261,7 @@ def test_the_endpoint_hands_preprocess_the_server_switch(monkeypatch): server.model_config = None server.processor = None server.tool_parser = None + server.metrics_collector = None server.generator = SimpleNamespace( args=SimpleNamespace(reasoning_parser=None, num_postprocess_workers=0) ) @@ -318,17 +279,49 @@ async def fake_preprocess(**kwargs): raise RuntimeError("stop after the handoff") monkeypatch.setattr(server_module, "responses_api_request_preprocess", fake_preprocess) - - request = ResponsesRequest( - model="test-model", input="hi", previous_response_id="resp_prev", store=False - ) raw_request = SimpleNamespace( state=SimpleNamespace(), headers={}, url=SimpleNamespace(path="/v1/responses"), json=AsyncMock(return_value={}), ) + return server, raw_request, captured + + +def test_the_endpoint_hands_preprocess_the_server_switch(monkeypatch): + """A store=false request must not flip the flag preprocessing receives.""" + server, raw_request, captured = _server_up_to_preprocess(monkeypatch) + request = ResponsesRequest( + model="test-model", input="hi", previous_response_id="resp_prev", store=False + ) asyncio.run(server.openai_responses(request, raw_request)) assert captured.get("enable_store") is True assert captured["request"].store is False + + +def test_protected_disagg_fields_are_checked_before_anything_is_rendered(monkeypatch): + server, raw_request, captured = _server_up_to_preprocess(monkeypatch) + server._validate_internal_disagg_request = Mock(side_effect=ValueError("unsigned")) + request = ResponsesRequest( + model="test-model", + input="hi", + disaggregated_params=DisaggregatedParams(request_type="generation_only"), + ) + response = asyncio.run(server.openai_responses(request, raw_request)) + + assert response.status_code == 400 + assert captured == {} + + +@pytest.mark.parametrize( + "fields, warned", + [({}, False), ({"parallel_tool_calls": True}, False), ({"parallel_tool_calls": False}, True)], +) +def test_unenforced_parallel_tool_calls_is_logged_only_when_requested(monkeypatch, fields, warned): + server, raw_request, _ = _server_up_to_preprocess(monkeypatch) + request = ResponsesRequest(model="test-model", input="hi", **fields) + with patch("tensorrt_llm.serve.openai_server.logger") as mock_logger: + asyncio.run(server.openai_responses(request, raw_request)) + keys = [c.kwargs.get("key") for c in mock_logger.warning_once.call_args_list] + assert ("responses_parallel_tool_calls_unenforced" in keys) is warned diff --git a/tests/unittest/llmapi/apps/test_responses_stream_final_consistency.py b/tests/unittest/llmapi/apps/test_responses_stream_final_consistency.py index a3a5d883fa5a..f0d8339df79f 100644 --- a/tests/unittest/llmapi/apps/test_responses_stream_final_consistency.py +++ b/tests/unittest/llmapi/apps/test_responses_stream_final_consistency.py @@ -12,23 +12,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Offline tests: a streamed Responses turn and its final snapshot must agree. - -`response.completed` rebuilds `output` from the accumulated text in a second, -independent pass, and streaming events are numbered by whoever built them. -Three ways the two views of one generation disagreed, each measured on live -traffic before it was tested here: - -* The rebuild minted fresh ids for the reasoning and message items the stream - had already announced - 216 of 221 responses - so a client joining streamed - items with the snapshot by id saw phantom items. -* With postprocessing workers enabled a response's frames come from two - processes, each numbering from zero, so every streamed response carried - sequence numbers 0,1,0,1,2,... - 221 of 221 measured. -* A stream that ended inside an unterminated tool call streamed nothing for - it while the final rebuild kept the fragment as message text (trace - tr_8f312e9973954fa784ae7dc8f45e9d3e, 429 characters), so the snapshot held - text the stream never showed. +"""A streamed Responses turn and its final snapshot must agree. + +The terminal event's snapshot is built after the stream, so these tests drive +the real streaming processor and compare what it streamed with what its final +event reports. """ import json @@ -45,13 +33,8 @@ StreamedItem, _create_output_content, create_response_non_store, - stamp_sse_sequence_number, ) -from tensorrt_llm.serve.tool_parser.core_types import StreamingParseResult, ToolCallItem -# The CPU-* CI stages run pytest with -m 'cpu_only'. Without this marker every -# test in the file is deselected, which pytest reports as exit code 5 and the -# stage reports as a failure. pytestmark = pytest.mark.cpu_only @@ -63,23 +46,19 @@ def _event_data(frame): return json.loads(frame.split("data: ", 1)[1]) -def _exec_tool(): +def _tool(name="exec", properties=("input",)): return FunctionTool( - name="exec", + name=name, type="function", strict=False, - parameters={"type": "object", "properties": {"input": {"type": "string"}}}, + parameters={"type": "object", "properties": {p: {"type": "string"} for p in properties}}, ) -def _processor(tools=None, reasoning_parser="glm", tool_parser=None, tool_choice="auto"): - """A processor built exactly as the server builds one.""" +def _processor(tools=(), reasoning_parser="glm", tool_parser=None, tool_choice="auto"): + """A processor built as the server builds one.""" request = ResponsesRequest( - model="test-model", - input="hi", - stream=True, - tools=list(tools or []), - tool_choice=tool_choice, + model="test-model", input="hi", stream=True, tools=list(tools), tool_choice=tool_choice ) return ResponsesStreamingProcessor( request=request, @@ -91,17 +70,13 @@ def _processor(tools=None, reasoning_parser="glm", tool_parser=None, tool_choice ) -def _generation_chunk(accumulated, diff, done): - """One GenerationResult as the streaming loop sees it. - - The final chunk doubles as the finished result the completed-response - builder reads, so it carries finish_reason, token ids and prompt tokens. - """ +def _generation_chunk(accumulated, diff, done, finish_reason="stop"): + """One GenerationResult as the streaming loop sees it.""" output = SimpleNamespace( index=0, text=accumulated, text_diff=diff, - finish_reason="stop" if done else None, + finish_reason=finish_reason if done else None, token_ids=[1, 2, 3], disaggregated_params=None, ) @@ -109,11 +84,7 @@ def _generation_chunk(accumulated, diff, done): def _stream(processor, chunks): - """Drive the real processor over generation chunks. - - Returns the SSE frames it produced and the finished generation result, - which is what the server hands to get_final_response_non_store. - """ + """Drive the processor; returns its frames and the finished result.""" frames, accumulated, result = [], "", None for i, chunk in enumerate(chunks): accumulated += chunk @@ -122,767 +93,236 @@ def _stream(processor, chunks): return frames, result -def _streamed_items(frames, event_type="response.output_item.added"): +def _done_items(frames, item_type=None): + return [ + data["item"] + for data in map(_event_data, frames) + if data.get("type") == "response.output_item.done" + and item_type in (None, data["item"]["type"]) + ] + + +def _added_ids(frames): return [ (data["item"]["type"], data["item"]["id"]) for data in map(_event_data, frames) - if data.get("type") == event_type + if data.get("type") == "response.output_item.added" ] +def _message_texts(items): + return [item["content"][0]["text"] for item in items if item["type"] == "message"] + + def _final_output(processor, result): frame = processor.get_final_response_non_store(result) assert _event_type(frame) == "response.completed" return _event_data(frame)["response"]["output"] +def _snapshot(text, streamed_item_ids, reasoning_parser=None, outputs=None): + items, messages, _reasoning = _create_output_content( + SimpleNamespace(outputs=outputs or [SimpleNamespace(index=0, text=text)]), + reasoning_parser=reasoning_parser, + streamed_item_ids=streamed_item_ids, + ) + return items, messages + + # --------------------------------------------------------------------------- -# Item ids: the snapshot must answer to the ids the stream announced +# Item ids and texts # --------------------------------------------------------------------------- -def test_final_item_ids_match_the_streamed_item_ids(): - """The invariant, driven end to end through the real processor. - - Same shape and cure as the tool-call ids one commit earlier: the stream - records what it announced in `output_item.added`, the rebuild reuses it. - A client that keyed items off the streamed ids used to match nothing in - the snapshot - 216 of 221 measured responses. - """ +def test_final_items_are_the_streamed_items(): processor = _processor() - frames, result = _stream(processor, ["Check the reference.", "", "The answer is 42."]) + frames, result = _stream(processor, ["Check the reference.", "", " The answer. "]) - added = _streamed_items(frames) - done = _streamed_items(frames, "response.output_item.done") - assert added == done, "an item must close under the id it opened with" + added = _added_ids(frames) assert [item_type for item_type, _ in added] == ["reasoning", "message"] - assert all(item_id for _, item_id in added) - - final = [(item["type"], item["id"]) for item in _final_output(processor, result)] - assert final == added - - -def test_streamed_ids_are_reused_positionally_in_emission_order(): - """Two items of one type map onto the rebuild first-emitted first. - - Emission order is the only correspondence the two passes share, which is - also how the tool-call precedent matches its calls. Two outputs mean no - stream produced the record (streaming reads outputs[0] alone), so this is - the positional-reuse fallback, not the record assembly. - """ - items, _messages, _reasoning = _create_output_content( - SimpleNamespace( - outputs=[ - SimpleNamespace(index=0, text="first answer"), - SimpleNamespace(index=1, text="second answer"), - ] - ), - reasoning_parser=None, - streamed_item_ids=[ - StreamedItem("message", "msg_first"), - StreamedItem("message", "msg_second"), - ], - ) - assert [(i.type, i.id) for i in items] == [ - ("message", "msg_first"), - ("message", "msg_second"), - ] - - -def test_reasoning_and_message_ids_are_reused_per_type(): - items, _messages, _reasoning = _create_output_content( - SimpleNamespace(outputs=[SimpleNamespace(index=0, text="thinkanswer")]), - reasoning_parser="glm", - streamed_item_ids=[ - StreamedItem("reasoning", "rs_r1", "think"), - StreamedItem("message", "msg_m1", "answer"), - ], - ) - assert [(i.type, i.id) for i in items] == [ - ("reasoning", "rs_r1"), - ("message", "msg_m1"), - ] - - -def test_a_non_streaming_rebuild_still_mints_fresh_ids_silently(): - """None means no stream ran; fresh uuids are correct and nothing warns.""" - with patch("tensorrt_llm.serve.responses_utils.logger") as mock_logger: - items, _messages, _reasoning = _create_output_content( - SimpleNamespace(outputs=[SimpleNamespace(index=0, text="thinkanswer")]), - reasoning_parser="glm", - streamed_item_ids=None, - ) - assert items[0].id.startswith("rs_") - assert items[1].id.startswith("msg_") - assert not mock_logger.warning.called - - -def test_rebuilding_items_the_stream_never_opened_is_said_out_loud(): - """Overflow means the two views already diverged structurally. - - A fresh id cannot hide that and must not try to - that divergence is the - unterminated-call territory below, and papering over it silently would - bury the one signal that says the views differ. - """ - with patch("tensorrt_llm.serve.responses_utils.logger") as mock_logger: - items, _messages, _reasoning = _create_output_content( - SimpleNamespace(outputs=[SimpleNamespace(index=0, text="answer")]), - reasoning_parser=None, - streamed_item_ids=[], - ) - assert items[0].id.startswith("msg_") - assert mock_logger.warning.called - assert "message" in mock_logger.warning.call_args[0][0] - - -def test_streamed_items_the_rebuild_drops_are_said_out_loud_too(): - """The mismatch matters in both directions; the first id is still reused. - - The streamed texts here cannot account for the rebuilt content - the - second item's tail exists nowhere in the accumulated text - so this is a - genuine divergence, not a segmentation difference: the rebuild keeps its - own (complete) view, reuses what ids it can, and says so. - """ - with patch("tensorrt_llm.serve.responses_utils.logger") as mock_logger: - items, _messages, _reasoning = _create_output_content( - SimpleNamespace(outputs=[SimpleNamespace(index=0, text="answer")]), - reasoning_parser=None, - streamed_item_ids=[ - StreamedItem("message", "msg_first", "answer"), - StreamedItem("message", "msg_second", " and a tail the text never held"), - ], - ) - assert [i.id for i in items] == ["msg_first"] - assert mock_logger.warning.called + assert [(i["type"], i["id"]) for i in _done_items(frames)] == added + final = _final_output(processor, result) + assert [(item["type"], item["id"]) for item in final] == added + # Edge whitespace is part of the generation. + assert _message_texts(final) == [" The answer. "] -def test_matching_streamed_items_are_repeated_rather_than_rederived(): - """The record assembly itself, at the unit level. - When the streamed texts read the same characters the re-parse does, the - snapshot repeats the streamed items verbatim - count, boundaries, ids and - bytes - instead of re-deriving one merged item. The boundary between - "the" and " answer" exists only in the record; a re-parse of the - accumulated text has no way to find it. - """ +def test_streamed_segmentation_is_repeated_verbatim(): with patch("tensorrt_llm.serve.responses_utils.logger") as mock_logger: - items, messages, _reasoning = _create_output_content( - SimpleNamespace(outputs=[SimpleNamespace(index=0, text="the answer")]), - reasoning_parser=None, - streamed_item_ids=[ + items, messages = _snapshot( + "the answer", + [ StreamedItem("message", "msg_first", "the"), - StreamedItem("message", "msg_second", " answer"), + StreamedItem("message", "msg_2", " answer"), ], ) - assert [(i.type, i.id, i.content[0].text) for i in items] == [ - ("message", "msg_first", "the"), - ("message", "msg_second", " answer"), + assert [(i.id, i.content[0].text) for i in items] == [ + ("msg_first", "the"), + ("msg_2", " answer"), ] - # Same characters, same views: nothing to warn about. - assert not mock_logger.warning.called - # The conversation store flattens the items back into one assistant - # message, built from the streamed characters. assert messages == [{"role": "assistant", "content": "the answer"}] + assert not mock_logger.warning.called -# --------------------------------------------------------------------------- -# Sequence numbers: one monotonic counter per response, stamped at the egress -# --------------------------------------------------------------------------- - - -def test_egress_restamp_unifies_the_two_producers_counters(): - """The postproc-worker configuration, reproduced with a real pickle. - - The worker builds the per-token frames from a pickled copy of the - streaming processor - the same trip PostprocParams takes - whose counter - starts at zero, while the frontend's own counter numbered the two opening - events. The raw frames therefore restart mid-stream, and the egress - restamp is what turns them into one monotonic sequence. - """ - frontend = _processor() - worker = pickle.loads(pickle.dumps(frontend)) - - frames = frontend.get_initial_responses() - body, result = _stream(worker, ["Plan.", "", "Done."]) - frames += body - frames.append(worker.get_final_response_non_store(result)) - - produced = [_event_data(frame)["sequence_number"] for frame in frames] - # The defect stated raw: two counters, one restart inside one response. - assert produced[:3] == [0, 1, 0] - - stamped = [stamp_sse_sequence_number(frame, i) for i, frame in enumerate(frames)] - assert [_event_data(f)["sequence_number"] for f in stamped] == list(range(len(frames))) - # The restamp changes the number and nothing else. - for before, after in zip(frames, stamped): - assert _event_type(before) == _event_type(after) - b, a = _event_data(before), _event_data(after) - b.pop("sequence_number"), a.pop("sequence_number") - assert b == a - - -def test_in_process_numbering_already_matches_the_egress_stamp(): - """With workers off one instance numbers everything; the stamp is a no-op. - - This is what keeps the restamp safe to apply unconditionally at the - egress: the already-correct configuration comes out byte-equivalent. - """ - processor = _processor() - frames = processor.get_initial_responses() - body, result = _stream(processor, ["Plan.", "", "Done."]) - frames += body - frames.append(processor.get_final_response_non_store(result)) - - numbers = [_event_data(frame)["sequence_number"] for frame in frames] - assert numbers == list(range(len(frames))) - stamped = [stamp_sse_sequence_number(frame, i) for i, frame in enumerate(frames)] - assert [_event_data(f)["sequence_number"] for f in stamped] == numbers - - -def test_the_stamp_rewrites_the_field_and_not_the_model_text(): - """The payload can quote this very protocol; only the real field changes. - - A substring replace would have rewritten the model's own output here, - which is why the stamp parses the frame instead. - """ - inner = 'code: {"sequence_number": 999}' - frame = ( - "event: response.completed\ndata: " - + json.dumps( - { - "type": "response.completed", - "sequence_number": 1, - "response": {"output_text": inner}, - }, - separators=(",", ":"), - ) - + "\n\n" +def test_ids_are_reused_in_emission_order_when_the_record_does_not_cover(): + """Two outputs: streaming reads only outputs[0], so the record is not used.""" + items, _ = _snapshot( + None, + [StreamedItem("message", "msg_first"), StreamedItem("message", "msg_second")], + outputs=[SimpleNamespace(index=0, text="first"), SimpleNamespace(index=1, text="second")], ) - out = stamp_sse_sequence_number(frame, 41) - payload = _event_data(out) - assert payload["sequence_number"] == 41 - assert payload["response"]["output_text"] == inner - assert out.endswith("\n\n") and out.count("\n\n") == 1 - - -def test_frames_without_a_sequence_number_pass_through_untouched(): - for frame in ( - "event: ping\n\n", - "event: x\ndata: {}\n\n", - "event: x\ndata: not-json\n\n", - ): - assert stamp_sse_sequence_number(frame, 9) == frame - - -# --------------------------------------------------------------------------- -# An unterminated tool call: both views carry the same fallback text -# --------------------------------------------------------------------------- - -# The shape of the real 429-character fragment from trace -# tr_8f312e9973954fa784ae7dc8f45e9d3e, shortened: a GLM call whose value never -# closes, so no parser - incremental or whole-text - can read it as a call. -_UNTERMINATED = ( - "execinput" - "const cmd = await tools.exec_command({cmd: `cat metrics.json`" -) - - -def _message_texts(frames): - return [ - data["item"]["content"][0]["text"] - for data in map(_event_data, frames) - if data.get("type") == "response.output_item.done" and data["item"]["type"] == "message" + assert [(i.id, i.content[0].text) for i in items] == [ + ("msg_first", "first"), + ("msg_second", "second"), ] -def test_an_unterminated_call_reaches_both_views_as_the_same_text(): - """The stream must show the fallback text the final response keeps. - - Before the flush released it, the streamed view had no trace of the - fragment while the snapshot carried it as message text - the one turn - shape where the snapshot held text the stream never showed. Both views - now publish the same characters; the stream splits them across two - message items because its first message closed when the call was - announced, chunks before anyone could know the call would never finish. - """ - processor = _processor(tools=[_exec_tool()], reasoning_parser="glm47", tool_parser="glm47") - frames, result = _stream(processor, ["Plan it.", "Run it.", _UNTERMINATED]) - - streamed_texts = _message_texts(frames) - assert streamed_texts == ["Run it.", _UNTERMINATED] - # No half-built call on the stream: the fragment is text, not a call. - assert all(item_type != "function_call" for item_type, _ in _streamed_items(frames)) - - final = _final_output(processor, result) - # The snapshot repeats the stream's items: same count, same boundaries, - # same texts. A whole-text re-parse can only read the fallback as one - # merged message - its regex has no memory of where the first message - # closed - so the snapshot used to publish one item where the stream - # published two, and the second streamed item vanished from the record - # the client reconciles against (10 live cases a week). - assert [item["type"] for item in final] == ["reasoning", "message", "message"] - final_messages = [item for item in final if item["type"] == "message"] - assert [item["content"][0]["text"] for item in final_messages] == streamed_texts - - -def test_the_unterminated_fragment_keeps_every_streamed_id(): - """Item identity across the segmentation the fragment causes. - - The stream published two message items; the snapshot, assembled from the - stream's own record, answers to both ids - including the second, which - the old re-derive could not even represent. Nothing warns: the two views - read the same characters, and segmentation is the stream's to decide. - """ - processor = _processor(tools=[_exec_tool()], reasoning_parser="glm47", tool_parser="glm47") - frames, result = _stream(processor, ["Plan it.", "Run it.", _UNTERMINATED]) - streamed = _streamed_items(frames) - assert [item_type for item_type, _ in streamed] == [ - "reasoning", - "message", - "message", - ] - +def test_a_non_streamed_snapshot_mints_fresh_ids_and_keeps_whitespace(): with patch("tensorrt_llm.serve.responses_utils.logger") as mock_logger: - final = _final_output(processor, result) - - assert [(item["type"], item["id"]) for item in final] == streamed + items, _ = _snapshot("think Backbone ", None, reasoning_parser="glm") + assert [i.id[:3] for i in items] == ["rs_", "msg"] + assert items[1].content[0].text == " Backbone " assert not mock_logger.warning.called -# --------------------------------------------------------------------------- -# Edge whitespace: the snapshot keeps the characters the stream sent -# --------------------------------------------------------------------------- - - -def test_final_text_keeps_the_whitespace_the_stream_sent(): - """The live shape: stream ` Backbone`, snapshot `Backbone` (716/week). - - Leading and trailing whitespace is part of the generation; the stream - delivered it as deltas and the snapshot used to strip it, so the two - views disagreed about the model's own characters on roughly every turn - that started with a space token. - """ - processor = _processor() - frames, result = _stream(processor, ["Think hard.", "", " Backbone of the design. "]) - assert _message_texts(frames) == [" Backbone of the design. "] - - final = _final_output(processor, result) - messages = [item for item in final if item["type"] == "message"] - assert messages[0]["content"][0]["text"] == " Backbone of the design. " - - -def test_non_streaming_text_is_verbatim_too(): - """No stream ran, so there is no second view to match. - - The strip still falsified the generation: the whitespace was generated, - and the JSON body is the only account a non-streaming client gets. - """ - items, _messages, _reasoning = _create_output_content( - SimpleNamespace(outputs=[SimpleNamespace(index=0, text="think Backbone ")]), - reasoning_parser="glm", - streamed_item_ids=None, - ) - assert [i.type for i in items] == ["reasoning", "message"] - assert items[1].content[0].text == " Backbone " +@pytest.mark.parametrize( + "streamed", + [ + [], + [StreamedItem("message", "msg_first", "answer"), StreamedItem("message", "msg_2", "?")], + ], +) +def test_a_count_mismatch_is_logged(streamed): + with patch("tensorrt_llm.serve.responses_utils.logger") as mock_logger: + items, _ = _snapshot("answer", streamed) + assert [i.content[0].text for i in items] == ["answer"] + if streamed: + assert items[0].id == "msg_first" + assert mock_logger.warning.called # --------------------------------------------------------------------------- -# A dropped call must not donate its streamed identity to the next call +# Sequence numbers # --------------------------------------------------------------------------- -class _ScriptedToolParser: - """A parser whose verdicts this test controls. - - The identity plumbing under test is parser-agnostic; what matters is the - exact disagreement GLM-4.7 produces for an `` that never closes - (test_tool_parsers.py, "the non-streaming path is unchanged"): the - streaming pass assembles a first call whose arguments are invalid JSON - - which `_assembled_tool_calls` drops - and a second, valid one, while the - whole-text pass reports *both*, the first with empty arguments. Scripting - that shape pins the test to the disagreement itself rather than to one - parser's markup dialect. - """ - - def __init__(self): - self._buffer = "" - self._reported = False - self.current_tool_name_sent = False - - def parse_streaming_increment(self, text, tools): - if self._reported or not text: - return StreamingParseResult(normal_text=text, calls=[]) - self._reported = True - return StreamingParseResult( - normal_text="Run them. ", - calls=[ - ToolCallItem(tool_index=0, name="exec", parameters='{"cmd": }'), - ToolCallItem(tool_index=1, name="exec", parameters='{"cmd": "ls"}'), - ], - ) - - def finish(self, tools): - return StreamingParseResult() - - def has_tool_call(self, text): - return False - - def detect_and_parse(self, text, tools): - return StreamingParseResult( - normal_text="Run them. ", - calls=[ - ToolCallItem(tool_index=0, name="exec", parameters="{}"), - ToolCallItem(tool_index=1, name="exec", parameters='{"cmd": "ls"}'), - ], - ) +@pytest.mark.parametrize("postprocess_worker", [False, True]) +def test_sequence_numbers_run_contiguously_from_zero(postprocess_worker): + """A postprocessing worker gets a pickled copy before the opening pair.""" + frontend = _processor() + producer = pickle.loads(pickle.dumps(frontend)) if postprocess_worker else frontend + frames = frontend.get_initial_responses() + body, result = _stream(producer, ["Plan.", "", "Done."]) + frames += body + [producer.get_final_response_non_store(result)] -def _function_call_items(output): - return [item for item in output if item["type"] == "function_call"] - - -def test_a_dropped_call_does_not_donate_its_identity_to_the_next_one(): - """Two calls parsed, the first one's arguments invalid JSON. - - The stream delivers only the second call. The final rebuild's whole-text - parse still reports both - the first with empty arguments - and pairing - the streamed ids positionally over the *filtered* list handed the first - call the identity the stream had published for the second; the client's - tool output, keyed on that call_id, then answered a call with different - arguments. The record now holds one entry per announced call, a drop - consumes its own slot (and logs the id it retired), and the final response - both drops what the stream dropped and names the survivor what the stream - named it. - """ - processor = _processor(tools=[_exec_tool()], reasoning_parser=None, tool_parser="scripted") - with ( - patch( - "tensorrt_llm.serve.responses_utils.ToolParserFactory.create_tool_parser", - side_effect=lambda _parser_id: _ScriptedToolParser(), - ), - patch("tensorrt_llm.serve.responses_utils.logger") as mock_logger, - ): - frames, result = _stream(processor, ["Run them. "]) - streamed_calls = [ - data["item"] - for data in map(_event_data, frames) - if data.get("type") == "response.output_item.done" - and data["item"]["type"] == "function_call" - ] - # The stream refused the malformed call, and the drop named the id it - # consumed - which is the dropped call's own, not the survivor's. - assert [json.loads(c["arguments"]) for c in streamed_calls] == [{"cmd": "ls"}] - drop_warnings = [ - str(call.args[0]) - for call in mock_logger.warning.call_args_list - if "valid JSON" in str(call.args[0]) - ] - assert len(drop_warnings) == 1 - assert "call_" in drop_warnings[0] - assert streamed_calls[0]["call_id"] not in drop_warnings[0] - - final_calls = _function_call_items(_final_output(processor, result)) - - # The two views agree on membership, so no structural-divergence - # warning fires; the rebuild-side drop is said out loud instead. - warnings = [str(call.args[0]) for call in mock_logger.warning.call_args_list] - assert not any("the stream saw" in text for text in warnings) - assert any("drops the tool call" in text for text in warnings) - - # The final view matches the stream: one call, carrying the identity the - # stream published for *it* - not one shifted off the rejected call. - assert [(c["id"], c["call_id"]) for c in final_calls] == [ - (streamed_calls[0]["id"], streamed_calls[0]["call_id"]) - ] - assert json.loads(final_calls[0]["arguments"]) == {"cmd": "ls"} + assert [_event_data(frame)["sequence_number"] for frame in frames] == list(range(len(frames))) # --------------------------------------------------------------------------- -# One call entity, two parses: the snapshot repeats the streamed arguments +# Tool calls # --------------------------------------------------------------------------- -# Malformed GLM-4.7 markup, the live shape: the model emits two consecutive -# tags. The incremental state machine reads it as the key "input" -# (re-anchoring on the second tag); the whole-text pair regex swallows the -# first closing tag into the key and delivers the corrupted -# `{"inputcmd": "ls -la"}` - both valid JSON, both -# "successful" parses, different inputs for the same call identity. 2 live -# cases a week. -_DOUBLE_ARG_KEY = ( - "execinputcmd" - "ls -la" -) - +_EXEC_CALL = "execinputls" +_QWEN3_CALL = '\n{"name": "exec", "arguments": {"input": "ls"}}\n' -def _streamed_function_calls(frames): - return [ - data["item"] - for data in map(_event_data, frames) - if data.get("type") == "response.output_item.done" - and data["item"]["type"] == "function_call" - ] - -def test_final_call_arguments_are_byte_identical_to_the_streamed_ones(): - """The same call_id must not carry two different tool inputs. - - The stream already delivered - and the client possibly executed - its - version of the arguments; a snapshot that re-describes the same call_id - from its own re-parse hands the client a second, different input for one - identity (live case: stream `false`, final `{}`; another: stream a shell - script, final JSON under a corrupted key). The record the stream keeps - per call now carries the assembled arguments, and the snapshot repeats - them. - """ - tool = FunctionTool( - name="exec", - type="function", - strict=False, - parameters={ - "type": "object", - "properties": {"input": {"type": "string"}, "cmd": {"type": "string"}}, - }, - ) - processor = _processor(tools=[tool], reasoning_parser="glm47", tool_parser="glm47") - with patch("tensorrt_llm.serve.responses_utils.logger") as mock_logger: - frames, result = _stream( - processor, - ["Plan it.", "Run it. ", _DOUBLE_ARG_KEY[:40], _DOUBLE_ARG_KEY[40:]], - ) - streamed_calls = _streamed_function_calls(frames) - assert [c["arguments"] for c in streamed_calls] == ['{"input": "ls -la"}'] - - final_calls = _function_call_items(_final_output(processor, result)) - - # Byte-identical arguments and name, under the streamed identity. The - # whole-text re-parse of the same markup reads a corrupted key; it must - # reach no client. - assert [(c["id"], c["call_id"], c["name"], c["arguments"]) for c in final_calls] == [ - ( - streamed_calls[0]["id"], - streamed_calls[0]["call_id"], - streamed_calls[0]["name"], - streamed_calls[0]["arguments"], - ) - ] - # The disagreement is said out loud, naming both versions. - disagreements = [ - str(call.args[0]) - for call in mock_logger.warning.call_args_list - if "differently from the stream" in str(call.args[0]) +@pytest.mark.parametrize( + "parsers, chunks", + [ + (("glm47", "glm47"), ["Plan it.", "Run it. ", _EXEC_CALL]), + (("glm", "qwen3"), ["Plan it.", "Run it.\n", _QWEN3_CALL]), + ], + ids=["incremental", "whole_text"], +) +def test_final_tool_calls_are_the_streamed_ones(parsers, chunks): + processor = _processor(tools=[_tool()], reasoning_parser=parsers[0], tool_parser=parsers[1]) + frames, result = _stream(processor, chunks) + streamed_calls = _done_items(frames, "function_call") + assert [(c["name"], json.loads(c["arguments"])) for c in streamed_calls] == [ + ("exec", {"input": "ls"}) ] - assert len(disagreements) == 1 - assert "ls -la" in disagreements[0] - - -def test_the_streamed_message_text_keeps_its_whitespace_alongside_a_call(): - """The whitespace the glm47 whole-text parse strips survives via the record. - - The parser-internal strip only feeds the re-parse: the stream sent - `Run it. ` with its trailing space, and the snapshot repeats the stream. - """ - tool = _exec_tool() - processor = _processor(tools=[tool], reasoning_parser="glm47", tool_parser="glm47") - whole_call = "execinputls" - frames, result = _stream(processor, ["Plan it.", "Run it. ", whole_call]) - assert _message_texts(frames) == ["Run it. "] final = _final_output(processor, result) - messages = [item for item in final if item["type"] == "message"] - assert [m["content"][0]["text"] for m in messages] == ["Run it. "] - + assert [i for i in final if i["type"] == "function_call"] == streamed_calls + assert [i["type"] for i in final] == ["reasoning", "message", "function_call"] + assert _message_texts(final) == _message_texts(_done_items(frames, "message")) -def test_non_streaming_whole_text_parse_is_unchanged_by_the_record(): - """Pin: with no stream there is no record to prefer. - - The whole-text parse is the only account there is - including its - corrupted-key reading of this markup. Fixing that is a parser change, - deliberately out of scope here; this pin is what makes the streamed - fix's boundary visible. - """ - request = ResponsesRequest( - model="test-model", - input="hi", - stream=False, - tools=[ - FunctionTool( - name="exec", - type="function", - strict=False, - parameters={ - "type": "object", - "properties": { - "input": {"type": "string"}, - "cmd": {"type": "string"}, - }, - }, - ) - ], - ) - result = _generation_chunk("Plan it.Run it. " + _DOUBLE_ARG_KEY, "", True) - response = create_response_non_store( - generation_result=result, - request=request, - sampling_params=request.to_sampling_params(), - model_name="test-model", - use_harmony=False, - reasoning_parser="glm47", - tool_parser="glm47", - num_prompt_tokens=2, - ).model_dump() - calls = [i for i in response["output"] if i["type"] == "function_call"] - assert [c["arguments"] for c in calls] == ['{"inputcmd": "ls -la"}'] +def test_an_unterminated_call_is_text_in_both_views(): + unterminated = "execinputconst cmd = `cat m.json`" + processor = _processor(tools=[_tool()], reasoning_parser="glm47", tool_parser="glm47") + frames, result = _stream(processor, ["Plan it.", "Run it.", unterminated]) -# --------------------------------------------------------------------------- -# tool_choice="none": no call anywhere, the markup stays as text everywhere -# --------------------------------------------------------------------------- + assert _done_items(frames, "function_call") == [] + streamed = _done_items(frames) + assert "".join(_message_texts(streamed)) == "Run it." + unterminated -_WHOLE_CALL = "execinputls" + with patch("tensorrt_llm.serve.responses_utils.logger") as mock_logger: + final = _final_output(processor, result) + assert [(i["type"], i["id"]) for i in final] == [(i["type"], i["id"]) for i in streamed] + assert _message_texts(final) == _message_texts(streamed) + assert not mock_logger.warning.called def test_tool_choice_none_keeps_the_markup_as_text_in_both_views(): - """tool_choice="none" is enforced by not parsing, which loses nothing. - - The model's markup stays in the visible text verbatim - stream and - snapshot alike, or the two views would disagree about whether a call - exists - and neither view reports a tool call item. The other tool_choice - values stay unenforced (honouring them needs sampling-level constraints); - "none" is the one value where enforcement is lossless. - """ processor = _processor( - tools=[_exec_tool()], - reasoning_parser="glm47", - tool_parser="glm47", - tool_choice="none", + tools=[_tool()], reasoning_parser="glm47", tool_parser="glm47", tool_choice="none" ) - frames, result = _stream(processor, ["Plan it.", "Calling: ", _WHOLE_CALL]) - - assert all(item_type != "function_call" for item_type, _ in _streamed_items(frames)) - streamed_text = "".join(_message_texts(frames)) - assert streamed_text == "Calling: " + _WHOLE_CALL + frames, result = _stream(processor, ["Plan it.", "Calling: ", _EXEC_CALL]) + assert _message_texts(_done_items(frames)) == ["Calling: " + _EXEC_CALL] final = _final_output(processor, result) assert [item["type"] for item in final] == ["reasoning", "message"] - assert final[1]["content"][0]["text"] == streamed_text + assert _message_texts(final) == ["Calling: " + _EXEC_CALL] # --------------------------------------------------------------------------- -# A truncated generation must end in a terminal that agrees with itself +# Terminal event and status # --------------------------------------------------------------------------- -def test_a_max_token_truncated_stream_ends_with_a_consistent_terminal(): - """Status, terminal event and incomplete_details must tell one story. - - finish_reason="length" maps to status "incomplete", and the terminal - event used to say `response.completed` around it - an event whose name - asserts the one thing its payload denies, so a client believed the - truncated answer was the whole one. The spec-correct pairing: - `response.incomplete`, status "incomplete", and - incomplete_details.reason naming the token budget. - """ +@pytest.mark.parametrize( + "finish_reason, event_type, status, details", + [ + ("stop", "response.completed", "completed", None), + ("length", "response.incomplete", "incomplete", {"reason": "max_output_tokens"}), + ("timeout", "response.failed", "failed", None), + ("cancelled", "response.completed", "cancelled", None), + ], +) +def test_the_terminal_event_matches_the_status(finish_reason, event_type, status, details): processor = _processor(reasoning_parser=None) - result = _generation_chunk("Partial answer", "Partial answer", True) - result.outputs[0].finish_reason = "length" + result = _generation_chunk("Partial answer", "Partial answer", True, finish_reason) frames = processor.process_single_output(result) - assert all(_event_data(f).get("type") != "response.completed" for f in frames) + assert all(_event_type(f) != "response.completed" for f in frames) frame = processor.get_final_response_non_store(result) - assert _event_type(frame) == "response.incomplete" payload = _event_data(frame) - assert payload["type"] == "response.incomplete" - assert payload["response"]["status"] == "incomplete" - assert payload["response"]["incomplete_details"] == {"reason": "max_output_tokens"} - # The truncated text itself still ships in the snapshot. - messages = [i for i in payload["response"]["output"] if i["type"] == "message"] - assert messages and messages[0]["content"][0]["text"] == "Partial answer" - - -def test_a_completed_stream_still_ends_with_response_completed(): - """The happy path is untouched: stop means completed, and no details.""" - processor = _processor(reasoning_parser=None) - _frames, result = _stream(processor, ["Done."]) - - frame = processor.get_final_response_non_store(result) - assert _event_type(frame) == "response.completed" - assert _event_data(frame)["response"]["incomplete_details"] is None - - -def _non_streaming_response(finish_reason): - """The JSON body a non-streaming request gets, for one finish reason.""" + assert (_event_type(frame), payload["type"]) == (event_type, event_type) + assert payload["response"]["status"] == status + assert payload["response"]["incomplete_details"] == details + assert _message_texts(payload["response"]["output"]) == ["Partial answer"] + + +@pytest.mark.parametrize( + "finish_reason, status, details", + [ + ("stop", "completed", None), + ("length", "incomplete", {"reason": "max_output_tokens"}), + # A disaggregated context worker handing off is not a token budget. + ("not_finished", "incomplete", None), + ], +) +def test_a_non_streamed_response_explains_an_incomplete_status(finish_reason, status, details): request = ResponsesRequest(model="test-model", input="hi", stream=False) - result = _generation_chunk("Partial answer", "Partial answer", True) - result.outputs[0].finish_reason = finish_reason - return create_response_non_store( - generation_result=result, + response = create_response_non_store( + generation_result=_generation_chunk("Partial", "Partial", True, finish_reason), request=request, sampling_params=request.to_sampling_params(), model_name="test-model", use_harmony=False, num_prompt_tokens=2, - ) - - -def test_a_non_streaming_truncated_response_carries_incomplete_details(): - """The JSON body must explain its status the way the stream already does. - - The streaming terminal event has paired status "incomplete" with - incomplete_details.reason="max_output_tokens" since the response.incomplete - fix, but the non-streaming body said only "incomplete" - the field was - commented out of ResponsesResponse - so a client polling the JSON body had - the status without the reason the spec promises alongside it. - """ - payload = _non_streaming_response("length").model_dump() - assert payload["status"] == "incomplete" - assert payload["incomplete_details"] == {"reason": "max_output_tokens"} - - -def test_a_non_streaming_completed_response_has_no_details(): - payload = _non_streaming_response("stop").model_dump() - assert payload["status"] == "completed" - assert payload["incomplete_details"] is None - - -def test_a_disagg_handoff_gets_no_invented_reason(): - """A "not_finished" incomplete explains nothing, on purpose. - - finish_reason "not_finished" also maps to "incomplete", but it is a - context worker handing the request off, not a token budget; naming a - public reason for an internal handoff would be a guess presented as fact. - """ - payload = _non_streaming_response("not_finished").model_dump() - assert payload["status"] == "incomplete" - assert payload["incomplete_details"] is None - - -def test_a_failed_generation_terminates_with_response_failed(): - """A failed response must not ship under response.completed. - - A timeout maps to status "failed"; an event named response.completed - around it asserts the one thing its payload denies. The spec pairs the - status with response.failed. - """ - processor = _processor(reasoning_parser=None) - result = _generation_chunk("Partial answer", "Partial answer", True) - result.outputs[0].finish_reason = "timeout" - frame = processor.get_final_response_non_store(result) - assert _event_type(frame) == "response.failed" - payload = _event_data(frame) - assert payload["type"] == "response.failed" - assert payload["response"]["status"] == "failed" - - -def test_a_cancelled_generation_still_ends_with_response_completed(): - """Deliberate, not an oversight. - - Cancellation is client-initiated and the Responses spec pairs no terminal - event with it, so inventing one would hand SDK clients an event type they - cannot parse. See _stream_terminal_event. - """ - processor = _processor(reasoning_parser=None) - result = _generation_chunk("Partial answer", "Partial answer", True) - result.outputs[0].finish_reason = "cancelled" - frame = processor.get_final_response_non_store(result) - assert _event_type(frame) == "response.completed" - assert _event_data(frame)["response"]["status"] == "cancelled" + ).model_dump() + assert (response["status"], response["incomplete_details"]) == (status, details) diff --git a/tests/unittest/llmapi/apps/test_responses_streaming_events.py b/tests/unittest/llmapi/apps/test_responses_streaming_events.py index 8edab616ddfa..3df5ff27f779 100644 --- a/tests/unittest/llmapi/apps/test_responses_streaming_events.py +++ b/tests/unittest/llmapi/apps/test_responses_streaming_events.py @@ -25,15 +25,19 @@ import json from types import SimpleNamespace +import aiohttp import pytest +from openai.types.responses.tool import FunctionTool from tensorrt_llm.executor import EngineDeadError, RequestError from tensorrt_llm.serve.openai_protocol import ResponsesRequest from tensorrt_llm.serve.responses_utils import ( ResponsesStreamingEventsHelper, ResponsesStreamingProcessor, + _create_output_content, _generate_streaming_event, classify_stream_termination, + describe_stream_termination, guard_responses_stream, stream_error_event, ) @@ -138,99 +142,6 @@ def test_done_events_carry_the_open_item_id(): assert helper.get_content_part_done_event(_output_text("hi")).item_id == item_id -# --------------------------------------------------------------------------- -# Reasoning that shares a chunk with the answer -# --------------------------------------------------------------------------- - - -class _FakeOutput: - """The attributes the streaming path reads from a generation output. - - `text` is the whole generation so far and `text_diff` only the newest - chunk: the delta path parses the diff, while the done-event path re-parses - the accumulated text. Supplying only the diff makes the second one raise, - which is how the first version of this fake was wrong. - """ - - def __init__(self, text, text_diff, index=0): - self.text = text - self.text_diff = text_diff - self.index = index - - -class _FakeRequest: - tools = None - - -def _stream(chunks): - """Drive the real dispatch over a sequence of generation chunks. - - Returns (reasoning deltas, text deltas) as the client would receive them. - """ - helper = ResponsesStreamingEventsHelper() - parsers = {} - reasoning, text = [], [] - accumulated = "" - for i, chunk in enumerate(chunks): - accumulated += chunk - events = _generate_streaming_event( - output=_FakeOutput(accumulated, chunk), - request=_FakeRequest(), - finished_generation=(i == len(chunks) - 1), - streaming_events_helper=helper, - reasoning_parser_id="glm", - reasoning_parser_dict=parsers, - ) - for event in events: - kind = getattr(event, "type", "") - if kind == "response.reasoning_text.delta": - reasoning.append(event.delta) - elif kind == "response.output_text.delta": - text.append(event.delta) - return "".join(reasoning), "".join(text) - - -def test_reasoning_sharing_a_chunk_with_the_answer_is_not_dropped(): - """Regression: the reasoning half of the straddling chunk was discarded. - - A generation chunk whose raw text spans the closing think tag parses into - both a reasoning part and a content part. The dispatch chose its branch on - the content part alone, so the reasoning part fell into a branch that never - ran and was never emitted -- and closing the item cleared the flag the - done-event path is guarded on, so the done event carried the short text - too. Measured at 56% of reasoning items across four fleets. - """ - reasoning, text = _stream(["Let", " me check the reference.I will read the file."]) - assert reasoning == "Let me check the reference." - assert text == "I will read the file." - - -def test_reasoning_shorter_than_one_chunk_survives(): - """The worst case: the whole reasoning shares a chunk with the answer. - - Reasoning short enough to fit inside a single chunk was truncated to - nothing at all, which is why the losses looked like first words. - """ - reasoning, _ = _stream(["Sure.Here it is."]) - assert reasoning == "Sure." - - -def test_reasoning_ending_on_a_chunk_boundary_is_unaffected(): - """The case that always worked, kept so a fix cannot regress it.""" - reasoning, text = _stream( - ["Let", " me check the reference.", "", "I will read the file."] - ) - assert reasoning == "Let me check the reference." - assert text == "I will read the file." - - -def test_reasoning_with_no_answer_after_it_is_unaffected(): - """A turn that ends in a tool call emits no text, so the other branch ran.""" - reasoning, text = _stream(["Let", " me check the reference."]) - assert reasoning == "Let me check the reference." - assert text == "" - - def _output_text(text): from openai.types.responses import ResponseOutputText @@ -238,90 +149,136 @@ def _output_text(text): # --------------------------------------------------------------------------- -# Every content part that is opened must close, in OpenAI's order -# -# A part is opened with response.content_part.added and has to end with -# response.content_part.done before its item's response.output_item.done. -# The message close has always done this; the reasoning close yielded -# reasoning_text.done and output_item.done but never closed the part it had -# opened - one more content_part.added than content_part.done on every turn -# with reasoning, ~294k unpaired events a week - so a client tracking part -# state held a reasoning part that never ended. +# Reasoning and answer deltas # --------------------------------------------------------------------------- -def _event_stream(chunks): +def _event_stream(chunks, reasoning_parser="glm"): """All events the real dispatch emits for a sequence of chunks.""" helper = ResponsesStreamingEventsHelper() + request = SimpleNamespace(tools=None, tool_choice="auto") parsers = {} events, accumulated = [], "" for i, chunk in enumerate(chunks): accumulated += chunk events.extend( _generate_streaming_event( - output=_FakeOutput(accumulated, chunk), - request=_FakeRequest(), - finished_generation=(i == len(chunks) - 1), + output=SimpleNamespace(index=0, text=accumulated, text_diff=chunk), + request=request, + finished_generation=i == len(chunks) - 1, streaming_events_helper=helper, - reasoning_parser_id="glm", + reasoning_parser_id=reasoning_parser, reasoning_parser_dict=parsers, ) ) return events -def _assert_reasoning_part_closes(events, reasoning_text): - kinds = [getattr(e, "type", "") for e in events] - assert kinds.count("response.content_part.added") == kinds.count( - "response.content_part.done" - ), "every part opened must close" +def _deltas(events, event_type): + return "".join(e.delta for e in events if e.type == event_type) - # The reasoning item's own closing sequence, in OpenAI's order: - # reasoning_text.done, then content_part.done, then output_item.done. - done_index = kinds.index("response.reasoning_text.done") - assert kinds[done_index + 1] == "response.content_part.done" - assert kinds[done_index + 2] == "response.output_item.done" - part_done = events[done_index + 1] - assert part_done.part.type == "reasoning_text" - assert part_done.part.text == reasoning_text - # The part closes under the id of the item it was opened in. - reasoning_done = events[done_index + 2] - assert reasoning_done.item.type == "reasoning" - assert part_done.item_id == reasoning_done.item.id +@pytest.mark.parametrize( + "chunks, reasoning, answer", + [ + (["Let", " me check.I will read it."], "Let me check.", "I will read it."), + (["Sure.Here it is."], "Sure.", "Here it is."), + (["Let", " me check.", "", "I will read it."], "Let me check.", "I will read it."), + (["Let", " me check."], "Let me check.", ""), + ], +) +def test_reasoning_and_answer_deltas_survive_any_chunk_boundary(chunks, reasoning, answer): + events = _event_stream(chunks) + assert _deltas(events, "response.reasoning_text.delta") == reasoning + assert _deltas(events, "response.output_text.delta") == answer -def test_a_reasoning_part_closed_by_the_answer_gets_content_part_done(): - """The transition close (_close_open_item): reasoning ends, text begins.""" - events = _event_stream(["Plan the fix.", "", "Done."]) - _assert_reasoning_part_closes(events, "Plan the fix.") +def test_an_item_is_opened_before_a_whitespace_only_first_delta(): + events = _event_stream(["Thinking. ", "hi"]) + types = [e.type for e in events] + message_added = next( + i + for i, e in enumerate(events) + if e.type == "response.output_item.added" and e.item.type == "message" + ) + assert message_added < types.index("response.output_text.delta") + assert _deltas(events, "response.output_text.delta") == " hi" -def test_a_reasoning_part_closed_by_end_of_generation_gets_content_part_done(): - """The other closing branch: generation ends inside the reasoning item.""" - events = _event_stream(["All reasoning, no answer."]) - _assert_reasoning_part_closes(events, "All reasoning, no answer.") +@pytest.mark.parametrize( + "chunks, reasoning", + [ + (["Plan the fix.", "", "Done."], "Plan the fix."), + (["All reasoning, no answer."], "All reasoning, no answer."), + ], +) +def test_every_content_part_closes_in_order(chunks, reasoning): + """content_part.done comes between the text done event and item done.""" + events = _event_stream(chunks) + types = [e.type for e in events] + assert types.count("response.content_part.added") == types.count("response.content_part.done") + + for done_type, part_type, text in ( + ("response.reasoning_text.done", "reasoning_text", reasoning), + ("response.output_text.done", "output_text", "Done."), + ): + if done_type not in types: + continue + index = types.index(done_type) + assert types[index + 1 : index + 3] == [ + "response.content_part.done", + "response.output_item.done", + ] + part_done, item_done = events[index + 1], events[index + 2] + assert (part_done.part.type, part_done.part.text) == (part_type, text) + assert part_done.item_id == item_done.item.id + + +# --------------------------------------------------------------------------- +# Snapshot item order +# --------------------------------------------------------------------------- + +_THINK = "Check the reference first." +_ANSWER = "The answer is 42." +_TOOL_CALL = '\n{"name": "read_file", "arguments": {"path": "a.txt"}}\n' + + +def _read_file_tool(): + return FunctionTool( + name="read_file", + description="Read a file.", + parameters={"type": "object", "properties": {"path": {"type": "string"}}}, + strict=False, + type="function", + ) -def test_message_part_pairing_is_unchanged(): - """The branch that was always right, pinned against regression.""" - events = _event_stream(["Plan.", "", "Done."]) - kinds = [getattr(e, "type", "") for e in events] - text_done = kinds.index("response.output_text.done") - assert kinds[text_done + 1] == "response.content_part.done" - assert kinds[text_done + 2] == "response.output_item.done" - assert events[text_done + 1].part.type == "output_text" - assert events[text_done + 1].part.text == "Done." +@pytest.mark.parametrize( + "text, kinds", + [ + (f"{_THINK}{_ANSWER}", ["reasoning", "message"]), + ( + f"{_THINK}Reading it now.\n{_TOOL_CALL}", + ["reasoning", "message", "function_call"], + ), + (f"{_THINK}{_TOOL_CALL}", ["reasoning", "function_call"]), + (_ANSWER, ["message"]), + ], +) +def test_snapshot_lists_reasoning_then_message_then_calls(text, kinds): + items, _messages, _reasoning = _create_output_content( + SimpleNamespace(outputs=[SimpleNamespace(index=0, text=text)]), + reasoning_parser="qwen3", + tool_parser="qwen3", + tools=[_read_file_tool()], + ) + assert [item.type for item in items] == kinds + if "reasoning" in kinds: + assert items[0].content[0].text == _THINK # --------------------------------------------------------------------------- -# Streams that stop before response.completed -# -# response.completed is the only event that repeats the full text, so a stream -# cut before it leaves everything the turn produced living solely in deltas. -# Until these events existed such a stream simply stopped: no error event, no -# populated response.error, nothing in the bytes or the trace to tell a reader -# the stream was cut rather than still running. +# Streams that stop before their terminal event # --------------------------------------------------------------------------- @@ -338,7 +295,6 @@ def _event_data(frame): def _processor(): - """A processor built exactly as the server builds one.""" request = ResponsesRequest(model="test-model", input="hi", stream=True) return ResponsesStreamingProcessor( request=request, @@ -349,49 +305,54 @@ def _processor(): def _forbidden(cause, detail, events_sent): - raise AssertionError(f"terminal events were built for a stream that did not need them: {cause}") + raise AssertionError(f"terminal events built for a stream that did not need them: {cause}") -async def _drive(source, terminal_events, on_termination=None): - """Run a stream through the guard, returning (frames, raised exception).""" +async def _drive(frames, terminal_events, exc=None): + """Run frames, then optionally a raised exception, through the guard.""" + + async def source(): + for frame in frames: + yield frame + if exc is not None: + raise exc + seen = [] try: - async for chunk in guard_responses_stream(source, terminal_events, on_termination): + async for chunk in guard_responses_stream(source(), terminal_events): seen.append(chunk) - except BaseException as exc: # noqa: BLE001 - the test asserts on it - return seen, exc + except BaseException as raised: # noqa: BLE001 - the test asserts on it + return seen, raised return seen, None +_COMPLETED = [ + "event: response.created\ndata: {}\n\n", + "event: response.output_text.delta\ndata: {}\n\n", + "event: response.completed\ndata: {}\n\n", +] + + @pytest.mark.asyncio async def test_a_completed_stream_is_forwarded_unchanged(): - """The happy path must stay byte-identical: same frames, nothing appended.""" - frames = [ - "event: response.created\ndata: {}\n\n", - "event: response.output_text.delta\ndata: {}\n\n", - "event: response.completed\ndata: {}\n\n", - ] + assert await _drive(_COMPLETED, _forbidden) == (_COMPLETED, None) - async def source(): - for frame in frames: - yield frame - seen, raised = await _drive(source(), _forbidden) - assert seen == frames - assert raised is None +@pytest.mark.asyncio +async def test_a_failure_after_the_terminal_event_adds_nothing(): + seen, raised = await _drive(_COMPLETED, _forbidden, RuntimeError("reset")) + assert seen == _COMPLETED + assert isinstance(raised, RuntimeError) @pytest.mark.asyncio -async def test_an_exception_mid_stream_ends_the_stream_with_a_terminal_event(): +async def test_a_failure_mid_stream_ends_with_error_and_response_failed(): processor = _processor() opening = processor.get_initial_responses() - async def source(): - for frame in opening: - yield frame - raise RuntimeError("event construction blew up") - - seen, raised = await _drive(source(), processor.get_stream_failed_events) + seen, raised = await _drive( + opening, processor.get_stream_failed_events, ValueError("item had no id") + ) assert [_event_type(frame) for frame in seen] == [ "response.created", @@ -399,563 +360,142 @@ async def source(): "error", "response.failed", ] - # Re-raised, not swallowed: the server still logs the fault and the trace - # still records the response as an error rather than a clean finish. - assert isinstance(raised, RuntimeError) + assert [_event_data(frame)["sequence_number"] for frame in seen] == [0, 1, 2, 3] + error = _event_data(seen[2]) + assert (error["code"], error["message"]) == ("internal_error", "ValueError") + assert isinstance(raised, ValueError) @pytest.mark.asyncio -async def test_the_terminal_event_carries_the_cause(): - processor = _processor() - - async def source(): - for frame in processor.get_initial_responses(): - yield frame - raise ValueError("item had no id") - - seen, _ = await _drive(source(), processor.get_stream_failed_events) - error = _event_data(seen[-2]) - assert error["type"] == "error" - assert error["code"] == "internal_error" - assert error["message"] == "ValueError: item had no id" +async def test_a_completion_marker_inside_a_payload_does_not_end_the_watch(): + delta = json.dumps({"delta": "\n\nevent: response.completed\n"}) + frames = [f"event: response.output_text.delta\ndata: {delta}\n\n"] + seen, raised = await _drive(frames, stream_error_event, RuntimeError("cut")) -@pytest.mark.parametrize( - "exc, cause", - [ - (RequestError("request failed"), "engine_error"), - (EngineDeadError(), "engine_error"), - (ValueError("bad item"), "internal_error"), - (asyncio.CancelledError(), "client_disconnect"), - (GeneratorExit(), "client_disconnect"), - ], -) -def test_causes_the_code_can_tell_apart(exc, cause): - """An engine failure, a fault in this process, and a client hangup differ. - - Only the middle one leaves the accumulated text in memory, which is what - makes the distinction worth recording rather than lumping into "error". - """ - assert classify_stream_termination(exc) == cause - - -def test_an_upstream_transport_error_is_named_as_one(): - aiohttp = pytest.importorskip("aiohttp") - assert classify_stream_termination(aiohttp.ClientPayloadError("cut")) == "upstream_error" + assert [_event_type(frame) for frame in seen] == ["response.output_text.delta", "error"] + assert isinstance(raised, RuntimeError) @pytest.mark.asyncio -async def test_a_client_hangup_adds_nothing_to_the_stream(): - """Nobody is left to send to, and yielding here would raise. - - An async generator that yields while GeneratorExit is propagating dies - with "async generator ignored GeneratorExit", trading a diagnosable - truncation for an undiagnosable one. The cause is reported out of band. - """ - noted = [] - - async def source(): - yield "event: response.created\ndata: {}\n\n" - raise asyncio.CancelledError - - seen, raised = await _drive( - source(), _forbidden, lambda cause, detail: noted.append((cause, detail)) - ) - assert len(seen) == 1 +async def test_a_client_hangup_adds_nothing(): + seen, raised = await _drive(_COMPLETED[:1], _forbidden, asyncio.CancelledError()) + assert seen == _COMPLETED[:1] assert isinstance(raised, asyncio.CancelledError) - assert noted == [("client_disconnect", "CancelledError")] @pytest.mark.asyncio async def test_a_broken_reporter_does_not_replace_the_original_fault(): - """Reporting a failure must not become a second, more confusing one.""" - def broken(cause, detail, events_sent): raise KeyError("snapshot field missing") - async def source(): - yield "event: response.created\ndata: {}\n\n" - raise ValueError("the fault worth seeing") - - seen, raised = await _drive(source(), broken) - assert len(seen) == 1 + seen, raised = await _drive(_COMPLETED[:1], broken, ValueError("the real fault")) + assert seen == _COMPLETED[:1] assert isinstance(raised, ValueError) - assert str(raised) == "the fault worth seeing" - - -@pytest.mark.asyncio -async def test_a_failure_after_completion_adds_nothing(): - """Regression guard for the happy path. - - aiohttp raises at end of stream when the connector closes the connection, - which arrives after the last frame. Appending an error event there would - corrupt a stream that completed perfectly well. - """ - - async def source(): - yield "event: response.completed\ndata: {}\n\n" - raise RuntimeError("connection reset at end of stream") - - seen, raised = await _drive(source(), _forbidden) - assert len(seen) == 1 - assert isinstance(raised, RuntimeError) - - -def test_terminal_events_continue_the_sequence_numbering(): - processor = _processor() - opening = processor.get_initial_responses() - assert [_event_data(frame)["sequence_number"] for frame in opening] == [0, 1] - events = processor.get_stream_failed_events("internal_error", "ValueError: x") - assert [_event_data(frame)["sequence_number"] for frame in events] == [2, 3] +@pytest.mark.parametrize( + "exc, cause, detail", + [ + (RequestError("request failed"), "engine_error", "RequestError: request failed"), + (EngineDeadError(), "engine_error", "EngineDeadError: Engine has died"), + (ValueError("at 10.0.0.7:8001"), "internal_error", "ValueError"), + (aiohttp.ClientPayloadError("from 10.0.0.7:8001"), "upstream_error", "ClientPayloadError"), + ], +) +def test_termination_cause_and_client_facing_detail(exc, cause, detail): + assert classify_stream_termination(exc) == cause + assert describe_stream_termination(exc, cause) == detail -def test_numbering_follows_events_this_processor_did_not_number(): - """With postprocessing workers the per-token events are numbered elsewhere. - A pickled copy of this processor builds them in another process, so the - local counter only ever saw the opening two and would put the terminal - events back at the start of a sequence the client has already passed. - """ +@pytest.mark.parametrize("events_sent", [2, 610]) +def test_terminal_events_are_numbered_after_the_frames_sent(events_sent): processor = _processor() - processor.get_initial_responses() - - events = processor.get_stream_failed_events("internal_error", "ValueError: x", 610) - assert [_event_data(frame)["sequence_number"] for frame in events] == [610, 611] + events = processor.get_stream_failed_events("internal_error", "ValueError", events_sent) + assert [_event_data(frame)["sequence_number"] for frame in events] == [ + events_sent, + events_sent + 1, + ] def test_the_failed_snapshot_says_failed_and_carries_no_content(): - """response.failed, not response.incomplete. - - incomplete_details.reason is a closed enum -- max_output_tokens, - max_messages, content_filter, steered -- with no member for a server fault - or a client hangup, so response.incomplete could only be sent with - reason=null, which is indistinguishable from an ordinary truncation. - """ processor = _processor() - processor.get_initial_responses() - _, failed = processor.get_stream_failed_events("internal_error", "ValueError: x") + _, failed = processor.get_stream_failed_events("internal_error", "ValueError: x", 2) assert _event_type(failed) == "response.failed" response = _event_data(failed)["response"] assert response["status"] == "failed" - assert response["error"]["code"] == "server_error" - assert response["error"]["message"] == "ValueError: x" - # No reconstruction: the lost text stays lost, this only says so. + assert (response["error"]["code"], response["error"]["message"]) == ( + "server_error", + "ValueError: x", + ) assert response["output"] == [] assert response["id"] == processor.request.request_id @pytest.mark.asyncio -async def test_the_relayed_terminal_event_is_numbered_from_the_frames_forwarded(): - """The orchestrator forwards bytes and never sees the events it carries. - - It can still count them, including when a transport read splits the blank - line that ends one -- without the carry every sequence number after such a - split is one short. - """ - - async def source(): - yield b"event: response.created\ndata: {}\n\n" - yield b"event: response.in_progress\ndata: {}\n" - yield b"\nevent: response.output_text.delta\ndata: {}\n\n" - raise RuntimeError("upstream cut") - - seen, raised = await _drive(source(), stream_error_event) +async def test_a_relay_numbers_its_error_event_across_split_delimiters(): + frames = [ + b"event: response.created\ndata: {}\n\n", + b"event: response.in_progress\ndata: {}\n", + b"\nevent: response.output_text.delta\ndata: {}\n\n", + ] + seen, raised = await _drive(frames, stream_error_event, RuntimeError("upstream cut")) assert isinstance(raised, RuntimeError) - terminal = seen[-1] - assert isinstance(terminal, bytes) - assert _event_type(terminal) == "error" - assert _event_data(terminal)["sequence_number"] == 3 + assert _event_type(seen[-1]) == "error" + assert _event_data(seen[-1])["sequence_number"] == 3 @pytest.mark.asyncio -async def test_a_relay_that_never_started_reports_sequence_zero(): - """The failures that record a response with an entirely empty body. - - Nothing was forwarded, so the error event is the stream's first and the - sequence number is exactly right rather than a guess. - """ +async def test_a_relay_cut_mid_frame_terminates_that_frame_first(): + frames = [ + b"event: response.created\ndata: {}\n\n", + b"event: response.output_text.delta\ndata: {", + ] + seen, _ = await _drive(frames, stream_error_event, RuntimeError("upstream cut")) + assert seen[2] == b"\n\n" + assert _event_type(seen[3]) == "error" + assert _event_data(seen[3])["sequence_number"] == 1 - async def source(): - raise RuntimeError("no context worker available") - yield b"" # pragma: no cover - makes this an async generator - seen, raised = await _drive(source(), stream_error_event) +@pytest.mark.asyncio +async def test_a_relay_that_never_started_reports_sequence_zero(): + seen, raised = await _drive([], stream_error_event, RuntimeError("no context worker")) assert isinstance(raised, RuntimeError) assert len(seen) == 1 assert _event_data(seen[0])["sequence_number"] == 0 - assert _event_data(seen[0])["code"] == "internal_error" -# --------------------------------------------------------------------------- -# Order of the items in the completed-response snapshot -# -# These cover the non-streaming assembly in `_create_output_content`, but they -# live here because what they assert is a property of the stream: the snapshot -# in `response.output` has to list the items in the order the stream emitted -# them. A client that reads both must not be told two different stories about -# one generation. This is also the file the CPU CI stage runs (l0_cpu.yml). -# --------------------------------------------------------------------------- - -_THINK = "Check the reference first." -_ANSWER = "The answer is 42." -_TOOL_CALL = '\n{"name": "read_file", "arguments": {"path": "a.txt"}}\n' - - -def _tools(): - """A real tool definition, so the tool parser accepts the parsed call.""" - from openai.types.responses.tool import FunctionTool - - return [ - FunctionTool( - name="read_file", - description="Read a file.", - parameters={"type": "object", "properties": {"path": {"type": "string"}}}, - strict=False, - type="function", - ) - ] - - -def _snapshot_kinds(text, tools=None): - """Item types of the completed-response snapshot for one generation. - - Drives the real `_create_output_content` with real parsers - `qwen3` for - both, so text with no `` is an ordinary answer rather than an - unterminated reasoning block. - """ - from tensorrt_llm.serve.responses_utils import _create_output_content - - items, _messages, _reasoning_texts = _create_output_content( - _FakeRequestOutput(text), - reasoning_parser="qwen3", - tool_parser="qwen3" if tools else None, - tools=tools, +def test_streamed_responses_use_wire_field_names(): + request = ResponsesRequest.model_validate( + { + "model": "m", + "input": "hi", + "stream": True, + "text": { + "format": {"type": "json_schema", "name": "out", "schema": {"type": "object"}} + }, + } ) - return items, [item.type for item in items] - - -class _FakeRequestOutput: - """The one attribute `_create_output_content` reads off a result. - - It iterates `final_res.outputs` and takes `.index` and `.text` from each; - everything else it needs it derives from the parsers. - """ - - def __init__(self, text): - self.outputs = [_FakeOutput(text, text)] - - -def test_snapshot_lists_reasoning_before_the_message(): - """Regression: the snapshot appended the message first, inverting cause and effect. - - The stream emits reasoning then message, but the snapshot for the same - generation listed message then reasoning - so a client replaying - `response.output` read the answer before the reasoning that produced it. - Measured on 99.99% of responses carrying both items across four fleets. - """ - items, kinds = _snapshot_kinds(f"{_THINK}{_ANSWER}") - assert kinds == ["reasoning", "message"] - # Reordering must move the items, not relabel them. - assert items[0].content[0].text == _THINK - assert items[1].content[0].text == _ANSWER - - -def test_snapshot_lists_reasoning_then_message_then_tool_call(): - items, kinds = _snapshot_kinds( - f"{_THINK}Reading it now.\n{_TOOL_CALL}", tools=_tools() - ) - assert kinds == ["reasoning", "message", "function_call"] - assert items[2].name == "read_file" - - -def test_snapshot_lists_reasoning_before_a_tool_call_with_no_message(): - """A turn that ends in a tool call emits no message item at all.""" - _items, kinds = _snapshot_kinds(f"{_THINK}{_TOOL_CALL}", tools=_tools()) - assert kinds == ["reasoning", "function_call"] - - -def test_snapshot_of_a_plain_answer_is_just_the_message(): - """No reasoning to order: the message must not be dropped or displaced.""" - items, kinds = _snapshot_kinds(_ANSWER) - assert kinds == ["message"] - assert items[0].content[0].text == _ANSWER - - -def test_snapshot_order_matches_the_streamed_order(): - """The two paths must agree; this is the invariant the bug violated. - - Both sides are driven for real - `_generate_streaming_event` for the - stream and `_create_output_content` for the snapshot - over the same - generation and the same reasoning parser, rather than comparing the - snapshot against a hardcoded expectation of what the stream does. - """ - from tensorrt_llm.serve.responses_utils import ( - ResponsesStreamingEventsHelper, - _create_output_content, - ) - - # "glm" is reasoning_at_start, so the generation opens inside the - # reasoning block with no tag - the same setup as the streaming - # tests above, and what the thinking chat templates actually render. - chunks = [_THINK, "", _ANSWER] - - helper = ResponsesStreamingEventsHelper() - parsers = {} - accumulated = "" - streamed = [] - for i, chunk in enumerate(chunks): - accumulated += chunk - for event in _generate_streaming_event( - output=_FakeOutput(accumulated, chunk), - request=_FakeRequest(), - finished_generation=(i == len(chunks) - 1), - streaming_events_helper=helper, - reasoning_parser_id="glm", - reasoning_parser_dict=parsers, - ): - if getattr(event, "type", "") == "response.output_item.done": - streamed.append(event.item.type) - - snapshot_items, _messages, _reasoning_texts = _create_output_content( - _FakeRequestOutput(accumulated), reasoning_parser="glm" + processor = ResponsesStreamingProcessor( + request=request, + sampling_params=request.to_sampling_params(), + model_name="m", + use_harmony=False, ) - - assert streamed == ["reasoning", "message"] - assert [item.type for item in snapshot_items] == streamed - - -def _finished_generation(text): - """A result complete enough to assemble a whole final response from. - - `_create_output_content` reads only index and text, which is all - `_FakeRequestOutput` carries; building the response around it also reads - `finish_reason`, and usage reads `token_ids` and `prompt_token_ids`. - """ output = SimpleNamespace( index=0, - text=text, - text_diff=text, + text="{}", + text_diff="{}", finish_reason="stop", - token_ids=[1, 2, 3], + token_ids=[1], disaggregated_params=None, ) - return SimpleNamespace(outputs=[output], prompt_token_ids=[1, 2], cached_tokens=0) - - -def test_final_response_is_assembled_without_touching_a_missing_attribute(): - """Regression: the final streaming event raised AttributeError. - - `get_final_response_non_store` reached for `self.emitted_tool_call_ids` - on the processor. That list lives on the events helper's state tracker, - not on the processor, so every streaming Responses request died building - its last event. The client never saw a server error -- the chunked body - simply stopped, surfacing as `TransferEncodingError: Not enough data to - satisfy transfer length header` one hop upstream, which reads like a - network fault. Measured on a live instance: 286 of 287 requests. - - `py_compile` passes on that bug, and so does an md5 comparison against the - file it came from. Only executing the call catches it, which is what this - test does: it asks for the final event and requires a real one back. - """ - processor = _processor() - frame = processor.get_final_response_non_store(_finished_generation("hello")) - - assert "response.completed" in frame - payload = _event_data(frame) - assert payload["type"] == "response.completed" - assert payload["response"]["status"] == "completed" - - -def test_final_response_reuses_the_call_ids_the_stream_published(): - """The ids a client matches its tool outputs against must not be reminted. - - `response.completed` rebuilds `output` from the generation, and - `_tool_call_output_item` mints a fresh random id every time it runs, so - the snapshot used to disagree with the stream on every call -- 1224 of - 1224 measured before the fix. A client keying tool results off the - streamed `call_id` then matches nothing in the final response. - """ - processor = _processor() - helper = processor.streaming_events_helper - helper.record_emitted_tool_call(SimpleNamespace(id="fc_fixed", call_id="call_fixed")) - - assert [(r.item_id, r.call_id) for r in helper.emitted_tool_call_ids] == [ - ("fc_fixed", "call_fixed") - ] - # And the processor reaches it by the path the final response uses. - assert [ - (r.item_id, r.call_id) for r in processor.streaming_events_helper.emitted_tool_call_ids - ] == [("fc_fixed", "call_fixed")] - - -def _processor_with_tools(): - """A processor that can actually produce a tool call. - - `_processor()` declares no tools and sets no parsers, so nothing it builds - ever contains one -- which is the case the next test needs. - """ - request = ResponsesRequest(model="test-model", input="hi", stream=True, tools=_tools()) - return ResponsesStreamingProcessor( - request=request, - sampling_params=request.to_sampling_params(), - model_name="test-model", - use_harmony=False, - reasoning_parser="glm", - tool_parser="qwen3", - ) - - -def test_final_response_reuses_a_streamed_call_id_end_to_end(): - """The invariant the fix exists for, driven through the real final event. - - This is the test that distinguishes the fix from the obvious-looking wrong - one. `AttributeError: no attribute 'emitted_tool_call_ids'` invites adding - the attribute to the processor -- the name would resolve and the crash - would stop. But `record_emitted_tool_call` appends to the state tracker's - list, so a new list on the processor stays empty forever, the final - response mints fresh ids, and the call_id mismatch is silently back. - - The crash tests pass in that world. This one does not. - """ - processor = _processor_with_tools() - processor.streaming_events_helper.record_emitted_tool_call( - SimpleNamespace(id="fc_streamed", call_id="call_streamed") - ) - - frame = processor.get_final_response_non_store( - _finished_generation(f"{_THINK}{_TOOL_CALL}") - ) - calls = [ - i - for i in _event_data(frame)["response"]["output"] - if i["type"] in ("function_call", "custom_tool_call") - ] - - assert calls, "the final response should carry the tool call" - assert calls[0]["call_id"] == "call_streamed" - assert calls[0]["id"] == "fc_streamed" - - -def test_nothing_streamed_means_fresh_ids_rather_than_a_crash(): - """The other half: an empty list is a legal state, not an error. - - A non-streaming request, or a stream cut off before any tool call, has - nothing to reuse. That must produce usable fresh ids -- keeping this - separate is what makes the test above read as "ids were not reused" - instead of "ids were missing". - """ - processor = _processor_with_tools() - frame = processor.get_final_response_non_store( - _finished_generation(f"{_THINK}{_TOOL_CALL}") - ) - calls = [ - i - for i in _event_data(frame)["response"]["output"] - if i["type"] in ("function_call", "custom_tool_call") - ] - - assert calls - assert calls[0]["call_id"].startswith("call_") - assert calls[0]["call_id"] != "call_streamed" - - -# --------------------------------------------------------------------------- -# Non-streaming rebuild: no phantom {}-argument calls -# --------------------------------------------------------------------------- -# -# The whole-text parse reports arguments `{}` both for a genuine zero-argument -# call and for a call whose argument markup its pair regex could not read - a -# GLM-4.7 block that opens and never closes it. The streamed -# assembly of the same text refuses the second (its arguments assemble to -# `{"cmd": }`), so a non-streaming request used to deliver a call the model -# never made while a streaming request of the same text delivered none. -# _verify_empty_calls_against_streaming reconciles the two when no stream ran. - - -def _glm47_tools(): - from openai.types.responses.tool import FunctionTool - - return [ - FunctionTool( - name="exec_command", - description="Run a command.", - parameters={"type": "object", "properties": {"cmd": {"type": "string"}}}, - strict=False, - type="function", - ), - FunctionTool( - name="get_time", description="Now.", parameters=None, strict=False, type="function" - ), - ] - - -def _glm47_snapshot(text, streamed_tool_call_ids=None): - from tensorrt_llm.serve.responses_utils import _create_output_content - - items, _messages, _reasoning_texts = _create_output_content( - _FakeRequestOutput(text), - reasoning_parser=None, - tool_parser="glm47", - tools=_glm47_tools(), - streamed_tool_call_ids=streamed_tool_call_ids, - ) - calls = [(i.name, i.arguments) for i in items if i.type == "function_call"] - texts = [i.content[0].text for i in items if i.type == "message"] - return calls, texts - - -_UNREADABLE_ARGS = ( - "exec_commandcmdls -la" -) - - -def test_non_streaming_unreadable_arg_markup_is_text_not_an_empty_call(): - """The auditor's shape: terminated call, never closed. - - The pair regex reads nothing out of the block, so the call used to ship - with arguments `{}` - and the client ran the tool with arguments the model - never wrote. The streamed view refuses the call; the whole-text view now - agrees, and the markup falls back into the message text so nothing the - model generated is lost. - """ - calls, texts = _glm47_snapshot("Let me check. " + _UNREADABLE_ARGS) - assert calls == [] - assert texts and "ls -la" in texts[0] - assert texts[0].startswith("Let me check.") - - -def test_non_streaming_unterminated_markup_stays_text(): - """Pin: markup with no closing tag already fell back to text on both views.""" - unterminated = "exec_commandcmdls -la" - calls, texts = _glm47_snapshot(unterminated) - assert calls == [] - assert texts == [unterminated] - - -def test_non_streaming_zero_argument_call_is_still_delivered(): - """Pin: `{}` from a genuinely argument-free call is the model's own call.""" - calls, texts = _glm47_snapshot("get_time") - assert calls == [("get_time", "{}")] - assert texts == [] - - -def test_non_streaming_real_call_survives_a_phantom_neighbor(): - good = ( - "exec_commandcmdpytest -q" - ) - calls, _texts = _glm47_snapshot(good + _UNREADABLE_ARGS) - assert calls == [("exec_command", '{"cmd": "pytest -q"}')] - - -def test_a_streamed_rebuild_is_untouched_by_the_reconciler(): - """A streamed request keeps its entity-pairing semantics. - - When a stream ran, the per-entity id record already settles the drop, and - the snapshot must keep describing the stream - which showed neither the - call nor its markup. - """ - calls, texts = _glm47_snapshot(_UNREADABLE_ARGS, streamed_tool_call_ids=[None]) - assert calls == [] - assert texts == [] + result = SimpleNamespace(outputs=[output], _done=True, prompt_token_ids=[1], cached_tokens=0) + frames = processor.get_initial_responses() + processor.process_single_output(result) + frames.append(processor.get_final_response_non_store(result)) + + for frame in (frames[0], frames[-1]): + text_format = _event_data(frame)["response"]["text"]["format"] + assert text_format["schema"] == {"type": "object"} + assert "schema_" not in text_format diff --git a/tests/unittest/llmapi/apps/test_responses_streaming_tool_calls.py b/tests/unittest/llmapi/apps/test_responses_streaming_tool_calls.py index ef7b9771b2b0..1ee074888e2c 100644 --- a/tests/unittest/llmapi/apps/test_responses_streaming_tool_calls.py +++ b/tests/unittest/llmapi/apps/test_responses_streaming_tool_calls.py @@ -12,21 +12,16 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +"""Unit tests for Responses API streaming tool call emission. -"""Unit tests for Responses API streaming tool call emission (TRTLLM-9605). - -These drive `_generate_streaming_event` with the real GLM-4.7 reasoning and -tool parsers rather than with mocked ones. The defect these tests exist for - -tool-call markup published as the assistant's message - lived entirely in the -disagreement between what the incremental parser had streamed and what a -whole-text re-parse concluded afterwards, so a test that mocks the parsers -cannot see it. +These drive `_generate_streaming_event` with real reasoning and tool parsers: +what they check is the agreement between the incremental and the whole-text +parse, which mocked parsers cannot exhibit. """ import json import random from types import SimpleNamespace -from unittest.mock import patch import pytest from openai.types.responses.tool import FunctionTool @@ -35,33 +30,20 @@ ResponsesStreamingEventsHelper, _accumulate_tool_call_fragments, _assembled_tool_calls, + _create_output_content, _generate_streaming_event, + _get_chat_completion_function_tools, ) from tensorrt_llm.serve.tool_parser.core_types import ToolCallItem +from tensorrt_llm.serve.tool_parser.tool_parser_factory import ToolParserFactory -# The CPU-* CI stages run pytest with -m 'cpu_only'. Without this marker every -# test in the file is deselected, which pytest reports as exit code 5 and the -# stage reports as a failure. pytestmark = pytest.mark.cpu_only -_REASONING_PARSER = "glm47" -_TOOL_PARSER = "glm47" - -# Markup that must never reach a client as assistant text - with one -# sanctioned exception: a stream that ends inside an unterminated call -# releases the raw markup as message text, because the final response's -# whole-text re-parse keeps it as message text too and the two views must -# agree (see _flush_tool_parser and TestEndOfStream below). _MARKUP = ("", "", "", "") -# One real GLM-4.7 response, verbatim, from a recorded agent run: eight stream -# frames carrying reasoning, a sentence of prose, and two tool calls. Feeding -# frames 0-6 used to publish 296 characters of `normal_text` beginning with the -# prose and continuing straight into `functions.exec...`, -# cut off mid-word at `yield`; the whole response yields 116 clean characters -# and two calls. The frame boundaries matter - they are what put one closed -# call and one open call in the accumulated text at the same time - so they are -# kept exactly as recorded. +# A recorded GLM-4.7 response in its original stream frames: reasoning, one +# sentence, then two calls to a namespaced tool. The frame boundaries put one +# closed and one open call in the parser's buffer at the same time. _REAL_FRAMES = [ "Let", ( @@ -102,80 +84,60 @@ "_time_ms: 5000});\ntext(out.output);\n", ] -# Everything the frames above put between `` and the first -# ``: the only visible assistant text in that response. +# The only visible assistant text in that response. _REAL_TEXT = ( "I'll start by reading the problem definition and understanding " "the schema, then dive into the kernel implementation." ) +_GET_TIME_CALL = "get_timecity{}" -def _make_output(text: str, text_diff: str, index: int = 0): - """A stand-in for one RequestOutput as the streaming loop sees it.""" - return SimpleNamespace(index=index, text=text, text_diff=text_diff) +def _get_time_call(city): + return _GET_TIME_CALL.format(city) -def _namespace_tool(namespace: str = "functions", name: str = "exec"): - """A namespaced tool, offered to the model as `.`. - This is the shape the recorded response was produced with, and the reason - its calls are named `functions.exec`. - """ +def _namespace_tool(namespace="functions", name="exec"): + """A namespaced tool, offered to the model as `.`.""" inner = SimpleNamespace( name=name, type="function", description="run a command", - parameters={ - "type": "object", - "properties": {"input": {"type": "string"}}, - }, + parameters={"type": "object", "properties": {"input": {"type": "string"}}}, ) return SimpleNamespace(name=namespace, type="namespace", description="tools", tools=[inner]) -def _function_tool(name: str = "get_time"): +def _function_tool(name="get_time", properties=("city",)): return FunctionTool( name=name, type="function", strict=False, - parameters={ - "type": "object", - "properties": {"city": {"type": "string"}}, - }, + parameters={"type": "object", "properties": {p: {"type": "string"} for p in properties}}, ) -def _make_request(tools=None): - """A minimal ResponsesRequest-like object. - - SimpleNamespace rather than a nested class: a class body does not close - over the enclosing function's locals (unlike a nested function), so - ``tools = tools or []`` inside one raises NameError on the right-hand name. - """ - return SimpleNamespace(tools=list(tools) if tools else []) - - def _drive( frames, tools=None, finish=True, + tool_parser_id="glm47", + reasoning_parser_id="glm47", helper=None, - tool_parser_id=_TOOL_PARSER, - reasoning_parser_id=_REASONING_PARSER, + parsers=None, ): - """Feed `frames` through the streaming event generator, one chunk each. - - Returns the flat list of events, which is what a client would receive. - """ + """Feed `frames` through the streaming event generator, one chunk each.""" helper = helper or ResponsesStreamingEventsHelper() - request = _make_request(tools if tools is not None else [_namespace_tool()]) - reasoning_parser_dict, tool_parser_dict = {}, {} + request = SimpleNamespace( + tools=list(tools) if tools is not None else [_namespace_tool()], tool_choice="auto" + ) + reasoning_parser_dict, tool_parser_dict = parsers or ({}, {}) events, accumulated = [], "" for i, frame in enumerate(frames): accumulated += frame events.extend( _generate_streaming_event( - output=_make_output(accumulated, frame), + output=SimpleNamespace(index=0, text=accumulated, text_diff=frame), request=request, finished_generation=finish and i == len(frames) - 1, streaming_events_helper=helper, @@ -188,476 +150,213 @@ def _drive( return events -def _of_type(events, event_type): - return [e for e in events if getattr(e, "type", None) == event_type] - - def _payloads(events, event_type): - return [e.text for e in _of_type(events, event_type)] + return [e.text for e in events if e.type == event_type] -def _done_payloads(events): - """Every terminal text payload a client would display, text or reasoning.""" - return _payloads(events, "response.output_text.done") + _payloads( - events, "response.reasoning_text.done" - ) +def _texts(events): + return _payloads(events, "response.output_text.done") def _items(events, event_type, item_type): - return [ - e.item for e in _of_type(events, event_type) if getattr(e.item, "type", None) == item_type - ] + return [e.item for e in events if e.type == event_type and e.item.type == item_type] def _call_items(events): return _items(events, "response.output_item.done", "function_call") -def _assert_no_markup(events): - for payload in _done_payloads(events): - for marker in _MARKUP: - assert marker not in payload, ( - f"{marker!r} reached the client as assistant text: {payload!r}" - ) +def _first_index(events, event_type, item_type): + return next( + i for i, e in enumerate(events) if e.type == event_type and e.item.type == item_type + ) -class TestTheDefect: - """Edge case 1: two or more calls in one response.""" +def _assert_no_markup(payloads): + for payload in payloads: + for marker in _MARKUP: + assert marker not in payload, f"{marker!r} reached the client as text: {payload!r}" - def test_text_item_closes_at_the_first_call_with_no_markup(self): - """The whole recorded response, replayed frame by frame. - Exactly one message item, holding only the prose that preceded the - first call, and both calls recovered. Before the fix this same replay - produced a 296-character message item running from the prose straight - into `functions.execinput`. - """ +class TestRecordedResponse: + def test_one_message_item_then_both_calls(self): events = _drive(_REAL_FRAMES) - _assert_no_markup(events) - assert _payloads(events, "response.output_text.done") == [_REAL_TEXT] - + assert _texts(events) == [_REAL_TEXT] + assert _first_index(events, "response.output_item.done", "message") < _first_index( + events, "response.output_item.added", "function_call" + ) calls = _call_items(events) - assert [c.name for c in calls] == ["exec", "exec"] - assert [c.namespace for c in calls] == ["functions", "functions"] - for call in calls: - # Arguments a client cannot parse are arguments it cannot run. - assert json.loads(call.arguments)["input"] - - def test_the_prose_is_one_item_not_two(self): - """The chunk carrying both the prose and the call start is one turn. - - The delta has to be appended to the open item before the call closes - it. If the close ran first, the prose in that chunk would land in a - second message item - the split happens mid-sentence, and a client - rendering only the last item shows only the fragment. - """ - events = _drive(_REAL_FRAMES) - assert len(_items(events, "response.output_item.done", "message")) == 1 - - def test_no_prefix_of_the_response_leaks_markup(self): - """Feeding frames 0..k for every k must never *leak* markup. - - The failing window was narrow - it opened once one call had closed - while another was still open - so the whole prefix family is replayed - rather than just the end state. + assert [(c.name, c.namespace) for c in calls] == [("exec", "functions")] * 2 + assert all(json.loads(c.arguments)["input"] for c in calls) - One shape is a release, not a leak: a prefix that finishes inside an - unterminated call flushes the raw remainder as its trailing message - item, because the final response's whole-text re-parse keeps that - text too and the two views of one generation must agree (see - _flush_tool_parser). That remainder is pinned exactly - the bytes - from the last unclosed `` to the end - and everything - else still has to be clean; an unfinished stream (finish=False) - releases nothing at all. - """ - - def pending_markup(accumulated): - start = accumulated.rfind("") - if start == -1 or "" in accumulated[start:]: - return None - return accumulated[start:] - - for k in range(1, len(_REAL_FRAMES) + 1): - for finish in (False, True): - events = _drive(_REAL_FRAMES[:k], finish=finish) - released = pending_markup("".join(_REAL_FRAMES[:k])) if finish else None - for payload in _done_payloads(events): - if released is not None and payload == released: - continue - for marker in _MARKUP: - assert marker not in payload, ( - f"k={k} finish={finish}: {marker!r} reached the " - f"client as assistant text: {payload!r}" - ) - - def test_done_payload_is_exactly_the_deltas_that_were_streamed(self): - """The invariant, stated directly, for every prefix. - - Every enumerated case is a consequence of this one property, and it - catches the regressions the enumerated cases miss: once a done payload - is defined as the sum of the deltas already sent, text that was never - streamed cannot appear in it. - """ + @pytest.mark.parametrize("finish", [False, True]) + def test_every_prefix_streams_done_payloads_equal_to_their_deltas(self, finish): + """Only a stream finishing inside a call may release markup, as text.""" delta_of = { "response.output_text.done": "response.output_text.delta", "response.reasoning_text.done": "response.reasoning_text.delta", } for k in range(1, len(_REAL_FRAMES) + 1): - for finish in (False, True): - events = _drive(_REAL_FRAMES[:k], finish=finish) - streamed: dict[tuple[str, str], str] = {} - for event in events: - event_type = getattr(event, "type", None) - if event_type in delta_of.values(): - key = (event.item_id, event_type) - streamed[key] = streamed.get(key, "") + event.delta - elif event_type in delta_of: - key = (event.item_id, delta_of[event_type]) - assert event.text == streamed.get(key, ""), ( - f"k={k} finish={finish}: done payload differs from " - f"the deltas sent for {event.item_id}" - ) - - def test_any_chunking_of_the_response_gives_the_same_result(self): - """The recorded frame boundaries are one chunking out of many. - - Whether a call arrives whole in one chunk or spread over ten is a - property of the decoder's timing, not of the response, so every - chunking has to produce the same text and the same two calls. Replayed - against the previous implementation this fails on 1742 of 2000 - randomised chunkings; the boundaries are seeded so a failure is - reproducible. - """ + accumulated = "".join(_REAL_FRAMES[:k]) + events = _drive(_REAL_FRAMES[:k], finish=finish) + streamed = {} + for event in events: + if event.type in delta_of.values(): + key = (event.item_id, event.type) + streamed[key] = streamed.get(key, "") + event.delta + elif event.type in delta_of: + assert event.text == streamed.get((event.item_id, delta_of[event.type]), "") + + start = accumulated.rfind("") + texts = _texts(events) + if finish and start != -1 and "" not in accumulated[start:]: + assert texts[-1].endswith(accumulated[start:]) + texts[-1] = texts[-1][: -len(accumulated[start:])] + _assert_no_markup(texts) + + def test_any_chunking_gives_the_same_result(self): full_text = "".join(_REAL_FRAMES) rng = random.Random(20260918) - for _ in range(30): cuts = sorted(rng.sample(range(1, len(full_text)), rng.randint(1, 40))) frames = [full_text[a:b] for a, b in zip([0] + cuts, cuts + [len(full_text)])] events = _drive(frames) - - _assert_no_markup(events) - assert _payloads(events, "response.output_text.done") == [_REAL_TEXT] + assert _texts(events) == [_REAL_TEXT] assert [c.name for c in _call_items(events)] == ["exec", "exec"] - def test_two_whole_calls_in_one_chunk_are_both_reported(self): - """A parser reports at most one finished call per increment. - - Glm47ToolParser returns as soon as it sees a `` and keeps - the rest buffered for the next increment. When the stream ends on that - chunk there is no next increment, so without draining the parser at - end of stream the second call is silently lost. - """ - events = _drive( - [ - "Thinking.", - "get_timecity" - "Beijing" - "get_timecity" - "Paris", - ], - tools=[_function_tool()], - ) + def test_a_stream_cut_off_mid_call_releases_the_markup_as_text(self): + raw = "".join(_REAL_FRAMES[:3]) + events = _drive(_REAL_FRAMES[:3]) - calls = _call_items(events) - assert [c.name for c in calls] == ["get_time", "get_time"] - assert [json.loads(c.arguments)["city"] for c in calls] == ["Beijing", "Paris"] + assert _texts(events) == [_REAL_TEXT + raw[raw.index("") :]] + assert _call_items(events) == [] + def test_a_stream_cut_off_after_one_call_keeps_that_call(self): + events = _drive(_REAL_FRAMES[:7]) + assert [c.name for c in _call_items(events)] == ["exec"] -class TestTransitions: - """Edge cases 2, 3 and 4: what closes, and what opens, around a call.""" + def test_a_finished_output_seen_twice_emits_its_calls_once(self): + helper, parsers = ResponsesStreamingEventsHelper(), ({}, {}) + assert len(_call_items(_drive(_REAL_FRAMES, helper=helper, parsers=parsers))) == 2 + assert _call_items(_drive([""], helper=helper, parsers=parsers)) == [] - def test_call_straight_after_reasoning_emits_no_message_item(self): - """Edge case 2/3: `` closes the reasoning item. - There is no text item to close, and none may be invented: an empty - message item is a turn the model never took. - """ +class TestTransitions: + def test_a_call_straight_after_reasoning_closes_the_reasoning_item(self): events = _drive( - [ - "Deciding what to do.", - "get_timecity" - "Beijing", - ], - tools=[_function_tool()], + ["Deciding what to do.", _get_time_call("Beijing")], tools=[_function_tool()] ) - assert _items(events, "response.output_item.done", "message") == [] assert _payloads(events, "response.reasoning_text.done") == ["Deciding what to do."] - assert [c.name for c in _call_items(events)] == ["get_time"] - - def test_reasoning_item_closes_before_the_call_item_is_added(self): - """Edge case 3, on ordering: no call is nested inside another item. - - A client that is still inside a reasoning item when a function_call - item arrives attributes the call to the reasoning. - """ - events = _drive( - [ - "Deciding what to do.", - "get_timecity" - "Beijing", - ], - tools=[_function_tool()], - ) - - types = [getattr(e, "type", None) for e in events] - reasoning_done = types.index("response.reasoning_text.done") - call_added = next( - i - for i, e in enumerate(events) - if getattr(e, "type", None) == "response.output_item.added" - and getattr(e.item, "type", None) == "function_call" - ) - assert reasoning_done < call_added - - def test_text_after_a_call_opens_a_new_message_item(self): - """Edge case 4, constructed - no recorded response has text after a call. - - All 35 multi-call records have nothing between `` and the - next ``, and nothing after the last one, so this shape only - exists here. - """ - events = _drive( - [ - "Thinking.Before. ", - "get_timecity" - "Beijing", - "After.", - ], - tools=[_function_tool()], + assert _first_index(events, "response.output_item.done", "reasoning") < _first_index( + events, "response.output_item.added", "function_call" ) + assert [c.name for c in _call_items(events)] == ["get_time"] - _assert_no_markup(events) - assert _payloads(events, "response.output_text.done") == ["Before. ", "After."] - - def test_text_in_the_same_chunk_as_the_closing_call_tag(self): - """Edge case 4 again, with the call and the trailing text in one chunk. - - The parser keeps everything after `` in its buffer and only - returns it on the next increment, so a stream that ends on that chunk - relies on the end-of-stream flush to release it. Without the flush the - sentence is silently dropped. - """ - events = _drive( - [ - "Thinking.Before. ", - "get_timecity" - "BeijingAfter.", - ], - tools=[_function_tool()], - ) + @pytest.mark.parametrize( + "frames", + [ + ["Thinking.Before. ", _get_time_call("Beijing"), "After."], + ["Thinking.Before. ", _get_time_call("Beijing") + "After."], + ], + ) + def test_text_after_a_call_opens_a_new_message_item(self, frames): + events = _drive(frames, tools=[_function_tool()]) + assert _texts(events) == ["Before. ", "After."] + assert len(_call_items(events)) == 1 - _assert_no_markup(events) - assert _payloads(events, "response.output_text.done") == ["Before. ", "After."] - - def test_whitespace_only_delta_between_calls_opens_a_message_item(self): - r"""Edge case 5, recording what happens rather than what should. - - SGLang suppresses a whitespace-only message between calls because - qwen3-coder separates its calls with `\n`. GLM does not, so no GLM - traffic produces this, and suppressing it here would contradict the - rule directly above it in _generate_streaming_event: a whitespace-only - *first* delta must still open an item, or Codex CLI reports - "OutputTextDelta without active item" and prints nothing at all. - Reconciling the two needs its own decision, so this test pins the - current behaviour and will fail loudly if someone changes it by - accident. - """ + def test_two_whole_calls_in_the_last_chunk_are_both_reported(self): events = _drive( - [ - "Thinking.", - "get_timecity" - "Beijing", - "\n", - "get_timecity" - "Paris", - ], + ["Thinking.", _get_time_call("Beijing") + _get_time_call("Paris")], tools=[_function_tool()], ) - - _assert_no_markup(events) - assert _payloads(events, "response.output_text.done") == ["\n"] - assert len(_call_items(events)) == 2 - - -class TestEndOfStream: - """Edge cases 6 and 7: streams that stop before the model was done.""" - - def test_stream_cut_off_mid_call_releases_the_markup_as_text(self): - """Edge case 6: the parser is still holding an unterminated call. - - Dropping those bytes was the first design, but the final response - never dropped them: its whole-text re-parse needs the closing tag, - fails to read the fragment as a call, and falls back to publishing it - as message text - so the snapshot carried a 429-character message the - stream never showed (trace tr_8f312e9973954fa784ae7dc8f45e9d3e). The - views have to agree, and of the two ways to agree, keeping the text - wins: these turns are recorded as training data, where silent loss is - worse than a call that visibly never closed. So the flush releases - the raw buffer as a trailing message item. The text already streamed - is unaffected, and the item carrying the fragment is separate because - the message item closed when the call was announced, chunks earlier. - """ - raw = "".join(_REAL_FRAMES[:3]) - unterminated = raw[raw.index("") :] - - with patch("tensorrt_llm.serve.responses_utils.logger") as mock_logger: - events = _drive(_REAL_FRAMES[:3]) - - assert _payloads(events, "response.output_text.done") == [ - _REAL_TEXT, - unterminated, + assert [json.loads(c.arguments)["city"] for c in _call_items(events)] == [ + "Beijing", + "Paris", ] - # The release has to be visible in the log, not silent: the model - # emitted a call it never finished, and someone reading the recorded - # turn should find that said somewhere. - assert any("tool call" in call.args[0] for call in mock_logger.warning.call_args_list) - - def test_stream_cut_off_mid_call_reports_no_half_built_call(self): - """A call whose arguments never closed is not a call a client can run. - - Its accumulated arguments are a JSON prefix. The non-streaming - endpoint reports no call for the same text, and the two must agree. - """ - events = _drive(_REAL_FRAMES[:3]) - assert _call_items(events) == [] - - def test_stream_cut_off_after_one_call_keeps_that_call(self): - """Dropping the unfinished call must not drop the finished ones. - Frames 0-6 close the first call and open the second, so exactly one - call survives. - """ - events = _drive(_REAL_FRAMES[:7]) - assert [c.name for c in _call_items(events)] == ["exec"] - assert json.loads(_call_items(events)[0].arguments)["input"] - - def test_abort_closes_the_open_item(self): - """Edge case 7: an aborted stream still finalises what it had. - - An abort surfaces as a final chunk carrying no new text, and an item - left open has no terminal state - Codex CLI echoes it back on the next - turn without a `status`, which the next request is rejected for. - """ + def test_an_abort_closes_the_open_item(self): events = _drive(["Thinking.Partial answer", ""]) - - assert _payloads(events, "response.output_text.done") == ["Partial answer"] + assert _texts(events) == ["Partial answer"] assert len(_items(events, "response.output_item.done", "message")) == 1 - -class TestUnchangedPaths: - """Edge cases 10 and 11, and the regressions the structure already fixed.""" - - def test_no_tool_parser_leaves_the_text_alone(self): - """Edge case 10: with no tool parser, nothing is a call. - - The markup is ordinary text on this path, and it still has to come - back as exactly what was streamed. - """ + def test_no_tool_parser_leaves_the_markup_as_text(self): events = _drive(["Thinking.Answer x"], tool_parser_id=None) - - assert _payloads(events, "response.output_text.done") == ["Answer x"] + assert _texts(events) == ["Answer x"] assert _call_items(events) == [] - def test_empty_delta_opens_nothing(self): - """Edge case 11.""" + def test_an_empty_delta_opens_nothing(self): assert _drive([""], finish=False) == [] - def test_reasoning_and_text_in_one_chunk_still_emits_the_reasoning(self): - """Regression: the chunk that spans `` carries both halves. - - The reasoning part has to be flushed in the delta branch or it is - never emitted; measured at 56% of reasoning items across four fleets, - and 100% of reasoning short enough to fit one chunk. - """ - events = _drive(["I should answer.Answer."]) - - assert _payloads(events, "response.reasoning_text.done") == ["I should answer."] - assert _payloads(events, "response.output_text.done") == ["Answer."] - - def test_item_is_opened_before_a_whitespace_only_first_delta(self): - """Regression: a delta with no item open makes a client drop the turn. - - Codex CLI reports "OutputTextDelta without active item" and prints - nothing. Short replies hit it, because a leading whitespace token is - more likely to be the whole first delta. - """ - events = _drive(["Thinking. ", "hi"]) - - types = [getattr(e, "type", None) for e in events] - first_delta = types.index("response.output_text.delta") - message_added = next( - i - for i, e in enumerate(events) - if getattr(e, "type", None) == "response.output_item.added" - and getattr(e.item, "type", None) == "message" - ) - assert message_added < first_delta - assert _payloads(events, "response.output_text.done") == [" hi"] - def test_message_item_is_closed_before_the_call_item_is_added(self): - """Regression: a call must never be nested inside a message item.""" - events = _drive(_REAL_FRAMES) +_CALLS_BEGIN, _CALLS_END = "<｜tool▁calls▁begin｜>", "<｜tool▁calls▁end｜>" +_CALL_BEGIN, _CALL_END, _SEP = "<｜tool▁call▁begin｜>", "<｜tool▁call▁end｜>", "<｜tool▁sep｜>" +_NYC, _AI = json.dumps({"location": "NYC"}), json.dumps({"query": "AI"}) - message_done = next( - i - for i, e in enumerate(events) - if getattr(e, "type", None) == "response.output_item.done" - and getattr(e.item, "type", None) == "message" - ) - call_added = next( - i - for i, e in enumerate(events) - if getattr(e, "type", None) == "response.output_item.added" - and getattr(e.item, "type", None) == "function_call" - ) - assert message_done < call_added - - def test_calls_are_emitted_once_if_the_finished_output_arrives_twice(self): - """Regression: double emission must stay impossible. +# Parsers whose increments are not guaranteed to match their whole-text parse +# take their calls from one whole-text parse at the end of the stream. +_WHOLE_TEXT_PARSER_OUTPUTS = { + "qwen3": ( + 'Checking.\n\n{"name": "get_weather", "arguments": {"location": "NYC"}}\n' + '\n\n{"name": "search_web", "arguments": {"query": "AI"}}\n' + "" + ), + "qwen3_coder": "Checking.\n\n\n\n", + "deepseek_v3": ( + f"Checking.{_CALLS_BEGIN}{_CALL_BEGIN}function{_SEP}get_weather\n```json\n{_NYC}\n```" + f"{_CALL_END}{_CALL_BEGIN}function{_SEP}search_web\n```json\n{_AI}\n```{_CALL_END}" + f"{_CALLS_END}" + ), + "deepseek_v31": ( + f"Checking.{_CALLS_BEGIN}{_CALL_BEGIN}get_weather{_SEP}{_NYC}{_CALL_END}" + f"{_CALL_BEGIN}search_web{_SEP}{_AI}{_CALL_END}{_CALLS_END}" + ), + "minimax_m2": ( + 'Checking.' + 'NYCAI' + "" + ), + "gemma4": 'Checking.<|tool_call>call:get_weather{location:<|"|>NYC<|"|>}', +} - `emitted_tool_calls` existed because the old per-chunk re-parse - re-reported the same calls on every chunk. Emission is driven by - accumulated fragments now, so the counter guards a different thing - - the emission block running for more than one chunk of the same - output - but it still has to guard it. - """ - helper = ResponsesStreamingEventsHelper() - request = _make_request([_namespace_tool()]) - reasoning_parser_dict, tool_parser_dict = {}, {} - - def step(text_diff, accumulated, finished): - return list( - _generate_streaming_event( - output=_make_output(accumulated, text_diff), - request=request, - finished_generation=finished, - streaming_events_helper=helper, - reasoning_parser_id=_REASONING_PARSER, - tool_parser_id=_TOOL_PARSER, - reasoning_parser_dict=reasoning_parser_dict, - tool_parser_dict=tool_parser_dict, - ) - ) - events, accumulated = [], "" - for frame in _REAL_FRAMES: - accumulated += frame - events += step(frame, accumulated, frame is _REAL_FRAMES[-1]) - assert len(_call_items(events)) == 2 +@pytest.mark.parametrize("parser_id", sorted(_WHOLE_TEXT_PARSER_OUTPUTS)) +def test_whole_text_parser_calls_are_streamed_once_and_repeated_in_the_snapshot(parser_id): + text = _WHOLE_TEXT_PARSER_OUTPUTS[parser_id] + tools = [ + _function_tool("get_weather", ("location",)), + _function_tool("search_web", ("query",)), + _function_tool("get_time", ()), + ] + expected = ToolParserFactory.create_tool_parser(parser_id).detect_and_parse( + text, _get_chat_completion_function_tools(tools) + ) + expected_calls = [(c.name, json.loads(c.parameters or "{}")) for c in expected.calls] + assert expected_calls - # A second finished chunk for the same output carries no new text, and - # must not re-report the calls already emitted for it. - assert _call_items(step("", accumulated, True)) == [] + for size in (1, 3, 8, len(text)): + helper = ResponsesStreamingEventsHelper() + frames = [text[i : i + size] for i in range(0, len(text), size)] + events = _drive( + frames, tools=tools, tool_parser_id=parser_id, reasoning_parser_id=None, helper=helper + ) + calls = _call_items(events) + assert [(c.name, json.loads(c.arguments)) for c in calls] == expected_calls, size + + items, _messages, _reasoning = _create_output_content( + SimpleNamespace(outputs=[SimpleNamespace(index=0, text=text)]), + tool_parser=parser_id, + tools=tools, + streamed_tool_calls=helper.emitted_tool_call_items, + streamed_item_ids=helper.emitted_item_ids, + ) + assert [i for i in items if i.type == "function_call"] == calls class TestFragmentAssembly: - """The accumulator that turns parser fragments back into whole calls.""" - - def test_fragments_are_concatenated_in_order(self): - fragments: dict = {} + def test_fragments_concatenate_in_order_under_the_first_name(self): + fragments = {} _accumulate_tool_call_fragments( fragments, [ @@ -666,76 +365,34 @@ def test_fragments_are_concatenated_in_order(self): ToolCallItem(tool_index=0, name=None, parameters='is"}'), ], ) - assert _assembled_tool_calls(fragments) == [ ToolCallItem(tool_index=0, name="get_time", parameters='{"city": "Paris"}') ] - def test_a_later_fragment_does_not_erase_the_name(self): - """Only a call's first fragment carries its name; the rest carry None.""" - fragments: dict = {} - _accumulate_tool_call_fragments( - fragments, - [ - ToolCallItem(tool_index=0, name="get_time", parameters=""), - ToolCallItem(tool_index=0, name=None, parameters="{}"), - ], - ) - assert _assembled_tool_calls(fragments)[0].name == "get_time" - - def test_the_unfinished_call_is_dropped_and_the_finished_ones_kept(self): - fragments: dict = {} - _accumulate_tool_call_fragments( - fragments, - [ - ToolCallItem(tool_index=0, name="get_time", parameters='{"a": 1}'), - ToolCallItem(tool_index=1, name="get_time", parameters='{"a": '), - ], - ) - - assembled = _assembled_tool_calls(fragments, unfinished_tool_index=1) - assert [c.parameters for c in assembled] == ['{"a": 1}'] - - def test_a_call_whose_name_never_arrived_is_dropped(self): - """Matches parse_base_json on the non-streaming path.""" - fragments: dict = {} - _accumulate_tool_call_fragments( - fragments, [ToolCallItem(tool_index=0, parameters='{"a": 1}')] - ) - assert _assembled_tool_calls(fragments) == [] - - def test_fragments_are_kept_apart_per_output(self): - """Edge case 8: one parser instance per output, one fragment set each. - - `tool_parser_dict` is keyed by output index, so two outputs of the same - request number their calls independently - both start at tool_index 0 - - and merging them would concatenate one output's arguments onto the - other's. - """ - helper = ResponsesStreamingEventsHelper() - _accumulate_tool_call_fragments( - helper.tool_call_fragments(0), - [ToolCallItem(tool_index=0, name="first", parameters='{"a": 1}')], - ) - _accumulate_tool_call_fragments( - helper.tool_call_fragments(1), - [ToolCallItem(tool_index=0, name="second", parameters='{"b": 2}')], - ) - - assert [c.name for c in _assembled_tool_calls(helper.tool_call_fragments(0))] == ["first"] - assert [c.name for c in _assembled_tool_calls(helper.tool_call_fragments(1))] == ["second"] - - def test_two_helpers_do_not_share_fragments(self): - """Two requests in flight at once must not share an accumulator. - - A mutable default on the state tracker's class body would be one - object shared by every request, so two concurrent streams would - accumulate into each other's calls. - """ - first, second = (ResponsesStreamingEventsHelper(), ResponsesStreamingEventsHelper()) - _accumulate_tool_call_fragments( - first.tool_call_fragments(0), - [ToolCallItem(tool_index=0, name="only_mine", parameters="{}")], - ) - + @pytest.mark.parametrize( + "name, parameters", + [ + ("get_time", '{"a": '), + ("get_time", '{"cmd": }'), + ("get_time", '{"a": NaN}'), + ("get_time", ""), + (None, '{"a": 1}'), + ], + ) + def test_calls_a_client_cannot_run_are_skipped(self, name, parameters): + fragments = {} + good = ToolCallItem(tool_index=0, name="get_time", parameters='{"a": 1}') + bad = ToolCallItem(tool_index=1, name=name, parameters=parameters) + _accumulate_tool_call_fragments(fragments, [good, bad]) + assert _assembled_tool_calls(fragments) == [good] + + def test_fragments_are_kept_apart_per_output_and_per_request(self): + first, second = ResponsesStreamingEventsHelper(), ResponsesStreamingEventsHelper() + for index, name in ((0, "first"), (1, "second")): + _accumulate_tool_call_fragments( + first.tool_call_fragments(index), + [ToolCallItem(tool_index=0, name=name, parameters="{}")], + ) + assert [c.name for c in _assembled_tool_calls(first.tool_call_fragments(0))] == ["first"] + assert [c.name for c in _assembled_tool_calls(first.tool_call_fragments(1))] == ["second"] assert second.tool_call_fragments(0) == {} diff --git a/tests/unittest/llmapi/apps/test_responses_usage.py b/tests/unittest/llmapi/apps/test_responses_usage.py index e85755a2d284..df3d391e7403 100644 --- a/tests/unittest/llmapi/apps/test_responses_usage.py +++ b/tests/unittest/llmapi/apps/test_responses_usage.py @@ -20,11 +20,18 @@ long session keeps appending turns until it overflows the context window. """ +import json from types import SimpleNamespace import pytest -from tensorrt_llm.serve.responses_utils import _create_usage +from tensorrt_llm.serve.openai_protocol import PromptTokensDetails, ResponsesRequest, UsageInfo +from tensorrt_llm.serve.responses_utils import ( + ResponsesStreamingProcessor, + _count_reasoning_tokens, + _create_output_content, + _create_usage, +) # The CPU-* CI stages run pytest with -m 'cpu_only'. Without this marker every # test in the file is deselected, which pytest reports as exit code 5 and the @@ -134,205 +141,156 @@ def test_usage_validates_inside_a_streamed_completion_event(): # --------------------------------------------------------------------------- -# reasoning_tokens -# -# Reported as a hardcoded zero until now. The count is rebuilt from the text -# the reasoning parser itself claimed, rather than by searching the generated -# token ids for ``, and these tests are the reason: GLM breaks a -# marker search three separate ways under agent workloads, and each of them -# was observed live before it was tested here. +# Disaggregated serving: the context phase's usage comes with the handoff # --------------------------------------------------------------------------- -class _WordTokenizer: - """One token per whitespace-separated word. Enough to count with.""" - - def encode(self, text, add_special_tokens=False): - return text.split() - - -def _reasoning_tokens(text, parser="glm", tokenizer=_WordTokenizer(), tools=None): - """Run the real parser, then count what it called reasoning.""" - from tensorrt_llm.serve.responses_utils import _count_reasoning_tokens, _create_output_content - - _items, _messages, reasoning_texts = _create_output_content( - SimpleNamespace(outputs=[SimpleNamespace(index=0, text=text)]), - reasoning_parser=parser, - tool_parser=None, - tools=tools, +def _ctx_usage(prompt_tokens, cached_tokens): + return UsageInfo( + prompt_tokens=prompt_tokens, + completion_tokens=1, + total_tokens=prompt_tokens + 1, + prompt_tokens_details=PromptTokensDetails(cached_tokens=cached_tokens), ) - return _count_reasoning_tokens(tokenizer, reasoning_texts, 10_000) -def test_reasoning_tokens_counts_up_to_the_closing_tag(): - # The ordinary shape: reasoning, close, answer. - assert _reasoning_tokens("one two threefour five") == 3 +def _handed_off(ctx_usage, prompt_tokens=100, completion_tokens=5): + """A generation-worker result: the whole prompt arrived as transferred KV.""" + result = _generation(prompt_tokens, completion_tokens, cached_tokens=prompt_tokens) + result.outputs[0].disaggregated_params = SimpleNamespace(ctx_usage=ctx_usage) + return result -def test_reasoning_tokens_stop_at_the_first_close_not_the_last(): - # GLM closes the block more than once (reasoning_parser.py:370-388). The - # parser splits at the first close and drops the stray one; a search for - # `` in token space that took the last match -- which is what - # ThinkingBudgetLogitsProcessor does, for its own good reasons -- would - # count the middle stretch as reasoning too. - text = "one twothree four fivex" - assert _reasoning_tokens(text) == 2 +@pytest.mark.parametrize( + "ctx_usage, input_tokens, cached_tokens", + [ + (_ctx_usage(96, 32), 96, 32), + (_ctx_usage(100, 0), 100, 0), + (_ctx_usage(100, 40).model_dump(), 100, 40), + (UsageInfo(prompt_tokens=100, completion_tokens=1, total_tokens=101), 100, 0), + ], + ids=["warm", "cold", "as_dict", "no_details"], +) +def test_the_context_phase_decides_prompt_and_cached_tokens(ctx_usage, input_tokens, cached_tokens): + usage = _create_usage(_handed_off(ctx_usage), num_prompt_tokens=100) + assert usage.input_tokens == input_tokens + assert usage.input_tokens_details.cached_tokens == cached_tokens + assert usage.output_tokens == 5 -def test_reasoning_tokens_end_at_an_implicit_tool_call(): - # Observed live: 26,055 characters over 169 frames with no closing tag at - # all and one well-formed . Counting to the missing `` - # would have called the entire turn reasoning. - text = "planning the call{}" - assert _reasoning_tokens(text) == 3 - +# --------------------------------------------------------------------------- +# reasoning_tokens: re-encoded from the text the reasoning parser claimed +# --------------------------------------------------------------------------- -def test_reasoning_tokens_covers_a_turn_that_never_leaves_the_block(): - # `` is prefilled by the template, so output starts inside the - # block; with no terminator the whole turn really is reasoning. - assert _reasoning_tokens("still thinking about it") == 4 +class _WordTokenizer: + """One token per whitespace-separated word.""" -def test_reasoning_tokens_are_zero_without_a_reasoning_parser(): - assert _reasoning_tokens("one two three", parser=None) == 0 + def encode(self, text, add_special_tokens=False): + return text.split() -def test_reasoning_tokens_are_zero_without_a_tokenizer(): - assert _reasoning_tokens("one two three", tokenizer=None) == 0 +@pytest.mark.parametrize( + "text, parser, tokenizer, expected", + [ + ("one two threefour five", "glm", _WordTokenizer(), 3), + ("one twothree fourx", "glm", _WordTokenizer(), 2), + ("planning the call{}", "glm", _WordTokenizer(), 3), + ("still thinking about it", "glm", _WordTokenizer(), 4), + ("one two three", None, _WordTokenizer(), 0), + ("one two three", "glm", None, 0), + ], + ids=["closed", "first_close_counts", "implicit_end", "never_closed", "no_parser", "no_tok"], +) +def test_reasoning_tokens_count_what_the_parser_called_reasoning(text, parser, tokenizer, expected): + _items, _messages, reasoning_texts = _create_output_content( + SimpleNamespace(outputs=[SimpleNamespace(index=0, text=text)]), reasoning_parser=parser + ) + assert _count_reasoning_tokens(tokenizer, reasoning_texts, 10_000) == expected def test_reasoning_tokens_never_exceed_generated_tokens(): - # Re-encoding a detokenized substring need not reproduce the tokenization - # it came from, and usage claiming more reasoning than output is visibly - # wrong to a client budgeting a context window. - from tensorrt_llm.serve.responses_utils import _count_reasoning_tokens - assert _count_reasoning_tokens(_WordTokenizer(), ["a b c d e"], 3) == 3 -def test_usage_reports_the_reasoning_tokens_it_counts(): +def test_usage_reports_reasoning_tokens_within_the_output_tokens(): usage = _create_usage( _generation(prompt_tokens=7, completion_tokens=9), tokenizer=_WordTokenizer(), reasoning_texts=["one two three four"], ) assert usage.output_tokens_details.reasoning_tokens == 4 - # Reasoning tokens are part of the generated total, not additional to it. - assert usage.output_tokens == 9 - assert usage.total_tokens == 16 - - -def test_usage_still_defaults_to_zero_reasoning_tokens(): - usage = _create_usage(_generation()) - assert usage.output_tokens_details.reasoning_tokens == 0 + assert (usage.output_tokens, usage.total_tokens) == (9, 16) + assert _create_usage(_generation()).output_tokens_details.reasoning_tokens == 0 + + +def test_a_streamed_response_counts_reasoning_tokens(): + request = ResponsesRequest(model="m", input="hi", stream=True) + processor = ResponsesStreamingProcessor( + request=request, + sampling_params=request.to_sampling_params(), + model_name="m", + use_harmony=False, + reasoning_parser="glm", + ) + output = SimpleNamespace( + index=0, + text="one twoanswer", + text_diff="one twoanswer", + finish_reason="stop", + token_ids=[1, 2, 3], + disaggregated_params=None, + ) + result = SimpleNamespace(outputs=[output], _done=True, prompt_token_ids=[1], cached_tokens=0) + processor.process_single_output(result) + frame = processor.get_final_response_non_store(result, tokenizer=_WordTokenizer()) + usage = json.loads(frame.split("data: ", 1)[1])["response"]["usage"] + assert usage["output_tokens_details"]["reasoning_tokens"] == 2 # --------------------------------------------------------------------------- -# input_image degradation +# input_image parts degrade to a text placeholder # --------------------------------------------------------------------------- +_PNG = "data:image/png;base64,AAAA" -def test_an_image_part_degrades_to_a_text_placeholder(): - """Regression: an input_image part 400'd the whole request. - - Codex attaches screenshots as `input_image` (base64). The input union has - no image member, so validation fell through to ResponseInputTextParam and - rejected the request -- deterministically, so the client's retries all - failed and the campaign died on its backoff limit. Measured: 2 of the - first 182 Kernel-Trace campaigns, each with the full base64 body echoed - into the stop reason. - """ - from tensorrt_llm.serve.openai_protocol import ResponsesRequest - - req = ResponsesRequest( - model="m", - input=[ - { - "role": "user", - "content": [ - {"type": "input_text", "text": "look at this"}, - {"type": "input_image", "image_url": "data:image/png;base64,AAAA"}, - ], - }, - # And as a bare top-level part, which clients also send. - {"type": "input_image", "image_url": "data:image/png;base64,BB"}, - ], - ) - dumped = req.model_dump()["input"] - texts = [] - for item in dumped: - parts = item.get("content") if isinstance(item.get("content"), list) else [item] - for p in parts: - assert p.get("type") != "input_image", "image part survived" - if isinstance(p.get("text"), str): - texts.append(p["text"]) - assert any("image omitted" in t and "image/png" in t for t in texts) - # The placeholder must not carry the base64 payload. - assert not any("AAAA" in t for t in texts) - - -def test_an_image_in_a_tool_output_degrades_too(): - """Regression: tool results carry parts under "output", not "content". - - An agent that plots something gets the PNG back through the tool: Codex - sends `custom_tool_call_output` whose `output` list ends with an - `input_image` part. The first degrade pass only walked `content` and the - top level, so this shape still 400'd the whole request -- measured at 114 - Kernel-Trace campaigns in the 24h after the content/top-level fix went - live (2026-09-23). Shape taken verbatim from a traced rejected request. - """ - from tensorrt_llm.serve.openai_protocol import ResponsesRequest - req = ResponsesRequest( - model="m", - input=[ +@pytest.mark.parametrize( + "item, key", + [ + ( + {"role": "user", "content": [{"type": "input_image", "image_url": _PNG}]}, + "content", + ), + ( { "role": "user", - "content": [ - {"type": "input_text", "text": "plot it"}, - ], - }, - { - "type": "custom_tool_call", - "id": "ctc_1", - "call_id": "call_1", - "name": "exec", - "input": "python plot.py", + "content": [{"type": "input_image", "image_url": {"url": _PNG}}], }, + "content", + ), + ( { "type": "custom_tool_call_output", "call_id": "call_1", - "output": [ - {"type": "input_text", "text": "Script completed\n"}, - {"type": "input_text", "text": "layout-a2 image"}, - {"type": "input_image", "image_url": "data:image/png;base64,CCCC"}, - ], - }, - ], - ) - out = req.model_dump()["input"][2]["output"] - assert all(p.get("type") != "input_image" for p in out), ( - "image part survived inside tool output" - ) - joined = " ".join(p.get("text", "") for p in out) - assert "image omitted" in joined and "image/png" in joined - assert "CCCC" not in joined - # A plain-string tool output must pass through untouched. - req2 = ResponsesRequest( - model="m", - input=[ - { - "type": "custom_tool_call", - "id": "ctc_2", - "call_id": "call_2", - "name": "exec", - "input": "true", + "output": [{"type": "input_image", "image_url": _PNG}], }, - { - "type": "custom_tool_call_output", - "call_id": "call_2", - "output": "Script completed\n", - }, - ], - ) - assert req2.model_dump()["input"][1]["output"] == "Script completed\n" + "output", + ), + ({"type": "input_image", "image_url": _PNG}, None), + ], + ids=["message_part", "url_object", "tool_output", "top_level"], +) +def test_an_image_degrades_to_a_text_placeholder(item, key): + request = ResponsesRequest(model="m", input=[item]) + dumped = request.model_dump()["input"][0] + (part,) = dumped[key] if key else [dumped] + assert part["type"] == "input_text" + assert part["text"].startswith("[image omitted: image/png, ") + assert "AAAA" not in part["text"] + + +def test_a_string_tool_output_is_not_degraded(): + item = {"type": "custom_tool_call_output", "call_id": "call_2", "output": "done\n"} + request = ResponsesRequest(model="m", input=[item]) + assert request.model_dump()["input"][0]["output"] == "done\n" diff --git a/tests/unittest/llmapi/apps/test_responses_usage_disagg.py b/tests/unittest/llmapi/apps/test_responses_usage_disagg.py deleted file mode 100644 index ceb5235e813b..000000000000 --- a/tests/unittest/llmapi/apps/test_responses_usage_disagg.py +++ /dev/null @@ -1,145 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Usage accounting for the Responses API under disaggregated serving. - -The generation worker cannot measure prefix reuse: the whole prompt reaches it -as KV transferred from the context worker, so its own `cached_tokens` equals the -prompt length. Reporting that says every request was a complete cache hit, which -is what a client watching its context window would act on. The real number comes -across with the handoff, and chat completions has read it from there since -PR #14177. -""" - -import unittest -from types import SimpleNamespace - -import pytest - -from tensorrt_llm.serve.openai_protocol import PromptTokensDetails, UsageInfo -from tensorrt_llm.serve.responses_utils import _create_usage - -# The CPU-* CI stages run pytest with -m 'cpu_only', and tests/unittest/conftest.py -# drops every file without this marker there. -pytestmark = pytest.mark.cpu_only - - -def result(prompt_len, output_len, cached, ctx_usage=None): - """A finished generation, shaped as the postprocessor sees one.""" - output = SimpleNamespace( - token_ids=list(range(output_len)), - disaggregated_params=( - SimpleNamespace(ctx_usage=ctx_usage) if ctx_usage is not None else None - ), - ) - return SimpleNamespace( - outputs=[output], - prompt_token_ids=list(range(prompt_len)), - cached_tokens=cached, - ) - - -def ctx(prompt_tokens, cached_tokens): - return UsageInfo( - prompt_tokens=prompt_tokens, - completion_tokens=1, - total_tokens=prompt_tokens + 1, - prompt_tokens_details=PromptTokensDetails(cached_tokens=cached_tokens), - ) - - -class Aggregated(unittest.TestCase): - """No handoff: the local counts are the only ones, and they are correct.""" - - def test_local_counts_are_used(self): - usage = _create_usage(result(100, 5, cached=32)) - self.assertEqual(100, usage.input_tokens) - self.assertEqual(5, usage.output_tokens) - self.assertEqual(32, usage.input_tokens_details.cached_tokens) - - def test_no_cache_hit_reports_zero(self): - usage = _create_usage(result(100, 5, cached=0)) - self.assertEqual(0, usage.input_tokens_details.cached_tokens) - - -class Disaggregated(unittest.TestCase): - def test_the_context_phase_decides_the_cached_count(self): - """The 100 the generation worker reports is the prompt length, not reuse.""" - usage = _create_usage(result(100, 5, cached=100, ctx_usage=ctx(100, 32))) - self.assertEqual(32, usage.input_tokens_details.cached_tokens) - self.assertEqual(100, usage.input_tokens) - - def test_a_cold_prompt_is_not_reported_as_fully_cached(self): - """The failure this fixes: cached == input on every single request.""" - usage = _create_usage(result(4096, 10, cached=4096, ctx_usage=ctx(4096, 0))) - self.assertEqual(0, usage.input_tokens_details.cached_tokens) - self.assertNotEqual(usage.input_tokens, usage.input_tokens_details.cached_tokens) - - def test_the_context_phase_also_decides_the_prompt_length(self): - usage = _create_usage( - result(100, 5, cached=100, ctx_usage=ctx(96, 64)), num_prompt_tokens=100 - ) - self.assertEqual(96, usage.input_tokens) - self.assertEqual(64, usage.input_tokens_details.cached_tokens) - - def test_output_tokens_stay_local(self): - """Only the generation worker knows what it generated.""" - usage = _create_usage(result(100, 7, cached=100, ctx_usage=ctx(100, 50))) - self.assertEqual(7, usage.output_tokens) - - def test_a_handoff_carrying_no_details_reports_no_reuse(self): - bare = UsageInfo(prompt_tokens=100, completion_tokens=1, total_tokens=101) - usage = _create_usage(result(100, 5, cached=100, ctx_usage=bare)) - self.assertEqual(0, usage.input_tokens_details.cached_tokens) - - def test_a_handoff_arriving_as_a_dict_is_accepted(self): - """It crosses a process boundary, so it may not still be a model.""" - as_dict = ctx(100, 40).model_dump() - usage = _create_usage(result(100, 5, cached=100, ctx_usage=as_dict)) - self.assertEqual(40, usage.input_tokens_details.cached_tokens) - - -class ProxyHandoff(unittest.TestCase): - """The proxy converts the context reply's usage into the handoff shape.""" - - def test_cached_tokens_survive_the_conversion(self): - from tensorrt_llm.serve.openai_disagg_service import _ctx_usage_info - - # Shaped by hand rather than through the SDK model: the two carry the - # same field names but not the same required set, and this test is - # about what the proxy copies, not about the SDK's schema. - response = SimpleNamespace( - usage=SimpleNamespace( - input_tokens=96, - input_tokens_details=SimpleNamespace(cached_tokens=64), - output_tokens=1, - total_tokens=97, - ) - ) - carried = _ctx_usage_info(response) - self.assertEqual(96, carried.prompt_tokens) - self.assertEqual(1, carried.completion_tokens) - self.assertEqual(64, carried.prompt_tokens_details.cached_tokens) - - def test_a_reply_without_details_still_converts(self): - from tensorrt_llm.serve.openai_disagg_service import _ctx_usage_info - - response = SimpleNamespace( - usage=SimpleNamespace( - input_tokens=96, - input_tokens_details=None, - output_tokens=1, - total_tokens=97, - ) - ) - carried = _ctx_usage_info(response) - self.assertEqual(0, carried.prompt_tokens_details.cached_tokens) - - def test_a_usage_that_is_already_in_the_handoff_shape_passes_through(self): - from tensorrt_llm.serve.openai_disagg_service import _ctx_usage_info - - already = ctx(96, 64) - self.assertIs(already, _ctx_usage_info(SimpleNamespace(usage=already))) - - -if __name__ == "__main__": - unittest.main(verbosity=2) diff --git a/tests/unittest/llmapi/apps/test_web_search_shared.py b/tests/unittest/llmapi/apps/test_web_search_shared.py index 832d08db4da4..9eeeed218a27 100644 --- a/tests/unittest/llmapi/apps/test_web_search_shared.py +++ b/tests/unittest/llmapi/apps/test_web_search_shared.py @@ -20,6 +20,7 @@ """ import asyncio +from types import SimpleNamespace import pytest @@ -289,63 +290,20 @@ def test_a_real_request_carrying_web_search_is_recognised(): assert web_search_rejection_reason(plain.tools) is None -def test_external_access_disabled_is_not_rejected(): - """Regression: Codex was locked out of the server entirely. - - Codex attaches {"type": "web_search", "external_web_access": false} to - every turn. That declares it does not want the open web, so there is no - live result the server fails to deliver and nothing the client can be - misled about - but the guard refused the whole request anyway, and Codex - treats the 400 as retryable, so the actionable message never surfaced. - """ - from types import SimpleNamespace - - from tensorrt_llm.serve.responses_web_search import web_search_rejection_reason - - tool = SimpleNamespace( - type="web_search", external_web_access=False, name=None, filters=None, max_uses=None - ) - assert web_search_rejection_reason([tool]) is None - assert ( - web_search_rejection_reason([{"type": "web_search", "external_web_access": False}]) is None - ) - - -def test_external_access_requested_is_still_rejected(): - """The case BowenFu raised must keep failing loudly.""" - from types import SimpleNamespace - - from tensorrt_llm.serve.responses_web_search import web_search_rejection_reason - - for external in (True, None): - tool = SimpleNamespace( - type="web_search", external_web_access=external, name=None, filters=None, max_uses=None - ) - assert web_search_rejection_reason([tool]) is not None, external - - -def test_rejection_names_the_tool_and_the_deciding_field(): - """The message has to carry what the decision turned on. - - Whoever has to act on this error needs to know which tool triggered it and - what ``external_web_access`` arrived as - a client may attach web_search - implicitly, and "absent" is a different situation from an explicit True. - Diagnosing a real case without these took several wrong guesses. - """ - from types import SimpleNamespace - +@pytest.mark.parametrize( + "tool, rejected", + [ + ({"type": "web_search", "external_web_access": False}, False), + (SimpleNamespace(type="web_search", external_web_access=False, filters=None), False), + (SimpleNamespace(type="web_search", external_web_access=True, filters=None), True), + ({"type": "web_search"}, True), + ], +) +def test_web_search_is_accepted_only_without_external_access(tool, rejected): from tensorrt_llm.serve.responses_web_search import web_search_rejection_reason - reason = web_search_rejection_reason( - [ - SimpleNamespace( - type="web_search", external_web_access=True, name=None, filters=None, max_uses=None - ) - ] - ) - assert "web_search" in reason - assert "external_web_access=True" in reason - - # Absent is reported as such rather than silently reading as True. - reason = web_search_rejection_reason([{"type": "web_search"}]) - assert "external_web_access='absent'" in reason + reason = web_search_rejection_reason([tool]) + assert (reason is not None) == rejected + if rejected: + external = tool.get("external_web_access") if isinstance(tool, dict) else True + assert f"tool type 'web_search', external_web_access={external!r}" in reason From c13a8f09fda52e99332fdfa07a4cd9bde0ae1883 Mon Sep 17 00:00:00 2001 From: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> Date: Wed, 7 Oct 2026 02:28:17 +0000 Subject: [PATCH 28/43] [None][feat] Route Responses requests on the disaggregated server The routers assumed a completion or chat request: the KV-cache-aware router failed on a Responses request, load balancing by tokens could not count one, and the conversation router hashed every Responses request to the same text, so all of them went to one server. All routers now handle ResponsesRequest. They use the prompt ids relayed from the context worker when present; otherwise the KV-cache-aware router renders the prompt the way a worker does (Harmony included) and the conversation router reads the request's messages as role:content lines. Routing ids are not written onto the request, so the context worker still renders the prompt itself. Signed-off-by: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> --- tensorrt_llm/serve/router.py | 11 +- tensorrt_llm/serve/router_utils.py | 60 ++++++++++- tests/unittest/disaggregated/test_router.py | 108 +++++++++++++++++++- 3 files changed, 173 insertions(+), 6 deletions(-) diff --git a/tensorrt_llm/serve/router.py b/tensorrt_llm/serve/router.py index 03524723b5c4..894ed367d46d 100644 --- a/tensorrt_llm/serve/router.py +++ b/tensorrt_llm/serve/router.py @@ -27,14 +27,16 @@ from tensorrt_llm.logger import logger from tensorrt_llm.serve.conversation_id import get_request_routing_id from tensorrt_llm.serve.metadata_server import JsonDictionary -from tensorrt_llm.serve.openai_protocol import ChatCompletionRequest +from tensorrt_llm.serve.openai_protocol import (ChatCompletionRequest, + ResponsesRequest) # Shared tokenization / block-hashing utilities (single source of truth). # Re-exported here for backward compat. from tensorrt_llm.serve.router_utils import ( # noqa: F401 KV_CACHE_HASH_ALGO_DEFAULT, KV_CACHE_HASH_ALGO_V1, KV_CACHE_HASH_ALGO_V2, KV_CACHE_HASH_ALGO_V2_SHA256_64, KV_CACHE_HASH_ALGOS, BlockHash, BlockHashMixin, OpenAIRequest, block_key_hasher, get_cache_salt_id, - get_request_num_tokens, hash_v1_block_key, truncate_sha256_hash_to_int64) + get_request_num_tokens, hash_v1_block_key, responses_request_text, + truncate_sha256_hash_to_int64) _MSGPACK_HEADERS = {"Content-Type": "application/msgpack"} COORDINATOR_SELECT_MAX_ATTEMPTS = 2 @@ -1430,6 +1432,8 @@ def _on_servers_updated(self, old_servers, new_servers): @staticmethod def _extract_text(request: OpenAIRequest) -> str: """Return a canonical text representation of the request content.""" + if isinstance(request, ResponsesRequest): + return responses_request_text(request) if isinstance(request, ChatCompletionRequest): parts = [] for msg in request.messages: @@ -1454,6 +1458,9 @@ def _try_extract_token_ids( Returns ``None`` when the request does not already carry them. """ + if isinstance(request, ResponsesRequest): + token_ids = request.relayed_prompt_token_ids() + return None if token_ids is None else [token_ids] if isinstance(request, ChatCompletionRequest): if request.prompt_token_ids is not None: return [request.prompt_token_ids] diff --git a/tensorrt_llm/serve/router_utils.py b/tensorrt_llm/serve/router_utils.py index 45b3ca6829ab..03d95584b8b0 100644 --- a/tensorrt_llm/serve/router_utils.py +++ b/tensorrt_llm/serve/router_utils.py @@ -29,8 +29,13 @@ from tensorrt_llm.serve.chat_tokenization import ( resolve_model_type_from_config, tokenize_chat_request_for_serving, + uses_harmony_tokenization, +) +from tensorrt_llm.serve.openai_protocol import ( + ChatCompletionRequest, + CompletionRequest, + ResponsesRequest, ) -from tensorrt_llm.serve.openai_protocol import ChatCompletionRequest, CompletionRequest KV_CACHE_HASH_ALGO_DEFAULT = kv_cache_hash.KV_CACHE_HASH_ALGO_DEFAULT KV_CACHE_HASH_ALGO_V1 = kv_cache_hash.KV_CACHE_HASH_ALGO_V1 @@ -47,7 +52,7 @@ hash_v1_block_key = kv_cache_hash.hash_v1_block_key truncate_sha256_hash_to_int64 = kv_cache_hash.truncate_sha256_hash_to_int64 -OpenAIRequest = Union[CompletionRequest, ChatCompletionRequest] +OpenAIRequest = Union[CompletionRequest, ChatCompletionRequest, ResponsesRequest] BlockHash = Union[int, str] __all__ = [ @@ -61,6 +66,7 @@ "truncate_sha256_hash_to_int64", "OpenAIRequest", "BlockHash", + "responses_request_text", "get_request_num_tokens", "block_key_hasher", "BlockHashMixin", @@ -122,10 +128,27 @@ def has_worker(self, owner_id: str) -> bool: return bool(self._blocks) +def responses_request_text(request: ResponsesRequest) -> str: + """Return the instructions and input as ``role:content`` lines, like a chat request's text. + + A later turn re-sends the earlier turns' input, so its text extends theirs. + """ + from tensorrt_llm.serve.responses_utils import chat_messages_from_responses_input + + messages = chat_messages_from_responses_input(request, []) + return "\n".join(f"{m.get('role', '')}:{m.get('content', '')}" for m in messages) + + def get_request_num_tokens(request: Optional[OpenAIRequest]) -> int: if request is None: return 0 + if isinstance(request, ResponsesRequest): + # Without relayed token ids the text is counted in characters, as a + # string completion prompt is below. + token_ids = request.relayed_prompt_token_ids() + return len(responses_request_text(request) if token_ids is None else token_ids) + if ( request.disaggregated_params is None or request.disaggregated_params.request_type == "context_only" @@ -256,6 +279,9 @@ def _get_model_type(self) -> Optional[str]: return self._model_types[model_path] def _tokenize(self, request: OpenAIRequest) -> list[list[int]]: + if isinstance(request, ResponsesRequest): + return [self._tokenize_responses_request(request)] + # Handle ChatCompletionRequest (has messages, not prompt) if isinstance(request, ChatCompletionRequest): @@ -305,6 +331,36 @@ def encode_rendered(rendered: str, tokenizer: object) -> list[int]: request.prompt = token_lists if len(token_lists) > 1 else token_lists[0] return token_lists + def _tokenize_responses_request(self, request: ResponsesRequest) -> list[int]: + """Return the token ids of the prompt a worker builds for a Responses request. + + The ids only drive routing and are not stored on the request, so the + context worker still renders the prompt itself. + """ + token_ids = request.relayed_prompt_token_ids() + if token_ids is not None: + return token_ids + from tensorrt_llm.serve import responses_utils + + # A disaggregated server rejects previous_response_id, so there is no + # stored history: the input is the whole conversation. + if uses_harmony_tokenization( + use_harmony=self._use_harmony, model_type_resolver=self._get_model_type + ): + return responses_utils._render_for_completion( + responses_utils._construct_harmony_messages(request, None) + ) + chat_request = ChatCompletionRequest.model_construct( + model=request.model, + messages=responses_utils.chat_messages_from_responses_input(request, []), + tools=responses_utils._get_chat_completion_function_tools(request.tools), + chat_template_kwargs=responses_utils.reasoning_chat_template_kwargs(request), + injected_chat_template_kwargs=list( + responses_utils.reasoning_injected_chat_template_keys(request) + ), + ) + return self._tokenize(chat_request)[0] + def _compute_block_hashes( self, token_lists: list[list[int]], diff --git a/tests/unittest/disaggregated/test_router.py b/tests/unittest/disaggregated/test_router.py index 9b4f10e72b14..34dc94e64480 100644 --- a/tests/unittest/disaggregated/test_router.py +++ b/tests/unittest/disaggregated/test_router.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 import asyncio +import base64 import copy import random import threading @@ -11,6 +12,7 @@ import aiohttp import msgpack +import numpy as np import pytest from tensorrt_llm.llmapi.disagg_utils import RouterConfig @@ -25,7 +27,9 @@ CompletionRequest, ConversationParams, DisaggregatedParams, - FunctionDefinition) + FunctionDefinition, + ResponsesRequest) +from tensorrt_llm.serve.responses_utils import request_preprocess from tensorrt_llm.serve.router import (COORDINATOR_SELECT_MAX_ATTEMPTS, KV_CACHE_HASH_ALGO_V1, KV_CACHE_HASH_ALGO_V2, @@ -35,7 +39,8 @@ KvCacheAwareRouter, KvCacheAwareServerState, LoadBalancingRouter, RoundRobinRouter, - block_key_hasher, create_router) + block_key_hasher, create_router, + responses_request_text) # yapf: enable @@ -2577,3 +2582,102 @@ def test_v2_sha256_64_block_hashes_are_big_endian_prefix_of_frozen_values( for key in _V2_GOLDEN_PARTIAL_TAIL ] assert block_hashes == [expected] + + +# ── ResponsesRequest routing ── + + +def _message(role: str, content: str) -> dict[str, str]: + return {"role": role, "content": content} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_harmony", [False, True]) +async def test_kv_cache_aware_router_routes_responses_turns( + servers, use_harmony): + """A later turn of a Responses conversation follows its cached prefix.""" + router = KvCacheAwareRouter(server_role=None, + servers=servers, + tokens_per_block=4, + use_harmony=use_harmony) + history = [_message("user", "plan a trip " * 10)] + turn1 = ResponsesRequest(model="m", instructions="Be brief.", input=history) + turn2 = ResponsesRequest( + model="m", + instructions="Be brief.", + input=history + + [_message("assistant", "day one"), + _message("user", "add food")]) + with mock.patch.object(router, + "_get_tokenizer", + return_value=_PrefixCacheFakeTokenizer()): + home, _ = await router.get_next_server(turn1) + await router.finish_request(turn1) + server, info = await router.get_next_server(turn2) + assert server == home and info["match_length"] > 0 + # Routing ids are not stored on the request: the context worker renders + # the prompt itself. + assert turn1.prompt_token_ids is None + if use_harmony: + worker_ids, _ = await request_preprocess(turn2, + None, + None, + enable_store=False, + use_harmony=True) + assert router._tokenize(turn2) == [worker_ids] + + +@pytest.mark.asyncio +async def test_conversation_router_tells_responses_conversations_apart(): + router = ConversationRouter(server_role=None, + servers=["server1", "server2", "server3"]) + home_a, _ = await router.get_next_server( + ResponsesRequest(model="m", input="topic A " * 200)) + home_b, _ = await router.get_next_server( + ResponsesRequest(model="m", input="topic B " * 200)) + later_a = ResponsesRequest(model="m", + input=[ + _message("user", "topic A " * 200), + _message("assistant", "done"), + _message("user", "again") + ]) + assert home_b != home_a + assert (await router.get_next_server(later_a))[0] == home_a + + +@pytest.mark.asyncio +@pytest.mark.parametrize("relay", + [None, "prompt_token_ids", "prompt_token_ids_b64"]) +async def test_responses_request_relayed_prompt_token_ids(relay): + """Routers use relayed prompt ids, plain or base64, and count the text without them.""" + token_ids = list(range(40)) + request = ResponsesRequest( + model="m", + input="hello " * 10, + disaggregated_params=DisaggregatedParams( + request_type="generation_only" if relay else "context_only")) + if relay == "prompt_token_ids_b64": + request.prompt_token_ids_b64 = base64.b64encode( + np.asarray(token_ids, dtype=np.int32).tobytes()).decode() + elif relay: + request.prompt_token_ids = token_ids + relayed = [token_ids] if relay else None + + router = LoadBalancingRouter(server_role=None, + servers=["server1"], + use_tokens=True) + await router.get_next_server(request) + assert router._server_state["server1"]._num_active_tokens == ( + len(token_ids) if relay else len(responses_request_text(request))) + await router.finish_request(request) + assert router._server_state["server1"]._num_active_tokens == 0 + assert ConversationRouter._try_extract_token_ids(request) == relayed + if relay: + kv_router = KvCacheAwareRouter(server_role=None, servers=["server1"]) + with mock.patch.object(kv_router, + "_get_tokenizer", + side_effect=AssertionError): + assert kv_router._tokenize(request) == relayed + # A base64 relay is decoded for routing, not expanded onto the request. + assert request.prompt_token_ids == (token_ids if relay == "prompt_token_ids" + else None) From a4c91dcd7a7e4a4adddae916af92dfee1a7693c0 Mon Sep 17 00:00:00 2001 From: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> Date: Wed, 7 Oct 2026 05:53:02 +0000 Subject: [PATCH 29/43] [None][fix] Responses: keep a processor's numbering when no frame count is given get_stream_failed_events numbered its error and response.failed events from zero when the caller passed no events_sent. The stream guard always passes the count, but any other caller would have restarted a sequence the client had already passed. Without a count the events now continue the processor's own numbering. Signed-off-by: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> --- tensorrt_llm/serve/responses_utils.py | 17 ++++++++++------- .../apps/test_responses_streaming_events.py | 10 ++++------ 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/tensorrt_llm/serve/responses_utils.py b/tensorrt_llm/serve/responses_utils.py index b77dd0a7b208..d7db885ce726 100644 --- a/tensorrt_llm/serve/responses_utils.py +++ b/tensorrt_llm/serve/responses_utils.py @@ -2911,16 +2911,19 @@ def process_single_output(self, res: GenerationResult) -> list[str]: return [self._send_event(event) for event in event_generator] - def get_stream_failed_events(self, - cause: str, - detail: str, - events_sent: int = 0) -> List[str]: + def get_stream_failed_events( + self, + cause: str, + detail: str, + events_sent: Optional[int] = None) -> List[str]: """``error`` and ``response.failed`` for a stream that stopped early. - ``events_sent`` is the number of frames that reached the wire; the - terminal events are numbered after them. + ``events_sent`` is the number of frames that reached the wire, which + the terminal events are numbered after; without it they continue this + processor's own numbering. """ - self.sequence_number = events_sent + if events_sent is not None: + self.sequence_number = events_sent error_event = self._send_event( ResponseErrorEvent( type="error", diff --git a/tests/unittest/llmapi/apps/test_responses_streaming_events.py b/tests/unittest/llmapi/apps/test_responses_streaming_events.py index 3df5ff27f779..45f0474475c1 100644 --- a/tests/unittest/llmapi/apps/test_responses_streaming_events.py +++ b/tests/unittest/llmapi/apps/test_responses_streaming_events.py @@ -408,14 +408,12 @@ def test_termination_cause_and_client_facing_detail(exc, cause, detail): assert describe_stream_termination(exc, cause) == detail -@pytest.mark.parametrize("events_sent", [2, 610]) -def test_terminal_events_are_numbered_after_the_frames_sent(events_sent): +@pytest.mark.parametrize("events_sent, expected", [(None, [2, 3]), (610, [610, 611])]) +def test_terminal_events_are_numbered_after_the_frames_sent(events_sent, expected): processor = _processor() + processor.get_initial_responses() events = processor.get_stream_failed_events("internal_error", "ValueError", events_sent) - assert [_event_data(frame)["sequence_number"] for frame in events] == [ - events_sent, - events_sent + 1, - ] + assert [_event_data(frame)["sequence_number"] for frame in events] == expected def test_the_failed_snapshot_says_failed_and_carries_no_content(): From 95e275a02378ebcd8e655c761542d6c0a9f1a216 Mon Sep 17 00:00:00 2001 From: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> Date: Fri, 9 Oct 2026 11:16:35 +0000 Subject: [PATCH 30/43] [None][test] Responses: fix the two tests CI run 76914 failed - _test_openai_perf_metrics.py: the streaming Responses tests cap the output at 2 tokens, so the stream now ends with response.incomplete, as this PR reports a length-truncated response. The tests check the perf-metrics event, so they accept either terminal event. - test_router.py: the Harmony case of test_kv_cache_aware_router_routes_responses_turns tried to download the Harmony vocab, which the offline CPU arm runner cannot. It now loads it from the datasets root, as the other Harmony unit tests do. Signed-off-by: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> --- tests/unittest/disaggregated/test_router.py | 9 ++++++++- tests/unittest/llmapi/apps/_test_openai_perf_metrics.py | 9 +++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/tests/unittest/disaggregated/test_router.py b/tests/unittest/disaggregated/test_router.py index 34dc94e64480..0e91d49c7f7f 100644 --- a/tests/unittest/disaggregated/test_router.py +++ b/tests/unittest/disaggregated/test_router.py @@ -4,6 +4,7 @@ import asyncio import base64 import copy +import os import random import threading from pathlib import Path @@ -14,6 +15,7 @@ import msgpack import numpy as np import pytest +from utils.llm_data import llm_datasets_root from tensorrt_llm.llmapi.disagg_utils import RouterConfig from tensorrt_llm.runtime.kv_cache_hash import (get_cache_salt_id, @@ -2594,8 +2596,13 @@ def _message(role: str, content: str) -> dict[str, str]: @pytest.mark.asyncio @pytest.mark.parametrize("use_harmony", [False, True]) async def test_kv_cache_aware_router_routes_responses_turns( - servers, use_harmony): + servers, use_harmony, monkeypatch): """A later turn of a Responses conversation follows its cached prefix.""" + if use_harmony: + # Load the Harmony vocab from the datasets root instead of downloading it. + cache_dir = os.path.join(llm_datasets_root(), "tiktoken_vocab") + monkeypatch.setenv("TIKTOKEN_RS_CACHE_DIR", cache_dir) + monkeypatch.setenv("TIKTOKEN_ENCODINGS_BASE", cache_dir) router = KvCacheAwareRouter(server_role=None, servers=servers, tokens_per_block=4, diff --git a/tests/unittest/llmapi/apps/_test_openai_perf_metrics.py b/tests/unittest/llmapi/apps/_test_openai_perf_metrics.py index e8ab10b13c2c..59d4bee3fe2e 100644 --- a/tests/unittest/llmapi/apps/_test_openai_perf_metrics.py +++ b/tests/unittest/llmapi/apps/_test_openai_perf_metrics.py @@ -19,6 +19,11 @@ logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) +# The streaming Responses tests cap the output at 2 tokens, so the stream +# usually ends incomplete; either terminal event ends it. +_RESPONSES_TERMINAL_EVENT = re.compile( + r"^event: response\.(completed|incomplete)$", re.MULTILINE) + @pytest.fixture(scope="module", ids=["Qwen3-0.6B"]) def model_name(): @@ -233,7 +238,7 @@ def test_responses_streaming_metrics_require_request_opt_in( json=payload, timeout=120) assert response.status_code == 200 - assert "event: response.completed" in response.text + assert _RESPONSES_TERMINAL_EVENT.search(response.text) assert f"event: {perf_metrics.SSE_METRICS_EVENT}" not in response.text wait_for_perf_metrics_jsonl(perf_metrics_output_dir, expected_count=num_existing_records + 1) @@ -245,7 +250,7 @@ def test_responses_streaming_metrics_require_request_opt_in( timeout=120, ) assert response.status_code == 200 - assert "event: response.completed" in response.text + assert _RESPONSES_TERMINAL_EVENT.search(response.text) assert f"event: {perf_metrics.SSE_METRICS_EVENT}" in response.text expected_count = num_existing_records + 2 records = wait_for_perf_metrics_jsonl(perf_metrics_output_dir, From f487fa642c1040e9aa92e89c60d69498ccfcfe22 Mon Sep 17 00:00:00 2001 From: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> Date: Sat, 10 Oct 2026 03:19:29 +0000 Subject: [PATCH 31/43] [None][fix] Responses: context-only prompt ids with postprocessing workers; route developer messages as the worker renders them - A context-only /v1/responses reply built by a postprocessing worker had no prompt_token_ids, because the worker's result carries no reference to its request, so the generation worker had to render the prompt again. The handler now takes them from the promise, as the chat path does. - The KV-cache-aware router tokenized a Responses request without the worker's developer-as-system rewrite, so for a chat template with no developer role (Qwen3, for one) the routing tokens diverged from the worker's at the first message and a Codex turn lost its cache affinity. The router now applies the same rewrite. Tests: a context-only request on a server with postprocessing workers returns the prompt ids, plain and base64; with Qwen3-0.6B the router's tokens for a developer message equal the worker's. Both fail before this change. Signed-off-by: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> --- tensorrt_llm/serve/openai_server.py | 7 ++ tensorrt_llm/serve/router_utils.py | 13 +++- .../test_disagg_responses_api.py | 72 +++++++++++++++++++ tests/unittest/disaggregated/test_router.py | 32 ++++++++- 4 files changed, 121 insertions(+), 3 deletions(-) diff --git a/tensorrt_llm/serve/openai_server.py b/tensorrt_llm/serve/openai_server.py index d8226fa04018..303b06415904 100644 --- a/tensorrt_llm/serve/openai_server.py +++ b/tensorrt_llm/serve/openai_server.py @@ -3149,6 +3149,13 @@ async def create_response( num_prompt_tokens=args.num_prompt_tokens, tokenizer=self.tokenizer, ) + # A postprocessing worker's result has no prompt token ids, and the + # context-only handoff needs them, so take them from the promise as + # the chat path does. + if (request.disaggregated_params is not None + and request.disaggregated_params.request_type + == "context_only"): + response.prompt_token_ids = promise.prompt_token_ids await self._extract_metrics(promise, raw_request) return response diff --git a/tensorrt_llm/serve/router_utils.py b/tensorrt_llm/serve/router_utils.py index 03d95584b8b0..614f0302fd94 100644 --- a/tensorrt_llm/serve/router_utils.py +++ b/tensorrt_llm/serve/router_utils.py @@ -350,10 +350,19 @@ def _tokenize_responses_request(self, request: ResponsesRequest) -> list[int]: return responses_utils._render_for_completion( responses_utils._construct_harmony_messages(request, None) ) + tools = responses_utils._get_chat_completion_function_tools(request.tools) + # As on the worker, developer messages render as system for a chat + # template that has no developer role. + messages = responses_utils._render_developer_as_system( + responses_utils.chat_messages_from_responses_input(request, []), + self._get_tokenizer(request.model), + None, + [tool.model_dump() for tool in tools], + ) chat_request = ChatCompletionRequest.model_construct( model=request.model, - messages=responses_utils.chat_messages_from_responses_input(request, []), - tools=responses_utils._get_chat_completion_function_tools(request.tools), + messages=messages, + tools=tools, chat_template_kwargs=responses_utils.reasoning_chat_template_kwargs(request), injected_chat_template_kwargs=list( responses_utils.reasoning_injected_chat_template_keys(request) diff --git a/tests/unittest/disaggregated/test_disagg_responses_api.py b/tests/unittest/disaggregated/test_disagg_responses_api.py index b237ce88695c..ea873601daac 100644 --- a/tests/unittest/disaggregated/test_disagg_responses_api.py +++ b/tests/unittest/disaggregated/test_disagg_responses_api.py @@ -227,6 +227,78 @@ def test_a_context_only_response_carries_the_handoff(use_harmony): assert (response.finish_reason, response.prompt_token_ids) == ("length", [1, 2]) +@pytest.mark.parametrize("as_b64", [False, True]) +def test_a_context_worker_with_postprocessing_workers_returns_the_prompt(monkeypatch, as_b64): + """A postprocessing worker's result has no prompt ids; the handoff still does.""" + import tensorrt_llm.serve.openai_server as server_module + from tensorrt_llm.serve.openai_server import OpenAIServer + + request = ResponsesRequest( + model="m", + input="hi", + store=False, + disaggregated_params=DisaggregatedParams( + request_type="context_only", return_prompt_token_ids_b64=as_b64 + ), + ) + output = SimpleNamespace( + index=0, text="", token_ids=[7], finish_reason="length", disaggregated_params=None + ) + # What a postprocessing worker builds: its result carries no prompt_token_ids. + postprocessed = create_response_non_store( + generation_result=SimpleNamespace(outputs=[output], cached_tokens=0), + request=request, + sampling_params=request.to_sampling_params(), + model_name="m", + use_harmony=False, + num_prompt_tokens=3, + ) + assert postprocessed.prompt_token_ids is None + + async def nothing(*args, **kwargs): + return None + + promise = SimpleNamespace( + outputs=[SimpleNamespace(_postprocess_result=postprocessed)], + prompt_token_ids=[5, 6, 7], + aresult=nothing, + ) + server = object.__new__(OpenAIServer) + server.model = "m" + server.enable_store = False + server._is_visual_gen = False + server.use_harmony = False + server.tokenizer = None + server.model_config = None + server.processor = None + server.tool_parser = None + server.metrics_collector = None + server.conversation_store = None + server.generator = SimpleNamespace( + args=SimpleNamespace(reasoning_parser=None, num_postprocess_workers=1), + generate_async=lambda **kwargs: promise, + ) + server._validate_internal_disagg_request = lambda *args: None + server.await_disconnected = nothing + server._extract_metrics = nothing + + async def preprocess(**kwargs): + return [5, 6, 7], kwargs["request"].to_sampling_params() + + monkeypatch.setattr(server_module, "responses_api_request_preprocess", preprocess) + raw_request = SimpleNamespace( + state=SimpleNamespace(), headers={}, url=SimpleNamespace(path="/v1/responses") + ) + + body = json.loads(asyncio.run(server.openai_responses(request, raw_request)).body) + + if as_b64: + ids = np.frombuffer(base64.b64decode(body["prompt_token_ids_b64"]), dtype=np.int32) + assert (ids.tolist(), body["prompt_token_ids"]) == ([5, 6, 7], None) + else: + assert body["prompt_token_ids"] == [5, 6, 7] + + # --------------------------------------------------------------------------- # Streaming a request the context phase finished # --------------------------------------------------------------------------- diff --git a/tests/unittest/disaggregated/test_router.py b/tests/unittest/disaggregated/test_router.py index 0e91d49c7f7f..13daa8ae3128 100644 --- a/tests/unittest/disaggregated/test_router.py +++ b/tests/unittest/disaggregated/test_router.py @@ -15,7 +15,7 @@ import msgpack import numpy as np import pytest -from utils.llm_data import llm_datasets_root +from utils.llm_data import llm_datasets_root, llm_models_root from tensorrt_llm.llmapi.disagg_utils import RouterConfig from tensorrt_llm.runtime.kv_cache_hash import (get_cache_salt_id, @@ -2634,6 +2634,36 @@ async def test_kv_cache_aware_router_routes_responses_turns( assert router._tokenize(turn2) == [worker_ids] +@pytest.mark.asyncio +async def test_responses_routing_tokens_match_the_worker_for_a_developer_message( + servers): + """Qwen3's chat template has no developer role; both sides render it as system.""" + from transformers import AutoConfig + + from tensorrt_llm.tokenizer import tokenizer_factory + + model_dir = os.path.join(llm_models_root(), "Qwen3", "Qwen3-0.6B") + router = KvCacheAwareRouter(server_role=None, + servers=servers, + tokens_per_block=4, + tokenizer_dir=model_dir, + use_harmony=False) + request = ResponsesRequest(model="m", + input=[ + _message("developer", "Answer in French."), + _message("user", "plan a trip") + ]) + worker_ids, _ = await request_preprocess( + request, + None, + None, + enable_store=False, + use_harmony=False, + tokenizer=tokenizer_factory(model_dir), + model_config=AutoConfig.from_pretrained(model_dir)) + assert router._tokenize(request) == [worker_ids] + + @pytest.mark.asyncio async def test_conversation_router_tells_responses_conversations_apart(): router = ConversationRouter(server_role=None, From 363e4be2610f8dd12f8dc1056bfc35f33e067732 Mon Sep 17 00:00:00 2001 From: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> Date: Sat, 10 Oct 2026 03:28:05 +0000 Subject: [PATCH 32/43] [None][test] Responses: a list-mode context-only reply carries no base64 prompt ids Signed-off-by: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> --- tests/unittest/disaggregated/test_disagg_responses_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unittest/disaggregated/test_disagg_responses_api.py b/tests/unittest/disaggregated/test_disagg_responses_api.py index ea873601daac..d12c4417a2b6 100644 --- a/tests/unittest/disaggregated/test_disagg_responses_api.py +++ b/tests/unittest/disaggregated/test_disagg_responses_api.py @@ -296,7 +296,7 @@ async def preprocess(**kwargs): ids = np.frombuffer(base64.b64decode(body["prompt_token_ids_b64"]), dtype=np.int32) assert (ids.tolist(), body["prompt_token_ids"]) == ([5, 6, 7], None) else: - assert body["prompt_token_ids"] == [5, 6, 7] + assert (body["prompt_token_ids"], body["prompt_token_ids_b64"]) == ([5, 6, 7], None) # --------------------------------------------------------------------------- From 4399d535dd39e46b2d756091e0e014095ae97d4b Mon Sep 17 00:00:00 2001 From: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> Date: Sat, 10 Oct 2026 20:12:39 +0000 Subject: [PATCH 33/43] [None][fix] Responses: address review of the input, stream and disagg paths - A chat template is now asked whether it renders a developer message by rendering one, rather than by finding the word "developer" in its source. A template that names the role only to reject it, or mentions it in a comment, no longer keeps the role and loses the message. When a template renders neither a developer nor a system message the roles are left alone. Results are cached per template. - At the end of a stream, the text the whole-text parse reads beyond the streamed message is released even when the parse stripped leading whitespace the stream already sent; a real divergence is logged instead of dropped silently. - An encrypted_content part is read as text only when it is not one long base64 token, so a real ciphertext (OpenAI's Fernet tokens) is not put into the prompt. - The unreachable storing stream finalizer (get_final_response and process_streaming_events, which main removed in #16262) is deleted, so it cannot be wired back in its stale form. Streamed responses are not stored. - Generation-first scheduling of a Responses request is tested, streaming and not. - The router's character-versus-token weighting is explained where it is computed. Signed-off-by: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> --- tensorrt_llm/serve/responses_utils.py | 138 ++++++++---------- tensorrt_llm/serve/router_utils.py | 5 +- .../test_disagg_responses_api.py | 87 +++++++++++ .../apps/test_responses_input_preprocess.py | 40 ++++- ...test_responses_stream_final_consistency.py | 33 +++++ 5 files changed, 225 insertions(+), 78 deletions(-) diff --git a/tensorrt_llm/serve/responses_utils.py b/tensorrt_llm/serve/responses_utils.py index d7db885ce726..66270ed4147b 100644 --- a/tensorrt_llm/serve/responses_utils.py +++ b/tensorrt_llm/serve/responses_utils.py @@ -5,6 +5,7 @@ import asyncio import json import os +import re import time import uuid # yapf: disable @@ -15,6 +16,7 @@ from typing import (Any, Callable, List, Literal, Optional, OrderedDict, Tuple, Union) +from jinja2.exceptions import TemplateError from openai.types.responses import (ResponseCompletedEvent, ResponseContentPartAddedEvent, ResponseContentPartDoneEvent, @@ -48,6 +50,7 @@ StreamState, SystemContent, TextContent, ToolDescription, load_harmony_encoding) from transformers import AutoProcessor, PretrainedConfig +from transformers.utils.chat_template_utils import render_jinja_template from tensorrt_llm._utils import \ get_steady_clock_now_in_seconds # noqa: F401 (re-export) @@ -715,6 +718,11 @@ def finish_reason_mapping(finish_reason: str) -> str: f"Unhandled finish reason {finish_reason!r} in finish_reason_mapping") +# One long base64/base64url token, such as a real ciphertext (OpenAI's +# encrypted content is a Fernet token, gAAAAA...), rather than readable text. +_OPAQUE_TOKEN = re.compile(r"[A-Za-z0-9+/_=-]{32,}") + + def _item_text(item: dict) -> str: """The text carried by an input item, whatever shape it uses.""" content = item.get("content") @@ -728,9 +736,10 @@ def _item_text(item: dict) -> str: text = part.get("text") if not text and part.get("type") == "encrypted_content": # Some clients carry readable text in an `encrypted_content` - # part; only a string value is taken as text. + # part; a string is taken as text unless it is an opaque token. value = part.get("encrypted_content") - if isinstance(value, str): + if isinstance(value, + str) and not _OPAQUE_TOKEN.fullmatch(value): text = value if text: parts.append(text) @@ -802,8 +811,9 @@ def _render_developer_as_system( ) -> list[ChatCompletionMessageParam]: """Render ``developer`` messages as ``system`` where the template lacks them. - A chat template without a ``developer`` branch renders those messages to - nothing. The DeepSeek tokenizers render ``developer`` themselves. + A chat template that does not render a ``developer`` message (it has no + branch for the role, or rejects it) loses that message unless it is sent + as ``system``. The DeepSeek tokenizers render ``developer`` themselves. """ if not any(message.get("role") == "developer" for message in messages): return messages @@ -814,7 +824,11 @@ def _render_developer_as_system( processor, chat_template=None, tools=tools) - if not isinstance(template, str) or "developer" in template: + if not isinstance(template, str) or _template_renders( + template, "developer"): + return messages + if not _template_renders(template, "system"): + # The probe cannot tell what the template does; leave the roles alone. return messages return [{ **message, "role": "system" @@ -822,6 +836,39 @@ def _render_developer_as_system( for message in messages] +_ROLE_PROBE = "trtllm-chat-template-role-probe" +_TEMPLATE_RENDERS_ROLE: dict[tuple[str, str], bool] = {} + + +def _template_renders(template: str, role: str) -> bool: + """Whether ``template`` renders the content of a message with ``role``. + + Decided by rendering one such message rather than by searching the source + for the role's name: a template may name a role only to reject it. A + template that raises renders nothing. Cached per template and role. + """ + key = (template, role) + renders = _TEMPLATE_RENDERS_ROLE.get(key) + if renders is None: + conversation = [{ + "role": role, + "content": _ROLE_PROBE + }, { + "role": "user", + "content": "hi" + }] + try: + rendered, _ = render_jinja_template(conversations=[conversation], + chat_template=template, + add_generation_prompt=True) + renders = _ROLE_PROBE in rendered[0] + except (TemplateError, TypeError, KeyError, IndexError, ValueError, + AttributeError): + renders = False + _TEMPLATE_RENDERS_ROLE[key] = renders + return renders + + def _response_output_item_to_chat_completion_message( item: Union[dict, ResponseInputOutputItem] ) -> Optional[ChatCompletionMessageParam]: @@ -1514,9 +1561,18 @@ def _whole_text_tool_calls( content, False) streamed = "".join(item.text for item in helper.emitted_item_ids if item.item_type == "message") - owed = normal_text[len(streamed):] if normal_text.startswith( - streamed) else "" - return owed, calls + if normal_text.startswith(streamed): + return normal_text[len(streamed):], calls + # The whole-text parse may strip edge whitespace the stream already sent. + text, sent = normal_text.lstrip(), streamed.lstrip() + if text.startswith(sent): + return text[len(sent):], calls + if normal_text.strip() != streamed.strip(): + logger.warning( + f"the whole-text parse reads {len(normal_text)} character(s) of " + f"message text that do not continue the {len(streamed)} the stream " + f"published; the stream releases none of it") + return "", calls def _effective_tool_parser(tool_parser_id: Optional[str], @@ -2827,33 +2883,6 @@ def get_initial_responses(self) -> List[str]: in_progress.sequence_number = 1 return [_format_sse_event(created), _format_sse_event(in_progress)] - async def get_final_response( - self, - final_res: RequestOutput, - num_prompt_tokens: Optional[int] = None, - ) -> str: - final_response = await create_response( - generator=None, - request=self.request, - sampling_params=self.sampling_params, - model_name=self.model_name, - conversation_store=self.conversation_store, - generation_result=final_res, - enable_store=self.enable_store, - use_harmony=self.use_harmony, - create_time=self.response_creation_time, - reasoning_parser=self.reasoning_parser, - tool_parser=self.tool_parser, - num_prompt_tokens=num_prompt_tokens, - ) - - return self._send_event( - ResponseCompletedEvent( - type="response.completed", - sequence_number=-1, - response=final_response.model_dump(), - )) - def get_final_response_non_store( self, final_res: RequestOutput, @@ -3061,45 +3090,6 @@ async def guard_responses_stream( raise -async def process_streaming_events( - generator, - request: ResponsesRequest, - sampling_params: SamplingParams, - model_name: str, - conversation_store: ConversationHistoryStore, - enable_store: bool = False, - use_harmony: bool = True, - create_time: Optional[int] = None, - reasoning_parser: Optional[str] = None, - tool_parser: Optional[str] = None, -) -> AsyncGenerator[str, None]: - streaming_processor = ResponsesStreamingProcessor( - request=request, - sampling_params=sampling_params, - model_name=model_name, - create_time=create_time, - conversation_store=conversation_store, - enable_store=enable_store, - use_harmony=use_harmony, - reasoning_parser=reasoning_parser, - tool_parser=tool_parser, - ) - - initial_responses = streaming_processor.get_initial_responses() - for initial_response in initial_responses: - yield initial_response - - async for res in generator: - final_res = res - events = streaming_processor.process_single_output(res) - for event in events: - yield event - - final_response = await streaming_processor.get_final_response(final_res) - - yield final_response - - class ServerArrivalTimeMiddleware: """ Custom ASGI middleware to track server arrival time. diff --git a/tensorrt_llm/serve/router_utils.py b/tensorrt_llm/serve/router_utils.py index 614f0302fd94..d9835540b98e 100644 --- a/tensorrt_llm/serve/router_utils.py +++ b/tensorrt_llm/serve/router_utils.py @@ -145,7 +145,10 @@ def get_request_num_tokens(request: Optional[OpenAIRequest]) -> int: if isinstance(request, ResponsesRequest): # Without relayed token ids the text is counted in characters, as a - # string completion prompt is below. + # string completion prompt is below, which weighs a request several + # times what its token count would. A router sees one kind only: the + # context router routes before any ids exist, and the generation + # router gets them relayed (context-first) or never (generation-first). token_ids = request.relayed_prompt_token_ids() return len(responses_request_text(request) if token_ids is None else token_ids) diff --git a/tests/unittest/disaggregated/test_disagg_responses_api.py b/tests/unittest/disaggregated/test_disagg_responses_api.py index d12c4417a2b6..814b88c48b0d 100644 --- a/tests/unittest/disaggregated/test_disagg_responses_api.py +++ b/tests/unittest/disaggregated/test_disagg_responses_api.py @@ -38,6 +38,7 @@ from tensorrt_llm.serve.openai_protocol import ( ChatCompletionResponse, DisaggregatedParams, + DisaggScheduleStyle, ModelCard, ModelList, PromptTokensDetails, @@ -299,6 +300,92 @@ async def preprocess(**kwargs): assert (body["prompt_token_ids"], body["prompt_token_ids_b64"]) == ([5, 6, 7], None) +# --------------------------------------------------------------------------- +# Generation-first scheduling +# --------------------------------------------------------------------------- + + +class _RecordingClient: + def __init__(self, response): + self.requests = [] + self._response = response + + async def send_request(self, request, server=None, hooks=None, req_id=None): + self.requests.append(request.model_copy(deep=True)) + return self._response + + +def _gen_first_service(ctx_response, gen_response): + async def disagg_request_id(): + return 11 + + async def ctx_server(request, req_id=None): + return "ctx0", None + + service = _service( + _schedule_style=DisaggScheduleStyle.GENERATION_FIRST, + _coordinator=SimpleNamespace(get_disagg_request_id=disagg_request_id), + _ctx_router=SimpleNamespace(get_next_server=ctx_server), + _ctx_client=_RecordingClient(ctx_response), + _gen_client=_RecordingClient(gen_response), + _tokids_ctxbytes=False, + _subagent_affinity_scope="both", + ) + service._send_disagg_request = service._send_disagg_request_gen_first + return service + + +def _assert_generation_first_handoff(service): + (ctx_request,) = service._ctx_client.requests + (gen_request,) = service._gen_client.requests + assert (ctx_request.disaggregated_params.request_type, ctx_request.stream) == ( + "context_only", + False, + ) + params = gen_request.disaggregated_params + assert (params.request_type, params.schedule_style, params.disagg_request_id) == ( + "generation_only", + DisaggScheduleStyle.GENERATION_FIRST, + 11, + ) + # Nothing is relayed: the generation worker renders the prompt itself. + assert gen_request.relayed_prompt_token_ids() is None + assert gen_request.input == ctx_request.input + + +def test_a_generation_first_response_comes_from_the_generation_worker(): + final = _responses_response(finish_reason="stop", status="completed") + service = _gen_first_service(ctx_response=_responses_response(), gen_response=final) + + response = asyncio.run(service.openai_responses(ResponsesRequest(model="m", input="hi"))) + + assert response is final + _assert_generation_first_handoff(service) + + +def test_a_generation_first_stream_relays_the_generation_frames_in_order(): + frames = [ + b"event: response.created\ndata: {}\n\n", + b"event: response.output_text.delta\ndata: {}\n\n", + b"event: response.completed\ndata: {}\n\n", + ] + + async def gen_frames(): + for frame in frames: + yield frame + + service = _gen_first_service(ctx_response=_responses_response(), gen_response=gen_frames()) + + async def relay(): + stream = await service.openai_responses( + ResponsesRequest(model="m", input="hi", stream=True) + ) + return [frame async for frame in stream] + + assert asyncio.run(relay()) == frames + _assert_generation_first_handoff(service) + + # --------------------------------------------------------------------------- # Streaming a request the context phase finished # --------------------------------------------------------------------------- diff --git a/tests/unittest/llmapi/apps/test_responses_input_preprocess.py b/tests/unittest/llmapi/apps/test_responses_input_preprocess.py index 738aef9f3998..c0fa188ac829 100644 --- a/tests/unittest/llmapi/apps/test_responses_input_preprocess.py +++ b/tests/unittest/llmapi/apps/test_responses_input_preprocess.py @@ -432,6 +432,7 @@ def test_meaningful_values_survive_the_null_scrub(): {"type": "input_text", "text": "Payload:"}, {"type": "encrypted_content", "encrypted_content": "write the file"}, {"type": "encrypted_content", "encrypted_content": {"blob": "aGk="}}, + {"type": "encrypted_content", "encrypted_content": "gAAAAABo" + "Zm9v" * 16}, ], }, {"role": "user", "content": "Payload:\nwrite the file"}, @@ -553,14 +554,47 @@ def get_chat_template(self, chat_template, tools=None): _DEVELOPER_TURN = [{"role": "developer", "content": "brief"}, {"role": "user", "content": "q"}] +def _template(body): + return _TemplateTokenizer("{% for m in messages %}" + body + "{% endfor %}") + + @pytest.mark.parametrize( "tokenizer, role", [ - (_TemplateTokenizer("{% if m.role == 'system' %}{{ m.content }}{% endif %}"), "system"), - (_TemplateTokenizer("{% if m.role in ('system', 'developer') %}{% endif %}"), "developer"), + (_template("{% if m.role in ('system', 'user') %}{{ m.content }}{% endif %}"), "system"), + ( + _template( + "{% if m.role in ('system', 'developer', 'user') %}{{ m.content }}{% endif %}" + ), + "developer", + ), + # Naming the role is not handling it. + ( + _template( + "{% if m.role == 'developer' %}" + "{{ raise_exception('Unsupported role: developer') }}{% endif %}{{ m.content }}" + ), + "system", + ), + ( + _TemplateTokenizer( + "{# developer messages are not supported #}{% for m in messages %}" + "{% if m.role != 'developer' %}{{ m.content }}{% endif %}{% endfor %}" + ), + "system", + ), + # A template that renders neither role tells nothing; the roles stay. + (_template("{{ m.content.missing.attribute }}"), "developer"), (DeepseekV4Tokenizer.__new__(DeepseekV4Tokenizer), "developer"), ], - ids=["no_developer_branch", "developer_branch", "deepseek_v4"], + ids=[ + "no_developer_branch", + "developer_branch", + "rejects_developer", + "developer_only_in_a_comment", + "renders_neither", + "deepseek_v4", + ], ) def test_developer_renders_as_system_only_where_the_template_lacks_it(tokenizer, role): messages = _render_developer_as_system(_DEVELOPER_TURN, tokenizer, None, None) diff --git a/tests/unittest/llmapi/apps/test_responses_stream_final_consistency.py b/tests/unittest/llmapi/apps/test_responses_stream_final_consistency.py index f0d8339df79f..90612ea34547 100644 --- a/tests/unittest/llmapi/apps/test_responses_stream_final_consistency.py +++ b/tests/unittest/llmapi/apps/test_responses_stream_final_consistency.py @@ -32,6 +32,7 @@ ResponsesStreamingProcessor, StreamedItem, _create_output_content, + _whole_text_tool_calls, create_response_non_store, ) @@ -326,3 +327,35 @@ def test_a_non_streamed_response_explains_an_incomplete_status(finish_reason, st num_prompt_tokens=2, ).model_dump() assert (response["status"], response["incomplete_details"]) == (status, details) + + +@pytest.mark.parametrize( + "streamed, whole, owed, warned", + [ + ("Hello", "Hello world", " world", False), + # The whole-text parse dropped the leading whitespace the stream sent. + (" Hello", "Hello world", " world", False), + ("Hello ", "Hello", "", False), + ("Hello", "Goodbye", "", True), + ], +) +def test_the_stream_releases_what_the_whole_text_parse_reads_beyond_it( + streamed, whole, owed, warned +): + helper = SimpleNamespace(emitted_item_ids=[StreamedItem("message", "msg_1", streamed)]) + request = ResponsesRequest(model="test-model", input="hi", stream=True) + with ( + patch( + "tensorrt_llm.serve.responses_utils._apply_reasoning_parser", + return_value=(whole, ""), + ), + patch( + "tensorrt_llm.serve.responses_utils._apply_tool_parser", + return_value=(whole, []), + ), + patch("tensorrt_llm.serve.responses_utils.logger") as logger, + ): + released, calls = _whole_text_tool_calls( + SimpleNamespace(index=0, text=whole), request, helper, None, "qwen3", [] + ) + assert (released, calls, logger.warning.called) == (owed, [], warned) From 0c38b38a9cc30d53ada175c8869de00e74f5c3ec Mon Sep 17 00:00:00 2001 From: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> Date: Sat, 10 Oct 2026 20:12:57 +0000 Subject: [PATCH 34/43] [None][doc] Responses API on the disaggregated server; release notes disagg-serving.md gains a Responses API section: the endpoints served, the two schedule styles, previous_response_id rejected and store ignored, and the limitations that apply on every server (unenforced parallel_tool_calls=false and tool_choice, images replaced by a text placeholder). The release notes list the feature and the response.incomplete terminal for a response cut short by max_output_tokens. Signed-off-by: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> --- docs/source/features/disagg-serving.md | 12 ++++++++++++ docs/source/release-notes.md | 6 ++++++ 2 files changed, 18 insertions(+) diff --git a/docs/source/features/disagg-serving.md b/docs/source/features/disagg-serving.md index 2e06df609aed..3c46c0f03b16 100644 --- a/docs/source/features/disagg-serving.md +++ b/docs/source/features/disagg-serving.md @@ -224,6 +224,18 @@ curl http://localhost:8000/v1/completions \ }' -w "\n" ``` +#### Responses API + +The disaggregated server also serves `/v1/responses`, with both `context_first` and `generation_first` scheduling, and `/v1/models`, which it answers from a context worker. A streamed response carries the generation worker's events unchanged. Unlike a single `trtllm-serve` instance, the disaggregated server keeps no response history: + +- A request with `previous_response_id` is rejected with a 400, because a stored response lives in one worker's memory and a later request may reach another worker. Send the earlier turns in `input` instead. +- `store` is ignored: nothing is persisted, and the server logs a warning once when a client asks for it. + +Two limitations apply to `/v1/responses` on every server, disaggregated or not: + +- `parallel_tool_calls: false`, and a `tool_choice` other than `auto` or `none` on non-Harmony models, are accepted but not enforced. The server logs a warning once. +- An image in the input, in a message or in a tool output, is replaced by a text placeholder that names what was dropped, since the endpoint accepts text only. + #### Launching disaggregated servers on SLURM clusters Please refer to [Disaggregated Inference Benchmark Scripts](source:examples/disaggregated/slurm). diff --git a/docs/source/release-notes.md b/docs/source/release-notes.md index c974a4648385..dcac5edfe6f0 100644 --- a/docs/source/release-notes.md +++ b/docs/source/release-notes.md @@ -6,8 +6,14 @@ All published functionality in the Release Notes has been fully tested and verif ## TensorRT-LLM Release 1.3 +### Key Features and Enhancements + +- Disaggregated serving supports the Responses API: `trtllm-serve disaggregated` serves `/v1/responses` with context-first and generation-first scheduling, and `/v1/models`. `previous_response_id` is rejected and `store` is ignored there, since no response history is kept. See [Disaggregated Serving](features/disagg-serving.md#responses-api). + ### API Changes +- `/v1/responses`: a response cut short by `max_output_tokens` now reports `incomplete_details.reason` `max_output_tokens`, and when streamed it ends with `response.incomplete` instead of `response.completed`, as in the OpenAI API. + - **[DEPRECATION]** The TRITON MoE backend (`TritonFusedMoE`, `moe_config.backend="TRITON"`) is deprecated as of TensorRT-LLM 1.3 (2026-09) and will be removed after the 3-month migration period. Its only remaining role is a modest performance edge for GPT-OSS on Hopper with `W4A16_MXFP4` β€” the single configuration `AUTO` resolves to TRITON, and the format an MXFP4 GPT-OSS checkpoint takes on SM90. As the model set and the supported platforms keep growing, a single-scenario MoE path is no longer worth its maintenance cost. `moe_config.backend="CUTLASS"` replaces it functionally on Hopper: it serves `W4A16_MXFP4` on SM90 along with the unquantized BF16 and FP8 per-tensor paths, and MoE backend resolution already degrades to it automatically when TRITON declines a layer. During the migration period TRITON keeps working and logs a one-time warning. See the [deprecation policy](https://github.com/NVIDIA/TensorRT-LLM#deprecation-policy). ## TensorRT-LLM Release 1.2 From 1ed6fb2e8e114ec821ad2b80e8a3e1c6432d2201 Mon Sep 17 00:00:00 2001 From: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> Date: Sat, 10 Oct 2026 20:29:00 +0000 Subject: [PATCH 35/43] [None][fix] Responses: folding a tool call does not change the request's input For an untyped input item whose content is not a list, the converted message is the request's own dict, and folding a following function_call into it wrote tool_calls into request.input. Every later conversion of the same request - the router's, then the worker's after the forwarded body - folded the calls again and duplicated them. The assistant message is now replaced by a copy. Signed-off-by: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> --- tensorrt_llm/serve/responses_utils.py | 9 +++++---- .../llmapi/apps/test_responses_input_preprocess.py | 11 +++++++++++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/tensorrt_llm/serve/responses_utils.py b/tensorrt_llm/serve/responses_utils.py index e7faedaaf4ba..b15837f51db2 100644 --- a/tensorrt_llm/serve/responses_utils.py +++ b/tensorrt_llm/serve/responses_utils.py @@ -1035,10 +1035,11 @@ def _fold_tool_calls_into_open_assistant_turn( last = messages[-1] if last.get("role") != "assistant": return False - # Rebuilt: the target's list may be None or owned by the caller. - last["tool_calls"] = [ - *(last.get("tool_calls") or []), *message["tool_calls"] - ] + # Replaced, not updated: the target may be the caller's own input item. + messages[-1] = { + **last, "tool_calls": + [*(last.get("tool_calls") or []), *message["tool_calls"]] + } return True diff --git a/tests/unittest/llmapi/apps/test_responses_input_preprocess.py b/tests/unittest/llmapi/apps/test_responses_input_preprocess.py index c0fa188ac829..e10cd96a6efa 100644 --- a/tests/unittest/llmapi/apps/test_responses_input_preprocess.py +++ b/tests/unittest/llmapi/apps/test_responses_input_preprocess.py @@ -672,6 +672,17 @@ def test_calls_fold_onto_a_reasoning_message(): assert [c["id"] for c in messages[0]["tool_calls"]] == ["call_0", "call_1"] +def test_folding_leaves_the_request_unchanged(): + request = ResponsesRequest( + model="m", + input=[{"role": "assistant", "content": "Running."}, _function_call_item(0)], + ) + for _ in range(2): + messages = asyncio.run(_create_input_messages(request=request, prev_msgs=[])) + assert [c["id"] for c in messages[0]["tool_calls"]] == ["call_0"] + assert "tool_calls" not in request.input[0] + + def test_calls_never_fold_into_replayed_history(): request = ResponsesRequest(model="m", input=[_function_call_item(0)]) history = [{"role": "assistant", "content": "a stored turn"}] From 571a1e32281b3a0250d8f10b74da335f8b0e8c94 Mon Sep 17 00:00:00 2001 From: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> Date: Sun, 11 Oct 2026 14:02:15 +0000 Subject: [PATCH 36/43] [None][fix] Responses: reject a malformed prompt_token_ids_b64 at validation The relayed prompt is decoded while the router picks a server and in the worker's preprocessing, both after request validation, so a client value that is not base64, or not a whole number of int32 ids, failed there with an internal error. ResponsesRequest now checks the field when the request is validated, so such a request gets a 400. Signed-off-by: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> --- tensorrt_llm/serve/openai_protocol.py | 17 +++++++++++++++++ .../disaggregated/test_disagg_responses_api.py | 14 ++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/tensorrt_llm/serve/openai_protocol.py b/tensorrt_llm/serve/openai_protocol.py index 3b70f7edcb28..69ed2f4023bc 100644 --- a/tensorrt_llm/serve/openai_protocol.py +++ b/tensorrt_llm/serve/openai_protocol.py @@ -1685,6 +1685,23 @@ def _materialize_lazy_item_content(cls, value): # doc: end-responses-extra-params + @field_validator("prompt_token_ids_b64") + @classmethod + def _check_prompt_token_ids_b64(cls, value: Optional[str]) -> Optional[str]: + # Decoded during routing and preprocessing, after validation; a bad + # value must be a 400 here rather than an internal error there. + if not value: + return value + try: + raw = base64.b64decode(value, validate=True) + except ValueError as error: + raise ValueError( + "prompt_token_ids_b64 is not valid base64") from error + if len(raw) % 4: + raise ValueError( + "prompt_token_ids_b64 is not a buffer of int32 token ids") + return value + def relayed_prompt_token_ids(self) -> Optional[List[int]]: """The prompt the orchestrator relayed from the context worker, if any.""" if self.prompt_token_ids is not None: diff --git a/tests/unittest/disaggregated/test_disagg_responses_api.py b/tests/unittest/disaggregated/test_disagg_responses_api.py index 814b88c48b0d..477676fe2806 100644 --- a/tests/unittest/disaggregated/test_disagg_responses_api.py +++ b/tests/unittest/disaggregated/test_disagg_responses_api.py @@ -28,6 +28,7 @@ import numpy as np import pytest from fastapi import HTTPException +from pydantic import ValidationError from tensorrt_llm.serve.openai_disagg_server import OpenAIDisaggServer from tensorrt_llm.serve.openai_disagg_service import ( @@ -167,6 +168,19 @@ def test_the_generation_worker_uses_the_relayed_prompt(fields, expected): assert request.prompt_token_ids == fields.get("prompt_token_ids") +@pytest.mark.parametrize( + "encoded, reason", + [ + ("abc", "not valid base64"), + ("AQ==!", "not valid base64"), + (base64.b64encode(b"\x01\x02\x03").decode("ascii"), "int32"), + ], +) +def test_a_malformed_relayed_prompt_is_a_validation_error(encoded, reason): + with pytest.raises(ValidationError, match=reason): + ResponsesRequest(model="m", input="hi", prompt_token_ids_b64=encoded) + + @pytest.mark.parametrize( "usage, expected", [ From 9fb0942fb376f15225ceeb26bcce2c40f1d4f80d Mon Sep 17 00:00:00 2001 From: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> Date: Mon, 12 Oct 2026 08:38:13 +0000 Subject: [PATCH 37/43] [None][doc] Responses: file the release notes under Release 1.4 Main now carries a Release 1.4 section, the release this change ships in. The disaggregated Responses API entry and the response.incomplete API change move there from Release 1.3. Signed-off-by: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> --- docs/source/release-notes.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/docs/source/release-notes.md b/docs/source/release-notes.md index 16c75fb87c39..e1cd8d7d1e6f 100644 --- a/docs/source/release-notes.md +++ b/docs/source/release-notes.md @@ -8,24 +8,22 @@ All published functionality in the Release Notes has been fully tested and verif ### Key Features and Enhancements +- Disaggregated serving supports the Responses API: `trtllm-serve disaggregated` serves `/v1/responses` with context-first and generation-first scheduling, and `/v1/models`. `previous_response_id` is rejected and `store` is ignored there, since no response history is kept. See [Disaggregated Serving](features/disagg-serving.md#responses-api). + ### Infrastructure Changes ### API Changes +- `/v1/responses`: a response cut short by `max_output_tokens` now reports `incomplete_details.reason` `max_output_tokens`, and when streamed it ends with `response.incomplete` instead of `response.completed`, as in the OpenAI API. + ### Fixed Issues ### Known Issues ## TensorRT-LLM Release 1.3 -### Key Features and Enhancements - -- Disaggregated serving supports the Responses API: `trtllm-serve disaggregated` serves `/v1/responses` with context-first and generation-first scheduling, and `/v1/models`. `previous_response_id` is rejected and `store` is ignored there, since no response history is kept. See [Disaggregated Serving](features/disagg-serving.md#responses-api). - ### API Changes -- `/v1/responses`: a response cut short by `max_output_tokens` now reports `incomplete_details.reason` `max_output_tokens`, and when streamed it ends with `response.incomplete` instead of `response.completed`, as in the OpenAI API. - - NVFP4 cold-page compression remains opt-in and disabled by default. When enabled for DeepSeek-V4, NoPE uses NVFP4 and RoPE now defaults to 2FP4 (`nvfp4_residual_dim=64`). Set `skip_rope_quantization=True` to keep RoPE at its original precision, as before, or `nvfp4_residual_dim=0` to use single NVFP4. The active GPU KV-cache dtype is unchanged. See the [NVFP4 cold-page compression example](source:examples/kv_cache_compression/nvfp4_cold_page.md#deepseek-v4). - **[DEPRECATION]** The TRITON MoE backend (`TritonFusedMoE`, `moe_config.backend="TRITON"`) is deprecated as of TensorRT-LLM 1.3 (2026-09) and will be removed after the 3-month migration period. Its only remaining role is a modest performance edge for GPT-OSS on Hopper with `W4A16_MXFP4` β€” the single configuration `AUTO` resolves to TRITON, and the format an MXFP4 GPT-OSS checkpoint takes on SM90. As the model set and the supported platforms keep growing, a single-scenario MoE path is no longer worth its maintenance cost. `moe_config.backend="CUTLASS"` replaces it functionally on Hopper: it serves `W4A16_MXFP4` on SM90 along with the unquantized BF16 and FP8 per-tensor paths, and MoE backend resolution already degrades to it automatically when TRITON declines a layer. During the migration period TRITON keeps working and logs a one-time warning. See the [deprecation policy](https://github.com/NVIDIA/TensorRT-LLM#deprecation-policy). From 2283d923ad205ad21e5287cd4321191f92b97a60 Mon Sep 17 00:00:00 2001 From: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> Date: Tue, 13 Oct 2026 06:15:11 +0000 Subject: [PATCH 38/43] [None][fix] Responses: keep the disaggregated handoff off client requests and responses - ResponsesRequest.relayed_prompt_token_ids() returns the relayed prompt only on a generation-only handoff. A client request that sets prompt_token_ids or prompt_token_ids_b64 used to skip rendering its input, instructions and tools while the response still echoed them; it now renders its own prompt, and the routers count it the same way. - ResponsesResponse leaves its handoff fields (disaggregated_params, prompt_token_ids, prompt_token_ids_b64, finish_reason) out of the serialized body when they are unset, instead of emitting four nulls in every response body, stream event and stored response. A context-only response still carries them to the orchestrator. Signed-off-by: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> --- tensorrt_llm/serve/openai_protocol.py | 28 ++++++++- .../test_disagg_responses_api.py | 59 ++++++++++++++++++- 2 files changed, 83 insertions(+), 4 deletions(-) diff --git a/tensorrt_llm/serve/openai_protocol.py b/tensorrt_llm/serve/openai_protocol.py index 69ed2f4023bc..d323eedcf480 100644 --- a/tensorrt_llm/serve/openai_protocol.py +++ b/tensorrt_llm/serve/openai_protocol.py @@ -1703,7 +1703,15 @@ def _check_prompt_token_ids_b64(cls, value: Optional[str]) -> Optional[str]: return value def relayed_prompt_token_ids(self) -> Optional[List[int]]: - """The prompt the orchestrator relayed from the context worker, if any.""" + """The prompt the orchestrator relayed from the context worker, if any. + + Only a generation-only handoff carries one. On any other request the + fields are ignored, so the prompt is rendered from the request and the + response echoes what was actually prefilled. + """ + params = self.disaggregated_params + if params is None or params.request_type != "generation_only": + return None if self.prompt_token_ids is not None: return self.prompt_token_ids if not self.prompt_token_ids_b64: @@ -1863,6 +1871,13 @@ class ResponseUsage(OpenAIBaseModel): ResponseCodeInterpreterCallCompletedEvent, ] +_RESPONSES_HANDOFF_FIELDS = ( + "disaggregated_params", + "prompt_token_ids", + "prompt_token_ids_b64", + "finish_reason", +) + class ResponsesResponse(OpenAIBaseModel): id: str = Field(default_factory=lambda: f"resp_{str(uuid.uuid4().hex)}") @@ -1902,6 +1917,17 @@ class ResponsesResponse(OpenAIBaseModel): prompt_token_ids_b64: Optional[str] = None finish_reason: Optional[str] = None + @model_serializer(mode="wrap") + def _omit_absent_handoff(self, handler: Any) -> Any: + # The handoff fields are unset on every response a client sees, so they + # are left out rather than serialized as null in each body and event. + data = handler(self) + if isinstance(data, dict): + for key in _RESPONSES_HANDOFF_FIELDS: + if data.get(key, ...) is None: + data.pop(key) + return data + @classmethod def from_request( cls, diff --git a/tests/unittest/disaggregated/test_disagg_responses_api.py b/tests/unittest/disaggregated/test_disagg_responses_api.py index 477676fe2806..651f27689bc2 100644 --- a/tests/unittest/disaggregated/test_disagg_responses_api.py +++ b/tests/unittest/disaggregated/test_disagg_responses_api.py @@ -28,6 +28,7 @@ import numpy as np import pytest from fastapi import HTTPException +from openai.types.responses import ResponseCreatedEvent from pydantic import ValidationError from tensorrt_llm.serve.openai_disagg_server import OpenAIDisaggServer @@ -134,6 +135,41 @@ def test_a_finished_context_response_goes_to_the_client_without_the_handoff( assert handoff == (None, None, None, None) +_HANDOFF_KEYS = { + "disaggregated_params", + "prompt_token_ids", + "prompt_token_ids_b64", + "finish_reason", +} + + +def test_a_response_without_the_handoff_serializes_without_its_keys(): + """Bodies, stream events and stored copies carry no null handoff keys.""" + response = _responses_response(finish_reason="stop") + _service()._need_gen(response) # a finished context response drops the handoff + + assert not _HANDOFF_KEYS & response.model_dump(by_alias=True).keys() + assert not _HANDOFF_KEYS & json.loads(response.model_dump_json(by_alias=True)).keys() + # The stream wraps a dump of the response, as get_initial_responses does. + created = ResponseCreatedEvent( + type="response.created", sequence_number=0, response=response.model_dump(by_alias=True) + ) + assert not _HANDOFF_KEYS & json.loads(created.model_dump_json(by_alias=True))["response"].keys() + + async def frames(): + return [frame async for frame in responses_done_generator(response)] + + for frame in asyncio.run(frames()): + event = json.loads(frame.decode().split("data: ", 1)[1]) + assert not _HANDOFF_KEYS & event["response"].keys() + + +def test_a_context_response_still_carries_its_handoff(): + body = _responses_response(finish_reason="length").model_dump(by_alias=True) + assert _HANDOFF_KEYS <= body.keys() + assert ResponsesResponse(**body).disaggregated_params.ctx_request_id == 7 + + @pytest.mark.parametrize( "finish_reason, status", [("not_finished", "incomplete"), ("length", "incomplete"), ("stop", "completed")], @@ -163,11 +199,28 @@ def test_an_unknown_finish_reason_names_itself(): ], ) def test_the_generation_worker_uses_the_relayed_prompt(fields, expected): - request = ResponsesRequest(model="m", input="hi", **fields) + request = ResponsesRequest( + model="m", + input="hi", + disaggregated_params=DisaggregatedParams(request_type="generation_only"), + **fields, + ) assert request.relayed_prompt_token_ids() == expected assert request.prompt_token_ids == fields.get("prompt_token_ids") +@pytest.mark.parametrize("request_type", [None, "context_only", "context_and_generation"]) +@pytest.mark.parametrize("field", ["prompt_token_ids", "prompt_token_ids_b64"]) +def test_prompt_ids_outside_a_generation_handoff_are_ignored(request_type, field): + """A request that is not a generation-only handoff renders its own prompt.""" + value = [1, 2, 3] + if field == "prompt_token_ids_b64": + value = base64.b64encode(np.asarray(value, dtype=np.int32).tobytes()).decode("ascii") + params = None if request_type is None else DisaggregatedParams(request_type=request_type) + request = ResponsesRequest(model="m", input="hi", disaggregated_params=params, **{field: value}) + assert request.relayed_prompt_token_ids() is None + + @pytest.mark.parametrize( "encoded, reason", [ @@ -309,9 +362,9 @@ async def preprocess(**kwargs): if as_b64: ids = np.frombuffer(base64.b64decode(body["prompt_token_ids_b64"]), dtype=np.int32) - assert (ids.tolist(), body["prompt_token_ids"]) == ([5, 6, 7], None) + assert (ids.tolist(), "prompt_token_ids" in body) == ([5, 6, 7], False) else: - assert (body["prompt_token_ids"], body["prompt_token_ids_b64"]) == ([5, 6, 7], None) + assert (body["prompt_token_ids"], "prompt_token_ids_b64" in body) == ([5, 6, 7], False) # --------------------------------------------------------------------------- From 05e644f22ea04701166db3e0b0933ea6a7927cf1 Mon Sep 17 00:00:00 2001 From: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> Date: Tue, 13 Oct 2026 19:51:24 +0000 Subject: [PATCH 39/43] [None][fix] Responses disagg: drop client prompt ids; complete a stream only on a whole terminal frame The orchestrator now clears client-supplied prompt_token_ids and prompt_token_ids_b64 before dispatch, so the generation worker consumes exactly the prompt the context worker prefilled in both schedule styles. guard_responses_stream marks a stream complete only once the terminal frame's closing delimiter arrives, so a relay cut inside that frame still ends with an error event. Signed-off-by: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> --- tensorrt_llm/serve/openai_disagg_service.py | 4 +++ tensorrt_llm/serve/responses_utils.py | 15 ++++++++- .../test_disagg_responses_api.py | 31 +++++++++++++++++++ .../apps/test_responses_streaming_events.py | 25 +++++++++++++++ 4 files changed, 74 insertions(+), 1 deletion(-) diff --git a/tensorrt_llm/serve/openai_disagg_service.py b/tensorrt_llm/serve/openai_disagg_service.py index 6eb94209598a..d8045026efc0 100644 --- a/tensorrt_llm/serve/openai_disagg_service.py +++ b/tensorrt_llm/serve/openai_disagg_service.py @@ -224,6 +224,10 @@ async def openai_responses( key="disagg_responses_store_ignored", ) request.store = False + # The generation worker must consume the prompt the context worker + # prefilled, so only the context response may supply prompt ids. + request.prompt_token_ids = None + request.prompt_token_ids_b64 = None return await self._send_disagg_request(request, hooks) diff --git a/tensorrt_llm/serve/responses_utils.py b/tensorrt_llm/serve/responses_utils.py index b15837f51db2..0584218e49a3 100644 --- a/tensorrt_llm/serve/responses_utils.py +++ b/tensorrt_llm/serve/responses_utils.py @@ -3061,6 +3061,7 @@ async def guard_responses_stream( """ events_sent = 0 tail = None + in_terminal_frame = False completed = False try: async for chunk in stream: @@ -3070,7 +3071,19 @@ async def guard_responses_stream( text = tail + chunk delimiter, markers = _sse_delimiter_and_markers(text) events_sent += text.count(delimiter) - tail.count(delimiter) - completed = any(marker in text for marker in markers) + if in_terminal_frame: + # A delimiter that ends the frame takes a byte of this chunk. + frame_rest = max(len(tail) - len(delimiter) + 1, 0) + else: + starts = [text.find(marker) for marker in markers] + starts = [start for start in starts if start >= 0] + if starts: + in_terminal_frame = True + frame_rest = min(starts) + len(delimiter) + # Only a whole terminal frame completes the stream; a client + # cut off inside it has not received the response. + if in_terminal_frame: + completed = text.find(delimiter, frame_rest) >= 0 tail = text[-_SSE_TAIL_LENGTH:] yield chunk except Exception as exc: diff --git a/tests/unittest/disaggregated/test_disagg_responses_api.py b/tests/unittest/disaggregated/test_disagg_responses_api.py index 651f27689bc2..0d08f50537bc 100644 --- a/tests/unittest/disaggregated/test_disagg_responses_api.py +++ b/tests/unittest/disaggregated/test_disagg_responses_api.py @@ -503,6 +503,37 @@ async def send(request, hooks): assert mock_logger.warning_once.called is warned +@pytest.mark.parametrize( + "fields", + [ + {"prompt_token_ids": [9, 9, 9]}, + {"prompt_token_ids_b64": "CQAAAA=="}, + {"prompt_token_ids": [9, 9, 9], "prompt_token_ids_b64": "CQAAAA=="}, + ], +) +def test_client_prompt_ids_are_dropped_before_dispatch(fields): + """Only the context response supplies the prompt the generation worker uses.""" + sent = [] + + async def send(request, hooks): + sent.append(request) + return "relayed" + + request = ResponsesRequest(model="m", input="hi", **fields) + asyncio.run(_service(_send_disagg_request=send).openai_responses(request)) + assert (sent[0].prompt_token_ids, sent[0].prompt_token_ids_b64) == (None, None) + + +def test_a_generation_first_worker_ignores_client_prompt_ids(): + final = _responses_response(finish_reason="stop", status="completed") + service = _gen_first_service(ctx_response=_responses_response(), gen_response=final) + + request = ResponsesRequest(model="m", input="hi", prompt_token_ids=[9, 9, 9]) + asyncio.run(service.openai_responses(request)) + + _assert_generation_first_handoff(service) + + def test_previous_response_id_is_rejected_with_a_400(): request = ResponsesRequest(model="m", input="hi", previous_response_id="resp_1") with pytest.raises(HTTPException) as raised: diff --git a/tests/unittest/llmapi/apps/test_responses_streaming_events.py b/tests/unittest/llmapi/apps/test_responses_streaming_events.py index 45f0474475c1..f7428e3fe9bd 100644 --- a/tests/unittest/llmapi/apps/test_responses_streaming_events.py +++ b/tests/unittest/llmapi/apps/test_responses_streaming_events.py @@ -456,6 +456,31 @@ async def test_a_relay_cut_mid_frame_terminates_that_frame_first(): assert _event_data(seen[3])["sequence_number"] == 1 +@pytest.mark.parametrize( + "terminal_cut", [b"event: response.completed\n", b"event: response.failed\ndata: {"] +) +@pytest.mark.asyncio +async def test_a_relay_cut_inside_its_terminal_frame_still_reports_the_error(terminal_cut): + frames = [b"event: response.created\ndata: {}\n\n", terminal_cut] + seen, raised = await _drive(frames, stream_error_event, RuntimeError("upstream cut")) + assert isinstance(raised, RuntimeError) + assert seen[:3] == [*frames, b"\n\n"] + assert _event_type(seen[3]) == "error" + + +@pytest.mark.asyncio +async def test_a_terminal_frame_split_across_chunks_completes_the_stream(): + frames = [ + b"event: response.created\ndata: {}\n\n", + b"event: response.completed\ndata: {", + b"}\n", + b"\n", + ] + seen, raised = await _drive(frames, _forbidden, RuntimeError("reset")) + assert seen == frames + assert isinstance(raised, RuntimeError) + + @pytest.mark.asyncio async def test_a_relay_that_never_started_reports_sequence_zero(): seen, raised = await _drive([], stream_error_event, RuntimeError("no context worker")) From 8851a434149924d30b87f46999eb501ea3a19a27 Mon Sep 17 00:00:00 2001 From: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> Date: Tue, 20 Oct 2026 16:41:34 +0000 Subject: [PATCH 40/43] [None][fix] Responses: leave image input to main's handling Main now passes Responses input images to multimodal models, so this change no longer replaces images with a text placeholder: request validation keeps them, and the input conversion is main's. Untyped messages are translated by main's code, so the duplicate test goes too. Signed-off-by: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> --- docs/source/features/disagg-serving.md | 3 +- tensorrt_llm/serve/openai_protocol.py | 41 +--------- tensorrt_llm/serve/responses_utils.py | 61 +-------------- .../apps/test_responses_input_preprocess.py | 20 ----- .../llmapi/apps/test_responses_usage.py | 77 ------------------- 5 files changed, 6 insertions(+), 196 deletions(-) diff --git a/docs/source/features/disagg-serving.md b/docs/source/features/disagg-serving.md index 8225f616ad31..93f284b5659f 100644 --- a/docs/source/features/disagg-serving.md +++ b/docs/source/features/disagg-serving.md @@ -231,10 +231,9 @@ The disaggregated server also serves `/v1/responses`, with both `context_first` - A request with `previous_response_id` is rejected with a 400, because a stored response lives in one worker's memory and a later request may reach another worker. Send the earlier turns in `input` instead. - `store` is ignored: nothing is persisted, and the server logs a warning once when a client asks for it. -Two limitations apply to `/v1/responses` on every server, disaggregated or not: +One limitation applies to `/v1/responses` on every server, disaggregated or not: - `parallel_tool_calls: false`, and a `tool_choice` other than `auto` or `none` on non-Harmony models, are accepted but not enforced. The server logs a warning once. -- An image in the input, in a message or in a tool output, is replaced by a text placeholder that names what was dropped, since the endpoint accepts text only. #### Launching disaggregated servers on SLURM clusters diff --git a/tensorrt_llm/serve/openai_protocol.py b/tensorrt_llm/serve/openai_protocol.py index 6bcdccfc1d55..c8550a756b64 100644 --- a/tensorrt_llm/serve/openai_protocol.py +++ b/tensorrt_llm/serve/openai_protocol.py @@ -1504,29 +1504,6 @@ def _materialize_validator_iterators(value, _depth=0): return value -def input_image_placeholder(part: Any) -> Any: - """Replace an ``input_image`` part with a text placeholder; keep others. - - Used where an image cannot reach the model; the placeholder says what was - dropped, so the model does not answer as if it had seen nothing. - """ - if isinstance(part, dict) and part.get("type") == "input_image": - url = part.get("image_url") or "" - if isinstance(url, dict): - url = url.get("url") or "" - if not isinstance(url, str): - url = "" - kind = url.split(";", 1)[0].removeprefix("data:") or "image" - return { - "type": - "input_text", - "text": - "[image omitted: %s, %d bytes as sent; this " - "model accepts text only]" % (kind, len(url)), - } - return part - - class ResponsesRequest(OpenAIBaseModel): # Ordered by official OpenAI API documentation # https://platform.openai.com/docs/api-reference/responses/create @@ -1579,25 +1556,11 @@ def _with_annotations(part): return {**part, "annotations": []} return part - def _with_image_detail(part): - """A tool result's input_image requires detail; clients omit it.""" - if (isinstance(part, dict) and part.get("type") == "input_image" - and "detail" not in part): - return {**part, "detail": "auto"} - return part - cleaned = [] for item in value: # A client may send a bare content part as a top-level item, not - # only nested inside a message. No input item type holds a bare - # image, so one is kept as a placeholder rather than failing the - # whole request. - item = _with_annotations(input_image_placeholder(item)) - if isinstance(item, dict) and isinstance(item.get("output"), list): - item = { - **item, "output": - [_with_image_detail(part) for part in item["output"]] - } + # only nested inside a message. + item = _with_annotations(item) if isinstance(item, dict) and item.get("type") in (None, "message"): role = item.get("role") if "id" in item and role in _ID_STRIPPED_ROLES: diff --git a/tensorrt_llm/serve/responses_utils.py b/tensorrt_llm/serve/responses_utils.py index c03d4849da6e..9c51808bb302 100644 --- a/tensorrt_llm/serve/responses_utils.py +++ b/tensorrt_llm/serve/responses_utils.py @@ -59,7 +59,6 @@ RequestError) from tensorrt_llm.inputs import PromptInputs, prompt_inputs from tensorrt_llm.inputs.multimodal import MultimodalServerConfig -from tensorrt_llm.inputs.registry import MULTIMODAL_PLACEHOLDER_REGISTRY from tensorrt_llm.inputs.utils import (async_apply_chat_template, resolve_hf_chat_template) from tensorrt_llm.llmapi import SamplingParams @@ -89,7 +88,6 @@ StreamingResponsesResponse, UCompletionRequest, UCompletionResponse, - input_image_placeholder, to_disaggregated_params) from tensorrt_llm.serve.responses_web_search import is_web_search_tool from tensorrt_llm.serve.tool_parser.base_tool_parser import ( @@ -1131,65 +1129,14 @@ async def _create_input_messages( request: ResponsesRequest, prev_msgs: list[ChatCompletionMessageParam], ) -> list[ChatCompletionMessageParam]: - return chat_messages_from_responses_input(request, - prev_msgs, - keep_images=True) - - -def _reads_images(request: ResponsesRequest, - model_config: PretrainedConfig) -> bool: - """Whether an image in ``request`` can be shown to the model. - - Not when the model takes no image input, nor for a disaggregated request, - whose context and generation workers exchange token ids only. - """ - if request.disaggregated_params is not None: - return False - return MULTIMODAL_PLACEHOLDER_REGISTRY.is_valid( - resolve_top_level_model_type(model_config), "image") - - -def _has_image_parts(messages: list[ChatCompletionMessageParam]) -> bool: - return any( - isinstance(part, dict) and part.get("type") == "image_url" - for message in messages if isinstance(message.get("content"), list) - for part in message["content"]) - - -def _without_image_parts( - messages: list[ChatCompletionMessageParam] -) -> list[ChatCompletionMessageParam]: - """``messages`` with each image part replaced by a text placeholder.""" - - def _part(part): - if isinstance(part, dict) and part.get("type") == "image_url": - return { - "type": - "text", - "text": - input_image_placeholder({ - "type": "input_image", - "image_url": part.get("image_url") - })["text"], - } - return part - - return [{ - **message, "content": [_part(part) for part in message["content"]] - } if isinstance(message.get("content"), list) else message - for message in messages] + return chat_messages_from_responses_input(request, prev_msgs) def chat_messages_from_responses_input( request: ResponsesRequest, prev_msgs: list[ChatCompletionMessageParam], - *, - keep_images: bool = False, ) -> list[ChatCompletionMessageParam]: - """Convert a Responses request's instructions, history and input to chat messages. - - An image becomes a text placeholder unless ``keep_images`` is set. - """ + """Convert a Responses request's instructions, history and input to chat messages.""" messages: list[ChatCompletionMessageParam] = [] if request.instructions: messages.append({ @@ -1227,7 +1174,7 @@ def chat_messages_from_responses_input( continue messages.append(message) - return messages if keep_images else _without_image_parts(messages) + return messages def _stored_tool_arguments(call) -> str: @@ -1460,8 +1407,6 @@ async def _create_input_tokens( request=request, prev_msgs=prev_msgs, ) - if _has_image_parts(messages) and not _reads_images(request, model_config): - messages = _without_image_parts(messages) if enable_store and request.store: await conversation_store.store_messages(request.request_id, messages, diff --git a/tests/unittest/llmapi/apps/test_responses_input_preprocess.py b/tests/unittest/llmapi/apps/test_responses_input_preprocess.py index 517d66c37593..012f0846fda9 100644 --- a/tests/unittest/llmapi/apps/test_responses_input_preprocess.py +++ b/tests/unittest/llmapi/apps/test_responses_input_preprocess.py @@ -445,26 +445,6 @@ def test_item_text_is_read_from_every_shape_that_carries_it(item, expected): assert _response_output_item_to_chat_completion_message(item) == expected -_IMAGE = {"type": "image_url", "image_url": {"url": "http://example/x.png"}} - - -@pytest.mark.parametrize( - "content, expected", - [ - ([{"type": "input_text", "text": "hello"}], [{"type": "text", "text": "hello"}]), - ([{"type": "output_text", "text": "hi"}], [{"type": "text", "text": "hi"}]), - ( - [{"type": "input_text", "text": "what is this"}, _IMAGE], - [{"type": "text", "text": "what is this"}, _IMAGE], - ), - ("hello", "hello"), - ], -) -def test_untyped_message_parts_are_translated_to_chat_parts(content, expected): - msg = _response_output_item_to_chat_completion_message({"role": "user", "content": content}) - assert msg == {"role": "user", "content": expected} - - @pytest.mark.parametrize("item_type", ["function_call_output", "custom_tool_call_output"]) @pytest.mark.parametrize( "output, expected", diff --git a/tests/unittest/llmapi/apps/test_responses_usage.py b/tests/unittest/llmapi/apps/test_responses_usage.py index fd417c021467..13c7490da625 100644 --- a/tests/unittest/llmapi/apps/test_responses_usage.py +++ b/tests/unittest/llmapi/apps/test_responses_usage.py @@ -31,8 +31,6 @@ _count_reasoning_tokens, _create_output_content, _create_usage, - _reads_images, - chat_messages_from_responses_input, ) # The CPU-* CI stages run pytest with -m 'cpu_only'. Without this marker every @@ -248,78 +246,3 @@ def test_a_streamed_response_counts_reasoning_tokens(): frame = processor.get_final_response_non_store(result, tokenizer=_WordTokenizer()) usage = json.loads(frame.split("data: ", 1)[1])["response"]["usage"] assert usage["output_tokens_details"]["reasoning_tokens"] == 2 - - -# --------------------------------------------------------------------------- -# An image the model cannot be shown degrades to a text placeholder -# --------------------------------------------------------------------------- - -_PNG = "data:image/png;base64,AAAA" - -_IMAGE_ITEMS = pytest.mark.parametrize( - "item", - [ - {"role": "user", "content": [{"type": "input_image", "image_url": _PNG}]}, - {"role": "user", "content": [{"type": "input_image", "image_url": {"url": _PNG}}]}, - { - "type": "custom_tool_call_output", - "call_id": "call_1", - "output": [{"type": "input_image", "image_url": _PNG}], - }, - ], - ids=["message_part", "url_object", "tool_output"], -) - - -def _assert_placeholder(part): - assert part["text"].startswith("[image omitted: image/png, ") - assert "AAAA" not in part["text"] - - -def test_a_top_level_image_degrades_at_validation(): - """No input item type holds a bare image part, so it would fail the request.""" - request = ResponsesRequest(model="m", input=[{"type": "input_image", "image_url": _PNG}]) - part = request.model_dump()["input"][0] - assert part["type"] == "input_text" - _assert_placeholder(part) - - -@_IMAGE_ITEMS -def test_an_image_survives_validation(item): - """Whether the model can be shown the image is decided later, per model.""" - dumped = ResponsesRequest(model="m", input=[item]).model_dump()["input"][0] - (part,) = dumped["output"] if "output" in dumped else dumped["content"] - assert part["type"] == "input_image" - - -@_IMAGE_ITEMS -def test_an_image_degrades_unless_it_can_be_shown(item): - request = ResponsesRequest(model="m", input=[item]) - (message,) = chat_messages_from_responses_input(request, []) - (part,) = message["content"] - assert part["type"] == "text" - _assert_placeholder(part) - - (message,) = chat_messages_from_responses_input(request, [], keep_images=True) - assert message["content"][0]["type"] == "image_url" - - -def test_images_are_not_shown_to_a_text_only_model(): - request = ResponsesRequest(model="m", input="hi") - assert not _reads_images(request, SimpleNamespace(model_type="llama")) - - -def test_images_are_not_shown_on_a_disaggregated_request(): - """The context and generation workers exchange token ids only.""" - request = ResponsesRequest( - model="m", - input="hi", - disaggregated_params={"request_type": "context_only"}, - ) - assert not _reads_images(request, SimpleNamespace(model_type="qwen2_vl")) - - -def test_a_string_tool_output_is_not_degraded(): - item = {"type": "custom_tool_call_output", "call_id": "call_2", "output": "done\n"} - request = ResponsesRequest(model="m", input=[item]) - assert request.model_dump()["input"][0]["output"] == "done\n" From 4ee109c6ba6f17e6b1ba0c7f34407c87b40f4b1d Mon Sep 17 00:00:00 2001 From: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> Date: Wed, 21 Oct 2026 04:16:23 +0000 Subject: [PATCH 41/43] [None][fix] Responses disagg: time the first token from the first output frame A worker's Responses stream opens with response.created and response.in_progress before the model generates anything, so the disaggregated server's on_first_token and first_token_latency_seconds measured those frames, which made TTFT near zero under generation-first. The first token is now the first chunk carrying an event past them. Signed-off-by: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> --- tensorrt_llm/serve/openai_client.py | 19 +++++++++- .../test_disagg_openai_client.py | 36 +++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/tensorrt_llm/serve/openai_client.py b/tensorrt_llm/serve/openai_client.py index 05ee7aa5040c..419d2d62bdb7 100644 --- a/tensorrt_llm/serve/openai_client.py +++ b/tensorrt_llm/serve/openai_client.py @@ -77,6 +77,18 @@ _msgpack_encoder = msgspec.msgpack.Encoder() MSGPACK_HEADERS = {"Content-Type": "application/json", "X-TRTLLM-Msgpack": "1"} +# A Responses stream opens with these events before the model emits anything. +_RESPONSES_OPENING_EVENTS = frozenset((b"response.created", b"response.in_progress")) + + +def _has_responses_output_event(chunk: bytes) -> bool: + """Whether an SSE chunk of a Responses stream carries an event past its opening.""" + return any( + line.startswith(b"event: ") + and line[len(b"event: ") :].strip() not in _RESPONSES_OPENING_EVENTS + for line in chunk.split(b"\n") + ) + # post_json's request and response are ordinary pydantic models, but not the # UCompletion* union: it carries auxiliary payloads such as count-tokens. The @@ -575,13 +587,18 @@ async def _response_generator( try: last_token_time = start_time chunk_count = 0 + # The first token of a Responses stream is the first frame after + # its opening events, which the worker sends before generating. + awaiting_output = isinstance(request, ResponsesRequest) marker = f"event: {SSE_METRICS_EVENT}\n".encode() event_delimiter = b"\n\n" pending = b"" metrics_event = b"" async for chunk in http_response.content.iter_any(): now_time = get_steady_clock_now_in_seconds() - if chunk: + if awaiting_output and chunk: + awaiting_output = not _has_responses_output_event(chunk) + if chunk and not awaiting_output: if chunk_count == 0 and hooks: hooks.on_first_token(server, request) latency = now_time - last_token_time diff --git a/tests/unittest/disaggregated/test_disagg_openai_client.py b/tests/unittest/disaggregated/test_disagg_openai_client.py index 7c0d55d74df4..8a16bf92857f 100644 --- a/tests/unittest/disaggregated/test_disagg_openai_client.py +++ b/tests/unittest/disaggregated/test_disagg_openai_client.py @@ -490,6 +490,42 @@ async def mock_iter_any(): streaming_completion_request, mock_session, success=True ) + @pytest.mark.asyncio + @pytest.mark.parametrize("together", [False, True], ids=["separate", "same-chunk"]) + async def test_responses_first_token_follows_the_opening_events( + self, openai_client, mock_session, together + ): + """The worker sends response.created and response.in_progress before generating.""" + request = ResponsesRequest(model="test-model", input="hi", stream=True) + opening = b"event: response.created\ndata: {}\n\nevent: response.in_progress\ndata: {}\n\n" + delta = b"event: response.output_text.delta\ndata: {}\n\n" + sent = [opening + delta] if together else [opening, delta] + + async def mock_iter_any(): + for data in sent: + yield data + + mock_http_response = AsyncMock() + mock_http_response.status = 200 + mock_http_response.headers = {"Content-Type": "text/event-stream"} + mock_http_response.content = AsyncMock() + mock_http_response.content.iter_any = mock_iter_any + mock_http_response.__aenter__ = AsyncMock(return_value=mock_http_response) + mock_http_response.__aexit__ = AsyncMock() + mock_session.post.return_value = mock_http_response + received = [] + hooks = MagicMock(spec=ResponseHooks) + first_token_after = [] + hooks.on_first_token.side_effect = lambda *args: first_token_after.append(len(received)) + + response_generator = await openai_client.send_request(request, hooks=hooks) + async for chunk in response_generator: + received.append(chunk) + + assert received == sent + # Fired on the chunk carrying the first output, not on the opening pair. + assert first_token_after == [len(sent) - 1] + @pytest.mark.asyncio async def test_streaming_perf_metrics_preserve_sse_event_boundaries( self, openai_client, streaming_completion_request, mock_session, mock_router From 771bb8d8c4a329b7c0ecd19f67e59f03119be9c2 Mon Sep 17 00:00:00 2001 From: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> Date: Fri, 23 Oct 2026 03:03:27 +0000 Subject: [PATCH 42/43] [None][fix] Responses: final snapshot keeps the status the stream reported The stream's output_item.done events close reasoning and function-call items with status "completed". The final snapshot built the same items without a status, so they read null in response.completed and in the non-streamed body: a consumer that filters items by status reconstructed a different turn from each view. Build them as completed on the streamed-record path, the re-derived path and the Harmony path, as the stream does. Signed-off-by: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> --- tensorrt_llm/serve/responses_utils.py | 17 +++++--- ...test_responses_stream_final_consistency.py | 41 +++++++++++++++++++ 2 files changed, 53 insertions(+), 5 deletions(-) diff --git a/tensorrt_llm/serve/responses_utils.py b/tensorrt_llm/serve/responses_utils.py index 60af2f00571e..bde4713a1660 100644 --- a/tensorrt_llm/serve/responses_utils.py +++ b/tensorrt_llm/serve/responses_utils.py @@ -656,7 +656,7 @@ def _parse_output_message_harmony(message: Message) -> list[ResponseOutputItem]: summary=[], type="reasoning", content=[Content(text=content.text, type="reasoning_text")], - status=None, + status="completed", ) output_items.append(reasoning_item) elif message.channel == "commentary": @@ -671,6 +671,7 @@ def _parse_output_message_harmony(message: Message) -> list[ResponseOutputItem]: type="function_call", name=function_name, id=f"fc_{_random_uuid()}", + status="completed", ) output_items.append(response_item) elif message.recipient.startswith( @@ -681,7 +682,7 @@ def _parse_output_message_harmony(message: Message) -> list[ResponseOutputItem]: summary=[], type="reasoning", content=[Content(text=content.text, type="reasoning_text")], - status=None, + status="completed", ) output_items.append(reasoning_item) else: @@ -1845,7 +1846,9 @@ def _streamed_or_fresh_id(item_type: str) -> str: content=[ Content(text=record.text, type="reasoning_text") ], - status=None, + # The stream closed this item as completed; the + # snapshot reports the same status. + status="completed", )) else: output_items.append( @@ -1878,7 +1881,7 @@ def _streamed_or_fresh_id(item_type: str) -> str: content=[ Content(text=reasoning_text, type="reasoning_text") ], - status=None, + status="completed", )) stored_reasoning = reasoning_text @@ -1905,8 +1908,12 @@ def _streamed_or_fresh_id(item_type: str) -> str: tool_calls_item = list(streamed_tool_calls) else: tool_resolution = _tool_resolution(tools) + # As streamed: the stream delivers every call it announces as + # completed. tool_calls_item = [ - _tool_call_output_item(call, tool_resolution) for call in calls + _tool_call_output_item(call, + tool_resolution, + status="completed") for call in calls ] output_items.extend(tool_calls_item) diff --git a/tests/unittest/llmapi/apps/test_responses_stream_final_consistency.py b/tests/unittest/llmapi/apps/test_responses_stream_final_consistency.py index 90612ea34547..502e07a85000 100644 --- a/tests/unittest/llmapi/apps/test_responses_stream_final_consistency.py +++ b/tests/unittest/llmapi/apps/test_responses_stream_final_consistency.py @@ -359,3 +359,44 @@ def test_the_stream_releases_what_the_whole_text_parse_reads_beyond_it( SimpleNamespace(index=0, text=whole), request, helper, None, "qwen3", [] ) assert (released, calls, logger.warning.called) == (owed, [], warned) + + +# --------------------------------------------------------------------------- +# Item status: the snapshot reports what the stream closed each item with +# --------------------------------------------------------------------------- + + +def test_final_items_report_the_status_the_stream_closed_them_with(): + """Reasoning and function-call items stay `completed` in the snapshot. + + Their done events say `completed`; a snapshot that built them without a + status read `null` for the same items, so a consumer that filters items by + status reconstructed a different turn from each view. + """ + processor = _processor(tools=[_tool()], reasoning_parser="glm47", tool_parser="glm47") + frames, result = _stream(processor, ["Plan it.", "Calling: ", _EXEC_CALL]) + + streamed = {item["type"]: item.get("status") for item in _done_items(frames)} + final = {item["type"]: item.get("status") for item in _final_output(processor, result)} + + assert streamed == { + "reasoning": "completed", + "message": "completed", + "function_call": "completed", + } + assert final == streamed + + +def test_non_streaming_items_are_completed_too(): + """No stream ran; the body is the only account, and it says completed.""" + items, _messages, _reasoning = _create_output_content( + SimpleNamespace(outputs=[SimpleNamespace(index=0, text="Plan it." + _EXEC_CALL)]), + reasoning_parser="glm47", + tool_parser="glm47", + tools=[_tool()], + streamed_item_ids=None, + ) + assert [(item.type, item.status) for item in items] == [ + ("reasoning", "completed"), + ("function_call", "completed"), + ] From 4faabf148bd6e49471080b823fdcc09387c47013 Mon Sep 17 00:00:00 2001 From: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> Date: Mon, 2 Nov 2026 21:26:58 +0000 Subject: [PATCH 43/43] [None][test] Check the status of Harmony reasoning and function-call items The Harmony branches of _parse_output_message_harmony now mark reasoning and function-call items completed, but no test read their status. A new parametrized test builds an analysis message, a commentary call to a function and a commentary message to python, and expects each item to be completed. Signed-off-by: JunyiXu-nv <219237550+JunyiXu-nv@users.noreply.github.com> --- ...test_responses_stream_final_consistency.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/unittest/llmapi/apps/test_responses_stream_final_consistency.py b/tests/unittest/llmapi/apps/test_responses_stream_final_consistency.py index 874906d13007..bf789648c8e2 100644 --- a/tests/unittest/llmapi/apps/test_responses_stream_final_consistency.py +++ b/tests/unittest/llmapi/apps/test_responses_stream_final_consistency.py @@ -32,6 +32,7 @@ ResponsesStreamingProcessor, StreamedItem, _create_output_content, + _parse_output_message_harmony, _whole_text_tool_calls, create_response_non_store, ) @@ -426,3 +427,26 @@ def test_non_streaming_items_are_completed_too(): ("reasoning", "completed"), ("function_call", "completed"), ] + + +@pytest.mark.parametrize( + "channel, recipient, item_type", + [ + ("analysis", None, "reasoning"), + ("commentary", "functions.get_weather", "function_call"), + ("commentary", "python", "reasoning"), + ], + ids=["analysis", "function_call", "python_commentary"], +) +def test_harmony_reasoning_and_function_call_items_are_completed(channel, recipient, item_type): + openai_harmony = pytest.importorskip("openai_harmony") + message = openai_harmony.Message.from_role_and_content( + openai_harmony.Role.ASSISTANT, '{"city": "Rome"}' + ).with_channel(channel) + if recipient is not None: + message = message.with_recipient(recipient) + + (item,) = _parse_output_message_harmony(message) + + assert item.type == item_type + assert item.status == "completed"