From 28abbf7c764cd2fe792fb971ea9ff374d298594b Mon Sep 17 00:00:00 2001 From: furionw Date: Fri, 23 Oct 2026 10:18:37 -0700 Subject: [PATCH] fix: MiniMax M3.1 Flash serving fixes and public vLLM/SGLang runtime bases Bring the MiniMax M3.1 Flash serving fixes to the release branch, and build the vLLM and SGLang runtimes on the public MiniMax M3.1 Flash images: - MiniMax M3: the reasoning parser takes its start state from the rendered prompt, so a template that ignores thinking_mode no longer leaks reasoning into content; guided_* output is delivered as content, and the parser is finalized at end of stream. - Responses: commentary and final answer are separate message items with a phase, and reasoning items are always returned. - TensorRT-LLM multimodal: frontend-rendered prompts reach processors without the token-ID fast path as rendered text with chat_template_applied, and video is decoded in the input processor's preferred format. - vLLM: import chat_utils from its new location and the engine-launch helpers lazily. - Runtimes: vLLM on vllm/vllm-openai:minimax-m31-flash and SGLang on lmsysorg/sglang:dev-minimax-m31-flash, pinned by index digest (CUDA 13.4.1, amd64 and arm64); vLLM-Omni off; NIXL 1.5.0 only when the base ships none; CUDA 13 and 13.4 license classifications and three Python license overrides. TensorRT-LLM keeps its base. Co-Authored-By: Claude Opus 5.5 Signed-off-by: furionw --- components/src/dynamo/frontend/prepost.py | 5 +- .../src/dynamo/frontend/vllm_processor.py | 7 +- .../src/dynamo/trtllm/multimodal_processor.py | 86 +- .../trtllm/request_handlers/handler_base.py | 6 + .../tests/test_trtllm_multimodal_processor.py | 127 ++ .../dynamo/vllm/embedding_worker_processes.py | 42 +- .../src/dynamo/vllm/realtime/factories.py | 6 +- .../vllm/tests/test_vllm_prompt_embeds.py | 3 + container/compliance/license_overrides.yaml | 82 + container/context.yaml | 52 +- container/templates/sglang_runtime.Dockerfile | 15 +- container/templates/vllm_runtime.Dockerfile | 16 +- lib/llm/PREPROCESSOR_CASES.md | 2 +- lib/llm/src/preprocessor.rs | 1606 ++++++++++++++++- lib/llm/src/preprocessor/structural_tag.rs | 50 + lib/llm/src/preprocessor/tool_choice.rs | 4 +- lib/llm/src/protocols/openai/responses/mod.rs | 310 +++- .../openai/responses/stream_converter.rs | 749 ++++++-- ...http_replay__responses_streaming_text.snap | 2 + ...ses_http_replay__responses_unary_text.snap | 1 + tests/report_pytest_markers.py | 1 + tests/utils/payloads.py | 48 +- 22 files changed, 2908 insertions(+), 312 deletions(-) diff --git a/components/src/dynamo/frontend/prepost.py b/components/src/dynamo/frontend/prepost.py index 83803e97eaa3..c80df1b8b48d 100644 --- a/components/src/dynamo/frontend/prepost.py +++ b/components/src/dynamo/frontend/prepost.py @@ -14,7 +14,10 @@ from functools import lru_cache, partial from typing import TYPE_CHECKING, Any, Protocol, TypeGuard, cast -from vllm.entrypoints.chat_utils import make_tool_call_id +try: + from vllm.renderers.chat_utils import make_tool_call_id +except ImportError: # vLLM before vllm-project/vllm#60522 moved chat_utils + from vllm.entrypoints.chat_utils import make_tool_call_id from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionNamedFunction, ChatCompletionNamedToolChoiceParam, diff --git a/components/src/dynamo/frontend/vllm_processor.py b/components/src/dynamo/frontend/vllm_processor.py index cf0063e7135a..03c2885ac180 100644 --- a/components/src/dynamo/frontend/vllm_processor.py +++ b/components/src/dynamo/frontend/vllm_processor.py @@ -18,7 +18,12 @@ from msgspec.structs import replace as msgspec_replace from vllm.config import CacheConfig, LoadConfig, ModelConfig, VllmConfig -from vllm.entrypoints.chat_utils import load_chat_template + +try: + from vllm.renderers.chat_utils import load_chat_template +except ImportError: # vLLM before vllm-project/vllm#60522 moved chat_utils + from vllm.entrypoints.chat_utils import load_chat_template + from vllm.exceptions import VLLMClientError from vllm.reasoning import ReasoningParser, ReasoningParserManager from vllm.sampling_params import RequestOutputKind, SamplingParams diff --git a/components/src/dynamo/trtllm/multimodal_processor.py b/components/src/dynamo/trtllm/multimodal_processor.py index f79d19be141f..926f72c9df53 100644 --- a/components/src/dynamo/trtllm/multimodal_processor.py +++ b/components/src/dynamo/trtllm/multimodal_processor.py @@ -25,7 +25,7 @@ from safetensors.torch import load as safetensors_load from safetensors.torch import load_file as safetensors_load_file from tensorrt_llm.inputs.multimodal_data import VideoData -from tensorrt_llm.inputs.utils import async_load_video +from tensorrt_llm.inputs.utils import async_load_video, load_video from tensorrt_llm.llmapi.tokenizer import tokenizer_factory from dynamo.common.http import HttpConfigurationError, HttpStatusError, fetch_bytes @@ -58,19 +58,46 @@ _EMBEDDING_FETCH_READ_TIMEOUT_S = 300.0 -def _nvdec_video_data(content: bytes, num_frames: int) -> VideoData: - """Decode H.264/H.265 via NVDEC into TRT-LLM's ``VideoData`` (format="pt"). +def _preferred_video_format(input_processor: Any) -> str: + """The input processor's preferred video decode format: "np" or "pt". - Mirrors ``tensorrt_llm.inputs.media_io._load_video_by_cv2``'s "pt" output for - the pinned v1.3.0rc21: RGB, a list of ``(C, H, W)`` float32 [0,1] tensors, - with the same metadata keys. Synchronous -- call via ``asyncio.to_thread``. + trtllm-serve seeds its media-IO defaults from + ``get_preferred_media_io_kwargs()``. MiniMax-M3 VL asks for uint8 "np" + frames because its video processor rescales by 1/255 itself; float [0,1] + "pt" frames would be rescaled twice and reach the vision tower near black. + Anything other than "np" keeps the "pt" frames. + """ + get_prefs = getattr(input_processor, "get_preferred_media_io_kwargs", None) + if get_prefs is None: + return "pt" + try: + prefs = get_prefs() or {} + except Exception as e: + logging.warning("Preferred media IO kwargs unavailable: %s", e) + return "pt" + return "np" if (prefs.get("video") or {}).get("format") == "np" else "pt" + + +def _nvdec_video_data( + content: bytes, num_frames: int, video_format: str = "pt" +) -> VideoData: + """Decode H.264/H.265 via NVDEC into TRT-LLM's ``VideoData``. + + Mirrors ``tensorrt_llm.inputs.media_io._load_video_by_cv2``: "pt" is a list + of ``(C, H, W)`` float32 [0,1] tensors, "np" the ``(N, H, W, 3)`` uint8 RGB + stack, with the same metadata keys. Synchronous -- call via + ``asyncio.to_thread``. """ from dynamo.common.multimodal.nvdec_decoder import decode_video_nvdec frames_np, meta = decode_video_nvdec(content, num_frames) # (N,H,W,3) uint8 RGB - stacked = frames_np.astype("float32") * (1.0 / 255.0) - nchw = torch.from_numpy(stacked).permute(0, 3, 1, 2).contiguous() - frames_pt = list(torch.unbind(nchw, dim=0)) + frames: Any + if video_format == "np": + frames = frames_np + else: + stacked = frames_np.astype("float32") * (1.0 / 255.0) + nchw = torch.from_numpy(stacked).permute(0, 3, 1, 2).contiguous() + frames = list(torch.unbind(nchw, dim=0)) fps = float(meta.get("fps") or 0.0) total = int(meta["total_num_frames"]) metadata = { @@ -79,7 +106,7 @@ def _nvdec_video_data(content: bytes, num_frames: int) -> VideoData: "duration": (total / fps) if fps > 0 else 0.0, "frames_indices": meta["frames_indices"], } - return VideoData(frames=frames_pt, metadata=metadata, audio=None) + return VideoData(frames=frames, metadata=metadata, audio=None) class TokenizerProtocol(Protocol): @@ -192,6 +219,7 @@ def __init__( self.input_processor = create_input_processor(model_dir, self.tokenizer) except Exception as e: logging.warning("Input processor unavailable for max_tokens sizing: %s", e) + self.video_format = _preferred_video_format(self.input_processor) def _expanded_prompt_len( self, token_ids: List[int], images: Optional[List[Any]] @@ -217,6 +245,18 @@ def _expanded_prompt_len( logging.warning("Could not compute expanded prompt length: %s", e) return None + async def _load_video_with_vendor_decoder(self, video: str) -> VideoData: + """Decode with TRT-LLM's cv2 loader in ``self.video_format``. + + ``async_load_video`` accepts only "pt" and "pil", so "np" goes through + the synchronous ``load_video`` in a worker thread. + """ + if self.video_format == "np": + return await asyncio.to_thread( + load_video, video, self.num_video_frames, format="np" + ) + return await async_load_video(video, self.num_video_frames) + def is_url(self, path: str) -> bool: """Check if a path is a URL.""" parsed = urlparse(path) @@ -600,7 +640,10 @@ async def process_openai_request( if should_use_nvdec(codec): try: nvdec_video = await asyncio.to_thread( - _nvdec_video_data, content, self.num_video_frames + _nvdec_video_data, + content, + self.num_video_frames, + self.video_format, ) except Exception as exc: # noqa: BLE001 - fall back logging.warning( @@ -618,8 +661,8 @@ async def process_openai_request( await asyncio.to_thread(video_file.flush) try: videos.append( - await async_load_video( - video_file.name, self.num_video_frames + await self._load_video_with_vendor_decoder( + video_file.name ) ) except ImportError as exc: @@ -638,8 +681,8 @@ async def process_openai_request( else: try: videos.append( - await async_load_video( - normalized_url, self.num_video_frames + await self._load_video_with_vendor_decoder( + normalized_url ) ) except ImportError as exc: @@ -712,6 +755,19 @@ async def process_openai_request( # Post-expansion prompt length, so an omitted max_tokens can be sized # against the real context usage rather than the unexpanded placeholders. mm_data = processed_inputs.get("multi_modal_data") + + # The token IDs are the frontend's rendering of the chat template, one + # placeholder per media item already in place. A TRT-LLM processor + # without the tokenized+MM fast path gets them detokenized by TRT-LLM, + # which drops `chat_template_applied`, so a processor that templates a + # raw prompt (MiniMax-M3 VL) adds a second placeholder and rejects the + # request. Hand such processors the rendered text with the flag, as + # trtllm-serve does; fast-path processors keep the token IDs alone. + if mm_data and not getattr( + self.input_processor, "supports_token_id_mm_expansion", True + ): + processed_inputs["prompt"] = self.tokenizer.decode(token_ids) + processed_inputs["chat_template_applied"] = True # Skipped when the request overrides the processor: the sizing # calculator is not override-aware (Qwen2-VL ignores the kwargs while # counting) and is not guaranteed non-mutating (Gemma-4 writes them diff --git a/components/src/dynamo/trtllm/request_handlers/handler_base.py b/components/src/dynamo/trtllm/request_handlers/handler_base.py index 9cf3e07f6a15..55e5f06e660d 100644 --- a/components/src/dynamo/trtllm/request_handlers/handler_base.py +++ b/components/src/dynamo/trtllm/request_handlers/handler_base.py @@ -1142,6 +1142,12 @@ async def _generate_locally_impl( sampling_params = self._override_sampling_params( self.default_sampling_params, request ) + # A rendered prompt already carries its special tokens; TRT-LLM's own + # detokenize path clears this flag for the same reason. + if isinstance(processed_input, dict) and processed_input.get( + "chat_template_applied" + ): + sampling_params.add_special_tokens = False # Additional sampling params in output options output_options = request.get("output_options", {}) diff --git a/components/src/dynamo/trtllm/tests/test_trtllm_multimodal_processor.py b/components/src/dynamo/trtllm/tests/test_trtllm_multimodal_processor.py index 779b62d08bcf..5283c2bb53a5 100644 --- a/components/src/dynamo/trtllm/tests/test_trtllm_multimodal_processor.py +++ b/components/src/dynamo/trtllm/tests/test_trtllm_multimodal_processor.py @@ -188,6 +188,94 @@ def test_nvdec_video_data_builds_pt_videodata(monkeypatch) -> None: assert vd.audio is None +def test_nvdec_video_data_keeps_uint8_frames_for_np(monkeypatch) -> None: + """ "np" hands over NVDEC's (T,H,W,3) uint8 stack untouched, as the vendor + loader's "np" format does; processors that rescale by 1/255 need it.""" + import numpy as np + + from dynamo.common.multimodal import nvdec_decoder + + frames_np = np.full((2, 4, 5, 3), 200, dtype=np.uint8) + meta = {"total_num_frames": 2, "fps": 1.0, "frames_indices": [0, 1]} + monkeypatch.setattr( + nvdec_decoder, + "decode_video_nvdec", + lambda content, num_frames: (frames_np, meta), + ) + + vd = mmp._nvdec_video_data(b"h264-bytes", 2, "np") + + assert vd.frames is frames_np + assert vd.metadata["frames_indices"] == [0, 1] + + +@pytest.mark.parametrize( + "input_processor, expected", + [ + (None, "pt"), + (SimpleNamespace(), "pt"), + (SimpleNamespace(get_preferred_media_io_kwargs=lambda: {}), "pt"), + ( + SimpleNamespace( + get_preferred_media_io_kwargs=lambda: {"image": {"format": "np"}} + ), + "pt", + ), + ( + SimpleNamespace( + get_preferred_media_io_kwargs=lambda: {"video": {"format": "np"}} + ), + "np", + ), + ], +) +def test_preferred_video_format(input_processor, expected) -> None: + assert mmp._preferred_video_format(input_processor) == expected + + +@pytest.mark.asyncio +@pytest.mark.parametrize("codec", ["h264", "vp9"]) +async def test_video_decodes_in_the_preferred_format(codec, monkeypatch) -> None: + """With "np" preferred, NVDEC is asked for "np" and the vendor fallback goes + through load_video(format="np"), since async_load_video rejects "np".""" + monkeypatch.setenv("DYN_MM_ALLOW_INTERNAL", "1") + processor = MultimodalRequestProcessor( + model_type="multimodal", + model_dir="unused", + max_file_size_mb=10, + tokenizer=MagicMock(), + ) + processor.video_format = "np" + monkeypatch.setattr(mmp, "fetch_bytes", AsyncMock(return_value=b"video bytes")) + monkeypatch.setattr(mmp, "probe_video_codec", lambda content: codec) + monkeypatch.setattr(mmp, "should_use_nvdec", lambda codec: codec == "h264") + nvdec = MagicMock(return_value=object()) + monkeypatch.setattr(mmp, "_nvdec_video_data", nvdec) + sync_load = MagicMock(return_value=object()) + monkeypatch.setattr(mmp, "load_video", sync_load) + async_load = AsyncMock(return_value=object()) + monkeypatch.setattr(mmp, "async_load_video", async_load) + + await processor.process_openai_request( + { + "multi_modal_data": { + "video_url": [{"Url": "http://169.254.169.254/v.mp4"}] + }, + "token_ids": [1], + }, + embeddings=None, + ep_disaggregated_params=None, + ) + + async_load.assert_not_awaited() + if codec == "h264": + assert nvdec.call_args.args[2] == "np" + sync_load.assert_not_called() + else: + nvdec.assert_not_called() + assert sync_load.call_args.kwargs == {"format": "np"} + + @pytest.mark.asyncio async def test_h264_video_routes_through_nvdec(monkeypatch) -> None: """H.264 video input is hardware-decoded via NVDEC; the vendor @@ -529,6 +617,45 @@ async def test_mm_processor_kwargs_forwarded(request_extra, expected) -> None: assert processed["mm_processor_kwargs"] == expected +@pytest.mark.asyncio +@pytest.mark.parametrize("fast_path", [False, True]) +async def test_rendered_prompt_reaches_processors_without_the_token_fast_path( + fast_path, +) -> None: + """Without the tokenized+MM fast path, media requests carry the rendered + text and chat_template_applied; with it, the token IDs alone.""" + tokenizer = MagicMock() + tokenizer.decode.return_value = "rendered prompt" + processor = MultimodalRequestProcessor( + model_type="multimodal", + model_dir="unused", + max_file_size_mb=10, + tokenizer=tokenizer, + ) + processor.input_processor = SimpleNamespace( + supports_token_id_mm_expansion=fast_path + ) + processor.image_loader.load_image_batch = AsyncMock(return_value=[object()]) + + processed = await processor.process_openai_request( + { + "multi_modal_data": {"image_url": [{"Url": "https://example.com/x.png"}]}, + "token_ids": [1, 2, 3], + }, + embeddings=None, + ep_disaggregated_params=None, + ) + + assert processed["prompt_token_ids"] == [1, 2, 3] + if fast_path: + assert "prompt" not in processed + assert "chat_template_applied" not in processed + else: + assert processed["prompt"] == "rendered prompt" + assert processed["chat_template_applied"] is True + tokenizer.decode.assert_called_once_with([1, 2, 3]) + + @pytest.mark.asyncio async def test_mm_processor_kwargs_non_object_is_rejected() -> None: """A non-object value is a client error, not a silently ignored field.""" diff --git a/components/src/dynamo/vllm/embedding_worker_processes.py b/components/src/dynamo/vllm/embedding_worker_processes.py index 06d0c2343d62..7947759a0b26 100644 --- a/components/src/dynamo/vllm/embedding_worker_processes.py +++ b/components/src/dynamo/vllm/embedding_worker_processes.py @@ -13,6 +13,7 @@ from __future__ import annotations import copy +import importlib import json import logging import os @@ -29,9 +30,43 @@ from vllm.config import VllmConfig from vllm.usage.usage_lib import UsageContext from vllm.v1.engine.async_llm import AsyncLLM -from vllm.v1.engine.utils import get_engine_zmq_addresses, launch_core_engines from vllm.v1.executor import Executor +# Resolved on first use, not at import: vLLM renamed get_engine_zmq_addresses in +# vllm-project/vllm#54113, and this module is imported by every dynamo.vllm worker +# while only --embedding-worker-processes > 1 calls into it. Module attributes so +# tests can substitute them. +get_engine_zmq_addresses: Callable[..., Any] | None = None +launch_core_engines: Callable[..., Any] | None = None + + +def _resolve_engine_utils() -> tuple[Callable[..., Any], Callable[..., Any]]: + """Bind the vLLM engine-launch helpers this pool needs, failing with a clear message. + + Returns ``(get_engine_zmq_addresses, launch_core_engines)`` so callers hold + non-optional references; the module attributes are kept in sync for tests. + """ + global get_engine_zmq_addresses, launch_core_engines + engine_utils = importlib.import_module("vllm.v1.engine.utils") + + if launch_core_engines is None: + launch_core_engines = engine_utils.launch_core_engines + if get_engine_zmq_addresses is None: + resolved = getattr(engine_utils, "get_engine_zmq_addresses", None) + if resolved is None: + # vllm#54113: the supervisor now binds the frontend listeners up front and + # children adopt them; this pool still lets children bind the addresses + # themselves, so refuse rather than hand out addresses that are already bound. + raise RuntimeError( + "--embedding-worker-processes > 1 is not yet supported on this vLLM: " + f"vLLM {vllm.__version__} has no get_engine_zmq_addresses (it binds " + "frontend listeners in the supervisor since vllm-project/vllm#54113). " + "Run with --embedding-worker-processes 1 until the pool adopts inherited listeners." + ) + get_engine_zmq_addresses = resolved + return get_engine_zmq_addresses, launch_core_engines + + logger = logging.getLogger(__name__) _ROLE_ENV = "DYN_VLLM_EMBEDDING_PROCESS_ROLE" @@ -463,7 +498,8 @@ def create_shared_embedding_engine_client( vllm_config.parallel_config._api_process_count = process_count vllm_config.parallel_config._api_process_rank = -1 executor_class = Executor.get_class(vllm_config) - addresses = get_engine_zmq_addresses(vllm_config, process_count) + engine_zmq_addresses, core_engines_launcher = _resolve_engine_utils() + addresses = engine_zmq_addresses(vllm_config, process_count) parent_logger_config = _client_config( vllm_config, @@ -476,7 +512,7 @@ def create_shared_embedding_engine_client( with ThreadPoolExecutor( max_workers=1, thread_name_prefix="dynamo-embedding-client" ) as client_pool: - with launch_core_engines( + with core_engines_launcher( vllm_config, executor_class, not disable_log_stats, diff --git a/components/src/dynamo/vllm/realtime/factories.py b/components/src/dynamo/vllm/realtime/factories.py index caedaf2ccc67..7d30d308328c 100644 --- a/components/src/dynamo/vllm/realtime/factories.py +++ b/components/src/dynamo/vllm/realtime/factories.py @@ -89,7 +89,11 @@ def build_realtime_text_factories( consumed without starting speculative generation. """ from vllm.engine.protocol import StreamingInput - from vllm.entrypoints.chat_utils import load_chat_template + + try: + from vllm.renderers.chat_utils import load_chat_template + except ImportError: # vLLM before vllm-project/vllm#60522 moved chat_utils + from vllm.entrypoints.chat_utils import load_chat_template from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat from vllm.entrypoints.serve.engine.protocol import ErrorResponse diff --git a/components/src/dynamo/vllm/tests/test_vllm_prompt_embeds.py b/components/src/dynamo/vllm/tests/test_vllm_prompt_embeds.py index ccec17b5573f..58cd0d7e8011 100644 --- a/components/src/dynamo/vllm/tests/test_vllm_prompt_embeds.py +++ b/components/src/dynamo/vllm/tests/test_vllm_prompt_embeds.py @@ -33,6 +33,9 @@ class MockHandler: handler.model_config = Mock(enable_prompt_embeds=True) handler.model_config.dtype = torch.float32 handler.model_config.get_hidden_size.return_value = 4096 + # No attributes: vLLM's loader prefers hf_text_config.word_embed_proj_dim + # when present, and a bare Mock would invent one. + handler.model_config.hf_text_config = Mock(spec=[]) handler._decode_prompt_embeds = BaseWorkerHandler._decode_prompt_embeds.__get__( # type: ignore handler ) diff --git a/container/compliance/license_overrides.yaml b/container/compliance/license_overrides.yaml index 3e74190b4a2c..d98456b4e75a 100644 --- a/container/compliance/license_overrides.yaml +++ b/container/compliance/license_overrides.yaml @@ -264,6 +264,7 @@ overrides: # Same NVIDIA EULA as the 12.9 variants above; the cuda-dl-base 25.11 # image bundles cuda-gdb so DEP-5 parser misclassifies these as GPL-3. - { ecosystem: dpkg, name: cuda-cccl-13-0, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } + - { ecosystem: dpkg, name: cuda-compiler-13-0, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA (metapackage)" } - { ecosystem: dpkg, name: cuda-crt-13-0, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } - { ecosystem: dpkg, name: cuda-cudart-13-0, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } - { ecosystem: dpkg, name: cuda-cudart-dev-13-0, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } @@ -272,8 +273,13 @@ overrides: - { ecosystem: dpkg, name: cuda-cuobjdump-13-0, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } - { ecosystem: dpkg, name: cuda-cupti-13-0, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } - { ecosystem: dpkg, name: cuda-cupti-dev-13-0, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } + - { ecosystem: dpkg, name: cuda-cuxxfilt-13-0, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA; its only GPL v3 grant (Attachment B) covers the CUDA-GDB component" } - { ecosystem: dpkg, name: cuda-driver-dev-13-0, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } - { ecosystem: dpkg, name: cuda-gdb-13-0, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA — cuda-gdb proper, not GDB upstream" } + - { ecosystem: dpkg, name: cuda-libraries-13-0, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA (metapackage)" } + - { ecosystem: dpkg, name: cuda-libraries-dev-13-0, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA (metapackage)" } + - { ecosystem: dpkg, name: cuda-minimal-build-13-0, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA (metapackage)" } + - { ecosystem: dpkg, name: cuda-nsight-compute-13-0, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA (metapackage)" } - { ecosystem: dpkg, name: cuda-nvcc-13-0, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } - { ecosystem: dpkg, name: cuda-nvdisasm-13-0, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } - { ecosystem: dpkg, name: cuda-nvml-dev-13-0, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } @@ -281,7 +287,9 @@ overrides: - { ecosystem: dpkg, name: cuda-nvrtc-13-0, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } - { ecosystem: dpkg, name: cuda-nvrtc-dev-13-0, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } - { ecosystem: dpkg, name: cuda-nvtx-13-0, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } + - { ecosystem: dpkg, name: cuda-opencl-13-0, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA; its only GPL v3 grant (Attachment B) covers the CUDA-GDB component" } - { ecosystem: dpkg, name: cuda-profiler-api-13-0, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } + - { ecosystem: dpkg, name: cuda-sandbox-dev-13-0, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA; its only GPL v3 grant (Attachment B) covers the CUDA-GDB component" } - { ecosystem: dpkg, name: cuda-sanitizer-13-0, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } - { ecosystem: dpkg, name: cuda-toolkit-13-0-config-common, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } - { ecosystem: dpkg, name: cuda-toolkit-13-config-common, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } @@ -302,12 +310,79 @@ overrides: - { ecosystem: dpkg, name: libnpp-13-0, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } - { ecosystem: dpkg, name: libnpp-dev-13-0, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } - { ecosystem: dpkg, name: libnvfatbin-13-0, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } + - { ecosystem: dpkg, name: libnvfatbin-dev-13-0, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA; its only GPL v3 grant (Attachment B) covers the CUDA-GDB component" } - { ecosystem: dpkg, name: libnvjitlink-13-0, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } - { ecosystem: dpkg, name: libnvjitlink-dev-13-0, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } - { ecosystem: dpkg, name: libnvjpeg-13-0, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } - { ecosystem: dpkg, name: libnvjpeg-dev-13-0, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } - { ecosystem: dpkg, name: libnvptxcompiler-13-0, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } - { ecosystem: dpkg, name: libnvvm-13-0, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } + + # --- CUDA 13.4 toolkit dpkgs (CUDA 13.4.1 vLLM and SGLang bases) --- + # The 13.0 entries above under their 13-4 names, plus the packages new in 13.4. Same NVIDIA + # CUDA Toolkit EULA; its only GPL v3 grant (Attachment B) covers the CUDA-GDB component. + - { ecosystem: dpkg, name: cuda-cccl-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } + - { ecosystem: dpkg, name: cuda-command-line-tools-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA (metapackage)" } + - { ecosystem: dpkg, name: cuda-compat-orin-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA (arm64 only; same copyright file as libcublas-13-4)" } + - { ecosystem: dpkg, name: cuda-compiler-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA (metapackage)" } + - { ecosystem: dpkg, name: cuda-crt-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } + - { ecosystem: dpkg, name: cuda-ctadvisor-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } + - { ecosystem: dpkg, name: cuda-cudart-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } + - { ecosystem: dpkg, name: cuda-cudart-dev-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } + - { ecosystem: dpkg, name: cuda-culibos-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } + - { ecosystem: dpkg, name: cuda-culibos-dev-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } + - { ecosystem: dpkg, name: cuda-cuobjdump-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } + - { ecosystem: dpkg, name: cuda-cupti-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } + - { ecosystem: dpkg, name: cuda-cupti-dev-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } + - { ecosystem: dpkg, name: cuda-cuxxfilt-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA; its only GPL v3 grant (Attachment B) covers the CUDA-GDB component" } + - { ecosystem: dpkg, name: cuda-driver-dev-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } + - { ecosystem: dpkg, name: cuda-gdb-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA — cuda-gdb proper, not GDB upstream" } + - { ecosystem: dpkg, name: cuda-libraries-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA (metapackage)" } + - { ecosystem: dpkg, name: cuda-libraries-dev-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA (metapackage)" } + - { ecosystem: dpkg, name: cuda-minimal-build-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA (metapackage)" } + - { ecosystem: dpkg, name: cuda-nsight-compute-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA (metapackage)" } + - { ecosystem: dpkg, name: cuda-nvcc-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } + - { ecosystem: dpkg, name: cuda-nvdisasm-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } + - { ecosystem: dpkg, name: cuda-nvml-dev-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } + - { ecosystem: dpkg, name: cuda-nvprune-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } + - { ecosystem: dpkg, name: cuda-nvrtc-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } + - { ecosystem: dpkg, name: cuda-nvrtc-dev-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } + - { ecosystem: dpkg, name: cuda-nvtx-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } + - { ecosystem: dpkg, name: cuda-opencl-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA; its only GPL v3 grant (Attachment B) covers the CUDA-GDB component" } + - { ecosystem: dpkg, name: cuda-opencl-dev-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } + - { ecosystem: dpkg, name: cuda-profiler-api-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } + - { ecosystem: dpkg, name: cuda-sandbox-dev-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA; its only GPL v3 grant (Attachment B) covers the CUDA-GDB component" } + - { ecosystem: dpkg, name: cuda-sanitizer-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } + - { ecosystem: dpkg, name: cuda-tileiras-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } + - { ecosystem: dpkg, name: cuda-toolkit-13-4-config-common, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } + - { ecosystem: dpkg, name: libcublas-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } + - { ecosystem: dpkg, name: libcublas-dev-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } + - { ecosystem: dpkg, name: libcudla-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA (arm64 only; same copyright file as libcublas-13-4)" } + - { ecosystem: dpkg, name: libcudla-dev-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA (arm64 only; same copyright file as libcublas-13-4)" } + - { ecosystem: dpkg, name: libcufft-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } + - { ecosystem: dpkg, name: libcufft-dev-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } + - { ecosystem: dpkg, name: libcufile-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } + - { ecosystem: dpkg, name: libcufile-dev-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } + - { ecosystem: dpkg, name: libcuobjclient-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } + - { ecosystem: dpkg, name: libcuobjclient-dev-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } + - { ecosystem: dpkg, name: libcurand-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } + - { ecosystem: dpkg, name: libcurand-dev-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } + - { ecosystem: dpkg, name: libcusolver-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } + - { ecosystem: dpkg, name: libcusolver-dev-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } + - { ecosystem: dpkg, name: libcusparse-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } + - { ecosystem: dpkg, name: libcusparse-dev-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } + - { ecosystem: dpkg, name: libnpp-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } + - { ecosystem: dpkg, name: libnpp-dev-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } + - { ecosystem: dpkg, name: libnvfatbin-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } + - { ecosystem: dpkg, name: libnvfatbin-dev-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA; its only GPL v3 grant (Attachment B) covers the CUDA-GDB component" } + - { ecosystem: dpkg, name: libnvjitlink-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } + - { ecosystem: dpkg, name: libnvjitlink-dev-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } + - { ecosystem: dpkg, name: libnvjpeg-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } + - { ecosystem: dpkg, name: libnvjpeg-dev-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } + - { ecosystem: dpkg, name: libnvptxcompiler-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } + - { ecosystem: dpkg, name: libnvvm-13-4, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA CUDA Toolkit EULA" } + - { ecosystem: dpkg, name: nsight-compute-2026.3.0, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA Nsight EULA" } + - { ecosystem: python, name: cupti-python, license: LicenseRef-NVIDIA-Proprietary, source: "NVIDIA Software License Agreement — https://docs.nvidia.com/cupti-python/nvidia-cupti-python-license.html" } - { ecosystem: dpkg, name: mlnx-dpdk, license: "BSD-3-Clause", source: "Mellanox build of upstream DPDK — https://www.dpdk.org/about/legal/" } # TRT-LLM 1.3.0rc27's base removes this package and keeps its ld.so.conf.d conffile. # The .deb (DOCA 3.5.0 repo) ships no copyright file; its only SPDX tags are BSD-3-Clause. @@ -338,6 +413,7 @@ overrides: - { ecosystem: dpkg, name: libgc1, license: "BSD-3-Clause", source: "Boehm-GC upstream — http://www.hboehm.info/gc/license.txt (MIT-like, mapped to closest SPDX)" } - { ecosystem: dpkg, name: numactl, license: "GPL-2.0-or-later", source: "Linux NUMA tools, GPL-2+" } - { ecosystem: dpkg, name: ubuntu-keyring, license: "GPL-2.0-or-later", source: "Ubuntu keyring; copyright file references common-licenses/GPL-2" } + - { ecosystem: dpkg, name: cuda-keyring, license: "MIT", source: "NVIDIA upstream — https://github.com/NVIDIA/apt-packaging-cuda-keyring/blob/1f6469f225a49c22ba791028cd0588a6eeb5895b/LICENSE; the 1.1-1 .deb ships no copyright file" } # --- Sglang cuda12.9 — dpkg additions on top of cuda:12.9.1-cudnn-devel --- # X11 / libdrm / common system libs whose syft DEP-5 parse came back UNKNOWN. @@ -1049,3 +1125,9 @@ overrides: - { ecosystem: python, name: instanttensor, license: "Apache-2.0", source: "https://github.com/scitix/InstantTensor/blob/v0.1.9/LICENSE" } - { ecosystem: python, name: sigstore-models, license: "MIT", source: "https://github.com/astral-sh/sigstore-models/blob/v0.0.6/LICENSE" } - { ecosystem: rust, name: tsp-asn1, license: "Apache-2.0", source: "https://github.com/trailofbits/rfc3161-client/blob/v1.0.8/LICENSE" } + + # --- MiniMax M3.1 Flash public SGLang base (lmsysorg/sglang:dev-minimax-m31-flash): Python + # packages whose METADATA has no License field, so the generator reports UNKNOWN --- + - { ecosystem: python, name: fastokens, license: "Apache-2.0", source: "Crusoe fastokens — https://github.com/crusoecloud/fastokens/blob/main/LICENSE (Apache-2.0); PyPI metadata has no license" } + - { ecosystem: python, name: minimax-msa, license: "MIT", source: "MiniMax Sparse Attention: MIT License (Copyright (c) 2026 MiniMax) shipped as the package's License-File, minimax_msa-0.1.0.dist-info/licenses/LICENSE, in lmsysorg/sglang:dev-minimax-m31-flash@sha256:7470e685; METADATA has no License field" } + - { ecosystem: python, name: sglang-msa-mono, license: "MIT", source: "SGLang plugin built from the minimax-msa repository (integrations/sglang), whose README states 'This repository uses the MIT license'; METADATA has no License field" } diff --git a/container/context.yaml b/container/context.yaml index 0d380e1c7459..f2772f02fa95 100644 --- a/container/context.yaml +++ b/container/context.yaml @@ -93,18 +93,22 @@ dynamo: vllm: cuda13.0: base_image: nvcr.io/nvidia/cuda-dl-base + # Layer the Dynamo runtime on the public MiniMax M3.1 Flash vLLM image + # (amd64 + arm64, CUDA 13.4.1) instead of a stable vllm/vllm-openai release: + # no release serves the model yet. The tag is mutable, so pin the index digest. runtime_image: vllm/vllm-openai base_image_tag: 25.11-cuda13.0-devel-ubuntu24.04 - runtime_image_tag: v0.31.0-ubuntu2404 + runtime_image_tag: minimax-m31-flash@sha256:86f582dae25ea92802353727164ae4eb03b37b18baf9119b805ffaec2a7ef07c vllm_omni_ref: "v0.31.0rc1" - # Baseline is vllm-openai's TRUE base, nvidia/cuda:13.0.2-base-ubuntu24.04 - # (Docker Hub), NOT a self-baseline of the third-party vllm/vllm-openai image. - # vllm-openai is built FROM nvidia/cuda, so this attributes everything vLLM - # adds over the clean NVIDIA CUDA floor (uv/rust stack, framework, codecs) - # rather than hiding it. Per-arch stem; the licenses stage appends - # -${TARGETARCH}.cdx.json. (GPL media codecs from vllm-openai are purged in - # the runtime image — see the codec-purge follow-up.) - baseline_sbom: cuda@2ab6381d + # No vLLM-Omni: it backs only the separate dynamo.vllm.omni entry point. Its + # dependencies cannot resolve against this base's torch nightly, and its + # Transformers <5.15 pin is below this vLLM's own floor (>=5.16.1), so the + # runtime keeps the base's Transformers and tokenizers. + enable_omni: "false" + # No baseline_sbom: the captured cuda@2ab6381d baseline is the CUDA 13.0 + # floor of stable vllm-openai, not this CUDA 13.4 base, so subtracting it + # could drop shipped components. Leaving it unset attributes the whole image + # in /legal. xpu: base_image: vllm/vllm-openai-xpu runtime_image: vllm/vllm-openai-xpu @@ -121,7 +125,10 @@ vllm: # Omni requires Transformers <5.15, which requires tokenizers <=0.23.0. transformers_version: "5.14.1" tokenizers_version: "0.22.2" - nixl_ref: v1.3.2 + # Installed only when the vLLM base ships no NIXL; the MiniMax M3.1 NIXL KV + # connector requires NIXL >= 1.5.0, and with 1.3.2 every P/D worker fails + # engine-core init. + nixl_ref: v1.5.0 max_jobs: "10" enable_media_ffmpeg: "true" enable_gpu_memory_service: "true" @@ -134,20 +141,21 @@ vllm: sglang: cuda13.0: base_image: nvcr.io/nvidia/cuda-dl-base + # Layer the Dynamo runtime on the public MiniMax M3.1 Flash SGLang image + # (amd64 + arm64, CUDA 13.4.1) instead of a stable lmsysorg/sglang release: + # no release serves the model yet, and no -runtime variant of it is + # published. The tag is mutable, so pin the index digest. runtime_image: lmsysorg/sglang base_image_tag: 25.11-cuda13.0-devel-ubuntu24.04 - runtime_image_tag: v0.5.21-cu130-runtime - # Baseline is the TRUE base of the third-party lmsysorg/sglang image, - # nvidia/cuda:13.0.3-cudnn-devel-ubuntu24.04 (Docker Hub; CUDA_VERSION=13.0.3, - # ubuntu24.04), NOT a self-baseline of lmsysorg/sglang. - # lmsysorg/sglang is built FROM this nvidia/cuda cudnn-devel base, so this - # attributes everything sglang adds over the clean NVIDIA CUDA+cuDNN floor - # (framework, python stack, system libs) rather than hiding it. Per-arch stem; - # the licenses stage appends -${TARGETARCH}.cdx.json. SGLang's codec-bearing - # Python wheels are purged in sglang_runtime.Dockerfile; only the in-tree VP9 - # ffmpeg is copied in. The CUDA Dynamo Rust wheel links against that same - # codec-limited build for frontend video decoding; XPU remains codec-free. - baseline_sbom: cuda@0230b7f2 + runtime_image_tag: dev-minimax-m31-flash@sha256:7470e685c9e488f4c99431e22ca2e9a9a6de61a182a582aa2659a0b22f51ac76 + # No baseline_sbom: the captured cuda@0230b7f2 baseline is the CUDA 13.0 + # floor of stable lmsysorg/sglang, not this CUDA 13.4 base, so subtracting it + # could drop shipped components. Leaving it unset attributes the whole image + # in /legal. + # SGLang's codec-bearing Python wheels are purged in sglang_runtime.Dockerfile; + # only the in-tree VP9 ffmpeg is copied in. The CUDA Dynamo Rust wheel links + # against that same codec-limited build for frontend video decoding; XPU + # remains codec-free. xpu: base_image: intel/deep-learning-essentials base_image_tag: 2025.3.2-0-devel-ubuntu24.04 diff --git a/container/templates/sglang_runtime.Dockerfile b/container/templates/sglang_runtime.Dockerfile index d2b3d70afeb6..4532250cd76f 100644 --- a/container/templates/sglang_runtime.Dockerfile +++ b/container/templates/sglang_runtime.Dockerfile @@ -247,9 +247,18 @@ RUN set -eux; \ # dependency closure. Remove the packages (and their dpkg metadata) before # copying Dynamo's separately built VP9-only ffmpeg below. File deletion # alone is insufficient because the compliance generator inventories dpkg. - apt-get purge -y --auto-remove \ - ffmpeg \ - libwayland-server0; \ + # Purge only the named packages that are installed: apt cannot resolve an + # absent package once the base has deleted its package lists, and some + # bases ship ffmpeg without libwayland-server0. + purge=""; \ + for pkg in ffmpeg libwayland-server0; do \ + if dpkg-query -W -f='${Status}' "$pkg" 2>/dev/null | grep -q 'install ok installed'; then \ + purge="$purge $pkg"; \ + fi; \ + done; \ + if [ -n "$purge" ]; then \ + apt-get purge -y --auto-remove $purge; \ + fi; \ rm -rf /var/lib/apt/lists/*; \ python3 -m pip uninstall --yes \ av \ diff --git a/container/templates/vllm_runtime.Dockerfile b/container/templates/vllm_runtime.Dockerfile index 1264205eb85a..fc97e00df391 100644 --- a/container/templates/vllm_runtime.Dockerfile +++ b/container/templates/vllm_runtime.Dockerfile @@ -65,7 +65,11 @@ ENV NIXL_PREFIX=/opt/dynamo/nixl \ NIXL_LIB_DIR=/opt/dynamo/nixl \ NIXL_PLUGIN_DIR=/opt/dynamo/nixl/plugins COPY --chmod=755 container/deps/vllm/install_nixl_from_wheel.sh /usr/local/bin/install_nixl_from_wheel -RUN install_nixl_from_wheel \ +RUN if [ ! -d "${SITE_PACKAGES}/.nixl_cu${CUDA_MAJOR}.mesonpy.libs" ]; then \ + python3 -m pip install --no-deps \ + "nixl==${NIXL_REF#v}" "nixl-cu${CUDA_MAJOR}==${NIXL_REF#v}"; \ + fi && \ + install_nixl_from_wheel \ --cuda-major "${CUDA_MAJOR}" \ --site-packages "${SITE_PACKAGES}" \ --prefix "${NIXL_PREFIX}" \ @@ -177,6 +181,8 @@ COPY --chmod=775 --chown=dynamo:0 --from=wheel_builder /opt/dynamo/dist/*.whl /o {% set pip_target = "--system" if device == "cuda" else "--python /opt/venv/bin/python" %} {% set python_executable = "python3" if device == "cuda" else "/opt/venv/bin/python" %} +{# enable_omni: "false" skips vLLM-Omni and the Transformers/tokenizers versions it requires. #} +{% set enable_omni = context.vllm[device_key].get("enable_omni", "true") == "true" %} {% if device != "cuda" %} {% set vllm_rs_link = "${VIRTUAL_ENV}/bin/vllm-rs" %} {# vLLM <0.31 rejects the extra output fields added by Omni. #} @@ -184,11 +190,13 @@ COPY --chmod=775 --chown=dynamo:0 --from=wheel_builder /opt/dynamo/dist/*.whl /o {% set vllm_rs_plugins = "modelexpress" if context.vllm.enable_modelexpress == "true" else "" %} {% endif %} +{% if enable_omni %} # Align Transformers and tokenizers before freezing Omni's protected dependencies. RUN --mount=type=cache,id=uv-root-{{ context.dynamo.uv_version }},target=/root/.cache/uv,sharing=locked \ export UV_CACHE_DIR=/root/.cache/uv && \ uv pip install {{ pip_target }} --no-deps \ "transformers==${TRANSFORMERS_VERSION}" "tokenizers==${TOKENIZERS_VERSION}" +{% endif %} {% if device != "cuda" %} # NIXL meta package always tries to find a cuda-backend @@ -257,6 +265,7 @@ RUN set -eux; \ ldconfig -p | grep -q 'libturbojpeg.so.0'; \ rm -rf /var/lib/apt/lists/* +{% if enable_omni %} # Layer the released vLLM-Omni package matching the pinned upstream ref while # constraining packages already solved in the upstream vLLM image. RUN --mount=type=bind,source=./container/deps/vllm/protected_packages.txt,target=/tmp/vllm_omni_protected_packages.txt \ @@ -266,6 +275,7 @@ RUN --mount=type=bind,source=./container/deps/vllm/protected_packages.txt,target export UV_CACHE_DIR=/root/.cache/uv; \ export VLLM_OMNI_TARGET_DEVICE={{ device }}; \ bash /tmp/install_vllm_omni.sh +{% endif %} {% if device == "xpu" %} # Remove conflicting standard triton package for XPU and reinstall triton-xpu @@ -587,6 +597,7 @@ assert eps, 'modelexpress vllm.general_plugins entry point not found'; \ [ep.load()() for ep in eps]" {% endif %} +{% if enable_omni %} # Check that later package layers preserve the Omni-compatible versions. RUN {{ python_executable }} - "${TRANSFORMERS_VERSION}" "${TOKENIZERS_VERSION}" <<'PY' import importlib.metadata as md @@ -597,10 +608,13 @@ for package, expected in zip(("transformers", "tokenizers"), sys.argv[1:]): if actual != expected: raise RuntimeError(f"expected {package} {expected}, found {actual}") PY +{% endif %} {% if device == "cuda" %} +{% if enable_omni %} # Omni's soxr dependency is only needed for unsupported Breeze voice cloning. RUN python3 -m pip uninstall --yes soxr +{% endif %} RUN vllm-rs --help >/dev/null {% else %} diff --git a/lib/llm/PREPROCESSOR_CASES.md b/lib/llm/PREPROCESSOR_CASES.md index 12d431045af3..3449e2941936 100644 --- a/lib/llm/PREPROCESSOR_CASES.md +++ b/lib/llm/PREPROCESSOR_CASES.md @@ -154,7 +154,7 @@ if you know the answer, fill it in. | `mistral` (tool + reasoning) | **YES** for reasoning | disabled unless `reasoning_effort` is present and not `none` | — | — | `[TOOL_CALLS]` tool text; `[THINK]...[/THINK]` reasoning. | | `phi4` (tool) | NO | — | — | — | `functools[...]` plain text. | | `minimax_m2` (tool + reasoning) | NO | `thinking=false` / `enable_thinking=false` | — | — | Force reasoning with `...`; dynamically distinguishes bare guided JSON from reasoning followed by an XML tool payload. | -| `minimax_m3` (tool + reasoning) | YES | `thinking_mode=disabled` | — | — | Prompt-prefilled `...` reasoning with MiniMax namespace-token XML tool calls. | +| `minimax_m3` (tool + reasoning) | YES | `thinking_mode=disabled`, when the rendered prompt ends with `` or a grammar applies | — | — | Prompt-prefilled `...` reasoning with MiniMax namespace-token XML tool calls. The parser is seeded inside reasoning when the rendered prompt ends with `` (except explicit `guided_*` without a held grammar), and outside it when the prompt ends at the generation header or ``. | | `minimax_append_think` (reasoning) | NO | — | — | OFF on `tool_choice=required/named` (universal, PRE.4) | Deprecated legacy MiniMax pass-through; does not populate `reasoning_content`. Use `minimax_m2` for MiniMax M2 tool-calling deployments. | | `glm47` (tool) | NO | — | — | — | Plain XML. | | `jamba` (tool) | NO | — | — | — | `` plain text wrapper. | diff --git a/lib/llm/src/preprocessor.rs b/lib/llm/src/preprocessor.rs index 30f623f8da4c..76b01060b95d 100644 --- a/lib/llm/src/preprocessor.rs +++ b/lib/llm/src/preprocessor.rs @@ -198,6 +198,13 @@ pub struct PromptReasoningPrefill { legacy: bool, unified: bool, prefix: Option<&'static str>, + /// MiniMax M3 with explicit `guided_*` only: the backend holds the grammar + /// until the prompt-opened reasoning block closes, so the output after the + /// prefill starts as reasoning. + backend_holds_guided_grammar: bool, + /// MiniMax M3 only: how the rendered prompt leaves the reasoning block. + /// `None` when no rendered prompt is known or its end is not recognized. + minimax_m3_prompt_tail: Option, } impl From for PromptReasoningPrefill { @@ -207,6 +214,37 @@ impl From for PromptReasoningPrefill { legacy: value, unified: value, prefix: None, + backend_holds_guided_grammar: false, + minimax_m3_prompt_tail: None, + } + } +} + +/// The end of a rendered MiniMax M3 prompt. The template decides whether the +/// completion starts inside reasoning, and some templates ignore +/// `thinking_mode`, so this, not the request, is authoritative. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum MiniMaxM3PromptTail { + /// Ends with ``: the completion starts inside reasoning. + Open, + /// Ends with ``: the reasoning block is closed (`disabled`). + Closed, + /// Ends at the generation header: the model decides whether to think. + Header, +} + +impl MiniMaxM3PromptTail { + /// `None` for any other end, such as a continued assistant message. + fn from_prompt(prompt: &str) -> Option { + let prompt = prompt.trim_end(); + if prompt.ends_with("") { + Some(Self::Open) + } else if prompt.ends_with("") { + Some(Self::Closed) + } else if prompt.ends_with("]~b]ai") { + Some(Self::Header) + } else { + None } } } @@ -684,18 +722,39 @@ fn drain_deferred_reasoning(state: &mut ChoiceReasoningState) -> (Option } } +/// Finalize a choice's parser without the deferral, returning `(content, +/// reasoning_content)` by the parser's own split: held marker-free text is +/// content, a held partial marker inside reasoning stays reasoning. Shared by +/// the terminal-chunk finalize and the end-of-stream fallback so the two +/// cannot disagree. Finalizes at most once per choice; see `parser_finished`. +fn finish_reasoning_parser(state: &mut ChoiceReasoningState) -> (Option, Option) { + if state.parser_finished { + return (None, None); + } + state.parser_finished = true; + let result = state.parser.finish_reasoning_stream(); + ( + (!result.normal_text.is_empty()).then_some(result.normal_text), + (!result.reasoning_text.is_empty()).then_some(result.reasoning_text), + ) +} + struct ReasoningState { stream: Pin> + Send>>, parser_name: String, - prompt_injected_reasoning: bool, + // Each choice's parser starts here; `None` leaves the parser to infer it. + initial_in_reasoning: Option, bypass_bare_guided_json: bool, choices: HashMap, - // Last emitted content-bearing response, reused as the envelope to carry any + // An emitted response with choices, reused as the envelope to carry any // text the parsers are still buffering when the upstream stream ends. Only - // retained when `defer_reasoning_for_nonempty_content` is set, to keep the - // per-token clone off the common reasoning hot path. Chunks with no choices - // (the trailing usage-only chunk) are never retained — that envelope has no - // delta slot to attach the flushed bytes to, so the flush would be dropped. + // retained when an end-of-stream flush may follow, to keep the per-token + // clone off the common reasoning hot path: under + // `defer_reasoning_for_nonempty_content` it is the latest such response; + // under `finalize_parser_at_finish` alone it is cloned once, while a parsed + // choice's parser is unfinished. Chunks with no choices (the trailing + // usage-only chunk) are never retained — that envelope has no delta slot + // to attach the flushed bytes to, so the flush would be dropped. last_response: Option>, // Nemotron force-reasoning parsers start inside the reasoning block, so // leading model output with no `` is reported as reasoning even when @@ -728,6 +787,14 @@ struct ReasoningState { // into `content` in the first place. Scoped to the flag so no other parser // pays it. defer_reasoning_for_nonempty_content: bool, + // Finalize each choice's parser on its terminal chunk even without the + // deferral above, for parsers that retain undecided text until EOF + // (MiniMax M3), and at end of stream for a choice whose parser is still + // unfinished: it sent no `finish_reason`, or its terminal chunk carried + // content parts. Otherwise a turn whose output never shows a marker, such + // as grammar-constrained output under an adaptive prompt, is dropped + // whole with the parser state. See `is_minimax_m3_reasoning_parser`. + finalize_parser_at_finish: bool, // Set once the backend has sent an error annotation. The end-of-stream // flush must not run after that: an error is terminal, and emitting // recovered reasoning as ordinary `content` after it would put text on the @@ -2429,8 +2496,12 @@ impl OpenAIPreprocessor { ) } - fn has_minimax_m3_constrained_generation(request: &R) -> bool { - Self::has_structured_response_format(request) || Self::has_guided_tool_choice(request) + /// A structured `response_format`, top-level `guided_*`, and a forced tool + /// choice each constrain output from the first token. + fn has_minimax_m3_constrained_generation(request: &NvCreateChatCompletionRequest) -> bool { + Self::has_structured_response_format(request) + || Self::has_guided_tool_choice(request) + || crate::preprocessor::tool_choice::has_explicit_guided_decoding(request) } /// True when `tool_choice` forces a tool call. @@ -5316,16 +5387,22 @@ impl OpenAIPreprocessor { | Some(ChatCompletionToolChoiceOption::Named(_)) ); let is_structured_response = Self::has_structured_response_format(request); - let is_guided_output = is_guided_tool_choice || is_structured_response; let reasoning_parser = self.runtime_config.reasoning_parser.as_deref(); + let is_minimax_m3_explicit_guided_output = + Self::is_minimax_m3_family(reasoning_parser, self.tool_call_parser.as_deref()) + && crate::preprocessor::tool_choice::has_explicit_guided_decoding(request); + let is_guided_output = + is_guided_tool_choice || is_structured_response || is_minimax_m3_explicit_guided_output; // Force parsers opt in by capability; prompt-seeded parsers opt in per request. // Structural-tag tool formats do not use this guided-JSON detection path. let inspect_force_reasoning_guided_output = is_guided_output && !uses_tool_call_structural_tag && Self::supports_reasoning_before_guided_json(reasoning_parser); + // A held grammar means the output starts as reasoning, never bare JSON. let inspect_prompt_injected_guided_output = is_guided_output && prompt_injected_reasoning && !uses_tool_call_structural_tag + && !prompt_prefill.backend_holds_guided_grammar && (Self::skips_guided_json_when_prompt_injected(reasoning_parser) || (is_structured_response && Self::skips_structured_response_when_prompt_injected(reasoning_parser))); @@ -5341,10 +5418,18 @@ impl OpenAIPreprocessor { && Self::is_force_reasoning_parser(reasoning_parser) && !inspect_force_reasoning_guided_output; + // MiniMax M3 `disabled` only rules out reasoning when the template + // closed the reasoning block in the prompt, or a grammar owns the output + // from token 0. A prompt that ends at the generation header leaves the + // model free to think (a template that ignores `thinking_mode`), so its + // markers must still be parsed. + let is_minimax_m3_reasoning_possible = prompt_prefill.minimax_m3_prompt_tail + == Some(MiniMaxM3PromptTail::Header) + && !Self::has_minimax_m3_constrained_generation(request); let reasoning_disabled_by_request = Self::is_reasoning_disabled_by_request( self.runtime_config.reasoning_parser.as_deref(), request.chat_template_args.as_ref(), - ); + ) && !is_minimax_m3_reasoning_possible; // Try to parse reasoning content only if parser is configured. let should_parse_reasoning = self.runtime_config.reasoning_parser.is_some() @@ -5380,16 +5465,37 @@ impl OpenAIPreprocessor { Box::pin(stream) }; + // On MiniMax M3 a top-level `guided_*` grammar can own the output from + // token 0, so a prefilled `` does not prove the output is + // reasoning: leave the parser unseeded and let its finalize decide. + // Keep the seed when the backend holds the grammar until reasoning + // ends: the output then starts as reasoning, even when cut off. + let seed_reasoning_parser = prompt_injected_reasoning + && (prompt_prefill.backend_holds_guided_grammar + || !(Self::is_minimax_m3_reasoning_parser(reasoning_parser) + && crate::preprocessor::tool_choice::has_explicit_guided_decoding(request))); + // A rendered M3 prompt that ends at the generation header or a closed + // block means the output starts outside reasoning. Left unset, the + // parser cannot rule out an implicit opener and holds marker-free text + // until EOF instead of streaming it. + let initial_in_reasoning = match prompt_prefill.minimax_m3_prompt_tail { + _ if seed_reasoning_parser => Some(true), + Some(MiniMaxM3PromptTail::Header | MiniMaxM3PromptTail::Closed) => Some(false), + _ => None, + }; + // Only a force_nonempty_content request needs the deferral and the EOF // flush of a truncated `` prefix; gating on the request keeps the // per-token clone and the finish_reasoning_stream() flush off every - // other request's path. Same predicate the aggregator uses, so the - // streaming and non-streaming paths cannot disagree. + // other request's path (a MiniMax M3 parser is still finalized on its + // terminal chunk, or at end of stream without one; see + // `finalize_parser_at_finish`). Same predicate the aggregator uses, so + // the streaming and non-streaming paths cannot disagree. let stream: Pin + Send>> = if should_parse_reasoning { Box::pin(Self::parse_reasoning_content_from_stream_inner( stream, self.runtime_config.reasoning_parser.clone().unwrap(), // Safety: We already checked that parser is some, so gtg - prompt_injected_reasoning, + initial_in_reasoning, bypass_reasoning_for_bare_guided_json, defer_reasoning_for_nonempty_content, )) @@ -5400,12 +5506,16 @@ impl OpenAIPreprocessor { } else { Box::pin(stream) }; - let stream: Pin + Send>> = - if defer_reasoning_for_nonempty_content || should_strip_disabled_reasoning_start { - Box::pin(Self::hold_usage_until_stream_end(stream)) - } else { - stream - }; + // A MiniMax M3 parser can also release held text in an end-of-stream + // chunk (see `finalize_parser_at_finish`), so its usage is held too. + let stream: Pin + Send>> = if defer_reasoning_for_nonempty_content + || should_strip_disabled_reasoning_start + || (should_parse_reasoning && Self::is_minimax_m3_reasoning_parser(reasoning_parser)) + { + Box::pin(Self::hold_usage_until_stream_end(stream)) + } else { + stream + }; let stream: Pin + Send>> = if should_parse_reasoning { Box::pin(annotate_reasoning_usage(stream)) } else { @@ -6499,7 +6609,8 @@ impl OpenAIPreprocessor { /// /// `enable_thinking` is intentionally not consulted here: when thinking is /// off, reasoning parsing is disabled, so `reasoning_content` is always empty - /// and the move is vacuous rather than suppressed. + /// and the move is vacuous rather than suppressed. (MiniMax M3 keeps parsing + /// a `disabled` request whose rendered prompt leaves reasoning open.) pub(crate) fn wants_reasoning_as_content_when_empty( chat_template_args: Option<&std::collections::HashMap>, ) -> bool { @@ -6526,6 +6637,8 @@ impl OpenAIPreprocessor { /// nothing is withheld. That direction is conservative — extra comment /// frames on an opt-in path rather than a silent stream — and closing it /// needs the guided-output derivation lifted out of the stream builder. + /// MiniMax M3 `disabled` keeps the heartbeat for the same reason: whether + /// its parser runs depends on the rendered prompt, which is not known here. pub(crate) fn stream_can_defer_all_output( tool_call_parser: Option<&str>, reasoning_parser: Option<&str>, @@ -6540,7 +6653,8 @@ impl OpenAIPreprocessor { .is_some(); has_reasoning_decoder && Self::wants_reasoning_as_content_when_empty(chat_template_args) - && !Self::is_reasoning_disabled_by_request(reasoning_parser, chat_template_args) + && (Self::is_minimax_m3_reasoning_parser(reasoning_parser) + || !Self::is_reasoning_disabled_by_request(reasoning_parser, chat_template_args)) } /// Parsers that begin streaming in reasoning mode (force_reasoning=true). @@ -6612,6 +6726,30 @@ impl OpenAIPreprocessor { ) } + /// MiniMax M3's reasoning parser recovers a reasoning block whose opener + /// is not visible (implicit tool-start recovery), so it retains + /// marker-free text until a close marker, a tool marker, or EOF decides + /// whether that text was reasoning. Its EOF decision only reaches the + /// client when the stream finalizes it. When the frontend rendered the + /// prompt, the parser is told where it starts instead (see + /// `MiniMaxM3PromptTail`), and marker-free text streams as it arrives. + fn is_minimax_m3_reasoning_parser(reasoning_parser: Option<&str>) -> bool { + matches!(reasoning_parser, Some("minimax_m3" | "minimax-m3")) + } + + /// For MiniMax M3 with explicit `guided_*`: whether the backend holds the + /// grammar until the prompt-opened reasoning block closes. Pass the final + /// `require_reasoning`, after tool-choice guidance has set it. + fn minimax_m3_guided_grammar_after_reasoning( + &self, + request: &NvCreateChatCompletionRequest, + require_reasoning: bool, + ) -> bool { + Self::is_minimax_m3_reasoning_parser(self.runtime_config.reasoning_parser.as_deref()) + && crate::preprocessor::tool_choice::has_explicit_guided_decoding(request) + && self.backend_holds_guided_grammar(require_reasoning) + } + fn skips_structured_response_when_prompt_injected(reasoning_parser: Option<&str>) -> bool { matches!(reasoning_parser, Some("qwen3" | "kimi_k3" | "kimi-k3")) || Self::skips_guided_json_when_prompt_injected(reasoning_parser) @@ -6635,7 +6773,9 @@ impl OpenAIPreprocessor { }; match reasoning_parser { - Some("minimax_m3") | Some("minimax-m3") => prompt.ends_with(""), + Some("minimax_m3") | Some("minimax-m3") => { + MiniMaxM3PromptTail::from_prompt(prompt) == Some(MiniMaxM3PromptTail::Open) + } Some("kimi_k3") | Some("kimi-k3") => { let mut suffix = prompt.chars().rev().filter(|c| !c.is_whitespace()); "<|open|>think<|sep|>" @@ -6697,10 +6837,15 @@ impl OpenAIPreprocessor { } else { None }; + let minimax_m3_prompt_tail = formatted_prompt + .filter(|_| Self::is_minimax_m3_reasoning_parser(reasoning_parser)) + .and_then(MiniMaxM3PromptTail::from_prompt); PromptReasoningPrefill { legacy, unified: unified || prefix.is_some(), prefix, + backend_holds_guided_grammar: false, + minimax_m3_prompt_tail, } } @@ -6847,7 +6992,7 @@ impl OpenAIPreprocessor { Self::parse_reasoning_content_from_stream_inner( stream, parser_name, - prompt_injected_reasoning, + prompt_injected_reasoning.then_some(true), false, false, ) @@ -6856,7 +7001,7 @@ impl OpenAIPreprocessor { fn parse_reasoning_content_from_stream_inner( stream: S, parser_name: String, - prompt_injected_reasoning: bool, + initial_in_reasoning: Option, bypass_bare_guided_json: bool, defer_reasoning_for_nonempty_content: bool, ) -> impl Stream> + Send @@ -6865,14 +7010,17 @@ impl OpenAIPreprocessor { { // Parsers and bypass decisions are created lazily per `choice.index` // inside the unfold loop, so `n > 1` choices never share state. + let finalize_parser_at_finish = + Self::is_minimax_m3_reasoning_parser(Some(parser_name.as_str())); let state = ReasoningState { stream: Box::pin(stream), parser_name, - prompt_injected_reasoning, + initial_in_reasoning, bypass_bare_guided_json, choices: HashMap::new(), last_response: None, defer_reasoning_for_nonempty_content, + finalize_parser_at_finish, saw_terminal_error: false, }; @@ -6892,16 +7040,18 @@ impl OpenAIPreprocessor { let processed_response = { let ReasoningState { parser_name, - prompt_injected_reasoning, + initial_in_reasoning, bypass_bare_guided_json, choices, defer_reasoning_for_nonempty_content, + finalize_parser_at_finish, .. } = &mut state; let parser_name = &*parser_name; - let prompt_injected_reasoning = *prompt_injected_reasoning; + let initial_in_reasoning = *initial_in_reasoning; let bypass_bare_guided_json = *bypass_bare_guided_json; let defer_reasoning = *defer_reasoning_for_nonempty_content; + let finalize_parser_at_finish = *finalize_parser_at_finish; response.map_data(|mut data| { for choice in data.inner.choices.iter_mut() { @@ -6911,8 +7061,8 @@ impl OpenAIPreprocessor { parser_name, )) as Box; - if prompt_injected_reasoning { - parser.set_in_reasoning(true); + if let Some(in_reasoning) = initial_in_reasoning { + parser.set_in_reasoning(in_reasoning); } ChoiceReasoningState { parser, @@ -7071,17 +7221,60 @@ impl OpenAIPreprocessor { ); } } + + // Without the deferral, a parser that retains + // undecided text until EOF is finalized here, on + // the same terminal delta and under the same + // conditions; see `finish_reasoning_parser`. + if !defer_reasoning + && finalize_parser_at_finish + && choice.finish_reason.is_some() + && !terminal_carries_parts + && bypass_decision == Some(false) + { + let (content, reasoning) = finish_reasoning_parser(choice_state); + if let Some(content) = content { + let content = match choice.delta.content.take() { + Some(ChatCompletionMessageContent::Text(existing)) => { + existing + &content + } + // `None`; parts were excluded above. + _ => content, + }; + choice.delta.content = + Some(ChatCompletionMessageContent::Text(content)); + } + if let Some(reasoning) = reasoning { + choice.delta.reasoning_content = Some( + choice.delta.reasoning_content.take().unwrap_or_default() + + &reasoning, + ); + } + } } Ok(data) }) }; // Retain a spare envelope only when an EOF flush may follow, so - // the common reasoning path avoids a full per-token clone. Skip + // the common reasoning path avoids a full per-token clone: the + // latest chunk under the deferral, or, without it, one chunk + // taken while a parsed choice still has a parser to finalize (a + // bypassed choice never fed its parser, and a finished one has + // nothing left). One is enough there: the flush rebuilds every + // choice field and scrubs the per-chunk metadata, and the rest + // of the envelope is the same for the whole response. Skip // chunks with no choices (the trailing usage-only chunk): they // carry no delta slot, so using one as the flush envelope would // drop the very bytes the flush exists to preserve. - if state.defer_reasoning_for_nonempty_content + let retain_envelope = state.defer_reasoning_for_nonempty_content + || (state.finalize_parser_at_finish + && state.last_response.is_none() + && state.choices.values().any(|choice_state| { + choice_state.guided_json_bypass_decision == Some(false) + && !choice_state.parser_finished + })); + if retain_envelope && processed_response .data .as_ref() @@ -7090,26 +7283,31 @@ impl OpenAIPreprocessor { state.last_response = Some(processed_response.clone()); } Some((processed_response, state)) - } else if !state.defer_reasoning_for_nonempty_content || state.saw_terminal_error { + } else if state.saw_terminal_error + || !(state.defer_reasoning_for_nonempty_content || state.finalize_parser_at_finish) + { // After a backend error the buffered bytes are dropped rather // than surfaced: the request failed, so there is no answer to // complete. See `saw_terminal_error`. None } else { // Upstream ended with no answer for the choices still holding a - // pending buffer. Normally the terminal chunk already drained - // them, so this is the fallback for a stream that ends without - // any `finish_reason` — an aborted or truncated generation. - // The synthetic chunk it builds is the only case where recovered - // bytes arrive after the last upstream chunk, which is - // unavoidable when there was no terminal chunk to attach them - // to. Only the force_nonempty_content path reaches this branch; - // every other parser took the `None` branch above and keeps its - // original no-flush EOF behavior. Taking the envelope below - // rather than cloning it makes this branch one-shot: once it is - // gone the next poll ends the stream. + // pending buffer or an unfinished parser. Normally the terminal + // chunk already drained or finalized them, so this is the + // fallback for a stream that ends without any `finish_reason` — + // an aborted or truncated generation. The synthetic chunk it + // builds is the only case where recovered bytes arrive after the + // last upstream chunk, which is unavoidable when there was no + // terminal chunk to attach them to. Only the + // force_nonempty_content path and a parser finalized at finish + // (MiniMax M3) reach this branch; every other parser took the + // `None` branch above and keeps its original no-flush EOF + // behavior. Taking the envelope below rather than cloning it + // makes this branch one-shot: once it is gone the next poll ends + // the stream. // Sorted so the emitted choice order is deterministic rather // than following HashMap iteration order. + let defer_reasoning = state.defer_reasoning_for_nonempty_content; let mut indices: Vec = state.choices.keys().copied().collect(); indices.sort_unstable(); #[allow(clippy::type_complexity)] @@ -7120,7 +7318,11 @@ impl OpenAIPreprocessor { if choice_state.guided_json_bypass_decision != Some(false) { return None; } - let (content, reasoning) = drain_deferred_reasoning(choice_state); + let (content, reasoning) = if defer_reasoning { + drain_deferred_reasoning(choice_state) + } else { + finish_reasoning_parser(choice_state) + }; (content.is_some() || reasoning.is_some()) .then_some((index, content, reasoning)) }) @@ -7663,6 +7865,9 @@ impl )?; let tool_processing_route = self.tool_processing_route(&request, &guided_tool_constraint)?; + let mut prompt_injected_reasoning = prompt_injected_reasoning; + prompt_injected_reasoning.backend_holds_guided_grammar = self + .minimax_m3_guided_grammar_after_reasoning(&request, common_request.require_reasoning); let payload_parsing_options = payload_handle.as_ref().map(|_| { context .get_optional::(REQUEST_PARSING_OPTIONS_CONTEXT_KEY) @@ -14758,6 +14963,1319 @@ mod tests { } } + /// Top-level `guided_*` requests and the bytes a grammar-constrained + /// backend emits for them from token 0. + fn minimax_m3_explicit_guided_cases() -> Vec<(&'static str, serde_json::Value, &'static str)> { + vec![ + ( + "guided_json", + serde_json::json!({"guided_json": { + "type": "object", + "properties": {"city": {"type": "string"}, "country": {"type": "string"}}, + "required": ["city", "country"] + }}), + r#"{"city":"Paris","country":"France"}"#, + ), + ( + "guided_regex", + serde_json::json!({"guided_regex": "Paris|London"}), + "Paris", + ), + ( + "guided_choice", + serde_json::json!({"guided_choice": ["Paris", "London"]}), + "Paris", + ), + ( + "guided_grammar", + serde_json::json!({"guided_grammar": "root ::= \"Paris\" | \"London\""}), + "Paris", + ), + ] + } + + #[test] + fn test_normalize_thinking_arg_m3_defaults_disabled_for_explicit_guided_decoding() { + // Top-level `guided_*` installs the same backend grammar as a structured + // `response_format`, so M3 must render it with thinking disabled too. + for (reasoning_parser, tool_call_parser) in [ + (Some("minimax_m3"), None), + (Some("minimax-m3"), None), + (None, Some("minimax_m3")), + (Some("minimax_m3"), Some("minimax_m3")), + ] { + for (field, body, _) in minimax_m3_explicit_guided_cases() { + let mut request = minimax_m3_request(body); + let desc = format!("{field} with {reasoning_parser:?}/{tool_call_parser:?}"); + assert!( + OpenAIPreprocessor::has_minimax_m3_constrained_generation(&request), + "{desc} is constrained generation" + ); + if reasoning_parser.is_some() { + assert!( + OpenAIPreprocessor::request_disables_reasoning_with_family( + &request, + tool_call_parser, + reasoning_parser, + None, + None, + ), + "{desc} must not run the reasoning parser" + ); + } + OpenAIPreprocessor::normalize_thinking_arg( + &mut request, + reasoning_parser, + tool_call_parser, + ); + let args = request.chat_template_args.as_ref(); + assert_eq!( + args.and_then(|args| args.get("thinking_mode")), + Some(&serde_json::json!("disabled")), + "{desc} must render the M3 template with thinking disabled" + ); + assert_eq!( + args.and_then(|args| args.get("thinking")), + Some(&serde_json::json!(false)) + ); + assert_eq!( + args.and_then(|args| args.get("enable_thinking")), + Some(&serde_json::json!(false)) + ); + } + } + } + + #[test] + fn test_normalize_thinking_arg_m3_explicit_guided_decoding_scope() { + // Fields that install no grammar leave plain M3 chat untouched. + for body in [ + serde_json::json!({"guided_choice": []}), + serde_json::json!({"guided_decoding_backend": "xgrammar"}), + serde_json::json!({"guided_whitespace_pattern": "[ ]?"}), + ] { + let mut request = minimax_m3_request(body.clone()); + assert!( + !OpenAIPreprocessor::has_minimax_m3_constrained_generation(&request), + "{body} installs no grammar" + ); + OpenAIPreprocessor::normalize_thinking_arg(&mut request, Some("minimax_m3"), None); + assert!(request.chat_template_args.is_none(), "{body}"); + } + + // Explicit client thinking still wins, as it does for `response_format`. + for (field, mut body, _) in minimax_m3_explicit_guided_cases() { + body.as_object_mut().unwrap().insert( + "chat_template_kwargs".to_string(), + serde_json::json!({"thinking_mode": "enabled"}), + ); + let mut request = minimax_m3_request(body); + OpenAIPreprocessor::normalize_thinking_arg(&mut request, Some("minimax_m3"), None); + assert_eq!( + request + .chat_template_args + .as_ref() + .and_then(|args| args.get("thinking_mode")), + Some(&serde_json::json!("enabled")), + "{field} + explicit thinking_mode=enabled must keep client intent" + ); + } + + // Non-M3 parsers keep their existing policy for `guided_*`. + for parser in ["qwen3", "minimax_m2", "deepseek_v3_1", "glm45"] { + let mut request = minimax_m3_request(serde_json::json!({ + "guided_json": {"type": "object"} + })); + OpenAIPreprocessor::normalize_thinking_arg(&mut request, Some(parser), None); + assert!( + request.chat_template_args.is_none(), + "parser {parser:?} must not receive M3 thinking defaults" + ); + } + } + + /// The generation-prompt tail of the MiniMax M3 chat template (its + /// `add_generation_prompt` block): only an exact `thinking_mode == + /// "disabled"` prefills `` and only an exact `"enabled"` + /// prefills ``; `"adaptive"`, any other value, or an undefined + /// mode adds no prefix. + fn minimax_m3_generation_prompt_tail( + chat_template_args: Option<&std::collections::HashMap>, + ) -> &'static str { + match chat_template_args + .and_then(|args| args.get("thinking_mode")) + .and_then(serde_json::Value::as_str) + { + Some("disabled") => "]~b]ai\n", + Some("enabled") => "]~b]ai\n", + _ => "]~b]ai\n", + } + } + + /// The chat template that renders the M3 prompt in a round trip. + #[derive(Clone, Copy)] + enum MiniMaxM3Template { + /// The public MiniMax M3 template: `thinking_mode` picks the prefill. + Public, + /// The MiniMax M3.1 Flash v2 template: no `thinking_mode`. Every + /// prompt ends after the generation header, and its instructions + /// mention both markers. + FlashV2, + } + + impl MiniMaxM3Template { + fn rendered_prompt( + self, + chat_template_args: Option<&std::collections::HashMap>, + ) -> &'static str { + match self { + Self::Public => minimax_m3_generation_prompt_tail(chat_template_args), + Self::FlashV2 => concat!( + "\nWhen thinking is enabled, wrap your reasoning in ", + " tags before your response. When thinking is disabled, ", + "begin your response directly after the prefix.\n", + "Current thinking mode: adaptive.\n[e~[\n", + "]~b]user\nWhat is 2 + 2?[e~[\n]~b]ai\n", + ), + } + } + } + + /// The worker and chat template an M3 round trip runs against. + #[derive(Clone, Copy)] + struct MiniMaxM3Setup { + /// The worker advertises a grammar held until reasoning ends, as a + /// vLLM worker with an engine-side reasoning parser does. + backend_holds_grammar: bool, + template: MiniMaxM3Template, + } + + impl Default for MiniMaxM3Setup { + fn default() -> Self { + Self { + backend_holds_grammar: false, + template: MiniMaxM3Template::Public, + } + } + } + + struct MiniMaxM3RoundTrip { + /// The parsing stream's output chunks, before aggregation. + chunks: Vec, + /// `(content, reasoning_content)` after non-streaming aggregation. + message: (String, String), + } + + /// How the backend ends an M3 stream after its content chunks. + #[derive(Clone, Copy)] + enum MiniMaxM3StreamEnd { + /// A terminal chunk with `finish_reason: "stop"`. + Stop, + /// Nothing more: every `finish_reason` stays null, as when the + /// generation is aborted or truncated. + NoFinishReason, + /// A usage-only chunk (no choices), and no `finish_reason`. + UsageOnlyNoFinishReason, + /// A backend error, and no `finish_reason`. + Error, + } + + /// Run an M3 request through the frontend steps `generate` applies around + /// the backend: the HTTP boundary's reasoning-control normalization, the + /// thinking defaults, prompt-prefill detection, then the response parsing + /// stream. The backend sends one content chunk per element of + /// `backend_chunks`, then ends as `end` says. + async fn minimax_m3_parsing_stream_output( + body: serde_json::Value, + backend_chunks: &[&str], + end: MiniMaxM3StreamEnd, + ) -> Vec> { + minimax_m3_parsing_stream_output_with_backend( + body, + backend_chunks, + end, + MiniMaxM3Setup::default(), + ) + .await + } + + /// `minimax_m3_parsing_stream_output` on the worker and chat template + /// `setup` describes. + async fn minimax_m3_parsing_stream_output_with_backend( + body: serde_json::Value, + backend_chunks: &[&str], + end: MiniMaxM3StreamEnd, + setup: MiniMaxM3Setup, + ) -> Vec> { + let model_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/data/sample-models/mock-llama-3.1-8b-instruct"); + let mut card = ModelDeploymentCard::load_from_disk(model_path, None).unwrap(); + card.runtime_config.reasoning_parser = Some("minimax_m3".to_string()); + card.runtime_config.tool_call_parser = Some("minimax_m3".to_string()); + if setup.backend_holds_grammar { + card.runtime_config + .set_engine_specific( + crate::local_model::runtime_config::TOOL_CALL_STRUCTURAL_TAG_EXCLUDES_REASONING_RUNTIME_KEY, + true, + ) + .unwrap(); + } + let preprocessor = OpenAIPreprocessor::new(card).unwrap(); + + let mut request = minimax_m3_request(body); + request.normalize_reasoning_template_args().unwrap(); + let thinking_control_from_client = + OpenAIPreprocessor::request_has_client_thinking_control(&request); + preprocessor.apply_default_thinking_mode(&mut request); + OpenAIPreprocessor::normalize_thinking_arg_with_source( + &mut request, + Some("minimax_m3"), + Some("minimax_m3"), + thinking_control_from_client, + ); + let mut prefill = OpenAIPreprocessor::prompt_reasoning_prefill_for_parsers( + Some("minimax_m3"), + Some("minimax_m3"), + Some( + setup + .template + .rendered_prompt(request.chat_template_args.as_ref()), + ), + ); + prefill.backend_holds_guided_grammar = preprocessor + .minimax_m3_guided_grammar_after_reasoning( + &request, + OpenAIPreprocessor::request_requires_reasoning(&request, Some("minimax_m3"), false), + ); + + let chunk = |choices: serde_json::Value| { + serde_json::json!({"id": "test", "object": "chat.completion.chunk", + "created": 0, "model": "test", "choices": choices}) + }; + let mut backend = backend_chunks + .iter() + .enumerate() + .map(|(i, text)| { + let delta = if i == 0 { + serde_json::json!({"role": "assistant", "content": text}) + } else { + serde_json::json!({"content": text}) + }; + chunk(serde_json::json!([{"index": 0, "delta": delta, "finish_reason": null}])) + }) + .collect::>(); + match end { + MiniMaxM3StreamEnd::Stop => backend.push(chunk( + serde_json::json!([{"index": 0, "delta": {}, "finish_reason": "stop"}]), + )), + MiniMaxM3StreamEnd::UsageOnlyNoFinishReason => { + let mut usage = chunk(serde_json::json!([])); + usage["usage"] = serde_json::json!( + {"prompt_tokens": 8, "completion_tokens": 4, "total_tokens": 12} + ); + backend.push(usage); + } + MiniMaxM3StreamEnd::NoFinishReason | MiniMaxM3StreamEnd::Error => {} + } + let mut backend = backend + .into_iter() + .map(|chunk| { + Annotated::from_data( + serde_json::from_value::(chunk).unwrap(), + ) + }) + .collect::>(); + if matches!(end, MiniMaxM3StreamEnd::Error) { + backend.push(Annotated::from_error("backend failed")); + } + preprocessor + .postprocessor_parsing_stream(stream::iter(backend), &request, prefill, false) + .unwrap() + .collect() + .await + } + + /// `minimax_m3_parsing_stream_output`, then non-streaming aggregation. + async fn minimax_m3_round_trip( + body: serde_json::Value, + backend_chunks: &[&str], + end: MiniMaxM3StreamEnd, + ) -> MiniMaxM3RoundTrip { + minimax_m3_round_trip_with_backend(body, backend_chunks, end, MiniMaxM3Setup::default()) + .await + } + + /// `minimax_m3_round_trip` on the worker and chat template `setup` describes. + async fn minimax_m3_round_trip_with_backend( + body: serde_json::Value, + backend_chunks: &[&str], + end: MiniMaxM3StreamEnd, + setup: MiniMaxM3Setup, + ) -> MiniMaxM3RoundTrip { + let output = + minimax_m3_parsing_stream_output_with_backend(body, backend_chunks, end, setup).await; + let chunks = output + .iter() + .filter_map(|annotated| annotated.data.clone()) + .collect(); + let response = + crate::protocols::openai::chat_completions::aggregator::DeltaAggregator::apply( + stream::iter(output), + crate::protocols::openai::ParsingOptions::default(), + ) + .await + .unwrap(); + let message = &response.inner.choices[0].message; + let content = match message.content.as_ref() { + Some(ChatCompletionMessageContent::Text(text)) => text.clone(), + Some(ChatCompletionMessageContent::Parts(_)) => panic!("unexpected content parts"), + None => String::new(), + }; + MiniMaxM3RoundTrip { + chunks, + message: ( + content, + message.reasoning_content.clone().unwrap_or_default(), + ), + } + } + + /// `(content, reasoning_content, finish_reason set)` for the first choice + /// of each chunk that carries one. + fn minimax_m3_chunk_placement( + chunks: &[NvCreateChatCompletionStreamResponse], + ) -> Vec<(String, String, bool)> { + chunks + .iter() + .filter(|chunk| !chunk.inner.choices.is_empty()) + .map(|chunk| { + let choice = &chunk.inner.choices[0]; + let content = match choice.delta.content.as_ref() { + Some(ChatCompletionMessageContent::Text(text)) => text.clone(), + _ => String::new(), + }; + let reasoning = choice.delta.reasoning_content.clone().unwrap_or_default(); + (content, reasoning, choice.finish_reason.is_some()) + }) + .collect() + } + + /// The model output as one backend chunk, and as 2-byte chunks so every + /// marker and JSON boundary also arrives split. + fn minimax_m3_backend_chunkings(model_output: &str) -> Vec<(&'static str, Vec<&str>)> { + assert!(model_output.is_ascii()); + let pieces = model_output + .as_bytes() + .chunks(2) + .map(|piece| std::str::from_utf8(piece).unwrap()) + .collect(); + vec![("whole", vec![model_output]), ("2-byte", pieces)] + } + + /// Run every `(label, body, model_output, expected)` row under both + /// chunkings, ending each stream as `end` says, and return each row whose + /// `(content, reasoning_content)` differs from `expected`, with what it + /// produced, so one run reports every failing row. + async fn minimax_m3_mismatched_rows( + rows: Vec<(String, serde_json::Value, &str, (&str, &str))>, + end: MiniMaxM3StreamEnd, + ) -> Vec<(String, (String, String))> { + let mut mismatched = Vec::new(); + for (label, body, model_output, (content, reasoning)) in rows { + for (chunking, chunks) in minimax_m3_backend_chunkings(model_output) { + let actual = minimax_m3_round_trip(body.clone(), &chunks, end) + .await + .message; + if actual != (content.to_string(), reasoning.to_string()) { + mismatched.push((format!("{label} [{chunking}]"), actual)); + } + } + } + mismatched + } + + /// `response_format` and every top-level `guided_*` form, with the bytes a + /// backend whose grammar applies from token 0 emits for each. + fn minimax_m3_constrained_output_cases() -> Vec<(&'static str, serde_json::Value, &'static str)> + { + let mut cases = vec![( + "response_format", + serde_json::json!({"response_format": { + "type": "json_schema", + "json_schema": {"name": "s", "schema": {"type": "object"}} + }}), + r#"{"city":"Paris","country":"France"}"#, + )]; + cases.extend(minimax_m3_explicit_guided_cases()); + cases + } + + /// Every way a client can set thinking, keyed by a label. "no thinking + /// control" sends none, so the constrained default applies. + fn minimax_m3_thinking_controls() -> Vec<(&'static str, serde_json::Value)> { + vec![ + ("no thinking control", serde_json::json!({})), + ( + "thinking_mode=disabled", + serde_json::json!({"chat_template_kwargs": {"thinking_mode": "disabled"}}), + ), + ( + "thinking_mode=adaptive", + serde_json::json!({"chat_template_kwargs": {"thinking_mode": "adaptive"}}), + ), + ( + "thinking={type:adaptive}", + serde_json::json!({"thinking": {"type": "adaptive"}}), + ), + ( + "thinking_mode=enabled", + serde_json::json!({"chat_template_kwargs": {"thinking_mode": "enabled"}}), + ), + ( + "reasoning_effort=high", + serde_json::json!({"reasoning_effort": "high"}), + ), + ] + } + + fn minimax_m3_merge_body( + mut body: serde_json::Value, + extra: &serde_json::Value, + ) -> serde_json::Value { + let object = body.as_object_mut().unwrap(); + for (key, value) in extra.as_object().unwrap() { + object.insert(key.clone(), value.clone()); + } + body + } + + #[tokio::test] + async fn test_minimax_m3_explicit_guided_output_is_content() { + // The parser version is process-global; run with the default (no DYN_PARSER_VERSION). + if crate::test_utils::run_isolated( + concat!( + module_path!(), + "::test_minimax_m3_explicit_guided_output_is_content" + ), + &[], + ) { + return; + } + // Whatever thinking control the client sends, output that a grammar + // forced from token 0 is the answer: it must arrive as content. + let mut rows = Vec::new(); + for (control, control_body) in minimax_m3_thinking_controls() { + for (field, body, model_output) in minimax_m3_constrained_output_cases() { + rows.push(( + format!("{field} + {control}"), + minimax_m3_merge_body(body, &control_body), + model_output, + (model_output, ""), + )); + } + } + assert_eq!( + minimax_m3_mismatched_rows(rows, MiniMaxM3StreamEnd::Stop).await, + Vec::<(String, (String, String))>::new(), + "rows whose (content, reasoning_content) was not (model output, \"\")" + ); + } + + #[tokio::test] + async fn test_minimax_m3_reasoning_before_guided_output_still_splits() { + // The parser version is process-global; run with the default (no DYN_PARSER_VERSION). + if crate::test_utils::run_isolated( + concat!( + module_path!(), + "::test_minimax_m3_reasoning_before_guided_output_still_splits" + ), + &[], + ) { + return; + } + // A worker that delays the grammar until reasoning ends (vLLM-native + // reasoning parser) emits reasoning first. That split must survive. + let json = r#"{"city":"Paris","country":"France"}"#; + let enabled = serde_json::json!({"chat_template_kwargs": {"thinking_mode": "enabled"}}); + let adaptive = serde_json::json!({"chat_template_kwargs": {"thinking_mode": "adaptive"}}); + let effort = serde_json::json!({"reasoning_effort": "high"}); + let mut rows = Vec::new(); + for (field, body, answer) in minimax_m3_constrained_output_cases() { + rows.push(( + format!("{field} + thinking_mode=enabled"), + minimax_m3_merge_body(body.clone(), &enabled), + if answer == json { + "Let me think.{\"city\":\"Paris\",\"country\":\"France\"}" + } else { + "Let me think.Paris" + }, + (answer, "Let me think."), + )); + rows.push(( + format!("{field} + reasoning_effort=high"), + minimax_m3_merge_body(body.clone(), &effort), + if answer == json { + "Let me think.{\"city\":\"Paris\",\"country\":\"France\"}" + } else { + "Let me think.Paris" + }, + (answer, "Let me think."), + )); + rows.push(( + format!("{field} + thinking_mode=adaptive"), + minimax_m3_merge_body(body, &adaptive), + if answer == json { + "Let me think.{\"city\":\"Paris\",\"country\":\"France\"}" + } else { + "Let me think.Paris" + }, + (answer, "Let me think."), + )); + } + assert_eq!( + minimax_m3_mismatched_rows(rows, MiniMaxM3StreamEnd::Stop).await, + Vec::<(String, (String, String))>::new(), + "rows whose reasoning/answer split changed" + ); + } + + #[tokio::test] + async fn test_minimax_m3_guided_reasoning_stays_reasoning_when_backend_holds_grammar() { + // The parser version is process-global; run with the default (no DYN_PARSER_VERSION). + if crate::test_utils::run_isolated( + concat!( + module_path!(), + "::test_minimax_m3_guided_reasoning_stays_reasoning_when_backend_holds_grammar" + ), + &[], + ) { + return; + } + // A worker that holds the grammar until reasoning ends lets the model + // reason after the prefilled ``, so the seeded parser keeps + // that text in `reasoning_content`, even when the generation is cut off + // before `` or the reasoning opens with a bracket. + let body = serde_json::json!({ + "guided_regex": "Paris|London", + "chat_template_kwargs": {"thinking_mode": "enabled"} + }); + let mut mismatched = Vec::new(); + for (label, output, end, expected) in [ + ( + "complete", + "Let me think.Paris", + MiniMaxM3StreamEnd::Stop, + ("Paris", "Let me think."), + ), + ( + "cut off", + "Let me think about", + MiniMaxM3StreamEnd::NoFinishReason, + ("", "Let me think about"), + ), + ( + "bracket-led reasoning", + "[plan] compare capitalsParis", + MiniMaxM3StreamEnd::Stop, + ("Paris", "[plan] compare capitals"), + ), + ] { + for (chunking, chunks) in minimax_m3_backend_chunkings(output) { + let actual = minimax_m3_round_trip_with_backend( + body.clone(), + &chunks, + end, + MiniMaxM3Setup { + backend_holds_grammar: true, + ..Default::default() + }, + ) + .await + .message; + if actual != (expected.0.to_string(), expected.1.to_string()) { + mismatched.push((format!("{label} [{chunking}]"), actual)); + } + } + } + assert_eq!( + mismatched, + Vec::<(String, (String, String))>::new(), + "rows that left the reasoning/answer split" + ); + + // Seeded, the reasoning streams as it is generated instead of arriving + // in one delta at ``. + let chunks = minimax_m3_round_trip_with_backend( + body, + &["Let me", " think.", "", "Paris"], + MiniMaxM3StreamEnd::Stop, + MiniMaxM3Setup { + backend_holds_grammar: true, + ..Default::default() + }, + ) + .await + .chunks; + let placement = minimax_m3_chunk_placement(&chunks); + assert_eq!( + placement.first(), + Some(&(String::new(), "Let me".to_string(), false)), + "{placement:?}" + ); + } + + #[tokio::test] + async fn test_minimax_m3_adaptive_turn_without_markers_is_content() { + // The parser version is process-global; run with the default (no DYN_PARSER_VERSION). + if crate::test_utils::run_isolated( + concat!( + module_path!(), + "::test_minimax_m3_adaptive_turn_without_markers_is_content" + ), + &[], + ) { + return; + } + // Plain M3 chat (no constraint, no thinking control) renders adaptive: + // the prompt opens no reasoning, so the parser starts outside it and + // marker-free text is content. + let plain = serde_json::json!({}); + let rows = vec![ + ( + "no marker".to_string(), + plain.clone(), + "Hello there", + ("Hello there", ""), + ), + ( + "think block".to_string(), + plain.clone(), + "hmmHello", + ("Hello", "hmm"), + ), + ( + "answer ends in a marker-prefix byte".to_string(), + plain.clone(), + "hmm1 <", + ("1 <", "hmm"), + ), + ( + "truncated inside reasoning".to_string(), + plain, + "hmm::new(), + "rows whose (content, reasoning_content) was wrong" + ); + } + + #[tokio::test] + async fn test_minimax_m3_guided_output_stream_placement() { + // The parser version is process-global; run with the default (no DYN_PARSER_VERSION). + if crate::test_utils::run_isolated( + concat!( + module_path!(), + "::test_minimax_m3_guided_output_stream_placement" + ), + &[], + ) { + return; + } + // An adaptive prompt opens no reasoning, so the parser starts outside + // it and the guided output streams as content as it arrives. + let regex = minimax_m3_merge_body( + serde_json::json!({"guided_regex": "Paris|London"}), + &serde_json::json!({"chat_template_kwargs": {"thinking_mode": "adaptive"}}), + ); + let placed = minimax_m3_chunk_placement( + &minimax_m3_round_trip(regex, &["Par", "is"], MiniMaxM3StreamEnd::Stop) + .await + .chunks, + ); + assert_eq!( + placed, + vec![ + ("Par".to_string(), String::new(), false), + ("is".to_string(), String::new(), false), + (String::new(), String::new(), true), + ], + "guided_regex + adaptive: placement per chunk" + ); + + // Under an enabled prompt the parser is left unseeded for explicit + // `guided_*`, so text it holds to EOF rides the terminal chunk, never a + // chunk after `finish_reason`. + let held = minimax_m3_merge_body( + serde_json::json!({"guided_regex": "Paris|London"}), + &serde_json::json!({"chat_template_kwargs": {"thinking_mode": "enabled"}}), + ); + let placed = minimax_m3_chunk_placement( + &minimax_m3_round_trip(held, &["Par", "is"], MiniMaxM3StreamEnd::Stop) + .await + .chunks, + ); + assert_eq!( + placed, + vec![ + (String::new(), String::new(), false), + (String::new(), String::new(), false), + ("Paris".to_string(), String::new(), true), + ], + "guided_regex + thinking_mode=enabled, no reasoning: placement per chunk" + ); + + // With thinking enabled, bare guided JSON bypasses the parser and + // streams as content as it arrives, as `response_format` does. + let json = minimax_m3_merge_body( + serde_json::json!({"guided_json": {"type": "object"}}), + &serde_json::json!({"chat_template_kwargs": {"thinking_mode": "enabled"}}), + ); + let placed = minimax_m3_chunk_placement( + &minimax_m3_round_trip( + json, + &[r#"{"city":"#, r#""Paris"}"#], + MiniMaxM3StreamEnd::Stop, + ) + .await + .chunks, + ); + assert_eq!( + placed, + vec![ + (r#"{"city":"#.to_string(), String::new(), false), + (r#""Paris"}"#.to_string(), String::new(), false), + (String::new(), String::new(), true), + ], + "guided_json + thinking_mode=enabled: placement per chunk" + ); + + // With thinking enabled, a worker that delays the grammar until + // reasoning ends emits reasoning first. The parser is not seeded for + // explicit `guided_*`, so it holds that text until `` + // decides it: the reasoning arrives in one delta at the close marker, + // and the answer streams after it. + let delayed = minimax_m3_merge_body( + serde_json::json!({"guided_regex": "Paris|London"}), + &serde_json::json!({"chat_template_kwargs": {"thinking_mode": "enabled"}}), + ); + let placed = minimax_m3_chunk_placement( + &minimax_m3_round_trip( + delayed, + &["Let me think.", "", "Paris"], + MiniMaxM3StreamEnd::Stop, + ) + .await + .chunks, + ); + assert_eq!( + placed, + vec![ + (String::new(), String::new(), false), + (String::new(), "Let me think.".to_string(), false), + ("Paris".to_string(), String::new(), false), + (String::new(), String::new(), true), + ], + "guided_regex + thinking_mode=enabled, reasoning first: placement per chunk" + ); + } + + /// Turns whose last bytes the M3 parser still holds when the backend stops + /// sending: each non-JSON top-level `guided_*` form under an enabled prompt + /// (explicit `guided_*` leaves the parser unseeded there, and bare JSON + /// bypasses it), and plain adaptive chat with an answer ending in a + /// marker-prefix byte and with reasoning truncated inside its close marker. + fn minimax_m3_held_text_rows() -> Vec<( + String, + serde_json::Value, + &'static str, + (&'static str, &'static str), + )> { + let enabled = serde_json::json!({"chat_template_kwargs": {"thinking_mode": "enabled"}}); + let mut rows: Vec<_> = minimax_m3_explicit_guided_cases() + .into_iter() + .filter(|(field, _, _)| *field != "guided_json") + .map(|(field, body, output)| { + ( + format!("{field} + thinking_mode=enabled"), + minimax_m3_merge_body(body, &enabled), + output, + (output, ""), + ) + }) + .collect(); + let plain = serde_json::json!({}); + rows.extend([ + ( + "plain, answer ends in a marker-prefix byte".to_string(), + plain.clone(), + "hmm1 <", + ("1 <", "hmm"), + ), + ( + "plain, truncated inside reasoning".to_string(), + plain, + "hmm::new(), + "rows whose (content, reasoning_content) was wrong without a finish_reason" + ); + + // The held text arrives once, in one chunk after the last backend + // chunk, with no `finish_reason` because the backend sent none. + let regex = minimax_m3_merge_body( + serde_json::json!({"guided_regex": "Paris|London"}), + &serde_json::json!({"chat_template_kwargs": {"thinking_mode": "enabled"}}), + ); + let placed = minimax_m3_chunk_placement( + &minimax_m3_round_trip(regex, &["Par", "is"], MiniMaxM3StreamEnd::NoFinishReason) + .await + .chunks, + ); + assert_eq!( + placed, + vec![ + (String::new(), String::new(), false), + (String::new(), String::new(), false), + ("Paris".to_string(), String::new(), false), + ], + "guided_regex + thinking_mode=enabled without a finish_reason: placement per chunk" + ); + } + + #[tokio::test] + async fn test_minimax_m3_terminal_finalize_is_not_repeated_at_end_of_stream() { + // The parser version is process-global; run with the default (no DYN_PARSER_VERSION). + if crate::test_utils::run_isolated( + concat!( + module_path!(), + "::test_minimax_m3_terminal_finalize_is_not_repeated_at_end_of_stream" + ), + &[], + ) { + return; + } + // A parser finalized on its terminal chunk is finished: the end of + // stream that follows must not emit its text a second time. + for (label, body, model_output, (content, reasoning)) in minimax_m3_held_text_rows() { + for (chunking, backend_chunks) in minimax_m3_backend_chunkings(model_output) { + let desc = format!("{label} [{chunking}]"); + let round_trip = + minimax_m3_round_trip(body.clone(), &backend_chunks, MiniMaxM3StreamEnd::Stop) + .await; + let placed = minimax_m3_chunk_placement(&round_trip.chunks); + assert!( + placed.last().is_some_and(|(_, _, finished)| *finished) + && placed.iter().filter(|(_, _, finished)| *finished).count() == 1, + "{desc}: no chunk may follow the terminal chunk: {placed:?}" + ); + let streamed = placed.iter().fold( + (String::new(), String::new()), + |(all_content, all_reasoning), (delta_content, delta_reasoning, _)| { + (all_content + delta_content, all_reasoning + delta_reasoning) + }, + ); + assert_eq!( + streamed, + (content.to_string(), reasoning.to_string()), + "{desc}: streamed text" + ); + assert_eq!(round_trip.message, streamed, "{desc}: aggregated text"); + } + } + } + + #[tokio::test] + async fn test_minimax_m3_backend_error_drops_held_text() { + // The parser version is process-global; run with the default (no DYN_PARSER_VERSION). + if crate::test_utils::run_isolated( + concat!( + module_path!(), + "::test_minimax_m3_backend_error_drops_held_text" + ), + &[], + ) { + return; + } + // An error is terminal, so text the parser still holds is dropped + // rather than emitted after the error as if it were the answer. These + // rows hold their whole output until the parser is finalized. + let rows = minimax_m3_held_text_rows() + .into_iter() + .filter(|(_, _, model_output, _)| !model_output.contains("")); + for (label, body, model_output, _) in rows { + for (chunking, backend_chunks) in minimax_m3_backend_chunkings(model_output) { + let desc = format!("{label} [{chunking}]"); + let output = minimax_m3_parsing_stream_output( + body.clone(), + &backend_chunks, + MiniMaxM3StreamEnd::Error, + ) + .await; + assert!( + output.last().is_some_and(|item| item.is_error()), + "{desc}: the error must reach the client last" + ); + let placed = minimax_m3_chunk_placement( + &output + .iter() + .filter_map(|item| item.data.clone()) + .collect::>(), + ); + assert!( + placed + .iter() + .all(|(content, reasoning, _)| content.is_empty() && reasoning.is_empty()), + "{desc}: held text must be dropped: {placed:?}" + ); + } + } + } + + #[tokio::test] + async fn test_minimax_m3_usage_only_chunk_is_not_the_flush_envelope() { + // The parser version is process-global; run with the default (no DYN_PARSER_VERSION). + if crate::test_utils::run_isolated( + concat!( + module_path!(), + "::test_minimax_m3_usage_only_chunk_is_not_the_flush_envelope" + ), + &[], + ) { + return; + } + // A usage-only chunk has no choice to carry the held text, so the + // end-of-stream flush rides a copy of an earlier chunk that had one, + // and the usage chunk, held until the stream ends, still comes last. + assert_eq!( + minimax_m3_mismatched_rows( + minimax_m3_held_text_rows(), + MiniMaxM3StreamEnd::UsageOnlyNoFinishReason + ) + .await, + Vec::<(String, (String, String))>::new(), + "rows whose (content, reasoning_content) was wrong after a usage-only chunk" + ); + + let regex = minimax_m3_merge_body( + serde_json::json!({"guided_regex": "Paris|London"}), + &serde_json::json!({"chat_template_kwargs": {"thinking_mode": "enabled"}}), + ); + let chunks = minimax_m3_round_trip( + regex, + &["Par", "is"], + MiniMaxM3StreamEnd::UsageOnlyNoFinishReason, + ) + .await + .chunks; + let (usage, before_usage) = chunks.split_last().expect("the usage chunk"); + assert!( + usage.inner.choices.is_empty() && usage.inner.usage.is_some(), + "the usage chunk must come last: {chunks:?}" + ); + let flushed = before_usage.last().expect("the flushed chunk"); + assert_eq!(flushed.inner.choices.len(), 1); + assert!( + matches!( + &flushed.inner.choices[0].delta.content, + Some(ChatCompletionMessageContent::Text(text)) if text == "Paris" + ), + "the held text must be flushed: {flushed:?}" + ); + assert!( + flushed.inner.choices[0].delta.role.is_none() + && flushed.inner.choices[0].finish_reason.is_none(), + "the flushed chunk must not copy the envelope's role or finish_reason: {flushed:?}" + ); + assert!( + flushed.inner.usage.is_none(), + "the flushed chunk must not repeat usage" + ); + assert_eq!( + chunks + .iter() + .filter(|chunk| chunk.inner.usage.is_some()) + .count(), + 1, + "usage must be reported once" + ); + } + + #[tokio::test] + async fn test_minimax_m3_end_of_stream_flush_skips_finished_choices() { + fn chunk(choices: serde_json::Value) -> Annotated { + Annotated::from_data( + serde_json::from_value(serde_json::json!({"id": "test", + "object": "chat.completion.chunk", "created": 0, "model": "test", + "choices": choices})) + .unwrap(), + ) + } + /// `(index, content, finish_reason set)` per choice, chunk by chunk. + async fn placement( + backend: Vec>, + ) -> Vec> { + OpenAIPreprocessor::parse_reasoning_content_from_stream( + stream::iter(backend), + "minimax_m3".to_string(), + false, + ) + .filter_map(|item| async move { item.data }) + .map(|data| { + data.inner + .choices + .iter() + .map(|choice| { + let content = match choice.delta.content.as_ref() { + Some(ChatCompletionMessageContent::Text(text)) => text.clone(), + _ => String::new(), + }; + (choice.index, content, choice.finish_reason.is_some()) + }) + .collect() + }) + .collect() + .await + } + let world = chunk(serde_json::json!( + [{"index": 1, "delta": {"content": "World"}, "finish_reason": null}] + )); + let hello = chunk(serde_json::json!( + [{"index": 0, "delta": {"content": "Hello"}, "finish_reason": null}] + )); + let stop = chunk(serde_json::json!( + [{"index": 0, "delta": {}, "finish_reason": "stop"}] + )); + + // Choice 0 finished on its terminal chunk, so the end-of-stream flush + // carries only choice 1. + assert_eq!( + placement(vec![world.clone(), hello.clone(), stop]).await, + vec![ + vec![(1, String::new(), false)], + vec![(0, String::new(), false)], + vec![(0, "Hello".to_string(), true)], + vec![(1, "World".to_string(), false)], + ] + ); + + // With neither finished, one flush chunk carries both, in index order. + assert_eq!( + placement(vec![world, hello]).await, + vec![ + vec![(1, String::new(), false)], + vec![(0, String::new(), false)], + vec![ + (0, "Hello".to_string(), false), + (1, "World".to_string(), false) + ], + ] + ); + } + + #[tokio::test] + async fn test_minimax_m3_parser_starts_where_the_rendered_prompt_leaves_it() { + // The parser version is process-global; run with the default (no DYN_PARSER_VERSION). + if crate::test_utils::run_isolated( + concat!( + module_path!(), + "::test_minimax_m3_parser_starts_where_the_rendered_prompt_leaves_it" + ), + &[], + ) { + return; + } + let flash_v2 = MiniMaxM3Setup { + template: MiniMaxM3Template::FlashV2, + ..Default::default() + }; + let thinking = + |mode: &str| serde_json::json!({"chat_template_kwargs": {"thinking_mode": mode}}); + let schema = serde_json::json!({"response_format": { + "type": "json_schema", + "json_schema": {"name": "s", "schema": {"type": "object"}} + }}); + + // The prompt opens no reasoning, so the parser starts outside it and + // releases marker-free output chunk by chunk, not on the terminal chunk. + let answer = [r#"{""#, "answer", r#"":"#, " ", "51", "}"]; + let placed = minimax_m3_chunk_placement( + &minimax_m3_round_trip_with_backend( + minimax_m3_merge_body(schema.clone(), &thinking("enabled")), + &answer, + MiniMaxM3StreamEnd::Stop, + flash_v2, + ) + .await + .chunks, + ); + let streamed: Vec<&str> = placed + .iter() + .filter(|(content, _, finished)| !finished && !content.is_empty()) + .map(|(content, _, _)| content.as_str()) + .collect(); + assert_eq!( + streamed, answer, + "enabled + response_format: content before the terminal chunk: {placed:?}" + ); + + // The template ignores `disabled` and the model thinks: its markers are + // parsed instead of leaking into content. Under a grammar the output + // cannot open reasoning, so a disabled request stays unparsed and a + // marker inside a JSON string stays in the content. + let rows = [ + ( + "disabled", + thinking("disabled"), + "Need 2+2.4", + ("4", "Need 2+2."), + ), + ( + "disabled + response_format", + minimax_m3_merge_body(schema, &thinking("disabled")), + r#"{"note":""}"#, + (r#"{"note":""}"#, ""), + ), + ]; + for (label, body, model_output, (content, reasoning)) in rows { + for (chunking, backend_chunks) in minimax_m3_backend_chunkings(model_output) { + let message = minimax_m3_round_trip_with_backend( + body.clone(), + &backend_chunks, + MiniMaxM3StreamEnd::Stop, + flash_v2, + ) + .await + .message; + assert_eq!( + message, + (content.to_string(), reasoning.to_string()), + "{label} [{chunking}]" + ); + } + } + } + + #[test] + fn test_minimax_m3_prompt_tail_by_rendered_prompt() { + use MiniMaxM3PromptTail::{Closed, Header, Open}; + let tail = |reasoning_parser: &str, prompt: &str| { + OpenAIPreprocessor::prompt_reasoning_prefill_for_parsers( + None, + Some(reasoning_parser), + Some(prompt), + ) + .minimax_m3_prompt_tail + }; + for (prompt, expected) in [ + ("[e~[\n]~b]ai\n", Some(Open)), + ("[e~[\n]~b]ai\n", Some(Closed)), + ("[e~[\n]~b]ai\n", Some(Header)), + // A continued assistant message ends mid-text: the start is unknown. + ("[e~[\n]~b]ai\nLet me", None), + // No generation prompt. + ("]~b]user\nWhat is 2 + 2?[e~[\n", None), + ] { + assert_eq!(tail("minimax_m3", prompt), expected, "{prompt:?}"); + } + assert_eq!(tail("qwen3", "[e~[\n]~b]ai\n"), None); + } + + #[tokio::test] + async fn test_minimax_m3_parser_told_it_starts_outside_reasoning() { + /// `(content, reasoning_content)` the M3 parsing stage streams for + /// `backend_chunks` from `initial_in_reasoning`. + async fn split( + initial_in_reasoning: Option, + backend_chunks: &[&str], + ) -> (String, String) { + let backend = backend_chunks + .iter() + .map(|text| { + Annotated::from_data( + serde_json::from_value::( + serde_json::json!({"id": "test", "object": "chat.completion.chunk", + "created": 0, "model": "test", "choices": [ + {"index": 0, "delta": {"content": text}, "finish_reason": null} + ]}), + ) + .unwrap(), + ) + }) + .collect::>(); + OpenAIPreprocessor::parse_reasoning_content_from_stream_inner( + stream::iter(backend), + "minimax_m3".to_string(), + initial_in_reasoning, + false, + false, + ) + .filter_map(|item| async move { item.data }) + .fold( + (String::new(), String::new()), + |(content, reasoning), data| async move { + let delta = &data.inner.choices[0].delta; + let delta_content = match delta.content.as_ref() { + Some(ChatCompletionMessageContent::Text(text)) => text.as_str(), + _ => "", + }; + ( + content + delta_content, + reasoning + delta.reasoning_content.as_deref().unwrap_or_default(), + ) + }, + ) + .await + } + + // Guards the dynamo-parsers contract the prompt seed relies on: told + // it starts outside, the M3 parser no longer infers a prompt-opened + // block, so text before a tool marker is the visible preamble. + let preamble = ["I'll check.", "]<]minimax[>["]; + assert_eq!( + split(Some(false), &preamble).await, + ( + "I'll check.]<]minimax[>[".to_string(), + String::new() + ) + ); + + // The no-thinking shape, a leading close marker, still splits. + assert_eq!( + split(Some(false), &["", "4"]).await, + ("4".to_string(), String::new()) + ); + } + /// PRE.2 — Per-request reasoning gate. See `lib/llm/PREPROCESSOR_CASES.md`. #[test] fn test_is_reasoning_disabled_by_request() { diff --git a/lib/llm/src/preprocessor/structural_tag.rs b/lib/llm/src/preprocessor/structural_tag.rs index d3228acc7cb4..ab69bad55935 100644 --- a/lib/llm/src/preprocessor/structural_tag.rs +++ b/lib/llm/src/preprocessor/structural_tag.rs @@ -363,6 +363,21 @@ impl OpenAIPreprocessor { Ok(applied) } + /// Whether the worker holds a guided grammar until the prompt-opened + /// reasoning block closes, as it advertises through + /// `TOOL_CALL_STRUCTURAL_TAG_EXCLUDES_REASONING_RUNTIME_KEY` (every request) + /// or `TOOL_CALL_STRUCTURAL_TAG_REASONING_GATE_RUNTIME_KEY` (requests that + /// set `require_reasoning`). Operator structural-tag settings govern tool + /// tags only, so they do not count here. + pub(super) fn backend_holds_guided_grammar(&self, require_reasoning: bool) -> bool { + self.structural_tag_reasoning_metadata( + TOOL_CALL_STRUCTURAL_TAG_EXCLUDES_REASONING_RUNTIME_KEY, + ) || (require_reasoning + && self.structural_tag_reasoning_metadata( + TOOL_CALL_STRUCTURAL_TAG_REASONING_GATE_RUNTIME_KEY, + )) + } + fn structural_tag_reasoning_metadata(&self, key: &str) -> bool { match self.runtime_config.get_engine_specific::(key) { Ok(Some(excludes_reasoning)) => excludes_reasoning, @@ -1142,6 +1157,41 @@ mod tests { assert_eq!(structural_tag["format"]["content"], expected_content); } + #[test] + fn backend_holds_guided_grammar_follows_worker_metadata() { + let model_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/data/sample-models/mock-llama-3.1-8b-instruct"); + for (metadata, require_reasoning, expected) in [ + (None, true, false), + ( + Some(TOOL_CALL_STRUCTURAL_TAG_EXCLUDES_REASONING_RUNTIME_KEY), + false, + true, + ), + ( + Some(TOOL_CALL_STRUCTURAL_TAG_REASONING_GATE_RUNTIME_KEY), + false, + false, + ), + ( + Some(TOOL_CALL_STRUCTURAL_TAG_REASONING_GATE_RUNTIME_KEY), + true, + true, + ), + ] { + let mut mdc = ModelDeploymentCard::load_from_disk(&model_path, None).unwrap(); + if let Some(key) = metadata { + mdc.runtime_config.set_engine_specific(key, true).unwrap(); + } + let preprocessor = OpenAIPreprocessor::new(mdc).unwrap(); + assert_eq!( + preprocessor.backend_holds_guided_grammar(require_reasoning), + expected, + "metadata={metadata:?}, require_reasoning={require_reasoning}" + ); + } + } + #[test] fn kimi_k3_reasoning_gate_avoids_duplicate_prefix_per_request() { let model_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) diff --git a/lib/llm/src/preprocessor/tool_choice.rs b/lib/llm/src/preprocessor/tool_choice.rs index a03f38c2c81a..f89045e95236 100644 --- a/lib/llm/src/preprocessor/tool_choice.rs +++ b/lib/llm/src/preprocessor/tool_choice.rs @@ -203,7 +203,9 @@ impl OpenAIPreprocessor { } } -fn has_explicit_guided_decoding(request: &NvCreateChatCompletionRequest) -> bool { +/// Whether top-level `guided_*` fields install a grammar; mirrors +/// `GuidedDecodingOptions::from_optional`. +pub(super) fn has_explicit_guided_decoding(request: &NvCreateChatCompletionRequest) -> bool { request.common.guided_json.is_some() || request.common.guided_regex.is_some() || request diff --git a/lib/llm/src/protocols/openai/responses/mod.rs b/lib/llm/src/protocols/openai/responses/mod.rs index 3144d66242a5..2058c1e17b29 100644 --- a/lib/llm/src/protocols/openai/responses/mod.rs +++ b/lib/llm/src/protocols/openai/responses/mod.rs @@ -11,12 +11,12 @@ use std::collections::{HashMap, HashSet}; use dynamo_protocols::types::responses::{ AssistantRole, FunctionCallOutput, FunctionToolCall, IncludeEnum, IncompleteDetails, InputContent, InputImageContent, InputItem, InputOutputMessageContent, InputParam, InputRole, - InputTokenDetails, Instructions, Item, MessageItem, NamespaceToolParamTool, OutputItem, - OutputMessage, OutputMessageContent, OutputStatus, OutputTextContent, OutputTokenDetails, - PromptCacheRetention, Reasoning, ReasoningItem, ReasoningItemContent, ReasoningTextContent, - Response, ResponseTextParam, ResponseUsage, Role as ResponseRole, ServiceTierResponses, Status, - SummaryPart, TextResponseFormatConfiguration, Tool, ToolChoiceAllowed, ToolChoiceAllowedMode, - ToolChoiceOptions, ToolChoiceParam, Truncation, + InputTokenDetails, Instructions, Item, MessageItem, MessagePhase, NamespaceToolParamTool, + OutputItem, OutputMessage, OutputMessageContent, OutputStatus, OutputTextContent, + OutputTokenDetails, PromptCacheRetention, Reasoning, ReasoningItem, ReasoningItemContent, + ReasoningTextContent, Response, ResponseTextParam, ResponseUsage, Role as ResponseRole, + ServiceTierResponses, Status, SummaryPart, SummaryTextContent, TextResponseFormatConfiguration, + Tool, ToolChoiceAllowed, ToolChoiceAllowedMode, ToolChoiceOptions, ToolChoiceParam, Truncation, }; use dynamo_protocols::types::{ ChatCompletionMessageToolCall, ChatCompletionNamedToolChoice, @@ -425,9 +425,15 @@ fn convert_function_call_output_content( /// run. Without it, a standalone assistant message with empty text (or only /// Refusal content parts that this converter currently strips) would be lost /// entirely — breaking turn boundaries the model relies on. +/// +/// `content_phase` records the `phase` of the accumulated text, so commentary +/// and a final answer echoed back as separate messages are not merged. +/// `text_reasoning_len` marks how much reasoning preceded the latest text. #[derive(Default)] struct PendingAssistant { content: Option, + content_phase: Option, + text_reasoning_len: usize, reasoning_segments: Vec, pending_reasoning: String, tool_calls: Vec, @@ -435,11 +441,39 @@ struct PendingAssistant { } impl PendingAssistant { - fn push_text(&mut self, text: &str) { + /// Text in a different known phase starts a new assistant turn. Never split + /// while tool calls are pending: their tool messages must follow the + /// assistant message that carries the calls. + fn is_phase_change(&self, phase: Option) -> bool { + self.tool_calls.is_empty() + && matches!((self.content_phase, phase), (Some(current), Some(next)) if current != next) + } + + /// Flush before text in a different phase. Reasoning echoed after the + /// earlier text moves to the new turn, ahead of the text that follows it. + fn split_on_phase_change( + &mut self, + phase: Option, + out: &mut Vec, + ) { + if !self.is_phase_change(phase) { + return; + } + let split_at = self.text_reasoning_len.min(self.pending_reasoning.len()); + let later_reasoning = self.pending_reasoning.split_off(split_at); + std::mem::take(self).flush_into(out); + self.pending_reasoning = later_reasoning; + } + + fn push_text(&mut self, text: &str, phase: Option) { self.touched = true; + if phase.is_some() { + self.content_phase = phase; + } if text.is_empty() { return; } + self.text_reasoning_len = self.pending_reasoning.len(); match self.content.as_mut() { Some(existing) => existing.push_str(text), None => self.content = Some(text.to_string()), @@ -568,7 +602,8 @@ fn convert_input_items_to_messages( }) .collect::>() .join(""); - pending.push_text(&text); + pending.split_on_phase_change(out_msg.phase, &mut messages); + pending.push_text(&text, out_msg.phase); } }, Item::FunctionCall(fc) => { @@ -687,7 +722,8 @@ fn convert_input_items_to_messages( convert_input_content_to_text(parts) } }; - pending.push_text(&text); + pending.split_on_phase_change(easy.phase, &mut messages); + pending.push_text(&text, easy.phase); } } } @@ -1181,12 +1217,12 @@ pub(super) fn normalize_tools(tools: Vec) -> Vec { } /// Build an assistant text message output item. -fn make_text_message(id: String, text: String) -> OutputItem { +fn make_text_message(id: String, text: String, phase: MessagePhase) -> OutputItem { OutputItem::Message(OutputMessage { id, role: AssistantRole::Assistant, status: OutputStatus::Completed, - phase: None, + phase: Some(phase), content: vec![OutputMessageContent::OutputText(OutputTextContent { text, annotations: vec![], @@ -1216,14 +1252,21 @@ pub fn chat_completion_to_response( if let Some(choice) = choice { incomplete_reason = responses_incomplete_reason(choice.finish_reason); - // Reasoning precedes tool calls so output order matches the decoded turn. + // Reasoning precedes visible output so output order matches the decoded turn. + // The trace is always returned as content; `reasoning.summary` mirrors it. if let Some(reasoning_text) = choice.message.reasoning_content && !reasoning_text.is_empty() - && params.reasoning_summary_requested() { + let summary = if params.reasoning_summary_requested() { + vec![SummaryPart::SummaryText(SummaryTextContent { + text: reasoning_text.clone(), + })] + } else { + vec![] + }; output.push(OutputItem::Reasoning(ReasoningItem { id: Some(format!("rs_{}", Uuid::new_v4().simple())), - summary: vec![], + summary, content: Some(vec![ReasoningItemContent::ReasoningText( ReasoningTextContent { text: reasoning_text, @@ -1235,6 +1278,7 @@ pub fn chat_completion_to_response( } // Handle structured tool calls + let mut function_calls = Vec::new(); if let Some(tool_calls) = choice.message.tool_calls { if let Some(disallowed) = tool_calls.iter().find(|tc| { allowed_names @@ -1248,7 +1292,7 @@ pub fn chat_completion_to_response( } for tc in &tool_calls { let (namespace, name) = names.decode(&tc.function.name); - output.push(OutputItem::FunctionCall(FunctionToolCall { + function_calls.push(OutputItem::FunctionCall(FunctionToolCall { caller: None, r#async: None, arguments: tc.function.arguments.clone(), @@ -1273,15 +1317,31 @@ pub fn chat_completion_to_response( if let Some(content_text) = content_text && !content_text.is_empty() { - output.push(make_text_message(message_id.clone(), content_text)); + // A unary response does not say where text fell relative to the + // calls, so text alongside calls is commentary placed before them. + let phase = if function_calls.is_empty() { + MessagePhase::FinalAnswer + } else { + MessagePhase::Commentary + }; + output.push(make_text_message(message_id.clone(), content_text, phase)); } + output.extend(function_calls); if output.is_empty() { - output.push(make_text_message(message_id, String::new())); + output.push(make_text_message( + message_id, + String::new(), + MessagePhase::FinalAnswer, + )); } } else { tracing::warn!("No choices in chat completion response, using empty content"); - output.push(make_text_message(message_id, String::new())); + output.push(make_text_message( + message_id, + String::new(), + MessagePhase::FinalAnswer, + )); } // Apply `include` filtering: strip logprobs from output text unless @@ -2876,6 +2936,111 @@ mod tests { assert!(matches!(messages[3], ChatCompletionRequestMessage::Tool(_))); } + fn convert_input_json(input: serde_json::Value) -> Vec { + let req: NvCreateResponse = + serde_json::from_value(serde_json::json!({"model": "m", "input": input})).unwrap(); + NvCreateChatCompletionRequest::try_from(req) + .unwrap() + .inner + .messages + } + + fn assistant_texts(messages: &[ChatCompletionRequestMessage]) -> Vec<&str> { + messages + .iter() + .filter_map(|message| match message { + ChatCompletionRequestMessage::Assistant(assistant) => match &assistant.content { + Some(ChatCompletionRequestAssistantMessageContent::Text(text)) => { + Some(text.as_str()) + } + _ => None, + }, + _ => None, + }) + .collect() + } + + /// Echoed commentary and final-answer messages stay separate assistant + /// turns, while phase-less assistant text keeps the legacy merge. + #[test] + fn test_assistant_phase_change_starts_a_new_turn() { + let split = convert_input_json(serde_json::json!([ + {"role": "user", "content": "hi"}, + {"type": "message", "role": "assistant", "phase": "commentary", + "content": [{"type": "output_text", "text": "Looking."}]}, + {"role": "assistant", "phase": "final_answer", "content": "Done."} + ])); + assert_eq!(split.len(), 3); + assert_eq!(assistant_texts(&split), vec!["Looking.", "Done."]); + + let legacy = convert_input_json(serde_json::json!([ + {"role": "user", "content": "hi"}, + {"type": "message", "role": "assistant", + "content": [{"type": "output_text", "text": "Looking."}]}, + {"role": "assistant", "content": "Done."} + ])); + assert_eq!(legacy.len(), 2); + assert_eq!(assistant_texts(&legacy), vec!["Looking.Done."]); + } + + /// Reasoning echoed between commentary and the final answer belongs to the + /// final-answer turn, not to the commentary it follows. + #[test] + fn test_phase_change_moves_later_reasoning_to_the_new_turn() { + let reasoning = |text: &str| { + serde_json::json!({"type": "reasoning", "summary": [], + "content": [{"type": "reasoning_text", "text": text}]}) + }; + let messages = convert_input_json(serde_json::json!([ + {"role": "user", "content": "hi"}, + reasoning("r1"), + {"type": "message", "role": "assistant", "phase": "commentary", + "content": [{"type": "output_text", "text": "A"}]}, + reasoning("r2"), + {"type": "message", "role": "assistant", "phase": "final_answer", + "content": [{"type": "output_text", "text": "B"}]} + ])); + let turns: Vec<_> = messages + .iter() + .filter_map(|message| match message { + ChatCompletionRequestMessage::Assistant(assistant) => { + Some(assistant.reasoning_content.clone()) + } + _ => None, + }) + .collect(); + assert_eq!( + turns, + vec![ + Some(ReasoningContent::Text("r1".into())), + Some(ReasoningContent::Text("r2".into())), + ] + ); + assert_eq!(assistant_texts(&messages), vec!["A", "B"]); + } + + /// Commentary shares the assistant turn with the calls it precedes, and a + /// phase change never separates pending calls from their tool messages. + #[test] + fn test_phase_change_keeps_calls_adjacent_to_outputs() { + let messages = convert_input_json(serde_json::json!([ + {"role": "user", "content": "weather?"}, + {"type": "message", "role": "assistant", "phase": "commentary", + "content": [{"type": "output_text", "text": "Checking."}]}, + {"type": "function_call", "call_id": "c", "name": "f", "arguments": "{}"}, + {"type": "message", "role": "assistant", "phase": "final_answer", + "content": [{"type": "output_text", "text": "Done."}]}, + {"type": "function_call_output", "call_id": "c", "output": "ok"} + ])); + assert_eq!(messages.len(), 3, "expected [user, assistant, tool]"); + let ChatCompletionRequestMessage::Assistant(assistant) = &messages[1] else { + panic!("expected one assistant turn carrying the call"); + }; + assert_eq!(assistant.tool_calls.as_ref().unwrap()[0].id, "c"); + assert_eq!(assistant_texts(&messages), vec!["Checking.Done."]); + assert!(matches!(messages[2], ChatCompletionRequestMessage::Tool(_))); + } + #[test] fn test_refusal_content_folded_into_assistant_text() { // Refusal parts in a prior assistant turn must survive to the next @@ -3321,6 +3486,7 @@ mod tests { _ => panic!("Expected Message variant"), }; assert_eq!(msg.role, AssistantRole::Assistant); + assert_eq!(msg.phase, Some(MessagePhase::FinalAnswer)); match &msg.content[0] { OutputMessageContent::OutputText(txt) => { @@ -3383,7 +3549,10 @@ mod tests { let OutputItem::Reasoning(reasoning) = &wrapped.inner.output[0] else { panic!("Expected Reasoning output before the tool call"); }; - assert!(reasoning.summary.is_empty()); + assert!(matches!( + reasoning.summary.as_slice(), + [SummaryPart::SummaryText(summary)] if summary.text == "Need the weather tool" + )); assert_eq!(reasoning_text(reasoning), "Need the weather tool"); match &wrapped.inner.output[1] { OutputItem::FunctionCall(fc) => { @@ -3496,16 +3665,17 @@ mod tests { chat.inner.choices[0].finish_reason = Some(FinishReason::ToolCalls); let response = chat_completion_to_response(chat, ¶ms, None).unwrap(); assert_eq!(response.inner.output.len(), 2); - let OutputItem::FunctionCall(call) = &response.inner.output[0] else { + let OutputItem::FunctionCall(call) = &response.inner.output[1] else { panic!("expected the structured function call"); }; assert_eq!(call.call_id, "call_original"); assert_eq!(call.name, "get_weather"); assert_eq!(call.namespace.as_deref(), Some("weather")); assert_eq!(call.arguments, r#"{"city":"Paris"}"#); - let OutputItem::Message(message) = &response.inner.output[1] else { + let OutputItem::Message(message) = &response.inner.output[0] else { panic!("expected the original text"); }; + assert_eq!(message.phase, Some(MessagePhase::Commentary)); let OutputMessageContent::OutputText(content) = &message.content[0] else { panic!("expected output text"); }; @@ -3958,38 +4128,92 @@ mod tests { } #[test] - fn test_reasoning_text_requires_explicit_request() { + fn test_reasoning_text_is_returned_and_summary_follows_request() { let unrequested = chat_completion_to_response( - make_chat_resp_with_reasoning("private reasoning"), + make_chat_resp_with_reasoning("raw reasoning"), &ResponseParams::default(), None, ) .unwrap(); - assert!( - unrequested - .inner - .output - .iter() - .all(|item| !matches!(item, OutputItem::Reasoning(_))) - ); + let [ + OutputItem::Reasoning(reasoning), + OutputItem::Message(message), + ] = unrequested.inner.output.as_slice() + else { + panic!("expected reasoning before the message"); + }; + assert!(reasoning.summary.is_empty()); + assert_eq!(reasoning_text(reasoning), "raw reasoning"); + assert_eq!(message.phase, Some(MessagePhase::FinalAnswer)); let params = requested_reasoning_params(); let requested = chat_completion_to_response(make_chat_resp_with_reasoning("summary"), ¶ms, None) .unwrap(); - let reasoning = requested - .inner - .output - .iter() - .find_map(|item| match item { - OutputItem::Reasoning(reasoning) => Some(reasoning), - _ => None, - }) - .expect("requested reasoning output"); - assert!(reasoning.summary.is_empty()); + let OutputItem::Reasoning(reasoning) = &requested.inner.output[0] else { + panic!("expected reasoning output"); + }; + assert!(matches!( + reasoning.summary.as_slice(), + [SummaryPart::SummaryText(summary)] if summary.text == "summary" + )); assert_eq!(reasoning_text(reasoning), "summary"); } + #[test] + fn test_reasoning_only_response_has_no_placeholder_message() { + let mut chat_resp = make_chat_resp_with_reasoning("only thinking"); + chat_resp.inner.choices[0].message.content = None; + + let response = + chat_completion_to_response(chat_resp, &ResponseParams::default(), None).unwrap(); + assert!(matches!( + response.inner.output.as_slice(), + [OutputItem::Reasoning(_)] + )); + + let mut empty = make_chat_resp_with_text(""); + empty.inner.choices[0].message.content = None; + let response = + chat_completion_to_response(empty, &ResponseParams::default(), None).unwrap(); + let [OutputItem::Message(placeholder)] = response.inner.output.as_slice() else { + panic!("expected only the placeholder message"); + }; + assert_eq!(placeholder.phase, Some(MessagePhase::FinalAnswer)); + } + + /// Text that accompanies calls is commentary placed before them, and a + /// length stop marks only the last call incomplete. + #[test] + fn test_unary_output_orders_reasoning_commentary_then_calls() { + let mut chat_resp = make_chat_resp_with_tool_calls( + dynamo_protocols::types::FinishReason::Length, + &[r#"{"location":"SF"}"#, r#"{"location":"NY"#], + ); + let message = &mut chat_resp.inner.choices[0].message; + message.reasoning_content = Some("check both cities".into()); + message.content = Some(dynamo_protocols::types::ChatCompletionMessageContent::Text( + "Checking.".into(), + )); + + let response = + chat_completion_to_response(chat_resp, &ResponseParams::default(), None).unwrap(); + let [ + OutputItem::Reasoning(reasoning), + OutputItem::Message(message), + OutputItem::FunctionCall(first), + OutputItem::FunctionCall(last), + ] = response.inner.output.as_slice() + else { + panic!("expected reasoning, message, then calls"); + }; + assert_eq!(reasoning.status, Some(OutputStatus::Completed)); + assert_eq!(message.phase, Some(MessagePhase::Commentary)); + assert_eq!(message.status, OutputStatus::Completed); + assert_eq!(first.status, Some(OutputStatus::Completed)); + assert_eq!(last.status, Some(OutputStatus::Incomplete)); + } + #[test] fn test_reasoning_summary_precedes_structured_tool_calls() { use dynamo_protocols::types::responses::{Reasoning, ReasoningSummary}; @@ -4327,12 +4551,16 @@ mod tests { assert!(json["output"][0].get("id").is_some()); assert!(json["output"][0].get("status").is_some()); assert_eq!(json["output"][0]["type"], "reasoning"); - assert_eq!(json["output"][0]["summary"], serde_json::json!([])); + assert_eq!( + json["output"][0]["summary"], + serde_json::json!([{"type": "summary_text", "text": "raw reasoning"}]) + ); assert_eq!( json["output"][0]["content"][0], serde_json::json!({"type": "reasoning_text", "text": "raw reasoning"}) ); assert_eq!(json["output"][1]["type"], "message"); + assert_eq!(json["output"][1]["phase"], "final_answer"); // Nullable-required fields must be present as null (not missing). for key in [ diff --git a/lib/llm/src/protocols/openai/responses/stream_converter.rs b/lib/llm/src/protocols/openai/responses/stream_converter.rs index ea920b6581c2..d98d5038df26 100644 --- a/lib/llm/src/protocols/openai/responses/stream_converter.rs +++ b/lib/llm/src/protocols/openai/responses/stream_converter.rs @@ -4,10 +4,15 @@ //! Converts a stream of chat completion SSE chunks into Responses API SSE events. //! //! The event sequence follows the OpenAI Responses API streaming spec: -//! `response.created` -> `response.in_progress` -> `response.output_item.added` -> -//! `response.content_part.added` -> N x `response.output_text.delta` -> -//! `response.output_text.done` -> `response.content_part.done` -> -//! `response.output_item.done` -> `response.completed` -> `[DONE]` +//! `response.created` -> `response.in_progress` -> output items -> +//! `response.completed` -> `[DONE]`. Each output item streams as +//! `response.output_item.added` -> part and delta events -> `response.output_item.done`. +//! +//! Output items appear in generation order. Reasoning is always a `reasoning` item; +//! a requested `reasoning.summary` streams it as `reasoning_summary_*` events instead +//! of `reasoning_text` events. A turn's text can span several `message` items: text +//! closed by a tool call or by resumed reasoning is `commentary`, and text still open +//! at the end is the `final_answer`. `phase` is set on `.done`, not on `.added`. use std::collections::HashSet; use std::time::{SystemTime, UNIX_EPOCH}; @@ -15,16 +20,19 @@ use std::time::{SystemTime, UNIX_EPOCH}; use axum::response::sse::Event; use dynamo_protocols::types::responses::{ AssistantRole, FunctionToolCall, IncompleteDetails, InputTokenDetails, Instructions, - OutputContent, OutputItem, OutputMessage, OutputMessageContent, OutputStatus, + MessagePhase, OutputContent, OutputItem, OutputMessage, OutputMessageContent, OutputStatus, OutputTextContent, OutputTokenDetails, ReasoningItem, ReasoningItemContent, ReasoningTextContent, Response, ResponseCompletedEvent, ResponseContentPartAddedEvent, ResponseContentPartDoneEvent, ResponseCreatedEvent, ResponseError, ResponseErrorCode, ResponseFailedEvent, ResponseFunctionCallArgumentsDeltaEvent, ResponseFunctionCallArgumentsDoneEvent, ResponseInProgressEvent, ResponseIncompleteEvent, - ResponseOutputItemAddedEvent, ResponseOutputItemDoneEvent, ResponseReasoningTextDeltaEvent, - ResponseReasoningTextDoneEvent, ResponseStreamEvent, ResponseTextDeltaEvent, - ResponseTextDoneEvent, ResponseTextParam, ResponseUsage, ServiceTierResponses, Status, - TextResponseFormatConfiguration, ToolChoiceOptions, ToolChoiceParam, Truncation, + ResponseOutputItemAddedEvent, ResponseOutputItemDoneEvent, + ResponseReasoningSummaryPartAddedEvent, ResponseReasoningSummaryPartDoneEvent, + ResponseReasoningSummaryTextDeltaEvent, ResponseReasoningSummaryTextDoneEvent, + ResponseReasoningTextDeltaEvent, ResponseReasoningTextDoneEvent, ResponseStreamEvent, + ResponseTextDeltaEvent, ResponseTextDoneEvent, ResponseTextParam, ResponseUsage, + ServiceTierResponses, Status, SummaryPart, SummaryTextContent, TextResponseFormatConfiguration, + ToolChoiceOptions, ToolChoiceParam, Truncation, }; use serde::{ Serialize, @@ -45,18 +53,23 @@ pub struct ResponseStreamConverter { params: ResponseParams, tool_names: super::ToolNameMap, allowed_names: Option>, + /// `reasoning.summary` was requested: reasoning streams as summary events and + /// reasoning items also carry the trace as their summary. + summary_requested: bool, /// Preserved Responses API-specific request context for faithful response reconstruction. api_context: Option, created_at: u64, sequence_number: u64, - // Text message tracking + // Open text message tracking. A tool call or resumed reasoning closes the + // message as commentary; a later text span opens a new message item. message_item_id: String, message_started: bool, message_output_index: u32, message_output_status: Option, accumulated_text: String, + closed_messages: Vec<(u32, OutputMessage)>, // Ordered reasoning spans; a new item opens when reasoning resumes after - // a tool call. + // a tool call or text. reasoning_items: Vec, active_reasoning_index: Option, // Function call tracking @@ -78,10 +91,22 @@ struct ReasoningState { } impl ReasoningState { - fn completed_item(&self, output_status: OutputStatus) -> ReasoningItem { + /// `summary_requested` mirrors the trace into `summary`; `content` always carries it. + fn completed_item( + &self, + output_status: OutputStatus, + summary_requested: bool, + ) -> ReasoningItem { + let summary = if summary_requested { + vec![SummaryPart::SummaryText(SummaryTextContent { + text: self.accumulated_text.clone(), + })] + } else { + vec![] + }; ReasoningItem { id: Some(self.item_id.clone()), - summary: vec![], + summary, content: Some(vec![ReasoningItemContent::ReasoningText( ReasoningTextContent { text: self.accumulated_text.clone(), @@ -125,21 +150,24 @@ impl ResponseStreamConverter { super::ToolNameMap::new(params.tools.as_deref().unwrap_or_default(), None) }); let allowed_names = params.allowed_backend_names(&tool_names); + let summary_requested = params.reasoning_summary_requested(); Self { response_id: format!("resp_{}", Uuid::new_v4().simple()), model, tool_names, allowed_names, + summary_requested, params, api_context: None, created_at, sequence_number: 0, - message_item_id: format!("msg_{}", Uuid::new_v4().simple()), + message_item_id: String::new(), message_started: false, message_output_index: 0, message_output_status: None, accumulated_text: String::new(), + closed_messages: Vec::new(), reasoning_items: Vec::new(), active_reasoning_index: None, function_call_items: Vec::new(), @@ -170,8 +198,14 @@ impl ResponseStreamConverter { let state_index = match self.active_reasoning_index { Some(state_index) => state_index, None => { - // Resumed reasoning proves pending tool calls completed, even if - // this chunk reports that the new reasoning exhausted the budget. + // Resumed reasoning proves the open message and pending tool calls + // completed, even if this chunk reports that the new reasoning + // exhausted the budget. Text followed by more reasoning is commentary. + self.close_open_message_item( + events, + OutputStatus::Completed, + MessagePhase::Commentary, + ); self.append_pending_function_call_done_events( events, OutputStatus::Completed, @@ -204,7 +238,21 @@ impl ResponseStreamConverter { }); events.push(self.make_sse_event(&item_added)); - let part_added = + // Like upstream SGLang, a requested summary streams as the summary + // part instead of the reasoning_text part. + let part_added = if self.summary_requested { + ResponseStreamEvent::ResponseReasoningSummaryPartAdded( + ResponseReasoningSummaryPartAddedEvent { + sequence_number: self.next_seq(), + item_id, + output_index, + summary_index: 0, + part: SummaryPart::SummaryText(SummaryTextContent { + text: String::new(), + }), + }, + ) + } else { ResponseStreamEvent::ResponseContentPartAdded(ResponseContentPartAddedEvent { sequence_number: self.next_seq(), item_id, @@ -213,7 +261,8 @@ impl ResponseStreamConverter { part: OutputContent::ReasoningText(ReasoningTextContent { text: String::new(), }), - }); + }) + }; events.push(self.make_sse_event(&part_added)); state_index @@ -225,14 +274,25 @@ impl ResponseStreamConverter { state.accumulated_text.push_str(reasoning); (state.item_id.clone(), state.output_index) }; - let delta = + let delta = if self.summary_requested { + ResponseStreamEvent::ResponseReasoningSummaryTextDelta( + ResponseReasoningSummaryTextDeltaEvent { + sequence_number: self.next_seq(), + item_id, + output_index, + summary_index: 0, + delta: reasoning.to_string(), + }, + ) + } else { ResponseStreamEvent::ResponseReasoningTextDelta(ResponseReasoningTextDeltaEvent { sequence_number: self.next_seq(), item_id, output_index, content_index: 0, delta: reasoning.to_string(), - }); + }) + }; events.push(self.make_sse_event(&delta)); } @@ -254,29 +314,55 @@ impl ResponseStreamConverter { state.item_id.clone(), state.output_index, state.accumulated_text.clone(), - state.completed_item(output_status), + state.completed_item(output_status, self.summary_requested), ) }; - let text_done = - ResponseStreamEvent::ResponseReasoningTextDone(ResponseReasoningTextDoneEvent { - sequence_number: self.next_seq(), - item_id: item_id.clone(), - output_index, - content_index: 0, - text: text.clone(), - }); - events.push(self.make_sse_event(&text_done)); + if self.summary_requested { + let text_done = ResponseStreamEvent::ResponseReasoningSummaryTextDone( + ResponseReasoningSummaryTextDoneEvent { + sequence_number: self.next_seq(), + item_id: item_id.clone(), + output_index, + summary_index: 0, + text: text.clone(), + }, + ); + events.push(self.make_sse_event(&text_done)); - let part_done = - ResponseStreamEvent::ResponseContentPartDone(ResponseContentPartDoneEvent { - sequence_number: self.next_seq(), - item_id: item_id.clone(), - output_index, - content_index: 0, - part: OutputContent::ReasoningText(ReasoningTextContent { text }), - }); - events.push(self.make_sse_event(&part_done)); + let part_done = ResponseStreamEvent::ResponseReasoningSummaryPartDone( + ResponseReasoningSummaryPartDoneEvent { + sequence_number: self.next_seq(), + item_id, + output_index, + summary_index: 0, + part: SummaryPart::SummaryText(SummaryTextContent { text }), + status: (output_status == OutputStatus::Incomplete) + .then(|| "incomplete".to_string()), + }, + ); + events.push(self.make_sse_event(&part_done)); + } else { + let text_done = + ResponseStreamEvent::ResponseReasoningTextDone(ResponseReasoningTextDoneEvent { + sequence_number: self.next_seq(), + item_id: item_id.clone(), + output_index, + content_index: 0, + text: text.clone(), + }); + events.push(self.make_sse_event(&text_done)); + + let part_done = + ResponseStreamEvent::ResponseContentPartDone(ResponseContentPartDoneEvent { + sequence_number: self.next_seq(), + item_id, + output_index, + content_index: 0, + part: OutputContent::ReasoningText(ReasoningTextContent { text }), + }); + events.push(self.make_sse_event(&part_done)); + } let item_done = ResponseStreamEvent::ResponseOutputItemDone(ResponseOutputItemDoneEvent { sequence_number: self.next_seq(), @@ -437,14 +523,6 @@ impl ResponseStreamConverter { self.incomplete_reason = Some(reason); } - if let Some(reasoning) = delta.reasoning_content.as_deref() - && !reasoning.is_empty() - && !self.message_started - && self.params.reasoning_summary_requested() - { - self.append_reasoning_delta(reasoning, events); - } - // Handle text content deltas — extract text from the enum let content_text = match &delta.content { Some(ChatCompletionMessageContent::Text(text)) => Some(text.as_str()), @@ -453,18 +531,39 @@ impl ResponseStreamConverter { None } None => None, - }; - if let Some(content) = content_text - && !content.is_empty() - { + } + .filter(|content| !content.is_empty()); + let reasoning = delta + .reasoning_content + .as_deref() + .filter(|reasoning| !reasoning.is_empty()); + // A delta carrying both fields while text is open continues that text + // before the reasoning that follows it. + let reasoning_after_text = content_text.is_some() + && self.message_started + && self.active_reasoning_index.is_none(); + + if !reasoning_after_text && let Some(reasoning) = reasoning { + self.append_reasoning_delta(reasoning, events); + } + + if let Some(content) = content_text { // Starting the answer is an explicit reasoning phase boundary. // The reasoning item completed even when this same chunk also // reports that the answer exhausted the output budget. self.append_active_reasoning_done_events(events, OutputStatus::Completed); - // Emit output_item.added + content_part.added on first text + // Emit output_item.added + content_part.added on first text. + // The phase is decided when the message closes. if !self.message_started { + // Text after tool calls proves those calls completed. + self.append_pending_function_call_done_events( + events, + OutputStatus::Completed, + false, + ); self.message_started = true; + self.message_item_id = format!("msg_{}", Uuid::new_v4().simple()); self.message_output_index = self.next_output_index; let output_index = self.message_output_index; self.next_output_index += 1; @@ -514,6 +613,10 @@ impl ResponseStreamConverter { events.push(self.make_sse_event(&text_delta)); } + if reasoning_after_text && let Some(reasoning) = reasoning { + self.append_reasoning_delta(reasoning, events); + } + // Handle tool call deltas if let Some(tool_calls) = &delta.tool_calls { // Match the chat-completions post-parse fallback: when the @@ -594,15 +697,23 @@ impl ResponseStreamConverter { "Backend returned function '{name}' outside allowed_tools" ), }; - let terminal_event = self.append_error_events(error, events); + // Open text preceded this call, so it is commentary. + let terminal_event = + self.append_failed_events(error, events, MessagePhase::Commentary); events.push(terminal_event); self.terminal_failure_emitted = true; return true; } if should_start { // Starting an allowed tool call is an explicit reasoning - // phase boundary, independent of the finish reason. + // phase boundary, independent of the finish reason. Text + // before the call is commentary and closes before it. self.append_active_reasoning_done_events(events, OutputStatus::Completed); + self.close_open_message_item( + events, + OutputStatus::Completed, + MessagePhase::Commentary, + ); } let new_output_index = should_start.then(|| { let output_index = self.next_output_index; @@ -820,7 +931,7 @@ impl ResponseStreamConverter { self.output_with_status(self.output_status()) } - fn message_output(&self, output_status: OutputStatus) -> OutputMessage { + fn message_output(&self, output_status: OutputStatus, phase: MessagePhase) -> OutputMessage { OutputMessage { id: self.message_item_id.clone(), content: vec![OutputMessageContent::OutputText(OutputTextContent { @@ -829,7 +940,7 @@ impl ResponseStreamConverter { logprobs: Some(vec![]), })], role: AssistantRole::Assistant, - phase: None, + phase: Some(phase), status: self.message_output_status.unwrap_or(output_status), } } @@ -841,18 +952,24 @@ impl ResponseStreamConverter { for reasoning in &self.reasoning_items { output.push(( reasoning.output_index, - OutputItem::Reasoning(reasoning.completed_item(output_status)), + OutputItem::Reasoning( + reasoning.completed_item(output_status, self.summary_requested), + ), )); } + for (output_index, message) in &self.closed_messages { + output.push((*output_index, OutputItem::Message(message.clone()))); + } if self.message_started { let message_status = if terminal_only { self.item_output_status(self.message_output_index) } else { output_status }; + // A message still open when the response ends is the final answer. output.push(( self.message_output_index, - OutputItem::Message(self.message_output(message_status)), + OutputItem::Message(self.message_output(message_status, MessagePhase::FinalAnswer)), )); } for function_call in &self.function_call_items { @@ -881,21 +998,28 @@ impl ResponseStreamConverter { output.into_iter().map(|(_, item)| item).collect() } + /// Close the open message with its phase and reset the text state so a + /// later text span opens a new message item. fn close_open_message_item( &mut self, events: &mut Vec>, output_status: OutputStatus, + phase: MessagePhase, ) { if !self.message_started { return; } + self.message_started = false; + let item_id = std::mem::take(&mut self.message_item_id); + let text = std::mem::take(&mut self.accumulated_text); + let status = self.message_output_status.take().unwrap_or(output_status); let text_done = ResponseStreamEvent::ResponseOutputTextDone(ResponseTextDoneEvent { sequence_number: self.next_seq(), - item_id: self.message_item_id.clone(), + item_id: item_id.clone(), output_index: self.message_output_index, content_index: 0, - text: self.accumulated_text.clone(), + text: text.clone(), logprobs: Some(vec![]), }); events.push(self.make_sse_event(&text_done)); @@ -903,23 +1027,36 @@ impl ResponseStreamConverter { let part_done = ResponseStreamEvent::ResponseContentPartDone(ResponseContentPartDoneEvent { sequence_number: self.next_seq(), - item_id: self.message_item_id.clone(), + item_id: item_id.clone(), output_index: self.message_output_index, content_index: 0, part: OutputContent::OutputText(OutputTextContent { - text: self.accumulated_text.clone(), + text: text.clone(), annotations: vec![], logprobs: Some(vec![]), }), }); events.push(self.make_sse_event(&part_done)); + let message = OutputMessage { + id: item_id, + content: vec![OutputMessageContent::OutputText(OutputTextContent { + text, + annotations: vec![], + logprobs: Some(vec![]), + })], + role: AssistantRole::Assistant, + phase: Some(phase), + status, + }; let item_done = ResponseStreamEvent::ResponseOutputItemDone(ResponseOutputItemDoneEvent { sequence_number: self.next_seq(), output_index: self.message_output_index, - item: OutputItem::Message(self.message_output(output_status)), + item: OutputItem::Message(message.clone()), }); events.push(self.make_sse_event(&item_done)); + self.closed_messages + .push((self.message_output_index, message)); } /// Emit remaining output completion events and `response.completed` at stream end. @@ -943,7 +1080,11 @@ impl ResponseStreamConverter { // Only the terminal item was cut short, and the terminal response reports it // that way. The `output_item.done` event has to agree, or a client sees one // item marked incomplete in the stream and completed in the final response. - self.close_open_message_item(events, self.item_output_status(self.message_output_index)); + self.close_open_message_item( + events, + self.item_output_status(self.message_output_index), + MessagePhase::FinalAnswer, + ); // Fallback for backends that end the transport without a finish-reason chunk. self.append_pending_function_call_done_events(events, output_status, true); @@ -973,15 +1114,25 @@ impl ResponseStreamConverter { } /// Append completion events for open output items and return the terminal - /// `response.failed` event. + /// `response.failed` event. Open text at a backend error is the final answer. pub fn append_error_events( &mut self, error: ResponseError, events: &mut Vec>, + ) -> Result { + self.append_failed_events(error, events, MessagePhase::FinalAnswer) + } + + /// Like `append_error_events`, labelling open text with `message_phase`. + fn append_failed_events( + &mut self, + error: ResponseError, + events: &mut Vec>, + message_phase: MessagePhase, ) -> Result { let output_status = OutputStatus::Incomplete; self.append_active_reasoning_done_events(events, output_status); - self.close_open_message_item(events, output_status); + self.close_open_message_item(events, output_status, message_phase); self.append_pending_function_call_done_events(events, output_status, false); let mut response = @@ -1490,6 +1641,25 @@ mod tests { events.iter().map(event_type).collect() } + /// Render one event through the SSE encoder and parse its JSON data. + fn event_json(event: &Result) -> serde_json::Value { + use axum::response::{IntoResponse, Sse}; + + let event = event.as_ref().unwrap().clone(); + let body = Sse::new(futures::stream::iter([Ok::<_, std::convert::Infallible>( + event, + )])) + .into_response() + .into_body(); + let bytes = futures::executor::block_on(axum::body::to_bytes(body, usize::MAX)).unwrap(); + let data = std::str::from_utf8(&bytes) + .unwrap() + .lines() + .find_map(|line| line.strip_prefix("data: ")) + .unwrap(); + serde_json::from_str(data).unwrap() + } + fn legacy_event_json( event: &ResponseStreamEvent, params: &ResponseParams, @@ -1581,8 +1751,8 @@ mod tests { event_types(&reasoning_events), vec![ "response.output_item.added".to_string(), - "response.content_part.added".to_string(), - "response.reasoning_text.delta".to_string(), + "response.reasoning_summary_part.added".to_string(), + "response.reasoning_summary_text.delta".to_string(), ] ); @@ -1595,12 +1765,17 @@ mod tests { assert_eq!( event_types(&tool_events), vec![ - "response.reasoning_text.done".to_string(), - "response.content_part.done".to_string(), + "response.reasoning_summary_text.done".to_string(), + "response.reasoning_summary_part.done".to_string(), "response.output_item.done".to_string(), "response.failed".to_string(), ] ); + assert_eq!( + event_json(&tool_events[1])["status"], + "incomplete", + "interrupted summary part must report incomplete" + ); assert_eq!( conv.reasoning_items[0].output_status, Some(OutputStatus::Incomplete) @@ -1666,6 +1841,47 @@ mod tests { panic!("expected message output"); }; assert_eq!(message.status, OutputStatus::Completed); + assert_eq!(message.phase, Some(MessagePhase::FinalAnswer)); + } + + #[test] + fn disallowed_function_call_closes_text_as_commentary() { + let params = ResponseParams { + tool_choice: Some( + serde_json::from_value(serde_json::json!({ + "type": "allowed_tools", + "mode": "auto", + "tools": [{"type": "function", "name": "read_file"}] + })) + .unwrap(), + ), + ..default_params() + }; + let mut conv = ResponseStreamConverter::new("test-model".into(), params); + let _ = conv.process_chunk(&text_chunk("Deleting it.")); + let events = conv.process_chunk(&tool_call_chunk( + 0, + Some("call-1"), + Some("delete_file"), + Some("{}"), + )); + assert_eq!( + event_types(&events), + vec![ + "response.output_text.done".to_string(), + "response.content_part.done".to_string(), + "response.output_item.done".to_string(), + "response.failed".to_string(), + ] + ); + assert_eq!(event_json(&events[2])["item"]["phase"], "commentary"); + + let output = conv.completed_output(); + let [OutputItem::Message(message)] = output.as_slice() else { + panic!("expected only the message output"); + }; + assert_eq!(message.phase, Some(MessagePhase::Commentary)); + assert_eq!(message.status, OutputStatus::Incomplete); } #[test] @@ -1939,8 +2155,8 @@ mod tests { } /// The message item is not the terminal item here, so the terminal response reports - /// it completed. `append_end_events` now closes it with this same per-item status, - /// so the `output_item.done` event cannot disagree with the final response. + /// it completed. The tool call closes it as completed commentary, so the + /// `output_item.done` event cannot disagree with the final response. #[test] fn test_length_leaves_a_non_terminal_message_item_completed() { let mut conv = ResponseStreamConverter::new("test-model".into(), default_params()); @@ -1953,18 +2169,21 @@ mod tests { )); let _ = conv.process_chunk(&finish_chunk(FinishReason::Length)); - assert_eq!( - conv.item_output_status(conv.message_output_index), - OutputStatus::Completed, - "only the terminal item was cut short" + let end_types = event_types(&conv.emit_end_events()); + assert!( + !end_types.contains(&"response.output_text.done".to_string()), + "the tool call already closed the message: {end_types:?}" ); - - let _ = conv.emit_end_events(); let response = conv.make_response(conv.terminal_status(), conv.completed_output()); let OutputItem::Message(message) = &response.output[0] else { panic!("expected message output"); }; assert_eq!(message.status, OutputStatus::Completed); + assert_eq!(message.phase, Some(MessagePhase::Commentary)); + let OutputItem::FunctionCall(call) = &response.output[1] else { + panic!("expected function call output"); + }; + assert_eq!(call.status, Some(OutputStatus::Incomplete)); } #[test] @@ -2026,8 +2245,8 @@ mod tests { assert_eq!( event_types(&events), vec![ - "response.reasoning_text.done".to_string(), - "response.content_part.done".to_string(), + "response.reasoning_summary_text.done".to_string(), + "response.reasoning_summary_part.done".to_string(), "response.output_item.done".to_string(), "response.output_item.added".to_string(), "response.content_part.added".to_string(), @@ -2113,8 +2332,8 @@ mod tests { event_types(&reasoning_events), vec![ "response.output_item.added".to_string(), - "response.content_part.added".to_string(), - "response.reasoning_text.delta".to_string(), + "response.reasoning_summary_part.added".to_string(), + "response.reasoning_summary_text.delta".to_string(), ] ); @@ -2122,58 +2341,108 @@ mod tests { assert_eq!( event_types(&text_events), vec![ - "response.reasoning_text.done".to_string(), - "response.content_part.done".to_string(), + "response.reasoning_summary_text.done".to_string(), + "response.reasoning_summary_part.done".to_string(), "response.output_item.done".to_string(), "response.output_item.added".to_string(), "response.content_part.added".to_string(), "response.output_text.delta".to_string(), ] ); + let reasoning_done = event_json(&text_events[2]); + assert_eq!( + reasoning_done["item"]["summary"], + serde_json::json!([{"type": "summary_text", "text": "thinking"}]) + ); + assert_eq!( + reasoning_done["item"]["content"], + serde_json::json!([{"type": "reasoning_text", "text": "thinking"}]) + ); let response = conv.make_response(Status::Completed, conv.completed_output()); assert_eq!(response.output.len(), 2); let OutputItem::Reasoning(reasoning) = &response.output[0] else { panic!("expected reasoning output before message"); }; - assert!(reasoning.summary.is_empty()); + assert!(matches!( + reasoning.summary.as_slice(), + [SummaryPart::SummaryText(summary)] if summary.text == "thinking" + )); assert_eq!(reasoning_text(reasoning), "thinking"); assert!(matches!(response.output[1], OutputItem::Message(_))); } #[test] - fn test_reasoning_without_requested_summary_emits_no_events() { + fn test_reasoning_without_requested_summary_streams_reasoning_text() { let mut conv = ResponseStreamConverter::new("test-model".into(), default_params()); - let events = conv.process_chunk(&reasoning_chunk("private reasoning")); + let events = conv.process_chunk(&reasoning_chunk("raw reasoning")); + assert_eq!( + event_types(&events), + vec![ + "response.output_item.added".to_string(), + "response.content_part.added".to_string(), + "response.reasoning_text.delta".to_string(), + ] + ); + let end_types = event_types(&conv.emit_end_events()); + assert_eq!( + end_types, + vec![ + "response.reasoning_text.done".to_string(), + "response.content_part.done".to_string(), + "response.output_item.done".to_string(), + "response.completed".to_string(), + ] + ); - assert!(events.is_empty()); - assert!(conv.completed_output().is_empty()); + let output = conv.completed_output(); + let [OutputItem::Reasoning(reasoning)] = output.as_slice() else { + panic!("expected only the reasoning output"); + }; + assert!(reasoning.summary.is_empty()); + assert_eq!(reasoning_text(reasoning), "raw reasoning"); } + /// Reasoning that resumes after text closes the text as commentary and opens + /// a new reasoning item; the next text span is a new final-answer message. #[test] - fn test_reasoning_text_ignores_updates_after_visible_output() { - use dynamo_protocols::types::responses::{Reasoning, ReasoningSummary}; + fn test_reasoning_after_text_is_emitted_and_splits_messages() { + let mut conv = ResponseStreamConverter::new("test-model".into(), default_params()); - let params = ResponseParams { - reasoning: Some(Reasoning { - effort: None, - summary: Some(ReasoningSummary::Auto), - ..Default::default() - }), - ..default_params() - }; - let mut conv = ResponseStreamConverter::new("test-model".into(), params); + let _ = conv.process_chunk(&reasoning_chunk("plan")); + let _ = conv.process_chunk(&text_chunk("Let me think again.")); + let late_events = conv.process_chunk(&reasoning_chunk("second thought")); + assert_eq!( + event_types(&late_events), + vec![ + "response.output_text.done".to_string(), + "response.content_part.done".to_string(), + "response.output_item.done".to_string(), + "response.output_item.added".to_string(), + "response.content_part.added".to_string(), + "response.reasoning_text.delta".to_string(), + ] + ); + assert_eq!(event_json(&late_events[2])["item"]["phase"], "commentary"); + let _ = conv.process_chunk(&text_chunk("Answer.")); + let _ = conv.emit_end_events(); - let _ = conv.process_chunk(&reasoning_chunk("summary")); - let _ = conv.process_chunk(&text_chunk("answer")); - let late_events = conv.process_chunk(&reasoning_chunk(" must not be appended")); - assert!(late_events.is_empty()); let output = conv.completed_output(); - let OutputItem::Reasoning(reasoning) = &output[0] else { - panic!("expected reasoning output"); + let [ + OutputItem::Reasoning(first), + OutputItem::Message(commentary), + OutputItem::Reasoning(second), + OutputItem::Message(answer), + ] = output.as_slice() + else { + panic!("expected reasoning, commentary, reasoning, answer: {output:?}"); }; - assert_eq!(reasoning_text(reasoning), "summary"); + assert_eq!(reasoning_text(first), "plan"); + assert_eq!(reasoning_text(second), "second thought"); + assert_eq!(commentary.phase, Some(MessagePhase::Commentary)); + assert_eq!(answer.phase, Some(MessagePhase::FinalAnswer)); + assert_ne!(commentary.id, answer.id); } #[test] @@ -2196,37 +2465,48 @@ mod tests { assert_eq!( event_types(&tool_events), vec![ - "response.reasoning_text.done".to_string(), - "response.content_part.done".to_string(), + "response.reasoning_summary_text.done".to_string(), + "response.reasoning_summary_part.done".to_string(), "response.output_item.done".to_string(), "response.output_item.added".to_string(), ] ); } + /// One delta can end open text and start reasoning; the text is appended + /// to the open message before the reasoning closes it. #[test] - fn test_reasoning_text_does_not_start_after_visible_output() { - use dynamo_protocols::types::responses::{Reasoning, ReasoningSummary}; - - let params = ResponseParams { - reasoning: Some(Reasoning { - effort: None, - summary: Some(ReasoningSummary::Auto), - ..Default::default() - }), - ..default_params() - }; - let mut conv = ResponseStreamConverter::new("test-model".into(), params); - - let _ = conv.process_chunk(&text_chunk("answer")); - let late_events = conv.process_chunk(&reasoning_chunk("out of order")); + fn test_delta_with_text_and_reasoning_continues_open_text_first() { + let mut conv = ResponseStreamConverter::new("test-model".into(), default_params()); + let _ = conv.process_chunk(&text_chunk("Let me")); - assert!(late_events.is_empty()); - assert!( - conv.completed_output() - .iter() - .all(|item| !matches!(item, OutputItem::Reasoning(_))) + let mut both = text_chunk(" check."); + both.inner.choices[0].delta.reasoning_content = Some("second thought".into()); + let events = conv.process_chunk(&both); + assert_eq!( + event_types(&events), + vec![ + "response.output_text.delta".to_string(), + "response.output_text.done".to_string(), + "response.content_part.done".to_string(), + "response.output_item.done".to_string(), + "response.output_item.added".to_string(), + "response.content_part.added".to_string(), + "response.reasoning_text.delta".to_string(), + ] ); + assert_eq!(event_json(&events[1])["text"], "Let me check."); + + let output = conv.completed_output(); + let [ + OutputItem::Message(message), + OutputItem::Reasoning(reasoning), + ] = output.as_slice() + else { + panic!("expected commentary before the reasoning: {output:?}"); + }; + assert_eq!(message.phase, Some(MessagePhase::Commentary)); + assert_eq!(reasoning_text(reasoning), "second thought"); } #[test] @@ -2549,34 +2829,181 @@ mod tests { end_types.contains(&"response.completed".to_string()), "completed in end events: {end_types:?}" ); + let output = conv.completed_output(); + let [OutputItem::Message(message)] = output.as_slice() else { + panic!("expected only the message output"); + }; + assert_eq!(message.phase, Some(MessagePhase::FinalAnswer)); } - /// Text followed by tool call: both handled correctly. + /// Text before tool calls is commentary: its message closes once, before the + /// first call is added. Text after the calls finishes them and opens a new + /// final-answer message. #[test] - fn test_text_then_tool_call() { + fn test_text_then_tool_calls_then_text() { let mut conv = ResponseStreamConverter::new("test-model".into(), default_params()); - let _ = conv.emit_start_events(); + let mut events = conv.emit_start_events(); + events.extend(conv.process_chunk(&text_chunk("Checking."))); - let text_events = conv.process_chunk(&text_chunk("Let me check that.")); - let text_types = event_types(&text_events); - assert!( - text_types.contains(&"response.output_item.added".to_string()), - "text message started: {text_types:?}" - ); - - let tool_events = conv.process_chunk(&tool_call_chunk( + let first_call = conv.process_chunk(&tool_call_chunk( 0, Some("call-1"), Some("search"), - Some("{\"q\":\"rust\"}"), + Some("{}"), + )); + assert_eq!( + event_types(&first_call), + vec![ + "response.output_text.done".to_string(), + "response.content_part.done".to_string(), + "response.output_item.done".to_string(), + "response.output_item.added".to_string(), + "response.function_call_arguments.delta".to_string(), + ] + ); + assert_eq!(event_json(&first_call[2])["item"]["phase"], "commentary"); + events.extend(first_call); + + let second_call = conv.process_chunk(&tool_call_chunk( + 1, + Some("call-2"), + Some("search"), + Some("{}"), )); - let tool_types = event_types(&tool_events); - assert!(!tool_types.contains(&"response.function_call_arguments.done".to_string())); - assert!(!tool_types.contains(&"response.output_item.done".to_string())); + assert_eq!( + event_types(&second_call), + vec![ + "response.output_item.added".to_string(), + "response.function_call_arguments.delta".to_string(), + ] + ); + events.extend(second_call); - let end_types = event_types(&conv.emit_end_events()); - assert!(end_types.contains(&"response.function_call_arguments.done".to_string())); - assert!(end_types.contains(&"response.output_item.done".to_string())); + let resumed = conv.process_chunk(&text_chunk("Done.")); + assert_eq!( + event_types(&resumed), + vec![ + "response.function_call_arguments.done".to_string(), + "response.output_item.done".to_string(), + "response.function_call_arguments.done".to_string(), + "response.output_item.done".to_string(), + "response.output_item.added".to_string(), + "response.content_part.added".to_string(), + "response.output_text.delta".to_string(), + ] + ); + events.extend(resumed); + let end = conv.emit_end_events(); + assert_eq!(event_json(&end[2])["item"]["phase"], "final_answer"); + events.extend(end); + + let jsons: Vec<_> = events.iter().map(event_json).collect(); + let sequence: Vec<_> = jsons + .iter() + .map(|json| json["sequence_number"].as_u64().unwrap()) + .collect(); + assert_eq!(sequence, (0..events.len() as u64).collect::>()); + let added_indices: Vec<_> = jsons + .iter() + .filter(|json| json["type"] == "response.output_item.added") + .map(|json| json["output_index"].as_u64().unwrap()) + .collect(); + assert_eq!(added_indices, vec![0, 1, 2, 3]); + + let output = conv.completed_output(); + let [ + OutputItem::Message(commentary), + OutputItem::FunctionCall(_), + OutputItem::FunctionCall(_), + OutputItem::Message(answer), + ] = output.as_slice() + else { + panic!("expected commentary, two calls, final answer: {output:?}"); + }; + assert_eq!(commentary.phase, Some(MessagePhase::Commentary)); + assert_eq!(commentary.status, OutputStatus::Completed); + assert_eq!(answer.phase, Some(MessagePhase::FinalAnswer)); + assert_ne!(commentary.id, answer.id); + let texts: Vec<_> = [commentary, answer] + .iter() + .map(|message| match &message.content[0] { + OutputMessageContent::OutputText(text) => text.text.as_str(), + other => panic!("expected output text, got {other:?}"), + }) + .collect(); + assert_eq!(texts, vec!["Checking.", "Done."]); + } + + /// Every output item that is added is done exactly once, whether the stream + /// completes or fails. + #[test] + fn test_every_added_output_item_is_done_once() { + for (params, fail) in [ + (default_params(), false), + (reasoning_params(), false), + (default_params(), true), + ] { + let mut conv = ResponseStreamConverter::new("test-model".into(), params); + let mut events = Vec::new(); + for chunk in [ + reasoning_chunk("plan"), + text_chunk("Checking."), + tool_call_chunk(0, Some("call-1"), Some("search"), Some("{}")), + tool_call_chunk(1, Some("call-2"), Some("search"), Some("{}")), + reasoning_chunk("results"), + text_chunk("Done."), + ] { + events.extend(conv.process_chunk(&chunk)); + } + events.extend(if fail { + conv.emit_error_events(ResponseError { + misalignment: None, + code: ResponseErrorCode::ServerError, + message: "backend error".to_string(), + }) + } else { + conv.emit_end_events() + }); + + let indices = |kind: &str| { + let mut indices: Vec<_> = events + .iter() + .map(event_json) + .filter(|json| json["type"] == kind) + .map(|json| json["output_index"].as_u64().unwrap()) + .collect(); + indices.sort_unstable(); + indices + }; + let expected: Vec = (0..6).collect(); + assert_eq!(indices("response.output_item.added"), expected); + assert_eq!(indices("response.output_item.done"), expected); + } + } + + #[test] + fn test_filtered_parallel_tool_call_does_not_close_text() { + let params = ResponseParams { + parallel_tool_calls: Some(false), + ..default_params() + }; + let mut conv = ResponseStreamConverter::new("test-model".into(), params); + let _ = conv.process_chunk(&text_chunk("Answer.")); + + let filtered = conv.process_chunk(&tool_call_chunk( + 1, + Some("call-2"), + Some("get_time"), + Some("{}"), + )); + assert!(filtered.is_empty()); + let _ = conv.emit_end_events(); + + let output = conv.completed_output(); + let [OutputItem::Message(message)] = output.as_slice() else { + panic!("expected only the message output"); + }; + assert_eq!(message.phase, Some(MessagePhase::FinalAnswer)); } #[test] @@ -2588,21 +3015,21 @@ mod tests { event_types(&conv.process_chunk(&reasoning_chunk("Let me"))), vec![ "response.output_item.added".to_string(), - "response.content_part.added".to_string(), - "response.reasoning_text.delta".to_string(), + "response.reasoning_summary_part.added".to_string(), + "response.reasoning_summary_text.delta".to_string(), ] ); assert_eq!( event_types(&conv.process_chunk(&reasoning_chunk(" think"))), - vec!["response.reasoning_text.delta".to_string()] + vec!["response.reasoning_summary_text.delta".to_string()] ); let end_events = conv.emit_end_events(); assert_eq!( event_types(&end_events), vec![ - "response.reasoning_text.done".to_string(), - "response.content_part.done".to_string(), + "response.reasoning_summary_text.done".to_string(), + "response.reasoning_summary_part.done".to_string(), "response.output_item.done".to_string(), "response.completed".to_string(), ] @@ -2616,7 +3043,10 @@ mod tests { let OutputItem::Reasoning(reasoning) = &output[0] else { panic!("expected reasoning output"); }; - assert!(reasoning.summary.is_empty()); + assert!(matches!( + reasoning.summary.as_slice(), + [SummaryPart::SummaryText(summary)] if summary.text == "Let me think" + )); assert_eq!(reasoning_text(reasoning), "Let me think"); } @@ -2645,15 +3075,15 @@ mod tests { assert_eq!( &transition_types[..3], [ - "response.reasoning_text.done", - "response.content_part.done", + "response.reasoning_summary_text.done", + "response.reasoning_summary_part.done", "response.output_item.done", ] ); assert!( !event_types(&conv.emit_end_events()) .iter() - .any(|event| event == "response.reasoning_text.done") + .any(|event| event == "response.reasoning_summary_text.done") ); let output = conv.completed_output(); @@ -2681,8 +3111,8 @@ mod tests { "response.function_call_arguments.done".to_string(), "response.output_item.done".to_string(), "response.output_item.added".to_string(), - "response.content_part.added".to_string(), - "response.reasoning_text.delta".to_string(), + "response.reasoning_summary_part.added".to_string(), + "response.reasoning_summary_text.delta".to_string(), ] ); let _ = conv.emit_end_events(); @@ -2818,6 +3248,9 @@ mod tests { assert_eq!( event_types(&events), vec![ + "response.output_text.done".to_string(), + "response.content_part.done".to_string(), + "response.output_item.done".to_string(), "response.output_item.added".to_string(), "response.function_call_arguments.delta".to_string(), ] diff --git a/lib/llm/tests/snapshots/responses_http_replay__responses_streaming_text.snap b/lib/llm/tests/snapshots/responses_http_replay__responses_streaming_text.snap index 6bce34eb8afb..e9349552804f 100644 --- a/lib/llm/tests/snapshots/responses_http_replay__responses_streaming_text.snap +++ b/lib/llm/tests/snapshots/responses_http_replay__responses_streaming_text.snap @@ -188,6 +188,7 @@ expression: "canonicalize(serde_json::to_value(events).unwrap())" ], "id": "[ID-2]", "role": "assistant", + "phase": "final_answer", "status": "completed" } } @@ -225,6 +226,7 @@ expression: "canonicalize(serde_json::to_value(events).unwrap())" ], "id": "[ID-2]", "role": "assistant", + "phase": "final_answer", "status": "completed" } ], diff --git a/lib/llm/tests/snapshots/responses_http_replay__responses_unary_text.snap b/lib/llm/tests/snapshots/responses_http_replay__responses_unary_text.snap index a88a2243c79e..0fbe101d0f3c 100644 --- a/lib/llm/tests/snapshots/responses_http_replay__responses_unary_text.snap +++ b/lib/llm/tests/snapshots/responses_http_replay__responses_unary_text.snap @@ -24,6 +24,7 @@ expression: canonicalize(body) ], "id": "[ID-2]", "role": "assistant", + "phase": "final_answer", "status": "completed" } ], diff --git a/tests/report_pytest_markers.py b/tests/report_pytest_markers.py index ef4f5da25c6c..7bec6b7e9a93 100755 --- a/tests/report_pytest_markers.py +++ b/tests/report_pytest_markers.py @@ -228,6 +228,7 @@ "vllm.reasoning.mistral_reasoning_parser", "vllm.reasoning.qwen3_engine_reasoning_parser", "vllm.renderers", + "vllm.renderers.chat_utils", "vllm.renderers.embed_utils", "vllm.sampling_params", "vllm.tokenizers", diff --git a/tests/utils/payloads.py b/tests/utils/payloads.py index 38cb5f2d44d3..3377042bd75a 100644 --- a/tests/utils/payloads.py +++ b/tests/utils/payloads.py @@ -1059,10 +1059,15 @@ def extract_content(response): output = result.get("output", []) assert len(output) > 0, "Response output is empty" - msg = output[0] + # Reasoning items and commentary can precede the final answer. + messages = [item for item in output if item.get("type") == "message"] assert ( - msg.get("type") == "message" - ), f"Expected output[0].type='message', got {msg.get('type')}" + messages + ), f"Expected a message output item, got types {[item.get('type') for item in output]}" + msg = next( + (item for item in messages if item.get("phase") == "final_answer"), + messages[-1], + ) assert ( msg.get("role") == "assistant" ), f"Expected role='assistant', got {msg.get('role')}" @@ -1126,28 +1131,31 @@ def extract_content(response): non_done[-1] == "response.completed" ), f"Last real event should be response.completed, got {non_done[-1]}" - # Validate text content events - assert "response.output_item.added" in event_types, "Missing output_item.added" - assert ( - "response.content_part.added" in event_types - ), "Missing content_part.added" + # Validate text content events. Reasoning items emit their own item and + # part events, so require a message item specifically. + assert any( + name == "response.output_item.added" + and data["item"].get("type") == "message" + for name, data in events + ), "Missing message output_item.added" assert "response.output_text.delta" in event_types, "Missing output_text.delta" assert "response.output_text.done" in event_types, "Missing output_text.done" assert "response.content_part.done" in event_types, "Missing content_part.done" assert "response.output_item.done" in event_types, "Missing output_item.done" - # Verify text deltas concatenate to the final text - deltas = [e[1]["delta"] for e in events if e[0] == "response.output_text.delta"] - done_events = [e for e in events if e[0] == "response.output_text.done"] - assert ( - len(done_events) == 1 - ), f"Expected 1 output_text.done, got {len(done_events)}" - full_text = "".join(deltas) - assert ( - done_events[0][1]["text"] == full_text - ), "Concatenated deltas don't match output_text.done text" - - return full_text + # Commentary and the final answer are separate message items; each + # item's deltas must concatenate to its output_text.done text. + deltas: dict[str, list[str]] = {} + for name, data in events: + if name == "response.output_text.delta": + deltas.setdefault(data["item_id"], []).append(data["delta"]) + for name, data in events: + if name == "response.output_text.done": + assert data["text"] == "".join( + deltas.get(data["item_id"], []) + ), "Concatenated deltas don't match output_text.done text" + + return "".join("".join(parts) for parts in deltas.values()) def response_handler(self, response: Any) -> str: return ResponsesStreamPayload.extract_content(response)