From 876d03a916618cf844378cc5cf73e06130f6bd52 Mon Sep 17 00:00:00 2001 From: Piotr Tarasiewicz Date: Thu, 20 Aug 2026 13:10:37 +0200 Subject: [PATCH 01/18] feat(vllm): add audio transcription serving Signed-off-by: Piotr Tarasiewicz Ported onto main 9cc8c3650487 by Inworld; conflicts resolved in: lib/bindings/python/src/dynamo/_core.pyi, lib/llm/src/discovery/model.rs, lib/llm/src/discovery/model_manager.rs, lib/llm/src/discovery/watcher.rs, lib/llm/src/discovery/worker_set.rs, lib/llm/src/http/service/openai.rs, lib/llm/src/model_type.rs (Cargo.lock and lib/bindings/python/Cargo.lock regenerated with cargo) Signed-off-by: Nurullah Morshed Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 18 ++ Cargo.toml | 2 +- components/src/dynamo/vllm/args.py | 2 + components/src/dynamo/vllm/backend_args.py | 51 ++++ components/src/dynamo/vllm/health_check.py | 31 ++ .../vllm/tests/test_vllm_backend_args.py | 38 +++ .../tests/test_vllm_health_check_payloads.py | 25 ++ .../vllm/tests/test_vllm_transcription.py | 152 ++++++++++ .../src/dynamo/vllm/tests/test_vllm_unit.py | 2 + .../vllm/tests/test_vllm_worker_factory.py | 11 + .../src/dynamo/vllm/transcription_handler.py | 171 +++++++++++ components/src/dynamo/vllm/worker_factory.py | 68 ++++- .../backends/vllm/launch/agg_transcription.sh | 54 ++++ lib/bindings/python/Cargo.lock | 18 ++ lib/bindings/python/rust/lib.rs | 4 + lib/bindings/python/src/dynamo/_core.pyi | 4 +- lib/llm/src/discovery/model.rs | 16 +- lib/llm/src/discovery/model_manager.rs | 54 +++- lib/llm/src/discovery/watcher.rs | 16 +- lib/llm/src/discovery/worker_set.rs | 22 +- lib/llm/src/endpoint_type.rs | 10 + lib/llm/src/http/service.rs | 1 + lib/llm/src/http/service/metrics.rs | 5 + lib/llm/src/http/service/openai.rs | 21 ++ lib/llm/src/http/service/service_v2.rs | 14 + lib/llm/src/http/service/transcriptions.rs | 270 ++++++++++++++++++ lib/llm/src/model_type.rs | 25 ++ lib/llm/src/protocols/openai.rs | 1 + .../src/protocols/openai/stream_aggregator.rs | 68 ++++- .../src/protocols/openai/transcriptions.rs | 128 +++++++++ .../openai/transcriptions/aggregator.rs | 18 ++ lib/llm/src/types.rs | 16 ++ lib/llm/tests/http-service.rs | 1 + lib/llm/tests/http-transcriptions.rs | 184 ++++++++++++ 34 files changed, 1512 insertions(+), 9 deletions(-) create mode 100644 components/src/dynamo/vllm/tests/test_vllm_transcription.py create mode 100644 components/src/dynamo/vllm/transcription_handler.py create mode 100755 examples/backends/vllm/launch/agg_transcription.sh create mode 100644 lib/llm/src/http/service/transcriptions.rs create mode 100644 lib/llm/src/protocols/openai/transcriptions.rs create mode 100644 lib/llm/src/protocols/openai/transcriptions/aggregator.rs create mode 100644 lib/llm/tests/http-transcriptions.rs diff --git a/Cargo.lock b/Cargo.lock index 08778ed532b5..765f440139f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -926,6 +926,7 @@ dependencies = [ "matchit 0.8.4", "memchr", "mime", + "multer", "percent-encoding", "pin-project-lite", "rustversion", @@ -6284,6 +6285,23 @@ dependencies = [ "pxfm", ] +[[package]] +name = "multer" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83e87776546dc87511aa5ee218730c92b666d7264ab6ed41f9d215af9cd5224b" +dependencies = [ + "bytes", + "encoding_rs", + "futures-util", + "http 1.4.1", + "httparse", + "memchr", + "mime", + "spin 0.9.8", + "version_check", +] + [[package]] name = "multimap" version = "0.10.1" diff --git a/Cargo.toml b/Cargo.toml index 43308867dc74..98d39315046c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -161,7 +161,7 @@ tokio-stream = { version = "0.1" } tokio-util = { version = "0.7.17", features = ["codec", "net", "rt", "io-util"] } tower = { version = "0.5", features = ["util"] } tower-http = { version = "0.6", features = ["trace"] } -axum = { version = "=0.8.4", features = ["macros", "ws"] } +axum = { version = "=0.8.4", features = ["macros", "multipart", "ws"] } axum-core = { version = "0.5.2" } hyper = { version = "=1.7.0" } hyper-util = { version = "=0.1.17" } diff --git a/components/src/dynamo/vllm/args.py b/components/src/dynamo/vllm/args.py index 2cd982f385cc..c4305bf70b01 100644 --- a/components/src/dynamo/vllm/args.py +++ b/components/src/dynamo/vllm/args.py @@ -325,6 +325,8 @@ def _unsupported_fpm_trace_role(dynamo_config: Config) -> Optional[str]: return "embedding" if dynamo_config.classify_worker: return "classify" + if dynamo_config.transcription_worker: + return "transcription" if dynamo_config.headless: return "headless" if dynamo_config.disaggregation_mode == DisaggregationMode.ENCODE: diff --git a/components/src/dynamo/vllm/backend_args.py b/components/src/dynamo/vllm/backend_args.py index 1f38aa8d9334..cce83e214a68 100644 --- a/components/src/dynamo/vllm/backend_args.py +++ b/components/src/dynamo/vllm/backend_args.py @@ -272,6 +272,15 @@ def add_arguments(self, parser) -> None: "supported with more than one process.", ) + add_negatable_bool_argument( + g, + flag_name="--transcription-worker", + env_var="DYN_VLLM_TRANSCRIPTION_WORKER", + default=False, + help="Run as an aggregated speech-to-text worker using vLLM's native " + "OpenAI transcription serving layer.", + ) + add_negatable_bool_argument( g, flag_name="--realtime", @@ -604,6 +613,7 @@ class DynamoVllmConfig(ConfigBase): embedding_worker: bool = False embedding_frontend_tokenization: bool = False embedding_worker_processes: int = 1 + transcription_worker: bool = False realtime: bool = False classify_worker: bool = False @@ -667,6 +677,7 @@ def validate(self) -> None: self._validate_embedding_frontend_tokenization() self._validate_embedding_worker_exclusivity() self._validate_embedding_worker_processes() + self._validate_transcription_worker_exclusivity() self._validate_realtime_worker_exclusivity() self._validate_classify_worker_exclusivity() self._validate_custom_encoder() @@ -1031,6 +1042,46 @@ def _validate_realtime_worker_exclusivity(self) -> None: if getattr(getattr(self, "engine_args", None), "enable_lora", False): raise ValueError("--realtime cannot be combined with --enable-lora.") + def _validate_transcription_worker_exclusivity(self) -> None: + """Transcription serving uses a dedicated aggregated ASR worker.""" + if not self.transcription_worker: + return + if self.disaggregation_mode != DisaggregationMode.AGGREGATED: + mode = ( + self.disaggregation_mode.value + if isinstance(self.disaggregation_mode, DisaggregationMode) + else self.disaggregation_mode + ) + raise ValueError( + "--transcription-worker is only valid with " + f"--disaggregation-mode=agg (got {mode})." + ) + for enabled, option in ( + (self.embedding_worker, "--embedding-worker"), + (self.classify_worker, "--classify-worker"), + (self.realtime, "--realtime"), + (bool(self.custom_encoder_class), "--custom-encoder-class"), + (self.gms_shadow_mode, "--gms-shadow-mode"), + (self.enable_rl, "--enable-rl"), + (self.headless, "--headless"), + ): + if enabled: + raise ValueError( + f"--transcription-worker cannot be combined with {option}." + ) + if self.enable_multimodal: + raise ValueError( + "--transcription-worker cannot be combined with multimodal worker flags." + ) + if self.benchmark_mode is not None: + raise ValueError( + "--transcription-worker cannot be combined with --benchmark-mode." + ) + if getattr(getattr(self, "engine_args", None), "enable_lora", False): + raise ValueError( + "--transcription-worker cannot be combined with --enable-lora." + ) + def _validate_classify_worker_exclusivity(self) -> None: """Classify worker is aggregated-only and exclusive of multimodal / embedding roles. Mirrors the embedding-worker constraints — both are diff --git a/components/src/dynamo/vllm/health_check.py b/components/src/dynamo/vllm/health_check.py index 9ef929e4f808..8b915995606b 100644 --- a/components/src/dynamo/vllm/health_check.py +++ b/components/src/dynamo/vllm/health_check.py @@ -7,7 +7,10 @@ This module defines the default health check payload for vLLM backends. """ +import base64 +import io import logging +import wave from typing import TYPE_CHECKING, Any, Optional from dynamo.health_check import HEALTH_CHECK_KEY, HealthCheckPayload @@ -152,6 +155,34 @@ def to_dict(self) -> dict[str, Any]: return _layer_probe_marker(super().to_dict()) +def _silent_wav_b64() -> str: + buffer = io.BytesIO() + with wave.open(buffer, "wb") as wav: + wav.setnchannels(1) + wav.setsampwidth(2) + wav.setframerate(16_000) + wav.writeframes(b"\x00" * 3_200) + return base64.b64encode(buffer.getvalue()).decode("ascii") + + +class VllmTranscriptionHealthCheckPayload(HealthCheckPayload): + """Real, short ASR probe matching TranscriptionWorkerHandler input.""" + + def __init__(self, model_name: Optional[str] = None): + self.default_payload: dict[str, Any] = { + "audio_b64": _silent_wav_b64(), + "filename": "health-check.wav", + "language": "en", + "response_format": "json", + } + if model_name is not None: + self.default_payload["model"] = model_name + super().__init__() + + def to_dict(self) -> dict[str, Any]: + return _layer_probe_marker(super().to_dict()) + + class VllmPrefillHealthCheckPayload(HealthCheckPayload): """ vLLM-specific health check payload for prefill workers in disaggregated mode. diff --git a/components/src/dynamo/vllm/tests/test_vllm_backend_args.py b/components/src/dynamo/vllm/tests/test_vllm_backend_args.py index f8a78afa386f..d877ba20528e 100644 --- a/components/src/dynamo/vllm/tests/test_vllm_backend_args.py +++ b/components/src/dynamo/vllm/tests/test_vllm_backend_args.py @@ -49,6 +49,12 @@ def create_config() -> DynamoVllmConfig: config.embedding_worker = False config.embedding_frontend_tokenization = False config.embedding_worker_processes = 1 + config.transcription_worker = False + config.realtime = False + config.classify_worker = False + config.custom_encoder_class = None + config.gms_shadow_mode = False + config.enable_rl = False config.headless = False config.benchmark_mode = None config.use_vllm_tokenizer = False @@ -318,6 +324,38 @@ def test_environment_rejects_invalid_value(self, monkeypatch): DynamoVllmArgGroup().add_arguments(parser) +class TestTranscriptionWorkerExclusivity: + def test_baseline_aggregated_is_accepted(self): + config = create_config() + config.transcription_worker = True + config.disaggregation_mode = DisaggregationMode.AGGREGATED + config._validate_transcription_worker_exclusivity() + + def test_non_aggregated_mode_is_rejected(self): + config = create_config() + config.transcription_worker = True + config.disaggregation_mode = DisaggregationMode.DECODE + with pytest.raises(ValueError, match="disaggregation-mode=agg"): + config._validate_transcription_worker_exclusivity() + + @pytest.mark.parametrize( + ("option", "value"), + [ + ("embedding_worker", True), + ("classify_worker", True), + ("realtime", True), + ("headless", True), + ], + ) + def test_standalone_role_combinations_are_rejected(self, option, value): + config = create_config() + config.transcription_worker = True + config.disaggregation_mode = DisaggregationMode.AGGREGATED + setattr(config, option, value) + with pytest.raises(ValueError, match=option.replace("_", "-")): + config._validate_transcription_worker_exclusivity() + + class TestRealtimeWorkerExclusivity: def test_baseline_aggregated_is_accepted(self): config = create_config() diff --git a/components/src/dynamo/vllm/tests/test_vllm_health_check_payloads.py b/components/src/dynamo/vllm/tests/test_vllm_health_check_payloads.py index 01e46f46e8f2..7b7ba47f7e5c 100644 --- a/components/src/dynamo/vllm/tests/test_vllm_health_check_payloads.py +++ b/components/src/dynamo/vllm/tests/test_vllm_health_check_payloads.py @@ -9,7 +9,10 @@ parity with trtllm/sglang for any future marker-gated behavior. """ +import base64 +import io import json +import wave import pytest @@ -19,11 +22,13 @@ VllmHealthCheckPayload, VllmOmniHealthCheckPayload, VllmPrefillHealthCheckPayload, + VllmTranscriptionHealthCheckPayload, ) pytestmark = [ pytest.mark.unit, pytest.mark.vllm, + pytest.mark.core, pytest.mark.gpu_0, pytest.mark.pre_merge, ] @@ -32,6 +37,10 @@ pytest.param(VllmHealthCheckPayload, id="VllmHealthCheckPayload"), pytest.param(VllmPrefillHealthCheckPayload, id="VllmPrefillHealthCheckPayload"), pytest.param(VllmOmniHealthCheckPayload, id="VllmOmniHealthCheckPayload"), + pytest.param( + VllmTranscriptionHealthCheckPayload, + id="VllmTranscriptionHealthCheckPayload", + ), pytest.param( VllmEmbeddingHealthCheckPayload, id="VllmEmbeddingHealthCheckPayload", @@ -89,3 +98,19 @@ def test_embedding_payload_omits_model_when_no_name(): assert "model" not in payload assert payload.get("input") == "probe" assert payload.get(HEALTH_CHECK_KEY) is True + + +def test_transcription_payload_contains_valid_short_wav(): + payload = VllmTranscriptionHealthCheckPayload(model_name="whisper").to_dict() + + assert payload.get("model") == "whisper" + assert payload.get("language") == "en" + assert payload.get("response_format") == "json" + assert payload.get(HEALTH_CHECK_KEY) is True + + audio = base64.b64decode(payload["audio_b64"], validate=True) + with wave.open(io.BytesIO(audio), "rb") as wav: + assert wav.getnchannels() == 1 + assert wav.getsampwidth() == 2 + assert wav.getframerate() == 16_000 + assert wav.getnframes() == 1_600 diff --git a/components/src/dynamo/vllm/tests/test_vllm_transcription.py b/components/src/dynamo/vllm/tests/test_vllm_transcription.py new file mode 100644 index 000000000000..d4d8c43d7018 --- /dev/null +++ b/components/src/dynamo/vllm/tests/test_vllm_transcription.py @@ -0,0 +1,152 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import asyncio +import base64 +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from dynamo.vllm.transcription_handler import TranscriptionWorkerHandler + +pytestmark = [ + pytest.mark.unit, + pytest.mark.vllm, + pytest.mark.multimodal, + pytest.mark.gpu_0, + pytest.mark.pre_merge, +] + + +def make_handler() -> TranscriptionWorkerHandler: + handler = TranscriptionWorkerHandler.__new__(TranscriptionWorkerHandler) + handler.config = SimpleNamespace( + model="openai/whisper-tiny", + served_model_name="whisper", + ) + return handler + + +def test_decode_audio_rejects_invalid_base64(): + with pytest.raises(ValueError, match="invalid base64"): + make_handler()._decode_audio({"audio_b64": "not base64"}) + + +def test_native_request_forwards_supported_openai_fields(): + request = make_handler()._native_request( + { + "filename": "sample.wav", + "model": "whisper", + "language": "en", + "prompt": "NVIDIA Dynamo", + "response_format": "verbose_json", + "temperature": 0.2, + "timestamp_granularities": ["word"], + } + ) + + assert request.file.filename == "sample.wav" + assert request.model == "whisper" + assert request.language == "en" + assert request.prompt == "NVIDIA Dynamo" + assert request.response_format == "verbose_json" + assert request.timestamp_granularities == ["word"] + + +def test_generate_delegates_to_native_vllm_serving(): + asyncio.run(_assert_generate_delegates_to_native_vllm_serving()) + + +async def _assert_generate_delegates_to_native_vllm_serving(): + class NativeResponse: + def model_dump(self, **_kwargs): + return { + "text": "hello", + "usage": {"type": "duration", "seconds": 1}, + } + + handler = make_handler() + handler._run_with_cancellation = AsyncMock(return_value=NativeResponse()) + context = object() + chunks = [ + chunk + async for chunk in handler.generate( + { + "audio_b64": base64.b64encode(b"audio").decode("ascii"), + "filename": "sample.wav", + "response_format": "json", + }, + context, + ) + ] + + assert chunks == [{"text": "hello", "usage": {"type": "duration", "seconds": 1}}] + handler._run_with_cancellation.assert_awaited_once() + + +def test_outer_cancellation_stops_native_vllm_inference(): + asyncio.run(_assert_outer_cancellation_stops_native_vllm_inference()) + + +async def _assert_outer_cancellation_stops_native_vllm_inference(): + inference_started = asyncio.Event() + inference_cancelled = asyncio.Event() + + async def create_transcription(*_args, **_kwargs): + inference_started.set() + try: + await asyncio.Event().wait() + finally: + inference_cancelled.set() + + async def wait_for_context_cancellation(): + await asyncio.Event().wait() + + handler = make_handler() + handler.serving = SimpleNamespace(create_transcription=create_transcription) + context = SimpleNamespace(async_killed_or_stopped=wait_for_context_cancellation) + + task = asyncio.create_task( + handler._run_with_cancellation(b"audio", object(), context) + ) + await inference_started.wait() + task.cancel() + + with pytest.raises(asyncio.CancelledError): + await task + await asyncio.wait_for(inference_cancelled.wait(), timeout=1) + + +def test_context_cancellation_stops_native_vllm_inference(): + asyncio.run(_assert_context_cancellation_stops_native_vllm_inference()) + + +async def _assert_context_cancellation_stops_native_vllm_inference(): + inference_started = asyncio.Event() + inference_cancelled = asyncio.Event() + context_cancelled = asyncio.Event() + + async def create_transcription(*_args, **_kwargs): + inference_started.set() + try: + await asyncio.Event().wait() + finally: + inference_cancelled.set() + + async def wait_for_context_cancellation(): + await context_cancelled.wait() + + handler = make_handler() + handler.serving = SimpleNamespace(create_transcription=create_transcription) + context = SimpleNamespace(async_killed_or_stopped=wait_for_context_cancellation) + + task = asyncio.create_task( + handler._run_with_cancellation(b"audio", object(), context) + ) + await inference_started.wait() + context_cancelled.set() + + with pytest.raises(asyncio.CancelledError): + await task + await asyncio.wait_for(inference_cancelled.wait(), timeout=1) diff --git a/components/src/dynamo/vllm/tests/test_vllm_unit.py b/components/src/dynamo/vllm/tests/test_vllm_unit.py index d0b2d179674c..9159a87c5e75 100644 --- a/components/src/dynamo/vllm/tests/test_vllm_unit.py +++ b/components/src/dynamo/vllm/tests/test_vllm_unit.py @@ -1666,6 +1666,7 @@ def _make_dynamo_config(**overrides): "enable_local_indexer": True, "embedding_worker": False, "classify_worker": False, + "transcription_worker": False, "headless": False, "enable_multimodal": False, "fpm_trace": False, @@ -1908,6 +1909,7 @@ def __init__(self, **kwargs): [ ({"embedding_worker": True}, "embedding"), ({"classify_worker": True}, "classify"), + ({"transcription_worker": True}, "transcription"), ({"headless": True}, "headless"), ( {"disaggregation_mode": DisaggregationMode.ENCODE}, diff --git a/components/src/dynamo/vllm/tests/test_vllm_worker_factory.py b/components/src/dynamo/vllm/tests/test_vllm_worker_factory.py index 2f705794baf4..18089d726bd1 100644 --- a/components/src/dynamo/vllm/tests/test_vllm_worker_factory.py +++ b/components/src/dynamo/vllm/tests/test_vllm_worker_factory.py @@ -65,6 +65,7 @@ def _make_config(**overrides) -> Mock: "disaggregation_mode": DisaggregationMode.AGGREGATED, "embedding_worker": False, "embedding_frontend_tokenization": False, + "transcription_worker": False, # Pin to the real Config default: an auto-created Mock attribute is # truthy, which enables the GMS shadow-mode path and imports the # optional gpu_memory_service package (absent in some test images). @@ -838,6 +839,7 @@ def factory(self) -> WorkerFactory: factory._create_prefill_worker = AsyncMock() # type: ignore[assignment] factory._create_decode_worker = AsyncMock() # type: ignore[assignment] factory._create_embedding_worker = AsyncMock() # type: ignore[assignment] + factory._create_transcription_worker = AsyncMock() # type: ignore[assignment] factory._create_realtime_worker = AsyncMock() # type: ignore[assignment] factory._create_classify_worker = AsyncMock() # type: ignore[assignment] return factory @@ -891,6 +893,15 @@ async def test_encode(self, factory: WorkerFactory, route_to_encode: bool) -> No factory._create_multimodal_encode_worker.assert_called_once() # type: ignore[union-attr] + async def test_transcription_worker_takes_priority( + self, factory: WorkerFactory + ) -> None: + config = _make_config(transcription_worker=True) + await factory.create(Mock(), config, asyncio.Event(), []) + + factory._create_transcription_worker.assert_called_once() # type: ignore[union-attr] + factory._create_decode_worker.assert_not_called() # type: ignore[union-attr] + async def test_embedding_worker_takes_priority( self, factory: WorkerFactory ) -> None: diff --git a/components/src/dynamo/vllm/transcription_handler.py b/components/src/dynamo/vllm/transcription_handler.py new file mode 100644 index 000000000000..498485cae193 --- /dev/null +++ b/components/src/dynamo/vllm/transcription_handler.py @@ -0,0 +1,171 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Dynamo adapter for vLLM's native OpenAI transcription serving layer.""" + +from __future__ import annotations + +import asyncio +import base64 +import binascii +import logging +from io import BytesIO +from typing import Any, AsyncIterator + +from dynamo._core import Context + +from .args import Config +from .engine_monitor import VllmEngineMonitor + +_TRANSCRIPTION_IMPORT_ERROR: ImportError | None = None +try: + from fastapi import UploadFile + from vllm.entrypoints.openai.engine.protocol import ErrorResponse + from vllm.entrypoints.openai.models.protocol import BaseModelPath + from vllm.entrypoints.openai.models.serving import OpenAIServingModels + from vllm.entrypoints.speech_to_text.transcription.protocol import ( + TranscriptionRequest, + ) + from vllm.entrypoints.speech_to_text.transcription.serving import ( + OpenAIServingTranscription, + ) +except ImportError as error: + UploadFile = None # type: ignore[assignment, misc] + ErrorResponse = None # type: ignore[assignment, misc] + BaseModelPath = None # type: ignore[assignment, misc] + OpenAIServingModels = None # type: ignore[assignment, misc] + TranscriptionRequest = None # type: ignore[assignment, misc] + OpenAIServingTranscription = None # type: ignore[assignment, misc] + _TRANSCRIPTION_IMPORT_ERROR = error + +logger = logging.getLogger(__name__) + +_TRANSCRIPTION_FIELDS = ( + "language", + "prompt", + "response_format", + "temperature", + "timestamp_granularities", +) + + +def _require_transcription_dependencies() -> None: + if _TRANSCRIPTION_IMPORT_ERROR is not None: + raise RuntimeError( + "vLLM transcription dependencies are unavailable" + ) from _TRANSCRIPTION_IMPORT_ERROR + + +class TranscriptionWorkerHandler: + """Serve batch STT requests using vLLM's model-native implementation.""" + + def __init__( + self, + runtime: Any, + engine: Any, + config: Config, + shutdown_event: asyncio.Event | None = None, + ) -> None: + _require_transcription_dependencies() + self.runtime = runtime + self.engine_client = engine + self.config = config + self.shutdown_event = shutdown_event + self.engine_monitor = VllmEngineMonitor(runtime, engine, shutdown_event) + + model_name = config.served_model_name or config.model + models = OpenAIServingModels( + engine, + [BaseModelPath(name=model_name, model_path=config.model)], + ) + self.serving = OpenAIServingTranscription( + engine, + models, + request_logger=None, + ) + logger.info("Transcription worker handler initialized for %s", model_name) + + def cleanup(self) -> None: + """Release handler-owned resources. + + The worker factory and runtime own the AsyncLLM lifecycle. The engine + monitor cancels its background tasks when this handler is released. + """ + return None + + @staticmethod + def _decode_audio(request: dict[str, Any]) -> bytes: + encoded = request.get("audio_b64") + if not isinstance(encoded, str) or not encoded: + raise ValueError("Transcription request missing required 'audio_b64' field") + try: + audio = base64.b64decode(encoded, validate=True) + except (binascii.Error, ValueError) as error: + raise ValueError( + "Transcription request contains invalid base64 audio" + ) from error + if not audio: + raise ValueError("Transcription audio must not be empty") + return audio + + def _native_request(self, request: dict[str, Any]) -> TranscriptionRequest: + _require_transcription_dependencies() + filename = request.get("filename") or "audio" + upload = UploadFile(file=BytesIO(b""), filename=filename) + values = { + field: request[field] + for field in _TRANSCRIPTION_FIELDS + if request.get(field) is not None + } + granularities = values.pop("timestamp_granularities", None) + if granularities is not None: + values["timestamp_granularities[]"] = granularities + values["file"] = upload + values["model"] = ( + request.get("model") or self.config.served_model_name or self.config.model + ) + return TranscriptionRequest(**values) + + async def _run_with_cancellation( + self, + audio: bytes, + request: TranscriptionRequest, + context: Context, + ) -> Any: + inference = asyncio.create_task( + self.serving.create_transcription(audio, request, raw_request=None) + ) + cancelled = asyncio.ensure_future(context.async_killed_or_stopped()) + try: + done, _ = await asyncio.wait( + (inference, cancelled), return_when=asyncio.FIRST_COMPLETED + ) + if cancelled in done and not inference.done(): + raise asyncio.CancelledError + return await inference + finally: + if not inference.done(): + inference.cancel() + await asyncio.gather(inference, return_exceptions=True) + if not cancelled.done(): + cancelled.cancel() + await asyncio.gather(cancelled, return_exceptions=True) + + async def generate( + self, request: dict[str, Any], context: Context + ) -> AsyncIterator[dict[str, Any]]: + audio = self._decode_audio(request) + native_request = self._native_request(request) + if native_request.response_format not in ("json", "verbose_json"): + raise ValueError( + "Transcription response_format must be 'json' or 'verbose_json'" + ) + + result = await self._run_with_cancellation(audio, native_request, context) + if ErrorResponse is not None and isinstance(result, ErrorResponse): + raise ValueError(result.message) + if not hasattr(result, "model_dump"): + raise TypeError( + f"Unexpected vLLM transcription response type: {type(result).__name__}" + ) + yield result.model_dump(mode="json", exclude_none=True) diff --git a/components/src/dynamo/vllm/worker_factory.py b/components/src/dynamo/vllm/worker_factory.py index 6fb22bbeca5e..45776ae312c0 100644 --- a/components/src/dynamo/vllm/worker_factory.py +++ b/components/src/dynamo/vllm/worker_factory.py @@ -47,6 +47,7 @@ VllmEmbeddingHealthCheckPayload, VllmHealthCheckPayload, VllmPrefillHealthCheckPayload, + VllmTranscriptionHealthCheckPayload, ) from .instrumented_scheduler import ( ENV_FPM_BENCHMARK_OUTPUT_PATH, @@ -59,6 +60,7 @@ from .publisher import StatLoggerFactory from .realtime import RealtimeHandler, RealtimeTextHandler, RealtimeTranscriptionHandler from .state_agent import StateAgentLifecycle, state_agent_settings +from .transcription_handler import TranscriptionWorkerHandler def _register_request_cache_metrics( @@ -1731,7 +1733,7 @@ def cleanup(self) -> None: class WorkerFactory: - """Factory for creating and initializing multimodal vLLM workers.""" + """Factory for creating and initializing vLLM workers.""" def __init__( self, @@ -1794,7 +1796,7 @@ async def create( shutdown_endpoints: list, snapshot_engine: Optional[SnapshotEngineSetupResult] = None, ) -> None: - """Create the appropriate multimodal worker based on config flags.""" + """Create the appropriate worker based on config flags.""" if config.realtime: await self._create_realtime_worker( @@ -1806,6 +1808,12 @@ async def create( ) return + if config.transcription_worker: + await self._create_transcription_worker( + runtime, config, shutdown_event, shutdown_endpoints + ) + return + # Embedding worker is selected first because it crosses worker shapes # (pooling AsyncLLM, ModelType.Embedding) rather than being a variant # of decode. Aggregated-only — exclusivity with disagg modes is @@ -2017,6 +2025,62 @@ async def _create_multimodal_encode_worker( finally: handler.cleanup() + async def _create_transcription_worker( + self, + runtime: DistributedRuntime, + config: Config, + shutdown_event: asyncio.Event, + shutdown_endpoints: list, + ) -> None: + generate_endpoint = runtime.endpoint( + f"{config.namespace}.{config.component}.{config.endpoint}" + ) + shutdown_endpoints[:] = [generate_endpoint] + + engine_client, vllm_config, _, _, _ = self.setup_vllm_engine( + config, + None, + fpm_worker_id=str(generate_endpoint.connection_id()), + ) + handler = TranscriptionWorkerHandler( + runtime=runtime, + engine=engine_client, + config=config, + shutdown_event=shutdown_event, + ) + model_name = config.served_model_name or config.model + health_payload = VllmTranscriptionHealthCheckPayload(model_name).to_dict() + metric_labels = [ + (prometheus_names.labels.MODEL, model_name), + (prometheus_names.labels.MODEL_NAME, model_name), + ] + + register_model_taint_route(runtime, generate_endpoint) + logger.info("Starting to serve the transcription worker endpoint...") + try: + await asyncio.gather( + generate_endpoint.serve_endpoint( + handler.generate, + metrics_labels=metric_labels, + health_check_payload=health_payload, + ), + self.register_vllm_model( + ModelInput.Text, + ModelType.Transcriptions, + generate_endpoint, + config, + engine_client, + vllm_config, + worker_type=WorkerType.Aggregated, + needs=[], + ), + ) + except Exception: + logger.exception("Failed to serve transcription worker endpoint") + raise + finally: + handler.cleanup() + async def _create_embedding_worker( self, runtime: DistributedRuntime, diff --git a/examples/backends/vllm/launch/agg_transcription.sh b/examples/backends/vllm/launch/agg_transcription.sh new file mode 100755 index 000000000000..890d6244cad1 --- /dev/null +++ b/examples/backends/vllm/launch/agg_transcription.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Aggregated speech-to-text model serving. +# GPUs: 1 + +set -e +trap 'echo Cleaning up...; kill 0' EXIT + +SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" +source "$SCRIPT_DIR/../../../common/gpu_utils.sh" +source "$SCRIPT_DIR/../../../common/launch_utils.sh" + +MODEL="openai/whisper-tiny" +EXTRA_ARGS=() +while [[ $# -gt 0 ]]; do + case $1 in + --model) + MODEL="$2" + shift 2 + ;; + -h|--help) + echo "Usage: $0 [--model ] [vLLM options]" + exit 0 + ;; + *) + EXTRA_ARGS+=("$1") + shift + ;; + esac +done + +HTTP_PORT="${DYN_HTTP_PORT:-8000}" +GPU_MEM_ARGS=$(build_vllm_gpu_mem_args) +print_launch_banner --no-curl "Launching Transcription Worker (1 GPU)" "$MODEL" "$HTTP_PORT" +print_curl_footer < bool { + self.worker_sets + .iter() + .any(|entry| entry.value().has_transcriptions_engine()) + } + /// Check if any WorkerSet has a realtime engine. pub fn has_realtime_engine(&self) -> bool { self.worker_sets @@ -637,6 +644,13 @@ impl Model { .ok_or_else(|| self.engine_error(self.has_audios_engine())) } + pub fn get_transcriptions_engine( + &self, + ) -> Result { + self.select_worker_set_with(|ws| ws.transcriptions_engine.clone()) + .ok_or_else(|| self.engine_error(self.has_transcriptions_engine())) + } + pub fn get_tensor_engine(&self) -> Result { self.select_worker_set_with(|ws| ws.tensor_engine.clone()) .ok_or_else(|| self.engine_error(self.has_tensor_engine())) diff --git a/lib/llm/src/discovery/model_manager.rs b/lib/llm/src/discovery/model_manager.rs index e314cd49a2b4..c39deeb42332 100644 --- a/lib/llm/src/discovery/model_manager.rs +++ b/lib/llm/src/discovery/model_manager.rs @@ -55,7 +55,9 @@ use crate::{ classify::OpenAIClassifyStreamingEngine, completions::OpenAICompletionsStreamingEngine, embeddings::OpenAIEmbeddingsStreamingEngine, generate::GenerateStreamingEngine, images::OpenAIImagesStreamingEngine, pooling::OpenAIPoolingStreamingEngine, - rerank::OpenAIRerankStreamingEngine, videos::OpenAIVideosStreamingEngine, + rerank::OpenAIRerankStreamingEngine, + transcriptions::OpenAITranscriptionsStreamingEngine, + videos::OpenAIVideosStreamingEngine, }, }, worker_type::WorkerType, @@ -1312,6 +1314,16 @@ impl ModelManager { .collect() } + pub fn list_transcriptions_models(&self) -> Vec { + self.catalog + .load() + .models + .iter() + .filter(|(_, model)| model.has_transcriptions_engine()) + .map(|(name, _)| name.clone()) + .collect() + } + pub fn list_videos_models(&self) -> Vec { self.catalog .load() @@ -1379,6 +1391,8 @@ impl ModelManager { || (model_type.contains(ModelType::Embedding) && model.has_embeddings_engine()) || (model_type.contains(ModelType::Images) && model.has_images_engine()) || (model_type.contains(ModelType::Audios) && model.has_audios_engine()) + || (model_type.contains(ModelType::Transcriptions) + && model.has_transcriptions_engine()) || (model_type.contains(ModelType::Videos) && model.has_videos_engine()) || (model_type.contains(ModelType::TensorBased) && model.has_tensor_engine()) || (model_type.contains(ModelType::Realtime) && model.has_realtime_engine()) @@ -1508,6 +1522,18 @@ impl ModelManager { .get_audios_engine() } + pub fn get_transcriptions_engine( + &self, + model: &str, + ) -> Result { + self.catalog + .load() + .models + .get(model) + .ok_or_else(|| ModelManagerError::ModelNotFound(model.to_string()))? + .get_transcriptions_engine() + } + pub fn get_realtime_engine( &self, model: &str, @@ -1873,6 +1899,30 @@ impl ModelManager { Ok(()) } + pub fn add_transcriptions_model( + &self, + model: &str, + card_checksum: &str, + engine: OpenAITranscriptionsStreamingEngine, + ) -> Result<(), ModelManagerError> { + let _reservation = self.reservation_lock.lock(); + self.ensure_name_not_alias(model)?; + let model_entry = self.get_or_create_model(model); + if model_entry.has_transcriptions_engine() { + return Err(ModelManagerError::ModelAlreadyExists(model.to_string())); + } + let namespace = format!("__local_transcriptions_{}", model); + let mut ws = WorkerSet::new( + namespace.clone(), + card_checksum.to_string(), + Self::aggregated_local_card(), + ); + ws.transcriptions_engine = Some(engine); + model_entry.add_worker_set(namespace, Arc::new(ws)); + self.publish_catalog_locked(); + Ok(()) + } + pub fn add_realtime_model( &self, model: &str, @@ -4419,6 +4469,8 @@ mod tests { manager.add_images_model(name, "ck", Arc::new(UncalledEngine)) } else if model_type == ModelType::Audios { manager.add_audios_model(name, "ck", Arc::new(UncalledEngine)) + } else if model_type == ModelType::Transcriptions { + manager.add_transcriptions_model(name, "ck", Arc::new(UncalledEngine)) } else if model_type == ModelType::Videos { manager.add_videos_model(name, "ck", Arc::new(UncalledEngine)) } else if model_type == ModelType::TensorBased { diff --git a/lib/llm/src/discovery/watcher.rs b/lib/llm/src/discovery/watcher.rs index b2aa15bda31e..bd71d79a6dad 100644 --- a/lib/llm/src/discovery/watcher.rs +++ b/lib/llm/src/discovery/watcher.rs @@ -52,6 +52,7 @@ use crate::{ images::{NvCreateImageRequest, NvImagesResponse}, pooling::{NvCreatePoolingRequest, NvCreatePoolingResponse}, rerank::{NvCreateRerankRequest, NvCreateRerankResponse}, + transcriptions::{NvAudioTranscriptionResponse, NvCreateAudioTranscriptionRequest}, videos::{NvCreateVideoRequest, NvVideosResponse}, }, tensor::{NvCreateTensorRequest, NvCreateTensorResponse}, @@ -230,6 +231,7 @@ const ALL_MODEL_TYPES: &[ModelType] = &[ ModelType::Embedding, ModelType::Images, ModelType::Audios, + ModelType::Transcriptions, ModelType::Videos, ModelType::TensorBased, ModelType::Realtime, @@ -922,6 +924,17 @@ impl ModelWatcher { worker_set.audios_engine = Some(Arc::new(audios_router)); } + if card.model_type.supports_transcriptions() { + let transcriptions_router = PushRouter::< + NvCreateAudioTranscriptionRequest, + Annotated, + >::from_client_with_monitor( + client.clone(), router_config.router_mode, None + ) + .await?; + worker_set.transcriptions_engine = Some(Arc::new(transcriptions_router)); + } + if card.model_type.supports_realtime() { // `Text` is overloaded for Realtime; its I/O passes through. let realtime_router = PushRouter::< @@ -1010,7 +1023,7 @@ impl ModelWatcher { // prefill is routed off `worker_type`.) anyhow::bail!( "Unsupported model configuration: {} with {} input. Supported combinations: \ - Tokens+(Chat|Completions), Text+(Chat|Completions|Images|Audios|Videos|Embeddings|Classify|Pooling|Rerank|Realtime), \ + Tokens+(Chat|Completions), Text+(Chat|Completions|Images|Audios|Transcriptions|Videos|Embeddings|Classify|Pooling|Rerank|Realtime), \ Tokens+Embeddings, Tensor+TensorBased", card.model_type, card.model_input.as_str() @@ -2825,6 +2838,7 @@ request_classifier: assert!(is_model_type_list_empty(&mm, ModelType::Embedding)); assert!(is_model_type_list_empty(&mm, ModelType::Images)); assert!(is_model_type_list_empty(&mm, ModelType::Audios)); + assert!(is_model_type_list_empty(&mm, ModelType::Transcriptions)); assert!(is_model_type_list_empty(&mm, ModelType::Videos)); assert!(is_model_type_list_empty(&mm, ModelType::TensorBased)); assert!(is_model_type_list_empty(&mm, ModelType::Realtime)); diff --git a/lib/llm/src/discovery/worker_set.rs b/lib/llm/src/discovery/worker_set.rs index eda75975f5cd..f333b5c79976 100644 --- a/lib/llm/src/discovery/worker_set.rs +++ b/lib/llm/src/discovery/worker_set.rs @@ -30,7 +30,9 @@ use crate::{ classify::OpenAIClassifyStreamingEngine, completions::OpenAICompletionsStreamingEngine, embeddings::OpenAIEmbeddingsStreamingEngine, generate::GenerateStreamingEngine, images::OpenAIImagesStreamingEngine, pooling::OpenAIPoolingStreamingEngine, - rerank::OpenAIRerankStreamingEngine, videos::OpenAIVideosStreamingEngine, + rerank::OpenAIRerankStreamingEngine, + transcriptions::OpenAITranscriptionsStreamingEngine, + videos::OpenAIVideosStreamingEngine, }, }, }; @@ -222,6 +224,7 @@ pub struct WorkerSet { pub(crate) images_engine: Option, pub(crate) videos_engine: Option, pub(crate) audios_engine: Option, + pub(crate) transcriptions_engine: Option, pub(crate) tensor_engine: Option, pub(crate) realtime_engine: Option, pub(crate) generate_engine: Option, @@ -269,6 +272,7 @@ impl WorkerSet { images_engine: None, videos_engine: None, audios_engine: None, + transcriptions_engine: None, tensor_engine: None, realtime_engine: None, generate_engine: None, @@ -353,6 +357,10 @@ impl WorkerSet { self.audios_engine.is_some() } + pub fn has_transcriptions_engine(&self) -> bool { + self.transcriptions_engine.is_some() + } + pub fn has_tensor_engine(&self) -> bool { self.tensor_engine.is_some() } @@ -392,6 +400,7 @@ impl WorkerSet { || self.has_tensor_engine() || self.has_videos_engine() || self.has_audios_engine() + || self.has_transcriptions_engine() || self.has_realtime_engine() || self.has_generate_engine() } @@ -506,6 +515,7 @@ impl WorkerSet { retain_for_requests!(images_engine); retain_for_requests!(videos_engine); retain_for_requests!(audios_engine); + retain_for_requests!(transcriptions_engine); retain_for_requests!(tensor_engine); retain_for_requests!(realtime_engine); retain_for_requests!(generate_engine); @@ -540,6 +550,7 @@ impl WorkerSet { images_engine: lora_context_engine(&self.images_engine, &lora_name), videos_engine: lora_context_engine(&self.videos_engine, &lora_name), audios_engine: lora_context_engine(&self.audios_engine, &lora_name), + transcriptions_engine: lora_context_engine(&self.transcriptions_engine, &lora_name), tensor_engine: lora_context_engine(&self.tensor_engine, &lora_name), // Realtime is bidirectional, so the server-streaming LoRA context wrapper cannot // inject the adapter identity. Fail closed instead of serving the base weights. @@ -592,6 +603,9 @@ mod tests { use crate::types::openai::images::{NvCreateImageRequest, NvImagesResponse}; use crate::types::openai::pooling::{NvCreatePoolingRequest, NvCreatePoolingResponse}; use crate::types::openai::rerank::{NvCreateRerankRequest, NvCreateRerankResponse}; + use crate::types::openai::transcriptions::{ + NvAudioTranscriptionResponse, NvCreateAudioTranscriptionRequest, + }; use crate::types::openai::videos::{NvCreateVideoRequest, NvVideosResponse}; use async_trait::async_trait; use dynamo_runtime::engine::AsyncEngine; @@ -843,6 +857,12 @@ mod tests { StubEngine::::new(), "audios" ); + check!( + transcriptions_engine, + has_transcriptions_engine, + StubEngine::::new(), + "transcriptions" + ); check!( tensor_engine, has_tensor_engine, diff --git a/lib/llm/src/endpoint_type.rs b/lib/llm/src/endpoint_type.rs index 8cf450b4548b..195670220977 100644 --- a/lib/llm/src/endpoint_type.rs +++ b/lib/llm/src/endpoint_type.rs @@ -16,6 +16,8 @@ pub enum EndpointType { Images, /// Audios API (speech/audio generation) Audios, + /// Audio Transcriptions API (speech-to-text) + Transcriptions, /// Videos API (video generation) Videos, /// Realtime API (bidirectional streaming over WebSocket) @@ -44,6 +46,7 @@ impl EndpointType { Self::Embedding => "embedding", Self::Images => "images", Self::Audios => "audios", + Self::Transcriptions => "transcriptions", Self::Videos => "videos", Self::Realtime => "realtime", Self::Classify => "classify", @@ -63,6 +66,7 @@ impl EndpointType { Self::Embedding, Self::Images, Self::Audios, + Self::Transcriptions, Self::Videos, Self::Realtime, Self::Classify, @@ -100,6 +104,12 @@ mod tests { assert!(EndpointType::all().contains(&EndpointType::Generate)); } + #[test] + fn transcriptions_is_registered() { + assert_eq!(EndpointType::Transcriptions.as_str(), "transcriptions"); + assert!(EndpointType::all().contains(&EndpointType::Transcriptions)); + } + #[test] fn batch_as_str() { assert_eq!(EndpointType::Batch.as_str(), "batch"); diff --git a/lib/llm/src/http/service.rs b/lib/llm/src/http/service.rs index 47d863c2db58..af6a9c3d56dc 100644 --- a/lib/llm/src/http/service.rs +++ b/lib/llm/src/http/service.rs @@ -22,6 +22,7 @@ mod anthropic; mod batch_proxy; pub mod metadata; mod openai; +mod transcriptions; mod write_timeout; pub mod busy_threshold; diff --git a/lib/llm/src/http/service/metrics.rs b/lib/llm/src/http/service/metrics.rs index 89f900f0f218..a037d91411c4 100644 --- a/lib/llm/src/http/service/metrics.rs +++ b/lib/llm/src/http/service/metrics.rs @@ -723,6 +723,9 @@ pub enum Endpoint { /// OAI Audio Speech Audios, + /// OAI Audio Transcriptions + Transcriptions, + /// OAI Responses Responses, @@ -1883,6 +1886,7 @@ impl std::fmt::Display for Endpoint { Endpoint::Images => write!(f, "images"), Endpoint::Videos => write!(f, "videos"), Endpoint::Audios => write!(f, "audios"), + Endpoint::Transcriptions => write!(f, "transcriptions"), Endpoint::Responses => write!(f, "responses"), Endpoint::AnthropicMessages => write!(f, "anthropic_messages"), Endpoint::Tensor => write!(f, "tensor"), @@ -1903,6 +1907,7 @@ impl Endpoint { Endpoint::Images => "images", Endpoint::Videos => "videos", Endpoint::Audios => "audios", + Endpoint::Transcriptions => "transcriptions", Endpoint::Responses => "responses", Endpoint::AnthropicMessages => "anthropic_messages", Endpoint::Tensor => "tensor", diff --git a/lib/llm/src/http/service/openai.rs b/lib/llm/src/http/service/openai.rs index bc7c23658f5a..2e991ede820d 100644 --- a/lib/llm/src/http/service/openai.rs +++ b/lib/llm/src/http/service/openai.rs @@ -465,6 +465,10 @@ pub(crate) fn find_invalid_argument_in_chain<'a>( None } +pub(super) fn error_is_invalid_argument(err: &(dyn std::error::Error + 'static)) -> bool { + find_invalid_argument_in_chain(err).is_some() +} + fn find_queue_rejection_in_chain<'a>( err: &'a (dyn std::error::Error + 'static), ) -> Option<&'a dynamo_kv_router::scheduling::QueueRejection> { @@ -581,6 +585,23 @@ impl ErrorMessage { ) } + /// Invalid request (400). Recorded as an `InvalidRequest` local failure, + /// like the other client-error constructors. + pub fn bad_request(msg: T) -> ErrorResponse { + let code = StatusCode::BAD_REQUEST; + record_local_failure(ErrorClass::InvalidRequest); + ( + code, + Json(ErrorMessage { + message: msg.to_string(), + error_type: bad_request_error_type(), + code: code.as_u16(), + details: None, + metric_error_type: None, + }), + ) + } + /// Internal Service Error /// Return this error when the service encounters an internal error. /// We should return a generic message to the client instead of the real error. diff --git a/lib/llm/src/http/service/service_v2.rs b/lib/llm/src/http/service/service_v2.rs index 43c1f8c06e8d..301d65f14101 100644 --- a/lib/llm/src/http/service/service_v2.rs +++ b/lib/llm/src/http/service/service_v2.rs @@ -447,6 +447,7 @@ struct StateFlags { images_endpoints_enabled: AtomicBool, videos_endpoints_enabled: AtomicBool, audios_endpoints_enabled: AtomicBool, + transcriptions_endpoints_enabled: AtomicBool, realtime_endpoints_enabled: AtomicBool, responses_endpoints_enabled: AtomicBool, anthropic_endpoints_enabled: AtomicBool, @@ -466,6 +467,9 @@ impl StateFlags { EndpointType::Images => self.images_endpoints_enabled.load(Ordering::Relaxed), EndpointType::Videos => self.videos_endpoints_enabled.load(Ordering::Relaxed), EndpointType::Audios => self.audios_endpoints_enabled.load(Ordering::Relaxed), + EndpointType::Transcriptions => self + .transcriptions_endpoints_enabled + .load(Ordering::Relaxed), EndpointType::Realtime => self.realtime_endpoints_enabled.load(Ordering::Relaxed), EndpointType::Responses => self.responses_endpoints_enabled.load(Ordering::Relaxed), EndpointType::AnthropicMessages => { @@ -505,6 +509,9 @@ impl StateFlags { EndpointType::Audios => self .audios_endpoints_enabled .store(enabled, Ordering::Relaxed), + EndpointType::Transcriptions => self + .transcriptions_endpoints_enabled + .store(enabled, Ordering::Relaxed), EndpointType::Realtime => self .realtime_endpoints_enabled .store(enabled, Ordering::Relaxed), @@ -547,6 +554,7 @@ impl State { images_endpoints_enabled: AtomicBool::new(false), videos_endpoints_enabled: AtomicBool::new(false), audios_endpoints_enabled: AtomicBool::new(false), + transcriptions_endpoints_enabled: AtomicBool::new(false), realtime_endpoints_enabled: AtomicBool::new(false), responses_endpoints_enabled: AtomicBool::new(false), anthropic_endpoints_enabled: AtomicBool::new(false), @@ -1696,6 +1704,8 @@ impl HttpServiceConfigBuilder { let (images_docs, images_route) = super::openai::images_router(state.clone(), None); let (videos_docs, videos_route) = super::openai::videos_router(state.clone(), None); let (audios_docs, audios_route) = super::openai::audios_router(state.clone(), None); + let (transcriptions_docs, transcriptions_route) = + super::transcriptions::transcriptions_router(state.clone(), None); let (realtime_docs, realtime_route) = super::realtime::realtime_router(state.clone(), None); let (responses_docs, responses_route) = super::openai::responses_router( state.clone(), @@ -1712,6 +1722,10 @@ impl HttpServiceConfigBuilder { endpoint_routes.insert(EndpointType::Images, (images_docs, images_route)); endpoint_routes.insert(EndpointType::Videos, (videos_docs, videos_route)); endpoint_routes.insert(EndpointType::Audios, (audios_docs, audios_route)); + endpoint_routes.insert( + EndpointType::Transcriptions, + (transcriptions_docs, transcriptions_route), + ); endpoint_routes.insert(EndpointType::Realtime, (realtime_docs, realtime_route)); endpoint_routes.insert(EndpointType::Responses, (responses_docs, responses_route)); diff --git a/lib/llm/src/http/service/transcriptions.rs b/lib/llm/src/http/service/transcriptions.rs new file mode 100644 index 000000000000..e7d3ce712907 --- /dev/null +++ b/lib/llm/src/http/service/transcriptions.rs @@ -0,0 +1,270 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::sync::Arc; + +use axum::{ + Json, Router, + extract::{Multipart, State, multipart::MultipartRejection}, + http::{HeaderMap, StatusCode}, + response::{IntoResponse, Response}, + routing::post, +}; +use base64::Engine as _; +use futures::StreamExt; + +use super::{ + RouteDoc, + metrics::{Endpoint, ErrorType, process_response_and_observe_metrics, request_was_rejected}, + openai::{ + ErrorMessage, ErrorResponse, check_model_serving_ready, check_ready, context_from_headers, + error_is_invalid_argument, extract_error_type_from_response, get_body_limit, + get_or_create_request_id, + }, + service_v2, +}; +use crate::protocols::openai::transcriptions::{ + NvAudioTranscriptionResponse, NvCreateAudioTranscriptionRequest, +}; + +#[derive(Default)] +struct TranscriptionForm { + audio_b64: Option, + filename: Option, + model: Option, + language: Option, + prompt: Option, + response_format: Option, + temperature: Option, + timestamp_granularities: Vec, +} + +impl TranscriptionForm { + async fn parse(mut multipart: Multipart) -> Result { + let mut form = Self::default(); + while let Some(field) = multipart.next_field().await.map_err(|error| { + ErrorMessage::bad_request(format!("Invalid multipart body: {error}")) + })? { + let name = field.name().unwrap_or_default().to_string(); + if name == "file" { + form.filename = Some(field.file_name().unwrap_or("audio").to_string()); + let bytes = field.bytes().await.map_err(|error| { + ErrorMessage::bad_request(format!("Failed to read audio file: {error}")) + })?; + if bytes.is_empty() { + return Err(ErrorMessage::bad_request( + "The audio file must not be empty", + )); + } + form.audio_b64 = Some(base64::engine::general_purpose::STANDARD.encode(bytes)); + continue; + } + + let value = field.text().await.map_err(|error| { + ErrorMessage::bad_request(format!("Failed to read form field `{name}`: {error}")) + })?; + match name.as_str() { + "model" => form.model = nonempty(value), + "language" => form.language = nonempty(value), + "prompt" => form.prompt = Some(value), + "response_format" => form.response_format = nonempty(value), + "temperature" => form.temperature = Some(parse_field(&name, &value)?), + "timestamp_granularities" | "timestamp_granularities[]" => { + if !value.is_empty() { + form.timestamp_granularities.push(value); + } + } + "stream" if value.eq_ignore_ascii_case("true") || value == "1" => { + return Err(ErrorMessage::bad_request( + "Streaming transcriptions are not supported yet", + )); + } + _ => {} + } + } + Ok(form) + } + + fn into_request(self) -> Result { + let audio_b64 = self + .audio_b64 + .ok_or_else(|| ErrorMessage::bad_request("Missing required `file` field"))?; + let response_format = self.response_format.as_deref().unwrap_or("json"); + if !matches!(response_format, "json" | "verbose_json") { + return Err(ErrorMessage::bad_request(format!( + "Unsupported response_format `{response_format}`; expected `json` or `verbose_json`" + ))); + } + if self + .timestamp_granularities + .iter() + .any(|value| !matches!(value.as_str(), "word" | "segment")) + { + return Err(ErrorMessage::bad_request( + "timestamp_granularities must contain only `word` or `segment`", + )); + } + + Ok(NvCreateAudioTranscriptionRequest { + audio_b64, + filename: self.filename.unwrap_or_else(|| "audio".to_string()), + model: self.model, + language: self.language, + prompt: self.prompt, + response_format: Some(response_format.to_string()), + temperature: self.temperature, + timestamp_granularities: (!self.timestamp_granularities.is_empty()) + .then_some(self.timestamp_granularities), + }) + } +} + +fn nonempty(value: String) -> Option { + (!value.is_empty()).then_some(value) +} + +fn parse_field(name: &str, value: &str) -> Result +where + T: std::str::FromStr, + T::Err: std::fmt::Display, +{ + value.parse().map_err(|error| { + ErrorMessage::bad_request(format!("Invalid `{name}` value `{value}`: {error}")) + }) +} + +async fn audio_transcription( + State(state): State>, + headers: HeaderMap, + multipart: Result, +) -> Result { + check_ready(&state)?; + let multipart = multipart + .map_err(|error| ErrorMessage::bad_request(format!("Invalid multipart body: {error}")))?; + let mut request = TranscriptionForm::parse(multipart).await?.into_request()?; + + let model = match request.model.clone() { + Some(model) => model, + None => state + .manager() + .list_transcriptions_models() + .into_iter() + .find(|model| state.manager().is_model_ready_to_serve(model)) + .unwrap_or_default(), + }; + request.model = Some(model.clone()); + check_model_serving_ready(&state, &model)?; + + let metric_model = state.manager().metric_model_for(&model).to_string(); + let request_id = get_or_create_request_id(&headers); + let request = context_from_headers(request, request_id, &headers)?; + let request_id = request.id().to_string(); + let mut inflight = state.metrics_clone().create_inflight_guard( + &metric_model, + Endpoint::Transcriptions, + false, + &request_id, + ); + let http_queue_guard = state.metrics_clone().create_http_queue_guard(&metric_model); + let engine = state + .manager() + .get_transcriptions_engine(&model) + .map_err(|error| { + let response = ErrorMessage::from_model_error(&error); + inflight.mark_error(extract_error_type_from_response(&response)); + response + })?; + let mut response_collector = state + .metrics_clone() + .create_response_collector(&metric_model); + + let stream = engine.generate(request).await.map_err(|error| { + let is_validation = error_is_invalid_argument(error.as_ref()); + if request_was_rejected(error.as_ref()) { + state + .metrics_clone() + .inc_rejection(&metric_model, Endpoint::Transcriptions); + } + let response = ErrorMessage::from_anyhow(error, "Failed to transcribe audio"); + inflight.mark_error(if is_validation { + ErrorType::Validation + } else { + extract_error_type_from_response(&response) + }); + response + })?; + let mut http_queue_guard = Some(http_queue_guard); + let stream = stream.inspect(move |response| { + process_response_and_observe_metrics( + response, + &mut response_collector, + &mut http_queue_guard, + ); + }); + let response = NvAudioTranscriptionResponse::from_annotated_stream(stream) + .await + .map_err(|error| { + tracing::error!(request_id, %error, "Failed to fold transcription response"); + let is_validation = error_is_invalid_argument(&error); + let response = ErrorMessage::from_anyhow( + anyhow::Error::new(error), + "Failed to fold transcription response", + ); + inflight.mark_error(if is_validation { + ErrorType::Validation + } else { + extract_error_type_from_response(&response) + }); + response + })?; + + inflight.mark_ok(); + Ok((StatusCode::OK, Json(response)).into_response()) +} + +pub fn transcriptions_router( + state: Arc, + path: Option, +) -> (Vec, Router) { + let path = path.unwrap_or_else(|| "/v1/audio/transcriptions".to_string()); + let doc = RouteDoc::new(axum::http::Method::POST, &path); + let router = Router::new() + .route(&path, post(audio_transcription)) + .layer(axum::extract::DefaultBodyLimit::max(get_body_limit())) + .with_state(state); + (vec![doc], router) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rejects_non_json_response_formats() { + let form = TranscriptionForm { + audio_b64: Some("AQ==".to_string()), + response_format: Some("srt".to_string()), + ..Default::default() + }; + assert_eq!(form.into_request().unwrap_err().0, StatusCode::BAD_REQUEST); + } + + #[test] + fn builds_transport_neutral_request() { + let form = TranscriptionForm { + audio_b64: Some("YXVkaW8=".to_string()), + filename: Some("sample.wav".to_string()), + language: Some("en".to_string()), + timestamp_granularities: vec!["word".to_string()], + ..Default::default() + }; + let request = form.into_request().unwrap(); + assert_eq!(request.filename, "sample.wav"); + assert_eq!(request.audio_b64, "YXVkaW8="); + assert_eq!(request.response_format.as_deref(), Some("json")); + assert_eq!( + request.timestamp_granularities, + Some(vec!["word".to_string()]) + ); + } +} diff --git a/lib/llm/src/model_type.rs b/lib/llm/src/model_type.rs index 5d730158c77d..4ff2cd02388f 100644 --- a/lib/llm/src/model_type.rs +++ b/lib/llm/src/model_type.rs @@ -64,6 +64,9 @@ bitflags! { const Pooling = 1 << 10; /// Cross-encoder relevance scoring served on `/v1/rerank`. const Rerank = 1 << 11; + /// Batch speech-to-text through the OpenAI `/v1/audio/transcriptions` + /// endpoint. + const Transcriptions = 1 << 12; } } @@ -103,6 +106,9 @@ impl ModelType { pub fn supports_realtime(&self) -> bool { self.contains(ModelType::Realtime) } + pub fn supports_transcriptions(&self) -> bool { + self.contains(ModelType::Transcriptions) + } pub fn supports_classify(&self) -> bool { self.contains(ModelType::Classify) } @@ -136,6 +142,9 @@ impl ModelType { if self.supports_audios() { result.push("audios"); } + if self.supports_transcriptions() { + result.push("transcriptions"); + } if self.supports_videos() { result.push("videos"); } @@ -179,6 +188,9 @@ impl ModelType { if self.supports_audios() { result.push(ModelType::Audios); } + if self.supports_transcriptions() { + result.push(ModelType::Transcriptions); + } if self.supports_videos() { result.push(ModelType::Videos); } @@ -232,6 +244,9 @@ impl ModelType { if self.contains(Self::Audios) { endpoint_types.push(crate::endpoint_type::EndpointType::Audios); } + if self.contains(Self::Transcriptions) { + endpoint_types.push(crate::endpoint_type::EndpointType::Transcriptions); + } if self.contains(Self::Videos) { endpoint_types.push(crate::endpoint_type::EndpointType::Videos); } @@ -391,6 +406,16 @@ mod tests { ); } + #[test] + fn transcriptions_capability_and_endpoint_mapping() { + assert_eq!(ModelType::Transcriptions.bits(), 1 << 12); + assert!(ModelType::Transcriptions.supports_transcriptions()); + assert_eq!( + ModelType::Transcriptions.as_endpoint_types(), + vec![EndpointType::Transcriptions] + ); + } + #[test] fn realtime_combines_with_other_endpoints() { let endpoints = (ModelType::Chat | ModelType::Realtime).as_endpoint_types(); diff --git a/lib/llm/src/protocols/openai.rs b/lib/llm/src/protocols/openai.rs index 60ec42056d93..aa69e65ceea3 100644 --- a/lib/llm/src/protocols/openai.rs +++ b/lib/llm/src/protocols/openai.rs @@ -31,6 +31,7 @@ pub mod responses; pub mod stream_aggregator; mod strict_schema; pub mod tools; +pub mod transcriptions; pub mod validate; pub mod videos; diff --git a/lib/llm/src/protocols/openai/stream_aggregator.rs b/lib/llm/src/protocols/openai/stream_aggregator.rs index 1bf2f18ecea6..75f3633d4ecb 100644 --- a/lib/llm/src/protocols/openai/stream_aggregator.rs +++ b/lib/llm/src/protocols/openai/stream_aggregator.rs @@ -1,10 +1,10 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +use dynamo_runtime::error::DynamoError; use futures::{Stream, StreamExt}; use crate::types::Annotated; -use dynamo_runtime::error::DynamoError; /// Response types whose `Annotated` streams can be folded into a single `T` /// using shared aggregation infrastructure. @@ -39,6 +39,31 @@ where Ok(response.unwrap_or_else(T::empty)) } +/// Collect exactly one data item while preserving typed backend errors. +/// +/// Unary APIs use Dynamo's streaming request plane internally even though +/// their HTTP response is singular. Empty and multi-item streams violate that +/// worker contract and are rejected. +pub async fn collect_unary_stream(stream: S) -> Result +where + S: Stream>, +{ + let mut stream = std::pin::pin!(stream); + let mut response = None; + + while let Some(item) = stream.next().await { + if let Some(data) = item.into_data()? { + if response.replace(data).is_some() { + return Err(DynamoError::msg( + "unary response stream produced more than one data item", + )); + } + } + } + + response.ok_or_else(|| DynamoError::msg("unary response stream produced no data")) +} + #[cfg(test)] mod tests { use super::*; @@ -94,4 +119,45 @@ mod tests { ); assert_eq!(error.message(), "invalid argument"); } + + #[tokio::test] + async fn unary_collector_preserves_backend_error_type() { + let stream = stream::iter(vec![Annotated::::from_err( + DynamoError::builder() + .error_type(ErrorType::Backend(BackendError::InvalidArgument)) + .message("invalid language") + .build(), + )]); + + let error = collect_unary_stream(stream).await.unwrap_err(); + + assert_eq!( + error.error_type(), + ErrorType::Backend(BackendError::InvalidArgument) + ); + } + + #[tokio::test] + async fn unary_collector_rejects_empty_and_multi_item_streams() { + let empty = stream::empty::>(); + assert!( + collect_unary_stream(empty) + .await + .unwrap_err() + .message() + .contains("produced no data") + ); + + let multiple = stream::iter(vec![ + Annotated::from_data(1_usize), + Annotated::from_data(2_usize), + ]); + assert!( + collect_unary_stream(multiple) + .await + .unwrap_err() + .message() + .contains("more than one") + ); + } } diff --git a/lib/llm/src/protocols/openai/transcriptions.rs b/lib/llm/src/protocols/openai/transcriptions.rs new file mode 100644 index 000000000000..5315aa805aba --- /dev/null +++ b/lib/llm/src/protocols/openai/transcriptions.rs @@ -0,0 +1,128 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use dynamo_runtime::protocols::annotated::AnnotationsProvider; +use serde::{Deserialize, Serialize}; +use validator::Validate; + +mod aggregator; + +/// Transport-neutral form of an OpenAI audio transcription request. +/// +/// The HTTP frontend converts the multipart upload to base64 so the request can +/// cross any Dynamo request plane without relying on HTTP-specific types. +#[derive(Serialize, Deserialize, Validate, Debug, Clone)] +pub struct NvCreateAudioTranscriptionRequest { + /// Base64-encoded contents of the uploaded audio file. + pub audio_b64: String, + + /// Original upload filename, used by decoders for format detection. + pub filename: String, + + #[serde(skip_serializing_if = "Option::is_none")] + pub model: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub language: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub prompt: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub response_format: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub temperature: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub timestamp_granularities: Option>, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] +pub struct NvAudioTranscriptionUsage { + #[serde(rename = "type")] + pub usage_type: String, + pub seconds: f64, +} + +/// JSON response for the OpenAI audio transcription endpoint. +#[derive(Serialize, Deserialize, Validate, Debug, Clone, PartialEq)] +pub struct NvAudioTranscriptionResponse { + pub text: String, + + #[serde(skip_serializing_if = "Option::is_none")] + pub usage: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub language: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub duration: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub segments: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub words: Option, +} + +impl NvAudioTranscriptionResponse { + pub fn empty() -> Self { + Self { + text: String::new(), + usage: None, + language: None, + duration: None, + segments: None, + words: None, + } + } +} + +impl AnnotationsProvider for NvCreateAudioTranscriptionRequest { + fn annotations(&self) -> Option> { + None + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn request_round_trips_openai_fields() { + let request: NvCreateAudioTranscriptionRequest = + serde_json::from_value(serde_json::json!({ + "audio_b64": "UklGRg==", + "filename": "sample.wav", + "model": "openai/whisper-tiny", + "language": "en", + "response_format": "verbose_json", + "timestamp_granularities": ["word"] + })) + .unwrap(); + + assert_eq!(request.filename, "sample.wav"); + assert_eq!(request.language.as_deref(), Some("en")); + assert_eq!( + request.timestamp_granularities, + Some(vec!["word".to_string()]) + ); + } + + #[test] + fn response_uses_openai_usage_field_name() { + let response = NvAudioTranscriptionResponse { + text: "hello".to_string(), + usage: Some(NvAudioTranscriptionUsage { + usage_type: "duration".to_string(), + seconds: 1.25, + }), + ..NvAudioTranscriptionResponse::empty() + }; + + let value = serde_json::to_value(response).unwrap(); + assert_eq!(value["usage"]["type"], "duration"); + assert_eq!(value["usage"]["seconds"], 1.25); + } +} diff --git a/lib/llm/src/protocols/openai/transcriptions/aggregator.rs b/lib/llm/src/protocols/openai/transcriptions/aggregator.rs new file mode 100644 index 000000000000..9f4dbc354392 --- /dev/null +++ b/lib/llm/src/protocols/openai/transcriptions/aggregator.rs @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use dynamo_runtime::error::DynamoError; +use futures::Stream; + +use crate::protocols::openai::stream_aggregator::collect_unary_stream; +use crate::types::Annotated; + +use super::NvAudioTranscriptionResponse; + +impl NvAudioTranscriptionResponse { + pub async fn from_annotated_stream( + stream: impl Stream>, + ) -> Result { + collect_unary_stream(stream).await + } +} diff --git a/lib/llm/src/types.rs b/lib/llm/src/types.rs index 1d51c0321c9f..9926bd0b20f7 100644 --- a/lib/llm/src/types.rs +++ b/lib/llm/src/types.rs @@ -168,6 +168,22 @@ pub mod openai { pub type OpenAIAudiosStreamingEngine = ServerStreamingEngine>; } + + pub mod transcriptions { + use super::*; + + pub use protocols::openai::transcriptions::{ + NvAudioTranscriptionResponse, NvCreateAudioTranscriptionRequest, + }; + + pub type OpenAITranscriptionsUnaryEngine = + UnaryEngine; + + pub type OpenAITranscriptionsStreamingEngine = ServerStreamingEngine< + NvCreateAudioTranscriptionRequest, + Annotated, + >; + } } pub mod generic { diff --git a/lib/llm/tests/http-service.rs b/lib/llm/tests/http-service.rs index 4e843736cfc2..0bffe381884c 100644 --- a/lib/llm/tests/http-service.rs +++ b/lib/llm/tests/http-service.rs @@ -622,6 +622,7 @@ fn compute_index(endpoint: &Endpoint, request_type: &RequestType, status: &Statu Endpoint::Images => todo!(), Endpoint::Videos => todo!(), Endpoint::Audios => todo!(), + Endpoint::Transcriptions => todo!(), Endpoint::Generate => todo!(), }; diff --git a/lib/llm/tests/http-transcriptions.rs b/lib/llm/tests/http-transcriptions.rs new file mode 100644 index 000000000000..c71b289f7c4e --- /dev/null +++ b/lib/llm/tests/http-transcriptions.rs @@ -0,0 +1,184 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::sync::Arc; +use std::time::Duration; + +use anyhow::Error; +use base64::Engine as _; +use dynamo_llm::endpoint_type::EndpointType; +use dynamo_llm::http::service::service_v2::HttpService; +use dynamo_llm::protocols::Annotated; +use dynamo_llm::protocols::openai::transcriptions::{ + NvAudioTranscriptionResponse, NvCreateAudioTranscriptionRequest, +}; +use dynamo_runtime::CancellationToken; +use dynamo_runtime::error::{BackendError, DynamoError, ErrorType}; +use dynamo_runtime::pipeline::{ + AsyncEngine, AsyncEngineContextProvider, ManyOut, ResponseStream, SingleIn, async_trait, +}; +use reqwest::StatusCode; + +const MODEL: &str = "test-whisper"; + +struct TestTranscriptionEngine { + fail: bool, +} + +#[async_trait] +impl + AsyncEngine< + SingleIn, + ManyOut>, + Error, + > for TestTranscriptionEngine +{ + async fn generate( + &self, + request: SingleIn, + ) -> Result>, Error> { + let (request, context) = request.transfer(()); + let response = if self.fail { + let error = DynamoError::builder() + .error_type(ErrorType::Backend(BackendError::InvalidArgument)) + .message("unsupported transcription language") + .build(); + Annotated { + data: None, + id: None, + event: Some("error".to_string()), + comment: None, + error: Some(error), + } + } else { + let audio = base64::engine::general_purpose::STANDARD + .decode(&request.audio_b64) + .expect("frontend should produce valid base64"); + Annotated::from_data(NvAudioTranscriptionResponse { + text: format!("{}:{}", request.filename, audio.len()), + usage: None, + language: request.language, + duration: None, + segments: None, + words: None, + }) + }; + let stream = futures::stream::iter([response]); + + Ok(ResponseStream::new(Box::pin(stream), context.context())) + } +} + +struct TestService { + base_url: String, + client: reqwest::Client, + cancel: CancellationToken, + join: tokio::task::JoinHandle>, +} + +impl TestService { + async fn start(fail: bool) -> Self { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("failed to bind test listener"); + let port = listener + .local_addr() + .expect("test listener has no local address") + .port(); + let service = HttpService::builder() + .host("127.0.0.1") + .port(port) + .build() + .expect("failed to build HTTP service"); + service + .enable_model_endpoint(EndpointType::Transcriptions, true) + .expect("failed to enable transcription endpoint"); + service + .model_manager() + .add_transcriptions_model(MODEL, "0", Arc::new(TestTranscriptionEngine { fail })) + .expect("failed to register transcription model"); + + let cancel = CancellationToken::new(); + let join = service.spawn_with_listener(cancel.clone(), listener).await; + let client = reqwest::Client::builder() + .no_proxy() + .build() + .expect("failed to build HTTP client"); + let base_url = format!("http://127.0.0.1:{port}"); + + tokio::time::timeout(Duration::from_secs(5), async { + loop { + if client + .get(format!("{base_url}/health")) + .send() + .await + .is_ok_and(|response| response.status().is_success()) + { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("HTTP service did not become healthy"); + + Self { + base_url, + client, + cancel, + join, + } + } + + async fn shutdown(self) { + self.cancel.cancel(); + tokio::time::timeout(Duration::from_secs(2), self.join) + .await + .expect("HTTP service did not stop") + .expect("HTTP service task panicked") + .expect("HTTP service returned an error"); + } + + async fn transcribe(&self) -> reqwest::Response { + let form = reqwest::multipart::Form::new() + .part( + "file", + reqwest::multipart::Part::bytes(b"audio".to_vec()) + .file_name("sample.wav") + .mime_str("audio/wav") + .unwrap(), + ) + .text("language", "en") + .text("response_format", "json"); + + self.client + .post(format!("{}/v1/audio/transcriptions", self.base_url)) + .multipart(form) + .send() + .await + .expect("transcription request failed") + } +} + +#[tokio::test] +async fn multipart_request_uses_default_model_and_returns_json() { + let service = TestService::start(false).await; + + let response = service.transcribe().await; + + assert_eq!(response.status(), StatusCode::OK); + let response: NvAudioTranscriptionResponse = response.json().await.unwrap(); + assert_eq!(response.text, "sample.wav:5"); + assert_eq!(response.language.as_deref(), Some("en")); + service.shutdown().await; +} + +#[tokio::test] +async fn annotated_invalid_argument_returns_bad_request() { + let service = TestService::start(true).await; + + let response = service.transcribe().await; + + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + service.shutdown().await; +} From 52277698a9639f0642fad9000e8c06de0a50680f Mon Sep 17 00:00:00 2001 From: Piotr Tarasiewicz Date: Thu, 20 Aug 2026 13:18:45 +0200 Subject: [PATCH 02/18] chore: refresh transcription generated artifacts Signed-off-by: Piotr Tarasiewicz Ported onto main 9cc8c3650487 by Inworld; conflicts resolved in: none (lib/bindings/kvbm/Cargo.lock regenerated with cargo rather than cherry-picked) Signed-off-by: Nurullah Morshed Co-Authored-By: Claude Opus 5.5 --- lib/bindings/kvbm/Cargo.lock | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/lib/bindings/kvbm/Cargo.lock b/lib/bindings/kvbm/Cargo.lock index a5e153cdfa4e..e3e3d512a83e 100644 --- a/lib/bindings/kvbm/Cargo.lock +++ b/lib/bindings/kvbm/Cargo.lock @@ -840,6 +840,7 @@ dependencies = [ "matchit", "memchr", "mime", + "multer", "percent-encoding", "pin-project-lite", "rustversion", @@ -4804,6 +4805,23 @@ dependencies = [ "pxfm", ] +[[package]] +name = "multer" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83e87776546dc87511aa5ee218730c92b666d7264ab6ed41f9d215af9cd5224b" +dependencies = [ + "bytes", + "encoding_rs", + "futures-util", + "http 1.4.1", + "httparse", + "memchr", + "mime", + "spin 0.9.8", + "version_check", +] + [[package]] name = "multimap" version = "0.10.1" From 02270eef131060f3ccf9dd3d20c9519eb9fe873f Mon Sep 17 00:00:00 2001 From: Piotr Tarasiewicz Date: Thu, 20 Aug 2026 13:37:21 +0200 Subject: [PATCH 03/18] fix(vllm): address transcription review feedback Signed-off-by: Piotr Tarasiewicz Ported onto main 9cc8c3650487 by Inworld; conflicts resolved in: components/src/dynamo/vllm/tests/test_vllm_worker_factory.py Signed-off-by: Nurullah Morshed Co-Authored-By: Claude Opus 5.5 --- .../vllm/tests/test_vllm_transcription.py | 2 + .../vllm/tests/test_vllm_worker_factory.py | 55 +++++++++++++++++++ components/src/dynamo/vllm/worker_factory.py | 28 ++++++++-- .../backends/vllm/launch/agg_transcription.sh | 4 ++ lib/llm/src/http/service/transcriptions.rs | 37 ++++++++++++- 5 files changed, 119 insertions(+), 7 deletions(-) diff --git a/components/src/dynamo/vllm/tests/test_vllm_transcription.py b/components/src/dynamo/vllm/tests/test_vllm_transcription.py index d4d8c43d7018..c5a0925b3926 100644 --- a/components/src/dynamo/vllm/tests/test_vllm_transcription.py +++ b/components/src/dynamo/vllm/tests/test_vllm_transcription.py @@ -85,6 +85,7 @@ def model_dump(self, **_kwargs): handler._run_with_cancellation.assert_awaited_once() +@pytest.mark.timeout(5) def test_outer_cancellation_stops_native_vllm_inference(): asyncio.run(_assert_outer_cancellation_stops_native_vllm_inference()) @@ -118,6 +119,7 @@ async def wait_for_context_cancellation(): await asyncio.wait_for(inference_cancelled.wait(), timeout=1) +@pytest.mark.timeout(5) def test_context_cancellation_stops_native_vllm_inference(): asyncio.run(_assert_context_cancellation_stops_native_vllm_inference()) diff --git a/components/src/dynamo/vllm/tests/test_vllm_worker_factory.py b/components/src/dynamo/vllm/tests/test_vllm_worker_factory.py index 18089d726bd1..5827d236d18b 100644 --- a/components/src/dynamo/vllm/tests/test_vllm_worker_factory.py +++ b/components/src/dynamo/vllm/tests/test_vllm_worker_factory.py @@ -4864,3 +4864,58 @@ async def collective_rpc(method, timeout=None): assert sent == rpcs assert merged["engine"]["resolution"] == resolution + + +@pytest.mark.asyncio +async def test_transcription_worker_reuses_snapshot_engine( + monkeypatch: pytest.MonkeyPatch, +) -> None: + endpoint = Mock() + endpoint.connection_id.return_value = "worker-1" + endpoint.serve_endpoint = AsyncMock() + runtime = Mock() + runtime.endpoint.return_value = endpoint + + engine_client = Mock() + vllm_config = Mock() + snapshot_engine: EngineSetupResult = ( + engine_client, + vllm_config, + Mock(), + None, + Mock(), + ) + setup_vllm_engine = Mock() + register_vllm_model = AsyncMock() + factory = _make_factory( + setup_vllm_engine_fn=setup_vllm_engine, + register_vllm_model_fn=register_vllm_model, + ) + handler = Mock() + monkeypatch.setattr( + "dynamo.vllm.worker_factory.TranscriptionWorkerHandler", + Mock(return_value=handler), + ) + monkeypatch.setattr("dynamo.vllm.worker_factory.register_model_taint_route", Mock()) + monkeypatch.setenv("DYN_FPM_WORKER_ID", "snapshot") + config = SimpleNamespace( + namespace="ns", + component="worker", + endpoint="generate", + served_model_name="whisper", + model="openai/whisper-tiny", + ) + + await factory._create_transcription_worker( + runtime, + config, + asyncio.Event(), + [], + snapshot_engine=snapshot_engine, + ) + + setup_vllm_engine.assert_not_called() + assert register_vllm_model.await_args.args[4] is engine_client + assert register_vllm_model.await_args.args[5] is vllm_config + endpoint.serve_endpoint.assert_awaited_once() + handler.cleanup.assert_called_once() diff --git a/components/src/dynamo/vllm/worker_factory.py b/components/src/dynamo/vllm/worker_factory.py index 45776ae312c0..4c3cb106c0ad 100644 --- a/components/src/dynamo/vllm/worker_factory.py +++ b/components/src/dynamo/vllm/worker_factory.py @@ -1810,7 +1810,11 @@ async def create( if config.transcription_worker: await self._create_transcription_worker( - runtime, config, shutdown_event, shutdown_endpoints + runtime, + config, + shutdown_event, + shutdown_endpoints, + snapshot_engine=snapshot_engine, ) return @@ -2031,17 +2035,29 @@ async def _create_transcription_worker( config: Config, shutdown_event: asyncio.Event, shutdown_endpoints: list, + snapshot_engine: Optional[EngineSetupResult] = None, ) -> None: generate_endpoint = runtime.endpoint( f"{config.namespace}.{config.component}.{config.endpoint}" ) shutdown_endpoints[:] = [generate_endpoint] - engine_client, vllm_config, _, _, _ = self.setup_vllm_engine( - config, - None, - fpm_worker_id=str(generate_endpoint.connection_id()), - ) + fpm_worker_id = str(generate_endpoint.connection_id()) + if snapshot_engine is not None: + ( + engine_client, + vllm_config, + _default_sampling_params, + _prometheus_temp_dir, + _component_gauges, + ) = snapshot_engine + os.environ[ENV_FPM_WORKER_ID] = fpm_worker_id + else: + engine_client, vllm_config, _, _, _ = self.setup_vllm_engine( + config, + None, + fpm_worker_id=fpm_worker_id, + ) handler = TranscriptionWorkerHandler( runtime=runtime, engine=engine_client, diff --git a/examples/backends/vllm/launch/agg_transcription.sh b/examples/backends/vllm/launch/agg_transcription.sh index 890d6244cad1..31c48a4e1175 100755 --- a/examples/backends/vllm/launch/agg_transcription.sh +++ b/examples/backends/vllm/launch/agg_transcription.sh @@ -17,6 +17,10 @@ EXTRA_ARGS=() while [[ $# -gt 0 ]]; do case $1 in --model) + if [[ $# -lt 2 || -z "$2" ]]; then + echo "Error: --model requires a model name." >&2 + exit 2 + fi MODEL="$2" shift 2 ;; diff --git a/lib/llm/src/http/service/transcriptions.rs b/lib/llm/src/http/service/transcriptions.rs index e7d3ce712907..b0f448f04b89 100644 --- a/lib/llm/src/http/service/transcriptions.rs +++ b/lib/llm/src/http/service/transcriptions.rs @@ -95,6 +95,13 @@ impl TranscriptionForm { "Unsupported response_format `{response_format}`; expected `json` or `verbose_json`" ))); } + if self.temperature.is_some_and(|temperature| { + !temperature.is_finite() || !(0.0..=1.0).contains(&temperature) + }) { + return Err(ErrorMessage::bad_request( + "temperature must be a finite number between 0 and 1", + )); + } if self .timestamp_granularities .iter() @@ -104,6 +111,11 @@ impl TranscriptionForm { "timestamp_granularities must contain only `word` or `segment`", )); } + if !self.timestamp_granularities.is_empty() && response_format != "verbose_json" { + return Err(ErrorMessage::bad_request( + "timestamp_granularities requires response_format `verbose_json`", + )); + } Ok(NvCreateAudioTranscriptionRequest { audio_b64, @@ -255,16 +267,39 @@ mod tests { audio_b64: Some("YXVkaW8=".to_string()), filename: Some("sample.wav".to_string()), language: Some("en".to_string()), + response_format: Some("verbose_json".to_string()), timestamp_granularities: vec!["word".to_string()], ..Default::default() }; let request = form.into_request().unwrap(); assert_eq!(request.filename, "sample.wav"); assert_eq!(request.audio_b64, "YXVkaW8="); - assert_eq!(request.response_format.as_deref(), Some("json")); + assert_eq!(request.response_format.as_deref(), Some("verbose_json")); assert_eq!( request.timestamp_granularities, Some(vec!["word".to_string()]) ); } + + #[test] + fn rejects_invalid_temperatures() { + for temperature in [-0.1, 1.1, f64::NAN, f64::INFINITY, f64::NEG_INFINITY] { + let form = TranscriptionForm { + audio_b64: Some("AQ==".to_string()), + temperature: Some(temperature), + ..Default::default() + }; + assert_eq!(form.into_request().unwrap_err().0, StatusCode::BAD_REQUEST); + } + } + + #[test] + fn rejects_timestamp_granularities_without_verbose_json() { + let form = TranscriptionForm { + audio_b64: Some("AQ==".to_string()), + timestamp_granularities: vec!["word".to_string()], + ..Default::default() + }; + assert_eq!(form.into_request().unwrap_err().0, StatusCode::BAD_REQUEST); + } } From 64484fb9cef41cfee0ce97157c35bcd4509add98 Mon Sep 17 00:00:00 2001 From: Piotr Tarasiewicz Date: Thu, 20 Aug 2026 13:41:01 +0200 Subject: [PATCH 04/18] chore: refresh runtime examples lockfile Signed-off-by: Piotr Tarasiewicz Ported onto main 9cc8c3650487 by Inworld; conflicts resolved in: none (lib/runtime/examples/Cargo.lock regenerated with cargo rather than cherry-picked; the newer encoding_rs on main also pulls simdutf8, core_detect and multiversion_no_op) Signed-off-by: Nurullah Morshed Co-Authored-By: Claude Opus 5.5 --- lib/runtime/examples/Cargo.lock | 56 +++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/lib/runtime/examples/Cargo.lock b/lib/runtime/examples/Cargo.lock index b101d7bc7630..3d237446cd19 100644 --- a/lib/runtime/examples/Cargo.lock +++ b/lib/runtime/examples/Cargo.lock @@ -211,6 +211,7 @@ dependencies = [ "matchit", "memchr", "mime", + "multer", "percent-encoding", "pin-project-lite", "rustversion", @@ -454,6 +455,12 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "core_detect" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -910,6 +917,20 @@ dependencies = [ "serde", ] +[[package]] +name = "encoding_rs" +version = "0.8.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e985e0451871ad22fb8d2b6b076e2028a502a0d3950998c2c5c0a4f9b5d9679" +dependencies = [ + "cfg-if", + "core_detect", + "multiversion_no_op", + "rustversion", + "scopeguard", + "simdutf8", +] + [[package]] name = "enum-ordinalize" version = "4.3.2" @@ -2026,12 +2047,35 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "multer" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83e87776546dc87511aa5ee218730c92b666d7264ab6ed41f9d215af9cd5224b" +dependencies = [ + "bytes", + "encoding_rs", + "futures-util", + "http", + "httparse", + "memchr", + "mime", + "spin", + "version_check", +] + [[package]] name = "multimap" version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" +[[package]] +name = "multiversion_no_op" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d" + [[package]] name = "neli" version = "0.7.4" @@ -3360,6 +3404,12 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + [[package]] name = "slab" version = "0.4.12" @@ -3392,6 +3442,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "spin" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" + [[package]] name = "spki" version = "0.7.3" From 769dc1c955d3544da7d9e7c9df375770a88f6b91 Mon Sep 17 00:00:00 2001 From: Piotr Tarasiewicz Date: Thu, 20 Aug 2026 15:34:18 +0200 Subject: [PATCH 05/18] fix(vllm): harden audio transcription serving Signed-off-by: Piotr Tarasiewicz Ported onto main 9cc8c3650487 by Inworld; conflicts resolved in: none Signed-off-by: Nurullah Morshed Co-Authored-By: Claude Opus 5.5 --- components/src/dynamo/vllm/args.py | 8 ++ .../vllm/tests/test_vllm_transcription.py | 66 +++++++++++++ .../src/dynamo/vllm/tests/test_vllm_unit.py | 18 ++++ .../src/dynamo/vllm/transcription_handler.py | 6 +- lib/llm/src/http/service/transcriptions.rs | 45 ++++++++- lib/llm/tests/http-transcriptions.rs | 92 ++++++++++++++++++- 6 files changed, 229 insertions(+), 6 deletions(-) diff --git a/components/src/dynamo/vllm/args.py b/components/src/dynamo/vllm/args.py index c4305bf70b01..5ee9081c1350 100644 --- a/components/src/dynamo/vllm/args.py +++ b/components/src/dynamo/vllm/args.py @@ -370,6 +370,14 @@ def update_engine_config_with_dynamo( "Pooling-family worker: defaulting --enable-prefix-caching to False" ) engine_config.enable_prefix_caching = False + elif dynamo_config.transcription_worker: + # Whisper is an encoder-decoder model. vLLM does not support + # prefix caching for cross-attention KV caches, and enabling it + # causes the first transcription request to terminate EngineCore. + logger.debug( + "Transcription worker: defaulting --enable-prefix-caching to False" + ) + engine_config.enable_prefix_caching = False else: logger.debug( "--enable-prefix-caching or --no-enable-prefix-caching not specified. " diff --git a/components/src/dynamo/vllm/tests/test_vllm_transcription.py b/components/src/dynamo/vllm/tests/test_vllm_transcription.py index c5a0925b3926..bde4468fb8d1 100644 --- a/components/src/dynamo/vllm/tests/test_vllm_transcription.py +++ b/components/src/dynamo/vllm/tests/test_vllm_transcription.py @@ -7,7 +7,10 @@ from unittest.mock import AsyncMock import pytest +from vllm.entrypoints.openai.engine.protocol import ErrorInfo, ErrorResponse +from dynamo.llm import HttpError +from dynamo.vllm import transcription_handler from dynamo.vllm.transcription_handler import TranscriptionWorkerHandler pytestmark = [ @@ -24,10 +27,42 @@ def make_handler() -> TranscriptionWorkerHandler: handler.config = SimpleNamespace( model="openai/whisper-tiny", served_model_name="whisper", + served_model_aliases=[], ) return handler +def test_handler_registers_served_model_aliases(monkeypatch): + registered = {} + + class FakeModels: + def __init__(self, _engine, base_model_paths): + registered["base_model_paths"] = base_model_paths + + monkeypatch.setattr(transcription_handler, "OpenAIServingModels", FakeModels) + monkeypatch.setattr( + transcription_handler, + "OpenAIServingTranscription", + lambda *_args, **_kwargs: object(), + ) + monkeypatch.setattr( + transcription_handler, + "VllmEngineMonitor", + lambda *_args, **_kwargs: object(), + ) + + config = SimpleNamespace( + model="openai/whisper-tiny", + served_model_name="whisper", + served_model_aliases=["whisper-alias", "stt"], + ) + TranscriptionWorkerHandler(runtime=object(), engine=object(), config=config) + + paths = registered["base_model_paths"] + assert [path.name for path in paths] == ["whisper", "whisper-alias", "stt"] + assert {path.model_path for path in paths} == {"openai/whisper-tiny"} + + def test_decode_audio_rejects_invalid_base64(): with pytest.raises(ValueError, match="invalid base64"): make_handler()._decode_audio({"audio_b64": "not base64"}) @@ -85,6 +120,37 @@ def model_dump(self, **_kwargs): handler._run_with_cancellation.assert_awaited_once() +def test_generate_preserves_native_vllm_error_status(): + asyncio.run(_assert_generate_preserves_native_vllm_error_status()) + + +async def _assert_generate_preserves_native_vllm_error_status(): + handler = make_handler() + handler._run_with_cancellation = AsyncMock( + return_value=ErrorResponse( + error=ErrorInfo( + message="The model does not exist", + type="NotFoundError", + code=404, + ) + ) + ) + + with pytest.raises(HttpError, match="The model does not exist") as error: + await anext( + handler.generate( + { + "audio_b64": base64.b64encode(b"audio").decode("ascii"), + "filename": "sample.wav", + "response_format": "json", + }, + object(), + ) + ) + + assert error.value.code == 404 + + @pytest.mark.timeout(5) def test_outer_cancellation_stops_native_vllm_inference(): asyncio.run(_assert_outer_cancellation_stops_native_vllm_inference()) diff --git a/components/src/dynamo/vllm/tests/test_vllm_unit.py b/components/src/dynamo/vllm/tests/test_vllm_unit.py index 9159a87c5e75..4f47a459b8b8 100644 --- a/components/src/dynamo/vllm/tests/test_vllm_unit.py +++ b/components/src/dynamo/vllm/tests/test_vllm_unit.py @@ -1743,6 +1743,24 @@ def test_generative_worker_still_defaults_to_enabled(self): assert engine_cfg.enable_prefix_caching is True +class TestTranscriptionWorkerPrefixCachingDefault: + def test_transcription_defaults_to_disabled(self): + dynamo_cfg = _make_dynamo_config(transcription_worker=True) + engine_cfg = _make_engine_config_with_runner(enable_prefix_caching=None) + + update_engine_config_with_dynamo(dynamo_cfg, engine_cfg) + + assert engine_cfg.enable_prefix_caching is False + + def test_explicit_enable_is_preserved(self): + dynamo_cfg = _make_dynamo_config(transcription_worker=True) + engine_cfg = _make_engine_config_with_runner(enable_prefix_caching=True) + + update_engine_config_with_dynamo(dynamo_cfg, engine_cfg) + + assert engine_cfg.enable_prefix_caching is True + + class TestRunnerPreservation: """update_engine_config_with_dynamo must not overwrite a user-set --runner.""" diff --git a/components/src/dynamo/vllm/transcription_handler.py b/components/src/dynamo/vllm/transcription_handler.py index 498485cae193..2e7797f0a96e 100644 --- a/components/src/dynamo/vllm/transcription_handler.py +++ b/components/src/dynamo/vllm/transcription_handler.py @@ -13,6 +13,7 @@ from typing import Any, AsyncIterator from dynamo._core import Context +from dynamo.llm import HttpError from .args import Config from .engine_monitor import VllmEngineMonitor @@ -74,9 +75,10 @@ def __init__( self.engine_monitor = VllmEngineMonitor(runtime, engine, shutdown_event) model_name = config.served_model_name or config.model + model_names = [model_name, *(config.served_model_aliases or [])] models = OpenAIServingModels( engine, - [BaseModelPath(name=model_name, model_path=config.model)], + [BaseModelPath(name=name, model_path=config.model) for name in model_names], ) self.serving = OpenAIServingTranscription( engine, @@ -163,7 +165,7 @@ async def generate( result = await self._run_with_cancellation(audio, native_request, context) if ErrorResponse is not None and isinstance(result, ErrorResponse): - raise ValueError(result.message) + raise HttpError(result.error.code, result.error.message) if not hasattr(result, "model_dump"): raise TypeError( f"Unexpected vLLM transcription response type: {type(result).__name__}" diff --git a/lib/llm/src/http/service/transcriptions.rs b/lib/llm/src/http/service/transcriptions.rs index b0f448f04b89..8973831262f3 100644 --- a/lib/llm/src/http/service/transcriptions.rs +++ b/lib/llm/src/http/service/transcriptions.rs @@ -11,11 +11,17 @@ use axum::{ routing::post, }; use base64::Engine as _; +use dynamo_runtime::pipeline::{AsyncEngineContextProvider, Context}; use futures::StreamExt; +use tracing::Instrument; use super::{ RouteDoc, - metrics::{Endpoint, ErrorType, process_response_and_observe_metrics, request_was_rejected}, + disconnect::create_connection_monitor, + metrics::{ + CancellationLabels, Endpoint, ErrorType, process_response_and_observe_metrics, + request_was_rejected, + }, openai::{ ErrorMessage, ErrorResponse, check_model_serving_ready, check_ready, context_from_headers, error_is_invalid_argument, extract_error_type_from_response, get_body_limit, @@ -155,7 +161,7 @@ async fn audio_transcription( .map_err(|error| ErrorMessage::bad_request(format!("Invalid multipart body: {error}")))?; let mut request = TranscriptionForm::parse(multipart).await?.into_request()?; - let model = match request.model.clone() { + let model = match request.model.take() { Some(model) => model, None => state .manager() @@ -171,6 +177,41 @@ async fn audio_transcription( let request_id = get_or_create_request_id(&headers); let request = context_from_headers(request, request_id, &headers)?; let request_id = request.id().to_string(); + let cancellation_labels = CancellationLabels { + model: metric_model.clone(), + endpoint: Endpoint::Transcriptions.to_string(), + request_type: "unary".to_string(), + }; + let (mut connection_handle, _stream_handle) = create_connection_monitor( + request.context(), + Some(state.metrics_clone()), + cancellation_labels, + ) + .await; + + let response = tokio::spawn( + dispatch_audio_transcription(state, request, model, metric_model, request_id) + .in_current_span(), + ) + .await + .map_err(|error| { + ErrorMessage::internal_server_error_with_details( + "Failed to await audio transcription task", + format!("{error:?}"), + ) + })?; + + connection_handle.disarm(); + response +} + +async fn dispatch_audio_transcription( + state: Arc, + request: Context, + model: String, + metric_model: String, + request_id: String, +) -> Result { let mut inflight = state.metrics_clone().create_inflight_guard( &metric_model, Endpoint::Transcriptions, diff --git a/lib/llm/tests/http-transcriptions.rs b/lib/llm/tests/http-transcriptions.rs index c71b289f7c4e..221671f165ad 100644 --- a/lib/llm/tests/http-transcriptions.rs +++ b/lib/llm/tests/http-transcriptions.rs @@ -1,7 +1,10 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -use std::sync::Arc; +use std::sync::{ + Arc, + atomic::{AtomicBool, Ordering}, +}; use std::time::Duration; use anyhow::Error; @@ -25,6 +28,22 @@ struct TestTranscriptionEngine { fail: bool, } +struct LongRunningTranscriptionEngine { + cancelled: Arc, +} + +impl LongRunningTranscriptionEngine { + fn new() -> Self { + Self { + cancelled: Arc::new(AtomicBool::new(false)), + } + } + + fn was_cancelled(&self) -> bool { + self.cancelled.load(Ordering::Acquire) + } +} + #[async_trait] impl AsyncEngine< @@ -69,6 +88,45 @@ impl } } +#[async_trait] +impl + AsyncEngine< + SingleIn, + ManyOut>, + Error, + > for LongRunningTranscriptionEngine +{ + async fn generate( + &self, + request: SingleIn, + ) -> Result>, Error> { + let (_request, context) = request.transfer(()); + let context = context.context(); + let cancelled = self.cancelled.clone(); + let stream_context = context.clone(); + + let stream = async_stream::stream! { + tokio::select! { + _ = tokio::time::sleep(Duration::from_secs(10)) => { + yield Annotated::from_data(NvAudioTranscriptionResponse { + text: "completed".to_string(), + usage: None, + language: None, + duration: None, + segments: None, + words: None, + }); + } + _ = stream_context.stopped() => { + cancelled.store(true, Ordering::Release); + } + } + }; + + Ok(ResponseStream::new(Box::pin(stream), context)) + } +} + struct TestService { base_url: String, client: reqwest::Client, @@ -78,6 +136,17 @@ struct TestService { impl TestService { async fn start(fail: bool) -> Self { + Self::start_with_engine(Arc::new(TestTranscriptionEngine { fail })).await + } + + async fn start_with_engine(engine: Arc) -> Self + where + E: AsyncEngine< + SingleIn, + ManyOut>, + Error, + > + 'static, + { let listener = tokio::net::TcpListener::bind("127.0.0.1:0") .await .expect("failed to bind test listener"); @@ -95,7 +164,7 @@ impl TestService { .expect("failed to enable transcription endpoint"); service .model_manager() - .add_transcriptions_model(MODEL, "0", Arc::new(TestTranscriptionEngine { fail })) + .add_transcriptions_model(MODEL, "0", engine) .expect("failed to register transcription model"); let cancel = CancellationToken::new(); @@ -182,3 +251,22 @@ async fn annotated_invalid_argument_returns_bad_request() { assert_eq!(response.status(), StatusCode::BAD_REQUEST); service.shutdown().await; } + +#[tokio::test] +async fn client_disconnect_cancels_transcription() { + let engine = Arc::new(LongRunningTranscriptionEngine::new()); + let service = TestService::start_with_engine(engine.clone()).await; + + let result = tokio::time::timeout(Duration::from_secs(1), service.transcribe()).await; + assert!(result.is_err(), "long-running request should time out"); + + tokio::time::timeout(Duration::from_secs(2), async { + while !engine.was_cancelled() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("client disconnect did not cancel transcription inference"); + + service.shutdown().await; +} From 2f2ef91548e41ec28f48e7dde25cb3c1d8cfb8a5 Mon Sep 17 00:00:00 2001 From: Piotr Tarasiewicz Date: Tue, 25 Aug 2026 17:28:52 +0200 Subject: [PATCH 06/18] fix(llm): harden transcription cancellation metrics Signed-off-by: Piotr Tarasiewicz Ported onto main 9cc8c3650487 by Inworld; conflicts resolved in: none Signed-off-by: Nurullah Morshed Co-Authored-By: Claude Opus 5.5 --- lib/llm/src/http/service/transcriptions.rs | 13 +++++-- .../src/protocols/openai/stream_aggregator.rs | 12 +++---- lib/llm/tests/http-transcriptions.rs | 34 ++++++++++++++++++- 3 files changed, 50 insertions(+), 9 deletions(-) diff --git a/lib/llm/src/http/service/transcriptions.rs b/lib/llm/src/http/service/transcriptions.rs index 8973831262f3..f716aa46f968 100644 --- a/lib/llm/src/http/service/transcriptions.rs +++ b/lib/llm/src/http/service/transcriptions.rs @@ -218,7 +218,12 @@ async fn dispatch_audio_transcription( false, &request_id, ); + // Treat an unexpectedly dropped unary request as cancelled unless a + // concrete backend or protocol error below gives us a more specific + // classification. This mirrors the streaming and audio-speech paths. + inflight.mark_error(ErrorType::Cancelled); let http_queue_guard = state.metrics_clone().create_http_queue_guard(&metric_model); + let request_context = request.context(); let engine = state .manager() .get_transcriptions_engine(&model) @@ -239,7 +244,9 @@ async fn dispatch_audio_transcription( .inc_rejection(&metric_model, Endpoint::Transcriptions); } let response = ErrorMessage::from_anyhow(error, "Failed to transcribe audio"); - inflight.mark_error(if is_validation { + inflight.mark_error(if request_context.is_stopped() { + ErrorType::Cancelled + } else if is_validation { ErrorType::Validation } else { extract_error_type_from_response(&response) @@ -263,7 +270,9 @@ async fn dispatch_audio_transcription( anyhow::Error::new(error), "Failed to fold transcription response", ); - inflight.mark_error(if is_validation { + inflight.mark_error(if request_context.is_stopped() { + ErrorType::Cancelled + } else if is_validation { ErrorType::Validation } else { extract_error_type_from_response(&response) diff --git a/lib/llm/src/protocols/openai/stream_aggregator.rs b/lib/llm/src/protocols/openai/stream_aggregator.rs index 75f3633d4ecb..de8b67c4fce9 100644 --- a/lib/llm/src/protocols/openai/stream_aggregator.rs +++ b/lib/llm/src/protocols/openai/stream_aggregator.rs @@ -52,12 +52,12 @@ where let mut response = None; while let Some(item) = stream.next().await { - if let Some(data) = item.into_data()? { - if response.replace(data).is_some() { - return Err(DynamoError::msg( - "unary response stream produced more than one data item", - )); - } + if let Some(data) = item.into_data()? + && response.replace(data).is_some() + { + return Err(DynamoError::msg( + "unary response stream produced more than one data item", + )); } } diff --git a/lib/llm/tests/http-transcriptions.rs b/lib/llm/tests/http-transcriptions.rs index 221671f165ad..27e90c47d284 100644 --- a/lib/llm/tests/http-transcriptions.rs +++ b/lib/llm/tests/http-transcriptions.rs @@ -10,7 +10,11 @@ use std::time::Duration; use anyhow::Error; use base64::Engine as _; use dynamo_llm::endpoint_type::EndpointType; -use dynamo_llm::http::service::service_v2::HttpService; +use dynamo_llm::http::service::{ + Metrics, + metrics::{Endpoint, ErrorType as MetricErrorType, RequestType, Status}, + service_v2::HttpService, +}; use dynamo_llm::protocols::Annotated; use dynamo_llm::protocols::openai::transcriptions::{ NvAudioTranscriptionResponse, NvCreateAudioTranscriptionRequest, @@ -130,6 +134,7 @@ impl struct TestService { base_url: String, client: reqwest::Client, + metrics: Arc, cancel: CancellationToken, join: tokio::task::JoinHandle>, } @@ -166,6 +171,7 @@ impl TestService { .model_manager() .add_transcriptions_model(MODEL, "0", engine) .expect("failed to register transcription model"); + let metrics = service.state_clone().metrics_clone(); let cancel = CancellationToken::new(); let join = service.spawn_with_listener(cancel.clone(), listener).await; @@ -194,6 +200,7 @@ impl TestService { Self { base_url, client, + metrics, cancel, join, } @@ -268,5 +275,30 @@ async fn client_disconnect_cancels_transcription() { .await .expect("client disconnect did not cancel transcription inference"); + tokio::time::timeout(Duration::from_secs(2), async { + while service.metrics.get_request_counter( + MODEL, + &Endpoint::Transcriptions, + &RequestType::Unary, + &Status::Error, + &MetricErrorType::Cancelled, + ) != 1 + { + tokio::task::yield_now().await; + } + }) + .await + .expect("client disconnect was not recorded as a cancelled request"); + assert_eq!( + service.metrics.get_request_counter( + MODEL, + &Endpoint::Transcriptions, + &RequestType::Unary, + &Status::Error, + &MetricErrorType::Internal, + ), + 0, + ); + service.shutdown().await; } From 4dd85a31b09560aad36e97930fe5f870e7fd2279 Mon Sep 17 00:00:00 2001 From: Piotr Tarasiewicz Date: Mon, 31 Aug 2026 12:35:02 +0200 Subject: [PATCH 07/18] fix(llm): use canonical transcription bool parser Signed-off-by: Piotr Tarasiewicz Ported onto main 9cc8c3650487 by Inworld; conflicts resolved in: none Signed-off-by: Nurullah Morshed Co-Authored-By: Claude Opus 5.5 --- lib/llm/src/http/service/transcriptions.rs | 18 ++++++++++----- lib/llm/tests/http-transcriptions.rs | 26 +++++++++++++++++++++- 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/lib/llm/src/http/service/transcriptions.rs b/lib/llm/src/http/service/transcriptions.rs index f716aa46f968..6942cf22f962 100644 --- a/lib/llm/src/http/service/transcriptions.rs +++ b/lib/llm/src/http/service/transcriptions.rs @@ -11,7 +11,10 @@ use axum::{ routing::post, }; use base64::Engine as _; -use dynamo_runtime::pipeline::{AsyncEngineContextProvider, Context}; +use dynamo_runtime::{ + config::parse_bool, + pipeline::{AsyncEngineContextProvider, Context}, +}; use futures::StreamExt; use tracing::Instrument; @@ -80,10 +83,15 @@ impl TranscriptionForm { form.timestamp_granularities.push(value); } } - "stream" if value.eq_ignore_ascii_case("true") || value == "1" => { - return Err(ErrorMessage::bad_request( - "Streaming transcriptions are not supported yet", - )); + "stream" => { + let stream = parse_bool(&value).map_err(|error| { + ErrorMessage::bad_request(format!("Invalid `stream` value: {error}")) + })?; + if stream { + return Err(ErrorMessage::bad_request( + "Streaming transcriptions are not supported yet", + )); + } } _ => {} } diff --git a/lib/llm/tests/http-transcriptions.rs b/lib/llm/tests/http-transcriptions.rs index 27e90c47d284..5fc6fe4dfdc5 100644 --- a/lib/llm/tests/http-transcriptions.rs +++ b/lib/llm/tests/http-transcriptions.rs @@ -216,7 +216,12 @@ impl TestService { } async fn transcribe(&self) -> reqwest::Response { - let form = reqwest::multipart::Form::new() + self.transcribe_with_form(reqwest::multipart::Form::new()) + .await + } + + async fn transcribe_with_form(&self, form: reqwest::multipart::Form) -> reqwest::Response { + let form = form .part( "file", reqwest::multipart::Part::bytes(b"audio".to_vec()) @@ -259,6 +264,25 @@ async fn annotated_invalid_argument_returns_bad_request() { service.shutdown().await; } +#[tokio::test] +async fn stream_field_is_parsed_strictly() { + let service = TestService::start(false).await; + + for value in ["true", "invalid"] { + let response = service + .transcribe_with_form(reqwest::multipart::Form::new().text("stream", value)) + .await; + assert_eq!(response.status(), StatusCode::BAD_REQUEST, "stream={value}"); + } + + let response = service + .transcribe_with_form(reqwest::multipart::Form::new().text("stream", "false")) + .await; + assert_eq!(response.status(), StatusCode::OK); + + service.shutdown().await; +} + #[tokio::test] async fn client_disconnect_cancels_transcription() { let engine = Arc::new(LongRunningTranscriptionEngine::new()); From 73c6b7773f172a40b1be2976d49948298e001528 Mon Sep 17 00:00:00 2001 From: Nurullah Morshed Date: Thu, 8 Oct 2026 09:57:41 -0700 Subject: [PATCH 08/18] fix(vllm): adapt transcription serving to main and vLLM 0.31.0 Follow-up to the ported ai-dynamo/dynamo#11826 commits for APIs that changed on main and in vLLM after the PR was written: - Import vLLM's ErrorResponse from vllm.entrypoints.serve.engine.protocol. vllm.entrypoints.openai.engine no longer exists in vLLM 0.31.0, so the old import made the whole guarded import block fail and the worker refuse to start. The test imports it lazily, as test_realtime_handler does, so pytest collection does not need vLLM's serving modules. - Unpack the snapshot engine as main's SnapshotEngineSetupResult ((engine setup, StatLoggerFactory)), bind the restored stat logger factory to the endpoint and sync the FPM worker id, as the realtime worker does. - Resolve --served-model-name aliases to the primary model in /v1/audio/transcriptions after the readiness gate, as /v1/audio/speech and /v1/rerank do on main, so alias requests route and are metered under the primary name. Add an HTTP test for it. - Read worker sets through main's copy-on-write snapshot (`worker_sets.load().values()`) in `Model::has_transcriptions_engine`, as the other engine checks do since #16119. Signed-off-by: Nurullah Morshed Co-Authored-By: Claude Opus 5.5 --- .../vllm/tests/test_vllm_transcription.py | 5 +++- .../vllm/tests/test_vllm_worker_factory.py | 11 ++++++- .../src/dynamo/vllm/transcription_handler.py | 2 +- components/src/dynamo/vllm/worker_factory.py | 12 ++++++-- lib/llm/src/discovery/model.rs | 6 ++-- lib/llm/src/http/service/transcriptions.rs | 6 +++- lib/llm/tests/http-transcriptions.rs | 30 +++++++++++++++++++ 7 files changed, 64 insertions(+), 8 deletions(-) diff --git a/components/src/dynamo/vllm/tests/test_vllm_transcription.py b/components/src/dynamo/vllm/tests/test_vllm_transcription.py index bde4468fb8d1..88cd5112e9e0 100644 --- a/components/src/dynamo/vllm/tests/test_vllm_transcription.py +++ b/components/src/dynamo/vllm/tests/test_vllm_transcription.py @@ -7,7 +7,6 @@ from unittest.mock import AsyncMock import pytest -from vllm.entrypoints.openai.engine.protocol import ErrorInfo, ErrorResponse from dynamo.llm import HttpError from dynamo.vllm import transcription_handler @@ -125,6 +124,10 @@ def test_generate_preserves_native_vllm_error_status(): async def _assert_generate_preserves_native_vllm_error_status(): + # Imported here, as in test_realtime_handler, so collection does not need + # vLLM's serving modules. + from vllm.entrypoints.serve.engine.protocol import ErrorInfo, ErrorResponse + handler = make_handler() handler._run_with_cancellation = AsyncMock( return_value=ErrorResponse( diff --git a/components/src/dynamo/vllm/tests/test_vllm_worker_factory.py b/components/src/dynamo/vllm/tests/test_vllm_worker_factory.py index 5827d236d18b..1f2a31f01d8d 100644 --- a/components/src/dynamo/vllm/tests/test_vllm_worker_factory.py +++ b/components/src/dynamo/vllm/tests/test_vllm_worker_factory.py @@ -4878,13 +4878,20 @@ async def test_transcription_worker_reuses_snapshot_engine( engine_client = Mock() vllm_config = Mock() - snapshot_engine: EngineSetupResult = ( + engine_setup: EngineSetupResult = ( engine_client, vllm_config, Mock(), None, Mock(), ) + stat_logger_factory = Mock() + snapshot_engine: SnapshotEngineSetupResult = (engine_setup, stat_logger_factory) + sync_fpm_worker_id = AsyncMock() + monkeypatch.setattr( + "dynamo.vllm.worker_factory._sync_fpm_worker_id_or_shutdown", + sync_fpm_worker_id, + ) setup_vllm_engine = Mock() register_vllm_model = AsyncMock() factory = _make_factory( @@ -4915,6 +4922,8 @@ async def test_transcription_worker_reuses_snapshot_engine( ) setup_vllm_engine.assert_not_called() + stat_logger_factory.bind_endpoint.assert_called_once_with(endpoint) + sync_fpm_worker_id.assert_awaited_once_with(engine_client, vllm_config, "worker-1") assert register_vllm_model.await_args.args[4] is engine_client assert register_vllm_model.await_args.args[5] is vllm_config endpoint.serve_endpoint.assert_awaited_once() diff --git a/components/src/dynamo/vllm/transcription_handler.py b/components/src/dynamo/vllm/transcription_handler.py index 2e7797f0a96e..efce5ad9e8b7 100644 --- a/components/src/dynamo/vllm/transcription_handler.py +++ b/components/src/dynamo/vllm/transcription_handler.py @@ -21,9 +21,9 @@ _TRANSCRIPTION_IMPORT_ERROR: ImportError | None = None try: from fastapi import UploadFile - from vllm.entrypoints.openai.engine.protocol import ErrorResponse from vllm.entrypoints.openai.models.protocol import BaseModelPath from vllm.entrypoints.openai.models.serving import OpenAIServingModels + from vllm.entrypoints.serve.engine.protocol import ErrorResponse from vllm.entrypoints.speech_to_text.transcription.protocol import ( TranscriptionRequest, ) diff --git a/components/src/dynamo/vllm/worker_factory.py b/components/src/dynamo/vllm/worker_factory.py index 4c3cb106c0ad..900be3467af5 100644 --- a/components/src/dynamo/vllm/worker_factory.py +++ b/components/src/dynamo/vllm/worker_factory.py @@ -2035,7 +2035,7 @@ async def _create_transcription_worker( config: Config, shutdown_event: asyncio.Event, shutdown_endpoints: list, - snapshot_engine: Optional[EngineSetupResult] = None, + snapshot_engine: Optional[SnapshotEngineSetupResult] = None, ) -> None: generate_endpoint = runtime.endpoint( f"{config.namespace}.{config.component}.{config.endpoint}" @@ -2044,14 +2044,22 @@ async def _create_transcription_worker( fpm_worker_id = str(generate_endpoint.connection_id()) if snapshot_engine is not None: + # A restored snapshot carries the engine together with the stat + # logger factory it was built with; bind that factory and retarget + # the FPM worker id the same way the realtime worker does. + engine_setup, stat_logger_factory = snapshot_engine ( engine_client, vllm_config, _default_sampling_params, _prometheus_temp_dir, _component_gauges, - ) = snapshot_engine + ) = engine_setup os.environ[ENV_FPM_WORKER_ID] = fpm_worker_id + stat_logger_factory.bind_endpoint(generate_endpoint) + await _sync_fpm_worker_id_or_shutdown( + engine_client, vllm_config, fpm_worker_id + ) else: engine_client, vllm_config, _, _, _ = self.setup_vllm_engine( config, diff --git a/lib/llm/src/discovery/model.rs b/lib/llm/src/discovery/model.rs index d26f3fc4fc3f..47e5baa2d21c 100644 --- a/lib/llm/src/discovery/model.rs +++ b/lib/llm/src/discovery/model.rs @@ -300,10 +300,12 @@ impl Model { .any(|worker_set| worker_set.has_audios_engine()) } + /// Check if any WorkerSet has a transcriptions engine. pub fn has_transcriptions_engine(&self) -> bool { self.worker_sets - .iter() - .any(|entry| entry.value().has_transcriptions_engine()) + .load() + .values() + .any(|worker_set| worker_set.has_transcriptions_engine()) } /// Check if any WorkerSet has a realtime engine. diff --git a/lib/llm/src/http/service/transcriptions.rs b/lib/llm/src/http/service/transcriptions.rs index 6942cf22f962..67f4e43e2d42 100644 --- a/lib/llm/src/http/service/transcriptions.rs +++ b/lib/llm/src/http/service/transcriptions.rs @@ -178,8 +178,12 @@ async fn audio_transcription( .find(|model| state.manager().is_model_ready_to_serve(model)) .unwrap_or_default(), }; - request.model = Some(model.clone()); + // Gate readiness on the requested name so a 503 quotes back what the caller + // asked for, then resolve a --served-model-name alias to its primary before + // it reaches routing, metrics, or the engine request (as /v1/audio/speech does). check_model_serving_ready(&state, &model)?; + let model = state.manager().resolve_canonical_name(&model); + request.model = Some(model.clone()); let metric_model = state.manager().metric_model_for(&model).to_string(); let request_id = get_or_create_request_id(&headers); diff --git a/lib/llm/tests/http-transcriptions.rs b/lib/llm/tests/http-transcriptions.rs index 5fc6fe4dfdc5..7edb43d69a32 100644 --- a/lib/llm/tests/http-transcriptions.rs +++ b/lib/llm/tests/http-transcriptions.rs @@ -27,6 +27,7 @@ use dynamo_runtime::pipeline::{ use reqwest::StatusCode; const MODEL: &str = "test-whisper"; +const ALIAS: &str = "test-whisper-alias"; struct TestTranscriptionEngine { fail: bool, @@ -171,6 +172,10 @@ impl TestService { .model_manager() .add_transcriptions_model(MODEL, "0", engine) .expect("failed to register transcription model"); + assert!( + service.model_manager().register_alias(ALIAS, MODEL), + "failed to register transcription model alias" + ); let metrics = service.state_clone().metrics_clone(); let cancel = CancellationToken::new(); @@ -254,6 +259,31 @@ async fn multipart_request_uses_default_model_and_returns_json() { service.shutdown().await; } +#[tokio::test] +async fn alias_request_routes_and_meters_under_primary_model() { + let service = TestService::start(false).await; + + let response = service + .transcribe_with_form(reqwest::multipart::Form::new().text("model", ALIAS)) + .await; + + assert_eq!(response.status(), StatusCode::OK); + let response: NvAudioTranscriptionResponse = response.json().await.unwrap(); + assert_eq!(response.text, "sample.wav:5"); + let counter = |model: &str| { + service.metrics.get_request_counter( + model, + &Endpoint::Transcriptions, + &RequestType::Unary, + &Status::Success, + &MetricErrorType::None, + ) + }; + assert_eq!(counter(MODEL), 1); + assert_eq!(counter(ALIAS), 0); + service.shutdown().await; +} + #[tokio::test] async fn annotated_invalid_argument_returns_bad_request() { let service = TestService::start(true).await; From 4a8cea36655f0d7b364f02679073d0ca58bf9ad2 Mon Sep 17 00:00:00 2001 From: Nurullah Morshed Date: Mon, 12 Oct 2026 20:59:09 -0700 Subject: [PATCH 09/18] fix(vllm): publish transcription worker metrics and clean up its temp dir Review follow-ups: - On a fresh start the transcription worker passed no StatLoggerFactory to setup_vllm_engine, so it published no engine metrics, and it discarded the Prometheus temp dir. Build the factory, set its GPU block count and publish it, as the realtime worker does, and clean up the temp dir when the worker exits (snapshot restores included). - Add ModelManager::remove_transcriptions_model, the counterpart of add_transcriptions_model that the other local model types have. - Skip the native vLLM error-status test where vLLM's serving protocol module is not installed. Tests: a fresh-start worker test (factory built, passed to engine setup, published; handler and temp dir cleaned up) and a transcription model add/get/remove round trip. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Nurullah Morshed --- .../vllm/tests/test_vllm_transcription.py | 5 +- .../vllm/tests/test_vllm_worker_factory.py | 54 +++++++++++++++++++ components/src/dynamo/vllm/worker_factory.py | 22 ++++++-- lib/llm/src/discovery/model_manager.rs | 23 ++++++++ 4 files changed, 99 insertions(+), 5 deletions(-) diff --git a/components/src/dynamo/vllm/tests/test_vllm_transcription.py b/components/src/dynamo/vllm/tests/test_vllm_transcription.py index 88cd5112e9e0..dd2ef82ea421 100644 --- a/components/src/dynamo/vllm/tests/test_vllm_transcription.py +++ b/components/src/dynamo/vllm/tests/test_vllm_transcription.py @@ -125,8 +125,9 @@ def test_generate_preserves_native_vllm_error_status(): async def _assert_generate_preserves_native_vllm_error_status(): # Imported here, as in test_realtime_handler, so collection does not need - # vLLM's serving modules. - from vllm.entrypoints.serve.engine.protocol import ErrorInfo, ErrorResponse + # vLLM's serving modules; skip where they are not installed. + protocol = pytest.importorskip("vllm.entrypoints.serve.engine.protocol") + ErrorInfo, ErrorResponse = protocol.ErrorInfo, protocol.ErrorResponse handler = make_handler() handler._run_with_cancellation = AsyncMock( diff --git a/components/src/dynamo/vllm/tests/test_vllm_worker_factory.py b/components/src/dynamo/vllm/tests/test_vllm_worker_factory.py index 1f2a31f01d8d..b086b9fd58d2 100644 --- a/components/src/dynamo/vllm/tests/test_vllm_worker_factory.py +++ b/components/src/dynamo/vllm/tests/test_vllm_worker_factory.py @@ -4928,3 +4928,57 @@ async def test_transcription_worker_reuses_snapshot_engine( assert register_vllm_model.await_args.args[5] is vllm_config endpoint.serve_endpoint.assert_awaited_once() handler.cleanup.assert_called_once() + + +@pytest.mark.asyncio +async def test_transcription_worker_publishes_metrics_and_cleans_up( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A fresh transcription worker builds and publishes a stat logger factory, + as the realtime worker does, and removes its Prometheus temp dir on exit.""" + endpoint = Mock() + endpoint.connection_id.return_value = "worker-1" + endpoint.serve_endpoint = AsyncMock() + runtime = Mock() + runtime.endpoint.return_value = endpoint + + prometheus_temp_dir = Mock() + vllm_config = Mock() + vllm_config.cache_config.num_gpu_blocks = 128 + setup_vllm_engine = Mock( + return_value=(Mock(), vllm_config, Mock(), prometheus_temp_dir, Mock()) + ) + stat_logger_factory = Mock() + stat_logger_factory_cls = Mock(return_value=stat_logger_factory) + monkeypatch.setattr( + "dynamo.vllm.worker_factory.StatLoggerFactory", stat_logger_factory_cls + ) + monkeypatch.setattr( + "dynamo.vllm.worker_factory.get_dp_range_for_worker", Mock(return_value=(0, 1)) + ) + factory = _make_factory( + setup_vllm_engine_fn=setup_vllm_engine, + register_vllm_model_fn=AsyncMock(), + ) + handler = Mock() + monkeypatch.setattr( + "dynamo.vllm.worker_factory.TranscriptionWorkerHandler", + Mock(return_value=handler), + ) + monkeypatch.setattr("dynamo.vllm.worker_factory.register_model_taint_route", Mock()) + config = SimpleNamespace( + namespace="ns", + component="worker", + endpoint="generate", + served_model_name="whisper", + model="openai/whisper-tiny", + ) + + await factory._create_transcription_worker(runtime, config, asyncio.Event(), []) + + stat_logger_factory_cls.assert_called_once_with(endpoint=endpoint) + assert setup_vllm_engine.call_args.args[1] is stat_logger_factory + stat_logger_factory.set_num_gpu_blocks_all.assert_called_once_with(128) + stat_logger_factory.init_publish.assert_called_once_with() + handler.cleanup.assert_called_once() + prometheus_temp_dir.cleanup.assert_called_once() diff --git a/components/src/dynamo/vllm/worker_factory.py b/components/src/dynamo/vllm/worker_factory.py index 900be3467af5..8d81541ae657 100644 --- a/components/src/dynamo/vllm/worker_factory.py +++ b/components/src/dynamo/vllm/worker_factory.py @@ -2052,7 +2052,7 @@ async def _create_transcription_worker( engine_client, vllm_config, _default_sampling_params, - _prometheus_temp_dir, + prometheus_temp_dir, _component_gauges, ) = engine_setup os.environ[ENV_FPM_WORKER_ID] = fpm_worker_id @@ -2061,11 +2061,25 @@ async def _create_transcription_worker( engine_client, vllm_config, fpm_worker_id ) else: - engine_client, vllm_config, _, _, _ = self.setup_vllm_engine( + # Publish engine metrics like the realtime worker: without a stat + # logger factory the worker would report none. + stat_logger_factory = StatLoggerFactory(endpoint=generate_endpoint) + ( + engine_client, + vllm_config, + _default_sampling_params, + prometheus_temp_dir, + _component_gauges, + ) = self.setup_vllm_engine( config, - None, + stat_logger_factory, fpm_worker_id=fpm_worker_id, ) + _, dp_size = get_dp_range_for_worker(vllm_config) + stat_logger_factory.set_num_gpu_blocks_all( + per_rank_kv_blocks(vllm_config.cache_config.num_gpu_blocks, dp_size) or 0 + ) + stat_logger_factory.init_publish() handler = TranscriptionWorkerHandler( runtime=runtime, engine=engine_client, @@ -2104,6 +2118,8 @@ async def _create_transcription_worker( raise finally: handler.cleanup() + if prometheus_temp_dir is not None: + prometheus_temp_dir.cleanup() async def _create_embedding_worker( self, diff --git a/lib/llm/src/discovery/model_manager.rs b/lib/llm/src/discovery/model_manager.rs index c39deeb42332..d3685ead2b15 100644 --- a/lib/llm/src/discovery/model_manager.rs +++ b/lib/llm/src/discovery/model_manager.rs @@ -2079,6 +2079,13 @@ impl ModelManager { .ok_or_else(|| ModelManagerError::ModelNotFound(model.to_string())) } + pub fn remove_transcriptions_model(&self, model: &str) -> Result<(), ModelManagerError> { + let namespace = format!("__local_transcriptions_{}", model); + self.remove_worker_set(model, &namespace) + .map(|_| ()) + .ok_or_else(|| ModelManagerError::ModelNotFound(model.to_string())) + } + pub fn remove_generate_model(&self, model: &str) -> Result<(), ModelManagerError> { let namespace = format!("__local_generate_{}", model); self.remove_worker_set(model, &namespace) @@ -4457,6 +4464,22 @@ mod tests { /// Registers one model that occupies `model_type`. Returns false when this test has no /// way to occupy that unit, which is the signal the caller turns into a failure. + #[test] + fn test_add_get_remove_transcriptions_model_round_trip() { + let mm = ModelManager::new(); + mm.add_transcriptions_model("stt", "0", Arc::new(UncalledEngine)) + .unwrap(); + assert!(mm.list_transcriptions_models().contains(&"stt".to_string())); + assert!(mm.get_transcriptions_engine("stt").is_ok()); + + mm.remove_transcriptions_model("stt").unwrap(); + assert!(!mm.list_transcriptions_models().contains(&"stt".to_string())); + assert!(matches!( + mm.remove_transcriptions_model("stt"), + Err(ModelManagerError::ModelNotFound(_)) + )); + } + fn register_model_occupying(manager: &ModelManager, model_type: ModelType) -> bool { let name = "occupancy-probe"; let outcome = if model_type == ModelType::Chat { From 00c4d3a7ad378371a634cb2cb141c0185d088ac4 Mon Sep 17 00:00:00 2001 From: Nurullah Morshed Date: Thu, 15 Oct 2026 15:06:11 -0700 Subject: [PATCH 10/18] fix(llm): count the transcriptions route in model HTTP readiness Review follow-up: State::model_http_routes_ready, added on main in #16174, lists each engine type with the route that serves it, and a model is ready only when it has at least one listed engine and every listed route is enabled. Transcriptions were missing from the list, so a transcription-only model never reported ready on /v1/models/{model}/ready. Add it. Test: a transcription-only model is not ready while the transcriptions route is disabled and ready once it is enabled; it fails without this change. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Nurullah Morshed --- lib/llm/src/http/service/service_v2.rs | 40 ++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/lib/llm/src/http/service/service_v2.rs b/lib/llm/src/http/service/service_v2.rs index 301d65f14101..d8ec29ff2ee2 100644 --- a/lib/llm/src/http/service/service_v2.rs +++ b/lib/llm/src/http/service/service_v2.rs @@ -620,6 +620,10 @@ impl State { (model.has_images_engine(), EndpointType::Images), (model.has_videos_engine(), EndpointType::Videos), (model.has_audios_engine(), EndpointType::Audios), + ( + model.has_transcriptions_engine(), + EndpointType::Transcriptions, + ), (model.has_realtime_engine(), EndpointType::Realtime), (model.has_generate_engine(), EndpointType::Generate), ]; @@ -2268,6 +2272,42 @@ mod tests { handle.abort(); } + #[tokio::test] + async fn model_readiness_waits_for_transcriptions_route() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let service = HttpService::builder().port(port).build().unwrap(); + let state = service.state_clone(); + let control = service.clone(); + state + .manager() + .add_transcriptions_model("whisper", "worker", Arc::new(ReadinessOnlyEngine)) + .unwrap(); + control + .enable_model_endpoint(EndpointType::Transcriptions, false) + .unwrap(); + let handle = tokio::spawn(async move { + service + .run_with_listener(CancellationToken::new(), listener) + .await + .unwrap(); + }); + let client = reqwest::Client::new(); + let url = format!("http://127.0.0.1:{port}/v1/models/whisper/ready"); + assert_eq!( + client.get(&url).send().await.unwrap().status(), + reqwest::StatusCode::SERVICE_UNAVAILABLE + ); + control + .enable_model_endpoint(EndpointType::Transcriptions, true) + .unwrap(); + assert_eq!( + client.get(&url).send().await.unwrap().status(), + reqwest::StatusCode::OK + ); + handle.abort(); + } + async fn spawn_derived_route_readiness_service( responses: bool, anthropic: bool, From 7eb91e97751b2ba670238bbdab8f2e64c4b94387 Mon Sep 17 00:00:00 2001 From: Nurullah Morshed Date: Sun, 25 Oct 2026 12:21:10 -0700 Subject: [PATCH 11/18] fix(llm): build the transcription metrics collector from the request guard main (#13187) removed Metrics::create_response_collector: each request's InflightGuard now owns a shared RequestMetrics handle and builds the response collector from it, so every metric family sees the same request transitions. The transcription endpoint still called the removed method and no longer compiled after merging main. Build its collector from the inflight guard, as the other unary endpoints do. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Nurullah Morshed --- lib/llm/src/http/service/transcriptions.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/llm/src/http/service/transcriptions.rs b/lib/llm/src/http/service/transcriptions.rs index 67f4e43e2d42..20b3b5ea58b4 100644 --- a/lib/llm/src/http/service/transcriptions.rs +++ b/lib/llm/src/http/service/transcriptions.rs @@ -244,9 +244,7 @@ async fn dispatch_audio_transcription( inflight.mark_error(extract_error_type_from_response(&response)); response })?; - let mut response_collector = state - .metrics_clone() - .create_response_collector(&metric_model); + let mut response_collector = inflight.response_collector(); let stream = engine.generate(request).await.map_err(|error| { let is_validation = error_is_invalid_argument(error.as_ref()); From 13cec37946172268a74e7806b7668496b9e54e29 Mon Sep 17 00:00:00 2001 From: Nurullah Morshed Date: Thu, 29 Oct 2026 18:40:28 -0700 Subject: [PATCH 12/18] fix(vllm): register the transcription worker's engine metric exporters Review follow-up: the transcription worker published routing and load data through StatLoggerFactory.init_publish() but never called setup_metrics_collection, so neither a fresh nor a snapshot-restored worker registered the Prometheus and OTLP callbacks, and its vllm:* and dynamo_component_* engine metrics were missing while it served requests. Call it once the handler exists, as the realtime worker does. Tests: both transcription worker tests (snapshot restore and fresh start) assert the call with the worker's config and endpoint. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Nurullah Morshed --- components/src/dynamo/vllm/tests/test_vllm_worker_factory.py | 4 +++- components/src/dynamo/vllm/worker_factory.py | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/components/src/dynamo/vllm/tests/test_vllm_worker_factory.py b/components/src/dynamo/vllm/tests/test_vllm_worker_factory.py index 3079d610b966..210c0174c1ed 100644 --- a/components/src/dynamo/vllm/tests/test_vllm_worker_factory.py +++ b/components/src/dynamo/vllm/tests/test_vllm_worker_factory.py @@ -8,7 +8,7 @@ import json import logging from types import SimpleNamespace -from unittest.mock import AsyncMock, Mock, patch +from unittest.mock import ANY, AsyncMock, Mock, patch import pytest from vllm.v1.core.sched.async_scheduler import AsyncScheduler @@ -5074,6 +5074,7 @@ async def test_transcription_worker_reuses_snapshot_engine( setup_vllm_engine.assert_not_called() stat_logger_factory.bind_endpoint.assert_called_once_with(endpoint) sync_fpm_worker_id.assert_awaited_once_with(engine_client, vllm_config, "worker-1") + factory.setup_metrics_collection.assert_called_once_with(config, endpoint, ANY) assert register_vllm_model.await_args.args[4] is engine_client assert register_vllm_model.await_args.args[5] is vllm_config endpoint.serve_endpoint.assert_awaited_once() @@ -5130,5 +5131,6 @@ async def test_transcription_worker_publishes_metrics_and_cleans_up( assert setup_vllm_engine.call_args.args[1] is stat_logger_factory stat_logger_factory.set_num_gpu_blocks_all.assert_called_once_with(128) stat_logger_factory.init_publish.assert_called_once_with() + factory.setup_metrics_collection.assert_called_once_with(config, endpoint, ANY) handler.cleanup.assert_called_once() prometheus_temp_dir.cleanup.assert_called_once() diff --git a/components/src/dynamo/vllm/worker_factory.py b/components/src/dynamo/vllm/worker_factory.py index e7c2f4c9ccdc..c2ad87ce4b47 100644 --- a/components/src/dynamo/vllm/worker_factory.py +++ b/components/src/dynamo/vllm/worker_factory.py @@ -2091,6 +2091,7 @@ async def _create_transcription_worker( config=config, shutdown_event=shutdown_event, ) + self.setup_metrics_collection(config, generate_endpoint, logger) model_name = config.served_model_name or config.model health_payload = VllmTranscriptionHealthCheckPayload(model_name).to_dict() metric_labels = [ From e190c32cf7409f596ebc2bb68c62894d37462f94 Mon Sep 17 00:00:00 2001 From: Nurullah Morshed Date: Thu, 29 Oct 2026 18:40:36 -0700 Subject: [PATCH 13/18] fix(llm): return 413 for an oversized transcription upload Review follow-up: the transcription endpoint turned every multipart read failure into a 400, so an upload over the HTTP body limit was reported as a malformed request and classified InvalidRequest instead of PayloadTooLarge. Clients could not tell an oversized valid upload from bad input. Keep the body limit's 413, with the same response JSON routes return (naming DYN_HTTP_BODY_LIMIT_MB), on the next-field, audio and text-field reads, and keep 400 for malformed multipart. Test: behind a 1 KiB body limit, with the body streamed without a Content-Length, a small upload parses, a 4 KiB upload gets 413 and a truncated body gets 400. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Nurullah Morshed --- lib/llm/src/http/service/openai.rs | 2 +- lib/llm/src/http/service/transcriptions.rs | 109 +++++++++++++++++++-- 2 files changed, 101 insertions(+), 10 deletions(-) diff --git a/lib/llm/src/http/service/openai.rs b/lib/llm/src/http/service/openai.rs index a2ab0c384e23..d9327901fa2c 100644 --- a/lib/llm/src/http/service/openai.rs +++ b/lib/llm/src/http/service/openai.rs @@ -2738,7 +2738,7 @@ fn unsupported_media_type_error() -> ErrorResponse { /// Returns the standard error response for a request body that exceeds the /// configured size limit. -fn payload_too_large_error() -> ErrorResponse { +pub(super) fn payload_too_large_error() -> ErrorResponse { ErrorMessage::from_http_error( ErrorClass::PayloadTooLarge, HttpError { diff --git a/lib/llm/src/http/service/transcriptions.rs b/lib/llm/src/http/service/transcriptions.rs index 20b3b5ea58b4..fe61fc064d29 100644 --- a/lib/llm/src/http/service/transcriptions.rs +++ b/lib/llm/src/http/service/transcriptions.rs @@ -5,7 +5,10 @@ use std::sync::Arc; use axum::{ Json, Router, - extract::{Multipart, State, multipart::MultipartRejection}, + extract::{ + Multipart, State, + multipart::{MultipartError, MultipartRejection}, + }, http::{HeaderMap, StatusCode}, response::{IntoResponse, Response}, routing::post, @@ -28,7 +31,7 @@ use super::{ openai::{ ErrorMessage, ErrorResponse, check_model_serving_ready, check_ready, context_from_headers, error_is_invalid_argument, extract_error_type_from_response, get_body_limit, - get_or_create_request_id, + get_or_create_request_id, payload_too_large_error, }, service_v2, }; @@ -51,15 +54,18 @@ struct TranscriptionForm { impl TranscriptionForm { async fn parse(mut multipart: Multipart) -> Result { let mut form = Self::default(); - while let Some(field) = multipart.next_field().await.map_err(|error| { - ErrorMessage::bad_request(format!("Invalid multipart body: {error}")) - })? { + while let Some(field) = multipart + .next_field() + .await + .map_err(|error| multipart_error("Invalid multipart body", error))? + { let name = field.name().unwrap_or_default().to_string(); if name == "file" { form.filename = Some(field.file_name().unwrap_or("audio").to_string()); - let bytes = field.bytes().await.map_err(|error| { - ErrorMessage::bad_request(format!("Failed to read audio file: {error}")) - })?; + let bytes = field + .bytes() + .await + .map_err(|error| multipart_error("Failed to read audio file", error))?; if bytes.is_empty() { return Err(ErrorMessage::bad_request( "The audio file must not be empty", @@ -70,7 +76,7 @@ impl TranscriptionForm { } let value = field.text().await.map_err(|error| { - ErrorMessage::bad_request(format!("Failed to read form field `{name}`: {error}")) + multipart_error(&format!("Failed to read form field `{name}`"), error) })?; match name.as_str() { "model" => form.model = nonempty(value), @@ -145,6 +151,16 @@ impl TranscriptionForm { } } +/// Keeps the body limit's 413 for an oversized upload; any other multipart failure is a +/// malformed request. +fn multipart_error(context: &str, error: MultipartError) -> ErrorResponse { + if error.status() == StatusCode::PAYLOAD_TOO_LARGE { + payload_too_large_error() + } else { + ErrorMessage::bad_request(format!("{context}: {error}")) + } +} + fn nonempty(value: String) -> Option { (!value.is_empty()).then_some(value) } @@ -310,6 +326,81 @@ pub fn transcriptions_router( #[cfg(test)] mod tests { use super::*; + use axum::{ + body::Body, + extract::DefaultBodyLimit, + http::{Request, header::CONTENT_TYPE}, + }; + use bytes::Bytes; + use tower::ServiceExt; + + fn multipart_body(audio: &[u8], closed: bool) -> Vec { + let mut body = b"--audio-boundary\r\nContent-Disposition: form-data; name=\"file\"; \ + filename=\"sample.wav\"\r\n\r\n" + .to_vec(); + body.extend_from_slice(audio); + if closed { + body.extend_from_slice(b"\r\n--audio-boundary--\r\n"); + } + body + } + + /// Parse the form behind `body_limit`, streamed without a Content-Length. + async fn parse_status(body: Vec, body_limit: usize) -> (StatusCode, serde_json::Value) { + let app = Router::new() + .route( + "/", + post(|multipart: Multipart| async move { + match TranscriptionForm::parse(multipart).await { + Ok(_) => StatusCode::OK.into_response(), + Err(error) => error.into_response(), + } + }), + ) + .layer(DefaultBodyLimit::max(body_limit)); + let chunks = body + .chunks(17) + .map(Bytes::copy_from_slice) + .map(Ok::<_, std::io::Error>) + .collect::>(); + let response = app + .oneshot( + Request::builder() + .method("POST") + .uri("/") + .header(CONTENT_TYPE, "multipart/form-data; boundary=audio-boundary") + .body(Body::from_stream(futures::stream::iter(chunks))) + .unwrap(), + ) + .await + .unwrap(); + let status = response.status(); + let bytes = axum::body::to_bytes(response.into_body(), 1024 * 1024) + .await + .unwrap(); + ( + status, + serde_json::from_slice(&bytes).unwrap_or(serde_json::Value::Null), + ) + } + + #[tokio::test] + async fn oversized_upload_is_payload_too_large_and_malformed_is_bad_request() { + let (status, _) = parse_status(multipart_body(&[1; 32], true), 1024).await; + assert_eq!(status, StatusCode::OK); + + let (status, error) = parse_status(multipart_body(&[1; 4096], true), 1024).await; + assert_eq!(status, StatusCode::PAYLOAD_TOO_LARGE); + assert!( + error["message"] + .as_str() + .is_some_and(|message| message.contains("DYN_HTTP_BODY_LIMIT_MB")), + "unexpected 413 body: {error}" + ); + + let (status, _) = parse_status(multipart_body(&[1; 32], false), 1024).await; + assert_eq!(status, StatusCode::BAD_REQUEST); + } #[test] fn rejects_non_json_response_formats() { From d6a823908f67ad21750e41361e8cc14193058103 Mon Sep 17 00:00:00 2001 From: Nurullah Morshed Date: Thu, 29 Oct 2026 23:23:28 -0700 Subject: [PATCH 14/18] test(llm): keep register_model_occupying's doc comment on the helper Review follow-up: the transcription model round-trip test sat between register_model_occupying and its doc comment, so the comment documented the test. Move the test above the comment. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Nurullah Morshed --- lib/llm/src/discovery/model_manager.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/llm/src/discovery/model_manager.rs b/lib/llm/src/discovery/model_manager.rs index ddb5632fb41b..30937cdceedd 100644 --- a/lib/llm/src/discovery/model_manager.rs +++ b/lib/llm/src/discovery/model_manager.rs @@ -4834,8 +4834,6 @@ mod tests { } } - /// Registers one model that occupies `model_type`. Returns false when this test has no - /// way to occupy that unit, which is the signal the caller turns into a failure. #[test] fn test_add_get_remove_transcriptions_model_round_trip() { let mm = ModelManager::new(); @@ -4852,6 +4850,8 @@ mod tests { )); } + /// Registers one model that occupies `model_type`. Returns false when this test has no + /// way to occupy that unit, which is the signal the caller turns into a failure. fn register_model_occupying(manager: &ModelManager, model_type: ModelType) -> bool { let name = "occupancy-probe"; let outcome = if model_type == ModelType::Chat { From b55edfdd17c721b66d6e0cb17ae15449ea947c62 Mon Sep 17 00:00:00 2001 From: Nurullah Morshed Date: Fri, 30 Oct 2026 08:50:27 -0700 Subject: [PATCH 15/18] fix(vllm): let mypy skip the transcription handler's fastapi import The dynamo-runtime image that runs mypy has no vLLM and no longer has fastapi, so mypy reported import-not-found for the guarded `from fastapi import UploadFile`. fastapi is a vLLM dependency and is present wherever the worker runs; the import is already guarded. Ignore the missing import on that line, as vllm/omni does for omegaconf. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Nurullah Morshed --- components/src/dynamo/vllm/transcription_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/src/dynamo/vllm/transcription_handler.py b/components/src/dynamo/vllm/transcription_handler.py index efce5ad9e8b7..982b78b3871a 100644 --- a/components/src/dynamo/vllm/transcription_handler.py +++ b/components/src/dynamo/vllm/transcription_handler.py @@ -20,7 +20,7 @@ _TRANSCRIPTION_IMPORT_ERROR: ImportError | None = None try: - from fastapi import UploadFile + from fastapi import UploadFile # type: ignore[import-not-found] from vllm.entrypoints.openai.models.protocol import BaseModelPath from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.serve.engine.protocol import ErrorResponse From 2568114dccea0bdb0fd52526a88c3ffd47c40378 Mon Sep 17 00:00:00 2001 From: Nurullah Morshed Date: Fri, 30 Oct 2026 16:55:59 -0700 Subject: [PATCH 16/18] fix(llm): cap transcription uploads below the request-plane frame Review follow-up: the audio crosses the request plane base64-encoded, at 4/3 of its size, so an upload above about 25 MB passed the 45 MiB HTTP body limit but exceeded the default 32 MiB TCP frame, and the client got a generic 400 "Invalid request" instead of a 413. Cap the route's body limit at 23 MiB, which leaves room for the envelope, as image_edits.rs keeps its request below the same frame. A 413 names whichever limit was hit: DYN_HTTP_BODY_LIMIT_MB when it is the lower one, otherwise the transcription cap. A compile-time assertion keeps the encoded cap below the frame. Test: a 24 MiB upload, under the default body limit, gets 413 naming the transcription limit. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Nurullah Morshed --- lib/llm/src/http/service/transcriptions.rs | 39 +++++++++++++++-- lib/llm/tests/http-transcriptions.rs | 32 ++++++++++++++ lib/runtime/examples/Cargo.lock | 50 ---------------------- 3 files changed, 68 insertions(+), 53 deletions(-) diff --git a/lib/llm/src/http/service/transcriptions.rs b/lib/llm/src/http/service/transcriptions.rs index fe61fc064d29..2941c1c09b8a 100644 --- a/lib/llm/src/http/service/transcriptions.rs +++ b/lib/llm/src/http/service/transcriptions.rs @@ -16,6 +16,7 @@ use axum::{ use base64::Engine as _; use dynamo_runtime::{ config::parse_bool, + error::ErrorClass, pipeline::{AsyncEngineContextProvider, Context}, }; use futures::StreamExt; @@ -24,6 +25,7 @@ use tracing::Instrument; use super::{ RouteDoc, disconnect::create_connection_monitor, + error::HttpError, metrics::{ CancellationLabels, Endpoint, ErrorType, process_response_and_observe_metrics, request_was_rejected, @@ -151,11 +153,42 @@ impl TranscriptionForm { } } +/// Largest request body this route accepts. The audio crosses the request plane +/// base64-encoded, at 4/3 of its size, so a larger upload than this would pass the HTTP body +/// limit but exceed the default 32 MiB TCP frame, and the client would get a generic 400 +/// instead of a 413. +const MAX_UPLOAD_BYTES: usize = 23 * 1024 * 1024; + +// The encoded upload must leave room for the request-plane envelope below the 32 MiB frame. +const _: () = assert!(MAX_UPLOAD_BYTES.div_ceil(3) * 4 + 1024 * 1024 < 32 * 1024 * 1024); + +/// The route's body limit: the HTTP body limit, capped at [`MAX_UPLOAD_BYTES`]. +fn upload_limit() -> usize { + get_body_limit().min(MAX_UPLOAD_BYTES) +} + +/// A 413 that names whichever limit the upload exceeded. +fn upload_too_large_error() -> ErrorResponse { + if get_body_limit() <= MAX_UPLOAD_BYTES { + return payload_too_large_error(); + } + ErrorMessage::from_http_error( + ErrorClass::PayloadTooLarge, + HttpError { + code: StatusCode::PAYLOAD_TOO_LARGE.as_u16(), + message: format!( + "Audio upload exceeds the transcription limit of {} MiB", + MAX_UPLOAD_BYTES / (1024 * 1024) + ), + }, + ) +} + /// Keeps the body limit's 413 for an oversized upload; any other multipart failure is a /// malformed request. fn multipart_error(context: &str, error: MultipartError) -> ErrorResponse { if error.status() == StatusCode::PAYLOAD_TOO_LARGE { - payload_too_large_error() + upload_too_large_error() } else { ErrorMessage::bad_request(format!("{context}: {error}")) } @@ -318,7 +351,7 @@ pub fn transcriptions_router( let doc = RouteDoc::new(axum::http::Method::POST, &path); let router = Router::new() .route(&path, post(audio_transcription)) - .layer(axum::extract::DefaultBodyLimit::max(get_body_limit())) + .layer(axum::extract::DefaultBodyLimit::max(upload_limit())) .with_state(state); (vec![doc], router) } @@ -394,7 +427,7 @@ mod tests { assert!( error["message"] .as_str() - .is_some_and(|message| message.contains("DYN_HTTP_BODY_LIMIT_MB")), + .is_some_and(|message| message.contains("limit")), "unexpected 413 body: {error}" ); diff --git a/lib/llm/tests/http-transcriptions.rs b/lib/llm/tests/http-transcriptions.rs index 7edb43d69a32..f52e9f910c74 100644 --- a/lib/llm/tests/http-transcriptions.rs +++ b/lib/llm/tests/http-transcriptions.rs @@ -313,6 +313,38 @@ async fn stream_field_is_parsed_strictly() { service.shutdown().await; } +#[tokio::test] +async fn upload_too_large_for_the_request_plane_returns_payload_too_large() { + let service = TestService::start(false).await; + + // Under the default HTTP body limit, but too large once base64-encoded for the + // request plane's default 32 MiB frame. + let form = reqwest::multipart::Form::new().part( + "file", + reqwest::multipart::Part::bytes(vec![1u8; 24 * 1024 * 1024]) + .file_name("large.wav") + .mime_str("audio/wav") + .unwrap(), + ); + let response = service + .client + .post(format!("{}/v1/audio/transcriptions", service.base_url)) + .multipart(form) + .send() + .await + .expect("transcription request failed"); + assert_eq!(response.status(), StatusCode::PAYLOAD_TOO_LARGE); + let body: serde_json::Value = response.json().await.unwrap(); + assert!( + body["message"] + .as_str() + .is_some_and(|message| message.contains("transcription limit")), + "unexpected 413 body: {body}" + ); + + service.shutdown().await; +} + #[tokio::test] async fn client_disconnect_cancels_transcription() { let engine = Arc::new(LongRunningTranscriptionEngine::new()); diff --git a/lib/runtime/examples/Cargo.lock b/lib/runtime/examples/Cargo.lock index 74ea3174c790..7fcc5f8d793a 100644 --- a/lib/runtime/examples/Cargo.lock +++ b/lib/runtime/examples/Cargo.lock @@ -212,7 +212,6 @@ dependencies = [ "matchit", "memchr", "mime", - "multer", "percent-encoding", "pin-project-lite", "rustversion", @@ -465,12 +464,6 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" -[[package]] -name = "core_detect" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" - [[package]] name = "cpufeatures" version = "0.2.17" @@ -929,20 +922,6 @@ dependencies = [ "serde", ] -[[package]] -name = "encoding_rs" -version = "0.8.42" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e985e0451871ad22fb8d2b6b076e2028a502a0d3950998c2c5c0a4f9b5d9679" -dependencies = [ - "cfg-if", - "core_detect", - "multiversion_no_op", - "rustversion", - "scopeguard", - "simdutf8", -] - [[package]] name = "enum-ordinalize" version = "4.3.2" @@ -2076,35 +2055,12 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "multer" -version = "3.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83e87776546dc87511aa5ee218730c92b666d7264ab6ed41f9d215af9cd5224b" -dependencies = [ - "bytes", - "encoding_rs", - "futures-util", - "http", - "httparse", - "memchr", - "mime", - "spin", - "version_check", -] - [[package]] name = "multimap" version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" -[[package]] -name = "multiversion_no_op" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d" - [[package]] name = "neli" version = "0.7.4" @@ -3456,12 +3412,6 @@ dependencies = [ "rand_core 0.6.4", ] -[[package]] -name = "simdutf8" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" - [[package]] name = "slab" version = "0.4.12" From bc0011ae37b5a038a2ade4a925b38de55b0a1861 Mon Sep 17 00:00:00 2001 From: Nurullah Morshed Date: Fri, 30 Oct 2026 16:55:59 -0700 Subject: [PATCH 17/18] fix(vllm): report vLLM transcription request errors as invalid arguments Review follow-up: vLLM raises VLLMValidationError, for example for a prompt longer than the model context, and it is not a ValueError, so the client got a 500. vLLM's other request errors came back as 400 without their reason. Re-raise ValueError and VLLMClientError (vLLM's 4xx base, which includes VLLMValidationError) as InvalidArgument, the only error that carries its message to the client, as handlers.py does. Test: a VLLMValidationError and a ValueError from vLLM both reach the client as InvalidArgument with their message. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Nurullah Morshed --- .../vllm/tests/test_vllm_transcription.py | 31 +++++++++++++++++++ .../src/dynamo/vllm/transcription_handler.py | 14 ++++++++- 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/components/src/dynamo/vllm/tests/test_vllm_transcription.py b/components/src/dynamo/vllm/tests/test_vllm_transcription.py index dd2ef82ea421..cef8204bed84 100644 --- a/components/src/dynamo/vllm/tests/test_vllm_transcription.py +++ b/components/src/dynamo/vllm/tests/test_vllm_transcription.py @@ -9,6 +9,7 @@ import pytest from dynamo.llm import HttpError +from dynamo.llm.exceptions import InvalidArgument from dynamo.vllm import transcription_handler from dynamo.vllm.transcription_handler import TranscriptionWorkerHandler @@ -155,6 +156,36 @@ async def _assert_generate_preserves_native_vllm_error_status(): assert error.value.code == 404 +@pytest.mark.parametrize("error_name", ["VLLMValidationError", "ValueError"]) +def test_generate_reports_vllm_request_errors_as_invalid_argument(error_name): + asyncio.run(_assert_generate_reports_vllm_request_error(error_name)) + + +async def _assert_generate_reports_vllm_request_error(error_name): + # VLLMValidationError (a prompt longer than the model context, for one) is not a + # ValueError; both must reach the client as a 400 with their message. + exceptions = pytest.importorskip("vllm.exceptions") + error_cls = ( + ValueError if error_name == "ValueError" else getattr(exceptions, error_name) + ) + handler = make_handler() + handler._run_with_cancellation = AsyncMock( + side_effect=error_cls("bad request detail") + ) + + with pytest.raises(InvalidArgument, match="bad request detail"): + await anext( + handler.generate( + { + "audio_b64": base64.b64encode(b"audio").decode("ascii"), + "filename": "sample.wav", + "response_format": "json", + }, + object(), + ) + ) + + @pytest.mark.timeout(5) def test_outer_cancellation_stops_native_vllm_inference(): asyncio.run(_assert_outer_cancellation_stops_native_vllm_inference()) diff --git a/components/src/dynamo/vllm/transcription_handler.py b/components/src/dynamo/vllm/transcription_handler.py index 982b78b3871a..cf761092fb1c 100644 --- a/components/src/dynamo/vllm/transcription_handler.py +++ b/components/src/dynamo/vllm/transcription_handler.py @@ -14,11 +14,15 @@ from dynamo._core import Context from dynamo.llm import HttpError +from dynamo.llm.exceptions import InvalidArgument from .args import Config from .engine_monitor import VllmEngineMonitor _TRANSCRIPTION_IMPORT_ERROR: ImportError | None = None +# Errors vLLM raises for a bad request. VLLMClientError (vLLM's 4xx base, which +# includes VLLMValidationError) is not a ValueError. +_REQUEST_ERRORS: tuple[type[Exception], ...] = (ValueError,) try: from fastapi import UploadFile # type: ignore[import-not-found] from vllm.entrypoints.openai.models.protocol import BaseModelPath @@ -30,6 +34,9 @@ from vllm.entrypoints.speech_to_text.transcription.serving import ( OpenAIServingTranscription, ) + from vllm.exceptions import VLLMClientError + + _REQUEST_ERRORS = (ValueError, VLLMClientError) except ImportError as error: UploadFile = None # type: ignore[assignment, misc] ErrorResponse = None # type: ignore[assignment, misc] @@ -163,7 +170,12 @@ async def generate( "Transcription response_format must be 'json' or 'verbose_json'" ) - result = await self._run_with_cancellation(audio, native_request, context) + try: + result = await self._run_with_cancellation(audio, native_request, context) + except _REQUEST_ERRORS as error: + # Only InvalidArgument carries its message to the client, as in + # handlers.py; otherwise a VLLMClientError is reported as a 500. + raise InvalidArgument(str(error)) from error if ErrorResponse is not None and isinstance(result, ErrorResponse): raise HttpError(result.error.code, result.error.message) if not hasattr(result, "model_dump"): From 17895bde27eec496e2f643d7e62fa65040ff1e2a Mon Sep 17 00:00:00 2001 From: Nurullah Morshed Date: Fri, 30 Oct 2026 16:55:59 -0700 Subject: [PATCH 18/18] build: drop the workspace multipart feature of axum Review follow-up: lib/llm already enables axum's multipart feature, and dynamo-llm is the only crate that uses it. Enabling it on the workspace only added five packages to lib/runtime/examples/Cargo.lock. Drop it and restore main's copy of that lock. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Nurullah Morshed --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 4b12c38eff6f..faff41d914c5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -159,7 +159,7 @@ tokio-stream = { version = "0.1" } tokio-util = { version = "0.7.19", features = ["codec", "net", "rt", "io-util"] } tower = { version = "0.5", features = ["util"] } tower-http = { version = "0.6", features = ["trace"] } -axum = { version = "=0.8.4", features = ["macros", "multipart", "ws"] } +axum = { version = "=0.8.4", features = ["macros", "ws"] } axum-core = { version = "0.5.2" } hyper = { version = "=1.7.0" } hyper-util = { version = "=0.1.17" }