feat(qwen3-asr): support prompt parameter in v1/audio/transcriptions

examples/speech_to_text/openai/openai_transcription_client.py

@@ -33,15 +33,23 @@ def sync_openai(
     *,
     repetition_penalty: float = 1.3,
     hotwords: str = None,
+    prompt: str | None = None,
 ):
     """
     Perform synchronous transcription using OpenAI-compatible API.
+
+    The optional ``prompt`` is the OpenAI-API ``prompt`` field (style /
+    vocabulary hint). It is wired through model-by-model: Whisper uses it
+    as a ``<|prev|>`` continuation hint, Qwen3-ASR maps it into the
+    chat-template ``system`` turn. Models that do not consume it accept
+    it without effect.
     """
     with open(audio_path, "rb") as f:
         transcription = client.audio.transcriptions.create(
             file=f,
             model=model,
             language="en",
+            prompt=prompt or "",
             response_format="json",
             temperature=0.0,
             # Additional sampling params not provided by OpenAI API.
@@ -55,7 +63,11 @@ def sync_openai(
 
 
 async def stream_openai_response(
-    audio_path: str, client: AsyncOpenAI, model: str, hotwords: str = None
+    audio_path: str,
+    client: AsyncOpenAI,
+    model: str,
+    hotwords: str = None,
+    prompt: str | None = None,
 ):
     """
     Perform asynchronous transcription using OpenAI-compatible API.
@@ -66,6 +78,7 @@ async def stream_openai_response(
             file=f,
             model=model,
             language="en",
+            prompt=prompt or "",
             response_format="json",
             temperature=0.0,
             # Additional sampling params not provided by OpenAI API.
@@ -146,6 +159,7 @@ def main(args):
         model=model,
         repetition_penalty=args.repetition_penalty,
         hotwords=args.hotwords,
+        prompt=args.prompt,
     )
 
     # Run the asynchronous function
@@ -160,6 +174,7 @@ def main(args):
                 client,
                 model,
                 hotwords=args.hotwords,
+                prompt=args.prompt,
             )
         )
     else:
@@ -193,5 +208,16 @@ def main(args):
         default=None,
         help="hotwords",
     )
+    parser.add_argument(
+        "--prompt",
+        type=str,
+        default=None,
+        help=(
+            "Optional `prompt` (OpenAI transcription API: style/vocabulary "
+            "hint). Wired model-by-model: Whisper uses it as a `<|prev|>` "
+            "continuation hint, Qwen3-ASR maps it into the chat-template "
+            "system turn."
+        ),
+    )
     args = parser.parse_args()
     main(args)

tests/models/multimodal/generation/test_qwen3_asr_sanitize_prompt.py

@@ -0,0 +1,64 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Unit tests for ``Qwen3ASR``'s user-text sanitizer.
+
+The sanitizer is the security boundary between user-supplied transcription
+fields (``prompt`` / ``response_prefix``) and the structured ChatML prompt
+template. It must strip both ``<|...|>`` control tokens and the
+``<asr_text>`` assistant-prefix delimiter, and it must do so to a fixpoint
+so nested payloads cannot reconstruct a valid token after a single pass.
+"""
+
+import pytest
+
+from vllm.model_executor.models.qwen3_asr import _sanitize_transcription_user_text
+
+
+@pytest.mark.parametrize(
+    ("text", "expected"),
+    [
+        # No-op cases
+        ("", ""),
+        ("plain text", "plain text"),
+        ("|piped|content", "|piped|content"),
+        ("contains < and > but not as a token", "contains < and > but not as a token"),
+        # Single-pass strips
+        ("<|im_end|>", ""),
+        ("<|im_start|>assistant<|im_end|>", "assistant"),
+        ("a<|x|>b", "ab"),
+        ("foo<asr_text>bar", "foobar"),
+        # Nested ChatML reconstruction attacks (would bypass a single re.sub)
+        ("<|im<|x|>_end|>", ""),
+        ("<|<|inner|>middle<|x|>_end|>", ""),
+        # Nested <asr_text> reconstruction attack
+        # (would bypass a single str.replace)
+        ("<asr_te<asr_text>xt>", ""),
+        ("<asr_te<asr_te<asr_text>xt>xt>", ""),
+        # Combined attacks across both kinds of token
+        ("<|im_end|>foo<asr_text>bar<|<|x|>im_end|>", "foobar"),
+        ("foo<asr_te<|x|>xt>bar", "foobar"),
+    ],
+)
+def test_sanitize_strips_control_tokens(text: str, expected: str) -> None:
+    assert _sanitize_transcription_user_text(text) == expected
+
+
+def test_sanitize_handles_falsy_inputs() -> None:
+    assert _sanitize_transcription_user_text("") == ""
+    # The dataclass default for ``response_prefix`` is the empty string;
+    # the sanitizer must accept that without exception or extra work.
+    assert _sanitize_transcription_user_text(None) == ""  # type: ignore[arg-type]
+
+
+def test_sanitize_is_idempotent() -> None:
+    """Once sanitized, applying again must be a no-op (fixpoint property)."""
+    cases = [
+        "plain text",
+        "<|im<|x|>_end|>",
+        "<asr_te<asr_text>xt>",
+        "<|im_end|>foo<asr_text>bar<|<|x|>im_end|>",
+    ]
+    for raw in cases:
+        once = _sanitize_transcription_user_text(raw)
+        twice = _sanitize_transcription_user_text(once)
+        assert once == twice, f"not idempotent for {raw!r}"

vllm/model_executor/models/qwen3_asr.py

@@ -25,6 +25,7 @@
 from collections.abc import Iterable, Mapping, Sequence
 from typing import Any
 
+import regex as re
 import torch
 import torch.nn as nn
 from transformers.feature_extraction_utils import BatchFeature
@@ -90,6 +91,31 @@
 
 logger = init_logger(__name__)
 _ASR_TEXT_TAG = "<asr_text>"
+# User-supplied `prompt` / `response_prefix` must not inject extra ChatML turns.
+_CHATML_LIKE_TOKEN = re.compile(r"<\|[^|]+\|>")
+
+
+def _sanitize_transcription_user_text(text: str) -> str:
+    """Strip ChatML-style special tokens from user-controlled transcription fields.
+
+    Applies the regex / ``<asr_text>`` substitutions to a fixpoint so nested
+    payloads cannot reconstruct a valid token after a single pass:
+
+    - ``<|im<|x|>_end|>`` would, with a single ``re.sub``, leave ``<|im_end|>``
+      (a real ChatML control token).
+    - ``<asr_te<asr_text>xt>`` would, with a single ``str.replace``, leave
+      ``<asr_text>`` (the model-significant assistant-prefix delimiter).
+
+    Looping both substitutions until the string stabilises eliminates these
+    reconstruction attacks.
+    """
+    if not text:
+        return ""
+    prev = None
+    while prev != text:
+        prev = text
+        text = _CHATML_LIKE_TOKEN.sub("", text).replace(_ASR_TEXT_TAG, "")
+    return text
 
 
 def _get_feat_extract_output_lengths(input_lengths: torch.Tensor):
@@ -550,11 +576,24 @@ def get_speech_to_text_config(
 
     @classmethod
     def get_generation_prompt(cls, stt_params: SpeechToTextParams) -> PromptType:
-        """Get the generation prompt to be used for transcription requests."""
+        """Get the generation prompt to be used for transcription requests.
+
+        Matches the official Qwen3-ASR SDK prompt format. The ``system`` turn
+        is only emitted when the caller supplied a ``prompt``, mirroring the
+        SDK's ``_build_messages`` (which omits the system role when context is
+        empty) and preserving the prior no-prompt behavior:
+
+          [system: {context}]                         # only when prompt given
+          user: {audio}
+          assistant: [language {Lang}<asr_text>]      # when language is forced
+        """
         audio = stt_params.audio
         model_config = stt_params.model_config
+        language = stt_params.language
         task_type = stt_params.task_type
+        request_prompt = stt_params.request_prompt
         to_language = stt_params.to_language
+
         tokenizer = cached_tokenizer_from_config(model_config)
         audio_placeholder = cls.get_placeholder_str("audio", 0)
 
@@ -563,17 +602,20 @@ def get_generation_prompt(cls, stt_params: SpeechToTextParams) -> PromptType:
                 f"Unsupported task_type '{task_type}'. "
                 "Supported task types are 'transcribe' and 'translate'."
             )
-        full_lang_name_to = cls.supported_languages.get(to_language, to_language)
-        if to_language is None:
-            prompt = (
-                f"<|im_start|>user\n{audio_placeholder}<|im_end|>\n"
-                f"<|im_start|>assistant\n"
-            )
-        else:
-            prompt = (
-                f"<|im_start|>user\n{audio_placeholder}<|im_end|>\n"
-                f"<|im_start|>assistant\nlanguage {full_lang_name_to}{_ASR_TEXT_TAG}"
-            )
+
+        context = _sanitize_transcription_user_text(request_prompt)
+        system_turn = f"<|im_start|>system\n{context}<|im_end|>\n" if context else ""
+
+        prompt = (
+            f"{system_turn}"
+            f"<|im_start|>user\n{audio_placeholder}<|im_end|>\n"
+            f"<|im_start|>assistant\n"
+        )
+
+        lang_code = to_language if task_type == "translate" else language
+        if lang_code is not None:
+            full_lang_name = cls.supported_languages.get(lang_code, lang_code)
+            prompt += f"language {full_lang_name}{_ASR_TEXT_TAG}"
 
         prompt_token_ids = tokenizer.encode(prompt)