feat(trtllm): [cherry-pick 1.7.0] OpenEngine sidecar audio support (#16871)

.github/workflows/shared-test-trtllm-nightly-image.yml

@@ -27,7 +27,9 @@ jobs:
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
       - name: Test nightly selection and media harness
-        run: python3 -m unittest discover -s lib/sidecar/trtllm/ci/tests -v
+        run: |
+          python3 -m pip install 'PyYAML==6.0.3'
+          python3 -m unittest discover -s lib/sidecar/trtllm/ci/tests -v
       - name: Resolve latest upstream nightly
         id: config
         run: >-
@@ -61,7 +63,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        media: [image, video]
+        media: [image, video, audio]
     needs: build
     runs-on: [gpu-1, amd64, v2, us-west-2]
     timeout-minutes: 30

lib/sidecar/testkit/tests/support/trtllm.rs

@@ -42,9 +42,7 @@ pub struct Discovery {
     pub calls: AtomicUsize,
     pub unavailable: AtomicBool,
     pub hold: AtomicBool,
-    pub portable_multimodal: AtomicBool,
-    pub advertises_image: AtomicBool,
-    pub advertises_video: AtomicBool,
+    pub model_info: Mutex<Option<pb::ModelInfo>>,
     pub omit_context_length: AtomicBool,
     pub error: Mutex<Option<tonic::Code>>,
     release: tokio::sync::Notify,
@@ -213,32 +211,15 @@ impl Control for ControlledControl {
         if let Some(code) = *self.discovery.error.lock().unwrap() {
             return Err(Status::new(code, "model discovery rejected"));
         }
-        let mut response = self.inner.get_model_info(request).await?;
-        let info = response.get_mut();
+        let model_info = self.discovery.model_info.lock().unwrap().clone();
+        let mut response = if let Some(info) = model_info {
+            Response::new(info)
+        } else {
+            self.inner.get_model_info(request).await?
+        };
         if self.discovery.omit_context_length.load(Ordering::SeqCst) {
-            info.max_context_length = None;
+            response.get_mut().max_context_length = None;
         }
-        info.supports_multimodal = Some(self.discovery.portable_multimodal.load(Ordering::SeqCst));
-        let modalities: Vec<_> = [
-            (
-                self.discovery.advertises_image.load(Ordering::SeqCst),
-                "image",
-            ),
-            (
-                self.discovery.advertises_video.load(Ordering::SeqCst),
-                "video",
-            ),
-        ]
-        .into_iter()
-        .filter_map(|(enabled, modality)| enabled.then_some(modality))
-        .collect();
-        info.extra = Some(
-            dynamo_sidecar_common::json_to_struct(
-                serde_json::json!({"trtllm_supported_modalities": modalities}),
-                "test capabilities",
-            )
-            .unwrap(),
-        );
         Ok(response)
     }
 

lib/sidecar/testkit/tests/trtllm_multimodal.rs

@@ -72,15 +72,23 @@ fn finished() -> pb::GenerateResponse {
     }
 }
 
-fn advertise_images(fixture: &Fixture) {
-    fixture
-        .discovery
-        .portable_multimodal
-        .store(true, Ordering::SeqCst);
-    fixture
-        .discovery
-        .advertises_image
-        .store(true, Ordering::SeqCst);
+fn advertise_media(fixture: &Fixture, modalities: &[&str]) {
+    *fixture.discovery.model_info.lock().unwrap() = Some(pb::ModelInfo {
+        model_id: "mocker-model".into(),
+        served_model_name: "mocker-model".into(),
+        max_context_length: Some(32768),
+        supports_multimodal: Some(true),
+        supports_text_input: Some(true),
+        supports_token_ids_input: Some(true),
+        extra: Some(
+            dynamo_sidecar_common::json_to_struct(
+                serde_json::json!({"trtllm_supported_modalities": modalities}),
+                "test capabilities",
+            )
+            .unwrap(),
+        ),
+        ..Default::default()
+    });
 }
 
 async fn wait_for_calls(fixture: &Fixture, count: usize) {
@@ -96,7 +104,7 @@ async fn wait_for_calls(fixture: &Fixture, count: usize) {
 async fn images_preserve_order_uuids_and_engine_expanded_usage() {
     let control = Controller::<Adapter>::default();
     let mut fixture = Fixture::start(control.clone(), FixtureConfig::default()).await;
-    advertise_images(&fixture);
+    advertise_media(&fixture, &["image"]);
     let engine = fixture.engine().await;
     engine.start(0).await.unwrap();
     let ctx = mock_context();
@@ -193,7 +201,7 @@ async fn formatter_regrouped_tool_media_reaches_openengine_in_rendered_order() {
 
     let control = Controller::<Adapter>::default();
     let mut fixture = Fixture::start(control.clone(), FixtureConfig::default()).await;
-    advertise_images(&fixture);
+    advertise_media(&fixture, &["image"]);
     let engine = fixture.engine().await;
     engine.start(0).await.unwrap();
     let ctx = mock_context();
@@ -232,23 +240,19 @@ async fn both_media_capability_signals_are_required() {
         for (portable, advertised) in [(true, false), (false, true)] {
             let control = Controller::<Adapter>::default();
             let mut fixture = Fixture::start(control.clone(), FixtureConfig::default()).await;
+            let mut modalities = vec![if video { "image" } else { "video" }];
+            if advertised {
+                modalities.push(if video { "video" } else { "image" });
+            }
+            advertise_media(&fixture, &modalities);
             fixture
                 .discovery
-                .portable_multimodal
-                .store(portable, Ordering::SeqCst);
-            let (target, other) = if video {
-                (
-                    &fixture.discovery.advertises_video,
-                    &fixture.discovery.advertises_image,
-                )
-            } else {
-                (
-                    &fixture.discovery.advertises_image,
-                    &fixture.discovery.advertises_video,
-                )
-            };
-            target.store(advertised, Ordering::SeqCst);
-            other.store(true, Ordering::SeqCst);
+                .model_info
+                .lock()
+                .unwrap()
+                .as_mut()
+                .unwrap()
+                .supports_multimodal = Some(portable);
             let engine = fixture.engine().await;
             engine.start(0).await.unwrap();
             let ctx = mock_context();
@@ -278,14 +282,7 @@ async fn videos_recover_discovery_and_preserve_sources_uuids_and_usage() {
     let control = Controller::<Adapter>::default();
     let mut fixture = Fixture::start(control.clone(), FixtureConfig::default()).await;
     fixture.discovery.unavailable.store(true, Ordering::SeqCst);
-    fixture
-        .discovery
-        .portable_multimodal
-        .store(true, Ordering::SeqCst);
-    fixture
-        .discovery
-        .advertises_video
-        .store(true, Ordering::SeqCst);
+    advertise_media(&fixture, &["video"]);
     let mut startup = tokio::spawn(bootstrap(&fixture, Some(32768), 5));
     wait_for_calls(&fixture, 2).await;
     assert!(!startup.is_finished());
@@ -359,11 +356,7 @@ async fn startup_recovers_media_capabilities_before_registration() {
     for omit_window in [false, true] {
         let control = Controller::<Adapter>::default();
         let mut fixture = Fixture::start(control.clone(), FixtureConfig::default()).await;
-        advertise_images(&fixture);
-        fixture
-            .discovery
-            .advertises_video
-            .store(true, Ordering::SeqCst);
+        advertise_media(&fixture, &["image", "video"]);
         fixture
             .discovery
             .omit_context_length
@@ -508,11 +501,7 @@ async fn shutdown_cancels_retrying_and_stalled_bootstrap_discovery() {
 async fn media_window_override_is_rejected_before_dispatch() {
     let control = Controller::<Adapter>::default();
     let mut fixture = Fixture::start(control.clone(), FixtureConfig::default()).await;
-    advertise_images(&fixture);
-    fixture
-        .discovery
-        .advertises_video
-        .store(true, Ordering::SeqCst);
+    advertise_media(&fixture, &["image", "video"]);
     let engine = fixture.engine_with_context_length(4096);
     engine.start(0).await.unwrap();
     for (video, max_tokens) in [
@@ -538,6 +527,57 @@ async fn media_window_override_is_rejected_before_dispatch() {
     fixture.shutdown().await;
 }
 
+#[tokio::test]
+async fn text_only_audio_preserves_whisper_task_prompt_with_empty_token_ids() {
+    let control = Controller::<Adapter>::default();
+    let mut fixture = Fixture::start(control.clone(), FixtureConfig::default()).await;
+    advertise_media(&fixture, &["audio"]);
+    fixture
+        .discovery
+        .model_info
+        .lock()
+        .unwrap()
+        .as_mut()
+        .unwrap()
+        .supports_token_ids_input = Some(false);
+    let engine = fixture.engine().await;
+    engine.start(0).await.unwrap();
+    let uri = "data:audio/wav;base64,YXVkaW8=";
+    for prompt in [
+        "",
+        "<|startoftranscript|><|fr|><|translate|><|notimestamps|>",
+    ] {
+        let mut req = request("mocker-model", vec![], 16);
+        req.extra_args = Some(serde_json::json!({"formatted_prompt": prompt}));
+        req.multi_modal_data = Some(HashMap::from([(
+            "audio_url".into(),
+            vec![MultimodalData::RawUrl(uri.into())],
+        )]));
+        let ctx = mock_context();
+        let handle = control.request(ctx.id(), RequestPlan::default());
+        fixture.respond(ctx.id(), vec![finished()]);
+        let outputs = collect(&engine, req, GenerateContext::new(ctx, None)).await;
+        assert_eq!(
+            outputs.last().unwrap().as_ref().unwrap().finish_reason,
+            Some(FinishReason::Stop)
+        );
+        let native = handle.native_request().unwrap();
+        assert_eq!(
+            native.input,
+            Some(pb::generate_request::Input::Prompt(prompt.into()))
+        );
+        assert_eq!(native.media.len(), 1);
+        assert_eq!(native.media[0].modality, pb::Modality::Audio as i32);
+        assert_eq!(
+            native.media[0].source,
+            Some(pb::media_item::Source::DataUri(uri.into()))
+        );
+    }
+    assert_eq!(fixture.discovery.calls.load(Ordering::SeqCst), 1);
+    engine.cleanup().await.unwrap();
+    fixture.shutdown().await;
+}
+
 #[test]
 fn remote_url_policy_is_enforced_before_engine_dispatch() {
     // Isolate startup environment changes from the other parallel tests.
@@ -590,11 +630,7 @@ fn remote_url_policy_is_enforced_before_engine_dispatch() {
 async fn check_remote_url_policy(allow_remote: bool) {
     let control = Controller::<Adapter>::default();
     let mut fixture = Fixture::start(control.clone(), FixtureConfig::default()).await;
-    advertise_images(&fixture);
-    fixture
-        .discovery
-        .advertises_video
-        .store(true, Ordering::SeqCst);
+    advertise_media(&fixture, &["image", "video", "audio"]);
     let engine = fixture.engine().await;
     engine.start(0).await.unwrap();
     for (key, media) in [
@@ -630,6 +666,22 @@ async fn check_remote_url_policy(allow_remote: bool) {
             "video_url",
             MultimodalData::RawUrl("data:video/mp4;base64,dmlkZW8=".into()),
         ),
+        (
+            "audio_url",
+            MultimodalData::RawUrl("http://example.com/audio.wav".into()),
+        ),
+        (
+            "audio_url",
+            MultimodalData::RawUrl("HTTPS://example.com/audio.wav".into()),
+        ),
+        (
+            "audio_url",
+            MultimodalData::Url("https://example.com/audio.wav".parse().unwrap()),
+        ),
+        (
+            "audio_url",
+            MultimodalData::RawUrl("data:audio/wav;base64,YXVkaW8=".into()),
+        ),
     ] {
         let inline = matches!(&media, MultimodalData::RawUrl(url) if url.starts_with("data:"));
         let mut req = image_request();

lib/sidecar/trtllm/README.md

@@ -31,7 +31,7 @@ supported, even when `--context-length` is supplied.
 > The standalone launch and deployment examples use
 > `nvcr.io/nvidia/tensorrt-llm/release:1.4.0rc3.dev202610290007`, which includes
 > image/video URL passthrough, native KVCM2 event streaming, and OpenEngine routing load support.
-> The nightly CI job tests image and video passthrough against the latest regular upstream nightly;
+> The nightly CI job tests image, video, and audio passthrough against the latest regular upstream nightly;
 > it also supports manual dispatch. The examples below stay pinned to the validated version.
 
 ## Support matrix
@@ -67,6 +67,45 @@ curl -s localhost:8000/v1/chat/completions \
   -d '{"model":"Qwen/Qwen3-0.6B","messages":[{"role":"user","content":"Hello"}],"max_tokens":32}'
 ```
 
+### Audio (Whisper)
+
+Audio requires a runtime containing upstream audio support, such as the validated
+`nvcr.io/nvidia/tensorrt-llm/release:1.4.0rc3.dev202610300406`; the older image/video
+example pin above predates audio. The nightly audio lane runs real Whisper-tiny
+inference and checks the transcript and authoritative token usage. Full frontend
+HTTP CI is tracked separately from that adapter-to-engine test.
+
+Whisper-tiny has no chat template and accepts text task prefixes rather than
+token-ID input. Supply `--custom-jinja-template` to the sidecar. An empty template
+selects the checkpoint's default transcription task; a template may instead emit
+a complete language/task prefix, for example
+`<|startoftranscript|><|fr|><|translate|><|notimestamps|>`.
+
+Run these processes in separate terminals with the appropriate Dynamo and
+TensorRT-LLM environments, using `DYN_DISCOVERY_BACKEND=file` on a single host:
+
+```bash
+python3 -m dynamo.frontend --migration-limit 0
+
+# In the audio-capable TensorRT-LLM environment, from the repository root:
+python3 -m tensorrt_llm.commands.serve openai/whisper-tiny \
+  --grpc --grpc-protocol openengine --host 127.0.0.1 --port 30001 \
+  --max_seq_len 1500 --extra_llm_api_options lib/sidecar/trtllm/ci/whisper.yaml
+
+# In the Dynamo environment:
+WHISPER_TEMPLATE=$(mktemp --suffix=.jinja)
+dynamo-trtllm-sidecar --grpc-endpoint 127.0.0.1:30001 \
+  --model-path openai/whisper-tiny --custom-jinja-template "$WHISPER_TEMPLATE"
+```
+
+Send audio using a chat content item with `"type":"audio_url"` and
+`"audio_url":{"url":"https://.../speech.wav"}` (or a data URI).
+For text-input models, **keep frontend `--migration-limit 0`**: retry migration
+currently extends token IDs, while these engines receive the original formatted
+text, which can restart a partially delivered transcript. Worker-advertised
+migration restrictions need separate frontend support. The aggregated launcher
+preserves explicit engine settings, including `guided_decoding_backend: null`.
+
 ## Deploy on Kubernetes
 
 The manifests pin compatible Dynamo frontend and TensorRT-LLM runtime nightlies

lib/sidecar/trtllm/ci/nightly.json

@@ -1,5 +1,9 @@
 {
   "repository": "nvcr.io/nvidia/tensorrt-llm/release",
   "model": "Qwen/Qwen3-VL-2B-Instruct",
-  "model_revision": "89644892e4d85e24eaac8bacfd4f463576704203"
+  "model_revision": "89644892e4d85e24eaac8bacfd4f463576704203",
+  "audio": {
+    "model": "openai/whisper-tiny",
+    "model_revision": "169d4a4341b33bc18d8881c4b69c2e104e1cc0af"
+  }
 }