feat(trtllm): support audio through the OpenEngine sidecar

.github/filters.yaml

@@ -377,8 +377,9 @@ planner:
   - '!**/*.rst'
 
 sidecar_trtllm_image:
-  # PR CI runs resolver CPU tests; the GPU test runs independently in nightly CI.
+  # PR CI runs nightly resolver/harness CPU tests; GPU cases run in nightly CI.
   - 'lib/sidecar/trtllm/ci/**'
+  - 'lib/llm/tests/data/media/triangle_240p_10.mp4'
   - '.github/actions/changed-files/**'
   - '.github/filters.yaml'
   - '.github/workflows/pr.yaml'

.github/scripts/test-filters.js

@@ -60,6 +60,11 @@ function checkFilter(file, patterns) {
 // Test cases: [file, expectations, description]
 // expectations: { filterName: expectedValue, ... }
 const testCases = [
+  {
+    file: 'lib/llm/tests/data/media/triangle_240p_10.mp4',
+    expect: { sidecar_trtllm_image: true },
+    desc: 'the video fixture triggers the nightly harness checksum tests'
+  },
   {
     file: 'lib/sidecar/trtllm/ci/resolve_nightly.py',
     expect: { sidecar_trtllm_image: true },

.github/workflows/nightly-ci.yml

@@ -1032,7 +1032,7 @@ jobs:
       sidecar_binary_artifact: ${{ needs.sidecar-build.outputs.binary_artifact }}
     secrets: inherit
 
-  sidecar-trtllm-nightly-image-test:
+  sidecar-trtllm-nightly-media-test:
     needs: [create-fresh-builder, compute-release-mode]
     if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }}
     uses: $/.github/workflows/shared-test-trtllm-nightly-image.yml
@@ -1748,7 +1748,7 @@ jobs:
     name: Clean K8s builder if exists
     runs-on: prod-default-small-v2
     if: always()
-    needs: [vllm-build, sglang-build, trtllm-build, dynamo-pipeline, operator-build, planner-build, frontend-build, vllm-efa-build, sglang-efa-build, trtllm-efa-build, sidecar-build, sidecar-trtllm-nightly-image-test, create-fresh-builder]
+    needs: [vllm-build, sglang-build, trtllm-build, dynamo-pipeline, operator-build, planner-build, frontend-build, vllm-efa-build, sglang-efa-build, trtllm-efa-build, sidecar-build, sidecar-trtllm-nightly-media-test, create-fresh-builder]
     permissions:
       contents: read
     steps:
@@ -1770,7 +1770,7 @@ jobs:
   ############################## SLACK NOTIFICATION ##############################
   notify-slack:
     if: always()
-    needs: [vllm-test, vllm-multi-gpu-test, vllm-4-gpu-test, vllm-h100-test, sglang-test, sglang-multi-gpu-test, sglang-4-gpu-test, sglang-h100-test, trtllm-test, trtllm-multi-gpu-test, trtllm-h100-test, sidecar-build, sidecar-vllm-test, sidecar-sglang-test, sidecar-trtllm-test, sidecar-trtllm-nightly-image-test, deploy-cleanup, deepseek-v4-pro-agg-perf, deepseek-v4-pro-disagg-perf, rust-tests, kimi-k25-agg-perf, kimi-k25-disagg-perf]
+    needs: [vllm-test, vllm-multi-gpu-test, vllm-4-gpu-test, vllm-h100-test, sglang-test, sglang-multi-gpu-test, sglang-4-gpu-test, sglang-h100-test, trtllm-test, trtllm-multi-gpu-test, trtllm-h100-test, sidecar-build, sidecar-vllm-test, sidecar-sglang-test, sidecar-trtllm-test, sidecar-trtllm-nightly-media-test, deploy-cleanup, deepseek-v4-pro-agg-perf, deepseek-v4-pro-disagg-perf, rust-tests, kimi-k25-agg-perf, kimi-k25-disagg-perf]
     permissions:
       contents: read
       actions: read   # grant the reusable notifier read access to list this run's jobs

.github/workflows/shared-test-trtllm-nightly-image.yml

@@ -1,7 +1,7 @@
 # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-name: TRT-LLM Sidecar Upstream Nightly Image
+name: TRT-LLM Sidecar Upstream Nightly Media
 
 on:
   workflow_call:
@@ -19,14 +19,14 @@ permissions:
 
 jobs:
   build:
-    name: Build image integration client
+    name: Build media integration client
     runs-on: prod-builder-v3
     timeout-minutes: 60
     outputs:
       image: ${{ steps.config.outputs.image }}
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
-      - name: Test nightly selection
+      - name: Test nightly selection and media harness
         run: python3 -m unittest discover -s lib/sidecar/trtllm/ci/tests -v
       - name: Resolve latest upstream nightly
         id: config
@@ -57,7 +57,11 @@ jobs:
           retention-days: 7
 
   test:
-    name: Upstream TRT-LLM image integration (1 GPU)
+    name: Upstream TRT-LLM ${{ matrix.media }} integration (1 GPU)
+    strategy:
+      fail-fast: false
+      matrix:
+        media: [image, video, audio]
     needs: build
     runs-on: [gpu-1, amd64, v2, us-west-2]
     timeout-minutes: 30
@@ -84,18 +88,24 @@ jobs:
           set -euo pipefail
           source lib/sidecar/trtllm/launch/common.sh
           trtllm_ensure_openengine_bindings python3
-      - name: Run live image integration test
+      - name: Install TRT-LLM video decoder
+        if: matrix.media == 'video'
+        run: >-
+          python3 components/src/dynamo/common/utils/install_media_decoders.py trtllm
+      - name: Run live ${{ matrix.media }} integration test
         env:
           HF_TOKEN: ${{ secrets.HF_TOKEN }}
         run: >-
           python3 lib/sidecar/trtllm/ci/run_image_test.py
           --binary trtllm-image-tests/trtllm-image-tests
           --config trtllm-image-tests/nightly.json
+          --media ${{ matrix.media }}
+          --results test-results/trtllm-nightly-${{ matrix.media }}
       - name: Upload engine logs and test results
         if: always()
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f  # v6.0.0
         with:
-          name: trtllm-nightly-image-results-${{ github.sha }}
-          path: test-results/trtllm-nightly-image/
+          name: trtllm-nightly-${{ matrix.media }}-results-${{ github.sha }}
+          path: test-results/trtllm-nightly-${{ matrix.media }}/
           if-no-files-found: warn
           retention-days: 7

lib/backend-common/src/engine.rs

@@ -28,7 +28,8 @@ pub use dynamo_llm::protocols::common::llm_backend::{
 };
 pub use dynamo_llm::protocols::common::preprocessor::{
     BootstrapInfo, KV_HINT_TRANSFER_CAPABILITY_KEY, KvHint, KvHintAction, KvSourceLocationsPayload,
-    MultimodalData, MultimodalDataMap, PrefillResult, PreprocessedRequest, RoutingHints,
+    MultimodalData, MultimodalDataMap, MultimodalItemRef, PrefillResult, PreprocessedRequest,
+    RoutingHints,
 };
 pub use dynamo_llm::protocols::common::{
     FinishReason, GuidedDecodingOptions, OutputOptions, SamplingOptions, StopConditions,

lib/backend-common/src/lib.rs

@@ -38,8 +38,8 @@ pub use engine::{
     FinishReason, FirstTokenNotifier, GenerateContext, GuidedDecodingOptions, HEALTH_CHECK_KEY,
     KV_HINT_TRANSFER_CAPABILITY_KEY, KvEventPublisher, KvEventSource, KvHint, KvHintAction,
     KvSourceLocationsPayload, LLMEngine, LLMEngineOutput, LLMEngineOutputExt, LlmRegistration,
-    LogProbs, Metrics, MetricsBindings, MetricsCtx, MultimodalData, OnPublisherReady,
-    OnSnapshotPublisherReady, OutputOptions, PrefillResult, PreprocessedRequest,
+    LogProbs, Metrics, MetricsBindings, MetricsCtx, MultimodalData, MultimodalItemRef,
+    OnPublisherReady, OnSnapshotPublisherReady, OutputOptions, PrefillResult, PreprocessedRequest,
     PromptTokensDetails, RawEngine, SamplingOptions, StopConditions, StopReason, TopLogprob,
     TopLogprobs, chunk, usage,
 };

lib/llm/src/preprocessor.rs

@@ -71,7 +71,8 @@ use crate::model_card::{ModelDeploymentCard, ModelInfo, PromptFormatterArtifact}
 use crate::preprocessor::media::MediaFetcher;
 use crate::preprocessor::media::{MediaDecoder, MediaLoader, max_data_url_bytes};
 use crate::protocols::common::preprocessor::{
-    MultimodalData, MultimodalDataMap, MultimodalUuidMap, PreprocessedRequestBuilder, RoutingHints,
+    MultimodalData, MultimodalDataMap, MultimodalItemRef, MultimodalUuidMap,
+    PreprocessedRequestBuilder, RoutingHints,
 };
 use crate::protocols::common::timing::RequestTracker;
 use crate::tokenizers::Encoding;
@@ -4111,6 +4112,7 @@ impl OpenAIPreprocessor {
 
         let mut media_map: MultimodalDataMap = HashMap::new();
         let mut uuid_map: MultimodalUuidMap = HashMap::new();
+        let mut media_order: Vec<MultimodalItemRef> = Vec::new();
         let mut has_user_uuid = false;
         // Decoded results are written back into these reserved modality slots so
         // URL-backed and UUID-only inputs retain request order.
@@ -4190,6 +4192,10 @@ impl OpenAIPreprocessor {
 
                 let slots = media_map.entry(type_str.to_string()).or_default();
                 let slot_idx = slots.len();
+                media_order.push(MultimodalItemRef {
+                    modality: type_str.to_string(),
+                    index: slot_idx,
+                });
                 has_user_uuid |= uuid.is_some();
                 uuid_map
                     .entry(type_str.to_string())
@@ -4493,6 +4499,7 @@ impl OpenAIPreprocessor {
 
         if !media_map.is_empty() {
             builder.multi_modal_data(Some(media_map));
+            builder.multi_modal_order(Some(media_order));
             if has_user_uuid {
                 builder.multi_modal_uuids(Some(uuid_map));
             }
@@ -9286,6 +9293,60 @@ mod extra_args_media_copy_tests {
             .to_string()
     }
 
+    #[tokio::test]
+    async fn multimodal_order_tracks_the_rendered_media_sequence() {
+        let preprocessor = test_preprocessor();
+        let image = inline_data_url();
+        let request: NvCreateChatCompletionRequest = serde_json::from_value(serde_json::json!({
+            "model": "alias",
+            "messages": [{
+                "role": "user",
+                "content": [
+                    {"type": "video_url", "video_url": {"url": "data:video/mp4;base64,AAAA"}},
+                    {"type": "image_url", "image_url": {"url": image}},
+                    {"type": "audio_url", "audio_url": {"url": "data:audio/wav;base64,AAAA"}},
+                    {"type": "video_url", "video_url": {"url": "https://example.com/second.mp4"}}
+                ]
+            }]
+        }))
+        .unwrap();
+        let mut builder = PreprocessedRequestBuilder::default();
+        builder
+            .model("alias".into())
+            .token_ids(Vec::new())
+            .stop_conditions(Default::default())
+            .sampling_options(Default::default())
+            .output_options(Default::default());
+
+        preprocessor
+            .gather_multi_modal_data_with_image_tokens(&request, &mut builder, None, &[])
+            .await
+            .unwrap();
+        let result = builder.build().unwrap();
+
+        assert_eq!(
+            result.multi_modal_order.unwrap(),
+            [
+                MultimodalItemRef {
+                    modality: "video_url".to_string(),
+                    index: 0,
+                },
+                MultimodalItemRef {
+                    modality: "image_url".to_string(),
+                    index: 0,
+                },
+                MultimodalItemRef {
+                    modality: "audio_url".to_string(),
+                    index: 0,
+                },
+                MultimodalItemRef {
+                    modality: "video_url".to_string(),
+                    index: 1,
+                },
+            ]
+        );
+    }
+
     #[tokio::test]
     async fn deepseek_v41_display_name_fallback_orders_tool_media() {
         use crate::common::checked_file::CheckedFile;

lib/llm/src/protocols/common/preprocessor.rs

@@ -384,6 +384,13 @@ pub type MultimodalDataMap = std::collections::HashMap<String, Vec<MultimodalDat
 /// Backend cache UUIDs aligned positionally with multimodal data slots.
 pub type MultimodalUuidMap = std::collections::HashMap<String, Vec<Option<String>>>;
 
+/// A reference into [`MultimodalDataMap`] preserving the media traversal order.
+#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
+pub struct MultimodalItemRef {
+    pub modality: String,
+    pub index: usize,
+}
+
 /// [`PreprocessedRequest`] is the internal representation of an LLM request. The `dynamo.llm-preprocessor`
 /// crate is responsible for converting request from the public APIs to this internal representation.
 #[derive(Serialize, Deserialize, Clone, Builder)]
@@ -443,6 +450,13 @@ pub struct PreprocessedRequest {
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub multi_modal_uuids: Option<MultimodalUuidMap>,
 
+    /// Media references in gather order: content parts, after the formatter's
+    /// message ordering. A chat template that regroups media by modality
+    /// renders a different placeholder order.
+    #[builder(default)]
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub multi_modal_order: Option<Vec<MultimodalItemRef>>,
+
     /// Optional multimodal routing-only fields (separate from execution payload).
     #[builder(default)]
     #[serde(default, skip_serializing_if = "Option::is_none")]

lib/sidecar/testkit/tests/support/trtllm.rs

@@ -41,8 +41,7 @@ pub struct Discovery {
     pub calls: AtomicUsize,
     pub unavailable: AtomicBool,
     pub hold: AtomicBool,
-    pub portable_image: AtomicBool,
-    pub advertises_image: AtomicBool,
+    pub model_info: Mutex<Option<pb::ModelInfo>>,
     pub omit_context_length: AtomicBool,
     pub error: Mutex<Option<tonic::Code>>,
     release: tokio::sync::Notify,
@@ -195,24 +194,15 @@ impl Control for ControlledControl {
         if let Some(code) = *self.discovery.error.lock().unwrap() {
             return Err(Status::new(code, "model discovery rejected"));
         }
-        let mut response = self.inner.get_model_info(request).await?;
-        let info = response.get_mut();
-        if self.discovery.omit_context_length.load(Ordering::SeqCst) {
-            info.max_context_length = None;
-        }
-        info.supports_multimodal = Some(self.discovery.portable_image.load(Ordering::SeqCst));
-        let modalities = if self.discovery.advertises_image.load(Ordering::SeqCst) {
-            vec!["image"]
+        let model_info = self.discovery.model_info.lock().unwrap().clone();
+        let mut response = if let Some(info) = model_info {
+            Response::new(info)
         } else {
-            vec![]
+            self.inner.get_model_info(request).await?
         };
-        info.extra = Some(
-            dynamo_sidecar_common::json_to_struct(
-                serde_json::json!({"trtllm_supported_modalities": modalities}),
-                "test capabilities",
-            )
-            .unwrap(),
-        );
+        if self.discovery.omit_context_length.load(Ordering::SeqCst) {
+            response.get_mut().max_context_length = None;
+        }
         Ok(response)
     }