From 54af7a675ca1835e80a9a2212b4b289213a9ebb8 Mon Sep 17 00:00:00 2001 From: krishung5 Date: Thu, 15 Oct 2026 11:16:15 -0700 Subject: [PATCH 01/61] feat(trtllm): support image inputs in sidecar Signed-off-by: krishung5 --- lib/llm/src/backend.rs | 206 ++++++++++++++- lib/sidecar/trtllm/src/client.rs | 25 ++ lib/sidecar/trtllm/src/convert.rs | 240 ++++++++++++++++-- lib/sidecar/trtllm/src/tests.rs | 55 +++- .../trtllm/src/tests/convert_request.rs | 178 +++++++++++++ lib/sidecar/trtllm/src/tests/disagg.rs | 53 ++++ lib/sidecar/trtllm/src/tests/e2e.rs | 180 +++++++++++++ lib/sidecar/trtllm/src/tests/engine.rs | 58 +++++ 8 files changed, 961 insertions(+), 34 deletions(-) diff --git a/lib/llm/src/backend.rs b/lib/llm/src/backend.rs index 3a73e46a21b0..4c5a2d80e0d7 100644 --- a/lib/llm/src/backend.rs +++ b/lib/llm/src/backend.rs @@ -261,8 +261,8 @@ impl return state.pending_flush.pop().map(|output| (output, state)); } - let output = loop { - let output = state.stream.next().await; + let (output, trailing_usage) = loop { + let mut output = state.stream.next().await; let choice_already_finished = output.as_ref().is_some_and(|output| { !output.is_event() && output.data.as_ref().is_some_and(|data| { @@ -272,7 +272,50 @@ impl }) }); if !choice_already_finished { - break output; + break (output, false); + } + + let carries_usage = output + .as_ref() + .and_then(|output| output.data.as_ref()) + .is_some_and(|data| data.completion_usage.is_some()); + let backend_failed = output + .as_ref() + .and_then(|output| output.data.as_ref()) + .is_some_and(|data| matches!(data.finish_reason, Some(FinishReason::Error(_)))); + if carries_usage || backend_failed { + // Native backends can put authoritative cumulative usage + // on a terminal event immediately after the token Dynamo + // recognized locally as EOS. Preserve that metadata without + // emitting duplicate content or a second successful finish + // event. Preserve a backend error with or without usage: it + // is not duplicate success metadata and must reach the + // frontend. + if let Some(data) = output.as_mut().and_then(|output| output.data.as_mut()) { + data.token_ids.clear(); + data.tokens = None; + data.text = None; + data.cum_log_probs = None; + data.log_probs = None; + data.top_logprobs = None; + if !backend_failed { + data.finish_reason = None; + data.stop_reason = None; + } + } + break (output, true); + } + + let is_terminal = output + .as_ref() + .and_then(|output| output.data.as_ref()) + .is_some_and(|data| data.finish_reason.is_some()); + if !is_terminal && state.finished_choices.len() >= state.decoders.len() { + // Once every choice ended locally, tolerate only terminal + // metadata. A nonterminal suffix means the backend ignored + // the stop; cancel rather than burning the remaining budget. + state.stream.context().stop_generating(); + return None; } }; @@ -281,6 +324,11 @@ impl // move to state.process_output // handle any error conditions / unwraps here + if trailing_usage { + state.finished = state.finished_choices.len() >= state.decoders.len(); + return Some((output, state)); + } + // events are pass thru if output.is_event() || output.data.is_none() { // A genuine stream-level error ends generation for every choice at @@ -497,13 +545,12 @@ impl .unwrap_or((None, None)); // If we detected a local stop condition, mark this choice as finished. - // Once all expected choices are finished, stop the upstream generator. + // Keep the upstream stream alive just long enough to consume + // a native backend's following terminal-usage event. The + // loop above still cancels on any nonterminal suffix once + // every choice has stopped. if finish_reason.is_some() && data.finish_reason.is_none() { state.finished_choices.insert(choice_idx); - if state.finished_choices.len() >= decoders_count { - state.stream.context().stop_generating(); - state.finished = true; - } } let text = result.text; @@ -1921,6 +1968,149 @@ mod tests { assert_eq!(generator.get_usage().completion_tokens, 2); } + #[tokio::test] + async fn local_eos_preserves_a_following_authoritative_usage_trailer() { + let tokenizer: Arc = Arc::new(CandidateDecoder); + let backend = Backend::from_tokenizer(Tokenizer::from(tokenizer)); + let request = PreprocessedRequest::builder() + .model("test-model".to_string()) + .token_ids(vec![]) + .stop_conditions(StopConditions { + stop_token_ids_hidden: Some(vec![103]), + ..Default::default() + }) + .sampling_options(SamplingOptions::default()) + .output_options(OutputOptions::default()) + .build() + .expect("valid preprocessed request"); + let reported = dynamo_protocols::types::CompletionUsage { + prompt_tokens: 82, + completion_tokens: 2, + total_tokens: 84, + prompt_tokens_details: None, + completion_tokens_details: None, + }; + let engine: ServerStreamingEngine> = + Arc::new(SyntheticSglangStopEngine { + outputs: Some(vec![ + // Dynamo recognizes 103 as EOS before the native backend's + // separate terminal event arrives. + LLMEngineOutput { + token_ids: vec![101, 103], + index: Some(0), + ..Default::default() + }, + LLMEngineOutput { + index: Some(0), + finish_reason: Some(FinishReason::Stop), + completion_usage: Some(reported.clone()), + ..Default::default() + }, + ]), + }); + + let outputs = Operator::generate(backend.as_ref(), SingleIn::new(request), engine) + .await + .expect("backend generation succeeds") + .collect::>() + .await; + assert_eq!(outputs.len(), 2); + + let local_terminal = outputs[0].data.as_ref().expect("local terminal"); + assert_eq!(local_terminal.finish_reason, Some(FinishReason::Stop)); + let usage_trailer = outputs[1].data.as_ref().expect("usage trailer"); + assert!(usage_trailer.token_ids.is_empty()); + assert!(usage_trailer.finish_reason.is_none()); + assert_eq!(usage_trailer.completion_usage.as_ref(), Some(&reported)); + + let options = DeltaGeneratorOptions::new(None, None, false, None); + let mut generator = DeltaGenerator::new( + "test-model".to_string(), + options, + "test-request".to_string(), + ); + for output in outputs { + generator + .choice_from_postprocessor(output.data.expect("backend output")) + .expect("OpenAI response conversion succeeds"); + } + assert_eq!(generator.get_usage(), reported); + } + + #[tokio::test] + async fn local_eos_does_not_mask_a_trailing_backend_error() { + let tokenizer: Arc = Arc::new(CandidateDecoder); + let backend = Backend::from_tokenizer(Tokenizer::from(tokenizer)); + let request = PreprocessedRequest::builder() + .model("test-model".to_string()) + .token_ids(vec![]) + .stop_conditions(StopConditions { + stop_token_ids_hidden: Some(vec![103]), + ..Default::default() + }) + .sampling_options(SamplingOptions::default()) + .output_options(OutputOptions::default()) + .build() + .expect("valid preprocessed request"); + let reported = dynamo_protocols::types::CompletionUsage { + prompt_tokens: 82, + completion_tokens: 2, + total_tokens: 84, + prompt_tokens_details: None, + completion_tokens_details: None, + }; + for completion_usage in [None, Some(reported.clone())] { + let engine: ServerStreamingEngine> = + Arc::new(SyntheticSglangStopEngine { + outputs: Some(vec![ + LLMEngineOutput { + token_ids: vec![101, 103], + index: Some(0), + ..Default::default() + }, + LLMEngineOutput { + index: Some(0), + finish_reason: Some(FinishReason::Error("worker failed".to_string())), + completion_usage: completion_usage.clone(), + ..Default::default() + }, + ]), + }); + + let outputs = + Operator::generate(backend.as_ref(), SingleIn::new(request.clone()), engine) + .await + .expect("backend generation succeeds") + .collect::>() + .await; + assert_eq!(outputs.len(), 2); + let error_trailer = outputs[1].data.as_ref().expect("error trailer"); + assert!(error_trailer.token_ids.is_empty()); + assert_eq!( + error_trailer.finish_reason, + Some(FinishReason::Error("worker failed".to_string())) + ); + assert_eq!( + error_trailer.completion_usage.as_ref(), + completion_usage.as_ref() + ); + + let options = DeltaGeneratorOptions::new(None, None, false, None); + let mut generator = DeltaGenerator::new( + "test-model".to_string(), + options, + "test-request".to_string(), + ); + generator + .choice_from_postprocessor(outputs[0].data.clone().expect("local terminal")) + .expect("the local terminal converts"); + let error = generator + .choice_from_postprocessor(outputs[1].data.clone().expect("error trailer")) + .expect_err("the backend error must reach the OpenAI response layer"); + assert!(error.to_string().contains("worker failed")); + } + } + #[tokio::test] async fn test_no_stop_trim_raw_completion_output() { use crate::protocols::common::{ diff --git a/lib/sidecar/trtllm/src/client.rs b/lib/sidecar/trtllm/src/client.rs index d1f4f046a014..16cea9315dfd 100644 --- a/lib/sidecar/trtllm/src/client.rs +++ b/lib/sidecar/trtllm/src/client.rs @@ -38,6 +38,11 @@ pub(crate) struct ModelLimits { /// Cap on generated tokens, when the engine reports one. A context window /// alone can imply a larger budget than the engine will accept. pub(crate) max_output_tokens: Option, + /// Whether this loaded model advertises image input through both the + /// portable multimodal bit and TensorRT-LLM's modality list. Requiring both + /// fails closed when a server and client disagree about the finer-grained + /// capability. + pub(crate) supports_image: bool, } pub(crate) struct TrtllmClient { @@ -102,9 +107,29 @@ impl TrtllmClient { }) .await? .into_inner(); + let supports_image = info.supports_multimodal == Some(true) + && info.extra.as_ref().is_some_and(|extra| { + extra + .fields + .get("trtllm_supported_modalities") + .and_then(|value| value.kind.as_ref()) + .is_some_and(|kind| match kind { + prost_types::value::Kind::ListValue(modalities) => { + modalities.values.iter().any(|value| { + matches!( + value.kind.as_ref(), + Some(prost_types::value::Kind::StringValue(modality)) + if modality == "image" + ) + }) + } + _ => false, + }) + }); Ok(ModelLimits { context_length: info.max_context_length.filter(|len| *len > 0), max_output_tokens: info.max_output_tokens.filter(|cap| *cap > 0), + supports_image, }) } diff --git a/lib/sidecar/trtllm/src/convert.rs b/lib/sidecar/trtllm/src/convert.rs index 3360818e2869..0c3e3d59d2f3 100644 --- a/lib/sidecar/trtllm/src/convert.rs +++ b/lib/sidecar/trtllm/src/convert.rs @@ -4,9 +4,10 @@ //! Conversion between Dynamo's `PreprocessedRequest` / `LLMEngineOutput` and the //! TensorRT-LLM OpenEngine (`openengine.v1`) protobuf messages. //! -//! Scope: aggregated and disaggregated (prefill/decode) generation. Multimodal, -//! LoRA, beam search, and `n > 1` are rejected before dispatch — the sidecar -//! streams a single sequence. +//! Scope: aggregated and disaggregated (prefill/decode) generation, including +//! image inputs when the loaded model advertises them. Video, audio, LoRA, +//! beam search, and `n > 1` are rejected before dispatch — the sidecar streams +//! a single sequence. //! //! Disaggregation is driven by [`DisaggregationMode`]: a prefill worker marks //! its request `context_only` and returns the `PrefillReady` handoff as its @@ -17,7 +18,7 @@ use std::collections::BTreeSet; use dynamo_backend_common::{ CompletionUsage, DisaggregationMode, DynamoError, FinishReason, LLMEngineOutput, - PreprocessedRequest, PromptTokensDetails, StopReason, TopLogprob, usage, + MultimodalData, PreprocessedRequest, PromptTokensDetails, StopReason, TopLogprob, usage, }; use crate::client::{self, ModelLimits}; @@ -36,6 +37,7 @@ pub(crate) fn build_generate_request( mode: DisaggregationMode, ) -> Result { validate_request(request, mode)?; + let media = image_media(request, limits)?; let sampling = &request.sampling_options; let stop = &request.stop_conditions; @@ -50,7 +52,7 @@ pub(crate) fn build_generate_request( // reject requests a wider decode window would serve. Releasing the handoff // when decode does reject is the sound fix and belongs on that side. let max_tokens = if mode.is_prefill() { - 1 + Some(1) } else { max_tokens(request, limits)? }; @@ -92,7 +94,7 @@ pub(crate) fn build_generate_request( num_sequences: Some(1), }), stopping: Some(pb::StoppingOptions { - max_tokens: Some(max_tokens), + max_tokens, // Forwarded on a context-only request too. The minimum only masks // EOS until the sequence reaches it, so it cannot extend a context // phase that `max_tokens` already caps at one token. Dropping it @@ -123,8 +125,7 @@ pub(crate) fn build_generate_request( prompt_logprob_start: None, }), guided: guided_decoding(request)?, - // Text generation only: no multimodal media or LoRA selection. - media: Vec::new(), + media, lora_name: String::new(), kv, extra: Some(disagg::request_extra( @@ -139,13 +140,14 @@ pub(crate) fn build_generate_request( } // Temporary workaround: the OpenEngine contract makes `stopping.max_tokens` -// optional, but an omitted value falls through to TensorRT-LLM's small -// `SamplingParams` default rather than filling the context. The Dynamo frontend -// forwards an omitted `max_tokens` as `None` expecting the backend to default, -// so we mirror the in-process backend's text-only default, +// optional. The Dynamo frontend forwards an omitted `max_tokens` as `None` +// expecting the backend to default. For text requests we mirror the in-process +// backend's default, // `max(1, context_length - prompt_len)` (components/src/dynamo/trtllm -// `_default_max_tokens`); the sidecar rejects multimodal before dispatch, so -// `token_ids.len()` is the true prompt length. `context_length` is resolved in +// `_default_max_tokens`). For image requests, only TensorRT-LLM knows the true +// prompt length after expanding image markers, so leave the field unset and let +// the OpenEngine request mapper derive the budget from the expanded prompt. +// `context_length` is resolved in // `engine::start`, where `--context-length` wins over the `GetModelInfo` report. // Only `/v1/chat/completions` and `/v1/responses` reach this fallback: they set // `PRESERVE_OMITTED_MAX_TOKENS_CONTEXT_KEY`, which stops the frontend supplying @@ -161,9 +163,12 @@ pub(crate) fn build_generate_request( fn max_tokens( request: &PreprocessedRequest, limits: Option, -) -> Result { +) -> Result, DynamoError> { if let Some(max_tokens) = request.stop_conditions.max_tokens { - return Ok(max_tokens); + return Ok(Some(max_tokens)); + } + if has_media(request) { + return Ok(None); } let context_length = limits .and_then(|limits| limits.context_length) @@ -185,7 +190,198 @@ fn max_tokens( None => remaining, }; reject_impossible_minimum(request, derived, context_length, prompt_len)?; - Ok(derived) + Ok(Some(derived)) +} + +const IMAGE_KEY: &str = "image_url"; +const MM_HASHES_KEY: &str = "mm_hashes"; + +/// Convert Dynamo's image-only execution payload into the ordered OpenEngine +/// media list. The frontend groups content parts by modality, so ordering is +/// retained within the image vector but not across different modalities. This +/// PR therefore accepts images only and fails explicitly on every other +/// non-empty modality instead of inventing a cross-modality order. +fn image_media( + request: &PreprocessedRequest, + limits: Option, +) -> Result, DynamoError> { + reject_media_override("mm_processor_kwargs", request.mm_processor_kwargs.as_ref())?; + reject_media_override("media_io_kwargs", request.media_io_kwargs.as_ref())?; + + let media_map = request.multi_modal_data.as_ref(); + if let Some(media_map) = media_map { + for (modality, items) in media_map { + if modality != IMAGE_KEY && !items.is_empty() { + return Err(client::invalid_argument(format!( + "multimodal modality `{modality}` is not supported by the TensorRT-LLM sidecar; this image-only implementation accepts `{IMAGE_KEY}`" + ))); + } + } + } + + let images = media_map + .and_then(|media| media.get(IMAGE_KEY)) + .map(Vec::as_slice) + .unwrap_or_default(); + + let user_image_uuids = if let Some(uuid_map) = request.multi_modal_uuids.as_ref() { + for (modality, uuids) in uuid_map { + if modality != IMAGE_KEY && !uuids.is_empty() { + return Err(client::invalid_argument(format!( + "multimodal UUIDs for `{modality}` cannot be forwarded by the image-only TensorRT-LLM sidecar" + ))); + } + } + uuid_map.get(IMAGE_KEY).map(Vec::as_slice) + } else { + None + }; + + if let Some(uuids) = user_image_uuids + && uuids.len() != images.len() + { + return Err(client::invalid_argument(format!( + "`{IMAGE_KEY}` UUID count ({}) must match its media item count ({})", + uuids.len(), + images.len() + ))); + } + + if images.is_empty() { + return Ok(Vec::new()); + } + if !limits.is_some_and(|limits| limits.supports_image) { + return Err(client::invalid_argument( + "the loaded TensorRT-LLM model does not advertise image support through Control.GetModelInfo", + )); + } + + // Explicit caller UUIDs remain authoritative. Otherwise carry the exact + // hashes generated by Dynamo's MM-aware preprocessor to the backend. This + // is request-side preparation for KV-aware routing; the router-owned + // `mm_routing_info` itself never crosses the worker boundary. + let has_user_uuid = user_image_uuids.is_some_and(|uuids| { + uuids + .iter() + .any(|uuid| uuid.as_deref().is_some_and(|uuid| !uuid.is_empty())) + }); + let routing_hashes = if has_user_uuid { + None + } else { + routing_hashes(request, images.len())? + }; + + images + .iter() + .enumerate() + .map(|(index, image)| { + let source = match image { + MultimodalData::Url(url) => media_source(url.as_str(), index)?, + MultimodalData::RawUrl(url) => media_source(url, index)?, + MultimodalData::Decoded(_) => { + return Err(client::invalid_argument(format!( + "`{IMAGE_KEY}` item {index} is pre-decoded; OpenEngine requires a URL or data URI so TensorRT-LLM can own image decoding" + ))); + } + MultimodalData::UuidOnly(_) => { + return Err(client::invalid_argument(format!( + "`{IMAGE_KEY}` item {index} contains only a cache UUID; OpenEngine requires an image source" + ))); + } + }; + Ok(pb::MediaItem { + modality: pb::Modality::Image as i32, + source: Some(source), + mime_type: String::new(), + uuid: if has_user_uuid { + user_image_uuids + .and_then(|uuids| uuids[index].as_deref()) + .unwrap_or_default() + .to_string() + } else { + routing_hashes + .as_ref() + .map(|hashes| hashes[index].clone()) + .unwrap_or_default() + }, + }) + }) + .collect() +} + +fn routing_hashes( + request: &PreprocessedRequest, + image_count: usize, +) -> Result>, DynamoError> { + let hashes = match request.extra_args.as_ref() { + Some(serde_json::Value::Object(extra)) => extra.get(MM_HASHES_KEY), + _ => None, + }; + let Some(hashes) = hashes else { + return Ok(None); + }; + let hashes = hashes.as_array().ok_or_else(|| { + client::invalid_argument("extra_args.mm_hashes must be an array of strings") + })?; + if hashes.is_empty() { + return Ok(None); + } + if hashes.len() != image_count { + return Err(client::invalid_argument(format!( + "extra_args.mm_hashes count ({}) must match the image media item count ({image_count})", + hashes.len() + ))); + } + hashes + .iter() + .enumerate() + .map(|(index, hash)| { + hash.as_str() + .filter(|hash| !hash.is_empty()) + .map(ToOwned::to_owned) + .ok_or_else(|| { + client::invalid_argument(format!( + "extra_args.mm_hashes[{index}] must be a non-empty string" + )) + }) + }) + .collect::, _>>() + .map(Some) +} + +fn media_source(value: &str, index: usize) -> Result { + let scheme = value + .split_once(':') + .map(|(scheme, _)| scheme.to_ascii_lowercase()); + match scheme.as_deref() { + Some("http") | Some("https") => Ok(pb::media_item::Source::Url(value.to_string())), + Some("data") => Ok(pb::media_item::Source::DataUri(value.to_string())), + _ => Err(client::invalid_argument(format!( + "`{IMAGE_KEY}` item {index} must use an http, https, or data URI" + ))), + } +} + +fn reject_media_override(name: &str, value: Option<&serde_json::Value>) -> Result<(), DynamoError> { + let is_empty = match value { + None | Some(serde_json::Value::Null) => true, + Some(serde_json::Value::Object(fields)) => fields.is_empty(), + Some(_) => false, + }; + if is_empty { + Ok(()) + } else { + Err(client::invalid_argument(format!( + "{name} is not supported by the TensorRT-LLM OpenEngine sidecar" + ))) + } +} + +fn has_media(request: &PreprocessedRequest) -> bool { + request + .multi_modal_data + .as_ref() + .is_some_and(|media| media.values().any(|items| !items.is_empty())) } /// Both the output cap and the `.max(1)` floor can land under an explicit @@ -341,14 +537,14 @@ fn validate_request( "prompt embeddings are not supported by the TensorRT-LLM sidecar", )); } - if request.multi_modal_data.is_some() - || request.mm_routing_info.is_some() - || request.encoder_result.is_some() - { + if request.encoder_result.is_some() { return Err(client::invalid_argument( - "multimodal requests are not supported by the TensorRT-LLM sidecar", + "multimodal encoder handoffs are not supported by the TensorRT-LLM sidecar", )); } + // `mm_routing_info` is deliberately not forwarded: it is framework-owned + // metadata that the Dynamo router consumes before selecting this worker. + // The execution payload remains `multi_modal_data`, mapped by `image_media`. // Only a decode worker consumes a prefill handoff. Seeing one anywhere else // means the frontend routed a decode request to the wrong worker role. if request.prefill_result.is_some() && !mode.is_decode() { diff --git a/lib/sidecar/trtllm/src/tests.rs b/lib/sidecar/trtllm/src/tests.rs index 411aa819176e..d6d3c3146762 100644 --- a/lib/sidecar/trtllm/src/tests.rs +++ b/lib/sidecar/trtllm/src/tests.rs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -use std::collections::BTreeSet; +use std::collections::{BTreeSet, HashMap}; use std::net::SocketAddr; use std::num::NonZeroUsize; use std::pin::Pin; @@ -10,8 +10,8 @@ use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::time::Duration; use dynamo_backend_common::{ - DisaggregationMode, ErrorType, FinishReason, GenerateContext, LLMEngine, OutputOptions, - PreprocessedRequest, SamplingOptions, StopConditions, StopReason, + DisaggregationMode, ErrorType, FinishReason, GenerateContext, LLMEngine, MultimodalData, + OutputOptions, PreprocessedRequest, SamplingOptions, StopConditions, StopReason, }; use dynamo_sidecar_common::{GrpcEndpoint, GrpcTransportConfig}; use futures::{Stream, StreamExt}; @@ -60,6 +60,10 @@ struct FakeTrtllm { empty_model_info: Arc, /// Answers `UNAVAILABLE`, the shape of an engine that is not serving yet. unavailable_model_info: Arc, + /// Portable coarse-grained model capability. + supports_multimodal: Arc, + /// TensorRT-LLM's finer-grained modality advertisement. + advertises_image: Arc, model_info_calls: Arc, } @@ -109,7 +113,10 @@ impl pb::inference_server::Inference for FakeTrtllm { } let request_id = request.request_id.clone(); - let prompt_tokens = prompt_len(&request); + // Image markers expand inside TensorRT-LLM after Dynamo tokenizes the + // text prompt. Simulate that so the integration test proves Dynamo + // trusts the engine's terminal usage rather than the unexpanded input. + let prompt_tokens = prompt_len(&request) + if request.media.is_empty() { 0 } else { 64 }; let wants_logprobs = wants_logprobs(&request); let context_only = is_context_only(&request); let hang = self.hang.load(Ordering::SeqCst); @@ -249,6 +256,14 @@ impl pb::control_server::Control for FakeTrtllm { if self.unavailable_model_info.load(Ordering::SeqCst) { return Err(Status::unavailable("not serving yet")); } + let supports_multimodal = self.supports_multimodal.load(Ordering::SeqCst); + let modalities = if self.advertises_image.load(Ordering::SeqCst) { + vec![prost_types::Value { + kind: Some(prost_types::value::Kind::StringValue("image".to_string())), + }] + } else { + Vec::new() + }; Ok(Response::new(pb::ModelInfo { model_id: "fake-model".to_string(), max_context_length: if self.empty_model_info.load(Ordering::SeqCst) { @@ -256,6 +271,19 @@ impl pb::control_server::Control for FakeTrtllm { } else { Some(4096) }, + supports_multimodal: Some(supports_multimodal), + extra: Some(prost_types::Struct { + fields: [( + "trtllm_supported_modalities".to_string(), + prost_types::Value { + kind: Some(prost_types::value::Kind::ListValue( + prost_types::ListValue { values: modalities }, + )), + }, + )] + .into_iter() + .collect(), + }), ..Default::default() })) } @@ -441,9 +469,28 @@ fn limits(context_length: u32) -> Option { Some(ModelLimits { context_length: Some(context_length), max_output_tokens: None, + supports_image: false, + }) +} + +fn image_limits(context_length: u32) -> Option { + Some(ModelLimits { + context_length: Some(context_length), + max_output_tokens: None, + supports_image: true, }) } +fn add_images(request: &mut PreprocessedRequest, images: impl IntoIterator) { + request.multi_modal_data = Some(HashMap::from([( + "image_url".to_string(), + images + .into_iter() + .map(|image| MultimodalData::RawUrl(image.to_string())) + .collect(), + )])); +} + fn engine(endpoint: &str, connections: usize) -> TrtllmSidecarEngine { engine_in_mode(endpoint, connections, AGG) } diff --git a/lib/sidecar/trtllm/src/tests/convert_request.rs b/lib/sidecar/trtllm/src/tests/convert_request.rs index e8dd43d172df..138057cb5dc8 100644 --- a/lib/sidecar/trtllm/src/tests/convert_request.rs +++ b/lib/sidecar/trtllm/src/tests/convert_request.rs @@ -72,6 +72,183 @@ fn request_maps_sampling_stop_and_output_fields() { } } +#[test] +fn image_urls_data_uris_and_uuids_are_mapped_in_order() { + let mut req = request(); + // Routing-only multimodal metadata has already served its purpose by the + // time the request reaches a worker and must not block engine delivery. + req.mm_routing_info = Some(Default::default()); + add_images( + &mut req, + [ + "https://example.com/first.png", + "data:image/png;base64,aW1hZ2U=", + ], + ); + req.multi_modal_uuids = Some(HashMap::from([( + "image_url".to_string(), + vec![Some("first-image".to_string()), None], + )])); + req.extra_args = Some(json!({ + "mm_hashes": ["routing-hash-one", "routing-hash-two"] + })); + + let proto = build_generate_request(&req, "req", "model", image_limits(4096), AGG) + .expect("image request"); + assert_eq!(proto.media.len(), 2); + assert_eq!(proto.media[0].modality, pb::Modality::Image as i32); + assert_eq!( + proto.media[0].source, + Some(pb::media_item::Source::Url( + "https://example.com/first.png".to_string() + )) + ); + assert_eq!(proto.media[0].uuid, "first-image"); + assert_eq!( + proto.media[1].source, + Some(pb::media_item::Source::DataUri( + "data:image/png;base64,aW1hZ2U=".to_string() + )) + ); + assert!(proto.media[1].uuid.is_empty()); +} + +#[test] +fn mm_routing_hashes_are_forwarded_as_image_uuids_in_order() { + let mut req = request(); + add_images( + &mut req, + [ + "https://example.com/first.png", + "https://example.com/second.png", + ], + ); + req.extra_args = Some(json!({ + "mm_hashes": ["0123456789abcdef", "fedcba9876543210"] + })); + + let proto = build_generate_request(&req, "req", "model", image_limits(4096), AGG) + .expect("image request"); + assert_eq!(proto.media[0].uuid, "0123456789abcdef"); + assert_eq!(proto.media[1].uuid, "fedcba9876543210"); +} + +#[test] +fn malformed_or_misaligned_mm_routing_hashes_are_rejected() { + for (hashes, expected) in [ + (json!("not-an-array"), "array of strings"), + (json!(["only-one"]), "must match"), + (json!(["one", ""]), "non-empty string"), + (json!(["one", 2]), "non-empty string"), + ] { + let mut req = request(); + add_images( + &mut req, + [ + "https://example.com/first.png", + "https://example.com/second.png", + ], + ); + req.extra_args = Some(json!({"mm_hashes": hashes})); + let error = build_generate_request(&req, "req", "model", image_limits(4096), AGG) + .expect_err("malformed routing hashes must fail closed"); + assert!( + error.to_string().contains(expected), + "unexpected error: {error}" + ); + } +} + +#[test] +fn image_media_requires_the_loaded_models_advertised_capability() { + let mut req = request(); + add_images(&mut req, ["https://example.com/image.png"]); + + for model_limits in [None, limits(4096)] { + let error = build_generate_request(&req, "req", "model", model_limits, AGG) + .expect_err("a text-only or unknown model must reject images"); + assert!( + error + .to_string() + .contains("does not advertise image support"), + "unexpected error: {error}" + ); + } +} + +#[test] +fn unsupported_media_modalities_and_sources_are_rejected() { + for (key, expected) in [ + ("video_url", "video_url"), + ("audio_url", "audio_url"), + ("future_media", "future_media"), + ] { + let mut req = request(); + req.multi_modal_data = Some(HashMap::from([( + key.to_string(), + vec![MultimodalData::RawUrl( + "https://example.com/media".to_string(), + )], + )])); + let error = build_generate_request(&req, "req", "model", image_limits(4096), AGG) + .expect_err("only images are supported"); + assert!( + error.to_string().contains(expected), + "unexpected error: {error}" + ); + } + + let mut req = request(); + add_images(&mut req, ["file:///tmp/image.png"]); + let error = build_generate_request(&req, "req", "model", image_limits(4096), AGG) + .expect_err("the engine cannot fetch a local Dynamo path"); + assert!(error.to_string().contains("http, https, or data URI")); + + let mut req = request(); + req.multi_modal_data = Some(HashMap::from([( + "image_url".to_string(), + vec![MultimodalData::UuidOnly("cached-image".to_string())], + )])); + let error = build_generate_request(&req, "req", "model", image_limits(4096), AGG) + .expect_err("OpenEngine cannot resolve a source-less UUID"); + assert!(error.to_string().contains("requires an image source")); +} + +#[test] +fn image_metadata_that_cannot_be_honored_is_rejected() { + let mut req = request(); + add_images(&mut req, ["https://example.com/image.png"]); + req.multi_modal_uuids = Some(HashMap::from([( + "image_url".to_string(), + vec![Some("one".to_string()), Some("two".to_string())], + )])); + let error = build_generate_request(&req, "req", "model", image_limits(4096), AGG) + .expect_err("UUIDs must align positionally"); + assert!(error.to_string().contains("UUID count")); + + req.multi_modal_uuids = None; + req.mm_processor_kwargs = Some(serde_json::json!({"do_resize": false})); + let error = build_generate_request(&req, "req", "model", image_limits(4096), AGG) + .expect_err("processor overrides cannot be dropped"); + assert!(error.to_string().contains("mm_processor_kwargs")); + + req.mm_processor_kwargs = Some(serde_json::json!({})); + req.media_io_kwargs = Some(serde_json::json!({"image": {"mode": "RGB"}})); + let error = build_generate_request(&req, "req", "model", image_limits(4096), AGG) + .expect_err("media I/O overrides cannot be dropped"); + assert!(error.to_string().contains("media_io_kwargs")); +} + +#[test] +fn image_request_without_max_tokens_is_left_for_engine_derivation() { + let mut req = request(); + add_images(&mut req, ["https://example.com/image.png"]); + req.stop_conditions.max_tokens = None; + let proto = build_generate_request(&req, "req", "model", image_limits(4096), AGG) + .expect("the engine derives against the expanded prompt"); + assert_eq!(proto.stopping.expect("stopping").max_tokens, None); +} + #[test] fn omitted_max_tokens_without_context_length_is_rejected() { let mut req = request(); @@ -191,6 +368,7 @@ fn the_derived_max_tokens_respects_the_engines_output_cap() { Some(ModelLimits { context_length: Some(4096), max_output_tokens: Some(256), + supports_image: false, }), AGG, ) diff --git a/lib/sidecar/trtllm/src/tests/disagg.rs b/lib/sidecar/trtllm/src/tests/disagg.rs index 3436c037d346..612fda3f39fc 100644 --- a/lib/sidecar/trtllm/src/tests/disagg.rs +++ b/lib/sidecar/trtllm/src/tests/disagg.rs @@ -59,6 +59,59 @@ fn prefill_request_is_marked_context_only() { assert!(proto.kv.is_none(), "prefill carries no session to replay"); } +/// Image inputs must survive both RPC legs. TensorRT-LLM reconstructs its +/// multimodal position/hash state while resuming the KV session, so the decode +/// request needs the same ordered media metadata as the context request. +#[test] +fn image_media_is_preserved_across_prefill_and_decode_requests() { + let mut req = request(); + add_images( + &mut req, + [ + "https://example.com/first.png", + "data:image/png;base64,c2Vjb25k", + ], + ); + req.extra_args = Some(json!({ + "mm_hashes": ["0123456789abcdef", "fedcba9876543210"] + })); + + let prefill = build_generate_request( + &req, + "req", + "model", + image_limits(4096), + DisaggregationMode::Prefill, + ) + .expect("build image prefill"); + assert!(is_context_only(&prefill)); + assert_eq!(prefill.stopping.as_ref().unwrap().max_tokens, Some(1)); + assert_eq!(prefill.media.len(), 2); + assert_eq!(prefill.media[0].uuid, "0123456789abcdef"); + assert_eq!(prefill.media[1].uuid, "fedcba9876543210"); + + req.prefill_result = Some(dynamo_backend_common::PrefillResult { + disaggregated_params: crate::disagg::session_to_json(fake_session()) + .expect("encode handoff"), + prompt_tokens_details: None, + }); + let decode = build_generate_request( + &req, + "req", + "model", + image_limits(4096), + DisaggregationMode::Decode, + ) + .expect("build image decode"); + assert!(!is_context_only(&decode)); + assert_eq!(decode.media, prefill.media); + assert_eq!( + decode.kv.and_then(|kv| kv.session), + Some(fake_session()), + "decode must carry both the image metadata and the KV handoff" + ); +} + /// `PrefillReady` is the prefill worker's terminal chunk: no tokens, and the /// handoff the decode worker will replay. #[tokio::test] diff --git a/lib/sidecar/trtllm/src/tests/e2e.rs b/lib/sidecar/trtllm/src/tests/e2e.rs index 984391b2e9eb..f3b213378490 100644 --- a/lib/sidecar/trtllm/src/tests/e2e.rs +++ b/lib/sidecar/trtllm/src/tests/e2e.rs @@ -111,6 +111,107 @@ async fn e2e_real_disagg_handoff() { ); } +/// Exercises the same real prefill/decode handoff with an image request. Both +/// servers may share one sufficiently large GPU for local validation. +/// +/// ```text +/// TRTLLM_E2E_PREFILL_ENDPOINT=http://127.0.0.1:50051 \ +/// TRTLLM_E2E_DECODE_ENDPOINT=http://127.0.0.1:50052 \ +/// TRTLLM_E2E_MODEL=Qwen/Qwen3-VL-2B-Instruct \ +/// cargo test -p dynamo-trtllm-sidecar e2e_real_openengine_image_disagg -- --ignored --nocapture +/// ``` +#[tokio::test] +#[ignore = "requires two live image-capable OpenEngine servers with a KV transceiver"] +async fn e2e_real_openengine_image_disagg() { + let prefill_endpoint = std::env::var("TRTLLM_E2E_PREFILL_ENDPOINT") + .expect("set TRTLLM_E2E_PREFILL_ENDPOINT, e.g. http://127.0.0.1:50051"); + let decode_endpoint = std::env::var("TRTLLM_E2E_DECODE_ENDPOINT") + .expect("set TRTLLM_E2E_DECODE_ENDPOINT, e.g. http://127.0.0.1:50052"); + let model = std::env::var("TRTLLM_E2E_MODEL") + .unwrap_or_else(|_| "Qwen/Qwen3-VL-2B-Instruct".to_string()); + let configured = |source: String| ConfiguredModel { + source, + context_length: None, + }; + let prefill = TrtllmSidecarEngine::new( + GrpcEndpoint::parse(&prefill_endpoint, "--grpc-endpoint").expect("valid endpoint"), + transport(1), + configured(model.clone()), + DisaggregationMode::Prefill, + ); + let decode = TrtllmSidecarEngine::new( + GrpcEndpoint::parse(&decode_endpoint, "--grpc-endpoint").expect("valid endpoint"), + transport(1), + configured(model.clone()), + DisaggregationMode::Decode, + ); + prefill.start(0).await.expect("start image prefill worker"); + decode.start(0).await.expect("start image decode worker"); + + let token_ids = vec![ + 151644, 872, 198, 151652, 151655, 151653, 675, 279, 7987, 504, 2115, 311, 1290, 13, 151645, + 198, 151644, 77091, 198, + ]; + let image_request = || { + let mut request = PreprocessedRequest::builder() + .model(model.clone()) + .token_ids(token_ids.clone()) + .stop_conditions(StopConditions::default()) + .sampling_options(SamplingOptions { + temperature: Some(0.0), + ..Default::default() + }) + .output_options(OutputOptions::default()) + .build() + .expect("request"); + add_images( + &mut request, + [concat!( + "data:image/png;base64,", + "iVBORw0KGgoAAAANSUhEUgAAABwAAAAcCAIAAAD9b0jDAAAAK0lEQVR4nGP8z0AsYGQgVi0T0WaSAEYNHTV01NBRQ0cNHTV01NBRQwefoQBvcAI3xbX7+QAAAABJRU5ErkJggg==" + )], + ); + request + }; + + let prefill_outputs = collect(&prefill, image_request()).await; + assert!( + prefill_outputs + .iter() + .all(|output| output.token_ids.is_empty()), + "the image prefill worker must not stream tokens" + ); + let handoff = prefill_outputs + .iter() + .find_map(|output| output.disaggregated_params.clone()) + .expect("image prefill terminal carries the KV handoff"); + + let mut decode_request = image_request(); + decode_request.prefill_result = Some(dynamo_backend_common::PrefillResult { + disaggregated_params: handoff, + prompt_tokens_details: None, + }); + let decode_outputs = collect(&decode, decode_request).await; + let generated = decode_outputs + .iter() + .flat_map(|output| output.token_ids.iter().copied()) + .collect::>(); + let terminal = decode_outputs + .iter() + .find(|output| output.finish_reason.is_some()) + .expect("image decode terminal"); + let usage = terminal.completion_usage.as_ref().expect("terminal usage"); + + assert!( + !generated.is_empty(), + "the image decode worker must generate" + ); + assert!( + usage.prompt_tokens > token_ids.len() as u32, + "the disaggregated image prompt must report its expanded token count" + ); +} + /// Drives the real `TrtllmSidecarEngine` against a live OpenEngine gRPC server /// (a real TensorRT-LLM engine on a GPU). Ignored by default; run explicitly with /// a reachable endpoint: @@ -202,3 +303,82 @@ async fn e2e_real_openengine_server() { "logprobs were requested but none were surfaced" ); } + +/// Drives an image request through the real Rust sidecar conversion layer and +/// a live TensorRT-LLM OpenEngine server. The fixed token IDs are Qwen3-VL's +/// chat-template rendering of one image marker and a short question; the tiny +/// PNG is split red/blue so a manual run also produces an obvious answer. +/// +/// ```text +/// TRTLLM_E2E_ENDPOINT=http://127.0.0.1:50051 \ +/// TRTLLM_E2E_MODEL=Qwen/Qwen3-VL-2B-Instruct \ +/// cargo test -p dynamo-trtllm-sidecar e2e_real_openengine_image -- --ignored --nocapture +/// ``` +#[tokio::test] +#[ignore = "requires a live image-capable TensorRT-LLM OpenEngine server"] +async fn e2e_real_openengine_image() { + let endpoint = std::env::var("TRTLLM_E2E_ENDPOINT") + .expect("set TRTLLM_E2E_ENDPOINT, e.g. http://127.0.0.1:50051"); + let model = std::env::var("TRTLLM_E2E_MODEL") + .unwrap_or_else(|_| "Qwen/Qwen3-VL-2B-Instruct".to_string()); + + let engine = TrtllmSidecarEngine::new( + GrpcEndpoint::parse(&endpoint, "--grpc-endpoint").expect("valid endpoint"), + transport(1), + ConfiguredModel { + source: model.clone(), + context_length: None, + }, + AGG, + ); + engine + .start(0) + .await + .expect("start against the live image-capable server"); + + let token_ids = vec![ + 151644, 872, 198, 151652, 151655, 151653, 675, 279, 7987, 504, 2115, 311, 1290, 13, 151645, + 198, 151644, 77091, 198, + ]; + let mut request = PreprocessedRequest::builder() + .model(model) + .token_ids(token_ids.clone()) + // Leave max_tokens omitted: only TensorRT-LLM can derive the output + // budget after expanding the image placeholder. + .stop_conditions(StopConditions::default()) + .sampling_options(SamplingOptions { + temperature: Some(0.0), + ..Default::default() + }) + .output_options(OutputOptions::default()) + .build() + .expect("request"); + add_images( + &mut request, + [concat!( + "data:image/png;base64,", + "iVBORw0KGgoAAAANSUhEUgAAABwAAAAcCAIAAAD9b0jDAAAAK0lEQVR4nGP8z0AsYGQgVi0T0WaSAEYNHTV01NBRQ0cNHTV01NBRQwefoQBvcAI3xbX7+QAAAABJRU5ErkJggg==" + )], + ); + + let outputs = collect(&engine, request).await; + let generated = outputs + .iter() + .flat_map(|output| output.token_ids.iter().copied()) + .collect::>(); + let terminal = outputs + .iter() + .find(|output| output.finish_reason.is_some()) + .expect("terminal output"); + let usage = terminal.completion_usage.as_ref().expect("terminal usage"); + eprintln!( + "[e2e-image] generated={generated:?}; usage: prompt={}, completion={}", + usage.prompt_tokens, usage.completion_tokens + ); + + assert!(!generated.is_empty(), "expected image-conditioned output"); + assert!( + usage.prompt_tokens > token_ids.len() as u32, + "the engine's expanded image prompt usage must replace Dynamo's unexpanded token count" + ); +} diff --git a/lib/sidecar/trtllm/src/tests/engine.rs b/lib/sidecar/trtllm/src/tests/engine.rs index 415456ede063..149303a06d3b 100644 --- a/lib/sidecar/trtllm/src/tests/engine.rs +++ b/lib/sidecar/trtllm/src/tests/engine.rs @@ -41,6 +41,64 @@ async fn aggregated_generation_streams_delta_then_terminal() { } } +#[tokio::test] +async fn advertised_image_media_reaches_openengine_and_uses_expanded_usage() { + let service = FakeTrtllm::default(); + service.supports_multimodal.store(true, Ordering::SeqCst); + service.advertises_image.store(true, Ordering::SeqCst); + let server = FakeServer::start(service).await; + let engine = engine(&server.endpoint, 1); + engine.start(0).await.expect("start"); + + let mut req = request(); + req.stop_conditions.max_tokens = None; + add_images(&mut req, ["data:image/png;base64,aW1hZ2U="]); + let outputs = collect(&engine, req).await; + let usage = outputs + .last() + .and_then(|output| output.completion_usage.as_ref()) + .expect("terminal engine usage"); + assert_eq!(usage.prompt_tokens, 67, "3 text tokens + 64 image tokens"); + + let requests = server.service.requests.lock().await; + let sent = requests.first().expect("recorded request"); + assert_eq!( + sent.stopping.as_ref().expect("stopping").max_tokens, + None, + "the expanded image prompt budget belongs to TensorRT-LLM" + ); + assert_eq!(sent.media.len(), 1); + assert_eq!(sent.media[0].modality, pb::Modality::Image as i32); + assert!(matches!( + sent.media[0].source.as_ref(), + Some(pb::media_item::Source::DataUri(value)) if value.starts_with("data:image/png") + )); +} + +#[tokio::test] +async fn both_multimodal_capability_signals_are_required() { + for (coarse, image) in [(true, false), (false, true)] { + let service = FakeTrtllm::default(); + service.supports_multimodal.store(coarse, Ordering::SeqCst); + service.advertises_image.store(image, Ordering::SeqCst); + let server = FakeServer::start(service).await; + let engine = engine(&server.endpoint, 1); + engine.start(0).await.expect("start"); + + let mut req = request(); + add_images(&mut req, ["https://example.com/image.png"]); + let context = dynamo_backend_common::testing::mock_context(); + let result = engine + .generate(req, GenerateContext::new(context, None)) + .await; + assert!( + result.is_err(), + "capabilities ({coarse}, {image}) must fail closed" + ); + assert!(server.service.requests.lock().await.is_empty()); + } +} + #[tokio::test] async fn grpc_request_errors_are_propagated() { let service = FakeTrtllm::default(); From f369b565ce95c21fb0f9b82af52ca12356055df2 Mon Sep 17 00:00:00 2001 From: krishung5 Date: Thu, 15 Oct 2026 16:55:16 -0700 Subject: [PATCH 02/61] fix(trtllm): narrow image media cache identity Signed-off-by: krishung5 --- lib/llm/src/backend.rs | 206 +----------------- lib/sidecar/trtllm/src/client.rs | 14 +- lib/sidecar/trtllm/src/convert.rs | 86 ++------ lib/sidecar/trtllm/src/engine.rs | 8 +- lib/sidecar/trtllm/src/tests.rs | 10 +- .../trtllm/src/tests/convert_request.rs | 37 +--- lib/sidecar/trtllm/src/tests/disagg.rs | 14 +- 7 files changed, 51 insertions(+), 324 deletions(-) diff --git a/lib/llm/src/backend.rs b/lib/llm/src/backend.rs index 4c5a2d80e0d7..3a73e46a21b0 100644 --- a/lib/llm/src/backend.rs +++ b/lib/llm/src/backend.rs @@ -261,8 +261,8 @@ impl return state.pending_flush.pop().map(|output| (output, state)); } - let (output, trailing_usage) = loop { - let mut output = state.stream.next().await; + let output = loop { + let output = state.stream.next().await; let choice_already_finished = output.as_ref().is_some_and(|output| { !output.is_event() && output.data.as_ref().is_some_and(|data| { @@ -272,50 +272,7 @@ impl }) }); if !choice_already_finished { - break (output, false); - } - - let carries_usage = output - .as_ref() - .and_then(|output| output.data.as_ref()) - .is_some_and(|data| data.completion_usage.is_some()); - let backend_failed = output - .as_ref() - .and_then(|output| output.data.as_ref()) - .is_some_and(|data| matches!(data.finish_reason, Some(FinishReason::Error(_)))); - if carries_usage || backend_failed { - // Native backends can put authoritative cumulative usage - // on a terminal event immediately after the token Dynamo - // recognized locally as EOS. Preserve that metadata without - // emitting duplicate content or a second successful finish - // event. Preserve a backend error with or without usage: it - // is not duplicate success metadata and must reach the - // frontend. - if let Some(data) = output.as_mut().and_then(|output| output.data.as_mut()) { - data.token_ids.clear(); - data.tokens = None; - data.text = None; - data.cum_log_probs = None; - data.log_probs = None; - data.top_logprobs = None; - if !backend_failed { - data.finish_reason = None; - data.stop_reason = None; - } - } - break (output, true); - } - - let is_terminal = output - .as_ref() - .and_then(|output| output.data.as_ref()) - .is_some_and(|data| data.finish_reason.is_some()); - if !is_terminal && state.finished_choices.len() >= state.decoders.len() { - // Once every choice ended locally, tolerate only terminal - // metadata. A nonterminal suffix means the backend ignored - // the stop; cancel rather than burning the remaining budget. - state.stream.context().stop_generating(); - return None; + break output; } }; @@ -324,11 +281,6 @@ impl // move to state.process_output // handle any error conditions / unwraps here - if trailing_usage { - state.finished = state.finished_choices.len() >= state.decoders.len(); - return Some((output, state)); - } - // events are pass thru if output.is_event() || output.data.is_none() { // A genuine stream-level error ends generation for every choice at @@ -545,12 +497,13 @@ impl .unwrap_or((None, None)); // If we detected a local stop condition, mark this choice as finished. - // Keep the upstream stream alive just long enough to consume - // a native backend's following terminal-usage event. The - // loop above still cancels on any nonterminal suffix once - // every choice has stopped. + // Once all expected choices are finished, stop the upstream generator. if finish_reason.is_some() && data.finish_reason.is_none() { state.finished_choices.insert(choice_idx); + if state.finished_choices.len() >= decoders_count { + state.stream.context().stop_generating(); + state.finished = true; + } } let text = result.text; @@ -1968,149 +1921,6 @@ mod tests { assert_eq!(generator.get_usage().completion_tokens, 2); } - #[tokio::test] - async fn local_eos_preserves_a_following_authoritative_usage_trailer() { - let tokenizer: Arc = Arc::new(CandidateDecoder); - let backend = Backend::from_tokenizer(Tokenizer::from(tokenizer)); - let request = PreprocessedRequest::builder() - .model("test-model".to_string()) - .token_ids(vec![]) - .stop_conditions(StopConditions { - stop_token_ids_hidden: Some(vec![103]), - ..Default::default() - }) - .sampling_options(SamplingOptions::default()) - .output_options(OutputOptions::default()) - .build() - .expect("valid preprocessed request"); - let reported = dynamo_protocols::types::CompletionUsage { - prompt_tokens: 82, - completion_tokens: 2, - total_tokens: 84, - prompt_tokens_details: None, - completion_tokens_details: None, - }; - let engine: ServerStreamingEngine> = - Arc::new(SyntheticSglangStopEngine { - outputs: Some(vec![ - // Dynamo recognizes 103 as EOS before the native backend's - // separate terminal event arrives. - LLMEngineOutput { - token_ids: vec![101, 103], - index: Some(0), - ..Default::default() - }, - LLMEngineOutput { - index: Some(0), - finish_reason: Some(FinishReason::Stop), - completion_usage: Some(reported.clone()), - ..Default::default() - }, - ]), - }); - - let outputs = Operator::generate(backend.as_ref(), SingleIn::new(request), engine) - .await - .expect("backend generation succeeds") - .collect::>() - .await; - assert_eq!(outputs.len(), 2); - - let local_terminal = outputs[0].data.as_ref().expect("local terminal"); - assert_eq!(local_terminal.finish_reason, Some(FinishReason::Stop)); - let usage_trailer = outputs[1].data.as_ref().expect("usage trailer"); - assert!(usage_trailer.token_ids.is_empty()); - assert!(usage_trailer.finish_reason.is_none()); - assert_eq!(usage_trailer.completion_usage.as_ref(), Some(&reported)); - - let options = DeltaGeneratorOptions::new(None, None, false, None); - let mut generator = DeltaGenerator::new( - "test-model".to_string(), - options, - "test-request".to_string(), - ); - for output in outputs { - generator - .choice_from_postprocessor(output.data.expect("backend output")) - .expect("OpenAI response conversion succeeds"); - } - assert_eq!(generator.get_usage(), reported); - } - - #[tokio::test] - async fn local_eos_does_not_mask_a_trailing_backend_error() { - let tokenizer: Arc = Arc::new(CandidateDecoder); - let backend = Backend::from_tokenizer(Tokenizer::from(tokenizer)); - let request = PreprocessedRequest::builder() - .model("test-model".to_string()) - .token_ids(vec![]) - .stop_conditions(StopConditions { - stop_token_ids_hidden: Some(vec![103]), - ..Default::default() - }) - .sampling_options(SamplingOptions::default()) - .output_options(OutputOptions::default()) - .build() - .expect("valid preprocessed request"); - let reported = dynamo_protocols::types::CompletionUsage { - prompt_tokens: 82, - completion_tokens: 2, - total_tokens: 84, - prompt_tokens_details: None, - completion_tokens_details: None, - }; - for completion_usage in [None, Some(reported.clone())] { - let engine: ServerStreamingEngine> = - Arc::new(SyntheticSglangStopEngine { - outputs: Some(vec![ - LLMEngineOutput { - token_ids: vec![101, 103], - index: Some(0), - ..Default::default() - }, - LLMEngineOutput { - index: Some(0), - finish_reason: Some(FinishReason::Error("worker failed".to_string())), - completion_usage: completion_usage.clone(), - ..Default::default() - }, - ]), - }); - - let outputs = - Operator::generate(backend.as_ref(), SingleIn::new(request.clone()), engine) - .await - .expect("backend generation succeeds") - .collect::>() - .await; - assert_eq!(outputs.len(), 2); - let error_trailer = outputs[1].data.as_ref().expect("error trailer"); - assert!(error_trailer.token_ids.is_empty()); - assert_eq!( - error_trailer.finish_reason, - Some(FinishReason::Error("worker failed".to_string())) - ); - assert_eq!( - error_trailer.completion_usage.as_ref(), - completion_usage.as_ref() - ); - - let options = DeltaGeneratorOptions::new(None, None, false, None); - let mut generator = DeltaGenerator::new( - "test-model".to_string(), - options, - "test-request".to_string(), - ); - generator - .choice_from_postprocessor(outputs[0].data.clone().expect("local terminal")) - .expect("the local terminal converts"); - let error = generator - .choice_from_postprocessor(outputs[1].data.clone().expect("error trailer")) - .expect_err("the backend error must reach the OpenAI response layer"); - assert!(error.to_string().contains("worker failed")); - } - } - #[tokio::test] async fn test_no_stop_trim_raw_completion_output() { use crate::protocols::common::{ diff --git a/lib/sidecar/trtllm/src/client.rs b/lib/sidecar/trtllm/src/client.rs index 16cea9315dfd..576bb2a48098 100644 --- a/lib/sidecar/trtllm/src/client.rs +++ b/lib/sidecar/trtllm/src/client.rs @@ -27,10 +27,10 @@ use crate::proto::inference_client::InferenceClient; /// `--grpc-startup-deadline-secs` instead. const RPC_TIMEOUT: Duration = Duration::from_secs(30); -/// The engine's limits, resolved at startup from `--context-length` and -/// `Control.GetModelInfo`. +/// Metadata resolved at startup from `--context-length` and +/// `Control.GetModelInfo`, including limits and media capabilities. #[derive(Clone, Copy, Debug, Default)] -pub(crate) struct ModelLimits { +pub(crate) struct ModelMetadata { /// Maximum input + output tokens; backs both the registered context window /// and the default `max_tokens`. Absent when neither source supplied one, /// which leaves requests that omit `max_tokens` to be rejected. @@ -99,7 +99,7 @@ impl TrtllmClient { /// unset rather than substituting its `max_input_len` default (measured /// against TensorRT-LLM main at 8bbaf66bd5), so `None` here means "the /// server did not say", and `--context-length` supplies it instead. - async fn get_model_info(&self, model: &str) -> Result { + async fn get_model_info(&self, model: &str) -> Result { let info = self .control() .get_model_info(pb::GetModelInfoRequest { @@ -126,7 +126,7 @@ impl TrtllmClient { _ => false, }) }); - Ok(ModelLimits { + Ok(ModelMetadata { context_length: info.max_context_length.filter(|len| *len > 0), max_output_tokens: info.max_output_tokens.filter(|cap| *cap > 0), supports_image, @@ -136,7 +136,7 @@ impl TrtllmClient { /// One `GetModelInfo` call, for when `--context-length` already supplies the /// window and the engine is consulted only to cross-check it and to learn /// its output cap. - pub(crate) async fn model_limits(&self, model: &str) -> Result { + pub(crate) async fn model_limits(&self, model: &str) -> Result { tokio::time::timeout(RPC_TIMEOUT, self.get_model_info(model)) .await .map_err(|_| { @@ -164,7 +164,7 @@ impl TrtllmClient { model: &str, deadline: Instant, retry_interval: Duration, - ) -> Result { + ) -> Result { let mut last = "it never answered".to_string(); loop { match timeout_at(deadline, self.get_model_info(model)).await { diff --git a/lib/sidecar/trtllm/src/convert.rs b/lib/sidecar/trtllm/src/convert.rs index 0c3e3d59d2f3..0a498a97aff0 100644 --- a/lib/sidecar/trtllm/src/convert.rs +++ b/lib/sidecar/trtllm/src/convert.rs @@ -21,7 +21,7 @@ use dynamo_backend_common::{ MultimodalData, PreprocessedRequest, PromptTokensDetails, StopReason, TopLogprob, usage, }; -use crate::client::{self, ModelLimits}; +use crate::client::{self, ModelMetadata}; use crate::disagg; use crate::proto as pb; @@ -33,7 +33,7 @@ pub(crate) fn build_generate_request( request: &PreprocessedRequest, request_id: &str, model: &str, - limits: Option, + limits: Option, mode: DisaggregationMode, ) -> Result { validate_request(request, mode)?; @@ -162,7 +162,7 @@ pub(crate) fn build_generate_request( // not govern. fn max_tokens( request: &PreprocessedRequest, - limits: Option, + limits: Option, ) -> Result, DynamoError> { if let Some(max_tokens) = request.stop_conditions.max_tokens { return Ok(Some(max_tokens)); @@ -194,7 +194,6 @@ fn max_tokens( } const IMAGE_KEY: &str = "image_url"; -const MM_HASHES_KEY: &str = "mm_hashes"; /// Convert Dynamo's image-only execution payload into the ordered OpenEngine /// media list. The frontend groups content parts by modality, so ordering is @@ -203,7 +202,7 @@ const MM_HASHES_KEY: &str = "mm_hashes"; /// non-empty modality instead of inventing a cross-modality order. fn image_media( request: &PreprocessedRequest, - limits: Option, + limits: Option, ) -> Result, DynamoError> { reject_media_override("mm_processor_kwargs", request.mm_processor_kwargs.as_ref())?; reject_media_override("media_io_kwargs", request.media_io_kwargs.as_ref())?; @@ -256,21 +255,6 @@ fn image_media( )); } - // Explicit caller UUIDs remain authoritative. Otherwise carry the exact - // hashes generated by Dynamo's MM-aware preprocessor to the backend. This - // is request-side preparation for KV-aware routing; the router-owned - // `mm_routing_info` itself never crosses the worker boundary. - let has_user_uuid = user_image_uuids.is_some_and(|uuids| { - uuids - .iter() - .any(|uuid| uuid.as_deref().is_some_and(|uuid| !uuid.is_empty())) - }); - let routing_hashes = if has_user_uuid { - None - } else { - routing_hashes(request, images.len())? - }; - images .iter() .enumerate() @@ -292,63 +276,23 @@ fn image_media( Ok(pb::MediaItem { modality: pb::Modality::Image as i32, source: Some(source), + // Dynamo's frontend representation carries URL/data-URI + // sources, not raw bytes. MIME type is only meaningful for the + // OpenEngine raw_bytes source used by direct clients. mime_type: String::new(), - uuid: if has_user_uuid { - user_image_uuids - .and_then(|uuids| uuids[index].as_deref()) - .unwrap_or_default() - .to_string() - } else { - routing_hashes - .as_ref() - .map(|hashes| hashes[index].clone()) - .unwrap_or_default() - }, + // UUIDs are caller-owned cache identities and are optional per + // item. Dynamo's router hashes are deliberately not substituted + // here: their compatibility with TensorRT-LLM cache identity is + // owned and validated by the MM-aware routing work. + uuid: user_image_uuids + .and_then(|uuids| uuids[index].as_deref()) + .unwrap_or_default() + .to_string(), }) }) .collect() } -fn routing_hashes( - request: &PreprocessedRequest, - image_count: usize, -) -> Result>, DynamoError> { - let hashes = match request.extra_args.as_ref() { - Some(serde_json::Value::Object(extra)) => extra.get(MM_HASHES_KEY), - _ => None, - }; - let Some(hashes) = hashes else { - return Ok(None); - }; - let hashes = hashes.as_array().ok_or_else(|| { - client::invalid_argument("extra_args.mm_hashes must be an array of strings") - })?; - if hashes.is_empty() { - return Ok(None); - } - if hashes.len() != image_count { - return Err(client::invalid_argument(format!( - "extra_args.mm_hashes count ({}) must match the image media item count ({image_count})", - hashes.len() - ))); - } - hashes - .iter() - .enumerate() - .map(|(index, hash)| { - hash.as_str() - .filter(|hash| !hash.is_empty()) - .map(ToOwned::to_owned) - .ok_or_else(|| { - client::invalid_argument(format!( - "extra_args.mm_hashes[{index}] must be a non-empty string" - )) - }) - }) - .collect::, _>>() - .map(Some) -} - fn media_source(value: &str, index: usize) -> Result { let scheme = value .split_once(':') diff --git a/lib/sidecar/trtllm/src/engine.rs b/lib/sidecar/trtllm/src/engine.rs index 66327bfb2d3f..2671ba619bbc 100644 --- a/lib/sidecar/trtllm/src/engine.rs +++ b/lib/sidecar/trtllm/src/engine.rs @@ -18,7 +18,7 @@ use tokio::sync::OnceCell; use tokio_util::sync::CancellationToken; use crate::args::Args; -use crate::client::{self, ModelLimits, TrtllmClient}; +use crate::client::{self, ModelMetadata, TrtllmClient}; use crate::convert::{ResponseState, build_generate_request}; use crate::model::ConfiguredModel; @@ -41,7 +41,7 @@ pub struct TrtllmSidecarEngine { /// Engine limits resolved at `start` from `--context-length` and /// `Control.GetModelInfo`, so `generate` can derive a default `max_tokens` /// for requests that omit one. - limits: OnceCell, + limits: OnceCell, cancel: CancellationToken, } @@ -220,7 +220,7 @@ impl LLMEngine for TrtllmSidecarEngine { ships no Control service or was started without \ --grpc-protocol openengine" ); - ModelLimits::default() + ModelMetadata::default() } }; if let Some(engine_context_length) = reported.context_length @@ -233,7 +233,7 @@ impl LLMEngine for TrtllmSidecarEngine { reported; using the configured --context-length" ); } - ModelLimits { + ModelMetadata { context_length: Some(configured), ..reported } diff --git a/lib/sidecar/trtllm/src/tests.rs b/lib/sidecar/trtllm/src/tests.rs index d6d3c3146762..0bcda724b744 100644 --- a/lib/sidecar/trtllm/src/tests.rs +++ b/lib/sidecar/trtllm/src/tests.rs @@ -21,7 +21,7 @@ use tokio::sync::{Mutex, oneshot}; use tokio_stream::wrappers::TcpListenerStream; use tonic::{Request, Response, Status}; -use crate::client::{ModelLimits, TrtllmClient}; +use crate::client::{ModelMetadata, TrtllmClient}; use crate::convert::{ResponseState, build_generate_request, engine_error}; use crate::engine::TrtllmSidecarEngine; use crate::model::ConfiguredModel; @@ -465,16 +465,16 @@ fn impatient_transport() -> GrpcTransportConfig { /// Engine limits as `Control.GetModelInfo` would report them, with no /// separate output cap. -fn limits(context_length: u32) -> Option { - Some(ModelLimits { +fn limits(context_length: u32) -> Option { + Some(ModelMetadata { context_length: Some(context_length), max_output_tokens: None, supports_image: false, }) } -fn image_limits(context_length: u32) -> Option { - Some(ModelLimits { +fn image_limits(context_length: u32) -> Option { + Some(ModelMetadata { context_length: Some(context_length), max_output_tokens: None, supports_image: true, diff --git a/lib/sidecar/trtllm/src/tests/convert_request.rs b/lib/sidecar/trtllm/src/tests/convert_request.rs index 138057cb5dc8..57007959e472 100644 --- a/lib/sidecar/trtllm/src/tests/convert_request.rs +++ b/lib/sidecar/trtllm/src/tests/convert_request.rs @@ -89,10 +89,6 @@ fn image_urls_data_uris_and_uuids_are_mapped_in_order() { "image_url".to_string(), vec![Some("first-image".to_string()), None], )])); - req.extra_args = Some(json!({ - "mm_hashes": ["routing-hash-one", "routing-hash-two"] - })); - let proto = build_generate_request(&req, "req", "model", image_limits(4096), AGG) .expect("image request"); assert_eq!(proto.media.len(), 2); @@ -114,7 +110,7 @@ fn image_urls_data_uris_and_uuids_are_mapped_in_order() { } #[test] -fn mm_routing_hashes_are_forwarded_as_image_uuids_in_order() { +fn mm_routing_hashes_are_not_used_as_engine_cache_uuids() { let mut req = request(); add_images( &mut req, @@ -129,34 +125,7 @@ fn mm_routing_hashes_are_forwarded_as_image_uuids_in_order() { let proto = build_generate_request(&req, "req", "model", image_limits(4096), AGG) .expect("image request"); - assert_eq!(proto.media[0].uuid, "0123456789abcdef"); - assert_eq!(proto.media[1].uuid, "fedcba9876543210"); -} - -#[test] -fn malformed_or_misaligned_mm_routing_hashes_are_rejected() { - for (hashes, expected) in [ - (json!("not-an-array"), "array of strings"), - (json!(["only-one"]), "must match"), - (json!(["one", ""]), "non-empty string"), - (json!(["one", 2]), "non-empty string"), - ] { - let mut req = request(); - add_images( - &mut req, - [ - "https://example.com/first.png", - "https://example.com/second.png", - ], - ); - req.extra_args = Some(json!({"mm_hashes": hashes})); - let error = build_generate_request(&req, "req", "model", image_limits(4096), AGG) - .expect_err("malformed routing hashes must fail closed"); - assert!( - error.to_string().contains(expected), - "unexpected error: {error}" - ); - } + assert!(proto.media.iter().all(|item| item.uuid.is_empty())); } #[test] @@ -365,7 +334,7 @@ fn the_derived_max_tokens_respects_the_engines_output_cap() { &req, "req", "model", - Some(ModelLimits { + Some(ModelMetadata { context_length: Some(4096), max_output_tokens: Some(256), supports_image: false, diff --git a/lib/sidecar/trtllm/src/tests/disagg.rs b/lib/sidecar/trtllm/src/tests/disagg.rs index 612fda3f39fc..a81c8dfe92c8 100644 --- a/lib/sidecar/trtllm/src/tests/disagg.rs +++ b/lib/sidecar/trtllm/src/tests/disagg.rs @@ -72,9 +72,13 @@ fn image_media_is_preserved_across_prefill_and_decode_requests() { "data:image/png;base64,c2Vjb25k", ], ); - req.extra_args = Some(json!({ - "mm_hashes": ["0123456789abcdef", "fedcba9876543210"] - })); + req.multi_modal_uuids = Some(HashMap::from([( + "image_url".to_string(), + vec![ + Some("first-image".to_string()), + Some("second-image".to_string()), + ], + )])); let prefill = build_generate_request( &req, @@ -87,8 +91,8 @@ fn image_media_is_preserved_across_prefill_and_decode_requests() { assert!(is_context_only(&prefill)); assert_eq!(prefill.stopping.as_ref().unwrap().max_tokens, Some(1)); assert_eq!(prefill.media.len(), 2); - assert_eq!(prefill.media[0].uuid, "0123456789abcdef"); - assert_eq!(prefill.media[1].uuid, "fedcba9876543210"); + assert_eq!(prefill.media[0].uuid, "first-image"); + assert_eq!(prefill.media[1].uuid, "second-image"); req.prefill_result = Some(dynamo_backend_common::PrefillResult { disaggregated_params: crate::disagg::session_to_json(fake_session()) From f59ffd89c183bc0bb039cf7a0324669ee4e58ff5 Mon Sep 17 00:00:00 2001 From: krishung5 Date: Thu, 15 Oct 2026 16:59:18 -0700 Subject: [PATCH 03/61] style(trtllm): tighten media comments Signed-off-by: krishung5 --- lib/sidecar/trtllm/src/client.rs | 8 ++--- lib/sidecar/trtllm/src/convert.rs | 51 +++++++------------------------ 2 files changed, 13 insertions(+), 46 deletions(-) diff --git a/lib/sidecar/trtllm/src/client.rs b/lib/sidecar/trtllm/src/client.rs index 576bb2a48098..e3ac0932f4ab 100644 --- a/lib/sidecar/trtllm/src/client.rs +++ b/lib/sidecar/trtllm/src/client.rs @@ -27,8 +27,7 @@ use crate::proto::inference_client::InferenceClient; /// `--grpc-startup-deadline-secs` instead. const RPC_TIMEOUT: Duration = Duration::from_secs(30); -/// Metadata resolved at startup from `--context-length` and -/// `Control.GetModelInfo`, including limits and media capabilities. +/// Startup metadata from `--context-length` and `Control.GetModelInfo`. #[derive(Clone, Copy, Debug, Default)] pub(crate) struct ModelMetadata { /// Maximum input + output tokens; backs both the registered context window @@ -38,10 +37,7 @@ pub(crate) struct ModelMetadata { /// Cap on generated tokens, when the engine reports one. A context window /// alone can imply a larger budget than the engine will accept. pub(crate) max_output_tokens: Option, - /// Whether this loaded model advertises image input through both the - /// portable multimodal bit and TensorRT-LLM's modality list. Requiring both - /// fails closed when a server and client disagree about the finer-grained - /// capability. + /// True when both portable and engine-specific capabilities advertise images. pub(crate) supports_image: bool, } diff --git a/lib/sidecar/trtllm/src/convert.rs b/lib/sidecar/trtllm/src/convert.rs index 0a498a97aff0..1f5c9b76210e 100644 --- a/lib/sidecar/trtllm/src/convert.rs +++ b/lib/sidecar/trtllm/src/convert.rs @@ -4,10 +4,8 @@ //! Conversion between Dynamo's `PreprocessedRequest` / `LLMEngineOutput` and the //! TensorRT-LLM OpenEngine (`openengine.v1`) protobuf messages. //! -//! Scope: aggregated and disaggregated (prefill/decode) generation, including -//! image inputs when the loaded model advertises them. Video, audio, LoRA, -//! beam search, and `n > 1` are rejected before dispatch — the sidecar streams -//! a single sequence. +//! Supports aggregated and disaggregated image generation. Video, audio, LoRA, +//! beam search, and `n > 1` are rejected before dispatch. //! //! Disaggregation is driven by [`DisaggregationMode`]: a prefill worker marks //! its request `context_only` and returns the `PrefillReady` handoff as its @@ -139,27 +137,10 @@ pub(crate) fn build_generate_request( }) } -// Temporary workaround: the OpenEngine contract makes `stopping.max_tokens` -// optional. The Dynamo frontend forwards an omitted `max_tokens` as `None` -// expecting the backend to default. For text requests we mirror the in-process -// backend's default, -// `max(1, context_length - prompt_len)` (components/src/dynamo/trtllm -// `_default_max_tokens`). For image requests, only TensorRT-LLM knows the true -// prompt length after expanding image markers, so leave the field unset and let -// the OpenEngine request mapper derive the budget from the expanded prompt. -// `context_length` is resolved in -// `engine::start`, where `--context-length` wins over the `GetModelInfo` report. -// Only `/v1/chat/completions` and `/v1/responses` reach this fallback: they set -// `PRESERVE_OMITTED_MAX_TOKENS_CONTEXT_KEY`, which stops the frontend supplying -// its own default (`preprocessor::omitted_max_tokens_default`). `/v1/completions` -// is already defaulted by the frontend and never lands here. -// -// Remove when https://github.com/NVIDIA/TensorRT-LLM/issues/16549 lands (gRPC -// `max_tokens` made optional): drop this fallback and forward an omitted -// `max_tokens` as unset. Keep `--context-length` and the plumbing in -// `engine.rs` — they also feed `LlmRegistration.context_length`, which the -// frontend registers as the served context window and which this fallback does -// not govern. +// Derive omitted text max_tokens from the context window. For media, only +// TensorRT-LLM knows the prompt length after expansion, so leave it unset. +// Remove this fallback when NVIDIA/TensorRT-LLM#16549 lands; context_length +// remains necessary for LlmRegistration. fn max_tokens( request: &PreprocessedRequest, limits: Option, @@ -195,11 +176,8 @@ fn max_tokens( const IMAGE_KEY: &str = "image_url"; -/// Convert Dynamo's image-only execution payload into the ordered OpenEngine -/// media list. The frontend groups content parts by modality, so ordering is -/// retained within the image vector but not across different modalities. This -/// PR therefore accepts images only and fails explicitly on every other -/// non-empty modality instead of inventing a cross-modality order. +/// Map Dynamo's image payload to OpenEngine media, rejecting other modalities. +/// Per-modality vectors do not preserve cross-modality order. fn image_media( request: &PreprocessedRequest, limits: Option, @@ -207,6 +185,7 @@ fn image_media( reject_media_override("mm_processor_kwargs", request.mm_processor_kwargs.as_ref())?; reject_media_override("media_io_kwargs", request.media_io_kwargs.as_ref())?; + // mm_routing_info stays router-side; multi_modal_data is engine input. let media_map = request.multi_modal_data.as_ref(); if let Some(media_map) = media_map { for (modality, items) in media_map { @@ -276,14 +255,9 @@ fn image_media( Ok(pb::MediaItem { modality: pb::Modality::Image as i32, source: Some(source), - // Dynamo's frontend representation carries URL/data-URI - // sources, not raw bytes. MIME type is only meaningful for the - // OpenEngine raw_bytes source used by direct clients. + // MIME metadata belongs to raw_bytes, which this client does not send. mime_type: String::new(), - // UUIDs are caller-owned cache identities and are optional per - // item. Dynamo's router hashes are deliberately not substituted - // here: their compatibility with TensorRT-LLM cache identity is - // owned and validated by the MM-aware routing work. + // Do not substitute router-owned mm_hashes for caller cache identities. uuid: user_image_uuids .and_then(|uuids| uuids[index].as_deref()) .unwrap_or_default() @@ -486,9 +460,6 @@ fn validate_request( "multimodal encoder handoffs are not supported by the TensorRT-LLM sidecar", )); } - // `mm_routing_info` is deliberately not forwarded: it is framework-owned - // metadata that the Dynamo router consumes before selecting this worker. - // The execution payload remains `multi_modal_data`, mapped by `image_media`. // Only a decode worker consumes a prefill handoff. Seeing one anywhere else // means the frontend routed a decode request to the wrong worker role. if request.prefill_result.is_some() && !mode.is_decode() { From 9ef1535df8039ac6591658eb99610633a61b4d18 Mon Sep 17 00:00:00 2001 From: krishung5 Date: Thu, 15 Oct 2026 17:49:52 -0700 Subject: [PATCH 04/61] test(trtllm): gate sidecar image requests in CI Signed-off-by: krishung5 --- .github/filters.yaml | 1 + .github/scripts/test-filters.js | 12 +++--- .github/workflows/pr.yaml | 27 ++++++++++++ container/templates/trtllm_runtime.Dockerfile | 20 +++++++++ tests/serve/test_sidecar.py | 41 ++++++++++++++++--- 5 files changed, 90 insertions(+), 11 deletions(-) diff --git a/.github/filters.yaml b/.github/filters.yaml index 69c3f5b01c10..b49e6f46532d 100644 --- a/.github/filters.yaml +++ b/.github/filters.yaml @@ -404,6 +404,7 @@ trtllm: - 'container/deps/requirements.trtllm.txt' - 'examples/backends/trtllm/**' - 'components/src/dynamo/trtllm/**' + - 'lib/sidecar/trtllm/**' - 'container/templates/trtllm_*' - '!**/*.md' - '!**/*.rst' diff --git a/.github/scripts/test-filters.js b/.github/scripts/test-filters.js index e23b4cb0014c..a00ec9bd975d 100755 --- a/.github/scripts/test-filters.js +++ b/.github/scripts/test-filters.js @@ -13,7 +13,7 @@ * This validates that tj-actions/changed-files will correctly: * - Match backend-specific files to their respective filters (vllm, sglang, trtllm) * - Route docs-artifact tests to recipe-check without core E2E checks - * - Route sidecar files to sidecar/Rust checks without backend or core E2E checks + * - Route sidecar files to sidecar/Rust checks and framework-native E2E where needed * - Exclude doc files (*.md, *.rst, *.txt) from core via negation patterns * - Match CI/infrastructure changes to core * - (with --coverage) Ensure all files in repo are covered by at least one filter @@ -175,7 +175,7 @@ const testCases = [ desc: 'AISimulate Dynamo adapter triggers planner validation' }, - // Sidecar Rust and proto files should trigger Rust checks without unrelated E2E + // Sidecar Rust and proto files trigger Rust checks and their native E2E only. { file: 'lib/sidecar/common/src/lib.rs', expect: { sidecar: true, rust: true, core: false, frontend: false, vllm: false, sglang: false, trtllm: false }, @@ -183,8 +183,8 @@ const testCases = [ }, { file: 'lib/sidecar/trtllm/proto/openengine/v1/generation.proto', - expect: { sidecar: true, rust: true, core: false, frontend: false, vllm: false, sglang: false, trtllm: false }, - desc: 'sidecar proto contracts trigger Rust checks without backend E2E' + expect: { sidecar: true, rust: true, core: false, frontend: false, vllm: false, sglang: false, trtllm: true }, + desc: 'trtllm sidecar contracts trigger Rust checks and trtllm E2E' }, { file: 'lib/sidecar/sglang/src/lib.rs', @@ -193,8 +193,8 @@ const testCases = [ }, { file: 'lib/sidecar/trtllm/src/lib.rs', - expect: { sidecar: true, rust: true, core: false, frontend: false, vllm: false, sglang: false, trtllm: false }, - desc: 'trtllm sidecar source does not route to sglang or trtllm E2E' + expect: { sidecar: true, rust: true, core: false, frontend: false, vllm: false, sglang: false, trtllm: true }, + desc: 'trtllm sidecar source triggers Rust checks and trtllm E2E' }, { file: 'lib/sidecar/vllm/deploy/agg.yaml', diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index e6e2a925bbd1..58eb3e77f6af 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -147,6 +147,7 @@ jobs: - trtllm-build - trtllm-dev-build - trtllm-test + - sidecar-trtllm-test - trtllm-multi-gpu-test - trtllm-copy-to-acr - triton-build @@ -541,6 +542,11 @@ jobs: diff_current_branch: ${{ github.ref_name }} diff_base_branch: ${{ needs.changed-files.outputs.base_ref }} diff_merge_base_sha: ${{ needs.changed-files.outputs.merge_base_sha }} + # Drop these overrides after the upstream TensorRT-LLM change ships. + extra_build_args: | + RUNTIME_IMAGE_TAG=1.4.0rc1.dev202610140007@sha256:7a728cd307bfa1651b4ad0787e71e69f7506fb1a698a0f4f04aef719ad43525f + TRTLLM_SOURCE_REPOSITORY=https://github.com/krishung5/TensorRT-LLM + TRTLLM_SOURCE_COMMIT=ab07b1649f638924908aed565722f4dd0f8547f8 # actions: read lets the build download a prior run's baseline compliance # artifact to diff against (a reusable workflow's token can't exceed the # caller's). Missing -> the diff soft-degrades, build stays green. @@ -928,6 +934,27 @@ jobs: gpu_parallel_max_vram_gib: '24' secrets: inherit + sidecar-trtllm-test: + name: sidecar-runtime + needs: [changed-files, trtllm-build, sidecar-build] + if: needs.changed-files.outputs.sidecar == 'true' && needs.changed-files.outputs.trtllm == 'true' + uses: $/.github/workflows/shared-test.yml + with: + test_suite_name: trtllm + framework: trtllm + test_type: TensorRT-LLM Sidecar 1-GPU E2E Test + amd_runner: prod-tester-amd-gpu-v2 + amd_runner_labels: '["gpu-1","amd64","v2","us-west-2"]' + target_tag_plain: ${{ needs.trtllm-build.outputs.target_tag_plain }} + cuda_version: '["13.1"]' + platform: '["amd64"]' + run_sanity_check: false + gpu_test_markers: pre_merge and sidecar and trtllm and gpu_1 + gpu_test_timeout_minutes: 45 + sidecar_backend: trtllm + sidecar_binary_artifact: ${{ needs.sidecar-build.outputs.binary_artifact }} + secrets: inherit + trtllm-multi-gpu-test: name: trtllm-runtime # This name overlaps with other trtllm jobs to group them in the UI needs: [changed-files, trtllm-build] diff --git a/container/templates/trtllm_runtime.Dockerfile b/container/templates/trtllm_runtime.Dockerfile index 3b147a085b55..860698519fbb 100644 --- a/container/templates/trtllm_runtime.Dockerfile +++ b/container/templates/trtllm_runtime.Dockerfile @@ -41,6 +41,26 @@ ARG ENABLE_KVBM ARG ENABLE_GPU_MEMORY_SERVICE ARG TARGETARCH ARG NIXL_REF +ARG TRTLLM_SOURCE_REPOSITORY +ARG TRTLLM_SOURCE_COMMIT + +# Optional exact-source overlay for cross-repository CI. +RUN if [ -n "${TRTLLM_SOURCE_COMMIT}" ]; then \ + test -n "${TRTLLM_SOURCE_REPOSITORY}" && \ + source_dir=$(mktemp -d) && \ + wget --quiet --tries=3 \ + "${TRTLLM_SOURCE_REPOSITORY}/archive/${TRTLLM_SOURCE_COMMIT}.tar.gz" \ + -O "${source_dir}/source.tar.gz" && \ + tar -xzf "${source_dir}/source.tar.gz" -C "${source_dir}" \ + --strip-components=1 && \ + package_dir=$(/usr/bin/python3 -c \ + 'import importlib.util; print(next(iter(importlib.util.find_spec("tensorrt_llm").submodule_search_locations)))') && \ + cp -a "${source_dir}/tensorrt_llm/grpc/openengine/." \ + "${package_dir}/grpc/openengine/" && \ + install -m 0644 "${source_dir}/tensorrt_llm/inputs/media_io.py" \ + "${package_dir}/inputs/media_io.py" && \ + rm -rf "${source_dir}"; \ + fi # Remove the upstream runtime's Git LFS package without pruning shared dependencies. RUN apt-get purge -y git-lfs && rm -rf /var/lib/apt/lists/* diff --git a/tests/serve/test_sidecar.py b/tests/serve/test_sidecar.py index 038446530fc0..55720fee9b12 100644 --- a/tests/serve/test_sidecar.py +++ b/tests/serve/test_sidecar.py @@ -31,6 +31,7 @@ disaggregated_chat_payload, disaggregated_token_count_payload, http_cancellation_payloads, + multimodal_payload_default, ) from tests.utils.payloads import ( ChatPayload, @@ -115,10 +116,7 @@ def _disaggregated_payloads( ] -# Sequential stage only: no profiled_vram_gib mark yet, since actual peak VRAM -# has not been profiled for the sidecar launch path. Add one once measured, to -# admit these into the parallel stage alongside the equivalent dynamo.{backend} -# scenarios in tests/serve/test_{vllm,sglang,trtllm}.py. +# Text cases stay sequential until their sidecar launch paths are profiled. sidecar_configs = { "vllm_aggregated": EngineConfig( name="vllm_aggregated", @@ -126,6 +124,7 @@ def _disaggregated_payloads( script_name="agg.sh", marks=[ pytest.mark.vllm, + pytest.mark.core, pytest.mark.gpu_1, # Let the 600s health check report failure before pytest times out. pytest.mark.timeout(780), @@ -149,6 +148,7 @@ def _disaggregated_payloads( script_args=["--enable-metrics", "--decode-log-interval", "1"], marks=[ pytest.mark.sglang, + pytest.mark.core, pytest.mark.gpu_1, pytest.mark.timeout(780), pytest.mark.post_merge, @@ -179,6 +179,7 @@ def _disaggregated_payloads( script_name="agg.sh", marks=[ pytest.mark.trtllm, + pytest.mark.core, pytest.mark.gpu_1, pytest.mark.timeout(780), pytest.mark.post_merge, @@ -198,12 +199,41 @@ def _disaggregated_payloads( chat_payload_default(), ], ), + "trtllm_aggregated_image": EngineConfig( + name="trtllm_aggregated_image", + directory=trtllm_sidecar_dir, + script_name="agg.sh", + marks=[ + pytest.mark.trtllm, + pytest.mark.multimodal, + pytest.mark.gpu_1, + pytest.mark.profiled_vram_gib(12.0), + pytest.mark.requested_trtllm_kv_tokens(32768), + pytest.mark.timeout(960), + pytest.mark.pre_merge, + pytest.mark.skipif( + not _trtllm_serves_openengine(), + reason=TRTLLM_OPENENGINE_SKIP_REASON, + ), + ], + model="Qwen/Qwen3-VL-2B-Instruct", + timeout=900, + delayed_start=60, + env={"PYTHONUNBUFFERED": "1"}, + request_payloads=[ + multimodal_payload_default( + text="Describe what you see in this image.", + expected_response=["mountain", "rock", "trees", "road"], + ) + ], + ), "trtllm_disaggregated": EngineConfig( name="trtllm_disaggregated", directory=trtllm_sidecar_dir, script_name="disagg.sh", marks=[ pytest.mark.trtllm, + pytest.mark.core, # Both engines share one GPU: the handoff is what this covers, and # a co-resident pair keeps it on the existing 1-GPU sidecar runner. # `requested_trtllm_kv_tokens` caps each engine's KV pool so the @@ -250,6 +280,7 @@ def _disaggregated_payloads( script_name="disagg.sh", marks=[ pytest.mark.vllm, + pytest.mark.core, pytest.mark.gpu_1, pytest.mark.post_merge, pytest.mark.timeout(1200), @@ -282,6 +313,7 @@ def _disaggregated_payloads( ], marks=[ pytest.mark.sglang, + pytest.mark.core, pytest.mark.gpu_1, pytest.mark.post_merge, pytest.mark.timeout(1200), @@ -320,7 +352,6 @@ def sidecar_config_test(request, monkeypatch, discovery_backend): return config -@pytest.mark.core @pytest.mark.sidecar @pytest.mark.e2e @pytest.mark.engine_ports From de12e0206b953ecc3bdf27ed5fab6f9d95291d89 Mon Sep 17 00:00:00 2001 From: krishung5 Date: Thu, 15 Oct 2026 20:37:16 -0700 Subject: [PATCH 05/61] feat(trtllm): pass video inputs through OpenEngine Signed-off-by: krishung5 --- lib/backend-common/src/engine.rs | 3 +- lib/backend-common/src/lib.rs | 4 +- lib/llm/src/preprocessor.rs | 63 +++++- lib/llm/src/protocols/common/preprocessor.rs | 12 ++ lib/sidecar/trtllm/src/client.rs | 62 +++--- lib/sidecar/trtllm/src/convert.rs | 187 +++++++++++++----- lib/sidecar/trtllm/src/tests.rs | 41 ++-- .../trtllm/src/tests/convert_request.rs | 139 ++++++++++++- lib/sidecar/trtllm/src/tests/e2e.rs | 74 +++++++ lib/sidecar/trtllm/src/tests/engine.rs | 30 +++ 10 files changed, 522 insertions(+), 93 deletions(-) diff --git a/lib/backend-common/src/engine.rs b/lib/backend-common/src/engine.rs index 2c5daf1643e9..10e2583879d8 100644 --- a/lib/backend-common/src/engine.rs +++ b/lib/backend-common/src/engine.rs @@ -28,7 +28,8 @@ pub use dynamo_llm::protocols::common::llm_backend::{ }; pub use dynamo_llm::protocols::common::preprocessor::{ BootstrapInfo, KV_HINT_TRANSFER_CAPABILITY_KEY, KvHint, KvHintAction, KvSourceLocationsPayload, - MultimodalData, MultimodalDataMap, PrefillResult, PreprocessedRequest, RoutingHints, + MultimodalData, MultimodalDataMap, MultimodalItemRef, PrefillResult, PreprocessedRequest, + RoutingHints, }; pub use dynamo_llm::protocols::common::{ FinishReason, GuidedDecodingOptions, OutputOptions, SamplingOptions, StopConditions, diff --git a/lib/backend-common/src/lib.rs b/lib/backend-common/src/lib.rs index fd749de7e1ab..affa584da728 100644 --- a/lib/backend-common/src/lib.rs +++ b/lib/backend-common/src/lib.rs @@ -38,8 +38,8 @@ pub use engine::{ FinishReason, FirstTokenNotifier, GenerateContext, GuidedDecodingOptions, HEALTH_CHECK_KEY, KV_HINT_TRANSFER_CAPABILITY_KEY, KvEventPublisher, KvEventSource, KvHint, KvHintAction, KvSourceLocationsPayload, LLMEngine, LLMEngineOutput, LLMEngineOutputExt, LlmRegistration, - LogProbs, Metrics, MetricsBindings, MetricsCtx, MultimodalData, OnPublisherReady, - OnSnapshotPublisherReady, OutputOptions, PrefillResult, PreprocessedRequest, + LogProbs, Metrics, MetricsBindings, MetricsCtx, MultimodalData, MultimodalItemRef, + OnPublisherReady, OnSnapshotPublisherReady, OutputOptions, PrefillResult, PreprocessedRequest, PromptTokensDetails, RawEngine, SamplingOptions, StopConditions, StopReason, TopLogprob, TopLogprobs, chunk, usage, }; diff --git a/lib/llm/src/preprocessor.rs b/lib/llm/src/preprocessor.rs index e6a67b6f3b45..1ffe83b3cc83 100644 --- a/lib/llm/src/preprocessor.rs +++ b/lib/llm/src/preprocessor.rs @@ -70,7 +70,8 @@ use crate::model_card::{ModelDeploymentCard, ModelInfo, PromptFormatterArtifact} use crate::preprocessor::media::MediaFetcher; use crate::preprocessor::media::{MediaDecoder, MediaLoader, max_data_url_bytes}; use crate::protocols::common::preprocessor::{ - MultimodalData, MultimodalDataMap, MultimodalUuidMap, PreprocessedRequestBuilder, RoutingHints, + MultimodalData, MultimodalDataMap, MultimodalItemRef, MultimodalUuidMap, + PreprocessedRequestBuilder, RoutingHints, }; use crate::protocols::common::timing::RequestTracker; use crate::tokenizers::Encoding; @@ -3614,6 +3615,7 @@ impl OpenAIPreprocessor { let mut media_map: MultimodalDataMap = HashMap::new(); let mut uuid_map: MultimodalUuidMap = HashMap::new(); + let mut media_order: Vec = Vec::new(); let mut has_user_uuid = false; // Decoded results are written back into these reserved modality slots so // URL-backed and UUID-only inputs retain request order. @@ -3693,6 +3695,10 @@ impl OpenAIPreprocessor { let slots = media_map.entry(type_str.to_string()).or_default(); let slot_idx = slots.len(); + media_order.push(MultimodalItemRef { + modality: type_str.to_string(), + index: slot_idx, + }); has_user_uuid |= uuid.is_some(); uuid_map .entry(type_str.to_string()) @@ -3970,6 +3976,7 @@ impl OpenAIPreprocessor { if !media_map.is_empty() { builder.multi_modal_data(Some(media_map)); + builder.multi_modal_order(Some(media_order)); if has_user_uuid { builder.multi_modal_uuids(Some(uuid_map)); } @@ -8476,6 +8483,60 @@ mod extra_args_media_copy_tests { .to_string() } + #[tokio::test] + async fn multimodal_order_tracks_the_rendered_media_sequence() { + let preprocessor = test_preprocessor(); + let image = inline_data_url(); + let request: NvCreateChatCompletionRequest = serde_json::from_value(serde_json::json!({ + "model": "alias", + "messages": [{ + "role": "user", + "content": [ + {"type": "video_url", "video_url": {"url": "data:video/mp4;base64,AAAA"}}, + {"type": "image_url", "image_url": {"url": image}}, + {"type": "audio_url", "audio_url": {"url": "data:audio/wav;base64,AAAA"}}, + {"type": "video_url", "video_url": {"url": "https://example.com/second.mp4"}} + ] + }] + })) + .unwrap(); + let mut builder = PreprocessedRequestBuilder::default(); + builder + .model("alias".into()) + .token_ids(Vec::new()) + .stop_conditions(Default::default()) + .sampling_options(Default::default()) + .output_options(Default::default()); + + preprocessor + .gather_multi_modal_data_with_image_tokens(&request, &mut builder, None, &[]) + .await + .unwrap(); + let result = builder.build().unwrap(); + + assert_eq!( + result.multi_modal_order.unwrap(), + [ + MultimodalItemRef { + modality: "video_url".to_string(), + index: 0, + }, + MultimodalItemRef { + modality: "image_url".to_string(), + index: 0, + }, + MultimodalItemRef { + modality: "audio_url".to_string(), + index: 0, + }, + MultimodalItemRef { + modality: "video_url".to_string(), + index: 1, + }, + ] + ); + } + #[tokio::test] async fn deepseek_v41_display_name_fallback_orders_tool_media() { use crate::common::checked_file::CheckedFile; diff --git a/lib/llm/src/protocols/common/preprocessor.rs b/lib/llm/src/protocols/common/preprocessor.rs index ed84d5571835..45c8bd2f2d90 100644 --- a/lib/llm/src/protocols/common/preprocessor.rs +++ b/lib/llm/src/protocols/common/preprocessor.rs @@ -362,6 +362,13 @@ pub type MultimodalDataMap = std::collections::HashMap>>; +/// A reference into [`MultimodalDataMap`] preserving the media traversal order. +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +pub struct MultimodalItemRef { + pub modality: String, + pub index: usize, +} + /// [`PreprocessedRequest`] is the internal representation of an LLM request. The `dynamo.llm-preprocessor` /// crate is responsible for converting request from the public APIs to this internal representation. #[derive(Serialize, Deserialize, Clone, Builder)] @@ -421,6 +428,11 @@ pub struct PreprocessedRequest { #[serde(default, skip_serializing_if = "Option::is_none")] pub multi_modal_uuids: Option, + /// Media references in the order used to render the prompt. + #[builder(default)] + #[serde(default, skip_serializing_if = "Option::is_none")] + pub multi_modal_order: Option>, + /// Optional multimodal routing-only fields (separate from execution payload). #[builder(default)] #[serde(default, skip_serializing_if = "Option::is_none")] diff --git a/lib/sidecar/trtllm/src/client.rs b/lib/sidecar/trtllm/src/client.rs index e3ac0932f4ab..0efbf801be1e 100644 --- a/lib/sidecar/trtllm/src/client.rs +++ b/lib/sidecar/trtllm/src/client.rs @@ -27,6 +27,23 @@ use crate::proto::inference_client::InferenceClient; /// `--grpc-startup-deadline-secs` instead. const RPC_TIMEOUT: Duration = Duration::from_secs(30); +/// Media modalities confirmed by both OpenEngine capability signals. +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct SupportedMedia { + pub(crate) image: bool, + pub(crate) video: bool, +} + +impl SupportedMedia { + pub(crate) fn contains(self, modality: &str) -> bool { + match modality { + "image" => self.image, + "video" => self.video, + _ => false, + } + } +} + /// Startup metadata from `--context-length` and `Control.GetModelInfo`. #[derive(Clone, Copy, Debug, Default)] pub(crate) struct ModelMetadata { @@ -37,8 +54,8 @@ pub(crate) struct ModelMetadata { /// Cap on generated tokens, when the engine reports one. A context window /// alone can imply a larger budget than the engine will accept. pub(crate) max_output_tokens: Option, - /// True when both portable and engine-specific capabilities advertise images. - pub(crate) supports_image: bool, + /// Modalities advertised by both the portable and engine-specific capabilities. + pub(crate) supported_media: SupportedMedia, } pub(crate) struct TrtllmClient { @@ -103,29 +120,30 @@ impl TrtllmClient { }) .await? .into_inner(); - let supports_image = info.supports_multimodal == Some(true) - && info.extra.as_ref().is_some_and(|extra| { - extra - .fields - .get("trtllm_supported_modalities") - .and_then(|value| value.kind.as_ref()) - .is_some_and(|kind| match kind { - prost_types::value::Kind::ListValue(modalities) => { - modalities.values.iter().any(|value| { - matches!( - value.kind.as_ref(), - Some(prost_types::value::Kind::StringValue(modality)) - if modality == "image" - ) - }) - } - _ => false, - }) - }); + let mut supported_media = SupportedMedia::default(); + if info.supports_multimodal == Some(true) + && let Some(prost_types::value::Kind::ListValue(modalities)) = info + .extra + .as_ref() + .and_then(|extra| extra.fields.get("trtllm_supported_modalities")) + .and_then(|value| value.kind.as_ref()) + { + for value in &modalities.values { + let Some(prost_types::value::Kind::StringValue(modality)) = value.kind.as_ref() + else { + continue; + }; + match modality.as_str() { + "image" => supported_media.image = true, + "video" => supported_media.video = true, + _ => {} + } + } + } Ok(ModelMetadata { context_length: info.max_context_length.filter(|len| *len > 0), max_output_tokens: info.max_output_tokens.filter(|cap| *cap > 0), - supports_image, + supported_media, }) } diff --git a/lib/sidecar/trtllm/src/convert.rs b/lib/sidecar/trtllm/src/convert.rs index 1f5c9b76210e..811ed083b731 100644 --- a/lib/sidecar/trtllm/src/convert.rs +++ b/lib/sidecar/trtllm/src/convert.rs @@ -4,15 +4,15 @@ //! Conversion between Dynamo's `PreprocessedRequest` / `LLMEngineOutput` and the //! TensorRT-LLM OpenEngine (`openengine.v1`) protobuf messages. //! -//! Supports aggregated and disaggregated image generation. Video, audio, LoRA, -//! beam search, and `n > 1` are rejected before dispatch. +//! Supports aggregated and disaggregated image and video generation. Audio, +//! LoRA, beam search, and `n > 1` are rejected before dispatch. //! //! Disaggregation is driven by [`DisaggregationMode`]: a prefill worker marks //! its request `context_only` and returns the `PrefillReady` handoff as its //! terminal chunk, and a decode worker replays that handoff in `kv.session`. //! See [`crate::disagg`] for the codec. -use std::collections::BTreeSet; +use std::collections::{BTreeSet, HashSet}; use dynamo_backend_common::{ CompletionUsage, DisaggregationMode, DynamoError, FinishReason, LLMEngineOutput, @@ -35,7 +35,7 @@ pub(crate) fn build_generate_request( mode: DisaggregationMode, ) -> Result { validate_request(request, mode)?; - let media = image_media(request, limits)?; + let media = media(request, limits)?; let sampling = &request.sampling_options; let stop = &request.stop_conditions; @@ -175,10 +175,34 @@ fn max_tokens( } const IMAGE_KEY: &str = "image_url"; +const VIDEO_KEY: &str = "video_url"; -/// Map Dynamo's image payload to OpenEngine media, rejecting other modalities. -/// Per-modality vectors do not preserve cross-modality order. -fn image_media( +#[derive(Clone, Copy)] +struct MediaKind { + key: &'static str, + name: &'static str, + modality: pb::Modality, +} + +const MEDIA_KINDS: [MediaKind; 2] = [ + MediaKind { + key: IMAGE_KEY, + name: "image", + modality: pb::Modality::Image, + }, + MediaKind { + key: VIDEO_KEY, + name: "video", + modality: pb::Modality::Video, + }, +]; + +fn media_kind(key: &str) -> Option { + MEDIA_KINDS.iter().copied().find(|kind| kind.key == key) +} + +/// Map Dynamo's grouped media payload to OpenEngine's globally ordered list. +fn media( request: &PreprocessedRequest, limits: Option, ) -> Result, DynamoError> { @@ -189,76 +213,139 @@ fn image_media( let media_map = request.multi_modal_data.as_ref(); if let Some(media_map) = media_map { for (modality, items) in media_map { - if modality != IMAGE_KEY && !items.is_empty() { + if media_kind(modality).is_none() && !items.is_empty() { return Err(client::invalid_argument(format!( - "multimodal modality `{modality}` is not supported by the TensorRT-LLM sidecar; this image-only implementation accepts `{IMAGE_KEY}`" + "multimodal modality `{modality}` is not supported by the TensorRT-LLM sidecar" ))); } } } - let images = media_map - .and_then(|media| media.get(IMAGE_KEY)) - .map(Vec::as_slice) - .unwrap_or_default(); - - let user_image_uuids = if let Some(uuid_map) = request.multi_modal_uuids.as_ref() { + let uuid_map = request.multi_modal_uuids.as_ref(); + if let Some(uuid_map) = uuid_map { for (modality, uuids) in uuid_map { - if modality != IMAGE_KEY && !uuids.is_empty() { + if media_kind(modality).is_none() && !uuids.is_empty() { return Err(client::invalid_argument(format!( - "multimodal UUIDs for `{modality}` cannot be forwarded by the image-only TensorRT-LLM sidecar" + "multimodal UUIDs for unsupported modality `{modality}` cannot be forwarded" ))); } } - uuid_map.get(IMAGE_KEY).map(Vec::as_slice) - } else { - None - }; + } - if let Some(uuids) = user_image_uuids - && uuids.len() != images.len() - { - return Err(client::invalid_argument(format!( - "`{IMAGE_KEY}` UUID count ({}) must match its media item count ({})", - uuids.len(), - images.len() - ))); + let mut active_kinds = Vec::new(); + let mut total_items = 0; + for kind in MEDIA_KINDS { + let items = media_map + .and_then(|media| media.get(kind.key)) + .map(Vec::as_slice) + .unwrap_or_default(); + if let Some(uuids) = uuid_map.and_then(|uuids| uuids.get(kind.key)) + && uuids.len() != items.len() + { + return Err(client::invalid_argument(format!( + "`{}` UUID count ({}) must match its media item count ({})", + kind.key, + uuids.len(), + items.len() + ))); + } + if !items.is_empty() { + if !limits.is_some_and(|limits| limits.supported_media.contains(kind.name)) { + return Err(client::invalid_argument(format!( + "the loaded TensorRT-LLM model does not advertise {} support through Control.GetModelInfo", + kind.name + ))); + } + active_kinds.push(kind); + total_items += items.len(); + } } - if images.is_empty() { + if total_items == 0 { return Ok(Vec::new()); } - if !limits.is_some_and(|limits| limits.supports_image) { - return Err(client::invalid_argument( - "the loaded TensorRT-LLM model does not advertise image support through Control.GetModelInfo", - )); - } - images - .iter() - .enumerate() - .map(|(index, image)| { - let source = match image { - MultimodalData::Url(url) => media_source(url.as_str(), index)?, - MultimodalData::RawUrl(url) => media_source(url, index)?, + let ordered_items: Vec<(MediaKind, usize)> = match request.multi_modal_order.as_ref() { + Some(order) => { + if order.len() != total_items { + return Err(client::invalid_argument(format!( + "multi_modal_order has {} entries for {total_items} media items", + order.len() + ))); + } + let mut seen = HashSet::with_capacity(order.len()); + order + .iter() + .map(|item| { + let kind = media_kind(&item.modality).ok_or_else(|| { + client::invalid_argument(format!( + "multi_modal_order references unsupported modality `{}`", + item.modality + )) + })?; + let item_count = media_map + .and_then(|media| media.get(kind.key)) + .map(Vec::len) + .unwrap_or_default(); + if item.index >= item_count { + return Err(client::invalid_argument(format!( + "multi_modal_order references missing `{}` item {}", + kind.key, item.index + ))); + } + if !seen.insert((kind.key, item.index)) { + return Err(client::invalid_argument(format!( + "multi_modal_order repeats `{}` item {}", + kind.key, item.index + ))); + } + Ok((kind, item.index)) + }) + .collect::>()? + } + None if active_kinds.len() == 1 => { + let kind = active_kinds[0]; + let item_count = media_map + .and_then(|media| media.get(kind.key)) + .map(Vec::len) + .unwrap_or_default(); + (0..item_count).map(|index| (kind, index)).collect() + } + None => { + return Err(client::invalid_argument( + "mixed multimodal input requires multi_modal_order to preserve prompt order", + )); + } + }; + + ordered_items + .into_iter() + .map(|(kind, index)| { + let item = &media_map.expect("media exists")[kind.key][index]; + let source = match item { + MultimodalData::Url(url) => media_source(url.as_str(), kind.key, index)?, + MultimodalData::RawUrl(url) => media_source(url, kind.key, index)?, MultimodalData::Decoded(_) => { return Err(client::invalid_argument(format!( - "`{IMAGE_KEY}` item {index} is pre-decoded; OpenEngine requires a URL or data URI so TensorRT-LLM can own image decoding" + "`{}` item {index} is pre-decoded; OpenEngine requires a URL or data URI so TensorRT-LLM can own {} decoding", + kind.key, kind.name ))); } MultimodalData::UuidOnly(_) => { return Err(client::invalid_argument(format!( - "`{IMAGE_KEY}` item {index} contains only a cache UUID; OpenEngine requires an image source" + "`{}` item {index} contains only a cache UUID; OpenEngine requires a source for {} input", + kind.key, kind.name ))); } }; Ok(pb::MediaItem { - modality: pb::Modality::Image as i32, + modality: kind.modality as i32, source: Some(source), // MIME metadata belongs to raw_bytes, which this client does not send. mime_type: String::new(), // Do not substitute router-owned mm_hashes for caller cache identities. - uuid: user_image_uuids + uuid: uuid_map + .and_then(|uuids| uuids.get(kind.key)) .and_then(|uuids| uuids[index].as_deref()) .unwrap_or_default() .to_string(), @@ -267,7 +354,11 @@ fn image_media( .collect() } -fn media_source(value: &str, index: usize) -> Result { +fn media_source( + value: &str, + key: &str, + index: usize, +) -> Result { let scheme = value .split_once(':') .map(|(scheme, _)| scheme.to_ascii_lowercase()); @@ -275,7 +366,7 @@ fn media_source(value: &str, index: usize) -> Result Ok(pb::media_item::Source::Url(value.to_string())), Some("data") => Ok(pb::media_item::Source::DataUri(value.to_string())), _ => Err(client::invalid_argument(format!( - "`{IMAGE_KEY}` item {index} must use an http, https, or data URI" + "`{key}` item {index} must use an http, https, or data URI" ))), } } diff --git a/lib/sidecar/trtllm/src/tests.rs b/lib/sidecar/trtllm/src/tests.rs index 0bcda724b744..d9b97d6e8386 100644 --- a/lib/sidecar/trtllm/src/tests.rs +++ b/lib/sidecar/trtllm/src/tests.rs @@ -11,7 +11,8 @@ use std::time::Duration; use dynamo_backend_common::{ DisaggregationMode, ErrorType, FinishReason, GenerateContext, LLMEngine, MultimodalData, - OutputOptions, PreprocessedRequest, SamplingOptions, StopConditions, StopReason, + MultimodalItemRef, OutputOptions, PreprocessedRequest, SamplingOptions, StopConditions, + StopReason, }; use dynamo_sidecar_common::{GrpcEndpoint, GrpcTransportConfig}; use futures::{Stream, StreamExt}; @@ -21,7 +22,7 @@ use tokio::sync::{Mutex, oneshot}; use tokio_stream::wrappers::TcpListenerStream; use tonic::{Request, Response, Status}; -use crate::client::{ModelMetadata, TrtllmClient}; +use crate::client::{ModelMetadata, SupportedMedia, TrtllmClient}; use crate::convert::{ResponseState, build_generate_request, engine_error}; use crate::engine::TrtllmSidecarEngine; use crate::model::ConfiguredModel; @@ -64,6 +65,7 @@ struct FakeTrtllm { supports_multimodal: Arc, /// TensorRT-LLM's finer-grained modality advertisement. advertises_image: Arc, + advertises_video: Arc, model_info_calls: Arc, } @@ -257,13 +259,16 @@ impl pb::control_server::Control for FakeTrtllm { return Err(Status::unavailable("not serving yet")); } let supports_multimodal = self.supports_multimodal.load(Ordering::SeqCst); - let modalities = if self.advertises_image.load(Ordering::SeqCst) { - vec![prost_types::Value { - kind: Some(prost_types::value::Kind::StringValue("image".to_string())), - }] - } else { - Vec::new() - }; + let modalities = [ + (self.advertises_image.load(Ordering::SeqCst), "image"), + (self.advertises_video.load(Ordering::SeqCst), "video"), + ] + .into_iter() + .filter(|(advertised, _)| *advertised) + .map(|(_, modality)| prost_types::Value { + kind: Some(prost_types::value::Kind::StringValue(modality.to_string())), + }) + .collect(); Ok(Response::new(pb::ModelInfo { model_id: "fake-model".to_string(), max_context_length: if self.empty_model_info.load(Ordering::SeqCst) { @@ -469,7 +474,7 @@ fn limits(context_length: u32) -> Option { Some(ModelMetadata { context_length: Some(context_length), max_output_tokens: None, - supports_image: false, + supported_media: SupportedMedia::default(), }) } @@ -477,7 +482,21 @@ fn image_limits(context_length: u32) -> Option { Some(ModelMetadata { context_length: Some(context_length), max_output_tokens: None, - supports_image: true, + supported_media: SupportedMedia { + image: true, + ..Default::default() + }, + }) +} + +fn image_video_limits(context_length: u32) -> Option { + Some(ModelMetadata { + context_length: Some(context_length), + max_output_tokens: None, + supported_media: SupportedMedia { + image: true, + video: true, + }, }) } diff --git a/lib/sidecar/trtllm/src/tests/convert_request.rs b/lib/sidecar/trtllm/src/tests/convert_request.rs index 57007959e472..233331a4fc8a 100644 --- a/lib/sidecar/trtllm/src/tests/convert_request.rs +++ b/lib/sidecar/trtllm/src/tests/convert_request.rs @@ -109,6 +109,120 @@ fn image_urls_data_uris_and_uuids_are_mapped_in_order() { assert!(proto.media[1].uuid.is_empty()); } +#[test] +fn mixed_image_and_video_media_follow_prompt_order() { + let mut req = request(); + req.multi_modal_data = Some(HashMap::from([ + ( + "image_url".to_string(), + vec![ + MultimodalData::RawUrl("https://example.com/first.png".to_string()), + MultimodalData::RawUrl("data:image/png;base64,aW1hZ2U=".to_string()), + ], + ), + ( + "video_url".to_string(), + vec![ + MultimodalData::RawUrl("https://example.com/first.mp4".to_string()), + MultimodalData::RawUrl("data:video/mp4;base64,dmlkZW8=".to_string()), + ], + ), + ])); + req.multi_modal_uuids = Some(HashMap::from([ + ( + "image_url".to_string(), + vec![Some("image-0".to_string()), None], + ), + ( + "video_url".to_string(), + vec![Some("video-0".to_string()), Some("video-1".to_string())], + ), + ])); + req.multi_modal_order = Some(vec![ + MultimodalItemRef { + modality: "video_url".to_string(), + index: 0, + }, + MultimodalItemRef { + modality: "image_url".to_string(), + index: 0, + }, + MultimodalItemRef { + modality: "video_url".to_string(), + index: 1, + }, + MultimodalItemRef { + modality: "image_url".to_string(), + index: 1, + }, + ]); + + let proto = build_generate_request(&req, "req", "model", image_video_limits(4096), AGG) + .expect("mixed media request"); + + assert_eq!( + proto + .media + .iter() + .map(|item| item.modality) + .collect::>(), + [ + pb::Modality::Video as i32, + pb::Modality::Image as i32, + pb::Modality::Video as i32, + pb::Modality::Image as i32, + ] + ); + assert_eq!( + proto + .media + .iter() + .map(|item| item.uuid.as_str()) + .collect::>(), + ["video-0", "image-0", "video-1", ""] + ); +} + +#[test] +fn mixed_media_without_a_complete_unique_order_is_rejected() { + let mixed_media = || { + HashMap::from([ + ( + "image_url".to_string(), + vec![MultimodalData::RawUrl( + "https://example.com/image.png".to_string(), + )], + ), + ( + "video_url".to_string(), + vec![MultimodalData::RawUrl( + "https://example.com/video.mp4".to_string(), + )], + ), + ]) + }; + + let mut req = request(); + req.multi_modal_data = Some(mixed_media()); + let error = build_generate_request(&req, "req", "model", image_video_limits(4096), AGG) + .expect_err("mixed legacy request has lost its order"); + assert!(error.to_string().contains("requires multi_modal_order")); + + req.multi_modal_order = Some(vec![ + MultimodalItemRef { + modality: "image_url".to_string(), + index: 0, + }, + MultimodalItemRef { + modality: "image_url".to_string(), + index: 0, + }, + ]); + let error = build_generate_request(&req, "req", "model", image_video_limits(4096), AGG) + .expect_err("duplicate order entry must fail"); + assert!(error.to_string().contains("repeats `image_url` item 0")); +} + #[test] fn mm_routing_hashes_are_not_used_as_engine_cache_uuids() { let mut req = request(); @@ -147,11 +261,7 @@ fn image_media_requires_the_loaded_models_advertised_capability() { #[test] fn unsupported_media_modalities_and_sources_are_rejected() { - for (key, expected) in [ - ("video_url", "video_url"), - ("audio_url", "audio_url"), - ("future_media", "future_media"), - ] { + for (key, expected) in [("audio_url", "audio_url"), ("future_media", "future_media")] { let mut req = request(); req.multi_modal_data = Some(HashMap::from([( key.to_string(), @@ -160,7 +270,7 @@ fn unsupported_media_modalities_and_sources_are_rejected() { )], )])); let error = build_generate_request(&req, "req", "model", image_limits(4096), AGG) - .expect_err("only images are supported"); + .expect_err("unsupported modalities must be rejected"); assert!( error.to_string().contains(expected), "unexpected error: {error}" @@ -180,7 +290,20 @@ fn unsupported_media_modalities_and_sources_are_rejected() { )])); let error = build_generate_request(&req, "req", "model", image_limits(4096), AGG) .expect_err("OpenEngine cannot resolve a source-less UUID"); - assert!(error.to_string().contains("requires an image source")); + assert!( + error + .to_string() + .contains("requires a source for image input") + ); + + let mut req = request(); + req.multi_modal_data = Some(HashMap::from([( + "video_url".to_string(), + vec![MultimodalData::RawUrl("file:///tmp/video.mp4".to_string())], + )])); + let error = build_generate_request(&req, "req", "model", image_video_limits(4096), AGG) + .expect_err("the engine cannot fetch a local Dynamo path"); + assert!(error.to_string().contains("`video_url` item 0")); } #[test] @@ -337,7 +460,7 @@ fn the_derived_max_tokens_respects_the_engines_output_cap() { Some(ModelMetadata { context_length: Some(4096), max_output_tokens: Some(256), - supports_image: false, + supported_media: SupportedMedia::default(), }), AGG, ) diff --git a/lib/sidecar/trtllm/src/tests/e2e.rs b/lib/sidecar/trtllm/src/tests/e2e.rs index f3b213378490..69eaab35ae89 100644 --- a/lib/sidecar/trtllm/src/tests/e2e.rs +++ b/lib/sidecar/trtllm/src/tests/e2e.rs @@ -382,3 +382,77 @@ async fn e2e_real_openengine_image() { "the engine's expanded image prompt usage must replace Dynamo's unexpanded token count" ); } + +/// Drives a video data URI through the real sidecar and OpenEngine server. +/// `TRTLLM_E2E_VIDEO_DATA_URI` must contain a complete `data:video/...` URI. +#[tokio::test] +#[ignore = "requires a live video-capable TensorRT-LLM OpenEngine server"] +async fn e2e_real_openengine_video() { + let endpoint = std::env::var("TRTLLM_E2E_ENDPOINT") + .expect("set TRTLLM_E2E_ENDPOINT, e.g. http://127.0.0.1:50051"); + let model = std::env::var("TRTLLM_E2E_MODEL") + .unwrap_or_else(|_| "Qwen/Qwen3-VL-2B-Instruct".to_string()); + let video = std::env::var("TRTLLM_E2E_VIDEO_DATA_URI") + .expect("set TRTLLM_E2E_VIDEO_DATA_URI to a video data URI"); + let engine = TrtllmSidecarEngine::new( + GrpcEndpoint::parse(&endpoint, "--grpc-endpoint").expect("valid endpoint"), + transport(1), + ConfiguredModel { + source: model.clone(), + context_length: None, + }, + AGG, + ); + engine + .start(0) + .await + .expect("start against the live video-capable server"); + + let token_ids = vec![ + 151644, 872, 198, 151652, 151656, 151653, 74785, 279, 7218, 6083, 323, 1181, 5106, 304, + 825, 2805, 11652, 13, 151645, 198, 151644, 77091, 198, + ]; + let mut request = PreprocessedRequest::builder() + .model(model) + .token_ids(token_ids.clone()) + .stop_conditions(StopConditions { + max_tokens: Some(64), + ..Default::default() + }) + .sampling_options(SamplingOptions { + temperature: Some(0.0), + ..Default::default() + }) + .output_options(OutputOptions::default()) + .build() + .expect("request"); + request.multi_modal_data = Some(HashMap::from([( + "video_url".to_string(), + vec![MultimodalData::RawUrl(video)], + )])); + request.multi_modal_order = Some(vec![MultimodalItemRef { + modality: "video_url".to_string(), + index: 0, + }]); + + let outputs = collect(&engine, request).await; + let generated = outputs + .iter() + .flat_map(|output| output.token_ids.iter().copied()) + .collect::>(); + let terminal = outputs + .iter() + .find(|output| output.finish_reason.is_some()) + .expect("terminal output"); + let usage = terminal.completion_usage.as_ref().expect("terminal usage"); + eprintln!( + "[e2e-video] generated={generated:?}; usage: prompt={}, completion={}", + usage.prompt_tokens, usage.completion_tokens + ); + + assert!(!generated.is_empty(), "expected video-conditioned output"); + assert!( + usage.prompt_tokens > token_ids.len() as u32, + "the engine must report the expanded video prompt" + ); +} diff --git a/lib/sidecar/trtllm/src/tests/engine.rs b/lib/sidecar/trtllm/src/tests/engine.rs index 149303a06d3b..4464fec80a18 100644 --- a/lib/sidecar/trtllm/src/tests/engine.rs +++ b/lib/sidecar/trtllm/src/tests/engine.rs @@ -75,6 +75,36 @@ async fn advertised_image_media_reaches_openengine_and_uses_expanded_usage() { )); } +#[tokio::test] +async fn advertised_video_media_reaches_openengine() { + let service = FakeTrtllm::default(); + service.supports_multimodal.store(true, Ordering::SeqCst); + service.advertises_video.store(true, Ordering::SeqCst); + let server = FakeServer::start(service).await; + let engine = engine(&server.endpoint, 1); + engine.start(0).await.expect("start"); + + let mut req = request(); + req.multi_modal_data = Some(HashMap::from([( + "video_url".to_string(), + vec![MultimodalData::RawUrl( + "https://example.com/video.mp4".to_string(), + )], + )])); + collect(&engine, req).await; + + let requests = server.service.requests.lock().await; + let sent = requests.first().expect("recorded request"); + assert_eq!(sent.media.len(), 1); + assert_eq!(sent.media[0].modality, pb::Modality::Video as i32); + assert_eq!( + sent.media[0].source, + Some(pb::media_item::Source::Url( + "https://example.com/video.mp4".to_string() + )) + ); +} + #[tokio::test] async fn both_multimodal_capability_signals_are_required() { for (coarse, image) in [(true, false), (false, true)] { From 7ff6c3c2616bd85bea784c28e2cc110dcc22a8e1 Mon Sep 17 00:00:00 2001 From: krishung5 Date: Thu, 15 Oct 2026 21:05:00 -0700 Subject: [PATCH 06/61] feat(trtllm): pass audio inputs through OpenEngine Signed-off-by: krishung5 --- lib/sidecar/trtllm/src/client.rs | 3 + lib/sidecar/trtllm/src/convert.rs | 10 ++- lib/sidecar/trtllm/src/tests.rs | 15 ++++ .../trtllm/src/tests/convert_request.rs | 72 ++++++++++++++----- lib/sidecar/trtllm/src/tests/e2e.rs | 62 ++++++++++++++++ lib/sidecar/trtllm/src/tests/engine.rs | 28 ++++++++ 6 files changed, 169 insertions(+), 21 deletions(-) diff --git a/lib/sidecar/trtllm/src/client.rs b/lib/sidecar/trtllm/src/client.rs index 0efbf801be1e..dea7f721442b 100644 --- a/lib/sidecar/trtllm/src/client.rs +++ b/lib/sidecar/trtllm/src/client.rs @@ -32,6 +32,7 @@ const RPC_TIMEOUT: Duration = Duration::from_secs(30); pub(crate) struct SupportedMedia { pub(crate) image: bool, pub(crate) video: bool, + pub(crate) audio: bool, } impl SupportedMedia { @@ -39,6 +40,7 @@ impl SupportedMedia { match modality { "image" => self.image, "video" => self.video, + "audio" => self.audio, _ => false, } } @@ -136,6 +138,7 @@ impl TrtllmClient { match modality.as_str() { "image" => supported_media.image = true, "video" => supported_media.video = true, + "audio" => supported_media.audio = true, _ => {} } } diff --git a/lib/sidecar/trtllm/src/convert.rs b/lib/sidecar/trtllm/src/convert.rs index 811ed083b731..01c0816bc85c 100644 --- a/lib/sidecar/trtllm/src/convert.rs +++ b/lib/sidecar/trtllm/src/convert.rs @@ -4,7 +4,7 @@ //! Conversion between Dynamo's `PreprocessedRequest` / `LLMEngineOutput` and the //! TensorRT-LLM OpenEngine (`openengine.v1`) protobuf messages. //! -//! Supports aggregated and disaggregated image and video generation. Audio, +//! Supports aggregated and disaggregated image, video, and audio generation. //! LoRA, beam search, and `n > 1` are rejected before dispatch. //! //! Disaggregation is driven by [`DisaggregationMode`]: a prefill worker marks @@ -176,6 +176,7 @@ fn max_tokens( const IMAGE_KEY: &str = "image_url"; const VIDEO_KEY: &str = "video_url"; +const AUDIO_KEY: &str = "audio_url"; #[derive(Clone, Copy)] struct MediaKind { @@ -184,7 +185,7 @@ struct MediaKind { modality: pb::Modality, } -const MEDIA_KINDS: [MediaKind; 2] = [ +const MEDIA_KINDS: [MediaKind; 3] = [ MediaKind { key: IMAGE_KEY, name: "image", @@ -195,6 +196,11 @@ const MEDIA_KINDS: [MediaKind; 2] = [ name: "video", modality: pb::Modality::Video, }, + MediaKind { + key: AUDIO_KEY, + name: "audio", + modality: pb::Modality::Audio, + }, ]; fn media_kind(key: &str) -> Option { diff --git a/lib/sidecar/trtllm/src/tests.rs b/lib/sidecar/trtllm/src/tests.rs index d9b97d6e8386..a7e52b951f80 100644 --- a/lib/sidecar/trtllm/src/tests.rs +++ b/lib/sidecar/trtllm/src/tests.rs @@ -66,6 +66,7 @@ struct FakeTrtllm { /// TensorRT-LLM's finer-grained modality advertisement. advertises_image: Arc, advertises_video: Arc, + advertises_audio: Arc, model_info_calls: Arc, } @@ -262,6 +263,7 @@ impl pb::control_server::Control for FakeTrtllm { let modalities = [ (self.advertises_image.load(Ordering::SeqCst), "image"), (self.advertises_video.load(Ordering::SeqCst), "video"), + (self.advertises_audio.load(Ordering::SeqCst), "audio"), ] .into_iter() .filter(|(advertised, _)| *advertised) @@ -496,6 +498,19 @@ fn image_video_limits(context_length: u32) -> Option { supported_media: SupportedMedia { image: true, video: true, + ..Default::default() + }, + }) +} + +fn all_media_limits(context_length: u32) -> Option { + Some(ModelMetadata { + context_length: Some(context_length), + max_output_tokens: None, + supported_media: SupportedMedia { + image: true, + video: true, + audio: true, }, }) } diff --git a/lib/sidecar/trtllm/src/tests/convert_request.rs b/lib/sidecar/trtllm/src/tests/convert_request.rs index 233331a4fc8a..6a6d63d86f1b 100644 --- a/lib/sidecar/trtllm/src/tests/convert_request.rs +++ b/lib/sidecar/trtllm/src/tests/convert_request.rs @@ -110,7 +110,7 @@ fn image_urls_data_uris_and_uuids_are_mapped_in_order() { } #[test] -fn mixed_image_and_video_media_follow_prompt_order() { +fn mixed_media_follow_prompt_order() { let mut req = request(); req.multi_modal_data = Some(HashMap::from([ ( @@ -127,6 +127,12 @@ fn mixed_image_and_video_media_follow_prompt_order() { MultimodalData::RawUrl("data:video/mp4;base64,dmlkZW8=".to_string()), ], ), + ( + "audio_url".to_string(), + vec![MultimodalData::RawUrl( + "data:audio/wav;base64,YXVkaW8=".to_string(), + )], + ), ])); req.multi_modal_uuids = Some(HashMap::from([ ( @@ -137,6 +143,7 @@ fn mixed_image_and_video_media_follow_prompt_order() { "video_url".to_string(), vec![Some("video-0".to_string()), Some("video-1".to_string())], ), + ("audio_url".to_string(), vec![Some("audio-0".to_string())]), ])); req.multi_modal_order = Some(vec![ MultimodalItemRef { @@ -147,6 +154,10 @@ fn mixed_image_and_video_media_follow_prompt_order() { modality: "image_url".to_string(), index: 0, }, + MultimodalItemRef { + modality: "audio_url".to_string(), + index: 0, + }, MultimodalItemRef { modality: "video_url".to_string(), index: 1, @@ -157,7 +168,7 @@ fn mixed_image_and_video_media_follow_prompt_order() { }, ]); - let proto = build_generate_request(&req, "req", "model", image_video_limits(4096), AGG) + let proto = build_generate_request(&req, "req", "model", all_media_limits(4096), AGG) .expect("mixed media request"); assert_eq!( @@ -169,6 +180,7 @@ fn mixed_image_and_video_media_follow_prompt_order() { [ pb::Modality::Video as i32, pb::Modality::Image as i32, + pb::Modality::Audio as i32, pb::Modality::Video as i32, pb::Modality::Image as i32, ] @@ -179,7 +191,7 @@ fn mixed_image_and_video_media_follow_prompt_order() { .iter() .map(|item| item.uuid.as_str()) .collect::>(), - ["video-0", "image-0", "video-1", ""] + ["video-0", "image-0", "audio-0", "video-1", ""] ); } @@ -243,7 +255,7 @@ fn mm_routing_hashes_are_not_used_as_engine_cache_uuids() { } #[test] -fn image_media_requires_the_loaded_models_advertised_capability() { +fn media_requires_the_loaded_models_advertised_capability() { let mut req = request(); add_images(&mut req, ["https://example.com/image.png"]); @@ -257,25 +269,38 @@ fn image_media_requires_the_loaded_models_advertised_capability() { "unexpected error: {error}" ); } + + req.multi_modal_data = Some(HashMap::from([( + "audio_url".to_string(), + vec![MultimodalData::RawUrl( + "https://example.com/audio.wav".to_string(), + )], + )])); + let error = build_generate_request(&req, "req", "model", image_video_limits(4096), AGG) + .expect_err("a model without audio support must reject audio"); + assert!( + error + .to_string() + .contains("does not advertise audio support"), + "unexpected error: {error}" + ); } #[test] fn unsupported_media_modalities_and_sources_are_rejected() { - for (key, expected) in [("audio_url", "audio_url"), ("future_media", "future_media")] { - let mut req = request(); - req.multi_modal_data = Some(HashMap::from([( - key.to_string(), - vec![MultimodalData::RawUrl( - "https://example.com/media".to_string(), - )], - )])); - let error = build_generate_request(&req, "req", "model", image_limits(4096), AGG) - .expect_err("unsupported modalities must be rejected"); - assert!( - error.to_string().contains(expected), - "unexpected error: {error}" - ); - } + let mut req = request(); + req.multi_modal_data = Some(HashMap::from([( + "future_media".to_string(), + vec![MultimodalData::RawUrl( + "https://example.com/media".to_string(), + )], + )])); + let error = build_generate_request(&req, "req", "model", all_media_limits(4096), AGG) + .expect_err("unsupported modalities must be rejected"); + assert!( + error.to_string().contains("future_media"), + "unexpected error: {error}" + ); let mut req = request(); add_images(&mut req, ["file:///tmp/image.png"]); @@ -304,6 +329,15 @@ fn unsupported_media_modalities_and_sources_are_rejected() { let error = build_generate_request(&req, "req", "model", image_video_limits(4096), AGG) .expect_err("the engine cannot fetch a local Dynamo path"); assert!(error.to_string().contains("`video_url` item 0")); + + let mut req = request(); + req.multi_modal_data = Some(HashMap::from([( + "audio_url".to_string(), + vec![MultimodalData::RawUrl("file:///tmp/audio.wav".to_string())], + )])); + let error = build_generate_request(&req, "req", "model", all_media_limits(4096), AGG) + .expect_err("the engine cannot fetch a local Dynamo path"); + assert!(error.to_string().contains("`audio_url` item 0")); } #[test] diff --git a/lib/sidecar/trtllm/src/tests/e2e.rs b/lib/sidecar/trtllm/src/tests/e2e.rs index 69eaab35ae89..68a06fc03173 100644 --- a/lib/sidecar/trtllm/src/tests/e2e.rs +++ b/lib/sidecar/trtllm/src/tests/e2e.rs @@ -456,3 +456,65 @@ async fn e2e_real_openengine_video() { "the engine must report the expanded video prompt" ); } + +/// Drives an audio data URI through the real sidecar and OpenEngine server. +/// `TRTLLM_E2E_AUDIO_DATA_URI` must contain a complete `data:audio/...` URI. +#[tokio::test] +#[ignore = "requires a live audio-capable TensorRT-LLM OpenEngine server"] +async fn e2e_real_openengine_audio() { + let endpoint = std::env::var("TRTLLM_E2E_ENDPOINT") + .expect("set TRTLLM_E2E_ENDPOINT, e.g. http://127.0.0.1:50051"); + let model = + std::env::var("TRTLLM_E2E_MODEL").unwrap_or_else(|_| "openai/whisper-tiny".to_string()); + let audio = std::env::var("TRTLLM_E2E_AUDIO_DATA_URI") + .expect("set TRTLLM_E2E_AUDIO_DATA_URI to an audio data URI"); + let engine = TrtllmSidecarEngine::new( + GrpcEndpoint::parse(&endpoint, "--grpc-endpoint").expect("valid endpoint"), + transport(1), + ConfiguredModel { + source: model.clone(), + context_length: None, + }, + AGG, + ); + engine + .start(0) + .await + .expect("start against the live audio-capable server"); + + let mut request = PreprocessedRequest::builder() + .model(model) + .token_ids(vec![50258, 50259, 50359, 50363]) + .stop_conditions(StopConditions { + max_tokens: Some(96), + ..Default::default() + }) + .sampling_options(SamplingOptions { + temperature: Some(0.0), + ..Default::default() + }) + .output_options(OutputOptions::default()) + .build() + .expect("request"); + request.multi_modal_data = Some(HashMap::from([( + "audio_url".to_string(), + vec![MultimodalData::RawUrl(audio)], + )])); + + let outputs = collect(&engine, request).await; + let generated = outputs + .iter() + .flat_map(|output| output.token_ids.iter().copied()) + .collect::>(); + let terminal = outputs + .iter() + .find(|output| output.finish_reason.is_some()) + .expect("terminal output"); + let usage = terminal.completion_usage.as_ref().expect("terminal usage"); + eprintln!( + "[e2e-audio] generated={generated:?}; usage: prompt={}, completion={}", + usage.prompt_tokens, usage.completion_tokens + ); + + assert!(!generated.is_empty(), "expected audio-conditioned output"); +} diff --git a/lib/sidecar/trtllm/src/tests/engine.rs b/lib/sidecar/trtllm/src/tests/engine.rs index 4464fec80a18..b7adf80da20a 100644 --- a/lib/sidecar/trtllm/src/tests/engine.rs +++ b/lib/sidecar/trtllm/src/tests/engine.rs @@ -105,6 +105,34 @@ async fn advertised_video_media_reaches_openengine() { ); } +#[tokio::test] +async fn advertised_audio_media_reaches_openengine() { + let service = FakeTrtllm::default(); + service.supports_multimodal.store(true, Ordering::SeqCst); + service.advertises_audio.store(true, Ordering::SeqCst); + let server = FakeServer::start(service).await; + let engine = engine(&server.endpoint, 1); + engine.start(0).await.expect("start"); + + let mut req = request(); + req.multi_modal_data = Some(HashMap::from([( + "audio_url".to_string(), + vec![MultimodalData::RawUrl( + "data:audio/wav;base64,YXVkaW8=".to_string(), + )], + )])); + collect(&engine, req).await; + + let requests = server.service.requests.lock().await; + let sent = requests.first().expect("recorded request"); + assert_eq!(sent.media.len(), 1); + assert_eq!(sent.media[0].modality, pb::Modality::Audio as i32); + assert!(matches!( + sent.media[0].source.as_ref(), + Some(pb::media_item::Source::DataUri(value)) if value.starts_with("data:audio/wav") + )); +} + #[tokio::test] async fn both_multimodal_capability_signals_are_required() { for (coarse, image) in [(true, false), (false, true)] { From 78b93ab422f3569375906a5582490d953e5ae4e8 Mon Sep 17 00:00:00 2001 From: krishung5 Date: Thu, 15 Oct 2026 11:16:15 -0700 Subject: [PATCH 07/61] feat(trtllm): support image inputs in sidecar Signed-off-by: krishung5 --- lib/llm/src/backend.rs | 206 ++++++++++++++- lib/sidecar/trtllm/src/client.rs | 25 ++ lib/sidecar/trtllm/src/convert.rs | 240 ++++++++++++++++-- lib/sidecar/trtllm/src/tests.rs | 55 +++- .../trtllm/src/tests/convert_request.rs | 178 +++++++++++++ lib/sidecar/trtllm/src/tests/disagg.rs | 53 ++++ lib/sidecar/trtllm/src/tests/e2e.rs | 180 +++++++++++++ lib/sidecar/trtllm/src/tests/engine.rs | 58 +++++ 8 files changed, 961 insertions(+), 34 deletions(-) diff --git a/lib/llm/src/backend.rs b/lib/llm/src/backend.rs index 3a73e46a21b0..4c5a2d80e0d7 100644 --- a/lib/llm/src/backend.rs +++ b/lib/llm/src/backend.rs @@ -261,8 +261,8 @@ impl return state.pending_flush.pop().map(|output| (output, state)); } - let output = loop { - let output = state.stream.next().await; + let (output, trailing_usage) = loop { + let mut output = state.stream.next().await; let choice_already_finished = output.as_ref().is_some_and(|output| { !output.is_event() && output.data.as_ref().is_some_and(|data| { @@ -272,7 +272,50 @@ impl }) }); if !choice_already_finished { - break output; + break (output, false); + } + + let carries_usage = output + .as_ref() + .and_then(|output| output.data.as_ref()) + .is_some_and(|data| data.completion_usage.is_some()); + let backend_failed = output + .as_ref() + .and_then(|output| output.data.as_ref()) + .is_some_and(|data| matches!(data.finish_reason, Some(FinishReason::Error(_)))); + if carries_usage || backend_failed { + // Native backends can put authoritative cumulative usage + // on a terminal event immediately after the token Dynamo + // recognized locally as EOS. Preserve that metadata without + // emitting duplicate content or a second successful finish + // event. Preserve a backend error with or without usage: it + // is not duplicate success metadata and must reach the + // frontend. + if let Some(data) = output.as_mut().and_then(|output| output.data.as_mut()) { + data.token_ids.clear(); + data.tokens = None; + data.text = None; + data.cum_log_probs = None; + data.log_probs = None; + data.top_logprobs = None; + if !backend_failed { + data.finish_reason = None; + data.stop_reason = None; + } + } + break (output, true); + } + + let is_terminal = output + .as_ref() + .and_then(|output| output.data.as_ref()) + .is_some_and(|data| data.finish_reason.is_some()); + if !is_terminal && state.finished_choices.len() >= state.decoders.len() { + // Once every choice ended locally, tolerate only terminal + // metadata. A nonterminal suffix means the backend ignored + // the stop; cancel rather than burning the remaining budget. + state.stream.context().stop_generating(); + return None; } }; @@ -281,6 +324,11 @@ impl // move to state.process_output // handle any error conditions / unwraps here + if trailing_usage { + state.finished = state.finished_choices.len() >= state.decoders.len(); + return Some((output, state)); + } + // events are pass thru if output.is_event() || output.data.is_none() { // A genuine stream-level error ends generation for every choice at @@ -497,13 +545,12 @@ impl .unwrap_or((None, None)); // If we detected a local stop condition, mark this choice as finished. - // Once all expected choices are finished, stop the upstream generator. + // Keep the upstream stream alive just long enough to consume + // a native backend's following terminal-usage event. The + // loop above still cancels on any nonterminal suffix once + // every choice has stopped. if finish_reason.is_some() && data.finish_reason.is_none() { state.finished_choices.insert(choice_idx); - if state.finished_choices.len() >= decoders_count { - state.stream.context().stop_generating(); - state.finished = true; - } } let text = result.text; @@ -1921,6 +1968,149 @@ mod tests { assert_eq!(generator.get_usage().completion_tokens, 2); } + #[tokio::test] + async fn local_eos_preserves_a_following_authoritative_usage_trailer() { + let tokenizer: Arc = Arc::new(CandidateDecoder); + let backend = Backend::from_tokenizer(Tokenizer::from(tokenizer)); + let request = PreprocessedRequest::builder() + .model("test-model".to_string()) + .token_ids(vec![]) + .stop_conditions(StopConditions { + stop_token_ids_hidden: Some(vec![103]), + ..Default::default() + }) + .sampling_options(SamplingOptions::default()) + .output_options(OutputOptions::default()) + .build() + .expect("valid preprocessed request"); + let reported = dynamo_protocols::types::CompletionUsage { + prompt_tokens: 82, + completion_tokens: 2, + total_tokens: 84, + prompt_tokens_details: None, + completion_tokens_details: None, + }; + let engine: ServerStreamingEngine> = + Arc::new(SyntheticSglangStopEngine { + outputs: Some(vec![ + // Dynamo recognizes 103 as EOS before the native backend's + // separate terminal event arrives. + LLMEngineOutput { + token_ids: vec![101, 103], + index: Some(0), + ..Default::default() + }, + LLMEngineOutput { + index: Some(0), + finish_reason: Some(FinishReason::Stop), + completion_usage: Some(reported.clone()), + ..Default::default() + }, + ]), + }); + + let outputs = Operator::generate(backend.as_ref(), SingleIn::new(request), engine) + .await + .expect("backend generation succeeds") + .collect::>() + .await; + assert_eq!(outputs.len(), 2); + + let local_terminal = outputs[0].data.as_ref().expect("local terminal"); + assert_eq!(local_terminal.finish_reason, Some(FinishReason::Stop)); + let usage_trailer = outputs[1].data.as_ref().expect("usage trailer"); + assert!(usage_trailer.token_ids.is_empty()); + assert!(usage_trailer.finish_reason.is_none()); + assert_eq!(usage_trailer.completion_usage.as_ref(), Some(&reported)); + + let options = DeltaGeneratorOptions::new(None, None, false, None); + let mut generator = DeltaGenerator::new( + "test-model".to_string(), + options, + "test-request".to_string(), + ); + for output in outputs { + generator + .choice_from_postprocessor(output.data.expect("backend output")) + .expect("OpenAI response conversion succeeds"); + } + assert_eq!(generator.get_usage(), reported); + } + + #[tokio::test] + async fn local_eos_does_not_mask_a_trailing_backend_error() { + let tokenizer: Arc = Arc::new(CandidateDecoder); + let backend = Backend::from_tokenizer(Tokenizer::from(tokenizer)); + let request = PreprocessedRequest::builder() + .model("test-model".to_string()) + .token_ids(vec![]) + .stop_conditions(StopConditions { + stop_token_ids_hidden: Some(vec![103]), + ..Default::default() + }) + .sampling_options(SamplingOptions::default()) + .output_options(OutputOptions::default()) + .build() + .expect("valid preprocessed request"); + let reported = dynamo_protocols::types::CompletionUsage { + prompt_tokens: 82, + completion_tokens: 2, + total_tokens: 84, + prompt_tokens_details: None, + completion_tokens_details: None, + }; + for completion_usage in [None, Some(reported.clone())] { + let engine: ServerStreamingEngine> = + Arc::new(SyntheticSglangStopEngine { + outputs: Some(vec![ + LLMEngineOutput { + token_ids: vec![101, 103], + index: Some(0), + ..Default::default() + }, + LLMEngineOutput { + index: Some(0), + finish_reason: Some(FinishReason::Error("worker failed".to_string())), + completion_usage: completion_usage.clone(), + ..Default::default() + }, + ]), + }); + + let outputs = + Operator::generate(backend.as_ref(), SingleIn::new(request.clone()), engine) + .await + .expect("backend generation succeeds") + .collect::>() + .await; + assert_eq!(outputs.len(), 2); + let error_trailer = outputs[1].data.as_ref().expect("error trailer"); + assert!(error_trailer.token_ids.is_empty()); + assert_eq!( + error_trailer.finish_reason, + Some(FinishReason::Error("worker failed".to_string())) + ); + assert_eq!( + error_trailer.completion_usage.as_ref(), + completion_usage.as_ref() + ); + + let options = DeltaGeneratorOptions::new(None, None, false, None); + let mut generator = DeltaGenerator::new( + "test-model".to_string(), + options, + "test-request".to_string(), + ); + generator + .choice_from_postprocessor(outputs[0].data.clone().expect("local terminal")) + .expect("the local terminal converts"); + let error = generator + .choice_from_postprocessor(outputs[1].data.clone().expect("error trailer")) + .expect_err("the backend error must reach the OpenAI response layer"); + assert!(error.to_string().contains("worker failed")); + } + } + #[tokio::test] async fn test_no_stop_trim_raw_completion_output() { use crate::protocols::common::{ diff --git a/lib/sidecar/trtllm/src/client.rs b/lib/sidecar/trtllm/src/client.rs index d1f4f046a014..16cea9315dfd 100644 --- a/lib/sidecar/trtllm/src/client.rs +++ b/lib/sidecar/trtllm/src/client.rs @@ -38,6 +38,11 @@ pub(crate) struct ModelLimits { /// Cap on generated tokens, when the engine reports one. A context window /// alone can imply a larger budget than the engine will accept. pub(crate) max_output_tokens: Option, + /// Whether this loaded model advertises image input through both the + /// portable multimodal bit and TensorRT-LLM's modality list. Requiring both + /// fails closed when a server and client disagree about the finer-grained + /// capability. + pub(crate) supports_image: bool, } pub(crate) struct TrtllmClient { @@ -102,9 +107,29 @@ impl TrtllmClient { }) .await? .into_inner(); + let supports_image = info.supports_multimodal == Some(true) + && info.extra.as_ref().is_some_and(|extra| { + extra + .fields + .get("trtllm_supported_modalities") + .and_then(|value| value.kind.as_ref()) + .is_some_and(|kind| match kind { + prost_types::value::Kind::ListValue(modalities) => { + modalities.values.iter().any(|value| { + matches!( + value.kind.as_ref(), + Some(prost_types::value::Kind::StringValue(modality)) + if modality == "image" + ) + }) + } + _ => false, + }) + }); Ok(ModelLimits { context_length: info.max_context_length.filter(|len| *len > 0), max_output_tokens: info.max_output_tokens.filter(|cap| *cap > 0), + supports_image, }) } diff --git a/lib/sidecar/trtllm/src/convert.rs b/lib/sidecar/trtllm/src/convert.rs index 3360818e2869..0c3e3d59d2f3 100644 --- a/lib/sidecar/trtllm/src/convert.rs +++ b/lib/sidecar/trtllm/src/convert.rs @@ -4,9 +4,10 @@ //! Conversion between Dynamo's `PreprocessedRequest` / `LLMEngineOutput` and the //! TensorRT-LLM OpenEngine (`openengine.v1`) protobuf messages. //! -//! Scope: aggregated and disaggregated (prefill/decode) generation. Multimodal, -//! LoRA, beam search, and `n > 1` are rejected before dispatch — the sidecar -//! streams a single sequence. +//! Scope: aggregated and disaggregated (prefill/decode) generation, including +//! image inputs when the loaded model advertises them. Video, audio, LoRA, +//! beam search, and `n > 1` are rejected before dispatch — the sidecar streams +//! a single sequence. //! //! Disaggregation is driven by [`DisaggregationMode`]: a prefill worker marks //! its request `context_only` and returns the `PrefillReady` handoff as its @@ -17,7 +18,7 @@ use std::collections::BTreeSet; use dynamo_backend_common::{ CompletionUsage, DisaggregationMode, DynamoError, FinishReason, LLMEngineOutput, - PreprocessedRequest, PromptTokensDetails, StopReason, TopLogprob, usage, + MultimodalData, PreprocessedRequest, PromptTokensDetails, StopReason, TopLogprob, usage, }; use crate::client::{self, ModelLimits}; @@ -36,6 +37,7 @@ pub(crate) fn build_generate_request( mode: DisaggregationMode, ) -> Result { validate_request(request, mode)?; + let media = image_media(request, limits)?; let sampling = &request.sampling_options; let stop = &request.stop_conditions; @@ -50,7 +52,7 @@ pub(crate) fn build_generate_request( // reject requests a wider decode window would serve. Releasing the handoff // when decode does reject is the sound fix and belongs on that side. let max_tokens = if mode.is_prefill() { - 1 + Some(1) } else { max_tokens(request, limits)? }; @@ -92,7 +94,7 @@ pub(crate) fn build_generate_request( num_sequences: Some(1), }), stopping: Some(pb::StoppingOptions { - max_tokens: Some(max_tokens), + max_tokens, // Forwarded on a context-only request too. The minimum only masks // EOS until the sequence reaches it, so it cannot extend a context // phase that `max_tokens` already caps at one token. Dropping it @@ -123,8 +125,7 @@ pub(crate) fn build_generate_request( prompt_logprob_start: None, }), guided: guided_decoding(request)?, - // Text generation only: no multimodal media or LoRA selection. - media: Vec::new(), + media, lora_name: String::new(), kv, extra: Some(disagg::request_extra( @@ -139,13 +140,14 @@ pub(crate) fn build_generate_request( } // Temporary workaround: the OpenEngine contract makes `stopping.max_tokens` -// optional, but an omitted value falls through to TensorRT-LLM's small -// `SamplingParams` default rather than filling the context. The Dynamo frontend -// forwards an omitted `max_tokens` as `None` expecting the backend to default, -// so we mirror the in-process backend's text-only default, +// optional. The Dynamo frontend forwards an omitted `max_tokens` as `None` +// expecting the backend to default. For text requests we mirror the in-process +// backend's default, // `max(1, context_length - prompt_len)` (components/src/dynamo/trtllm -// `_default_max_tokens`); the sidecar rejects multimodal before dispatch, so -// `token_ids.len()` is the true prompt length. `context_length` is resolved in +// `_default_max_tokens`). For image requests, only TensorRT-LLM knows the true +// prompt length after expanding image markers, so leave the field unset and let +// the OpenEngine request mapper derive the budget from the expanded prompt. +// `context_length` is resolved in // `engine::start`, where `--context-length` wins over the `GetModelInfo` report. // Only `/v1/chat/completions` and `/v1/responses` reach this fallback: they set // `PRESERVE_OMITTED_MAX_TOKENS_CONTEXT_KEY`, which stops the frontend supplying @@ -161,9 +163,12 @@ pub(crate) fn build_generate_request( fn max_tokens( request: &PreprocessedRequest, limits: Option, -) -> Result { +) -> Result, DynamoError> { if let Some(max_tokens) = request.stop_conditions.max_tokens { - return Ok(max_tokens); + return Ok(Some(max_tokens)); + } + if has_media(request) { + return Ok(None); } let context_length = limits .and_then(|limits| limits.context_length) @@ -185,7 +190,198 @@ fn max_tokens( None => remaining, }; reject_impossible_minimum(request, derived, context_length, prompt_len)?; - Ok(derived) + Ok(Some(derived)) +} + +const IMAGE_KEY: &str = "image_url"; +const MM_HASHES_KEY: &str = "mm_hashes"; + +/// Convert Dynamo's image-only execution payload into the ordered OpenEngine +/// media list. The frontend groups content parts by modality, so ordering is +/// retained within the image vector but not across different modalities. This +/// PR therefore accepts images only and fails explicitly on every other +/// non-empty modality instead of inventing a cross-modality order. +fn image_media( + request: &PreprocessedRequest, + limits: Option, +) -> Result, DynamoError> { + reject_media_override("mm_processor_kwargs", request.mm_processor_kwargs.as_ref())?; + reject_media_override("media_io_kwargs", request.media_io_kwargs.as_ref())?; + + let media_map = request.multi_modal_data.as_ref(); + if let Some(media_map) = media_map { + for (modality, items) in media_map { + if modality != IMAGE_KEY && !items.is_empty() { + return Err(client::invalid_argument(format!( + "multimodal modality `{modality}` is not supported by the TensorRT-LLM sidecar; this image-only implementation accepts `{IMAGE_KEY}`" + ))); + } + } + } + + let images = media_map + .and_then(|media| media.get(IMAGE_KEY)) + .map(Vec::as_slice) + .unwrap_or_default(); + + let user_image_uuids = if let Some(uuid_map) = request.multi_modal_uuids.as_ref() { + for (modality, uuids) in uuid_map { + if modality != IMAGE_KEY && !uuids.is_empty() { + return Err(client::invalid_argument(format!( + "multimodal UUIDs for `{modality}` cannot be forwarded by the image-only TensorRT-LLM sidecar" + ))); + } + } + uuid_map.get(IMAGE_KEY).map(Vec::as_slice) + } else { + None + }; + + if let Some(uuids) = user_image_uuids + && uuids.len() != images.len() + { + return Err(client::invalid_argument(format!( + "`{IMAGE_KEY}` UUID count ({}) must match its media item count ({})", + uuids.len(), + images.len() + ))); + } + + if images.is_empty() { + return Ok(Vec::new()); + } + if !limits.is_some_and(|limits| limits.supports_image) { + return Err(client::invalid_argument( + "the loaded TensorRT-LLM model does not advertise image support through Control.GetModelInfo", + )); + } + + // Explicit caller UUIDs remain authoritative. Otherwise carry the exact + // hashes generated by Dynamo's MM-aware preprocessor to the backend. This + // is request-side preparation for KV-aware routing; the router-owned + // `mm_routing_info` itself never crosses the worker boundary. + let has_user_uuid = user_image_uuids.is_some_and(|uuids| { + uuids + .iter() + .any(|uuid| uuid.as_deref().is_some_and(|uuid| !uuid.is_empty())) + }); + let routing_hashes = if has_user_uuid { + None + } else { + routing_hashes(request, images.len())? + }; + + images + .iter() + .enumerate() + .map(|(index, image)| { + let source = match image { + MultimodalData::Url(url) => media_source(url.as_str(), index)?, + MultimodalData::RawUrl(url) => media_source(url, index)?, + MultimodalData::Decoded(_) => { + return Err(client::invalid_argument(format!( + "`{IMAGE_KEY}` item {index} is pre-decoded; OpenEngine requires a URL or data URI so TensorRT-LLM can own image decoding" + ))); + } + MultimodalData::UuidOnly(_) => { + return Err(client::invalid_argument(format!( + "`{IMAGE_KEY}` item {index} contains only a cache UUID; OpenEngine requires an image source" + ))); + } + }; + Ok(pb::MediaItem { + modality: pb::Modality::Image as i32, + source: Some(source), + mime_type: String::new(), + uuid: if has_user_uuid { + user_image_uuids + .and_then(|uuids| uuids[index].as_deref()) + .unwrap_or_default() + .to_string() + } else { + routing_hashes + .as_ref() + .map(|hashes| hashes[index].clone()) + .unwrap_or_default() + }, + }) + }) + .collect() +} + +fn routing_hashes( + request: &PreprocessedRequest, + image_count: usize, +) -> Result>, DynamoError> { + let hashes = match request.extra_args.as_ref() { + Some(serde_json::Value::Object(extra)) => extra.get(MM_HASHES_KEY), + _ => None, + }; + let Some(hashes) = hashes else { + return Ok(None); + }; + let hashes = hashes.as_array().ok_or_else(|| { + client::invalid_argument("extra_args.mm_hashes must be an array of strings") + })?; + if hashes.is_empty() { + return Ok(None); + } + if hashes.len() != image_count { + return Err(client::invalid_argument(format!( + "extra_args.mm_hashes count ({}) must match the image media item count ({image_count})", + hashes.len() + ))); + } + hashes + .iter() + .enumerate() + .map(|(index, hash)| { + hash.as_str() + .filter(|hash| !hash.is_empty()) + .map(ToOwned::to_owned) + .ok_or_else(|| { + client::invalid_argument(format!( + "extra_args.mm_hashes[{index}] must be a non-empty string" + )) + }) + }) + .collect::, _>>() + .map(Some) +} + +fn media_source(value: &str, index: usize) -> Result { + let scheme = value + .split_once(':') + .map(|(scheme, _)| scheme.to_ascii_lowercase()); + match scheme.as_deref() { + Some("http") | Some("https") => Ok(pb::media_item::Source::Url(value.to_string())), + Some("data") => Ok(pb::media_item::Source::DataUri(value.to_string())), + _ => Err(client::invalid_argument(format!( + "`{IMAGE_KEY}` item {index} must use an http, https, or data URI" + ))), + } +} + +fn reject_media_override(name: &str, value: Option<&serde_json::Value>) -> Result<(), DynamoError> { + let is_empty = match value { + None | Some(serde_json::Value::Null) => true, + Some(serde_json::Value::Object(fields)) => fields.is_empty(), + Some(_) => false, + }; + if is_empty { + Ok(()) + } else { + Err(client::invalid_argument(format!( + "{name} is not supported by the TensorRT-LLM OpenEngine sidecar" + ))) + } +} + +fn has_media(request: &PreprocessedRequest) -> bool { + request + .multi_modal_data + .as_ref() + .is_some_and(|media| media.values().any(|items| !items.is_empty())) } /// Both the output cap and the `.max(1)` floor can land under an explicit @@ -341,14 +537,14 @@ fn validate_request( "prompt embeddings are not supported by the TensorRT-LLM sidecar", )); } - if request.multi_modal_data.is_some() - || request.mm_routing_info.is_some() - || request.encoder_result.is_some() - { + if request.encoder_result.is_some() { return Err(client::invalid_argument( - "multimodal requests are not supported by the TensorRT-LLM sidecar", + "multimodal encoder handoffs are not supported by the TensorRT-LLM sidecar", )); } + // `mm_routing_info` is deliberately not forwarded: it is framework-owned + // metadata that the Dynamo router consumes before selecting this worker. + // The execution payload remains `multi_modal_data`, mapped by `image_media`. // Only a decode worker consumes a prefill handoff. Seeing one anywhere else // means the frontend routed a decode request to the wrong worker role. if request.prefill_result.is_some() && !mode.is_decode() { diff --git a/lib/sidecar/trtllm/src/tests.rs b/lib/sidecar/trtllm/src/tests.rs index 411aa819176e..d6d3c3146762 100644 --- a/lib/sidecar/trtllm/src/tests.rs +++ b/lib/sidecar/trtllm/src/tests.rs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -use std::collections::BTreeSet; +use std::collections::{BTreeSet, HashMap}; use std::net::SocketAddr; use std::num::NonZeroUsize; use std::pin::Pin; @@ -10,8 +10,8 @@ use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::time::Duration; use dynamo_backend_common::{ - DisaggregationMode, ErrorType, FinishReason, GenerateContext, LLMEngine, OutputOptions, - PreprocessedRequest, SamplingOptions, StopConditions, StopReason, + DisaggregationMode, ErrorType, FinishReason, GenerateContext, LLMEngine, MultimodalData, + OutputOptions, PreprocessedRequest, SamplingOptions, StopConditions, StopReason, }; use dynamo_sidecar_common::{GrpcEndpoint, GrpcTransportConfig}; use futures::{Stream, StreamExt}; @@ -60,6 +60,10 @@ struct FakeTrtllm { empty_model_info: Arc, /// Answers `UNAVAILABLE`, the shape of an engine that is not serving yet. unavailable_model_info: Arc, + /// Portable coarse-grained model capability. + supports_multimodal: Arc, + /// TensorRT-LLM's finer-grained modality advertisement. + advertises_image: Arc, model_info_calls: Arc, } @@ -109,7 +113,10 @@ impl pb::inference_server::Inference for FakeTrtllm { } let request_id = request.request_id.clone(); - let prompt_tokens = prompt_len(&request); + // Image markers expand inside TensorRT-LLM after Dynamo tokenizes the + // text prompt. Simulate that so the integration test proves Dynamo + // trusts the engine's terminal usage rather than the unexpanded input. + let prompt_tokens = prompt_len(&request) + if request.media.is_empty() { 0 } else { 64 }; let wants_logprobs = wants_logprobs(&request); let context_only = is_context_only(&request); let hang = self.hang.load(Ordering::SeqCst); @@ -249,6 +256,14 @@ impl pb::control_server::Control for FakeTrtllm { if self.unavailable_model_info.load(Ordering::SeqCst) { return Err(Status::unavailable("not serving yet")); } + let supports_multimodal = self.supports_multimodal.load(Ordering::SeqCst); + let modalities = if self.advertises_image.load(Ordering::SeqCst) { + vec![prost_types::Value { + kind: Some(prost_types::value::Kind::StringValue("image".to_string())), + }] + } else { + Vec::new() + }; Ok(Response::new(pb::ModelInfo { model_id: "fake-model".to_string(), max_context_length: if self.empty_model_info.load(Ordering::SeqCst) { @@ -256,6 +271,19 @@ impl pb::control_server::Control for FakeTrtllm { } else { Some(4096) }, + supports_multimodal: Some(supports_multimodal), + extra: Some(prost_types::Struct { + fields: [( + "trtllm_supported_modalities".to_string(), + prost_types::Value { + kind: Some(prost_types::value::Kind::ListValue( + prost_types::ListValue { values: modalities }, + )), + }, + )] + .into_iter() + .collect(), + }), ..Default::default() })) } @@ -441,9 +469,28 @@ fn limits(context_length: u32) -> Option { Some(ModelLimits { context_length: Some(context_length), max_output_tokens: None, + supports_image: false, + }) +} + +fn image_limits(context_length: u32) -> Option { + Some(ModelLimits { + context_length: Some(context_length), + max_output_tokens: None, + supports_image: true, }) } +fn add_images(request: &mut PreprocessedRequest, images: impl IntoIterator) { + request.multi_modal_data = Some(HashMap::from([( + "image_url".to_string(), + images + .into_iter() + .map(|image| MultimodalData::RawUrl(image.to_string())) + .collect(), + )])); +} + fn engine(endpoint: &str, connections: usize) -> TrtllmSidecarEngine { engine_in_mode(endpoint, connections, AGG) } diff --git a/lib/sidecar/trtllm/src/tests/convert_request.rs b/lib/sidecar/trtllm/src/tests/convert_request.rs index e8dd43d172df..138057cb5dc8 100644 --- a/lib/sidecar/trtllm/src/tests/convert_request.rs +++ b/lib/sidecar/trtllm/src/tests/convert_request.rs @@ -72,6 +72,183 @@ fn request_maps_sampling_stop_and_output_fields() { } } +#[test] +fn image_urls_data_uris_and_uuids_are_mapped_in_order() { + let mut req = request(); + // Routing-only multimodal metadata has already served its purpose by the + // time the request reaches a worker and must not block engine delivery. + req.mm_routing_info = Some(Default::default()); + add_images( + &mut req, + [ + "https://example.com/first.png", + "data:image/png;base64,aW1hZ2U=", + ], + ); + req.multi_modal_uuids = Some(HashMap::from([( + "image_url".to_string(), + vec![Some("first-image".to_string()), None], + )])); + req.extra_args = Some(json!({ + "mm_hashes": ["routing-hash-one", "routing-hash-two"] + })); + + let proto = build_generate_request(&req, "req", "model", image_limits(4096), AGG) + .expect("image request"); + assert_eq!(proto.media.len(), 2); + assert_eq!(proto.media[0].modality, pb::Modality::Image as i32); + assert_eq!( + proto.media[0].source, + Some(pb::media_item::Source::Url( + "https://example.com/first.png".to_string() + )) + ); + assert_eq!(proto.media[0].uuid, "first-image"); + assert_eq!( + proto.media[1].source, + Some(pb::media_item::Source::DataUri( + "data:image/png;base64,aW1hZ2U=".to_string() + )) + ); + assert!(proto.media[1].uuid.is_empty()); +} + +#[test] +fn mm_routing_hashes_are_forwarded_as_image_uuids_in_order() { + let mut req = request(); + add_images( + &mut req, + [ + "https://example.com/first.png", + "https://example.com/second.png", + ], + ); + req.extra_args = Some(json!({ + "mm_hashes": ["0123456789abcdef", "fedcba9876543210"] + })); + + let proto = build_generate_request(&req, "req", "model", image_limits(4096), AGG) + .expect("image request"); + assert_eq!(proto.media[0].uuid, "0123456789abcdef"); + assert_eq!(proto.media[1].uuid, "fedcba9876543210"); +} + +#[test] +fn malformed_or_misaligned_mm_routing_hashes_are_rejected() { + for (hashes, expected) in [ + (json!("not-an-array"), "array of strings"), + (json!(["only-one"]), "must match"), + (json!(["one", ""]), "non-empty string"), + (json!(["one", 2]), "non-empty string"), + ] { + let mut req = request(); + add_images( + &mut req, + [ + "https://example.com/first.png", + "https://example.com/second.png", + ], + ); + req.extra_args = Some(json!({"mm_hashes": hashes})); + let error = build_generate_request(&req, "req", "model", image_limits(4096), AGG) + .expect_err("malformed routing hashes must fail closed"); + assert!( + error.to_string().contains(expected), + "unexpected error: {error}" + ); + } +} + +#[test] +fn image_media_requires_the_loaded_models_advertised_capability() { + let mut req = request(); + add_images(&mut req, ["https://example.com/image.png"]); + + for model_limits in [None, limits(4096)] { + let error = build_generate_request(&req, "req", "model", model_limits, AGG) + .expect_err("a text-only or unknown model must reject images"); + assert!( + error + .to_string() + .contains("does not advertise image support"), + "unexpected error: {error}" + ); + } +} + +#[test] +fn unsupported_media_modalities_and_sources_are_rejected() { + for (key, expected) in [ + ("video_url", "video_url"), + ("audio_url", "audio_url"), + ("future_media", "future_media"), + ] { + let mut req = request(); + req.multi_modal_data = Some(HashMap::from([( + key.to_string(), + vec![MultimodalData::RawUrl( + "https://example.com/media".to_string(), + )], + )])); + let error = build_generate_request(&req, "req", "model", image_limits(4096), AGG) + .expect_err("only images are supported"); + assert!( + error.to_string().contains(expected), + "unexpected error: {error}" + ); + } + + let mut req = request(); + add_images(&mut req, ["file:///tmp/image.png"]); + let error = build_generate_request(&req, "req", "model", image_limits(4096), AGG) + .expect_err("the engine cannot fetch a local Dynamo path"); + assert!(error.to_string().contains("http, https, or data URI")); + + let mut req = request(); + req.multi_modal_data = Some(HashMap::from([( + "image_url".to_string(), + vec![MultimodalData::UuidOnly("cached-image".to_string())], + )])); + let error = build_generate_request(&req, "req", "model", image_limits(4096), AGG) + .expect_err("OpenEngine cannot resolve a source-less UUID"); + assert!(error.to_string().contains("requires an image source")); +} + +#[test] +fn image_metadata_that_cannot_be_honored_is_rejected() { + let mut req = request(); + add_images(&mut req, ["https://example.com/image.png"]); + req.multi_modal_uuids = Some(HashMap::from([( + "image_url".to_string(), + vec![Some("one".to_string()), Some("two".to_string())], + )])); + let error = build_generate_request(&req, "req", "model", image_limits(4096), AGG) + .expect_err("UUIDs must align positionally"); + assert!(error.to_string().contains("UUID count")); + + req.multi_modal_uuids = None; + req.mm_processor_kwargs = Some(serde_json::json!({"do_resize": false})); + let error = build_generate_request(&req, "req", "model", image_limits(4096), AGG) + .expect_err("processor overrides cannot be dropped"); + assert!(error.to_string().contains("mm_processor_kwargs")); + + req.mm_processor_kwargs = Some(serde_json::json!({})); + req.media_io_kwargs = Some(serde_json::json!({"image": {"mode": "RGB"}})); + let error = build_generate_request(&req, "req", "model", image_limits(4096), AGG) + .expect_err("media I/O overrides cannot be dropped"); + assert!(error.to_string().contains("media_io_kwargs")); +} + +#[test] +fn image_request_without_max_tokens_is_left_for_engine_derivation() { + let mut req = request(); + add_images(&mut req, ["https://example.com/image.png"]); + req.stop_conditions.max_tokens = None; + let proto = build_generate_request(&req, "req", "model", image_limits(4096), AGG) + .expect("the engine derives against the expanded prompt"); + assert_eq!(proto.stopping.expect("stopping").max_tokens, None); +} + #[test] fn omitted_max_tokens_without_context_length_is_rejected() { let mut req = request(); @@ -191,6 +368,7 @@ fn the_derived_max_tokens_respects_the_engines_output_cap() { Some(ModelLimits { context_length: Some(4096), max_output_tokens: Some(256), + supports_image: false, }), AGG, ) diff --git a/lib/sidecar/trtllm/src/tests/disagg.rs b/lib/sidecar/trtllm/src/tests/disagg.rs index 3436c037d346..612fda3f39fc 100644 --- a/lib/sidecar/trtllm/src/tests/disagg.rs +++ b/lib/sidecar/trtllm/src/tests/disagg.rs @@ -59,6 +59,59 @@ fn prefill_request_is_marked_context_only() { assert!(proto.kv.is_none(), "prefill carries no session to replay"); } +/// Image inputs must survive both RPC legs. TensorRT-LLM reconstructs its +/// multimodal position/hash state while resuming the KV session, so the decode +/// request needs the same ordered media metadata as the context request. +#[test] +fn image_media_is_preserved_across_prefill_and_decode_requests() { + let mut req = request(); + add_images( + &mut req, + [ + "https://example.com/first.png", + "data:image/png;base64,c2Vjb25k", + ], + ); + req.extra_args = Some(json!({ + "mm_hashes": ["0123456789abcdef", "fedcba9876543210"] + })); + + let prefill = build_generate_request( + &req, + "req", + "model", + image_limits(4096), + DisaggregationMode::Prefill, + ) + .expect("build image prefill"); + assert!(is_context_only(&prefill)); + assert_eq!(prefill.stopping.as_ref().unwrap().max_tokens, Some(1)); + assert_eq!(prefill.media.len(), 2); + assert_eq!(prefill.media[0].uuid, "0123456789abcdef"); + assert_eq!(prefill.media[1].uuid, "fedcba9876543210"); + + req.prefill_result = Some(dynamo_backend_common::PrefillResult { + disaggregated_params: crate::disagg::session_to_json(fake_session()) + .expect("encode handoff"), + prompt_tokens_details: None, + }); + let decode = build_generate_request( + &req, + "req", + "model", + image_limits(4096), + DisaggregationMode::Decode, + ) + .expect("build image decode"); + assert!(!is_context_only(&decode)); + assert_eq!(decode.media, prefill.media); + assert_eq!( + decode.kv.and_then(|kv| kv.session), + Some(fake_session()), + "decode must carry both the image metadata and the KV handoff" + ); +} + /// `PrefillReady` is the prefill worker's terminal chunk: no tokens, and the /// handoff the decode worker will replay. #[tokio::test] diff --git a/lib/sidecar/trtllm/src/tests/e2e.rs b/lib/sidecar/trtllm/src/tests/e2e.rs index 9b5f857cd1df..3f97fe73b7f4 100644 --- a/lib/sidecar/trtllm/src/tests/e2e.rs +++ b/lib/sidecar/trtllm/src/tests/e2e.rs @@ -111,6 +111,107 @@ async fn e2e_real_disagg_handoff() { ); } +/// Exercises the same real prefill/decode handoff with an image request. Both +/// servers may share one sufficiently large GPU for local validation. +/// +/// ```text +/// TRTLLM_E2E_PREFILL_ENDPOINT=http://127.0.0.1:50051 \ +/// TRTLLM_E2E_DECODE_ENDPOINT=http://127.0.0.1:50052 \ +/// TRTLLM_E2E_MODEL=Qwen/Qwen3-VL-2B-Instruct \ +/// cargo test -p dynamo-trtllm-sidecar e2e_real_openengine_image_disagg -- --ignored --nocapture +/// ``` +#[tokio::test] +#[ignore = "requires two live image-capable OpenEngine servers with a KV transceiver"] +async fn e2e_real_openengine_image_disagg() { + let prefill_endpoint = std::env::var("TRTLLM_E2E_PREFILL_ENDPOINT") + .expect("set TRTLLM_E2E_PREFILL_ENDPOINT, e.g. http://127.0.0.1:50051"); + let decode_endpoint = std::env::var("TRTLLM_E2E_DECODE_ENDPOINT") + .expect("set TRTLLM_E2E_DECODE_ENDPOINT, e.g. http://127.0.0.1:50052"); + let model = std::env::var("TRTLLM_E2E_MODEL") + .unwrap_or_else(|_| "Qwen/Qwen3-VL-2B-Instruct".to_string()); + let configured = |source: String| ConfiguredModel { + source, + context_length: None, + }; + let prefill = TrtllmSidecarEngine::new( + GrpcEndpoint::parse(&prefill_endpoint, "--grpc-endpoint").expect("valid endpoint"), + transport(1), + configured(model.clone()), + DisaggregationMode::Prefill, + ); + let decode = TrtllmSidecarEngine::new( + GrpcEndpoint::parse(&decode_endpoint, "--grpc-endpoint").expect("valid endpoint"), + transport(1), + configured(model.clone()), + DisaggregationMode::Decode, + ); + prefill.start(0).await.expect("start image prefill worker"); + decode.start(0).await.expect("start image decode worker"); + + let token_ids = vec![ + 151644, 872, 198, 151652, 151655, 151653, 675, 279, 7987, 504, 2115, 311, 1290, 13, 151645, + 198, 151644, 77091, 198, + ]; + let image_request = || { + let mut request = PreprocessedRequest::builder() + .model(model.clone()) + .token_ids(token_ids.clone()) + .stop_conditions(StopConditions::default()) + .sampling_options(SamplingOptions { + temperature: Some(0.0), + ..Default::default() + }) + .output_options(OutputOptions::default()) + .build() + .expect("request"); + add_images( + &mut request, + [concat!( + "data:image/png;base64,", + "iVBORw0KGgoAAAANSUhEUgAAABwAAAAcCAIAAAD9b0jDAAAAK0lEQVR4nGP8z0AsYGQgVi0T0WaSAEYNHTV01NBRQ0cNHTV01NBRQwefoQBvcAI3xbX7+QAAAABJRU5ErkJggg==" + )], + ); + request + }; + + let prefill_outputs = collect(&prefill, image_request()).await; + assert!( + prefill_outputs + .iter() + .all(|output| output.token_ids.is_empty()), + "the image prefill worker must not stream tokens" + ); + let handoff = prefill_outputs + .iter() + .find_map(|output| output.disaggregated_params.clone()) + .expect("image prefill terminal carries the KV handoff"); + + let mut decode_request = image_request(); + decode_request.prefill_result = Some(dynamo_backend_common::PrefillResult { + disaggregated_params: handoff, + prompt_tokens_details: None, + }); + let decode_outputs = collect(&decode, decode_request).await; + let generated = decode_outputs + .iter() + .flat_map(|output| output.token_ids.iter().copied()) + .collect::>(); + let terminal = decode_outputs + .iter() + .find(|output| output.finish_reason.is_some()) + .expect("image decode terminal"); + let usage = terminal.completion_usage.as_ref().expect("terminal usage"); + + assert!( + !generated.is_empty(), + "the image decode worker must generate" + ); + assert!( + usage.prompt_tokens > token_ids.len() as u32, + "the disaggregated image prompt must report its expanded token count" + ); +} + /// Drives the real `TrtllmSidecarEngine` against a live OpenEngine gRPC server /// (a real TensorRT-LLM engine on a GPU). Ignored by default; run explicitly with /// a reachable endpoint: @@ -202,3 +303,82 @@ async fn e2e_real_openengine_server() { "logprobs were requested but none were surfaced" ); } + +/// Drives an image request through the real Rust sidecar conversion layer and +/// a live TensorRT-LLM OpenEngine server. The fixed token IDs are Qwen3-VL's +/// chat-template rendering of one image marker and a short question; the tiny +/// PNG is split red/blue so a manual run also produces an obvious answer. +/// +/// ```text +/// TRTLLM_E2E_ENDPOINT=http://127.0.0.1:50051 \ +/// TRTLLM_E2E_MODEL=Qwen/Qwen3-VL-2B-Instruct \ +/// cargo test -p dynamo-trtllm-sidecar e2e_real_openengine_image -- --ignored --nocapture +/// ``` +#[tokio::test] +#[ignore = "requires a live image-capable TensorRT-LLM OpenEngine server"] +async fn e2e_real_openengine_image() { + let endpoint = std::env::var("TRTLLM_E2E_ENDPOINT") + .expect("set TRTLLM_E2E_ENDPOINT, e.g. http://127.0.0.1:50051"); + let model = std::env::var("TRTLLM_E2E_MODEL") + .unwrap_or_else(|_| "Qwen/Qwen3-VL-2B-Instruct".to_string()); + + let engine = TrtllmSidecarEngine::new( + GrpcEndpoint::parse(&endpoint, "--grpc-endpoint").expect("valid endpoint"), + transport(1), + ConfiguredModel { + source: model.clone(), + context_length: None, + }, + AGG, + ); + engine + .start(0) + .await + .expect("start against the live image-capable server"); + + let token_ids = vec![ + 151644, 872, 198, 151652, 151655, 151653, 675, 279, 7987, 504, 2115, 311, 1290, 13, 151645, + 198, 151644, 77091, 198, + ]; + let mut request = PreprocessedRequest::builder() + .model(model) + .token_ids(token_ids.clone()) + // Leave max_tokens omitted: only TensorRT-LLM can derive the output + // budget after expanding the image placeholder. + .stop_conditions(StopConditions::default()) + .sampling_options(SamplingOptions { + temperature: Some(0.0), + ..Default::default() + }) + .output_options(OutputOptions::default()) + .build() + .expect("request"); + add_images( + &mut request, + [concat!( + "data:image/png;base64,", + "iVBORw0KGgoAAAANSUhEUgAAABwAAAAcCAIAAAD9b0jDAAAAK0lEQVR4nGP8z0AsYGQgVi0T0WaSAEYNHTV01NBRQ0cNHTV01NBRQwefoQBvcAI3xbX7+QAAAABJRU5ErkJggg==" + )], + ); + + let outputs = collect(&engine, request).await; + let generated = outputs + .iter() + .flat_map(|output| output.token_ids.iter().copied()) + .collect::>(); + let terminal = outputs + .iter() + .find(|output| output.finish_reason.is_some()) + .expect("terminal output"); + let usage = terminal.completion_usage.as_ref().expect("terminal usage"); + eprintln!( + "[e2e-image] generated={generated:?}; usage: prompt={}, completion={}", + usage.prompt_tokens, usage.completion_tokens + ); + + assert!(!generated.is_empty(), "expected image-conditioned output"); + assert!( + usage.prompt_tokens > token_ids.len() as u32, + "the engine's expanded image prompt usage must replace Dynamo's unexpanded token count" + ); +} diff --git a/lib/sidecar/trtllm/src/tests/engine.rs b/lib/sidecar/trtllm/src/tests/engine.rs index 415456ede063..149303a06d3b 100644 --- a/lib/sidecar/trtllm/src/tests/engine.rs +++ b/lib/sidecar/trtllm/src/tests/engine.rs @@ -41,6 +41,64 @@ async fn aggregated_generation_streams_delta_then_terminal() { } } +#[tokio::test] +async fn advertised_image_media_reaches_openengine_and_uses_expanded_usage() { + let service = FakeTrtllm::default(); + service.supports_multimodal.store(true, Ordering::SeqCst); + service.advertises_image.store(true, Ordering::SeqCst); + let server = FakeServer::start(service).await; + let engine = engine(&server.endpoint, 1); + engine.start(0).await.expect("start"); + + let mut req = request(); + req.stop_conditions.max_tokens = None; + add_images(&mut req, ["data:image/png;base64,aW1hZ2U="]); + let outputs = collect(&engine, req).await; + let usage = outputs + .last() + .and_then(|output| output.completion_usage.as_ref()) + .expect("terminal engine usage"); + assert_eq!(usage.prompt_tokens, 67, "3 text tokens + 64 image tokens"); + + let requests = server.service.requests.lock().await; + let sent = requests.first().expect("recorded request"); + assert_eq!( + sent.stopping.as_ref().expect("stopping").max_tokens, + None, + "the expanded image prompt budget belongs to TensorRT-LLM" + ); + assert_eq!(sent.media.len(), 1); + assert_eq!(sent.media[0].modality, pb::Modality::Image as i32); + assert!(matches!( + sent.media[0].source.as_ref(), + Some(pb::media_item::Source::DataUri(value)) if value.starts_with("data:image/png") + )); +} + +#[tokio::test] +async fn both_multimodal_capability_signals_are_required() { + for (coarse, image) in [(true, false), (false, true)] { + let service = FakeTrtllm::default(); + service.supports_multimodal.store(coarse, Ordering::SeqCst); + service.advertises_image.store(image, Ordering::SeqCst); + let server = FakeServer::start(service).await; + let engine = engine(&server.endpoint, 1); + engine.start(0).await.expect("start"); + + let mut req = request(); + add_images(&mut req, ["https://example.com/image.png"]); + let context = dynamo_backend_common::testing::mock_context(); + let result = engine + .generate(req, GenerateContext::new(context, None)) + .await; + assert!( + result.is_err(), + "capabilities ({coarse}, {image}) must fail closed" + ); + assert!(server.service.requests.lock().await.is_empty()); + } +} + #[tokio::test] async fn grpc_request_errors_are_propagated() { let service = FakeTrtllm::default(); From c7896b4a8356867d868daf266374abb1771621b2 Mon Sep 17 00:00:00 2001 From: krishung5 Date: Thu, 15 Oct 2026 16:55:16 -0700 Subject: [PATCH 08/61] fix(trtllm): narrow image media cache identity Signed-off-by: krishung5 --- lib/llm/src/backend.rs | 206 +----------------- lib/sidecar/trtllm/src/client.rs | 14 +- lib/sidecar/trtllm/src/convert.rs | 86 ++------ lib/sidecar/trtllm/src/engine.rs | 8 +- lib/sidecar/trtllm/src/tests.rs | 10 +- .../trtllm/src/tests/convert_request.rs | 37 +--- lib/sidecar/trtllm/src/tests/disagg.rs | 14 +- 7 files changed, 51 insertions(+), 324 deletions(-) diff --git a/lib/llm/src/backend.rs b/lib/llm/src/backend.rs index 4c5a2d80e0d7..3a73e46a21b0 100644 --- a/lib/llm/src/backend.rs +++ b/lib/llm/src/backend.rs @@ -261,8 +261,8 @@ impl return state.pending_flush.pop().map(|output| (output, state)); } - let (output, trailing_usage) = loop { - let mut output = state.stream.next().await; + let output = loop { + let output = state.stream.next().await; let choice_already_finished = output.as_ref().is_some_and(|output| { !output.is_event() && output.data.as_ref().is_some_and(|data| { @@ -272,50 +272,7 @@ impl }) }); if !choice_already_finished { - break (output, false); - } - - let carries_usage = output - .as_ref() - .and_then(|output| output.data.as_ref()) - .is_some_and(|data| data.completion_usage.is_some()); - let backend_failed = output - .as_ref() - .and_then(|output| output.data.as_ref()) - .is_some_and(|data| matches!(data.finish_reason, Some(FinishReason::Error(_)))); - if carries_usage || backend_failed { - // Native backends can put authoritative cumulative usage - // on a terminal event immediately after the token Dynamo - // recognized locally as EOS. Preserve that metadata without - // emitting duplicate content or a second successful finish - // event. Preserve a backend error with or without usage: it - // is not duplicate success metadata and must reach the - // frontend. - if let Some(data) = output.as_mut().and_then(|output| output.data.as_mut()) { - data.token_ids.clear(); - data.tokens = None; - data.text = None; - data.cum_log_probs = None; - data.log_probs = None; - data.top_logprobs = None; - if !backend_failed { - data.finish_reason = None; - data.stop_reason = None; - } - } - break (output, true); - } - - let is_terminal = output - .as_ref() - .and_then(|output| output.data.as_ref()) - .is_some_and(|data| data.finish_reason.is_some()); - if !is_terminal && state.finished_choices.len() >= state.decoders.len() { - // Once every choice ended locally, tolerate only terminal - // metadata. A nonterminal suffix means the backend ignored - // the stop; cancel rather than burning the remaining budget. - state.stream.context().stop_generating(); - return None; + break output; } }; @@ -324,11 +281,6 @@ impl // move to state.process_output // handle any error conditions / unwraps here - if trailing_usage { - state.finished = state.finished_choices.len() >= state.decoders.len(); - return Some((output, state)); - } - // events are pass thru if output.is_event() || output.data.is_none() { // A genuine stream-level error ends generation for every choice at @@ -545,12 +497,13 @@ impl .unwrap_or((None, None)); // If we detected a local stop condition, mark this choice as finished. - // Keep the upstream stream alive just long enough to consume - // a native backend's following terminal-usage event. The - // loop above still cancels on any nonterminal suffix once - // every choice has stopped. + // Once all expected choices are finished, stop the upstream generator. if finish_reason.is_some() && data.finish_reason.is_none() { state.finished_choices.insert(choice_idx); + if state.finished_choices.len() >= decoders_count { + state.stream.context().stop_generating(); + state.finished = true; + } } let text = result.text; @@ -1968,149 +1921,6 @@ mod tests { assert_eq!(generator.get_usage().completion_tokens, 2); } - #[tokio::test] - async fn local_eos_preserves_a_following_authoritative_usage_trailer() { - let tokenizer: Arc = Arc::new(CandidateDecoder); - let backend = Backend::from_tokenizer(Tokenizer::from(tokenizer)); - let request = PreprocessedRequest::builder() - .model("test-model".to_string()) - .token_ids(vec![]) - .stop_conditions(StopConditions { - stop_token_ids_hidden: Some(vec![103]), - ..Default::default() - }) - .sampling_options(SamplingOptions::default()) - .output_options(OutputOptions::default()) - .build() - .expect("valid preprocessed request"); - let reported = dynamo_protocols::types::CompletionUsage { - prompt_tokens: 82, - completion_tokens: 2, - total_tokens: 84, - prompt_tokens_details: None, - completion_tokens_details: None, - }; - let engine: ServerStreamingEngine> = - Arc::new(SyntheticSglangStopEngine { - outputs: Some(vec![ - // Dynamo recognizes 103 as EOS before the native backend's - // separate terminal event arrives. - LLMEngineOutput { - token_ids: vec![101, 103], - index: Some(0), - ..Default::default() - }, - LLMEngineOutput { - index: Some(0), - finish_reason: Some(FinishReason::Stop), - completion_usage: Some(reported.clone()), - ..Default::default() - }, - ]), - }); - - let outputs = Operator::generate(backend.as_ref(), SingleIn::new(request), engine) - .await - .expect("backend generation succeeds") - .collect::>() - .await; - assert_eq!(outputs.len(), 2); - - let local_terminal = outputs[0].data.as_ref().expect("local terminal"); - assert_eq!(local_terminal.finish_reason, Some(FinishReason::Stop)); - let usage_trailer = outputs[1].data.as_ref().expect("usage trailer"); - assert!(usage_trailer.token_ids.is_empty()); - assert!(usage_trailer.finish_reason.is_none()); - assert_eq!(usage_trailer.completion_usage.as_ref(), Some(&reported)); - - let options = DeltaGeneratorOptions::new(None, None, false, None); - let mut generator = DeltaGenerator::new( - "test-model".to_string(), - options, - "test-request".to_string(), - ); - for output in outputs { - generator - .choice_from_postprocessor(output.data.expect("backend output")) - .expect("OpenAI response conversion succeeds"); - } - assert_eq!(generator.get_usage(), reported); - } - - #[tokio::test] - async fn local_eos_does_not_mask_a_trailing_backend_error() { - let tokenizer: Arc = Arc::new(CandidateDecoder); - let backend = Backend::from_tokenizer(Tokenizer::from(tokenizer)); - let request = PreprocessedRequest::builder() - .model("test-model".to_string()) - .token_ids(vec![]) - .stop_conditions(StopConditions { - stop_token_ids_hidden: Some(vec![103]), - ..Default::default() - }) - .sampling_options(SamplingOptions::default()) - .output_options(OutputOptions::default()) - .build() - .expect("valid preprocessed request"); - let reported = dynamo_protocols::types::CompletionUsage { - prompt_tokens: 82, - completion_tokens: 2, - total_tokens: 84, - prompt_tokens_details: None, - completion_tokens_details: None, - }; - for completion_usage in [None, Some(reported.clone())] { - let engine: ServerStreamingEngine> = - Arc::new(SyntheticSglangStopEngine { - outputs: Some(vec![ - LLMEngineOutput { - token_ids: vec![101, 103], - index: Some(0), - ..Default::default() - }, - LLMEngineOutput { - index: Some(0), - finish_reason: Some(FinishReason::Error("worker failed".to_string())), - completion_usage: completion_usage.clone(), - ..Default::default() - }, - ]), - }); - - let outputs = - Operator::generate(backend.as_ref(), SingleIn::new(request.clone()), engine) - .await - .expect("backend generation succeeds") - .collect::>() - .await; - assert_eq!(outputs.len(), 2); - let error_trailer = outputs[1].data.as_ref().expect("error trailer"); - assert!(error_trailer.token_ids.is_empty()); - assert_eq!( - error_trailer.finish_reason, - Some(FinishReason::Error("worker failed".to_string())) - ); - assert_eq!( - error_trailer.completion_usage.as_ref(), - completion_usage.as_ref() - ); - - let options = DeltaGeneratorOptions::new(None, None, false, None); - let mut generator = DeltaGenerator::new( - "test-model".to_string(), - options, - "test-request".to_string(), - ); - generator - .choice_from_postprocessor(outputs[0].data.clone().expect("local terminal")) - .expect("the local terminal converts"); - let error = generator - .choice_from_postprocessor(outputs[1].data.clone().expect("error trailer")) - .expect_err("the backend error must reach the OpenAI response layer"); - assert!(error.to_string().contains("worker failed")); - } - } - #[tokio::test] async fn test_no_stop_trim_raw_completion_output() { use crate::protocols::common::{ diff --git a/lib/sidecar/trtllm/src/client.rs b/lib/sidecar/trtllm/src/client.rs index 16cea9315dfd..576bb2a48098 100644 --- a/lib/sidecar/trtllm/src/client.rs +++ b/lib/sidecar/trtllm/src/client.rs @@ -27,10 +27,10 @@ use crate::proto::inference_client::InferenceClient; /// `--grpc-startup-deadline-secs` instead. const RPC_TIMEOUT: Duration = Duration::from_secs(30); -/// The engine's limits, resolved at startup from `--context-length` and -/// `Control.GetModelInfo`. +/// Metadata resolved at startup from `--context-length` and +/// `Control.GetModelInfo`, including limits and media capabilities. #[derive(Clone, Copy, Debug, Default)] -pub(crate) struct ModelLimits { +pub(crate) struct ModelMetadata { /// Maximum input + output tokens; backs both the registered context window /// and the default `max_tokens`. Absent when neither source supplied one, /// which leaves requests that omit `max_tokens` to be rejected. @@ -99,7 +99,7 @@ impl TrtllmClient { /// unset rather than substituting its `max_input_len` default (measured /// against TensorRT-LLM main at 8bbaf66bd5), so `None` here means "the /// server did not say", and `--context-length` supplies it instead. - async fn get_model_info(&self, model: &str) -> Result { + async fn get_model_info(&self, model: &str) -> Result { let info = self .control() .get_model_info(pb::GetModelInfoRequest { @@ -126,7 +126,7 @@ impl TrtllmClient { _ => false, }) }); - Ok(ModelLimits { + Ok(ModelMetadata { context_length: info.max_context_length.filter(|len| *len > 0), max_output_tokens: info.max_output_tokens.filter(|cap| *cap > 0), supports_image, @@ -136,7 +136,7 @@ impl TrtllmClient { /// One `GetModelInfo` call, for when `--context-length` already supplies the /// window and the engine is consulted only to cross-check it and to learn /// its output cap. - pub(crate) async fn model_limits(&self, model: &str) -> Result { + pub(crate) async fn model_limits(&self, model: &str) -> Result { tokio::time::timeout(RPC_TIMEOUT, self.get_model_info(model)) .await .map_err(|_| { @@ -164,7 +164,7 @@ impl TrtllmClient { model: &str, deadline: Instant, retry_interval: Duration, - ) -> Result { + ) -> Result { let mut last = "it never answered".to_string(); loop { match timeout_at(deadline, self.get_model_info(model)).await { diff --git a/lib/sidecar/trtllm/src/convert.rs b/lib/sidecar/trtllm/src/convert.rs index 0c3e3d59d2f3..0a498a97aff0 100644 --- a/lib/sidecar/trtllm/src/convert.rs +++ b/lib/sidecar/trtllm/src/convert.rs @@ -21,7 +21,7 @@ use dynamo_backend_common::{ MultimodalData, PreprocessedRequest, PromptTokensDetails, StopReason, TopLogprob, usage, }; -use crate::client::{self, ModelLimits}; +use crate::client::{self, ModelMetadata}; use crate::disagg; use crate::proto as pb; @@ -33,7 +33,7 @@ pub(crate) fn build_generate_request( request: &PreprocessedRequest, request_id: &str, model: &str, - limits: Option, + limits: Option, mode: DisaggregationMode, ) -> Result { validate_request(request, mode)?; @@ -162,7 +162,7 @@ pub(crate) fn build_generate_request( // not govern. fn max_tokens( request: &PreprocessedRequest, - limits: Option, + limits: Option, ) -> Result, DynamoError> { if let Some(max_tokens) = request.stop_conditions.max_tokens { return Ok(Some(max_tokens)); @@ -194,7 +194,6 @@ fn max_tokens( } const IMAGE_KEY: &str = "image_url"; -const MM_HASHES_KEY: &str = "mm_hashes"; /// Convert Dynamo's image-only execution payload into the ordered OpenEngine /// media list. The frontend groups content parts by modality, so ordering is @@ -203,7 +202,7 @@ const MM_HASHES_KEY: &str = "mm_hashes"; /// non-empty modality instead of inventing a cross-modality order. fn image_media( request: &PreprocessedRequest, - limits: Option, + limits: Option, ) -> Result, DynamoError> { reject_media_override("mm_processor_kwargs", request.mm_processor_kwargs.as_ref())?; reject_media_override("media_io_kwargs", request.media_io_kwargs.as_ref())?; @@ -256,21 +255,6 @@ fn image_media( )); } - // Explicit caller UUIDs remain authoritative. Otherwise carry the exact - // hashes generated by Dynamo's MM-aware preprocessor to the backend. This - // is request-side preparation for KV-aware routing; the router-owned - // `mm_routing_info` itself never crosses the worker boundary. - let has_user_uuid = user_image_uuids.is_some_and(|uuids| { - uuids - .iter() - .any(|uuid| uuid.as_deref().is_some_and(|uuid| !uuid.is_empty())) - }); - let routing_hashes = if has_user_uuid { - None - } else { - routing_hashes(request, images.len())? - }; - images .iter() .enumerate() @@ -292,63 +276,23 @@ fn image_media( Ok(pb::MediaItem { modality: pb::Modality::Image as i32, source: Some(source), + // Dynamo's frontend representation carries URL/data-URI + // sources, not raw bytes. MIME type is only meaningful for the + // OpenEngine raw_bytes source used by direct clients. mime_type: String::new(), - uuid: if has_user_uuid { - user_image_uuids - .and_then(|uuids| uuids[index].as_deref()) - .unwrap_or_default() - .to_string() - } else { - routing_hashes - .as_ref() - .map(|hashes| hashes[index].clone()) - .unwrap_or_default() - }, + // UUIDs are caller-owned cache identities and are optional per + // item. Dynamo's router hashes are deliberately not substituted + // here: their compatibility with TensorRT-LLM cache identity is + // owned and validated by the MM-aware routing work. + uuid: user_image_uuids + .and_then(|uuids| uuids[index].as_deref()) + .unwrap_or_default() + .to_string(), }) }) .collect() } -fn routing_hashes( - request: &PreprocessedRequest, - image_count: usize, -) -> Result>, DynamoError> { - let hashes = match request.extra_args.as_ref() { - Some(serde_json::Value::Object(extra)) => extra.get(MM_HASHES_KEY), - _ => None, - }; - let Some(hashes) = hashes else { - return Ok(None); - }; - let hashes = hashes.as_array().ok_or_else(|| { - client::invalid_argument("extra_args.mm_hashes must be an array of strings") - })?; - if hashes.is_empty() { - return Ok(None); - } - if hashes.len() != image_count { - return Err(client::invalid_argument(format!( - "extra_args.mm_hashes count ({}) must match the image media item count ({image_count})", - hashes.len() - ))); - } - hashes - .iter() - .enumerate() - .map(|(index, hash)| { - hash.as_str() - .filter(|hash| !hash.is_empty()) - .map(ToOwned::to_owned) - .ok_or_else(|| { - client::invalid_argument(format!( - "extra_args.mm_hashes[{index}] must be a non-empty string" - )) - }) - }) - .collect::, _>>() - .map(Some) -} - fn media_source(value: &str, index: usize) -> Result { let scheme = value .split_once(':') diff --git a/lib/sidecar/trtllm/src/engine.rs b/lib/sidecar/trtllm/src/engine.rs index 66327bfb2d3f..2671ba619bbc 100644 --- a/lib/sidecar/trtllm/src/engine.rs +++ b/lib/sidecar/trtllm/src/engine.rs @@ -18,7 +18,7 @@ use tokio::sync::OnceCell; use tokio_util::sync::CancellationToken; use crate::args::Args; -use crate::client::{self, ModelLimits, TrtllmClient}; +use crate::client::{self, ModelMetadata, TrtllmClient}; use crate::convert::{ResponseState, build_generate_request}; use crate::model::ConfiguredModel; @@ -41,7 +41,7 @@ pub struct TrtllmSidecarEngine { /// Engine limits resolved at `start` from `--context-length` and /// `Control.GetModelInfo`, so `generate` can derive a default `max_tokens` /// for requests that omit one. - limits: OnceCell, + limits: OnceCell, cancel: CancellationToken, } @@ -220,7 +220,7 @@ impl LLMEngine for TrtllmSidecarEngine { ships no Control service or was started without \ --grpc-protocol openengine" ); - ModelLimits::default() + ModelMetadata::default() } }; if let Some(engine_context_length) = reported.context_length @@ -233,7 +233,7 @@ impl LLMEngine for TrtllmSidecarEngine { reported; using the configured --context-length" ); } - ModelLimits { + ModelMetadata { context_length: Some(configured), ..reported } diff --git a/lib/sidecar/trtllm/src/tests.rs b/lib/sidecar/trtllm/src/tests.rs index d6d3c3146762..0bcda724b744 100644 --- a/lib/sidecar/trtllm/src/tests.rs +++ b/lib/sidecar/trtllm/src/tests.rs @@ -21,7 +21,7 @@ use tokio::sync::{Mutex, oneshot}; use tokio_stream::wrappers::TcpListenerStream; use tonic::{Request, Response, Status}; -use crate::client::{ModelLimits, TrtllmClient}; +use crate::client::{ModelMetadata, TrtllmClient}; use crate::convert::{ResponseState, build_generate_request, engine_error}; use crate::engine::TrtllmSidecarEngine; use crate::model::ConfiguredModel; @@ -465,16 +465,16 @@ fn impatient_transport() -> GrpcTransportConfig { /// Engine limits as `Control.GetModelInfo` would report them, with no /// separate output cap. -fn limits(context_length: u32) -> Option { - Some(ModelLimits { +fn limits(context_length: u32) -> Option { + Some(ModelMetadata { context_length: Some(context_length), max_output_tokens: None, supports_image: false, }) } -fn image_limits(context_length: u32) -> Option { - Some(ModelLimits { +fn image_limits(context_length: u32) -> Option { + Some(ModelMetadata { context_length: Some(context_length), max_output_tokens: None, supports_image: true, diff --git a/lib/sidecar/trtllm/src/tests/convert_request.rs b/lib/sidecar/trtllm/src/tests/convert_request.rs index 138057cb5dc8..57007959e472 100644 --- a/lib/sidecar/trtllm/src/tests/convert_request.rs +++ b/lib/sidecar/trtllm/src/tests/convert_request.rs @@ -89,10 +89,6 @@ fn image_urls_data_uris_and_uuids_are_mapped_in_order() { "image_url".to_string(), vec![Some("first-image".to_string()), None], )])); - req.extra_args = Some(json!({ - "mm_hashes": ["routing-hash-one", "routing-hash-two"] - })); - let proto = build_generate_request(&req, "req", "model", image_limits(4096), AGG) .expect("image request"); assert_eq!(proto.media.len(), 2); @@ -114,7 +110,7 @@ fn image_urls_data_uris_and_uuids_are_mapped_in_order() { } #[test] -fn mm_routing_hashes_are_forwarded_as_image_uuids_in_order() { +fn mm_routing_hashes_are_not_used_as_engine_cache_uuids() { let mut req = request(); add_images( &mut req, @@ -129,34 +125,7 @@ fn mm_routing_hashes_are_forwarded_as_image_uuids_in_order() { let proto = build_generate_request(&req, "req", "model", image_limits(4096), AGG) .expect("image request"); - assert_eq!(proto.media[0].uuid, "0123456789abcdef"); - assert_eq!(proto.media[1].uuid, "fedcba9876543210"); -} - -#[test] -fn malformed_or_misaligned_mm_routing_hashes_are_rejected() { - for (hashes, expected) in [ - (json!("not-an-array"), "array of strings"), - (json!(["only-one"]), "must match"), - (json!(["one", ""]), "non-empty string"), - (json!(["one", 2]), "non-empty string"), - ] { - let mut req = request(); - add_images( - &mut req, - [ - "https://example.com/first.png", - "https://example.com/second.png", - ], - ); - req.extra_args = Some(json!({"mm_hashes": hashes})); - let error = build_generate_request(&req, "req", "model", image_limits(4096), AGG) - .expect_err("malformed routing hashes must fail closed"); - assert!( - error.to_string().contains(expected), - "unexpected error: {error}" - ); - } + assert!(proto.media.iter().all(|item| item.uuid.is_empty())); } #[test] @@ -365,7 +334,7 @@ fn the_derived_max_tokens_respects_the_engines_output_cap() { &req, "req", "model", - Some(ModelLimits { + Some(ModelMetadata { context_length: Some(4096), max_output_tokens: Some(256), supports_image: false, diff --git a/lib/sidecar/trtllm/src/tests/disagg.rs b/lib/sidecar/trtllm/src/tests/disagg.rs index 612fda3f39fc..a81c8dfe92c8 100644 --- a/lib/sidecar/trtllm/src/tests/disagg.rs +++ b/lib/sidecar/trtllm/src/tests/disagg.rs @@ -72,9 +72,13 @@ fn image_media_is_preserved_across_prefill_and_decode_requests() { "data:image/png;base64,c2Vjb25k", ], ); - req.extra_args = Some(json!({ - "mm_hashes": ["0123456789abcdef", "fedcba9876543210"] - })); + req.multi_modal_uuids = Some(HashMap::from([( + "image_url".to_string(), + vec![ + Some("first-image".to_string()), + Some("second-image".to_string()), + ], + )])); let prefill = build_generate_request( &req, @@ -87,8 +91,8 @@ fn image_media_is_preserved_across_prefill_and_decode_requests() { assert!(is_context_only(&prefill)); assert_eq!(prefill.stopping.as_ref().unwrap().max_tokens, Some(1)); assert_eq!(prefill.media.len(), 2); - assert_eq!(prefill.media[0].uuid, "0123456789abcdef"); - assert_eq!(prefill.media[1].uuid, "fedcba9876543210"); + assert_eq!(prefill.media[0].uuid, "first-image"); + assert_eq!(prefill.media[1].uuid, "second-image"); req.prefill_result = Some(dynamo_backend_common::PrefillResult { disaggregated_params: crate::disagg::session_to_json(fake_session()) From bf4904f7924988b543ed8e365254688bdc3b664f Mon Sep 17 00:00:00 2001 From: krishung5 Date: Thu, 15 Oct 2026 16:59:18 -0700 Subject: [PATCH 09/61] style(trtllm): tighten media comments Signed-off-by: krishung5 --- lib/sidecar/trtllm/src/client.rs | 8 ++--- lib/sidecar/trtllm/src/convert.rs | 51 +++++++------------------------ 2 files changed, 13 insertions(+), 46 deletions(-) diff --git a/lib/sidecar/trtllm/src/client.rs b/lib/sidecar/trtllm/src/client.rs index 576bb2a48098..e3ac0932f4ab 100644 --- a/lib/sidecar/trtllm/src/client.rs +++ b/lib/sidecar/trtllm/src/client.rs @@ -27,8 +27,7 @@ use crate::proto::inference_client::InferenceClient; /// `--grpc-startup-deadline-secs` instead. const RPC_TIMEOUT: Duration = Duration::from_secs(30); -/// Metadata resolved at startup from `--context-length` and -/// `Control.GetModelInfo`, including limits and media capabilities. +/// Startup metadata from `--context-length` and `Control.GetModelInfo`. #[derive(Clone, Copy, Debug, Default)] pub(crate) struct ModelMetadata { /// Maximum input + output tokens; backs both the registered context window @@ -38,10 +37,7 @@ pub(crate) struct ModelMetadata { /// Cap on generated tokens, when the engine reports one. A context window /// alone can imply a larger budget than the engine will accept. pub(crate) max_output_tokens: Option, - /// Whether this loaded model advertises image input through both the - /// portable multimodal bit and TensorRT-LLM's modality list. Requiring both - /// fails closed when a server and client disagree about the finer-grained - /// capability. + /// True when both portable and engine-specific capabilities advertise images. pub(crate) supports_image: bool, } diff --git a/lib/sidecar/trtllm/src/convert.rs b/lib/sidecar/trtllm/src/convert.rs index 0a498a97aff0..1f5c9b76210e 100644 --- a/lib/sidecar/trtllm/src/convert.rs +++ b/lib/sidecar/trtllm/src/convert.rs @@ -4,10 +4,8 @@ //! Conversion between Dynamo's `PreprocessedRequest` / `LLMEngineOutput` and the //! TensorRT-LLM OpenEngine (`openengine.v1`) protobuf messages. //! -//! Scope: aggregated and disaggregated (prefill/decode) generation, including -//! image inputs when the loaded model advertises them. Video, audio, LoRA, -//! beam search, and `n > 1` are rejected before dispatch — the sidecar streams -//! a single sequence. +//! Supports aggregated and disaggregated image generation. Video, audio, LoRA, +//! beam search, and `n > 1` are rejected before dispatch. //! //! Disaggregation is driven by [`DisaggregationMode`]: a prefill worker marks //! its request `context_only` and returns the `PrefillReady` handoff as its @@ -139,27 +137,10 @@ pub(crate) fn build_generate_request( }) } -// Temporary workaround: the OpenEngine contract makes `stopping.max_tokens` -// optional. The Dynamo frontend forwards an omitted `max_tokens` as `None` -// expecting the backend to default. For text requests we mirror the in-process -// backend's default, -// `max(1, context_length - prompt_len)` (components/src/dynamo/trtllm -// `_default_max_tokens`). For image requests, only TensorRT-LLM knows the true -// prompt length after expanding image markers, so leave the field unset and let -// the OpenEngine request mapper derive the budget from the expanded prompt. -// `context_length` is resolved in -// `engine::start`, where `--context-length` wins over the `GetModelInfo` report. -// Only `/v1/chat/completions` and `/v1/responses` reach this fallback: they set -// `PRESERVE_OMITTED_MAX_TOKENS_CONTEXT_KEY`, which stops the frontend supplying -// its own default (`preprocessor::omitted_max_tokens_default`). `/v1/completions` -// is already defaulted by the frontend and never lands here. -// -// Remove when https://github.com/NVIDIA/TensorRT-LLM/issues/16549 lands (gRPC -// `max_tokens` made optional): drop this fallback and forward an omitted -// `max_tokens` as unset. Keep `--context-length` and the plumbing in -// `engine.rs` — they also feed `LlmRegistration.context_length`, which the -// frontend registers as the served context window and which this fallback does -// not govern. +// Derive omitted text max_tokens from the context window. For media, only +// TensorRT-LLM knows the prompt length after expansion, so leave it unset. +// Remove this fallback when NVIDIA/TensorRT-LLM#16549 lands; context_length +// remains necessary for LlmRegistration. fn max_tokens( request: &PreprocessedRequest, limits: Option, @@ -195,11 +176,8 @@ fn max_tokens( const IMAGE_KEY: &str = "image_url"; -/// Convert Dynamo's image-only execution payload into the ordered OpenEngine -/// media list. The frontend groups content parts by modality, so ordering is -/// retained within the image vector but not across different modalities. This -/// PR therefore accepts images only and fails explicitly on every other -/// non-empty modality instead of inventing a cross-modality order. +/// Map Dynamo's image payload to OpenEngine media, rejecting other modalities. +/// Per-modality vectors do not preserve cross-modality order. fn image_media( request: &PreprocessedRequest, limits: Option, @@ -207,6 +185,7 @@ fn image_media( reject_media_override("mm_processor_kwargs", request.mm_processor_kwargs.as_ref())?; reject_media_override("media_io_kwargs", request.media_io_kwargs.as_ref())?; + // mm_routing_info stays router-side; multi_modal_data is engine input. let media_map = request.multi_modal_data.as_ref(); if let Some(media_map) = media_map { for (modality, items) in media_map { @@ -276,14 +255,9 @@ fn image_media( Ok(pb::MediaItem { modality: pb::Modality::Image as i32, source: Some(source), - // Dynamo's frontend representation carries URL/data-URI - // sources, not raw bytes. MIME type is only meaningful for the - // OpenEngine raw_bytes source used by direct clients. + // MIME metadata belongs to raw_bytes, which this client does not send. mime_type: String::new(), - // UUIDs are caller-owned cache identities and are optional per - // item. Dynamo's router hashes are deliberately not substituted - // here: their compatibility with TensorRT-LLM cache identity is - // owned and validated by the MM-aware routing work. + // Do not substitute router-owned mm_hashes for caller cache identities. uuid: user_image_uuids .and_then(|uuids| uuids[index].as_deref()) .unwrap_or_default() @@ -486,9 +460,6 @@ fn validate_request( "multimodal encoder handoffs are not supported by the TensorRT-LLM sidecar", )); } - // `mm_routing_info` is deliberately not forwarded: it is framework-owned - // metadata that the Dynamo router consumes before selecting this worker. - // The execution payload remains `multi_modal_data`, mapped by `image_media`. // Only a decode worker consumes a prefill handoff. Seeing one anywhere else // means the frontend routed a decode request to the wrong worker role. if request.prefill_result.is_some() && !mode.is_decode() { From 95b3a0f55bfbd57623b26a968bb412a4b0578a1f Mon Sep 17 00:00:00 2001 From: krishung5 Date: Thu, 15 Oct 2026 17:49:52 -0700 Subject: [PATCH 10/61] test(trtllm): gate sidecar image requests in CI Signed-off-by: krishung5 --- .github/filters.yaml | 1 + .github/scripts/test-filters.js | 12 +++--- .github/workflows/pr.yaml | 27 ++++++++++++ container/templates/trtllm_runtime.Dockerfile | 20 +++++++++ tests/serve/test_sidecar.py | 41 ++++++++++++++++--- 5 files changed, 90 insertions(+), 11 deletions(-) diff --git a/.github/filters.yaml b/.github/filters.yaml index 69c3f5b01c10..b49e6f46532d 100644 --- a/.github/filters.yaml +++ b/.github/filters.yaml @@ -404,6 +404,7 @@ trtllm: - 'container/deps/requirements.trtllm.txt' - 'examples/backends/trtllm/**' - 'components/src/dynamo/trtllm/**' + - 'lib/sidecar/trtllm/**' - 'container/templates/trtllm_*' - '!**/*.md' - '!**/*.rst' diff --git a/.github/scripts/test-filters.js b/.github/scripts/test-filters.js index e23b4cb0014c..a00ec9bd975d 100755 --- a/.github/scripts/test-filters.js +++ b/.github/scripts/test-filters.js @@ -13,7 +13,7 @@ * This validates that tj-actions/changed-files will correctly: * - Match backend-specific files to their respective filters (vllm, sglang, trtllm) * - Route docs-artifact tests to recipe-check without core E2E checks - * - Route sidecar files to sidecar/Rust checks without backend or core E2E checks + * - Route sidecar files to sidecar/Rust checks and framework-native E2E where needed * - Exclude doc files (*.md, *.rst, *.txt) from core via negation patterns * - Match CI/infrastructure changes to core * - (with --coverage) Ensure all files in repo are covered by at least one filter @@ -175,7 +175,7 @@ const testCases = [ desc: 'AISimulate Dynamo adapter triggers planner validation' }, - // Sidecar Rust and proto files should trigger Rust checks without unrelated E2E + // Sidecar Rust and proto files trigger Rust checks and their native E2E only. { file: 'lib/sidecar/common/src/lib.rs', expect: { sidecar: true, rust: true, core: false, frontend: false, vllm: false, sglang: false, trtllm: false }, @@ -183,8 +183,8 @@ const testCases = [ }, { file: 'lib/sidecar/trtllm/proto/openengine/v1/generation.proto', - expect: { sidecar: true, rust: true, core: false, frontend: false, vllm: false, sglang: false, trtllm: false }, - desc: 'sidecar proto contracts trigger Rust checks without backend E2E' + expect: { sidecar: true, rust: true, core: false, frontend: false, vllm: false, sglang: false, trtllm: true }, + desc: 'trtllm sidecar contracts trigger Rust checks and trtllm E2E' }, { file: 'lib/sidecar/sglang/src/lib.rs', @@ -193,8 +193,8 @@ const testCases = [ }, { file: 'lib/sidecar/trtllm/src/lib.rs', - expect: { sidecar: true, rust: true, core: false, frontend: false, vllm: false, sglang: false, trtllm: false }, - desc: 'trtllm sidecar source does not route to sglang or trtllm E2E' + expect: { sidecar: true, rust: true, core: false, frontend: false, vllm: false, sglang: false, trtllm: true }, + desc: 'trtllm sidecar source triggers Rust checks and trtllm E2E' }, { file: 'lib/sidecar/vllm/deploy/agg.yaml', diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index e6e2a925bbd1..58eb3e77f6af 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -147,6 +147,7 @@ jobs: - trtllm-build - trtllm-dev-build - trtllm-test + - sidecar-trtllm-test - trtllm-multi-gpu-test - trtllm-copy-to-acr - triton-build @@ -541,6 +542,11 @@ jobs: diff_current_branch: ${{ github.ref_name }} diff_base_branch: ${{ needs.changed-files.outputs.base_ref }} diff_merge_base_sha: ${{ needs.changed-files.outputs.merge_base_sha }} + # Drop these overrides after the upstream TensorRT-LLM change ships. + extra_build_args: | + RUNTIME_IMAGE_TAG=1.4.0rc1.dev202610140007@sha256:7a728cd307bfa1651b4ad0787e71e69f7506fb1a698a0f4f04aef719ad43525f + TRTLLM_SOURCE_REPOSITORY=https://github.com/krishung5/TensorRT-LLM + TRTLLM_SOURCE_COMMIT=ab07b1649f638924908aed565722f4dd0f8547f8 # actions: read lets the build download a prior run's baseline compliance # artifact to diff against (a reusable workflow's token can't exceed the # caller's). Missing -> the diff soft-degrades, build stays green. @@ -928,6 +934,27 @@ jobs: gpu_parallel_max_vram_gib: '24' secrets: inherit + sidecar-trtllm-test: + name: sidecar-runtime + needs: [changed-files, trtllm-build, sidecar-build] + if: needs.changed-files.outputs.sidecar == 'true' && needs.changed-files.outputs.trtllm == 'true' + uses: $/.github/workflows/shared-test.yml + with: + test_suite_name: trtllm + framework: trtllm + test_type: TensorRT-LLM Sidecar 1-GPU E2E Test + amd_runner: prod-tester-amd-gpu-v2 + amd_runner_labels: '["gpu-1","amd64","v2","us-west-2"]' + target_tag_plain: ${{ needs.trtllm-build.outputs.target_tag_plain }} + cuda_version: '["13.1"]' + platform: '["amd64"]' + run_sanity_check: false + gpu_test_markers: pre_merge and sidecar and trtllm and gpu_1 + gpu_test_timeout_minutes: 45 + sidecar_backend: trtllm + sidecar_binary_artifact: ${{ needs.sidecar-build.outputs.binary_artifact }} + secrets: inherit + trtllm-multi-gpu-test: name: trtllm-runtime # This name overlaps with other trtllm jobs to group them in the UI needs: [changed-files, trtllm-build] diff --git a/container/templates/trtllm_runtime.Dockerfile b/container/templates/trtllm_runtime.Dockerfile index 3b147a085b55..860698519fbb 100644 --- a/container/templates/trtllm_runtime.Dockerfile +++ b/container/templates/trtllm_runtime.Dockerfile @@ -41,6 +41,26 @@ ARG ENABLE_KVBM ARG ENABLE_GPU_MEMORY_SERVICE ARG TARGETARCH ARG NIXL_REF +ARG TRTLLM_SOURCE_REPOSITORY +ARG TRTLLM_SOURCE_COMMIT + +# Optional exact-source overlay for cross-repository CI. +RUN if [ -n "${TRTLLM_SOURCE_COMMIT}" ]; then \ + test -n "${TRTLLM_SOURCE_REPOSITORY}" && \ + source_dir=$(mktemp -d) && \ + wget --quiet --tries=3 \ + "${TRTLLM_SOURCE_REPOSITORY}/archive/${TRTLLM_SOURCE_COMMIT}.tar.gz" \ + -O "${source_dir}/source.tar.gz" && \ + tar -xzf "${source_dir}/source.tar.gz" -C "${source_dir}" \ + --strip-components=1 && \ + package_dir=$(/usr/bin/python3 -c \ + 'import importlib.util; print(next(iter(importlib.util.find_spec("tensorrt_llm").submodule_search_locations)))') && \ + cp -a "${source_dir}/tensorrt_llm/grpc/openengine/." \ + "${package_dir}/grpc/openengine/" && \ + install -m 0644 "${source_dir}/tensorrt_llm/inputs/media_io.py" \ + "${package_dir}/inputs/media_io.py" && \ + rm -rf "${source_dir}"; \ + fi # Remove the upstream runtime's Git LFS package without pruning shared dependencies. RUN apt-get purge -y git-lfs && rm -rf /var/lib/apt/lists/* diff --git a/tests/serve/test_sidecar.py b/tests/serve/test_sidecar.py index 038446530fc0..55720fee9b12 100644 --- a/tests/serve/test_sidecar.py +++ b/tests/serve/test_sidecar.py @@ -31,6 +31,7 @@ disaggregated_chat_payload, disaggregated_token_count_payload, http_cancellation_payloads, + multimodal_payload_default, ) from tests.utils.payloads import ( ChatPayload, @@ -115,10 +116,7 @@ def _disaggregated_payloads( ] -# Sequential stage only: no profiled_vram_gib mark yet, since actual peak VRAM -# has not been profiled for the sidecar launch path. Add one once measured, to -# admit these into the parallel stage alongside the equivalent dynamo.{backend} -# scenarios in tests/serve/test_{vllm,sglang,trtllm}.py. +# Text cases stay sequential until their sidecar launch paths are profiled. sidecar_configs = { "vllm_aggregated": EngineConfig( name="vllm_aggregated", @@ -126,6 +124,7 @@ def _disaggregated_payloads( script_name="agg.sh", marks=[ pytest.mark.vllm, + pytest.mark.core, pytest.mark.gpu_1, # Let the 600s health check report failure before pytest times out. pytest.mark.timeout(780), @@ -149,6 +148,7 @@ def _disaggregated_payloads( script_args=["--enable-metrics", "--decode-log-interval", "1"], marks=[ pytest.mark.sglang, + pytest.mark.core, pytest.mark.gpu_1, pytest.mark.timeout(780), pytest.mark.post_merge, @@ -179,6 +179,7 @@ def _disaggregated_payloads( script_name="agg.sh", marks=[ pytest.mark.trtllm, + pytest.mark.core, pytest.mark.gpu_1, pytest.mark.timeout(780), pytest.mark.post_merge, @@ -198,12 +199,41 @@ def _disaggregated_payloads( chat_payload_default(), ], ), + "trtllm_aggregated_image": EngineConfig( + name="trtllm_aggregated_image", + directory=trtllm_sidecar_dir, + script_name="agg.sh", + marks=[ + pytest.mark.trtllm, + pytest.mark.multimodal, + pytest.mark.gpu_1, + pytest.mark.profiled_vram_gib(12.0), + pytest.mark.requested_trtllm_kv_tokens(32768), + pytest.mark.timeout(960), + pytest.mark.pre_merge, + pytest.mark.skipif( + not _trtllm_serves_openengine(), + reason=TRTLLM_OPENENGINE_SKIP_REASON, + ), + ], + model="Qwen/Qwen3-VL-2B-Instruct", + timeout=900, + delayed_start=60, + env={"PYTHONUNBUFFERED": "1"}, + request_payloads=[ + multimodal_payload_default( + text="Describe what you see in this image.", + expected_response=["mountain", "rock", "trees", "road"], + ) + ], + ), "trtllm_disaggregated": EngineConfig( name="trtllm_disaggregated", directory=trtllm_sidecar_dir, script_name="disagg.sh", marks=[ pytest.mark.trtllm, + pytest.mark.core, # Both engines share one GPU: the handoff is what this covers, and # a co-resident pair keeps it on the existing 1-GPU sidecar runner. # `requested_trtllm_kv_tokens` caps each engine's KV pool so the @@ -250,6 +280,7 @@ def _disaggregated_payloads( script_name="disagg.sh", marks=[ pytest.mark.vllm, + pytest.mark.core, pytest.mark.gpu_1, pytest.mark.post_merge, pytest.mark.timeout(1200), @@ -282,6 +313,7 @@ def _disaggregated_payloads( ], marks=[ pytest.mark.sglang, + pytest.mark.core, pytest.mark.gpu_1, pytest.mark.post_merge, pytest.mark.timeout(1200), @@ -320,7 +352,6 @@ def sidecar_config_test(request, monkeypatch, discovery_backend): return config -@pytest.mark.core @pytest.mark.sidecar @pytest.mark.e2e @pytest.mark.engine_ports From 2e545566d98b8d82da8a9866c53dc78e459ede7a Mon Sep 17 00:00:00 2001 From: krishung5 Date: Fri, 16 Oct 2026 11:24:34 -0700 Subject: [PATCH 11/61] fix(trtllm): address multimodal review findings Signed-off-by: krishung5 --- .github/filters.yaml | 1 + .github/scripts/test-filters.js | 5 + .github/workflows/pr.yaml | 2 +- .github/workflows/shared-test.yml | 1 + container/compliance/policy/codec_policy.yaml | 26 +++-- container/templates/trtllm_runtime.Dockerfile | 48 +++++----- lib/sidecar/trtllm/src/client.rs | 32 ++++--- lib/sidecar/trtllm/src/convert.rs | 75 ++++++++------- lib/sidecar/trtllm/src/engine.rs | 94 +++++++++++++++---- lib/sidecar/trtllm/src/tests.rs | 6 +- .../trtllm/src/tests/convert_request.rs | 46 ++++----- lib/sidecar/trtllm/src/tests/disagg.rs | 10 +- lib/sidecar/trtllm/src/tests/engine.rs | 58 ++++++++++++ tests/serve/test_sidecar.py | 8 +- 14 files changed, 272 insertions(+), 140 deletions(-) diff --git a/.github/filters.yaml b/.github/filters.yaml index b49e6f46532d..d5e9ae3d5075 100644 --- a/.github/filters.yaml +++ b/.github/filters.yaml @@ -405,6 +405,7 @@ trtllm: - 'examples/backends/trtllm/**' - 'components/src/dynamo/trtllm/**' - 'lib/sidecar/trtllm/**' + - 'tests/serve/test_sidecar.py' - 'container/templates/trtllm_*' - '!**/*.md' - '!**/*.rst' diff --git a/.github/scripts/test-filters.js b/.github/scripts/test-filters.js index a00ec9bd975d..d5d45a4ce863 100755 --- a/.github/scripts/test-filters.js +++ b/.github/scripts/test-filters.js @@ -196,6 +196,11 @@ const testCases = [ expect: { sidecar: true, rust: true, core: false, frontend: false, vllm: false, sglang: false, trtllm: true }, desc: 'trtllm sidecar source triggers Rust checks and trtllm E2E' }, + { + file: 'tests/serve/test_sidecar.py', + expect: { sidecar: true, trtllm: true }, + desc: 'shared sidecar test changes trigger the TensorRT-LLM sidecar E2E' + }, { file: 'lib/sidecar/vllm/deploy/agg.yaml', expect: { sidecar: true, rust: false, core: false, frontend: false, vllm: false, sglang: false, trtllm: false }, diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 58eb3e77f6af..4eb2b9312608 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -546,7 +546,7 @@ jobs: extra_build_args: | RUNTIME_IMAGE_TAG=1.4.0rc1.dev202610140007@sha256:7a728cd307bfa1651b4ad0787e71e69f7506fb1a698a0f4f04aef719ad43525f TRTLLM_SOURCE_REPOSITORY=https://github.com/krishung5/TensorRT-LLM - TRTLLM_SOURCE_COMMIT=ab07b1649f638924908aed565722f4dd0f8547f8 + TRTLLM_SOURCE_COMMIT=5d2e3b2c758b4ccc0fa0a3eb0e01cdef934e8089 # actions: read lets the build download a prior run's baseline compliance # artifact to diff against (a reusable workflow's token can't exceed the # caller's). Missing -> the diff soft-degrades, build stays green. diff --git a/.github/workflows/shared-test.yml b/.github/workflows/shared-test.yml index 951c36ee5d6e..8669d55143b1 100644 --- a/.github/workflows/shared-test.yml +++ b/.github/workflows/shared-test.yml @@ -259,6 +259,7 @@ jobs: install -m755 "sidecar-artifact/dynamo-${SIDECAR_BACKEND}-sidecar" \ "${bin_dir}/dynamo-${SIDECAR_BACKEND}-sidecar" echo "DYNAMO_SIDECAR_BIN=${bin_dir}/dynamo-${SIDECAR_BACKEND}-sidecar" >> "${GITHUB_ENV}" + echo "DYNAMO_TEST_SIDECAR_BACKEND=${SIDECAR_BACKEND}" >> "${GITHUB_ENV}" if [ -z "${CONTAINER_WORKSPACE}" ]; then echo "::error::container_workspace is empty; refusing to replace test files" exit 1 diff --git a/container/compliance/policy/codec_policy.yaml b/container/compliance/policy/codec_policy.yaml index 9e4023b0e2e3..61b8f39e19a0 100644 --- a/container/compliance/policy/codec_policy.yaml +++ b/container/compliance/policy/codec_policy.yaml @@ -108,9 +108,9 @@ exceptions: (NVIDIA/DALI_deps#162, NVIDIA/DALI#6352, May 2026), first released in 2.1.1 -- as a cherry-pick, so checking whether the merge commit is an ancestor of the tag gives the wrong answer; the release notes are the - reliable source. Upgrading the wheel in place to 2.1.1 or 2.2.0 leaves 440 - decoders with all five absent, vp8/vp9/mjpeg/av1 still present, and the - image importing and passing its unit suite. + reliable source. Upgrading the wheel in place to 2.1.1, 2.2.0, or 2.3.0 + leaves 440 decoders with all five absent, vp8/vp9/mjpeg/av1 still present, + and the image importing and passing its unit suite. This waiver cannot be narrowed to the non-codec companions, which an earlier revision of this note claimed. libavcodec, libavfilter, libavformat, @@ -120,13 +120,12 @@ exceptions: set is all-or-nothing with the package. The entry stays even so. trtllm_runtime.Dockerfile pins - nvidia-dali-cuda130==2.2.0, which is the version the 1.3.0rc27 base image - already carries, so the pin now holds a version rather than moving one. The - vendored libraries do not register those decoders -- but they are still on - disk and still match the deny globs above, which gate on presence rather - than content. What changed is the reason for the waiver: it now covers a - vendored ffmpeg that carries no software h264/hevc/aac, rather than one - that does. + nvidia-dali-cuda130 at an explicitly measured version: 2.2.0 in the + release base or 2.3.0 in the pinned development base. The vendored + libraries do not register those decoders -- but they are still on disk and + still match the deny globs above, which gate on presence rather than + content. What changed is the reason for the waiver: it now covers a vendored + ffmpeg that carries no software h264/hevc/aac, rather than one that does. Enforcement lives at build time, not here. Two guards in trtllm_runtime.Dockerfile enumerate every bundled libavcodec -- one in @@ -142,10 +141,9 @@ exceptions: subtracted from the generated delta, so such an entry would never match anything and would read as a check while doing nothing. - Re-measure when the vendored version moves. Last measured against - nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc27, whose base carries 2.2.0: - 440 decoders, h264, hevc, aac, aac_fixed and aac_latm absent, vp8, vp9, - mjpeg and av1 present. + Re-measure when the vendored version moves. Last measured against 2.2.0 and + 2.3.0: 440 decoders, h264, hevc, aac, aac_fixed and aac_latm absent, vp8, + vp9, mjpeg and av1 present. owner: DALI # PyNvVideoCodec (added for NVDEC hardware video decode) vendors libavutil and # libavformat under its package dir to demux input containers before handing the diff --git a/container/templates/trtllm_runtime.Dockerfile b/container/templates/trtllm_runtime.Dockerfile index 860698519fbb..b71d05bdb1e9 100644 --- a/container/templates/trtllm_runtime.Dockerfile +++ b/container/templates/trtllm_runtime.Dockerfile @@ -403,10 +403,9 @@ RUN /usr/bin/python3 -m pip uninstall -y --break-system-packages opencv-python-h # Hold DALI at a version that carries its own media-codec cleanup. Upstream # restricted DALI's vendored ffmpeg build to drop the software h264/hevc/aac # decoders (NVIDIA/DALI_deps#162, NVIDIA/DALI#6352), first released in 2.1.1. -# The base image carried 2.1.0 through 1.3.0rc26, which predates it, and carries -# 2.2.0 from 1.3.0rc27, which does not. So on this base the install below is a -# no-op and what the block still buys is the two checks around it: the version -# tripwire and the decoder enumeration. +# The supported base images carry either 2.2.0 or 2.3.0. Both were measured +# without the restricted codecs; older versions are upgraded to 2.2.0 and any +# other newer version stops the build for re-measurement. # # Pin rather than remove. The vendored libav*/libsw* set cannot be trimmed # on its own -- every one of them is DT_NEEDED by libdali.so and its siblings, so @@ -416,23 +415,17 @@ RUN /usr/bin/python3 -m pip uninstall -y --break-system-packages opencv-python-h # out of someone else's image. # # Measured on the shipped image: 2.1.0 registers 446 decoders including h264, -# hevc, aac, aac_fixed and aac_latm; 2.1.1 and 2.2.0 each register 440 with all -# five absent and vp8/vp9/mjpeg/av1 retained. 2.2.0 measured again on -# tensorrt-llm/release:1.3.0rc27 itself, now that DALI arrives with the base. +# hevc, aac, aac_fixed and aac_latm; 2.1.1, 2.2.0, and 2.3.0 each register 440 +# with all five absent and vp8/vp9/mjpeg/av1 retained. # # Pinned exactly, not a floor. `>=2.1.1` resolves to whatever is newest, which is # a jump into a component this repo does not own, taken silently at build time. # The pin names the version that was measured, so moving it means measuring the # new one. # -# The pin can go stale in the one direction that matters: if TensorRT-LLM later -# ships a base image with a newer DALI, installing 2.2.0 over it is a downgrade. -# The check below turns that into a build failure with instructions rather than a -# silent regression. The pin now equals the base version, so the next base image -# that moves DALI past 2.2.0 stops the build, which is the point: a human -# re-measures and moves the pin, or deletes this RUN and lets the base version -# stand. A base that moves DALI back below 2.2.0 passes that check and the -# install below puts the measured version back, which is what it is there for. +# A base below 2.2.0 is upgraded to the measured 2.2.0 wheel. The two measured +# versions otherwise stay in place; any unmeasured newer version fails rather +# than being downgraded or accepted silently. # # THIS BLOCK IS TRANSITIONAL, and on this base it is already down to its checks. # The guards after it and the bundled-libavcodec assertion in @@ -457,20 +450,23 @@ RUN --mount=type=bind,source=./container/compliance/enumerate_bundled_decoders.p set -eu; \ before=$(/usr/bin/python3 -c 'import importlib.metadata as m; print(m.version("nvidia-dali-cuda130"))'); \ echo "DALI in base image: $before"; \ - newest=$(printf '%s\n2.2.0\n' "$before" | sort -V | tail -1); \ - if [ "$newest" != "2.2.0" ]; then \ - echo "ERROR: base image already carries DALI $before, so pinning 2.2.0 would" >&2; \ - echo " downgrade it. Upstream has caught up -- delete this RUN and let" >&2; \ - echo " the base version stand. Keep the guards below and the bundled-" >&2; \ - echo " libavcodec assertion in tests/dependencies/, which are what stop" >&2; \ - echo " this from regressing." >&2; \ - exit 1; \ + case "$before" in \ + 2.3.0) target=2.3.0 ;; \ + *) \ + newest=$(printf '%s\n2.2.0\n' "$before" | sort -V | tail -1); \ + if [ "$newest" != "2.2.0" ]; then \ + echo "ERROR: base image carries unmeasured DALI $before" >&2; \ + exit 1; \ + fi; \ + target=2.2.0 ;; \ + esac; \ + if [ "$before" != "$target" ]; then \ + /usr/bin/python3 -m pip install --break-system-packages --no-cache-dir \ + --extra-index-url https://pypi.nvidia.com "nvidia-dali-cuda130==$target"; \ fi; \ - /usr/bin/python3 -m pip install --break-system-packages --no-cache-dir \ - --extra-index-url https://pypi.nvidia.com 'nvidia-dali-cuda130==2.2.0'; \ v=$(/usr/bin/python3 -c 'import importlib.metadata as m; print(m.version("nvidia-dali-cuda130"))'); \ echo "DALI version: $v"; \ - [ "$v" = "2.2.0" ] || { echo "ERROR: wanted DALI 2.2.0, got $v" >&2; exit 1; }; \ + [ "$v" = "$target" ] || { echo "ERROR: wanted DALI $target, got $v" >&2; exit 1; }; \ libs=$(find /usr/local/lib/python3.12/dist-packages/nvidia/dali/.libs -name 'libavcodec*.so*'); \ if [ -z "$libs" ]; then \ echo "ERROR: DALI $v ships no libavcodec under .libs, so the enumeration" >&2; \ diff --git a/lib/sidecar/trtllm/src/client.rs b/lib/sidecar/trtllm/src/client.rs index e3ac0932f4ab..f3daac1910c7 100644 --- a/lib/sidecar/trtllm/src/client.rs +++ b/lib/sidecar/trtllm/src/client.rs @@ -37,8 +37,14 @@ pub(crate) struct ModelMetadata { /// Cap on generated tokens, when the engine reports one. A context window /// alone can imply a larger budget than the engine will accept. pub(crate) max_output_tokens: Option, - /// True when both portable and engine-specific capabilities advertise images. - pub(crate) supports_image: bool, + /// Whether both portable and engine-specific capabilities advertise images. + /// `None` means startup discovery failed transiently and may be retried. + pub(crate) supports_image: Option, +} + +pub(crate) enum ModelInfoResponse { + Available(ModelMetadata), + Unsupported(tonic::Status), } pub(crate) struct TrtllmClient { @@ -125,22 +131,24 @@ impl TrtllmClient { Ok(ModelMetadata { context_length: info.max_context_length.filter(|len| *len > 0), max_output_tokens: info.max_output_tokens.filter(|cap| *cap > 0), - supports_image, + supports_image: Some(supports_image), }) } /// One `GetModelInfo` call, for when `--context-length` already supplies the /// window and the engine is consulted only to cross-check it and to learn /// its output cap. - pub(crate) async fn model_limits(&self, model: &str) -> Result { - tokio::time::timeout(RPC_TIMEOUT, self.get_model_info(model)) - .await - .map_err(|_| { - connection_timeout(format!( - "GetModelInfo did not respond within {RPC_TIMEOUT:?}" - )) - })? - .map_err(|status| status_to_dynamo("GetModelInfo", status)) + pub(crate) async fn model_limits(&self, model: &str) -> Result { + match tokio::time::timeout(RPC_TIMEOUT, self.get_model_info(model)).await { + Ok(Ok(metadata)) => Ok(ModelInfoResponse::Available(metadata)), + Ok(Err(status)) if is_request_itself_wrong(&status) => { + Ok(ModelInfoResponse::Unsupported(status)) + } + Ok(Err(status)) => Err(status_to_dynamo("GetModelInfo", status)), + Err(_) => Err(connection_timeout(format!( + "GetModelInfo did not respond within {RPC_TIMEOUT:?}" + ))), + } } /// Polls `GetModelInfo` until the engine reports a usable context length or diff --git a/lib/sidecar/trtllm/src/convert.rs b/lib/sidecar/trtllm/src/convert.rs index 1f5c9b76210e..81661405df03 100644 --- a/lib/sidecar/trtllm/src/convert.rs +++ b/lib/sidecar/trtllm/src/convert.rs @@ -13,6 +13,7 @@ //! See [`crate::disagg`] for the codec. use std::collections::BTreeSet; +use std::sync::Arc; use dynamo_backend_common::{ CompletionUsage, DisaggregationMode, DynamoError, FinishReason, LLMEngineOutput, @@ -28,14 +29,15 @@ use crate::proto as pb; type MappedTokens = (Vec, Option>, Option>>); pub(crate) fn build_generate_request( - request: &PreprocessedRequest, + mut request: PreprocessedRequest, request_id: &str, model: &str, limits: Option, mode: DisaggregationMode, ) -> Result { - validate_request(request, mode)?; - let media = image_media(request, limits)?; + validate_request(&request, mode)?; + let request_has_media = has_media(&request); + let media = image_media(&mut request, limits)?; let sampling = &request.sampling_options; let stop = &request.stop_conditions; @@ -52,7 +54,7 @@ pub(crate) fn build_generate_request( let max_tokens = if mode.is_prefill() { Some(1) } else { - max_tokens(request, limits)? + max_tokens(&request, limits, request_has_media)? }; // The decode worker replays the prefill worker's session; the prefill worker // marks its request `context_only` through `extra`. @@ -71,6 +73,15 @@ pub(crate) fn build_generate_request( }, DisaggregationMode::Aggregated | DisaggregationMode::Encode => None, }; + let conditions = stop_conditions(&request); + let guided = guided_decoding(&request)?; + let session_id = request + .agent_context + .as_ref() + .map(|context| context.session_id.trim()) + .filter(|session_id| !session_id.is_empty()) + .map(str::to_string); + let token_ids = Arc::unwrap_or_clone(request.token_ids); Ok(pb::GenerateRequest { request_id: request_id.to_string(), @@ -78,7 +89,7 @@ pub(crate) fn build_generate_request( // served by the loaded model (single-model server). model: model.to_string(), input: Some(pb::generate_request::Input::TokenIds(pb::TokenIds { - ids: request.token_ids.as_ref().clone(), + ids: token_ids, })), sampling: Some(pb::SamplingParams { temperature: sampling.temperature.map(f64::from), @@ -100,7 +111,7 @@ pub(crate) fn build_generate_request( // `Stop` instead of `PrefillReady`, and leave the frontend treating // a request that never met its minimum as complete. min_tokens: stop.min_tokens, - conditions: stop_conditions(request), + conditions, ignore_eos: stop.ignore_eos, // `include_stop_in_output` retains matched stop *strings*; the // request-level flag is rejected in `validate_request`, so leave it @@ -122,17 +133,13 @@ pub(crate) fn build_generate_request( prompt_candidates: None, prompt_logprob_start: None, }), - guided: guided_decoding(request)?, + guided, media, lora_name: String::new(), kv, extra: Some(disagg::request_extra( mode.is_prefill(), - request - .agent_context - .as_ref() - .map(|context| context.session_id.trim()) - .filter(|session_id| !session_id.is_empty()), + session_id.as_deref(), )), }) } @@ -144,11 +151,12 @@ pub(crate) fn build_generate_request( fn max_tokens( request: &PreprocessedRequest, limits: Option, + has_media: bool, ) -> Result, DynamoError> { if let Some(max_tokens) = request.stop_conditions.max_tokens { return Ok(Some(max_tokens)); } - if has_media(request) { + if has_media { return Ok(None); } let context_length = limits @@ -176,18 +184,17 @@ fn max_tokens( const IMAGE_KEY: &str = "image_url"; -/// Map Dynamo's image payload to OpenEngine media, rejecting other modalities. /// Per-modality vectors do not preserve cross-modality order. fn image_media( - request: &PreprocessedRequest, + request: &mut PreprocessedRequest, limits: Option, ) -> Result, DynamoError> { reject_media_override("mm_processor_kwargs", request.mm_processor_kwargs.as_ref())?; reject_media_override("media_io_kwargs", request.media_io_kwargs.as_ref())?; // mm_routing_info stays router-side; multi_modal_data is engine input. - let media_map = request.multi_modal_data.as_ref(); - if let Some(media_map) = media_map { + let mut media_map = request.multi_modal_data.take(); + if let Some(media_map) = media_map.as_ref() { for (modality, items) in media_map { if modality != IMAGE_KEY && !items.is_empty() { return Err(client::invalid_argument(format!( @@ -198,24 +205,25 @@ fn image_media( } let images = media_map - .and_then(|media| media.get(IMAGE_KEY)) - .map(Vec::as_slice) + .as_mut() + .and_then(|media| media.remove(IMAGE_KEY)) .unwrap_or_default(); - let user_image_uuids = if let Some(uuid_map) = request.multi_modal_uuids.as_ref() { - for (modality, uuids) in uuid_map { + let mut uuid_map = request.multi_modal_uuids.take(); + let user_image_uuids = if let Some(uuid_map) = uuid_map.as_mut() { + for (modality, uuids) in uuid_map.iter() { if modality != IMAGE_KEY && !uuids.is_empty() { return Err(client::invalid_argument(format!( "multimodal UUIDs for `{modality}` cannot be forwarded by the image-only TensorRT-LLM sidecar" ))); } } - uuid_map.get(IMAGE_KEY).map(Vec::as_slice) + uuid_map.remove(IMAGE_KEY) } else { None }; - if let Some(uuids) = user_image_uuids + if let Some(uuids) = user_image_uuids.as_ref() && uuids.len() != images.len() { return Err(client::invalid_argument(format!( @@ -228,18 +236,20 @@ fn image_media( if images.is_empty() { return Ok(Vec::new()); } - if !limits.is_some_and(|limits| limits.supports_image) { + if limits.is_none_or(|limits| limits.supports_image != Some(true)) { return Err(client::invalid_argument( "the loaded TensorRT-LLM model does not advertise image support through Control.GetModelInfo", )); } + let uuids = user_image_uuids.unwrap_or_else(|| vec![None; images.len()]); images - .iter() + .into_iter() + .zip(uuids) .enumerate() - .map(|(index, image)| { + .map(|(index, (image, uuid))| { let source = match image { - MultimodalData::Url(url) => media_source(url.as_str(), index)?, + MultimodalData::Url(url) => media_source(url.to_string(), index)?, MultimodalData::RawUrl(url) => media_source(url, index)?, MultimodalData::Decoded(_) => { return Err(client::invalid_argument(format!( @@ -258,22 +268,19 @@ fn image_media( // MIME metadata belongs to raw_bytes, which this client does not send. mime_type: String::new(), // Do not substitute router-owned mm_hashes for caller cache identities. - uuid: user_image_uuids - .and_then(|uuids| uuids[index].as_deref()) - .unwrap_or_default() - .to_string(), + uuid: uuid.unwrap_or_default(), }) }) .collect() } -fn media_source(value: &str, index: usize) -> Result { +fn media_source(value: String, index: usize) -> Result { let scheme = value .split_once(':') .map(|(scheme, _)| scheme.to_ascii_lowercase()); match scheme.as_deref() { - Some("http") | Some("https") => Ok(pb::media_item::Source::Url(value.to_string())), - Some("data") => Ok(pb::media_item::Source::DataUri(value.to_string())), + Some("http") | Some("https") => Ok(pb::media_item::Source::Url(value)), + Some("data") => Ok(pb::media_item::Source::DataUri(value)), _ => Err(client::invalid_argument(format!( "`{IMAGE_KEY}` item {index} must use an http, https, or data URI" ))), diff --git a/lib/sidecar/trtllm/src/engine.rs b/lib/sidecar/trtllm/src/engine.rs index 2671ba619bbc..afd6fddf1f97 100644 --- a/lib/sidecar/trtllm/src/engine.rs +++ b/lib/sidecar/trtllm/src/engine.rs @@ -18,7 +18,7 @@ use tokio::sync::OnceCell; use tokio_util::sync::CancellationToken; use crate::args::Args; -use crate::client::{self, ModelMetadata, TrtllmClient}; +use crate::client::{self, ModelInfoResponse, ModelMetadata, TrtllmClient}; use crate::convert::{ResponseState, build_generate_request}; use crate::model::ConfiguredModel; @@ -42,6 +42,7 @@ pub struct TrtllmSidecarEngine { /// `Control.GetModelInfo`, so `generate` can derive a default `max_tokens` /// for requests that omit one. limits: OnceCell, + rediscovered_metadata: OnceCell, cancel: CancellationToken, } @@ -59,6 +60,7 @@ impl TrtllmSidecarEngine { mode, client: OnceCell::new(), limits: OnceCell::new(), + rediscovered_metadata: OnceCell::new(), cancel: CancellationToken::new(), } } @@ -167,6 +169,55 @@ impl TrtllmSidecarEngine { }; Ok((engine, config)) } + + async fn metadata_for_request( + &self, + client: &TrtllmClient, + request: &PreprocessedRequest, + ) -> Result { + let initial = self + .limits + .get() + .copied() + .ok_or_else(|| client::engine_shutdown("TensorRT-LLM sidecar is not started"))?; + if let Some(metadata) = self.rediscovered_metadata.get() { + return Ok(*metadata); + } + let has_images = request.multi_modal_data.as_ref().is_some_and(|media| { + media + .get("image_url") + .is_some_and(|images| !images.is_empty()) + }); + if !has_images || initial.supports_image.is_some() { + return Ok(initial); + } + + self.rediscovered_metadata + .get_or_try_init(|| async { + match client.model_limits(&self.model.source).await? { + ModelInfoResponse::Available(mut reported) => { + if let Some(engine_context_length) = reported.context_length + && Some(engine_context_length) != initial.context_length + { + tracing::warn!( + configured_context_length = ?initial.context_length, + engine_context_length, + "--context-length disagrees with the context length TensorRT-LLM \ + reported; using the configured --context-length" + ); + } + reported.context_length = initial.context_length; + Ok(reported) + } + ModelInfoResponse::Unsupported(_) => Ok(ModelMetadata { + supports_image: Some(false), + ..initial + }), + } + }) + .await + .copied() + } } #[async_trait] @@ -205,20 +256,26 @@ impl LLMEngine for TrtllmSidecarEngine { // `Watch`). Do not grow a second readiness protocol here. Some(configured) => { let reported = match client.model_limits(&model.source).await { - Ok(reported) => reported, + Ok(ModelInfoResponse::Available(reported)) => reported, + Ok(ModelInfoResponse::Unsupported(status)) => { + tracing::warn!( + code = ?status.code(), + message = status.message(), + configured_context_length = configured, + "Control.GetModelInfo is unavailable; using the configured \ + --context-length" + ); + ModelMetadata { + supports_image: Some(false), + ..Default::default() + } + } Err(error) => { - // Tolerated because some OpenEngine servicer builds ship - // without Control, which is what --context-length exists - // for. UNIMPLEMENTED also fits an engine left on the - // default --grpc-protocol smg, where Generate will fail - // too -- name both so the log points at the real fix. tracing::warn!( %error, configured_context_length = configured, - "Control.GetModelInfo failed; using the configured \ - --context-length. If this is UNIMPLEMENTED, the engine either \ - ships no Control service or was started without \ - --grpc-protocol openengine" + "Control.GetModelInfo failed transiently; using the configured \ + --context-length and deferring media capability discovery" ); ModelMetadata::default() } @@ -286,13 +343,7 @@ impl LLMEngine for TrtllmSidecarEngine { .get() .ok_or_else(|| client::engine_shutdown("TensorRT-LLM sidecar is not started"))?; let request_id = ctx.id().to_string(); - let proto_request = build_generate_request( - &request, - &request_id, - &self.model.source, - self.limits.get().copied(), - self.mode, - )?; + let limits = self.metadata_for_request(client, &request).await?; let mut state = ResponseState::new(&request, self.mode); // A decode request that took a handoff has KV transferred into it, and // the transceiver releases those blocks when the engine finishes the @@ -303,6 +354,13 @@ impl LLMEngine for TrtllmSidecarEngine { // consumer. A decode-mode request without a handoff ran locally and // has nothing to strand. let needs_cancellation_deferral = self.mode.is_decode() && request.prefill_result.is_some(); + let proto_request = build_generate_request( + request, + &request_id, + &self.model.source, + Some(limits), + self.mode, + )?; let stopped_ctx = ctx.inner_arc(); // Hoisted: `stopped()` is an async-trait method, so re-creating it per // streamed chunk costs a boxed future and a waker registration on every diff --git a/lib/sidecar/trtllm/src/tests.rs b/lib/sidecar/trtllm/src/tests.rs index 0bcda724b744..61daa182d2c9 100644 --- a/lib/sidecar/trtllm/src/tests.rs +++ b/lib/sidecar/trtllm/src/tests.rs @@ -469,7 +469,7 @@ fn limits(context_length: u32) -> Option { Some(ModelMetadata { context_length: Some(context_length), max_output_tokens: None, - supports_image: false, + supports_image: Some(false), }) } @@ -477,7 +477,7 @@ fn image_limits(context_length: u32) -> Option { Some(ModelMetadata { context_length: Some(context_length), max_output_tokens: None, - supports_image: true, + supports_image: Some(true), }) } @@ -542,7 +542,7 @@ async fn collect( fn assert_rejected(mutate: impl FnOnce(&mut PreprocessedRequest), expect: &str) { let mut req = request(); mutate(&mut req); - let error = build_generate_request(&req, "req", "model", None, AGG) + let error = build_generate_request(req, "req", "model", None, AGG) .expect_err("request must be rejected"); assert!( error.to_string().contains(expect), diff --git a/lib/sidecar/trtllm/src/tests/convert_request.rs b/lib/sidecar/trtllm/src/tests/convert_request.rs index 57007959e472..c43479a7cb40 100644 --- a/lib/sidecar/trtllm/src/tests/convert_request.rs +++ b/lib/sidecar/trtllm/src/tests/convert_request.rs @@ -8,8 +8,8 @@ use super::*; #[test] fn request_maps_sampling_stop_and_output_fields() { let request = request(); - let proto = build_generate_request(&request, "req-1", "served-model", None, AGG) - .expect("build request"); + let proto = + build_generate_request(request, "req-1", "served-model", None, AGG).expect("build request"); assert_eq!(proto.request_id, "req-1"); assert_eq!(proto.model, "served-model"); match proto.input.as_ref().expect("input") { @@ -89,7 +89,7 @@ fn image_urls_data_uris_and_uuids_are_mapped_in_order() { "image_url".to_string(), vec![Some("first-image".to_string()), None], )])); - let proto = build_generate_request(&req, "req", "model", image_limits(4096), AGG) + let proto = build_generate_request(req, "req", "model", image_limits(4096), AGG) .expect("image request"); assert_eq!(proto.media.len(), 2); assert_eq!(proto.media[0].modality, pb::Modality::Image as i32); @@ -123,7 +123,7 @@ fn mm_routing_hashes_are_not_used_as_engine_cache_uuids() { "mm_hashes": ["0123456789abcdef", "fedcba9876543210"] })); - let proto = build_generate_request(&req, "req", "model", image_limits(4096), AGG) + let proto = build_generate_request(req, "req", "model", image_limits(4096), AGG) .expect("image request"); assert!(proto.media.iter().all(|item| item.uuid.is_empty())); } @@ -134,7 +134,7 @@ fn image_media_requires_the_loaded_models_advertised_capability() { add_images(&mut req, ["https://example.com/image.png"]); for model_limits in [None, limits(4096)] { - let error = build_generate_request(&req, "req", "model", model_limits, AGG) + let error = build_generate_request(req.clone(), "req", "model", model_limits, AGG) .expect_err("a text-only or unknown model must reject images"); assert!( error @@ -159,7 +159,7 @@ fn unsupported_media_modalities_and_sources_are_rejected() { "https://example.com/media".to_string(), )], )])); - let error = build_generate_request(&req, "req", "model", image_limits(4096), AGG) + let error = build_generate_request(req, "req", "model", image_limits(4096), AGG) .expect_err("only images are supported"); assert!( error.to_string().contains(expected), @@ -169,7 +169,7 @@ fn unsupported_media_modalities_and_sources_are_rejected() { let mut req = request(); add_images(&mut req, ["file:///tmp/image.png"]); - let error = build_generate_request(&req, "req", "model", image_limits(4096), AGG) + let error = build_generate_request(req, "req", "model", image_limits(4096), AGG) .expect_err("the engine cannot fetch a local Dynamo path"); assert!(error.to_string().contains("http, https, or data URI")); @@ -178,7 +178,7 @@ fn unsupported_media_modalities_and_sources_are_rejected() { "image_url".to_string(), vec![MultimodalData::UuidOnly("cached-image".to_string())], )])); - let error = build_generate_request(&req, "req", "model", image_limits(4096), AGG) + let error = build_generate_request(req, "req", "model", image_limits(4096), AGG) .expect_err("OpenEngine cannot resolve a source-less UUID"); assert!(error.to_string().contains("requires an image source")); } @@ -191,19 +191,19 @@ fn image_metadata_that_cannot_be_honored_is_rejected() { "image_url".to_string(), vec![Some("one".to_string()), Some("two".to_string())], )])); - let error = build_generate_request(&req, "req", "model", image_limits(4096), AGG) + let error = build_generate_request(req.clone(), "req", "model", image_limits(4096), AGG) .expect_err("UUIDs must align positionally"); assert!(error.to_string().contains("UUID count")); req.multi_modal_uuids = None; req.mm_processor_kwargs = Some(serde_json::json!({"do_resize": false})); - let error = build_generate_request(&req, "req", "model", image_limits(4096), AGG) + let error = build_generate_request(req.clone(), "req", "model", image_limits(4096), AGG) .expect_err("processor overrides cannot be dropped"); assert!(error.to_string().contains("mm_processor_kwargs")); req.mm_processor_kwargs = Some(serde_json::json!({})); req.media_io_kwargs = Some(serde_json::json!({"image": {"mode": "RGB"}})); - let error = build_generate_request(&req, "req", "model", image_limits(4096), AGG) + let error = build_generate_request(req, "req", "model", image_limits(4096), AGG) .expect_err("media I/O overrides cannot be dropped"); assert!(error.to_string().contains("media_io_kwargs")); } @@ -213,7 +213,7 @@ fn image_request_without_max_tokens_is_left_for_engine_derivation() { let mut req = request(); add_images(&mut req, ["https://example.com/image.png"]); req.stop_conditions.max_tokens = None; - let proto = build_generate_request(&req, "req", "model", image_limits(4096), AGG) + let proto = build_generate_request(req, "req", "model", image_limits(4096), AGG) .expect("the engine derives against the expanded prompt"); assert_eq!(proto.stopping.expect("stopping").max_tokens, None); } @@ -222,7 +222,7 @@ fn image_request_without_max_tokens_is_left_for_engine_derivation() { fn omitted_max_tokens_without_context_length_is_rejected() { let mut req = request(); req.stop_conditions.max_tokens = None; - let error = build_generate_request(&req, "req", "model", None, AGG) + let error = build_generate_request(req, "req", "model", None, AGG) .expect_err("must require max_tokens"); assert!(error.to_string().contains("max_tokens")); } @@ -233,8 +233,8 @@ fn omitted_max_tokens_defaults_to_remaining_context() { req.stop_conditions.max_tokens = None; // request() carries three prompt tokens ([11, 22, 33]); the default fills the // remaining context: max(1, context_length - prompt_len). - let proto = build_generate_request(&req, "req", "model", limits(100), AGG) - .expect("build with fallback"); + let proto = + build_generate_request(req, "req", "model", limits(100), AGG).expect("build with fallback"); assert_eq!(proto.stopping.unwrap().max_tokens, Some(97)); } @@ -244,7 +244,7 @@ fn omitted_max_tokens_default_is_floored_at_one() { req.stop_conditions.max_tokens = None; // Prompt already fills (or exceeds) the context: default must not underflow to 0. let proto = - build_generate_request(&req, "req", "model", limits(2), AGG).expect("build with fallback"); + build_generate_request(req, "req", "model", limits(2), AGG).expect("build with fallback"); assert_eq!(proto.stopping.unwrap().max_tokens, Some(1)); } @@ -252,7 +252,7 @@ fn omitted_max_tokens_default_is_floored_at_one() { fn top_k_all_tokens_is_left_unset() { let mut req = request(); req.sampling_options.top_k = Some(-1); - let proto = build_generate_request(&req, "req", "model", None, AGG).expect("build"); + let proto = build_generate_request(req, "req", "model", None, AGG).expect("build"); assert_eq!(proto.sampling.unwrap().top_k, None); } @@ -301,7 +301,7 @@ fn logprobs_zero_keeps_selected_without_alternatives() { req.output_options.logprobs = Some(0); // The wire request must still ask TRT-LLM for one candidate so the selected // token's logprob is computed. - let proto = build_generate_request(&req, "req", "model", None, AGG).expect("build"); + let proto = build_generate_request(req.clone(), "req", "model", None, AGG).expect("build"); match proto .response .unwrap() @@ -331,13 +331,13 @@ fn the_derived_max_tokens_respects_the_engines_output_cap() { let mut req = request(); req.stop_conditions.max_tokens = None; let proto = build_generate_request( - &req, + req, "req", "model", Some(ModelMetadata { context_length: Some(4096), max_output_tokens: Some(256), - supports_image: false, + supports_image: Some(false), }), AGG, ) @@ -363,7 +363,7 @@ fn a_derived_budget_below_min_tokens_is_rejected() { let prompt_len = req.token_ids.len() as u32; let window = prompt_len + 8; - let error = build_generate_request(&req, "req", "model", limits(window), AGG) + let error = build_generate_request(req, "req", "model", limits(window), AGG) .expect_err("a minimum above the remaining budget cannot be served"); let message = error.to_string(); assert!( @@ -383,7 +383,7 @@ fn a_prefill_request_does_not_judge_a_minimum_against_its_own_window() { req.stop_conditions.min_tokens = Some(64); let window = req.token_ids.len() as u32 + 8; - build_generate_request(&req, "req", "model", limits(window), PREFILL) + build_generate_request(req, "req", "model", limits(window), PREFILL) .expect("only the decode worker's window governs the minimum"); } @@ -396,7 +396,7 @@ fn a_minimum_within_the_derived_budget_is_served() { req.stop_conditions.min_tokens = Some(8); let window = req.token_ids.len() as u32 + 64; - let proto = build_generate_request(&req, "req", "model", limits(window), AGG) + let proto = build_generate_request(req, "req", "model", limits(window), AGG) .expect("a minimum that fits is not a conflict"); let stopping = proto.stopping.expect("stopping options"); assert_eq!(stopping.min_tokens, Some(8)); diff --git a/lib/sidecar/trtllm/src/tests/disagg.rs b/lib/sidecar/trtllm/src/tests/disagg.rs index a81c8dfe92c8..45f93e949efd 100644 --- a/lib/sidecar/trtllm/src/tests/disagg.rs +++ b/lib/sidecar/trtllm/src/tests/disagg.rs @@ -17,7 +17,7 @@ fn prefill_request_is_marked_context_only() { req.stop_conditions.max_tokens = Some(128); req.stop_conditions.min_tokens = Some(8); req.output_options.logprobs = Some(1); - let proto = build_generate_request(&req, "req", "model", None, DisaggregationMode::Prefill) + let proto = build_generate_request(req, "req", "model", None, DisaggregationMode::Prefill) .expect("build prefill request"); assert!( @@ -81,7 +81,7 @@ fn image_media_is_preserved_across_prefill_and_decode_requests() { )])); let prefill = build_generate_request( - &req, + req.clone(), "req", "model", image_limits(4096), @@ -100,7 +100,7 @@ fn image_media_is_preserved_across_prefill_and_decode_requests() { prompt_tokens_details: None, }); let decode = build_generate_request( - &req, + req, "req", "model", image_limits(4096), @@ -220,7 +220,7 @@ async fn decode_request_replays_the_prefill_session() { #[test] fn decode_without_a_prefill_result_runs_the_whole_request() { let proto = build_generate_request( - &request(), + request(), "req", "model", limits(4096), @@ -251,7 +251,7 @@ fn prefill_result_on_a_non_decode_worker_is_rejected() { prompt_tokens_details: None, }); for mode in [DisaggregationMode::Aggregated, DisaggregationMode::Prefill] { - let error = build_generate_request(&req, "req", "model", None, mode) + let error = build_generate_request(req.clone(), "req", "model", None, mode) .expect_err("handoff must be rejected"); assert!( error diff --git a/lib/sidecar/trtllm/src/tests/engine.rs b/lib/sidecar/trtllm/src/tests/engine.rs index 149303a06d3b..aff3ef99a66e 100644 --- a/lib/sidecar/trtllm/src/tests/engine.rs +++ b/lib/sidecar/trtllm/src/tests/engine.rs @@ -441,6 +441,64 @@ async fn a_configured_context_length_survives_a_server_without_control() { config.llm.expect("llm registration").context_length, Some(8192) ); + + let mut image_request = request(); + add_images(&mut image_request, ["https://example.com/image.png"]); + let error = match engine + .generate( + image_request, + GenerateContext::new(dynamo_backend_common::testing::mock_context(), None), + ) + .await + { + Ok(_) => panic!("a server without capability discovery must reject images"), + Err(error) => error, + }; + assert!( + error + .to_string() + .contains("does not advertise image support") + ); + assert_eq!( + server.service.model_info_calls.load(Ordering::SeqCst), + 1, + "a definitive UNIMPLEMENTED response must not be retried" + ); +} + +#[tokio::test] +async fn image_request_retries_transient_capability_discovery() { + let service = FakeTrtllm::default(); + service.unavailable_model_info.store(true, Ordering::SeqCst); + let server = FakeServer::start(service).await; + let engine = engine_with(&server.endpoint, transport(1), Some(8192), AGG); + engine.start(0).await.expect("start with configured window"); + + server + .service + .unavailable_model_info + .store(false, Ordering::SeqCst); + server + .service + .supports_multimodal + .store(true, Ordering::SeqCst); + server + .service + .advertises_image + .store(true, Ordering::SeqCst); + let mut image_request = request(); + add_images(&mut image_request, ["https://example.com/image.png"]); + + let output = collect(&engine, image_request).await; + assert_eq!( + output.last().unwrap().finish_reason, + Some(FinishReason::Stop) + ); + assert_eq!( + server.service.model_info_calls.load(Ordering::SeqCst), + 2, + "the image request must retry the transient startup failure once" + ); } /// Shutdown has to terminate in-flight requests: the worker cannot drain if a diff --git a/tests/serve/test_sidecar.py b/tests/serve/test_sidecar.py index 55720fee9b12..0375aa862e0d 100644 --- a/tests/serve/test_sidecar.py +++ b/tests/serve/test_sidecar.py @@ -211,10 +211,6 @@ def _disaggregated_payloads( pytest.mark.requested_trtllm_kv_tokens(32768), pytest.mark.timeout(960), pytest.mark.pre_merge, - pytest.mark.skipif( - not _trtllm_serves_openengine(), - reason=TRTLLM_OPENENGINE_SKIP_REASON, - ), ], model="Qwen/Qwen3-VL-2B-Instruct", timeout=900, @@ -339,6 +335,10 @@ def _disaggregated_payloads( @pytest.fixture(params=params_with_model_mark(sidecar_configs)) def sidecar_config_test(request, monkeypatch, discovery_backend): config = sidecar_configs[request.param] + if config.name == "trtllm_aggregated_image" and not _trtllm_serves_openengine(): + if os.environ.get("DYNAMO_TEST_SIDECAR_BACKEND") == "trtllm": + pytest.fail(TRTLLM_OPENENGINE_SKIP_REASON) + pytest.skip(TRTLLM_OPENENGINE_SKIP_REASON) monkeypatch.setenv("MODEL", config.model) monkeypatch.setenv("DYN_DISCOVERY_BACKEND", discovery_backend) monkeypatch.setenv("DYN_NAMESPACE", f"sidecar-{generate_random_suffix()}") From 33fad484ef8e11d1feedcf9618d1fdb10250796a Mon Sep 17 00:00:00 2001 From: krishung5 Date: Fri, 16 Oct 2026 14:01:16 -0700 Subject: [PATCH 12/61] fix(ci): build sidecar image on compatible TRT-LLM base --- .github/workflows/pr.yaml | 3 +- container/compliance/policy/codec_policy.yaml | 26 ++++---- container/templates/trtllm_runtime.Dockerfile | 64 +++++++++++-------- 3 files changed, 54 insertions(+), 39 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 4eb2b9312608..f016a5acd227 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -542,9 +542,8 @@ jobs: diff_current_branch: ${{ github.ref_name }} diff_base_branch: ${{ needs.changed-files.outputs.base_ref }} diff_merge_base_sha: ${{ needs.changed-files.outputs.merge_base_sha }} - # Drop these overrides after the upstream TensorRT-LLM change ships. + # Drop this source overlay after the upstream TensorRT-LLM change ships. extra_build_args: | - RUNTIME_IMAGE_TAG=1.4.0rc1.dev202610140007@sha256:7a728cd307bfa1651b4ad0787e71e69f7506fb1a698a0f4f04aef719ad43525f TRTLLM_SOURCE_REPOSITORY=https://github.com/krishung5/TensorRT-LLM TRTLLM_SOURCE_COMMIT=5d2e3b2c758b4ccc0fa0a3eb0e01cdef934e8089 # actions: read lets the build download a prior run's baseline compliance diff --git a/container/compliance/policy/codec_policy.yaml b/container/compliance/policy/codec_policy.yaml index 61b8f39e19a0..9e4023b0e2e3 100644 --- a/container/compliance/policy/codec_policy.yaml +++ b/container/compliance/policy/codec_policy.yaml @@ -108,9 +108,9 @@ exceptions: (NVIDIA/DALI_deps#162, NVIDIA/DALI#6352, May 2026), first released in 2.1.1 -- as a cherry-pick, so checking whether the merge commit is an ancestor of the tag gives the wrong answer; the release notes are the - reliable source. Upgrading the wheel in place to 2.1.1, 2.2.0, or 2.3.0 - leaves 440 decoders with all five absent, vp8/vp9/mjpeg/av1 still present, - and the image importing and passing its unit suite. + reliable source. Upgrading the wheel in place to 2.1.1 or 2.2.0 leaves 440 + decoders with all five absent, vp8/vp9/mjpeg/av1 still present, and the + image importing and passing its unit suite. This waiver cannot be narrowed to the non-codec companions, which an earlier revision of this note claimed. libavcodec, libavfilter, libavformat, @@ -120,12 +120,13 @@ exceptions: set is all-or-nothing with the package. The entry stays even so. trtllm_runtime.Dockerfile pins - nvidia-dali-cuda130 at an explicitly measured version: 2.2.0 in the - release base or 2.3.0 in the pinned development base. The vendored - libraries do not register those decoders -- but they are still on disk and - still match the deny globs above, which gate on presence rather than - content. What changed is the reason for the waiver: it now covers a vendored - ffmpeg that carries no software h264/hevc/aac, rather than one that does. + nvidia-dali-cuda130==2.2.0, which is the version the 1.3.0rc27 base image + already carries, so the pin now holds a version rather than moving one. The + vendored libraries do not register those decoders -- but they are still on + disk and still match the deny globs above, which gate on presence rather + than content. What changed is the reason for the waiver: it now covers a + vendored ffmpeg that carries no software h264/hevc/aac, rather than one + that does. Enforcement lives at build time, not here. Two guards in trtllm_runtime.Dockerfile enumerate every bundled libavcodec -- one in @@ -141,9 +142,10 @@ exceptions: subtracted from the generated delta, so such an entry would never match anything and would read as a check while doing nothing. - Re-measure when the vendored version moves. Last measured against 2.2.0 and - 2.3.0: 440 decoders, h264, hevc, aac, aac_fixed and aac_latm absent, vp8, - vp9, mjpeg and av1 present. + Re-measure when the vendored version moves. Last measured against + nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc27, whose base carries 2.2.0: + 440 decoders, h264, hevc, aac, aac_fixed and aac_latm absent, vp8, vp9, + mjpeg and av1 present. owner: DALI # PyNvVideoCodec (added for NVDEC hardware video decode) vendors libavutil and # libavformat under its package dir to demux input containers before handing the diff --git a/container/templates/trtllm_runtime.Dockerfile b/container/templates/trtllm_runtime.Dockerfile index b71d05bdb1e9..82661874f3e6 100644 --- a/container/templates/trtllm_runtime.Dockerfile +++ b/container/templates/trtllm_runtime.Dockerfile @@ -44,7 +44,8 @@ ARG NIXL_REF ARG TRTLLM_SOURCE_REPOSITORY ARG TRTLLM_SOURCE_COMMIT -# Optional exact-source overlay for cross-repository CI. +# Optional source overlay for cross-repository CI. Keep the base image's server +# lifecycle entry points, which are coupled to that release's frontend runtime. RUN if [ -n "${TRTLLM_SOURCE_COMMIT}" ]; then \ test -n "${TRTLLM_SOURCE_REPOSITORY}" && \ source_dir=$(mktemp -d) && \ @@ -55,8 +56,17 @@ RUN if [ -n "${TRTLLM_SOURCE_COMMIT}" ]; then \ --strip-components=1 && \ package_dir=$(/usr/bin/python3 -c \ 'import importlib.util; print(next(iter(importlib.util.find_spec("tensorrt_llm").submodule_search_locations)))') && \ - cp -a "${source_dir}/tensorrt_llm/grpc/openengine/." \ - "${package_dir}/grpc/openengine/" && \ + for entry in "${source_dir}/tensorrt_llm/grpc/openengine/"*; do \ + case "$(basename "${entry}")" in \ + __init__.py|server.py) continue ;; \ + esac; \ + cp -a "${entry}" "${package_dir}/grpc/openengine/"; \ + done && \ + sed -i \ + 's/from openengine\.v1 import openengine_pb2_grpc/from .bindings import openengine_pb2_grpc/' \ + "${package_dir}/grpc/openengine/server.py" && \ + grep -Fxq 'from .bindings import openengine_pb2_grpc' \ + "${package_dir}/grpc/openengine/server.py" && \ install -m 0644 "${source_dir}/tensorrt_llm/inputs/media_io.py" \ "${package_dir}/inputs/media_io.py" && \ rm -rf "${source_dir}"; \ @@ -403,9 +413,10 @@ RUN /usr/bin/python3 -m pip uninstall -y --break-system-packages opencv-python-h # Hold DALI at a version that carries its own media-codec cleanup. Upstream # restricted DALI's vendored ffmpeg build to drop the software h264/hevc/aac # decoders (NVIDIA/DALI_deps#162, NVIDIA/DALI#6352), first released in 2.1.1. -# The supported base images carry either 2.2.0 or 2.3.0. Both were measured -# without the restricted codecs; older versions are upgraded to 2.2.0 and any -# other newer version stops the build for re-measurement. +# The base image carried 2.1.0 through 1.3.0rc26, which predates it, and carries +# 2.2.0 from 1.3.0rc27, which does not. So on this base the install below is a +# no-op and what the block still buys is the two checks around it: the version +# tripwire and the decoder enumeration. # # Pin rather than remove. The vendored libav*/libsw* set cannot be trimmed # on its own -- every one of them is DT_NEEDED by libdali.so and its siblings, so @@ -415,17 +426,23 @@ RUN /usr/bin/python3 -m pip uninstall -y --break-system-packages opencv-python-h # out of someone else's image. # # Measured on the shipped image: 2.1.0 registers 446 decoders including h264, -# hevc, aac, aac_fixed and aac_latm; 2.1.1, 2.2.0, and 2.3.0 each register 440 -# with all five absent and vp8/vp9/mjpeg/av1 retained. +# hevc, aac, aac_fixed and aac_latm; 2.1.1 and 2.2.0 each register 440 with all +# five absent and vp8/vp9/mjpeg/av1 retained. 2.2.0 measured again on +# tensorrt-llm/release:1.3.0rc27 itself, now that DALI arrives with the base. # # Pinned exactly, not a floor. `>=2.1.1` resolves to whatever is newest, which is # a jump into a component this repo does not own, taken silently at build time. # The pin names the version that was measured, so moving it means measuring the # new one. # -# A base below 2.2.0 is upgraded to the measured 2.2.0 wheel. The two measured -# versions otherwise stay in place; any unmeasured newer version fails rather -# than being downgraded or accepted silently. +# The pin can go stale in the one direction that matters: if TensorRT-LLM later +# ships a base image with a newer DALI, installing 2.2.0 over it is a downgrade. +# The check below turns that into a build failure with instructions rather than a +# silent regression. The pin now equals the base version, so the next base image +# that moves DALI past 2.2.0 stops the build, which is the point: a human +# re-measures and moves the pin, or deletes this RUN and lets the base version +# stand. A base that moves DALI back below 2.2.0 passes that check and the +# install below puts the measured version back, which is what it is there for. # # THIS BLOCK IS TRANSITIONAL, and on this base it is already down to its checks. # The guards after it and the bundled-libavcodec assertion in @@ -450,23 +467,20 @@ RUN --mount=type=bind,source=./container/compliance/enumerate_bundled_decoders.p set -eu; \ before=$(/usr/bin/python3 -c 'import importlib.metadata as m; print(m.version("nvidia-dali-cuda130"))'); \ echo "DALI in base image: $before"; \ - case "$before" in \ - 2.3.0) target=2.3.0 ;; \ - *) \ - newest=$(printf '%s\n2.2.0\n' "$before" | sort -V | tail -1); \ - if [ "$newest" != "2.2.0" ]; then \ - echo "ERROR: base image carries unmeasured DALI $before" >&2; \ - exit 1; \ - fi; \ - target=2.2.0 ;; \ - esac; \ - if [ "$before" != "$target" ]; then \ - /usr/bin/python3 -m pip install --break-system-packages --no-cache-dir \ - --extra-index-url https://pypi.nvidia.com "nvidia-dali-cuda130==$target"; \ + newest=$(printf '%s\n2.2.0\n' "$before" | sort -V | tail -1); \ + if [ "$newest" != "2.2.0" ]; then \ + echo "ERROR: base image already carries DALI $before, so pinning 2.2.0 would" >&2; \ + echo " downgrade it. Upstream has caught up -- delete this RUN and let" >&2; \ + echo " the base version stand. Keep the guards below and the bundled-" >&2; \ + echo " libavcodec assertion in tests/dependencies/, which are what stop" >&2; \ + echo " this from regressing." >&2; \ + exit 1; \ fi; \ + /usr/bin/python3 -m pip install --break-system-packages --no-cache-dir \ + --extra-index-url https://pypi.nvidia.com 'nvidia-dali-cuda130==2.2.0'; \ v=$(/usr/bin/python3 -c 'import importlib.metadata as m; print(m.version("nvidia-dali-cuda130"))'); \ echo "DALI version: $v"; \ - [ "$v" = "$target" ] || { echo "ERROR: wanted DALI $target, got $v" >&2; exit 1; }; \ + [ "$v" = "2.2.0" ] || { echo "ERROR: wanted DALI 2.2.0, got $v" >&2; exit 1; }; \ libs=$(find /usr/local/lib/python3.12/dist-packages/nvidia/dali/.libs -name 'libavcodec*.so*'); \ if [ -z "$libs" ]; then \ echo "ERROR: DALI $v ships no libavcodec under .libs, so the enumeration" >&2; \ From 22e0e0dc1954b80b39682d2068421b5be9a722e8 Mon Sep 17 00:00:00 2001 From: krishung5 Date: Fri, 16 Oct 2026 15:07:39 -0700 Subject: [PATCH 13/61] test(trtllm): remove redundant image cases --- .../trtllm/src/tests/convert_request.rs | 42 ++++++------------- 1 file changed, 13 insertions(+), 29 deletions(-) diff --git a/lib/sidecar/trtllm/src/tests/convert_request.rs b/lib/sidecar/trtllm/src/tests/convert_request.rs index c43479a7cb40..31d47129d23a 100644 --- a/lib/sidecar/trtllm/src/tests/convert_request.rs +++ b/lib/sidecar/trtllm/src/tests/convert_request.rs @@ -147,25 +147,19 @@ fn image_media_requires_the_loaded_models_advertised_capability() { #[test] fn unsupported_media_modalities_and_sources_are_rejected() { - for (key, expected) in [ - ("video_url", "video_url"), - ("audio_url", "audio_url"), - ("future_media", "future_media"), - ] { - let mut req = request(); - req.multi_modal_data = Some(HashMap::from([( - key.to_string(), - vec![MultimodalData::RawUrl( - "https://example.com/media".to_string(), - )], - )])); - let error = build_generate_request(req, "req", "model", image_limits(4096), AGG) - .expect_err("only images are supported"); - assert!( - error.to_string().contains(expected), - "unexpected error: {error}" - ); - } + let mut req = request(); + req.multi_modal_data = Some(HashMap::from([( + "video_url".to_string(), + vec![MultimodalData::RawUrl( + "https://example.com/media".to_string(), + )], + )])); + let error = build_generate_request(req, "req", "model", image_limits(4096), AGG) + .expect_err("only images are supported"); + assert!( + error.to_string().contains("video_url"), + "unexpected error: {error}" + ); let mut req = request(); add_images(&mut req, ["file:///tmp/image.png"]); @@ -208,16 +202,6 @@ fn image_metadata_that_cannot_be_honored_is_rejected() { assert!(error.to_string().contains("media_io_kwargs")); } -#[test] -fn image_request_without_max_tokens_is_left_for_engine_derivation() { - let mut req = request(); - add_images(&mut req, ["https://example.com/image.png"]); - req.stop_conditions.max_tokens = None; - let proto = build_generate_request(req, "req", "model", image_limits(4096), AGG) - .expect("the engine derives against the expanded prompt"); - assert_eq!(proto.stopping.expect("stopping").max_tokens, None); -} - #[test] fn omitted_max_tokens_without_context_length_is_rejected() { let mut req = request(); From da67ad6e9dcae083010f78c3630e5d1b4d93c58f Mon Sep 17 00:00:00 2001 From: krishung5 Date: Thu, 15 Oct 2026 11:16:15 -0700 Subject: [PATCH 14/61] feat(trtllm): support image inputs in sidecar Signed-off-by: krishung5 --- lib/llm/src/backend.rs | 206 ++++++++++++++- lib/sidecar/trtllm/src/client.rs | 25 ++ lib/sidecar/trtllm/src/convert.rs | 240 ++++++++++++++++-- lib/sidecar/trtllm/src/tests.rs | 55 +++- .../trtllm/src/tests/convert_request.rs | 178 +++++++++++++ lib/sidecar/trtllm/src/tests/disagg.rs | 53 ++++ lib/sidecar/trtllm/src/tests/e2e.rs | 180 +++++++++++++ lib/sidecar/trtllm/src/tests/engine.rs | 58 +++++ 8 files changed, 961 insertions(+), 34 deletions(-) diff --git a/lib/llm/src/backend.rs b/lib/llm/src/backend.rs index 3a73e46a21b0..4c5a2d80e0d7 100644 --- a/lib/llm/src/backend.rs +++ b/lib/llm/src/backend.rs @@ -261,8 +261,8 @@ impl return state.pending_flush.pop().map(|output| (output, state)); } - let output = loop { - let output = state.stream.next().await; + let (output, trailing_usage) = loop { + let mut output = state.stream.next().await; let choice_already_finished = output.as_ref().is_some_and(|output| { !output.is_event() && output.data.as_ref().is_some_and(|data| { @@ -272,7 +272,50 @@ impl }) }); if !choice_already_finished { - break output; + break (output, false); + } + + let carries_usage = output + .as_ref() + .and_then(|output| output.data.as_ref()) + .is_some_and(|data| data.completion_usage.is_some()); + let backend_failed = output + .as_ref() + .and_then(|output| output.data.as_ref()) + .is_some_and(|data| matches!(data.finish_reason, Some(FinishReason::Error(_)))); + if carries_usage || backend_failed { + // Native backends can put authoritative cumulative usage + // on a terminal event immediately after the token Dynamo + // recognized locally as EOS. Preserve that metadata without + // emitting duplicate content or a second successful finish + // event. Preserve a backend error with or without usage: it + // is not duplicate success metadata and must reach the + // frontend. + if let Some(data) = output.as_mut().and_then(|output| output.data.as_mut()) { + data.token_ids.clear(); + data.tokens = None; + data.text = None; + data.cum_log_probs = None; + data.log_probs = None; + data.top_logprobs = None; + if !backend_failed { + data.finish_reason = None; + data.stop_reason = None; + } + } + break (output, true); + } + + let is_terminal = output + .as_ref() + .and_then(|output| output.data.as_ref()) + .is_some_and(|data| data.finish_reason.is_some()); + if !is_terminal && state.finished_choices.len() >= state.decoders.len() { + // Once every choice ended locally, tolerate only terminal + // metadata. A nonterminal suffix means the backend ignored + // the stop; cancel rather than burning the remaining budget. + state.stream.context().stop_generating(); + return None; } }; @@ -281,6 +324,11 @@ impl // move to state.process_output // handle any error conditions / unwraps here + if trailing_usage { + state.finished = state.finished_choices.len() >= state.decoders.len(); + return Some((output, state)); + } + // events are pass thru if output.is_event() || output.data.is_none() { // A genuine stream-level error ends generation for every choice at @@ -497,13 +545,12 @@ impl .unwrap_or((None, None)); // If we detected a local stop condition, mark this choice as finished. - // Once all expected choices are finished, stop the upstream generator. + // Keep the upstream stream alive just long enough to consume + // a native backend's following terminal-usage event. The + // loop above still cancels on any nonterminal suffix once + // every choice has stopped. if finish_reason.is_some() && data.finish_reason.is_none() { state.finished_choices.insert(choice_idx); - if state.finished_choices.len() >= decoders_count { - state.stream.context().stop_generating(); - state.finished = true; - } } let text = result.text; @@ -1921,6 +1968,149 @@ mod tests { assert_eq!(generator.get_usage().completion_tokens, 2); } + #[tokio::test] + async fn local_eos_preserves_a_following_authoritative_usage_trailer() { + let tokenizer: Arc = Arc::new(CandidateDecoder); + let backend = Backend::from_tokenizer(Tokenizer::from(tokenizer)); + let request = PreprocessedRequest::builder() + .model("test-model".to_string()) + .token_ids(vec![]) + .stop_conditions(StopConditions { + stop_token_ids_hidden: Some(vec![103]), + ..Default::default() + }) + .sampling_options(SamplingOptions::default()) + .output_options(OutputOptions::default()) + .build() + .expect("valid preprocessed request"); + let reported = dynamo_protocols::types::CompletionUsage { + prompt_tokens: 82, + completion_tokens: 2, + total_tokens: 84, + prompt_tokens_details: None, + completion_tokens_details: None, + }; + let engine: ServerStreamingEngine> = + Arc::new(SyntheticSglangStopEngine { + outputs: Some(vec![ + // Dynamo recognizes 103 as EOS before the native backend's + // separate terminal event arrives. + LLMEngineOutput { + token_ids: vec![101, 103], + index: Some(0), + ..Default::default() + }, + LLMEngineOutput { + index: Some(0), + finish_reason: Some(FinishReason::Stop), + completion_usage: Some(reported.clone()), + ..Default::default() + }, + ]), + }); + + let outputs = Operator::generate(backend.as_ref(), SingleIn::new(request), engine) + .await + .expect("backend generation succeeds") + .collect::>() + .await; + assert_eq!(outputs.len(), 2); + + let local_terminal = outputs[0].data.as_ref().expect("local terminal"); + assert_eq!(local_terminal.finish_reason, Some(FinishReason::Stop)); + let usage_trailer = outputs[1].data.as_ref().expect("usage trailer"); + assert!(usage_trailer.token_ids.is_empty()); + assert!(usage_trailer.finish_reason.is_none()); + assert_eq!(usage_trailer.completion_usage.as_ref(), Some(&reported)); + + let options = DeltaGeneratorOptions::new(None, None, false, None); + let mut generator = DeltaGenerator::new( + "test-model".to_string(), + options, + "test-request".to_string(), + ); + for output in outputs { + generator + .choice_from_postprocessor(output.data.expect("backend output")) + .expect("OpenAI response conversion succeeds"); + } + assert_eq!(generator.get_usage(), reported); + } + + #[tokio::test] + async fn local_eos_does_not_mask_a_trailing_backend_error() { + let tokenizer: Arc = Arc::new(CandidateDecoder); + let backend = Backend::from_tokenizer(Tokenizer::from(tokenizer)); + let request = PreprocessedRequest::builder() + .model("test-model".to_string()) + .token_ids(vec![]) + .stop_conditions(StopConditions { + stop_token_ids_hidden: Some(vec![103]), + ..Default::default() + }) + .sampling_options(SamplingOptions::default()) + .output_options(OutputOptions::default()) + .build() + .expect("valid preprocessed request"); + let reported = dynamo_protocols::types::CompletionUsage { + prompt_tokens: 82, + completion_tokens: 2, + total_tokens: 84, + prompt_tokens_details: None, + completion_tokens_details: None, + }; + for completion_usage in [None, Some(reported.clone())] { + let engine: ServerStreamingEngine> = + Arc::new(SyntheticSglangStopEngine { + outputs: Some(vec![ + LLMEngineOutput { + token_ids: vec![101, 103], + index: Some(0), + ..Default::default() + }, + LLMEngineOutput { + index: Some(0), + finish_reason: Some(FinishReason::Error("worker failed".to_string())), + completion_usage: completion_usage.clone(), + ..Default::default() + }, + ]), + }); + + let outputs = + Operator::generate(backend.as_ref(), SingleIn::new(request.clone()), engine) + .await + .expect("backend generation succeeds") + .collect::>() + .await; + assert_eq!(outputs.len(), 2); + let error_trailer = outputs[1].data.as_ref().expect("error trailer"); + assert!(error_trailer.token_ids.is_empty()); + assert_eq!( + error_trailer.finish_reason, + Some(FinishReason::Error("worker failed".to_string())) + ); + assert_eq!( + error_trailer.completion_usage.as_ref(), + completion_usage.as_ref() + ); + + let options = DeltaGeneratorOptions::new(None, None, false, None); + let mut generator = DeltaGenerator::new( + "test-model".to_string(), + options, + "test-request".to_string(), + ); + generator + .choice_from_postprocessor(outputs[0].data.clone().expect("local terminal")) + .expect("the local terminal converts"); + let error = generator + .choice_from_postprocessor(outputs[1].data.clone().expect("error trailer")) + .expect_err("the backend error must reach the OpenAI response layer"); + assert!(error.to_string().contains("worker failed")); + } + } + #[tokio::test] async fn test_no_stop_trim_raw_completion_output() { use crate::protocols::common::{ diff --git a/lib/sidecar/trtllm/src/client.rs b/lib/sidecar/trtllm/src/client.rs index d1f4f046a014..16cea9315dfd 100644 --- a/lib/sidecar/trtllm/src/client.rs +++ b/lib/sidecar/trtllm/src/client.rs @@ -38,6 +38,11 @@ pub(crate) struct ModelLimits { /// Cap on generated tokens, when the engine reports one. A context window /// alone can imply a larger budget than the engine will accept. pub(crate) max_output_tokens: Option, + /// Whether this loaded model advertises image input through both the + /// portable multimodal bit and TensorRT-LLM's modality list. Requiring both + /// fails closed when a server and client disagree about the finer-grained + /// capability. + pub(crate) supports_image: bool, } pub(crate) struct TrtllmClient { @@ -102,9 +107,29 @@ impl TrtllmClient { }) .await? .into_inner(); + let supports_image = info.supports_multimodal == Some(true) + && info.extra.as_ref().is_some_and(|extra| { + extra + .fields + .get("trtllm_supported_modalities") + .and_then(|value| value.kind.as_ref()) + .is_some_and(|kind| match kind { + prost_types::value::Kind::ListValue(modalities) => { + modalities.values.iter().any(|value| { + matches!( + value.kind.as_ref(), + Some(prost_types::value::Kind::StringValue(modality)) + if modality == "image" + ) + }) + } + _ => false, + }) + }); Ok(ModelLimits { context_length: info.max_context_length.filter(|len| *len > 0), max_output_tokens: info.max_output_tokens.filter(|cap| *cap > 0), + supports_image, }) } diff --git a/lib/sidecar/trtllm/src/convert.rs b/lib/sidecar/trtllm/src/convert.rs index 3360818e2869..0c3e3d59d2f3 100644 --- a/lib/sidecar/trtllm/src/convert.rs +++ b/lib/sidecar/trtllm/src/convert.rs @@ -4,9 +4,10 @@ //! Conversion between Dynamo's `PreprocessedRequest` / `LLMEngineOutput` and the //! TensorRT-LLM OpenEngine (`openengine.v1`) protobuf messages. //! -//! Scope: aggregated and disaggregated (prefill/decode) generation. Multimodal, -//! LoRA, beam search, and `n > 1` are rejected before dispatch — the sidecar -//! streams a single sequence. +//! Scope: aggregated and disaggregated (prefill/decode) generation, including +//! image inputs when the loaded model advertises them. Video, audio, LoRA, +//! beam search, and `n > 1` are rejected before dispatch — the sidecar streams +//! a single sequence. //! //! Disaggregation is driven by [`DisaggregationMode`]: a prefill worker marks //! its request `context_only` and returns the `PrefillReady` handoff as its @@ -17,7 +18,7 @@ use std::collections::BTreeSet; use dynamo_backend_common::{ CompletionUsage, DisaggregationMode, DynamoError, FinishReason, LLMEngineOutput, - PreprocessedRequest, PromptTokensDetails, StopReason, TopLogprob, usage, + MultimodalData, PreprocessedRequest, PromptTokensDetails, StopReason, TopLogprob, usage, }; use crate::client::{self, ModelLimits}; @@ -36,6 +37,7 @@ pub(crate) fn build_generate_request( mode: DisaggregationMode, ) -> Result { validate_request(request, mode)?; + let media = image_media(request, limits)?; let sampling = &request.sampling_options; let stop = &request.stop_conditions; @@ -50,7 +52,7 @@ pub(crate) fn build_generate_request( // reject requests a wider decode window would serve. Releasing the handoff // when decode does reject is the sound fix and belongs on that side. let max_tokens = if mode.is_prefill() { - 1 + Some(1) } else { max_tokens(request, limits)? }; @@ -92,7 +94,7 @@ pub(crate) fn build_generate_request( num_sequences: Some(1), }), stopping: Some(pb::StoppingOptions { - max_tokens: Some(max_tokens), + max_tokens, // Forwarded on a context-only request too. The minimum only masks // EOS until the sequence reaches it, so it cannot extend a context // phase that `max_tokens` already caps at one token. Dropping it @@ -123,8 +125,7 @@ pub(crate) fn build_generate_request( prompt_logprob_start: None, }), guided: guided_decoding(request)?, - // Text generation only: no multimodal media or LoRA selection. - media: Vec::new(), + media, lora_name: String::new(), kv, extra: Some(disagg::request_extra( @@ -139,13 +140,14 @@ pub(crate) fn build_generate_request( } // Temporary workaround: the OpenEngine contract makes `stopping.max_tokens` -// optional, but an omitted value falls through to TensorRT-LLM's small -// `SamplingParams` default rather than filling the context. The Dynamo frontend -// forwards an omitted `max_tokens` as `None` expecting the backend to default, -// so we mirror the in-process backend's text-only default, +// optional. The Dynamo frontend forwards an omitted `max_tokens` as `None` +// expecting the backend to default. For text requests we mirror the in-process +// backend's default, // `max(1, context_length - prompt_len)` (components/src/dynamo/trtllm -// `_default_max_tokens`); the sidecar rejects multimodal before dispatch, so -// `token_ids.len()` is the true prompt length. `context_length` is resolved in +// `_default_max_tokens`). For image requests, only TensorRT-LLM knows the true +// prompt length after expanding image markers, so leave the field unset and let +// the OpenEngine request mapper derive the budget from the expanded prompt. +// `context_length` is resolved in // `engine::start`, where `--context-length` wins over the `GetModelInfo` report. // Only `/v1/chat/completions` and `/v1/responses` reach this fallback: they set // `PRESERVE_OMITTED_MAX_TOKENS_CONTEXT_KEY`, which stops the frontend supplying @@ -161,9 +163,12 @@ pub(crate) fn build_generate_request( fn max_tokens( request: &PreprocessedRequest, limits: Option, -) -> Result { +) -> Result, DynamoError> { if let Some(max_tokens) = request.stop_conditions.max_tokens { - return Ok(max_tokens); + return Ok(Some(max_tokens)); + } + if has_media(request) { + return Ok(None); } let context_length = limits .and_then(|limits| limits.context_length) @@ -185,7 +190,198 @@ fn max_tokens( None => remaining, }; reject_impossible_minimum(request, derived, context_length, prompt_len)?; - Ok(derived) + Ok(Some(derived)) +} + +const IMAGE_KEY: &str = "image_url"; +const MM_HASHES_KEY: &str = "mm_hashes"; + +/// Convert Dynamo's image-only execution payload into the ordered OpenEngine +/// media list. The frontend groups content parts by modality, so ordering is +/// retained within the image vector but not across different modalities. This +/// PR therefore accepts images only and fails explicitly on every other +/// non-empty modality instead of inventing a cross-modality order. +fn image_media( + request: &PreprocessedRequest, + limits: Option, +) -> Result, DynamoError> { + reject_media_override("mm_processor_kwargs", request.mm_processor_kwargs.as_ref())?; + reject_media_override("media_io_kwargs", request.media_io_kwargs.as_ref())?; + + let media_map = request.multi_modal_data.as_ref(); + if let Some(media_map) = media_map { + for (modality, items) in media_map { + if modality != IMAGE_KEY && !items.is_empty() { + return Err(client::invalid_argument(format!( + "multimodal modality `{modality}` is not supported by the TensorRT-LLM sidecar; this image-only implementation accepts `{IMAGE_KEY}`" + ))); + } + } + } + + let images = media_map + .and_then(|media| media.get(IMAGE_KEY)) + .map(Vec::as_slice) + .unwrap_or_default(); + + let user_image_uuids = if let Some(uuid_map) = request.multi_modal_uuids.as_ref() { + for (modality, uuids) in uuid_map { + if modality != IMAGE_KEY && !uuids.is_empty() { + return Err(client::invalid_argument(format!( + "multimodal UUIDs for `{modality}` cannot be forwarded by the image-only TensorRT-LLM sidecar" + ))); + } + } + uuid_map.get(IMAGE_KEY).map(Vec::as_slice) + } else { + None + }; + + if let Some(uuids) = user_image_uuids + && uuids.len() != images.len() + { + return Err(client::invalid_argument(format!( + "`{IMAGE_KEY}` UUID count ({}) must match its media item count ({})", + uuids.len(), + images.len() + ))); + } + + if images.is_empty() { + return Ok(Vec::new()); + } + if !limits.is_some_and(|limits| limits.supports_image) { + return Err(client::invalid_argument( + "the loaded TensorRT-LLM model does not advertise image support through Control.GetModelInfo", + )); + } + + // Explicit caller UUIDs remain authoritative. Otherwise carry the exact + // hashes generated by Dynamo's MM-aware preprocessor to the backend. This + // is request-side preparation for KV-aware routing; the router-owned + // `mm_routing_info` itself never crosses the worker boundary. + let has_user_uuid = user_image_uuids.is_some_and(|uuids| { + uuids + .iter() + .any(|uuid| uuid.as_deref().is_some_and(|uuid| !uuid.is_empty())) + }); + let routing_hashes = if has_user_uuid { + None + } else { + routing_hashes(request, images.len())? + }; + + images + .iter() + .enumerate() + .map(|(index, image)| { + let source = match image { + MultimodalData::Url(url) => media_source(url.as_str(), index)?, + MultimodalData::RawUrl(url) => media_source(url, index)?, + MultimodalData::Decoded(_) => { + return Err(client::invalid_argument(format!( + "`{IMAGE_KEY}` item {index} is pre-decoded; OpenEngine requires a URL or data URI so TensorRT-LLM can own image decoding" + ))); + } + MultimodalData::UuidOnly(_) => { + return Err(client::invalid_argument(format!( + "`{IMAGE_KEY}` item {index} contains only a cache UUID; OpenEngine requires an image source" + ))); + } + }; + Ok(pb::MediaItem { + modality: pb::Modality::Image as i32, + source: Some(source), + mime_type: String::new(), + uuid: if has_user_uuid { + user_image_uuids + .and_then(|uuids| uuids[index].as_deref()) + .unwrap_or_default() + .to_string() + } else { + routing_hashes + .as_ref() + .map(|hashes| hashes[index].clone()) + .unwrap_or_default() + }, + }) + }) + .collect() +} + +fn routing_hashes( + request: &PreprocessedRequest, + image_count: usize, +) -> Result>, DynamoError> { + let hashes = match request.extra_args.as_ref() { + Some(serde_json::Value::Object(extra)) => extra.get(MM_HASHES_KEY), + _ => None, + }; + let Some(hashes) = hashes else { + return Ok(None); + }; + let hashes = hashes.as_array().ok_or_else(|| { + client::invalid_argument("extra_args.mm_hashes must be an array of strings") + })?; + if hashes.is_empty() { + return Ok(None); + } + if hashes.len() != image_count { + return Err(client::invalid_argument(format!( + "extra_args.mm_hashes count ({}) must match the image media item count ({image_count})", + hashes.len() + ))); + } + hashes + .iter() + .enumerate() + .map(|(index, hash)| { + hash.as_str() + .filter(|hash| !hash.is_empty()) + .map(ToOwned::to_owned) + .ok_or_else(|| { + client::invalid_argument(format!( + "extra_args.mm_hashes[{index}] must be a non-empty string" + )) + }) + }) + .collect::, _>>() + .map(Some) +} + +fn media_source(value: &str, index: usize) -> Result { + let scheme = value + .split_once(':') + .map(|(scheme, _)| scheme.to_ascii_lowercase()); + match scheme.as_deref() { + Some("http") | Some("https") => Ok(pb::media_item::Source::Url(value.to_string())), + Some("data") => Ok(pb::media_item::Source::DataUri(value.to_string())), + _ => Err(client::invalid_argument(format!( + "`{IMAGE_KEY}` item {index} must use an http, https, or data URI" + ))), + } +} + +fn reject_media_override(name: &str, value: Option<&serde_json::Value>) -> Result<(), DynamoError> { + let is_empty = match value { + None | Some(serde_json::Value::Null) => true, + Some(serde_json::Value::Object(fields)) => fields.is_empty(), + Some(_) => false, + }; + if is_empty { + Ok(()) + } else { + Err(client::invalid_argument(format!( + "{name} is not supported by the TensorRT-LLM OpenEngine sidecar" + ))) + } +} + +fn has_media(request: &PreprocessedRequest) -> bool { + request + .multi_modal_data + .as_ref() + .is_some_and(|media| media.values().any(|items| !items.is_empty())) } /// Both the output cap and the `.max(1)` floor can land under an explicit @@ -341,14 +537,14 @@ fn validate_request( "prompt embeddings are not supported by the TensorRT-LLM sidecar", )); } - if request.multi_modal_data.is_some() - || request.mm_routing_info.is_some() - || request.encoder_result.is_some() - { + if request.encoder_result.is_some() { return Err(client::invalid_argument( - "multimodal requests are not supported by the TensorRT-LLM sidecar", + "multimodal encoder handoffs are not supported by the TensorRT-LLM sidecar", )); } + // `mm_routing_info` is deliberately not forwarded: it is framework-owned + // metadata that the Dynamo router consumes before selecting this worker. + // The execution payload remains `multi_modal_data`, mapped by `image_media`. // Only a decode worker consumes a prefill handoff. Seeing one anywhere else // means the frontend routed a decode request to the wrong worker role. if request.prefill_result.is_some() && !mode.is_decode() { diff --git a/lib/sidecar/trtllm/src/tests.rs b/lib/sidecar/trtllm/src/tests.rs index 411aa819176e..d6d3c3146762 100644 --- a/lib/sidecar/trtllm/src/tests.rs +++ b/lib/sidecar/trtllm/src/tests.rs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -use std::collections::BTreeSet; +use std::collections::{BTreeSet, HashMap}; use std::net::SocketAddr; use std::num::NonZeroUsize; use std::pin::Pin; @@ -10,8 +10,8 @@ use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::time::Duration; use dynamo_backend_common::{ - DisaggregationMode, ErrorType, FinishReason, GenerateContext, LLMEngine, OutputOptions, - PreprocessedRequest, SamplingOptions, StopConditions, StopReason, + DisaggregationMode, ErrorType, FinishReason, GenerateContext, LLMEngine, MultimodalData, + OutputOptions, PreprocessedRequest, SamplingOptions, StopConditions, StopReason, }; use dynamo_sidecar_common::{GrpcEndpoint, GrpcTransportConfig}; use futures::{Stream, StreamExt}; @@ -60,6 +60,10 @@ struct FakeTrtllm { empty_model_info: Arc, /// Answers `UNAVAILABLE`, the shape of an engine that is not serving yet. unavailable_model_info: Arc, + /// Portable coarse-grained model capability. + supports_multimodal: Arc, + /// TensorRT-LLM's finer-grained modality advertisement. + advertises_image: Arc, model_info_calls: Arc, } @@ -109,7 +113,10 @@ impl pb::inference_server::Inference for FakeTrtllm { } let request_id = request.request_id.clone(); - let prompt_tokens = prompt_len(&request); + // Image markers expand inside TensorRT-LLM after Dynamo tokenizes the + // text prompt. Simulate that so the integration test proves Dynamo + // trusts the engine's terminal usage rather than the unexpanded input. + let prompt_tokens = prompt_len(&request) + if request.media.is_empty() { 0 } else { 64 }; let wants_logprobs = wants_logprobs(&request); let context_only = is_context_only(&request); let hang = self.hang.load(Ordering::SeqCst); @@ -249,6 +256,14 @@ impl pb::control_server::Control for FakeTrtllm { if self.unavailable_model_info.load(Ordering::SeqCst) { return Err(Status::unavailable("not serving yet")); } + let supports_multimodal = self.supports_multimodal.load(Ordering::SeqCst); + let modalities = if self.advertises_image.load(Ordering::SeqCst) { + vec![prost_types::Value { + kind: Some(prost_types::value::Kind::StringValue("image".to_string())), + }] + } else { + Vec::new() + }; Ok(Response::new(pb::ModelInfo { model_id: "fake-model".to_string(), max_context_length: if self.empty_model_info.load(Ordering::SeqCst) { @@ -256,6 +271,19 @@ impl pb::control_server::Control for FakeTrtllm { } else { Some(4096) }, + supports_multimodal: Some(supports_multimodal), + extra: Some(prost_types::Struct { + fields: [( + "trtllm_supported_modalities".to_string(), + prost_types::Value { + kind: Some(prost_types::value::Kind::ListValue( + prost_types::ListValue { values: modalities }, + )), + }, + )] + .into_iter() + .collect(), + }), ..Default::default() })) } @@ -441,9 +469,28 @@ fn limits(context_length: u32) -> Option { Some(ModelLimits { context_length: Some(context_length), max_output_tokens: None, + supports_image: false, + }) +} + +fn image_limits(context_length: u32) -> Option { + Some(ModelLimits { + context_length: Some(context_length), + max_output_tokens: None, + supports_image: true, }) } +fn add_images(request: &mut PreprocessedRequest, images: impl IntoIterator) { + request.multi_modal_data = Some(HashMap::from([( + "image_url".to_string(), + images + .into_iter() + .map(|image| MultimodalData::RawUrl(image.to_string())) + .collect(), + )])); +} + fn engine(endpoint: &str, connections: usize) -> TrtllmSidecarEngine { engine_in_mode(endpoint, connections, AGG) } diff --git a/lib/sidecar/trtllm/src/tests/convert_request.rs b/lib/sidecar/trtllm/src/tests/convert_request.rs index e8dd43d172df..138057cb5dc8 100644 --- a/lib/sidecar/trtllm/src/tests/convert_request.rs +++ b/lib/sidecar/trtllm/src/tests/convert_request.rs @@ -72,6 +72,183 @@ fn request_maps_sampling_stop_and_output_fields() { } } +#[test] +fn image_urls_data_uris_and_uuids_are_mapped_in_order() { + let mut req = request(); + // Routing-only multimodal metadata has already served its purpose by the + // time the request reaches a worker and must not block engine delivery. + req.mm_routing_info = Some(Default::default()); + add_images( + &mut req, + [ + "https://example.com/first.png", + "data:image/png;base64,aW1hZ2U=", + ], + ); + req.multi_modal_uuids = Some(HashMap::from([( + "image_url".to_string(), + vec![Some("first-image".to_string()), None], + )])); + req.extra_args = Some(json!({ + "mm_hashes": ["routing-hash-one", "routing-hash-two"] + })); + + let proto = build_generate_request(&req, "req", "model", image_limits(4096), AGG) + .expect("image request"); + assert_eq!(proto.media.len(), 2); + assert_eq!(proto.media[0].modality, pb::Modality::Image as i32); + assert_eq!( + proto.media[0].source, + Some(pb::media_item::Source::Url( + "https://example.com/first.png".to_string() + )) + ); + assert_eq!(proto.media[0].uuid, "first-image"); + assert_eq!( + proto.media[1].source, + Some(pb::media_item::Source::DataUri( + "data:image/png;base64,aW1hZ2U=".to_string() + )) + ); + assert!(proto.media[1].uuid.is_empty()); +} + +#[test] +fn mm_routing_hashes_are_forwarded_as_image_uuids_in_order() { + let mut req = request(); + add_images( + &mut req, + [ + "https://example.com/first.png", + "https://example.com/second.png", + ], + ); + req.extra_args = Some(json!({ + "mm_hashes": ["0123456789abcdef", "fedcba9876543210"] + })); + + let proto = build_generate_request(&req, "req", "model", image_limits(4096), AGG) + .expect("image request"); + assert_eq!(proto.media[0].uuid, "0123456789abcdef"); + assert_eq!(proto.media[1].uuid, "fedcba9876543210"); +} + +#[test] +fn malformed_or_misaligned_mm_routing_hashes_are_rejected() { + for (hashes, expected) in [ + (json!("not-an-array"), "array of strings"), + (json!(["only-one"]), "must match"), + (json!(["one", ""]), "non-empty string"), + (json!(["one", 2]), "non-empty string"), + ] { + let mut req = request(); + add_images( + &mut req, + [ + "https://example.com/first.png", + "https://example.com/second.png", + ], + ); + req.extra_args = Some(json!({"mm_hashes": hashes})); + let error = build_generate_request(&req, "req", "model", image_limits(4096), AGG) + .expect_err("malformed routing hashes must fail closed"); + assert!( + error.to_string().contains(expected), + "unexpected error: {error}" + ); + } +} + +#[test] +fn image_media_requires_the_loaded_models_advertised_capability() { + let mut req = request(); + add_images(&mut req, ["https://example.com/image.png"]); + + for model_limits in [None, limits(4096)] { + let error = build_generate_request(&req, "req", "model", model_limits, AGG) + .expect_err("a text-only or unknown model must reject images"); + assert!( + error + .to_string() + .contains("does not advertise image support"), + "unexpected error: {error}" + ); + } +} + +#[test] +fn unsupported_media_modalities_and_sources_are_rejected() { + for (key, expected) in [ + ("video_url", "video_url"), + ("audio_url", "audio_url"), + ("future_media", "future_media"), + ] { + let mut req = request(); + req.multi_modal_data = Some(HashMap::from([( + key.to_string(), + vec![MultimodalData::RawUrl( + "https://example.com/media".to_string(), + )], + )])); + let error = build_generate_request(&req, "req", "model", image_limits(4096), AGG) + .expect_err("only images are supported"); + assert!( + error.to_string().contains(expected), + "unexpected error: {error}" + ); + } + + let mut req = request(); + add_images(&mut req, ["file:///tmp/image.png"]); + let error = build_generate_request(&req, "req", "model", image_limits(4096), AGG) + .expect_err("the engine cannot fetch a local Dynamo path"); + assert!(error.to_string().contains("http, https, or data URI")); + + let mut req = request(); + req.multi_modal_data = Some(HashMap::from([( + "image_url".to_string(), + vec![MultimodalData::UuidOnly("cached-image".to_string())], + )])); + let error = build_generate_request(&req, "req", "model", image_limits(4096), AGG) + .expect_err("OpenEngine cannot resolve a source-less UUID"); + assert!(error.to_string().contains("requires an image source")); +} + +#[test] +fn image_metadata_that_cannot_be_honored_is_rejected() { + let mut req = request(); + add_images(&mut req, ["https://example.com/image.png"]); + req.multi_modal_uuids = Some(HashMap::from([( + "image_url".to_string(), + vec![Some("one".to_string()), Some("two".to_string())], + )])); + let error = build_generate_request(&req, "req", "model", image_limits(4096), AGG) + .expect_err("UUIDs must align positionally"); + assert!(error.to_string().contains("UUID count")); + + req.multi_modal_uuids = None; + req.mm_processor_kwargs = Some(serde_json::json!({"do_resize": false})); + let error = build_generate_request(&req, "req", "model", image_limits(4096), AGG) + .expect_err("processor overrides cannot be dropped"); + assert!(error.to_string().contains("mm_processor_kwargs")); + + req.mm_processor_kwargs = Some(serde_json::json!({})); + req.media_io_kwargs = Some(serde_json::json!({"image": {"mode": "RGB"}})); + let error = build_generate_request(&req, "req", "model", image_limits(4096), AGG) + .expect_err("media I/O overrides cannot be dropped"); + assert!(error.to_string().contains("media_io_kwargs")); +} + +#[test] +fn image_request_without_max_tokens_is_left_for_engine_derivation() { + let mut req = request(); + add_images(&mut req, ["https://example.com/image.png"]); + req.stop_conditions.max_tokens = None; + let proto = build_generate_request(&req, "req", "model", image_limits(4096), AGG) + .expect("the engine derives against the expanded prompt"); + assert_eq!(proto.stopping.expect("stopping").max_tokens, None); +} + #[test] fn omitted_max_tokens_without_context_length_is_rejected() { let mut req = request(); @@ -191,6 +368,7 @@ fn the_derived_max_tokens_respects_the_engines_output_cap() { Some(ModelLimits { context_length: Some(4096), max_output_tokens: Some(256), + supports_image: false, }), AGG, ) diff --git a/lib/sidecar/trtllm/src/tests/disagg.rs b/lib/sidecar/trtllm/src/tests/disagg.rs index 3436c037d346..612fda3f39fc 100644 --- a/lib/sidecar/trtllm/src/tests/disagg.rs +++ b/lib/sidecar/trtllm/src/tests/disagg.rs @@ -59,6 +59,59 @@ fn prefill_request_is_marked_context_only() { assert!(proto.kv.is_none(), "prefill carries no session to replay"); } +/// Image inputs must survive both RPC legs. TensorRT-LLM reconstructs its +/// multimodal position/hash state while resuming the KV session, so the decode +/// request needs the same ordered media metadata as the context request. +#[test] +fn image_media_is_preserved_across_prefill_and_decode_requests() { + let mut req = request(); + add_images( + &mut req, + [ + "https://example.com/first.png", + "data:image/png;base64,c2Vjb25k", + ], + ); + req.extra_args = Some(json!({ + "mm_hashes": ["0123456789abcdef", "fedcba9876543210"] + })); + + let prefill = build_generate_request( + &req, + "req", + "model", + image_limits(4096), + DisaggregationMode::Prefill, + ) + .expect("build image prefill"); + assert!(is_context_only(&prefill)); + assert_eq!(prefill.stopping.as_ref().unwrap().max_tokens, Some(1)); + assert_eq!(prefill.media.len(), 2); + assert_eq!(prefill.media[0].uuid, "0123456789abcdef"); + assert_eq!(prefill.media[1].uuid, "fedcba9876543210"); + + req.prefill_result = Some(dynamo_backend_common::PrefillResult { + disaggregated_params: crate::disagg::session_to_json(fake_session()) + .expect("encode handoff"), + prompt_tokens_details: None, + }); + let decode = build_generate_request( + &req, + "req", + "model", + image_limits(4096), + DisaggregationMode::Decode, + ) + .expect("build image decode"); + assert!(!is_context_only(&decode)); + assert_eq!(decode.media, prefill.media); + assert_eq!( + decode.kv.and_then(|kv| kv.session), + Some(fake_session()), + "decode must carry both the image metadata and the KV handoff" + ); +} + /// `PrefillReady` is the prefill worker's terminal chunk: no tokens, and the /// handoff the decode worker will replay. #[tokio::test] diff --git a/lib/sidecar/trtllm/src/tests/e2e.rs b/lib/sidecar/trtllm/src/tests/e2e.rs index 9b5f857cd1df..3f97fe73b7f4 100644 --- a/lib/sidecar/trtllm/src/tests/e2e.rs +++ b/lib/sidecar/trtllm/src/tests/e2e.rs @@ -111,6 +111,107 @@ async fn e2e_real_disagg_handoff() { ); } +/// Exercises the same real prefill/decode handoff with an image request. Both +/// servers may share one sufficiently large GPU for local validation. +/// +/// ```text +/// TRTLLM_E2E_PREFILL_ENDPOINT=http://127.0.0.1:50051 \ +/// TRTLLM_E2E_DECODE_ENDPOINT=http://127.0.0.1:50052 \ +/// TRTLLM_E2E_MODEL=Qwen/Qwen3-VL-2B-Instruct \ +/// cargo test -p dynamo-trtllm-sidecar e2e_real_openengine_image_disagg -- --ignored --nocapture +/// ``` +#[tokio::test] +#[ignore = "requires two live image-capable OpenEngine servers with a KV transceiver"] +async fn e2e_real_openengine_image_disagg() { + let prefill_endpoint = std::env::var("TRTLLM_E2E_PREFILL_ENDPOINT") + .expect("set TRTLLM_E2E_PREFILL_ENDPOINT, e.g. http://127.0.0.1:50051"); + let decode_endpoint = std::env::var("TRTLLM_E2E_DECODE_ENDPOINT") + .expect("set TRTLLM_E2E_DECODE_ENDPOINT, e.g. http://127.0.0.1:50052"); + let model = std::env::var("TRTLLM_E2E_MODEL") + .unwrap_or_else(|_| "Qwen/Qwen3-VL-2B-Instruct".to_string()); + let configured = |source: String| ConfiguredModel { + source, + context_length: None, + }; + let prefill = TrtllmSidecarEngine::new( + GrpcEndpoint::parse(&prefill_endpoint, "--grpc-endpoint").expect("valid endpoint"), + transport(1), + configured(model.clone()), + DisaggregationMode::Prefill, + ); + let decode = TrtllmSidecarEngine::new( + GrpcEndpoint::parse(&decode_endpoint, "--grpc-endpoint").expect("valid endpoint"), + transport(1), + configured(model.clone()), + DisaggregationMode::Decode, + ); + prefill.start(0).await.expect("start image prefill worker"); + decode.start(0).await.expect("start image decode worker"); + + let token_ids = vec![ + 151644, 872, 198, 151652, 151655, 151653, 675, 279, 7987, 504, 2115, 311, 1290, 13, 151645, + 198, 151644, 77091, 198, + ]; + let image_request = || { + let mut request = PreprocessedRequest::builder() + .model(model.clone()) + .token_ids(token_ids.clone()) + .stop_conditions(StopConditions::default()) + .sampling_options(SamplingOptions { + temperature: Some(0.0), + ..Default::default() + }) + .output_options(OutputOptions::default()) + .build() + .expect("request"); + add_images( + &mut request, + [concat!( + "data:image/png;base64,", + "iVBORw0KGgoAAAANSUhEUgAAABwAAAAcCAIAAAD9b0jDAAAAK0lEQVR4nGP8z0AsYGQgVi0T0WaSAEYNHTV01NBRQ0cNHTV01NBRQwefoQBvcAI3xbX7+QAAAABJRU5ErkJggg==" + )], + ); + request + }; + + let prefill_outputs = collect(&prefill, image_request()).await; + assert!( + prefill_outputs + .iter() + .all(|output| output.token_ids.is_empty()), + "the image prefill worker must not stream tokens" + ); + let handoff = prefill_outputs + .iter() + .find_map(|output| output.disaggregated_params.clone()) + .expect("image prefill terminal carries the KV handoff"); + + let mut decode_request = image_request(); + decode_request.prefill_result = Some(dynamo_backend_common::PrefillResult { + disaggregated_params: handoff, + prompt_tokens_details: None, + }); + let decode_outputs = collect(&decode, decode_request).await; + let generated = decode_outputs + .iter() + .flat_map(|output| output.token_ids.iter().copied()) + .collect::>(); + let terminal = decode_outputs + .iter() + .find(|output| output.finish_reason.is_some()) + .expect("image decode terminal"); + let usage = terminal.completion_usage.as_ref().expect("terminal usage"); + + assert!( + !generated.is_empty(), + "the image decode worker must generate" + ); + assert!( + usage.prompt_tokens > token_ids.len() as u32, + "the disaggregated image prompt must report its expanded token count" + ); +} + /// Drives the real `TrtllmSidecarEngine` against a live OpenEngine gRPC server /// (a real TensorRT-LLM engine on a GPU). Ignored by default; run explicitly with /// a reachable endpoint: @@ -202,3 +303,82 @@ async fn e2e_real_openengine_server() { "logprobs were requested but none were surfaced" ); } + +/// Drives an image request through the real Rust sidecar conversion layer and +/// a live TensorRT-LLM OpenEngine server. The fixed token IDs are Qwen3-VL's +/// chat-template rendering of one image marker and a short question; the tiny +/// PNG is split red/blue so a manual run also produces an obvious answer. +/// +/// ```text +/// TRTLLM_E2E_ENDPOINT=http://127.0.0.1:50051 \ +/// TRTLLM_E2E_MODEL=Qwen/Qwen3-VL-2B-Instruct \ +/// cargo test -p dynamo-trtllm-sidecar e2e_real_openengine_image -- --ignored --nocapture +/// ``` +#[tokio::test] +#[ignore = "requires a live image-capable TensorRT-LLM OpenEngine server"] +async fn e2e_real_openengine_image() { + let endpoint = std::env::var("TRTLLM_E2E_ENDPOINT") + .expect("set TRTLLM_E2E_ENDPOINT, e.g. http://127.0.0.1:50051"); + let model = std::env::var("TRTLLM_E2E_MODEL") + .unwrap_or_else(|_| "Qwen/Qwen3-VL-2B-Instruct".to_string()); + + let engine = TrtllmSidecarEngine::new( + GrpcEndpoint::parse(&endpoint, "--grpc-endpoint").expect("valid endpoint"), + transport(1), + ConfiguredModel { + source: model.clone(), + context_length: None, + }, + AGG, + ); + engine + .start(0) + .await + .expect("start against the live image-capable server"); + + let token_ids = vec![ + 151644, 872, 198, 151652, 151655, 151653, 675, 279, 7987, 504, 2115, 311, 1290, 13, 151645, + 198, 151644, 77091, 198, + ]; + let mut request = PreprocessedRequest::builder() + .model(model) + .token_ids(token_ids.clone()) + // Leave max_tokens omitted: only TensorRT-LLM can derive the output + // budget after expanding the image placeholder. + .stop_conditions(StopConditions::default()) + .sampling_options(SamplingOptions { + temperature: Some(0.0), + ..Default::default() + }) + .output_options(OutputOptions::default()) + .build() + .expect("request"); + add_images( + &mut request, + [concat!( + "data:image/png;base64,", + "iVBORw0KGgoAAAANSUhEUgAAABwAAAAcCAIAAAD9b0jDAAAAK0lEQVR4nGP8z0AsYGQgVi0T0WaSAEYNHTV01NBRQ0cNHTV01NBRQwefoQBvcAI3xbX7+QAAAABJRU5ErkJggg==" + )], + ); + + let outputs = collect(&engine, request).await; + let generated = outputs + .iter() + .flat_map(|output| output.token_ids.iter().copied()) + .collect::>(); + let terminal = outputs + .iter() + .find(|output| output.finish_reason.is_some()) + .expect("terminal output"); + let usage = terminal.completion_usage.as_ref().expect("terminal usage"); + eprintln!( + "[e2e-image] generated={generated:?}; usage: prompt={}, completion={}", + usage.prompt_tokens, usage.completion_tokens + ); + + assert!(!generated.is_empty(), "expected image-conditioned output"); + assert!( + usage.prompt_tokens > token_ids.len() as u32, + "the engine's expanded image prompt usage must replace Dynamo's unexpanded token count" + ); +} diff --git a/lib/sidecar/trtllm/src/tests/engine.rs b/lib/sidecar/trtllm/src/tests/engine.rs index 415456ede063..149303a06d3b 100644 --- a/lib/sidecar/trtllm/src/tests/engine.rs +++ b/lib/sidecar/trtllm/src/tests/engine.rs @@ -41,6 +41,64 @@ async fn aggregated_generation_streams_delta_then_terminal() { } } +#[tokio::test] +async fn advertised_image_media_reaches_openengine_and_uses_expanded_usage() { + let service = FakeTrtllm::default(); + service.supports_multimodal.store(true, Ordering::SeqCst); + service.advertises_image.store(true, Ordering::SeqCst); + let server = FakeServer::start(service).await; + let engine = engine(&server.endpoint, 1); + engine.start(0).await.expect("start"); + + let mut req = request(); + req.stop_conditions.max_tokens = None; + add_images(&mut req, ["data:image/png;base64,aW1hZ2U="]); + let outputs = collect(&engine, req).await; + let usage = outputs + .last() + .and_then(|output| output.completion_usage.as_ref()) + .expect("terminal engine usage"); + assert_eq!(usage.prompt_tokens, 67, "3 text tokens + 64 image tokens"); + + let requests = server.service.requests.lock().await; + let sent = requests.first().expect("recorded request"); + assert_eq!( + sent.stopping.as_ref().expect("stopping").max_tokens, + None, + "the expanded image prompt budget belongs to TensorRT-LLM" + ); + assert_eq!(sent.media.len(), 1); + assert_eq!(sent.media[0].modality, pb::Modality::Image as i32); + assert!(matches!( + sent.media[0].source.as_ref(), + Some(pb::media_item::Source::DataUri(value)) if value.starts_with("data:image/png") + )); +} + +#[tokio::test] +async fn both_multimodal_capability_signals_are_required() { + for (coarse, image) in [(true, false), (false, true)] { + let service = FakeTrtllm::default(); + service.supports_multimodal.store(coarse, Ordering::SeqCst); + service.advertises_image.store(image, Ordering::SeqCst); + let server = FakeServer::start(service).await; + let engine = engine(&server.endpoint, 1); + engine.start(0).await.expect("start"); + + let mut req = request(); + add_images(&mut req, ["https://example.com/image.png"]); + let context = dynamo_backend_common::testing::mock_context(); + let result = engine + .generate(req, GenerateContext::new(context, None)) + .await; + assert!( + result.is_err(), + "capabilities ({coarse}, {image}) must fail closed" + ); + assert!(server.service.requests.lock().await.is_empty()); + } +} + #[tokio::test] async fn grpc_request_errors_are_propagated() { let service = FakeTrtllm::default(); From 8705d4c38abbc321f8b21204413571772f63ab1d Mon Sep 17 00:00:00 2001 From: krishung5 Date: Thu, 15 Oct 2026 16:55:16 -0700 Subject: [PATCH 15/61] fix(trtllm): narrow image media cache identity Signed-off-by: krishung5 --- lib/llm/src/backend.rs | 206 +----------------- lib/sidecar/trtllm/src/client.rs | 14 +- lib/sidecar/trtllm/src/convert.rs | 86 ++------ lib/sidecar/trtllm/src/engine.rs | 8 +- lib/sidecar/trtllm/src/tests.rs | 10 +- .../trtllm/src/tests/convert_request.rs | 37 +--- lib/sidecar/trtllm/src/tests/disagg.rs | 14 +- 7 files changed, 51 insertions(+), 324 deletions(-) diff --git a/lib/llm/src/backend.rs b/lib/llm/src/backend.rs index 4c5a2d80e0d7..3a73e46a21b0 100644 --- a/lib/llm/src/backend.rs +++ b/lib/llm/src/backend.rs @@ -261,8 +261,8 @@ impl return state.pending_flush.pop().map(|output| (output, state)); } - let (output, trailing_usage) = loop { - let mut output = state.stream.next().await; + let output = loop { + let output = state.stream.next().await; let choice_already_finished = output.as_ref().is_some_and(|output| { !output.is_event() && output.data.as_ref().is_some_and(|data| { @@ -272,50 +272,7 @@ impl }) }); if !choice_already_finished { - break (output, false); - } - - let carries_usage = output - .as_ref() - .and_then(|output| output.data.as_ref()) - .is_some_and(|data| data.completion_usage.is_some()); - let backend_failed = output - .as_ref() - .and_then(|output| output.data.as_ref()) - .is_some_and(|data| matches!(data.finish_reason, Some(FinishReason::Error(_)))); - if carries_usage || backend_failed { - // Native backends can put authoritative cumulative usage - // on a terminal event immediately after the token Dynamo - // recognized locally as EOS. Preserve that metadata without - // emitting duplicate content or a second successful finish - // event. Preserve a backend error with or without usage: it - // is not duplicate success metadata and must reach the - // frontend. - if let Some(data) = output.as_mut().and_then(|output| output.data.as_mut()) { - data.token_ids.clear(); - data.tokens = None; - data.text = None; - data.cum_log_probs = None; - data.log_probs = None; - data.top_logprobs = None; - if !backend_failed { - data.finish_reason = None; - data.stop_reason = None; - } - } - break (output, true); - } - - let is_terminal = output - .as_ref() - .and_then(|output| output.data.as_ref()) - .is_some_and(|data| data.finish_reason.is_some()); - if !is_terminal && state.finished_choices.len() >= state.decoders.len() { - // Once every choice ended locally, tolerate only terminal - // metadata. A nonterminal suffix means the backend ignored - // the stop; cancel rather than burning the remaining budget. - state.stream.context().stop_generating(); - return None; + break output; } }; @@ -324,11 +281,6 @@ impl // move to state.process_output // handle any error conditions / unwraps here - if trailing_usage { - state.finished = state.finished_choices.len() >= state.decoders.len(); - return Some((output, state)); - } - // events are pass thru if output.is_event() || output.data.is_none() { // A genuine stream-level error ends generation for every choice at @@ -545,12 +497,13 @@ impl .unwrap_or((None, None)); // If we detected a local stop condition, mark this choice as finished. - // Keep the upstream stream alive just long enough to consume - // a native backend's following terminal-usage event. The - // loop above still cancels on any nonterminal suffix once - // every choice has stopped. + // Once all expected choices are finished, stop the upstream generator. if finish_reason.is_some() && data.finish_reason.is_none() { state.finished_choices.insert(choice_idx); + if state.finished_choices.len() >= decoders_count { + state.stream.context().stop_generating(); + state.finished = true; + } } let text = result.text; @@ -1968,149 +1921,6 @@ mod tests { assert_eq!(generator.get_usage().completion_tokens, 2); } - #[tokio::test] - async fn local_eos_preserves_a_following_authoritative_usage_trailer() { - let tokenizer: Arc = Arc::new(CandidateDecoder); - let backend = Backend::from_tokenizer(Tokenizer::from(tokenizer)); - let request = PreprocessedRequest::builder() - .model("test-model".to_string()) - .token_ids(vec![]) - .stop_conditions(StopConditions { - stop_token_ids_hidden: Some(vec![103]), - ..Default::default() - }) - .sampling_options(SamplingOptions::default()) - .output_options(OutputOptions::default()) - .build() - .expect("valid preprocessed request"); - let reported = dynamo_protocols::types::CompletionUsage { - prompt_tokens: 82, - completion_tokens: 2, - total_tokens: 84, - prompt_tokens_details: None, - completion_tokens_details: None, - }; - let engine: ServerStreamingEngine> = - Arc::new(SyntheticSglangStopEngine { - outputs: Some(vec![ - // Dynamo recognizes 103 as EOS before the native backend's - // separate terminal event arrives. - LLMEngineOutput { - token_ids: vec![101, 103], - index: Some(0), - ..Default::default() - }, - LLMEngineOutput { - index: Some(0), - finish_reason: Some(FinishReason::Stop), - completion_usage: Some(reported.clone()), - ..Default::default() - }, - ]), - }); - - let outputs = Operator::generate(backend.as_ref(), SingleIn::new(request), engine) - .await - .expect("backend generation succeeds") - .collect::>() - .await; - assert_eq!(outputs.len(), 2); - - let local_terminal = outputs[0].data.as_ref().expect("local terminal"); - assert_eq!(local_terminal.finish_reason, Some(FinishReason::Stop)); - let usage_trailer = outputs[1].data.as_ref().expect("usage trailer"); - assert!(usage_trailer.token_ids.is_empty()); - assert!(usage_trailer.finish_reason.is_none()); - assert_eq!(usage_trailer.completion_usage.as_ref(), Some(&reported)); - - let options = DeltaGeneratorOptions::new(None, None, false, None); - let mut generator = DeltaGenerator::new( - "test-model".to_string(), - options, - "test-request".to_string(), - ); - for output in outputs { - generator - .choice_from_postprocessor(output.data.expect("backend output")) - .expect("OpenAI response conversion succeeds"); - } - assert_eq!(generator.get_usage(), reported); - } - - #[tokio::test] - async fn local_eos_does_not_mask_a_trailing_backend_error() { - let tokenizer: Arc = Arc::new(CandidateDecoder); - let backend = Backend::from_tokenizer(Tokenizer::from(tokenizer)); - let request = PreprocessedRequest::builder() - .model("test-model".to_string()) - .token_ids(vec![]) - .stop_conditions(StopConditions { - stop_token_ids_hidden: Some(vec![103]), - ..Default::default() - }) - .sampling_options(SamplingOptions::default()) - .output_options(OutputOptions::default()) - .build() - .expect("valid preprocessed request"); - let reported = dynamo_protocols::types::CompletionUsage { - prompt_tokens: 82, - completion_tokens: 2, - total_tokens: 84, - prompt_tokens_details: None, - completion_tokens_details: None, - }; - for completion_usage in [None, Some(reported.clone())] { - let engine: ServerStreamingEngine> = - Arc::new(SyntheticSglangStopEngine { - outputs: Some(vec![ - LLMEngineOutput { - token_ids: vec![101, 103], - index: Some(0), - ..Default::default() - }, - LLMEngineOutput { - index: Some(0), - finish_reason: Some(FinishReason::Error("worker failed".to_string())), - completion_usage: completion_usage.clone(), - ..Default::default() - }, - ]), - }); - - let outputs = - Operator::generate(backend.as_ref(), SingleIn::new(request.clone()), engine) - .await - .expect("backend generation succeeds") - .collect::>() - .await; - assert_eq!(outputs.len(), 2); - let error_trailer = outputs[1].data.as_ref().expect("error trailer"); - assert!(error_trailer.token_ids.is_empty()); - assert_eq!( - error_trailer.finish_reason, - Some(FinishReason::Error("worker failed".to_string())) - ); - assert_eq!( - error_trailer.completion_usage.as_ref(), - completion_usage.as_ref() - ); - - let options = DeltaGeneratorOptions::new(None, None, false, None); - let mut generator = DeltaGenerator::new( - "test-model".to_string(), - options, - "test-request".to_string(), - ); - generator - .choice_from_postprocessor(outputs[0].data.clone().expect("local terminal")) - .expect("the local terminal converts"); - let error = generator - .choice_from_postprocessor(outputs[1].data.clone().expect("error trailer")) - .expect_err("the backend error must reach the OpenAI response layer"); - assert!(error.to_string().contains("worker failed")); - } - } - #[tokio::test] async fn test_no_stop_trim_raw_completion_output() { use crate::protocols::common::{ diff --git a/lib/sidecar/trtllm/src/client.rs b/lib/sidecar/trtllm/src/client.rs index 16cea9315dfd..576bb2a48098 100644 --- a/lib/sidecar/trtllm/src/client.rs +++ b/lib/sidecar/trtllm/src/client.rs @@ -27,10 +27,10 @@ use crate::proto::inference_client::InferenceClient; /// `--grpc-startup-deadline-secs` instead. const RPC_TIMEOUT: Duration = Duration::from_secs(30); -/// The engine's limits, resolved at startup from `--context-length` and -/// `Control.GetModelInfo`. +/// Metadata resolved at startup from `--context-length` and +/// `Control.GetModelInfo`, including limits and media capabilities. #[derive(Clone, Copy, Debug, Default)] -pub(crate) struct ModelLimits { +pub(crate) struct ModelMetadata { /// Maximum input + output tokens; backs both the registered context window /// and the default `max_tokens`. Absent when neither source supplied one, /// which leaves requests that omit `max_tokens` to be rejected. @@ -99,7 +99,7 @@ impl TrtllmClient { /// unset rather than substituting its `max_input_len` default (measured /// against TensorRT-LLM main at 8bbaf66bd5), so `None` here means "the /// server did not say", and `--context-length` supplies it instead. - async fn get_model_info(&self, model: &str) -> Result { + async fn get_model_info(&self, model: &str) -> Result { let info = self .control() .get_model_info(pb::GetModelInfoRequest { @@ -126,7 +126,7 @@ impl TrtllmClient { _ => false, }) }); - Ok(ModelLimits { + Ok(ModelMetadata { context_length: info.max_context_length.filter(|len| *len > 0), max_output_tokens: info.max_output_tokens.filter(|cap| *cap > 0), supports_image, @@ -136,7 +136,7 @@ impl TrtllmClient { /// One `GetModelInfo` call, for when `--context-length` already supplies the /// window and the engine is consulted only to cross-check it and to learn /// its output cap. - pub(crate) async fn model_limits(&self, model: &str) -> Result { + pub(crate) async fn model_limits(&self, model: &str) -> Result { tokio::time::timeout(RPC_TIMEOUT, self.get_model_info(model)) .await .map_err(|_| { @@ -164,7 +164,7 @@ impl TrtllmClient { model: &str, deadline: Instant, retry_interval: Duration, - ) -> Result { + ) -> Result { let mut last = "it never answered".to_string(); loop { match timeout_at(deadline, self.get_model_info(model)).await { diff --git a/lib/sidecar/trtllm/src/convert.rs b/lib/sidecar/trtllm/src/convert.rs index 0c3e3d59d2f3..0a498a97aff0 100644 --- a/lib/sidecar/trtllm/src/convert.rs +++ b/lib/sidecar/trtllm/src/convert.rs @@ -21,7 +21,7 @@ use dynamo_backend_common::{ MultimodalData, PreprocessedRequest, PromptTokensDetails, StopReason, TopLogprob, usage, }; -use crate::client::{self, ModelLimits}; +use crate::client::{self, ModelMetadata}; use crate::disagg; use crate::proto as pb; @@ -33,7 +33,7 @@ pub(crate) fn build_generate_request( request: &PreprocessedRequest, request_id: &str, model: &str, - limits: Option, + limits: Option, mode: DisaggregationMode, ) -> Result { validate_request(request, mode)?; @@ -162,7 +162,7 @@ pub(crate) fn build_generate_request( // not govern. fn max_tokens( request: &PreprocessedRequest, - limits: Option, + limits: Option, ) -> Result, DynamoError> { if let Some(max_tokens) = request.stop_conditions.max_tokens { return Ok(Some(max_tokens)); @@ -194,7 +194,6 @@ fn max_tokens( } const IMAGE_KEY: &str = "image_url"; -const MM_HASHES_KEY: &str = "mm_hashes"; /// Convert Dynamo's image-only execution payload into the ordered OpenEngine /// media list. The frontend groups content parts by modality, so ordering is @@ -203,7 +202,7 @@ const MM_HASHES_KEY: &str = "mm_hashes"; /// non-empty modality instead of inventing a cross-modality order. fn image_media( request: &PreprocessedRequest, - limits: Option, + limits: Option, ) -> Result, DynamoError> { reject_media_override("mm_processor_kwargs", request.mm_processor_kwargs.as_ref())?; reject_media_override("media_io_kwargs", request.media_io_kwargs.as_ref())?; @@ -256,21 +255,6 @@ fn image_media( )); } - // Explicit caller UUIDs remain authoritative. Otherwise carry the exact - // hashes generated by Dynamo's MM-aware preprocessor to the backend. This - // is request-side preparation for KV-aware routing; the router-owned - // `mm_routing_info` itself never crosses the worker boundary. - let has_user_uuid = user_image_uuids.is_some_and(|uuids| { - uuids - .iter() - .any(|uuid| uuid.as_deref().is_some_and(|uuid| !uuid.is_empty())) - }); - let routing_hashes = if has_user_uuid { - None - } else { - routing_hashes(request, images.len())? - }; - images .iter() .enumerate() @@ -292,63 +276,23 @@ fn image_media( Ok(pb::MediaItem { modality: pb::Modality::Image as i32, source: Some(source), + // Dynamo's frontend representation carries URL/data-URI + // sources, not raw bytes. MIME type is only meaningful for the + // OpenEngine raw_bytes source used by direct clients. mime_type: String::new(), - uuid: if has_user_uuid { - user_image_uuids - .and_then(|uuids| uuids[index].as_deref()) - .unwrap_or_default() - .to_string() - } else { - routing_hashes - .as_ref() - .map(|hashes| hashes[index].clone()) - .unwrap_or_default() - }, + // UUIDs are caller-owned cache identities and are optional per + // item. Dynamo's router hashes are deliberately not substituted + // here: their compatibility with TensorRT-LLM cache identity is + // owned and validated by the MM-aware routing work. + uuid: user_image_uuids + .and_then(|uuids| uuids[index].as_deref()) + .unwrap_or_default() + .to_string(), }) }) .collect() } -fn routing_hashes( - request: &PreprocessedRequest, - image_count: usize, -) -> Result>, DynamoError> { - let hashes = match request.extra_args.as_ref() { - Some(serde_json::Value::Object(extra)) => extra.get(MM_HASHES_KEY), - _ => None, - }; - let Some(hashes) = hashes else { - return Ok(None); - }; - let hashes = hashes.as_array().ok_or_else(|| { - client::invalid_argument("extra_args.mm_hashes must be an array of strings") - })?; - if hashes.is_empty() { - return Ok(None); - } - if hashes.len() != image_count { - return Err(client::invalid_argument(format!( - "extra_args.mm_hashes count ({}) must match the image media item count ({image_count})", - hashes.len() - ))); - } - hashes - .iter() - .enumerate() - .map(|(index, hash)| { - hash.as_str() - .filter(|hash| !hash.is_empty()) - .map(ToOwned::to_owned) - .ok_or_else(|| { - client::invalid_argument(format!( - "extra_args.mm_hashes[{index}] must be a non-empty string" - )) - }) - }) - .collect::, _>>() - .map(Some) -} - fn media_source(value: &str, index: usize) -> Result { let scheme = value .split_once(':') diff --git a/lib/sidecar/trtllm/src/engine.rs b/lib/sidecar/trtllm/src/engine.rs index 66327bfb2d3f..2671ba619bbc 100644 --- a/lib/sidecar/trtllm/src/engine.rs +++ b/lib/sidecar/trtllm/src/engine.rs @@ -18,7 +18,7 @@ use tokio::sync::OnceCell; use tokio_util::sync::CancellationToken; use crate::args::Args; -use crate::client::{self, ModelLimits, TrtllmClient}; +use crate::client::{self, ModelMetadata, TrtllmClient}; use crate::convert::{ResponseState, build_generate_request}; use crate::model::ConfiguredModel; @@ -41,7 +41,7 @@ pub struct TrtllmSidecarEngine { /// Engine limits resolved at `start` from `--context-length` and /// `Control.GetModelInfo`, so `generate` can derive a default `max_tokens` /// for requests that omit one. - limits: OnceCell, + limits: OnceCell, cancel: CancellationToken, } @@ -220,7 +220,7 @@ impl LLMEngine for TrtllmSidecarEngine { ships no Control service or was started without \ --grpc-protocol openengine" ); - ModelLimits::default() + ModelMetadata::default() } }; if let Some(engine_context_length) = reported.context_length @@ -233,7 +233,7 @@ impl LLMEngine for TrtllmSidecarEngine { reported; using the configured --context-length" ); } - ModelLimits { + ModelMetadata { context_length: Some(configured), ..reported } diff --git a/lib/sidecar/trtllm/src/tests.rs b/lib/sidecar/trtllm/src/tests.rs index d6d3c3146762..0bcda724b744 100644 --- a/lib/sidecar/trtllm/src/tests.rs +++ b/lib/sidecar/trtllm/src/tests.rs @@ -21,7 +21,7 @@ use tokio::sync::{Mutex, oneshot}; use tokio_stream::wrappers::TcpListenerStream; use tonic::{Request, Response, Status}; -use crate::client::{ModelLimits, TrtllmClient}; +use crate::client::{ModelMetadata, TrtllmClient}; use crate::convert::{ResponseState, build_generate_request, engine_error}; use crate::engine::TrtllmSidecarEngine; use crate::model::ConfiguredModel; @@ -465,16 +465,16 @@ fn impatient_transport() -> GrpcTransportConfig { /// Engine limits as `Control.GetModelInfo` would report them, with no /// separate output cap. -fn limits(context_length: u32) -> Option { - Some(ModelLimits { +fn limits(context_length: u32) -> Option { + Some(ModelMetadata { context_length: Some(context_length), max_output_tokens: None, supports_image: false, }) } -fn image_limits(context_length: u32) -> Option { - Some(ModelLimits { +fn image_limits(context_length: u32) -> Option { + Some(ModelMetadata { context_length: Some(context_length), max_output_tokens: None, supports_image: true, diff --git a/lib/sidecar/trtllm/src/tests/convert_request.rs b/lib/sidecar/trtllm/src/tests/convert_request.rs index 138057cb5dc8..57007959e472 100644 --- a/lib/sidecar/trtllm/src/tests/convert_request.rs +++ b/lib/sidecar/trtllm/src/tests/convert_request.rs @@ -89,10 +89,6 @@ fn image_urls_data_uris_and_uuids_are_mapped_in_order() { "image_url".to_string(), vec![Some("first-image".to_string()), None], )])); - req.extra_args = Some(json!({ - "mm_hashes": ["routing-hash-one", "routing-hash-two"] - })); - let proto = build_generate_request(&req, "req", "model", image_limits(4096), AGG) .expect("image request"); assert_eq!(proto.media.len(), 2); @@ -114,7 +110,7 @@ fn image_urls_data_uris_and_uuids_are_mapped_in_order() { } #[test] -fn mm_routing_hashes_are_forwarded_as_image_uuids_in_order() { +fn mm_routing_hashes_are_not_used_as_engine_cache_uuids() { let mut req = request(); add_images( &mut req, @@ -129,34 +125,7 @@ fn mm_routing_hashes_are_forwarded_as_image_uuids_in_order() { let proto = build_generate_request(&req, "req", "model", image_limits(4096), AGG) .expect("image request"); - assert_eq!(proto.media[0].uuid, "0123456789abcdef"); - assert_eq!(proto.media[1].uuid, "fedcba9876543210"); -} - -#[test] -fn malformed_or_misaligned_mm_routing_hashes_are_rejected() { - for (hashes, expected) in [ - (json!("not-an-array"), "array of strings"), - (json!(["only-one"]), "must match"), - (json!(["one", ""]), "non-empty string"), - (json!(["one", 2]), "non-empty string"), - ] { - let mut req = request(); - add_images( - &mut req, - [ - "https://example.com/first.png", - "https://example.com/second.png", - ], - ); - req.extra_args = Some(json!({"mm_hashes": hashes})); - let error = build_generate_request(&req, "req", "model", image_limits(4096), AGG) - .expect_err("malformed routing hashes must fail closed"); - assert!( - error.to_string().contains(expected), - "unexpected error: {error}" - ); - } + assert!(proto.media.iter().all(|item| item.uuid.is_empty())); } #[test] @@ -365,7 +334,7 @@ fn the_derived_max_tokens_respects_the_engines_output_cap() { &req, "req", "model", - Some(ModelLimits { + Some(ModelMetadata { context_length: Some(4096), max_output_tokens: Some(256), supports_image: false, diff --git a/lib/sidecar/trtllm/src/tests/disagg.rs b/lib/sidecar/trtllm/src/tests/disagg.rs index 612fda3f39fc..a81c8dfe92c8 100644 --- a/lib/sidecar/trtllm/src/tests/disagg.rs +++ b/lib/sidecar/trtllm/src/tests/disagg.rs @@ -72,9 +72,13 @@ fn image_media_is_preserved_across_prefill_and_decode_requests() { "data:image/png;base64,c2Vjb25k", ], ); - req.extra_args = Some(json!({ - "mm_hashes": ["0123456789abcdef", "fedcba9876543210"] - })); + req.multi_modal_uuids = Some(HashMap::from([( + "image_url".to_string(), + vec![ + Some("first-image".to_string()), + Some("second-image".to_string()), + ], + )])); let prefill = build_generate_request( &req, @@ -87,8 +91,8 @@ fn image_media_is_preserved_across_prefill_and_decode_requests() { assert!(is_context_only(&prefill)); assert_eq!(prefill.stopping.as_ref().unwrap().max_tokens, Some(1)); assert_eq!(prefill.media.len(), 2); - assert_eq!(prefill.media[0].uuid, "0123456789abcdef"); - assert_eq!(prefill.media[1].uuid, "fedcba9876543210"); + assert_eq!(prefill.media[0].uuid, "first-image"); + assert_eq!(prefill.media[1].uuid, "second-image"); req.prefill_result = Some(dynamo_backend_common::PrefillResult { disaggregated_params: crate::disagg::session_to_json(fake_session()) From 7b55c8d0bf4b660b8a94b09227ec91ff246ebc11 Mon Sep 17 00:00:00 2001 From: krishung5 Date: Thu, 15 Oct 2026 16:59:18 -0700 Subject: [PATCH 16/61] style(trtllm): tighten media comments Signed-off-by: krishung5 --- lib/sidecar/trtllm/src/client.rs | 8 ++--- lib/sidecar/trtllm/src/convert.rs | 51 +++++++------------------------ 2 files changed, 13 insertions(+), 46 deletions(-) diff --git a/lib/sidecar/trtllm/src/client.rs b/lib/sidecar/trtllm/src/client.rs index 576bb2a48098..e3ac0932f4ab 100644 --- a/lib/sidecar/trtllm/src/client.rs +++ b/lib/sidecar/trtllm/src/client.rs @@ -27,8 +27,7 @@ use crate::proto::inference_client::InferenceClient; /// `--grpc-startup-deadline-secs` instead. const RPC_TIMEOUT: Duration = Duration::from_secs(30); -/// Metadata resolved at startup from `--context-length` and -/// `Control.GetModelInfo`, including limits and media capabilities. +/// Startup metadata from `--context-length` and `Control.GetModelInfo`. #[derive(Clone, Copy, Debug, Default)] pub(crate) struct ModelMetadata { /// Maximum input + output tokens; backs both the registered context window @@ -38,10 +37,7 @@ pub(crate) struct ModelMetadata { /// Cap on generated tokens, when the engine reports one. A context window /// alone can imply a larger budget than the engine will accept. pub(crate) max_output_tokens: Option, - /// Whether this loaded model advertises image input through both the - /// portable multimodal bit and TensorRT-LLM's modality list. Requiring both - /// fails closed when a server and client disagree about the finer-grained - /// capability. + /// True when both portable and engine-specific capabilities advertise images. pub(crate) supports_image: bool, } diff --git a/lib/sidecar/trtllm/src/convert.rs b/lib/sidecar/trtllm/src/convert.rs index 0a498a97aff0..1f5c9b76210e 100644 --- a/lib/sidecar/trtllm/src/convert.rs +++ b/lib/sidecar/trtllm/src/convert.rs @@ -4,10 +4,8 @@ //! Conversion between Dynamo's `PreprocessedRequest` / `LLMEngineOutput` and the //! TensorRT-LLM OpenEngine (`openengine.v1`) protobuf messages. //! -//! Scope: aggregated and disaggregated (prefill/decode) generation, including -//! image inputs when the loaded model advertises them. Video, audio, LoRA, -//! beam search, and `n > 1` are rejected before dispatch — the sidecar streams -//! a single sequence. +//! Supports aggregated and disaggregated image generation. Video, audio, LoRA, +//! beam search, and `n > 1` are rejected before dispatch. //! //! Disaggregation is driven by [`DisaggregationMode`]: a prefill worker marks //! its request `context_only` and returns the `PrefillReady` handoff as its @@ -139,27 +137,10 @@ pub(crate) fn build_generate_request( }) } -// Temporary workaround: the OpenEngine contract makes `stopping.max_tokens` -// optional. The Dynamo frontend forwards an omitted `max_tokens` as `None` -// expecting the backend to default. For text requests we mirror the in-process -// backend's default, -// `max(1, context_length - prompt_len)` (components/src/dynamo/trtllm -// `_default_max_tokens`). For image requests, only TensorRT-LLM knows the true -// prompt length after expanding image markers, so leave the field unset and let -// the OpenEngine request mapper derive the budget from the expanded prompt. -// `context_length` is resolved in -// `engine::start`, where `--context-length` wins over the `GetModelInfo` report. -// Only `/v1/chat/completions` and `/v1/responses` reach this fallback: they set -// `PRESERVE_OMITTED_MAX_TOKENS_CONTEXT_KEY`, which stops the frontend supplying -// its own default (`preprocessor::omitted_max_tokens_default`). `/v1/completions` -// is already defaulted by the frontend and never lands here. -// -// Remove when https://github.com/NVIDIA/TensorRT-LLM/issues/16549 lands (gRPC -// `max_tokens` made optional): drop this fallback and forward an omitted -// `max_tokens` as unset. Keep `--context-length` and the plumbing in -// `engine.rs` — they also feed `LlmRegistration.context_length`, which the -// frontend registers as the served context window and which this fallback does -// not govern. +// Derive omitted text max_tokens from the context window. For media, only +// TensorRT-LLM knows the prompt length after expansion, so leave it unset. +// Remove this fallback when NVIDIA/TensorRT-LLM#16549 lands; context_length +// remains necessary for LlmRegistration. fn max_tokens( request: &PreprocessedRequest, limits: Option, @@ -195,11 +176,8 @@ fn max_tokens( const IMAGE_KEY: &str = "image_url"; -/// Convert Dynamo's image-only execution payload into the ordered OpenEngine -/// media list. The frontend groups content parts by modality, so ordering is -/// retained within the image vector but not across different modalities. This -/// PR therefore accepts images only and fails explicitly on every other -/// non-empty modality instead of inventing a cross-modality order. +/// Map Dynamo's image payload to OpenEngine media, rejecting other modalities. +/// Per-modality vectors do not preserve cross-modality order. fn image_media( request: &PreprocessedRequest, limits: Option, @@ -207,6 +185,7 @@ fn image_media( reject_media_override("mm_processor_kwargs", request.mm_processor_kwargs.as_ref())?; reject_media_override("media_io_kwargs", request.media_io_kwargs.as_ref())?; + // mm_routing_info stays router-side; multi_modal_data is engine input. let media_map = request.multi_modal_data.as_ref(); if let Some(media_map) = media_map { for (modality, items) in media_map { @@ -276,14 +255,9 @@ fn image_media( Ok(pb::MediaItem { modality: pb::Modality::Image as i32, source: Some(source), - // Dynamo's frontend representation carries URL/data-URI - // sources, not raw bytes. MIME type is only meaningful for the - // OpenEngine raw_bytes source used by direct clients. + // MIME metadata belongs to raw_bytes, which this client does not send. mime_type: String::new(), - // UUIDs are caller-owned cache identities and are optional per - // item. Dynamo's router hashes are deliberately not substituted - // here: their compatibility with TensorRT-LLM cache identity is - // owned and validated by the MM-aware routing work. + // Do not substitute router-owned mm_hashes for caller cache identities. uuid: user_image_uuids .and_then(|uuids| uuids[index].as_deref()) .unwrap_or_default() @@ -486,9 +460,6 @@ fn validate_request( "multimodal encoder handoffs are not supported by the TensorRT-LLM sidecar", )); } - // `mm_routing_info` is deliberately not forwarded: it is framework-owned - // metadata that the Dynamo router consumes before selecting this worker. - // The execution payload remains `multi_modal_data`, mapped by `image_media`. // Only a decode worker consumes a prefill handoff. Seeing one anywhere else // means the frontend routed a decode request to the wrong worker role. if request.prefill_result.is_some() && !mode.is_decode() { From 16dd1f276bd8830f5b780af800f0348840508c4e Mon Sep 17 00:00:00 2001 From: krishung5 Date: Thu, 15 Oct 2026 17:49:52 -0700 Subject: [PATCH 17/61] test(trtllm): gate sidecar image requests in CI Signed-off-by: krishung5 --- .github/filters.yaml | 1 + .github/scripts/test-filters.js | 12 +++--- .github/workflows/pr.yaml | 27 ++++++++++++ container/templates/trtllm_runtime.Dockerfile | 20 +++++++++ tests/serve/test_sidecar.py | 41 ++++++++++++++++--- 5 files changed, 90 insertions(+), 11 deletions(-) diff --git a/.github/filters.yaml b/.github/filters.yaml index 69c3f5b01c10..b49e6f46532d 100644 --- a/.github/filters.yaml +++ b/.github/filters.yaml @@ -404,6 +404,7 @@ trtllm: - 'container/deps/requirements.trtllm.txt' - 'examples/backends/trtllm/**' - 'components/src/dynamo/trtllm/**' + - 'lib/sidecar/trtllm/**' - 'container/templates/trtllm_*' - '!**/*.md' - '!**/*.rst' diff --git a/.github/scripts/test-filters.js b/.github/scripts/test-filters.js index e23b4cb0014c..a00ec9bd975d 100755 --- a/.github/scripts/test-filters.js +++ b/.github/scripts/test-filters.js @@ -13,7 +13,7 @@ * This validates that tj-actions/changed-files will correctly: * - Match backend-specific files to their respective filters (vllm, sglang, trtllm) * - Route docs-artifact tests to recipe-check without core E2E checks - * - Route sidecar files to sidecar/Rust checks without backend or core E2E checks + * - Route sidecar files to sidecar/Rust checks and framework-native E2E where needed * - Exclude doc files (*.md, *.rst, *.txt) from core via negation patterns * - Match CI/infrastructure changes to core * - (with --coverage) Ensure all files in repo are covered by at least one filter @@ -175,7 +175,7 @@ const testCases = [ desc: 'AISimulate Dynamo adapter triggers planner validation' }, - // Sidecar Rust and proto files should trigger Rust checks without unrelated E2E + // Sidecar Rust and proto files trigger Rust checks and their native E2E only. { file: 'lib/sidecar/common/src/lib.rs', expect: { sidecar: true, rust: true, core: false, frontend: false, vllm: false, sglang: false, trtllm: false }, @@ -183,8 +183,8 @@ const testCases = [ }, { file: 'lib/sidecar/trtllm/proto/openengine/v1/generation.proto', - expect: { sidecar: true, rust: true, core: false, frontend: false, vllm: false, sglang: false, trtllm: false }, - desc: 'sidecar proto contracts trigger Rust checks without backend E2E' + expect: { sidecar: true, rust: true, core: false, frontend: false, vllm: false, sglang: false, trtllm: true }, + desc: 'trtllm sidecar contracts trigger Rust checks and trtllm E2E' }, { file: 'lib/sidecar/sglang/src/lib.rs', @@ -193,8 +193,8 @@ const testCases = [ }, { file: 'lib/sidecar/trtllm/src/lib.rs', - expect: { sidecar: true, rust: true, core: false, frontend: false, vllm: false, sglang: false, trtllm: false }, - desc: 'trtllm sidecar source does not route to sglang or trtllm E2E' + expect: { sidecar: true, rust: true, core: false, frontend: false, vllm: false, sglang: false, trtllm: true }, + desc: 'trtllm sidecar source triggers Rust checks and trtllm E2E' }, { file: 'lib/sidecar/vllm/deploy/agg.yaml', diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index e6e2a925bbd1..58eb3e77f6af 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -147,6 +147,7 @@ jobs: - trtllm-build - trtllm-dev-build - trtllm-test + - sidecar-trtllm-test - trtllm-multi-gpu-test - trtllm-copy-to-acr - triton-build @@ -541,6 +542,11 @@ jobs: diff_current_branch: ${{ github.ref_name }} diff_base_branch: ${{ needs.changed-files.outputs.base_ref }} diff_merge_base_sha: ${{ needs.changed-files.outputs.merge_base_sha }} + # Drop these overrides after the upstream TensorRT-LLM change ships. + extra_build_args: | + RUNTIME_IMAGE_TAG=1.4.0rc1.dev202610140007@sha256:7a728cd307bfa1651b4ad0787e71e69f7506fb1a698a0f4f04aef719ad43525f + TRTLLM_SOURCE_REPOSITORY=https://github.com/krishung5/TensorRT-LLM + TRTLLM_SOURCE_COMMIT=ab07b1649f638924908aed565722f4dd0f8547f8 # actions: read lets the build download a prior run's baseline compliance # artifact to diff against (a reusable workflow's token can't exceed the # caller's). Missing -> the diff soft-degrades, build stays green. @@ -928,6 +934,27 @@ jobs: gpu_parallel_max_vram_gib: '24' secrets: inherit + sidecar-trtllm-test: + name: sidecar-runtime + needs: [changed-files, trtllm-build, sidecar-build] + if: needs.changed-files.outputs.sidecar == 'true' && needs.changed-files.outputs.trtllm == 'true' + uses: $/.github/workflows/shared-test.yml + with: + test_suite_name: trtllm + framework: trtllm + test_type: TensorRT-LLM Sidecar 1-GPU E2E Test + amd_runner: prod-tester-amd-gpu-v2 + amd_runner_labels: '["gpu-1","amd64","v2","us-west-2"]' + target_tag_plain: ${{ needs.trtllm-build.outputs.target_tag_plain }} + cuda_version: '["13.1"]' + platform: '["amd64"]' + run_sanity_check: false + gpu_test_markers: pre_merge and sidecar and trtllm and gpu_1 + gpu_test_timeout_minutes: 45 + sidecar_backend: trtllm + sidecar_binary_artifact: ${{ needs.sidecar-build.outputs.binary_artifact }} + secrets: inherit + trtllm-multi-gpu-test: name: trtllm-runtime # This name overlaps with other trtllm jobs to group them in the UI needs: [changed-files, trtllm-build] diff --git a/container/templates/trtllm_runtime.Dockerfile b/container/templates/trtllm_runtime.Dockerfile index 3b147a085b55..860698519fbb 100644 --- a/container/templates/trtllm_runtime.Dockerfile +++ b/container/templates/trtllm_runtime.Dockerfile @@ -41,6 +41,26 @@ ARG ENABLE_KVBM ARG ENABLE_GPU_MEMORY_SERVICE ARG TARGETARCH ARG NIXL_REF +ARG TRTLLM_SOURCE_REPOSITORY +ARG TRTLLM_SOURCE_COMMIT + +# Optional exact-source overlay for cross-repository CI. +RUN if [ -n "${TRTLLM_SOURCE_COMMIT}" ]; then \ + test -n "${TRTLLM_SOURCE_REPOSITORY}" && \ + source_dir=$(mktemp -d) && \ + wget --quiet --tries=3 \ + "${TRTLLM_SOURCE_REPOSITORY}/archive/${TRTLLM_SOURCE_COMMIT}.tar.gz" \ + -O "${source_dir}/source.tar.gz" && \ + tar -xzf "${source_dir}/source.tar.gz" -C "${source_dir}" \ + --strip-components=1 && \ + package_dir=$(/usr/bin/python3 -c \ + 'import importlib.util; print(next(iter(importlib.util.find_spec("tensorrt_llm").submodule_search_locations)))') && \ + cp -a "${source_dir}/tensorrt_llm/grpc/openengine/." \ + "${package_dir}/grpc/openengine/" && \ + install -m 0644 "${source_dir}/tensorrt_llm/inputs/media_io.py" \ + "${package_dir}/inputs/media_io.py" && \ + rm -rf "${source_dir}"; \ + fi # Remove the upstream runtime's Git LFS package without pruning shared dependencies. RUN apt-get purge -y git-lfs && rm -rf /var/lib/apt/lists/* diff --git a/tests/serve/test_sidecar.py b/tests/serve/test_sidecar.py index 038446530fc0..55720fee9b12 100644 --- a/tests/serve/test_sidecar.py +++ b/tests/serve/test_sidecar.py @@ -31,6 +31,7 @@ disaggregated_chat_payload, disaggregated_token_count_payload, http_cancellation_payloads, + multimodal_payload_default, ) from tests.utils.payloads import ( ChatPayload, @@ -115,10 +116,7 @@ def _disaggregated_payloads( ] -# Sequential stage only: no profiled_vram_gib mark yet, since actual peak VRAM -# has not been profiled for the sidecar launch path. Add one once measured, to -# admit these into the parallel stage alongside the equivalent dynamo.{backend} -# scenarios in tests/serve/test_{vllm,sglang,trtllm}.py. +# Text cases stay sequential until their sidecar launch paths are profiled. sidecar_configs = { "vllm_aggregated": EngineConfig( name="vllm_aggregated", @@ -126,6 +124,7 @@ def _disaggregated_payloads( script_name="agg.sh", marks=[ pytest.mark.vllm, + pytest.mark.core, pytest.mark.gpu_1, # Let the 600s health check report failure before pytest times out. pytest.mark.timeout(780), @@ -149,6 +148,7 @@ def _disaggregated_payloads( script_args=["--enable-metrics", "--decode-log-interval", "1"], marks=[ pytest.mark.sglang, + pytest.mark.core, pytest.mark.gpu_1, pytest.mark.timeout(780), pytest.mark.post_merge, @@ -179,6 +179,7 @@ def _disaggregated_payloads( script_name="agg.sh", marks=[ pytest.mark.trtllm, + pytest.mark.core, pytest.mark.gpu_1, pytest.mark.timeout(780), pytest.mark.post_merge, @@ -198,12 +199,41 @@ def _disaggregated_payloads( chat_payload_default(), ], ), + "trtllm_aggregated_image": EngineConfig( + name="trtllm_aggregated_image", + directory=trtllm_sidecar_dir, + script_name="agg.sh", + marks=[ + pytest.mark.trtllm, + pytest.mark.multimodal, + pytest.mark.gpu_1, + pytest.mark.profiled_vram_gib(12.0), + pytest.mark.requested_trtllm_kv_tokens(32768), + pytest.mark.timeout(960), + pytest.mark.pre_merge, + pytest.mark.skipif( + not _trtllm_serves_openengine(), + reason=TRTLLM_OPENENGINE_SKIP_REASON, + ), + ], + model="Qwen/Qwen3-VL-2B-Instruct", + timeout=900, + delayed_start=60, + env={"PYTHONUNBUFFERED": "1"}, + request_payloads=[ + multimodal_payload_default( + text="Describe what you see in this image.", + expected_response=["mountain", "rock", "trees", "road"], + ) + ], + ), "trtllm_disaggregated": EngineConfig( name="trtllm_disaggregated", directory=trtllm_sidecar_dir, script_name="disagg.sh", marks=[ pytest.mark.trtllm, + pytest.mark.core, # Both engines share one GPU: the handoff is what this covers, and # a co-resident pair keeps it on the existing 1-GPU sidecar runner. # `requested_trtllm_kv_tokens` caps each engine's KV pool so the @@ -250,6 +280,7 @@ def _disaggregated_payloads( script_name="disagg.sh", marks=[ pytest.mark.vllm, + pytest.mark.core, pytest.mark.gpu_1, pytest.mark.post_merge, pytest.mark.timeout(1200), @@ -282,6 +313,7 @@ def _disaggregated_payloads( ], marks=[ pytest.mark.sglang, + pytest.mark.core, pytest.mark.gpu_1, pytest.mark.post_merge, pytest.mark.timeout(1200), @@ -320,7 +352,6 @@ def sidecar_config_test(request, monkeypatch, discovery_backend): return config -@pytest.mark.core @pytest.mark.sidecar @pytest.mark.e2e @pytest.mark.engine_ports From e8c2e565779cda189035a64f8b5407f7282cc639 Mon Sep 17 00:00:00 2001 From: krishung5 Date: Fri, 16 Oct 2026 11:24:34 -0700 Subject: [PATCH 18/61] fix(trtllm): address multimodal review findings Signed-off-by: krishung5 --- .github/filters.yaml | 1 + .github/scripts/test-filters.js | 5 + .github/workflows/pr.yaml | 2 +- .github/workflows/shared-test.yml | 1 + container/compliance/policy/codec_policy.yaml | 26 +++-- container/templates/trtllm_runtime.Dockerfile | 48 +++++----- lib/sidecar/trtllm/src/client.rs | 32 ++++--- lib/sidecar/trtllm/src/convert.rs | 75 ++++++++------- lib/sidecar/trtllm/src/engine.rs | 94 +++++++++++++++---- lib/sidecar/trtllm/src/tests.rs | 6 +- .../trtllm/src/tests/convert_request.rs | 46 ++++----- lib/sidecar/trtllm/src/tests/disagg.rs | 10 +- lib/sidecar/trtllm/src/tests/engine.rs | 58 ++++++++++++ tests/serve/test_sidecar.py | 8 +- 14 files changed, 272 insertions(+), 140 deletions(-) diff --git a/.github/filters.yaml b/.github/filters.yaml index b49e6f46532d..d5e9ae3d5075 100644 --- a/.github/filters.yaml +++ b/.github/filters.yaml @@ -405,6 +405,7 @@ trtllm: - 'examples/backends/trtllm/**' - 'components/src/dynamo/trtllm/**' - 'lib/sidecar/trtllm/**' + - 'tests/serve/test_sidecar.py' - 'container/templates/trtllm_*' - '!**/*.md' - '!**/*.rst' diff --git a/.github/scripts/test-filters.js b/.github/scripts/test-filters.js index a00ec9bd975d..d5d45a4ce863 100755 --- a/.github/scripts/test-filters.js +++ b/.github/scripts/test-filters.js @@ -196,6 +196,11 @@ const testCases = [ expect: { sidecar: true, rust: true, core: false, frontend: false, vllm: false, sglang: false, trtllm: true }, desc: 'trtllm sidecar source triggers Rust checks and trtllm E2E' }, + { + file: 'tests/serve/test_sidecar.py', + expect: { sidecar: true, trtllm: true }, + desc: 'shared sidecar test changes trigger the TensorRT-LLM sidecar E2E' + }, { file: 'lib/sidecar/vllm/deploy/agg.yaml', expect: { sidecar: true, rust: false, core: false, frontend: false, vllm: false, sglang: false, trtllm: false }, diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 58eb3e77f6af..4eb2b9312608 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -546,7 +546,7 @@ jobs: extra_build_args: | RUNTIME_IMAGE_TAG=1.4.0rc1.dev202610140007@sha256:7a728cd307bfa1651b4ad0787e71e69f7506fb1a698a0f4f04aef719ad43525f TRTLLM_SOURCE_REPOSITORY=https://github.com/krishung5/TensorRT-LLM - TRTLLM_SOURCE_COMMIT=ab07b1649f638924908aed565722f4dd0f8547f8 + TRTLLM_SOURCE_COMMIT=5d2e3b2c758b4ccc0fa0a3eb0e01cdef934e8089 # actions: read lets the build download a prior run's baseline compliance # artifact to diff against (a reusable workflow's token can't exceed the # caller's). Missing -> the diff soft-degrades, build stays green. diff --git a/.github/workflows/shared-test.yml b/.github/workflows/shared-test.yml index 951c36ee5d6e..8669d55143b1 100644 --- a/.github/workflows/shared-test.yml +++ b/.github/workflows/shared-test.yml @@ -259,6 +259,7 @@ jobs: install -m755 "sidecar-artifact/dynamo-${SIDECAR_BACKEND}-sidecar" \ "${bin_dir}/dynamo-${SIDECAR_BACKEND}-sidecar" echo "DYNAMO_SIDECAR_BIN=${bin_dir}/dynamo-${SIDECAR_BACKEND}-sidecar" >> "${GITHUB_ENV}" + echo "DYNAMO_TEST_SIDECAR_BACKEND=${SIDECAR_BACKEND}" >> "${GITHUB_ENV}" if [ -z "${CONTAINER_WORKSPACE}" ]; then echo "::error::container_workspace is empty; refusing to replace test files" exit 1 diff --git a/container/compliance/policy/codec_policy.yaml b/container/compliance/policy/codec_policy.yaml index 9e4023b0e2e3..61b8f39e19a0 100644 --- a/container/compliance/policy/codec_policy.yaml +++ b/container/compliance/policy/codec_policy.yaml @@ -108,9 +108,9 @@ exceptions: (NVIDIA/DALI_deps#162, NVIDIA/DALI#6352, May 2026), first released in 2.1.1 -- as a cherry-pick, so checking whether the merge commit is an ancestor of the tag gives the wrong answer; the release notes are the - reliable source. Upgrading the wheel in place to 2.1.1 or 2.2.0 leaves 440 - decoders with all five absent, vp8/vp9/mjpeg/av1 still present, and the - image importing and passing its unit suite. + reliable source. Upgrading the wheel in place to 2.1.1, 2.2.0, or 2.3.0 + leaves 440 decoders with all five absent, vp8/vp9/mjpeg/av1 still present, + and the image importing and passing its unit suite. This waiver cannot be narrowed to the non-codec companions, which an earlier revision of this note claimed. libavcodec, libavfilter, libavformat, @@ -120,13 +120,12 @@ exceptions: set is all-or-nothing with the package. The entry stays even so. trtllm_runtime.Dockerfile pins - nvidia-dali-cuda130==2.2.0, which is the version the 1.3.0rc27 base image - already carries, so the pin now holds a version rather than moving one. The - vendored libraries do not register those decoders -- but they are still on - disk and still match the deny globs above, which gate on presence rather - than content. What changed is the reason for the waiver: it now covers a - vendored ffmpeg that carries no software h264/hevc/aac, rather than one - that does. + nvidia-dali-cuda130 at an explicitly measured version: 2.2.0 in the + release base or 2.3.0 in the pinned development base. The vendored + libraries do not register those decoders -- but they are still on disk and + still match the deny globs above, which gate on presence rather than + content. What changed is the reason for the waiver: it now covers a vendored + ffmpeg that carries no software h264/hevc/aac, rather than one that does. Enforcement lives at build time, not here. Two guards in trtllm_runtime.Dockerfile enumerate every bundled libavcodec -- one in @@ -142,10 +141,9 @@ exceptions: subtracted from the generated delta, so such an entry would never match anything and would read as a check while doing nothing. - Re-measure when the vendored version moves. Last measured against - nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc27, whose base carries 2.2.0: - 440 decoders, h264, hevc, aac, aac_fixed and aac_latm absent, vp8, vp9, - mjpeg and av1 present. + Re-measure when the vendored version moves. Last measured against 2.2.0 and + 2.3.0: 440 decoders, h264, hevc, aac, aac_fixed and aac_latm absent, vp8, + vp9, mjpeg and av1 present. owner: DALI # PyNvVideoCodec (added for NVDEC hardware video decode) vendors libavutil and # libavformat under its package dir to demux input containers before handing the diff --git a/container/templates/trtllm_runtime.Dockerfile b/container/templates/trtllm_runtime.Dockerfile index 860698519fbb..b71d05bdb1e9 100644 --- a/container/templates/trtllm_runtime.Dockerfile +++ b/container/templates/trtllm_runtime.Dockerfile @@ -403,10 +403,9 @@ RUN /usr/bin/python3 -m pip uninstall -y --break-system-packages opencv-python-h # Hold DALI at a version that carries its own media-codec cleanup. Upstream # restricted DALI's vendored ffmpeg build to drop the software h264/hevc/aac # decoders (NVIDIA/DALI_deps#162, NVIDIA/DALI#6352), first released in 2.1.1. -# The base image carried 2.1.0 through 1.3.0rc26, which predates it, and carries -# 2.2.0 from 1.3.0rc27, which does not. So on this base the install below is a -# no-op and what the block still buys is the two checks around it: the version -# tripwire and the decoder enumeration. +# The supported base images carry either 2.2.0 or 2.3.0. Both were measured +# without the restricted codecs; older versions are upgraded to 2.2.0 and any +# other newer version stops the build for re-measurement. # # Pin rather than remove. The vendored libav*/libsw* set cannot be trimmed # on its own -- every one of them is DT_NEEDED by libdali.so and its siblings, so @@ -416,23 +415,17 @@ RUN /usr/bin/python3 -m pip uninstall -y --break-system-packages opencv-python-h # out of someone else's image. # # Measured on the shipped image: 2.1.0 registers 446 decoders including h264, -# hevc, aac, aac_fixed and aac_latm; 2.1.1 and 2.2.0 each register 440 with all -# five absent and vp8/vp9/mjpeg/av1 retained. 2.2.0 measured again on -# tensorrt-llm/release:1.3.0rc27 itself, now that DALI arrives with the base. +# hevc, aac, aac_fixed and aac_latm; 2.1.1, 2.2.0, and 2.3.0 each register 440 +# with all five absent and vp8/vp9/mjpeg/av1 retained. # # Pinned exactly, not a floor. `>=2.1.1` resolves to whatever is newest, which is # a jump into a component this repo does not own, taken silently at build time. # The pin names the version that was measured, so moving it means measuring the # new one. # -# The pin can go stale in the one direction that matters: if TensorRT-LLM later -# ships a base image with a newer DALI, installing 2.2.0 over it is a downgrade. -# The check below turns that into a build failure with instructions rather than a -# silent regression. The pin now equals the base version, so the next base image -# that moves DALI past 2.2.0 stops the build, which is the point: a human -# re-measures and moves the pin, or deletes this RUN and lets the base version -# stand. A base that moves DALI back below 2.2.0 passes that check and the -# install below puts the measured version back, which is what it is there for. +# A base below 2.2.0 is upgraded to the measured 2.2.0 wheel. The two measured +# versions otherwise stay in place; any unmeasured newer version fails rather +# than being downgraded or accepted silently. # # THIS BLOCK IS TRANSITIONAL, and on this base it is already down to its checks. # The guards after it and the bundled-libavcodec assertion in @@ -457,20 +450,23 @@ RUN --mount=type=bind,source=./container/compliance/enumerate_bundled_decoders.p set -eu; \ before=$(/usr/bin/python3 -c 'import importlib.metadata as m; print(m.version("nvidia-dali-cuda130"))'); \ echo "DALI in base image: $before"; \ - newest=$(printf '%s\n2.2.0\n' "$before" | sort -V | tail -1); \ - if [ "$newest" != "2.2.0" ]; then \ - echo "ERROR: base image already carries DALI $before, so pinning 2.2.0 would" >&2; \ - echo " downgrade it. Upstream has caught up -- delete this RUN and let" >&2; \ - echo " the base version stand. Keep the guards below and the bundled-" >&2; \ - echo " libavcodec assertion in tests/dependencies/, which are what stop" >&2; \ - echo " this from regressing." >&2; \ - exit 1; \ + case "$before" in \ + 2.3.0) target=2.3.0 ;; \ + *) \ + newest=$(printf '%s\n2.2.0\n' "$before" | sort -V | tail -1); \ + if [ "$newest" != "2.2.0" ]; then \ + echo "ERROR: base image carries unmeasured DALI $before" >&2; \ + exit 1; \ + fi; \ + target=2.2.0 ;; \ + esac; \ + if [ "$before" != "$target" ]; then \ + /usr/bin/python3 -m pip install --break-system-packages --no-cache-dir \ + --extra-index-url https://pypi.nvidia.com "nvidia-dali-cuda130==$target"; \ fi; \ - /usr/bin/python3 -m pip install --break-system-packages --no-cache-dir \ - --extra-index-url https://pypi.nvidia.com 'nvidia-dali-cuda130==2.2.0'; \ v=$(/usr/bin/python3 -c 'import importlib.metadata as m; print(m.version("nvidia-dali-cuda130"))'); \ echo "DALI version: $v"; \ - [ "$v" = "2.2.0" ] || { echo "ERROR: wanted DALI 2.2.0, got $v" >&2; exit 1; }; \ + [ "$v" = "$target" ] || { echo "ERROR: wanted DALI $target, got $v" >&2; exit 1; }; \ libs=$(find /usr/local/lib/python3.12/dist-packages/nvidia/dali/.libs -name 'libavcodec*.so*'); \ if [ -z "$libs" ]; then \ echo "ERROR: DALI $v ships no libavcodec under .libs, so the enumeration" >&2; \ diff --git a/lib/sidecar/trtllm/src/client.rs b/lib/sidecar/trtllm/src/client.rs index e3ac0932f4ab..f3daac1910c7 100644 --- a/lib/sidecar/trtllm/src/client.rs +++ b/lib/sidecar/trtllm/src/client.rs @@ -37,8 +37,14 @@ pub(crate) struct ModelMetadata { /// Cap on generated tokens, when the engine reports one. A context window /// alone can imply a larger budget than the engine will accept. pub(crate) max_output_tokens: Option, - /// True when both portable and engine-specific capabilities advertise images. - pub(crate) supports_image: bool, + /// Whether both portable and engine-specific capabilities advertise images. + /// `None` means startup discovery failed transiently and may be retried. + pub(crate) supports_image: Option, +} + +pub(crate) enum ModelInfoResponse { + Available(ModelMetadata), + Unsupported(tonic::Status), } pub(crate) struct TrtllmClient { @@ -125,22 +131,24 @@ impl TrtllmClient { Ok(ModelMetadata { context_length: info.max_context_length.filter(|len| *len > 0), max_output_tokens: info.max_output_tokens.filter(|cap| *cap > 0), - supports_image, + supports_image: Some(supports_image), }) } /// One `GetModelInfo` call, for when `--context-length` already supplies the /// window and the engine is consulted only to cross-check it and to learn /// its output cap. - pub(crate) async fn model_limits(&self, model: &str) -> Result { - tokio::time::timeout(RPC_TIMEOUT, self.get_model_info(model)) - .await - .map_err(|_| { - connection_timeout(format!( - "GetModelInfo did not respond within {RPC_TIMEOUT:?}" - )) - })? - .map_err(|status| status_to_dynamo("GetModelInfo", status)) + pub(crate) async fn model_limits(&self, model: &str) -> Result { + match tokio::time::timeout(RPC_TIMEOUT, self.get_model_info(model)).await { + Ok(Ok(metadata)) => Ok(ModelInfoResponse::Available(metadata)), + Ok(Err(status)) if is_request_itself_wrong(&status) => { + Ok(ModelInfoResponse::Unsupported(status)) + } + Ok(Err(status)) => Err(status_to_dynamo("GetModelInfo", status)), + Err(_) => Err(connection_timeout(format!( + "GetModelInfo did not respond within {RPC_TIMEOUT:?}" + ))), + } } /// Polls `GetModelInfo` until the engine reports a usable context length or diff --git a/lib/sidecar/trtllm/src/convert.rs b/lib/sidecar/trtllm/src/convert.rs index 1f5c9b76210e..81661405df03 100644 --- a/lib/sidecar/trtllm/src/convert.rs +++ b/lib/sidecar/trtllm/src/convert.rs @@ -13,6 +13,7 @@ //! See [`crate::disagg`] for the codec. use std::collections::BTreeSet; +use std::sync::Arc; use dynamo_backend_common::{ CompletionUsage, DisaggregationMode, DynamoError, FinishReason, LLMEngineOutput, @@ -28,14 +29,15 @@ use crate::proto as pb; type MappedTokens = (Vec, Option>, Option>>); pub(crate) fn build_generate_request( - request: &PreprocessedRequest, + mut request: PreprocessedRequest, request_id: &str, model: &str, limits: Option, mode: DisaggregationMode, ) -> Result { - validate_request(request, mode)?; - let media = image_media(request, limits)?; + validate_request(&request, mode)?; + let request_has_media = has_media(&request); + let media = image_media(&mut request, limits)?; let sampling = &request.sampling_options; let stop = &request.stop_conditions; @@ -52,7 +54,7 @@ pub(crate) fn build_generate_request( let max_tokens = if mode.is_prefill() { Some(1) } else { - max_tokens(request, limits)? + max_tokens(&request, limits, request_has_media)? }; // The decode worker replays the prefill worker's session; the prefill worker // marks its request `context_only` through `extra`. @@ -71,6 +73,15 @@ pub(crate) fn build_generate_request( }, DisaggregationMode::Aggregated | DisaggregationMode::Encode => None, }; + let conditions = stop_conditions(&request); + let guided = guided_decoding(&request)?; + let session_id = request + .agent_context + .as_ref() + .map(|context| context.session_id.trim()) + .filter(|session_id| !session_id.is_empty()) + .map(str::to_string); + let token_ids = Arc::unwrap_or_clone(request.token_ids); Ok(pb::GenerateRequest { request_id: request_id.to_string(), @@ -78,7 +89,7 @@ pub(crate) fn build_generate_request( // served by the loaded model (single-model server). model: model.to_string(), input: Some(pb::generate_request::Input::TokenIds(pb::TokenIds { - ids: request.token_ids.as_ref().clone(), + ids: token_ids, })), sampling: Some(pb::SamplingParams { temperature: sampling.temperature.map(f64::from), @@ -100,7 +111,7 @@ pub(crate) fn build_generate_request( // `Stop` instead of `PrefillReady`, and leave the frontend treating // a request that never met its minimum as complete. min_tokens: stop.min_tokens, - conditions: stop_conditions(request), + conditions, ignore_eos: stop.ignore_eos, // `include_stop_in_output` retains matched stop *strings*; the // request-level flag is rejected in `validate_request`, so leave it @@ -122,17 +133,13 @@ pub(crate) fn build_generate_request( prompt_candidates: None, prompt_logprob_start: None, }), - guided: guided_decoding(request)?, + guided, media, lora_name: String::new(), kv, extra: Some(disagg::request_extra( mode.is_prefill(), - request - .agent_context - .as_ref() - .map(|context| context.session_id.trim()) - .filter(|session_id| !session_id.is_empty()), + session_id.as_deref(), )), }) } @@ -144,11 +151,12 @@ pub(crate) fn build_generate_request( fn max_tokens( request: &PreprocessedRequest, limits: Option, + has_media: bool, ) -> Result, DynamoError> { if let Some(max_tokens) = request.stop_conditions.max_tokens { return Ok(Some(max_tokens)); } - if has_media(request) { + if has_media { return Ok(None); } let context_length = limits @@ -176,18 +184,17 @@ fn max_tokens( const IMAGE_KEY: &str = "image_url"; -/// Map Dynamo's image payload to OpenEngine media, rejecting other modalities. /// Per-modality vectors do not preserve cross-modality order. fn image_media( - request: &PreprocessedRequest, + request: &mut PreprocessedRequest, limits: Option, ) -> Result, DynamoError> { reject_media_override("mm_processor_kwargs", request.mm_processor_kwargs.as_ref())?; reject_media_override("media_io_kwargs", request.media_io_kwargs.as_ref())?; // mm_routing_info stays router-side; multi_modal_data is engine input. - let media_map = request.multi_modal_data.as_ref(); - if let Some(media_map) = media_map { + let mut media_map = request.multi_modal_data.take(); + if let Some(media_map) = media_map.as_ref() { for (modality, items) in media_map { if modality != IMAGE_KEY && !items.is_empty() { return Err(client::invalid_argument(format!( @@ -198,24 +205,25 @@ fn image_media( } let images = media_map - .and_then(|media| media.get(IMAGE_KEY)) - .map(Vec::as_slice) + .as_mut() + .and_then(|media| media.remove(IMAGE_KEY)) .unwrap_or_default(); - let user_image_uuids = if let Some(uuid_map) = request.multi_modal_uuids.as_ref() { - for (modality, uuids) in uuid_map { + let mut uuid_map = request.multi_modal_uuids.take(); + let user_image_uuids = if let Some(uuid_map) = uuid_map.as_mut() { + for (modality, uuids) in uuid_map.iter() { if modality != IMAGE_KEY && !uuids.is_empty() { return Err(client::invalid_argument(format!( "multimodal UUIDs for `{modality}` cannot be forwarded by the image-only TensorRT-LLM sidecar" ))); } } - uuid_map.get(IMAGE_KEY).map(Vec::as_slice) + uuid_map.remove(IMAGE_KEY) } else { None }; - if let Some(uuids) = user_image_uuids + if let Some(uuids) = user_image_uuids.as_ref() && uuids.len() != images.len() { return Err(client::invalid_argument(format!( @@ -228,18 +236,20 @@ fn image_media( if images.is_empty() { return Ok(Vec::new()); } - if !limits.is_some_and(|limits| limits.supports_image) { + if limits.is_none_or(|limits| limits.supports_image != Some(true)) { return Err(client::invalid_argument( "the loaded TensorRT-LLM model does not advertise image support through Control.GetModelInfo", )); } + let uuids = user_image_uuids.unwrap_or_else(|| vec![None; images.len()]); images - .iter() + .into_iter() + .zip(uuids) .enumerate() - .map(|(index, image)| { + .map(|(index, (image, uuid))| { let source = match image { - MultimodalData::Url(url) => media_source(url.as_str(), index)?, + MultimodalData::Url(url) => media_source(url.to_string(), index)?, MultimodalData::RawUrl(url) => media_source(url, index)?, MultimodalData::Decoded(_) => { return Err(client::invalid_argument(format!( @@ -258,22 +268,19 @@ fn image_media( // MIME metadata belongs to raw_bytes, which this client does not send. mime_type: String::new(), // Do not substitute router-owned mm_hashes for caller cache identities. - uuid: user_image_uuids - .and_then(|uuids| uuids[index].as_deref()) - .unwrap_or_default() - .to_string(), + uuid: uuid.unwrap_or_default(), }) }) .collect() } -fn media_source(value: &str, index: usize) -> Result { +fn media_source(value: String, index: usize) -> Result { let scheme = value .split_once(':') .map(|(scheme, _)| scheme.to_ascii_lowercase()); match scheme.as_deref() { - Some("http") | Some("https") => Ok(pb::media_item::Source::Url(value.to_string())), - Some("data") => Ok(pb::media_item::Source::DataUri(value.to_string())), + Some("http") | Some("https") => Ok(pb::media_item::Source::Url(value)), + Some("data") => Ok(pb::media_item::Source::DataUri(value)), _ => Err(client::invalid_argument(format!( "`{IMAGE_KEY}` item {index} must use an http, https, or data URI" ))), diff --git a/lib/sidecar/trtllm/src/engine.rs b/lib/sidecar/trtllm/src/engine.rs index 2671ba619bbc..afd6fddf1f97 100644 --- a/lib/sidecar/trtllm/src/engine.rs +++ b/lib/sidecar/trtllm/src/engine.rs @@ -18,7 +18,7 @@ use tokio::sync::OnceCell; use tokio_util::sync::CancellationToken; use crate::args::Args; -use crate::client::{self, ModelMetadata, TrtllmClient}; +use crate::client::{self, ModelInfoResponse, ModelMetadata, TrtllmClient}; use crate::convert::{ResponseState, build_generate_request}; use crate::model::ConfiguredModel; @@ -42,6 +42,7 @@ pub struct TrtllmSidecarEngine { /// `Control.GetModelInfo`, so `generate` can derive a default `max_tokens` /// for requests that omit one. limits: OnceCell, + rediscovered_metadata: OnceCell, cancel: CancellationToken, } @@ -59,6 +60,7 @@ impl TrtllmSidecarEngine { mode, client: OnceCell::new(), limits: OnceCell::new(), + rediscovered_metadata: OnceCell::new(), cancel: CancellationToken::new(), } } @@ -167,6 +169,55 @@ impl TrtllmSidecarEngine { }; Ok((engine, config)) } + + async fn metadata_for_request( + &self, + client: &TrtllmClient, + request: &PreprocessedRequest, + ) -> Result { + let initial = self + .limits + .get() + .copied() + .ok_or_else(|| client::engine_shutdown("TensorRT-LLM sidecar is not started"))?; + if let Some(metadata) = self.rediscovered_metadata.get() { + return Ok(*metadata); + } + let has_images = request.multi_modal_data.as_ref().is_some_and(|media| { + media + .get("image_url") + .is_some_and(|images| !images.is_empty()) + }); + if !has_images || initial.supports_image.is_some() { + return Ok(initial); + } + + self.rediscovered_metadata + .get_or_try_init(|| async { + match client.model_limits(&self.model.source).await? { + ModelInfoResponse::Available(mut reported) => { + if let Some(engine_context_length) = reported.context_length + && Some(engine_context_length) != initial.context_length + { + tracing::warn!( + configured_context_length = ?initial.context_length, + engine_context_length, + "--context-length disagrees with the context length TensorRT-LLM \ + reported; using the configured --context-length" + ); + } + reported.context_length = initial.context_length; + Ok(reported) + } + ModelInfoResponse::Unsupported(_) => Ok(ModelMetadata { + supports_image: Some(false), + ..initial + }), + } + }) + .await + .copied() + } } #[async_trait] @@ -205,20 +256,26 @@ impl LLMEngine for TrtllmSidecarEngine { // `Watch`). Do not grow a second readiness protocol here. Some(configured) => { let reported = match client.model_limits(&model.source).await { - Ok(reported) => reported, + Ok(ModelInfoResponse::Available(reported)) => reported, + Ok(ModelInfoResponse::Unsupported(status)) => { + tracing::warn!( + code = ?status.code(), + message = status.message(), + configured_context_length = configured, + "Control.GetModelInfo is unavailable; using the configured \ + --context-length" + ); + ModelMetadata { + supports_image: Some(false), + ..Default::default() + } + } Err(error) => { - // Tolerated because some OpenEngine servicer builds ship - // without Control, which is what --context-length exists - // for. UNIMPLEMENTED also fits an engine left on the - // default --grpc-protocol smg, where Generate will fail - // too -- name both so the log points at the real fix. tracing::warn!( %error, configured_context_length = configured, - "Control.GetModelInfo failed; using the configured \ - --context-length. If this is UNIMPLEMENTED, the engine either \ - ships no Control service or was started without \ - --grpc-protocol openengine" + "Control.GetModelInfo failed transiently; using the configured \ + --context-length and deferring media capability discovery" ); ModelMetadata::default() } @@ -286,13 +343,7 @@ impl LLMEngine for TrtllmSidecarEngine { .get() .ok_or_else(|| client::engine_shutdown("TensorRT-LLM sidecar is not started"))?; let request_id = ctx.id().to_string(); - let proto_request = build_generate_request( - &request, - &request_id, - &self.model.source, - self.limits.get().copied(), - self.mode, - )?; + let limits = self.metadata_for_request(client, &request).await?; let mut state = ResponseState::new(&request, self.mode); // A decode request that took a handoff has KV transferred into it, and // the transceiver releases those blocks when the engine finishes the @@ -303,6 +354,13 @@ impl LLMEngine for TrtllmSidecarEngine { // consumer. A decode-mode request without a handoff ran locally and // has nothing to strand. let needs_cancellation_deferral = self.mode.is_decode() && request.prefill_result.is_some(); + let proto_request = build_generate_request( + request, + &request_id, + &self.model.source, + Some(limits), + self.mode, + )?; let stopped_ctx = ctx.inner_arc(); // Hoisted: `stopped()` is an async-trait method, so re-creating it per // streamed chunk costs a boxed future and a waker registration on every diff --git a/lib/sidecar/trtllm/src/tests.rs b/lib/sidecar/trtllm/src/tests.rs index 0bcda724b744..61daa182d2c9 100644 --- a/lib/sidecar/trtllm/src/tests.rs +++ b/lib/sidecar/trtllm/src/tests.rs @@ -469,7 +469,7 @@ fn limits(context_length: u32) -> Option { Some(ModelMetadata { context_length: Some(context_length), max_output_tokens: None, - supports_image: false, + supports_image: Some(false), }) } @@ -477,7 +477,7 @@ fn image_limits(context_length: u32) -> Option { Some(ModelMetadata { context_length: Some(context_length), max_output_tokens: None, - supports_image: true, + supports_image: Some(true), }) } @@ -542,7 +542,7 @@ async fn collect( fn assert_rejected(mutate: impl FnOnce(&mut PreprocessedRequest), expect: &str) { let mut req = request(); mutate(&mut req); - let error = build_generate_request(&req, "req", "model", None, AGG) + let error = build_generate_request(req, "req", "model", None, AGG) .expect_err("request must be rejected"); assert!( error.to_string().contains(expect), diff --git a/lib/sidecar/trtllm/src/tests/convert_request.rs b/lib/sidecar/trtllm/src/tests/convert_request.rs index 57007959e472..c43479a7cb40 100644 --- a/lib/sidecar/trtllm/src/tests/convert_request.rs +++ b/lib/sidecar/trtllm/src/tests/convert_request.rs @@ -8,8 +8,8 @@ use super::*; #[test] fn request_maps_sampling_stop_and_output_fields() { let request = request(); - let proto = build_generate_request(&request, "req-1", "served-model", None, AGG) - .expect("build request"); + let proto = + build_generate_request(request, "req-1", "served-model", None, AGG).expect("build request"); assert_eq!(proto.request_id, "req-1"); assert_eq!(proto.model, "served-model"); match proto.input.as_ref().expect("input") { @@ -89,7 +89,7 @@ fn image_urls_data_uris_and_uuids_are_mapped_in_order() { "image_url".to_string(), vec![Some("first-image".to_string()), None], )])); - let proto = build_generate_request(&req, "req", "model", image_limits(4096), AGG) + let proto = build_generate_request(req, "req", "model", image_limits(4096), AGG) .expect("image request"); assert_eq!(proto.media.len(), 2); assert_eq!(proto.media[0].modality, pb::Modality::Image as i32); @@ -123,7 +123,7 @@ fn mm_routing_hashes_are_not_used_as_engine_cache_uuids() { "mm_hashes": ["0123456789abcdef", "fedcba9876543210"] })); - let proto = build_generate_request(&req, "req", "model", image_limits(4096), AGG) + let proto = build_generate_request(req, "req", "model", image_limits(4096), AGG) .expect("image request"); assert!(proto.media.iter().all(|item| item.uuid.is_empty())); } @@ -134,7 +134,7 @@ fn image_media_requires_the_loaded_models_advertised_capability() { add_images(&mut req, ["https://example.com/image.png"]); for model_limits in [None, limits(4096)] { - let error = build_generate_request(&req, "req", "model", model_limits, AGG) + let error = build_generate_request(req.clone(), "req", "model", model_limits, AGG) .expect_err("a text-only or unknown model must reject images"); assert!( error @@ -159,7 +159,7 @@ fn unsupported_media_modalities_and_sources_are_rejected() { "https://example.com/media".to_string(), )], )])); - let error = build_generate_request(&req, "req", "model", image_limits(4096), AGG) + let error = build_generate_request(req, "req", "model", image_limits(4096), AGG) .expect_err("only images are supported"); assert!( error.to_string().contains(expected), @@ -169,7 +169,7 @@ fn unsupported_media_modalities_and_sources_are_rejected() { let mut req = request(); add_images(&mut req, ["file:///tmp/image.png"]); - let error = build_generate_request(&req, "req", "model", image_limits(4096), AGG) + let error = build_generate_request(req, "req", "model", image_limits(4096), AGG) .expect_err("the engine cannot fetch a local Dynamo path"); assert!(error.to_string().contains("http, https, or data URI")); @@ -178,7 +178,7 @@ fn unsupported_media_modalities_and_sources_are_rejected() { "image_url".to_string(), vec![MultimodalData::UuidOnly("cached-image".to_string())], )])); - let error = build_generate_request(&req, "req", "model", image_limits(4096), AGG) + let error = build_generate_request(req, "req", "model", image_limits(4096), AGG) .expect_err("OpenEngine cannot resolve a source-less UUID"); assert!(error.to_string().contains("requires an image source")); } @@ -191,19 +191,19 @@ fn image_metadata_that_cannot_be_honored_is_rejected() { "image_url".to_string(), vec![Some("one".to_string()), Some("two".to_string())], )])); - let error = build_generate_request(&req, "req", "model", image_limits(4096), AGG) + let error = build_generate_request(req.clone(), "req", "model", image_limits(4096), AGG) .expect_err("UUIDs must align positionally"); assert!(error.to_string().contains("UUID count")); req.multi_modal_uuids = None; req.mm_processor_kwargs = Some(serde_json::json!({"do_resize": false})); - let error = build_generate_request(&req, "req", "model", image_limits(4096), AGG) + let error = build_generate_request(req.clone(), "req", "model", image_limits(4096), AGG) .expect_err("processor overrides cannot be dropped"); assert!(error.to_string().contains("mm_processor_kwargs")); req.mm_processor_kwargs = Some(serde_json::json!({})); req.media_io_kwargs = Some(serde_json::json!({"image": {"mode": "RGB"}})); - let error = build_generate_request(&req, "req", "model", image_limits(4096), AGG) + let error = build_generate_request(req, "req", "model", image_limits(4096), AGG) .expect_err("media I/O overrides cannot be dropped"); assert!(error.to_string().contains("media_io_kwargs")); } @@ -213,7 +213,7 @@ fn image_request_without_max_tokens_is_left_for_engine_derivation() { let mut req = request(); add_images(&mut req, ["https://example.com/image.png"]); req.stop_conditions.max_tokens = None; - let proto = build_generate_request(&req, "req", "model", image_limits(4096), AGG) + let proto = build_generate_request(req, "req", "model", image_limits(4096), AGG) .expect("the engine derives against the expanded prompt"); assert_eq!(proto.stopping.expect("stopping").max_tokens, None); } @@ -222,7 +222,7 @@ fn image_request_without_max_tokens_is_left_for_engine_derivation() { fn omitted_max_tokens_without_context_length_is_rejected() { let mut req = request(); req.stop_conditions.max_tokens = None; - let error = build_generate_request(&req, "req", "model", None, AGG) + let error = build_generate_request(req, "req", "model", None, AGG) .expect_err("must require max_tokens"); assert!(error.to_string().contains("max_tokens")); } @@ -233,8 +233,8 @@ fn omitted_max_tokens_defaults_to_remaining_context() { req.stop_conditions.max_tokens = None; // request() carries three prompt tokens ([11, 22, 33]); the default fills the // remaining context: max(1, context_length - prompt_len). - let proto = build_generate_request(&req, "req", "model", limits(100), AGG) - .expect("build with fallback"); + let proto = + build_generate_request(req, "req", "model", limits(100), AGG).expect("build with fallback"); assert_eq!(proto.stopping.unwrap().max_tokens, Some(97)); } @@ -244,7 +244,7 @@ fn omitted_max_tokens_default_is_floored_at_one() { req.stop_conditions.max_tokens = None; // Prompt already fills (or exceeds) the context: default must not underflow to 0. let proto = - build_generate_request(&req, "req", "model", limits(2), AGG).expect("build with fallback"); + build_generate_request(req, "req", "model", limits(2), AGG).expect("build with fallback"); assert_eq!(proto.stopping.unwrap().max_tokens, Some(1)); } @@ -252,7 +252,7 @@ fn omitted_max_tokens_default_is_floored_at_one() { fn top_k_all_tokens_is_left_unset() { let mut req = request(); req.sampling_options.top_k = Some(-1); - let proto = build_generate_request(&req, "req", "model", None, AGG).expect("build"); + let proto = build_generate_request(req, "req", "model", None, AGG).expect("build"); assert_eq!(proto.sampling.unwrap().top_k, None); } @@ -301,7 +301,7 @@ fn logprobs_zero_keeps_selected_without_alternatives() { req.output_options.logprobs = Some(0); // The wire request must still ask TRT-LLM for one candidate so the selected // token's logprob is computed. - let proto = build_generate_request(&req, "req", "model", None, AGG).expect("build"); + let proto = build_generate_request(req.clone(), "req", "model", None, AGG).expect("build"); match proto .response .unwrap() @@ -331,13 +331,13 @@ fn the_derived_max_tokens_respects_the_engines_output_cap() { let mut req = request(); req.stop_conditions.max_tokens = None; let proto = build_generate_request( - &req, + req, "req", "model", Some(ModelMetadata { context_length: Some(4096), max_output_tokens: Some(256), - supports_image: false, + supports_image: Some(false), }), AGG, ) @@ -363,7 +363,7 @@ fn a_derived_budget_below_min_tokens_is_rejected() { let prompt_len = req.token_ids.len() as u32; let window = prompt_len + 8; - let error = build_generate_request(&req, "req", "model", limits(window), AGG) + let error = build_generate_request(req, "req", "model", limits(window), AGG) .expect_err("a minimum above the remaining budget cannot be served"); let message = error.to_string(); assert!( @@ -383,7 +383,7 @@ fn a_prefill_request_does_not_judge_a_minimum_against_its_own_window() { req.stop_conditions.min_tokens = Some(64); let window = req.token_ids.len() as u32 + 8; - build_generate_request(&req, "req", "model", limits(window), PREFILL) + build_generate_request(req, "req", "model", limits(window), PREFILL) .expect("only the decode worker's window governs the minimum"); } @@ -396,7 +396,7 @@ fn a_minimum_within_the_derived_budget_is_served() { req.stop_conditions.min_tokens = Some(8); let window = req.token_ids.len() as u32 + 64; - let proto = build_generate_request(&req, "req", "model", limits(window), AGG) + let proto = build_generate_request(req, "req", "model", limits(window), AGG) .expect("a minimum that fits is not a conflict"); let stopping = proto.stopping.expect("stopping options"); assert_eq!(stopping.min_tokens, Some(8)); diff --git a/lib/sidecar/trtllm/src/tests/disagg.rs b/lib/sidecar/trtllm/src/tests/disagg.rs index a81c8dfe92c8..45f93e949efd 100644 --- a/lib/sidecar/trtllm/src/tests/disagg.rs +++ b/lib/sidecar/trtllm/src/tests/disagg.rs @@ -17,7 +17,7 @@ fn prefill_request_is_marked_context_only() { req.stop_conditions.max_tokens = Some(128); req.stop_conditions.min_tokens = Some(8); req.output_options.logprobs = Some(1); - let proto = build_generate_request(&req, "req", "model", None, DisaggregationMode::Prefill) + let proto = build_generate_request(req, "req", "model", None, DisaggregationMode::Prefill) .expect("build prefill request"); assert!( @@ -81,7 +81,7 @@ fn image_media_is_preserved_across_prefill_and_decode_requests() { )])); let prefill = build_generate_request( - &req, + req.clone(), "req", "model", image_limits(4096), @@ -100,7 +100,7 @@ fn image_media_is_preserved_across_prefill_and_decode_requests() { prompt_tokens_details: None, }); let decode = build_generate_request( - &req, + req, "req", "model", image_limits(4096), @@ -220,7 +220,7 @@ async fn decode_request_replays_the_prefill_session() { #[test] fn decode_without_a_prefill_result_runs_the_whole_request() { let proto = build_generate_request( - &request(), + request(), "req", "model", limits(4096), @@ -251,7 +251,7 @@ fn prefill_result_on_a_non_decode_worker_is_rejected() { prompt_tokens_details: None, }); for mode in [DisaggregationMode::Aggregated, DisaggregationMode::Prefill] { - let error = build_generate_request(&req, "req", "model", None, mode) + let error = build_generate_request(req.clone(), "req", "model", None, mode) .expect_err("handoff must be rejected"); assert!( error diff --git a/lib/sidecar/trtllm/src/tests/engine.rs b/lib/sidecar/trtllm/src/tests/engine.rs index 149303a06d3b..aff3ef99a66e 100644 --- a/lib/sidecar/trtllm/src/tests/engine.rs +++ b/lib/sidecar/trtllm/src/tests/engine.rs @@ -441,6 +441,64 @@ async fn a_configured_context_length_survives_a_server_without_control() { config.llm.expect("llm registration").context_length, Some(8192) ); + + let mut image_request = request(); + add_images(&mut image_request, ["https://example.com/image.png"]); + let error = match engine + .generate( + image_request, + GenerateContext::new(dynamo_backend_common::testing::mock_context(), None), + ) + .await + { + Ok(_) => panic!("a server without capability discovery must reject images"), + Err(error) => error, + }; + assert!( + error + .to_string() + .contains("does not advertise image support") + ); + assert_eq!( + server.service.model_info_calls.load(Ordering::SeqCst), + 1, + "a definitive UNIMPLEMENTED response must not be retried" + ); +} + +#[tokio::test] +async fn image_request_retries_transient_capability_discovery() { + let service = FakeTrtllm::default(); + service.unavailable_model_info.store(true, Ordering::SeqCst); + let server = FakeServer::start(service).await; + let engine = engine_with(&server.endpoint, transport(1), Some(8192), AGG); + engine.start(0).await.expect("start with configured window"); + + server + .service + .unavailable_model_info + .store(false, Ordering::SeqCst); + server + .service + .supports_multimodal + .store(true, Ordering::SeqCst); + server + .service + .advertises_image + .store(true, Ordering::SeqCst); + let mut image_request = request(); + add_images(&mut image_request, ["https://example.com/image.png"]); + + let output = collect(&engine, image_request).await; + assert_eq!( + output.last().unwrap().finish_reason, + Some(FinishReason::Stop) + ); + assert_eq!( + server.service.model_info_calls.load(Ordering::SeqCst), + 2, + "the image request must retry the transient startup failure once" + ); } /// Shutdown has to terminate in-flight requests: the worker cannot drain if a diff --git a/tests/serve/test_sidecar.py b/tests/serve/test_sidecar.py index 55720fee9b12..0375aa862e0d 100644 --- a/tests/serve/test_sidecar.py +++ b/tests/serve/test_sidecar.py @@ -211,10 +211,6 @@ def _disaggregated_payloads( pytest.mark.requested_trtllm_kv_tokens(32768), pytest.mark.timeout(960), pytest.mark.pre_merge, - pytest.mark.skipif( - not _trtllm_serves_openengine(), - reason=TRTLLM_OPENENGINE_SKIP_REASON, - ), ], model="Qwen/Qwen3-VL-2B-Instruct", timeout=900, @@ -339,6 +335,10 @@ def _disaggregated_payloads( @pytest.fixture(params=params_with_model_mark(sidecar_configs)) def sidecar_config_test(request, monkeypatch, discovery_backend): config = sidecar_configs[request.param] + if config.name == "trtllm_aggregated_image" and not _trtllm_serves_openengine(): + if os.environ.get("DYNAMO_TEST_SIDECAR_BACKEND") == "trtllm": + pytest.fail(TRTLLM_OPENENGINE_SKIP_REASON) + pytest.skip(TRTLLM_OPENENGINE_SKIP_REASON) monkeypatch.setenv("MODEL", config.model) monkeypatch.setenv("DYN_DISCOVERY_BACKEND", discovery_backend) monkeypatch.setenv("DYN_NAMESPACE", f"sidecar-{generate_random_suffix()}") From c47b92e4d472ffcbcdd290cc58950a5982827ed7 Mon Sep 17 00:00:00 2001 From: krishung5 Date: Fri, 16 Oct 2026 14:01:16 -0700 Subject: [PATCH 19/61] fix(ci): build sidecar image on compatible TRT-LLM base --- .github/workflows/pr.yaml | 3 +- container/compliance/policy/codec_policy.yaml | 26 ++++---- container/templates/trtllm_runtime.Dockerfile | 64 +++++++++++-------- 3 files changed, 54 insertions(+), 39 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 4eb2b9312608..f016a5acd227 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -542,9 +542,8 @@ jobs: diff_current_branch: ${{ github.ref_name }} diff_base_branch: ${{ needs.changed-files.outputs.base_ref }} diff_merge_base_sha: ${{ needs.changed-files.outputs.merge_base_sha }} - # Drop these overrides after the upstream TensorRT-LLM change ships. + # Drop this source overlay after the upstream TensorRT-LLM change ships. extra_build_args: | - RUNTIME_IMAGE_TAG=1.4.0rc1.dev202610140007@sha256:7a728cd307bfa1651b4ad0787e71e69f7506fb1a698a0f4f04aef719ad43525f TRTLLM_SOURCE_REPOSITORY=https://github.com/krishung5/TensorRT-LLM TRTLLM_SOURCE_COMMIT=5d2e3b2c758b4ccc0fa0a3eb0e01cdef934e8089 # actions: read lets the build download a prior run's baseline compliance diff --git a/container/compliance/policy/codec_policy.yaml b/container/compliance/policy/codec_policy.yaml index 61b8f39e19a0..9e4023b0e2e3 100644 --- a/container/compliance/policy/codec_policy.yaml +++ b/container/compliance/policy/codec_policy.yaml @@ -108,9 +108,9 @@ exceptions: (NVIDIA/DALI_deps#162, NVIDIA/DALI#6352, May 2026), first released in 2.1.1 -- as a cherry-pick, so checking whether the merge commit is an ancestor of the tag gives the wrong answer; the release notes are the - reliable source. Upgrading the wheel in place to 2.1.1, 2.2.0, or 2.3.0 - leaves 440 decoders with all five absent, vp8/vp9/mjpeg/av1 still present, - and the image importing and passing its unit suite. + reliable source. Upgrading the wheel in place to 2.1.1 or 2.2.0 leaves 440 + decoders with all five absent, vp8/vp9/mjpeg/av1 still present, and the + image importing and passing its unit suite. This waiver cannot be narrowed to the non-codec companions, which an earlier revision of this note claimed. libavcodec, libavfilter, libavformat, @@ -120,12 +120,13 @@ exceptions: set is all-or-nothing with the package. The entry stays even so. trtllm_runtime.Dockerfile pins - nvidia-dali-cuda130 at an explicitly measured version: 2.2.0 in the - release base or 2.3.0 in the pinned development base. The vendored - libraries do not register those decoders -- but they are still on disk and - still match the deny globs above, which gate on presence rather than - content. What changed is the reason for the waiver: it now covers a vendored - ffmpeg that carries no software h264/hevc/aac, rather than one that does. + nvidia-dali-cuda130==2.2.0, which is the version the 1.3.0rc27 base image + already carries, so the pin now holds a version rather than moving one. The + vendored libraries do not register those decoders -- but they are still on + disk and still match the deny globs above, which gate on presence rather + than content. What changed is the reason for the waiver: it now covers a + vendored ffmpeg that carries no software h264/hevc/aac, rather than one + that does. Enforcement lives at build time, not here. Two guards in trtllm_runtime.Dockerfile enumerate every bundled libavcodec -- one in @@ -141,9 +142,10 @@ exceptions: subtracted from the generated delta, so such an entry would never match anything and would read as a check while doing nothing. - Re-measure when the vendored version moves. Last measured against 2.2.0 and - 2.3.0: 440 decoders, h264, hevc, aac, aac_fixed and aac_latm absent, vp8, - vp9, mjpeg and av1 present. + Re-measure when the vendored version moves. Last measured against + nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc27, whose base carries 2.2.0: + 440 decoders, h264, hevc, aac, aac_fixed and aac_latm absent, vp8, vp9, + mjpeg and av1 present. owner: DALI # PyNvVideoCodec (added for NVDEC hardware video decode) vendors libavutil and # libavformat under its package dir to demux input containers before handing the diff --git a/container/templates/trtllm_runtime.Dockerfile b/container/templates/trtllm_runtime.Dockerfile index b71d05bdb1e9..82661874f3e6 100644 --- a/container/templates/trtllm_runtime.Dockerfile +++ b/container/templates/trtllm_runtime.Dockerfile @@ -44,7 +44,8 @@ ARG NIXL_REF ARG TRTLLM_SOURCE_REPOSITORY ARG TRTLLM_SOURCE_COMMIT -# Optional exact-source overlay for cross-repository CI. +# Optional source overlay for cross-repository CI. Keep the base image's server +# lifecycle entry points, which are coupled to that release's frontend runtime. RUN if [ -n "${TRTLLM_SOURCE_COMMIT}" ]; then \ test -n "${TRTLLM_SOURCE_REPOSITORY}" && \ source_dir=$(mktemp -d) && \ @@ -55,8 +56,17 @@ RUN if [ -n "${TRTLLM_SOURCE_COMMIT}" ]; then \ --strip-components=1 && \ package_dir=$(/usr/bin/python3 -c \ 'import importlib.util; print(next(iter(importlib.util.find_spec("tensorrt_llm").submodule_search_locations)))') && \ - cp -a "${source_dir}/tensorrt_llm/grpc/openengine/." \ - "${package_dir}/grpc/openengine/" && \ + for entry in "${source_dir}/tensorrt_llm/grpc/openengine/"*; do \ + case "$(basename "${entry}")" in \ + __init__.py|server.py) continue ;; \ + esac; \ + cp -a "${entry}" "${package_dir}/grpc/openengine/"; \ + done && \ + sed -i \ + 's/from openengine\.v1 import openengine_pb2_grpc/from .bindings import openengine_pb2_grpc/' \ + "${package_dir}/grpc/openengine/server.py" && \ + grep -Fxq 'from .bindings import openengine_pb2_grpc' \ + "${package_dir}/grpc/openengine/server.py" && \ install -m 0644 "${source_dir}/tensorrt_llm/inputs/media_io.py" \ "${package_dir}/inputs/media_io.py" && \ rm -rf "${source_dir}"; \ @@ -403,9 +413,10 @@ RUN /usr/bin/python3 -m pip uninstall -y --break-system-packages opencv-python-h # Hold DALI at a version that carries its own media-codec cleanup. Upstream # restricted DALI's vendored ffmpeg build to drop the software h264/hevc/aac # decoders (NVIDIA/DALI_deps#162, NVIDIA/DALI#6352), first released in 2.1.1. -# The supported base images carry either 2.2.0 or 2.3.0. Both were measured -# without the restricted codecs; older versions are upgraded to 2.2.0 and any -# other newer version stops the build for re-measurement. +# The base image carried 2.1.0 through 1.3.0rc26, which predates it, and carries +# 2.2.0 from 1.3.0rc27, which does not. So on this base the install below is a +# no-op and what the block still buys is the two checks around it: the version +# tripwire and the decoder enumeration. # # Pin rather than remove. The vendored libav*/libsw* set cannot be trimmed # on its own -- every one of them is DT_NEEDED by libdali.so and its siblings, so @@ -415,17 +426,23 @@ RUN /usr/bin/python3 -m pip uninstall -y --break-system-packages opencv-python-h # out of someone else's image. # # Measured on the shipped image: 2.1.0 registers 446 decoders including h264, -# hevc, aac, aac_fixed and aac_latm; 2.1.1, 2.2.0, and 2.3.0 each register 440 -# with all five absent and vp8/vp9/mjpeg/av1 retained. +# hevc, aac, aac_fixed and aac_latm; 2.1.1 and 2.2.0 each register 440 with all +# five absent and vp8/vp9/mjpeg/av1 retained. 2.2.0 measured again on +# tensorrt-llm/release:1.3.0rc27 itself, now that DALI arrives with the base. # # Pinned exactly, not a floor. `>=2.1.1` resolves to whatever is newest, which is # a jump into a component this repo does not own, taken silently at build time. # The pin names the version that was measured, so moving it means measuring the # new one. # -# A base below 2.2.0 is upgraded to the measured 2.2.0 wheel. The two measured -# versions otherwise stay in place; any unmeasured newer version fails rather -# than being downgraded or accepted silently. +# The pin can go stale in the one direction that matters: if TensorRT-LLM later +# ships a base image with a newer DALI, installing 2.2.0 over it is a downgrade. +# The check below turns that into a build failure with instructions rather than a +# silent regression. The pin now equals the base version, so the next base image +# that moves DALI past 2.2.0 stops the build, which is the point: a human +# re-measures and moves the pin, or deletes this RUN and lets the base version +# stand. A base that moves DALI back below 2.2.0 passes that check and the +# install below puts the measured version back, which is what it is there for. # # THIS BLOCK IS TRANSITIONAL, and on this base it is already down to its checks. # The guards after it and the bundled-libavcodec assertion in @@ -450,23 +467,20 @@ RUN --mount=type=bind,source=./container/compliance/enumerate_bundled_decoders.p set -eu; \ before=$(/usr/bin/python3 -c 'import importlib.metadata as m; print(m.version("nvidia-dali-cuda130"))'); \ echo "DALI in base image: $before"; \ - case "$before" in \ - 2.3.0) target=2.3.0 ;; \ - *) \ - newest=$(printf '%s\n2.2.0\n' "$before" | sort -V | tail -1); \ - if [ "$newest" != "2.2.0" ]; then \ - echo "ERROR: base image carries unmeasured DALI $before" >&2; \ - exit 1; \ - fi; \ - target=2.2.0 ;; \ - esac; \ - if [ "$before" != "$target" ]; then \ - /usr/bin/python3 -m pip install --break-system-packages --no-cache-dir \ - --extra-index-url https://pypi.nvidia.com "nvidia-dali-cuda130==$target"; \ + newest=$(printf '%s\n2.2.0\n' "$before" | sort -V | tail -1); \ + if [ "$newest" != "2.2.0" ]; then \ + echo "ERROR: base image already carries DALI $before, so pinning 2.2.0 would" >&2; \ + echo " downgrade it. Upstream has caught up -- delete this RUN and let" >&2; \ + echo " the base version stand. Keep the guards below and the bundled-" >&2; \ + echo " libavcodec assertion in tests/dependencies/, which are what stop" >&2; \ + echo " this from regressing." >&2; \ + exit 1; \ fi; \ + /usr/bin/python3 -m pip install --break-system-packages --no-cache-dir \ + --extra-index-url https://pypi.nvidia.com 'nvidia-dali-cuda130==2.2.0'; \ v=$(/usr/bin/python3 -c 'import importlib.metadata as m; print(m.version("nvidia-dali-cuda130"))'); \ echo "DALI version: $v"; \ - [ "$v" = "$target" ] || { echo "ERROR: wanted DALI $target, got $v" >&2; exit 1; }; \ + [ "$v" = "2.2.0" ] || { echo "ERROR: wanted DALI 2.2.0, got $v" >&2; exit 1; }; \ libs=$(find /usr/local/lib/python3.12/dist-packages/nvidia/dali/.libs -name 'libavcodec*.so*'); \ if [ -z "$libs" ]; then \ echo "ERROR: DALI $v ships no libavcodec under .libs, so the enumeration" >&2; \ From 391fad4c91d70c5f7c9039b5935afa503046873f Mon Sep 17 00:00:00 2001 From: krishung5 Date: Fri, 16 Oct 2026 15:07:39 -0700 Subject: [PATCH 20/61] test(trtllm): remove redundant image cases --- .../trtllm/src/tests/convert_request.rs | 42 ++++++------------- 1 file changed, 13 insertions(+), 29 deletions(-) diff --git a/lib/sidecar/trtllm/src/tests/convert_request.rs b/lib/sidecar/trtllm/src/tests/convert_request.rs index c43479a7cb40..31d47129d23a 100644 --- a/lib/sidecar/trtllm/src/tests/convert_request.rs +++ b/lib/sidecar/trtllm/src/tests/convert_request.rs @@ -147,25 +147,19 @@ fn image_media_requires_the_loaded_models_advertised_capability() { #[test] fn unsupported_media_modalities_and_sources_are_rejected() { - for (key, expected) in [ - ("video_url", "video_url"), - ("audio_url", "audio_url"), - ("future_media", "future_media"), - ] { - let mut req = request(); - req.multi_modal_data = Some(HashMap::from([( - key.to_string(), - vec![MultimodalData::RawUrl( - "https://example.com/media".to_string(), - )], - )])); - let error = build_generate_request(req, "req", "model", image_limits(4096), AGG) - .expect_err("only images are supported"); - assert!( - error.to_string().contains(expected), - "unexpected error: {error}" - ); - } + let mut req = request(); + req.multi_modal_data = Some(HashMap::from([( + "video_url".to_string(), + vec![MultimodalData::RawUrl( + "https://example.com/media".to_string(), + )], + )])); + let error = build_generate_request(req, "req", "model", image_limits(4096), AGG) + .expect_err("only images are supported"); + assert!( + error.to_string().contains("video_url"), + "unexpected error: {error}" + ); let mut req = request(); add_images(&mut req, ["file:///tmp/image.png"]); @@ -208,16 +202,6 @@ fn image_metadata_that_cannot_be_honored_is_rejected() { assert!(error.to_string().contains("media_io_kwargs")); } -#[test] -fn image_request_without_max_tokens_is_left_for_engine_derivation() { - let mut req = request(); - add_images(&mut req, ["https://example.com/image.png"]); - req.stop_conditions.max_tokens = None; - let proto = build_generate_request(req, "req", "model", image_limits(4096), AGG) - .expect("the engine derives against the expanded prompt"); - assert_eq!(proto.stopping.expect("stopping").max_tokens, None); -} - #[test] fn omitted_max_tokens_without_context_length_is_rejected() { let mut req = request(); From 46eade0ac53cc22fe0e062e3fd78c7d90ba3072c Mon Sep 17 00:00:00 2001 From: krishung5 Date: Thu, 15 Oct 2026 20:37:16 -0700 Subject: [PATCH 21/61] feat(trtllm): pass video inputs through OpenEngine Signed-off-by: krishung5 --- .github/workflows/pr.yaml | 2 +- lib/backend-common/src/engine.rs | 3 +- lib/backend-common/src/lib.rs | 4 +- lib/llm/src/preprocessor.rs | 63 ++++- lib/llm/src/protocols/common/preprocessor.rs | 12 + lib/sidecar/trtllm/src/client.rs | 62 +++-- lib/sidecar/trtllm/src/convert.rs | 215 +++++++++++++----- lib/sidecar/trtllm/src/engine.rs | 16 +- lib/sidecar/trtllm/src/tests.rs | 41 +++- .../trtllm/src/tests/convert_request.rs | 137 ++++++++++- lib/sidecar/trtllm/src/tests/disagg.rs | 58 +++-- lib/sidecar/trtllm/src/tests/e2e.rs | 74 ++++++ lib/sidecar/trtllm/src/tests/engine.rs | 106 ++++++--- tests/serve/test_sidecar.py | 40 +++- 14 files changed, 672 insertions(+), 161 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index f016a5acd227..a528c4c4d48a 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -545,7 +545,7 @@ jobs: # Drop this source overlay after the upstream TensorRT-LLM change ships. extra_build_args: | TRTLLM_SOURCE_REPOSITORY=https://github.com/krishung5/TensorRT-LLM - TRTLLM_SOURCE_COMMIT=5d2e3b2c758b4ccc0fa0a3eb0e01cdef934e8089 + TRTLLM_SOURCE_COMMIT=9db976c9c2fb7f6a03fc5db96bf58c71d3b0d689 # actions: read lets the build download a prior run's baseline compliance # artifact to diff against (a reusable workflow's token can't exceed the # caller's). Missing -> the diff soft-degrades, build stays green. diff --git a/lib/backend-common/src/engine.rs b/lib/backend-common/src/engine.rs index 2c5daf1643e9..10e2583879d8 100644 --- a/lib/backend-common/src/engine.rs +++ b/lib/backend-common/src/engine.rs @@ -28,7 +28,8 @@ pub use dynamo_llm::protocols::common::llm_backend::{ }; pub use dynamo_llm::protocols::common::preprocessor::{ BootstrapInfo, KV_HINT_TRANSFER_CAPABILITY_KEY, KvHint, KvHintAction, KvSourceLocationsPayload, - MultimodalData, MultimodalDataMap, PrefillResult, PreprocessedRequest, RoutingHints, + MultimodalData, MultimodalDataMap, MultimodalItemRef, PrefillResult, PreprocessedRequest, + RoutingHints, }; pub use dynamo_llm::protocols::common::{ FinishReason, GuidedDecodingOptions, OutputOptions, SamplingOptions, StopConditions, diff --git a/lib/backend-common/src/lib.rs b/lib/backend-common/src/lib.rs index fd749de7e1ab..affa584da728 100644 --- a/lib/backend-common/src/lib.rs +++ b/lib/backend-common/src/lib.rs @@ -38,8 +38,8 @@ pub use engine::{ FinishReason, FirstTokenNotifier, GenerateContext, GuidedDecodingOptions, HEALTH_CHECK_KEY, KV_HINT_TRANSFER_CAPABILITY_KEY, KvEventPublisher, KvEventSource, KvHint, KvHintAction, KvSourceLocationsPayload, LLMEngine, LLMEngineOutput, LLMEngineOutputExt, LlmRegistration, - LogProbs, Metrics, MetricsBindings, MetricsCtx, MultimodalData, OnPublisherReady, - OnSnapshotPublisherReady, OutputOptions, PrefillResult, PreprocessedRequest, + LogProbs, Metrics, MetricsBindings, MetricsCtx, MultimodalData, MultimodalItemRef, + OnPublisherReady, OnSnapshotPublisherReady, OutputOptions, PrefillResult, PreprocessedRequest, PromptTokensDetails, RawEngine, SamplingOptions, StopConditions, StopReason, TopLogprob, TopLogprobs, chunk, usage, }; diff --git a/lib/llm/src/preprocessor.rs b/lib/llm/src/preprocessor.rs index e6a67b6f3b45..1ffe83b3cc83 100644 --- a/lib/llm/src/preprocessor.rs +++ b/lib/llm/src/preprocessor.rs @@ -70,7 +70,8 @@ use crate::model_card::{ModelDeploymentCard, ModelInfo, PromptFormatterArtifact} use crate::preprocessor::media::MediaFetcher; use crate::preprocessor::media::{MediaDecoder, MediaLoader, max_data_url_bytes}; use crate::protocols::common::preprocessor::{ - MultimodalData, MultimodalDataMap, MultimodalUuidMap, PreprocessedRequestBuilder, RoutingHints, + MultimodalData, MultimodalDataMap, MultimodalItemRef, MultimodalUuidMap, + PreprocessedRequestBuilder, RoutingHints, }; use crate::protocols::common::timing::RequestTracker; use crate::tokenizers::Encoding; @@ -3614,6 +3615,7 @@ impl OpenAIPreprocessor { let mut media_map: MultimodalDataMap = HashMap::new(); let mut uuid_map: MultimodalUuidMap = HashMap::new(); + let mut media_order: Vec = Vec::new(); let mut has_user_uuid = false; // Decoded results are written back into these reserved modality slots so // URL-backed and UUID-only inputs retain request order. @@ -3693,6 +3695,10 @@ impl OpenAIPreprocessor { let slots = media_map.entry(type_str.to_string()).or_default(); let slot_idx = slots.len(); + media_order.push(MultimodalItemRef { + modality: type_str.to_string(), + index: slot_idx, + }); has_user_uuid |= uuid.is_some(); uuid_map .entry(type_str.to_string()) @@ -3970,6 +3976,7 @@ impl OpenAIPreprocessor { if !media_map.is_empty() { builder.multi_modal_data(Some(media_map)); + builder.multi_modal_order(Some(media_order)); if has_user_uuid { builder.multi_modal_uuids(Some(uuid_map)); } @@ -8476,6 +8483,60 @@ mod extra_args_media_copy_tests { .to_string() } + #[tokio::test] + async fn multimodal_order_tracks_the_rendered_media_sequence() { + let preprocessor = test_preprocessor(); + let image = inline_data_url(); + let request: NvCreateChatCompletionRequest = serde_json::from_value(serde_json::json!({ + "model": "alias", + "messages": [{ + "role": "user", + "content": [ + {"type": "video_url", "video_url": {"url": "data:video/mp4;base64,AAAA"}}, + {"type": "image_url", "image_url": {"url": image}}, + {"type": "audio_url", "audio_url": {"url": "data:audio/wav;base64,AAAA"}}, + {"type": "video_url", "video_url": {"url": "https://example.com/second.mp4"}} + ] + }] + })) + .unwrap(); + let mut builder = PreprocessedRequestBuilder::default(); + builder + .model("alias".into()) + .token_ids(Vec::new()) + .stop_conditions(Default::default()) + .sampling_options(Default::default()) + .output_options(Default::default()); + + preprocessor + .gather_multi_modal_data_with_image_tokens(&request, &mut builder, None, &[]) + .await + .unwrap(); + let result = builder.build().unwrap(); + + assert_eq!( + result.multi_modal_order.unwrap(), + [ + MultimodalItemRef { + modality: "video_url".to_string(), + index: 0, + }, + MultimodalItemRef { + modality: "image_url".to_string(), + index: 0, + }, + MultimodalItemRef { + modality: "audio_url".to_string(), + index: 0, + }, + MultimodalItemRef { + modality: "video_url".to_string(), + index: 1, + }, + ] + ); + } + #[tokio::test] async fn deepseek_v41_display_name_fallback_orders_tool_media() { use crate::common::checked_file::CheckedFile; diff --git a/lib/llm/src/protocols/common/preprocessor.rs b/lib/llm/src/protocols/common/preprocessor.rs index ed84d5571835..45c8bd2f2d90 100644 --- a/lib/llm/src/protocols/common/preprocessor.rs +++ b/lib/llm/src/protocols/common/preprocessor.rs @@ -362,6 +362,13 @@ pub type MultimodalDataMap = std::collections::HashMap>>; +/// A reference into [`MultimodalDataMap`] preserving the media traversal order. +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] +pub struct MultimodalItemRef { + pub modality: String, + pub index: usize, +} + /// [`PreprocessedRequest`] is the internal representation of an LLM request. The `dynamo.llm-preprocessor` /// crate is responsible for converting request from the public APIs to this internal representation. #[derive(Serialize, Deserialize, Clone, Builder)] @@ -421,6 +428,11 @@ pub struct PreprocessedRequest { #[serde(default, skip_serializing_if = "Option::is_none")] pub multi_modal_uuids: Option, + /// Media references in the order used to render the prompt. + #[builder(default)] + #[serde(default, skip_serializing_if = "Option::is_none")] + pub multi_modal_order: Option>, + /// Optional multimodal routing-only fields (separate from execution payload). #[builder(default)] #[serde(default, skip_serializing_if = "Option::is_none")] diff --git a/lib/sidecar/trtllm/src/client.rs b/lib/sidecar/trtllm/src/client.rs index f3daac1910c7..699159722eb5 100644 --- a/lib/sidecar/trtllm/src/client.rs +++ b/lib/sidecar/trtllm/src/client.rs @@ -27,6 +27,23 @@ use crate::proto::inference_client::InferenceClient; /// `--grpc-startup-deadline-secs` instead. const RPC_TIMEOUT: Duration = Duration::from_secs(30); +/// Media modalities confirmed by both OpenEngine capability signals. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct SupportedMedia { + pub(crate) image: bool, + pub(crate) video: bool, +} + +impl SupportedMedia { + pub(crate) fn contains(self, modality: &str) -> bool { + match modality { + "image" => self.image, + "video" => self.video, + _ => false, + } + } +} + /// Startup metadata from `--context-length` and `Control.GetModelInfo`. #[derive(Clone, Copy, Debug, Default)] pub(crate) struct ModelMetadata { @@ -37,9 +54,9 @@ pub(crate) struct ModelMetadata { /// Cap on generated tokens, when the engine reports one. A context window /// alone can imply a larger budget than the engine will accept. pub(crate) max_output_tokens: Option, - /// Whether both portable and engine-specific capabilities advertise images. + /// Modalities advertised by both the portable and engine-specific capabilities. /// `None` means startup discovery failed transiently and may be retried. - pub(crate) supports_image: Option, + pub(crate) supported_media: Option, } pub(crate) enum ModelInfoResponse { @@ -109,29 +126,30 @@ impl TrtllmClient { }) .await? .into_inner(); - let supports_image = info.supports_multimodal == Some(true) - && info.extra.as_ref().is_some_and(|extra| { - extra - .fields - .get("trtllm_supported_modalities") - .and_then(|value| value.kind.as_ref()) - .is_some_and(|kind| match kind { - prost_types::value::Kind::ListValue(modalities) => { - modalities.values.iter().any(|value| { - matches!( - value.kind.as_ref(), - Some(prost_types::value::Kind::StringValue(modality)) - if modality == "image" - ) - }) - } - _ => false, - }) - }); + let mut supported_media = SupportedMedia::default(); + if info.supports_multimodal == Some(true) + && let Some(prost_types::value::Kind::ListValue(modalities)) = info + .extra + .as_ref() + .and_then(|extra| extra.fields.get("trtllm_supported_modalities")) + .and_then(|value| value.kind.as_ref()) + { + for value in &modalities.values { + let Some(prost_types::value::Kind::StringValue(modality)) = value.kind.as_ref() + else { + continue; + }; + match modality.as_str() { + "image" => supported_media.image = true, + "video" => supported_media.video = true, + _ => {} + } + } + } Ok(ModelMetadata { context_length: info.max_context_length.filter(|len| *len > 0), max_output_tokens: info.max_output_tokens.filter(|cap| *cap > 0), - supports_image: Some(supports_image), + supported_media: Some(supported_media), }) } diff --git a/lib/sidecar/trtllm/src/convert.rs b/lib/sidecar/trtllm/src/convert.rs index 81661405df03..6e42b03b41fd 100644 --- a/lib/sidecar/trtllm/src/convert.rs +++ b/lib/sidecar/trtllm/src/convert.rs @@ -4,15 +4,15 @@ //! Conversion between Dynamo's `PreprocessedRequest` / `LLMEngineOutput` and the //! TensorRT-LLM OpenEngine (`openengine.v1`) protobuf messages. //! -//! Supports aggregated and disaggregated image generation. Video, audio, LoRA, -//! beam search, and `n > 1` are rejected before dispatch. +//! Supports aggregated and disaggregated image and video generation. Audio, +//! LoRA, beam search, and `n > 1` are rejected before dispatch. //! //! Disaggregation is driven by [`DisaggregationMode`]: a prefill worker marks //! its request `context_only` and returns the `PrefillReady` handoff as its //! terminal chunk, and a decode worker replays that handoff in `kv.session`. //! See [`crate::disagg`] for the codec. -use std::collections::BTreeSet; +use std::collections::{BTreeSet, HashSet}; use std::sync::Arc; use dynamo_backend_common::{ @@ -37,7 +37,7 @@ pub(crate) fn build_generate_request( ) -> Result { validate_request(&request, mode)?; let request_has_media = has_media(&request); - let media = image_media(&mut request, limits)?; + let media = media(&mut request, limits)?; let sampling = &request.sampling_options; let stop = &request.stop_conditions; @@ -183,9 +183,34 @@ fn max_tokens( } const IMAGE_KEY: &str = "image_url"; +const VIDEO_KEY: &str = "video_url"; -/// Per-modality vectors do not preserve cross-modality order. -fn image_media( +#[derive(Clone, Copy)] +struct MediaKind { + key: &'static str, + name: &'static str, + modality: pb::Modality, +} + +const MEDIA_KINDS: [MediaKind; 2] = [ + MediaKind { + key: IMAGE_KEY, + name: "image", + modality: pb::Modality::Image, + }, + MediaKind { + key: VIDEO_KEY, + name: "video", + modality: pb::Modality::Video, + }, +]; + +fn media_kind(key: &str) -> Option { + MEDIA_KINDS.iter().copied().find(|kind| kind.key == key) +} + +/// Map Dynamo's grouped media payload to OpenEngine's globally ordered list. +fn media( request: &mut PreprocessedRequest, limits: Option, ) -> Result, DynamoError> { @@ -193,88 +218,176 @@ fn image_media( reject_media_override("media_io_kwargs", request.media_io_kwargs.as_ref())?; // mm_routing_info stays router-side; multi_modal_data is engine input. - let mut media_map = request.multi_modal_data.take(); + let media_map = request.multi_modal_data.take(); if let Some(media_map) = media_map.as_ref() { for (modality, items) in media_map { - if modality != IMAGE_KEY && !items.is_empty() { + if media_kind(modality).is_none() && !items.is_empty() { return Err(client::invalid_argument(format!( - "multimodal modality `{modality}` is not supported by the TensorRT-LLM sidecar; this image-only implementation accepts `{IMAGE_KEY}`" + "multimodal modality `{modality}` is not supported by the TensorRT-LLM sidecar" ))); } } } - let images = media_map - .as_mut() - .and_then(|media| media.remove(IMAGE_KEY)) - .unwrap_or_default(); - - let mut uuid_map = request.multi_modal_uuids.take(); - let user_image_uuids = if let Some(uuid_map) = uuid_map.as_mut() { - for (modality, uuids) in uuid_map.iter() { - if modality != IMAGE_KEY && !uuids.is_empty() { + let uuid_map = request.multi_modal_uuids.take(); + if let Some(uuid_map) = uuid_map.as_ref() { + for (modality, uuids) in uuid_map { + if media_kind(modality).is_none() && !uuids.is_empty() { return Err(client::invalid_argument(format!( - "multimodal UUIDs for `{modality}` cannot be forwarded by the image-only TensorRT-LLM sidecar" + "multimodal UUIDs for unsupported modality `{modality}` cannot be forwarded" ))); } } - uuid_map.remove(IMAGE_KEY) - } else { - None - }; + } - if let Some(uuids) = user_image_uuids.as_ref() - && uuids.len() != images.len() - { - return Err(client::invalid_argument(format!( - "`{IMAGE_KEY}` UUID count ({}) must match its media item count ({})", - uuids.len(), - images.len() - ))); + let mut active_kinds = Vec::new(); + let mut total_items = 0; + for kind in MEDIA_KINDS { + let items = media_map + .as_ref() + .and_then(|media| media.get(kind.key)) + .map(Vec::as_slice) + .unwrap_or_default(); + if let Some(uuids) = uuid_map.as_ref().and_then(|uuids| uuids.get(kind.key)) + && uuids.len() != items.len() + { + return Err(client::invalid_argument(format!( + "`{}` UUID count ({}) must match its media item count ({})", + kind.key, + uuids.len(), + items.len() + ))); + } + if !items.is_empty() { + if !limits + .and_then(|limits| limits.supported_media) + .is_some_and(|media| media.contains(kind.name)) + { + return Err(client::invalid_argument(format!( + "the loaded TensorRT-LLM model does not advertise {} support through Control.GetModelInfo", + kind.name + ))); + } + active_kinds.push(kind); + total_items += items.len(); + } } - if images.is_empty() { + if total_items == 0 { return Ok(Vec::new()); } - if limits.is_none_or(|limits| limits.supports_image != Some(true)) { - return Err(client::invalid_argument( - "the loaded TensorRT-LLM model does not advertise image support through Control.GetModelInfo", - )); - } + let ordered_items: Vec<(MediaKind, usize)> = match request.multi_modal_order.take() { + Some(order) => { + if order.len() != total_items { + return Err(client::invalid_argument(format!( + "multi_modal_order has {} entries for {total_items} media items", + order.len() + ))); + } + let mut seen = HashSet::with_capacity(order.len()); + order + .into_iter() + .map(|item| { + let kind = media_kind(&item.modality).ok_or_else(|| { + client::invalid_argument(format!( + "multi_modal_order references unsupported modality `{}`", + item.modality + )) + })?; + let item_count = media_map + .as_ref() + .and_then(|media| media.get(kind.key)) + .map(Vec::len) + .unwrap_or_default(); + if item.index >= item_count { + return Err(client::invalid_argument(format!( + "multi_modal_order references missing `{}` item {}", + kind.key, item.index + ))); + } + if !seen.insert((kind.key, item.index)) { + return Err(client::invalid_argument(format!( + "multi_modal_order repeats `{}` item {}", + kind.key, item.index + ))); + } + Ok((kind, item.index)) + }) + .collect::>()? + } + None if active_kinds.len() == 1 => { + let kind = active_kinds[0]; + let item_count = media_map + .as_ref() + .and_then(|media| media.get(kind.key)) + .map(Vec::len) + .unwrap_or_default(); + (0..item_count).map(|index| (kind, index)).collect() + } + None => { + return Err(client::invalid_argument( + "mixed multimodal input requires multi_modal_order to preserve prompt order", + )); + } + }; + + let mut media_slots = media_map + .unwrap_or_default() + .into_iter() + .map(|(key, items)| (key, items.into_iter().map(Some).collect::>())) + .collect::>(); + let mut uuid_slots = uuid_map + .unwrap_or_default() + .into_iter() + .map(|(key, uuids)| (key, uuids.into_iter().map(Some).collect::>())) + .collect::>(); - let uuids = user_image_uuids.unwrap_or_else(|| vec![None; images.len()]); - images + ordered_items .into_iter() - .zip(uuids) - .enumerate() - .map(|(index, (image, uuid))| { - let source = match image { - MultimodalData::Url(url) => media_source(url.to_string(), index)?, - MultimodalData::RawUrl(url) => media_source(url, index)?, + .map(|(kind, index)| { + let item = media_slots + .get_mut(kind.key) + .and_then(|items| items.get_mut(index)) + .and_then(Option::take) + .expect("media order was validated"); + let source = match item { + MultimodalData::Url(url) => media_source(url.to_string(), kind.key, index)?, + MultimodalData::RawUrl(url) => media_source(url, kind.key, index)?, MultimodalData::Decoded(_) => { return Err(client::invalid_argument(format!( - "`{IMAGE_KEY}` item {index} is pre-decoded; OpenEngine requires a URL or data URI so TensorRT-LLM can own image decoding" + "`{}` item {index} is pre-decoded; OpenEngine requires a URL or data URI so TensorRT-LLM can own {} decoding", + kind.key, kind.name ))); } MultimodalData::UuidOnly(_) => { return Err(client::invalid_argument(format!( - "`{IMAGE_KEY}` item {index} contains only a cache UUID; OpenEngine requires an image source" + "`{}` item {index} contains only a cache UUID; OpenEngine requires a source for {} input", + kind.key, kind.name ))); } }; Ok(pb::MediaItem { - modality: pb::Modality::Image as i32, + modality: kind.modality as i32, source: Some(source), // MIME metadata belongs to raw_bytes, which this client does not send. mime_type: String::new(), // Do not substitute router-owned mm_hashes for caller cache identities. - uuid: uuid.unwrap_or_default(), + uuid: uuid_slots + .get_mut(kind.key) + .and_then(|uuids| uuids.get_mut(index)) + .and_then(Option::take) + .flatten() + .unwrap_or_default(), }) }) .collect() } -fn media_source(value: String, index: usize) -> Result { +fn media_source( + value: String, + key: &str, + index: usize, +) -> Result { let scheme = value .split_once(':') .map(|(scheme, _)| scheme.to_ascii_lowercase()); @@ -282,7 +395,7 @@ fn media_source(value: String, index: usize) -> Result Ok(pb::media_item::Source::Url(value)), Some("data") => Ok(pb::media_item::Source::DataUri(value)), _ => Err(client::invalid_argument(format!( - "`{IMAGE_KEY}` item {index} must use an http, https, or data URI" + "`{key}` item {index} must use an http, https, or data URI" ))), } } diff --git a/lib/sidecar/trtllm/src/engine.rs b/lib/sidecar/trtllm/src/engine.rs index afd6fddf1f97..59e492cac07d 100644 --- a/lib/sidecar/trtllm/src/engine.rs +++ b/lib/sidecar/trtllm/src/engine.rs @@ -18,7 +18,7 @@ use tokio::sync::OnceCell; use tokio_util::sync::CancellationToken; use crate::args::Args; -use crate::client::{self, ModelInfoResponse, ModelMetadata, TrtllmClient}; +use crate::client::{self, ModelInfoResponse, ModelMetadata, SupportedMedia, TrtllmClient}; use crate::convert::{ResponseState, build_generate_request}; use crate::model::ConfiguredModel; @@ -183,12 +183,12 @@ impl TrtllmSidecarEngine { if let Some(metadata) = self.rediscovered_metadata.get() { return Ok(*metadata); } - let has_images = request.multi_modal_data.as_ref().is_some_and(|media| { - media - .get("image_url") - .is_some_and(|images| !images.is_empty()) + let has_supported_media = request.multi_modal_data.as_ref().is_some_and(|media| { + ["image_url", "video_url"] + .iter() + .any(|key| media.get(*key).is_some_and(|items| !items.is_empty())) }); - if !has_images || initial.supports_image.is_some() { + if !has_supported_media || initial.supported_media.is_some() { return Ok(initial); } @@ -210,7 +210,7 @@ impl TrtllmSidecarEngine { Ok(reported) } ModelInfoResponse::Unsupported(_) => Ok(ModelMetadata { - supports_image: Some(false), + supported_media: Some(SupportedMedia::default()), ..initial }), } @@ -266,7 +266,7 @@ impl LLMEngine for TrtllmSidecarEngine { --context-length" ); ModelMetadata { - supports_image: Some(false), + supported_media: Some(SupportedMedia::default()), ..Default::default() } } diff --git a/lib/sidecar/trtllm/src/tests.rs b/lib/sidecar/trtllm/src/tests.rs index 61daa182d2c9..121ef8e8da06 100644 --- a/lib/sidecar/trtllm/src/tests.rs +++ b/lib/sidecar/trtllm/src/tests.rs @@ -11,7 +11,8 @@ use std::time::Duration; use dynamo_backend_common::{ DisaggregationMode, ErrorType, FinishReason, GenerateContext, LLMEngine, MultimodalData, - OutputOptions, PreprocessedRequest, SamplingOptions, StopConditions, StopReason, + MultimodalItemRef, OutputOptions, PreprocessedRequest, SamplingOptions, StopConditions, + StopReason, }; use dynamo_sidecar_common::{GrpcEndpoint, GrpcTransportConfig}; use futures::{Stream, StreamExt}; @@ -21,7 +22,7 @@ use tokio::sync::{Mutex, oneshot}; use tokio_stream::wrappers::TcpListenerStream; use tonic::{Request, Response, Status}; -use crate::client::{ModelMetadata, TrtllmClient}; +use crate::client::{ModelMetadata, SupportedMedia, TrtllmClient}; use crate::convert::{ResponseState, build_generate_request, engine_error}; use crate::engine::TrtllmSidecarEngine; use crate::model::ConfiguredModel; @@ -64,6 +65,7 @@ struct FakeTrtllm { supports_multimodal: Arc, /// TensorRT-LLM's finer-grained modality advertisement. advertises_image: Arc, + advertises_video: Arc, model_info_calls: Arc, } @@ -257,13 +259,16 @@ impl pb::control_server::Control for FakeTrtllm { return Err(Status::unavailable("not serving yet")); } let supports_multimodal = self.supports_multimodal.load(Ordering::SeqCst); - let modalities = if self.advertises_image.load(Ordering::SeqCst) { - vec![prost_types::Value { - kind: Some(prost_types::value::Kind::StringValue("image".to_string())), - }] - } else { - Vec::new() - }; + let modalities = [ + (self.advertises_image.load(Ordering::SeqCst), "image"), + (self.advertises_video.load(Ordering::SeqCst), "video"), + ] + .into_iter() + .filter(|(advertised, _)| *advertised) + .map(|(_, modality)| prost_types::Value { + kind: Some(prost_types::value::Kind::StringValue(modality.to_string())), + }) + .collect(); Ok(Response::new(pb::ModelInfo { model_id: "fake-model".to_string(), max_context_length: if self.empty_model_info.load(Ordering::SeqCst) { @@ -469,7 +474,7 @@ fn limits(context_length: u32) -> Option { Some(ModelMetadata { context_length: Some(context_length), max_output_tokens: None, - supports_image: Some(false), + supported_media: Some(SupportedMedia::default()), }) } @@ -477,7 +482,21 @@ fn image_limits(context_length: u32) -> Option { Some(ModelMetadata { context_length: Some(context_length), max_output_tokens: None, - supports_image: Some(true), + supported_media: Some(SupportedMedia { + image: true, + ..Default::default() + }), + }) +} + +fn image_video_limits(context_length: u32) -> Option { + Some(ModelMetadata { + context_length: Some(context_length), + max_output_tokens: None, + supported_media: Some(SupportedMedia { + image: true, + video: true, + }), }) } diff --git a/lib/sidecar/trtllm/src/tests/convert_request.rs b/lib/sidecar/trtllm/src/tests/convert_request.rs index 31d47129d23a..4606376a4ee5 100644 --- a/lib/sidecar/trtllm/src/tests/convert_request.rs +++ b/lib/sidecar/trtllm/src/tests/convert_request.rs @@ -109,6 +109,120 @@ fn image_urls_data_uris_and_uuids_are_mapped_in_order() { assert!(proto.media[1].uuid.is_empty()); } +#[test] +fn mixed_image_and_video_media_follow_prompt_order() { + let mut req = request(); + req.multi_modal_data = Some(HashMap::from([ + ( + "image_url".to_string(), + vec![ + MultimodalData::RawUrl("https://example.com/first.png".to_string()), + MultimodalData::RawUrl("data:image/png;base64,aW1hZ2U=".to_string()), + ], + ), + ( + "video_url".to_string(), + vec![ + MultimodalData::RawUrl("https://example.com/first.mp4".to_string()), + MultimodalData::RawUrl("data:video/mp4;base64,dmlkZW8=".to_string()), + ], + ), + ])); + req.multi_modal_uuids = Some(HashMap::from([ + ( + "image_url".to_string(), + vec![Some("image-0".to_string()), None], + ), + ( + "video_url".to_string(), + vec![Some("video-0".to_string()), Some("video-1".to_string())], + ), + ])); + req.multi_modal_order = Some(vec![ + MultimodalItemRef { + modality: "video_url".to_string(), + index: 0, + }, + MultimodalItemRef { + modality: "image_url".to_string(), + index: 0, + }, + MultimodalItemRef { + modality: "video_url".to_string(), + index: 1, + }, + MultimodalItemRef { + modality: "image_url".to_string(), + index: 1, + }, + ]); + + let proto = build_generate_request(req, "req", "model", image_video_limits(4096), AGG) + .expect("mixed media request"); + + assert_eq!( + proto + .media + .iter() + .map(|item| item.modality) + .collect::>(), + [ + pb::Modality::Video as i32, + pb::Modality::Image as i32, + pb::Modality::Video as i32, + pb::Modality::Image as i32, + ] + ); + assert_eq!( + proto + .media + .iter() + .map(|item| item.uuid.as_str()) + .collect::>(), + ["video-0", "image-0", "video-1", ""] + ); +} + +#[test] +fn mixed_media_without_a_complete_unique_order_is_rejected() { + let mixed_media = || { + HashMap::from([ + ( + "image_url".to_string(), + vec![MultimodalData::RawUrl( + "https://example.com/image.png".to_string(), + )], + ), + ( + "video_url".to_string(), + vec![MultimodalData::RawUrl( + "https://example.com/video.mp4".to_string(), + )], + ), + ]) + }; + + let mut req = request(); + req.multi_modal_data = Some(mixed_media()); + let error = build_generate_request(req.clone(), "req", "model", image_video_limits(4096), AGG) + .expect_err("mixed legacy request has lost its order"); + assert!(error.to_string().contains("requires multi_modal_order")); + + req.multi_modal_order = Some(vec![ + MultimodalItemRef { + modality: "image_url".to_string(), + index: 0, + }, + MultimodalItemRef { + modality: "image_url".to_string(), + index: 0, + }, + ]); + let error = build_generate_request(req, "req", "model", image_video_limits(4096), AGG) + .expect_err("duplicate order entry must fail"); + assert!(error.to_string().contains("repeats `image_url` item 0")); +} + #[test] fn mm_routing_hashes_are_not_used_as_engine_cache_uuids() { let mut req = request(); @@ -149,15 +263,15 @@ fn image_media_requires_the_loaded_models_advertised_capability() { fn unsupported_media_modalities_and_sources_are_rejected() { let mut req = request(); req.multi_modal_data = Some(HashMap::from([( - "video_url".to_string(), + "audio_url".to_string(), vec![MultimodalData::RawUrl( "https://example.com/media".to_string(), )], )])); let error = build_generate_request(req, "req", "model", image_limits(4096), AGG) - .expect_err("only images are supported"); + .expect_err("audio is not supported yet"); assert!( - error.to_string().contains("video_url"), + error.to_string().contains("audio_url"), "unexpected error: {error}" ); @@ -174,7 +288,20 @@ fn unsupported_media_modalities_and_sources_are_rejected() { )])); let error = build_generate_request(req, "req", "model", image_limits(4096), AGG) .expect_err("OpenEngine cannot resolve a source-less UUID"); - assert!(error.to_string().contains("requires an image source")); + assert!( + error + .to_string() + .contains("requires a source for image input") + ); + + let mut req = request(); + req.multi_modal_data = Some(HashMap::from([( + "video_url".to_string(), + vec![MultimodalData::RawUrl("file:///tmp/video.mp4".to_string())], + )])); + let error = build_generate_request(req, "req", "model", image_video_limits(4096), AGG) + .expect_err("the engine cannot fetch a local Dynamo path"); + assert!(error.to_string().contains("`video_url` item 0")); } #[test] @@ -321,7 +448,7 @@ fn the_derived_max_tokens_respects_the_engines_output_cap() { Some(ModelMetadata { context_length: Some(4096), max_output_tokens: Some(256), - supports_image: Some(false), + supported_media: Some(SupportedMedia::default()), }), AGG, ) diff --git a/lib/sidecar/trtllm/src/tests/disagg.rs b/lib/sidecar/trtllm/src/tests/disagg.rs index 45f93e949efd..19c066d4199b 100644 --- a/lib/sidecar/trtllm/src/tests/disagg.rs +++ b/lib/sidecar/trtllm/src/tests/disagg.rs @@ -59,40 +59,56 @@ fn prefill_request_is_marked_context_only() { assert!(proto.kv.is_none(), "prefill carries no session to replay"); } -/// Image inputs must survive both RPC legs. TensorRT-LLM reconstructs its +/// Mixed media must survive both RPC legs. TensorRT-LLM reconstructs its /// multimodal position/hash state while resuming the KV session, so the decode /// request needs the same ordered media metadata as the context request. #[test] -fn image_media_is_preserved_across_prefill_and_decode_requests() { +fn mixed_media_is_preserved_across_prefill_and_decode_requests() { let mut req = request(); - add_images( - &mut req, - [ - "https://example.com/first.png", - "data:image/png;base64,c2Vjb25k", - ], - ); - req.multi_modal_uuids = Some(HashMap::from([( - "image_url".to_string(), - vec![ - Some("first-image".to_string()), - Some("second-image".to_string()), - ], - )])); + req.multi_modal_data = Some(HashMap::from([ + ( + "image_url".to_string(), + vec![MultimodalData::RawUrl( + "https://example.com/image.png".to_string(), + )], + ), + ( + "video_url".to_string(), + vec![MultimodalData::RawUrl( + "data:video/mp4;base64,dmlkZW8=".to_string(), + )], + ), + ])); + req.multi_modal_uuids = Some(HashMap::from([ + ("image_url".to_string(), vec![Some("image-id".to_string())]), + ("video_url".to_string(), vec![Some("video-id".to_string())]), + ])); + req.multi_modal_order = Some(vec![ + MultimodalItemRef { + modality: "video_url".to_string(), + index: 0, + }, + MultimodalItemRef { + modality: "image_url".to_string(), + index: 0, + }, + ]); let prefill = build_generate_request( req.clone(), "req", "model", - image_limits(4096), + image_video_limits(4096), DisaggregationMode::Prefill, ) .expect("build image prefill"); assert!(is_context_only(&prefill)); assert_eq!(prefill.stopping.as_ref().unwrap().max_tokens, Some(1)); assert_eq!(prefill.media.len(), 2); - assert_eq!(prefill.media[0].uuid, "first-image"); - assert_eq!(prefill.media[1].uuid, "second-image"); + assert_eq!(prefill.media[0].modality, pb::Modality::Video as i32); + assert_eq!(prefill.media[0].uuid, "video-id"); + assert_eq!(prefill.media[1].modality, pb::Modality::Image as i32); + assert_eq!(prefill.media[1].uuid, "image-id"); req.prefill_result = Some(dynamo_backend_common::PrefillResult { disaggregated_params: crate::disagg::session_to_json(fake_session()) @@ -103,7 +119,7 @@ fn image_media_is_preserved_across_prefill_and_decode_requests() { req, "req", "model", - image_limits(4096), + image_video_limits(4096), DisaggregationMode::Decode, ) .expect("build image decode"); @@ -112,7 +128,7 @@ fn image_media_is_preserved_across_prefill_and_decode_requests() { assert_eq!( decode.kv.and_then(|kv| kv.session), Some(fake_session()), - "decode must carry both the image metadata and the KV handoff" + "decode must carry both ordered media items and the KV handoff" ); } diff --git a/lib/sidecar/trtllm/src/tests/e2e.rs b/lib/sidecar/trtllm/src/tests/e2e.rs index 3f97fe73b7f4..6ae819b8a916 100644 --- a/lib/sidecar/trtllm/src/tests/e2e.rs +++ b/lib/sidecar/trtllm/src/tests/e2e.rs @@ -382,3 +382,77 @@ async fn e2e_real_openengine_image() { "the engine's expanded image prompt usage must replace Dynamo's unexpanded token count" ); } + +/// Drives a video data URI through the real sidecar and OpenEngine server. +/// `TRTLLM_E2E_VIDEO_DATA_URI` must contain a complete `data:video/...` URI. +#[tokio::test] +#[ignore = "requires a live video-capable TensorRT-LLM OpenEngine server"] +async fn e2e_real_openengine_video() { + let endpoint = std::env::var("TRTLLM_E2E_ENDPOINT") + .expect("set TRTLLM_E2E_ENDPOINT, e.g. http://127.0.0.1:50051"); + let model = std::env::var("TRTLLM_E2E_MODEL") + .unwrap_or_else(|_| "Qwen/Qwen3-VL-2B-Instruct".to_string()); + let video = std::env::var("TRTLLM_E2E_VIDEO_DATA_URI") + .expect("set TRTLLM_E2E_VIDEO_DATA_URI to a video data URI"); + let engine = TrtllmSidecarEngine::new( + GrpcEndpoint::parse(&endpoint, "--grpc-endpoint").expect("valid endpoint"), + transport(1), + ConfiguredModel { + source: model.clone(), + context_length: None, + }, + AGG, + ); + engine + .start(0) + .await + .expect("start against the live video-capable server"); + + let token_ids = vec![ + 151644, 872, 198, 151652, 151656, 151653, 74785, 279, 7218, 6083, 323, 1181, 5106, 304, + 825, 2805, 11652, 13, 151645, 198, 151644, 77091, 198, + ]; + let mut request = PreprocessedRequest::builder() + .model(model) + .token_ids(token_ids.clone()) + .stop_conditions(StopConditions { + max_tokens: Some(64), + ..Default::default() + }) + .sampling_options(SamplingOptions { + temperature: Some(0.0), + ..Default::default() + }) + .output_options(OutputOptions::default()) + .build() + .expect("request"); + request.multi_modal_data = Some(HashMap::from([( + "video_url".to_string(), + vec![MultimodalData::RawUrl(video)], + )])); + request.multi_modal_order = Some(vec![MultimodalItemRef { + modality: "video_url".to_string(), + index: 0, + }]); + + let outputs = collect(&engine, request).await; + let generated = outputs + .iter() + .flat_map(|output| output.token_ids.iter().copied()) + .collect::>(); + let terminal = outputs + .iter() + .find(|output| output.finish_reason.is_some()) + .expect("terminal output"); + let usage = terminal.completion_usage.as_ref().expect("terminal usage"); + eprintln!( + "[e2e-video] generated={generated:?}; usage: prompt={}, completion={}", + usage.prompt_tokens, usage.completion_tokens + ); + + assert!(!generated.is_empty(), "expected video-conditioned output"); + assert!( + usage.prompt_tokens > token_ids.len() as u32, + "the engine must report the expanded video prompt" + ); +} diff --git a/lib/sidecar/trtllm/src/tests/engine.rs b/lib/sidecar/trtllm/src/tests/engine.rs index aff3ef99a66e..b330223ab3eb 100644 --- a/lib/sidecar/trtllm/src/tests/engine.rs +++ b/lib/sidecar/trtllm/src/tests/engine.rs @@ -75,6 +75,36 @@ async fn advertised_image_media_reaches_openengine_and_uses_expanded_usage() { )); } +#[tokio::test] +async fn advertised_video_media_reaches_openengine() { + let service = FakeTrtllm::default(); + service.supports_multimodal.store(true, Ordering::SeqCst); + service.advertises_video.store(true, Ordering::SeqCst); + let server = FakeServer::start(service).await; + let engine = engine(&server.endpoint, 1); + engine.start(0).await.expect("start"); + + let mut req = request(); + req.multi_modal_data = Some(HashMap::from([( + "video_url".to_string(), + vec![MultimodalData::RawUrl( + "https://example.com/video.mp4".to_string(), + )], + )])); + collect(&engine, req).await; + + let requests = server.service.requests.lock().await; + let sent = requests.first().expect("recorded request"); + assert_eq!(sent.media.len(), 1); + assert_eq!(sent.media[0].modality, pb::Modality::Video as i32); + assert_eq!( + sent.media[0].source, + Some(pb::media_item::Source::Url( + "https://example.com/video.mp4".to_string() + )) + ); +} + #[tokio::test] async fn both_multimodal_capability_signals_are_required() { for (coarse, image) in [(true, false), (false, true)] { @@ -467,38 +497,50 @@ async fn a_configured_context_length_survives_a_server_without_control() { } #[tokio::test] -async fn image_request_retries_transient_capability_discovery() { - let service = FakeTrtllm::default(); - service.unavailable_model_info.store(true, Ordering::SeqCst); - let server = FakeServer::start(service).await; - let engine = engine_with(&server.endpoint, transport(1), Some(8192), AGG); - engine.start(0).await.expect("start with configured window"); - - server - .service - .unavailable_model_info - .store(false, Ordering::SeqCst); - server - .service - .supports_multimodal - .store(true, Ordering::SeqCst); - server - .service - .advertises_image - .store(true, Ordering::SeqCst); - let mut image_request = request(); - add_images(&mut image_request, ["https://example.com/image.png"]); - - let output = collect(&engine, image_request).await; - assert_eq!( - output.last().unwrap().finish_reason, - Some(FinishReason::Stop) - ); - assert_eq!( - server.service.model_info_calls.load(Ordering::SeqCst), - 2, - "the image request must retry the transient startup failure once" - ); +async fn media_requests_retry_transient_capability_discovery() { + for (key, url, image, video) in [ + ("image_url", "https://example.com/image.png", true, false), + ("video_url", "https://example.com/video.mp4", false, true), + ] { + let service = FakeTrtllm::default(); + service.unavailable_model_info.store(true, Ordering::SeqCst); + let server = FakeServer::start(service).await; + let engine = engine_with(&server.endpoint, transport(1), Some(8192), AGG); + engine.start(0).await.expect("start with configured window"); + + server + .service + .unavailable_model_info + .store(false, Ordering::SeqCst); + server + .service + .supports_multimodal + .store(true, Ordering::SeqCst); + server + .service + .advertises_image + .store(image, Ordering::SeqCst); + server + .service + .advertises_video + .store(video, Ordering::SeqCst); + let mut media_request = request(); + media_request.multi_modal_data = Some(HashMap::from([( + key.to_string(), + vec![MultimodalData::RawUrl(url.to_string())], + )])); + + let output = collect(&engine, media_request).await; + assert_eq!( + output.last().unwrap().finish_reason, + Some(FinishReason::Stop) + ); + assert_eq!( + server.service.model_info_calls.load(Ordering::SeqCst), + 2, + "the {key} request must retry the transient startup failure once" + ); + } } /// Shutdown has to terminate in-flight requests: the worker cannot drain if a diff --git a/tests/serve/test_sidecar.py b/tests/serve/test_sidecar.py index 0375aa862e0d..2cb78a46bb7b 100644 --- a/tests/serve/test_sidecar.py +++ b/tests/serve/test_sidecar.py @@ -10,8 +10,14 @@ import pytest +from dynamo.common.utils.install_media_decoders import VALIDATED_SPECS from tests.router.common import _test_frontend_kv_routing from tests.router.helper import generate_random_suffix +from tests.serve.conftest import ( + MULTIMODAL_IMG_URL, + MULTIMODAL_VIDEO_EXPECTED, + MULTIMODAL_VIDEO_URL, +) from tests.serve.common import ( WORKSPACE_DIR, params_with_model_mark, @@ -26,6 +32,7 @@ from tests.utils.engine_process import EngineConfig from tests.utils.gpu_args import map_cuda_visible_devices from tests.utils.payload_builder import ( + chat_payload, chat_logprobs_and_structured_output_payloads, chat_payload_default, disaggregated_chat_payload, @@ -199,8 +206,8 @@ def _disaggregated_payloads( chat_payload_default(), ], ), - "trtllm_aggregated_image": EngineConfig( - name="trtllm_aggregated_image", + "trtllm_aggregated_multimodal": EngineConfig( + name="trtllm_aggregated_multimodal", directory=trtllm_sidecar_dir, script_name="agg.sh", marks=[ @@ -211,16 +218,32 @@ def _disaggregated_payloads( pytest.mark.requested_trtllm_kv_tokens(32768), pytest.mark.timeout(960), pytest.mark.pre_merge, + pytest.mark.installs_extra_dependencies, ], model="Qwen/Qwen3-VL-2B-Instruct", timeout=900, delayed_start=60, - env={"PYTHONUNBUFFERED": "1"}, + env={ + "DYN_MM_ALLOW_INTERNAL": "1", + "DYN_TEST_ONLY_PIP_INSTALL": VALIDATED_SPECS["opencv-python-headless"], + "PYTHONUNBUFFERED": "1", + "TRTLLM_MEDIA_ALLOW_PRIVATE_URLS": "1", + }, request_payloads=[ multimodal_payload_default( + image_url=MULTIMODAL_IMG_URL, text="Describe what you see in this image.", - expected_response=["mountain", "rock", "trees", "road"], - ) + expected_response=["green", "square"], + ), + chat_payload( + [ + {"type": "text", "text": "Describe the video in detail."}, + {"type": "video_url", "video_url": {"url": MULTIMODAL_VIDEO_URL}}, + ], + expected_response=MULTIMODAL_VIDEO_EXPECTED, + max_tokens=100, + temperature=0.0, + ), ], ), "trtllm_disaggregated": EngineConfig( @@ -335,10 +358,15 @@ def _disaggregated_payloads( @pytest.fixture(params=params_with_model_mark(sidecar_configs)) def sidecar_config_test(request, monkeypatch, discovery_backend): config = sidecar_configs[request.param] - if config.name == "trtllm_aggregated_image" and not _trtllm_serves_openengine(): + if ( + config.name == "trtllm_aggregated_multimodal" + and not _trtllm_serves_openengine() + ): if os.environ.get("DYNAMO_TEST_SIDECAR_BACKEND") == "trtllm": pytest.fail(TRTLLM_OPENENGINE_SKIP_REASON) pytest.skip(TRTLLM_OPENENGINE_SKIP_REASON) + if config.name == "trtllm_aggregated_multimodal": + request.getfixturevalue("image_server") monkeypatch.setenv("MODEL", config.model) monkeypatch.setenv("DYN_DISCOVERY_BACKEND", discovery_backend) monkeypatch.setenv("DYN_NAMESPACE", f"sidecar-{generate_random_suffix()}") From b5dafda65c65e8698c12a9c9595782ae6bac9c23 Mon Sep 17 00:00:00 2001 From: krishung5 Date: Mon, 19 Oct 2026 11:21:48 -0700 Subject: [PATCH 22/61] test(trtllm): trim redundant test narration Signed-off-by: krishung5 --- .github/scripts/test-filters.js | 1 - lib/sidecar/trtllm/src/tests/disagg.rs | 5 ++--- lib/sidecar/trtllm/src/tests/e2e.rs | 10 ++++------ 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/.github/scripts/test-filters.js b/.github/scripts/test-filters.js index d5d45a4ce863..ec493457647d 100755 --- a/.github/scripts/test-filters.js +++ b/.github/scripts/test-filters.js @@ -175,7 +175,6 @@ const testCases = [ desc: 'AISimulate Dynamo adapter triggers planner validation' }, - // Sidecar Rust and proto files trigger Rust checks and their native E2E only. { file: 'lib/sidecar/common/src/lib.rs', expect: { sidecar: true, rust: true, core: false, frontend: false, vllm: false, sglang: false, trtllm: false }, diff --git a/lib/sidecar/trtllm/src/tests/disagg.rs b/lib/sidecar/trtllm/src/tests/disagg.rs index 45f93e949efd..e9eee5f3c4c9 100644 --- a/lib/sidecar/trtllm/src/tests/disagg.rs +++ b/lib/sidecar/trtllm/src/tests/disagg.rs @@ -59,9 +59,8 @@ fn prefill_request_is_marked_context_only() { assert!(proto.kv.is_none(), "prefill carries no session to replay"); } -/// Image inputs must survive both RPC legs. TensorRT-LLM reconstructs its -/// multimodal position/hash state while resuming the KV session, so the decode -/// request needs the same ordered media metadata as the context request. +/// TensorRT-LLM needs the same ordered media metadata to reconstruct +/// multimodal position/hash state when resuming the KV session. #[test] fn image_media_is_preserved_across_prefill_and_decode_requests() { let mut req = request(); diff --git a/lib/sidecar/trtllm/src/tests/e2e.rs b/lib/sidecar/trtllm/src/tests/e2e.rs index 3f97fe73b7f4..429d984d37e3 100644 --- a/lib/sidecar/trtllm/src/tests/e2e.rs +++ b/lib/sidecar/trtllm/src/tests/e2e.rs @@ -111,8 +111,7 @@ async fn e2e_real_disagg_handoff() { ); } -/// Exercises the same real prefill/decode handoff with an image request. Both -/// servers may share one sufficiently large GPU for local validation. +/// Both servers may share one sufficiently large GPU for local validation. /// /// ```text /// TRTLLM_E2E_PREFILL_ENDPOINT=http://127.0.0.1:50051 \ @@ -304,10 +303,9 @@ async fn e2e_real_openengine_server() { ); } -/// Drives an image request through the real Rust sidecar conversion layer and -/// a live TensorRT-LLM OpenEngine server. The fixed token IDs are Qwen3-VL's -/// chat-template rendering of one image marker and a short question; the tiny -/// PNG is split red/blue so a manual run also produces an obvious answer. +/// The fixed token IDs are Qwen3-VL's chat-template rendering of one image +/// marker and a short question. The tiny PNG is split red/blue so a manual run +/// produces an obvious answer. /// /// ```text /// TRTLLM_E2E_ENDPOINT=http://127.0.0.1:50051 \ From 17de5addfa1b1163b5b256e90b37cddeab1ccbce Mon Sep 17 00:00:00 2001 From: krishung5 Date: Mon, 19 Oct 2026 11:22:51 -0700 Subject: [PATCH 23/61] ci(trtllm): refresh OpenEngine source pin Signed-off-by: krishung5 --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 51f13e719ed1..b6deca9a9d01 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -546,7 +546,7 @@ jobs: # Drop this source overlay after the upstream TensorRT-LLM change ships. extra_build_args: | TRTLLM_SOURCE_REPOSITORY=https://github.com/krishung5/TensorRT-LLM - TRTLLM_SOURCE_COMMIT=5d2e3b2c758b4ccc0fa0a3eb0e01cdef934e8089 + TRTLLM_SOURCE_COMMIT=9dc3ab125422271b67aed519ca4c6d8109be3b15 # actions: read lets the build download a prior run's baseline compliance # artifact to diff against (a reusable workflow's token can't exceed the # caller's). Missing -> the diff soft-degrades, build stays green. From 480536fb953851ef1805f3f758c7907299cc13fc Mon Sep 17 00:00:00 2001 From: krishung5 Date: Mon, 19 Oct 2026 15:02:56 -0700 Subject: [PATCH 24/61] fix(ci): include OpenEngine model config in source overlay Signed-off-by: krishung5 --- .github/workflows/pr.yaml | 3 ++- container/templates/trtllm_runtime.Dockerfile | 11 +++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 4f0d38f54b33..ec7d985d30a6 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -543,7 +543,8 @@ jobs: diff_current_branch: ${{ github.ref_name }} diff_base_branch: ${{ needs.changed-files.outputs.base_ref }} diff_merge_base_sha: ${{ needs.changed-files.outputs.merge_base_sha }} - # Drop this source overlay after the upstream TensorRT-LLM change ships. + # Drop this source overlay after the pinned TRT-LLM runtime image contains + # the upstream change. extra_build_args: | TRTLLM_SOURCE_REPOSITORY=https://github.com/krishung5/TensorRT-LLM TRTLLM_SOURCE_COMMIT=dac2ab5daf6e0720d9425902e4190b3ac29aa7ed diff --git a/container/templates/trtllm_runtime.Dockerfile b/container/templates/trtllm_runtime.Dockerfile index 82661874f3e6..57cedd2df11f 100644 --- a/container/templates/trtllm_runtime.Dockerfile +++ b/container/templates/trtllm_runtime.Dockerfile @@ -46,6 +46,8 @@ ARG TRTLLM_SOURCE_COMMIT # Optional source overlay for cross-repository CI. Keep the base image's server # lifecycle entry points, which are coupled to that release's frontend runtime. +# capabilities imports model_config; install that helper with the media loader. +# Keep the base image's other input modules to avoid a partial framework upgrade. RUN if [ -n "${TRTLLM_SOURCE_COMMIT}" ]; then \ test -n "${TRTLLM_SOURCE_REPOSITORY}" && \ source_dir=$(mktemp -d) && \ @@ -67,8 +69,13 @@ RUN if [ -n "${TRTLLM_SOURCE_COMMIT}" ]; then \ "${package_dir}/grpc/openengine/server.py" && \ grep -Fxq 'from .bindings import openengine_pb2_grpc' \ "${package_dir}/grpc/openengine/server.py" && \ - install -m 0644 "${source_dir}/tensorrt_llm/inputs/media_io.py" \ - "${package_dir}/inputs/media_io.py" && \ + for module in media_io.py model_config.py; do \ + install -m 0644 "${source_dir}/tensorrt_llm/inputs/${module}" \ + "${package_dir}/inputs/${module}"; \ + done && \ + test -f "${package_dir}/inputs/model_config.py" && \ + grep -Fq 'from tensorrt_llm.inputs.model_config import resolve_top_level_model_type' \ + "${package_dir}/grpc/openengine/capabilities.py" && \ rm -rf "${source_dir}"; \ fi From 343de4be8313dcdbd0c865ec90e6eca607347e7e Mon Sep 17 00:00:00 2001 From: krishung5 Date: Mon, 19 Oct 2026 15:07:19 -0700 Subject: [PATCH 25/61] fix(trtllm): harden video media conversion Signed-off-by: krishung5 --- .github/workflows/pr.yaml | 2 +- lib/sidecar/trtllm/src/convert.rs | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 438ac7aa9b2a..7d3fbb33ee33 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -547,7 +547,7 @@ jobs: # the upstream change. extra_build_args: | TRTLLM_SOURCE_REPOSITORY=https://github.com/krishung5/TensorRT-LLM - TRTLLM_SOURCE_COMMIT=8745cae734447a4eb33241850953a3e4ea43356d + TRTLLM_SOURCE_COMMIT=b777c5c41c4d077efb03de2325c986d62f383e1c # actions: read lets the build download a prior run's baseline compliance # artifact to diff against (a reusable workflow's token can't exceed the # caller's). Missing -> the diff soft-degrades, build stays green. diff --git a/lib/sidecar/trtllm/src/convert.rs b/lib/sidecar/trtllm/src/convert.rs index 6e42b03b41fd..acf6e591799c 100644 --- a/lib/sidecar/trtllm/src/convert.rs +++ b/lib/sidecar/trtllm/src/convert.rs @@ -349,7 +349,12 @@ fn media( .get_mut(kind.key) .and_then(|items| items.get_mut(index)) .and_then(Option::take) - .expect("media order was validated"); + .ok_or_else(|| { + client::protocol_error(format!( + "validated media order lost `{}` item {index}", + kind.key + )) + })?; let source = match item { MultimodalData::Url(url) => media_source(url.to_string(), kind.key, index)?, MultimodalData::RawUrl(url) => media_source(url, kind.key, index)?, From a306889a891e1968c33bbcde8d7c09e3a7ba3b21 Mon Sep 17 00:00:00 2001 From: krishung5 Date: Mon, 19 Oct 2026 17:23:58 -0700 Subject: [PATCH 26/61] feat(trtllm): pass audio inputs through OpenEngine Signed-off-by: krishung5 --- .github/workflows/pr.yaml | 2 +- lib/sidecar/trtllm/launch/agg.sh | 29 ++++++--- lib/sidecar/trtllm/launch/common.sh | 46 ++++++++++++++ lib/sidecar/trtllm/src/client.rs | 3 + lib/sidecar/trtllm/src/convert.rs | 10 ++- lib/sidecar/trtllm/src/engine.rs | 2 +- lib/sidecar/trtllm/src/tests.rs | 15 +++++ .../trtllm/src/tests/convert_request.rs | 52 +++++++++++++--- lib/sidecar/trtllm/src/tests/e2e.rs | 62 +++++++++++++++++++ lib/sidecar/trtllm/src/tests/engine.rs | 57 ++++++++++++++++- tests/serve/fixtures/trtllm_whisper_agg.yaml | 19 ++++++ .../trtllm_whisper_chat_template.jinja | 3 + tests/serve/test_sidecar.py | 53 +++++++++++++--- 13 files changed, 321 insertions(+), 32 deletions(-) create mode 100644 tests/serve/fixtures/trtllm_whisper_agg.yaml create mode 100644 tests/serve/fixtures/trtllm_whisper_chat_template.jinja diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 7d3fbb33ee33..ead15823335c 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -547,7 +547,7 @@ jobs: # the upstream change. extra_build_args: | TRTLLM_SOURCE_REPOSITORY=https://github.com/krishung5/TensorRT-LLM - TRTLLM_SOURCE_COMMIT=b777c5c41c4d077efb03de2325c986d62f383e1c + TRTLLM_SOURCE_COMMIT=6844e5a9b026f52fee7e9d18113aeaa05c56267f # actions: read lets the build download a prior run's baseline compliance # artifact to diff against (a reusable workflow's token can't exceed the # caller's). Missing -> the diff soft-degrades, build stays green. diff --git a/lib/sidecar/trtllm/launch/agg.sh b/lib/sidecar/trtllm/launch/agg.sh index f5de0e11f400..6f04dcbf1be2 100755 --- a/lib/sidecar/trtllm/launch/agg.sh +++ b/lib/sidecar/trtllm/launch/agg.sh @@ -21,7 +21,7 @@ source "$SCRIPT_DIR/../../../../examples/common/gpu_utils.sh" # build_trtllm_o # shellcheck disable=SC1091 # Resolved relative to this script at runtime. source "$SCRIPT_DIR/../../../../examples/common/launch_utils.sh" # print_launch_banner, wait_any_exit # shellcheck disable=SC1091 # Resolved relative to this script at runtime. -source "$SCRIPT_DIR/common.sh" # trtllm_ensure_openengine_bindings, trtllm_resolve_context_length +source "$SCRIPT_DIR/common.sh" # OpenEngine bindings, context length, config merge MODEL="${MODEL:-Qwen/Qwen3-0.6B}" @@ -50,8 +50,11 @@ while [[ $# -gt 0 ]]; do echo " DYN_HTTP_PORT Dynamo frontend port (default: 8000)" echo " DYN_SYSTEM_PORT Dynamo sidecar system port (default: 8081)" echo " TRTLLM_GRPC_PORT TensorRT-LLM gRPC port (default: 30001)" + echo " AGG_ENGINE_ARGS Base TensorRT-LLM YAML/JSON options file" echo " TRTLLM_CONTEXT_LENGTH Model context length, applied to both the engine and the" echo " sidecar (default: 4096; unset when --max_seq_len is given)" + echo " TRTLLM_DISABLE_GUIDED_DECODING=1" + echo " Disable xgrammar for models that do not support it" exit 0 ;; *) @@ -78,15 +81,13 @@ TRTLLM_PYTHON="${TRTLLM_PYTHON:-python3}" TRTLLM_GRPC_PORT=$(dyn_port DYN_ENGINE_GRPC_PORT 1 "${TRTLLM_GRPC_PORT:-30001}") CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0}" -# `--extra_llm_api_options` (alias `--config`) is last-wins, not additive, so a -# forwarded copy would drop the guided-decoding backend and a required or named -# tool_choice would fail again. Refuse it rather than silently losing the setting. +# `--extra_llm_api_options` (alias `--config`) is last-wins, not additive. The +# launcher reserves it for required and profiler overrides; callers can supply +# base options through AGG_ENGINE_ARGS, which are merged below. for arg in "${EXTRA_ARGS[@]}"; do case "$arg" in --extra_llm_api_options|--extra_llm_api_options=*|--config|--config=*) - echo "Cannot forward ${arg%%=*}: this launcher needs it for" >&2 - echo "guided_decoding_backend. Add 'guided_decoding_backend: xgrammar'" >&2 - echo "to your file and run the engine yourself." >&2 + echo "Cannot forward ${arg%%=*}: use AGG_ENGINE_ARGS instead." >&2 exit 1 ;; esac @@ -100,9 +101,17 @@ HTTP_PORT="${DYN_HTTP_PORT:-8000}" # Dynamo sends a required or named tool_choice as a JSON schema, which # TensorRT-LLM enforces only with a guided-decoding backend. TRTLLM_EXTRA_CONFIG=$(mktemp "${TMPDIR:-/tmp}/dynamo-trtllm-sidecar.XXXXXX.yaml") -build_trtllm_override_args_with_mem \ - --merge-with-json '{"guided_decoding_backend": "xgrammar"}' \ - > "$TRTLLM_EXTRA_CONFIG" +TRTLLM_REQUIRED_OVERRIDES='{"guided_decoding_backend": "xgrammar"}' +if [[ "${TRTLLM_DISABLE_GUIDED_DECODING:-0}" == "1" ]]; then + TRTLLM_REQUIRED_OVERRIDES='{}' +fi +TRTLLM_OVERRIDES=$(build_trtllm_override_args_with_mem \ + --merge-with-json "$TRTLLM_REQUIRED_OVERRIDES") +trtllm_write_merged_config \ + "${AGG_ENGINE_ARGS:-}" \ + "$TRTLLM_OVERRIDES" \ + "$TRTLLM_EXTRA_CONFIG" \ + "$TRTLLM_PYTHON" print_launch_banner "Launching TensorRT-LLM OpenEngine-gRPC Sidecar (1 GPU)" "$MODEL" "$HTTP_PORT" \ "TensorRT-LLM gRPC: 127.0.0.1:${TRTLLM_GRPC_PORT}" \ diff --git a/lib/sidecar/trtllm/launch/common.sh b/lib/sidecar/trtllm/launch/common.sh index 21e4a901dacf..ce6c8b76115b 100755 --- a/lib/sidecar/trtllm/launch/common.sh +++ b/lib/sidecar/trtllm/launch/common.sh @@ -9,6 +9,7 @@ # Source it, then call: # trtllm_ensure_openengine_bindings "$TRTLLM_PYTHON" # trtllm_resolve_context_length "${EXTRA_ARGS[@]}" +# trtllm_write_merged_config "$BASE" "$OVERRIDES" "$OUTPUT" "$TRTLLM_PYTHON" # # `trtllm_resolve_context_length` sets TRTLLM_MAX_SEQ_LEN_ARGS, # TRTLLM_CONTEXT_LENGTH_ARGS, and TRTLLM_CONTEXT_LENGTH. @@ -78,3 +79,48 @@ trtllm_resolve_context_length() { TRTLLM_CONTEXT_LENGTH_ARGS=(--context-length "$TRTLLM_CONTEXT_LENGTH") fi } + +# Write a TensorRT-LLM options file by recursively applying JSON overrides to +# an optional YAML/JSON base. Recursive merging preserves sibling settings in +# nested blocks such as kv_cache_config when the test profiler supplies a +# max_tokens override. +trtllm_write_merged_config() { + local base_config="$1" + local overrides_json="$2" + local output="$3" + local python="$4" + + if [[ -z "$base_config" ]]; then + printf '%s\n' "$overrides_json" > "$output" + return 0 + fi + if [[ ! -r "$base_config" ]]; then + echo "TensorRT-LLM engine config is not readable: $base_config" >&2 + return 1 + fi + + "$python" - "$base_config" "$overrides_json" "$output" <<'PY' +import json +import sys + +import yaml + + +def merge(base, overrides): + for key, value in overrides.items(): + if isinstance(value, dict) and isinstance(base.get(key), dict): + merge(base[key], value) + else: + base[key] = value + + +base_path, overrides_json, output_path = sys.argv[1:] +with open(base_path, encoding="utf-8") as source: + config = yaml.safe_load(source) or {} +if not isinstance(config, dict): + raise TypeError(f"TensorRT-LLM engine config must be a mapping: {base_path}") +merge(config, json.loads(overrides_json)) +with open(output_path, "w", encoding="utf-8") as destination: + yaml.safe_dump(config, destination, sort_keys=False) +PY +} diff --git a/lib/sidecar/trtllm/src/client.rs b/lib/sidecar/trtllm/src/client.rs index 699159722eb5..1e627b4e355b 100644 --- a/lib/sidecar/trtllm/src/client.rs +++ b/lib/sidecar/trtllm/src/client.rs @@ -32,6 +32,7 @@ const RPC_TIMEOUT: Duration = Duration::from_secs(30); pub(crate) struct SupportedMedia { pub(crate) image: bool, pub(crate) video: bool, + pub(crate) audio: bool, } impl SupportedMedia { @@ -39,6 +40,7 @@ impl SupportedMedia { match modality { "image" => self.image, "video" => self.video, + "audio" => self.audio, _ => false, } } @@ -142,6 +144,7 @@ impl TrtllmClient { match modality.as_str() { "image" => supported_media.image = true, "video" => supported_media.video = true, + "audio" => supported_media.audio = true, _ => {} } } diff --git a/lib/sidecar/trtllm/src/convert.rs b/lib/sidecar/trtllm/src/convert.rs index acf6e591799c..dfb9745e0f41 100644 --- a/lib/sidecar/trtllm/src/convert.rs +++ b/lib/sidecar/trtllm/src/convert.rs @@ -4,7 +4,7 @@ //! Conversion between Dynamo's `PreprocessedRequest` / `LLMEngineOutput` and the //! TensorRT-LLM OpenEngine (`openengine.v1`) protobuf messages. //! -//! Supports aggregated and disaggregated image and video generation. Audio, +//! Supports aggregated and disaggregated image, video, and audio generation. //! LoRA, beam search, and `n > 1` are rejected before dispatch. //! //! Disaggregation is driven by [`DisaggregationMode`]: a prefill worker marks @@ -184,6 +184,7 @@ fn max_tokens( const IMAGE_KEY: &str = "image_url"; const VIDEO_KEY: &str = "video_url"; +const AUDIO_KEY: &str = "audio_url"; #[derive(Clone, Copy)] struct MediaKind { @@ -192,7 +193,7 @@ struct MediaKind { modality: pb::Modality, } -const MEDIA_KINDS: [MediaKind; 2] = [ +const MEDIA_KINDS: [MediaKind; 3] = [ MediaKind { key: IMAGE_KEY, name: "image", @@ -203,6 +204,11 @@ const MEDIA_KINDS: [MediaKind; 2] = [ name: "video", modality: pb::Modality::Video, }, + MediaKind { + key: AUDIO_KEY, + name: "audio", + modality: pb::Modality::Audio, + }, ]; fn media_kind(key: &str) -> Option { diff --git a/lib/sidecar/trtllm/src/engine.rs b/lib/sidecar/trtllm/src/engine.rs index 59e492cac07d..c711a639069d 100644 --- a/lib/sidecar/trtllm/src/engine.rs +++ b/lib/sidecar/trtllm/src/engine.rs @@ -184,7 +184,7 @@ impl TrtllmSidecarEngine { return Ok(*metadata); } let has_supported_media = request.multi_modal_data.as_ref().is_some_and(|media| { - ["image_url", "video_url"] + ["image_url", "video_url", "audio_url"] .iter() .any(|key| media.get(*key).is_some_and(|items| !items.is_empty())) }); diff --git a/lib/sidecar/trtllm/src/tests.rs b/lib/sidecar/trtllm/src/tests.rs index 121ef8e8da06..4dbe7a386241 100644 --- a/lib/sidecar/trtllm/src/tests.rs +++ b/lib/sidecar/trtllm/src/tests.rs @@ -66,6 +66,7 @@ struct FakeTrtllm { /// TensorRT-LLM's finer-grained modality advertisement. advertises_image: Arc, advertises_video: Arc, + advertises_audio: Arc, model_info_calls: Arc, } @@ -262,6 +263,7 @@ impl pb::control_server::Control for FakeTrtllm { let modalities = [ (self.advertises_image.load(Ordering::SeqCst), "image"), (self.advertises_video.load(Ordering::SeqCst), "video"), + (self.advertises_audio.load(Ordering::SeqCst), "audio"), ] .into_iter() .filter(|(advertised, _)| *advertised) @@ -496,6 +498,19 @@ fn image_video_limits(context_length: u32) -> Option { supported_media: Some(SupportedMedia { image: true, video: true, + ..Default::default() + }), + }) +} + +fn all_media_limits(context_length: u32) -> Option { + Some(ModelMetadata { + context_length: Some(context_length), + max_output_tokens: None, + supported_media: Some(SupportedMedia { + image: true, + video: true, + audio: true, }), }) } diff --git a/lib/sidecar/trtllm/src/tests/convert_request.rs b/lib/sidecar/trtllm/src/tests/convert_request.rs index 4606376a4ee5..5a8a4d710978 100644 --- a/lib/sidecar/trtllm/src/tests/convert_request.rs +++ b/lib/sidecar/trtllm/src/tests/convert_request.rs @@ -110,7 +110,7 @@ fn image_urls_data_uris_and_uuids_are_mapped_in_order() { } #[test] -fn mixed_image_and_video_media_follow_prompt_order() { +fn mixed_media_follow_prompt_order() { let mut req = request(); req.multi_modal_data = Some(HashMap::from([ ( @@ -127,6 +127,12 @@ fn mixed_image_and_video_media_follow_prompt_order() { MultimodalData::RawUrl("data:video/mp4;base64,dmlkZW8=".to_string()), ], ), + ( + "audio_url".to_string(), + vec![MultimodalData::RawUrl( + "data:audio/wav;base64,YXVkaW8=".to_string(), + )], + ), ])); req.multi_modal_uuids = Some(HashMap::from([ ( @@ -137,6 +143,7 @@ fn mixed_image_and_video_media_follow_prompt_order() { "video_url".to_string(), vec![Some("video-0".to_string()), Some("video-1".to_string())], ), + ("audio_url".to_string(), vec![Some("audio-0".to_string())]), ])); req.multi_modal_order = Some(vec![ MultimodalItemRef { @@ -147,6 +154,10 @@ fn mixed_image_and_video_media_follow_prompt_order() { modality: "image_url".to_string(), index: 0, }, + MultimodalItemRef { + modality: "audio_url".to_string(), + index: 0, + }, MultimodalItemRef { modality: "video_url".to_string(), index: 1, @@ -157,7 +168,7 @@ fn mixed_image_and_video_media_follow_prompt_order() { }, ]); - let proto = build_generate_request(req, "req", "model", image_video_limits(4096), AGG) + let proto = build_generate_request(req, "req", "model", all_media_limits(4096), AGG) .expect("mixed media request"); assert_eq!( @@ -169,6 +180,7 @@ fn mixed_image_and_video_media_follow_prompt_order() { [ pb::Modality::Video as i32, pb::Modality::Image as i32, + pb::Modality::Audio as i32, pb::Modality::Video as i32, pb::Modality::Image as i32, ] @@ -179,7 +191,7 @@ fn mixed_image_and_video_media_follow_prompt_order() { .iter() .map(|item| item.uuid.as_str()) .collect::>(), - ["video-0", "image-0", "video-1", ""] + ["video-0", "image-0", "audio-0", "video-1", ""] ); } @@ -243,7 +255,7 @@ fn mm_routing_hashes_are_not_used_as_engine_cache_uuids() { } #[test] -fn image_media_requires_the_loaded_models_advertised_capability() { +fn media_requires_the_loaded_models_advertised_capability() { let mut req = request(); add_images(&mut req, ["https://example.com/image.png"]); @@ -257,21 +269,36 @@ fn image_media_requires_the_loaded_models_advertised_capability() { "unexpected error: {error}" ); } + + req.multi_modal_data = Some(HashMap::from([( + "audio_url".to_string(), + vec![MultimodalData::RawUrl( + "https://example.com/audio.wav".to_string(), + )], + )])); + let error = build_generate_request(req, "req", "model", image_video_limits(4096), AGG) + .expect_err("a model without audio support must reject audio"); + assert!( + error + .to_string() + .contains("does not advertise audio support"), + "unexpected error: {error}" + ); } #[test] fn unsupported_media_modalities_and_sources_are_rejected() { let mut req = request(); req.multi_modal_data = Some(HashMap::from([( - "audio_url".to_string(), + "future_media".to_string(), vec![MultimodalData::RawUrl( "https://example.com/media".to_string(), )], )])); - let error = build_generate_request(req, "req", "model", image_limits(4096), AGG) - .expect_err("audio is not supported yet"); + let error = build_generate_request(req, "req", "model", all_media_limits(4096), AGG) + .expect_err("unsupported modalities must be rejected"); assert!( - error.to_string().contains("audio_url"), + error.to_string().contains("future_media"), "unexpected error: {error}" ); @@ -302,6 +329,15 @@ fn unsupported_media_modalities_and_sources_are_rejected() { let error = build_generate_request(req, "req", "model", image_video_limits(4096), AGG) .expect_err("the engine cannot fetch a local Dynamo path"); assert!(error.to_string().contains("`video_url` item 0")); + + let mut req = request(); + req.multi_modal_data = Some(HashMap::from([( + "audio_url".to_string(), + vec![MultimodalData::RawUrl("file:///tmp/audio.wav".to_string())], + )])); + let error = build_generate_request(req, "req", "model", all_media_limits(4096), AGG) + .expect_err("the engine cannot fetch a local Dynamo path"); + assert!(error.to_string().contains("`audio_url` item 0")); } #[test] diff --git a/lib/sidecar/trtllm/src/tests/e2e.rs b/lib/sidecar/trtllm/src/tests/e2e.rs index a5a532966ad5..03764559a7f9 100644 --- a/lib/sidecar/trtllm/src/tests/e2e.rs +++ b/lib/sidecar/trtllm/src/tests/e2e.rs @@ -454,3 +454,65 @@ async fn e2e_real_openengine_video() { "the engine must report the expanded video prompt" ); } + +/// Drives an audio data URI through the real sidecar and OpenEngine server. +/// `TRTLLM_E2E_AUDIO_DATA_URI` must contain a complete `data:audio/...` URI. +#[tokio::test] +#[ignore = "requires a live audio-capable TensorRT-LLM OpenEngine server"] +async fn e2e_real_openengine_audio() { + let endpoint = std::env::var("TRTLLM_E2E_ENDPOINT") + .expect("set TRTLLM_E2E_ENDPOINT, e.g. http://127.0.0.1:50051"); + let model = + std::env::var("TRTLLM_E2E_MODEL").unwrap_or_else(|_| "openai/whisper-tiny".to_string()); + let audio = std::env::var("TRTLLM_E2E_AUDIO_DATA_URI") + .expect("set TRTLLM_E2E_AUDIO_DATA_URI to an audio data URI"); + let engine = TrtllmSidecarEngine::new( + GrpcEndpoint::parse(&endpoint, "--grpc-endpoint").expect("valid endpoint"), + transport(1), + ConfiguredModel { + source: model.clone(), + context_length: None, + }, + AGG, + ); + engine + .start(0) + .await + .expect("start against the live audio-capable server"); + + let mut request = PreprocessedRequest::builder() + .model(model) + .token_ids(vec![50258, 50259, 50359, 50363]) + .stop_conditions(StopConditions { + max_tokens: Some(96), + ..Default::default() + }) + .sampling_options(SamplingOptions { + temperature: Some(0.0), + ..Default::default() + }) + .output_options(OutputOptions::default()) + .build() + .expect("request"); + request.multi_modal_data = Some(HashMap::from([( + "audio_url".to_string(), + vec![MultimodalData::RawUrl(audio)], + )])); + + let outputs = collect(&engine, request).await; + let generated = outputs + .iter() + .flat_map(|output| output.token_ids.iter().copied()) + .collect::>(); + let terminal = outputs + .iter() + .find(|output| output.finish_reason.is_some()) + .expect("terminal output"); + let usage = terminal.completion_usage.as_ref().expect("terminal usage"); + eprintln!( + "[e2e-audio] generated={generated:?}; usage: prompt={}, completion={}", + usage.prompt_tokens, usage.completion_tokens + ); + + assert!(!generated.is_empty(), "expected audio-conditioned output"); +} diff --git a/lib/sidecar/trtllm/src/tests/engine.rs b/lib/sidecar/trtllm/src/tests/engine.rs index b330223ab3eb..a2ea02f5d117 100644 --- a/lib/sidecar/trtllm/src/tests/engine.rs +++ b/lib/sidecar/trtllm/src/tests/engine.rs @@ -105,6 +105,34 @@ async fn advertised_video_media_reaches_openengine() { ); } +#[tokio::test] +async fn advertised_audio_media_reaches_openengine() { + let service = FakeTrtllm::default(); + service.supports_multimodal.store(true, Ordering::SeqCst); + service.advertises_audio.store(true, Ordering::SeqCst); + let server = FakeServer::start(service).await; + let engine = engine(&server.endpoint, 1); + engine.start(0).await.expect("start"); + + let mut req = request(); + req.multi_modal_data = Some(HashMap::from([( + "audio_url".to_string(), + vec![MultimodalData::RawUrl( + "data:audio/wav;base64,YXVkaW8=".to_string(), + )], + )])); + collect(&engine, req).await; + + let requests = server.service.requests.lock().await; + let sent = requests.first().expect("recorded request"); + assert_eq!(sent.media.len(), 1); + assert_eq!(sent.media[0].modality, pb::Modality::Audio as i32); + assert!(matches!( + sent.media[0].source.as_ref(), + Some(pb::media_item::Source::DataUri(value)) if value.starts_with("data:audio/wav") + )); +} + #[tokio::test] async fn both_multimodal_capability_signals_are_required() { for (coarse, image) in [(true, false), (false, true)] { @@ -498,9 +526,28 @@ async fn a_configured_context_length_survives_a_server_without_control() { #[tokio::test] async fn media_requests_retry_transient_capability_discovery() { - for (key, url, image, video) in [ - ("image_url", "https://example.com/image.png", true, false), - ("video_url", "https://example.com/video.mp4", false, true), + for (key, url, image, video, audio) in [ + ( + "image_url", + "https://example.com/image.png", + true, + false, + false, + ), + ( + "video_url", + "https://example.com/video.mp4", + false, + true, + false, + ), + ( + "audio_url", + "https://example.com/audio.wav", + false, + false, + true, + ), ] { let service = FakeTrtllm::default(); service.unavailable_model_info.store(true, Ordering::SeqCst); @@ -524,6 +571,10 @@ async fn media_requests_retry_transient_capability_discovery() { .service .advertises_video .store(video, Ordering::SeqCst); + server + .service + .advertises_audio + .store(audio, Ordering::SeqCst); let mut media_request = request(); media_request.multi_modal_data = Some(HashMap::from([( key.to_string(), diff --git a/tests/serve/fixtures/trtllm_whisper_agg.yaml b/tests/serve/fixtures/trtllm_whisper_agg.yaml new file mode 100644 index 000000000000..7bfa6f1e2c19 --- /dev/null +++ b/tests/serve/fixtures/trtllm_whisper_agg.yaml @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +tensor_parallel_size: 1 +max_batch_size: 1 +max_input_len: 1500 +max_num_tokens: 1500 +backend: pytorch +disable_overlap_scheduler: true +enable_chunked_prefill: false + +scheduler_config: + use_python_scheduler: true + +kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.2 + cross_kv_cache_fraction: 0.5 + use_kv_cache_manager_v2: true diff --git a/tests/serve/fixtures/trtllm_whisper_chat_template.jinja b/tests/serve/fixtures/trtllm_whisper_chat_template.jinja new file mode 100644 index 000000000000..a61ec520b339 --- /dev/null +++ b/tests/serve/fixtures/trtllm_whisper_chat_template.jinja @@ -0,0 +1,3 @@ +{# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. #} +{# SPDX-License-Identifier: Apache-2.0 #} +{{ bos_token }} diff --git a/tests/serve/test_sidecar.py b/tests/serve/test_sidecar.py index 2cb78a46bb7b..f3558d31dfd6 100644 --- a/tests/serve/test_sidecar.py +++ b/tests/serve/test_sidecar.py @@ -13,16 +13,16 @@ from dynamo.common.utils.install_media_decoders import VALIDATED_SPECS from tests.router.common import _test_frontend_kv_routing from tests.router.helper import generate_random_suffix -from tests.serve.conftest import ( - MULTIMODAL_IMG_URL, - MULTIMODAL_VIDEO_EXPECTED, - MULTIMODAL_VIDEO_URL, -) from tests.serve.common import ( WORKSPACE_DIR, params_with_model_mark, run_serve_deployment, ) +from tests.serve.conftest import ( + MULTIMODAL_IMG_URL, + MULTIMODAL_VIDEO_EXPECTED, + MULTIMODAL_VIDEO_URL, +) from tests.utils.constants import DynamoPortRange from tests.utils.engine_metrics import ( EngineMetrics, @@ -31,9 +31,10 @@ ) from tests.utils.engine_process import EngineConfig from tests.utils.gpu_args import map_cuda_visible_devices +from tests.utils.multimodal import make_audio_payload from tests.utils.payload_builder import ( - chat_payload, chat_logprobs_and_structured_output_payloads, + chat_payload, chat_payload_default, disaggregated_chat_payload, disaggregated_token_count_payload, @@ -56,6 +57,12 @@ trtllm_sidecar_dir = os.environ.get("TRTLLM_SIDECAR_DIR") or os.path.join( WORKSPACE_DIR, "lib/sidecar/trtllm" ) +trtllm_whisper_chat_template = os.path.join( + WORKSPACE_DIR, "tests/serve/fixtures/trtllm_whisper_chat_template.jinja" +) +trtllm_whisper_engine_args = os.path.join( + WORKSPACE_DIR, "tests/serve/fixtures/trtllm_whisper_agg.yaml" +) def _trtllm_serves_openengine() -> bool: @@ -246,6 +253,34 @@ def _disaggregated_payloads( ), ], ), + "trtllm_aggregated_audio": EngineConfig( + name="trtllm_aggregated_audio", + directory=trtllm_sidecar_dir, + script_name="agg.sh", + marks=[ + pytest.mark.trtllm, + pytest.mark.multimodal, + pytest.mark.gpu_1, + pytest.mark.profiled_vram_gib(6.0), + pytest.mark.requested_trtllm_kv_tokens(2048), + pytest.mark.timeout(960), + # This is the only test that covers audio URL handling across the + # frontend, sidecar, and OpenEngine boundary. + pytest.mark.pre_merge, + ], + model="openai/whisper-tiny", + timeout=900, + delayed_start=60, + env={ + "AGG_ENGINE_ARGS": trtllm_whisper_engine_args, + "DYN_CUSTOM_JINJA_TEMPLATE": trtllm_whisper_chat_template, + "MODALITY": "multimodal", + "PYTHONUNBUFFERED": "1", + "TRTLLM_CONTEXT_LENGTH": "1500", + "TRTLLM_DISABLE_GUIDED_DECODING": "1", + }, + request_payloads=[make_audio_payload(["thoughts affected Hester"])], + ), "trtllm_disaggregated": EngineConfig( name="trtllm_disaggregated", directory=trtllm_sidecar_dir, @@ -359,7 +394,11 @@ def _disaggregated_payloads( def sidecar_config_test(request, monkeypatch, discovery_backend): config = sidecar_configs[request.param] if ( - config.name == "trtllm_aggregated_multimodal" + config.name + in { + "trtllm_aggregated_multimodal", + "trtllm_aggregated_audio", + } and not _trtllm_serves_openengine() ): if os.environ.get("DYNAMO_TEST_SIDECAR_BACKEND") == "trtllm": From 09fb93c3e0704b98abbbcbf0f0e1f3c0c2859092 Mon Sep 17 00:00:00 2001 From: krishung5 Date: Mon, 19 Oct 2026 20:55:37 -0700 Subject: [PATCH 27/61] test(trtllm): tighten video sidecar coverage Signed-off-by: krishung5 --- lib/sidecar/trtllm/src/convert/request_tests.rs | 9 --------- lib/sidecar/trtllm/src/tests/e2e.rs | 1 - tests/serve/test_sidecar.py | 12 ++++++------ 3 files changed, 6 insertions(+), 16 deletions(-) diff --git a/lib/sidecar/trtllm/src/convert/request_tests.rs b/lib/sidecar/trtllm/src/convert/request_tests.rs index c9cb8af8687e..1c3d7368163f 100644 --- a/lib/sidecar/trtllm/src/convert/request_tests.rs +++ b/lib/sidecar/trtllm/src/convert/request_tests.rs @@ -299,15 +299,6 @@ fn unsupported_media_modalities_and_sources_are_rejected() { .to_string() .contains("requires a source for image input") ); - - let mut req = request(); - req.multi_modal_data = Some(HashMap::from([( - "video_url".to_string(), - vec![MultimodalData::RawUrl("file:///tmp/video.mp4".to_string())], - )])); - let error = build_generate_request(req, "req", "model", image_video_limits(4096), AGG) - .expect_err("the engine cannot fetch a local Dynamo path"); - assert!(error.to_string().contains("`video_url` item 0")); } #[test] diff --git a/lib/sidecar/trtllm/src/tests/e2e.rs b/lib/sidecar/trtllm/src/tests/e2e.rs index b959969aef96..43c5e4a7d99b 100644 --- a/lib/sidecar/trtllm/src/tests/e2e.rs +++ b/lib/sidecar/trtllm/src/tests/e2e.rs @@ -383,7 +383,6 @@ async fn e2e_real_openengine_image() { ); } -/// Drives a video data URI through the real sidecar and OpenEngine server. /// `TRTLLM_E2E_VIDEO_DATA_URI` must contain a complete `data:video/...` URI. #[tokio::test] #[ignore = "requires a live video-capable TensorRT-LLM OpenEngine server"] diff --git a/tests/serve/test_sidecar.py b/tests/serve/test_sidecar.py index 2cb78a46bb7b..72e6435a3a3b 100644 --- a/tests/serve/test_sidecar.py +++ b/tests/serve/test_sidecar.py @@ -13,16 +13,16 @@ from dynamo.common.utils.install_media_decoders import VALIDATED_SPECS from tests.router.common import _test_frontend_kv_routing from tests.router.helper import generate_random_suffix -from tests.serve.conftest import ( - MULTIMODAL_IMG_URL, - MULTIMODAL_VIDEO_EXPECTED, - MULTIMODAL_VIDEO_URL, -) from tests.serve.common import ( WORKSPACE_DIR, params_with_model_mark, run_serve_deployment, ) +from tests.serve.conftest import ( + MULTIMODAL_IMG_URL, + MULTIMODAL_VIDEO_EXPECTED, + MULTIMODAL_VIDEO_URL, +) from tests.utils.constants import DynamoPortRange from tests.utils.engine_metrics import ( EngineMetrics, @@ -32,8 +32,8 @@ from tests.utils.engine_process import EngineConfig from tests.utils.gpu_args import map_cuda_visible_devices from tests.utils.payload_builder import ( - chat_payload, chat_logprobs_and_structured_output_payloads, + chat_payload, chat_payload_default, disaggregated_chat_payload, disaggregated_token_count_payload, From 4870a57360883eb5539ae96a84400afd2a75e6f3 Mon Sep 17 00:00:00 2001 From: krishung5 Date: Mon, 19 Oct 2026 20:57:14 -0700 Subject: [PATCH 28/61] fix(trtllm): disable guided decoding explicitly for audio Signed-off-by: krishung5 --- lib/sidecar/trtllm/launch/agg.sh | 2 +- lib/sidecar/trtllm/src/tests.rs | 29 +++-------------------------- lib/sidecar/trtllm/src/tests/e2e.rs | 1 - 3 files changed, 4 insertions(+), 28 deletions(-) diff --git a/lib/sidecar/trtllm/launch/agg.sh b/lib/sidecar/trtllm/launch/agg.sh index 6f04dcbf1be2..d76fafbdc6ee 100755 --- a/lib/sidecar/trtllm/launch/agg.sh +++ b/lib/sidecar/trtllm/launch/agg.sh @@ -103,7 +103,7 @@ HTTP_PORT="${DYN_HTTP_PORT:-8000}" TRTLLM_EXTRA_CONFIG=$(mktemp "${TMPDIR:-/tmp}/dynamo-trtllm-sidecar.XXXXXX.yaml") TRTLLM_REQUIRED_OVERRIDES='{"guided_decoding_backend": "xgrammar"}' if [[ "${TRTLLM_DISABLE_GUIDED_DECODING:-0}" == "1" ]]; then - TRTLLM_REQUIRED_OVERRIDES='{}' + TRTLLM_REQUIRED_OVERRIDES='{"guided_decoding_backend": null}' fi TRTLLM_OVERRIDES=$(build_trtllm_override_args_with_mem \ --merge-with-json "$TRTLLM_REQUIRED_OVERRIDES") diff --git a/lib/sidecar/trtllm/src/tests.rs b/lib/sidecar/trtllm/src/tests.rs index e51a363c9b51..0b0c9b3812b2 100644 --- a/lib/sidecar/trtllm/src/tests.rs +++ b/lib/sidecar/trtllm/src/tests.rs @@ -1006,28 +1006,9 @@ async fn a_configured_context_length_survives_a_server_without_control() { #[tokio::test] async fn media_requests_retry_transient_capability_discovery() { - for (key, url, image, video, audio) in [ - ( - "image_url", - "https://example.com/image.png", - true, - false, - false, - ), - ( - "video_url", - "https://example.com/video.mp4", - false, - true, - false, - ), - ( - "audio_url", - "https://example.com/audio.wav", - false, - false, - true, - ), + for (key, url, image, video) in [ + ("image_url", "https://example.com/image.png", true, false), + ("video_url", "https://example.com/video.mp4", false, true), ] { let service = FakeTrtllm::default(); service.unavailable_model_info.store(true, Ordering::SeqCst); @@ -1051,10 +1032,6 @@ async fn media_requests_retry_transient_capability_discovery() { .service .advertises_video .store(video, Ordering::SeqCst); - server - .service - .advertises_audio - .store(audio, Ordering::SeqCst); let mut media_request = request(); media_request.multi_modal_data = Some(HashMap::from([( key.to_string(), diff --git a/lib/sidecar/trtllm/src/tests/e2e.rs b/lib/sidecar/trtllm/src/tests/e2e.rs index cffdb0078bc3..73696a8981bb 100644 --- a/lib/sidecar/trtllm/src/tests/e2e.rs +++ b/lib/sidecar/trtllm/src/tests/e2e.rs @@ -456,7 +456,6 @@ async fn e2e_real_openengine_video() { ); } -/// Drives an audio data URI through the real sidecar and OpenEngine server. /// `TRTLLM_E2E_AUDIO_DATA_URI` must contain a complete `data:audio/...` URI. #[tokio::test] #[ignore = "requires a live audio-capable TensorRT-LLM OpenEngine server"] From 66bd6aaff3af384e1457055c9ac8b0206dc2c101 Mon Sep 17 00:00:00 2001 From: krishung5 Date: Mon, 19 Oct 2026 21:02:09 -0700 Subject: [PATCH 29/61] ci(trtllm): refresh image source overlay Signed-off-by: krishung5 --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index b22da614073d..821a768c9989 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -547,7 +547,7 @@ jobs: # the upstream change. extra_build_args: | TRTLLM_SOURCE_REPOSITORY=https://github.com/krishung5/TensorRT-LLM - TRTLLM_SOURCE_COMMIT=cfcc7b860e7fa56c4f42f209997260e4927a401b + TRTLLM_SOURCE_COMMIT=1e3f7f85497657ae1fc7e1edf125f6dc26ccecbf # actions: read lets the build download a prior run's baseline compliance # artifact to diff against (a reusable workflow's token can't exceed the # caller's). Missing -> the diff soft-degrades, build stays green. From 4b7c82280fe3633745963ad8e4e61c4bacc8eef1 Mon Sep 17 00:00:00 2001 From: krishung5 Date: Tue, 20 Oct 2026 09:51:07 -0700 Subject: [PATCH 30/61] ci(trtllm): refresh video source overlay Signed-off-by: krishung5 --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 96689226708d..54c037d008fc 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -547,7 +547,7 @@ jobs: # the upstream change. extra_build_args: | TRTLLM_SOURCE_REPOSITORY=https://github.com/krishung5/TensorRT-LLM - TRTLLM_SOURCE_COMMIT=36e555272e66b1d2db9b61da80dea5c129fa539d + TRTLLM_SOURCE_COMMIT=b591693204e2443324f3f61e5d9e2bc1207c5fa1 # actions: read lets the build download a prior run's baseline compliance # artifact to diff against (a reusable workflow's token can't exceed the # caller's). Missing -> the diff soft-degrades, build stays green. From f783f9ddae13a40351781a0d853591684dce48b0 Mon Sep 17 00:00:00 2001 From: krishung5 Date: Tue, 20 Oct 2026 12:38:45 -0700 Subject: [PATCH 31/61] fix(trtllm): reject non-mapping engine configs Signed-off-by: krishung5 --- lib/sidecar/trtllm/launch/common.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/sidecar/trtllm/launch/common.sh b/lib/sidecar/trtllm/launch/common.sh index ce6c8b76115b..5831d4a7b6ed 100755 --- a/lib/sidecar/trtllm/launch/common.sh +++ b/lib/sidecar/trtllm/launch/common.sh @@ -116,7 +116,9 @@ def merge(base, overrides): base_path, overrides_json, output_path = sys.argv[1:] with open(base_path, encoding="utf-8") as source: - config = yaml.safe_load(source) or {} + config = yaml.safe_load(source) +if config is None: + config = {} if not isinstance(config, dict): raise TypeError(f"TensorRT-LLM engine config must be a mapping: {base_path}") merge(config, json.loads(overrides_json)) From 6f49127c2b0a4d523bed7ec1a94938392b6c75e0 Mon Sep 17 00:00:00 2001 From: krishung5 Date: Tue, 20 Oct 2026 12:38:49 -0700 Subject: [PATCH 32/61] fix(trtllm): preserve text-only model prompts Signed-off-by: krishung5 --- .github/workflows/pr.yaml | 2 +- lib/sidecar/trtllm/src/client.rs | 20 ++++++ lib/sidecar/trtllm/src/convert.rs | 54 ++++++++++++---- .../trtllm/src/convert/request_tests.rs | 62 ++++++++++++++++++- lib/sidecar/trtllm/src/test_fixtures.rs | 4 ++ lib/sidecar/trtllm/src/tests.rs | 20 ++++++ .../trtllm_whisper_chat_template.jinja | 6 +- 7 files changed, 152 insertions(+), 16 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 5272bb2764f5..38fb81308d94 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -547,7 +547,7 @@ jobs: # the upstream change. extra_build_args: | TRTLLM_SOURCE_REPOSITORY=https://github.com/krishung5/TensorRT-LLM - TRTLLM_SOURCE_COMMIT=8978a9eedee4974a78100871631e362f99f61c7d + TRTLLM_SOURCE_COMMIT=4e4c9d14f73646ac9184dd510326b79a4bafe833 # actions: read lets the build download a prior run's baseline compliance # artifact to diff against (a reusable workflow's token can't exceed the # caller's). Missing -> the diff soft-degrades, build stays green. diff --git a/lib/sidecar/trtllm/src/client.rs b/lib/sidecar/trtllm/src/client.rs index 120a87635828..e0f57e078e2c 100644 --- a/lib/sidecar/trtllm/src/client.rs +++ b/lib/sidecar/trtllm/src/client.rs @@ -35,6 +35,13 @@ pub(crate) struct SupportedMedia { pub(crate) audio: bool, } +/// Input forms explicitly advertised by `Control.GetModelInfo`. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct SupportedInputs { + pub(crate) text: bool, + pub(crate) token_ids: bool, +} + impl SupportedMedia { pub(crate) fn contains(self, modality: &str) -> bool { match modality { @@ -56,6 +63,9 @@ pub(crate) struct ModelLimits { /// Cap on generated tokens, when the engine reports one. A context window /// alone can imply a larger budget than the engine will accept. pub(crate) max_output_tokens: Option, + /// `None` preserves token-ID behavior for older servers that omit both + /// optional input-capability fields. + pub(crate) supported_inputs: Option, /// Modalities advertised by both the portable and engine-specific capabilities. /// `None` means startup discovery failed transiently and may be retried. pub(crate) supported_media: Option, @@ -128,6 +138,15 @@ impl TrtllmClient { .await? .into_inner(); let mut supported_media = SupportedMedia::default(); + let supported_inputs = + if info.supports_text_input.is_none() && info.supports_token_ids_input.is_none() { + None + } else { + Some(SupportedInputs { + text: info.supports_text_input == Some(true), + token_ids: info.supports_token_ids_input == Some(true), + }) + }; if info.supports_multimodal == Some(true) && let Some(prost_types::value::Kind::ListValue(modalities)) = info .extra @@ -151,6 +170,7 @@ impl TrtllmClient { Ok(ModelLimits { context_length: info.max_context_length.filter(|len| *len > 0), max_output_tokens: info.max_output_tokens.filter(|cap| *cap > 0), + supported_inputs, supported_media: Some(supported_media), }) } diff --git a/lib/sidecar/trtllm/src/convert.rs b/lib/sidecar/trtllm/src/convert.rs index 2092af604156..693349ee874a 100644 --- a/lib/sidecar/trtllm/src/convert.rs +++ b/lib/sidecar/trtllm/src/convert.rs @@ -39,10 +39,6 @@ pub(crate) fn build_generate_request( let request_has_media = has_media(&request); let media = media(&mut request, limits)?; - let sampling = &request.sampling_options; - let stop = &request.stop_conditions; - let output = &request.output_options; - // A prefill worker only needs the context phase; TensorRT-LLM still requires // a positive budget, and one token is what the context phase produces. // No minimum check here: the minimum applies to the decode worker's budget, @@ -81,16 +77,17 @@ pub(crate) fn build_generate_request( .map(|context| context.session_id.trim()) .filter(|session_id| !session_id.is_empty()) .map(str::to_string); - let token_ids = Arc::unwrap_or_clone(request.token_ids); + let input = request_input(&mut request, limits)?; + let sampling = &request.sampling_options; + let stop = &request.stop_conditions; + let output = &request.output_options; Ok(pb::GenerateRequest { request_id: request_id.to_string(), // The OpenEngine server rejects an empty model; any non-empty name is // served by the loaded model (single-model server). model: model.to_string(), - input: Some(pb::generate_request::Input::TokenIds(pb::TokenIds { - ids: token_ids, - })), + input: Some(input), sampling: Some(pb::SamplingParams { temperature: sampling.temperature.map(f64::from), top_p: sampling.top_p.map(f64::from), @@ -144,6 +141,44 @@ pub(crate) fn build_generate_request( }) } +fn request_input( + request: &mut PreprocessedRequest, + limits: Option, +) -> Result { + match limits.and_then(|limits| limits.supported_inputs) { + Some(inputs) if inputs.token_ids => token_ids_input(request), + Some(inputs) if inputs.text => { + let prompt = request + .extra_args + .as_ref() + .and_then(|extra| extra.get("formatted_prompt")) + .and_then(serde_json::Value::as_str) + .ok_or_else(|| { + client::invalid_argument( + "the loaded TensorRT-LLM model requires text input, but the request has no string extra_args.formatted_prompt", + ) + })?; + Ok(pb::generate_request::Input::Prompt(prompt.to_string())) + } + Some(_) => Err(client::invalid_argument( + "the loaded TensorRT-LLM model advertises neither text nor token-ID input support", + )), + // Older OpenEngine servers leave both optional capability fields unset. + None => token_ids_input(request), + } +} + +fn token_ids_input( + request: &mut PreprocessedRequest, +) -> Result { + if request.token_ids.is_empty() { + return Err(client::invalid_argument("token_ids must not be empty")); + } + Ok(pb::generate_request::Input::TokenIds(pb::TokenIds { + ids: Arc::unwrap_or_clone(std::mem::take(&mut request.token_ids)), + })) +} + // Derive omitted text max_tokens from the context window. For media, only // TensorRT-LLM knows the prompt length after expansion, so leave it unset. // Remove this fallback when NVIDIA/TensorRT-LLM#16549 lands; context_length @@ -578,9 +613,6 @@ fn validate_request( request: &PreprocessedRequest, mode: DisaggregationMode, ) -> Result<(), DynamoError> { - if request.token_ids.is_empty() { - return Err(client::invalid_argument("token_ids must not be empty")); - } if request.prompt_embeds.is_some() { return Err(client::invalid_argument( "prompt embeddings are not supported by the TensorRT-LLM sidecar", diff --git a/lib/sidecar/trtllm/src/convert/request_tests.rs b/lib/sidecar/trtllm/src/convert/request_tests.rs index 1801b9d460ea..ea94746236c5 100644 --- a/lib/sidecar/trtllm/src/convert/request_tests.rs +++ b/lib/sidecar/trtllm/src/convert/request_tests.rs @@ -6,7 +6,7 @@ use std::collections::HashMap; use super::*; -use crate::client::SupportedMedia; +use crate::client::{SupportedInputs, SupportedMedia}; use crate::test_fixtures::*; use dynamo_backend_common::MultimodalItemRef; use serde_json::json; @@ -78,6 +78,65 @@ fn request_maps_sampling_stop_and_output_fields() { } } +#[test] +fn text_input_model_receives_the_formatted_audio_prompt() { + let mut req = request(); + req.token_ids = Arc::new(Vec::new()); + req.multi_modal_data = Some(HashMap::from([( + "audio_url".to_string(), + vec![MultimodalData::RawUrl( + "https://example.com/audio.wav".to_string(), + )], + )])); + let task_prompt = "<|startoftranscript|><|fr|><|translate|><|notimestamps|>"; + req.extra_args = Some(json!({"formatted_prompt": task_prompt})); + let mut model_limits = all_media_limits(4096).expect("limits"); + model_limits.supported_inputs = Some(SupportedInputs { + text: true, + token_ids: false, + }); + + let proto = build_generate_request(req, "req", "model", Some(model_limits), AGG) + .expect("text-input audio request"); + assert_eq!( + proto.input, + Some(pb::generate_request::Input::Prompt(task_prompt.to_string())) + ); +} + +#[test] +fn text_input_model_accepts_an_empty_formatted_prompt() { + let mut req = request(); + req.token_ids = Arc::new(Vec::new()); + req.extra_args = Some(json!({"formatted_prompt": ""})); + let mut model_limits = limits(4096).expect("limits"); + model_limits.supported_inputs = Some(SupportedInputs { + text: true, + token_ids: false, + }); + + let proto = build_generate_request(req, "req", "model", Some(model_limits), AGG) + .expect("an empty prompt lets the engine select its model default"); + assert_eq!( + proto.input, + Some(pb::generate_request::Input::Prompt(String::new())) + ); +} + +#[test] +fn text_input_model_requires_a_formatted_prompt() { + let req = request(); + let mut model_limits = limits(4096).expect("limits"); + model_limits.supported_inputs = Some(SupportedInputs { + text: true, + token_ids: false, + }); + + let error = build_generate_request(req, "req", "model", Some(model_limits), AGG) + .expect_err("a token-only preprocessed request cannot serve a text-input model"); + assert!(error.to_string().contains("extra_args.formatted_prompt")); +} + #[test] fn image_urls_data_uris_and_uuids_are_mapped_in_order() { let mut req = request(); @@ -472,6 +531,7 @@ fn the_derived_max_tokens_respects_the_engines_output_cap() { Some(ModelLimits { context_length: Some(4096), max_output_tokens: Some(256), + supported_inputs: None, supported_media: Some(SupportedMedia::default()), }), AGG, diff --git a/lib/sidecar/trtllm/src/test_fixtures.rs b/lib/sidecar/trtllm/src/test_fixtures.rs index 54f7cea97617..199c5379534c 100644 --- a/lib/sidecar/trtllm/src/test_fixtures.rs +++ b/lib/sidecar/trtllm/src/test_fixtures.rs @@ -55,6 +55,7 @@ pub(crate) fn limits(context_length: u32) -> Option { Some(ModelLimits { context_length: Some(context_length), max_output_tokens: None, + supported_inputs: None, supported_media: Some(SupportedMedia::default()), }) } @@ -63,6 +64,7 @@ pub(crate) fn image_limits(context_length: u32) -> Option { Some(ModelLimits { context_length: Some(context_length), max_output_tokens: None, + supported_inputs: None, supported_media: Some(SupportedMedia { image: true, ..Default::default() @@ -74,6 +76,7 @@ pub(crate) fn image_video_limits(context_length: u32) -> Option { Some(ModelLimits { context_length: Some(context_length), max_output_tokens: None, + supported_inputs: None, supported_media: Some(SupportedMedia { image: true, video: true, @@ -86,6 +89,7 @@ pub(crate) fn all_media_limits(context_length: u32) -> Option { Some(ModelLimits { context_length: Some(context_length), max_output_tokens: None, + supported_inputs: None, supported_media: Some(SupportedMedia { image: true, video: true, diff --git a/lib/sidecar/trtllm/src/tests.rs b/lib/sidecar/trtllm/src/tests.rs index 0b0c9b3812b2..50a1efce8c43 100644 --- a/lib/sidecar/trtllm/src/tests.rs +++ b/lib/sidecar/trtllm/src/tests.rs @@ -56,6 +56,9 @@ struct FakeTrtllm { advertises_image: Arc, advertises_video: Arc, advertises_audio: Arc, + advertises_input_capabilities: Arc, + supports_text_input: Arc, + supports_token_ids_input: Arc, model_info_calls: Arc, } @@ -191,6 +194,8 @@ impl pb::control_server::Control for FakeTrtllm { return Err(Status::unavailable("not serving yet")); } let supports_multimodal = self.supports_multimodal.load(Ordering::SeqCst); + let advertises_input_capabilities = + self.advertises_input_capabilities.load(Ordering::SeqCst); let modalities = [ (self.advertises_image.load(Ordering::SeqCst), "image"), (self.advertises_video.load(Ordering::SeqCst), "video"), @@ -209,6 +214,10 @@ impl pb::control_server::Control for FakeTrtllm { } else { Some(4096) }, + supports_text_input: advertises_input_capabilities + .then(|| self.supports_text_input.load(Ordering::SeqCst)), + supports_token_ids_input: advertises_input_capabilities + .then(|| self.supports_token_ids_input.load(Ordering::SeqCst)), supports_multimodal: Some(supports_multimodal), extra: Some(prost_types::Struct { fields: [( @@ -611,11 +620,18 @@ async fn advertised_audio_media_reaches_openengine() { let service = FakeTrtllm::default(); service.supports_multimodal.store(true, Ordering::SeqCst); service.advertises_audio.store(true, Ordering::SeqCst); + service + .advertises_input_capabilities + .store(true, Ordering::SeqCst); + service.supports_text_input.store(true, Ordering::SeqCst); let server = FakeServer::start(service).await; let engine = engine(&server.endpoint, 1); engine.start(0).await.expect("start"); let mut req = request(); + req.token_ids = Arc::new(Vec::new()); + let task_prompt = "<|startoftranscript|><|fr|><|translate|><|notimestamps|>"; + req.extra_args = Some(json!({"formatted_prompt": task_prompt})); req.multi_modal_data = Some(HashMap::from([( "audio_url".to_string(), vec![MultimodalData::RawUrl( @@ -626,6 +642,10 @@ async fn advertised_audio_media_reaches_openengine() { let requests = server.service.requests.lock().await; let sent = requests.first().expect("recorded request"); + assert_eq!( + sent.input, + Some(pb::generate_request::Input::Prompt(task_prompt.to_string())) + ); assert_eq!(sent.media.len(), 1); assert_eq!(sent.media[0].modality, pb::Modality::Audio as i32); assert!(matches!( diff --git a/tests/serve/fixtures/trtllm_whisper_chat_template.jinja b/tests/serve/fixtures/trtllm_whisper_chat_template.jinja index a61ec520b339..85f926c7bb4d 100644 --- a/tests/serve/fixtures/trtllm_whisper_chat_template.jinja +++ b/tests/serve/fixtures/trtllm_whisper_chat_template.jinja @@ -1,3 +1,3 @@ -{# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. #} -{# SPDX-License-Identifier: Apache-2.0 #} -{{ bos_token }} +{#- SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -#} +{#- SPDX-License-Identifier: Apache-2.0 -#} +{#- An empty prompt selects the checkpoint's default language and transcription task. -#} From 612b9482022c19825251007ca48209117bd344da Mon Sep 17 00:00:00 2001 From: krishung5 Date: Wed, 21 Oct 2026 12:54:50 -0700 Subject: [PATCH 33/61] fix(trtllm): address image sidecar discovery and overlay review Signed-off-by: krishung5 --- .github/workflows/pr.yaml | 2 +- .../deps/trtllm/validate_media_overlay.py | 94 ++++++ container/templates/trtllm_runtime.Dockerfile | 7 + lib/sidecar/testkit/tests/support/trtllm.rs | 164 +++++++++-- .../testkit/tests/trtllm_multimodal.rs | 274 ++++++++++++++++++ lib/sidecar/trtllm/src/client.rs | 3 + lib/sidecar/trtllm/src/convert.rs | 12 +- .../trtllm/src/convert/request_tests.rs | 22 ++ lib/sidecar/trtllm/src/engine.rs | 57 +++- lib/sidecar/trtllm/src/test_fixtures.rs | 1 + lib/sidecar/trtllm/src/tests.rs | 116 +------- tests/serve/test_sidecar.py | 8 + 12 files changed, 614 insertions(+), 146 deletions(-) create mode 100644 container/deps/trtllm/validate_media_overlay.py create mode 100644 lib/sidecar/testkit/tests/trtllm_multimodal.rs diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 90837cb9b245..e64fe4655932 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -547,7 +547,7 @@ jobs: # the upstream change. extra_build_args: | TRTLLM_SOURCE_REPOSITORY=https://github.com/krishung5/TensorRT-LLM - TRTLLM_SOURCE_COMMIT=1e3f7f85497657ae1fc7e1edf125f6dc26ccecbf + TRTLLM_SOURCE_COMMIT=c0e8499a981b28defb27e0d3e5a44faff17ae509 # actions: read lets the build download a prior run's baseline compliance # artifact to diff against (a reusable workflow's token can't exceed the # caller's). Missing -> the diff soft-degrades, build stays green. diff --git a/container/deps/trtllm/validate_media_overlay.py b/container/deps/trtllm/validate_media_overlay.py new file mode 100644 index 000000000000..b0a68b602dce --- /dev/null +++ b/container/deps/trtllm/validate_media_overlay.py @@ -0,0 +1,94 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Exercise retained runtime helpers against the overlaid fetch signature.""" + +import ast +import asyncio +import importlib.util +import inspect +import sys +from io import BytesIO +from pathlib import Path +from unittest.mock import patch + + +def check_signatures(): + package = Path( + next(iter(importlib.util.find_spec("tensorrt_llm").submodule_search_locations)) + ) + loader = ast.parse((package / "inputs/media_io.py").read_text()) + fetch = next( + node + for node in loader.body + if isinstance(node, ast.AsyncFunctionDef) and node.name == "_safe_aiohttp_get" + ) + accepted = {arg.arg for arg in fetch.args.args + fetch.args.kwonlyargs} + callers = ast.parse((package / "inputs/utils.py").read_text()) + checked = set() + for function in callers.body: + if not isinstance(function, ast.AsyncFunctionDef): + continue + for node in ast.walk(function): + if ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "_safe_aiohttp_get" + ): + assert all( + keyword.arg in accepted for keyword in node.keywords + ), function.name + checked.add(function.name) + assert {"async_load_image", "async_load_video", "async_load_audio"} <= checked + print("TRT-LLM overlay: retained media helper signatures passed") + + +async def validate(): + import numpy as np + import soundfile + from PIL import Image + from tensorrt_llm.inputs import media_io, utils + + image_bytes = BytesIO() + Image.new("RGB", (32, 32), "red").save(image_bytes, format="PNG") + audio_bytes = BytesIO() + soundfile.write(audio_bytes, np.zeros(1600), 16000, format="WAV") + payloads = { + "image": image_bytes.getvalue(), + "video": b"video", + "audio": audio_bytes.getvalue(), + } + calls = [] + + async def fetch(url, timeout_sec=30): + # Enforce the installed loader's signature without fetching external media. + inspect.signature(media_io._safe_aiohttp_get).bind(url, timeout_sec=timeout_sec) + name = url.rsplit("/", 1)[-1] + calls.append(name) + return payloads[name] + + def decode_video(path, *args, **kwargs): + # OpenCV is optional in the base image; this checks the URL caller, not decoding. + assert Path(path).read_bytes() == payloads["video"] + return "video-decoded" + + with ( + patch.object(utils, "_safe_aiohttp_get", fetch), + patch.object(utils, "_load_video_by_cv2", decode_video), + ): + image = await utils.async_load_image("https://example.com/image", format="pil") + assert image.size == (32, 32) + video = await utils.async_load_video( + "https://example.com/video", num_frames=2, format="pil" + ) + assert video == "video-decoded" + audio, sample_rate = await utils.async_load_audio("https://example.com/audio") + assert len(audio) == 1600 and sample_rate == 16000 + assert calls == ["image", "video", "audio"] + print("TRT-LLM overlay: image, video, and audio helpers passed") + + +if __name__ == "__main__": + check_signatures() + if "--check-signatures-only" not in sys.argv[1:]: + asyncio.run(validate()) diff --git a/container/templates/trtllm_runtime.Dockerfile b/container/templates/trtllm_runtime.Dockerfile index a88b71bc4d75..65c4056c6393 100644 --- a/container/templates/trtllm_runtime.Dockerfile +++ b/container/templates/trtllm_runtime.Dockerfile @@ -43,6 +43,8 @@ ARG NIXL_REF ARG TRTLLM_SOURCE_REPOSITORY ARG TRTLLM_SOURCE_COMMIT +COPY container/deps/trtllm/validate_media_overlay.py /tmp/validate_media_overlay.py + # Optional source overlay for cross-repository CI. Keep the base image's server # lifecycle entry points, which are coupled to that release's frontend runtime. # capabilities imports model_config; install that helper with the media loader. @@ -72,9 +74,14 @@ RUN if [ -n "${TRTLLM_SOURCE_COMMIT}" ]; then \ install -m 0644 "${source_dir}/tensorrt_llm/inputs/${module}" \ "${package_dir}/inputs/${module}"; \ done && \ + sed -i \ + -e '/^[[:space:]]*session = await _get_aiohttp_session()$/d' \ + -e 's/, session=session)/)/g' \ + "${package_dir}/inputs/utils.py" && \ test -f "${package_dir}/inputs/model_config.py" && \ grep -Fq 'from tensorrt_llm.inputs.model_config import resolve_top_level_model_type' \ "${package_dir}/grpc/openengine/capabilities.py" && \ + /usr/bin/python3 /tmp/validate_media_overlay.py --check-signatures-only && \ rm -rf "${source_dir}"; \ fi diff --git a/lib/sidecar/testkit/tests/support/trtllm.rs b/lib/sidecar/testkit/tests/support/trtllm.rs index 3b543ea468e6..8f020c96762a 100644 --- a/lib/sidecar/testkit/tests/support/trtllm.rs +++ b/lib/sidecar/testkit/tests/support/trtllm.rs @@ -3,6 +3,7 @@ use std::collections::HashMap; use std::process::Command; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; use dynamo_backend_common::{BackendError, DisaggregationMode, PreprocessedRequest}; @@ -15,7 +16,7 @@ use dynamo_trtllm_mocker::{MockerServerConfig, ServerMode, TrtllmMockerService}; use dynamo_trtllm_sidecar::TrtllmSidecarEngine; use dynamo_trtllm_sidecar::proto::{ self as pb, - control_server::ControlServer, + control_server::{Control, ControlServer}, inference_server::{Inference, InferenceServer}, }; use futures::stream::BoxStream; @@ -32,9 +33,51 @@ pub struct Fixture { service: TrtllmMockerService, server: TestServer, scripted: Arc>>>, + pub discovery: Arc, +} + +#[derive(Default)] +pub struct Discovery { + pub calls: AtomicUsize, + pub unavailable: AtomicBool, + pub hold: AtomicBool, + pub portable_image: AtomicBool, + pub advertises_image: AtomicBool, + release: tokio::sync::Notify, } impl Fixture { + pub fn release_discovery(&self) { + self.discovery.hold.store(false, Ordering::SeqCst); + self.discovery.release.notify_waiters(); + } + + pub fn engine_with_context_length(&self, context_length: u32) -> TrtllmSidecarEngine { + self.make_engine(Some(context_length)) + } + + fn make_engine(&self, context_length: Option) -> TrtllmSidecarEngine { + let mut args = vec![ + "dynamo-trtllm-sidecar".into(), + "--grpc-endpoint".into(), + self.server.endpoint(), + "--model-path".into(), + self.config.model.clone(), + "--disaggregation-mode".into(), + self.config.disaggregation_mode.to_string(), + "--grpc-connections".into(), + self.config.connections.to_string(), + "--grpc-startup-deadline-secs".into(), + "5".into(), + "--grpc-connect-attempt-timeout-secs".into(), + "1".into(), + ]; + if let Some(context_length) = context_length { + args.extend(["--context-length".into(), context_length.to_string()]); + } + TrtllmSidecarEngine::from_args(args).unwrap().0 + } + pub fn respond(&self, request_id: &str, responses: Vec) { assert!( self.scripted @@ -74,7 +117,11 @@ impl SidecarFixture for Fixture { inner: service.clone(), control, }; - let control_service = service.clone(); + let discovery = Arc::new(Discovery::default()); + let control_service = ControlledControl { + inner: service.clone(), + discovery: discovery.clone(), + }; let server = TestServer::start(move |listener, shutdown| async move { tonic::transport::Server::builder() .add_service(InferenceServer::new(controlled)) @@ -92,27 +139,12 @@ impl SidecarFixture for Fixture { service, server, scripted, + discovery, } } async fn engine(&self) -> Self::Engine { - TrtllmSidecarEngine::from_args(vec![ - "dynamo-trtllm-sidecar".into(), - "--grpc-endpoint".into(), - self.server.endpoint(), - "--model-path".into(), - self.config.model.clone(), - "--disaggregation-mode".into(), - self.config.disaggregation_mode.to_string(), - "--grpc-connections".into(), - self.config.connections.to_string(), - "--grpc-startup-deadline-secs".into(), - "5".into(), - "--grpc-connect-attempt-timeout-secs".into(), - "1".into(), - ]) - .unwrap() - .0 + self.make_engine(None) } fn eof_error() -> BackendError { @@ -139,6 +171,100 @@ impl SidecarFixture for Fixture { } } +struct ControlledControl { + inner: TrtllmMockerService, + discovery: Arc, +} + +#[tonic::async_trait] +impl Control for ControlledControl { + async fn get_model_info( + &self, + request: Request, + ) -> Result, Status> { + let released = self.discovery.release.notified(); + self.discovery.calls.fetch_add(1, Ordering::SeqCst); + if self.discovery.hold.load(Ordering::SeqCst) { + released.await; + } + if self.discovery.unavailable.load(Ordering::SeqCst) { + return Err(Status::unavailable("discovery temporarily unavailable")); + } + let mut response = self.inner.get_model_info(request).await?; + let info = response.get_mut(); + info.supports_multimodal = Some(self.discovery.portable_image.load(Ordering::SeqCst)); + let modalities = if self.discovery.advertises_image.load(Ordering::SeqCst) { + vec!["image"] + } else { + vec![] + }; + info.extra = Some( + dynamo_sidecar_common::json_to_struct( + serde_json::json!({"trtllm_supported_modalities": modalities}), + "test capabilities", + ) + .unwrap(), + ); + Ok(response) + } + + async fn get_server_info( + &self, + request: Request, + ) -> Result, Status> { + self.inner.get_server_info(request).await + } + async fn get_load( + &self, + request: Request, + ) -> Result, Status> { + self.inner.get_load(request).await + } + async fn health( + &self, + request: Request, + ) -> Result, Status> { + self.inner.health(request).await + } + async fn abort( + &self, + request: Request, + ) -> Result, Status> { + self.inner.abort(request).await + } + async fn load_lora( + &self, + request: Request, + ) -> Result, Status> { + self.inner.load_lora(request).await + } + async fn unload_lora( + &self, + request: Request, + ) -> Result, Status> { + self.inner.unload_lora(request).await + } + async fn list_loras( + &self, + request: Request, + ) -> Result, Status> { + self.inner.list_loras(request).await + } + async fn get_kv_event_sources( + &self, + request: Request, + ) -> Result, Status> { + self.inner.get_kv_event_sources(request).await + } + type SubscribeKvEventsStream = ::SubscribeKvEventsStream; + async fn subscribe_kv_events( + &self, + request: Request, + ) -> Result, Status> { + self.inner.subscribe_kv_events(request).await + } +} + impl WireFixture for Fixture { fn assert_stream( handle: &RequestHandle, diff --git a/lib/sidecar/testkit/tests/trtllm_multimodal.rs b/lib/sidecar/testkit/tests/trtllm_multimodal.rs new file mode 100644 index 000000000000..067dc304e012 --- /dev/null +++ b/lib/sidecar/testkit/tests/trtllm_multimodal.rs @@ -0,0 +1,274 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::collections::HashMap; +use std::sync::atomic::Ordering; +use std::time::Duration; + +use dynamo_backend_common::testing::mock_context; +use dynamo_backend_common::{ + BackendError, ErrorType, FinishReason, GenerateContext, LLMEngine, MultimodalData, + PreprocessedRequest, +}; +use dynamo_sidecar_testkit::bounded; +use dynamo_sidecar_testkit::control::{Controller, RequestPlan}; +use dynamo_sidecar_testkit::fixtures::{collect, request}; +use dynamo_trtllm_sidecar::proto as pb; +use futures::future::join_all; + +#[allow(dead_code)] +mod support; +use support::trtllm::{Adapter, Fixture}; +use support::{FixtureConfig, SidecarFixture}; + +fn image_request() -> PreprocessedRequest { + let mut request = request("mocker-model", vec![11, 22, 33], 1); + request.stop_conditions.max_tokens = None; + request.multi_modal_data = Some(HashMap::from([( + "image_url".into(), + vec![ + MultimodalData::RawUrl("https://example.com/first.png".into()), + MultimodalData::RawUrl("data:image/png;base64,aW1hZ2U=".into()), + ], + )])); + request.multi_modal_uuids = Some(HashMap::from([( + "image_url".into(), + vec![Some("first".into()), None], + )])); + request +} + +fn finished() -> pb::GenerateResponse { + pb::GenerateResponse { + event: Some(pb::generate_response::Event::Finished( + pb::GenerationFinished { + output_index: Some(0), + reason: pb::FinishReason::Stop as i32, + ..Default::default() + }, + )), + usage: Some(pb::Usage { + prompt_tokens: 131, + total_tokens: 131, + ..Default::default() + }), + ..Default::default() + } +} + +fn advertise_images(fixture: &Fixture) { + fixture + .discovery + .portable_image + .store(true, Ordering::SeqCst); + fixture + .discovery + .advertises_image + .store(true, Ordering::SeqCst); +} + +async fn wait_for_calls(fixture: &Fixture, count: usize) { + bounded("Control discovery reached", async { + while fixture.discovery.calls.load(Ordering::SeqCst) < count { + tokio::task::yield_now().await; + } + }) + .await; +} + +#[tokio::test] +async fn images_preserve_order_uuids_and_engine_expanded_usage() { + let control = Controller::::default(); + let mut fixture = Fixture::start(control.clone(), FixtureConfig::default()).await; + advertise_images(&fixture); + let engine = fixture.engine().await; + engine.start(0).await.unwrap(); + let ctx = mock_context(); + let handle = control.request(ctx.id(), RequestPlan::default()); + fixture.respond(ctx.id(), vec![finished()]); + let outputs = collect(&engine, image_request(), GenerateContext::new(ctx, None)).await; + let output = outputs.last().unwrap().as_ref().unwrap(); + assert_eq!(output.finish_reason, Some(FinishReason::Stop)); + assert_eq!(output.completion_usage.as_ref().unwrap().prompt_tokens, 131); + let native = handle.native_request().unwrap(); + assert_eq!(native.stopping.unwrap().max_tokens, None); + assert_eq!(native.media.len(), 2); + assert!( + native + .media + .iter() + .all(|item| item.modality == pb::Modality::Image as i32) + ); + assert_eq!(native.media[0].uuid, "first"); + assert_eq!(native.media[1].uuid, ""); + assert_eq!( + native.media[0].source, + Some(pb::media_item::Source::Url( + "https://example.com/first.png".into() + )) + ); + assert_eq!( + native.media[1].source, + Some(pb::media_item::Source::DataUri( + "data:image/png;base64,aW1hZ2U=".into() + )) + ); + engine.cleanup().await.unwrap(); + fixture.shutdown().await; +} + +#[tokio::test] +async fn both_image_capability_signals_are_required() { + for (portable, image) in [(true, false), (false, true)] { + let control = Controller::::default(); + let mut fixture = Fixture::start(control.clone(), FixtureConfig::default()).await; + fixture + .discovery + .portable_image + .store(portable, Ordering::SeqCst); + fixture + .discovery + .advertises_image + .store(image, Ordering::SeqCst); + let engine = fixture.engine().await; + engine.start(0).await.unwrap(); + let ctx = mock_context(); + let handle = control.request(ctx.id(), RequestPlan::default()); + assert!( + engine + .generate(image_request(), GenerateContext::new(ctx, None)) + .await + .is_err() + ); + assert!(handle.native_request().is_none()); + engine.cleanup().await.unwrap(); + fixture.shutdown().await; + } +} + +#[tokio::test] +async fn concurrent_discovery_failure_is_shared_and_later_recovers() { + let control = Controller::::default(); + let mut fixture = Fixture::start(control.clone(), FixtureConfig::default()).await; + fixture.discovery.unavailable.store(true, Ordering::SeqCst); + let engine = fixture.engine_with_context_length(32768); + engine.start(0).await.unwrap(); + fixture.discovery.hold.store(true, Ordering::SeqCst); + let requests = (0..4) + .map(|_| engine.generate(image_request(), GenerateContext::new(mock_context(), None))); + let (results, ()) = tokio::join!(join_all(requests), async { + wait_for_calls(&fixture, 2).await; + fixture.release_discovery(); + }); + for result in results { + let error = result.err().expect("discovery must fail for every waiter"); + assert_eq!( + error.error_type(), + ErrorType::Backend(BackendError::CannotConnect) + ); + } + assert_eq!(fixture.discovery.calls.load(Ordering::SeqCst), 2); + // Failures are retained for one second; recovery is not permanently latched. + fixture.discovery.unavailable.store(false, Ordering::SeqCst); + advertise_images(&fixture); + tokio::time::sleep(Duration::from_millis(1100)).await; + for _ in 0..2 { + let ctx = mock_context(); + control.request(ctx.id(), RequestPlan::default()); + fixture.respond(ctx.id(), vec![finished()]); + let outputs = collect(&engine, image_request(), GenerateContext::new(ctx, None)).await; + assert_eq!(outputs.len(), 1); + assert_eq!( + outputs[0].as_ref().unwrap().finish_reason, + Some(FinishReason::Stop) + ); + } + assert_eq!(fixture.discovery.calls.load(Ordering::SeqCst), 3); + engine.cleanup().await.unwrap(); + fixture.shutdown().await; +} + +#[tokio::test] +async fn stalled_discovery_observes_request_and_shutdown_cancellation() { + for shutdown in [false, true] { + let control = Controller::::default(); + let mut fixture = Fixture::start(control, FixtureConfig::default()).await; + fixture.discovery.unavailable.store(true, Ordering::SeqCst); + let engine = fixture.engine_with_context_length(32768); + engine.start(0).await.unwrap(); + fixture.discovery.hold.store(true, Ordering::SeqCst); + let ctx = mock_context(); + let cancelling_ctx = ctx.clone(); + let (outputs, ()) = bounded("cancel stalled discovery", async { + tokio::join!( + collect(&engine, image_request(), GenerateContext::new(ctx, None)), + async { + wait_for_calls(&fixture, 2).await; + if shutdown { + engine.cleanup().await.unwrap(); + } else { + cancelling_ctx.stop_generating(); + } + } + ) + }) + .await; + assert_eq!(outputs.len(), 1); + assert_eq!( + outputs[0].as_ref().unwrap().finish_reason, + Some(FinishReason::Cancelled) + ); + fixture.release_discovery(); + engine.cleanup().await.unwrap(); + fixture.shutdown().await; + } +} + +#[tokio::test] +async fn unresponsive_discovery_has_one_bounded_timeout_for_all_waiters() { + let control = Controller::::default(); + let mut fixture = Fixture::start(control, FixtureConfig::default()).await; + fixture.discovery.unavailable.store(true, Ordering::SeqCst); + let engine = fixture.engine_with_context_length(32768); + engine.start(0).await.unwrap(); + fixture.discovery.hold.store(true, Ordering::SeqCst); + let requests = (0..4) + .map(|_| engine.generate(image_request(), GenerateContext::new(mock_context(), None))); + let results = tokio::time::timeout(Duration::from_secs(35), join_all(requests)) + .await + .expect("all four requests must share the 30-second RPC timeout"); + for result in results { + let error = result + .err() + .expect("discovery must time out for every waiter"); + assert_eq!( + error.error_type(), + ErrorType::Backend(BackendError::ConnectionTimeout) + ); + } + assert_eq!(fixture.discovery.calls.load(Ordering::SeqCst), 2); + fixture.release_discovery(); + engine.cleanup().await.unwrap(); + fixture.shutdown().await; +} + +#[tokio::test] +async fn image_window_override_is_rejected_before_dispatch() { + let control = Controller::::default(); + let mut fixture = Fixture::start(control.clone(), FixtureConfig::default()).await; + advertise_images(&fixture); + let engine = fixture.engine_with_context_length(4096); + engine.start(0).await.unwrap(); + for max_tokens in [None, Some(1)] { + let ctx = mock_context(); + let handle = control.request(ctx.id(), RequestPlan::default()); + let mut req = image_request(); + req.stop_conditions.max_tokens = max_tokens; + let result = engine.generate(req, GenerateContext::new(ctx, None)).await; + let error = result.err().expect("window mismatch must fail"); + assert!(error.to_string().contains("--context-length to match")); + assert!(handle.native_request().is_none()); + } + engine.cleanup().await.unwrap(); + fixture.shutdown().await; +} diff --git a/lib/sidecar/trtllm/src/client.rs b/lib/sidecar/trtllm/src/client.rs index 5fa7f32da17a..39fabd20ff5e 100644 --- a/lib/sidecar/trtllm/src/client.rs +++ b/lib/sidecar/trtllm/src/client.rs @@ -34,6 +34,8 @@ pub(crate) struct ModelLimits { /// and the default `max_tokens`. Absent when neither source supplied one, /// which leaves requests that omit `max_tokens` to be rejected. pub(crate) context_length: Option, + /// Engine window before any operator override; media expansion uses this window. + pub(crate) engine_context_length: Option, /// Cap on generated tokens, when the engine reports one. A context window /// alone can imply a larger budget than the engine will accept. pub(crate) max_output_tokens: Option, @@ -129,6 +131,7 @@ impl TrtllmClient { }); Ok(ModelLimits { context_length: info.max_context_length.filter(|len| *len > 0), + engine_context_length: info.max_context_length.filter(|len| *len > 0), max_output_tokens: info.max_output_tokens.filter(|cap| *cap > 0), supports_image: Some(supports_image), }) diff --git a/lib/sidecar/trtllm/src/convert.rs b/lib/sidecar/trtllm/src/convert.rs index 803b8b320e47..64aa7b1178be 100644 --- a/lib/sidecar/trtllm/src/convert.rs +++ b/lib/sidecar/trtllm/src/convert.rs @@ -81,6 +81,7 @@ pub(crate) fn build_generate_request( .map(|context| context.session_id.trim()) .filter(|session_id| !session_id.is_empty()) .map(str::to_string); + let include_stop_in_output = has_visible_stop_tokens(&request).then_some(true); let token_ids = Arc::unwrap_or_clone(request.token_ids); Ok(pb::GenerateRequest { @@ -116,7 +117,7 @@ pub(crate) fn build_generate_request( // Keeps a matched visible stop token (e.g. Harmony's `<|call|>`) in // the output. A visible token equal to `end_id` finishes as END_ID // and may still be dropped. - include_stop_in_output: has_visible_stop_tokens(request).then_some(true), + include_stop_in_output, }), response: Some(pb::ResponseOptions { // A prefill worker streams no tokens to the client, but it must @@ -241,6 +242,15 @@ fn image_media( "the loaded TensorRT-LLM model does not advertise image support through Control.GetModelInfo", )); } + if let Some(limits) = limits + && (limits.engine_context_length.is_none() + || limits.context_length != limits.engine_context_length) + { + return Err(client::invalid_argument( + "image requests require --context-length to match the window reported by \ + TensorRT-LLM; the sidecar cannot enforce a different window after media expansion", + )); + } let uuids = user_image_uuids.unwrap_or_else(|| vec![None; images.len()]); images diff --git a/lib/sidecar/trtllm/src/convert/request_tests.rs b/lib/sidecar/trtllm/src/convert/request_tests.rs index cd59bfcc68ff..59a9fb844dbf 100644 --- a/lib/sidecar/trtllm/src/convert/request_tests.rs +++ b/lib/sidecar/trtllm/src/convert/request_tests.rs @@ -149,6 +149,26 @@ fn image_media_requires_the_loaded_models_advertised_capability() { } } +#[test] +fn images_reject_unknown_or_overridden_engine_windows() { + for engine_window in [None, Some(32768)] { + for max_tokens in [None, Some(1)] { + let mut req = request(); + add_images(&mut req, ["https://example.com/image.png"]); + req.stop_conditions.max_tokens = max_tokens; + let limits = Some(ModelLimits { + context_length: Some(4096), + engine_context_length: engine_window, + supports_image: Some(true), + ..Default::default() + }); + let error = build_generate_request(req, "req", "model", limits, AGG) + .expect_err("media expansion cannot honor a different window"); + assert!(error.to_string().contains("--context-length to match")); + } + } +} + #[test] fn unsupported_media_modalities_and_sources_are_rejected() { let mut req = request(); @@ -354,6 +374,7 @@ fn the_derived_max_tokens_respects_the_engines_output_cap() { context_length: Some(4096), max_output_tokens: Some(256), supports_image: Some(false), + ..Default::default() }), AGG, ) @@ -505,6 +526,7 @@ fn image_media_is_preserved_across_prefill_and_decode_requests() { )])); let image_limits = Some(ModelLimits { context_length: Some(4096), + engine_context_length: Some(4096), supports_image: Some(true), ..Default::default() }); diff --git a/lib/sidecar/trtllm/src/engine.rs b/lib/sidecar/trtllm/src/engine.rs index 545b071e6bd6..311bcb319b8f 100644 --- a/lib/sidecar/trtllm/src/engine.rs +++ b/lib/sidecar/trtllm/src/engine.rs @@ -4,6 +4,7 @@ //! Dynamo backend for TensorRT-LLM's OpenEngine (`openengine.v1`) gRPC server. use std::sync::Arc; +use std::time::Duration; use async_trait::async_trait; use dynamo_backend_common::{ @@ -14,7 +15,8 @@ use dynamo_sidecar_common::{ EngineBootstrapResult, GrpcEndpoint, GrpcTransportConfig, SidecarStartupError, startup_deadline, }; use futures::stream::BoxStream; -use tokio::sync::OnceCell; +use tokio::sync::{Mutex, OnceCell}; +use tokio::time::Instant; use tokio_util::sync::CancellationToken; use crate::args::Args; @@ -23,6 +25,12 @@ use crate::convert::{ResponseState, build_generate_request}; use crate::model::ConfiguredModel; const ALREADY_STARTED: &str = "TensorRT-LLM sidecar has already started"; +const DISCOVERY_RETRY_DELAY: Duration = Duration::from_secs(1); + +struct FailedDiscovery { + error: DynamoError, + retry_at: Instant, +} /// Terminal output emitted when a request is cancelled, carrying the usage /// accumulated so far. @@ -41,6 +49,7 @@ pub struct TrtllmSidecarEngine { connection: OnceCell<(TrtllmClient, ModelLimits)>, started: OnceCell<()>, rediscovered_limits: OnceCell, + discovery_failure: Mutex>, cancel: CancellationToken, } @@ -59,6 +68,7 @@ impl TrtllmSidecarEngine { connection: OnceCell::new(), started: OnceCell::new(), rediscovered_limits: OnceCell::new(), + discovery_failure: Mutex::new(None), cancel: CancellationToken::new(), } } @@ -185,7 +195,17 @@ impl TrtllmSidecarEngine { return Ok(initial); } - self.rediscovered_limits + // Serialize attempts and retain failures through the retry delay, so waiters + // share a failed RPC instead of each spending another RPC timeout. + let arrived_at = Instant::now(); + let mut failure = self.discovery_failure.lock().await; + if let Some(failure) = &*failure + && arrived_at < failure.retry_at + { + return Err(failure.error.clone()); + } + let result: Result = self + .rediscovered_limits .get_or_try_init(|| async { match client.model_limits(&self.model.source).await? { ModelInfoResponse::Available(mut reported) => { @@ -209,7 +229,12 @@ impl TrtllmSidecarEngine { } }) .await - .copied() + .copied(); + *failure = result.as_ref().err().map(|error| FailedDiscovery { + error: error.clone(), + retry_at: Instant::now() + DISCOVERY_RETRY_DELAY, + }); + result } async fn bootstrap( @@ -356,7 +381,6 @@ impl LLMEngine for TrtllmSidecarEngine { .filter(|_| self.started.initialized()) .ok_or_else(|| client::engine_shutdown("TensorRT-LLM sidecar is not started"))?; let request_id = ctx.id().to_string(); - let limits = self.limits_for_request(client, *limits, &request).await?; let mut state = ResponseState::new(&request, self.mode); // A decode request that took a handoff has KV transferred into it, and // the transceiver releases those blocks when the engine finishes the @@ -367,13 +391,6 @@ impl LLMEngine for TrtllmSidecarEngine { // consumer. A decode-mode request without a handoff ran locally and // has nothing to strand. let needs_cancellation_deferral = self.mode.is_decode() && request.prefill_result.is_some(); - let proto_request = build_generate_request( - request, - &request_id, - &self.model.source, - Some(limits), - self.mode, - )?; let stopped_ctx = ctx.inner_arc(); // Hoisted: `stopped()` is an async-trait method, so re-creating it per // streamed chunk costs a boxed future and a waker registration on every @@ -382,6 +399,24 @@ impl LLMEngine for TrtllmSidecarEngine { let shutdown = self.cancel.child_token(); let mut shutdown_cancellation = Box::pin(async move { shutdown.cancelled().await }); + let limits = tokio::select! { + biased; + _ = &mut request_cancellation, if !needs_cancellation_deferral => None, + _ = &mut shutdown_cancellation => None, + result = self.limits_for_request(client, *limits, &request) => Some(result?), + }; + let Some(limits) = limits else { + let output = cancelled(&state); + return Ok(Box::pin(futures::stream::once(async move { Ok(output) }))); + }; + let proto_request = build_generate_request( + request, + &request_id, + &self.model.source, + Some(limits), + self.mode, + )?; + // The same deferral applies here, not just to the streaming loop below. // `generate` sends the request and then awaits response headers, so // losing this race can drop a request the engine has already accepted diff --git a/lib/sidecar/trtllm/src/test_fixtures.rs b/lib/sidecar/trtllm/src/test_fixtures.rs index 30c142f64242..b95bcc314827 100644 --- a/lib/sidecar/trtllm/src/test_fixtures.rs +++ b/lib/sidecar/trtllm/src/test_fixtures.rs @@ -62,6 +62,7 @@ pub(crate) fn limits(context_length: u32) -> Option { pub(crate) fn image_limits(context_length: u32) -> Option { Some(ModelLimits { context_length: Some(context_length), + engine_context_length: Some(context_length), supports_image: Some(true), ..Default::default() }) diff --git a/lib/sidecar/trtllm/src/tests.rs b/lib/sidecar/trtllm/src/tests.rs index 85877e08fbc9..d7a0bb6f35fb 100644 --- a/lib/sidecar/trtllm/src/tests.rs +++ b/lib/sidecar/trtllm/src/tests.rs @@ -49,10 +49,6 @@ struct FakeTrtllm { empty_model_info: Arc, /// Answers `UNAVAILABLE`, the shape of an engine that is not serving yet. unavailable_model_info: Arc, - /// Portable coarse-grained model capability. - supports_multimodal: Arc, - /// TensorRT-LLM's finer-grained modality advertisement. - advertises_image: Arc, model_info_calls: Arc, } @@ -86,10 +82,7 @@ impl pb::inference_server::Inference for FakeTrtllm { self.requests.lock().await.push(request.clone()); let request_id = request.request_id.clone(); - // Image markers expand inside TensorRT-LLM after Dynamo tokenizes the - // text prompt. Simulate that so the integration test proves Dynamo - // trusts the engine's terminal usage rather than the unexpanded input. - let prompt_tokens = prompt_len(&request) + if request.media.is_empty() { 0 } else { 64 }; + let prompt_tokens = prompt_len(&request); let wants_logprobs = wants_logprobs(&request); let context_only = is_context_only(&request); let hang = self.hang.load(Ordering::SeqCst); @@ -197,14 +190,6 @@ impl pb::control_server::Control for FakeTrtllm { if self.unavailable_model_info.load(Ordering::SeqCst) { return Err(Status::unavailable("not serving yet")); } - let supports_multimodal = self.supports_multimodal.load(Ordering::SeqCst); - let modalities = if self.advertises_image.load(Ordering::SeqCst) { - vec![prost_types::Value { - kind: Some(prost_types::value::Kind::StringValue("image".to_string())), - }] - } else { - Vec::new() - }; Ok(Response::new(pb::ModelInfo { model_id: "fake-model".to_string(), max_context_length: if self.empty_model_info.load(Ordering::SeqCst) { @@ -212,19 +197,7 @@ impl pb::control_server::Control for FakeTrtllm { } else { Some(4096) }, - supports_multimodal: Some(supports_multimodal), - extra: Some(prost_types::Struct { - fields: [( - "trtllm_supported_modalities".to_string(), - prost_types::Value { - kind: Some(prost_types::value::Kind::ListValue( - prost_types::ListValue { values: modalities }, - )), - }, - )] - .into_iter() - .collect(), - }), + supports_multimodal: Some(false), ..Default::default() })) } @@ -549,60 +522,6 @@ async fn aggregated_generation_streams_delta_then_terminal() { } } -#[tokio::test] -async fn advertised_image_media_reaches_openengine_and_uses_expanded_usage() { - let service = FakeTrtllm::default(); - service.supports_multimodal.store(true, Ordering::SeqCst); - service.advertises_image.store(true, Ordering::SeqCst); - let server = FakeServer::start(service).await; - let engine = engine(&server.endpoint, 1); - engine.start(0).await.expect("start"); - - let mut req = request(); - req.stop_conditions.max_tokens = None; - add_images(&mut req, ["data:image/png;base64,aW1hZ2U="]); - let outputs = collect(&engine, req).await; - let usage = outputs - .last() - .and_then(|output| output.completion_usage.as_ref()) - .expect("terminal engine usage"); - assert_eq!(usage.prompt_tokens, 67, "3 text tokens + 64 image tokens"); - - let requests = server.service.requests.lock().await; - let sent = requests.first().expect("recorded request"); - assert_eq!(sent.stopping.as_ref().expect("stopping").max_tokens, None); - assert_eq!(sent.media.len(), 1); - assert_eq!(sent.media[0].modality, pb::Modality::Image as i32); - assert!(matches!( - sent.media[0].source.as_ref(), - Some(pb::media_item::Source::DataUri(value)) if value.starts_with("data:image/png") - )); -} - -#[tokio::test] -async fn both_multimodal_capability_signals_are_required() { - for (coarse, image) in [(true, false), (false, true)] { - let service = FakeTrtllm::default(); - service.supports_multimodal.store(coarse, Ordering::SeqCst); - service.advertises_image.store(image, Ordering::SeqCst); - let server = FakeServer::start(service).await; - let engine = engine(&server.endpoint, 1); - engine.start(0).await.expect("start"); - - let mut req = request(); - add_images(&mut req, ["https://example.com/image.png"]); - let context = dynamo_backend_common::testing::mock_context(); - let result = engine - .generate(req, GenerateContext::new(context, None)) - .await; - assert!( - result.is_err(), - "capabilities ({coarse}, {image}) must fail closed" - ); - assert!(server.service.requests.lock().await.is_empty()); - } -} - #[tokio::test] async fn cancellation_yields_a_cancelled_terminal() { let service = FakeTrtllm::default(); @@ -978,37 +897,6 @@ async fn a_configured_context_length_survives_a_server_without_control() { assert_eq!(server.service.model_info_calls.load(Ordering::SeqCst), 1); } -#[tokio::test] -async fn image_request_retries_transient_capability_discovery() { - let service = FakeTrtllm::default(); - service.unavailable_model_info.store(true, Ordering::SeqCst); - let server = FakeServer::start(service).await; - let engine = engine_with(&server.endpoint, transport(1), Some(8192), AGG); - engine.start(0).await.expect("start with configured window"); - - server - .service - .unavailable_model_info - .store(false, Ordering::SeqCst); - server - .service - .supports_multimodal - .store(true, Ordering::SeqCst); - server - .service - .advertises_image - .store(true, Ordering::SeqCst); - let mut image_request = request(); - add_images(&mut image_request, ["https://example.com/image.png"]); - - let output = collect(&engine, image_request).await; - assert_eq!( - output.last().unwrap().finish_reason, - Some(FinishReason::Stop) - ); - assert_eq!(server.service.model_info_calls.load(Ordering::SeqCst), 2); -} - /// Shutdown has to terminate in-flight requests: the worker cannot drain if a /// stream waits forever on an engine that will never answer. #[tokio::test] diff --git a/tests/serve/test_sidecar.py b/tests/serve/test_sidecar.py index b802ee686e9a..ee2c21aaded8 100644 --- a/tests/serve/test_sidecar.py +++ b/tests/serve/test_sidecar.py @@ -6,6 +6,7 @@ import importlib.util import os import pathlib +import runpy from dataclasses import dataclass, field import pytest @@ -585,6 +586,13 @@ def test_serve_deployment( predownload_models, ): """Launch a sidecar deployment and run its configured request payloads.""" + if sidecar_config_test.name == "trtllm_aggregated_image": + runpy.run_path( + os.path.join( + WORKSPACE_DIR, "container/deps/trtllm/validate_media_overlay.py" + ), + run_name="__main__", + ) assert ( num_system_ports >= 2 ), "serve tests require at least SYSTEM_PORT1 + SYSTEM_PORT2" From 9a810c92041d7e795e82fe8cbce4ba4f7314640d Mon Sep 17 00:00:00 2001 From: krishung5 Date: Wed, 21 Oct 2026 14:18:58 -0700 Subject: [PATCH 34/61] fix(ci): package TRT-LLM media smoke helper with tests Signed-off-by: krishung5 --- .github/filters.yaml | 2 ++ .github/scripts/test-filters.js | 5 +++++ container/templates/trtllm_runtime.Dockerfile | 2 +- tests/serve/test_sidecar.py | 4 +--- .../utils/trtllm_media_overlay.py | 0 5 files changed, 9 insertions(+), 4 deletions(-) rename container/deps/trtllm/validate_media_overlay.py => tests/utils/trtllm_media_overlay.py (100%) diff --git a/.github/filters.yaml b/.github/filters.yaml index f947a34e8216..57fcde11a43b 100644 --- a/.github/filters.yaml +++ b/.github/filters.yaml @@ -372,6 +372,7 @@ sidecar: - 'lib/sidecar/**' - '.github/workflows/shared-build-sidecar.yml' - 'tests/serve/test_sidecar.py' + - 'tests/utils/trtllm_media_overlay.py' - 'tests/router/common.py' - 'tests/router/helper.py' - 'tests/router/router_process.py' @@ -413,6 +414,7 @@ trtllm: - 'components/src/dynamo/trtllm/**' - 'lib/sidecar/trtllm/**' - 'tests/serve/test_sidecar.py' + - 'tests/utils/trtllm_media_overlay.py' - 'container/templates/trtllm_*' - '!**/*.md' - '!**/*.rst' diff --git a/.github/scripts/test-filters.js b/.github/scripts/test-filters.js index eea6e9e0ed41..c34ab96b0eba 100755 --- a/.github/scripts/test-filters.js +++ b/.github/scripts/test-filters.js @@ -205,6 +205,11 @@ const testCases = [ expect: { sidecar: true, trtllm: true }, desc: 'shared sidecar test changes trigger the TensorRT-LLM sidecar E2E' }, + { + file: 'tests/utils/trtllm_media_overlay.py', + expect: { sidecar: true, trtllm: true }, + desc: 'media helper changes trigger the TensorRT-LLM sidecar E2E' + }, { file: 'lib/sidecar/vllm/deploy/agg.yaml', expect: { sidecar: true, rust: false, core: false, frontend: false, vllm: false, sglang: false, trtllm: false }, diff --git a/container/templates/trtllm_runtime.Dockerfile b/container/templates/trtllm_runtime.Dockerfile index 65c4056c6393..2ba670a480a7 100644 --- a/container/templates/trtllm_runtime.Dockerfile +++ b/container/templates/trtllm_runtime.Dockerfile @@ -43,7 +43,7 @@ ARG NIXL_REF ARG TRTLLM_SOURCE_REPOSITORY ARG TRTLLM_SOURCE_COMMIT -COPY container/deps/trtllm/validate_media_overlay.py /tmp/validate_media_overlay.py +COPY tests/utils/trtllm_media_overlay.py /tmp/validate_media_overlay.py # Optional source overlay for cross-repository CI. Keep the base image's server # lifecycle entry points, which are coupled to that release's frontend runtime. diff --git a/tests/serve/test_sidecar.py b/tests/serve/test_sidecar.py index ee2c21aaded8..6f51c8f4b0a8 100644 --- a/tests/serve/test_sidecar.py +++ b/tests/serve/test_sidecar.py @@ -588,9 +588,7 @@ def test_serve_deployment( """Launch a sidecar deployment and run its configured request payloads.""" if sidecar_config_test.name == "trtllm_aggregated_image": runpy.run_path( - os.path.join( - WORKSPACE_DIR, "container/deps/trtllm/validate_media_overlay.py" - ), + os.path.join(WORKSPACE_DIR, "tests/utils/trtllm_media_overlay.py"), run_name="__main__", ) assert ( diff --git a/container/deps/trtllm/validate_media_overlay.py b/tests/utils/trtllm_media_overlay.py similarity index 100% rename from container/deps/trtllm/validate_media_overlay.py rename to tests/utils/trtllm_media_overlay.py From cc234ed1ad691f54adc2462dbbbd97d4372b6a08 Mon Sep 17 00:00:00 2001 From: krishung5 Date: Wed, 21 Oct 2026 15:52:23 -0700 Subject: [PATCH 35/61] fix(trtllm): clarify video ordering and decoder requirements Signed-off-by: krishung5 --- .github/workflows/pr.yaml | 2 +- lib/llm/src/protocols/common/preprocessor.rs | 4 ++- lib/sidecar/trtllm/src/convert.rs | 9 ++++-- .../trtllm/src/convert/request_tests.rs | 29 ++++++++++--------- 4 files changed, 27 insertions(+), 17 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index d25ea166e74b..f591327afe8e 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -547,7 +547,7 @@ jobs: # the upstream change. extra_build_args: | TRTLLM_SOURCE_REPOSITORY=https://github.com/krishung5/TensorRT-LLM - TRTLLM_SOURCE_COMMIT=b591693204e2443324f3f61e5d9e2bc1207c5fa1 + TRTLLM_SOURCE_COMMIT=88f22f88e0a08a6304459d96001b45e0a19fa056 # actions: read lets the build download a prior run's baseline compliance # artifact to diff against (a reusable workflow's token can't exceed the # caller's). Missing -> the diff soft-degrades, build stays green. diff --git a/lib/llm/src/protocols/common/preprocessor.rs b/lib/llm/src/protocols/common/preprocessor.rs index 45c8bd2f2d90..4a558b69d7f5 100644 --- a/lib/llm/src/protocols/common/preprocessor.rs +++ b/lib/llm/src/protocols/common/preprocessor.rs @@ -428,7 +428,9 @@ pub struct PreprocessedRequest { #[serde(default, skip_serializing_if = "Option::is_none")] pub multi_modal_uuids: Option, - /// Media references in the order used to render the prompt. + /// Media references in gather order: content parts, after the formatter's + /// message ordering. A chat template that regroups media by modality + /// renders a different placeholder order. #[builder(default)] #[serde(default, skip_serializing_if = "Option::is_none")] pub multi_modal_order: Option>, diff --git a/lib/sidecar/trtllm/src/convert.rs b/lib/sidecar/trtllm/src/convert.rs index c2a5a944442a..a4c876ba3170 100644 --- a/lib/sidecar/trtllm/src/convert.rs +++ b/lib/sidecar/trtllm/src/convert.rs @@ -264,9 +264,14 @@ fn media( .and_then(|limits| limits.supported_media) .is_some_and(|media| media.contains(kind.name)) { + let requirement = if kind.name == "video" { + ". Video decoding requires OpenCV (cv2) in the TensorRT-LLM server environment" + } else { + "" + }; return Err(client::invalid_argument(format!( - "the loaded TensorRT-LLM model does not advertise {} support through Control.GetModelInfo", - kind.name + "the loaded TensorRT-LLM model does not advertise {} support through Control.GetModelInfo{requirement}", + kind.name, ))); } active_kinds.push(kind); diff --git a/lib/sidecar/trtllm/src/convert/request_tests.rs b/lib/sidecar/trtllm/src/convert/request_tests.rs index e4ef05e18da1..56844f5b917d 100644 --- a/lib/sidecar/trtllm/src/convert/request_tests.rs +++ b/lib/sidecar/trtllm/src/convert/request_tests.rs @@ -249,19 +249,22 @@ fn mm_routing_hashes_are_not_used_as_engine_cache_uuids() { } #[test] -fn image_media_requires_the_loaded_models_advertised_capability() { - let mut req = request(); - add_images(&mut req, ["https://example.com/image.png"]); - - for model_limits in [None, limits(4096)] { - let error = build_generate_request(req.clone(), "req", "model", model_limits, AGG) - .expect_err("a text-only or unknown model must reject images"); - assert!( - error - .to_string() - .contains("does not advertise image support"), - "unexpected error: {error}" - ); +fn media_requires_the_loaded_models_advertised_capability() { + for (key, modality) in [("image_url", "image"), ("video_url", "video")] { + let mut req = request(); + req.multi_modal_data = Some(HashMap::from([( + key.to_string(), + vec![MultimodalData::RawUrl( + "https://example.com/media".to_string(), + )], + )])); + for model_limits in [None, limits(4096)] { + let error = build_generate_request(req.clone(), "req", "model", model_limits, AGG) + .expect_err("a text-only or unknown model must reject media") + .to_string(); + assert!(error.contains(&format!("does not advertise {modality} support"))); + assert_eq!(error.contains("OpenCV (cv2)"), modality == "video"); + } } } From 106f4f371322e582db1b5348ebf45e24e1ff5ad4 Mon Sep 17 00:00:00 2001 From: krishung5 Date: Wed, 21 Oct 2026 19:38:13 -0700 Subject: [PATCH 36/61] ci(trtllm): pin reviewed video memory-budget fix Signed-off-by: krishung5 --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index f591327afe8e..f5da30b54df1 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -547,7 +547,7 @@ jobs: # the upstream change. extra_build_args: | TRTLLM_SOURCE_REPOSITORY=https://github.com/krishung5/TensorRT-LLM - TRTLLM_SOURCE_COMMIT=88f22f88e0a08a6304459d96001b45e0a19fa056 + TRTLLM_SOURCE_COMMIT=2d3a2a2d57395529e4e7e485598314705e2c12c4 # actions: read lets the build download a prior run's baseline compliance # artifact to diff against (a reusable workflow's token can't exceed the # caller's). Missing -> the diff soft-degrades, build stays green. From 5b18885c00dd15cb273b34e9bcc372f8c8257eed Mon Sep 17 00:00:00 2001 From: krishung5 Date: Wed, 21 Oct 2026 19:38:23 -0700 Subject: [PATCH 37/61] ci(trtllm): pin reviewed audio decoding and capability fixes Signed-off-by: krishung5 --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index b48773c013c0..ddd2f3e44d81 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -547,7 +547,7 @@ jobs: # the upstream change. extra_build_args: | TRTLLM_SOURCE_REPOSITORY=https://github.com/krishung5/TensorRT-LLM - TRTLLM_SOURCE_COMMIT=4e4c9d14f73646ac9184dd510326b79a4bafe833 + TRTLLM_SOURCE_COMMIT=8646ea88f85148c1a1bc6116c147b7941ee43af8 # actions: read lets the build download a prior run's baseline compliance # artifact to diff against (a reusable workflow's token can't exceed the # caller's). Missing -> the diff soft-degrades, build stays green. From 50d2d243faf92d20de1fd20acc1bdb13e517f9a5 Mon Sep 17 00:00:00 2001 From: krishung5 Date: Thu, 22 Oct 2026 14:13:05 -0700 Subject: [PATCH 38/61] perf(trtllm): move owned media URLs into OpenEngine requests Signed-off-by: krishung5 --- lib/sidecar/trtllm/src/convert.rs | 2 +- .../trtllm/src/convert/request_tests.rs | 29 +++++++++++++++++++ lib/sidecar/trtllm/src/tests/e2e.rs | 1 + 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/lib/sidecar/trtllm/src/convert.rs b/lib/sidecar/trtllm/src/convert.rs index 582a0d6f1f9f..897863a93ad1 100644 --- a/lib/sidecar/trtllm/src/convert.rs +++ b/lib/sidecar/trtllm/src/convert.rs @@ -394,7 +394,7 @@ fn media( )) })?; let source = match item { - MultimodalData::Url(url) => media_source(url.to_string(), kind.key, index)?, + MultimodalData::Url(url) => media_source(String::from(url), kind.key, index)?, MultimodalData::RawUrl(url) => media_source(url, kind.key, index)?, MultimodalData::Decoded(_) => { return Err(client::invalid_argument(format!( diff --git a/lib/sidecar/trtllm/src/convert/request_tests.rs b/lib/sidecar/trtllm/src/convert/request_tests.rs index 16a35d528f05..3e0e0b036c73 100644 --- a/lib/sidecar/trtllm/src/convert/request_tests.rs +++ b/lib/sidecar/trtllm/src/convert/request_tests.rs @@ -115,6 +115,35 @@ fn image_urls_data_uris_and_uuids_are_mapped_in_order() { assert!(proto.media[1].uuid.is_empty()); } +#[test] +fn inline_media_conversion_preserves_the_payload_allocation() { + for (key, mime) in [("image_url", "image/png"), ("video_url", "video/mp4")] { + for parsed in [false, true] { + let uri = format!("data:{mime};base64,{}", "AAAA".repeat(16384)); + let item = if parsed { + MultimodalData::Url(uri.parse().unwrap()) + } else { + MultimodalData::RawUrl(uri) + }; + let original = match &item { + MultimodalData::Url(url) => url.as_str().as_ptr(), + MultimodalData::RawUrl(url) => url.as_ptr(), + _ => unreachable!(), + }; + let mut req = request(); + req.multi_modal_data = Some(HashMap::from([(key.to_string(), vec![item])])); + + let proto = build_generate_request(req, "req", "model", image_video_limits(4096), AGG) + .expect("inline media request"); + + let Some(pb::media_item::Source::DataUri(value)) = &proto.media[0].source else { + panic!("expected inline media"); + }; + assert_eq!(value.as_ptr(), original, "{key}, parsed={parsed}"); + } + } +} + #[test] fn mixed_image_and_video_media_follow_prompt_order() { let mut req = request(); diff --git a/lib/sidecar/trtllm/src/tests/e2e.rs b/lib/sidecar/trtllm/src/tests/e2e.rs index 90946269950b..35282adbd67a 100644 --- a/lib/sidecar/trtllm/src/tests/e2e.rs +++ b/lib/sidecar/trtllm/src/tests/e2e.rs @@ -425,6 +425,7 @@ async fn e2e_real_openengine_video() { ConfiguredModel { source: model.clone(), context_length: None, + ..Default::default() }, AGG, ); From 98f6af130fc7c0640ace184037cd7dcfd3f68850 Mon Sep 17 00:00:00 2001 From: krishung5 Date: Thu, 22 Oct 2026 14:13:05 -0700 Subject: [PATCH 39/61] ci(trtllm): carry video preparation support into the release overlay Signed-off-by: krishung5 --- .github/workflows/pr.yaml | 2 +- container/templates/trtllm_runtime.Dockerfile | 6 +- tests/utils/trtllm_media_overlay.py | 104 ++++++++++++++++-- 3 files changed, 102 insertions(+), 10 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index f5da30b54df1..ab259dfb3266 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -547,7 +547,7 @@ jobs: # the upstream change. extra_build_args: | TRTLLM_SOURCE_REPOSITORY=https://github.com/krishung5/TensorRT-LLM - TRTLLM_SOURCE_COMMIT=2d3a2a2d57395529e4e7e485598314705e2c12c4 + TRTLLM_SOURCE_COMMIT=2e1b47c206189bcac3817a73a4f30f1bd4a67a0c # actions: read lets the build download a prior run's baseline compliance # artifact to diff against (a reusable workflow's token can't exceed the # caller's). Missing -> the diff soft-degrades, build stays green. diff --git a/container/templates/trtllm_runtime.Dockerfile b/container/templates/trtllm_runtime.Dockerfile index 2ba670a480a7..637653fbeae5 100644 --- a/container/templates/trtllm_runtime.Dockerfile +++ b/container/templates/trtllm_runtime.Dockerfile @@ -48,7 +48,8 @@ COPY tests/utils/trtllm_media_overlay.py /tmp/validate_media_overlay.py # Optional source overlay for cross-repository CI. Keep the base image's server # lifecycle entry points, which are coupled to that release's frontend runtime. # capabilities imports model_config; install that helper with the media loader. -# Keep the base image's other input modules to avoid a partial framework upgrade. +# Install the matching preparation dependencies and server class, while retaining +# the release launcher and unrelated input helpers. RUN if [ -n "${TRTLLM_SOURCE_COMMIT}" ]; then \ test -n "${TRTLLM_SOURCE_REPOSITORY}" && \ source_dir=$(mktemp -d) && \ @@ -81,7 +82,8 @@ RUN if [ -n "${TRTLLM_SOURCE_COMMIT}" ]; then \ test -f "${package_dir}/inputs/model_config.py" && \ grep -Fq 'from tensorrt_llm.inputs.model_config import resolve_top_level_model_type' \ "${package_dir}/grpc/openengine/capabilities.py" && \ - /usr/bin/python3 /tmp/validate_media_overlay.py --check-signatures-only && \ + /usr/bin/python3 /tmp/validate_media_overlay.py \ + --install-preparation-support "${source_dir}" --check-signatures-only && \ rm -rf "${source_dir}"; \ fi diff --git a/tests/utils/trtllm_media_overlay.py b/tests/utils/trtllm_media_overlay.py index b0a68b602dce..b5c84d5443d8 100644 --- a/tests/utils/trtllm_media_overlay.py +++ b/tests/utils/trtllm_media_overlay.py @@ -3,16 +3,85 @@ """Exercise retained runtime helpers against the overlaid fetch signature.""" +import argparse import ast import asyncio import importlib.util import inspect -import sys +import shutil from io import BytesIO from pathlib import Path +from types import SimpleNamespace from unittest.mock import patch +def install_preparation_support(source_root): + package = Path( + next(iter(importlib.util.find_spec("tensorrt_llm").submodule_search_locations)) + ) + source = Path(source_root) / "tensorrt_llm" + for name in ( + "registry.py", + "multimodal.py", + "multimodal_data.py", + "content_format.py", + ): + shutil.copy2(source / "inputs" / name, package / "inputs" / name) + shutil.copytree( + source / "_torch/shared_tensor", + package / "_torch/shared_tensor", + dirs_exist_ok=True, + ignore=shutil.ignore_patterns("__pycache__"), + ) + + # Keep the release launcher; only the serving class owns preparation pools. + target = package / "grpc/openengine/server.py" + current = target.read_text() + replacement = (source / "grpc/openengine/server.py").read_text() + old_class = next( + n + for n in ast.parse(current).body + if isinstance(n, ast.ClassDef) and n.name == "OpenEngineServer" + ) + new_class = next( + n + for n in ast.parse(replacement).body + if isinstance(n, ast.ClassDef) and n.name == "OpenEngineServer" + ) + lines = current.splitlines(keepends=True) + lines[old_class.lineno - 1 : old_class.end_lineno] = [ + "from concurrent.futures import ThreadPoolExecutor\n", + "from .coordinator import FrontendClient\n", + "from .kv_events import KvEventsUnavailableError, events_config\n\n", + ast.get_source_segment(replacement, new_class) + "\n", + ] + target.write_text("".join(lines)) + + target = package / "inputs/utils.py" + current = target.read_text() + replacement = (source / "inputs/utils.py").read_text() + names = {"load_base64_video", "load_video", "async_load_video"} + definitions = { + n.name: n + for n in ast.parse(replacement).body + if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) and n.name in names + } + lines = current.splitlines(keepends=True) + for node in reversed(ast.parse(current).body): + if ( + isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + and node.name in names + ): + lines[node.lineno - 1 : node.end_lineno] = [ + ast.get_source_segment(replacement, definitions.pop(node.name)) + "\n" + ] + assert not definitions, f"release image lacks video helpers: {sorted(definitions)}" + lines.append( + "\nfrom tensorrt_llm.inputs.media_io import VideoMediaIO, _decode_base64_with_limit\n" + ) + target.write_text("".join(lines)) + + def check_signatures(): package = Path( next(iter(importlib.util.find_spec("tensorrt_llm").submodule_search_locations)) @@ -35,9 +104,9 @@ def check_signatures(): and isinstance(node.func, ast.Name) and node.func.id == "_safe_aiohttp_get" ): - assert all( - keyword.arg in accepted for keyword in node.keywords - ), function.name + assert all(keyword.arg in accepted for keyword in node.keywords), ( + function.name + ) checked.add(function.name) assert {"async_load_image", "async_load_video", "async_load_audio"} <= checked print("TRT-LLM overlay: retained media helper signatures passed") @@ -47,7 +116,20 @@ async def validate(): import numpy as np import soundfile from PIL import Image + from tensorrt_llm.grpc.openengine.server import OpenEngineServer from tensorrt_llm.inputs import media_io, utils + from tensorrt_llm.inputs.multimodal import MultimodalParams + from tensorrt_llm.inputs.registry import BaseMultimodalInputProcessor + + assert callable(BaseMultimodalInputProcessor.requires_input_placeholders) + assert callable(MultimodalParams.release_handles) + llm = SimpleNamespace(args=SimpleNamespace(guided_decoding_backend=None)) + server = OpenEngineServer("127.0.0.1", 0, llm, "overlay-smoke") + try: + assert server._input_executor._max_workers == 4 + assert server._media_executor._max_workers == 8 + finally: + await server.stop(grace=0) image_bytes = BytesIO() Image.new("RGB", (32, 32), "red").save(image_bytes, format="PNG") @@ -69,12 +151,14 @@ async def fetch(url, timeout_sec=30): def decode_video(path, *args, **kwargs): # OpenCV is optional in the base image; this checks the URL caller, not decoding. - assert Path(path).read_bytes() == payloads["video"] + assert (Path(path).read_bytes() if isinstance(path, str) else path) == payloads[ + "video" + ] return "video-decoded" with ( patch.object(utils, "_safe_aiohttp_get", fetch), - patch.object(utils, "_load_video_by_cv2", decode_video), + patch.object(media_io, "_load_video_by_cv2", decode_video), ): image = await utils.async_load_image("https://example.com/image", format="pil") assert image.size == (32, 32) @@ -89,6 +173,12 @@ def decode_video(path, *args, **kwargs): if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--install-preparation-support") + parser.add_argument("--check-signatures-only", action="store_true") + args = parser.parse_args() + if args.install_preparation_support: + install_preparation_support(args.install_preparation_support) check_signatures() - if "--check-signatures-only" not in sys.argv[1:]: + if not args.check_signatures_only: asyncio.run(validate()) From e28f0b0242fce8795486643eb774e268177e4dc9 Mon Sep 17 00:00:00 2001 From: krishung5 <43719498+krishung5@users.noreply.github.com> Date: Mon, 26 Oct 2026 09:27:37 -0700 Subject: [PATCH 40/61] ci(trtllm): refresh OpenEngine image PR overlay Signed-off-by: krishung5 <43719498+krishung5@users.noreply.github.com> --- .github/workflows/pr.yaml | 2 +- container/templates/trtllm_runtime.Dockerfile | 34 +---- tests/utils/test_trtllm_media_overlay.py | 46 +++++++ tests/utils/trtllm_media_overlay.py | 123 +++++++++++++++++- 4 files changed, 172 insertions(+), 33 deletions(-) create mode 100644 tests/utils/test_trtllm_media_overlay.py diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index e64fe4655932..26b4e1919a57 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -547,7 +547,7 @@ jobs: # the upstream change. extra_build_args: | TRTLLM_SOURCE_REPOSITORY=https://github.com/krishung5/TensorRT-LLM - TRTLLM_SOURCE_COMMIT=c0e8499a981b28defb27e0d3e5a44faff17ae509 + TRTLLM_SOURCE_COMMIT=750f17d503ec3224f4792c78ad465a18060b9f34 # actions: read lets the build download a prior run's baseline compliance # artifact to diff against (a reusable workflow's token can't exceed the # caller's). Missing -> the diff soft-degrades, build stays green. diff --git a/container/templates/trtllm_runtime.Dockerfile b/container/templates/trtllm_runtime.Dockerfile index 2ba670a480a7..466a966b3227 100644 --- a/container/templates/trtllm_runtime.Dockerfile +++ b/container/templates/trtllm_runtime.Dockerfile @@ -45,10 +45,9 @@ ARG TRTLLM_SOURCE_COMMIT COPY tests/utils/trtllm_media_overlay.py /tmp/validate_media_overlay.py -# Optional source overlay for cross-repository CI. Keep the base image's server -# lifecycle entry points, which are coupled to that release's frontend runtime. -# capabilities imports model_config; install that helper with the media loader. -# Keep the base image's other input modules to avoid a partial framework upgrade. +# Optional source overlay for cross-repository CI. Retain the release-coupled +# server launcher and framework, installing the pinned media hooks and ownership +# helpers alongside OpenEngine's worker pools. RUN if [ -n "${TRTLLM_SOURCE_COMMIT}" ]; then \ test -n "${TRTLLM_SOURCE_REPOSITORY}" && \ source_dir=$(mktemp -d) && \ @@ -57,31 +56,8 @@ RUN if [ -n "${TRTLLM_SOURCE_COMMIT}" ]; then \ -O "${source_dir}/source.tar.gz" && \ tar -xzf "${source_dir}/source.tar.gz" -C "${source_dir}" \ --strip-components=1 && \ - package_dir=$(/usr/bin/python3 -c \ - 'import importlib.util; print(next(iter(importlib.util.find_spec("tensorrt_llm").submodule_search_locations)))') && \ - for entry in "${source_dir}/tensorrt_llm/grpc/openengine/"*; do \ - case "$(basename "${entry}")" in \ - __init__.py|server.py) continue ;; \ - esac; \ - cp -a "${entry}" "${package_dir}/grpc/openengine/"; \ - done && \ - sed -i \ - 's/from openengine\.v1 import openengine_pb2_grpc/from .bindings import openengine_pb2_grpc/' \ - "${package_dir}/grpc/openengine/server.py" && \ - grep -Fxq 'from .bindings import openengine_pb2_grpc' \ - "${package_dir}/grpc/openengine/server.py" && \ - for module in media_io.py model_config.py; do \ - install -m 0644 "${source_dir}/tensorrt_llm/inputs/${module}" \ - "${package_dir}/inputs/${module}"; \ - done && \ - sed -i \ - -e '/^[[:space:]]*session = await _get_aiohttp_session()$/d' \ - -e 's/, session=session)/)/g' \ - "${package_dir}/inputs/utils.py" && \ - test -f "${package_dir}/inputs/model_config.py" && \ - grep -Fq 'from tensorrt_llm.inputs.model_config import resolve_top_level_model_type' \ - "${package_dir}/grpc/openengine/capabilities.py" && \ - /usr/bin/python3 /tmp/validate_media_overlay.py --check-signatures-only && \ + /usr/bin/python3 /tmp/validate_media_overlay.py \ + --install-source "${source_dir}" --check-signatures-only && \ rm -rf "${source_dir}"; \ fi diff --git a/tests/utils/test_trtllm_media_overlay.py b/tests/utils/test_trtllm_media_overlay.py new file mode 100644 index 000000000000..d74cd7eb94dc --- /dev/null +++ b/tests/utils/test_trtllm_media_overlay.py @@ -0,0 +1,46 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Source overlay edits preserve unrelated release definitions.""" + +import pytest + +from tests.utils.trtllm_media_overlay import _replace_definition + +pytestmark = [pytest.mark.unit, pytest.mark.gpu_0] + + +@pytest.mark.parametrize("existing", [False, True]) +def test_replace_processor_method_preserves_release_class(tmp_path, existing): + destination = tmp_path / "installed.py" + source = tmp_path / "source.py" + destination.write_text( + "class Processor:\n release_value = 7\n" + + (" def feature(self):\n return 0\n" if existing else "") + + "\ndef launcher():\n return 'release'\n" + ) + source.write_text( + "class Processor:\n @classmethod\n" + " def feature(cls):\n return cls.release_value\n" + ) + _replace_definition(destination, source, "Processor", ["feature"]) + namespace = {} + exec(destination.read_text(), namespace) + assert namespace["Processor"].feature() == 7 + assert namespace["launcher"]() == "release" + + +def test_replace_server_class_preserves_release_launcher(tmp_path): + destination = tmp_path / "installed.py" + source = tmp_path / "source.py" + destination.write_text( + "class Server:\n workers = 0\n\ndef launcher():\n return 'release'\n" + ) + source.write_text( + "class Server:\n workers = 4\n\ndef launcher():\n return 'new-main'\n" + ) + _replace_definition(destination, source, None, ["Server"]) + namespace = {} + exec(destination.read_text(), namespace) + assert namespace["Server"].workers == 4 + assert namespace["launcher"]() == "release" diff --git a/tests/utils/trtllm_media_overlay.py b/tests/utils/trtllm_media_overlay.py index b0a68b602dce..9f14560620dd 100644 --- a/tests/utils/trtllm_media_overlay.py +++ b/tests/utils/trtllm_media_overlay.py @@ -1,18 +1,129 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Exercise retained runtime helpers against the overlaid fetch signature.""" +"""Install and validate the pinned OpenEngine media overlay for CI.""" +import argparse import ast import asyncio import importlib.util import inspect -import sys +import shutil from io import BytesIO from pathlib import Path from unittest.mock import patch +def _replace_definition(destination, source, class_name, names): + """Overlay selected Python definitions without upgrading the framework.""" + original = destination.read_text() + updated = source.read_text() + original_tree, updated_tree = ast.parse(original), ast.parse(updated) + + def definitions(tree): + if class_name is not None: + tree = next( + node + for node in tree.body + if isinstance(node, ast.ClassDef) and node.name == class_name + ) + return {node.name: node for node in tree.body if hasattr(node, "name")} + + existing, replacements = definitions(original_tree), definitions(updated_tree) + lines, source_lines = original.splitlines(keepends=True), updated.splitlines( + keepends=True + ) + edits = [] + for name in names: + node = replacements[name] + start = min([node.lineno] + [item.lineno for item in node.decorator_list]) - 1 + replacement = source_lines[start : node.end_lineno] + if name in existing: + old = existing[name] + begin = min([old.lineno] + [item.lineno for item in old.decorator_list]) - 1 + edits.append((begin, old.end_lineno, replacement)) + else: + owner = next( + node + for node in original_tree.body + if isinstance(node, ast.ClassDef) and node.name == class_name + ) + edits.append((owner.end_lineno, owner.end_lineno, ["\n"] + replacement)) + for start, end, replacement in sorted(edits, reverse=True): + lines[start:end] = replacement + result = "".join(lines) + ast.parse(result) + destination.write_text(result) + + +def install_overlay(source): + """Install the pinned media implementation into a release runtime.""" + package = Path( + next(iter(importlib.util.find_spec("tensorrt_llm").submodule_search_locations)) + ) + source = Path(source) / "tensorrt_llm" + for entry in (source / "grpc/openengine").iterdir(): + if entry.name not in {"__init__.py", "server.py", "__pycache__"}: + target = package / "grpc/openengine" / entry.name + if entry.is_dir(): + shutil.copytree(entry, target, dirs_exist_ok=True) + else: + shutil.copy2(entry, target) + for module in ("media_io.py", "model_config.py", "multimodal_data.py"): + shutil.copy2(source / "inputs" / module, package / "inputs" / module) + shutil.copy2( + source / "_torch/shared_tensor/shared_tensor.py", + package / "_torch/shared_tensor/shared_tensor.py", + ) + # Retain release-coupled launch_server; use the pinned class's worker pools + # and cleanup rather than silently running preprocessing on the default pool. + server = package / "grpc/openengine/server.py" + _replace_definition( + server, source / "grpc/openengine/server.py", None, ["OpenEngineServer"] + ) + server.write_text( + server.read_text() + .replace( + "from typing import Any", + "from typing import Any\nfrom concurrent.futures import ThreadPoolExecutor\n" + "from .coordinator import FrontendClient", + ) + .replace( + "from openengine.v1 import openengine_pb2_grpc", + "from .bindings import openengine_pb2_grpc", + ) + ) + registry = package / "inputs/registry.py" + _replace_definition( + registry, + source / "inputs/registry.py", + "BaseMultimodalInputProcessor", + ["requires_input_placeholders"], + ) + registry.write_text( + registry.read_text().replace( + "from .content_format import ContentFormat", + "from .content_format import ContentFormat\n" + "from .model_config import resolve_top_level_model_type", + ) + ) + _replace_definition( + package / "inputs/multimodal.py", + source / "inputs/multimodal.py", + "MultimodalParams", + ["_apply_tensor_operation", "release_handles"], + ) + utils = package / "inputs/utils.py" + utils.write_text( + "".join( + line + for line in utils.read_text().splitlines(keepends=True) + if line.strip() != "session = await _get_aiohttp_session()" + ).replace(", session=session)", ")") + ) + check_signatures() + + def check_signatures(): package = Path( next(iter(importlib.util.find_spec("tensorrt_llm").submodule_search_locations)) @@ -89,6 +200,12 @@ def decode_video(path, *args, **kwargs): if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--install-source", type=Path) + parser.add_argument("--check-signatures-only", action="store_true") + args, _ = parser.parse_known_args() + if args.install_source: + install_overlay(args.install_source) check_signatures() - if "--check-signatures-only" not in sys.argv[1:]: + if not args.check_signatures_only: asyncio.run(validate()) From b2793bd33aa65e34da87e2f6e712d6dbcf77e759 Mon Sep 17 00:00:00 2001 From: krishung5 <43719498+krishung5@users.noreply.github.com> Date: Mon, 26 Oct 2026 09:54:12 -0700 Subject: [PATCH 41/61] test(trtllm): mark media overlay regressions for pre-merge Signed-off-by: krishung5 <43719498+krishung5@users.noreply.github.com> --- tests/utils/test_trtllm_media_overlay.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/utils/test_trtllm_media_overlay.py b/tests/utils/test_trtllm_media_overlay.py index d74cd7eb94dc..617ff3b9e0cd 100644 --- a/tests/utils/test_trtllm_media_overlay.py +++ b/tests/utils/test_trtllm_media_overlay.py @@ -7,7 +7,7 @@ from tests.utils.trtllm_media_overlay import _replace_definition -pytestmark = [pytest.mark.unit, pytest.mark.gpu_0] +pytestmark = [pytest.mark.unit, pytest.mark.pre_merge, pytest.mark.gpu_0] @pytest.mark.parametrize("existing", [False, True]) From 2915cfd198923c74ff49eb908ff9d647c2a697c5 Mon Sep 17 00:00:00 2001 From: krishung5 <43719498+krishung5@users.noreply.github.com> Date: Mon, 26 Oct 2026 11:17:12 -0700 Subject: [PATCH 42/61] fix(ci): validate video helpers through the shared media loader Signed-off-by: krishung5 <43719498+krishung5@users.noreply.github.com> --- tests/utils/trtllm_media_overlay.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/utils/trtllm_media_overlay.py b/tests/utils/trtllm_media_overlay.py index 3022c4718029..1386c611c353 100644 --- a/tests/utils/trtllm_media_overlay.py +++ b/tests/utils/trtllm_media_overlay.py @@ -186,14 +186,14 @@ async def fetch(url, timeout_sec=30): calls.append(name) return payloads[name] - def decode_video(path, *args, **kwargs): + def decode_video(self, data): # OpenCV is optional in the base image; this checks the URL caller, not decoding. - assert Path(path).read_bytes() == payloads["video"] + assert data == payloads["video"] return "video-decoded" with ( patch.object(utils, "_safe_aiohttp_get", fetch), - patch.object(utils, "_load_video_by_cv2", decode_video), + patch.object(media_io.VideoMediaIO, "load_bytes", decode_video), ): image = await utils.async_load_image("https://example.com/image", format="pil") assert image.size == (32, 32) From 70373cd1a883b5c2927d473b8166d4d1a6da5d39 Mon Sep 17 00:00:00 2001 From: krishung5 <43719498+krishung5@users.noreply.github.com> Date: Mon, 26 Oct 2026 11:23:29 -0700 Subject: [PATCH 43/61] test(trtllm): share video RPC coverage through the sidecar testkit Signed-off-by: krishung5 <43719498+krishung5@users.noreply.github.com> --- lib/sidecar/testkit/tests/support/trtllm.rs | 23 +- .../testkit/tests/trtllm_multimodal.rs | 288 ++++++++++++++++-- lib/sidecar/trtllm/src/tests.rs | 167 +--------- lib/sidecar/trtllm/src/tests/e2e.rs | 2 + 4 files changed, 278 insertions(+), 202 deletions(-) diff --git a/lib/sidecar/testkit/tests/support/trtllm.rs b/lib/sidecar/testkit/tests/support/trtllm.rs index 3e1041bc93b8..22d14fea6765 100644 --- a/lib/sidecar/testkit/tests/support/trtllm.rs +++ b/lib/sidecar/testkit/tests/support/trtllm.rs @@ -41,8 +41,9 @@ pub struct Discovery { pub calls: AtomicUsize, pub unavailable: AtomicBool, pub hold: AtomicBool, - pub portable_image: AtomicBool, + pub portable_multimodal: AtomicBool, pub advertises_image: AtomicBool, + pub advertises_video: AtomicBool, release: tokio::sync::Notify, } @@ -192,12 +193,20 @@ impl Control for ControlledControl { } let mut response = self.inner.get_model_info(request).await?; let info = response.get_mut(); - info.supports_multimodal = Some(self.discovery.portable_image.load(Ordering::SeqCst)); - let modalities = if self.discovery.advertises_image.load(Ordering::SeqCst) { - vec!["image"] - } else { - vec![] - }; + info.supports_multimodal = Some(self.discovery.portable_multimodal.load(Ordering::SeqCst)); + let modalities: Vec<_> = [ + ( + self.discovery.advertises_image.load(Ordering::SeqCst), + "image", + ), + ( + self.discovery.advertises_video.load(Ordering::SeqCst), + "video", + ), + ] + .into_iter() + .filter_map(|(enabled, modality)| enabled.then_some(modality)) + .collect(); info.extra = Some( dynamo_sidecar_common::json_to_struct( serde_json::json!({"trtllm_supported_modalities": modalities}), diff --git a/lib/sidecar/testkit/tests/trtllm_multimodal.rs b/lib/sidecar/testkit/tests/trtllm_multimodal.rs index 067dc304e012..b9f7b2cb0286 100644 --- a/lib/sidecar/testkit/tests/trtllm_multimodal.rs +++ b/lib/sidecar/testkit/tests/trtllm_multimodal.rs @@ -38,6 +38,22 @@ fn image_request() -> PreprocessedRequest { request } +fn video_request() -> PreprocessedRequest { + let mut req = image_request(); + req.multi_modal_data = Some(HashMap::from([( + "video_url".into(), + vec![ + MultimodalData::RawUrl("https://example.com/first.mp4".into()), + MultimodalData::RawUrl("data:video/mp4;base64,dmlkZW8=".into()), + ], + )])); + req.multi_modal_uuids = Some(HashMap::from([( + "video_url".into(), + vec![Some("first-video".into()), None], + )])); + req +} + fn finished() -> pb::GenerateResponse { pb::GenerateResponse { event: Some(pb::generate_response::Event::Finished( @@ -59,7 +75,7 @@ fn finished() -> pb::GenerateResponse { fn advertise_images(fixture: &Fixture) { fixture .discovery - .portable_image + .portable_multimodal .store(true, Ordering::SeqCst); fixture .discovery @@ -118,32 +134,197 @@ async fn images_preserve_order_uuids_and_engine_expanded_usage() { } #[tokio::test] -async fn both_image_capability_signals_are_required() { - for (portable, image) in [(true, false), (false, true)] { - let control = Controller::::default(); - let mut fixture = Fixture::start(control.clone(), FixtureConfig::default()).await; - fixture - .discovery - .portable_image - .store(portable, Ordering::SeqCst); - fixture - .discovery - .advertises_image - .store(image, Ordering::SeqCst); - let engine = fixture.engine().await; - engine.start(0).await.unwrap(); +async fn formatter_regrouped_tool_media_reaches_openengine_in_rendered_order() { + use dynamo_llm::common::checked_file::CheckedFile; + use dynamo_llm::model_card::{ModelDeploymentCard, ModelInfoType, TokenizerKind}; + use dynamo_llm::preprocessor::OpenAIPreprocessor; + use dynamo_llm::protocols::openai::chat_completions::NvCreateChatCompletionRequest; + + let model_path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../llm/tests/data/sample-models/mock-llama-3.1-8b-instruct"); + let mut card = ModelDeploymentCard::load_from_disk(model_path, None).unwrap(); + let root = tempfile::tempdir().unwrap(); + let config = root.path().join("config.json"); + std::fs::write( + &config, + r#"{"architectures":[],"model_type":"","eos_token_id":128009}"#, + ) + .unwrap(); + card.model_info = Some(ModelInfoType::HfConfigJson( + CheckedFile::from_disk(&config).unwrap(), + )); + let tokenizer = root.path().join("tokenizer.json"); + std::fs::write(&tokenizer, r#"{"version":"1.0","truncation":null,"padding":null,"added_tokens":[],"normalizer":null,"pre_tokenizer":{"type":"Whitespace"},"post_processor":null,"decoder":null,"model":{"type":"WordLevel","vocab":{"[UNK]":0,"Result":1,"A":2,"B":3},"unk_token":"[UNK]"}}"#).unwrap(); + card.tokenizer = Some(TokenizerKind::HfTokenizerJson( + CheckedFile::from_disk(&tokenizer).unwrap(), + )); + card.display_name = "DeepSeek-V4.1-Flash".into(); + let preprocessor = OpenAIPreprocessor::new(card).unwrap(); + // The real formatter regroups tool replies by call order, not arrival order. + let request: NvCreateChatCompletionRequest = serde_json::from_value(serde_json::json!({ + "model":"mocker-model", "max_tokens":16, "messages":[ + {"role":"assistant","tool_calls":[ + {"id":"a","type":"function","function":{"name":"image","arguments":"{}"}}, + {"id":"b","type":"function","function":{"name":"image","arguments":"{}"}} + ]}, + {"role":"tool","tool_call_id":"b","content":[ + {"type":"image_url","image_url":{"url":"https://example.com/b.png"},"uuid":"b"}, + {"type":"text","text":"Result B"} + ]}, + {"role":"tool","tool_call_id":"a","content":[ + {"type":"image_url","image_url":{"url":"https://example.com/a.png"},"uuid":"a"}, + {"type":"text","text":"Result A"} + ]} + ] + })) + .unwrap(); + let (mut prepared, _, _) = preprocessor + .preprocess_request(&request, None) + .await + .unwrap(); + let prompt = prepared.extra_args.as_ref().unwrap()["formatted_prompt"] + .as_str() + .unwrap(); + assert!(prompt.find("Result A").unwrap() < prompt.find("Result B").unwrap()); + assert_eq!(prompt.matches("<｜deepseek_image｜>").count(), 2); + let expected_tokens = prepared.token_ids.clone(); + assert!(!expected_tokens.is_empty()); + prepared.model = "mocker-model".into(); + + let control = Controller::::default(); + let mut fixture = Fixture::start(control.clone(), FixtureConfig::default()).await; + advertise_images(&fixture); + let engine = fixture.engine().await; + engine.start(0).await.unwrap(); + let ctx = mock_context(); + let handle = control.request(ctx.id(), RequestPlan::default()); + fixture.respond(ctx.id(), vec![finished()]); + let outputs = collect(&engine, prepared, GenerateContext::new(ctx, None)).await; + assert_eq!( + outputs.last().unwrap().as_ref().unwrap().finish_reason, + Some(FinishReason::Stop) + ); + let native = handle.native_request().unwrap(); + assert_eq!( + native.input, + Some(pb::generate_request::Input::TokenIds(pb::TokenIds { + ids: expected_tokens.to_vec() + })) + ); + assert_eq!(native.media.len(), 2); + for (item, id) in native.media.iter().zip(["a", "b"]) { + assert_eq!(item.modality, pb::Modality::Image as i32); + assert_eq!(item.uuid, id); + assert_eq!( + item.source, + Some(pb::media_item::Source::Url(format!( + "https://example.com/{id}.png" + ))) + ); + } + engine.cleanup().await.unwrap(); + fixture.shutdown().await; +} + +#[tokio::test] +async fn both_media_capability_signals_are_required() { + for video in [false, true] { + for (portable, advertised) in [(true, false), (false, true)] { + let control = Controller::::default(); + let mut fixture = Fixture::start(control.clone(), FixtureConfig::default()).await; + fixture + .discovery + .portable_multimodal + .store(portable, Ordering::SeqCst); + let (target, other) = if video { + ( + &fixture.discovery.advertises_video, + &fixture.discovery.advertises_image, + ) + } else { + ( + &fixture.discovery.advertises_image, + &fixture.discovery.advertises_video, + ) + }; + target.store(advertised, Ordering::SeqCst); + other.store(true, Ordering::SeqCst); + let engine = fixture.engine().await; + engine.start(0).await.unwrap(); + let ctx = mock_context(); + let handle = control.request(ctx.id(), RequestPlan::default()); + assert!( + engine + .generate( + if video { + video_request() + } else { + image_request() + }, + GenerateContext::new(ctx, None) + ) + .await + .is_err() + ); + assert!(handle.native_request().is_none()); + engine.cleanup().await.unwrap(); + fixture.shutdown().await; + } + } +} + +#[tokio::test] +async fn videos_recover_discovery_and_preserve_sources_uuids_and_usage() { + let control = Controller::::default(); + let mut fixture = Fixture::start(control.clone(), FixtureConfig::default()).await; + fixture.discovery.unavailable.store(true, Ordering::SeqCst); + let engine = fixture.engine_with_context_length(32768); + engine.start(0).await.unwrap(); + assert_eq!(fixture.discovery.calls.load(Ordering::SeqCst), 1); + fixture.discovery.unavailable.store(false, Ordering::SeqCst); + fixture + .discovery + .portable_multimodal + .store(true, Ordering::SeqCst); + fixture + .discovery + .advertises_video + .store(true, Ordering::SeqCst); + for _ in 0..2 { let ctx = mock_context(); let handle = control.request(ctx.id(), RequestPlan::default()); + fixture.respond(ctx.id(), vec![finished()]); + let outputs = collect(&engine, video_request(), GenerateContext::new(ctx, None)).await; + let output = outputs.last().unwrap().as_ref().unwrap(); + assert_eq!(output.finish_reason, Some(FinishReason::Stop)); + assert_eq!(output.completion_usage.as_ref().unwrap().prompt_tokens, 131); + let native = handle.native_request().unwrap(); + assert_eq!(native.stopping.unwrap().max_tokens, None); + assert_eq!(native.media.len(), 2); assert!( - engine - .generate(image_request(), GenerateContext::new(ctx, None)) - .await - .is_err() + native + .media + .iter() + .all(|item| item.modality == pb::Modality::Video as i32) + ); + assert_eq!(native.media[0].uuid, "first-video"); + assert_eq!(native.media[1].uuid, ""); + assert_eq!( + native.media[0].source, + Some(pb::media_item::Source::Url( + "https://example.com/first.mp4".into() + )) + ); + assert_eq!( + native.media[1].source, + Some(pb::media_item::Source::DataUri( + "data:video/mp4;base64,dmlkZW8=".into() + )) ); - assert!(handle.native_request().is_none()); - engine.cleanup().await.unwrap(); - fixture.shutdown().await; } + assert_eq!(fixture.discovery.calls.load(Ordering::SeqCst), 2); + engine.cleanup().await.unwrap(); + fixture.shutdown().await; } #[tokio::test] @@ -154,8 +335,16 @@ async fn concurrent_discovery_failure_is_shared_and_later_recovers() { let engine = fixture.engine_with_context_length(32768); engine.start(0).await.unwrap(); fixture.discovery.hold.store(true, Ordering::SeqCst); - let requests = (0..4) - .map(|_| engine.generate(image_request(), GenerateContext::new(mock_context(), None))); + let requests = (0..4).map(|index| { + engine.generate( + if index % 2 == 0 { + image_request() + } else { + video_request() + }, + GenerateContext::new(mock_context(), None), + ) + }); let (results, ()) = tokio::join!(join_all(requests), async { wait_for_calls(&fixture, 2).await; fixture.release_discovery(); @@ -171,12 +360,16 @@ async fn concurrent_discovery_failure_is_shared_and_later_recovers() { // Failures are retained for one second; recovery is not permanently latched. fixture.discovery.unavailable.store(false, Ordering::SeqCst); advertise_images(&fixture); + fixture + .discovery + .advertises_video + .store(true, Ordering::SeqCst); tokio::time::sleep(Duration::from_millis(1100)).await; - for _ in 0..2 { + for req in [image_request(), video_request()] { let ctx = mock_context(); control.request(ctx.id(), RequestPlan::default()); fixture.respond(ctx.id(), vec![finished()]); - let outputs = collect(&engine, image_request(), GenerateContext::new(ctx, None)).await; + let outputs = collect(&engine, req, GenerateContext::new(ctx, None)).await; assert_eq!(outputs.len(), 1); assert_eq!( outputs[0].as_ref().unwrap().finish_reason, @@ -190,7 +383,7 @@ async fn concurrent_discovery_failure_is_shared_and_later_recovers() { #[tokio::test] async fn stalled_discovery_observes_request_and_shutdown_cancellation() { - for shutdown in [false, true] { + for (video, shutdown) in [(false, false), (false, true), (true, false), (true, true)] { let control = Controller::::default(); let mut fixture = Fixture::start(control, FixtureConfig::default()).await; fixture.discovery.unavailable.store(true, Ordering::SeqCst); @@ -201,7 +394,15 @@ async fn stalled_discovery_observes_request_and_shutdown_cancellation() { let cancelling_ctx = ctx.clone(); let (outputs, ()) = bounded("cancel stalled discovery", async { tokio::join!( - collect(&engine, image_request(), GenerateContext::new(ctx, None)), + collect( + &engine, + if video { + video_request() + } else { + image_request() + }, + GenerateContext::new(ctx, None) + ), async { wait_for_calls(&fixture, 2).await; if shutdown { @@ -232,8 +433,16 @@ async fn unresponsive_discovery_has_one_bounded_timeout_for_all_waiters() { let engine = fixture.engine_with_context_length(32768); engine.start(0).await.unwrap(); fixture.discovery.hold.store(true, Ordering::SeqCst); - let requests = (0..4) - .map(|_| engine.generate(image_request(), GenerateContext::new(mock_context(), None))); + let requests = (0..4).map(|index| { + engine.generate( + if index % 2 == 0 { + image_request() + } else { + video_request() + }, + GenerateContext::new(mock_context(), None), + ) + }); let results = tokio::time::timeout(Duration::from_secs(35), join_all(requests)) .await .expect("all four requests must share the 30-second RPC timeout"); @@ -253,16 +462,29 @@ async fn unresponsive_discovery_has_one_bounded_timeout_for_all_waiters() { } #[tokio::test] -async fn image_window_override_is_rejected_before_dispatch() { +async fn media_window_override_is_rejected_before_dispatch() { let control = Controller::::default(); let mut fixture = Fixture::start(control.clone(), FixtureConfig::default()).await; advertise_images(&fixture); + fixture + .discovery + .advertises_video + .store(true, Ordering::SeqCst); let engine = fixture.engine_with_context_length(4096); engine.start(0).await.unwrap(); - for max_tokens in [None, Some(1)] { + for (video, max_tokens) in [ + (false, None), + (false, Some(1)), + (true, None), + (true, Some(1)), + ] { let ctx = mock_context(); let handle = control.request(ctx.id(), RequestPlan::default()); - let mut req = image_request(); + let mut req = if video { + video_request() + } else { + image_request() + }; req.stop_conditions.max_tokens = max_tokens; let result = engine.generate(req, GenerateContext::new(ctx, None)).await; let error = result.err().expect("window mismatch must fail"); diff --git a/lib/sidecar/trtllm/src/tests.rs b/lib/sidecar/trtllm/src/tests.rs index bb20ad2413ef..a8bef9a3ec0e 100644 --- a/lib/sidecar/trtllm/src/tests.rs +++ b/lib/sidecar/trtllm/src/tests.rs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -use std::collections::{BTreeSet, HashMap}; +use std::collections::BTreeSet; use std::net::SocketAddr; use std::num::NonZeroUsize; use std::pin::Pin; @@ -10,9 +10,8 @@ use std::sync::atomic::{AtomicBool, AtomicU32, AtomicUsize, Ordering}; use std::time::Duration; use dynamo_backend_common::{ - DisaggregationMode, ErrorType, FinishReason, GenerateContext, LLMEngine, MultimodalData, - MultimodalItemRef, OutputOptions, PreprocessedRequest, SamplingOptions, StopConditions, - StopReason, + DisaggregationMode, ErrorType, FinishReason, GenerateContext, LLMEngine, OutputOptions, + PreprocessedRequest, SamplingOptions, StopConditions, StopReason, }; use dynamo_sidecar_common::{GrpcEndpoint, GrpcTransportConfig}; use futures::{Stream, StreamExt}; @@ -53,11 +52,6 @@ struct FakeTrtllm { empty_model_info: Arc, /// Answers `UNAVAILABLE`, the shape of an engine that is not serving yet. unavailable_model_info: Arc, - /// Portable coarse-grained model capability. - supports_multimodal: Arc, - /// TensorRT-LLM's finer-grained modality advertisement. - advertises_image: Arc, - advertises_video: Arc, model_info_calls: Arc, kv_routing: Arc, server_info_error: Arc>>, @@ -120,7 +114,7 @@ impl pb::inference_server::Inference for FakeTrtllm { self.requests.lock().await.push(request.clone()); let request_id = request.request_id.clone(); - let prompt_tokens = prompt_len(&request) + 64 * request.media.len() as u32; + let prompt_tokens = prompt_len(&request); let wants_logprobs = wants_logprobs(&request); let context_only = is_context_only(&request); let hang = self.hang.load(Ordering::SeqCst); @@ -228,17 +222,6 @@ impl pb::control_server::Control for FakeTrtllm { if self.unavailable_model_info.load(Ordering::SeqCst) { return Err(Status::unavailable("not serving yet")); } - let supports_multimodal = self.supports_multimodal.load(Ordering::SeqCst); - let modalities = [ - (self.advertises_image.load(Ordering::SeqCst), "image"), - (self.advertises_video.load(Ordering::SeqCst), "video"), - ] - .into_iter() - .filter(|(advertised, _)| *advertised) - .map(|(_, modality)| prost_types::Value { - kind: Some(prost_types::value::Kind::StringValue(modality.to_string())), - }) - .collect(); Ok(Response::new(pb::ModelInfo { model_id: "fake-model".to_string(), max_context_length: if self.empty_model_info.load(Ordering::SeqCst) { @@ -246,17 +229,7 @@ impl pb::control_server::Control for FakeTrtllm { } else { Some(4096) }, - supports_multimodal: Some(supports_multimodal), - extra: Some(prost_types::Struct { - fields: std::collections::BTreeMap::from([( - "trtllm_supported_modalities".to_string(), - prost_types::Value { - kind: Some(prost_types::value::Kind::ListValue( - prost_types::ListValue { values: modalities }, - )), - }, - )]), - }), + supports_multimodal: Some(false), ..Default::default() })) } @@ -713,90 +686,6 @@ async fn aggregated_generation_streams_delta_then_terminal() { } } -#[tokio::test] -async fn advertised_image_media_reaches_openengine_and_uses_expanded_usage() { - let service = FakeTrtllm::default(); - service.supports_multimodal.store(true, Ordering::SeqCst); - service.advertises_image.store(true, Ordering::SeqCst); - let server = FakeServer::start(service).await; - let engine = engine(&server.endpoint, 1); - engine.start(0).await.expect("start"); - - let mut req = request(); - req.stop_conditions.max_tokens = None; - add_images(&mut req, ["data:image/png;base64,aW1hZ2U="]); - let outputs = collect(&engine, req).await; - let usage = outputs - .last() - .and_then(|output| output.completion_usage.as_ref()) - .expect("terminal engine usage"); - assert_eq!(usage.prompt_tokens, 67, "3 text tokens + 64 image tokens"); - - let requests = server.service.requests.lock().await; - let sent = requests.first().expect("recorded request"); - assert_eq!(sent.stopping.as_ref().expect("stopping").max_tokens, None); - assert_eq!(sent.media.len(), 1); - assert_eq!(sent.media[0].modality, pb::Modality::Image as i32); - assert!(matches!( - sent.media[0].source.as_ref(), - Some(pb::media_item::Source::DataUri(value)) if value.starts_with("data:image/png") - )); -} - -#[tokio::test] -async fn advertised_video_media_reaches_openengine() { - let service = FakeTrtllm::default(); - service.supports_multimodal.store(true, Ordering::SeqCst); - service.advertises_video.store(true, Ordering::SeqCst); - let server = FakeServer::start(service).await; - let engine = engine(&server.endpoint, 1); - engine.start(0).await.expect("start"); - - let mut req = request(); - req.multi_modal_data = Some(HashMap::from([( - "video_url".to_string(), - vec![MultimodalData::RawUrl( - "https://example.com/video.mp4".to_string(), - )], - )])); - collect(&engine, req).await; - - let requests = server.service.requests.lock().await; - let sent = requests.first().expect("recorded request"); - assert_eq!(sent.media.len(), 1); - assert_eq!(sent.media[0].modality, pb::Modality::Video as i32); - assert_eq!( - sent.media[0].source, - Some(pb::media_item::Source::Url( - "https://example.com/video.mp4".to_string() - )) - ); -} - -#[tokio::test] -async fn both_multimodal_capability_signals_are_required() { - for (coarse, image) in [(true, false), (false, true)] { - let service = FakeTrtllm::default(); - service.supports_multimodal.store(coarse, Ordering::SeqCst); - service.advertises_image.store(image, Ordering::SeqCst); - let server = FakeServer::start(service).await; - let engine = engine(&server.endpoint, 1); - engine.start(0).await.expect("start"); - - let mut req = request(); - add_images(&mut req, ["https://example.com/image.png"]); - let context = dynamo_backend_common::testing::mock_context(); - let result = engine - .generate(req, GenerateContext::new(context, None)) - .await; - assert!( - result.is_err(), - "capabilities ({coarse}, {image}) must fail closed" - ); - assert!(server.service.requests.lock().await.is_empty()); - } -} - #[tokio::test] async fn cancellation_yields_a_cancelled_terminal() { let service = FakeTrtllm::default(); @@ -1176,52 +1065,6 @@ async fn a_configured_context_length_survives_a_server_without_control() { assert_eq!(server.service.model_info_calls.load(Ordering::SeqCst), 1); } -#[tokio::test] -async fn media_requests_retry_transient_capability_discovery() { - for (key, url, image, video) in [ - ("image_url", "https://example.com/image.png", true, false), - ("video_url", "https://example.com/video.mp4", false, true), - ] { - let service = FakeTrtllm::default(); - service.unavailable_model_info.store(true, Ordering::SeqCst); - let server = FakeServer::start(service).await; - let engine = engine_with(&server.endpoint, transport(1), Some(4096), AGG); - engine.start(0).await.expect("start with configured window"); - - server - .service - .unavailable_model_info - .store(false, Ordering::SeqCst); - server - .service - .supports_multimodal - .store(true, Ordering::SeqCst); - server - .service - .advertises_image - .store(image, Ordering::SeqCst); - server - .service - .advertises_video - .store(video, Ordering::SeqCst); - let mut media_request = request(); - media_request.multi_modal_data = Some(HashMap::from([( - key.to_string(), - vec![MultimodalData::RawUrl(url.to_string())], - )])); - - let output = collect(&engine, media_request).await; - assert_eq!( - output.last().unwrap().finish_reason, - Some(FinishReason::Stop) - ); - assert_eq!( - server.service.model_info_calls.load(Ordering::SeqCst), - 2, - "the {key} request must retry the transient startup failure once" - ); - } -} /// Shutdown has to terminate in-flight requests: the worker cannot drain if a /// stream waits forever on an engine that will never answer. #[tokio::test] diff --git a/lib/sidecar/trtllm/src/tests/e2e.rs b/lib/sidecar/trtllm/src/tests/e2e.rs index 35282adbd67a..7062400c9caf 100644 --- a/lib/sidecar/trtllm/src/tests/e2e.rs +++ b/lib/sidecar/trtllm/src/tests/e2e.rs @@ -5,6 +5,8 @@ use std::collections::HashMap; +use dynamo_backend_common::{MultimodalData, MultimodalItemRef}; + use super::*; /// Drives a real disaggregated prefill -> decode handoff against two live From 52a01cfbd88ca89a07ac1eb4ecfde951c3cb1856 Mon Sep 17 00:00:00 2001 From: krishung5 <43719498+krishung5@users.noreply.github.com> Date: Mon, 26 Oct 2026 13:56:22 -0700 Subject: [PATCH 44/61] fix(ci): include KV event dependencies in TRT-LLM media overlay Retain the release launcher while importing the KV event symbols used by the pinned server class. Exercise installed server startup with KV events disabled, available, and unavailable before loading model weights. Signed-off-by: krishung5 <43719498+krishung5@users.noreply.github.com> --- tests/utils/test_trtllm_media_overlay.py | 34 ++++++++++- tests/utils/trtllm_media_overlay.py | 72 ++++++++++++++++++------ 2 files changed, 89 insertions(+), 17 deletions(-) diff --git a/tests/utils/test_trtllm_media_overlay.py b/tests/utils/test_trtllm_media_overlay.py index 617ff3b9e0cd..460ee01a1efe 100644 --- a/tests/utils/test_trtllm_media_overlay.py +++ b/tests/utils/test_trtllm_media_overlay.py @@ -3,9 +3,12 @@ """Source overlay edits preserve unrelated release definitions.""" +import sys +from types import ModuleType + import pytest -from tests.utils.trtllm_media_overlay import _replace_definition +from tests.utils.trtllm_media_overlay import _overlay_server, _replace_definition pytestmark = [pytest.mark.unit, pytest.mark.pre_merge, pytest.mark.gpu_0] @@ -44,3 +47,32 @@ def test_replace_server_class_preserves_release_launcher(tmp_path): exec(destination.read_text(), namespace) assert namespace["Server"].workers == 4 assert namespace["launcher"]() == "release" + + +def test_overlay_server_installs_startup_dependencies(tmp_path, monkeypatch): + destination = tmp_path / "installed.py" + source = tmp_path / "source.py" + destination.write_text( + "from typing import Any\n" + "class OpenEngineServer:\n workers = 0\n" + "\ndef launcher():\n return 'release'\n" + ) + source.write_text("class OpenEngineServer:\n workers = 4\n") + coordinator = ModuleType("overlay_test.coordinator") + coordinator.FrontendClient = object() + events = ModuleType("overlay_test.kv_events") + events.KvEventsUnavailableError = type( + "KvEventsUnavailableError", (RuntimeError,), {} + ) + events.events_config = object() + monkeypatch.setitem(sys.modules, coordinator.__name__, coordinator) + monkeypatch.setitem(sys.modules, events.__name__, events) + + _overlay_server(destination, source) + namespace = {"__package__": "overlay_test"} + exec(destination.read_text(), namespace) + assert namespace["events_config"] is events.events_config + assert namespace["KvEventsUnavailableError"] is events.KvEventsUnavailableError + assert namespace["FrontendClient"] is coordinator.FrontendClient + assert namespace["OpenEngineServer"].workers == 4 + assert namespace["launcher"]() == "release" diff --git a/tests/utils/trtllm_media_overlay.py b/tests/utils/trtllm_media_overlay.py index 9f14560620dd..8c06473fd0d3 100644 --- a/tests/utils/trtllm_media_overlay.py +++ b/tests/utils/trtllm_media_overlay.py @@ -11,7 +11,8 @@ import shutil from io import BytesIO from pathlib import Path -from unittest.mock import patch +from types import SimpleNamespace +from unittest.mock import AsyncMock, Mock, patch def _replace_definition(destination, source, class_name, names): @@ -56,6 +57,24 @@ def definitions(tree): destination.write_text(result) +def _overlay_server(server, source): + """Keep the release launcher while installing the class and its imports.""" + _replace_definition(server, source, None, ["OpenEngineServer"]) + server.write_text( + server.read_text() + .replace( + "from typing import Any", + "from typing import Any\nfrom concurrent.futures import ThreadPoolExecutor\n" + "from .coordinator import FrontendClient\n" + "from .kv_events import KvEventsUnavailableError, events_config", + ) + .replace( + "from openengine.v1 import openengine_pb2_grpc", + "from .bindings import openengine_pb2_grpc", + ) + ) + + def install_overlay(source): """Install the pinned media implementation into a release runtime.""" package = Path( @@ -78,21 +97,7 @@ def install_overlay(source): # Retain release-coupled launch_server; use the pinned class's worker pools # and cleanup rather than silently running preprocessing on the default pool. server = package / "grpc/openengine/server.py" - _replace_definition( - server, source / "grpc/openengine/server.py", None, ["OpenEngineServer"] - ) - server.write_text( - server.read_text() - .replace( - "from typing import Any", - "from typing import Any\nfrom concurrent.futures import ThreadPoolExecutor\n" - "from .coordinator import FrontendClient", - ) - .replace( - "from openengine.v1 import openengine_pb2_grpc", - "from .bindings import openengine_pb2_grpc", - ) - ) + _overlay_server(server, source / "grpc/openengine/server.py") registry = package / "inputs/registry.py" _replace_definition( registry, @@ -154,12 +159,47 @@ def check_signatures(): print("TRT-LLM overlay: retained media helper signatures passed") +async def check_server_startup(): + """Exercise the installed startup method without loading model weights.""" + from tensorrt_llm.grpc.openengine.kv_events import KvEventsUnavailableError + from tensorrt_llm.grpc.openengine.server import OpenEngineServer + + for configured, available in ((False, False), (True, True), (True, False)): + events = SimpleNamespace(enable_kv_cache_events=configured, publisher="zmq") + capacity = Mock(return_value={"kvEventsEnabled": available}) + server = OpenEngineServer.__new__(OpenEngineServer) + server._llm = SimpleNamespace( + args=SimpleNamespace( + kv_cache_config=SimpleNamespace(kv_events_config=events) + ), + _executor=SimpleNamespace(get_kv_cache_capacity=capacity), + ) + server._server = AsyncMock() + server.host, server.port = "127.0.0.1", 0 + if configured and not available: + try: + await server.start() + except KvEventsUnavailableError: + pass + else: + raise AssertionError( + "Startup accepted an unavailable KV event publisher" + ) + server._server.start.assert_not_awaited() + else: + await server.start() + server._server.start.assert_awaited_once() + assert capacity.call_count == int(configured) + print("TRT-LLM overlay: server startup with and without KV events passed") + + async def validate(): import numpy as np import soundfile from PIL import Image from tensorrt_llm.inputs import media_io, utils + await check_server_startup() image_bytes = BytesIO() Image.new("RGB", (32, 32), "red").save(image_bytes, format="PNG") audio_bytes = BytesIO() From bd460efa49d805ece5eaf5b2572976ed1a7cf99a Mon Sep 17 00:00:00 2001 From: krishung5 Date: Tue, 27 Oct 2026 10:25:33 -0700 Subject: [PATCH 45/61] fix(ci): fail required TRT-LLM sidecar tests when OpenEngine is missing Signed-off-by: krishung5 --- .github/workflows/shared-test.yml | 1 + tests/serve/test_sidecar.py | 45 ++-------------------------- tests/utils/sidecar_runtime.py | 30 +++++++++++++++++++ tests/utils/test_sidecar_runtime.py | 46 +++++++++++++++++++++++++++++ 4 files changed, 80 insertions(+), 42 deletions(-) create mode 100644 tests/utils/sidecar_runtime.py create mode 100644 tests/utils/test_sidecar_runtime.py diff --git a/.github/workflows/shared-test.yml b/.github/workflows/shared-test.yml index 4659fd663fe6..dde5beff676c 100644 --- a/.github/workflows/shared-test.yml +++ b/.github/workflows/shared-test.yml @@ -272,6 +272,7 @@ jobs: install -m755 "sidecar-artifact/dynamo-${SIDECAR_BACKEND}-sidecar" \ "${bin_dir}/dynamo-${SIDECAR_BACKEND}-sidecar" echo "DYNAMO_SIDECAR_BIN=${bin_dir}/dynamo-${SIDECAR_BACKEND}-sidecar" >> "${GITHUB_ENV}" + echo "DYNAMO_TEST_SIDECAR_BACKEND=${SIDECAR_BACKEND}" >> "${GITHUB_ENV}" if [ -z "${CONTAINER_WORKSPACE}" ]; then echo "::error::container_workspace is empty; refusing to replace test files" exit 1 diff --git a/tests/serve/test_sidecar.py b/tests/serve/test_sidecar.py index f0e279a19853..e5ef558aab18 100644 --- a/tests/serve/test_sidecar.py +++ b/tests/serve/test_sidecar.py @@ -3,9 +3,7 @@ """E2E coverage for native-gRPC sidecar launch scripts.""" -import importlib.util import os -import pathlib from dataclasses import dataclass, field import pytest @@ -35,6 +33,7 @@ ) from tests.utils.payloads import KvTransferPayload, OmittedMaxTokensChatPayload from tests.utils.port_utils import allocate_contiguous_ports, deallocate_ports +from tests.utils.sidecar_runtime import require_trtllm_openengine vllm_sidecar_dir = os.environ.get("VLLM_SIDECAR_DIR") or os.path.join( WORKSPACE_DIR, "lib/sidecar/vllm" @@ -47,38 +46,6 @@ ) -def _trtllm_serves_openengine() -> bool: - """Is the installed TensorRT-LLM new enough to serve `openengine.v1`? - - `lib/sidecar/trtllm` speaks OpenEngine, which the servicer provides from - release 1.3.0rc27 onward. `container/context.yaml` pins that release, so - this passes in the stock container and the case runs; it guards the - off-container and downgraded-image paths, where - `trtllm-serve --grpc-protocol openengine` does not exist and the launcher - cannot come up at all. - - Probing for the servicer module beats comparing version strings: it is the - thing the sidecar actually needs, and it tracks the pinned image by itself. - `find_spec` on the parent does not import TensorRT-LLM, so this stays cheap - and works on a host without a GPU. - """ - spec = importlib.util.find_spec("tensorrt_llm") - locations = getattr(spec, "submodule_search_locations", None) if spec else None - if not locations: - return False - return any( - (pathlib.Path(root) / "grpc" / "openengine" / "servicer.py").is_file() - for root in locations - ) - - -TRTLLM_OPENENGINE_SKIP_REASON = ( - "the installed TensorRT-LLM has no OpenEngine servicer; the sidecar needs " - "release 1.3.0rc27 or newer, which container/context.yaml pins. Seeing this " - "skip in CI means the pin moved backwards." -) - - CANCELLATION_MAX_TOKENS = 2048 CANCELLATION_CONTEXT_LEN = 4096 @@ -236,10 +203,6 @@ def _on_first_gpu(*names: str) -> dict[str, tuple[int, ...]]: pytest.mark.requested_trtllm_kv_tokens(4096), pytest.mark.timeout(780), pytest.mark.post_merge, - pytest.mark.skipif( - not _trtllm_serves_openengine(), - reason=TRTLLM_OPENENGINE_SKIP_REASON, - ), ], model="Qwen/Qwen3-0.6B", env={ @@ -280,10 +243,6 @@ def _on_first_gpu(*names: str) -> dict[str, tuple[int, ...]]: # at the single-engine budget and then idle until the kill timer. pytest.mark.timeout(1200), pytest.mark.post_merge, - pytest.mark.skipif( - not _trtllm_serves_openengine(), - reason=TRTLLM_OPENENGINE_SKIP_REASON, - ), ], model="Qwen/Qwen3-0.6B", timeout=1000, @@ -527,6 +486,8 @@ def _on_first_gpu(*names: str) -> dict[str, tuple[int, ...]]: @pytest.fixture(params=params_with_model_mark(sidecar_configs)) def sidecar_config_test(request, monkeypatch, discovery_backend): config = sidecar_configs[request.param] + if config.name.startswith("trtllm_"): + require_trtllm_openengine() monkeypatch.setenv("MODEL", config.model) monkeypatch.setenv("DYN_DISCOVERY_BACKEND", discovery_backend) monkeypatch.setenv("DYN_NAMESPACE", f"sidecar-{generate_random_suffix()}") diff --git a/tests/utils/sidecar_runtime.py b/tests/utils/sidecar_runtime.py new file mode 100644 index 000000000000..e248f0a4de89 --- /dev/null +++ b/tests/utils/sidecar_runtime.py @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Runtime requirements for native sidecar serving tests.""" + +import importlib.util +import os +from pathlib import Path + +import pytest + + +def trtllm_serves_openengine() -> bool: + """Probe the installed package without importing TensorRT-LLM.""" + spec = importlib.util.find_spec("tensorrt_llm") + locations = getattr(spec, "submodule_search_locations", None) if spec else None + return any( + (Path(root) / "grpc/openengine/servicer.py").is_file() + for root in locations or () + ) + + +def require_trtllm_openengine() -> None: + """Fail required TRT-LLM CI, or skip optional local runs, if unavailable.""" + if trtllm_serves_openengine(): + return + reason = "the installed TensorRT-LLM runtime has no OpenEngine servicer" + if os.environ.get("DYNAMO_TEST_SIDECAR_BACKEND") == "trtllm": + pytest.fail(reason) + pytest.skip(reason) diff --git a/tests/utils/test_sidecar_runtime.py b/tests/utils/test_sidecar_runtime.py new file mode 100644 index 000000000000..81799cc1f497 --- /dev/null +++ b/tests/utils/test_sidecar_runtime.py @@ -0,0 +1,46 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Required sidecar lanes must exercise their runtime instead of skipping.""" + +from types import SimpleNamespace + +import pytest + +from tests.utils import sidecar_runtime + +pytestmark = [pytest.mark.unit, pytest.mark.pre_merge, pytest.mark.gpu_0] + + +@pytest.mark.parametrize("backend", [None, "trtllm", "vllm"]) +@pytest.mark.parametrize("available", [False, True]) +def test_required_ci_fails_missing_runtime_and_local_runs_skip( + monkeypatch, backend, available +): + monkeypatch.delenv("DYNAMO_TEST_SIDECAR_BACKEND", raising=False) + if backend is not None: + monkeypatch.setenv("DYNAMO_TEST_SIDECAR_BACKEND", backend) + monkeypatch.setattr(sidecar_runtime, "trtllm_serves_openengine", lambda: available) + if available: + sidecar_runtime.require_trtllm_openengine() + else: + error = pytest.fail.Exception if backend == "trtllm" else pytest.skip.Exception + with pytest.raises(error, match="no OpenEngine servicer"): + sidecar_runtime.require_trtllm_openengine() + + +@pytest.mark.parametrize("layout", ["absent", "old", "openengine"]) +def test_probe_requires_servicer_file_without_importing_backend( + monkeypatch, tmp_path, layout +): + spec = ( + None + if layout == "absent" + else SimpleNamespace(submodule_search_locations=[str(tmp_path)]) + ) + if layout == "openengine": + servicer = tmp_path / "grpc/openengine/servicer.py" + servicer.parent.mkdir(parents=True) + servicer.write_text("raise AssertionError('backend must not be imported')\n") + monkeypatch.setattr(sidecar_runtime.importlib.util, "find_spec", lambda name: spec) + assert sidecar_runtime.trtllm_serves_openengine() is (layout == "openengine") From 1911e1ce4bb6ca7bfdeb9e19f3b2e9f7f9524149 Mon Sep 17 00:00:00 2001 From: krishung5 Date: Tue, 27 Oct 2026 10:43:39 -0700 Subject: [PATCH 46/61] Separate container E2E coverage from TRT-LLM image wiring Signed-off-by: krishung5 --- .github/workflows/shared-test.yml | 1 - tests/serve/test_sidecar.py | 70 +++++++++++++++++------------ tests/utils/sidecar_runtime.py | 30 ------------- tests/utils/test_sidecar_runtime.py | 46 ------------------- 4 files changed, 42 insertions(+), 105 deletions(-) delete mode 100644 tests/utils/sidecar_runtime.py delete mode 100644 tests/utils/test_sidecar_runtime.py diff --git a/.github/workflows/shared-test.yml b/.github/workflows/shared-test.yml index dde5beff676c..4659fd663fe6 100644 --- a/.github/workflows/shared-test.yml +++ b/.github/workflows/shared-test.yml @@ -272,7 +272,6 @@ jobs: install -m755 "sidecar-artifact/dynamo-${SIDECAR_BACKEND}-sidecar" \ "${bin_dir}/dynamo-${SIDECAR_BACKEND}-sidecar" echo "DYNAMO_SIDECAR_BIN=${bin_dir}/dynamo-${SIDECAR_BACKEND}-sidecar" >> "${GITHUB_ENV}" - echo "DYNAMO_TEST_SIDECAR_BACKEND=${SIDECAR_BACKEND}" >> "${GITHUB_ENV}" if [ -z "${CONTAINER_WORKSPACE}" ]; then echo "::error::container_workspace is empty; refusing to replace test files" exit 1 diff --git a/tests/serve/test_sidecar.py b/tests/serve/test_sidecar.py index f49c1e7af3dc..f0e279a19853 100644 --- a/tests/serve/test_sidecar.py +++ b/tests/serve/test_sidecar.py @@ -3,7 +3,9 @@ """E2E coverage for native-gRPC sidecar launch scripts.""" +import importlib.util import os +import pathlib from dataclasses import dataclass, field import pytest @@ -28,13 +30,11 @@ disaggregated_token_count_payload, http_cancellation_payloads, kv_routing_payload, - multimodal_payload_default, sglang_generate_cancellation_payloads, stop_string_payloads, ) from tests.utils.payloads import KvTransferPayload, OmittedMaxTokensChatPayload from tests.utils.port_utils import allocate_contiguous_ports, deallocate_ports -from tests.utils.sidecar_runtime import require_trtllm_openengine vllm_sidecar_dir = os.environ.get("VLLM_SIDECAR_DIR") or os.path.join( WORKSPACE_DIR, "lib/sidecar/vllm" @@ -47,6 +47,38 @@ ) +def _trtllm_serves_openengine() -> bool: + """Is the installed TensorRT-LLM new enough to serve `openengine.v1`? + + `lib/sidecar/trtllm` speaks OpenEngine, which the servicer provides from + release 1.3.0rc27 onward. `container/context.yaml` pins that release, so + this passes in the stock container and the case runs; it guards the + off-container and downgraded-image paths, where + `trtllm-serve --grpc-protocol openengine` does not exist and the launcher + cannot come up at all. + + Probing for the servicer module beats comparing version strings: it is the + thing the sidecar actually needs, and it tracks the pinned image by itself. + `find_spec` on the parent does not import TensorRT-LLM, so this stays cheap + and works on a host without a GPU. + """ + spec = importlib.util.find_spec("tensorrt_llm") + locations = getattr(spec, "submodule_search_locations", None) if spec else None + if not locations: + return False + return any( + (pathlib.Path(root) / "grpc" / "openengine" / "servicer.py").is_file() + for root in locations + ) + + +TRTLLM_OPENENGINE_SKIP_REASON = ( + "the installed TensorRT-LLM has no OpenEngine servicer; the sidecar needs " + "release 1.3.0rc27 or newer, which container/context.yaml pins. Seeing this " + "skip in CI means the pin moved backwards." +) + + CANCELLATION_MAX_TOKENS = 2048 CANCELLATION_CONTEXT_LEN = 4096 @@ -204,6 +236,10 @@ def _on_first_gpu(*names: str) -> dict[str, tuple[int, ...]]: pytest.mark.requested_trtllm_kv_tokens(4096), pytest.mark.timeout(780), pytest.mark.post_merge, + pytest.mark.skipif( + not _trtllm_serves_openengine(), + reason=TRTLLM_OPENENGINE_SKIP_REASON, + ), ], model="Qwen/Qwen3-0.6B", env={ @@ -219,30 +255,6 @@ def _on_first_gpu(*names: str) -> dict[str, tuple[int, ...]]: *http_cancellation_payloads(None, max_tokens=CANCELLATION_MAX_TOKENS), ], ), - "trtllm_aggregated_image": SidecarConfig( - name="trtllm_aggregated_image", - directory=trtllm_sidecar_dir, - script_name="agg.sh", - marks=[ - pytest.mark.trtllm, - pytest.mark.multimodal, - pytest.mark.gpu_1, - pytest.mark.profiled_vram_gib(12.0), - pytest.mark.requested_trtllm_kv_tokens(32768), - pytest.mark.timeout(960), - pytest.mark.post_merge, - ], - model="Qwen/Qwen3-VL-2B-Instruct", - timeout=900, - delayed_start=60, - env={"PYTHONUNBUFFERED": "1"}, - request_payloads=[ - multimodal_payload_default( - text="Describe what you see in this image.", - expected_response=["mountain", "rock", "trees", "road"], - ) - ], - ), "trtllm_disaggregated": SidecarConfig( name="trtllm_disaggregated", directory=trtllm_sidecar_dir, @@ -268,6 +280,10 @@ def _on_first_gpu(*names: str) -> dict[str, tuple[int, ...]]: # at the single-engine budget and then idle until the kill timer. pytest.mark.timeout(1200), pytest.mark.post_merge, + pytest.mark.skipif( + not _trtllm_serves_openengine(), + reason=TRTLLM_OPENENGINE_SKIP_REASON, + ), ], model="Qwen/Qwen3-0.6B", timeout=1000, @@ -511,8 +527,6 @@ def _on_first_gpu(*names: str) -> dict[str, tuple[int, ...]]: @pytest.fixture(params=params_with_model_mark(sidecar_configs)) def sidecar_config_test(request, monkeypatch, discovery_backend): config = sidecar_configs[request.param] - if config.name.startswith("trtllm_"): - require_trtllm_openengine() monkeypatch.setenv("MODEL", config.model) monkeypatch.setenv("DYN_DISCOVERY_BACKEND", discovery_backend) monkeypatch.setenv("DYN_NAMESPACE", f"sidecar-{generate_random_suffix()}") diff --git a/tests/utils/sidecar_runtime.py b/tests/utils/sidecar_runtime.py deleted file mode 100644 index e248f0a4de89..000000000000 --- a/tests/utils/sidecar_runtime.py +++ /dev/null @@ -1,30 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Runtime requirements for native sidecar serving tests.""" - -import importlib.util -import os -from pathlib import Path - -import pytest - - -def trtllm_serves_openengine() -> bool: - """Probe the installed package without importing TensorRT-LLM.""" - spec = importlib.util.find_spec("tensorrt_llm") - locations = getattr(spec, "submodule_search_locations", None) if spec else None - return any( - (Path(root) / "grpc/openengine/servicer.py").is_file() - for root in locations or () - ) - - -def require_trtllm_openengine() -> None: - """Fail required TRT-LLM CI, or skip optional local runs, if unavailable.""" - if trtllm_serves_openengine(): - return - reason = "the installed TensorRT-LLM runtime has no OpenEngine servicer" - if os.environ.get("DYNAMO_TEST_SIDECAR_BACKEND") == "trtllm": - pytest.fail(reason) - pytest.skip(reason) diff --git a/tests/utils/test_sidecar_runtime.py b/tests/utils/test_sidecar_runtime.py deleted file mode 100644 index 81799cc1f497..000000000000 --- a/tests/utils/test_sidecar_runtime.py +++ /dev/null @@ -1,46 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Required sidecar lanes must exercise their runtime instead of skipping.""" - -from types import SimpleNamespace - -import pytest - -from tests.utils import sidecar_runtime - -pytestmark = [pytest.mark.unit, pytest.mark.pre_merge, pytest.mark.gpu_0] - - -@pytest.mark.parametrize("backend", [None, "trtllm", "vllm"]) -@pytest.mark.parametrize("available", [False, True]) -def test_required_ci_fails_missing_runtime_and_local_runs_skip( - monkeypatch, backend, available -): - monkeypatch.delenv("DYNAMO_TEST_SIDECAR_BACKEND", raising=False) - if backend is not None: - monkeypatch.setenv("DYNAMO_TEST_SIDECAR_BACKEND", backend) - monkeypatch.setattr(sidecar_runtime, "trtllm_serves_openengine", lambda: available) - if available: - sidecar_runtime.require_trtllm_openengine() - else: - error = pytest.fail.Exception if backend == "trtllm" else pytest.skip.Exception - with pytest.raises(error, match="no OpenEngine servicer"): - sidecar_runtime.require_trtllm_openengine() - - -@pytest.mark.parametrize("layout", ["absent", "old", "openengine"]) -def test_probe_requires_servicer_file_without_importing_backend( - monkeypatch, tmp_path, layout -): - spec = ( - None - if layout == "absent" - else SimpleNamespace(submodule_search_locations=[str(tmp_path)]) - ) - if layout == "openengine": - servicer = tmp_path / "grpc/openengine/servicer.py" - servicer.parent.mkdir(parents=True) - servicer.write_text("raise AssertionError('backend must not be imported')\n") - monkeypatch.setattr(sidecar_runtime.importlib.util, "find_spec", lambda name: spec) - assert sidecar_runtime.trtllm_serves_openengine() is (layout == "openengine") From 2609107474fd53de6573adde8cc103333d1d7094 Mon Sep 17 00:00:00 2001 From: krishung5 Date: Wed, 28 Oct 2026 13:23:52 -0700 Subject: [PATCH 47/61] fix(trtllm): resolve sidecar image capabilities during startup Signed-off-by: krishung5 --- lib/sidecar/testkit/tests/support/trtllm.rs | 8 + .../testkit/tests/trtllm_multimodal.rs | 249 +++++++++++------- lib/sidecar/trtllm/src/client.rs | 98 +++---- lib/sidecar/trtllm/src/engine.rs | 202 ++------------ 4 files changed, 240 insertions(+), 317 deletions(-) diff --git a/lib/sidecar/testkit/tests/support/trtllm.rs b/lib/sidecar/testkit/tests/support/trtllm.rs index 76227f35f398..1f78bdb2037f 100644 --- a/lib/sidecar/testkit/tests/support/trtllm.rs +++ b/lib/sidecar/testkit/tests/support/trtllm.rs @@ -43,6 +43,8 @@ pub struct Discovery { pub hold: AtomicBool, pub portable_image: AtomicBool, pub advertises_image: AtomicBool, + pub omit_context_length: AtomicBool, + pub error: Mutex>, release: tokio::sync::Notify, } @@ -190,8 +192,14 @@ impl Control for ControlledControl { if self.discovery.unavailable.load(Ordering::SeqCst) { return Err(Status::unavailable("discovery temporarily unavailable")); } + if let Some(code) = *self.discovery.error.lock().unwrap() { + return Err(Status::new(code, "model discovery rejected")); + } let mut response = self.inner.get_model_info(request).await?; let info = response.get_mut(); + if self.discovery.omit_context_length.load(Ordering::SeqCst) { + info.max_context_length = None; + } info.supports_multimodal = Some(self.discovery.portable_image.load(Ordering::SeqCst)); let modalities = if self.discovery.advertises_image.load(Ordering::SeqCst) { vec!["image"] diff --git a/lib/sidecar/testkit/tests/trtllm_multimodal.rs b/lib/sidecar/testkit/tests/trtllm_multimodal.rs index 067dc304e012..753da8aeffcb 100644 --- a/lib/sidecar/testkit/tests/trtllm_multimodal.rs +++ b/lib/sidecar/testkit/tests/trtllm_multimodal.rs @@ -7,19 +7,19 @@ use std::time::Duration; use dynamo_backend_common::testing::mock_context; use dynamo_backend_common::{ - BackendError, ErrorType, FinishReason, GenerateContext, LLMEngine, MultimodalData, - PreprocessedRequest, + FinishReason, GenerateContext, LLMEngine, MultimodalData, PreprocessedRequest, }; +use dynamo_sidecar_common::EngineBootstrapResult; use dynamo_sidecar_testkit::bounded; use dynamo_sidecar_testkit::control::{Controller, RequestPlan}; use dynamo_sidecar_testkit::fixtures::{collect, request}; +use dynamo_trtllm_sidecar::TrtllmSidecarEngine; use dynamo_trtllm_sidecar::proto as pb; -use futures::future::join_all; #[allow(dead_code)] mod support; use support::trtllm::{Adapter, Fixture}; -use support::{FixtureConfig, SidecarFixture}; +use support::{FixtureConfig, ProcessFixture, SidecarFixture}; fn image_request() -> PreprocessedRequest { let mut request = request("mocker-model", vec![11, 22, 33], 1); @@ -146,110 +146,171 @@ async fn both_image_capability_signals_are_required() { } } -#[tokio::test] -async fn concurrent_discovery_failure_is_shared_and_later_recovers() { - let control = Controller::::default(); - let mut fixture = Fixture::start(control.clone(), FixtureConfig::default()).await; - fixture.discovery.unavailable.store(true, Ordering::SeqCst); - let engine = fixture.engine_with_context_length(32768); - engine.start(0).await.unwrap(); - fixture.discovery.hold.store(true, Ordering::SeqCst); - let requests = (0..4) - .map(|_| engine.generate(image_request(), GenerateContext::new(mock_context(), None))); - let (results, ()) = tokio::join!(join_all(requests), async { - wait_for_calls(&fixture, 2).await; - fixture.release_discovery(); - }); - for result in results { - let error = result.err().expect("discovery must fail for every waiter"); - assert_eq!( - error.error_type(), - ErrorType::Backend(BackendError::CannotConnect) - ); - } - assert_eq!(fixture.discovery.calls.load(Ordering::SeqCst), 2); - // Failures are retained for one second; recovery is not permanently latched. - fixture.discovery.unavailable.store(false, Ordering::SeqCst); - advertise_images(&fixture); - tokio::time::sleep(Duration::from_millis(1100)).await; - for _ in 0..2 { - let ctx = mock_context(); - control.request(ctx.id(), RequestPlan::default()); - fixture.respond(ctx.id(), vec![finished()]); - let outputs = collect(&engine, image_request(), GenerateContext::new(ctx, None)).await; - assert_eq!(outputs.len(), 1); - assert_eq!( - outputs[0].as_ref().unwrap().finish_reason, - Some(FinishReason::Stop) - ); +fn bootstrap( + fixture: &Fixture, + context_length: Option, + deadline_secs: u32, +) -> impl std::future::Future> + use<> { + let mut args = vec![ + "dynamo-trtllm-sidecar".into(), + "--grpc-endpoint".into(), + fixture.endpoint(), + "--model-path".into(), + "mocker-model".into(), + "--grpc-startup-deadline-secs".into(), + deadline_secs.to_string(), + ]; + if let Some(context_length) = context_length { + args.extend(["--context-length".into(), context_length.to_string()]); } - assert_eq!(fixture.discovery.calls.load(Ordering::SeqCst), 3); - engine.cleanup().await.unwrap(); - fixture.shutdown().await; + TrtllmSidecarEngine::try_from_args_async(args).unwrap().1 } #[tokio::test] -async fn stalled_discovery_observes_request_and_shutdown_cancellation() { - for shutdown in [false, true] { +async fn startup_recovers_image_capabilities_before_registration() { + for omit_window in [false, true] { let control = Controller::::default(); - let mut fixture = Fixture::start(control, FixtureConfig::default()).await; + let mut fixture = Fixture::start(control.clone(), FixtureConfig::default()).await; + advertise_images(&fixture); + fixture + .discovery + .omit_context_length + .store(omit_window, Ordering::SeqCst); fixture.discovery.unavailable.store(true, Ordering::SeqCst); - let engine = fixture.engine_with_context_length(32768); - engine.start(0).await.unwrap(); - fixture.discovery.hold.store(true, Ordering::SeqCst); - let ctx = mock_context(); - let cancelling_ctx = ctx.clone(); - let (outputs, ()) = bounded("cancel stalled discovery", async { - tokio::join!( - collect(&engine, image_request(), GenerateContext::new(ctx, None)), - async { - wait_for_calls(&fixture, 2).await; - if shutdown { - engine.cleanup().await.unwrap(); - } else { - cancelling_ctx.stop_generating(); - } - } - ) - }) - .await; - assert_eq!(outputs.len(), 1); - assert_eq!( - outputs[0].as_ref().unwrap().finish_reason, - Some(FinishReason::Cancelled) + let mut startup = tokio::spawn(bootstrap(&fixture, Some(32768), 5)); + wait_for_calls(&fixture, 2).await; + assert!( + !startup.is_finished(), + "unresolved capabilities must prevent registration" ); - fixture.release_discovery(); + fixture.discovery.unavailable.store(false, Ordering::SeqCst); + let (engine, _) = bounded("recover startup discovery", &mut startup) + .await + .unwrap() + .unwrap(); + let config = engine.start(0).await.unwrap(); + assert_eq!(config.llm.unwrap().context_length, Some(32768)); + let calls = fixture.discovery.calls.load(Ordering::SeqCst); + // Serving must use startup metadata, including image support when the + // engine supplied no window. No Generate may rediscover capabilities. + fixture.discovery.unavailable.store(true, Ordering::SeqCst); + for _ in 0..2 { + let ctx = mock_context(); + let handle = control.request(ctx.id(), RequestPlan::default()); + if omit_window { + let error = engine + .generate(image_request(), GenerateContext::new(ctx, None)) + .await + .err() + .expect("media still needs a reported engine window"); + assert!( + error.to_string().contains("--context-length to match"), + "{error}" + ); + assert!(handle.native_request().is_none()); + } else { + fixture.respond(ctx.id(), vec![finished()]); + let outputs = + collect(&engine, image_request(), GenerateContext::new(ctx, None)).await; + assert_eq!( + outputs.last().unwrap().as_ref().unwrap().finish_reason, + Some(FinishReason::Stop) + ); + assert_eq!(handle.native_request().unwrap().media.len(), 2); + } + } + assert_eq!(fixture.discovery.calls.load(Ordering::SeqCst), calls); engine.cleanup().await.unwrap(); fixture.shutdown().await; } } #[tokio::test] -async fn unresponsive_discovery_has_one_bounded_timeout_for_all_waiters() { - let control = Controller::::default(); - let mut fixture = Fixture::start(control, FixtureConfig::default()).await; - fixture.discovery.unavailable.store(true, Ordering::SeqCst); - let engine = fixture.engine_with_context_length(32768); - engine.start(0).await.unwrap(); - fixture.discovery.hold.store(true, Ordering::SeqCst); - let requests = (0..4) - .map(|_| engine.generate(image_request(), GenerateContext::new(mock_context(), None))); - let results = tokio::time::timeout(Duration::from_secs(35), join_all(requests)) - .await - .expect("all four requests must share the 30-second RPC timeout"); - for result in results { - let error = result - .err() - .expect("discovery must time out for every waiter"); - assert_eq!( - error.error_type(), - ErrorType::Backend(BackendError::ConnectionTimeout) - ); +async fn discovery_model_and_authentication_errors_fail_startup_without_retry() { + for configured in [None, Some(32768)] { + for code in [ + tonic::Code::InvalidArgument, + tonic::Code::NotFound, + tonic::Code::Unauthenticated, + tonic::Code::PermissionDenied, + ] { + let mut fixture = + Fixture::start(Controller::::default(), FixtureConfig::default()).await; + *fixture.discovery.error.lock().unwrap() = Some(code); + let result = bounded( + "terminal discovery error", + bootstrap(&fixture, configured, 60), + ) + .await; + let error = result + .err() + .expect("model/authentication errors must prevent registration"); + assert!( + error.to_string().contains("model discovery rejected"), + "{error}" + ); + assert!(error.to_string().contains(&format!("{code:?}")), "{error}"); + assert_eq!(fixture.discovery.calls.load(Ordering::SeqCst), 1); + fixture.shutdown().await; + } + } +} + +#[tokio::test] +async fn startup_deadline_bounds_retrying_and_stalled_discovery() { + for configured in [None, Some(32768)] { + for held in [false, true] { + let mut fixture = + Fixture::start(Controller::::default(), FixtureConfig::default()).await; + fixture.discovery.unavailable.store(true, Ordering::SeqCst); + fixture.discovery.hold.store(held, Ordering::SeqCst); + let result = + tokio::time::timeout(Duration::from_secs(4), bootstrap(&fixture, configured, 2)) + .await + .expect("discovery must respect the startup deadline"); + let error = result + .err() + .expect("unresolved capabilities must prevent registration"); + assert!(error.to_string().contains("startup deadline"), "{error}"); + let calls = fixture.discovery.calls.load(Ordering::SeqCst); + if held { + assert_eq!(calls, 1); + } else { + assert!(calls >= 2); + } + fixture.release_discovery(); + fixture.shutdown().await; + } + } +} + +#[tokio::test] +async fn shutdown_cancels_retrying_and_stalled_bootstrap_discovery() { + for held in [false, true] { + let mut fixture = + Fixture::start(Controller::::default(), FixtureConfig::default()).await; + fixture.discovery.unavailable.store(true, Ordering::SeqCst); + fixture.discovery.hold.store(held, Ordering::SeqCst); + let shutdown = tokio_util::sync::CancellationToken::new(); + let cancel = shutdown.clone(); + let startup = bootstrap(&fixture, Some(32768), 60); + // Mirrors the shared runner's cancellation of the bootstrap future. + let task = tokio::spawn(async move { + tokio::select! { + biased; + _ = shutdown.cancelled() => {}, + _ = startup => panic!("unresolved discovery must not register a worker"), + } + }); + wait_for_calls(&fixture, 1).await; + cancel.cancel(); + bounded("shutdown cancels bootstrap", task).await.unwrap(); + let calls = fixture.discovery.calls.load(Ordering::SeqCst); + fixture.release_discovery(); + tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!(fixture.discovery.calls.load(Ordering::SeqCst), calls); + fixture.shutdown().await; } - assert_eq!(fixture.discovery.calls.load(Ordering::SeqCst), 2); - fixture.release_discovery(); - engine.cleanup().await.unwrap(); - fixture.shutdown().await; } #[tokio::test] diff --git a/lib/sidecar/trtllm/src/client.rs b/lib/sidecar/trtllm/src/client.rs index 204d582090bf..4b0039776bb6 100644 --- a/lib/sidecar/trtllm/src/client.rs +++ b/lib/sidecar/trtllm/src/client.rs @@ -46,15 +46,10 @@ pub(crate) struct ModelLimits { /// alone can imply a larger budget than the engine will accept. pub(crate) max_output_tokens: Option, /// Whether both portable and engine-specific capabilities advertise images. - /// `None` means startup discovery failed transiently and may be retried. + /// `None` is reserved for unresolved/test metadata; startup always resolves this. pub(crate) supports_image: Option, } -pub(crate) enum ModelInfoResponse { - Available(ModelLimits), - Unsupported(tonic::Status), -} - const TARGET_DP_RANK_METADATA_KEY: &str = "openengine-target-dp-rank"; #[derive(Clone)] @@ -215,22 +210,6 @@ impl TrtllmClient { }) } - /// One `GetModelInfo` call, for when `--context-length` already supplies the - /// window and the engine is consulted only to cross-check it and to learn - /// its output cap. - pub(crate) async fn model_limits(&self, model: &str) -> Result { - match tokio::time::timeout(RPC_TIMEOUT, self.get_model_info(model)).await { - Ok(Ok(metadata)) => Ok(ModelInfoResponse::Available(metadata)), - Ok(Err(status)) if is_request_itself_wrong(&status) => { - Ok(ModelInfoResponse::Unsupported(status)) - } - Ok(Err(status)) => Err(status_to_dynamo("GetModelInfo", status)), - Err(_) => Err(connection_timeout(format!( - "GetModelInfo did not respond within {RPC_TIMEOUT:?}" - ))), - } - } - pub(crate) async fn server_info(&self) -> Result, DynamoError> { let response = tokio::time::timeout( RPC_TIMEOUT, @@ -296,37 +275,62 @@ impl TrtllmClient { } } - /// Polls `GetModelInfo` until the engine reports a usable context length or - /// `deadline` passes. - /// - /// Used only when no `--context-length` was supplied, which makes the engine - /// the sole source. TensorRT-LLM binds its gRPC port only once the model has - /// loaded, so waiting is for the engine not being up yet, and only transport - /// failures are worth retrying. An engine that answers has finished loading: - /// if that answer carries no window it was started without `--max_seq_len` - /// and will say the same thing at the deadline, so fail now with the fix - /// rather than polling for half an hour and then blaming a slow load. An - /// answer that says the request itself is wrong ends the wait for the same - /// reason. + /// Resolve capabilities before registration, retrying temporary failures within + /// the shared startup deadline. Only an older engine's UNIMPLEMENTED response + /// permits text-only operation with an operator-supplied context window. pub(crate) async fn wait_for_model_limits( &self, model: &str, + configured_context_length: Option, deadline: Instant, retry_interval: Duration, ) -> Result { let mut last = "it never answered".to_string(); loop { match timeout_at(deadline, self.get_model_info(model)).await { - Ok(Ok(limits)) if limits.context_length.is_some() => return Ok(limits), - Ok(Ok(_)) => { + Ok(Ok(mut limits)) => { + if let Some(configured) = configured_context_length { + if let Some(reported) = limits.context_length + && reported != configured + { + tracing::warn!( + configured_context_length = configured, + engine_context_length = reported, + "--context-length disagrees with the context length TensorRT-LLM \ + reported; using the configured --context-length" + ); + } + limits.context_length = Some(configured); + } + if limits.context_length.is_some() { + return Ok(limits); + } return Err(protocol_error(format!( "TensorRT-LLM answered GetModelInfo for {model} with no \ - max_context_length, which means it was started without \ - --max_seq_len. Start the engine with --max_seq_len, or give the \ + max_context_length. Start the engine with --max_seq_len, or give the \ sidecar --context-length." ))); } - Ok(Err(status)) if is_request_itself_wrong(&status) => { + Ok(Err(status)) if status.code() == tonic::Code::Unimplemented => { + if let Some(configured) = configured_context_length { + tracing::warn!( + message = status.message(), + "Control.GetModelInfo is not implemented; using --context-length \ + with text-only support" + ); + return Ok(ModelLimits { + context_length: Some(configured), + supports_image: Some(false), + ..Default::default() + }); + } + return Err(protocol_error(format!( + "Control.GetModelInfo is not implemented: {}. Supply --context-length \ + for text-only operation with this engine.", + status.message() + ))); + } + Ok(Err(status)) if is_terminal_discovery_error(&status) => { return Err(status_to_dynamo("GetModelInfo", status)); } Ok(Err(status)) => last = format!("{}: {}", status.code(), status.message()), @@ -340,9 +344,8 @@ impl TrtllmClient { sleep_until(next_attempt).await; } Err(connection_timeout(format!( - "TensorRT-LLM never answered GetModelInfo before the gRPC startup deadline \ - ({last}). Raise --grpc-startup-deadline-secs if the engine is still starting, \ - or pin the window with --context-length." + "TensorRT-LLM did not resolve GetModelInfo before the gRPC startup deadline \ + ({last}). Raise --grpc-startup-deadline-secs if the engine is still starting." ))) } @@ -382,13 +385,14 @@ fn metadata_status_to_dynamo(rpc: &str, status: tonic::Status) -> DynamoError { } } -/// Whether the server answered that the request itself is wrong -- a model it -/// does not serve, or no Control service at all. Waiting cannot change any of -/// these, so a startup probe stops rather than burning its whole deadline. -fn is_request_itself_wrong(status: &tonic::Status) -> bool { +/// These model/configuration/authentication errors cannot recover by waiting. +fn is_terminal_discovery_error(status: &tonic::Status) -> bool { matches!( status.code(), - tonic::Code::InvalidArgument | tonic::Code::NotFound | tonic::Code::Unimplemented + tonic::Code::InvalidArgument + | tonic::Code::NotFound + | tonic::Code::Unauthenticated + | tonic::Code::PermissionDenied ) } diff --git a/lib/sidecar/trtllm/src/engine.rs b/lib/sidecar/trtllm/src/engine.rs index 8568c8b62bb8..ed3e0efc2269 100644 --- a/lib/sidecar/trtllm/src/engine.rs +++ b/lib/sidecar/trtllm/src/engine.rs @@ -17,25 +17,18 @@ use dynamo_sidecar_common::{ EngineBootstrapResult, GrpcEndpoint, GrpcTransportConfig, SidecarStartupError, startup_deadline, }; use futures::stream::BoxStream; -use tokio::sync::{Mutex as AsyncMutex, OnceCell}; +use tokio::sync::OnceCell; use tokio::task::JoinHandle; -use tokio::time::{Instant, timeout}; +use tokio::time::timeout; use tokio_util::sync::CancellationToken; use crate::args::Args; -use crate::client::{self, ModelInfoResponse, ModelLimits, TrtllmClient}; +use crate::client::{self, ModelLimits, TrtllmClient}; use crate::convert::{ResponseState, build_generate_request_with_routing}; use crate::model::ConfiguredModel; use crate::proto as pb; const ALREADY_STARTED: &str = "TensorRT-LLM sidecar has already started"; -const DISCOVERY_RETRY_DELAY: Duration = Duration::from_secs(1); - -struct FailedDiscovery { - error: DynamoError, - retry_at: Instant, -} - const LOAD_STALENESS_LIMIT: Duration = Duration::from_secs(30); const LOAD_POLL_INTERVAL: Duration = Duration::from_millis(100); const MAX_INITIAL_LOAD_AGE: Duration = Duration::from_secs(5); @@ -67,8 +60,6 @@ pub struct TrtllmSidecarEngine { /// Connection and limits may be discovered in the cancellable sidecar bootstrap. connection: OnceCell<(TrtllmClient, ModelLimits)>, started: OnceCell<()>, - rediscovered_limits: OnceCell, - discovery_failure: AsyncMutex>, routing: OnceCell, metrics_task: Arc>>>, cancel: CancellationToken, @@ -90,8 +81,6 @@ impl TrtllmSidecarEngine { subagent_affinity_auth_key: None, connection: OnceCell::new(), started: OnceCell::new(), - rediscovered_limits: OnceCell::new(), - discovery_failure: AsyncMutex::new(None), routing: OnceCell::new(), metrics_task: Arc::new(Mutex::new(None)), cancel: CancellationToken::new(), @@ -238,66 +227,6 @@ impl TrtllmSidecarEngine { }; Ok((engine, config)) } - async fn limits_for_request( - &self, - client: &TrtllmClient, - initial: ModelLimits, - request: &PreprocessedRequest, - ) -> Result { - if let Some(limits) = self.rediscovered_limits.get() { - return Ok(*limits); - } - let has_images = request.multi_modal_data.as_ref().is_some_and(|media| { - media - .get("image_url") - .is_some_and(|images| !images.is_empty()) - }); - if !has_images || initial.supports_image.is_some() { - return Ok(initial); - } - - // Serialize attempts and retain failures through the retry delay, so waiters - // share a failed RPC instead of each spending another RPC timeout. - let arrived_at = Instant::now(); - let mut failure = self.discovery_failure.lock().await; - if let Some(failure) = &*failure - && arrived_at < failure.retry_at - { - return Err(failure.error.clone()); - } - let result: Result = self - .rediscovered_limits - .get_or_try_init(|| async { - match client.model_limits(&self.model.source).await? { - ModelInfoResponse::Available(mut reported) => { - if let Some(engine_context_length) = reported.context_length - && Some(engine_context_length) != initial.context_length - { - tracing::warn!( - configured_context_length = ?initial.context_length, - engine_context_length, - "--context-length disagrees with the context length TensorRT-LLM \ - reported; using the configured --context-length" - ); - } - reported.context_length = initial.context_length; - Ok(reported) - } - ModelInfoResponse::Unsupported(_) => Ok(ModelLimits { - supports_image: Some(false), - ..initial - }), - } - }) - .await - .copied(); - *failure = result.as_ref().err().map(|error| FailedDiscovery { - error: error.clone(), - retry_at: Instant::now() + DISCOVERY_RETRY_DELAY, - }); - result - } - pub(crate) async fn bootstrap( (engine, config): (Self, WorkerConfig), ) -> Result<(Self, WorkerConfig), DynamoError> { @@ -330,84 +259,14 @@ impl TrtllmSidecarEngine { )); } - // `--context-length` wins over what the engine reports, and is the - // only source when the engine reports nothing usable. The resolved - // value backs both the registered window and the default-`max_tokens` - // path in `convert::max_tokens`. - let model = &self.model; - let limits = match model.context_length { - // Configured: the engine is consulted once, to cross-check the - // value and to learn its output cap. It may not answer at all -- - // an older server has no Control service -- and that must not stop - // a worker whose window the operator already supplied. - // - // Neither branch waits for the engine to be *ready*, only to answer. - // Readiness belongs to the engine's health service, which is what - // the Kubernetes probes use and what DEP #14897 standardises on - // (`grpc.health.v1` Check/Watch, with registration driven by - // `Watch`). Do not grow a second readiness protocol here. - Some(configured) => { - let reported = match client.model_limits(&model.source).await { - Ok(ModelInfoResponse::Available(reported)) => reported, - Ok(ModelInfoResponse::Unsupported(status)) => { - tracing::warn!( - code = ?status.code(), - message = status.message(), - configured_context_length = configured, - "Control.GetModelInfo is unavailable; using the configured \ - --context-length" - ); - ModelLimits { - supports_image: Some(false), - ..Default::default() - } - } - Err(error) => { - tracing::warn!( - %error, - configured_context_length = configured, - "Control.GetModelInfo failed transiently; using the configured \ - --context-length and deferring media capability discovery" - ); - ModelLimits::default() - } - }; - if let Some(engine_context_length) = reported.context_length - && engine_context_length != configured - { - tracing::warn!( - configured_context_length = configured, - engine_context_length, - "--context-length disagrees with the context length TensorRT-LLM \ - reported; using the configured --context-length" - ); - } - ModelLimits { - context_length: Some(configured), - ..reported - } - } - // Nothing configured: the engine is the only source. It binds its - // gRPC port only after the model has loaded, but the sidecar and the - // engine start independently, so wait for it either way. - None => { - // Failing here beats registering a worker that advertises - // capacity and then rejects, with a non-migratable 4xx, every - // request that omits `max_tokens` -- which is most chat - // traffic. The operator gets one startup error naming the fix - // instead of a worker that looks healthy and serves half. - client - .wait_for_model_limits(&model.source, deadline, self.transport.retry_interval) - .await - .map_err(|error| { - client::invalid_argument(format!( - "no context length is available: {error}. TensorRT-LLM reports one \ - only when it was started with --max_seq_len; otherwise supply \ - --context-length." - )) - })? - } - }; + let limits = client + .wait_for_model_limits( + &self.model.source, + self.model.context_length, + deadline, + self.transport.retry_interval, + ) + .await?; Ok((client, limits)) } } @@ -608,25 +467,6 @@ impl LLMEngine for TrtllmSidecarEngine { let shutdown = self.cancel.child_token(); let mut shutdown_cancellation = Box::pin(async move { shutdown.cancelled().await }); - let limits = tokio::select! { - biased; - _ = &mut request_cancellation, if !needs_cancellation_deferral => None, - _ = &mut shutdown_cancellation => None, - result = self.limits_for_request(client, *limits, &request) => Some(result?), - }; - let Some(limits) = limits else { - let output = cancelled(&state); - return Ok(Box::pin(futures::stream::once(async move { Ok(output) }))); - }; - let proto_request = build_generate_request_with_routing( - request, - &request_id, - &self.model.source, - Some(limits), - self.mode, - use_dp_targeting, - )?; - // The same deferral applies here, not just to the streaming loop below. // `generate` sends the request and then awaits response headers, so // losing this race can drop a request the engine has already accepted @@ -638,11 +478,21 @@ impl LLMEngine for TrtllmSidecarEngine { biased; _ = &mut request_cancellation, if !needs_cancellation_deferral => None, _ = &mut shutdown_cancellation => None, - result = client.generate( - proto_request, - target_dp_rank, - parent_affinity.as_deref().zip(self.subagent_affinity_auth_key.as_deref()), - ) => Some(result?), + result = async { + let proto_request = build_generate_request_with_routing( + request, + &request_id, + &self.model.source, + Some(*limits), + self.mode, + use_dp_targeting, + )?; + client.generate( + proto_request, + target_dp_rank, + parent_affinity.as_deref().zip(self.subagent_affinity_auth_key.as_deref()), + ).await + } => Some(result?), }; let Some(mut stream) = stream else { let output = cancelled(&state); From d9c040cfed9b5bdba44d3611cedea5fdf56c2266 Mon Sep 17 00:00:00 2001 From: krishung5 Date: Wed, 28 Oct 2026 13:24:33 -0700 Subject: [PATCH 48/61] ci(trtllm): test sidecar images against latest published nightly Signed-off-by: krishung5 --- .github/actions/changed-files/action.yml | 3 + .github/filters.yaml | 21 ++ .github/workflows/nightly-ci.yml | 12 +- .github/workflows/pr.yaml | 14 + .../shared-test-trtllm-nightly-image.yml | 97 +++++++ lib/sidecar/Dockerfile | 13 + lib/sidecar/trtllm/ci/build.sh | 24 ++ lib/sidecar/trtllm/ci/nightly.json | 5 + lib/sidecar/trtllm/ci/resolve_nightly.py | 126 +++++++++ lib/sidecar/trtllm/ci/run_image_test.py | 258 ++++++++++++++++++ .../trtllm/ci/tests/test_resolve_nightly.py | 111 ++++++++ lib/sidecar/trtllm/src/tests/e2e.rs | 13 + 12 files changed, 695 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/shared-test-trtllm-nightly-image.yml create mode 100755 lib/sidecar/trtllm/ci/build.sh create mode 100644 lib/sidecar/trtllm/ci/nightly.json create mode 100755 lib/sidecar/trtllm/ci/resolve_nightly.py create mode 100755 lib/sidecar/trtllm/ci/run_image_test.py create mode 100644 lib/sidecar/trtllm/ci/tests/test_resolve_nightly.py diff --git a/.github/actions/changed-files/action.yml b/.github/actions/changed-files/action.yml index 9a91ae7a48d9..3c3b0dc0a19c 100644 --- a/.github/actions/changed-files/action.yml +++ b/.github/actions/changed-files/action.yml @@ -61,6 +61,9 @@ outputs: planner: description: 'Whether planner files changed' value: ${{ steps.filter.outputs.planner_any_modified }} + sidecar_trtllm_image: + description: 'Whether the standalone TRT-LLM image integration test is affected' + value: ${{ steps.filter.outputs.sidecar_trtllm_image_any_modified }} sidecar: description: 'Whether sidecar files changed' value: ${{ steps.filter.outputs.sidecar_any_modified }} diff --git a/.github/filters.yaml b/.github/filters.yaml index e205c56c43ed..e5e69ec561ab 100644 --- a/.github/filters.yaml +++ b/.github/filters.yaml @@ -367,11 +367,32 @@ planner: - '!**/*.md' - '!**/*.rst' +sidecar_trtllm_image: + - 'lib/sidecar/trtllm/**' + - 'lib/sidecar/common/**' + - 'lib/sidecar/Dockerfile' + - 'lib/backend-common/**' + - 'lib/llm/**' + - 'lib/runtime/**' + - 'Cargo.toml' + - 'Cargo.lock' + - 'rust-toolchain.toml' + - '.cargo/**' + - '.github/actions/builder-refresher/**' + - '.github/actions/changed-files/**' + - '.github/filters.yaml' + - '.github/workflows/shared-test-trtllm-nightly-image.yml' + - '.github/workflows/pr.yaml' + - '.github/workflows/nightly-ci.yml' + - '!**/*.md' + - '!**/*.rst' + sidecar: # TODO: Expand this to every Dockerfile and composite-action input after the # acceptable CI fan-out is agreed. Keep the remaining scope narrow for now. - 'lib/sidecar/**' - '.github/workflows/shared-build-sidecar.yml' + - '.github/workflows/shared-test-trtllm-nightly-image.yml' - 'tests/serve/test_sidecar.py' - 'tests/router/common.py' - 'tests/router/helper.py' diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml index 9bb345554369..fffdaa37727b 100644 --- a/.github/workflows/nightly-ci.yml +++ b/.github/workflows/nightly-ci.yml @@ -1032,6 +1032,14 @@ jobs: sidecar_binary_artifact: ${{ needs.sidecar-build.outputs.binary_artifact }} secrets: inherit + sidecar-trtllm-nightly-image-test: + needs: [create-fresh-builder, compute-release-mode] + if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }} + uses: $/.github/workflows/shared-test-trtllm-nightly-image.yml + with: + builder_name: ${{ needs.create-fresh-builder.outputs.builder_name }} + secrets: inherit + sidecar-trtllm-test: name: sidecar-runtime needs: [trtllm-build, sidecar-build, resolve-source-sha, compute-release-mode] @@ -1740,7 +1748,7 @@ jobs: name: Clean K8s builder if exists runs-on: prod-default-small-v2 if: always() - needs: [vllm-build, sglang-build, trtllm-build, dynamo-pipeline, operator-build, planner-build, frontend-build, vllm-efa-build, sglang-efa-build, trtllm-efa-build, sidecar-build, create-fresh-builder] + needs: [vllm-build, sglang-build, trtllm-build, dynamo-pipeline, operator-build, planner-build, frontend-build, vllm-efa-build, sglang-efa-build, trtllm-efa-build, sidecar-build, sidecar-trtllm-nightly-image-test, create-fresh-builder] permissions: contents: read steps: @@ -1762,7 +1770,7 @@ jobs: ############################## SLACK NOTIFICATION ############################## notify-slack: if: always() - needs: [vllm-test, vllm-multi-gpu-test, vllm-4-gpu-test, vllm-h100-test, sglang-test, sglang-multi-gpu-test, sglang-4-gpu-test, sglang-h100-test, trtllm-test, trtllm-multi-gpu-test, trtllm-h100-test, sidecar-build, sidecar-vllm-test, sidecar-sglang-test, sidecar-trtllm-test, deploy-cleanup, deepseek-v4-pro-agg-perf, deepseek-v4-pro-disagg-perf, rust-tests, kimi-k25-agg-perf, kimi-k25-disagg-perf] + needs: [vllm-test, vllm-multi-gpu-test, vllm-4-gpu-test, vllm-h100-test, sglang-test, sglang-multi-gpu-test, sglang-4-gpu-test, sglang-h100-test, trtllm-test, trtllm-multi-gpu-test, trtllm-h100-test, sidecar-build, sidecar-vllm-test, sidecar-sglang-test, sidecar-trtllm-test, sidecar-trtllm-nightly-image-test, deploy-cleanup, deepseek-v4-pro-agg-perf, deepseek-v4-pro-disagg-perf, rust-tests, kimi-k25-agg-perf, kimi-k25-disagg-perf] permissions: contents: read actions: read # grant the reusable notifier read access to list this run's jobs diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 0bc0c96e214a..3eec513b33b8 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -33,6 +33,7 @@ jobs: snapshot_trtllm: ${{ steps.changes.outputs.snapshot_trtllm }} power_agent: ${{ steps.changes.outputs.power_agent }} sidecar: ${{ steps.changes.outputs.sidecar }} + sidecar_trtllm_image: ${{ steps.changes.outputs.sidecar_trtllm_image }} deploy: ${{ steps.changes.outputs.deploy }} dgdr: ${{ steps.changes.outputs.dgdr }} vllm: ${{ steps.changes.outputs.vllm }} @@ -131,6 +132,7 @@ jobs: - helm-chart-tests - power-agent - sidecar-build + - sidecar-trtllm-nightly-image-test - vllm-build - vllm-local-dev-build - sglang-local-dev-build @@ -165,6 +167,9 @@ jobs: - name: Check all dependent jobs run: | echo '${{ toJson(needs) }}' | jq -e 'to_entries | map(.value.result) | all(. as $result | ["success", "skipped"] | any($result == .))' + if [[ '${{ needs.changed-files.outputs.sidecar_trtllm_image }}' == 'true' ]]; then + test '${{ needs.sidecar-trtllm-nightly-image-test.result }}' = 'success' + fi deploy-status-check: runs-on: ubuntu-24.04 @@ -340,6 +345,14 @@ jobs: # Sidecar # ============================================================================ + sidecar-trtllm-nightly-image-test: + needs: changed-files + if: needs.changed-files.outputs.sidecar_trtllm_image == 'true' + uses: $/.github/workflows/shared-test-trtllm-nightly-image.yml + with: + builder_name: ${{ needs.changed-files.outputs.builder_name }} + secrets: inherit + sidecar-build: name: dynamo-sidecar needs: changed-files @@ -2028,6 +2041,7 @@ jobs: - operator - power-agent - sidecar-build + - sidecar-trtllm-nightly-image-test - planner-test - planner-copy-to-acr - frontend-build diff --git a/.github/workflows/shared-test-trtllm-nightly-image.yml b/.github/workflows/shared-test-trtllm-nightly-image.yml new file mode 100644 index 000000000000..a88384aff53f --- /dev/null +++ b/.github/workflows/shared-test-trtllm-nightly-image.yml @@ -0,0 +1,97 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: TRT-LLM Sidecar Upstream Nightly Image + +on: + workflow_call: + inputs: + builder_name: + required: true + type: string + secrets: + HF_TOKEN: + required: false + +permissions: + contents: read + actions: read + +jobs: + build: + name: Build image integration client + runs-on: prod-builder-v3 + timeout-minutes: 60 + outputs: + image: ${{ steps.config.outputs.image }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Test nightly selection + run: python3 -m unittest discover -s lib/sidecar/trtllm/ci/tests -v + - name: Resolve latest upstream nightly + id: config + run: >- + python3 lib/sidecar/trtllm/ci/resolve_nightly.py + --output resolved-nightly.json + --github-output "$GITHUB_OUTPUT" + - name: Refresh BuildKit builder + uses: $/.github/actions/builder-refresher + with: + builder_name: ${{ inputs.builder_name }} + flavor: general + arch: linux/amd64,linux/arm64 + namespace: ${{ vars.BUILDKIT_NAMESPACE }} + - name: Build test executable + shell: bash + run: | + set -euo pipefail + docker buildx build --target sidecar-trtllm-image-tests \ + --platform linux/amd64 -f lib/sidecar/Dockerfile \ + --output type=local,dest=trtllm-image-tests . + cp resolved-nightly.json trtllm-image-tests/nightly.json + - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: trtllm-image-tests-${{ github.sha }} + path: trtllm-image-tests/ + if-no-files-found: error + retention-days: 7 + + test: + name: Upstream TRT-LLM image integration (1 GPU) + needs: build + runs-on: [gpu-1, amd64, v2, us-west-2] + timeout-minutes: 30 + container: + image: ${{ needs.build.outputs.image }} + env: + HOME: /__w/dynamo/dynamo + USER: dynamo + PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION: upb + TRTLLM_NO_USAGE_STATS: '1' + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: trtllm-image-tests-${{ github.sha }} + path: trtllm-image-tests + - name: Install pinned OpenEngine bindings + shell: bash + run: | + set -euo pipefail + source lib/sidecar/trtllm/launch/common.sh + trtllm_ensure_openengine_bindings python3 + - name: Run live image integration test + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + run: >- + python3 lib/sidecar/trtllm/ci/run_image_test.py + --binary trtllm-image-tests/trtllm-image-tests + --config trtllm-image-tests/nightly.json + - name: Upload engine logs and test results + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: trtllm-nightly-image-results-${{ github.sha }} + path: test-results/trtllm-nightly-image/ + if-no-files-found: warn + retention-days: 7 diff --git a/lib/sidecar/Dockerfile b/lib/sidecar/Dockerfile index d20d1396900c..292465d219fb 100644 --- a/lib/sidecar/Dockerfile +++ b/lib/sidecar/Dockerfile @@ -240,6 +240,19 @@ COPY --from=licenses /legal/ / FROM scratch AS sidecar-binaries COPY --from=sidecar-builder /out/bin/ / +# ---- CI-only: live image integration test ----------------------------------- +# The live image test runs inside the upstream engine image, independently of +# Dynamo's shared runtime. Keep test dependencies out of production artifacts. +FROM builder-base AS trtllm-image-test-builder +ARG TARGETARCH +RUN --mount=type=cache,id=dynamo-sidecar-cargo-registry-${TARGETARCH},target=/usr/local/cargo/registry,sharing=locked \ + --mount=type=cache,id=dynamo-sidecar-cargo-git-${TARGETARCH},target=/usr/local/cargo/git,sharing=locked \ + --mount=type=cache,id=dynamo-sidecar-image-tests-${TARGETARCH},target=/src/target,sharing=locked \ + bash lib/sidecar/trtllm/ci/build.sh + +FROM scratch AS sidecar-trtllm-image-tests +COPY --from=trtllm-image-test-builder /out/tests/ / + # ---- shipped runtime --------------------------------------------------------- FROM runtime-base AS runtime diff --git a/lib/sidecar/trtllm/ci/build.sh b/lib/sidecar/trtllm/ci/build.sh new file mode 100755 index 000000000000..9c5aadf80421 --- /dev/null +++ b/lib/sidecar/trtllm/ci/build.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +set -euo pipefail + +CARGO_PROFILE_DEV_DEBUG=0 CARGO_INCREMENTAL=0 \ + cargo test --locked -p dynamo-trtllm-sidecar --lib --no-run \ + --message-format=json > /tmp/trtllm-test-artifacts.json +python3 - <<'PY' +import json +import pathlib +import shutil + +artifacts = [json.loads(line) for line in pathlib.Path('/tmp/trtllm-test-artifacts.json').read_text().splitlines()] +executables = [item['executable'] for item in artifacts + if item.get('reason') == 'compiler-artifact' + and item.get('profile', {}).get('test') + and item.get('target', {}).get('name') == 'dynamo_trtllm_sidecar' + and item.get('executable')] +assert len(executables) == 1, executables +pathlib.Path('/out/tests').mkdir(parents=True, exist_ok=True) +shutil.copy2(executables[0], '/out/tests/trtllm-image-tests') +PY +strip /out/tests/trtllm-image-tests diff --git a/lib/sidecar/trtllm/ci/nightly.json b/lib/sidecar/trtllm/ci/nightly.json new file mode 100644 index 000000000000..67b4986ed17c --- /dev/null +++ b/lib/sidecar/trtllm/ci/nightly.json @@ -0,0 +1,5 @@ +{ + "repository": "nvcr.io/nvidia/tensorrt-llm/release", + "model": "Qwen/Qwen3-VL-2B-Instruct", + "model_revision": "89644892e4d85e24eaac8bacfd4f463576704203" +} diff --git a/lib/sidecar/trtllm/ci/resolve_nightly.py b/lib/sidecar/trtllm/ci/resolve_nightly.py new file mode 100755 index 000000000000..daecd9428e55 --- /dev/null +++ b/lib/sidecar/trtllm/ci/resolve_nightly.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Resolve the latest published regular TRT-LLM nightly to an immutable image.""" + +import argparse +import hashlib +import json +import re +import urllib.error +import urllib.parse +import urllib.request +from pathlib import Path + +REPOSITORY = "nvcr.io/nvidia/tensorrt-llm/release" +REGISTRY_PATH = "https://nvcr.io/v2/nvidia/tensorrt-llm/release" +NIGHTLY_TAG = re.compile(r"\d+\.\d+\.\d+(?:(?:a|b|rc)\d+)?\.dev(\d{12})") +MANIFEST_TYPES = ", ".join( + [ + "application/vnd.oci.image.index.v1+json", + "application/vnd.oci.image.manifest.v1+json", + "application/vnd.docker.distribution.manifest.list.v2+json", + "application/vnd.docker.distribution.manifest.v2+json", + ] +) + + +def latest_tag(tags): + """Select by publication timestamp, excluding feature and release aliases.""" + regular = [tag for tag in tags if NIGHTLY_TAG.fullmatch(tag)] + if not regular: + raise ValueError("No regular TRT-LLM nightly tags were published") + return max(regular, key=lambda tag: (NIGHTLY_TAG.fullmatch(tag)[1], tag)) + + +class Registry: + def __init__(self): + self.token = None + + def get(self, path): + headers = {"Accept": MANIFEST_TYPES} + if self.token: + headers["Authorization"] = f"Bearer {self.token}" + request = urllib.request.Request(f"{REGISTRY_PATH}/{path}", headers=headers) + try: + response = urllib.request.urlopen(request, timeout=30) + except urllib.error.HTTPError as error: + if error.code != 401 or self.token: + raise + challenge = dict( + re.findall(r'(\w+)="([^"]+)"', error.headers["WWW-Authenticate"]) + ) + realm = challenge.pop("realm") + if urllib.parse.urlparse(realm).netloc != "nvcr.io" or not realm.startswith( + "https://" + ): + raise ValueError("Unexpected NGC authentication endpoint") from error + with urllib.request.urlopen( + realm + "?" + urllib.parse.urlencode(challenge), timeout=30 + ) as auth_response: + auth = json.load(auth_response) + self.token = auth.get("token") or auth["access_token"] + return self.get(path) + with response: + return response.read(), response.headers + + def manifest(self, reference): + data, headers = self.get(f"manifests/{reference}") + digest = "sha256:" + hashlib.sha256(data).hexdigest() + if headers.get("Docker-Content-Digest", digest) != digest: + raise ValueError("Registry manifest digest does not match its content") + if reference.startswith("sha256:") and reference != digest: + raise ValueError("Registry returned a different manifest than requested") + return json.loads(data), digest + + +def resolve(config, registry): + if config["repository"] != REPOSITORY: + raise ValueError("Expected the official TRT-LLM release repository") + data, _ = registry.get("tags/list") + tag = latest_tag(json.loads(data)["tags"]) + manifest, digest = registry.manifest(tag) + if "manifests" in manifest: + amd64 = [ + item + for item in manifest["manifests"] + if item.get("platform", {}).get("os") == "linux" + and item.get("platform", {}).get("architecture") == "amd64" + ] + if len(amd64) != 1: + raise ValueError("Expected exactly one linux/amd64 nightly image") + manifest, _ = registry.manifest(amd64[0]["digest"]) + data, _ = registry.get(f"blobs/{manifest['config']['digest']}") + environment = dict( + entry.split("=", 1) for entry in json.loads(data)["config"]["Env"] + ) + commit = environment.get("TRT_LLM_GIT_COMMIT", "") + if not re.fullmatch(r"[0-9a-f]{40}", commit): + raise ValueError("Nightly image does not identify its TRT-LLM source commit") + return { + **config, + "image": f"{REPOSITORY}:{tag}@{digest}", + "upstream_commit": commit, + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--config", type=Path, default=Path(__file__).with_name("nightly.json") + ) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--github-output", type=Path) + args = parser.parse_args() + resolved = resolve(json.loads(args.config.read_text()), Registry()) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(resolved, indent=2) + "\n") + if args.github_output: + with args.github_output.open("a") as output: + output.write(f"image={resolved['image']}\n") + print(json.dumps(resolved, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/lib/sidecar/trtllm/ci/run_image_test.py b/lib/sidecar/trtllm/ci/run_image_test.py new file mode 100755 index 000000000000..b8c124b3829c --- /dev/null +++ b/lib/sidecar/trtllm/ci/run_image_test.py @@ -0,0 +1,258 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run the Rust adapter against an unmodified upstream TRT-LLM image.""" + +import argparse +import importlib.metadata +import json +import os +import signal +import socket +import subprocess +import sys +import time +import traceback +import xml.etree.ElementTree as ET +from pathlib import Path + +TEST = "tests::e2e::e2e_real_openengine_image" + + +def wait_for_engine(process, port, timeout=600): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if process.poll() is not None: + raise RuntimeError( + f"Engine exited with {process.returncode}; see engine.log" + ) + try: + with socket.create_connection(("127.0.0.1", port), timeout=1): + return + except OSError: + time.sleep(1) + raise TimeoutError("Engine did not become ready within 600 seconds") + + +def stop_engine(process): + # The serving CLI may spawn MPI workers; reap the entire process group. + if process is not None: + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + return + try: + process.wait(timeout=30) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.wait(timeout=10) + + +def run(args, results): + from huggingface_hub import snapshot_download + from transformers import AutoTokenizer + + config = json.loads(args.config.read_text()) + identity = { + **config, + "installed_version": importlib.metadata.version("tensorrt_llm"), + "installed_commit": os.environ.get("TRT_LLM_GIT_COMMIT"), + } + (results / "runtime.json").write_text(json.dumps(identity, indent=2) + "\n") + if identity["installed_commit"] != config["upstream_commit"]: + raise RuntimeError( + f"Unexpected upstream commit: {identity['installed_commit']}" + ) + + binary = args.binary.resolve() + binary.chmod(binary.stat().st_mode | 0o111) + listing = subprocess.check_output( + [str(binary), "--list", "--exact", TEST, "--format", "terse"], text=True + ) + if listing.splitlines().count(f"{TEST}: test") != 1: + raise RuntimeError( + f"Expected exactly one image integration test, got: {listing}" + ) + + model_path = args.model_path or snapshot_download( + config["model"], revision=config["model_revision"] + ) + tokenizer = AutoTokenizer.from_pretrained(model_path, local_files_only=True) + # The fixed Rust fixture uses Qwen3-VL's rendering of this exact message. + # Check it against the pinned tokenizer so a template change fails loudly. + token_ids = tokenizer.apply_chat_template( + [ + { + "role": "user", + "content": [ + {"type": "image"}, + {"type": "text", "text": "Name the colors from left to right."}, + ], + } + ], + tokenize=True, + add_generation_prompt=True, + ) + expected_ids = [ + 151644, + 872, + 198, + 151652, + 151655, + 151653, + 675, + 279, + 7987, + 504, + 2115, + 311, + 1290, + 13, + 151645, + 198, + 151644, + 77091, + 198, + ] + token_ids = ( + token_ids["input_ids"] + if isinstance(token_ids, dict) or hasattr(token_ids, "input_ids") + else token_ids + ) + if token_ids != expected_ids: + raise RuntimeError(f"Image fixture tokenizer mismatch: {token_ids}") + + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + port = listener.getsockname()[1] + options = results / "engine.yaml" + options.write_text( + "max_batch_size: 1\nmax_num_tokens: 2048\n" + "kv_cache_config:\n max_tokens: 2048\n" + " free_gpu_memory_fraction: 0.15\n" + ) + result_path = results / "response.json" + engine = None + try: + with (results / "engine.log").open("w") as log: + engine = subprocess.Popen( + [ + sys.executable, + "-m", + "tensorrt_llm.commands.serve", + str(model_path), + "--grpc", + "--grpc-protocol", + "openengine", + "--host", + "127.0.0.1", + "--port", + str(port), + "--max_seq_len", + "2048", + "--extra_llm_api_options", + str(options), + ], + stdout=log, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + wait_for_engine(engine, port) + env = dict( + os.environ, + TRTLLM_E2E_ENDPOINT=f"http://127.0.0.1:{port}", + TRTLLM_E2E_MODEL=config["model"], + TRTLLM_E2E_RESULT=str(result_path), + ) + completed = subprocess.run( + [ + str(binary), + "--exact", + TEST, + "--ignored", + "--nocapture", + "--color", + "never", + ], + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + timeout=240, + ) + (results / "rust-test.log").write_text(completed.stdout) + print(completed.stdout, flush=True) + completed.check_returncode() + if "1 passed; 0 failed; 0 ignored;" not in completed.stdout: + raise RuntimeError( + "Image integration test did not execute exactly once" + ) + finally: + stop_engine(engine) + + response = json.loads(result_path.read_text()) + caption = tokenizer.decode(response["token_ids"], skip_special_tokens=True) + response["text"] = caption + result_path.write_text(json.dumps(response, indent=2) + "\n") + print(f"Image response: {caption}", flush=True) + words = caption.lower().replace("-", " ").split() + # Accept punctuation and phrasing differences, but require both fixture colors. + colors = {word.strip(".,:;!?()\"'") for word in words} + if not {"red", "blue"} <= colors: + raise AssertionError( + f"Expected red and blue in the image description, got: {caption}" + ) + if response["completion_tokens"] <= 0 or response["prompt_tokens"] <= len( + expected_ids + ): + raise AssertionError(f"Missing authoritative image token usage: {response}") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--binary", type=Path, required=True) + parser.add_argument( + "--config", type=Path, required=True, help="Resolved nightly image manifest" + ) + parser.add_argument( + "--results", type=Path, default=Path("test-results/trtllm-nightly-image") + ) + parser.add_argument( + "--model-path", type=Path, help="Use an already downloaded model snapshot" + ) + args = parser.parse_args() + results = args.results.resolve() + results.mkdir(parents=True, exist_ok=True) + start = time.monotonic() + failure = None + try: + run(args, results) + except Exception: + failure = traceback.format_exc() + print(failure, file=sys.stderr) + duration = str(time.monotonic() - start) + suite = ET.Element( + "testsuite", + name="trtllm-nightly-image", + tests="1", + failures=str(int(failure is not None)), + skipped="0", + time=duration, + ) + case = ET.SubElement( + suite, "testcase", name=TEST, classname="trtllm.sidecar", time=duration + ) + if failure: + ET.SubElement(case, "failure", message=failure.splitlines()[-1]).text = failure + rust_log = results / "rust-test.log" + if rust_log.exists(): + ET.SubElement(case, "system-out").text = rust_log.read_text() + ET.ElementTree(suite).write( + results / "junit.xml", encoding="utf-8", xml_declaration=True + ) + return int(failure is not None) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/lib/sidecar/trtllm/ci/tests/test_resolve_nightly.py b/lib/sidecar/trtllm/ci/tests/test_resolve_nightly.py new file mode 100644 index 000000000000..72b028623979 --- /dev/null +++ b/lib/sidecar/trtllm/ci/tests/test_resolve_nightly.py @@ -0,0 +1,111 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import importlib.util +import json +import unittest +from pathlib import Path +from unittest.mock import Mock + +SPEC = importlib.util.spec_from_file_location( + "resolve_nightly", Path(__file__).resolve().parents[1] / "resolve_nightly.py" +) +resolver = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(resolver) + + +class NightlySelectionTests(unittest.TestCase): + def test_latest_means_timestamp_not_version_or_feature_branch(self): + self.assertEqual( + resolver.latest_tag( + [ + "1.9.0rc1.dev202610260007", + "1.4.0rc2.dev202610270008", + "feat-qwen3.5-opt-1.4.0a0.dev202610281318", + "1.4.0rc3", + "latest", + "nightly", + ] + ), + "1.4.0rc2.dev202610270008", + ) + + def test_no_regular_nightly_fails(self): + with self.assertRaisesRegex(ValueError, "No regular"): + resolver.latest_tag(["nightly", "feat-test-1.4.0a0.dev202610281318"]) + + def test_selection_records_exact_digest_and_source_from_image(self): + registry = Mock() + digest = "sha256:" + "a" * 64 + registry.get.side_effect = [ + (json.dumps({"tags": ["1.4.0rc2.dev202610270008"]}).encode(), {}), + ( + json.dumps( + {"config": {"Env": ["TRT_LLM_GIT_COMMIT=" + "b" * 40]}} + ).encode(), + {}, + ), + ] + registry.manifest.return_value = ( + {"config": {"digest": "sha256:config"}}, + digest, + ) + resolved = resolver.resolve({"repository": resolver.REPOSITORY}, registry) + self.assertEqual( + resolved["image"], + f"{resolver.REPOSITORY}:1.4.0rc2.dev202610270008@{digest}", + ) + self.assertEqual(resolved["upstream_commit"], "b" * 40) + registry.get.assert_called_with("blobs/sha256:config") + + def test_multiarch_pin_retains_index_digest_and_reads_amd64_metadata(self): + registry = Mock() + registry.get.side_effect = [ + (json.dumps({"tags": ["1.4.0rc2.dev202610270008"]}).encode(), {}), + ( + json.dumps( + {"config": {"Env": ["TRT_LLM_GIT_COMMIT=" + "c" * 40]}} + ).encode(), + {}, + ), + ] + registry.manifest.side_effect = [ + ( + { + "manifests": [ + { + "platform": {"os": "linux", "architecture": "arm64"}, + "digest": "sha256:arm", + }, + { + "platform": {"os": "linux", "architecture": "amd64"}, + "digest": "sha256:amd", + }, + ] + }, + "sha256:index", + ), + ({"config": {"digest": "sha256:config"}}, "sha256:amd"), + ] + resolved = resolver.resolve({"repository": resolver.REPOSITORY}, registry) + self.assertTrue(resolved["image"].endswith("@sha256:index")) + self.assertEqual(resolved["upstream_commit"], "c" * 40) + registry.manifest.assert_called_with("sha256:amd") + + def test_registry_failure_does_not_select_an_older_image(self): + registry = Mock() + registry.get.side_effect = OSError("registry unavailable") + with self.assertRaisesRegex(OSError, "registry unavailable"): + resolver.resolve({"repository": resolver.REPOSITORY}, registry) + + def test_manifest_digest_mismatch_fails(self): + registry = resolver.Registry() + registry.get = Mock( + return_value=(b"{}", {"Docker-Content-Digest": "sha256:wrong"}) + ) + with self.assertRaisesRegex(ValueError, "digest"): + registry.manifest("1.4.0rc2.dev202610270008") + + +if __name__ == "__main__": + unittest.main() diff --git a/lib/sidecar/trtllm/src/tests/e2e.rs b/lib/sidecar/trtllm/src/tests/e2e.rs index 524afcae7132..58522646f8fd 100644 --- a/lib/sidecar/trtllm/src/tests/e2e.rs +++ b/lib/sidecar/trtllm/src/tests/e2e.rs @@ -400,6 +400,19 @@ async fn e2e_real_openengine_image() { usage.prompt_tokens, usage.completion_tokens ); + if let Ok(path) = std::env::var("TRTLLM_E2E_RESULT") { + std::fs::write( + path, + serde_json::to_vec(&json!({ + "token_ids": generated, + "prompt_tokens": usage.prompt_tokens, + "completion_tokens": usage.completion_tokens, + })) + .expect("serialize image result"), + ) + .expect("write image result"); + } + assert!(!generated.is_empty(), "expected image-conditioned output"); assert!( usage.prompt_tokens > token_ids.len() as u32, From e702d2eda2ab81d9c480f9751f2a40ef4809fae3 Mon Sep 17 00:00:00 2001 From: krishung5 Date: Wed, 28 Oct 2026 19:05:42 -0700 Subject: [PATCH 49/61] ci(trtllm): keep nightly image GPU test out of PR checks Signed-off-by: krishung5 --- .github/workflows/pr.yaml | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 3eec513b33b8..d37eb69ae883 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -132,7 +132,7 @@ jobs: - helm-chart-tests - power-agent - sidecar-build - - sidecar-trtllm-nightly-image-test + - sidecar-trtllm-nightly-config-test - vllm-build - vllm-local-dev-build - sglang-local-dev-build @@ -168,7 +168,7 @@ jobs: run: | echo '${{ toJson(needs) }}' | jq -e 'to_entries | map(.value.result) | all(. as $result | ["success", "skipped"] | any($result == .))' if [[ '${{ needs.changed-files.outputs.sidecar_trtllm_image }}' == 'true' ]]; then - test '${{ needs.sidecar-trtllm-nightly-image-test.result }}' = 'success' + test '${{ needs.sidecar-trtllm-nightly-config-test.result }}' = 'success' fi deploy-status-check: @@ -345,13 +345,18 @@ jobs: # Sidecar # ============================================================================ - sidecar-trtllm-nightly-image-test: + sidecar-trtllm-nightly-config-test: + name: TRT-LLM nightly configuration (CPU) needs: changed-files if: needs.changed-files.outputs.sidecar_trtllm_image == 'true' - uses: $/.github/workflows/shared-test-trtllm-nightly-image.yml - with: - builder_name: ${{ needs.changed-files.outputs.builder_name }} - secrets: inherit + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + contents: read + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Test nightly selection + run: python3 -m unittest discover -s lib/sidecar/trtllm/ci/tests -v sidecar-build: name: dynamo-sidecar @@ -2041,7 +2046,6 @@ jobs: - operator - power-agent - sidecar-build - - sidecar-trtllm-nightly-image-test - planner-test - planner-copy-to-acr - frontend-build From f9324e009fbcb76de1a3396369fe89a8d9dada90 Mon Sep 17 00:00:00 2001 From: krishung5 Date: Thu, 29 Oct 2026 14:25:59 -0700 Subject: [PATCH 50/61] ci(trtllm): allow MPI workers in upstream nightly image test Signed-off-by: krishung5 --- .github/workflows/shared-test-trtllm-nightly-image.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/shared-test-trtllm-nightly-image.yml b/.github/workflows/shared-test-trtllm-nightly-image.yml index a88384aff53f..dc8f9d2cb005 100644 --- a/.github/workflows/shared-test-trtllm-nightly-image.yml +++ b/.github/workflows/shared-test-trtllm-nightly-image.yml @@ -66,6 +66,8 @@ jobs: env: HOME: /__w/dynamo/dynamo USER: dynamo + PRTE_ALLOW_RUN_AS_ROOT: '1' + PRTE_ALLOW_RUN_AS_ROOT_CONFIRM: '1' PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION: upb TRTLLM_NO_USAGE_STATS: '1' steps: From c2d5a21c712036104877ae14202101f74ff2ae92 Mon Sep 17 00:00:00 2001 From: krishung5 Date: Thu, 29 Oct 2026 14:59:58 -0700 Subject: [PATCH 51/61] ci(trtllm): shorten MPI hostname for upstream nightly test Signed-off-by: krishung5 --- .github/workflows/shared-test-trtllm-nightly-image.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/shared-test-trtllm-nightly-image.yml b/.github/workflows/shared-test-trtllm-nightly-image.yml index dc8f9d2cb005..2461a810f680 100644 --- a/.github/workflows/shared-test-trtllm-nightly-image.yml +++ b/.github/workflows/shared-test-trtllm-nightly-image.yml @@ -68,6 +68,8 @@ jobs: USER: dynamo PRTE_ALLOW_RUN_AS_ROOT: '1' PRTE_ALLOW_RUN_AS_ROOT_CONFIRM: '1' + # Open MPI 5 singleton worker spawning fails with long K8s hostnames. + PMIX_HOSTNAME: mpi-node0 PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION: upb TRTLLM_NO_USAGE_STATS: '1' steps: From 943e5962b0c06c46c83200fb0e16a9076a759fe1 Mon Sep 17 00:00:00 2001 From: krishung5 Date: Thu, 29 Oct 2026 17:14:07 -0700 Subject: [PATCH 52/61] fix(trtllm): address nightly image CI review findings Signed-off-by: krishung5 --- .github/filters.yaml | 15 +----- .github/scripts/test-filters.js | 19 ++++++- lib/sidecar/Dockerfile | 1 - lib/sidecar/trtllm/README.md | 10 ++-- lib/sidecar/trtllm/ci/build.sh | 5 ++ lib/sidecar/trtllm/ci/resolve_nightly.py | 2 +- lib/sidecar/trtllm/ci/run_image_test.py | 8 +-- .../trtllm/ci/tests/test_resolve_nightly.py | 50 +++++++++++++++++++ lib/sidecar/trtllm/deploy/agg.yaml | 2 +- lib/sidecar/trtllm/deploy/agg_kv_router.yaml | 2 +- lib/sidecar/trtllm/deploy/disagg.yaml | 4 +- .../trtllm/deploy/disagg_kv_router.yaml | 4 +- lib/sidecar/trtllm/launch/agg.sh | 6 +-- lib/sidecar/trtllm/launch/agg_kv_router.sh | 2 + lib/sidecar/trtllm/launch/disagg.sh | 6 +-- lib/sidecar/trtllm/launch/disagg_kv_router.sh | 2 + 16 files changed, 99 insertions(+), 39 deletions(-) diff --git a/.github/filters.yaml b/.github/filters.yaml index b322b9d32ac1..1d0ee6169250 100644 --- a/.github/filters.yaml +++ b/.github/filters.yaml @@ -377,22 +377,11 @@ planner: - '!**/*.rst' sidecar_trtllm_image: - - 'lib/sidecar/trtllm/**' - - 'lib/sidecar/common/**' - - 'lib/sidecar/Dockerfile' - - 'lib/backend-common/**' - - 'lib/llm/**' - - 'lib/runtime/**' - - 'Cargo.toml' - - 'Cargo.lock' - - 'rust-toolchain.toml' - - '.cargo/**' - - '.github/actions/builder-refresher/**' + # PR CI runs resolver CPU tests; the GPU test runs independently in nightly CI. + - 'lib/sidecar/trtllm/ci/**' - '.github/actions/changed-files/**' - '.github/filters.yaml' - - '.github/workflows/shared-test-trtllm-nightly-image.yml' - '.github/workflows/pr.yaml' - - '.github/workflows/nightly-ci.yml' - '!**/*.md' - '!**/*.rst' diff --git a/.github/scripts/test-filters.js b/.github/scripts/test-filters.js index 7a8b5654ff39..cf7473ffadfe 100755 --- a/.github/scripts/test-filters.js +++ b/.github/scripts/test-filters.js @@ -60,6 +60,21 @@ function checkFilter(file, patterns) { // Test cases: [file, expectations, description] // expectations: { filterName: expectedValue, ... } const testCases = [ + { + file: 'lib/sidecar/trtllm/ci/resolve_nightly.py', + expect: { sidecar_trtllm_image: true }, + desc: 'nightly resolver changes run its CPU tests' + }, + { + file: '.github/workflows/pr.yaml', + expect: { sidecar_trtllm_image: true }, + desc: 'PR workflow wiring runs the nightly resolver CPU tests' + }, + { + file: 'lib/llm/src/lib.rs', + expect: { sidecar_trtllm_image: false }, + desc: 'unrelated Rust changes do not run resolver CPU tests' + }, { file: 'scripts/protocol_compatibility/composition/openai.py', expect: { ignore: true, rust: false, core: false, frontend: false }, @@ -203,7 +218,7 @@ const testCases = [ }, { file: 'lib/sidecar/trtllm/src/lib.rs', - expect: { sidecar: true, rust: true, core: false, frontend: false, vllm: false, sglang: false, trtllm: false }, + expect: { sidecar: true, sidecar_trtllm_image: false, rust: true, core: false, frontend: false, vllm: false, sglang: false, trtllm: false }, desc: 'trtllm sidecar source does not route to sglang or trtllm E2E' }, { @@ -233,7 +248,7 @@ const testCases = [ }, { file: 'Cargo.lock', - expect: { sidecar: true, rust: true }, + expect: { sidecar: true, sidecar_trtllm_image: false, rust: true }, desc: 'root lockfile is a sidecar image build input' }, diff --git a/lib/sidecar/Dockerfile b/lib/sidecar/Dockerfile index 5486dd0bd063..124fb06f3f5e 100644 --- a/lib/sidecar/Dockerfile +++ b/lib/sidecar/Dockerfile @@ -253,7 +253,6 @@ FROM builder-base AS trtllm-image-test-builder ARG TARGETARCH RUN --mount=type=cache,id=dynamo-sidecar-cargo-registry-${TARGETARCH},target=/usr/local/cargo/registry,sharing=locked \ --mount=type=cache,id=dynamo-sidecar-cargo-git-${TARGETARCH},target=/usr/local/cargo/git,sharing=locked \ - --mount=type=cache,id=dynamo-sidecar-image-tests-${TARGETARCH},target=/src/target,sharing=locked \ bash lib/sidecar/trtllm/ci/build.sh FROM scratch AS sidecar-trtllm-image-tests diff --git a/lib/sidecar/trtllm/README.md b/lib/sidecar/trtllm/README.md index f165d1ba1699..9e1287e3e72f 100644 --- a/lib/sidecar/trtllm/README.md +++ b/lib/sidecar/trtllm/README.md @@ -28,9 +28,11 @@ supported, even when `--context-length` is supplied. > which carries the latest gRPC server updates. Nightly builds are the > `nvcr.io/nvidia/tensorrt-llm/release:.dev` tags on > [NGC](https://catalog.ngc.nvidia.com/orgs/nvidia/tensorrt-llm/containers/release/-). -> KV routing is available in -> `nvcr.io/nvidia/tensorrt-llm/release:1.4.0a0.dev202610180820`, which includes -> native KVCM2 event streaming and OpenEngine routing load support. +> The standalone launch and deployment examples use +> `nvcr.io/nvidia/tensorrt-llm/release:1.4.0rc3.dev202610290007`, which includes +> image URL passthrough, native KVCM2 event streaming, and OpenEngine routing load support. +> The nightly CI job tests image passthrough against the latest regular upstream nightly; +> it also supports manual dispatch. The examples below stay pinned to the validated version. ## Support matrix @@ -79,7 +81,7 @@ kubectl apply -f lib/sidecar/trtllm/deploy/agg.yaml -n kubectl port-forward -n svc/trtllm-sidecar-agg-frontend 8000:8000 ``` -For KV routing, use `deploy/agg_kv_router.yaml` or `deploy/disagg_kv_router.yaml`. These examples pin matching Dynamo frontend and sidecar nightlies (`20261023-75d0686`) and `nvcr.io/nvidia/tensorrt-llm/release:1.4.0a0.dev202610180820` for the engine. +For KV routing, use `deploy/agg_kv_router.yaml` or `deploy/disagg_kv_router.yaml`. These examples pin matching Dynamo frontend and sidecar nightlies (`20261023-75d0686`) and `nvcr.io/nvidia/tensorrt-llm/release:1.4.0rc3.dev202610290007` for the engine. ## Topologies diff --git a/lib/sidecar/trtllm/ci/build.sh b/lib/sidecar/trtllm/ci/build.sh index 9c5aadf80421..0cc10944dc02 100755 --- a/lib/sidecar/trtllm/ci/build.sh +++ b/lib/sidecar/trtllm/ci/build.sh @@ -3,6 +3,11 @@ # SPDX-License-Identifier: Apache-2.0 set -euo pipefail +# Never reuse compiled artifacts across checkouts on shared builders. +CARGO_TARGET_DIR="$(mktemp -d)" +export CARGO_TARGET_DIR +trap 'rm -rf "$CARGO_TARGET_DIR"' EXIT + CARGO_PROFILE_DEV_DEBUG=0 CARGO_INCREMENTAL=0 \ cargo test --locked -p dynamo-trtllm-sidecar --lib --no-run \ --message-format=json > /tmp/trtllm-test-artifacts.json diff --git a/lib/sidecar/trtllm/ci/resolve_nightly.py b/lib/sidecar/trtllm/ci/resolve_nightly.py index daecd9428e55..8d58a05a1ae9 100755 --- a/lib/sidecar/trtllm/ci/resolve_nightly.py +++ b/lib/sidecar/trtllm/ci/resolve_nightly.py @@ -15,7 +15,7 @@ REPOSITORY = "nvcr.io/nvidia/tensorrt-llm/release" REGISTRY_PATH = "https://nvcr.io/v2/nvidia/tensorrt-llm/release" -NIGHTLY_TAG = re.compile(r"\d+\.\d+\.\d+(?:(?:a|b|rc)\d+)?\.dev(\d{12})") +NIGHTLY_TAG = re.compile(r"\d+\.\d+\.\d+(?:rc\d+)?\.dev(\d{12})") MANIFEST_TYPES = ", ".join( [ "application/vnd.oci.image.index.v1+json", diff --git a/lib/sidecar/trtllm/ci/run_image_test.py b/lib/sidecar/trtllm/ci/run_image_test.py index b8c124b3829c..44bf35154878 100755 --- a/lib/sidecar/trtllm/ci/run_image_test.py +++ b/lib/sidecar/trtllm/ci/run_image_test.py @@ -197,11 +197,11 @@ def run(args, results): result_path.write_text(json.dumps(response, indent=2) + "\n") print(f"Image response: {caption}", flush=True) words = caption.lower().replace("-", " ").split() - # Accept punctuation and phrasing differences, but require both fixture colors. - colors = {word.strip(".,:;!?()\"'") for word in words} - if not {"red", "blue"} <= colors: + # Accept phrasing differences, but require the fixture colors left to right. + colors = [word.strip(".,:;!?()\"'") for word in words] + if [color for color in colors if color in ("red", "blue")][:2] != ["red", "blue"]: raise AssertionError( - f"Expected red and blue in the image description, got: {caption}" + f"Expected red, then blue, in the image description, got: {caption}" ) if response["completion_tokens"] <= 0 or response["prompt_tokens"] <= len( expected_ids diff --git a/lib/sidecar/trtllm/ci/tests/test_resolve_nightly.py b/lib/sidecar/trtllm/ci/tests/test_resolve_nightly.py index 72b028623979..dc30850075e1 100644 --- a/lib/sidecar/trtllm/ci/tests/test_resolve_nightly.py +++ b/lib/sidecar/trtllm/ci/tests/test_resolve_nightly.py @@ -22,6 +22,8 @@ def test_latest_means_timestamp_not_version_or_feature_branch(self): "1.9.0rc1.dev202610260007", "1.4.0rc2.dev202610270008", "feat-qwen3.5-opt-1.4.0a0.dev202610281318", + "1.3.0a0.dev202610280406", + "1.4.0b1.dev202610290406", "1.4.0rc3", "latest", "nightly", @@ -34,6 +36,12 @@ def test_no_regular_nightly_fails(self): with self.assertRaisesRegex(ValueError, "No regular"): resolver.latest_tag(["nightly", "feat-test-1.4.0a0.dev202610281318"]) + def test_final_version_nightly_is_accepted(self): + self.assertEqual( + resolver.latest_tag(["1.4.0rc3.dev202610280007", "1.4.0.dev202610290007"]), + "1.4.0.dev202610290007", + ) + def test_selection_records_exact_digest_and_source_from_image(self): registry = Mock() digest = "sha256:" + "a" * 64 @@ -106,6 +114,48 @@ def test_manifest_digest_mismatch_fails(self): with self.assertRaisesRegex(ValueError, "digest"): registry.manifest("1.4.0rc2.dev202610270008") + def test_manifest_by_digest_must_match_the_request(self): + registry = resolver.Registry() + registry.get = Mock(return_value=(b"{}", {})) + with self.assertRaisesRegex(ValueError, "different manifest"): + registry.manifest("sha256:" + "0" * 64) + + def test_missing_or_invalid_source_commit_fails(self): + for commit in (None, "", "abc", "g" * 40): + with self.subTest(commit=commit): + registry = Mock() + env = [] if commit is None else ["TRT_LLM_GIT_COMMIT=" + commit] + registry.get.side_effect = [ + (json.dumps({"tags": ["1.4.0rc2.dev202610270008"]}).encode(), {}), + (json.dumps({"config": {"Env": env}}).encode(), {}), + ] + registry.manifest.return_value = ( + {"config": {"digest": "sha256:config"}}, + "sha256:" + "a" * 64, + ) + with self.assertRaisesRegex(ValueError, "source commit"): + resolver.resolve({"repository": resolver.REPOSITORY}, registry) + + def test_index_without_exactly_one_amd64_image_fails(self): + amd64 = {"platform": {"os": "linux", "architecture": "amd64"}} + for manifests in ( + [], + [{"platform": {"os": "linux", "architecture": "arm64"}}], + [dict(amd64, digest="sha256:a"), dict(amd64, digest="sha256:b")], + ): + with self.subTest(manifests=manifests): + registry = Mock() + registry.get.return_value = ( + json.dumps({"tags": ["1.4.0rc2.dev202610270008"]}).encode(), + {}, + ) + registry.manifest.return_value = ( + {"manifests": manifests}, + "sha256:index", + ) + with self.assertRaisesRegex(ValueError, "exactly one linux/amd64"): + resolver.resolve({"repository": resolver.REPOSITORY}, registry) + if __name__ == "__main__": unittest.main() diff --git a/lib/sidecar/trtllm/deploy/agg.yaml b/lib/sidecar/trtllm/deploy/agg.yaml index 93a781dc3fb5..63cf233e1809 100644 --- a/lib/sidecar/trtllm/deploy/agg.yaml +++ b/lib/sidecar/trtllm/deploy/agg.yaml @@ -46,7 +46,7 @@ spec: spec: containers: - name: main - image: nvcr.io/nvidia/tensorrt-llm/release:1.4.0rc1.dev202610140007 + image: nvcr.io/nvidia/tensorrt-llm/release:1.4.0rc3.dev202610290007@sha256:14aa63789ee878885c55200c5ae8f44e4d15b90fe84bd869315ae13ef1299ac3 # Readiness via the OpenEngine Control.Health RPC, which reports the # engine's own state (grpc + model, plus kv_connector when configured) # rather than just a connectable socket. diff --git a/lib/sidecar/trtllm/deploy/agg_kv_router.yaml b/lib/sidecar/trtllm/deploy/agg_kv_router.yaml index b43c7d18423b..f47f1baec047 100644 --- a/lib/sidecar/trtllm/deploy/agg_kv_router.yaml +++ b/lib/sidecar/trtllm/deploy/agg_kv_router.yaml @@ -48,7 +48,7 @@ spec: spec: containers: - name: main - image: nvcr.io/nvidia/tensorrt-llm/release:1.4.0a0.dev202610180820@sha256:b8c862fea7c4306043dd2dbda06175af0fc8af4f336435597c07ec92f6fff3bd + image: nvcr.io/nvidia/tensorrt-llm/release:1.4.0rc3.dev202610290007@sha256:14aa63789ee878885c55200c5ae8f44e4d15b90fe84bd869315ae13ef1299ac3 # Probe Control.Health without importing TensorRT-LLM; field 1 is state, READY = 2. startupProbe: exec: diff --git a/lib/sidecar/trtllm/deploy/disagg.yaml b/lib/sidecar/trtllm/deploy/disagg.yaml index fde53b08d708..87b86ce34861 100644 --- a/lib/sidecar/trtllm/deploy/disagg.yaml +++ b/lib/sidecar/trtllm/deploy/disagg.yaml @@ -92,7 +92,7 @@ spec: sizeLimit: 10Gi containers: - name: main - image: nvcr.io/nvidia/tensorrt-llm/release:1.4.0rc1.dev202610140007 + image: nvcr.io/nvidia/tensorrt-llm/release:1.4.0rc3.dev202610290007@sha256:14aa63789ee878885c55200c5ae8f44e4d15b90fe84bd869315ae13ef1299ac3 # Readiness via the OpenEngine Control.Health RPC, which reports the # engine's own state rather than just a connectable socket. # The pinned engine ships OpenEngine bindings under tensorrt_llm, @@ -230,7 +230,7 @@ spec: sizeLimit: 10Gi containers: - name: main - image: nvcr.io/nvidia/tensorrt-llm/release:1.4.0rc1.dev202610140007 + image: nvcr.io/nvidia/tensorrt-llm/release:1.4.0rc3.dev202610290007@sha256:14aa63789ee878885c55200c5ae8f44e4d15b90fe84bd869315ae13ef1299ac3 startupProbe: exec: command: *engineReadyCommand diff --git a/lib/sidecar/trtllm/deploy/disagg_kv_router.yaml b/lib/sidecar/trtllm/deploy/disagg_kv_router.yaml index 1b53fc4855f5..69175fd1a37e 100644 --- a/lib/sidecar/trtllm/deploy/disagg_kv_router.yaml +++ b/lib/sidecar/trtllm/deploy/disagg_kv_router.yaml @@ -81,7 +81,7 @@ spec: sizeLimit: 10Gi containers: - name: main - image: nvcr.io/nvidia/tensorrt-llm/release:1.4.0a0.dev202610180820@sha256:b8c862fea7c4306043dd2dbda06175af0fc8af4f336435597c07ec92f6fff3bd + image: nvcr.io/nvidia/tensorrt-llm/release:1.4.0rc3.dev202610290007@sha256:14aa63789ee878885c55200c5ae8f44e4d15b90fe84bd869315ae13ef1299ac3 # Probe Control.Health without importing TensorRT-LLM; field 1 is state, READY = 2. startupProbe: exec: @@ -185,7 +185,7 @@ spec: sizeLimit: 10Gi containers: - name: main - image: nvcr.io/nvidia/tensorrt-llm/release:1.4.0a0.dev202610180820@sha256:b8c862fea7c4306043dd2dbda06175af0fc8af4f336435597c07ec92f6fff3bd + image: nvcr.io/nvidia/tensorrt-llm/release:1.4.0rc3.dev202610290007@sha256:14aa63789ee878885c55200c5ae8f44e4d15b90fe84bd869315ae13ef1299ac3 startupProbe: exec: command: *engineReadyCommand diff --git a/lib/sidecar/trtllm/launch/agg.sh b/lib/sidecar/trtllm/launch/agg.sh index f5de0e11f400..c5a9f817342c 100755 --- a/lib/sidecar/trtllm/launch/agg.sh +++ b/lib/sidecar/trtllm/launch/agg.sh @@ -4,10 +4,8 @@ # # Aggregated serving through TensorRT-LLM's OpenEngine gRPC server (1 GPU). # -# Run this where `TRTLLM_PYTHON` has TensorRT-LLM installed -- -# `nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc27.dev202609170000` or newer, the -# first releases carrying the OpenEngine servicer. The bindings it needs are not -# in that image; the pip step below adds them. +# Run with `TRTLLM_PYTHON` from `nvcr.io/nvidia/tensorrt-llm/release:1.4.0rc3.dev202610290007`. +# The bindings it needs are installed by the helper below. set -e diff --git a/lib/sidecar/trtllm/launch/agg_kv_router.sh b/lib/sidecar/trtllm/launch/agg_kv_router.sh index ccbd57bfc0fa..b556118f5f14 100755 --- a/lib/sidecar/trtllm/launch/agg_kv_router.sh +++ b/lib/sidecar/trtllm/launch/agg_kv_router.sh @@ -4,6 +4,8 @@ # # Aggregated KV-aware serving (2 GPUs). +# Use `TRTLLM_PYTHON` from `nvcr.io/nvidia/tensorrt-llm/release:1.4.0rc3.dev202610290007`. + set -e SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" diff --git a/lib/sidecar/trtllm/launch/disagg.sh b/lib/sidecar/trtllm/launch/disagg.sh index bee8c2434a2d..2a589232401c 100755 --- a/lib/sidecar/trtllm/launch/disagg.sh +++ b/lib/sidecar/trtllm/launch/disagg.sh @@ -4,10 +4,8 @@ # # Disaggregated serving through two TensorRT-LLM OpenEngine gRPC servers (2 workers). # -# Run this where `TRTLLM_PYTHON` has TensorRT-LLM installed -- -# `nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc27.dev202609170000` or newer, the -# first releases carrying the OpenEngine servicer. The bindings it needs are not -# in that image; the pip step below adds them. +# Run with `TRTLLM_PYTHON` from `nvcr.io/nvidia/tensorrt-llm/release:1.4.0rc3.dev202610290007`. +# The bindings it needs are installed by the helper below. # # KV cache moves prefill -> decode engine-to-engine over the cache transceiver, # not through the sidecars. Dynamo carries only the handoff: the prefill worker diff --git a/lib/sidecar/trtllm/launch/disagg_kv_router.sh b/lib/sidecar/trtllm/launch/disagg_kv_router.sh index d61ae53188a0..e6bad5490c53 100755 --- a/lib/sidecar/trtllm/launch/disagg_kv_router.sh +++ b/lib/sidecar/trtllm/launch/disagg_kv_router.sh @@ -4,6 +4,8 @@ # # Disaggregated KV-aware serving (2 prefill, 2 decode GPUs). +# Use `TRTLLM_PYTHON` from `nvcr.io/nvidia/tensorrt-llm/release:1.4.0rc3.dev202610290007`. + set -e SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" From 5b7673af9c3f24229c86f42c7acc79903494d5ec Mon Sep 17 00:00:00 2001 From: krishung5 Date: Thu, 29 Oct 2026 17:14:29 -0700 Subject: [PATCH 53/61] chore(trtllm): align multinode examples with validated nightly Signed-off-by: krishung5 --- lib/sidecar/trtllm/deploy/multinode_dp.yaml | 4 ++-- lib/sidecar/trtllm/deploy/multinode_tp.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/sidecar/trtllm/deploy/multinode_dp.yaml b/lib/sidecar/trtllm/deploy/multinode_dp.yaml index 7b3fd334c20a..86ee5500305c 100644 --- a/lib/sidecar/trtllm/deploy/multinode_dp.yaml +++ b/lib/sidecar/trtllm/deploy/multinode_dp.yaml @@ -84,7 +84,7 @@ spec: # Requires OpenEngine node-local discovery and KV events (NVIDIA/TensorRT-LLM#19440) # and the grpc.health.v1 service used by the probes below (NVIDIA/TensorRT-LLM#20437, # nightlies from 2026-10-27). - image: nvcr.io/nvidia/tensorrt-llm/release:1.4.0rc2.dev202610270008@sha256:c1e774f48f8c9b5e06bb036c1798b80c9ab5201ebeb1013044ba4f865bb779a7 + image: nvcr.io/nvidia/tensorrt-llm/release:1.4.0rc3.dev202610290007@sha256:14aa63789ee878885c55200c5ae8f44e4d15b90fe84bd869315ae13ef1299ac3 # The liveness service fails only on a fatal engine error. The inference service tracks the executor. startupProbe: grpc: @@ -168,7 +168,7 @@ spec: optional: true containers: - name: main - image: nvcr.io/nvidia/tensorrt-llm/release:1.4.0rc2.dev202610270008@sha256:c1e774f48f8c9b5e06bb036c1798b80c9ab5201ebeb1013044ba4f865bb779a7 + image: nvcr.io/nvidia/tensorrt-llm/release:1.4.0rc3.dev202610290007@sha256:14aa63789ee878885c55200c5ae8f44e4d15b90fe84bd869315ae13ef1299ac3 # No probes: sshd is the only process here. The leader waits for it, then # launches and supervises this node's ranks through mpirun. resources: diff --git a/lib/sidecar/trtllm/deploy/multinode_tp.yaml b/lib/sidecar/trtllm/deploy/multinode_tp.yaml index ec9ac033a55a..e16be9b0c798 100644 --- a/lib/sidecar/trtllm/deploy/multinode_tp.yaml +++ b/lib/sidecar/trtllm/deploy/multinode_tp.yaml @@ -81,7 +81,7 @@ spec: # Requires OpenEngine node-local discovery and KV events (NVIDIA/TensorRT-LLM#19440) # and the grpc.health.v1 service used by the probes below (NVIDIA/TensorRT-LLM#20437, # nightlies from 2026-10-27). - image: nvcr.io/nvidia/tensorrt-llm/release:1.4.0rc2.dev202610270008@sha256:c1e774f48f8c9b5e06bb036c1798b80c9ab5201ebeb1013044ba4f865bb779a7 + image: nvcr.io/nvidia/tensorrt-llm/release:1.4.0rc3.dev202610290007@sha256:14aa63789ee878885c55200c5ae8f44e4d15b90fe84bd869315ae13ef1299ac3 # The liveness service fails only on a fatal engine error. The inference service tracks the executor. startupProbe: grpc: @@ -155,7 +155,7 @@ spec: spec: containers: - name: main - image: nvcr.io/nvidia/tensorrt-llm/release:1.4.0rc2.dev202610270008@sha256:c1e774f48f8c9b5e06bb036c1798b80c9ab5201ebeb1013044ba4f865bb779a7 + image: nvcr.io/nvidia/tensorrt-llm/release:1.4.0rc3.dev202610290007@sha256:14aa63789ee878885c55200c5ae8f44e4d15b90fe84bd869315ae13ef1299ac3 # No probes: sshd is the only process here. The leader waits for it, then # launches and supervises this node's ranks through mpirun. resources: From c498dd741771b64280c55e31a991d6aea9c5bab7 Mon Sep 17 00:00:00 2001 From: krishung5 Date: Fri, 30 Oct 2026 00:22:32 -0700 Subject: [PATCH 54/61] ci(trtllm): validate sidecar videos against upstream nightlies Signed-off-by: krishung5 --- .github/filters.yaml | 3 +- .github/scripts/test-filters.js | 5 + .github/workflows/nightly-ci.yml | 6 +- .../shared-test-trtllm-nightly-image.yml | 20 +- lib/sidecar/trtllm/README.md | 4 +- lib/sidecar/trtllm/ci/run_image_test.py | 216 ++++++++++++------ .../trtllm/ci/tests/test_run_media_test.py | 123 ++++++++++ lib/sidecar/trtllm/src/tests/e2e.rs | 20 +- tests/serve/test_sidecar.py | 56 ----- 9 files changed, 310 insertions(+), 143 deletions(-) create mode 100644 lib/sidecar/trtllm/ci/tests/test_run_media_test.py diff --git a/.github/filters.yaml b/.github/filters.yaml index 1d0ee6169250..af03837abb0a 100644 --- a/.github/filters.yaml +++ b/.github/filters.yaml @@ -377,8 +377,9 @@ planner: - '!**/*.rst' sidecar_trtllm_image: - # PR CI runs resolver CPU tests; the GPU test runs independently in nightly CI. + # PR CI runs nightly resolver/harness CPU tests; GPU cases run in nightly CI. - 'lib/sidecar/trtllm/ci/**' + - 'lib/llm/tests/data/media/triangle_240p_10.mp4' - '.github/actions/changed-files/**' - '.github/filters.yaml' - '.github/workflows/pr.yaml' diff --git a/.github/scripts/test-filters.js b/.github/scripts/test-filters.js index cf7473ffadfe..3d992df7c589 100755 --- a/.github/scripts/test-filters.js +++ b/.github/scripts/test-filters.js @@ -60,6 +60,11 @@ function checkFilter(file, patterns) { // Test cases: [file, expectations, description] // expectations: { filterName: expectedValue, ... } const testCases = [ + { + file: 'lib/llm/tests/data/media/triangle_240p_10.mp4', + expect: { sidecar_trtllm_image: true }, + desc: 'the video fixture triggers the nightly harness checksum tests' + }, { file: 'lib/sidecar/trtllm/ci/resolve_nightly.py', expect: { sidecar_trtllm_image: true }, diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml index fffdaa37727b..47499f8ef5fd 100644 --- a/.github/workflows/nightly-ci.yml +++ b/.github/workflows/nightly-ci.yml @@ -1032,7 +1032,7 @@ jobs: sidecar_binary_artifact: ${{ needs.sidecar-build.outputs.binary_artifact }} secrets: inherit - sidecar-trtllm-nightly-image-test: + sidecar-trtllm-nightly-media-test: needs: [create-fresh-builder, compute-release-mode] if: ${{ needs.compute-release-mode.outputs.run_tests == 'true' }} uses: $/.github/workflows/shared-test-trtllm-nightly-image.yml @@ -1748,7 +1748,7 @@ jobs: name: Clean K8s builder if exists runs-on: prod-default-small-v2 if: always() - needs: [vllm-build, sglang-build, trtllm-build, dynamo-pipeline, operator-build, planner-build, frontend-build, vllm-efa-build, sglang-efa-build, trtllm-efa-build, sidecar-build, sidecar-trtllm-nightly-image-test, create-fresh-builder] + needs: [vllm-build, sglang-build, trtllm-build, dynamo-pipeline, operator-build, planner-build, frontend-build, vllm-efa-build, sglang-efa-build, trtllm-efa-build, sidecar-build, sidecar-trtllm-nightly-media-test, create-fresh-builder] permissions: contents: read steps: @@ -1770,7 +1770,7 @@ jobs: ############################## SLACK NOTIFICATION ############################## notify-slack: if: always() - needs: [vllm-test, vllm-multi-gpu-test, vllm-4-gpu-test, vllm-h100-test, sglang-test, sglang-multi-gpu-test, sglang-4-gpu-test, sglang-h100-test, trtllm-test, trtllm-multi-gpu-test, trtllm-h100-test, sidecar-build, sidecar-vllm-test, sidecar-sglang-test, sidecar-trtllm-test, sidecar-trtllm-nightly-image-test, deploy-cleanup, deepseek-v4-pro-agg-perf, deepseek-v4-pro-disagg-perf, rust-tests, kimi-k25-agg-perf, kimi-k25-disagg-perf] + needs: [vllm-test, vllm-multi-gpu-test, vllm-4-gpu-test, vllm-h100-test, sglang-test, sglang-multi-gpu-test, sglang-4-gpu-test, sglang-h100-test, trtllm-test, trtllm-multi-gpu-test, trtllm-h100-test, sidecar-build, sidecar-vllm-test, sidecar-sglang-test, sidecar-trtllm-test, sidecar-trtllm-nightly-media-test, deploy-cleanup, deepseek-v4-pro-agg-perf, deepseek-v4-pro-disagg-perf, rust-tests, kimi-k25-agg-perf, kimi-k25-disagg-perf] permissions: contents: read actions: read # grant the reusable notifier read access to list this run's jobs diff --git a/.github/workflows/shared-test-trtllm-nightly-image.yml b/.github/workflows/shared-test-trtllm-nightly-image.yml index 2461a810f680..12059b72ae86 100644 --- a/.github/workflows/shared-test-trtllm-nightly-image.yml +++ b/.github/workflows/shared-test-trtllm-nightly-image.yml @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -name: TRT-LLM Sidecar Upstream Nightly Image +name: TRT-LLM Sidecar Upstream Nightly Media on: workflow_call: @@ -19,14 +19,14 @@ permissions: jobs: build: - name: Build image integration client + name: Build media integration client runs-on: prod-builder-v3 timeout-minutes: 60 outputs: image: ${{ steps.config.outputs.image }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - name: Test nightly selection + - name: Test nightly selection and media harness run: python3 -m unittest discover -s lib/sidecar/trtllm/ci/tests -v - name: Resolve latest upstream nightly id: config @@ -57,7 +57,11 @@ jobs: retention-days: 7 test: - name: Upstream TRT-LLM image integration (1 GPU) + name: Upstream TRT-LLM ${{ matrix.media }} integration (1 GPU) + strategy: + fail-fast: false + matrix: + media: [image, video] needs: build runs-on: [gpu-1, amd64, v2, us-west-2] timeout-minutes: 30 @@ -84,18 +88,20 @@ jobs: set -euo pipefail source lib/sidecar/trtllm/launch/common.sh trtllm_ensure_openengine_bindings python3 - - name: Run live image integration test + - name: Run live ${{ matrix.media }} integration test env: HF_TOKEN: ${{ secrets.HF_TOKEN }} run: >- python3 lib/sidecar/trtllm/ci/run_image_test.py --binary trtllm-image-tests/trtllm-image-tests --config trtllm-image-tests/nightly.json + --media ${{ matrix.media }} + --results test-results/trtllm-nightly-${{ matrix.media }} - name: Upload engine logs and test results if: always() uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: - name: trtllm-nightly-image-results-${{ github.sha }} - path: test-results/trtllm-nightly-image/ + name: trtllm-nightly-${{ matrix.media }}-results-${{ github.sha }} + path: test-results/trtllm-nightly-${{ matrix.media }}/ if-no-files-found: warn retention-days: 7 diff --git a/lib/sidecar/trtllm/README.md b/lib/sidecar/trtllm/README.md index cf93d0903f08..251d17708358 100644 --- a/lib/sidecar/trtllm/README.md +++ b/lib/sidecar/trtllm/README.md @@ -30,8 +30,8 @@ supported, even when `--context-length` is supplied. > [NGC](https://catalog.ngc.nvidia.com/orgs/nvidia/tensorrt-llm/containers/release/-). > The standalone launch and deployment examples use > `nvcr.io/nvidia/tensorrt-llm/release:1.4.0rc3.dev202610290007`, which includes -> image URL passthrough, native KVCM2 event streaming, and OpenEngine routing load support. -> The nightly CI job tests image passthrough against the latest regular upstream nightly; +> image/video URL passthrough, native KVCM2 event streaming, and OpenEngine routing load support. +> The nightly CI job tests image and video passthrough against the latest regular upstream nightly; > it also supports manual dispatch. The examples below stay pinned to the validated version. ## Support matrix diff --git a/lib/sidecar/trtllm/ci/run_image_test.py b/lib/sidecar/trtllm/ci/run_image_test.py index 44bf35154878..09de9a343801 100755 --- a/lib/sidecar/trtllm/ci/run_image_test.py +++ b/lib/sidecar/trtllm/ci/run_image_test.py @@ -5,9 +5,12 @@ """Run the Rust adapter against an unmodified upstream TRT-LLM image.""" import argparse +import base64 +import hashlib import importlib.metadata import json import os +import re import signal import socket import subprocess @@ -17,7 +20,90 @@ import xml.etree.ElementTree as ET from pathlib import Path -TEST = "tests::e2e::e2e_real_openengine_image" +FIXTURES = { + "image": { + "prompt": "Name the colors from left to right.", + "token_ids": [ + 151644, + 872, + 198, + 151652, + 151655, + 151653, + 675, + 279, + 7987, + 504, + 2115, + 311, + 1290, + 13, + 151645, + 198, + 151644, + 77091, + 198, + ], + "context_length": 2048, + }, + "video": { + "prompt": "Describe the shape and its motion in this video.", + "token_ids": [ + 151644, + 872, + 198, + 151652, + 151656, + 151653, + 74785, + 279, + 6083, + 323, + 1181, + 11379, + 304, + 419, + 2766, + 13, + 151645, + 198, + 151644, + 77091, + 198, + ], + "context_length": 2048, + }, +} +VIDEO_PATH = ( + Path(__file__).resolve().parents[3] / "llm/tests/data/media/triangle_240p_10.mp4" +) +VIDEO_SHA256 = "04171a85ee3ba5ac9118b0a5858adf0892b49474cdf15320e32599a3d1ab9919" + + +def video_data_uri(path=VIDEO_PATH): + data = path.read_bytes() + if hashlib.sha256(data).hexdigest() != VIDEO_SHA256: + raise ValueError( + "Video fixture changed; revalidate its content and expected response" + ) + return "data:video/mp4;base64," + base64.b64encode(data).decode("ascii") + + +def validate_response(media, caption, response, input_token_count): + words = re.findall(r"[a-z]+", caption.lower()) + if media == "image": + colors = [word for word in words if word in ("red", "blue")] + if colors[:2] != ["red", "blue"]: + raise AssertionError(f"Expected red, then blue, got: {caption}") + elif not {"triangle", "triangular"}.intersection(words): + raise AssertionError( + f"Expected a triangle in the video description, got: {caption}" + ) + if ( + response["completion_tokens"] <= 0 + or response["prompt_tokens"] <= input_token_count + ): + raise AssertionError(f"Missing authoritative {media} token usage: {response}") def wait_for_engine(process, port, timeout=600): @@ -49,13 +135,41 @@ def stop_engine(process): process.wait(timeout=10) +def run_rust_test(binary, env, test, results): + completed = subprocess.run( + [ + str(binary), + "--exact", + test, + "--ignored", + "--nocapture", + "--color", + "never", + ], + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + timeout=240, + ) + (results / "rust-test.log").write_text(completed.stdout) + print(completed.stdout, flush=True) + completed.check_returncode() + if "1 passed; 0 failed; 0 ignored;" not in completed.stdout: + raise RuntimeError(f"{test} did not execute exactly once") + + def run(args, results): + # GPU-only dependencies stay out of the CPU harness tests. from huggingface_hub import snapshot_download from transformers import AutoTokenizer + fixture = FIXTURES[args.media] + test = f"tests::e2e::e2e_real_openengine_{args.media}" config = json.loads(args.config.read_text()) identity = { **config, + "media": args.media, "installed_version": importlib.metadata.version("tensorrt_llm"), "installed_commit": os.environ.get("TRT_LLM_GIT_COMMIT"), } @@ -65,14 +179,16 @@ def run(args, results): f"Unexpected upstream commit: {identity['installed_commit']}" ) + video = video_data_uri() if args.media == "video" else None + binary = args.binary.resolve() binary.chmod(binary.stat().st_mode | 0o111) listing = subprocess.check_output( - [str(binary), "--list", "--exact", TEST, "--format", "terse"], text=True + [str(binary), "--list", "--exact", test, "--format", "terse"], text=True ) - if listing.splitlines().count(f"{TEST}: test") != 1: + if listing.splitlines().count(f"{test}: test") != 1: raise RuntimeError( - f"Expected exactly one image integration test, got: {listing}" + f"Expected exactly one {args.media} integration test, got: {listing}" ) model_path = args.model_path or snapshot_download( @@ -86,50 +202,31 @@ def run(args, results): { "role": "user", "content": [ - {"type": "image"}, - {"type": "text", "text": "Name the colors from left to right."}, + {"type": args.media}, + {"type": "text", "text": fixture["prompt"]}, ], } ], tokenize=True, add_generation_prompt=True, ) - expected_ids = [ - 151644, - 872, - 198, - 151652, - 151655, - 151653, - 675, - 279, - 7987, - 504, - 2115, - 311, - 1290, - 13, - 151645, - 198, - 151644, - 77091, - 198, - ] + expected_ids = fixture["token_ids"] token_ids = ( token_ids["input_ids"] if isinstance(token_ids, dict) or hasattr(token_ids, "input_ids") else token_ids ) if token_ids != expected_ids: - raise RuntimeError(f"Image fixture tokenizer mismatch: {token_ids}") + raise RuntimeError(f"{args.media} fixture tokenizer mismatch: {token_ids}") with socket.socket() as listener: listener.bind(("127.0.0.1", 0)) port = listener.getsockname()[1] options = results / "engine.yaml" + context_length = fixture["context_length"] options.write_text( - "max_batch_size: 1\nmax_num_tokens: 2048\n" - "kv_cache_config:\n max_tokens: 2048\n" + f"max_batch_size: 1\nmax_num_tokens: {context_length}\n" + f"kv_cache_config:\n max_tokens: {context_length}\n" " free_gpu_memory_fraction: 0.15\n" ) result_path = results / "response.json" @@ -150,7 +247,7 @@ def run(args, results): "--port", str(port), "--max_seq_len", - "2048", + str(context_length), "--extra_llm_api_options", str(options), ], @@ -165,29 +262,9 @@ def run(args, results): TRTLLM_E2E_MODEL=config["model"], TRTLLM_E2E_RESULT=str(result_path), ) - completed = subprocess.run( - [ - str(binary), - "--exact", - TEST, - "--ignored", - "--nocapture", - "--color", - "never", - ], - env=env, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - timeout=240, - ) - (results / "rust-test.log").write_text(completed.stdout) - print(completed.stdout, flush=True) - completed.check_returncode() - if "1 passed; 0 failed; 0 ignored;" not in completed.stdout: - raise RuntimeError( - "Image integration test did not execute exactly once" - ) + if video is not None: + env["TRTLLM_E2E_VIDEO_DATA_URI"] = video + run_rust_test(binary, env, test, results) finally: stop_engine(engine) @@ -195,34 +272,25 @@ def run(args, results): caption = tokenizer.decode(response["token_ids"], skip_special_tokens=True) response["text"] = caption result_path.write_text(json.dumps(response, indent=2) + "\n") - print(f"Image response: {caption}", flush=True) - words = caption.lower().replace("-", " ").split() - # Accept phrasing differences, but require the fixture colors left to right. - colors = [word.strip(".,:;!?()\"'") for word in words] - if [color for color in colors if color in ("red", "blue")][:2] != ["red", "blue"]: - raise AssertionError( - f"Expected red, then blue, in the image description, got: {caption}" - ) - if response["completion_tokens"] <= 0 or response["prompt_tokens"] <= len( - expected_ids - ): - raise AssertionError(f"Missing authoritative image token usage: {response}") + print(f"{args.media} response: {caption}", flush=True) + validate_response(args.media, caption, response, len(expected_ids)) def main(): parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--media", choices=FIXTURES, default="image") parser.add_argument("--binary", type=Path, required=True) parser.add_argument( "--config", type=Path, required=True, help="Resolved nightly image manifest" ) - parser.add_argument( - "--results", type=Path, default=Path("test-results/trtllm-nightly-image") - ) + parser.add_argument("--results", type=Path) parser.add_argument( "--model-path", type=Path, help="Use an already downloaded model snapshot" ) args = parser.parse_args() - results = args.results.resolve() + results = ( + args.results or Path(f"test-results/trtllm-nightly-{args.media}") + ).resolve() results.mkdir(parents=True, exist_ok=True) start = time.monotonic() failure = None @@ -234,14 +302,18 @@ def main(): duration = str(time.monotonic() - start) suite = ET.Element( "testsuite", - name="trtllm-nightly-image", + name=f"trtllm-nightly-{args.media}", tests="1", failures=str(int(failure is not None)), skipped="0", time=duration, ) case = ET.SubElement( - suite, "testcase", name=TEST, classname="trtllm.sidecar", time=duration + suite, + "testcase", + name=f"tests::e2e::e2e_real_openengine_{args.media}", + classname="trtllm.sidecar", + time=duration, ) if failure: ET.SubElement(case, "failure", message=failure.splitlines()[-1]).text = failure diff --git a/lib/sidecar/trtllm/ci/tests/test_run_media_test.py b/lib/sidecar/trtllm/ci/tests/test_run_media_test.py new file mode 100644 index 000000000000..4237da921f35 --- /dev/null +++ b/lib/sidecar/trtllm/ci/tests/test_run_media_test.py @@ -0,0 +1,123 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import base64 +import contextlib +import importlib.util +import io +import subprocess +import tempfile +import unittest +import xml.etree.ElementTree as ET +from pathlib import Path +from unittest.mock import patch + +SPEC = importlib.util.spec_from_file_location( + "run_media_test", Path(__file__).resolve().parents[1] / "run_image_test.py" +) +runner = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(runner) + + +class MediaResponseTests(unittest.TestCase): + def test_image_requires_left_to_right_color_order(self): + response = {"prompt_tokens": 82, "completion_tokens": 20} + runner.validate_response("image", "- Red\n- Blue", response, 19) + for caption in ("Blue, red.", "Red.", "A green square."): + with self.subTest(caption=caption), self.assertRaises(AssertionError): + runner.validate_response("image", caption, response, 19) + + def test_video_requires_the_fixture_subject(self): + response = {"prompt_tokens": 1024, "completion_tokens": 30} + for caption in ("A rotating triangle.", "A triangular shape moves."): + runner.validate_response("video", caption, response, 21) + with self.assertRaisesRegex(AssertionError, "triangle"): + runner.validate_response("video", "A red square moves.", response, 21) + + def test_video_requires_expanded_prompt_and_completion_usage(self): + for response in ( + {"prompt_tokens": 21, "completion_tokens": 30}, + {"prompt_tokens": 1024, "completion_tokens": 0}, + ): + with ( + self.subTest(response=response), + self.assertRaisesRegex(AssertionError, "token usage"), + ): + runner.validate_response("video", "A rotating triangle.", response, 21) + + def test_video_data_uri_contains_the_checked_in_clip(self): + uri = runner.video_data_uri() + header, encoded = uri.split(",", 1) + self.assertEqual(header, "data:video/mp4;base64") + self.assertEqual(base64.b64decode(encoded), runner.VIDEO_PATH.read_bytes()) + + def test_changed_video_fixture_is_rejected(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "video.mp4" + path.write_bytes(b"not the validated video") + with self.assertRaisesRegex(ValueError, "Video fixture changed"): + runner.video_data_uri(path) + + def test_zero_or_ignored_rust_tests_cannot_pass(self): + for stdout in ( + "test result: ok. 0 passed; 0 failed; 0 ignored;", + "test result: ok. 0 passed; 0 failed; 1 ignored;", + ): + with self.subTest(stdout=stdout), tempfile.TemporaryDirectory() as tmp: + result = subprocess.CompletedProcess([], 0, stdout=stdout) + with ( + patch.object(runner.subprocess, "run", return_value=result), + contextlib.redirect_stdout(io.StringIO()), + self.assertRaisesRegex( + RuntimeError, "did not execute exactly once" + ), + ): + runner.run_rust_test(Path("binary"), {}, "video-test", Path(tmp)) + self.assertEqual((Path(tmp) / "rust-test.log").read_text(), stdout) + + def test_rust_video_test_is_explicitly_selected_and_executed(self): + with tempfile.TemporaryDirectory() as tmp: + result = subprocess.CompletedProcess( + [], 0, stdout="test result: ok. 1 passed; 0 failed; 0 ignored;" + ) + with ( + patch.object(runner.subprocess, "run", return_value=result) as run, + contextlib.redirect_stdout(io.StringIO()), + ): + runner.run_rust_test(Path("binary"), {}, "video-test", Path(tmp)) + self.assertEqual( + run.call_args.args[0][:4], + ["binary", "--exact", "video-test", "--ignored"], + ) + + def test_video_failures_are_recorded_as_failures_in_junit(self): + with tempfile.TemporaryDirectory() as tmp: + argv = [ + "run_image_test.py", + "--media", + "video", + "--binary", + "unused", + "--config", + "unused", + "--results", + tmp, + ] + with ( + patch("sys.argv", argv), + patch.object(runner, "run", side_effect=RuntimeError("engine failed")), + contextlib.redirect_stderr(io.StringIO()), + ): + self.assertEqual(runner.main(), 1) + suite = ET.parse(Path(tmp) / "junit.xml").getroot() + self.assertEqual(suite.attrib["failures"], "1") + self.assertEqual(suite.attrib["skipped"], "0") + case = suite.find("testcase") + self.assertEqual( + case.attrib["name"], "tests::e2e::e2e_real_openengine_video" + ) + self.assertIn("engine failed", case.find("failure").text) + + +if __name__ == "__main__": + unittest.main() diff --git a/lib/sidecar/trtllm/src/tests/e2e.rs b/lib/sidecar/trtllm/src/tests/e2e.rs index 0b276f007abf..dd03683375c6 100644 --- a/lib/sidecar/trtllm/src/tests/e2e.rs +++ b/lib/sidecar/trtllm/src/tests/e2e.rs @@ -450,8 +450,8 @@ async fn e2e_real_openengine_video() { .expect("start against the live video-capable server"); let token_ids = vec![ - 151644, 872, 198, 151652, 151656, 151653, 74785, 279, 7218, 6083, 323, 1181, 5106, 304, - 825, 2805, 11652, 13, 151645, 198, 151644, 77091, 198, + 151644, 872, 198, 151652, 151656, 151653, 74785, 279, 6083, 323, 1181, 11379, 304, 419, + 2766, 13, 151645, 198, 151644, 77091, 198, ]; let mut request = PreprocessedRequest::builder() .model(model) @@ -491,6 +491,22 @@ async fn e2e_real_openengine_video() { usage.prompt_tokens, usage.completion_tokens ); + if let Ok(path) = std::env::var("TRTLLM_E2E_RESULT") { + std::fs::write( + path, + serde_json::to_vec(&json!({ + "token_ids": generated, + "prompt_tokens": usage.prompt_tokens, + "completion_tokens": usage.completion_tokens, + })) + .expect("serialize video result"), + ) + .expect("write video result"); + } + assert!( + usage.completion_tokens > 0, + "expected completion token usage" + ); assert!(!generated.is_empty(), "expected video-conditioned output"); assert!( usage.prompt_tokens > token_ids.len() as u32, diff --git a/tests/serve/test_sidecar.py b/tests/serve/test_sidecar.py index 896a6f4a2609..f0e279a19853 100644 --- a/tests/serve/test_sidecar.py +++ b/tests/serve/test_sidecar.py @@ -10,18 +10,12 @@ import pytest -from dynamo.common.utils.install_media_decoders import VALIDATED_SPECS from tests.router.helper import generate_random_suffix from tests.serve.common import ( WORKSPACE_DIR, params_with_model_mark, run_serve_deployment, ) -from tests.serve.conftest import ( - MULTIMODAL_IMG_URL, - MULTIMODAL_VIDEO_EXPECTED, - MULTIMODAL_VIDEO_URL, -) from tests.utils.constants import DynamoPortRange from tests.utils.engine_metrics import ( EngineMetrics, @@ -32,7 +26,6 @@ from tests.utils.gpu_args import map_cuda_visible_devices from tests.utils.payload_builder import ( chat_logprobs_and_structured_output_payloads, - chat_payload, disaggregated_kv_routing_payload, disaggregated_token_count_payload, http_cancellation_payloads, @@ -262,46 +255,6 @@ def _on_first_gpu(*names: str) -> dict[str, tuple[int, ...]]: *http_cancellation_payloads(None, max_tokens=CANCELLATION_MAX_TOKENS), ], ), - "trtllm_aggregated_multimodal": SidecarConfig( - name="trtllm_aggregated_multimodal", - directory=trtllm_sidecar_dir, - script_name="agg.sh", - marks=[ - pytest.mark.trtllm, - pytest.mark.multimodal, - pytest.mark.gpu_1, - pytest.mark.profiled_vram_gib(12.0), - pytest.mark.requested_trtllm_kv_tokens(32768), - pytest.mark.timeout(960), - pytest.mark.post_merge, - pytest.mark.installs_extra_dependencies, - ], - model="Qwen/Qwen3-VL-2B-Instruct", - timeout=900, - delayed_start=60, - env={ - "DYN_MM_ALLOW_INTERNAL": "1", - "DYN_TEST_ONLY_PIP_INSTALL": VALIDATED_SPECS["opencv-python-headless"], - "PYTHONUNBUFFERED": "1", - "TRTLLM_MEDIA_ALLOW_PRIVATE_URLS": "1", - }, - request_payloads=[ - multimodal_payload_default( - image_url=MULTIMODAL_IMG_URL, - text="Describe what you see in this image.", - expected_response=["green", "square"], - ), - chat_payload( - [ - {"type": "text", "text": "Describe the video in detail."}, - {"type": "video_url", "video_url": {"url": MULTIMODAL_VIDEO_URL}}, - ], - expected_response=MULTIMODAL_VIDEO_EXPECTED, - max_tokens=100, - temperature=0.0, - ), - ], - ), "trtllm_disaggregated": SidecarConfig( name="trtllm_disaggregated", directory=trtllm_sidecar_dir, @@ -574,15 +527,6 @@ def _on_first_gpu(*names: str) -> dict[str, tuple[int, ...]]: @pytest.fixture(params=params_with_model_mark(sidecar_configs)) def sidecar_config_test(request, monkeypatch, discovery_backend): config = sidecar_configs[request.param] - if ( - config.name == "trtllm_aggregated_multimodal" - and not _trtllm_serves_openengine() - ): - if os.environ.get("DYNAMO_TEST_SIDECAR_BACKEND") == "trtllm": - pytest.fail(TRTLLM_OPENENGINE_SKIP_REASON) - pytest.skip(TRTLLM_OPENENGINE_SKIP_REASON) - if config.name == "trtllm_aggregated_multimodal": - request.getfixturevalue("image_server") monkeypatch.setenv("MODEL", config.model) monkeypatch.setenv("DYN_DISCOVERY_BACKEND", discovery_backend) monkeypatch.setenv("DYN_NAMESPACE", f"sidecar-{generate_random_suffix()}") From 78c66924b927a83ca927ac54ab64073d6b5372f0 Mon Sep 17 00:00:00 2001 From: krishung5 Date: Fri, 30 Oct 2026 01:01:20 -0700 Subject: [PATCH 55/61] ci(trtllm): install optional decoder for sidecar video tests Signed-off-by: krishung5 --- .github/workflows/shared-test-trtllm-nightly-image.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/shared-test-trtllm-nightly-image.yml b/.github/workflows/shared-test-trtllm-nightly-image.yml index 12059b72ae86..7409afc5f7bb 100644 --- a/.github/workflows/shared-test-trtllm-nightly-image.yml +++ b/.github/workflows/shared-test-trtllm-nightly-image.yml @@ -88,6 +88,10 @@ jobs: set -euo pipefail source lib/sidecar/trtllm/launch/common.sh trtllm_ensure_openengine_bindings python3 + - name: Install TRT-LLM video decoder + if: matrix.media == 'video' + run: >- + python3 components/src/dynamo/common/utils/install_media_decoders.py trtllm - name: Run live ${{ matrix.media }} integration test env: HF_TOKEN: ${{ secrets.HF_TOKEN }} From 3711414c5e4e0e18b3525c30dd3aa446969a1f58 Mon Sep 17 00:00:00 2001 From: krishung5 Date: Fri, 30 Oct 2026 08:55:46 -0700 Subject: [PATCH 56/61] test(trtllm): share audio input capability RPC coverage Signed-off-by: krishung5 --- lib/sidecar/testkit/tests/support/trtllm.rs | 35 ++--- .../testkit/tests/trtllm_multimodal.rs | 128 ++++++++++++------ 2 files changed, 92 insertions(+), 71 deletions(-) diff --git a/lib/sidecar/testkit/tests/support/trtllm.rs b/lib/sidecar/testkit/tests/support/trtllm.rs index 61b283b4f4b5..d19821a62ef0 100644 --- a/lib/sidecar/testkit/tests/support/trtllm.rs +++ b/lib/sidecar/testkit/tests/support/trtllm.rs @@ -41,9 +41,7 @@ pub struct Discovery { pub calls: AtomicUsize, pub unavailable: AtomicBool, pub hold: AtomicBool, - pub portable_multimodal: AtomicBool, - pub advertises_image: AtomicBool, - pub advertises_video: AtomicBool, + pub model_info: Mutex>, pub omit_context_length: AtomicBool, pub error: Mutex>, release: tokio::sync::Notify, @@ -196,32 +194,15 @@ impl Control for ControlledControl { if let Some(code) = *self.discovery.error.lock().unwrap() { return Err(Status::new(code, "model discovery rejected")); } - let mut response = self.inner.get_model_info(request).await?; - let info = response.get_mut(); + let model_info = self.discovery.model_info.lock().unwrap().clone(); + let mut response = if let Some(info) = model_info { + Response::new(info) + } else { + self.inner.get_model_info(request).await? + }; if self.discovery.omit_context_length.load(Ordering::SeqCst) { - info.max_context_length = None; + response.get_mut().max_context_length = None; } - info.supports_multimodal = Some(self.discovery.portable_multimodal.load(Ordering::SeqCst)); - let modalities: Vec<_> = [ - ( - self.discovery.advertises_image.load(Ordering::SeqCst), - "image", - ), - ( - self.discovery.advertises_video.load(Ordering::SeqCst), - "video", - ), - ] - .into_iter() - .filter_map(|(enabled, modality)| enabled.then_some(modality)) - .collect(); - info.extra = Some( - dynamo_sidecar_common::json_to_struct( - serde_json::json!({"trtllm_supported_modalities": modalities}), - "test capabilities", - ) - .unwrap(), - ); Ok(response) } diff --git a/lib/sidecar/testkit/tests/trtllm_multimodal.rs b/lib/sidecar/testkit/tests/trtllm_multimodal.rs index 56b10dd53c04..a91b97004e3c 100644 --- a/lib/sidecar/testkit/tests/trtllm_multimodal.rs +++ b/lib/sidecar/testkit/tests/trtllm_multimodal.rs @@ -72,15 +72,23 @@ fn finished() -> pb::GenerateResponse { } } -fn advertise_images(fixture: &Fixture) { - fixture - .discovery - .portable_multimodal - .store(true, Ordering::SeqCst); - fixture - .discovery - .advertises_image - .store(true, Ordering::SeqCst); +fn advertise_media(fixture: &Fixture, modalities: &[&str]) { + *fixture.discovery.model_info.lock().unwrap() = Some(pb::ModelInfo { + model_id: "mocker-model".into(), + served_model_name: "mocker-model".into(), + max_context_length: Some(32768), + supports_multimodal: Some(true), + supports_text_input: Some(true), + supports_token_ids_input: Some(true), + extra: Some( + dynamo_sidecar_common::json_to_struct( + serde_json::json!({"trtllm_supported_modalities": modalities}), + "test capabilities", + ) + .unwrap(), + ), + ..Default::default() + }); } async fn wait_for_calls(fixture: &Fixture, count: usize) { @@ -96,7 +104,7 @@ async fn wait_for_calls(fixture: &Fixture, count: usize) { async fn images_preserve_order_uuids_and_engine_expanded_usage() { let control = Controller::::default(); let mut fixture = Fixture::start(control.clone(), FixtureConfig::default()).await; - advertise_images(&fixture); + advertise_media(&fixture, &["image"]); let engine = fixture.engine().await; engine.start(0).await.unwrap(); let ctx = mock_context(); @@ -193,7 +201,7 @@ async fn formatter_regrouped_tool_media_reaches_openengine_in_rendered_order() { let control = Controller::::default(); let mut fixture = Fixture::start(control.clone(), FixtureConfig::default()).await; - advertise_images(&fixture); + advertise_media(&fixture, &["image"]); let engine = fixture.engine().await; engine.start(0).await.unwrap(); let ctx = mock_context(); @@ -232,23 +240,19 @@ async fn both_media_capability_signals_are_required() { for (portable, advertised) in [(true, false), (false, true)] { let control = Controller::::default(); let mut fixture = Fixture::start(control.clone(), FixtureConfig::default()).await; + let mut modalities = vec![if video { "image" } else { "video" }]; + if advertised { + modalities.push(if video { "video" } else { "image" }); + } + advertise_media(&fixture, &modalities); fixture .discovery - .portable_multimodal - .store(portable, Ordering::SeqCst); - let (target, other) = if video { - ( - &fixture.discovery.advertises_video, - &fixture.discovery.advertises_image, - ) - } else { - ( - &fixture.discovery.advertises_image, - &fixture.discovery.advertises_video, - ) - }; - target.store(advertised, Ordering::SeqCst); - other.store(true, Ordering::SeqCst); + .model_info + .lock() + .unwrap() + .as_mut() + .unwrap() + .supports_multimodal = Some(portable); let engine = fixture.engine().await; engine.start(0).await.unwrap(); let ctx = mock_context(); @@ -278,14 +282,7 @@ async fn videos_recover_discovery_and_preserve_sources_uuids_and_usage() { let control = Controller::::default(); let mut fixture = Fixture::start(control.clone(), FixtureConfig::default()).await; fixture.discovery.unavailable.store(true, Ordering::SeqCst); - fixture - .discovery - .portable_multimodal - .store(true, Ordering::SeqCst); - fixture - .discovery - .advertises_video - .store(true, Ordering::SeqCst); + advertise_media(&fixture, &["video"]); let mut startup = tokio::spawn(bootstrap(&fixture, Some(32768), 5)); wait_for_calls(&fixture, 2).await; assert!(!startup.is_finished()); @@ -359,11 +356,7 @@ async fn startup_recovers_media_capabilities_before_registration() { for omit_window in [false, true] { let control = Controller::::default(); let mut fixture = Fixture::start(control.clone(), FixtureConfig::default()).await; - advertise_images(&fixture); - fixture - .discovery - .advertises_video - .store(true, Ordering::SeqCst); + advertise_media(&fixture, &["image", "video"]); fixture .discovery .omit_context_length @@ -508,11 +501,7 @@ async fn shutdown_cancels_retrying_and_stalled_bootstrap_discovery() { async fn media_window_override_is_rejected_before_dispatch() { let control = Controller::::default(); let mut fixture = Fixture::start(control.clone(), FixtureConfig::default()).await; - advertise_images(&fixture); - fixture - .discovery - .advertises_video - .store(true, Ordering::SeqCst); + advertise_media(&fixture, &["image", "video"]); let engine = fixture.engine_with_context_length(4096); engine.start(0).await.unwrap(); for (video, max_tokens) in [ @@ -537,3 +526,54 @@ async fn media_window_override_is_rejected_before_dispatch() { engine.cleanup().await.unwrap(); fixture.shutdown().await; } + +#[tokio::test] +async fn text_only_audio_preserves_whisper_task_prompt_with_empty_token_ids() { + let control = Controller::::default(); + let mut fixture = Fixture::start(control.clone(), FixtureConfig::default()).await; + advertise_media(&fixture, &["audio"]); + fixture + .discovery + .model_info + .lock() + .unwrap() + .as_mut() + .unwrap() + .supports_token_ids_input = Some(false); + let engine = fixture.engine().await; + engine.start(0).await.unwrap(); + let uri = "data:audio/wav;base64,YXVkaW8="; + for prompt in [ + "", + "<|startoftranscript|><|fr|><|translate|><|notimestamps|>", + ] { + let mut req = request("mocker-model", vec![], 16); + req.extra_args = Some(serde_json::json!({"formatted_prompt": prompt})); + req.multi_modal_data = Some(HashMap::from([( + "audio_url".into(), + vec![MultimodalData::RawUrl(uri.into())], + )])); + let ctx = mock_context(); + let handle = control.request(ctx.id(), RequestPlan::default()); + fixture.respond(ctx.id(), vec![finished()]); + let outputs = collect(&engine, req, GenerateContext::new(ctx, None)).await; + assert_eq!( + outputs.last().unwrap().as_ref().unwrap().finish_reason, + Some(FinishReason::Stop) + ); + let native = handle.native_request().unwrap(); + assert_eq!( + native.input, + Some(pb::generate_request::Input::Prompt(prompt.into())) + ); + assert_eq!(native.media.len(), 1); + assert_eq!(native.media[0].modality, pb::Modality::Audio as i32); + assert_eq!( + native.media[0].source, + Some(pb::media_item::Source::DataUri(uri.into())) + ); + } + assert_eq!(fixture.discovery.calls.load(Ordering::SeqCst), 1); + engine.cleanup().await.unwrap(); + fixture.shutdown().await; +} From ff4aae6a701afa3a7e5ace93c81892cfb1528d76 Mon Sep 17 00:00:00 2001 From: krishung5 Date: Fri, 30 Oct 2026 08:55:47 -0700 Subject: [PATCH 57/61] ci(trtllm): validate sidecar audio against upstream nightlies Signed-off-by: krishung5 --- .../shared-test-trtllm-nightly-image.yml | 6 +- lib/sidecar/trtllm/ci/nightly.json | 6 +- lib/sidecar/trtllm/ci/run_image_test.py | 101 ++++++++++++------ .../trtllm/ci/tests/test_run_media_test.py | 39 +++++++ .../sidecar/trtllm/ci/whisper.yaml | 1 + lib/sidecar/trtllm/src/tests/e2e.rs | 31 +++++- .../trtllm_whisper_chat_template.jinja | 3 - 7 files changed, 147 insertions(+), 40 deletions(-) rename tests/serve/fixtures/trtllm_whisper_agg.yaml => lib/sidecar/trtllm/ci/whisper.yaml (94%) delete mode 100644 tests/serve/fixtures/trtllm_whisper_chat_template.jinja diff --git a/.github/workflows/shared-test-trtllm-nightly-image.yml b/.github/workflows/shared-test-trtllm-nightly-image.yml index 12059b72ae86..c5b5e01a1083 100644 --- a/.github/workflows/shared-test-trtllm-nightly-image.yml +++ b/.github/workflows/shared-test-trtllm-nightly-image.yml @@ -61,7 +61,7 @@ jobs: strategy: fail-fast: false matrix: - media: [image, video] + media: [image, video, audio] needs: build runs-on: [gpu-1, amd64, v2, us-west-2] timeout-minutes: 30 @@ -88,6 +88,10 @@ jobs: set -euo pipefail source lib/sidecar/trtllm/launch/common.sh trtllm_ensure_openengine_bindings python3 + - name: Install TRT-LLM video decoder + if: matrix.media == 'video' + run: >- + python3 components/src/dynamo/common/utils/install_media_decoders.py trtllm - name: Run live ${{ matrix.media }} integration test env: HF_TOKEN: ${{ secrets.HF_TOKEN }} diff --git a/lib/sidecar/trtllm/ci/nightly.json b/lib/sidecar/trtllm/ci/nightly.json index 67b4986ed17c..40f4fd76f6e6 100644 --- a/lib/sidecar/trtllm/ci/nightly.json +++ b/lib/sidecar/trtllm/ci/nightly.json @@ -1,5 +1,9 @@ { "repository": "nvcr.io/nvidia/tensorrt-llm/release", "model": "Qwen/Qwen3-VL-2B-Instruct", - "model_revision": "89644892e4d85e24eaac8bacfd4f463576704203" + "model_revision": "89644892e4d85e24eaac8bacfd4f463576704203", + "audio": { + "model": "openai/whisper-tiny", + "model_revision": "169d4a4341b33bc18d8881c4b69c2e104e1cc0af" + } } diff --git a/lib/sidecar/trtllm/ci/run_image_test.py b/lib/sidecar/trtllm/ci/run_image_test.py index 09de9a343801..2c71cc983817 100755 --- a/lib/sidecar/trtllm/ci/run_image_test.py +++ b/lib/sidecar/trtllm/ci/run_image_test.py @@ -17,6 +17,7 @@ import sys import time import traceback +import urllib.request import xml.etree.ElementTree as ET from pathlib import Path @@ -73,12 +74,40 @@ ], "context_length": 2048, }, + "audio": { + "prompt": "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>", + "token_ids": [], + "context_length": 1500, + }, } VIDEO_PATH = ( Path(__file__).resolve().parents[3] / "llm/tests/data/media/triangle_240p_10.mp4" ) VIDEO_SHA256 = "04171a85ee3ba5ac9118b0a5858adf0892b49474cdf15320e32599a3d1ab9919" +AUDIO_URL = ( + "https://raw.githubusercontent.com/yuekaizhang/Triton-ASR-Client/" + "8e35ed77aeece087fa53abf2a68a282dc13efc8c/" + "datasets/mini_en/wav/1221-135766-0002.wav" +) +AUDIO_SHA256 = "65b222837919ccbb924a4e1077413ea7cc6af3e68b663b012a9539d5c05850f0" + + +def prepare_audio(results): + with urllib.request.urlopen(AUDIO_URL, timeout=30) as response: + data = response.read(1024 * 1024 + 1) + if hashlib.sha256(data).hexdigest() != AUDIO_SHA256: + raise ValueError("Audio fixture does not match the validated speech sample") + path = results / "audio-data-uri.txt" + path.write_text("data:audio/wav;base64," + base64.b64encode(data).decode("ascii")) + return path + + +def model_config(config, media): + if media == "audio": + return {**config, **config["audio"]} + return config + def video_data_uri(path=VIDEO_PATH): data = path.read_bytes() @@ -95,13 +124,15 @@ def validate_response(media, caption, response, input_token_count): colors = [word for word in words if word in ("red", "blue")] if colors[:2] != ["red", "blue"]: raise AssertionError(f"Expected red, then blue, got: {caption}") + elif media == "audio": + if "thoughts affected hester" not in " ".join(words): + raise AssertionError(f"Unexpected speech transcription: {caption}") elif not {"triangle", "triangular"}.intersection(words): raise AssertionError( f"Expected a triangle in the video description, got: {caption}" ) - if ( - response["completion_tokens"] <= 0 - or response["prompt_tokens"] <= input_token_count + if response["completion_tokens"] <= 0 or ( + media != "audio" and response["prompt_tokens"] <= input_token_count ): raise AssertionError(f"Missing authoritative {media} token usage: {response}") @@ -166,7 +197,7 @@ def run(args, results): fixture = FIXTURES[args.media] test = f"tests::e2e::e2e_real_openengine_{args.media}" - config = json.loads(args.config.read_text()) + config = model_config(json.loads(args.config.read_text()), args.media) identity = { **config, "media": args.media, @@ -180,6 +211,7 @@ def run(args, results): ) video = video_data_uri() if args.media == "video" else None + audio_path = prepare_audio(results) if args.media == "audio" else None binary = args.binary.resolve() binary.chmod(binary.stat().st_mode | 0o111) @@ -195,40 +227,44 @@ def run(args, results): config["model"], revision=config["model_revision"] ) tokenizer = AutoTokenizer.from_pretrained(model_path, local_files_only=True) - # The fixed Rust fixture uses Qwen3-VL's rendering of this exact message. - # Check it against the pinned tokenizer so a template change fails loudly. - token_ids = tokenizer.apply_chat_template( - [ - { - "role": "user", - "content": [ - {"type": args.media}, - {"type": "text", "text": fixture["prompt"]}, - ], - } - ], - tokenize=True, - add_generation_prompt=True, - ) expected_ids = fixture["token_ids"] - token_ids = ( - token_ids["input_ids"] - if isinstance(token_ids, dict) or hasattr(token_ids, "input_ids") - else token_ids - ) - if token_ids != expected_ids: - raise RuntimeError(f"{args.media} fixture tokenizer mismatch: {token_ids}") + if args.media != "audio": + # The fixed Rust fixture uses Qwen3-VL's rendering of this exact message. + # Check it against the pinned tokenizer so a template change fails loudly. + token_ids = tokenizer.apply_chat_template( + [ + { + "role": "user", + "content": [ + {"type": args.media}, + {"type": "text", "text": fixture["prompt"]}, + ], + } + ], + tokenize=True, + add_generation_prompt=True, + ) + token_ids = ( + token_ids["input_ids"] + if isinstance(token_ids, dict) or hasattr(token_ids, "input_ids") + else token_ids + ) + if token_ids != expected_ids: + raise RuntimeError(f"{args.media} fixture tokenizer mismatch: {token_ids}") with socket.socket() as listener: listener.bind(("127.0.0.1", 0)) port = listener.getsockname()[1] options = results / "engine.yaml" context_length = fixture["context_length"] - options.write_text( - f"max_batch_size: 1\nmax_num_tokens: {context_length}\n" - f"kv_cache_config:\n max_tokens: {context_length}\n" - " free_gpu_memory_fraction: 0.15\n" - ) + if args.media == "audio": + options.write_text(Path(__file__).with_name("whisper.yaml").read_text()) + else: + options.write_text( + f"max_batch_size: 1\nmax_num_tokens: {context_length}\n" + f"kv_cache_config:\n max_tokens: {context_length}\n" + " free_gpu_memory_fraction: 0.15\n" + ) result_path = results / "response.json" engine = None try: @@ -264,6 +300,9 @@ def run(args, results): ) if video is not None: env["TRTLLM_E2E_VIDEO_DATA_URI"] = video + if audio_path is not None: + env["TRTLLM_E2E_AUDIO_DATA_URI_FILE"] = str(audio_path) + env["TRTLLM_E2E_AUDIO_PROMPT"] = fixture["prompt"] run_rust_test(binary, env, test, results) finally: stop_engine(engine) diff --git a/lib/sidecar/trtllm/ci/tests/test_run_media_test.py b/lib/sidecar/trtllm/ci/tests/test_run_media_test.py index 4237da921f35..c464a4edbf74 100644 --- a/lib/sidecar/trtllm/ci/tests/test_run_media_test.py +++ b/lib/sidecar/trtllm/ci/tests/test_run_media_test.py @@ -45,6 +45,45 @@ def test_video_requires_expanded_prompt_and_completion_usage(self): ): runner.validate_response("video", "A rotating triangle.", response, 21) + def test_audio_requires_expected_transcription_and_completion(self): + response = {"prompt_tokens": 0, "completion_tokens": 17} + runner.validate_response("audio", "Thoughts affected Hester.", response, 0) + with self.assertRaisesRegex(AssertionError, "transcription"): + runner.validate_response("audio", "An unrelated sentence.", response, 0) + with self.assertRaisesRegex(AssertionError, "token usage"): + runner.validate_response( + "audio", + "Thoughts affected Hester.", + {"prompt_tokens": 0, "completion_tokens": 0}, + 0, + ) + + def test_audio_selects_whisper_without_changing_image_identity(self): + config = { + "model": "vision", + "model_revision": "vision-revision", + "image": "official@sha256:digest", + "upstream_commit": "commit", + "audio": {"model": "whisper", "model_revision": "audio-revision"}, + } + selected = runner.model_config(config, "audio") + self.assertEqual(selected["model"], "whisper") + self.assertEqual(selected["model_revision"], "audio-revision") + self.assertEqual(selected["image"], config["image"]) + self.assertEqual(config["model"], "vision") + + def test_changed_audio_fixture_is_rejected(self): + with tempfile.TemporaryDirectory() as tmp: + with ( + patch.object(runner.urllib.request, "urlopen") as fetch, + self.assertRaisesRegex(ValueError, "Audio fixture"), + ): + fetch.return_value.__enter__.return_value.read.return_value = ( + b"wrong audio" + ) + runner.prepare_audio(Path(tmp)) + self.assertFalse((Path(tmp) / "audio-data-uri.txt").exists()) + def test_video_data_uri_contains_the_checked_in_clip(self): uri = runner.video_data_uri() header, encoded = uri.split(",", 1) diff --git a/tests/serve/fixtures/trtllm_whisper_agg.yaml b/lib/sidecar/trtllm/ci/whisper.yaml similarity index 94% rename from tests/serve/fixtures/trtllm_whisper_agg.yaml rename to lib/sidecar/trtllm/ci/whisper.yaml index 7bfa6f1e2c19..a66d9fb01331 100644 --- a/tests/serve/fixtures/trtllm_whisper_agg.yaml +++ b/lib/sidecar/trtllm/ci/whisper.yaml @@ -17,3 +17,4 @@ kv_cache_config: free_gpu_memory_fraction: 0.2 cross_kv_cache_fraction: 0.5 use_kv_cache_manager_v2: true +guided_decoding_backend: null diff --git a/lib/sidecar/trtllm/src/tests/e2e.rs b/lib/sidecar/trtllm/src/tests/e2e.rs index 8a0f314ff1ae..c7c8487aab46 100644 --- a/lib/sidecar/trtllm/src/tests/e2e.rs +++ b/lib/sidecar/trtllm/src/tests/e2e.rs @@ -514,7 +514,7 @@ async fn e2e_real_openengine_video() { ); } -/// `TRTLLM_E2E_AUDIO_DATA_URI` must contain a complete `data:audio/...` URI. +/// Set `TRTLLM_E2E_AUDIO_DATA_URI` or `TRTLLM_E2E_AUDIO_DATA_URI_FILE` to a data URI. #[tokio::test] #[ignore = "requires a live audio-capable TensorRT-LLM OpenEngine server"] async fn e2e_real_openengine_audio() { @@ -522,14 +522,20 @@ async fn e2e_real_openengine_audio() { .expect("set TRTLLM_E2E_ENDPOINT, e.g. http://127.0.0.1:50051"); let model = std::env::var("TRTLLM_E2E_MODEL").unwrap_or_else(|_| "openai/whisper-tiny".to_string()); - let audio = std::env::var("TRTLLM_E2E_AUDIO_DATA_URI") - .expect("set TRTLLM_E2E_AUDIO_DATA_URI to an audio data URI"); + let audio = match std::env::var("TRTLLM_E2E_AUDIO_DATA_URI_FILE") { + Ok(path) => std::fs::read_to_string(path).expect("read audio data URI"), + Err(_) => std::env::var("TRTLLM_E2E_AUDIO_DATA_URI") + .expect("set TRTLLM_E2E_AUDIO_DATA_URI or TRTLLM_E2E_AUDIO_DATA_URI_FILE"), + }; + let prompt = std::env::var("TRTLLM_E2E_AUDIO_PROMPT") + .unwrap_or_else(|_| "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>".into()); let engine = TrtllmSidecarEngine::new( GrpcEndpoint::parse(&endpoint, "--grpc-endpoint").expect("valid endpoint"), transport(1), ConfiguredModel { source: model.clone(), context_length: None, + ..Default::default() }, AGG, ); @@ -540,7 +546,7 @@ async fn e2e_real_openengine_audio() { let mut request = PreprocessedRequest::builder() .model(model) - .token_ids(vec![50258, 50259, 50359, 50363]) + .token_ids(vec![]) .stop_conditions(StopConditions { max_tokens: Some(96), ..Default::default() @@ -552,6 +558,7 @@ async fn e2e_real_openengine_audio() { .output_options(OutputOptions::default()) .build() .expect("request"); + request.extra_args = Some(json!({"formatted_prompt": prompt})); request.multi_modal_data = Some(HashMap::from([( "audio_url".to_string(), vec![MultimodalData::RawUrl(audio)], @@ -572,5 +579,21 @@ async fn e2e_real_openengine_audio() { usage.prompt_tokens, usage.completion_tokens ); + if let Ok(path) = std::env::var("TRTLLM_E2E_RESULT") { + std::fs::write( + path, + serde_json::to_vec(&json!({ + "token_ids": generated, + "prompt_tokens": usage.prompt_tokens, + "completion_tokens": usage.completion_tokens, + })) + .expect("serialize audio result"), + ) + .expect("write audio result"); + } + assert!( + usage.completion_tokens > 0, + "expected completion token usage" + ); assert!(!generated.is_empty(), "expected audio-conditioned output"); } diff --git a/tests/serve/fixtures/trtllm_whisper_chat_template.jinja b/tests/serve/fixtures/trtllm_whisper_chat_template.jinja deleted file mode 100644 index 85f926c7bb4d..000000000000 --- a/tests/serve/fixtures/trtllm_whisper_chat_template.jinja +++ /dev/null @@ -1,3 +0,0 @@ -{#- SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -#} -{#- SPDX-License-Identifier: Apache-2.0 -#} -{#- An empty prompt selects the checkpoint's default language and transcription task. -#} From a1f090a8f16e27d3c36fb27ddbd19bd2928395a3 Mon Sep 17 00:00:00 2001 From: krishung5 Date: Fri, 30 Oct 2026 13:30:10 -0700 Subject: [PATCH 58/61] test(trtllm): cover remote URL policy for video inputs Signed-off-by: krishung5 --- .../testkit/tests/trtllm_multimodal.rs | 46 ++++++++++++++++--- 1 file changed, 39 insertions(+), 7 deletions(-) diff --git a/lib/sidecar/testkit/tests/trtllm_multimodal.rs b/lib/sidecar/testkit/tests/trtllm_multimodal.rs index 1b937897ed51..9f77dae3c1b5 100644 --- a/lib/sidecar/testkit/tests/trtllm_multimodal.rs +++ b/lib/sidecar/testkit/tests/trtllm_multimodal.rs @@ -591,17 +591,49 @@ async fn check_remote_url_policy(allow_remote: bool) { let control = Controller::::default(); let mut fixture = Fixture::start(control.clone(), FixtureConfig::default()).await; advertise_images(&fixture); + fixture + .discovery + .advertises_video + .store(true, Ordering::SeqCst); let engine = fixture.engine().await; engine.start(0).await.unwrap(); - for image in [ - MultimodalData::RawUrl("http://example.com/image.png".into()), - MultimodalData::RawUrl("HTTPS://example.com/image.png".into()), - MultimodalData::Url("https://example.com/image.png".parse().unwrap()), - MultimodalData::RawUrl("data:image/png;base64,aW1hZ2U=".into()), + for (key, media) in [ + ( + "image_url", + MultimodalData::RawUrl("http://example.com/image.png".into()), + ), + ( + "image_url", + MultimodalData::RawUrl("HTTPS://example.com/image.png".into()), + ), + ( + "image_url", + MultimodalData::Url("https://example.com/image.png".parse().unwrap()), + ), + ( + "image_url", + MultimodalData::RawUrl("data:image/png;base64,aW1hZ2U=".into()), + ), + ( + "video_url", + MultimodalData::RawUrl("http://example.com/video.mp4".into()), + ), + ( + "video_url", + MultimodalData::RawUrl("HTTPS://example.com/video.mp4".into()), + ), + ( + "video_url", + MultimodalData::Url("https://example.com/video.mp4".parse().unwrap()), + ), + ( + "video_url", + MultimodalData::RawUrl("data:video/mp4;base64,dmlkZW8=".into()), + ), ] { - let inline = matches!(&image, MultimodalData::RawUrl(url) if url.starts_with("data:")); + let inline = matches!(&media, MultimodalData::RawUrl(url) if url.starts_with("data:")); let mut req = image_request(); - req.multi_modal_data = Some(HashMap::from([("image_url".into(), vec![image])])); + req.multi_modal_data = Some(HashMap::from([(key.into(), vec![media])])); req.multi_modal_uuids = None; let ctx = mock_context(); let handle = control.request(ctx.id(), RequestPlan::default()); From 3b118475749cb924e3388807f2e90408ad65f645 Mon Sep 17 00:00:00 2001 From: krishung5 Date: Fri, 30 Oct 2026 14:14:32 -0700 Subject: [PATCH 59/61] test(trtllm): cover remote URL policy for audio inputs Signed-off-by: krishung5 --- lib/sidecar/testkit/tests/trtllm_multimodal.rs | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/lib/sidecar/testkit/tests/trtllm_multimodal.rs b/lib/sidecar/testkit/tests/trtllm_multimodal.rs index f7abc9e67921..0206c877299c 100644 --- a/lib/sidecar/testkit/tests/trtllm_multimodal.rs +++ b/lib/sidecar/testkit/tests/trtllm_multimodal.rs @@ -630,7 +630,7 @@ fn remote_url_policy_is_enforced_before_engine_dispatch() { async fn check_remote_url_policy(allow_remote: bool) { let control = Controller::::default(); let mut fixture = Fixture::start(control.clone(), FixtureConfig::default()).await; - advertise_media(&fixture, &["image", "video"]); + advertise_media(&fixture, &["image", "video", "audio"]); let engine = fixture.engine().await; engine.start(0).await.unwrap(); for (key, media) in [ @@ -666,6 +666,22 @@ async fn check_remote_url_policy(allow_remote: bool) { "video_url", MultimodalData::RawUrl("data:video/mp4;base64,dmlkZW8=".into()), ), + ( + "audio_url", + MultimodalData::RawUrl("http://example.com/audio.wav".into()), + ), + ( + "audio_url", + MultimodalData::RawUrl("HTTPS://example.com/audio.wav".into()), + ), + ( + "audio_url", + MultimodalData::Url("https://example.com/audio.wav".parse().unwrap()), + ), + ( + "audio_url", + MultimodalData::RawUrl("data:audio/wav;base64,YXVkaW8=".into()), + ), ] { let inline = matches!(&media, MultimodalData::RawUrl(url) if url.starts_with("data:")); let mut req = image_request(); From 19ae6c833c91386bff1eec28ee80e2638431c363 Mon Sep 17 00:00:00 2001 From: krishung5 Date: Fri, 30 Oct 2026 15:15:34 -0700 Subject: [PATCH 60/61] fix(trtllm): preserve terminal usage before hiding stop tokens Signed-off-by: krishung5 --- lib/sidecar/trtllm/src/convert.rs | 18 ++----- .../trtllm/src/convert/response_tests.rs | 47 +++++++++++++++++++ 2 files changed, 51 insertions(+), 14 deletions(-) diff --git a/lib/sidecar/trtllm/src/convert.rs b/lib/sidecar/trtllm/src/convert.rs index c6fc1285b7ef..9f6fab3448d0 100644 --- a/lib/sidecar/trtllm/src/convert.rs +++ b/lib/sidecar/trtllm/src/convert.rs @@ -595,17 +595,13 @@ fn has_visible_stop_tokens(request: &PreprocessedRequest) -> bool { .is_some_and(|ids| !ids.is_empty()) } -/// Stop IDs the engine trims by default but returns once -/// `include_stop_in_output` is set for visible stops. +/// Hidden stop IDs to remove when the engine confirms they ended generation. fn retained_hidden_stop_ids(request: &PreprocessedRequest) -> BTreeSet { let visible = request .stop_conditions .stop_token_ids_visible .as_deref() .unwrap_or_default(); - if visible.is_empty() { - return BTreeSet::new(); - } stop_token_ids(request) .into_iter() .filter(|id| !visible.contains(id)) @@ -790,10 +786,8 @@ pub(crate) struct ResponseState { prompt_tokens: u32, completion_tokens: u32, output_logprobs: Option, - /// Every stop ID when visible stops are set, empty otherwise. The engine - /// stops on these, so one at the end of a delta is held for the terminal. - /// OpenEngine sends the matched stop before the `finished` event carrying - /// usage, and the frontend decoder drops everything after a visible stop. + /// Hold a trailing stop until terminal usage arrives. The frontend decoder + /// can terminate on hidden or visible stops before reading that usage. held_stop_ids: BTreeSet, /// See [`retained_hidden_stop_ids`]. Dropped when the terminal matched it. hidden_stop_ids: BTreeSet, @@ -834,11 +828,7 @@ impl ResponseState { prompt_tokens: request.token_ids.len() as u32, completion_tokens: 0, output_logprobs: request.output_options.logprobs, - held_stop_ids: if has_visible_stop_tokens(request) { - stop_token_ids(request).into_iter().collect() - } else { - BTreeSet::new() - }, + held_stop_ids: stop_token_ids(request).into_iter().collect(), hidden_stop_ids: retained_hidden_stop_ids(request), pending_stop: None, phase: if mode.is_prefill() { diff --git a/lib/sidecar/trtllm/src/convert/response_tests.rs b/lib/sidecar/trtllm/src/convert/response_tests.rs index df43cb2ddb47..6dd770e54ef2 100644 --- a/lib/sidecar/trtllm/src/convert/response_tests.rs +++ b/lib/sidecar/trtllm/src/convert/response_tests.rs @@ -499,6 +499,53 @@ fn held_hidden_stop_token_is_released_unless_it_ended_the_sequence() { assert_eq!(terminal.completion_usage.unwrap().completion_tokens, 4); } +#[test] +fn hidden_stop_waits_for_terminal_cache_usage_without_visible_stops() { + let req = request(); + let mut state = ResponseState::new(&req, AGG); + let delta = state + .convert(token_response(vec![ + logprob_token(5, -0.1), + logprob_token(2, -0.2), + ])) + .expect("convert token") + .expect("delta"); + assert_eq!(delta.token_ids, [5]); + assert!(delta.finish_reason.is_none()); + + let terminal = state + .convert(pb::GenerateResponse { + request_id: "r".to_string(), + event: Some(pb::generate_response::Event::Finished( + pb::GenerationFinished { + output_index: Some(0), + reason: pb::FinishReason::Stop as i32, + message: String::new(), + stop_match: Some(pb::StopMatch { + r#match: Some(pb::stop_match::Match::EosTokenId(2)), + }), + }, + )), + usage: Some(pb::Usage { + prompt_tokens: 218, + completion_tokens: 2, + total_tokens: 220, + cached_prompt_tokens: Some(217), + reasoning_tokens: None, + }), + }) + .expect("convert finished") + .expect("terminal"); + assert!(terminal.token_ids.is_empty()); + assert_eq!(terminal.finish_reason, Some(FinishReason::Stop)); + let usage = terminal.completion_usage.unwrap(); + assert_eq!(usage.prompt_tokens, 218); + assert_eq!( + usage.prompt_tokens_details.unwrap().cached_tokens, + Some(217) + ); +} + #[test] fn visible_stop_token_arrives_with_the_terminal_usage() { let mut req = request(); From 024d807372bcb2f01796cbb3878762e0340fbe80 Mon Sep 17 00:00:00 2001 From: krishung5 Date: Mon, 2 Nov 2026 14:25:16 -0800 Subject: [PATCH 61/61] fix(trtllm): preserve audio launch settings and validate usage Signed-off-by: krishung5 --- .../shared-test-trtllm-nightly-image.yml | 4 +- lib/sidecar/trtllm/README.md | 41 ++++++- lib/sidecar/trtllm/ci/run_image_test.py | 5 +- .../trtllm/ci/tests/test_agg_launch.py | 102 ++++++++++++++++++ .../trtllm/ci/tests/test_run_media_test.py | 19 ++-- lib/sidecar/trtllm/launch/agg.sh | 5 +- lib/sidecar/trtllm/launch/common.sh | 24 +++-- 7 files changed, 175 insertions(+), 25 deletions(-) create mode 100644 lib/sidecar/trtllm/ci/tests/test_agg_launch.py diff --git a/.github/workflows/shared-test-trtllm-nightly-image.yml b/.github/workflows/shared-test-trtllm-nightly-image.yml index c5b5e01a1083..0382a42d7ee6 100644 --- a/.github/workflows/shared-test-trtllm-nightly-image.yml +++ b/.github/workflows/shared-test-trtllm-nightly-image.yml @@ -27,7 +27,9 @@ jobs: steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Test nightly selection and media harness - run: python3 -m unittest discover -s lib/sidecar/trtllm/ci/tests -v + run: | + python3 -m pip install 'PyYAML==6.0.3' + python3 -m unittest discover -s lib/sidecar/trtllm/ci/tests -v - name: Resolve latest upstream nightly id: config run: >- diff --git a/lib/sidecar/trtllm/README.md b/lib/sidecar/trtllm/README.md index 251d17708358..ff9411bec26e 100644 --- a/lib/sidecar/trtllm/README.md +++ b/lib/sidecar/trtllm/README.md @@ -31,7 +31,7 @@ supported, even when `--context-length` is supplied. > The standalone launch and deployment examples use > `nvcr.io/nvidia/tensorrt-llm/release:1.4.0rc3.dev202610290007`, which includes > image/video URL passthrough, native KVCM2 event streaming, and OpenEngine routing load support. -> The nightly CI job tests image and video passthrough against the latest regular upstream nightly; +> The nightly CI job tests image, video, and audio passthrough against the latest regular upstream nightly; > it also supports manual dispatch. The examples below stay pinned to the validated version. ## Support matrix @@ -65,6 +65,45 @@ curl -s localhost:8000/v1/chat/completions \ -d '{"model":"Qwen/Qwen3-0.6B","messages":[{"role":"user","content":"Hello"}],"max_tokens":32}' ``` +### Audio (Whisper) + +Audio requires a runtime containing upstream audio support, such as the validated +`nvcr.io/nvidia/tensorrt-llm/release:1.4.0rc3.dev202610300406`; the older image/video +example pin above predates audio. The nightly audio lane runs real Whisper-tiny +inference and checks the transcript and authoritative token usage. Full frontend +HTTP CI is tracked separately from that adapter-to-engine test. + +Whisper-tiny has no chat template and accepts text task prefixes rather than +token-ID input. Supply `--custom-jinja-template` to the sidecar. An empty template +selects the checkpoint's default transcription task; a template may instead emit +a complete language/task prefix, for example +`<|startoftranscript|><|fr|><|translate|><|notimestamps|>`. + +Run these processes in separate terminals with the appropriate Dynamo and +TensorRT-LLM environments, using `DYN_DISCOVERY_BACKEND=file` on a single host: + +```bash +python3 -m dynamo.frontend --migration-limit 0 + +# In the audio-capable TensorRT-LLM environment, from the repository root: +python3 -m tensorrt_llm.commands.serve openai/whisper-tiny \ + --grpc --grpc-protocol openengine --host 127.0.0.1 --port 30001 \ + --max_seq_len 1500 --extra_llm_api_options lib/sidecar/trtllm/ci/whisper.yaml + +# In the Dynamo environment: +WHISPER_TEMPLATE=$(mktemp --suffix=.jinja) +dynamo-trtllm-sidecar --grpc-endpoint 127.0.0.1:30001 \ + --model-path openai/whisper-tiny --custom-jinja-template "$WHISPER_TEMPLATE" +``` + +Send audio using a chat content item with `"type":"audio_url"` and +`"audio_url":{"url":"https://.../speech.wav"}` (or a data URI). +For text-input models, **keep frontend `--migration-limit 0`**: retry migration +currently extends token IDs, while these engines receive the original formatted +text, which can restart a partially delivered transcript. Worker-advertised +migration restrictions need separate frontend support. The aggregated launcher +preserves explicit engine settings, including `guided_decoding_backend: null`. + ## Deploy on Kubernetes The manifests pin compatible Dynamo frontend and TensorRT-LLM runtime nightlies diff --git a/lib/sidecar/trtllm/ci/run_image_test.py b/lib/sidecar/trtllm/ci/run_image_test.py index 2c71cc983817..bd230552fe61 100755 --- a/lib/sidecar/trtllm/ci/run_image_test.py +++ b/lib/sidecar/trtllm/ci/run_image_test.py @@ -131,8 +131,9 @@ def validate_response(media, caption, response, input_token_count): raise AssertionError( f"Expected a triangle in the video description, got: {caption}" ) - if response["completion_tokens"] <= 0 or ( - media != "audio" and response["prompt_tokens"] <= input_token_count + if ( + response["completion_tokens"] <= 0 + or response["prompt_tokens"] <= input_token_count ): raise AssertionError(f"Missing authoritative {media} token usage: {response}") diff --git a/lib/sidecar/trtllm/ci/tests/test_agg_launch.py b/lib/sidecar/trtllm/ci/tests/test_agg_launch.py new file mode 100644 index 000000000000..7991d9516623 --- /dev/null +++ b/lib/sidecar/trtllm/ci/tests/test_agg_launch.py @@ -0,0 +1,102 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import json +import os +import signal +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +import yaml + +ROOT = Path(__file__).resolve().parents[5] + + +class AggregatedLaunchTests(unittest.TestCase): + def test_engine_receives_base_config_defaults_and_profiler_overrides(self): + cases = [ + (None, False, "xgrammar"), + ("max_batch_size: 1\n", False, "xgrammar"), + ("guided_decoding_backend: guidance\n", False, "guidance"), + ((ROOT / "lib/sidecar/trtllm/ci/whisper.yaml").read_text(), False, None), + ("guided_decoding_backend: guidance\n", True, None), + ] + for config, disabled, expected in cases: + with self.subTest( + config=config, disabled=disabled + ), tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + wrapper = root / "python3" + wrapper.write_text( + f"#!{sys.executable}\n" + "import json, os, signal, sys\n" + "from pathlib import Path\n" + "import yaml\n" + "args = sys.argv[1:]\n" + "if args[:1] == ['-c']:\n" + " sys.exit(0)\n" + "if args[:2] == ['-m', 'tensorrt_llm.commands.serve']:\n" + " config = Path(args[args.index('--extra_llm_api_options') + 1])\n" + " Path(os.environ['CAPTURE']).write_text(json.dumps(yaml.safe_load(config.read_text())))\n" + " sys.exit(0)\n" + "if args[:1] == ['-m'] or '--grpc-endpoint' in args:\n" + " signal.pause()\n" + f"os.execv({sys.executable!r}, [{sys.executable!r}, *args])\n" + ) + wrapper.chmod(0o755) + capture = root / "captured.json" + env = { + key: value + for key, value in os.environ.items() + if not key.startswith(("TRTLLM_", "DYN_", "_PROFILE_OVERRIDE_")) + and key not in ("AGG_ENGINE_ARGS", "MODEL") + } + env.update( + { + "PATH": f"{root}:{os.environ['PATH']}", + "TRTLLM_PYTHON": str(wrapper), + "DYNAMO_SIDECAR_BIN": str(wrapper), + "CAPTURE": str(capture), + "TMPDIR": str(root), + "_PROFILE_OVERRIDE_TRTLLM_MAX_TOTAL_TOKENS": "512", + } + ) + if config is not None: + base = root / "base.yaml" + base.write_text(config) + env["AGG_ENGINE_ARGS"] = str(base) + if disabled: + env["TRTLLM_DISABLE_GUIDED_DECODING"] = "1" + process = subprocess.Popen( + ["bash", str(ROOT / "lib/sidecar/trtllm/launch/agg.sh")], + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + start_new_session=True, + ) + try: + stdout, stderr = process.communicate(timeout=10) + finally: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait(timeout=5) + self.assertEqual(process.returncode, 0, stdout + stderr) + actual = json.loads(capture.read_text()) + self.assertEqual(actual["guided_decoding_backend"], expected) + self.assertEqual(actual["kv_cache_config"]["max_tokens"], 512) + if config is not None: + for key, value in yaml.safe_load(config).items(): + if key == "kv_cache_config": + self.assertEqual(actual[key], {**value, "max_tokens": 512}) + elif key != "guided_decoding_backend": + self.assertEqual(actual[key], value) + + +if __name__ == "__main__": + unittest.main() diff --git a/lib/sidecar/trtllm/ci/tests/test_run_media_test.py b/lib/sidecar/trtllm/ci/tests/test_run_media_test.py index c464a4edbf74..642004292218 100644 --- a/lib/sidecar/trtllm/ci/tests/test_run_media_test.py +++ b/lib/sidecar/trtllm/ci/tests/test_run_media_test.py @@ -45,18 +45,19 @@ def test_video_requires_expanded_prompt_and_completion_usage(self): ): runner.validate_response("video", "A rotating triangle.", response, 21) - def test_audio_requires_expected_transcription_and_completion(self): - response = {"prompt_tokens": 0, "completion_tokens": 17} + def test_audio_requires_expected_transcription_and_authoritative_usage(self): + response = {"prompt_tokens": 4, "completion_tokens": 17} runner.validate_response("audio", "Thoughts affected Hester.", response, 0) with self.assertRaisesRegex(AssertionError, "transcription"): runner.validate_response("audio", "An unrelated sentence.", response, 0) - with self.assertRaisesRegex(AssertionError, "token usage"): - runner.validate_response( - "audio", - "Thoughts affected Hester.", - {"prompt_tokens": 0, "completion_tokens": 0}, - 0, - ) + for usage in ( + {"prompt_tokens": 0, "completion_tokens": 17}, + {"prompt_tokens": 4, "completion_tokens": 0}, + ): + with self.subTest(usage=usage), self.assertRaisesRegex( + AssertionError, "token usage" + ): + runner.validate_response("audio", "Thoughts affected Hester.", usage, 0) def test_audio_selects_whisper_without_changing_image_identity(self): config = { diff --git a/lib/sidecar/trtllm/launch/agg.sh b/lib/sidecar/trtllm/launch/agg.sh index c409591135ee..5b01ffee0551 100755 --- a/lib/sidecar/trtllm/launch/agg.sh +++ b/lib/sidecar/trtllm/launch/agg.sh @@ -99,7 +99,7 @@ HTTP_PORT="${DYN_HTTP_PORT:-8000}" # Dynamo sends a required or named tool_choice as a JSON schema, which # TensorRT-LLM enforces only with a guided-decoding backend. TRTLLM_EXTRA_CONFIG=$(mktemp "${TMPDIR:-/tmp}/dynamo-trtllm-sidecar.XXXXXX.yaml") -TRTLLM_REQUIRED_OVERRIDES='{"guided_decoding_backend": "xgrammar"}' +TRTLLM_REQUIRED_OVERRIDES='{}' if [[ "${TRTLLM_DISABLE_GUIDED_DECODING:-0}" == "1" ]]; then TRTLLM_REQUIRED_OVERRIDES='{"guided_decoding_backend": null}' fi @@ -109,7 +109,8 @@ trtllm_write_merged_config \ "${AGG_ENGINE_ARGS:-}" \ "$TRTLLM_OVERRIDES" \ "$TRTLLM_EXTRA_CONFIG" \ - "$TRTLLM_PYTHON" + "$TRTLLM_PYTHON" \ + '{"guided_decoding_backend":"xgrammar"}' print_launch_banner "Launching TensorRT-LLM OpenEngine-gRPC Sidecar (1 GPU)" "$MODEL" "$HTTP_PORT" \ "TensorRT-LLM gRPC: 127.0.0.1:${TRTLLM_GRPC_PORT}" \ diff --git a/lib/sidecar/trtllm/launch/common.sh b/lib/sidecar/trtllm/launch/common.sh index 5831d4a7b6ed..c622336527af 100755 --- a/lib/sidecar/trtllm/launch/common.sh +++ b/lib/sidecar/trtllm/launch/common.sh @@ -83,23 +83,23 @@ trtllm_resolve_context_length() { # Write a TensorRT-LLM options file by recursively applying JSON overrides to # an optional YAML/JSON base. Recursive merging preserves sibling settings in # nested blocks such as kv_cache_config when the test profiler supplies a -# max_tokens override. +# max_tokens override. Optional fifth argument supplies defaults for absent keys. trtllm_write_merged_config() { local base_config="$1" local overrides_json="$2" local output="$3" local python="$4" - - if [[ -z "$base_config" ]]; then - printf '%s\n' "$overrides_json" > "$output" - return 0 + local defaults_json='{}' + if [[ $# -ge 5 ]]; then + defaults_json="$5" fi - if [[ ! -r "$base_config" ]]; then + + if [[ -n "$base_config" && ! -r "$base_config" ]]; then echo "TensorRT-LLM engine config is not readable: $base_config" >&2 return 1 fi - "$python" - "$base_config" "$overrides_json" "$output" <<'PY' + "$python" - "$base_config" "$overrides_json" "$output" "$defaults_json" <<'PY' import json import sys @@ -114,13 +114,17 @@ def merge(base, overrides): base[key] = value -base_path, overrides_json, output_path = sys.argv[1:] -with open(base_path, encoding="utf-8") as source: - config = yaml.safe_load(source) +base_path, overrides_json, output_path, defaults_json = sys.argv[1:] +config = None +if base_path: + with open(base_path, encoding="utf-8") as source: + config = yaml.safe_load(source) if config is None: config = {} if not isinstance(config, dict): raise TypeError(f"TensorRT-LLM engine config must be a mapping: {base_path}") +for key, value in json.loads(defaults_json).items(): + config.setdefault(key, value) merge(config, json.loads(overrides_json)) with open(output_path, "w", encoding="utf-8") as destination: yaml.safe_dump(config, destination, sort_keys=False)