feat(sidecar): speak TensorRT-LLM's OpenEngine gRPC API

.github/filters.yaml

@@ -357,6 +357,7 @@ sidecar:
   - 'tests/router/helper.py'
   - 'tests/router/router_process.py'
   - 'tests/utils/gpu_args.py'
+  - 'tests/utils/payloads.py'
   - 'tests/utils/prometheus.py'
   - 'tests/utils/router_logs.py'
   - 'tests/utils/router_nvext.py'

.github/scripts/test-filters.js

@@ -127,7 +127,7 @@ const testCases = [
     desc: 'common sidecar source avoids unrelated build and E2E filters'
   },
   {
-    file: 'lib/sidecar/trtllm/proto/trtllm_service.proto',
+    file: 'lib/sidecar/trtllm/proto/openengine/v1/generation.proto',
     expect: { sidecar: true, rust: true, core: false, frontend: false, vllm: false, sglang: false, trtllm: false },
     desc: 'sidecar proto contracts trigger Rust checks without backend E2E'
   },

.pre-commit-config.yaml

@@ -14,7 +14,7 @@
 # limitations under the License.
 
 default_install_hook_types: [pre-commit, commit-msg]
-exclude: ^(src/grpc_generated|.*\.patch$|.*/connect/.*\.py|components/src/dynamo/planner/plugins/proto/v1/plugin_pb2(_grpc)?\.pyi?$|lib/sidecar/trtllm/proto/trtllm_service\.proto$)
+exclude: ^(src/grpc_generated|.*\.patch$|.*/connect/.*\.py|components/src/dynamo/planner/plugins/proto/v1/plugin_pb2(_grpc)?\.pyi?$|lib/sidecar/trtllm/proto/openengine/v1/.*\.proto$)
 repos:
 - repo: https://github.com/timothycrosley/isort
   rev: 5.12.0

docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/tensorrt-llm/sidecar.md

@@ -2,69 +2,72 @@
 # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 title: TensorRT-LLM Sidecar
-subtitle: Run Dynamo beside a stock TensorRT-LLM engine through native gRPC.
+subtitle: Run Dynamo beside a TensorRT-LLM engine through its OpenEngine gRPC API.
 ---
 
 > [!WARNING]
 > **Experimental.** The TensorRT-LLM sidecar, launcher, packaging, and feature
 > coverage can change without notice.
 
 `dynamo-trtllm-sidecar` is a CPU-only Dynamo worker that connects to
-TensorRT-LLM's native gRPC service. It preserves the upstream engine process
-and argument surface while using Dynamo for request handling and distributed
-serving. See the
+TensorRT-LLM's OpenEngine gRPC API (`openengine.v1`), served by
+`trtllm-serve --grpc --grpc-protocol openengine`. It preserves the upstream
+engine process and argument surface while using Dynamo for request handling and
+distributed serving. See the
 [Sidecar Backends](../../../concepts/system-architecture/sidecar-backends.md) page for the common
 architecture.
 
 ## Readiness
 
 | Deployment path | Aggregated | Disaggregated |
 |---|---|---|
-| Local launcher | Validated on one GPU | Not supported |
-| Kubernetes example | Validated | Not supported |
+| Local launcher | Validated on one GPU | Validated on one GPU, both engines co-located; the launcher defaults to two |
+| Kubernetes example | Validated | Validated, prefill and decode on separate pods |
 
 This table covers launch topology only. The
 [TensorRT-LLM feature matrix](overview.md#feature-support-matrix) describes the
-in-process backend; sidecar feature parity is still under evaluation. The
-current native gRPC contract does not provide the prefill/decode handoff needed
-for disaggregated serving. See the
+in-process backend; sidecar feature parity is still under evaluation.
+Disaggregated prefill/decode is supported over the OpenEngine contract: a
+prefill worker marks its request `context_only` and returns the `PrefillReady`
+KV handoff that a decode worker replays. Running it needs an engine with a KV
+cache transceiver configured on both legs. See the
 [TensorRT-LLM sidecar README](https://github.com/ai-dynamo/dynamo/blob/main/lib/sidecar/trtllm/README.md)
 for other protocol limitations.
 
 ## Launch Locally
 
-From a Dynamo source checkout, build or install Dynamo so
-`dynamo-trtllm-sidecar` is on `PATH`. Install a TensorRT-LLM release that
-provides `tensorrt_llm.commands.serve --grpc`.
-
-Start Dynamo's local discovery services, then run the aggregated launcher:
+Build or install Dynamo from a source checkout so `dynamo-trtllm-sidecar` is on
+`PATH`. The engine must serve `--grpc --grpc-protocol openengine`, which means
+`nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc27.dev202609170000` or newer; the
+launchers install the pinned OpenEngine Python bindings, which those releases do
+not ship.
 
 ```bash
-docker compose -f dev/docker-compose.yml up -d
+docker compose -f dev/docker-compose.yml up -d          # local discovery services
 ./lib/sidecar/trtllm/launch/agg.sh --model Qwen/Qwen3-0.6B
 ```
 
-The launcher starts the Dynamo frontend, TensorRT-LLM engine, and sidecar. It
-binds TensorRT-LLM's native gRPC endpoint to loopback.
-
-Verify the frontend:
+Each launcher starts the Dynamo frontend, the engine, and the sidecar, and binds
+the engine's gRPC endpoint to loopback — it is unauthenticated and plaintext.
+`launch/disagg.sh` is the same on two GPUs, with a NIXL cache transceiver on both
+engines for the KV handoff. Either way the frontend serves the usual endpoint:
 
 ```bash
-curl localhost:8000/v1/chat/completions \
-  -H "Content-Type: application/json" \
-  -d '{
-    "model": "Qwen/Qwen3-0.6B",
-    "messages": [{"role": "user", "content": "Hello"}],
-    "max_tokens": 32
-  }'
+curl localhost:8000/v1/chat/completions -H 'Content-Type: application/json' \
+  -d '{"model":"Qwen/Qwen3-0.6B","messages":[{"role":"user","content":"Hello"}],"max_tokens":32}'
 ```
 
 ## Deploy on Kubernetes
 
-No published sidecar image is available yet. Follow the
-[Kubernetes quick start](https://github.com/ai-dynamo/dynamo/blob/main/lib/sidecar/trtllm/README.md#deploy-on-kubernetes-quick-start)
-to build `dynamo-sidecar`, which contains all three engine-specific sidecar
-executables. The TensorRT-LLM manifest runs `dynamo-trtllm-sidecar` as the
-container command and pairs it with a stock upstream TensorRT-LLM image. The
-source tree includes an
-[aggregated deployment manifest](https://github.com/ai-dynamo/dynamo/blob/main/lib/sidecar/trtllm/deploy/agg.yaml).
+The source tree ships two
+[deployment manifests](https://github.com/ai-dynamo/dynamo/tree/main/lib/sidecar/trtllm/deploy):
+`agg.yaml`, and `disagg.yaml`, which runs prefill and decode as separate worker
+pods. Both need two images you build yourself: `dynamo-sidecar`, which carries
+all three engine-specific executables, and a TensorRT-LLM image with the pinned
+OpenEngine bindings layered on — the release ships the servicer but not the
+bindings that it and the manifests' health probes import.
+
+Read the disaggregated manifest's header before applying it: it runs the engines
+over TCP/CUDA-IPC and requests no `rdma/ib`, which you add on a fabric that
+provides it. The
+[README](https://github.com/ai-dynamo/dynamo/blob/main/lib/sidecar/trtllm/README.md#deploy-on-kubernetes) has the full walkthrough.

lib/backend-common/src/engine.rs

@@ -33,7 +33,7 @@ pub use dynamo_llm::protocols::common::preprocessor::{
 pub use dynamo_llm::protocols::common::{
     FinishReason, GuidedDecodingOptions, OutputOptions, SamplingOptions, StopConditions,
 };
-pub use dynamo_protocols::types::{CompletionUsage, StopReason};
+pub use dynamo_protocols::types::{CompletionUsage, PromptTokensDetails, StopReason};
 pub use dynamo_runtime::engine::AsyncEngineContext;
 
 /// Per-request handle wrapping the runtime context. `Deref`s to

lib/backend-common/src/lib.rs

@@ -39,8 +39,9 @@ pub use engine::{
     KV_HINT_TRANSFER_CAPABILITY_KEY, KvEventPublisher, KvEventSource, KvHint, KvHintAction,
     KvSourceLocationsPayload, LLMEngine, LLMEngineOutput, LLMEngineOutputExt, LlmRegistration,
     LogProbs, Metrics, MetricsBindings, MetricsCtx, MultimodalData, OnPublisherReady,
-    OnSnapshotPublisherReady, OutputOptions, PrefillResult, PreprocessedRequest, RawEngine,
-    SamplingOptions, StopConditions, StopReason, TopLogprob, TopLogprobs, chunk, usage,
+    OnSnapshotPublisherReady, OutputOptions, PrefillResult, PreprocessedRequest,
+    PromptTokensDetails, RawEngine, SamplingOptions, StopConditions, StopReason, TopLogprob,
+    TopLogprobs, chunk, usage,
 };
 pub use error::{BackendError, DynamoError, ErrorType};
 pub use metrics::{ComponentGauges, EngineMetrics, LifecycleGauges};

lib/sidecar/Dockerfile

@@ -74,6 +74,13 @@ COPY examples/router/custom-policy-example/ examples/router/custom-policy-exampl
 COPY deploy/inference-gateway/ext-proc/ deploy/inference-gateway/ext-proc/
 COPY deploy/inference-gateway/sidecar/ deploy/inference-gateway/sidecar/
 
+# The `/src/target` cache below is keyed by a name that changes whenever a
+# build-script *input* is renamed or removed. The TensorRT-LLM sidecar's
+# vendored protos moved from `proto/trtllm_service.proto` to
+# `proto/openengine/v1/`, and a cached build-script binary compiled against the
+# old layout keeps asking protoc for a file this tree no longer has. Bumping the
+# id costs one cold build and makes that impossible.
+
 # ---- vLLM builder -----------------------------------------------------------
 FROM builder-base AS vllm-builder
 
@@ -89,7 +96,7 @@ ARG TARGETARCH
 # build do not serialize against each other.
 RUN --mount=type=cache,id=dynamo-sidecar-cargo-registry-${TARGETARCH},target=/usr/local/cargo/registry,sharing=locked \
     --mount=type=cache,id=dynamo-sidecar-cargo-git-${TARGETARCH},target=/usr/local/cargo/git,sharing=locked \
-    --mount=type=cache,id=dynamo-sidecar-target-${TARGETARCH},target=/src/target,sharing=locked \
+    --mount=type=cache,id=dynamo-sidecar-target-${TARGETARCH}-openengine,target=/src/target,sharing=locked \
     cargo build --release --locked -p dynamo-vllm-sidecar \
     && strip target/release/dynamo-vllm-sidecar \
     && install -D target/release/dynamo-vllm-sidecar /out/dynamo-vllm-sidecar
@@ -101,7 +108,7 @@ ARG TARGETARCH
 
 RUN --mount=type=cache,id=dynamo-sidecar-cargo-registry-${TARGETARCH},target=/usr/local/cargo/registry,sharing=locked \
     --mount=type=cache,id=dynamo-sidecar-cargo-git-${TARGETARCH},target=/usr/local/cargo/git,sharing=locked \
-    --mount=type=cache,id=dynamo-sidecar-target-${TARGETARCH},target=/src/target,sharing=locked \
+    --mount=type=cache,id=dynamo-sidecar-target-${TARGETARCH}-openengine,target=/src/target,sharing=locked \
     cargo build --release --locked -p dynamo-sglang-sidecar \
     && strip target/release/dynamo-sglang-sidecar \
     && install -D target/release/dynamo-sglang-sidecar /out/dynamo-sglang-sidecar
@@ -113,7 +120,7 @@ ARG TARGETARCH
 
 RUN --mount=type=cache,id=dynamo-sidecar-cargo-registry-${TARGETARCH},target=/usr/local/cargo/registry,sharing=locked \
     --mount=type=cache,id=dynamo-sidecar-cargo-git-${TARGETARCH},target=/usr/local/cargo/git,sharing=locked \
-    --mount=type=cache,id=dynamo-sidecar-target-${TARGETARCH},target=/src/target,sharing=locked \
+    --mount=type=cache,id=dynamo-sidecar-target-${TARGETARCH}-openengine,target=/src/target,sharing=locked \
     cargo build --release --locked -p dynamo-trtllm-sidecar \
     && strip target/release/dynamo-trtllm-sidecar \
     && install -D target/release/dynamo-trtllm-sidecar /out/dynamo-trtllm-sidecar

lib/sidecar/common/Cargo.toml

@@ -11,13 +11,25 @@ homepage.workspace = true
 repository.workspace = true
 description = "Shared gRPC transport and CLI infrastructure for Dynamo sidecars"
 
+[package.metadata.cargo-machete]
+# Referenced only as `::prost_types_v14` inside the `json_codec!` macro body,
+# which cargo-machete's source scan does not expand.
+ignored = ["prost-types-v14"]
+
 [dependencies]
 dynamo-backend-common = { workspace = true }
 
 clap = { version = "4", features = ["derive", "env"] }
 futures = { workspace = true }
+prost-types = { workspace = true }
+prost-types-v14 = { workspace = true, optional = true }
+serde_json = { workspace = true }
 tokio = { workspace = true }
 tonic = { workspace = true }
 tonic-v14 = { workspace = true, optional = true }
 tracing = { workspace = true }
 url = { workspace = true }
+
+[features]
+# The v14 protobuf/tonic surface, for sidecars generated against prost 0.14.
+tonic-v14 = ["dep:tonic-v14", "dep:prost-types-v14"]

lib/sidecar/common/src/error.rs

@@ -80,6 +80,17 @@ pub fn connection_timeout(message: impl Into<String>) -> DynamoError {
     backend(BackendError::ConnectionTimeout, message)
 }
 
+fn typed(kind: ErrorType, message: impl Into<String>) -> DynamoError {
+    DynamoError::builder()
+        .error_type(kind)
+        .message(message)
+        .build()
+}
+
+pub fn cancelled(message: impl Into<String>) -> DynamoError {
+    typed(ErrorType::Cancelled, message)
+}
+
 pub fn status_to_dynamo(rpc: &str, status: tonic::Status) -> DynamoError {
     status_to_dynamo_parts(rpc, status.message(), status.code())
 }
@@ -125,6 +136,12 @@ mod tests {
                 BackendError::ConnectionTimeout,
             ),
             (tonic::Code::Internal, BackendError::Unknown),
+            // Not WorkerOverloaded: that type is migratable, and a shared
+            // helper cannot tell an overloaded engine from a per-request
+            // rejection such as an oversized message, which every worker would
+            // reject identically. A backend that knows its server means
+            // backpressure maps this itself.
+            (tonic::Code::ResourceExhausted, BackendError::Unknown),
         ] {
             let error = status_to_dynamo("Test", tonic::Status::new(code, "failure"));
             assert_eq!(error.error_type(), ErrorType::Backend(expected));