From 9729a36d96bf07b607d03d56298992f4598b400e Mon Sep 17 00:00:00 2001 From: jthomson04 Date: Fri, 9 Oct 2026 20:02:36 -0700 Subject: [PATCH] perf(trtllm): isolate sidecar request shutdown waiters Give each generation request a child of the engine shutdown token so concurrent streams stop contending on the shared token's waiter mutex on every streamed message, matching the vLLM and SGLang sidecars. Co-Authored-By: Claude Opus 5.5 Signed-off-by: jthomson04 --- lib/sidecar/trtllm/src/engine.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/sidecar/trtllm/src/engine.rs b/lib/sidecar/trtllm/src/engine.rs index f35498ef4255..66327bfb2d3f 100644 --- a/lib/sidecar/trtllm/src/engine.rs +++ b/lib/sidecar/trtllm/src/engine.rs @@ -308,7 +308,7 @@ impl LLMEngine for TrtllmSidecarEngine { // streamed chunk costs a boxed future and a waker registration on every // token. let mut request_cancellation = Box::pin(async move { stopped_ctx.stopped().await }); - let shutdown = self.cancel.clone(); + let shutdown = self.cancel.child_token(); let mut shutdown_cancellation = Box::pin(async move { shutdown.cancelled().await }); // The same deferral applies here, not just to the streaming loop below.