Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 4 additions & 0 deletions lib/backend-common/AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,10 @@ opt-out and lets `run.rs` stay non-generic.
reliable idle signal); SGLang overrides it by counting in-flight prefill
streams. The mode-gate, poll loop, and SIGTERM/SIGINT ownership live in the
`Worker`.
- `engine_lost(&self) -> DynamoError` — optional, default never resolves.
Resolves once the engine can no longer serve. The `Worker` then shuts down
as on SIGTERM (withdraw, grace period, drain, `cleanup`) and returns this
error, so the process exits and its supervisor restarts it.
- `cleanup(&self) -> Result<(), DynamoError>` — called exactly once.
Runs after `start()` returns Ok on shutdown (even if registration /
serve fails), **and** after `start()` raises — so implementations
Expand Down
13 changes: 13 additions & 0 deletions lib/backend-common/src/engine.rs
Original file line number Diff line number Diff line change
Expand Up @@ -280,6 +280,19 @@ pub trait LLMEngine: Send + Sync + 'static {
Ok(None)
}

/// Resolves once the engine can no longer serve, with the error the
/// `Worker` returns. The `Worker` then shuts down as on SIGTERM, withdrawing
/// from discovery, draining and running `cleanup`, so the process exits
/// and its supervisor can restart it. Default: never resolves.
///
/// The `Worker` may poll this before `start` completes, and drops the
/// future once it finishes or starts shutting down another way.
/// Implementations must allow polling during startup and must not use this
/// future to clean up.
async fn engine_lost(&self) -> DynamoError {
std::future::pending().await
}

/// Release all engine resources. Called exactly once.
///
/// `Worker` guarantees:
Expand Down
141 changes: 136 additions & 5 deletions lib/backend-common/src/worker.rs
Original file line number Diff line number Diff line change
Expand Up @@ -337,6 +337,14 @@ impl EngineKind {
}
}

/// See [`LLMEngine::engine_lost`].
async fn engine_lost(&self) -> DynamoError {
match self {
EngineKind::Llm(e) => e.engine_lost().await,
EngineKind::Raw(_) => std::future::pending().await,
}
}

async fn kv_event_sources(&self) -> Result<Vec<KvEventSource>, DynamoError> {
match self {
EngineKind::Llm(e) => e.kv_event_sources().await,
Expand Down Expand Up @@ -599,14 +607,25 @@ impl Worker {
// grace-period sleep, which is a fixed wait rather than a hang
// risk), otherwise we exit(911). Healthy long-running workers
// never hit this — the timer only starts after `shutdown_token`
// is cancelled.
// is cancelled. A lost engine starts the same shutdown.
let engine = self.engine.clone();
let outcome = {
let inner_fut = self.run_inner(runtime.clone(), drt, &shutdown_token);
tokio::pin!(inner_fut);

tokio::select! {
result = &mut inner_fut => result,
_ = shutdown_token.cancelled() => {
let (finished, lost) = tokio::select! {
biased;
result = &mut inner_fut => (Some(result), None),
_ = shutdown_token.cancelled() => (None, None),
error = engine.engine_lost() => (None, Some(error)),
};
match finished {
Some(result) => result,
None => {
if let Some(error) = &lost {
tracing::error!(%error, "Engine lost; shutting down the worker");
shutdown_token.cancel();
}
runtime.mark_shutting_down();
let timeout = graceful_shutdown_timeout();
let grace = grace_period_secs();
Expand All @@ -617,7 +636,7 @@ impl Worker {
timeout.as_secs(),
grace,
);
match tokio::time::timeout(deadline, &mut inner_fut).await {
let result = match tokio::time::timeout(deadline, &mut inner_fut).await {
Ok(result) => result,
Err(_) => {
tracing::error!(
Expand All @@ -627,6 +646,15 @@ impl Worker {
);
std::process::exit(911);
}
};
match lost {
Some(error) => {
if let Err(stopped) = result {
tracing::warn!(error = %stopped, "Worker shutdown failed");
}
Err(error)
}
None => result,
}
}
}
Expand Down Expand Up @@ -4353,6 +4381,109 @@ mod handoff_and_lifecycle_tests {
worker.begin_engine_route_shutdown().await;
}

/// A lost engine withdraws the serving worker, runs cleanup, and makes the
/// worker return the engine's error.
#[tokio::test]
#[serial_test::serial]
async fn lost_engine_withdraws_the_worker_and_returns_its_error() {
struct LostEngine {
lost: CancellationToken,
cleanup_calls: std::sync::atomic::AtomicUsize,
}

#[async_trait]
impl LLMEngine for LostEngine {
async fn start(&self, _worker_id: u64) -> Result<EngineConfig, DynamoError> {
Ok(EngineConfig {
model: "lost-mock".to_string(),
..EngineConfig::default()
})
}

async fn generate(
&self,
_request: PreprocessedRequest,
_ctx: crate::engine::GenerateContext,
) -> Result<
BoxStream<'static, Result<crate::engine::LLMEngineOutput, DynamoError>>,
DynamoError,
> {
unreachable!("not used in engine-loss tests")
}

async fn cleanup(&self) -> Result<(), DynamoError> {
self.cleanup_calls
.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
Ok(())
}

async fn engine_lost(&self) -> DynamoError {
self.lost.cancelled().await;
DynamoError::builder()
.error_type(ErrorType::Backend(BackendError::EngineShutdown))
.message("synthetic engine loss")
.build()
}
}

let runtime = dynamo_runtime::Runtime::from_current().unwrap();
let config = dynamo_runtime::distributed::DistributedConfig::process_local();
let drt = DistributedRuntime::new(runtime, config).await.unwrap();
let engine = Arc::new(LostEngine {
lost: CancellationToken::new(),
cleanup_calls: Default::default(),
});
let config = WorkerConfig::default();
let query = DiscoveryQuery::Endpoint {
namespace: config.namespace.clone(),
component: config.component.clone(),
endpoint: config.endpoint.clone(),
};
let shutdown = CancellationToken::new();
let run = tokio::spawn(
Worker::new(engine.clone(), config).run_with_drt(drt.clone(), shutdown.clone()),
);
tokio::time::timeout(Duration::from_secs(30), async {
while drt
.discovery()
.list(query.clone())
.await
.unwrap()
.is_empty()
{
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.expect("worker registers its endpoint");

engine.lost.cancel();
let error = tokio::time::timeout(Duration::from_secs(30), run)
.await
.expect("a lost engine stops the worker")
.unwrap()
.expect_err("a lost engine fails the worker");
assert_eq!(
error.error_type(),
ErrorType::Backend(BackendError::EngineShutdown)
);
assert!(
error.to_string().contains("synthetic engine loss"),
"{error}"
);
assert!(
drt.discovery().list(query).await.unwrap().is_empty(),
"a lost engine must leave no endpoint"
);
assert_eq!(
engine
.cleanup_calls
.load(std::sync::atomic::Ordering::SeqCst),
1
);
assert!(shutdown.is_cancelled(), "engine loss uses the SIGTERM path");
}

/// Ensures a payload-free Rust backend publishes readiness while it is
/// serviceable and withdraws it again on shutdown.
#[tokio::test]
Expand Down
1 change: 1 addition & 0 deletions lib/bindings/python/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions lib/sidecar/testkit/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ dynamo-trtllm-mocker = { path = "../../mocker/servers/trtllm" }
dynamo-lpx-sidecar = { path = "../lpx" }
tokio-stream = { workspace = true, features = ["net"] }
tonic = { workspace = true }
tonic-health = { workspace = true }
tonic-v14 = { workspace = true }
tonic-health-v14 = { workspace = true }
serde_json = { workspace = true }
Expand Down
48 changes: 48 additions & 0 deletions lib/sidecar/testkit/tests/router_sidecar_mocker_integration.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1392,6 +1392,54 @@ async fn trtllm_native_errors_reach_http_and_recover() {
.expect("process scenario exceeded its overall deadline");
}

#[tokio::test]
async fn trtllm_engine_loss_withdraws_worker_and_fails_the_process() {
tokio::time::timeout(
Duration::from_secs(60),
liveness_loss_withdraws_worker_and_fails_the_process(),
)
.await
.expect("process scenario exceeded its overall deadline");
}

/// A TensorRT-LLM engine lost after startup ends its `liveness` watch, which
/// withdraws the worker, and the sidecar then exits with an error.
async fn liveness_loss_withdraws_worker_and_fails_the_process() {
let env = Environment::new().await;
let mut peer = support::trtllm::Fixture::start(
Controller::default(),
FixtureConfig {
model: env.model.clone(),
..Default::default()
},
)
.await;
// The grace keeps the withdrawn process up while the test observes it.
let mut child = env.spawn_with_grace::<support::trtllm::Fixture>(
&peer.endpoint(),
DisaggregationMode::Aggregated,
5,
);
let router = env.ready("backend").await;
peer.shutdown().await;
tokio::time::timeout(Duration::from_secs(20), async {
while !env.registrations("backend").await.is_empty() {
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.unwrap_or_else(|_| panic!("engine loss did not withdraw the worker\n{}", child.logs()));
env.withdrawn("backend", &router).await;
assert!(child.is_running(), "withdrawal must precede process exit");
// 1 is the engine-loss error; a forced shutdown (exit 911) would read 143.
assert_eq!(child.exit().await.code(), Some(1), "{}", child.logs());
assert!(
child.logs().contains("TensorRT-LLM engine lost"),
"{}",
child.logs()
);
}

#[tokio::test]
async fn lpx_native_errors_reach_http_and_recover() {
tokio::time::timeout(
Expand Down
28 changes: 28 additions & 0 deletions lib/sidecar/testkit/tests/sidecar_mocker_integration.rs
Original file line number Diff line number Diff line change
Expand Up @@ -883,6 +883,34 @@ async fn trtllm_malformed_terminal_fails_then_recovers() {
.await;
}

/// TensorRT-LLM reporting `liveness` NOT_SERVING after startup resolves
/// `engine_lost`; the process test covers a dropped connection.
#[tokio::test]
async fn trtllm_engine_lost_on_liveness_not_serving() {
let mut fixture =
support::trtllm::Fixture::start(Controller::default(), FixtureConfig::default()).await;
let engine = fixture.engine().await;
engine.start(0).await.unwrap();
let lost = engine.engine_lost();
tokio::pin!(lost);
tokio::time::timeout(std::time::Duration::from_millis(300), &mut lost)
.await
.expect_err("a SERVING engine is not lost");
fixture.set_liveness_serving(false).await;
let error = bounded("engine lost on liveness NOT_SERVING", lost).await;
assert_eq!(
error.error_type(),
dynamo_backend_common::ErrorType::Backend(BackendError::EngineShutdown)
);
assert!(
error
.to_string()
.contains("TensorRT-LLM engine lost: liveness reported NOT_SERVING"),
"{error}"
);
finish(&mut fixture, &engine).await;
}

fn http_request() -> PreprocessedRequest {
let mut req = request("mocker-model", vec![11, 22, 33], 2);
req.extra_args = Some(json!({"sglang_tito": {"sampling_params": {"max_new_tokens": 2}}}));
Expand Down
17 changes: 17 additions & 0 deletions lib/sidecar/testkit/tests/support/trtllm.rs
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ pub struct Fixture {
server: TestServer,
scripted: Arc<Mutex<HashMap<String, Vec<pb::GenerateResponse>>>>,
pub discovery: Arc<Discovery>,
health: tonic_health::server::HealthReporter,
}

#[derive(Default)]
Expand All @@ -49,6 +50,16 @@ pub struct Discovery {
}

impl Fixture {
/// Sets what `Watch` reports for TensorRT-LLM's `liveness` health.
pub async fn set_liveness_serving(&self, is_serving: bool) {
let status = if is_serving {
tonic_health::ServingStatus::Serving
} else {
tonic_health::ServingStatus::NotServing
};
self.health.set_service_status("liveness", status).await;
}

pub fn release_discovery(&self) {
self.discovery.hold.store(false, Ordering::SeqCst);
self.discovery.release.notify_waiters();
Expand Down Expand Up @@ -124,8 +135,13 @@ impl SidecarFixture for Fixture {
inner: service.clone(),
discovery: discovery.clone(),
};
let (health, health_service) = tonic_health::server::health_reporter();
health
.set_service_status("liveness", tonic_health::ServingStatus::Serving)
.await;
let server = TestServer::start(move |listener, shutdown| async move {
tonic::transport::Server::builder()
.add_service(health_service)
.add_service(InferenceServer::new(controlled))
.add_service(ControlServer::new(control_service))
.serve_with_incoming_shutdown(TcpListenerStream::new(listener), async {
Expand All @@ -142,6 +158,7 @@ impl SidecarFixture for Fixture {
server,
scripted,
discovery,
health,
}
}

Expand Down
1 change: 1 addition & 0 deletions lib/sidecar/trtllm/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ tracing = { workspace = true }
prost = { workspace = true }
prost-types = { workspace = true }
tonic = { workspace = true }
tonic-health = { workspace = true }

[build-dependencies]
tonic-build = { workspace = true }
Expand Down
Loading
Loading