feat(backend-common): drain and exit the worker when its engine is lost

lib/backend-common/AGENTS.md

@@ -29,7 +29,7 @@ parse args,    start engine,    wire Prometheus    serve requests drain-poll  sh
 return engine  return metadata  (optional)         (concurrent)   predicate   release
 ```
 
-The trait has twelve methods. `from_args` is NOT on the trait — each
+The trait has seventeen methods. `from_args` is NOT on the trait — each
 backend exposes a backend-specific constructor (typically a sync
 `from_args(argv) -> Result<(Self, WorkerConfig)>` inherent method).
 This keeps the trait fully object-safe without a `where Self: Sized`
@@ -123,6 +123,20 @@ opt-out and lets `run.rs` stay non-generic.
   startup**. Engines that publish their own discovery records stash it
   (e.g. vLLM dynamic LoRA via `register_model` / `unregister_model`).
   Mirrors the `on_publisher_ready` handoff idiom.
+- `availability()` / `recover()` / `on_recovered()` — optional, for
+  engines that can fail while the worker lives (e.g. a sidecar's separate
+  engine process). `availability()` returns a `watch` of `Ready` /
+  `NotReady` / `Lost`; the default `None` keeps today's behavior. On
+  `NotReady` the Worker withdraws the endpoint; on `Lost` it also
+  withdraws the model card, calls `recover()`, registers the returned
+  metadata with the caller's taints, then calls `on_recovered()` so the
+  engine can restore records that need the card (e.g. LoRA adapters). It
+  re-registers the endpoint only while the engine is `Ready` and no
+  operator pause is in effect. In-flight requests are
+  never cancelled, and a `recover()` error stops the worker. Discovery
+  writes for pause, resume, taints, engine state and shutdown go through
+  one lock and are never cancelled mid-write; endpoint writes are skipped
+  when discovery already matches.
 
 ## Contract for `generate`
 

lib/backend-common/src/engine.rs

@@ -19,7 +19,7 @@ use async_trait::async_trait;
 use futures::stream::BoxStream;
 use tokio::sync::watch;
 
-use crate::error::DynamoError;
+use crate::error::{BackendError, DynamoError, ErrorType};
 
 pub use dynamo_llm::first_token::FirstTokenNotifier;
 pub use dynamo_llm::kv_router::publisher::KvEventPublisher;
@@ -184,6 +184,17 @@ pub struct EngineConfig {
     pub llm: Option<LlmRegistration>,
 }
 
+/// Whether the engine behind an [`LLMEngine`] can serve, as reported by
+/// [`LLMEngine::availability`].
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum EngineAvailability {
+    Ready,
+    /// Alive but not accepting work, for example while paused or asleep.
+    NotReady,
+    /// Gone or unreachable until [`LLMEngine::recover`] reconnects.
+    Lost,
+}
+
 /// Inference engine trait.
 ///
 /// Lifecycle:
@@ -422,6 +433,35 @@ pub trait LLMEngine: Send + Sync + 'static {
     ) -> Result<(), DynamoError> {
         Ok(())
     }
+
+    /// Live availability of an engine that can fail independently of the
+    /// worker. `None` (default) keeps the worker registered until shutdown.
+    /// On `NotReady` the worker withdraws its endpoint; on `Lost` it also
+    /// withdraws the model card and calls [`recover`](LLMEngine::recover).
+    /// `generate` must reject new requests while `Lost`.
+    fn availability(&self) -> Option<watch::Receiver<EngineAvailability>> {
+        None
+    }
+
+    /// Reconnect after [`EngineAvailability::Lost`] and return fresh
+    /// registration metadata once the engine serves again, with availability
+    /// no longer `Lost`. Runs while the worker is out of discovery and is
+    /// dropped if the worker shuts down; in-flight requests are not cancelled.
+    /// An error stops the worker, so the engine bounds its own retries. KV event
+    /// sources and metrics bindings from `start` are kept.
+    async fn recover(&self) -> Result<EngineConfig, DynamoError> {
+        Err(DynamoError::builder()
+            .error_type(ErrorType::Backend(BackendError::EngineShutdown))
+            .message("engine does not support recovery")
+            .build())
+    }
+
+    /// Called after recovery re-registers the model card and before the
+    /// endpoint is routable again, to restore discovery records that depend on
+    /// the card (for example LoRA adapters). An error is logged.
+    async fn on_recovered(&self) -> Result<(), DynamoError> {
+        Ok(())
+    }
 }
 
 /// Raw media-generation engine trait — the non-token sibling of [`LLMEngine`].

lib/backend-common/src/lib.rs

@@ -34,12 +34,12 @@ pub use args::CommonArgs;
 pub use disagg::DisaggregationMode;
 pub use dynamo_llm::model_type::ModelInput;
 pub use engine::{
-    AsyncEngineContext, BootstrapInfo, CompletionUsage, ComponentSnapshot, EngineConfig,
-    FinishReason, FirstTokenNotifier, GenerateContext, GuidedDecodingOptions, HEALTH_CHECK_KEY,
-    KV_HINT_TRANSFER_CAPABILITY_KEY, KvEventPublisher, KvEventSource, KvHint, KvHintAction,
-    KvSourceLocationsPayload, LLMEngine, LLMEngineOutput, LLMEngineOutputExt, LlmRegistration,
-    LogProbs, Metrics, MetricsBindings, MetricsCtx, MultimodalData, OnPublisherReady,
-    OnSnapshotPublisherReady, OutputOptions, PrefillResult, PreprocessedRequest,
+    AsyncEngineContext, BootstrapInfo, CompletionUsage, ComponentSnapshot, EngineAvailability,
+    EngineConfig, FinishReason, FirstTokenNotifier, GenerateContext, GuidedDecodingOptions,
+    HEALTH_CHECK_KEY, KV_HINT_TRANSFER_CAPABILITY_KEY, KvEventPublisher, KvEventSource, KvHint,
+    KvHintAction, KvSourceLocationsPayload, LLMEngine, LLMEngineOutput, LLMEngineOutputExt,
+    LlmRegistration, LogProbs, Metrics, MetricsBindings, MetricsCtx, MultimodalData,
+    OnPublisherReady, OnSnapshotPublisherReady, OutputOptions, PrefillResult, PreprocessedRequest,
     PromptTokensDetails, RawEngine, SamplingOptions, StopConditions, StopReason, TopLogprob,
     TopLogprobs, chunk, usage,
 };