From d55fc5f64aea65d9d88792e5e6841ee220f8f1ee Mon Sep 17 00:00:00 2001 From: Peter Pan Date: Wed, 19 Aug 2026 19:20:26 +0800 Subject: [PATCH 1/4] feat(epp): graceful shutdown on SIGTERM/SIGINT MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The EPP had no signal handling: kubelet termination sent SIGTERM and the process exited immediately, dropping in-flight ext_proc streams and the peer-discovery / KV-listener / replica-sync tasks without notice. On SIGTERM or SIGINT the EPP now flips the gRPC health service to NOT_SERVING (the gateway stops routing new requests to this replica), waits a bounded grace period (DYN_EPP_SHUTDOWN_GRACE_MS, default 5000 ms) for in-flight requests to drain, then stops accepting new connections and exits cleanly — destructors stop peer discovery, ZMQ KV listeners, and replica-sync tasks. The plaintext path uses tonic serve_with_shutdown; the TLS accept loop selects on the shutdown token before each accept. Per-connection HTTP/2 graceful drain is a follow-up. Signed-off-by: Peter Pan --- .../inference-gateway/ext-proc/src/runner.rs | 73 +++++++++++++++++-- 1 file changed, 68 insertions(+), 5 deletions(-) diff --git a/deploy/inference-gateway/ext-proc/src/runner.rs b/deploy/inference-gateway/ext-proc/src/runner.rs index 4b61ac1340cc..68e381a52f91 100644 --- a/deploy/inference-gateway/ext-proc/src/runner.rs +++ b/deploy/inference-gateway/ext-proc/src/runner.rs @@ -18,6 +18,7 @@ use dynamo_kv_router::config::{KvRouterConfig, try_kv_router_config_from_dynamo_ use dynamo_kv_router::services::selection::{SelectionService, WorkerSelectionPolicyRegistry}; use tokio::net::TcpListener; use tokio_rustls::TlsAcceptor; +use tokio_util::sync::CancellationToken; use crate::{EppMode, EppStandaloneConfig, ExtProcServer, Router, Selector, metrics}; @@ -28,6 +29,12 @@ const HEALTH_SERVICE_NAME: &str = "inference-extension"; /// connection flood from exhausting fds / memory. Tuned for an inference EPP /// where a single Envoy upstream typically holds <100 concurrent streams. const MAX_CONCURRENT_CONNECTIONS: usize = 1024; +/// Grace period after a shutdown signal before the server stops accepting new +/// connections, so in-flight ext_proc streams can drain (the gateway stops +/// routing to this EPP as soon as health flips to NOT_SERVING). Configurable +/// via `DYN_EPP_SHUTDOWN_GRACE_MS`. +const DEFAULT_SHUTDOWN_GRACE_MS: u64 = 5_000; +const SHUTDOWN_GRACE_ENV: &str = "DYN_EPP_SHUTDOWN_GRACE_MS"; /// Max time to wait for the TLS handshake to complete before dropping the /// connection. Without this, a client that finishes the TCP connect but /// stalls the TLS handshake holds a connection-limit permit indefinitely; @@ -219,6 +226,23 @@ fn init_tracing() { .try_init(); } +/// Wait for SIGTERM (kubelet pod termination) or SIGINT (Ctrl-C). +async fn wait_for_shutdown_signal() { + #[cfg(unix)] + { + let mut sigterm = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) + .expect("failed to install SIGTERM handler"); + tokio::select! { + _ = tokio::signal::ctrl_c() => {} + _ = sigterm.recv() => {} + } + } + #[cfg(not(unix))] + { + let _ = tokio::signal::ctrl_c().await; + } +} + async fn run_inner( mode: EppMode, standalone_selection_service: StandaloneSelectionService, @@ -263,6 +287,29 @@ async fn run_inner( }); } + // Graceful shutdown: on SIGTERM/SIGINT, flip health to NOT_SERVING (the + // gateway stops routing new requests to this EPP), let in-flight ext_proc + // streams drain for a bounded grace period, then stop accepting and let + // the process exit cleanly (destructors stop the peer discovery, ZMQ + // listeners, and replica-sync tasks). + let shutdown = CancellationToken::new(); + { + let health_reporter = health_reporter.clone(); + let shutdown = shutdown.clone(); + tokio::spawn(async move { + wait_for_shutdown_signal().await; + tracing::info!("Shutdown signal received; draining in-flight requests"); + health_reporter + .set_service_status(HEALTH_SERVICE_NAME, tonic_health::ServingStatus::NotServing) + .await; + let grace_ms = parse_env(SHUTDOWN_GRACE_ENV, DEFAULT_SHUTDOWN_GRACE_MS); + tracing::info!(grace_ms, "EPP health set to NOT_SERVING; draining"); + tokio::time::sleep(std::time::Duration::from_millis(grace_ms)).await; + shutdown.cancel(); + tracing::info!("EPP graceful shutdown complete"); + }); + } + if standalone { let selector_cfg = EppStandaloneConfig::from_env()?; tracing::info!( @@ -294,7 +341,13 @@ async fn run_inner( } }); let ready_router = router.clone(); - serve(router, move || ready_router.is_ready(), health_reporter).await + serve( + router, + move || ready_router.is_ready(), + health_reporter, + shutdown, + ) + .await } else { tracing::info!("Initializing KV-aware router from discovery..."); let router = Router::from_discovery(&config.namespace, &config.component).await?; @@ -304,6 +357,7 @@ async fn run_inner( Arc::new(router), move || ready.load(std::sync::atomic::Ordering::Acquire), health_reporter, + shutdown, ) .await } @@ -315,6 +369,7 @@ async fn serve( picker: Arc

, is_ready: impl Fn() -> bool + Send + 'static, health_reporter: tonic_health::server::HealthReporter, + shutdown: CancellationToken, ) -> Result<()> { // Continuously mirror readiness onto the health status. `is_ready()` is a // *live* signal that can flip both ways — standalone discovery clears it when @@ -368,9 +423,16 @@ async fn serve( loop { // Acquire permit before accept() so we backpressure the listener - // instead of accepting and immediately dropping connections. - let permit = conn_semaphore.clone().acquire_owned().await?; - let (tcp_stream, remote_addr) = listener.accept().await?; + // instead of accepting and immediately dropping connections. Stop + // accepting once the shutdown grace period has elapsed. + let permit = tokio::select! { + _ = shutdown.cancelled() => break, + permit = conn_semaphore.clone().acquire_owned() => permit?, + }; + let (tcp_stream, remote_addr) = tokio::select! { + _ = shutdown.cancelled() => break, + accepted = listener.accept() => accepted?, + }; let tls_acceptor = tls_acceptor.clone(); let svc = svc.clone(); @@ -409,11 +471,12 @@ async fn serve( } }); } + Ok(()) } else { tracing::info!(%addr, "Listening for ext_proc connections (plaintext h2)"); tonic::transport::Server::builder() .add_service(server.into_service()) - .serve(addr) + .serve_with_shutdown(addr, shutdown.cancelled_owned()) .await?; Ok(()) } From 9acfaccc60951647aa38f3591082f5f4ad98d28b Mon Sep 17 00:00:00 2001 From: Peter Pan Date: Thu, 20 Aug 2026 20:59:05 +0800 Subject: [PATCH 2/4] fix(epp): stop startup and readiness work on shutdown Signed-off-by: Peter Pan --- .../inference-gateway/ext-proc/src/runner.rs | 136 +++++++++++++----- 1 file changed, 100 insertions(+), 36 deletions(-) diff --git a/deploy/inference-gateway/ext-proc/src/runner.rs b/deploy/inference-gateway/ext-proc/src/runner.rs index 68e381a52f91..8072cb182898 100644 --- a/deploy/inference-gateway/ext-proc/src/runner.rs +++ b/deploy/inference-gateway/ext-proc/src/runner.rs @@ -292,16 +292,19 @@ async fn run_inner( // streams drain for a bounded grace period, then stop accepting and let // the process exit cleanly (destructors stop the peer discovery, ZMQ // listeners, and replica-sync tasks). + let draining = CancellationToken::new(); let shutdown = CancellationToken::new(); { - let health_reporter = health_reporter.clone(); + let draining = draining.clone(); let shutdown = shutdown.clone(); tokio::spawn(async move { wait_for_shutdown_signal().await; tracing::info!("Shutdown signal received; draining in-flight requests"); - health_reporter - .set_service_status(HEALTH_SERVICE_NAME, tonic_health::ServingStatus::NotServing) - .await; + // The readiness mirror owns the health transition so it cannot + // race with a final SERVING update. If initialization has not + // reached `serve` yet, health is already NOT_SERVING and the + // draining cancellation below makes initialization return. + draining.cancel(); let grace_ms = parse_env(SHUTDOWN_GRACE_ENV, DEFAULT_SHUTDOWN_GRACE_MS); tracing::info!(grace_ms, "EPP health set to NOT_SERVING; draining"); tokio::time::sleep(std::time::Duration::from_millis(grace_ms)).await; @@ -319,44 +322,68 @@ async fn run_inner( "Initializing standalone selector mode (no Dynamo runtime)..." ); metrics::set_served_model(&selector_cfg.model_name); - let router = Arc::new(match standalone_selection_service { - StandaloneSelectionService::Default => { - crate::EppRouter::from_selector(selector_cfg).await? - } - StandaloneSelectionService::Existing(service) => { - crate::EppRouter::from_selection_service(selector_cfg, service).await? + let router = tokio::select! { + _ = draining.cancelled() => { + tracing::info!("Shutdown received during standalone EPP initialization"); + return Ok(()); } - StandaloneSelectionService::LinkedWorkerSelectionPolicy { - kv_router_config, - factory, - } => { - let service = - Selector::build_selection_service_with_worker_selection_policy_factory( - &selector_cfg, - *kv_router_config, + router = async { + Ok::<_, anyhow::Error>(Arc::new(match standalone_selection_service { + StandaloneSelectionService::Default => { + crate::EppRouter::from_selector(selector_cfg).await? + } + StandaloneSelectionService::Existing(service) => { + crate::EppRouter::from_selection_service(selector_cfg, service).await? + } + StandaloneSelectionService::LinkedWorkerSelectionPolicy { + kv_router_config, factory, - ) - .await?; - crate::EppRouter::from_selection_service(selector_cfg, service).await? - } - }); + } => { + let service = + Selector::build_selection_service_with_worker_selection_policy_factory( + &selector_cfg, + *kv_router_config, + factory, + ) + .await?; + crate::EppRouter::from_selection_service(selector_cfg, service).await? + } + })) + } => router?, + }; + if draining.is_cancelled() { + tracing::info!("Shutdown received before standalone EPP serving started"); + return Ok(()); + } let ready_router = router.clone(); serve( router, move || ready_router.is_ready(), health_reporter, + draining, shutdown, ) .await } else { tracing::info!("Initializing KV-aware router from discovery..."); - let router = Router::from_discovery(&config.namespace, &config.component).await?; + let router = tokio::select! { + _ = draining.cancelled() => { + tracing::info!("Shutdown received during Dynamo discovery initialization"); + return Ok(()); + } + router = Router::from_discovery(&config.namespace, &config.component) => router?, + }; + if draining.is_cancelled() { + tracing::info!("Shutdown received before Dynamo discovery serving started"); + return Ok(()); + } metrics::set_served_model(router.served_model()); let ready = router.pod_store_ready(); serve( Arc::new(router), move || ready.load(std::sync::atomic::Ordering::Acquire), health_reporter, + draining, shutdown, ) .await @@ -369,6 +396,7 @@ async fn serve( picker: Arc

, is_ready: impl Fn() -> bool + Send + 'static, health_reporter: tonic_health::server::HealthReporter, + draining: CancellationToken, shutdown: CancellationToken, ) -> Result<()> { // Continuously mirror readiness onto the health status. `is_ready()` is a @@ -379,13 +407,25 @@ async fn serve( // transitions and moves the health status in lock-step, dropping out of // SERVING when readiness drops and recovering when it returns. Health starts // NOT_SERVING (set during startup); the mirror polls a cheap closure (atomic loads). - { + let readiness_task = { let health_reporter = health_reporter.clone(); + let draining = draining.clone(); tokio::spawn(async move { let mut last: Option = None; loop { + if draining.is_cancelled() { + health_reporter + .set_service_status( + HEALTH_SERVICE_NAME, + tonic_health::ServingStatus::NotServing, + ) + .await; + tracing::info!("EPP readiness mirror stopped; health set to NOT_SERVING"); + break; + } + let now = is_ready(); - if last != Some(now) { + if !draining.is_cancelled() && last != Some(now) { let status = if now { tonic_health::ServingStatus::Serving } else { @@ -394,13 +434,27 @@ async fn serve( health_reporter .set_service_status(HEALTH_SERVICE_NAME, status) .await; + if draining.is_cancelled() { + health_reporter + .set_service_status( + HEALTH_SERVICE_NAME, + tonic_health::ServingStatus::NotServing, + ) + .await; + tracing::info!("EPP readiness mirror stopped; health set to NOT_SERVING"); + break; + } tracing::info!(ready = now, "EPP readiness changed; health status updated"); last = Some(now); } - tokio::time::sleep(std::time::Duration::from_millis(500)).await; + + tokio::select! { + _ = draining.cancelled() => {} + _ = tokio::time::sleep(std::time::Duration::from_millis(500)) => {} + } } - }); - } + }) + }; let server = ExtProcServer::new(picker); // Default to TLS to match the Go EPP behavior. Verified working with @@ -421,17 +475,23 @@ async fn serve( "Listening for ext_proc connections (TLS)" ); - loop { + let result: Result<()> = loop { // Acquire permit before accept() so we backpressure the listener // instead of accepting and immediately dropping connections. Stop // accepting once the shutdown grace period has elapsed. let permit = tokio::select! { - _ = shutdown.cancelled() => break, - permit = conn_semaphore.clone().acquire_owned() => permit?, + _ = shutdown.cancelled() => break Ok(()), + permit = conn_semaphore.clone().acquire_owned() => match permit { + Ok(permit) => permit, + Err(error) => break Err(error.into()), + }, }; let (tcp_stream, remote_addr) = tokio::select! { - _ = shutdown.cancelled() => break, - accepted = listener.accept() => accepted?, + _ = shutdown.cancelled() => break Ok(()), + accepted = listener.accept() => match accepted { + Ok(accepted) => accepted, + Err(error) => break Err(error.into()), + }, }; let tls_acceptor = tls_acceptor.clone(); let svc = svc.clone(); @@ -470,14 +530,18 @@ async fn serve( tracing::debug!(%remote_addr, error = %e, "Connection ended"); } }); - } - Ok(()) + }; + readiness_task.abort(); + let _ = readiness_task.await; + result } else { tracing::info!(%addr, "Listening for ext_proc connections (plaintext h2)"); tonic::transport::Server::builder() .add_service(server.into_service()) .serve_with_shutdown(addr, shutdown.cancelled_owned()) .await?; + readiness_task.abort(); + let _ = readiness_task.await; Ok(()) } } From 446133043c3f44f00377055053cf2c9c0c65f919 Mon Sep 17 00:00:00 2001 From: Peter Pan Date: Fri, 21 Aug 2026 09:35:04 +0800 Subject: [PATCH 3/4] fix(epp): name shutdown propagation window in seconds Signed-off-by: Peter Pan --- .../inference-gateway/ext-proc/src/runner.rs | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/deploy/inference-gateway/ext-proc/src/runner.rs b/deploy/inference-gateway/ext-proc/src/runner.rs index 8072cb182898..f58c9d74da4e 100644 --- a/deploy/inference-gateway/ext-proc/src/runner.rs +++ b/deploy/inference-gateway/ext-proc/src/runner.rs @@ -29,12 +29,12 @@ const HEALTH_SERVICE_NAME: &str = "inference-extension"; /// connection flood from exhausting fds / memory. Tuned for an inference EPP /// where a single Envoy upstream typically holds <100 concurrent streams. const MAX_CONCURRENT_CONNECTIONS: usize = 1024; -/// Grace period after a shutdown signal before the server stops accepting new -/// connections, so in-flight ext_proc streams can drain (the gateway stops -/// routing to this EPP as soon as health flips to NOT_SERVING). Configurable -/// via `DYN_EPP_SHUTDOWN_GRACE_MS`. -const DEFAULT_SHUTDOWN_GRACE_MS: u64 = 5_000; -const SHUTDOWN_GRACE_ENV: &str = "DYN_EPP_SHUTDOWN_GRACE_MS"; +/// Propagation window after a shutdown signal before the server stops +/// accepting new connections. The gateway stops routing to this EPP as soon +/// as health flips to NOT_SERVING. Configurable via +/// `DYN_EPP_GRACEFUL_SHUTDOWN_PROPAGATION_SECS`. +const DEFAULT_GRACEFUL_SHUTDOWN_PROPAGATION_SECS: u64 = 5; +const GRACEFUL_SHUTDOWN_PROPAGATION_ENV: &str = "DYN_EPP_GRACEFUL_SHUTDOWN_PROPAGATION_SECS"; /// Max time to wait for the TLS handshake to complete before dropping the /// connection. Without this, a client that finishes the TCP connect but /// stalls the TLS handshake holds a connection-limit permit indefinitely; @@ -305,9 +305,15 @@ async fn run_inner( // reached `serve` yet, health is already NOT_SERVING and the // draining cancellation below makes initialization return. draining.cancel(); - let grace_ms = parse_env(SHUTDOWN_GRACE_ENV, DEFAULT_SHUTDOWN_GRACE_MS); - tracing::info!(grace_ms, "EPP health set to NOT_SERVING; draining"); - tokio::time::sleep(std::time::Duration::from_millis(grace_ms)).await; + let propagation_secs = parse_env( + GRACEFUL_SHUTDOWN_PROPAGATION_ENV, + DEFAULT_GRACEFUL_SHUTDOWN_PROPAGATION_SECS, + ); + tracing::info!( + propagation_secs, + "EPP health set to NOT_SERVING; allowing endpoint propagation" + ); + tokio::time::sleep(std::time::Duration::from_secs(propagation_secs)).await; shutdown.cancel(); tracing::info!("EPP graceful shutdown complete"); }); From 977473b4b47301e82d2fd8921c52b936c50b61b9 Mon Sep 17 00:00:00 2001 From: Peter Pan Date: Fri, 21 Aug 2026 10:03:06 +0800 Subject: [PATCH 4/4] fix(epp): clarify shutdown propagation semantics Signed-off-by: Peter Pan --- deploy/inference-gateway/ext-proc/src/runner.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/deploy/inference-gateway/ext-proc/src/runner.rs b/deploy/inference-gateway/ext-proc/src/runner.rs index f58c9d74da4e..e764d0fa0702 100644 --- a/deploy/inference-gateway/ext-proc/src/runner.rs +++ b/deploy/inference-gateway/ext-proc/src/runner.rs @@ -287,11 +287,11 @@ async fn run_inner( }); } - // Graceful shutdown: on SIGTERM/SIGINT, flip health to NOT_SERVING (the - // gateway stops routing new requests to this EPP), let in-flight ext_proc - // streams drain for a bounded grace period, then stop accepting and let - // the process exit cleanly (destructors stop the peer discovery, ZMQ - // listeners, and replica-sync tasks). + // Shutdown coordination: on SIGTERM/SIGINT, flip health to NOT_SERVING + // (the gateway stops routing new requests to this EPP), allow the endpoint + // propagation window to elapse, then stop accepting connections. The + // protocol-correct drain deadline and forced close of long-lived HTTP/2 + // connections are handled by the follow-up connection-lifecycle work. let draining = CancellationToken::new(); let shutdown = CancellationToken::new(); { @@ -299,7 +299,7 @@ async fn run_inner( let shutdown = shutdown.clone(); tokio::spawn(async move { wait_for_shutdown_signal().await; - tracing::info!("Shutdown signal received; draining in-flight requests"); + tracing::info!("Shutdown signal received; starting endpoint withdrawal"); // The readiness mirror owns the health transition so it cannot // race with a final SERVING update. If initialization has not // reached `serve` yet, health is already NOT_SERVING and the @@ -315,7 +315,7 @@ async fn run_inner( ); tokio::time::sleep(std::time::Duration::from_secs(propagation_secs)).await; shutdown.cancel(); - tracing::info!("EPP graceful shutdown complete"); + tracing::info!("EPP endpoint propagation complete; stopping accepts"); }); } @@ -484,7 +484,7 @@ async fn serve( let result: Result<()> = loop { // Acquire permit before accept() so we backpressure the listener // instead of accepting and immediately dropping connections. Stop - // accepting once the shutdown grace period has elapsed. + // accepting once the endpoint propagation window has elapsed. let permit = tokio::select! { _ = shutdown.cancelled() => break Ok(()), permit = conn_semaphore.clone().acquire_owned() => match permit {