From b4d60c16d319745d120b9289305ab80e319a3c2a Mon Sep 17 00:00:00 2001
From: "svc-glamr@nvidia.com" <svc-glamr@nvidia.com>
Date: Tue, 28 Jul 2026 14:31:40 -0700
Subject: [PATCH 01/22] fix(runtime): rebind system health server after CRIU
 listener loss

After a CRIU checkpoint/restore the system status server's TCP listener is
gone, but the accept loop kept the dead listener and retried against it
forever. The server reported healthy while accepting nothing.

Rebind the listener inside axum's `Listener` implementation rather than
outside it, so the rebind happens on the same path that observes the loss,
and surface accept failures instead of swallowing them.

Squashed from 21a7ccd50, 85d54b0ae, 2bb927819 and 1376069ad.

Signed-off-by: svc-glamr@nvidia.com <svc-glamr@nvidia.com>
Signed-off-by: Matej Kosec <mkosec@nvidia.com>
---
 lib/runtime/src/system_status_server.rs | 274 +++++++++++++++++++++---
 1 file changed, 239 insertions(+), 35 deletions(-)

diff --git a/lib/runtime/src/system_status_server.rs b/lib/runtime/src/system_status_server.rs
index 1a5d69d7749e..48e418f37bd3 100644
--- a/lib/runtime/src/system_status_server.rs
+++ b/lib/runtime/src/system_status_server.rs
@@ -18,13 +18,15 @@ use axum::{
     http::StatusCode,
     response::IntoResponse,
     routing::{any, delete, get, post},
+    serve::Listener,
 };
 use futures::StreamExt;
 use serde::{Deserialize, Serialize};
 use serde_json::json;
 use std::collections::HashMap;
+use std::io;
 use std::sync::{Arc, OnceLock};
-use std::time::Instant;
+use std::time::{Duration, Instant};
 use tokio::{net::TcpListener, task::JoinHandle};
 use tokio_util::sync::CancellationToken;
 use tower_http::trace::TraceLayer;
@@ -128,6 +130,8 @@ pub struct LoraResponse {
     pub count: Option<usize>,
 }
 
+const SYSTEM_STATUS_REBIND_BACKOFF: Duration = Duration::from_secs(1);
+
 /// Start system status server with metrics support
 pub async fn spawn_system_status_server(
     host: &str,
@@ -138,6 +142,36 @@ pub async fn spawn_system_status_server(
 ) -> anyhow::Result<(std::net::SocketAddr, tokio::task::JoinHandle<()>)> {
     // Create system status server state with the provided distributed runtime
     let server_state = Arc::new(SystemStatusState::new(drt, discovery_metadata)?);
+    let app = build_system_status_router(server_state);
+
+    let initial_bind_address = format!("{}:{}", host, port);
+    let (listener, actual_address) = bind_system_status_listener(initial_bind_address).await?;
+
+    // Hand axum a listener that rebinds `actual_address` for itself. Whatever
+    // leaves the listening socket unable to accept, the server re-binds the
+    // same address the caller was told about rather than retrying a dead file
+    // descriptor for the life of the process. Nothing here is specific to
+    // CRIU; a checkpoint/restore is just the case that motivated it, and the
+    // one where re-binding also lands the server in the restored network
+    // namespace.
+    let listener =
+        RebindingTcpListener::new(listener, actual_address, SYSTEM_STATUS_REBIND_BACKOFF);
+    let observer = cancel_token.child_token();
+
+    // Spawn the server in the background and return the handle
+    let handle = tokio::spawn(async move {
+        if let Err(e) = axum::serve(listener, app)
+            .with_graceful_shutdown(observer.cancelled_owned())
+            .await
+        {
+            tracing::error!("System status server error: {e}");
+        }
+    });
+
+    Ok((actual_address, handle))
+}
+
+fn build_system_status_router(server_state: Arc<SystemStatusState>) -> Router {
     let health_path = server_state
         .drt()
         .system_health()
@@ -219,53 +253,129 @@ pub async fn spawn_system_status_server(
     // The endpoint triple disambiguates multi-LocalModel-per-DRT; the
     // suffix segment (LoRA slug or `_base`) scopes per-registration so
     // detaching one doesn't wipe another's entries.
-    app = app.route(
+    app.route(
         "/v1/metadata/{namespace}/{component}/{endpoint}/{model_slug}/{model_suffix}/{*filename}",
         get({
             let state = Arc::clone(&server_state);
             move |path| metadata_file_handler(State(state), path)
         }),
+    )
+    .fallback(|| async {
+        tracing::info!("[fallback handler] called");
+        (StatusCode::NOT_FOUND, "Route not found").into_response()
+    })
+    .layer(TraceLayer::new_for_http().make_span_with(make_system_request_span))
+}
+
+async fn bind_system_status_listener(
+    address: String,
+) -> anyhow::Result<(TcpListener, std::net::SocketAddr)> {
+    tracing::info!("[spawn_system_status_server] binding to: {address}");
+
+    let listener = TcpListener::bind(&address).await.map_err(|e| {
+        tracing::error!("Failed to bind to address {}: {}", address, e);
+        anyhow::anyhow!("Failed to bind to address: {}", e)
+    })?;
+
+    let actual_address = listener.local_addr()?;
+    tracing::info!(
+        "[spawn_system_status_server] system status server bound to: {}",
+        actual_address
     );
 
-    let app = app
-        .fallback(|| async {
-            tracing::info!("[fallback handler] called");
-            (StatusCode::NOT_FOUND, "Route not found").into_response()
-        })
-        .layer(TraceLayer::new_for_http().make_span_with(make_system_request_span));
+    Ok((listener, actual_address))
+}
 
-    let address = format!("{}:{}", host, port);
-    tracing::info!("[spawn_system_status_server] binding to: {address}");
+/// A listener that owns its bind address and re-binds it when the socket
+/// underneath stops accepting connections.
+///
+/// CRIU closes the process's listening sockets when it checkpoints. After
+/// restore the file descriptor is still open but no longer usable, so `accept`
+/// fails with a non-transient error every time it is called and the system
+/// status server goes silent while the rest of the runtime carries on.
+///
+/// `axum::serve` cannot report that on its own. [`Listener::accept`] returns no
+/// `Result`, and axum's own `TcpListener` implementation logs the error, sleeps
+/// for a second and tries the same dead socket again, forever. Recovering
+/// inside the listener is the extension point the trait leaves open: the
+/// documentation for `accept` states that an implementation whose accept call
+/// can fail "must take care of logging and retrying".
+struct RebindingTcpListener {
+    /// The address to re-bind. This is the address the initial bind resolved
+    /// to, not the requested one, so a caller that asked for port 0 keeps the
+    /// concrete port it was told about.
+    address: std::net::SocketAddr,
+    listener: Option<TcpListener>,
+    rebind_backoff: Duration,
+}
 
-    let listener = match TcpListener::bind(&address).await {
-        Ok(listener) => {
-            // get the actual address and port, print in debug level
-            let actual_address = listener.local_addr()?;
-            tracing::info!(
-                "[spawn_system_status_server] system status server bound to: {}",
-                actual_address
-            );
-            (listener, actual_address)
-        }
-        Err(e) => {
-            tracing::error!("Failed to bind to address {}: {}", address, e);
-            return Err(anyhow::anyhow!("Failed to bind to address: {}", e));
+impl RebindingTcpListener {
+    fn new(listener: TcpListener, address: std::net::SocketAddr, rebind_backoff: Duration) -> Self {
+        Self {
+            address,
+            listener: Some(listener),
+            rebind_backoff,
         }
-    };
-    let (listener, actual_address) = listener;
+    }
+}
 
-    let observer = cancel_token.child_token();
-    // Spawn the server in the background and return the handle
-    let handle = tokio::spawn(async move {
-        if let Err(e) = axum::serve(listener, app)
-            .with_graceful_shutdown(observer.cancelled_owned())
-            .await
-        {
-            tracing::error!("System status server error: {e}");
+impl Listener for RebindingTcpListener {
+    type Io = tokio::net::TcpStream;
+    type Addr = std::net::SocketAddr;
+
+    async fn accept(&mut self) -> (Self::Io, Self::Addr) {
+        loop {
+            let Some(listener) = self.listener.as_ref() else {
+                match bind_system_status_listener(self.address.to_string()).await {
+                    Ok((listener, _)) => self.listener = Some(listener),
+                    Err(error) => {
+                        tracing::error!(
+                            "System status server failed to rebind {}; retrying after {:?}: {error}",
+                            self.address,
+                            self.rebind_backoff
+                        );
+                        tokio::time::sleep(self.rebind_backoff).await;
+                    }
+                }
+                continue;
+            };
+
+            match listener.accept().await {
+                Ok(accepted) => return accepted,
+                // A connection that died between the SYN and our accept says
+                // nothing about the listener; take the next one immediately.
+                Err(error) if is_transient_accept_error(&error) => {
+                    tracing::trace!("system status connection dropped before accept: {error}");
+                }
+                Err(error) => {
+                    tracing::error!(
+                        "System status listener stopped accepting on {}; rebinding after {:?}: {error}",
+                        self.address,
+                        self.rebind_backoff
+                    );
+                    // Dropping the old listener closes its file descriptor and
+                    // frees the address. A socket that has stopped accepting is
+                    // still bound, so rebinding before this would fail with
+                    // `EADDRINUSE`.
+                    self.listener = None;
+                    tokio::time::sleep(self.rebind_backoff).await;
+                }
+            }
         }
-    });
+    }
 
-    Ok((actual_address, handle))
+    fn local_addr(&self) -> io::Result<Self::Addr> {
+        Ok(self.address)
+    }
+}
+
+fn is_transient_accept_error(error: &io::Error) -> bool {
+    matches!(
+        error.kind(),
+        io::ErrorKind::ConnectionRefused
+            | io::ErrorKind::ConnectionAborted
+            | io::ErrorKind::ConnectionReset
+    )
 }
 
 /// Health handler with optional active health checking
@@ -712,6 +822,9 @@ mod tests {
     use super::*;
     use tokio::time::Duration;
 
+    #[cfg(unix)]
+    use std::os::fd::AsRawFd;
+
     // This is a basic test to verify the HTTP server is working before testing other more complicated tests
     #[tokio::test]
     async fn test_http_server_lifecycle() {
@@ -741,6 +854,97 @@ mod tests {
             "HTTP server should shut down when cancel token is cancelled"
         );
     }
+
+    /// A listener that has been shut down keeps failing `accept`, so the only
+    /// way a connection can be served afterwards is if the listener rebound.
+    ///
+    /// `shutdown(2)` on a listening socket is the closest stand-in for what
+    /// CRIU does to it. On platforms where that leaves the socket accepting,
+    /// the original listener answers and the test still passes — it just
+    /// stops proving the recovery path, so it is worth keeping honest by
+    /// checking the returned peer address belongs to our own connection.
+    #[cfg(unix)]
+    #[tokio::test]
+    async fn test_rebinding_listener_serves_again_after_listener_shutdown() {
+        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+        let address = listener.local_addr().unwrap();
+        let listener_fd = listener.as_raw_fd();
+
+        let mut rebinding = RebindingTcpListener::new(listener, address, Duration::from_millis(10));
+
+        let shutdown_result = unsafe { libc::shutdown(listener_fd, libc::SHUT_RDWR) };
+        assert_eq!(shutdown_result, 0, "listener shutdown should succeed");
+
+        let accepted = tokio::spawn(async move { rebinding.accept().await });
+
+        // The address is unbound between the old listener being dropped and the
+        // rebind completing, so connecting has to be retried.
+        let client = tokio::time::timeout(Duration::from_secs(5), async {
+            loop {
+                if let Ok(stream) = tokio::net::TcpStream::connect(address).await {
+                    return stream;
+                }
+                tokio::time::sleep(Duration::from_millis(10)).await;
+            }
+        })
+        .await
+        .expect("listener should rebind and accept connections again");
+
+        let (_io, peer) = tokio::time::timeout(Duration::from_secs(5), accepted)
+            .await
+            .expect("accept should return once the listener is rebound")
+            .expect("accept task should not panic");
+
+        assert_eq!(
+            peer,
+            client.local_addr().unwrap(),
+            "accepted connection should be the one we opened"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_rebinding_listener_reports_the_address_it_rebinds() {
+        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+        let address = listener.local_addr().unwrap();
+
+        let rebinding = RebindingTcpListener::new(listener, address, Duration::from_millis(10));
+
+        assert_ne!(address.port(), 0, "bind should resolve the ephemeral port");
+        assert_eq!(
+            rebinding.local_addr().unwrap(),
+            address,
+            "the resolved address is what gets rebound and advertised"
+        );
+    }
+
+    /// Recovery must not cost graceful shutdown: axum selects the shutdown
+    /// signal against `accept`, so a listener stuck in a rebind loop still
+    /// stops when the token is cancelled.
+    #[tokio::test]
+    async fn test_rebinding_listener_still_shuts_down_gracefully() {
+        let cancel_token = CancellationToken::new();
+        let app = Router::new().route("/test", get(|| async { (StatusCode::OK, "test") }));
+        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+        let address = listener.local_addr().unwrap();
+        let listener = RebindingTcpListener::new(listener, address, Duration::from_millis(10));
+
+        let server = tokio::spawn({
+            let cancel_token = cancel_token.clone();
+            async move {
+                axum::serve(listener, app)
+                    .with_graceful_shutdown(cancel_token.cancelled_owned())
+                    .await
+            }
+        });
+
+        cancel_token.cancel();
+
+        tokio::time::timeout(Duration::from_secs(5), server)
+            .await
+            .expect("server should shut down when the cancel token is cancelled")
+            .expect("server task should not panic")
+            .expect("graceful shutdown should not error");
+    }
 }
 
 // Integration tests: cargo test system_status_server --lib --features integration

From 87a366567872364b06d82e8845877b139f4fe83b Mon Sep 17 00:00:00 2001
From: Matej Kosec <mkosec@nvidia.com>
Date: Tue, 11 Aug 2026 19:56:41 -0700
Subject: [PATCH 02/22] fix(runtime): declare libc as a dev-dependency for the
 listener rebind test

The unix-gated test in system_status_server.rs calls libc::shutdown on a
listening socket. It compiled while this branch was behind, because something
else in dynamo-runtime's graph pulled libc in. Current main no longer does, so
after bringing the branch up to date clippy and rust-tests both failed with
E0433 'use of unresolved module or unlinked crate libc'.

libc is already declared in [workspace.dependencies], so the crate only needs to
opt in. It goes in [dev-dependencies] because only the test target uses it.

I updated Cargo.lock by hand to add the new edge, since CI runs cargo test
--locked. I have not compiled this locally; CI on this PR is the check.

Signed-off-by: Matej Kosec <mkosec@nvidia.com>
---
 Cargo.lock             | 1 +
 lib/runtime/Cargo.toml | 3 +++
 2 files changed, 4 insertions(+)

diff --git a/Cargo.lock b/Cargo.lock
index 4e2fe6a57ba2..056c1e07bb8e 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2942,6 +2942,7 @@ dependencies = [
  "jsonschema",
  "k8s-openapi",
  "kube",
+ "libc",
  "local-ip-address",
  "log",
  "lru 0.16.4",
diff --git a/lib/runtime/Cargo.toml b/lib/runtime/Cargo.toml
index 641319763cc3..d0fb611d49c4 100644
--- a/lib/runtime/Cargo.toml
+++ b/lib/runtime/Cargo.toml
@@ -117,6 +117,9 @@ k8s-openapi = { version = "0.26.0", features = ["v1_32"] }
 
 [dev-dependencies]
 criterion = { version = "0.5", features = ["async_tokio"] }
+# Used only by the unix-gated system_status_server test that calls shutdown(2)
+# on a listening socket to stand in for what CRIU does to it.
+libc = { workspace = true }
 rcgen = { workspace = true }
 rstest = { version = "0.23.0" }
 temp-env = { version = "0.3.6" , features=["async_closure"] }

From 8325b43295571b62e53af030233ad519808aa1b3 Mon Sep 17 00:00:00 2001
From: Matej Kosec <mkosec@nvidia.com>
Date: Wed, 12 Aug 2026 10:22:21 -0700
Subject: [PATCH 03/22] test(runtime): shut the listener down through socket2
 instead of libc

I added libc to [dev-dependencies] in the previous commit to make this test
compile, and both the fix and its explanation were wrong.

The explanation first. I wrote that libc had compiled before because something
else in dynamo-runtime's graph pulled it in. Rust never resolves a libc:: path
through a transitive dependency. libc was a direct dependency of lib/runtime
until #12620 deleted it, which is why the branch stopped compiling once I merged
main.

The section was wrong too. lib/runtime/src/nvtx.rs and
lib/runtime/src/pipeline/network/egress/tcp_client.rs both call libc:: outside
tests, under the nvtx and tcp-low-latency features. A dev-dependency does not
link into those builds, so my change would not have helped them.

This test needs no new dependency at all. socket2 0.5 is already a dependency of
this crate, socket2::SockRef::from accepts anything implementing AsFd, and
tokio's TcpListener implements it. SockRef::shutdown issues the same syscall
with no unsafe block and no raw descriptor, so I reverted lib/runtime/Cargo.toml
and Cargo.lock to main.

I also corrected the test's platform handling. It asserted the shutdown returned
0, and its doc comment predicted that a platform refusing the call would leave
the test passing. Darwin refuses shutdown on a listening socket with ENOTCONN,
so the test would have failed there rather than passing. It now returns early
when the platform refuses, because a listener that never broke cannot exercise
the rebind path.

libc missing from lib/runtime's [dependencies] still breaks the nvtx and
tcp-low-latency feature builds. That break came from #12620 and I have not
fixed it here.

I could not build this workspace locally. CI on this pull request is the check.

Signed-off-by: Matej Kosec <mkosec@nvidia.com>
---
 Cargo.lock                              |  1 -
 lib/runtime/Cargo.toml                  |  3 ---
 lib/runtime/src/system_status_server.rs | 31 +++++++++++++++++--------
 3 files changed, 21 insertions(+), 14 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 056c1e07bb8e..4e2fe6a57ba2 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2942,7 +2942,6 @@ dependencies = [
  "jsonschema",
  "k8s-openapi",
  "kube",
- "libc",
  "local-ip-address",
  "log",
  "lru 0.16.4",
diff --git a/lib/runtime/Cargo.toml b/lib/runtime/Cargo.toml
index d0fb611d49c4..641319763cc3 100644
--- a/lib/runtime/Cargo.toml
+++ b/lib/runtime/Cargo.toml
@@ -117,9 +117,6 @@ k8s-openapi = { version = "0.26.0", features = ["v1_32"] }
 
 [dev-dependencies]
 criterion = { version = "0.5", features = ["async_tokio"] }
-# Used only by the unix-gated system_status_server test that calls shutdown(2)
-# on a listening socket to stand in for what CRIU does to it.
-libc = { workspace = true }
 rcgen = { workspace = true }
 rstest = { version = "0.23.0" }
 temp-env = { version = "0.3.6" , features=["async_closure"] }
diff --git a/lib/runtime/src/system_status_server.rs b/lib/runtime/src/system_status_server.rs
index acb85c077b58..a7a9ac5d48cb 100644
--- a/lib/runtime/src/system_status_server.rs
+++ b/lib/runtime/src/system_status_server.rs
@@ -822,9 +822,6 @@ mod tests {
     use super::*;
     use tokio::time::Duration;
 
-    #[cfg(unix)]
-    use std::os::fd::AsRawFd;
-
     // This is a basic test to verify the HTTP server is working before testing other more complicated tests
     #[tokio::test]
     async fn test_http_server_lifecycle() {
@@ -859,21 +856,35 @@ mod tests {
     /// way a connection can be served afterwards is if the listener rebound.
     ///
     /// `shutdown(2)` on a listening socket is the closest stand-in for what
-    /// CRIU does to it. On platforms where that leaves the socket accepting,
-    /// the original listener answers and the test still passes — it just
-    /// stops proving the recovery path, so it is worth keeping honest by
-    /// checking the returned peer address belongs to our own connection.
+    /// CRIU does to it, and only some platforms allow it. Linux shuts the
+    /// listener down and every later `accept` fails, which is the case this
+    /// test exists for. Darwin instead rejects the call with `ENOTCONN`,
+    /// because it only shuts down connected sockets, so there is no failure to
+    /// recover from and the test returns early rather than asserting on a
+    /// recovery that was never provoked.
+    ///
+    /// `socket2::SockRef` issues the same syscall as a raw `libc::shutdown`
+    /// without an `unsafe` block or a raw descriptor, and `socket2` is already
+    /// a dependency of this crate.
     #[cfg(unix)]
     #[tokio::test]
     async fn test_rebinding_listener_serves_again_after_listener_shutdown() {
         let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
         let address = listener.local_addr().unwrap();
-        let listener_fd = listener.as_raw_fd();
+
+        // Take the reference before the listener moves into the wrapper.
+        let shutdown_result = socket2::SockRef::from(&listener).shutdown(std::net::Shutdown::Both);
 
         let mut rebinding = RebindingTcpListener::new(listener, address, Duration::from_millis(10));
 
-        let shutdown_result = unsafe { libc::shutdown(listener_fd, libc::SHUT_RDWR) };
-        assert_eq!(shutdown_result, 0, "listener shutdown should succeed");
+        if let Err(error) = shutdown_result {
+            // Nothing broke the listener, so `accept` still works and the
+            // rebind path is never reached. Asserting anything past this point
+            // would test the ordinary accept path under a name that promises
+            // otherwise.
+            eprintln!("skipping: this platform refused shutdown on a listening socket: {error}");
+            return;
+        }
 
         let accepted = tokio::spawn(async move { rebinding.accept().await });
 

From eb9eef4958e4bcecded6dfa215d1c9afa77a93e7 Mon Sep 17 00:00:00 2001
From: Matej Kosec <mkosec@nvidia.com>
Date: Wed, 12 Aug 2026 13:53:51 -0700
Subject: [PATCH 04/22] fix(runtime): keep the listener when the process runs
 out of descriptors
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

`is_transient_accept_error` reused axum's connection-error triple. Every other accept error took the rebind arm, including `EMFILE`, `ENFILE` and `ENOBUFS`. Those errors mean the process is short of a descriptor or a buffer. The socket still works. Axum retries the same descriptor on them. The rebind arm closed a working listening socket and discarded its accept backlog. It then tried to bind a replacement. That requires a new descriptor — the resource the process ran out of. The port stayed unbound until the shortage cleared.

The accept loop now has three arms. A dead connection retries immediately, as before. A resource shortage causes the loop to sleep and then accept on the same socket. Anything else drops the socket and rebinds. The CRIU behavior this branch targets does not change.

`ENOMEM` arrives as `io::ErrorKind::OutOfMemory`. `EMFILE`, `ENFILE` and `ENOBUFS` have no `ErrorKind` of their own. They arrive as `Uncategorized`. The predicate matches their raw codes.

Also here:

- A test fails a rebind and then lets a later one succeed. A second socket holds the address the listener wants. The first rebind fails with `EADDRINUSE`. Dropping the holder partway through lets a later attempt bind. That arm had no coverage.
- A unit test pins the classification of `ENFILE` and `EMFILE`.
- The call-site comment now bounds its network-namespace claim to the default `DYN_SYSTEM_HOST` of `0.0.0.0`. A deployment that names a concrete address the restored namespace lacks receives `EADDRNOTAVAIL` on every rebind.
- The 404 fallback logs at `debug` with the request URI instead of `info` with no path.

Signed-off-by: Matej Kosec <mkosec@nvidia.com>
---
 lib/runtime/src/system_status_server.rs | 147 +++++++++++++++++++++++-
 1 file changed, 142 insertions(+), 5 deletions(-)

diff --git a/lib/runtime/src/system_status_server.rs b/lib/runtime/src/system_status_server.rs
index a7a9ac5d48cb..b17b46fe3c14 100644
--- a/lib/runtime/src/system_status_server.rs
+++ b/lib/runtime/src/system_status_server.rs
@@ -153,7 +153,11 @@ pub async fn spawn_system_status_server(
     // descriptor for the life of the process. Nothing here is specific to
     // CRIU; a checkpoint/restore is just the case that motivated it, and the
     // one where re-binding also lands the server in the restored network
-    // namespace.
+    // namespace. That last part holds for the default `DYN_SYSTEM_HOST` of
+    // `0.0.0.0`, which any namespace can bind. A deployment that sets a
+    // concrete address the restored namespace does not have gets a rebind that
+    // fails with `EADDRNOTAVAIL` and a loop that retries it once a second —
+    // no worse than the stock listener, which never recovers either.
     let listener =
         RebindingTcpListener::new(listener, actual_address, SYSTEM_STATUS_REBIND_BACKOFF);
     let observer = cancel_token.child_token();
@@ -260,8 +264,8 @@ fn build_system_status_router(server_state: Arc<SystemStatusState>) -> Router {
             move |path| metadata_file_handler(State(state), path)
         }),
     )
-    .fallback(|| async {
-        tracing::info!("[fallback handler] called");
+    .fallback(|uri: axum::http::Uri| async move {
+        tracing::debug!(%uri, "system status server has no route for this request");
         (StatusCode::NOT_FOUND, "Route not found").into_response()
     })
     .layer(TraceLayer::new_for_http().make_span_with(make_system_request_span))
@@ -344,9 +348,19 @@ impl Listener for RebindingTcpListener {
                 Ok(accepted) => return accepted,
                 // A connection that died between the SYN and our accept says
                 // nothing about the listener; take the next one immediately.
-                Err(error) if is_transient_accept_error(&error) => {
+                Err(error) if is_dead_connection_error(&error) => {
                     tracing::trace!("system status connection dropped before accept: {error}");
                 }
+                // The socket still works, the process has run out of
+                // something. Keep the socket and wait, exactly as axum does.
+                Err(error) if is_resource_exhaustion_error(&error) => {
+                    tracing::error!(
+                        "System status listener cannot accept on {} for want of a process resource; retrying the same socket after {:?}: {error}",
+                        self.address,
+                        self.rebind_backoff
+                    );
+                    tokio::time::sleep(self.rebind_backoff).await;
+                }
                 Err(error) => {
                     tracing::error!(
                         "System status listener stopped accepting on {}; rebinding after {:?}: {error}",
@@ -369,7 +383,11 @@ impl Listener for RebindingTcpListener {
     }
 }
 
-fn is_transient_accept_error(error: &io::Error) -> bool {
+/// A connection that died between the SYN and our `accept` — the client hung
+/// up, or the kernel reset it. It says nothing about the listener, so the next
+/// `accept` runs straight away. These are the three kinds axum's own listener
+/// singles out for the same treatment.
+fn is_dead_connection_error(error: &io::Error) -> bool {
     matches!(
         error.kind(),
         io::ErrorKind::ConnectionRefused
@@ -378,6 +396,40 @@ fn is_transient_accept_error(error: &io::Error) -> bool {
     )
 }
 
+/// The process has run out of file descriptors, socket buffers or memory. That
+/// is a property of the process, not of the socket, so the listener is still
+/// good and rebinding would be the wrong move twice over: it closes a working
+/// socket and discards its accept backlog, and the `bind` that follows needs a
+/// descriptor, which is the thing the process may have none of. axum sleeps
+/// and retries the same descriptor here, and so does this listener.
+///
+/// The standard library decodes `ENOMEM` as [`io::ErrorKind::OutOfMemory`],
+/// but it has no kind for `EMFILE`, `ENFILE` or `ENOBUFS` — all three arrive
+/// as `Uncategorized`, which cannot be matched on — so match their raw codes.
+fn is_resource_exhaustion_error(error: &io::Error) -> bool {
+    if error.kind() == io::ErrorKind::OutOfMemory {
+        return true;
+    }
+
+    #[cfg(unix)]
+    {
+        // `ENFILE` and `EMFILE` carry these numbers on every Unix. `ENOBUFS`
+        // is 105 on Linux and 55 on the BSDs, macOS among them.
+        const ENFILE: i32 = 23;
+        const EMFILE: i32 = 24;
+        #[cfg(target_os = "linux")]
+        const ENOBUFS: i32 = 105;
+        #[cfg(not(target_os = "linux"))]
+        const ENOBUFS: i32 = 55;
+
+        if let Some(code) = error.raw_os_error() {
+            return matches!(code, ENFILE | EMFILE | ENOBUFS);
+        }
+    }
+
+    false
+}
+
 /// Health handler with optional active health checking
 #[tracing::instrument(skip_all, level = "trace")]
 async fn health_handler(state: Arc<SystemStatusState>) -> impl IntoResponse {
@@ -913,6 +965,91 @@ mod tests {
         );
     }
 
+    /// A rebind can fail — the address may be taken, or absent from the
+    /// network namespace the process was restored into. The listener has to
+    /// keep trying rather than give up, and it has to succeed once the
+    /// obstacle clears.
+    ///
+    /// The test arranges exactly that: the wrapper's address is a port a
+    /// second socket already holds, so the first rebind fails with
+    /// `EADDRINUSE`; dropping that holder part-way through lets a later one
+    /// through. Without the retry loop the connection below is never accepted.
+    #[cfg(unix)]
+    #[tokio::test]
+    async fn test_rebinding_listener_keeps_trying_after_a_rebind_fails() {
+        // The address the wrapper will try to rebind, held by a listener the
+        // test owns and will drop.
+        let holder = TcpListener::bind("127.0.0.1:0").await.unwrap();
+        let contested_address = holder.local_addr().unwrap();
+
+        // A separate socket, broken the way CRIU breaks one, so that the very
+        // first accept sends the wrapper down the rebind path.
+        let broken = TcpListener::bind("127.0.0.1:0").await.unwrap();
+        let shutdown_result = socket2::SockRef::from(&broken).shutdown(std::net::Shutdown::Both);
+
+        let mut rebinding =
+            RebindingTcpListener::new(broken, contested_address, Duration::from_millis(10));
+
+        if let Err(error) = shutdown_result {
+            eprintln!("skipping: this platform refused shutdown on a listening socket: {error}");
+            return;
+        }
+
+        let accepted = tokio::spawn(async move { rebinding.accept().await });
+
+        // Let the rebind fail against the holder a few times over, then let go.
+        tokio::time::sleep(Duration::from_millis(100)).await;
+        drop(holder);
+
+        let client = tokio::time::timeout(Duration::from_secs(5), async {
+            loop {
+                if let Ok(stream) = tokio::net::TcpStream::connect(contested_address).await {
+                    return stream;
+                }
+                tokio::time::sleep(Duration::from_millis(10)).await;
+            }
+        })
+        .await
+        .expect("listener should rebind once the address is free again");
+
+        let (_io, peer) = tokio::time::timeout(Duration::from_secs(5), accepted)
+            .await
+            .expect("accept should return once a later rebind succeeds")
+            .expect("accept task should not panic");
+
+        assert_eq!(
+            peer,
+            client.local_addr().unwrap(),
+            "accepted connection should be the one we opened"
+        );
+    }
+
+    /// A descriptor or buffer shortage is about the process, not the socket.
+    /// Classifying one as a dead listener would close a socket that still
+    /// works and then need a descriptor to bind its replacement.
+    #[cfg(unix)]
+    #[test]
+    fn test_resource_exhaustion_does_not_look_like_a_dead_listener() {
+        for code in [23 /* ENFILE */, 24 /* EMFILE */] {
+            let error = io::Error::from_raw_os_error(code);
+            assert!(
+                is_resource_exhaustion_error(&error),
+                "errno {code} should be treated as a process resource shortage, got {:?}",
+                error.kind()
+            );
+            assert!(
+                !is_dead_connection_error(&error),
+                "errno {code} is not a dead connection"
+            );
+        }
+
+        // The error the listener-shutdown test provokes must still rebind, so
+        // it must fall through both predicates.
+        let dead_listener = io::Error::from(io::ErrorKind::InvalidInput);
+        assert!(!is_resource_exhaustion_error(&dead_listener));
+        assert!(!is_dead_connection_error(&dead_listener));
+    }
+
     #[tokio::test]
     async fn test_rebinding_listener_reports_the_address_it_rebinds() {
         let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();

From b04fe603cd23cad9c11afe8de8f57179141a38bc Mon Sep 17 00:00:00 2001
From: Matej Kosec <mkosec@nvidia.com>
Date: Thu, 20 Aug 2026 15:51:45 -0700
Subject: [PATCH 05/22] refactor(runtime): trim listener recovery commentary

Signed-off-by: Matej Kosec <mkosec@nvidia.com>
---
 lib/runtime/src/system_status_server.rs | 106 +++---------------------
 1 file changed, 11 insertions(+), 95 deletions(-)

diff --git a/lib/runtime/src/system_status_server.rs b/lib/runtime/src/system_status_server.rs
index b17b46fe3c14..b5b8f7f02f17 100644
--- a/lib/runtime/src/system_status_server.rs
+++ b/lib/runtime/src/system_status_server.rs
@@ -147,17 +147,7 @@ pub async fn spawn_system_status_server(
     let initial_bind_address = format!("{}:{}", host, port);
     let (listener, actual_address) = bind_system_status_listener(initial_bind_address).await?;
 
-    // Hand axum a listener that rebinds `actual_address` for itself. Whatever
-    // leaves the listening socket unable to accept, the server re-binds the
-    // same address the caller was told about rather than retrying a dead file
-    // descriptor for the life of the process. Nothing here is specific to
-    // CRIU; a checkpoint/restore is just the case that motivated it, and the
-    // one where re-binding also lands the server in the restored network
-    // namespace. That last part holds for the default `DYN_SYSTEM_HOST` of
-    // `0.0.0.0`, which any namespace can bind. A deployment that sets a
-    // concrete address the restored namespace does not have gets a rebind that
-    // fails with `EADDRNOTAVAIL` and a loop that retries it once a second —
-    // no worse than the stock listener, which never recovers either.
+    // Reuse the concrete address so an ephemeral port remains stable across rebinds.
     let listener =
         RebindingTcpListener::new(listener, actual_address, SYSTEM_STATUS_REBIND_BACKOFF);
     let observer = cancel_token.child_token();
@@ -290,24 +280,10 @@ async fn bind_system_status_listener(
     Ok((listener, actual_address))
 }
 
-/// A listener that owns its bind address and re-binds it when the socket
-/// underneath stops accepting connections.
-///
-/// CRIU closes the process's listening sockets when it checkpoints. After
-/// restore the file descriptor is still open but no longer usable, so `accept`
-/// fails with a non-transient error every time it is called and the system
-/// status server goes silent while the rest of the runtime carries on.
-///
-/// `axum::serve` cannot report that on its own. [`Listener::accept`] returns no
-/// `Result`, and axum's own `TcpListener` implementation logs the error, sleeps
-/// for a second and tries the same dead socket again, forever. Recovering
-/// inside the listener is the extension point the trait leaves open: the
-/// documentation for `accept` states that an implementation whose accept call
-/// can fail "must take care of logging and retrying".
+/// Rebinds after fatal accept errors.
+/// Axum's [`Listener::accept`] cannot return an error, so recovery must happen here.
 struct RebindingTcpListener {
-    /// The address to re-bind. This is the address the initial bind resolved
-    /// to, not the requested one, so a caller that asked for port 0 keeps the
-    /// concrete port it was told about.
+    /// Concrete address returned by the initial bind, including an assigned ephemeral port.
     address: std::net::SocketAddr,
     listener: Option<TcpListener>,
     rebind_backoff: Duration,
@@ -346,13 +322,11 @@ impl Listener for RebindingTcpListener {
 
             match listener.accept().await {
                 Ok(accepted) => return accepted,
-                // A connection that died between the SYN and our accept says
-                // nothing about the listener; take the next one immediately.
+                // A dropped connection does not invalidate the listener.
                 Err(error) if is_dead_connection_error(&error) => {
                     tracing::trace!("system status connection dropped before accept: {error}");
                 }
-                // The socket still works, the process has run out of
-                // something. Keep the socket and wait, exactly as axum does.
+                // Resource exhaustion does not invalidate the listener.
                 Err(error) if is_resource_exhaustion_error(&error) => {
                     tracing::error!(
                         "System status listener cannot accept on {} for want of a process resource; retrying the same socket after {:?}: {error}",
@@ -367,10 +341,7 @@ impl Listener for RebindingTcpListener {
                         self.address,
                         self.rebind_backoff
                     );
-                    // Dropping the old listener closes its file descriptor and
-                    // frees the address. A socket that has stopped accepting is
-                    // still bound, so rebinding before this would fail with
-                    // `EADDRINUSE`.
+                    // Drop before rebinding or the address remains in use.
                     self.listener = None;
                     tokio::time::sleep(self.rebind_backoff).await;
                 }
@@ -383,10 +354,7 @@ impl Listener for RebindingTcpListener {
     }
 }
 
-/// A connection that died between the SYN and our `accept` — the client hung
-/// up, or the kernel reset it. It says nothing about the listener, so the next
-/// `accept` runs straight away. These are the three kinds axum's own listener
-/// singles out for the same treatment.
+/// Matches connection failures that leave the listener usable.
 fn is_dead_connection_error(error: &io::Error) -> bool {
     matches!(
         error.kind(),
@@ -396,16 +364,8 @@ fn is_dead_connection_error(error: &io::Error) -> bool {
     )
 }
 
-/// The process has run out of file descriptors, socket buffers or memory. That
-/// is a property of the process, not of the socket, so the listener is still
-/// good and rebinding would be the wrong move twice over: it closes a working
-/// socket and discards its accept backlog, and the `bind` that follows needs a
-/// descriptor, which is the thing the process may have none of. axum sleeps
-/// and retries the same descriptor here, and so does this listener.
-///
-/// The standard library decodes `ENOMEM` as [`io::ErrorKind::OutOfMemory`],
-/// but it has no kind for `EMFILE`, `ENFILE` or `ENOBUFS` — all three arrive
-/// as `Uncategorized`, which cannot be matched on — so match their raw codes.
+/// Matches shortages that require retrying the same socket.
+/// EMFILE, ENFILE, and ENOBUFS require raw errno checks because Rust has no distinct kinds.
 fn is_resource_exhaustion_error(error: &io::Error) -> bool {
     if error.kind() == io::ErrorKind::OutOfMemory {
         return true;
@@ -413,8 +373,7 @@ fn is_resource_exhaustion_error(error: &io::Error) -> bool {
 
     #[cfg(unix)]
     {
-        // `ENFILE` and `EMFILE` carry these numbers on every Unix. `ENOBUFS`
-        // is 105 on Linux and 55 on the BSDs, macOS among them.
+        // ENOBUFS differs between Linux and BSD-derived systems.
         const ENFILE: i32 = 23;
         const EMFILE: i32 = 24;
         #[cfg(target_os = "linux")]
@@ -904,44 +863,23 @@ mod tests {
         );
     }
 
-    /// A listener that has been shut down keeps failing `accept`, so the only
-    /// way a connection can be served afterwards is if the listener rebound.
-    ///
-    /// `shutdown(2)` on a listening socket is the closest stand-in for what
-    /// CRIU does to it, and only some platforms allow it. Linux shuts the
-    /// listener down and every later `accept` fails, which is the case this
-    /// test exists for. Darwin instead rejects the call with `ENOTCONN`,
-    /// because it only shuts down connected sockets, so there is no failure to
-    /// recover from and the test returns early rather than asserting on a
-    /// recovery that was never provoked.
-    ///
-    /// `socket2::SockRef` issues the same syscall as a raw `libc::shutdown`
-    /// without an `unsafe` block or a raw descriptor, and `socket2` is already
-    /// a dependency of this crate.
     #[cfg(unix)]
     #[tokio::test]
     async fn test_rebinding_listener_serves_again_after_listener_shutdown() {
         let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
         let address = listener.local_addr().unwrap();
 
-        // Take the reference before the listener moves into the wrapper.
         let shutdown_result = socket2::SockRef::from(&listener).shutdown(std::net::Shutdown::Both);
 
         let mut rebinding = RebindingTcpListener::new(listener, address, Duration::from_millis(10));
 
         if let Err(error) = shutdown_result {
-            // Nothing broke the listener, so `accept` still works and the
-            // rebind path is never reached. Asserting anything past this point
-            // would test the ordinary accept path under a name that promises
-            // otherwise.
             eprintln!("skipping: this platform refused shutdown on a listening socket: {error}");
             return;
         }
 
         let accepted = tokio::spawn(async move { rebinding.accept().await });
 
-        // The address is unbound between the old listener being dropped and the
-        // rebind completing, so connecting has to be retried.
         let client = tokio::time::timeout(Duration::from_secs(5), async {
             loop {
                 if let Ok(stream) = tokio::net::TcpStream::connect(address).await {
@@ -965,25 +903,12 @@ mod tests {
         );
     }
 
-    /// A rebind can fail — the address may be taken, or absent from the
-    /// network namespace the process was restored into. The listener has to
-    /// keep trying rather than give up, and it has to succeed once the
-    /// obstacle clears.
-    ///
-    /// The test arranges exactly that: the wrapper's address is a port a
-    /// second socket already holds, so the first rebind fails with
-    /// `EADDRINUSE`; dropping that holder part-way through lets a later one
-    /// through. Without the retry loop the connection below is never accepted.
     #[cfg(unix)]
     #[tokio::test]
     async fn test_rebinding_listener_keeps_trying_after_a_rebind_fails() {
-        // The address the wrapper will try to rebind, held by a listener the
-        // test owns and will drop.
         let holder = TcpListener::bind("127.0.0.1:0").await.unwrap();
         let contested_address = holder.local_addr().unwrap();
 
-        // A separate socket, broken the way CRIU breaks one, so that the very
-        // first accept sends the wrapper down the rebind path.
         let broken = TcpListener::bind("127.0.0.1:0").await.unwrap();
         let shutdown_result = socket2::SockRef::from(&broken).shutdown(std::net::Shutdown::Both);
 
@@ -997,7 +922,6 @@ mod tests {
 
         let accepted = tokio::spawn(async move { rebinding.accept().await });
 
-        // Let the rebind fail against the holder a few times over, then let go.
         tokio::time::sleep(Duration::from_millis(100)).await;
         drop(holder);
 
@@ -1024,9 +948,6 @@ mod tests {
         );
     }
 
-    /// A descriptor or buffer shortage is about the process, not the socket.
-    /// Classifying one as a dead listener would close a socket that still
-    /// works and then need a descriptor to bind its replacement.
     #[cfg(unix)]
     #[test]
     fn test_resource_exhaustion_does_not_look_like_a_dead_listener() {
@@ -1043,8 +964,6 @@ mod tests {
             );
         }
 
-        // The error the listener-shutdown test provokes must still rebind, so
-        // it must fall through both predicates.
         let dead_listener = io::Error::from(io::ErrorKind::InvalidInput);
         assert!(!is_resource_exhaustion_error(&dead_listener));
         assert!(!is_dead_connection_error(&dead_listener));
@@ -1065,9 +984,6 @@ mod tests {
         );
     }
 
-    /// Recovery must not cost graceful shutdown: axum selects the shutdown
-    /// signal against `accept`, so a listener stuck in a rebind loop still
-    /// stops when the token is cancelled.
     #[tokio::test]
     async fn test_rebinding_listener_still_shuts_down_gracefully() {
         let cancel_token = CancellationToken::new();

From db68507e9e902d4ae8d511b8c0751ea0171ff469 Mon Sep 17 00:00:00 2001
From: GLAMR <svc-glamr@nvidia.com>
Date: Sat, 12 Sep 2026 00:48:49 +0000
Subject: [PATCH 06/22] test(runtime): remove redundant rebinding listener
 assertion

Signed-off-by: GLAMR <svc-glamr@nvidia.com>
---
 lib/runtime/src/system_status_server.rs | 15 ---------------
 1 file changed, 15 deletions(-)

diff --git a/lib/runtime/src/system_status_server.rs b/lib/runtime/src/system_status_server.rs
index 8979779ecc0f..b2ceea920d0f 100644
--- a/lib/runtime/src/system_status_server.rs
+++ b/lib/runtime/src/system_status_server.rs
@@ -969,21 +969,6 @@ mod tests {
         assert!(!is_dead_connection_error(&dead_listener));
     }
 
-    #[tokio::test]
-    async fn test_rebinding_listener_reports_the_address_it_rebinds() {
-        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
-        let address = listener.local_addr().unwrap();
-
-        let rebinding = RebindingTcpListener::new(listener, address, Duration::from_millis(10));
-
-        assert_ne!(address.port(), 0, "bind should resolve the ephemeral port");
-        assert_eq!(
-            rebinding.local_addr().unwrap(),
-            address,
-            "the resolved address is what gets rebound and advertised"
-        );
-    }
-
     #[tokio::test]
     async fn test_rebinding_listener_still_shuts_down_gracefully() {
         let cancel_token = CancellationToken::new();

From 9c78688afb5682e956d77074920d7ef520c5fc14 Mon Sep 17 00:00:00 2001
From: GLAMR <svc-glamr@nvidia.com>
Date: Sat, 12 Sep 2026 01:50:20 +0000
Subject: [PATCH 07/22] docs(runtime): remove redundant listener comment

Signed-off-by: GLAMR <svc-glamr@nvidia.com>
---
 lib/runtime/src/system_status_server.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/runtime/src/system_status_server.rs b/lib/runtime/src/system_status_server.rs
index b2ceea920d0f..c3257ef9fd5d 100644
--- a/lib/runtime/src/system_status_server.rs
+++ b/lib/runtime/src/system_status_server.rs
@@ -280,7 +280,6 @@ async fn bind_system_status_listener(
     Ok((listener, actual_address))
 }
 
-/// Rebinds after fatal accept errors.
 /// Axum's [`Listener::accept`] cannot return an error, so recovery must happen here.
 struct RebindingTcpListener {
     /// Concrete address returned by the initial bind, including an assigned ephemeral port.

From ba8f22532804a43a7fe5defb5ae921db93613b39 Mon Sep 17 00:00:00 2001
From: GLAMR <svc-glamr@nvidia.com>
Date: Sat, 12 Sep 2026 02:07:17 +0000
Subject: [PATCH 08/22] test(runtime): avoid pre-rebind connection race

Signed-off-by: GLAMR <svc-glamr@nvidia.com>
---
 lib/runtime/src/system_status_server.rs | 24 ++++++++++++++----------
 1 file changed, 14 insertions(+), 10 deletions(-)

diff --git a/lib/runtime/src/system_status_server.rs b/lib/runtime/src/system_status_server.rs
index c3257ef9fd5d..ec13485e5dc4 100644
--- a/lib/runtime/src/system_status_server.rs
+++ b/lib/runtime/src/system_status_server.rs
@@ -879,25 +879,29 @@ mod tests {
 
         let accepted = tokio::spawn(async move { rebinding.accept().await });
 
-        let client = tokio::time::timeout(Duration::from_secs(5), async {
+        let mut accepted = accepted;
+        let (peer, client_addr) = tokio::time::timeout(Duration::from_secs(5), async {
             loop {
-                if let Ok(stream) = tokio::net::TcpStream::connect(address).await {
-                    return stream;
+                let Ok(stream) = tokio::net::TcpStream::connect(address).await else {
+                    tokio::time::sleep(Duration::from_millis(10)).await;
+                    continue;
+                };
+                let client_addr = stream.local_addr().unwrap();
+                tokio::select! {
+                    result = &mut accepted => {
+                        let (_io, peer) = result.expect("accept task should not panic");
+                        return (peer, client_addr);
+                    }
+                    _ = tokio::time::sleep(Duration::from_millis(50)) => {}
                 }
-                tokio::time::sleep(Duration::from_millis(10)).await;
             }
         })
         .await
         .expect("listener should rebind and accept connections again");
 
-        let (_io, peer) = tokio::time::timeout(Duration::from_secs(5), accepted)
-            .await
-            .expect("accept should return once the listener is rebound")
-            .expect("accept task should not panic");
-
         assert_eq!(
             peer,
-            client.local_addr().unwrap(),
+            client_addr,
             "accepted connection should be the one we opened"
         );
     }

From 6776b2fac7f4fff5ad8dbefa9793a02e5d827d7b Mon Sep 17 00:00:00 2001
From: GLAMR <svc-glamr@nvidia.com>
Date: Sat, 12 Sep 2026 02:10:03 +0000
Subject: [PATCH 09/22] style(runtime): format listener rebind test

Signed-off-by: GLAMR <svc-glamr@nvidia.com>
---
 lib/runtime/src/system_status_server.rs | 33 +++++++++++++------------
 1 file changed, 17 insertions(+), 16 deletions(-)

diff --git a/lib/runtime/src/system_status_server.rs b/lib/runtime/src/system_status_server.rs
index ec13485e5dc4..c659c24b9794 100644
--- a/lib/runtime/src/system_status_server.rs
+++ b/lib/runtime/src/system_status_server.rs
@@ -880,24 +880,25 @@ mod tests {
         let accepted = tokio::spawn(async move { rebinding.accept().await });
 
         let mut accepted = accepted;
-        let (peer, client_addr) = tokio::time::timeout(Duration::from_secs(5), async {
-            loop {
-                let Ok(stream) = tokio::net::TcpStream::connect(address).await else {
-                    tokio::time::sleep(Duration::from_millis(10)).await;
-                    continue;
-                };
-                let client_addr = stream.local_addr().unwrap();
-                tokio::select! {
-                    result = &mut accepted => {
-                        let (_io, peer) = result.expect("accept task should not panic");
-                        return (peer, client_addr);
+        let (peer, client_addr) =
+            tokio::time::timeout(Duration::from_secs(5), async {
+                loop {
+                    let Ok(stream) = tokio::net::TcpStream::connect(address).await else {
+                        tokio::time::sleep(Duration::from_millis(10)).await;
+                        continue;
+                    };
+                    let client_addr = stream.local_addr().unwrap();
+                    tokio::select! {
+                        result = &mut accepted => {
+                            let (_io, peer) = result.expect("accept task should not panic");
+                            return (peer, client_addr);
+                        }
+                        _ = tokio::time::sleep(Duration::from_millis(50)) => {}
                     }
-                    _ = tokio::time::sleep(Duration::from_millis(50)) => {}
                 }
-            }
-        })
-        .await
-        .expect("listener should rebind and accept connections again");
+            })
+            .await
+            .expect("listener should rebind and accept connections again");
 
         assert_eq!(
             peer,

From 6ade20a9ce290c35e1081ae7e2ab978a5799e9ae Mon Sep 17 00:00:00 2001
From: GLAMR <svc-glamr@nvidia.com>
Date: Sat, 12 Sep 2026 02:26:48 +0000
Subject: [PATCH 10/22] style(runtime): format rebinding listener code

Signed-off-by: GLAMR <svc-glamr@nvidia.com>
---
 lib/runtime/src/system_status_server.rs | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/lib/runtime/src/system_status_server.rs b/lib/runtime/src/system_status_server.rs
index c659c24b9794..0023366ed829 100644
--- a/lib/runtime/src/system_status_server.rs
+++ b/lib/runtime/src/system_status_server.rs
@@ -289,7 +289,11 @@ struct RebindingTcpListener {
 }
 
 impl RebindingTcpListener {
-    fn new(listener: TcpListener, address: std::net::SocketAddr, rebind_backoff: Duration) -> Self {
+    fn new(
+        listener: TcpListener,
+        address: std::net::SocketAddr,
+        rebind_backoff: Duration,
+    ) -> Self {
         Self {
             address,
             listener: Some(listener),
@@ -868,9 +872,11 @@ mod tests {
         let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
         let address = listener.local_addr().unwrap();
 
-        let shutdown_result = socket2::SockRef::from(&listener).shutdown(std::net::Shutdown::Both);
+        let shutdown_result =
+            socket2::SockRef::from(&listener).shutdown(std::net::Shutdown::Both);
 
-        let mut rebinding = RebindingTcpListener::new(listener, address, Duration::from_millis(10));
+        let mut rebinding =
+            RebindingTcpListener::new(listener, address, Duration::from_millis(10));
 
         if let Err(error) = shutdown_result {
             eprintln!("skipping: this platform refused shutdown on a listening socket: {error}");
@@ -914,7 +920,8 @@ mod tests {
         let contested_address = holder.local_addr().unwrap();
 
         let broken = TcpListener::bind("127.0.0.1:0").await.unwrap();
-        let shutdown_result = socket2::SockRef::from(&broken).shutdown(std::net::Shutdown::Both);
+        let shutdown_result =
+            socket2::SockRef::from(&broken).shutdown(std::net::Shutdown::Both);
 
         let mut rebinding =
             RebindingTcpListener::new(broken, contested_address, Duration::from_millis(10));

From b8b44f53e1491a4f936ea75c617d438160393127 Mon Sep 17 00:00:00 2001
From: GLAMR <svc-glamr@nvidia.com>
Date: Sat, 12 Sep 2026 02:46:42 +0000
Subject: [PATCH 11/22] fix(runtime): stabilize system health listener recovery
 tests

Signed-off-by: GLAMR <svc-glamr@nvidia.com>
---
 lib/runtime/src/system_status_server.rs | 72 +++++++++++++------------
 1 file changed, 39 insertions(+), 33 deletions(-)

diff --git a/lib/runtime/src/system_status_server.rs b/lib/runtime/src/system_status_server.rs
index 0023366ed829..a3d041fb465f 100644
--- a/lib/runtime/src/system_status_server.rs
+++ b/lib/runtime/src/system_status_server.rs
@@ -145,7 +145,16 @@ pub async fn spawn_system_status_server(
     let app = build_system_status_router(server_state);
 
     let initial_bind_address = format!("{}:{}", host, port);
-    let (listener, actual_address) = bind_system_status_listener(initial_bind_address).await?;
+    tracing::info!("[spawn_system_status_server] binding to: {initial_bind_address}");
+    let (listener, actual_address) = bind_system_status_listener(initial_bind_address.clone())
+        .await
+        .map_err(|error| {
+            tracing::error!("Failed to bind to address {initial_bind_address}: {error}");
+            error
+        })?;
+    tracing::info!(
+        "[spawn_system_status_server] system status server bound to: {actual_address}"
+    );
 
     // Reuse the concrete address so an ephemeral port remains stable across rebinds.
     let listener =
@@ -264,18 +273,11 @@ fn build_system_status_router(server_state: Arc<SystemStatusState>) -> Router {
 async fn bind_system_status_listener(
     address: String,
 ) -> anyhow::Result<(TcpListener, std::net::SocketAddr)> {
-    tracing::info!("[spawn_system_status_server] binding to: {address}");
-
-    let listener = TcpListener::bind(&address).await.map_err(|e| {
-        tracing::error!("Failed to bind to address {}: {}", address, e);
-        anyhow::anyhow!("Failed to bind to address: {}", e)
-    })?;
+    let listener = TcpListener::bind(&address)
+        .await
+        .map_err(|error| anyhow::anyhow!("Failed to bind to address: {error}"))?;
 
     let actual_address = listener.local_addr()?;
-    tracing::info!(
-        "[spawn_system_status_server] system status server bound to: {}",
-        actual_address
-    );
 
     Ok((listener, actual_address))
 }
@@ -309,8 +311,12 @@ impl Listener for RebindingTcpListener {
     async fn accept(&mut self) -> (Self::Io, Self::Addr) {
         loop {
             let Some(listener) = self.listener.as_ref() else {
+                tracing::info!("System status server rebinding to {}", self.address);
                 match bind_system_status_listener(self.address.to_string()).await {
-                    Ok((listener, _)) => self.listener = Some(listener),
+                    Ok((listener, actual_address)) => {
+                        tracing::info!("System status server rebound to {actual_address}");
+                        self.listener = Some(listener);
+                    }
                     Err(error) => {
                         tracing::error!(
                             "System status server failed to rebind {}; retrying after {:?}: {error}",
@@ -886,30 +892,30 @@ mod tests {
         let accepted = tokio::spawn(async move { rebinding.accept().await });
 
         let mut accepted = accepted;
-        let (peer, client_addr) =
-            tokio::time::timeout(Duration::from_secs(5), async {
-                loop {
-                    let Ok(stream) = tokio::net::TcpStream::connect(address).await else {
-                        tokio::time::sleep(Duration::from_millis(10)).await;
-                        continue;
-                    };
-                    let client_addr = stream.local_addr().unwrap();
-                    tokio::select! {
-                        result = &mut accepted => {
-                            let (_io, peer) = result.expect("accept task should not panic");
-                            return (peer, client_addr);
-                        }
-                        _ = tokio::time::sleep(Duration::from_millis(50)) => {}
+        let mut clients = Vec::new();
+        let peer = tokio::time::timeout(Duration::from_secs(5), async {
+            loop {
+                let Ok(stream) = tokio::net::TcpStream::connect(address).await else {
+                    tokio::time::sleep(Duration::from_millis(10)).await;
+                    continue;
+                };
+                let client_addr = stream.local_addr().unwrap();
+                clients.push((stream, client_addr));
+                tokio::select! {
+                    result = &mut accepted => {
+                        let (_io, peer) = result.expect("accept task should not panic");
+                        return peer;
                     }
+                    _ = tokio::time::sleep(Duration::from_millis(50)) => {}
                 }
-            })
-            .await
-            .expect("listener should rebind and accept connections again");
+            }
+        })
+        .await
+        .expect("listener should rebind and accept connections again");
 
-        assert_eq!(
-            peer,
-            client_addr,
-            "accepted connection should be the one we opened"
+        assert!(
+            clients.iter().any(|(_, client_addr)| *client_addr == peer),
+            "accepted connection should be one we opened"
         );
     }
 

From bb6ddb7f2ae1901dbd7662e302885321b11c1632 Mon Sep 17 00:00:00 2001
From: GLAMR <svc-glamr@nvidia.com>
Date: Sat, 12 Sep 2026 02:57:24 +0000
Subject: [PATCH 12/22] style(runtime): format system health server logging

Signed-off-by: GLAMR <svc-glamr@nvidia.com>
---
 lib/runtime/src/system_status_server.rs | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/lib/runtime/src/system_status_server.rs b/lib/runtime/src/system_status_server.rs
index a3d041fb465f..a44c71df83ff 100644
--- a/lib/runtime/src/system_status_server.rs
+++ b/lib/runtime/src/system_status_server.rs
@@ -152,9 +152,7 @@ pub async fn spawn_system_status_server(
             tracing::error!("Failed to bind to address {initial_bind_address}: {error}");
             error
         })?;
-    tracing::info!(
-        "[spawn_system_status_server] system status server bound to: {actual_address}"
-    );
+    tracing::info!("[spawn_system_status_server] system status server bound to: {actual_address}");
 
     // Reuse the concrete address so an ephemeral port remains stable across rebinds.
     let listener =

From 5c6f8ff897365709ed20e7cf0c891dd9068cf59b Mon Sep 17 00:00:00 2001
From: GLAMR <svc-glamr@nvidia.com>
Date: Sat, 12 Sep 2026 03:00:16 +0000
Subject: [PATCH 13/22] style(runtime): format status listener startup bind

Signed-off-by: GLAMR <svc-glamr@nvidia.com>
---
 lib/runtime/src/system_status_server.rs | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/lib/runtime/src/system_status_server.rs b/lib/runtime/src/system_status_server.rs
index a44c71df83ff..088c71725e77 100644
--- a/lib/runtime/src/system_status_server.rs
+++ b/lib/runtime/src/system_status_server.rs
@@ -146,12 +146,13 @@ pub async fn spawn_system_status_server(
 
     let initial_bind_address = format!("{}:{}", host, port);
     tracing::info!("[spawn_system_status_server] binding to: {initial_bind_address}");
-    let (listener, actual_address) = bind_system_status_listener(initial_bind_address.clone())
-        .await
-        .map_err(|error| {
-            tracing::error!("Failed to bind to address {initial_bind_address}: {error}");
-            error
-        })?;
+    let (listener, actual_address) =
+        bind_system_status_listener(initial_bind_address.clone())
+            .await
+            .map_err(|error| {
+                tracing::error!("Failed to bind to address {initial_bind_address}: {error}");
+                error
+            })?;
     tracing::info!("[spawn_system_status_server] system status server bound to: {actual_address}");
 
     // Reuse the concrete address so an ephemeral port remains stable across rebinds.

From 246e5b8e1b8dc61f93afc52f05f495a6c4898cbe Mon Sep 17 00:00:00 2001
From: GLAMR <svc-glamr@nvidia.com>
Date: Sat, 12 Sep 2026 03:03:45 +0000
Subject: [PATCH 14/22] style(runtime): format system health listener recovery

Signed-off-by: GLAMR <svc-glamr@nvidia.com>
---
 lib/runtime/src/system_status_server.rs | 28 +++++++++----------------
 1 file changed, 10 insertions(+), 18 deletions(-)

diff --git a/lib/runtime/src/system_status_server.rs b/lib/runtime/src/system_status_server.rs
index 088c71725e77..4c1300e4c21e 100644
--- a/lib/runtime/src/system_status_server.rs
+++ b/lib/runtime/src/system_status_server.rs
@@ -146,13 +146,12 @@ pub async fn spawn_system_status_server(
 
     let initial_bind_address = format!("{}:{}", host, port);
     tracing::info!("[spawn_system_status_server] binding to: {initial_bind_address}");
-    let (listener, actual_address) =
-        bind_system_status_listener(initial_bind_address.clone())
-            .await
-            .map_err(|error| {
-                tracing::error!("Failed to bind to address {initial_bind_address}: {error}");
-                error
-            })?;
+    let (listener, actual_address) = bind_system_status_listener(initial_bind_address.clone())
+        .await
+        .map_err(|error| {
+            tracing::error!("Failed to bind to address {initial_bind_address}: {error}");
+            error
+        })?;
     tracing::info!("[spawn_system_status_server] system status server bound to: {actual_address}");
 
     // Reuse the concrete address so an ephemeral port remains stable across rebinds.
@@ -290,11 +289,7 @@ struct RebindingTcpListener {
 }
 
 impl RebindingTcpListener {
-    fn new(
-        listener: TcpListener,
-        address: std::net::SocketAddr,
-        rebind_backoff: Duration,
-    ) -> Self {
+    fn new(listener: TcpListener, address: std::net::SocketAddr, rebind_backoff: Duration) -> Self {
         Self {
             address,
             listener: Some(listener),
@@ -877,11 +872,9 @@ mod tests {
         let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
         let address = listener.local_addr().unwrap();
 
-        let shutdown_result =
-            socket2::SockRef::from(&listener).shutdown(std::net::Shutdown::Both);
+        let shutdown_result = socket2::SockRef::from(&listener).shutdown(std::net::Shutdown::Both);
 
-        let mut rebinding =
-            RebindingTcpListener::new(listener, address, Duration::from_millis(10));
+        let mut rebinding = RebindingTcpListener::new(listener, address, Duration::from_millis(10));
 
         if let Err(error) = shutdown_result {
             eprintln!("skipping: this platform refused shutdown on a listening socket: {error}");
@@ -925,8 +918,7 @@ mod tests {
         let contested_address = holder.local_addr().unwrap();
 
         let broken = TcpListener::bind("127.0.0.1:0").await.unwrap();
-        let shutdown_result =
-            socket2::SockRef::from(&broken).shutdown(std::net::Shutdown::Both);
+        let shutdown_result = socket2::SockRef::from(&broken).shutdown(std::net::Shutdown::Both);
 
         let mut rebinding =
             RebindingTcpListener::new(broken, contested_address, Duration::from_millis(10));

From 975f4f0488c8c5e5b98d1fa7c184ce1d4dfe0586 Mon Sep 17 00:00:00 2001
From: GLAMR <svc-glamr@nvidia.com>
Date: Sat, 12 Sep 2026 04:08:39 +0000
Subject: [PATCH 15/22] test(runtime): remove redundant listener shutdown test

Signed-off-by: GLAMR <svc-glamr@nvidia.com>
---
 lib/runtime/src/system_status_server.rs | 25 -------------------------
 1 file changed, 25 deletions(-)

diff --git a/lib/runtime/src/system_status_server.rs b/lib/runtime/src/system_status_server.rs
index 4c1300e4c21e..ad38aa2badfa 100644
--- a/lib/runtime/src/system_status_server.rs
+++ b/lib/runtime/src/system_status_server.rs
@@ -977,31 +977,6 @@ mod tests {
         assert!(!is_dead_connection_error(&dead_listener));
     }
 
-    #[tokio::test]
-    async fn test_rebinding_listener_still_shuts_down_gracefully() {
-        let cancel_token = CancellationToken::new();
-        let app = Router::new().route("/test", get(|| async { (StatusCode::OK, "test") }));
-        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
-        let address = listener.local_addr().unwrap();
-        let listener = RebindingTcpListener::new(listener, address, Duration::from_millis(10));
-
-        let server = tokio::spawn({
-            let cancel_token = cancel_token.clone();
-            async move {
-                axum::serve(listener, app)
-                    .with_graceful_shutdown(cancel_token.cancelled_owned())
-                    .await
-            }
-        });
-
-        cancel_token.cancel();
-
-        tokio::time::timeout(Duration::from_secs(5), server)
-            .await
-            .expect("server should shut down when the cancel token is cancelled")
-            .expect("server task should not panic")
-            .expect("graceful shutdown should not error");
-    }
 }
 
 // Integration tests: cargo test system_status_server --lib --features integration

From ab03fa4e46cd419ab535d151847d2c8f2f7544c5 Mon Sep 17 00:00:00 2001
From: GLAMR <svc-glamr@nvidia.com>
Date: Sat, 12 Sep 2026 04:27:05 +0000
Subject: [PATCH 16/22] style(runtime): remove stale blank line

Signed-off-by: GLAMR <svc-glamr@nvidia.com>
---
 lib/runtime/src/system_status_server.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/runtime/src/system_status_server.rs b/lib/runtime/src/system_status_server.rs
index ad38aa2badfa..de2f0800ad90 100644
--- a/lib/runtime/src/system_status_server.rs
+++ b/lib/runtime/src/system_status_server.rs
@@ -976,7 +976,6 @@ mod tests {
         assert!(!is_resource_exhaustion_error(&dead_listener));
         assert!(!is_dead_connection_error(&dead_listener));
     }
-
 }
 
 // Integration tests: cargo test system_status_server --lib --features integration

From 60a525557fc81e483b85caf6b24b4f0931e308d1 Mon Sep 17 00:00:00 2001
From: GLAMR <svc-glamr@nvidia.com>
Date: Thu, 17 Sep 2026 19:10:33 +0000
Subject: [PATCH 17/22] fix(runtime): attempt listener recovery before backing
 off

Signed-off-by: GLAMR <svc-glamr@nvidia.com>
---
 lib/runtime/src/system_status_server.rs | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/lib/runtime/src/system_status_server.rs b/lib/runtime/src/system_status_server.rs
index de2f0800ad90..6835536b6f1c 100644
--- a/lib/runtime/src/system_status_server.rs
+++ b/lib/runtime/src/system_status_server.rs
@@ -303,8 +303,11 @@ impl Listener for RebindingTcpListener {
     type Addr = std::net::SocketAddr;
 
     async fn accept(&mut self) -> (Self::Io, Self::Addr) {
+        let mut next_rebind = tokio::time::Instant::now();
         loop {
             let Some(listener) = self.listener.as_ref() else {
+                tokio::time::sleep_until(next_rebind).await;
+                next_rebind = tokio::time::Instant::now() + self.rebind_backoff;
                 tracing::info!("System status server rebinding to {}", self.address);
                 match bind_system_status_listener(self.address.to_string()).await {
                     Ok((listener, actual_address)) => {
@@ -317,7 +320,6 @@ impl Listener for RebindingTcpListener {
                             self.address,
                             self.rebind_backoff
                         );
-                        tokio::time::sleep(self.rebind_backoff).await;
                     }
                 }
                 continue;
@@ -340,13 +342,11 @@ impl Listener for RebindingTcpListener {
                 }
                 Err(error) => {
                     tracing::error!(
-                        "System status listener stopped accepting on {}; rebinding after {:?}: {error}",
-                        self.address,
-                        self.rebind_backoff
+                        "System status listener stopped accepting on {}; rebinding: {error}",
+                        self.address
                     );
                     // Drop before rebinding or the address remains in use.
                     self.listener = None;
-                    tokio::time::sleep(self.rebind_backoff).await;
                 }
             }
         }

From 96df87143bf0065b90968a8692fe2e6ddfcbdb62 Mon Sep 17 00:00:00 2001
From: GLAMR <svc-glamr@nvidia.com>
Date: Thu, 17 Sep 2026 19:27:16 +0000
Subject: [PATCH 18/22] fix(ci): pass framework to the Triton PR test job

Apply the PR workflow fix from upstream f2becb5fa21fe74e9c9bbd2ec38666fafc656565 to unblock full CI startup.

Signed-off-by: GLAMR <svc-glamr@nvidia.com>
---
 .github/workflows/pr.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml
index 57936e0b4ed6..09428070b404 100644
--- a/.github/workflows/pr.yaml
+++ b/.github/workflows/pr.yaml
@@ -940,6 +940,7 @@ jobs:
     uses: $/.github/workflows/shared-test.yml
     with:
       test_suite_name: triton
+      framework: triton
       test_type: CPU Test
       amd_runner: prod-tester-amd-v2
       target_tag_plain: ${{ needs.triton-build.outputs.target_tag_plain }}

From 323db1be6f6520db1167d06b3569ec2768329269 Mon Sep 17 00:00:00 2001
From: GLAMR <svc-glamr@nvidia.com>
Date: Thu, 17 Sep 2026 19:54:31 +0000
Subject: [PATCH 19/22] fix(ci): include graph synthesis dependency in test
 images

Apply the networkx test dependency fix from upstream df09fa6a998e7177c6bc7a2e47e32a90d92d0cb2. Triton CPU CI otherwise fails whole-tree collection in benchmarks/router/tests/test_common.py.

Signed-off-by: GLAMR <svc-glamr@nvidia.com>
---
 container/deps/requirements.test.txt | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/container/deps/requirements.test.txt b/container/deps/requirements.test.txt
index 15a6091a0809..a9153b3848da 100644
--- a/container/deps/requirements.test.txt
+++ b/container/deps/requirements.test.txt
@@ -45,6 +45,8 @@ mistral-common>=1.11.2
 mypy==1.18.2
 # For NATS object store verification in router tests
 nats-py==2.12.0
+# Router benchmark tests import graph synthesis during whole-tree collection.
+networkx
 openai==2.32.0  # for Fault Tolerance migration tests
 # Protobuf definitions the OTLP parity receiver decodes; see grpcio above.
 # Version-matched to the opentelemetry-* pins in requirements.common.txt.

From 950a79699af9fd23882b59dcdeafcf222ff64b00 Mon Sep 17 00:00:00 2001
From: GLAMR <svc-glamr@nvidia.com>
Date: Mon, 28 Sep 2026 16:51:51 +0000
Subject: [PATCH 20/22] test(runtime): preserve listener backlog during
 descriptor exhaustion

Signed-off-by: GLAMR <svc-glamr@nvidia.com>
---
 lib/runtime/src/system_status_server.rs | 88 ++++++++++++++++++++++++-
 1 file changed, 85 insertions(+), 3 deletions(-)

diff --git a/lib/runtime/src/system_status_server.rs b/lib/runtime/src/system_status_server.rs
index e62b71214cb8..1abbc1851efc 100644
--- a/lib/runtime/src/system_status_server.rs
+++ b/lib/runtime/src/system_status_server.rs
@@ -1021,10 +1021,92 @@ mod tests {
                 "errno {code} is not a dead connection"
             );
         }
+    }
+
+    #[cfg(target_os = "linux")]
+    #[tokio::test]
+    async fn test_rebinding_listener_preserves_backlog_during_resource_exhaustion() {
+        const CHILD_ENV: &str = "DYNAMO_TEST_LISTENER_EMFILE_CHILD";
+        if std::env::var_os(CHILD_ENV).is_none() {
+            // Descriptor limits are process-wide; isolate this test from sibling tests.
+            let output = tokio::time::timeout(
+                Duration::from_secs(15),
+                tokio::process::Command::new(std::env::current_exe().unwrap())
+                    .args([
+                        "--exact",
+                        "system_status_server::tests::test_rebinding_listener_preserves_backlog_during_resource_exhaustion",
+                        "--nocapture",
+                    ])
+                    .env(CHILD_ENV, "1")
+                    .kill_on_drop(true)
+                    .output(),
+            )
+            .await
+            .expect("descriptor-exhaustion child should finish promptly")
+            .expect("test child should start");
+            assert!(
+                output.status.success(),
+                "descriptor-exhaustion child failed:\n{}\n{}",
+                String::from_utf8_lossy(&output.stdout),
+                String::from_utf8_lossy(&output.stderr),
+            );
+            assert!(String::from_utf8_lossy(&output.stdout).contains("1 passed"));
+            return;
+        }
+
+        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+        let address = listener.local_addr().unwrap();
+        let client = tokio::net::TcpStream::connect(address).await.unwrap();
+        let peer = client.local_addr().unwrap();
+
+        let mut limit = libc::rlimit {
+            rlim_cur: 0,
+            rlim_max: 0,
+        };
+        // SAFETY: limit is writable and correctly aligned. Only this child is affected.
+        assert_eq!(
+            unsafe { libc::getrlimit(libc::RLIMIT_NOFILE, &mut limit) },
+            0
+        );
+        limit.rlim_cur = limit.rlim_cur.min(96);
+        // SAFETY: limit is initialized; the soft limit is lowered without changing the hard limit.
+        assert_eq!(unsafe { libc::setrlimit(libc::RLIMIT_NOFILE, &limit) }, 0);
 
-        let dead_listener = io::Error::from(io::ErrorKind::InvalidInput);
-        assert!(!is_resource_exhaustion_error(&dead_listener));
-        assert!(!is_dead_connection_error(&dead_listener));
+        let mut descriptors = Vec::new();
+        loop {
+            match std::fs::File::open("/dev/null") {
+                Ok(file) => descriptors.push(file),
+                Err(error) => {
+                    assert_eq!(error.raw_os_error(), Some(libc::EMFILE));
+                    break;
+                }
+            }
+        }
+        let error = listener.accept().await.unwrap_err();
+        assert_eq!(error.raw_os_error(), Some(libc::EMFILE));
+
+        let mut rebinding =
+            RebindingTcpListener::new(listener, address, Duration::from_millis(200));
+        assert!(
+            tokio::time::timeout(Duration::from_millis(50), rebinding.accept())
+                .await
+                .is_err(),
+            "accept should wait during descriptor exhaustion"
+        );
+        drop(descriptors);
+
+        // A fresh bind rules out lingering descriptor pressure as the cause of AddrInUse.
+        let _control = TcpListener::bind("127.0.0.1:0").await.unwrap();
+        assert_eq!(
+            TcpListener::bind(address).await.unwrap_err().kind(),
+            io::ErrorKind::AddrInUse,
+            "the original listening address should remain held"
+        );
+        let (_stream, accepted_peer) =
+            tokio::time::timeout(Duration::from_secs(2), rebinding.accept())
+                .await
+                .expect("the queued connection should survive the shortage");
+        assert_eq!(accepted_peer, peer);
     }
 }
 

From f11a1846f18af9cdedd95bbf7b888c2ccf601fcf Mon Sep 17 00:00:00 2001
From: GLAMR <svc-glamr@nvidia.com>
Date: Mon, 5 Oct 2026 20:07:41 +0000
Subject: [PATCH 21/22] fix(runtime): retain status listener on pending network
 errors

Signed-off-by: GLAMR <svc-glamr@nvidia.com>
---
 lib/runtime/src/system_status_server.rs | 154 +++++++++++++++++-------
 1 file changed, 113 insertions(+), 41 deletions(-)

diff --git a/lib/runtime/src/system_status_server.rs b/lib/runtime/src/system_status_server.rs
index 160d34f6ac22..67b6e99034da 100644
--- a/lib/runtime/src/system_status_server.rs
+++ b/lib/runtime/src/system_status_server.rs
@@ -206,7 +206,7 @@ pub async fn spawn_system_status_server(
 
     let initial_bind_address = format!("{}:{}", host, port);
     tracing::info!("[spawn_system_status_server] binding to: {initial_bind_address}");
-    let (listener, actual_address) = bind_system_status_listener(initial_bind_address.clone())
+    let (listener, actual_address) = bind_system_status_listener(&initial_bind_address)
         .await
         .map_err(|error| {
             tracing::error!("Failed to bind to address {initial_bind_address}: {error}");
@@ -339,9 +339,9 @@ fn build_system_status_router(
 }
 
 async fn bind_system_status_listener(
-    address: String,
+    address: &str,
 ) -> anyhow::Result<(TcpListener, std::net::SocketAddr)> {
-    let listener = TcpListener::bind(&address)
+    let listener = TcpListener::bind(address)
         .await
         .map_err(|error| anyhow::anyhow!("Failed to bind to address: {error}"))?;
 
@@ -378,17 +378,18 @@ impl Listener for RebindingTcpListener {
             let Some(listener) = self.listener.as_ref() else {
                 tokio::time::sleep_until(next_rebind).await;
                 next_rebind = tokio::time::Instant::now() + self.rebind_backoff;
-                tracing::info!("System status server rebinding to {}", self.address);
-                match bind_system_status_listener(self.address.to_string()).await {
-                    Ok((listener, actual_address)) => {
-                        tracing::info!("System status server rebound to {actual_address}");
+                tracing::debug!(address = %self.address, "System status server rebinding");
+                match TcpListener::bind(self.address).await {
+                    Ok(listener) => {
+                        tracing::info!(address = %self.address, "System status server rebound");
                         self.listener = Some(listener);
                     }
                     Err(error) => {
                         tracing::error!(
-                            "System status server failed to rebind {}; retrying after {:?}: {error}",
-                            self.address,
-                            self.rebind_backoff
+                            address = %self.address,
+                            %error,
+                            retry_after = ?self.rebind_backoff,
+                            "System status server failed to rebind"
                         );
                     }
                 }
@@ -397,23 +398,29 @@ impl Listener for RebindingTcpListener {
 
             match listener.accept().await {
                 Ok(accepted) => return accepted,
-                // A dropped connection does not invalidate the listener.
-                Err(error) if is_dead_connection_error(&error) => {
-                    tracing::trace!("system status connection dropped before accept: {error}");
+                // Connection errors do not invalidate the listener.
+                Err(error) if is_connection_error(&error) => {
+                    tracing::trace!(
+                        address = %self.address,
+                        %error,
+                        "System status connection failed before accept"
+                    );
                 }
                 // Resource exhaustion does not invalidate the listener.
                 Err(error) if is_resource_exhaustion_error(&error) => {
                     tracing::error!(
-                        "System status listener cannot accept on {} for want of a process resource; retrying the same socket after {:?}: {error}",
-                        self.address,
-                        self.rebind_backoff
+                        address = %self.address,
+                        %error,
+                        retry_after = ?self.rebind_backoff,
+                        "System status listener lacks resources; retrying the same socket"
                     );
                     tokio::time::sleep(self.rebind_backoff).await;
                 }
                 Err(error) => {
                     tracing::error!(
-                        "System status listener stopped accepting on {}; rebinding: {error}",
-                        self.address
+                        address = %self.address,
+                        %error,
+                        "System status listener stopped accepting; rebinding"
                     );
                     // Drop before rebinding or the address remains in use.
                     self.listener = None;
@@ -427,17 +434,35 @@ impl Listener for RebindingTcpListener {
     }
 }
 
-/// Matches connection failures that leave the listener usable.
-fn is_dead_connection_error(error: &io::Error) -> bool {
-    matches!(
+fn is_connection_error(error: &io::Error) -> bool {
+    if matches!(
         error.kind(),
         io::ErrorKind::ConnectionRefused
             | io::ErrorKind::ConnectionAborted
             | io::ErrorKind::ConnectionReset
-    )
+    ) {
+        return true;
+    }
+
+    // Linux accept(2) also reports pending network errors for the queued connection.
+    #[cfg(target_os = "linux")]
+    if let Some(code) = error.raw_os_error() {
+        return matches!(
+            code,
+            libc::ENETDOWN
+                | libc::EPROTO
+                | libc::ENOPROTOOPT
+                | libc::EHOSTDOWN
+                | libc::ENONET
+                | libc::EHOSTUNREACH
+                | libc::EOPNOTSUPP
+                | libc::ENETUNREACH
+        );
+    }
+
+    false
 }
 
-/// Matches shortages that require retrying the same socket.
 /// EMFILE, ENFILE, and ENOBUFS require raw errno checks because Rust has no distinct kinds.
 fn is_resource_exhaustion_error(error: &io::Error) -> bool {
     if error.kind() == io::ErrorKind::OutOfMemory {
@@ -446,16 +471,8 @@ fn is_resource_exhaustion_error(error: &io::Error) -> bool {
 
     #[cfg(unix)]
     {
-        // ENOBUFS differs between Linux and BSD-derived systems.
-        const ENFILE: i32 = 23;
-        const EMFILE: i32 = 24;
-        #[cfg(target_os = "linux")]
-        const ENOBUFS: i32 = 105;
-        #[cfg(not(target_os = "linux"))]
-        const ENOBUFS: i32 = 55;
-
         if let Some(code) = error.raw_os_error() {
-            return matches!(code, ENFILE | EMFILE | ENOBUFS);
+            return matches!(code, libc::ENFILE | libc::EMFILE | libc::ENOBUFS);
         }
     }
 
@@ -1058,7 +1075,7 @@ mod tests {
     #[cfg(unix)]
     #[test]
     fn test_resource_exhaustion_does_not_look_like_a_dead_listener() {
-        for code in [23 /* ENFILE */, 24 /* EMFILE */] {
+        for code in [libc::ENFILE, libc::EMFILE, libc::ENOBUFS] {
             let error = io::Error::from_raw_os_error(code);
             assert!(
                 is_resource_exhaustion_error(&error),
@@ -1066,12 +1083,61 @@ mod tests {
                 error.kind()
             );
             assert!(
-                !is_dead_connection_error(&error),
+                !is_connection_error(&error),
                 "errno {code} is not a dead connection"
             );
         }
     }
 
+    #[cfg(target_os = "linux")]
+    #[test]
+    fn test_pending_network_errors_leave_the_listener_usable() {
+        for code in [
+            libc::ENETDOWN,
+            libc::EPROTO,
+            libc::ENOPROTOOPT,
+            libc::EHOSTDOWN,
+            libc::ENONET,
+            libc::EHOSTUNREACH,
+            libc::EOPNOTSUPP,
+            libc::ENETUNREACH,
+        ] {
+            let error = io::Error::from_raw_os_error(code);
+            assert!(
+                is_connection_error(&error),
+                "errno {code} should retry accept"
+            );
+        }
+    }
+
+    #[tokio::test]
+    async fn test_rebinding_listener_shutdown_interrupts_failed_rebinds() {
+        let holder = TcpListener::bind("127.0.0.1:0").await.unwrap();
+        let listener = RebindingTcpListener {
+            address: holder.local_addr().unwrap(),
+            listener: None,
+            rebind_backoff: Duration::from_secs(60),
+        };
+        let cancel_token = CancellationToken::new();
+        let observer = cancel_token.clone();
+        let server = tokio::spawn(async move {
+            axum::serve(listener, Router::new())
+                .with_graceful_shutdown(observer.cancelled_owned())
+                .await
+                .unwrap();
+        });
+        tokio::time::sleep(Duration::from_millis(50)).await;
+        assert!(
+            !server.is_finished(),
+            "server should keep retrying the held address"
+        );
+        cancel_token.cancel();
+        tokio::time::timeout(Duration::from_secs(1), server)
+            .await
+            .expect("shutdown should interrupt the 60-second rebind backoff")
+            .expect("server task should not panic");
+    }
+
     #[cfg(target_os = "linux")]
     #[tokio::test]
     async fn test_rebinding_listener_preserves_backlog_during_resource_exhaustion() {
@@ -1134,10 +1200,13 @@ mod tests {
         let error = listener.accept().await.unwrap_err();
         assert_eq!(error.raw_os_error(), Some(libc::EMFILE));
 
-        let mut rebinding =
-            RebindingTcpListener::new(listener, address, Duration::from_millis(200));
+        let backoff = Duration::from_millis(200);
+        let mut rebinding = RebindingTcpListener::new(listener, address, backoff);
+        let started = tokio::time::Instant::now();
+        let accepted = rebinding.accept();
+        tokio::pin!(accepted);
         assert!(
-            tokio::time::timeout(Duration::from_millis(50), rebinding.accept())
+            tokio::time::timeout(Duration::from_millis(50), accepted.as_mut())
                 .await
                 .is_err(),
             "accept should wait during descriptor exhaustion"
@@ -1151,10 +1220,13 @@ mod tests {
             io::ErrorKind::AddrInUse,
             "the original listening address should remain held"
         );
-        let (_stream, accepted_peer) =
-            tokio::time::timeout(Duration::from_secs(2), rebinding.accept())
-                .await
-                .expect("the queued connection should survive the shortage");
+        let (_stream, accepted_peer) = tokio::time::timeout(Duration::from_secs(2), accepted)
+            .await
+            .expect("the queued connection should survive the shortage");
+        assert!(
+            started.elapsed() >= backoff,
+            "accept should finish its resource-shortage backoff before retrying"
+        );
         assert_eq!(accepted_peer, peer);
     }
 }

From c56d94d023188c02dbb73ca1fa90c118d2bcb9dd Mon Sep 17 00:00:00 2001
From: GLAMR <svc-glamr@nvidia.com>
Date: Mon, 5 Oct 2026 20:29:38 +0000
Subject: [PATCH 22/22] test(runtime): remove redundant EMFILE classifier case

Signed-off-by: GLAMR <svc-glamr@nvidia.com>
---
 lib/runtime/src/system_status_server.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/runtime/src/system_status_server.rs b/lib/runtime/src/system_status_server.rs
index 67b6e99034da..3029ad1f121a 100644
--- a/lib/runtime/src/system_status_server.rs
+++ b/lib/runtime/src/system_status_server.rs
@@ -1075,7 +1075,7 @@ mod tests {
     #[cfg(unix)]
     #[test]
     fn test_resource_exhaustion_does_not_look_like_a_dead_listener() {
-        for code in [libc::ENFILE, libc::EMFILE, libc::ENOBUFS] {
+        for code in [libc::ENFILE, libc::ENOBUFS] {
             let error = io::Error::from_raw_os_error(code);
             assert!(
                 is_resource_exhaustion_error(&error),