From 08bbe559a0f91a1dd7a28d25a21156559f07fb5f Mon Sep 17 00:00:00 2001 From: Pei Li Date: Fri, 18 Sep 2026 16:54:11 -0700 Subject: [PATCH 1/3] fix(frontend): make the HTTP listen backlog configurable, default 4096 tokio's TcpListener::bind listens with a backlog of 128. A few thousand clients connecting within seconds overflow it, and on hosts with tcp_syncookies=1 the overflowed handshakes are reset instead of retried. Bind through TcpSocket and listen with DYN_HTTP_LISTEN_BACKLOG (default 4096, capped by net.core.somaxconn). Signed-off-by: Pei Li --- .../components/frontend-configuration.mdx | 9 +++ lib/llm/src/http/service/service_v2.rs | 57 ++++++++++++++++++- lib/runtime/src/config/environment_names.rs | 8 +++ 3 files changed, 73 insertions(+), 1 deletion(-) diff --git a/docs/fern/pages/reference/components/frontend-configuration.mdx b/docs/fern/pages/reference/components/frontend-configuration.mdx index 76179995cdbc..85c80c50c284 100644 --- a/docs/fern/pages/reference/components/frontend-configuration.mdx +++ b/docs/fern/pages/reference/components/frontend-configuration.mdx @@ -69,6 +69,15 @@ The Rust HTTP server also reads these environment variables, which are not expos See [Runtime Configuration](runtime-configuration.mdx) for `DYN_TCP_MAX_MESSAGE_SIZE`. + + Listen backlog of the frontend HTTP socket, capped by the kernel at + `net.core.somaxconn`. Raise it when many clients open connections within a few + seconds (for example a benchmark client with one connection per session): a + connection that arrives while the accept queue is full is dropped, and on hosts + with `net.ipv4.tcp_syncookies=1` the client sees `Connection reset by peer` + instead of a retry. + + Maximum time the Frontend waits for admitted HTTP response bodies and `/v1/realtime` WebSocket tasks to finish after shutdown begins. New inference requests are rejected while draining. diff --git a/lib/llm/src/http/service/service_v2.rs b/lib/llm/src/http/service/service_v2.rs index bcdbc5d06a5d..8ec51c551447 100644 --- a/lib/llm/src/http/service/service_v2.rs +++ b/lib/llm/src/http/service/service_v2.rs @@ -1002,7 +1002,7 @@ impl HttpService { let addr: SocketAddr = address .parse() .map_err(|e| anyhow::anyhow!("Invalid address '{}': {}", address, e))?; - tokio::net::TcpListener::bind(addr).await.map_err(|e| { + bind_listener(addr).map_err(|e| { tracing::error!( protocol = %protocol, address = %address, @@ -1132,6 +1132,34 @@ fn get_graceful_shutdown_timeout() -> usize { .unwrap_or(5) } +const DEFAULT_LISTEN_BACKLOG: u32 = 4096; + +fn parse_listen_backlog(value: Result) -> u32 { + value + .ok() + .and_then(|s| s.trim().parse::().ok()) + .filter(|n| *n > 0) + .unwrap_or(DEFAULT_LISTEN_BACKLOG) +} + +fn listen_backlog() -> u32 { + parse_listen_backlog(std::env::var(env_llm::DYN_HTTP_LISTEN_BACKLOG)) +} + +/// `tokio::net::TcpListener::bind` listens with a backlog of 128. A few thousand +/// clients connecting within seconds overflow that, and with `tcp_syncookies=1` +/// the overflowed handshakes are reset instead of retried. +fn bind_listener(addr: SocketAddr) -> std::io::Result { + let socket = if addr.is_ipv4() { + tokio::net::TcpSocket::new_v4()? + } else { + tokio::net::TcpSocket::new_v6()? + }; + socket.set_reuseaddr(true)?; + socket.bind(addr)?; + socket.listen(listen_backlog()) +} + /// Environment variable to set the metrics endpoint path (default: `/metrics`) static HTTP_SVC_METRICS_PATH_ENV: &str = "DYN_HTTP_SVC_METRICS_PATH"; /// Environment variable to set the models endpoint path (default: `/v1/models`) @@ -2702,6 +2730,33 @@ mod tests { assert_eq!(parse_sse_keep_alive(Ok(u64::MAX.to_string())), expected); } + #[test] + fn test_listen_backlog_env_var() { + assert_eq!( + parse_listen_backlog(Err(std::env::VarError::NotPresent)), + DEFAULT_LISTEN_BACKLOG + ); + assert_eq!( + parse_listen_backlog(Ok("0".to_string())), + DEFAULT_LISTEN_BACKLOG + ); + assert_eq!( + parse_listen_backlog(Ok("invalid".to_string())), + DEFAULT_LISTEN_BACKLOG + ); + assert_eq!(parse_listen_backlog(Ok(" 8192 ".to_string())), 8192); + } + + #[tokio::test] + async fn test_bind_listener_accepts_connections() { + let listener = bind_listener("127.0.0.1:0".parse().unwrap()).unwrap(); + let addr = listener.local_addr().unwrap(); + assert_ne!(addr.port(), 0); + let client = tokio::net::TcpStream::connect(addr).await.unwrap(); + let (_server_side, peer) = listener.accept().await.unwrap(); + assert_eq!(peer, client.local_addr().unwrap()); + } + #[test] fn test_sse_keep_alive_for_deferred_response() { let configured = Duration::from_millis(5000); diff --git a/lib/runtime/src/config/environment_names.rs b/lib/runtime/src/config/environment_names.rs index c6fd04d8a8aa..ab6533e4061b 100644 --- a/lib/runtime/src/config/environment_names.rs +++ b/lib/runtime/src/config/environment_names.rs @@ -356,6 +356,13 @@ pub mod llm { /// HTTP body size limit in MB pub const DYN_HTTP_BODY_LIMIT_MB: &str = "DYN_HTTP_BODY_LIMIT_MB"; + /// Listen backlog for the frontend HTTP socket. Defaults to 4096; the + /// kernel caps it at `net.core.somaxconn`. tokio's default is 128, which a + /// few thousand clients connecting within seconds can overflow, and with + /// `tcp_syncookies=1` the overflowed connections are reset rather than + /// retried. + pub const DYN_HTTP_LISTEN_BACKLOG: &str = "DYN_HTTP_LISTEN_BACKLOG"; + pub const DYN_HTTP_GRACEFUL_SHUTDOWN_TIMEOUT_SECS: &str = "DYN_HTTP_GRACEFUL_SHUTDOWN_TIMEOUT_SECS"; @@ -1070,6 +1077,7 @@ mod tests { kvbm::leader::DYN_KVBM_LEADER_ZMQ_ACK_PORT, // LLM llm::DYN_HTTP_BODY_LIMIT_MB, + llm::DYN_HTTP_LISTEN_BACKLOG, llm::DYN_HTTP_GRACEFUL_SHUTDOWN_TIMEOUT_SECS, llm::DYN_HTTP_OVERLOAD_STATUS_CODE, llm::DYN_HTTP_BACKEND_STREAM_TIMEOUT_SECS, From 3992593a9c9305f34918e62a2dc1700a8fb8dbc2 Mon Sep 17 00:00:00 2001 From: Pei Li Date: Fri, 18 Sep 2026 18:16:52 -0700 Subject: [PATCH 2/3] fix(frontend): apply the listen backlog to the TLS listener too; drop the SYN-cookie wording Bind the TLS socket through the same helper and serve it with axum_server::from_tcp_rustls, so DYN_HTTP_LISTEN_BACKLOG covers HTTP and HTTPS. Describe accept-queue overflow as delaying or failing connections rather than asserting a reset. Signed-off-by: Pei Li --- .../components/frontend-configuration.mdx | 11 ++++----- lib/llm/src/http/service/service_v2.rs | 24 ++++++++++++------- lib/runtime/src/config/environment_names.rs | 9 ++++--- 3 files changed, 25 insertions(+), 19 deletions(-) diff --git a/docs/fern/pages/reference/components/frontend-configuration.mdx b/docs/fern/pages/reference/components/frontend-configuration.mdx index 85c80c50c284..29175d0993ef 100644 --- a/docs/fern/pages/reference/components/frontend-configuration.mdx +++ b/docs/fern/pages/reference/components/frontend-configuration.mdx @@ -70,12 +70,11 @@ The Rust HTTP server also reads these environment variables, which are not expos - Listen backlog of the frontend HTTP socket, capped by the kernel at - `net.core.somaxconn`. Raise it when many clients open connections within a few - seconds (for example a benchmark client with one connection per session): a - connection that arrives while the accept queue is full is dropped, and on hosts - with `net.ipv4.tcp_syncookies=1` the client sees `Connection reset by peer` - instead of a retry. + Listen backlog of the frontend socket, for both plain HTTP and TLS, capped by + the kernel at `net.core.somaxconn`. Raise it when many clients open connections + within a few seconds (for example a load client with one connection per + session): connections that arrive while the accept queue is full are delayed + or, depending on the host's TCP settings, fail. diff --git a/lib/llm/src/http/service/service_v2.rs b/lib/llm/src/http/service/service_v2.rs index 8ec51c551447..78f67ae0b5b9 100644 --- a/lib/llm/src/http/service/service_v2.rs +++ b/lib/llm/src/http/service/service_v2.rs @@ -872,9 +872,7 @@ impl HttpService { /// Like [`spawn`], but uses a caller-provided pre-bound listener. Closes the TOCTOU /// port-allocation gap for tests that need to know the bound port up front. Not - /// supported in TLS mode: TLS uses `axum_server::bind_rustls`, which owns its own - /// bind, so a pre-bound listener cannot be threaded through and dropping it before - /// `bind_rustls` would just re-open the same race. Returns an error if invoked on a + /// supported in TLS mode, which binds internally. Returns an error if invoked on a /// service built with `enable_tls(true)`. /// /// [`spawn`]: HttpService::spawn @@ -924,8 +922,7 @@ impl HttpService { if self.enable_tls { if listener.is_some() { return Err(anyhow::anyhow!( - "Pre-bound listener is not supported in TLS mode; \ - axum_server::bind_rustls owns its own bind. \ + "Pre-bound listener is not supported in TLS mode. \ Use run()/spawn() (which bind internally) when enable_tls is set." )); } @@ -950,7 +947,18 @@ impl HttpService { let config = RustlsConfig::from_config(Arc::new(server_config)); let handle = tls_handle.unwrap_or_default(); - let server = axum_server::bind_rustls(addr, config) + let std_listener = bind_listener(addr) + .and_then(|l| l.into_std()) + .map_err(|e| { + tracing::error!( + protocol = %protocol, + address = %address, + error = %e, + "Failed to bind server to address" + ); + anyhow::anyhow!("Failed to start {} server on {}: {}", protocol, address, e) + })?; + let server = axum_server::from_tcp_rustls(std_listener, config) .handle(handle.clone()) .serve(router.into_make_service()); @@ -1147,8 +1155,8 @@ fn listen_backlog() -> u32 { } /// `tokio::net::TcpListener::bind` listens with a backlog of 128. A few thousand -/// clients connecting within seconds overflow that, and with `tcp_syncookies=1` -/// the overflowed handshakes are reset instead of retried. +/// clients connecting within seconds overflow that, and overflowed connections +/// are delayed or, depending on host TCP settings, fail. fn bind_listener(addr: SocketAddr) -> std::io::Result { let socket = if addr.is_ipv4() { tokio::net::TcpSocket::new_v4()? diff --git a/lib/runtime/src/config/environment_names.rs b/lib/runtime/src/config/environment_names.rs index ab6533e4061b..54b98412894f 100644 --- a/lib/runtime/src/config/environment_names.rs +++ b/lib/runtime/src/config/environment_names.rs @@ -356,11 +356,10 @@ pub mod llm { /// HTTP body size limit in MB pub const DYN_HTTP_BODY_LIMIT_MB: &str = "DYN_HTTP_BODY_LIMIT_MB"; - /// Listen backlog for the frontend HTTP socket. Defaults to 4096; the - /// kernel caps it at `net.core.somaxconn`. tokio's default is 128, which a - /// few thousand clients connecting within seconds can overflow, and with - /// `tcp_syncookies=1` the overflowed connections are reset rather than - /// retried. + /// Listen backlog for the frontend HTTP and HTTPS socket. Defaults to 4096; + /// the kernel caps it at `net.core.somaxconn`. tokio's default is 128, which + /// a few thousand clients connecting within seconds can overflow; overflowed + /// connections are delayed or, depending on host TCP settings, fail. pub const DYN_HTTP_LISTEN_BACKLOG: &str = "DYN_HTTP_LISTEN_BACKLOG"; pub const DYN_HTTP_GRACEFUL_SHUTDOWN_TIMEOUT_SECS: &str = From bb492e8f9934dffe197530ba9d5c10590472aec2 Mon Sep 17 00:00:00 2001 From: Pei Li Date: Fri, 18 Sep 2026 18:40:09 -0700 Subject: [PATCH 3/3] fix(frontend): reject listen backlog values above i32::MAX; trim env docstring listen(2) takes an int, so a u32 above i32::MAX would become a negative backlog. Fall back to the default for those values. Signed-off-by: Pei Li --- lib/llm/src/http/service/service_v2.rs | 11 ++++++++++- lib/runtime/src/config/environment_names.rs | 5 +---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/lib/llm/src/http/service/service_v2.rs b/lib/llm/src/http/service/service_v2.rs index 78f67ae0b5b9..28f91b230037 100644 --- a/lib/llm/src/http/service/service_v2.rs +++ b/lib/llm/src/http/service/service_v2.rs @@ -1142,11 +1142,12 @@ fn get_graceful_shutdown_timeout() -> usize { const DEFAULT_LISTEN_BACKLOG: u32 = 4096; +// `listen(2)` takes an `int`; anything above `i32::MAX` would go negative. fn parse_listen_backlog(value: Result) -> u32 { value .ok() .and_then(|s| s.trim().parse::().ok()) - .filter(|n| *n > 0) + .filter(|n| (1..=i32::MAX as u32).contains(n)) .unwrap_or(DEFAULT_LISTEN_BACKLOG) } @@ -2752,6 +2753,14 @@ mod tests { parse_listen_backlog(Ok("invalid".to_string())), DEFAULT_LISTEN_BACKLOG ); + assert_eq!( + parse_listen_backlog(Ok((i32::MAX as u32 + 1).to_string())), + DEFAULT_LISTEN_BACKLOG + ); + assert_eq!( + parse_listen_backlog(Ok(i32::MAX.to_string())), + i32::MAX as u32 + ); assert_eq!(parse_listen_backlog(Ok(" 8192 ".to_string())), 8192); } diff --git a/lib/runtime/src/config/environment_names.rs b/lib/runtime/src/config/environment_names.rs index 54b98412894f..27bb3adc4d8e 100644 --- a/lib/runtime/src/config/environment_names.rs +++ b/lib/runtime/src/config/environment_names.rs @@ -356,10 +356,7 @@ pub mod llm { /// HTTP body size limit in MB pub const DYN_HTTP_BODY_LIMIT_MB: &str = "DYN_HTTP_BODY_LIMIT_MB"; - /// Listen backlog for the frontend HTTP and HTTPS socket. Defaults to 4096; - /// the kernel caps it at `net.core.somaxconn`. tokio's default is 128, which - /// a few thousand clients connecting within seconds can overflow; overflowed - /// connections are delayed or, depending on host TCP settings, fail. + /// Listen backlog of the frontend HTTP/HTTPS socket (default 4096). pub const DYN_HTTP_LISTEN_BACKLOG: &str = "DYN_HTTP_LISTEN_BACKLOG"; pub const DYN_HTTP_GRACEFUL_SHUTDOWN_TIMEOUT_SECS: &str =