From 4808a8fd0eb4b612e828deefefc67f94c36344b9 Mon Sep 17 00:00:00 2001 From: Vaibhav Gupta Date: Fri, 11 Oct 2024 11:05:22 -0700 Subject: [PATCH] Add limit on connection pool to prevent stalling issues in pyo3 and other ffi boundaries --- engine/baml-runtime/src/request/mod.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/engine/baml-runtime/src/request/mod.rs b/engine/baml-runtime/src/request/mod.rs index d7fbd2f2e..5220af68b 100644 --- a/engine/baml-runtime/src/request/mod.rs +++ b/engine/baml-runtime/src/request/mod.rs @@ -12,7 +12,7 @@ fn builder() -> reqwest::ClientBuilder { // regularly have requests that take multiple minutes, due to how // long LLMs take .connect_timeout(Duration::from_secs(10)) - .danger_accept_invalid_certs(danger_accept_invalid_certs) + .danger_accept_invalid_certs(danger_accept_invalid_certs) .http2_keep_alive_interval(Some(Duration::from_secs(10))) } } @@ -27,9 +27,13 @@ pub(crate) fn create_tracing_client() -> Result { if #[cfg(target_arch = "wasm32")] { let cb = builder(); } else { - let cb =builder() + let cb = builder() // Wait up to 30s to send traces to the backend - .read_timeout(Duration::from_secs(30)); + .read_timeout(Duration::from_secs(30)) + // We don't want to keep idle connections around due to sometimes + // causing a stall in the connection pool across FFI boundaries + // https://github.com/seanmonstar/reqwest/issues/600 + .pool_max_idle_per_host(0); } }