From eb90e62ffe21109e0da1bd74439d36bb37246ec3 Mon Sep 17 00:00:00 2001
From: hellovai <vbv@boundaryml.com>
Date: Fri, 11 Oct 2024 11:07:50 -0700
Subject: [PATCH] Add limit on connection pool to prevent stalling issues in
 pyo3 and other ffi boundaries (#1027)

<!-- ELLIPSIS_HIDDEN -->



> [!IMPORTANT]
> Set `pool_max_idle_per_host(0)` in `create_tracing_client()` to
prevent connection pool stalling issues across FFI boundaries.
>
>   - **Behavior**:
> - Set `pool_max_idle_per_host(0)` in `create_tracing_client()` in
`mod.rs` to prevent stalling issues across FFI boundaries.
> - Addresses issue https://github.com/seanmonstar/reqwest/issues/600.
>   - **Misc**:
>     - Minor formatting change in `mod.rs`.
>
> <sup>This description was created by </sup>[<img alt="Ellipsis"
src="https://img.shields.io/badge/Ellipsis-blue?color=175173">](https://www.ellipsis.dev?ref=BoundaryML%2Fbaml&utm_source=github&utm_medium=referral)<sup>
for 4808a8fd0eb4b612e828deefefc67f94c36344b9. It will automatically
update as commits are pushed.</sup>

<!-- ELLIPSIS_HIDDEN -->
---
 engine/baml-runtime/src/request/mod.rs | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/engine/baml-runtime/src/request/mod.rs b/engine/baml-runtime/src/request/mod.rs
index d7fbd2f2e..5220af68b 100644
--- a/engine/baml-runtime/src/request/mod.rs
+++ b/engine/baml-runtime/src/request/mod.rs
@@ -12,7 +12,7 @@ fn builder() -> reqwest::ClientBuilder {
                 // regularly have requests that take multiple minutes, due to how
                 // long LLMs take
                 .connect_timeout(Duration::from_secs(10))
-                .danger_accept_invalid_certs(danger_accept_invalid_certs) 
+                .danger_accept_invalid_certs(danger_accept_invalid_certs)
                 .http2_keep_alive_interval(Some(Duration::from_secs(10)))
         }
     }
@@ -27,9 +27,13 @@ pub(crate) fn create_tracing_client() -> Result<reqwest::Client> {
         if #[cfg(target_arch = "wasm32")] {
             let cb = builder();
         } else {
-            let cb =builder()
+            let cb = builder()
                 // Wait up to 30s to send traces to the backend
-                .read_timeout(Duration::from_secs(30));
+                .read_timeout(Duration::from_secs(30))
+                // We don't want to keep idle connections around due to sometimes
+                // causing a stall in the connection pool across FFI boundaries
+                // https://github.com/seanmonstar/reqwest/issues/600
+                .pool_max_idle_per_host(0);
         }
     }