From e16cc7012fcb678362ef33560c38cb7d79d3018f Mon Sep 17 00:00:00 2001
From: Daniel Savu <23065004+daniel-savu@users.noreply.github.com>
Date: Mon, 19 Feb 2024 16:37:09 +0000
Subject: [PATCH] fix: lower rusoto timeout to 15s (#3283)

### Description

Applies the fix in
https://github.com/hyperlane-xyz/hyperlane-monorepo/pull/2384 everywhere
an `HttpClient` is constructed via rusoto.

It lowers the S3 timeout to 15s based on tips in [this
thread](https://github.com/hyperium/hyper/issues/2136#issuecomment-589488526),
to avoid `Error during dispatch: connection closed before message
completed` errors. Note that we'll probably still run into these issues,
but less frequently
([source](https://github.com/rusoto/rusoto/issues/1766#issuecomment-639299961)).


### Drive-by changes

<!--
Are there any minor or drive-by changes also included?
-->

### Related issues

<!--
- Fixes #[issue number here]
-->

### Backward compatibility

<!--
Are these changes backward compatible? Are there any infrastructure
implications, e.g. changes that would prohibit deploying older commits
using this infra tooling?

Yes/No
-->

### Testing

<!--
What kind of testing have these changes undergone?

None/Manual/Unit Tests
-->
---
 rust/hyperlane-base/src/settings/signers.rs | 10 +++-------
 rust/hyperlane-base/src/types/mod.rs        |  3 +++
 rust/hyperlane-base/src/types/s3_storage.rs |  7 ++++---
 rust/hyperlane-base/src/types/utils.rs      | 15 +++++++++++++++
 4 files changed, 25 insertions(+), 10 deletions(-)
 create mode 100644 rust/hyperlane-base/src/types/utils.rs

diff --git a/rust/hyperlane-base/src/settings/signers.rs b/rust/hyperlane-base/src/settings/signers.rs
index c48de65f731..d0642434f5b 100644
--- a/rust/hyperlane-base/src/settings/signers.rs
+++ b/rust/hyperlane-base/src/settings/signers.rs
@@ -1,5 +1,3 @@
-use std::time::Duration;
-
 use async_trait::async_trait;
 use ed25519_dalek::SecretKey;
 use ethers::prelude::{AwsSigner, LocalWallet};
@@ -7,11 +5,12 @@ use ethers::utils::hex::ToHex;
 use eyre::{bail, Context, Report};
 use hyperlane_core::H256;
 use hyperlane_sealevel::Keypair;
-use rusoto_core::{HttpClient, HttpConfig, Region};
+use rusoto_core::Region;
 use rusoto_kms::KmsClient;
 use tracing::instrument;
 
 use super::aws_credentials::AwsChainCredentialsProvider;
+use crate::types::utils;
 
 /// Signer types
 #[derive(Default, Debug, Clone)]
@@ -73,13 +72,10 @@ impl BuildableWithSignerConf for hyperlane_ethereum::Signers {
                 ),
             )),
             SignerConf::Aws { id, region } => {
-                let mut config = HttpConfig::new();
-                // see https://github.com/hyperium/hyper/issues/2136#issuecomment-589345238
-                config.pool_idle_timeout(Duration::from_secs(20));
                 let client = KmsClient::new_with_client(
                     rusoto_core::Client::new_with(
                         AwsChainCredentialsProvider::new(),
-                        HttpClient::new_with_config(config).unwrap(),
+                        utils::http_client_with_timeout().unwrap(),
                     ),
                     region.clone(),
                 );
diff --git a/rust/hyperlane-base/src/types/mod.rs b/rust/hyperlane-base/src/types/mod.rs
index bd07fe23c91..1f6b3b792f2 100644
--- a/rust/hyperlane-base/src/types/mod.rs
+++ b/rust/hyperlane-base/src/types/mod.rs
@@ -3,6 +3,9 @@ mod local_storage;
 mod multisig;
 mod s3_storage;
 
+/// Reusable logic for working with storage backends.
+pub mod utils;
+
 pub use gcs_storage::*;
 pub use local_storage::*;
 pub use multisig::*;
diff --git a/rust/hyperlane-base/src/types/s3_storage.rs b/rust/hyperlane-base/src/types/s3_storage.rs
index 2d09a18a864..4db179ad4da 100644
--- a/rust/hyperlane-base/src/types/s3_storage.rs
+++ b/rust/hyperlane-base/src/types/s3_storage.rs
@@ -8,11 +8,12 @@ use hyperlane_core::{SignedAnnouncement, SignedCheckpointWithMessageId};
 use prometheus::IntGauge;
 use rusoto_core::{
     credential::{Anonymous, AwsCredentials, StaticProvider},
-    HttpClient, Region, RusotoError,
+    Region, RusotoError,
 };
 use rusoto_s3::{GetObjectError, GetObjectRequest, PutObjectRequest, S3Client, S3};
 use tokio::time::timeout;
 
+use crate::types::utils;
 use crate::{settings::aws_credentials::AwsChainCredentialsProvider, CheckpointSyncer};
 
 /// The timeout for S3 requests. Rusoto doesn't offer timeout configuration
@@ -93,7 +94,7 @@ impl S3Storage {
     fn authenticated_client(&self) -> &S3Client {
         self.authenticated_client.get_or_init(|| {
             S3Client::new_with(
-                HttpClient::new().unwrap(),
+                utils::http_client_with_timeout().unwrap(),
                 AwsChainCredentialsProvider::new(),
                 self.region.clone(),
             )
@@ -113,7 +114,7 @@ impl S3Storage {
             assert!(credentials.is_anonymous(), "AWS credentials not anonymous");
 
             S3Client::new_with(
-                HttpClient::new().unwrap(),
+                utils::http_client_with_timeout().unwrap(),
                 StaticProvider::from(credentials),
                 self.region.clone(),
             )
diff --git a/rust/hyperlane-base/src/types/utils.rs b/rust/hyperlane-base/src/types/utils.rs
new file mode 100644
index 00000000000..a84c55d5ad1
--- /dev/null
+++ b/rust/hyperlane-base/src/types/utils.rs
@@ -0,0 +1,15 @@
+use std::time::Duration;
+
+use eyre::Result;
+use rusoto_core::{HttpClient, HttpConfig};
+
+/// See https://github.com/hyperium/hyper/issues/2136#issuecomment-589488526
+pub const HYPER_POOL_IDLE_TIMEOUT: Duration = Duration::from_secs(15);
+
+/// Create a new HTTP client with a timeout for the connection pool.
+/// This is a workaround for https://github.com/hyperium/hyper/issues/2136#issuecomment-589345238
+pub fn http_client_with_timeout() -> Result<HttpClient> {
+    let mut config = HttpConfig::new();
+    config.pool_idle_timeout(HYPER_POOL_IDLE_TIMEOUT);
+    Ok(HttpClient::new_with_config(config)?)
+}