fix: lower rusoto timeout to 15s (hyperlane-xyz#3283)

### Description

Applies the fix in
hyperlane-xyz#2384 everywhere
an `HttpClient` is constructed via rusoto.

It lowers the S3 timeout to 15s based on tips in [this
thread](hyperium/hyper#2136 (comment)),
to avoid `Error during dispatch: connection closed before message
completed` errors. Note that we'll probably still run into these issues,
but less frequently
([source](rusoto/rusoto#1766 (comment))).


### Drive-by changes

<!--
Are there any minor or drive-by changes also included?
-->

### Related issues

<!--
- Fixes #[issue number here]
-->

### Backward compatibility

<!--
Are these changes backward compatible? Are there any infrastructure
implications, e.g. changes that would prohibit deploying older commits
using this infra tooling?

Yes/No
-->

### Testing

<!--
What kind of testing have these changes undergone?

None/Manual/Unit Tests
-->

rust/hyperlane-base/src/settings/signers.rs

@@ -1,17 +1,16 @@
-use std::time::Duration;
-
 use async_trait::async_trait;
 use ed25519_dalek::SecretKey;
 use ethers::prelude::{AwsSigner, LocalWallet};
 use ethers::utils::hex::ToHex;
 use eyre::{bail, Context, Report};
 use hyperlane_core::H256;
 use hyperlane_sealevel::Keypair;
-use rusoto_core::{HttpClient, HttpConfig, Region};
+use rusoto_core::Region;
 use rusoto_kms::KmsClient;
 use tracing::instrument;
 
 use super::aws_credentials::AwsChainCredentialsProvider;
+use crate::types::utils;
 
 /// Signer types
 #[derive(Default, Debug, Clone)]
@@ -73,13 +72,10 @@ impl BuildableWithSignerConf for hyperlane_ethereum::Signers {
                 ),
             )),
             SignerConf::Aws { id, region } => {
-                let mut config = HttpConfig::new();
-                // see https://github.com/hyperium/hyper/issues/2136#issuecomment-589345238
-                config.pool_idle_timeout(Duration::from_secs(20));
                 let client = KmsClient::new_with_client(
                     rusoto_core::Client::new_with(
                         AwsChainCredentialsProvider::new(),
-                        HttpClient::new_with_config(config).unwrap(),
+                        utils::http_client_with_timeout().unwrap(),
                     ),
                     region.clone(),
                 );

rust/hyperlane-base/src/types/mod.rs

@@ -3,6 +3,9 @@ mod local_storage;
 mod multisig;
 mod s3_storage;
 
+/// Reusable logic for working with storage backends.
+pub mod utils;
+
 pub use gcs_storage::*;
 pub use local_storage::*;
 pub use multisig::*;

rust/hyperlane-base/src/types/s3_storage.rs

@@ -8,11 +8,12 @@ use hyperlane_core::{SignedAnnouncement, SignedCheckpointWithMessageId};
 use prometheus::IntGauge;
 use rusoto_core::{
     credential::{Anonymous, AwsCredentials, StaticProvider},
-    HttpClient, Region, RusotoError,
+    Region, RusotoError,
 };
 use rusoto_s3::{GetObjectError, GetObjectRequest, PutObjectRequest, S3Client, S3};
 use tokio::time::timeout;
 
+use crate::types::utils;
 use crate::{settings::aws_credentials::AwsChainCredentialsProvider, CheckpointSyncer};
 
 /// The timeout for S3 requests. Rusoto doesn't offer timeout configuration
@@ -93,7 +94,7 @@ impl S3Storage {
     fn authenticated_client(&self) -> &S3Client {
         self.authenticated_client.get_or_init(|| {
             S3Client::new_with(
-                HttpClient::new().unwrap(),
+                utils::http_client_with_timeout().unwrap(),
                 AwsChainCredentialsProvider::new(),
                 self.region.clone(),
             )
@@ -113,7 +114,7 @@ impl S3Storage {
             assert!(credentials.is_anonymous(), "AWS credentials not anonymous");
 
             S3Client::new_with(
-                HttpClient::new().unwrap(),
+                utils::http_client_with_timeout().unwrap(),
                 StaticProvider::from(credentials),
                 self.region.clone(),
             )

rust/hyperlane-base/src/types/utils.rs

@@ -0,0 +1,15 @@
+use std::time::Duration;
+
+use eyre::Result;
+use rusoto_core::{HttpClient, HttpConfig};
+
+/// See https://github.com/hyperium/hyper/issues/2136#issuecomment-589488526
+pub const HYPER_POOL_IDLE_TIMEOUT: Duration = Duration::from_secs(15);
+
+/// Create a new HTTP client with a timeout for the connection pool.
+/// This is a workaround for https://github.com/hyperium/hyper/issues/2136#issuecomment-589345238
+pub fn http_client_with_timeout() -> Result<HttpClient> {
+    let mut config = HttpConfig::new();
+    config.pool_idle_timeout(HYPER_POOL_IDLE_TIMEOUT);
+    Ok(HttpClient::new_with_config(config)?)
+}