Adding configurable connection count setting for S3 Sync Client

CHANGELOG.md

@@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - GHA to verify checklist items completion in PR descriptions ([#10800](https://github.com/opensearch-project/OpenSearch/pull/10800))
 - Allow to pass the list settings through environment variables (like [], ["a", "b", "c"], ...) ([#10625](https://github.com/opensearch-project/OpenSearch/pull/10625))
 - [Admission Control] Integrate CPU AC with ResourceUsageCollector and add CPU AC stats to nodes/stats ([#10887](https://github.com/opensearch-project/OpenSearch/pull/10887))
+- [S3 Repository] Add setting to control connection count for sync client ([#12028](https://github.com/opensearch-project/OpenSearch/pull/12028))
 
 ### Dependencies
 - Bump `log4j-core` from 2.18.0 to 2.19.0

plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3ClientSettings.java

@@ -201,6 +201,12 @@ final class S3ClientSettings {
         key -> Setting.intSetting(key, 500, Property.NodeScope)
     );
 
+    static final Setting.AffixSetting<Integer> MAX_SYNC_CONNECTIONS_SETTING = Setting.affixKeySetting(
+        PREFIX,
+        "max_sync_connections",
+        key -> Setting.intSetting(key, 500, Property.NodeScope)
+    );
+
     /** Connection acquisition timeout for new connections to S3. */
     static final Setting.AffixSetting<TimeValue> CONNECTION_ACQUISITION_TIMEOUT = Setting.affixKeySetting(
         PREFIX,
@@ -284,10 +290,13 @@ final class S3ClientSettings {
     /** The connection TTL for the s3 client */
     final int connectionTTLMillis;
 
-    /** The max number of connections for the s3 client */
+    /** The max number of connections for the s3 async client */
     final int maxConnections;
 
-    /** The connnection acquisition timeout for the s3 async client */
+    /** The max number of connections for the s3 sync client */
+    final int maxSyncConnections;
+
+    /** The connnection acquisition timeout for the s3 sync and async client */
     final int connectionAcquisitionTimeoutMillis;
 
     /** The number of retries to use for the s3 client. */
@@ -318,6 +327,7 @@ private S3ClientSettings(
         int connectionTimeoutMillis,
         int connectionTTLMillis,
         int maxConnections,
+        int maxSyncConnections,
         int connectionAcquisitionTimeoutMillis,
         int maxRetries,
         boolean throttleRetries,
@@ -336,6 +346,7 @@ private S3ClientSettings(
         this.connectionTimeoutMillis = connectionTimeoutMillis;
         this.connectionTTLMillis = connectionTTLMillis;
         this.maxConnections = maxConnections;
+        this.maxSyncConnections = maxSyncConnections;
         this.connectionAcquisitionTimeoutMillis = connectionAcquisitionTimeoutMillis;
         this.maxRetries = maxRetries;
         this.throttleRetries = throttleRetries;
@@ -386,6 +397,9 @@ S3ClientSettings refine(Settings repositorySettings) {
             ).millis()
         );
         final int newMaxConnections = Math.toIntExact(getRepoSettingOrDefault(MAX_CONNECTIONS_SETTING, normalizedSettings, maxConnections));
+        final int newMaxSyncConnections = Math.toIntExact(
+            getRepoSettingOrDefault(MAX_SYNC_CONNECTIONS_SETTING, normalizedSettings, maxConnections)
+        );
         final int newMaxRetries = getRepoSettingOrDefault(MAX_RETRIES_SETTING, normalizedSettings, maxRetries);
         final boolean newThrottleRetries = getRepoSettingOrDefault(USE_THROTTLE_RETRIES_SETTING, normalizedSettings, throttleRetries);
         final boolean newPathStyleAccess = getRepoSettingOrDefault(USE_PATH_STYLE_ACCESS, normalizedSettings, pathStyleAccess);
@@ -433,6 +447,7 @@ S3ClientSettings refine(Settings repositorySettings) {
             newConnectionTimeoutMillis,
             newConnectionTTLMillis,
             newMaxConnections,
+            newMaxSyncConnections,
             newConnectionAcquisitionTimeoutMillis,
             newMaxRetries,
             newThrottleRetries,
@@ -563,6 +578,7 @@ static S3ClientSettings getClientSettings(final Settings settings, final String
             Math.toIntExact(getConfigValue(settings, clientName, CONNECTION_TIMEOUT_SETTING).millis()),
             Math.toIntExact(getConfigValue(settings, clientName, CONNECTION_TTL_SETTING).millis()),
             Math.toIntExact(getConfigValue(settings, clientName, MAX_CONNECTIONS_SETTING)),
+            Math.toIntExact(getConfigValue(settings, clientName, MAX_SYNC_CONNECTIONS_SETTING)),
             Math.toIntExact(getConfigValue(settings, clientName, CONNECTION_ACQUISITION_TIMEOUT).millis()),
             getConfigValue(settings, clientName, MAX_RETRIES_SETTING),
             getConfigValue(settings, clientName, USE_THROTTLE_RETRIES_SETTING),

plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Service.java

@@ -279,6 +279,8 @@ protected PasswordAuthentication getPasswordAuthentication() {
         }
 
         clientBuilder.socketTimeout(Duration.ofMillis(clientSettings.readTimeoutMillis));
+        clientBuilder.maxConnections(clientSettings.maxSyncConnections);
+        clientBuilder.connectionAcquisitionTimeout(Duration.ofMillis(clientSettings.connectionAcquisitionTimeoutMillis));
 
         return clientBuilder;
     }

plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3ClientSettingsTests.java

@@ -74,6 +74,8 @@ public void testThereIsADefaultClientByDefault() {
         assertThat(defaultSettings.connectionTimeoutMillis, is(10 * 1000));
         assertThat(defaultSettings.connectionTTLMillis, is(5 * 1000));
         assertThat(defaultSettings.maxConnections, is(500));
+        assertThat(defaultSettings.maxSyncConnections, is(500));
+        assertThat(defaultSettings.connectionAcquisitionTimeoutMillis, is(15 * 60 * 1000));
         assertThat(defaultSettings.maxRetries, is(3));
         assertThat(defaultSettings.throttleRetries, is(true));
     }