HDDS-11576. Create a separate S3 client factory

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java

@@ -17,7 +17,6 @@
 
 package org.apache.hadoop.ozone;
 
-import com.amazonaws.services.s3.AmazonS3;
 import java.io.IOException;
 import java.util.List;
 import java.util.UUID;
@@ -39,7 +38,6 @@
 import org.apache.ozone.test.GenericTestUtils;
 import org.apache.ratis.util.ExitUtils;
 import org.apache.ratis.util.function.CheckedFunction;
-import software.amazon.awssdk.services.s3.S3Client;
 
 /**
  * Interface used for MiniOzoneClusters.
@@ -162,16 +160,6 @@ void waitForPipelineTobeReady(HddsProtos.ReplicationFactor factor,
    */
   OzoneClient newClient() throws IOException;
 
-  /**
-   * Returns an {@link AmazonS3} to use AWS SDK V1 to access the {@link MiniOzoneCluster}.
-   */
-  AmazonS3 newS3Client();
-
-  /**
-   * Returns an {@link S3Client} to use AWS SDK V2 to access the {@link MiniOzoneCluster}.
-   */
-  S3Client newS3ClientV2() throws Exception;
-
   /**
    * Returns StorageContainerLocationClient to communicate with
    * {@link StorageContainerManager} associated with the MiniOzoneCluster.

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java

@@ -23,29 +23,15 @@
 import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_DATANODE_ADDRESS_KEY;
 import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_HTTP_ADDRESS_KEY;
 import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_TASK_SAFEMODE_WAIT_THRESHOLD;
-import static org.apache.hadoop.hdds.server.http.HttpConfig.getHttpPolicy;
-import static org.apache.hadoop.hdds.server.http.HttpServer2.HTTPS_SCHEME;
-import static org.apache.hadoop.hdds.server.http.HttpServer2.HTTP_SCHEME;
 import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_DB_DIR;
 import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_OM_SNAPSHOT_DB_DIR;
 import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_SCM_DB_DIR;
-import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_HTTPS_ADDRESS_KEY;
-import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_HTTP_ADDRESS_KEY;
 import static org.apache.ozone.test.GenericTestUtils.PortAllocator.anyHostWithFreePort;
 import static org.apache.ozone.test.GenericTestUtils.PortAllocator.getFreePort;
 import static org.apache.ozone.test.GenericTestUtils.PortAllocator.localhostWithFreePort;
 
-import com.amazonaws.ClientConfiguration;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.auth.AWSStaticCredentialsProvider;
-import com.amazonaws.auth.BasicAWSCredentials;
-import com.amazonaws.client.builder.AwsClientBuilder;
-import com.amazonaws.regions.Regions;
-import com.amazonaws.services.s3.AmazonS3;
-import com.amazonaws.services.s3.AmazonS3ClientBuilder;
 import java.io.File;
 import java.io.IOException;
-import java.net.URI;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -82,7 +68,6 @@
 import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
 import org.apache.hadoop.hdds.security.symmetric.SecretKeyClient;
 import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
-import org.apache.hadoop.hdds.server.http.HttpConfig;
 import org.apache.hadoop.hdds.utils.IOUtils;
 import org.apache.hadoop.hdds.utils.db.CodecBuffer;
 import org.apache.hadoop.hdds.utils.db.CodecTestUtil;
@@ -107,10 +92,6 @@
 import org.apache.ozone.test.GenericTestUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
-import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
-import software.amazon.awssdk.regions.Region;
-import software.amazon.awssdk.services.s3.S3Client;
 
 /**
  * MiniOzoneCluster creates a complete in-process Ozone cluster suitable for
@@ -306,91 +287,6 @@ public OzoneClient newClient() throws IOException {
     return client;
   }
 
-  @Override
-  public AmazonS3 newS3Client() {
-    // TODO: Parameterize tests between Virtual host style and Path style
-    return createS3Client(true);
-  }
-
-  @Override
-  public S3Client newS3ClientV2() throws Exception {
-    return createS3ClientV2(true);
-  }
-
-  public AmazonS3 createS3Client(boolean enablePathStyle) {
-    final String accessKey = "user";
-    final String secretKey = "password";
-    final Regions region = Regions.DEFAULT_REGION;
-
-    final String protocol;
-    final HttpConfig.Policy webPolicy = getHttpPolicy(conf);
-    String host;
-
-    if (webPolicy.isHttpsEnabled()) {
-      // TODO: Currently HTTPS is disabled in the test, we can add HTTPS
-      //  integration in the future
-      protocol = HTTPS_SCHEME;
-      host = conf.get(OZONE_S3G_HTTPS_ADDRESS_KEY);
-    } else {
-      protocol = HTTP_SCHEME;
-      host = conf.get(OZONE_S3G_HTTP_ADDRESS_KEY);
-    }
-
-    String endpoint = protocol + "://" + host;
-
-    AWSCredentialsProvider credentials = new AWSStaticCredentialsProvider(
-        new BasicAWSCredentials(accessKey, secretKey)
-    );
-
-
-    ClientConfiguration clientConfiguration = new ClientConfiguration();
-    LOG.info("S3 Endpoint is {}", endpoint);
-
-    return AmazonS3ClientBuilder.standard()
-        .withPathStyleAccessEnabled(enablePathStyle)
-        .withEndpointConfiguration(
-            new AwsClientBuilder.EndpointConfiguration(
-                endpoint, region.getName()
-            )
-        )
-        .withClientConfiguration(clientConfiguration)
-        .withCredentials(credentials)
-        .build();
-  }
-
-  public S3Client createS3ClientV2(boolean enablePathStyle) throws Exception {
-    final String accessKey = "user";
-    final String secretKey = "password";
-    final Region region = Region.US_EAST_1;
-
-    final String protocol;
-    final HttpConfig.Policy webPolicy = getHttpPolicy(conf);
-    String host;
-
-    if (webPolicy.isHttpsEnabled()) {
-      // TODO: Currently HTTPS is disabled in the test, we can add HTTPS
-      //  integration in the future
-      protocol = HTTPS_SCHEME;
-      host = conf.get(OZONE_S3G_HTTPS_ADDRESS_KEY);
-    } else {
-      protocol = HTTP_SCHEME;
-      host = conf.get(OZONE_S3G_HTTP_ADDRESS_KEY);
-    }
-
-    String endpoint = protocol + "://" + host;
-
-    LOG.info("S3 Endpoint is {}", endpoint);
-
-    AwsBasicCredentials credentials = AwsBasicCredentials.create(accessKey, secretKey);
-
-    return S3Client.builder()
-        .region(region)
-        .endpointOverride(new URI(endpoint))
-        .credentialsProvider(StaticCredentialsProvider.create(credentials))
-        .forcePathStyle(enablePathStyle)
-        .build();
-  }
-
   protected OzoneClient createClient() throws IOException {
     return OzoneClientFactory.getRpcClient(conf);
   }

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/S3ClientFactory.java

@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.s3;
+
+import static org.apache.hadoop.hdds.server.http.HttpConfig.getHttpPolicy;
+import static org.apache.hadoop.hdds.server.http.HttpServer2.HTTPS_SCHEME;
+import static org.apache.hadoop.hdds.server.http.HttpServer2.HTTP_SCHEME;
+import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_HTTPS_ADDRESS_KEY;
+import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_HTTP_ADDRESS_KEY;
+
+import com.amazonaws.ClientConfiguration;
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.auth.AWSStaticCredentialsProvider;
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.client.builder.AwsClientBuilder;
+import com.amazonaws.regions.Regions;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+import java.net.URI;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.server.http.HttpConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3Client;
+
+/**
+ * Factory class for creating S3 clients.
+ */
+public class S3ClientFactory {
+  private static final Logger LOG = LoggerFactory.getLogger(S3ClientFactory.class);
+  private final OzoneConfiguration conf;
+
+  /**
+   * Constructor for S3ClientFactory.
+   *
+   * @param conf OzoneConfiguration
+   */
+  public S3ClientFactory(OzoneConfiguration conf) {
+    this.conf = conf;
+  }
+
+  /**
+   * Creates an AmazonS3 client (AWS SDK V1) with path style access enabled.
+   *
+   * @return AmazonS3 client
+   */
+  public AmazonS3 createS3Client() {
+    return createS3Client(true);
+  }
+
+  /**
+   * Creates an AmazonS3 client (AWS SDK V1).
+   *
+   * @param enablePathStyle whether to enable path style access
+   * @return AmazonS3 client
+   */
+  public AmazonS3 createS3Client(boolean enablePathStyle) {
+    final String accessKey = "user";
+    final String secretKey = "password";
+    final Regions region = Regions.DEFAULT_REGION;
+
+    final String protocol;
+    final HttpConfig.Policy webPolicy = getHttpPolicy(conf);
+    String host;
+
+    if (webPolicy.isHttpsEnabled()) {
+      // TODO: Currently HTTPS is disabled in the test, we can add HTTPS
+      // integration in the future
+      protocol = HTTPS_SCHEME;
+      host = conf.get(OZONE_S3G_HTTPS_ADDRESS_KEY);
+    } else {
+      protocol = HTTP_SCHEME;
+      host = conf.get(OZONE_S3G_HTTP_ADDRESS_KEY);
+    }
+
+    String endpoint = protocol + "://" + host;
+
+    AWSCredentialsProvider credentials = new AWSStaticCredentialsProvider(
+        new BasicAWSCredentials(accessKey, secretKey));
+
+    ClientConfiguration clientConfiguration = new ClientConfiguration();
+    LOG.info("S3 Endpoint is {}", endpoint);
+
+    return AmazonS3ClientBuilder.standard()
+        .withPathStyleAccessEnabled(enablePathStyle)
+        .withEndpointConfiguration(
+            new AwsClientBuilder.EndpointConfiguration(
+                endpoint, region.getName()))
+        .withClientConfiguration(clientConfiguration)
+        .withCredentials(credentials)
+        .build();
+  }
+
+  /**
+   * Creates an S3Client (AWS SDK V2) with path style access enabled.
+   *
+   * @return S3Client
+   * @throws Exception if there is an error creating the client
+   */
+  public S3Client createS3ClientV2() throws Exception {
+    return createS3ClientV2(true);
+  }
+
+  /**
+   * Creates an S3Client (AWS SDK V2).
+   *
+   * @param enablePathStyle whether to enable path style access
+   * @return S3Client
+   * @throws Exception if there is an error creating the client
+   */
+  public S3Client createS3ClientV2(boolean enablePathStyle) throws Exception {
+    final String accessKey = "user";
+    final String secretKey = "password";
+    final Region region = Region.US_EAST_1;
+
+    final String protocol;
+    final HttpConfig.Policy webPolicy = getHttpPolicy(conf);
+    String host;
+
+    if (webPolicy.isHttpsEnabled()) {
+      // TODO: Currently HTTPS is disabled in the test, we can add HTTPS
+      // integration in the future
+      protocol = HTTPS_SCHEME;
+      host = conf.get(OZONE_S3G_HTTPS_ADDRESS_KEY);
+    } else {
+      protocol = HTTP_SCHEME;
+      host = conf.get(OZONE_S3G_HTTP_ADDRESS_KEY);
+    }
+
+    String endpoint = protocol + "://" + host;
+
+    LOG.info("S3 Endpoint is {}", endpoint);
+
+    AwsBasicCredentials credentials = AwsBasicCredentials.create(accessKey, secretKey);
+
+    return S3Client.builder()
+        .region(region)
+        .endpointOverride(new URI(endpoint))
+        .credentialsProvider(StaticCredentialsProvider.create(credentials))
+        .forcePathStyle(enablePathStyle)
+        .build();
+  }
+}

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java

@@ -101,6 +101,7 @@
 import org.apache.hadoop.ozone.client.OzoneClientFactory;
 import org.apache.hadoop.ozone.client.OzoneVolume;
 import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.s3.S3ClientFactory;
 import org.apache.ozone.test.OzoneTestBase;
 import org.junit.jupiter.api.MethodOrderer;
 import org.junit.jupiter.api.Test;
@@ -186,7 +187,7 @@ static void startCluster(OzoneConfiguration conf) throws Exception {
         .setNumDatanodes(5)
         .build();
     cluster.waitForClusterToBeReady();
-    s3Client = cluster.newS3Client();
+    s3Client = new S3ClientFactory(cluster.getConf()).createS3Client();
   }
 
   /**

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java

@@ -39,6 +39,7 @@
 import javax.xml.bind.DatatypeConverter;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.s3.S3ClientFactory;
 import org.apache.ozone.test.OzoneTestBase;
 import org.junit.jupiter.api.MethodOrderer;
 import org.junit.jupiter.api.Test;
@@ -92,7 +93,7 @@ static void startCluster(OzoneConfiguration conf) throws Exception {
         .setNumDatanodes(5)
         .build();
     cluster.waitForClusterToBeReady();
-    s3Client = cluster.newS3ClientV2();
+    s3Client = new S3ClientFactory(cluster.getConf()).createS3ClientV2();
   }
 
   /**