diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java index 368d3df1c7f1..054165286c82 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java @@ -17,7 +17,6 @@ package org.apache.hadoop.ozone; -import com.amazonaws.services.s3.AmazonS3; import java.io.IOException; import java.util.List; import java.util.UUID; @@ -39,7 +38,6 @@ import org.apache.ozone.test.GenericTestUtils; import org.apache.ratis.util.ExitUtils; import org.apache.ratis.util.function.CheckedFunction; -import software.amazon.awssdk.services.s3.S3Client; /** * Interface used for MiniOzoneClusters. @@ -162,16 +160,6 @@ void waitForPipelineTobeReady(HddsProtos.ReplicationFactor factor, */ OzoneClient newClient() throws IOException; - /** - * Returns an {@link AmazonS3} to use AWS SDK V1 to access the {@link MiniOzoneCluster}. - */ - AmazonS3 newS3Client(); - - /** - * Returns an {@link S3Client} to use AWS SDK V2 to access the {@link MiniOzoneCluster}. - */ - S3Client newS3ClientV2() throws Exception; - /** * Returns StorageContainerLocationClient to communicate with * {@link StorageContainerManager} associated with the MiniOzoneCluster. diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java index 4e26e3d580fd..1d1ef33e06bb 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java @@ -23,29 +23,15 @@ import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_DATANODE_ADDRESS_KEY; import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_HTTP_ADDRESS_KEY; import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_TASK_SAFEMODE_WAIT_THRESHOLD; -import static org.apache.hadoop.hdds.server.http.HttpConfig.getHttpPolicy; -import static org.apache.hadoop.hdds.server.http.HttpServer2.HTTPS_SCHEME; -import static org.apache.hadoop.hdds.server.http.HttpServer2.HTTP_SCHEME; import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_DB_DIR; import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_OM_SNAPSHOT_DB_DIR; import static org.apache.hadoop.ozone.recon.ReconServerConfigKeys.OZONE_RECON_SCM_DB_DIR; -import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_HTTPS_ADDRESS_KEY; -import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_HTTP_ADDRESS_KEY; import static org.apache.ozone.test.GenericTestUtils.PortAllocator.anyHostWithFreePort; import static org.apache.ozone.test.GenericTestUtils.PortAllocator.getFreePort; import static org.apache.ozone.test.GenericTestUtils.PortAllocator.localhostWithFreePort; -import com.amazonaws.ClientConfiguration; -import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.auth.AWSStaticCredentialsProvider; -import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.client.builder.AwsClientBuilder; -import com.amazonaws.regions.Regions; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3ClientBuilder; import java.io.File; import java.io.IOException; -import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -82,7 +68,6 @@ import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.apache.hadoop.hdds.security.symmetric.SecretKeyClient; import org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient; -import org.apache.hadoop.hdds.server.http.HttpConfig; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.hdds.utils.db.CodecBuffer; import org.apache.hadoop.hdds.utils.db.CodecTestUtil; @@ -107,10 +92,6 @@ import org.apache.ozone.test.GenericTestUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; -import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.s3.S3Client; /** * MiniOzoneCluster creates a complete in-process Ozone cluster suitable for @@ -306,91 +287,6 @@ public OzoneClient newClient() throws IOException { return client; } - @Override - public AmazonS3 newS3Client() { - // TODO: Parameterize tests between Virtual host style and Path style - return createS3Client(true); - } - - @Override - public S3Client newS3ClientV2() throws Exception { - return createS3ClientV2(true); - } - - public AmazonS3 createS3Client(boolean enablePathStyle) { - final String accessKey = "user"; - final String secretKey = "password"; - final Regions region = Regions.DEFAULT_REGION; - - final String protocol; - final HttpConfig.Policy webPolicy = getHttpPolicy(conf); - String host; - - if (webPolicy.isHttpsEnabled()) { - // TODO: Currently HTTPS is disabled in the test, we can add HTTPS - // integration in the future - protocol = HTTPS_SCHEME; - host = conf.get(OZONE_S3G_HTTPS_ADDRESS_KEY); - } else { - protocol = HTTP_SCHEME; - host = conf.get(OZONE_S3G_HTTP_ADDRESS_KEY); - } - - String endpoint = protocol + "://" + host; - - AWSCredentialsProvider credentials = new AWSStaticCredentialsProvider( - new BasicAWSCredentials(accessKey, secretKey) - ); - - - ClientConfiguration clientConfiguration = new ClientConfiguration(); - LOG.info("S3 Endpoint is {}", endpoint); - - return AmazonS3ClientBuilder.standard() - .withPathStyleAccessEnabled(enablePathStyle) - .withEndpointConfiguration( - new AwsClientBuilder.EndpointConfiguration( - endpoint, region.getName() - ) - ) - .withClientConfiguration(clientConfiguration) - .withCredentials(credentials) - .build(); - } - - public S3Client createS3ClientV2(boolean enablePathStyle) throws Exception { - final String accessKey = "user"; - final String secretKey = "password"; - final Region region = Region.US_EAST_1; - - final String protocol; - final HttpConfig.Policy webPolicy = getHttpPolicy(conf); - String host; - - if (webPolicy.isHttpsEnabled()) { - // TODO: Currently HTTPS is disabled in the test, we can add HTTPS - // integration in the future - protocol = HTTPS_SCHEME; - host = conf.get(OZONE_S3G_HTTPS_ADDRESS_KEY); - } else { - protocol = HTTP_SCHEME; - host = conf.get(OZONE_S3G_HTTP_ADDRESS_KEY); - } - - String endpoint = protocol + "://" + host; - - LOG.info("S3 Endpoint is {}", endpoint); - - AwsBasicCredentials credentials = AwsBasicCredentials.create(accessKey, secretKey); - - return S3Client.builder() - .region(region) - .endpointOverride(new URI(endpoint)) - .credentialsProvider(StaticCredentialsProvider.create(credentials)) - .forcePathStyle(enablePathStyle) - .build(); - } - protected OzoneClient createClient() throws IOException { return OzoneClientFactory.getRpcClient(conf); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/S3ClientFactory.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/S3ClientFactory.java new file mode 100644 index 000000000000..5f80a0907c5f --- /dev/null +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/S3ClientFactory.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.s3; + +import static org.apache.hadoop.hdds.server.http.HttpConfig.getHttpPolicy; +import static org.apache.hadoop.hdds.server.http.HttpServer2.HTTPS_SCHEME; +import static org.apache.hadoop.hdds.server.http.HttpServer2.HTTP_SCHEME; +import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_HTTPS_ADDRESS_KEY; +import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_HTTP_ADDRESS_KEY; + +import com.amazonaws.ClientConfiguration; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.client.builder.AwsClientBuilder; +import com.amazonaws.regions.Regions; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import java.net.URI; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.server.http.HttpConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; + +/** + * Factory class for creating S3 clients. + */ +public class S3ClientFactory { + private static final Logger LOG = LoggerFactory.getLogger(S3ClientFactory.class); + private final OzoneConfiguration conf; + + /** + * Constructor for S3ClientFactory. + * + * @param conf OzoneConfiguration + */ + public S3ClientFactory(OzoneConfiguration conf) { + this.conf = conf; + } + + /** + * Creates an AmazonS3 client (AWS SDK V1) with path style access enabled. + * + * @return AmazonS3 client + */ + public AmazonS3 createS3Client() { + return createS3Client(true); + } + + /** + * Creates an AmazonS3 client (AWS SDK V1). + * + * @param enablePathStyle whether to enable path style access + * @return AmazonS3 client + */ + public AmazonS3 createS3Client(boolean enablePathStyle) { + final String accessKey = "user"; + final String secretKey = "password"; + final Regions region = Regions.DEFAULT_REGION; + + final String protocol; + final HttpConfig.Policy webPolicy = getHttpPolicy(conf); + String host; + + if (webPolicy.isHttpsEnabled()) { + // TODO: Currently HTTPS is disabled in the test, we can add HTTPS + // integration in the future + protocol = HTTPS_SCHEME; + host = conf.get(OZONE_S3G_HTTPS_ADDRESS_KEY); + } else { + protocol = HTTP_SCHEME; + host = conf.get(OZONE_S3G_HTTP_ADDRESS_KEY); + } + + String endpoint = protocol + "://" + host; + + AWSCredentialsProvider credentials = new AWSStaticCredentialsProvider( + new BasicAWSCredentials(accessKey, secretKey)); + + ClientConfiguration clientConfiguration = new ClientConfiguration(); + LOG.info("S3 Endpoint is {}", endpoint); + + return AmazonS3ClientBuilder.standard() + .withPathStyleAccessEnabled(enablePathStyle) + .withEndpointConfiguration( + new AwsClientBuilder.EndpointConfiguration( + endpoint, region.getName())) + .withClientConfiguration(clientConfiguration) + .withCredentials(credentials) + .build(); + } + + /** + * Creates an S3Client (AWS SDK V2) with path style access enabled. + * + * @return S3Client + * @throws Exception if there is an error creating the client + */ + public S3Client createS3ClientV2() throws Exception { + return createS3ClientV2(true); + } + + /** + * Creates an S3Client (AWS SDK V2). + * + * @param enablePathStyle whether to enable path style access + * @return S3Client + * @throws Exception if there is an error creating the client + */ + public S3Client createS3ClientV2(boolean enablePathStyle) throws Exception { + final String accessKey = "user"; + final String secretKey = "password"; + final Region region = Region.US_EAST_1; + + final String protocol; + final HttpConfig.Policy webPolicy = getHttpPolicy(conf); + String host; + + if (webPolicy.isHttpsEnabled()) { + // TODO: Currently HTTPS is disabled in the test, we can add HTTPS + // integration in the future + protocol = HTTPS_SCHEME; + host = conf.get(OZONE_S3G_HTTPS_ADDRESS_KEY); + } else { + protocol = HTTP_SCHEME; + host = conf.get(OZONE_S3G_HTTP_ADDRESS_KEY); + } + + String endpoint = protocol + "://" + host; + + LOG.info("S3 Endpoint is {}", endpoint); + + AwsBasicCredentials credentials = AwsBasicCredentials.create(accessKey, secretKey); + + return S3Client.builder() + .region(region) + .endpointOverride(new URI(endpoint)) + .credentialsProvider(StaticCredentialsProvider.create(credentials)) + .forcePathStyle(enablePathStyle) + .build(); + } +} diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java index 9efaafd290d7..23740c91301c 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java @@ -101,6 +101,7 @@ import org.apache.hadoop.ozone.client.OzoneClientFactory; import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.client.io.OzoneOutputStream; +import org.apache.hadoop.ozone.s3.S3ClientFactory; import org.apache.ozone.test.OzoneTestBase; import org.junit.jupiter.api.MethodOrderer; import org.junit.jupiter.api.Test; @@ -186,7 +187,7 @@ static void startCluster(OzoneConfiguration conf) throws Exception { .setNumDatanodes(5) .build(); cluster.waitForClusterToBeReady(); - s3Client = cluster.newS3Client(); + s3Client = new S3ClientFactory(cluster.getConf()).createS3Client(); } /** diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java index 53328f9e4350..5a57eb96146e 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java @@ -39,6 +39,7 @@ import javax.xml.bind.DatatypeConverter; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.hadoop.ozone.s3.S3ClientFactory; import org.apache.ozone.test.OzoneTestBase; import org.junit.jupiter.api.MethodOrderer; import org.junit.jupiter.api.Test; @@ -92,7 +93,7 @@ static void startCluster(OzoneConfiguration conf) throws Exception { .setNumDatanodes(5) .build(); cluster.waitForClusterToBeReady(); - s3Client = cluster.newS3ClientV2(); + s3Client = new S3ClientFactory(cluster.getConf()).createS3ClientV2(); } /**