diff --git a/services/s3/src/it/java/software/amazon/awssdk/services/s3/crossregion/S3CrossRegionAsyncIntegrationTest.java b/services/s3/src/it/java/software/amazon/awssdk/services/s3/crossregion/S3CrossRegionAsyncIntegrationTest.java new file mode 100644 index 00000000000..d986ee1e3e2 --- /dev/null +++ b/services/s3/src/it/java/software/amazon/awssdk/services/s3/crossregion/S3CrossRegionAsyncIntegrationTest.java @@ -0,0 +1,52 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.crossregion; + +import static software.amazon.awssdk.testutils.service.S3BucketUtils.temporaryBucketName; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import software.amazon.awssdk.services.s3.S3AsyncClient; + +public class S3CrossRegionAsyncIntegrationTest extends S3CrossRegionAsyncIntegrationTestBase { + private static final String BUCKET = temporaryBucketName(S3CrossRegionAsyncIntegrationTest.class); + + @BeforeAll + static void setUpClass() { + s3 = s3ClientBuilder().build(); + createBucket(BUCKET); + } + + @AfterAll + static void clearClass() { + deleteBucketAndAllContents(BUCKET); + } + + @BeforeEach + public void initialize() { + crossRegionS3Client = S3AsyncClient.builder() + .region(CROSS_REGION) + .serviceConfiguration(s -> s.crossRegionAccessEnabled(true)) + .build(); + } + + @Override + protected String bucketName() { + return BUCKET; + } + +} diff --git a/services/s3/src/it/java/software/amazon/awssdk/services/s3/crossregion/S3CrossRegionAsyncIntegrationTestBase.java b/services/s3/src/it/java/software/amazon/awssdk/services/s3/crossregion/S3CrossRegionAsyncIntegrationTestBase.java new file mode 100644 index 00000000000..ee1b82d4da4 --- /dev/null +++ b/services/s3/src/it/java/software/amazon/awssdk/services/s3/crossregion/S3CrossRegionAsyncIntegrationTestBase.java @@ -0,0 +1,77 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.crossregion; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.core.async.AsyncRequestBody; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectResponse; +import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.HeadBucketRequest; +import software.amazon.awssdk.services.s3.model.HeadBucketResponse; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.PutObjectResponse; +import software.amazon.awssdk.services.s3.model.S3Object; +import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Publisher; + +public abstract class S3CrossRegionAsyncIntegrationTestBase extends S3CrossRegionIntegrationTestBase{ + protected S3AsyncClient crossRegionS3Client; + + @Override + protected List paginatedAPICall(ListObjectsV2Request listObjectsV2Request) { + List resultObjects = new ArrayList<>(); + ListObjectsV2Publisher publisher = crossRegionS3Client.listObjectsV2Paginator(listObjectsV2Request); + CompletableFuture subscribe = publisher.subscribe(response -> { + response.contents().forEach(a -> resultObjects.add(a)); + }); + subscribe.join(); + return resultObjects; + } + + @Override + protected DeleteObjectsResponse postObjectAPICall(DeleteObjectsRequest deleteObjectsRequest) { + return crossRegionS3Client.deleteObjects(deleteObjectsRequest).join(); + } + + @Override + protected HeadBucketResponse headAPICall(HeadBucketRequest headBucketRequest) { + return crossRegionS3Client.headBucket(headBucketRequest).join(); + } + + @Override + protected DeleteObjectResponse deleteObjectAPICall(DeleteObjectRequest deleteObjectRequest) { + return crossRegionS3Client.deleteObject(deleteObjectRequest).join(); + } + + @Override + protected PutObjectResponse putAPICall(PutObjectRequest putObjectRequest, String testString) { + return crossRegionS3Client.putObject(putObjectRequest, AsyncRequestBody.fromString(testString)).join(); + } + + @Override + protected ResponseBytes getAPICall(GetObjectRequest getObjectRequest) { + return crossRegionS3Client.getObject(getObjectRequest, AsyncResponseTransformer.toBytes()).join(); + } +} diff --git a/services/s3/src/it/java/software/amazon/awssdk/services/s3/crossregion/S3CrossRegionCrtIntegrationTest.java b/services/s3/src/it/java/software/amazon/awssdk/services/s3/crossregion/S3CrossRegionCrtIntegrationTest.java new file mode 100644 index 00000000000..29c5d075be8 --- /dev/null +++ b/services/s3/src/it/java/software/amazon/awssdk/services/s3/crossregion/S3CrossRegionCrtIntegrationTest.java @@ -0,0 +1,51 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.crossregion; + +import static software.amazon.awssdk.testutils.service.S3BucketUtils.temporaryBucketName; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import software.amazon.awssdk.services.s3.S3AsyncClient; + +public class S3CrossRegionCrtIntegrationTest extends S3CrossRegionAsyncIntegrationTestBase { + private static final String BUCKET = temporaryBucketName(S3CrossRegionCrtIntegrationTest.class); + + @BeforeAll + static void setUpClass() { + s3 = s3ClientBuilder().build(); + createBucket(BUCKET); + } + + @AfterAll + static void clearClass() { + deleteBucketAndAllContents(BUCKET); + } + + @BeforeEach + public void initialize() { + crossRegionS3Client = S3AsyncClient.crtBuilder() + .region(CROSS_REGION) + .crossRegionAccessEnabled(true) + .build(); + } + @Override + protected String bucketName() { + return BUCKET; + } + +} diff --git a/services/s3/src/it/java/software/amazon/awssdk/services/s3/crossregion/S3CrossRegionIntegrationTestBase.java b/services/s3/src/it/java/software/amazon/awssdk/services/s3/crossregion/S3CrossRegionIntegrationTestBase.java new file mode 100644 index 00000000000..a8d391251bd --- /dev/null +++ b/services/s3/src/it/java/software/amazon/awssdk/services/s3/crossregion/S3CrossRegionIntegrationTestBase.java @@ -0,0 +1,141 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.crossregion; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.List; +import java.util.stream.IntStream; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3IntegrationTestBase; +import software.amazon.awssdk.services.s3.model.ChecksumAlgorithm; +import software.amazon.awssdk.services.s3.model.ChecksumMode; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectResponse; +import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.HeadBucketRequest; +import software.amazon.awssdk.services.s3.model.HeadBucketResponse; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.PutObjectResponse; +import software.amazon.awssdk.services.s3.model.S3Object; + +public abstract class S3CrossRegionIntegrationTestBase extends S3IntegrationTestBase { + + public static final String X_AMZ_BUCKET_REGION = "x-amz-bucket-region"; + + protected static final Region CROSS_REGION = Region.of("eu-central-1"); + + private static final String KEY = "key"; + + @Test + void getApi_CrossRegionCall() { + s3.putObject(p -> p.bucket(bucketName()).checksumAlgorithm(ChecksumAlgorithm.CRC32).key(KEY), RequestBody.fromString( + "TEST_STRING")); + GetObjectRequest getObjectRequest = + GetObjectRequest.builder().bucket(bucketName()).checksumMode(ChecksumMode.ENABLED).key(KEY).build(); + ResponseBytes response = getAPICall(getObjectRequest); + assertThat(new String(response.asByteArray())).isEqualTo("TEST_STRING"); + } + + @Test + void putApi_CrossRegionCall() { + s3.putObject(p -> p.bucket(bucketName()).checksumAlgorithm(ChecksumAlgorithm.CRC32).key(KEY), RequestBody.fromString( + "TEST_STRING")); + PutObjectRequest putObjectRequest = + PutObjectRequest.builder().bucket(bucketName()).checksumAlgorithm(ChecksumAlgorithm.CRC32).key(KEY).build(); + PutObjectResponse response = putAPICall(putObjectRequest, "TEST_STRING"); + assertThat(response.checksumCRC32()).isEqualTo("S9ke8w=="); + } + + @Test + void deleteApi_CrossRegionCall() { + s3.putObject(p -> p.bucket(bucketName()).checksumAlgorithm(ChecksumAlgorithm.CRC32).key(KEY), RequestBody.fromString( + "TEST_STRING")); + DeleteObjectRequest deleteObjectRequest = DeleteObjectRequest.builder().bucket(bucketName()).key(KEY).build(); + DeleteObjectResponse response = deleteObjectAPICall(deleteObjectRequest); + assertThat(response).isNotNull(); + } + + @Test + void postApi_CrossRegionCall() { + s3.putObject(p -> p.bucket(bucketName()).checksumAlgorithm(ChecksumAlgorithm.CRC32).key(KEY), RequestBody.fromString( + "TEST_STRING")); + s3.putObject(p -> p.bucket(bucketName()).checksumAlgorithm(ChecksumAlgorithm.CRC32).key(KEY + "_1"), + RequestBody.fromString("TEST_STRING")); + DeleteObjectsRequest deleteObjectsRequest = + DeleteObjectsRequest.builder().bucket(bucketName()).delete(d -> d.objects(o -> o.key(KEY), o -> o.key(KEY + "_1"))).build(); + DeleteObjectsResponse response = postObjectAPICall(deleteObjectsRequest); + assertThat(response).isNotNull(); + } + + @Test + void cachedRegionGetsUsed_when_CrossRegionCall() { + putAPICall(PutObjectRequest.builder().bucket(bucketName()).checksumAlgorithm(ChecksumAlgorithm.CRC32).key(KEY).build(), + "TEST_STRING"); + GetObjectRequest getObjectRequest = + GetObjectRequest.builder().bucket(bucketName()).checksumMode(ChecksumMode.ENABLED).key(KEY).build(); + ResponseBytes response = getAPICall(getObjectRequest); + assertThat(new String(response.asByteArray())).isEqualTo("TEST_STRING"); + } + + @Test + void paginatedApi_CrossRegionCall() { + s3.deleteObject(p -> p.bucket(bucketName()).key(KEY)); + int maxKeys = 3; + int totalKeys = maxKeys * 2 ; + IntStream.range(0, totalKeys ) + .forEach( + i -> + s3.putObject(p -> p.bucket(bucketName()).checksumAlgorithm(ChecksumAlgorithm.CRC32).key(KEY + "_" + i), + RequestBody.fromString("TEST_STRING")) + ); + ListObjectsV2Request listObjectsV2Request = ListObjectsV2Request.builder().bucket(bucketName()).maxKeys(maxKeys).build(); + List s3ObjectList = paginatedAPICall(listObjectsV2Request); + assertThat(s3ObjectList).hasSize(totalKeys); + IntStream.range(0, totalKeys ).forEach(i -> s3.deleteObject(p -> p.bucket(bucketName()).key(KEY + "_" + i))); + } + + @Test + void headApi_CrossRegionCall() { + s3.putObject(p -> p.bucket(bucketName()).checksumAlgorithm(ChecksumAlgorithm.CRC32).key(KEY), RequestBody.fromString( + "TEST_STRING")); + HeadBucketRequest headBucketRequest = HeadBucketRequest.builder().bucket(bucketName()).build(); + HeadBucketResponse response = headAPICall(headBucketRequest); + assertThat(response).isNotNull(); + } + + protected abstract List paginatedAPICall(ListObjectsV2Request listObjectsV2Request); + + protected abstract DeleteObjectsResponse postObjectAPICall(DeleteObjectsRequest deleteObjectsRequest); + + protected abstract HeadBucketResponse headAPICall(HeadBucketRequest headBucketRequest); + + protected abstract DeleteObjectResponse deleteObjectAPICall(DeleteObjectRequest deleteObjectRequest); + + protected abstract PutObjectResponse putAPICall(PutObjectRequest putObjectRequest, String testString); + + protected abstract ResponseBytes getAPICall(GetObjectRequest getObjectRequest); + + protected abstract String bucketName(); + +} diff --git a/services/s3/src/it/java/software/amazon/awssdk/services/s3/crossregion/S3CrossRegionSyncIntegrationTest.java b/services/s3/src/it/java/software/amazon/awssdk/services/s3/crossregion/S3CrossRegionSyncIntegrationTest.java new file mode 100644 index 00000000000..53813954c95 --- /dev/null +++ b/services/s3/src/it/java/software/amazon/awssdk/services/s3/crossregion/S3CrossRegionSyncIntegrationTest.java @@ -0,0 +1,107 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.crossregion; + +import static software.amazon.awssdk.testutils.service.S3BucketUtils.temporaryBucketName; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.core.sync.ResponseTransformer; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectResponse; +import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.HeadBucketRequest; +import software.amazon.awssdk.services.s3.model.HeadBucketResponse; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; +import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.PutObjectResponse; +import software.amazon.awssdk.services.s3.model.S3Object; + +public class S3CrossRegionSyncIntegrationTest extends S3CrossRegionIntegrationTestBase { + private static final String BUCKET = temporaryBucketName(S3CrossRegionSyncIntegrationTest.class); + private S3Client crossRegionS3Client; + + @BeforeAll + static void setUpClass() { + s3 = s3ClientBuilder().build(); + createBucket(BUCKET); + } + + @AfterAll + static void clearClass() { + deleteBucketAndAllContents(BUCKET); + } + + @BeforeEach + public void initialize() { + crossRegionS3Client = S3Client.builder() + .region(CROSS_REGION) + .serviceConfiguration(s -> s.crossRegionAccessEnabled(true)) + .build(); + } + + @Override + protected List paginatedAPICall(ListObjectsV2Request listObjectsV2Request) { + List resultS3Object = new ArrayList<>(); + Iterator v2ResponseIterator = + crossRegionS3Client.listObjectsV2Paginator(listObjectsV2Request).iterator(); + while (v2ResponseIterator.hasNext()) { + v2ResponseIterator.next().contents().forEach(a -> resultS3Object.add(a)); + } + return resultS3Object; + } + + @Override + protected DeleteObjectsResponse postObjectAPICall(DeleteObjectsRequest deleteObjectsRequest) { + return crossRegionS3Client.deleteObjects(deleteObjectsRequest); + } + + @Override + protected HeadBucketResponse headAPICall(HeadBucketRequest headBucketRequest) { + return crossRegionS3Client.headBucket(headBucketRequest); + } + + @Override + protected DeleteObjectResponse deleteObjectAPICall(DeleteObjectRequest deleteObjectRequest) { + return crossRegionS3Client.deleteObject(deleteObjectRequest); + } + + @Override + protected PutObjectResponse putAPICall(PutObjectRequest putObjectRequest, String testString) { + return crossRegionS3Client.putObject(putObjectRequest, RequestBody.fromString(testString)); + } + + @Override + protected ResponseBytes getAPICall(GetObjectRequest getObjectRequest) { + return crossRegionS3Client.getObject(getObjectRequest, ResponseTransformer.toBytes()); + } + + @Override + protected String bucketName() { + return BUCKET; + } +} diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClient.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClient.java index a6358b238a4..149471f3017 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClient.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/crt/S3CrtAsyncHttpClient.java @@ -49,7 +49,6 @@ import software.amazon.awssdk.utils.AttributeMap; import software.amazon.awssdk.utils.Logger; import software.amazon.awssdk.utils.NumericUtils; -import software.amazon.awssdk.utils.StringUtils; import software.amazon.awssdk.utils.http.SdkHttpUtils; /**