Integration test cases for Cross region Async and Sync Clients

services/s3/src/it/java/software/amazon/awssdk/services/s3/crossregion/CaptureInterceptor.java

@@ -0,0 +1,54 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.services.s3.crossregion;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import software.amazon.awssdk.core.interceptor.Context;
+import software.amazon.awssdk.core.interceptor.ExecutionAttributes;
+import software.amazon.awssdk.core.interceptor.ExecutionInterceptor;
+import software.amazon.awssdk.http.SdkHttpMethod;
+
+public final class CaptureInterceptor implements ExecutionInterceptor {
+    private final List<SdkHttpMethod> httpMethods = new ArrayList<>();
+    private List<String> hosts = new ArrayList<>();
+    private AtomicInteger serviceCalls = new AtomicInteger(0);
+
+    @Override
+    public void beforeTransmission(Context.BeforeTransmission context, ExecutionAttributes executionAttributes) {
+        hosts.add(context.httpRequest().host());
+        httpMethods.add(context.httpRequest().method());
+        serviceCalls.incrementAndGet();
+    }
+
+    public int getServiceCalls() {
+        return serviceCalls.get();
+    }
+
+    public List<String> hosts() {
+        return Collections.unmodifiableList(this.hosts);
+    }
+
+    public List<SdkHttpMethod> httpMethods() {
+        return Collections.unmodifiableList(this.httpMethods);
+    }
+
+    public void reset() {
+        this.hosts = new ArrayList<>();
+    }
+}

services/s3/src/it/java/software/amazon/awssdk/services/s3/crossregion/S3AsyncCrossRegionIntegrationTest.java

@@ -0,0 +1,117 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.services.s3.crossregion;
+
+import static software.amazon.awssdk.testutils.service.S3BucketUtils.temporaryBucketName;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import software.amazon.awssdk.core.ResponseBytes;
+import software.amazon.awssdk.core.async.AsyncRequestBody;
+import software.amazon.awssdk.core.async.AsyncResponseTransformer;
+import software.amazon.awssdk.services.s3.S3AsyncClient;
+import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
+import software.amazon.awssdk.services.s3.model.DeleteObjectResponse;
+import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
+import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectResponse;
+import software.amazon.awssdk.services.s3.model.HeadBucketRequest;
+import software.amazon.awssdk.services.s3.model.HeadBucketResponse;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+import software.amazon.awssdk.services.s3.model.PutObjectResponse;
+import software.amazon.awssdk.services.s3.model.S3Object;
+import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Publisher;
+
+public class S3AsyncCrossRegionIntegrationTest extends S3CrossRegionIntegrationTestBase {
+    private S3AsyncClient crossRegionS3Client ;
+
+    @BeforeEach
+    public void initialize() {
+        captureInterceptor = new CaptureInterceptor();
+        crossRegionS3Client = S3AsyncClient.builder()
+                                          .region(CROSS_REGION)
+                                           .serviceConfiguration(s -> s.crossRegionAccessEnabled(true))
+                                           .overrideConfiguration(o -> o.addExecutionInterceptor(captureInterceptor))
+                                           .build();
+    }
+
+    @AfterEach
+    void reset(){
+        captureInterceptor.reset();
+    }
+    @BeforeAll
+    static void setUpClass(){
+        // Bucket Created in CROSS Region
+        s3 = s3ClientBuilder().build();
+        createBucket(BUCKET);
+    }
+
+    @AfterAll
+    static void clearClass(){
+        deleteBucketAndAllContents(BUCKET);
+    }
+
+    private static final String BUCKET = temporaryBucketName(S3AsyncCrossRegionIntegrationTest.class);
+
+    @Override
+    protected List<S3Object> paginatedAPICall(ListObjectsV2Request listObjectsV2Request) {
+        List<S3Object> resultObjects = new ArrayList<>();
+        ListObjectsV2Publisher publisher = crossRegionS3Client.listObjectsV2Paginator(listObjectsV2Request);
+        CompletableFuture<Void> subscribe = publisher.subscribe(response -> {
+             response.contents().forEach(a -> resultObjects.add(a));
+        });
+        subscribe.join();
+        return resultObjects;
+    }
+
+    @Override
+    protected DeleteObjectsResponse postObjectAPICall(DeleteObjectsRequest deleteObjectsRequest) {
+        return crossRegionS3Client.deleteObjects(deleteObjectsRequest).join();
+    }
+
+    @Override
+    protected HeadBucketResponse headAPICall(HeadBucketRequest headBucketRequest) {
+        return crossRegionS3Client.headBucket(headBucketRequest).join();
+    }
+
+    @Override
+    protected DeleteObjectResponse deleteObjectAPICall(DeleteObjectRequest deleteObjectRequest) {
+        return crossRegionS3Client.deleteObject(deleteObjectRequest).join();
+    }
+
+    @Override
+    protected PutObjectResponse putAPICall(PutObjectRequest putObjectRequest, String testString) {
+        return crossRegionS3Client.putObject(putObjectRequest, AsyncRequestBody.fromString(testString)).join();
+    }
+
+    @Override
+    protected ResponseBytes<GetObjectResponse> getAPICall(GetObjectRequest getObjectRequest) {
+        return crossRegionS3Client.getObject(getObjectRequest, AsyncResponseTransformer.toBytes()).join();
+    }
+
+    @Override
+    protected String bucketName() {
+        return BUCKET;
+    }
+
+}

services/s3/src/it/java/software/amazon/awssdk/services/s3/crossregion/S3CrossRegionIntegrationTestBase.java

@@ -0,0 +1,179 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.services.s3.crossregion;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import org.junit.jupiter.api.Test;
+import software.amazon.awssdk.core.ResponseBytes;
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.http.SdkHttpMethod;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3IntegrationTestBase;
+import software.amazon.awssdk.services.s3.model.ChecksumAlgorithm;
+import software.amazon.awssdk.services.s3.model.ChecksumMode;
+import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
+import software.amazon.awssdk.services.s3.model.DeleteObjectResponse;
+import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest;
+import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectResponse;
+import software.amazon.awssdk.services.s3.model.HeadBucketRequest;
+import software.amazon.awssdk.services.s3.model.HeadBucketResponse;
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+import software.amazon.awssdk.services.s3.model.PutObjectResponse;
+import software.amazon.awssdk.services.s3.model.S3Object;
+
+public abstract class S3CrossRegionIntegrationTestBase extends S3IntegrationTestBase {
+
+    public static final String X_AMZ_BUCKET_REGION = "x-amz-bucket-region";
+
+    protected static final Region CROSS_REGION = Region.of("eu-central-1");
+
+    private static final String KEY = "key";
+
+    protected CaptureInterceptor captureInterceptor;
+
+    @Test
+    void getApi_CrossRegionCall() {
+        s3.putObject(p -> p.bucket(bucketName()).checksumAlgorithm(ChecksumAlgorithm.CRC32).key(KEY), RequestBody.fromString(
+            "TEST_STRING"));
+        GetObjectRequest getObjectRequest =
+            GetObjectRequest.builder().bucket(bucketName()).checksumMode(ChecksumMode.ENABLED).key(KEY).build();
+        ResponseBytes<GetObjectResponse> response = getAPICall(getObjectRequest);
+        assertThat(regionsIntercepted(captureInterceptor.hosts())).isEqualTo(Arrays.asList(CROSS_REGION.id(),
+                                                                                           DEFAULT_REGION.id()));
+        assertThat(captureInterceptor.httpMethods()).isEqualTo(Arrays.asList(SdkHttpMethod.GET, SdkHttpMethod.GET));
+        assertThat(new String(response.asByteArray())).isEqualTo("TEST_STRING");
+        assertThat(captureInterceptor.getServiceCalls()).isEqualTo(2);
+    }
+
+    @Test
+    void putApi_CrossRegionCall() {
+        s3.putObject(p -> p.bucket(bucketName()).checksumAlgorithm(ChecksumAlgorithm.CRC32).key(KEY), RequestBody.fromString(
+            "TEST_STRING"));
+        PutObjectRequest putObjectRequest =
+            PutObjectRequest.builder().bucket(bucketName()).checksumAlgorithm(ChecksumAlgorithm.CRC32).key(KEY).build();
+        PutObjectResponse response = putAPICall(putObjectRequest, "TEST_STRING");
+        assertThat(regionsIntercepted(captureInterceptor.hosts())).isEqualTo(Arrays.asList(CROSS_REGION.id(),
+                                                                                           DEFAULT_REGION.id()));
+        assertThat(captureInterceptor.httpMethods()).isEqualTo(Arrays.asList(SdkHttpMethod.PUT, SdkHttpMethod.PUT));
+        assertThat(response.checksumCRC32()).isEqualTo("S9ke8w==");
+        assertThat(captureInterceptor.getServiceCalls()).isEqualTo(2);
+    }
+
+    @Test
+    void deleteApi_CrossRegionCall() {
+        s3.putObject(p -> p.bucket(bucketName()).checksumAlgorithm(ChecksumAlgorithm.CRC32).key(KEY), RequestBody.fromString(
+            "TEST_STRING"));
+        DeleteObjectRequest deleteObjectRequest = DeleteObjectRequest.builder().bucket(bucketName()).key(KEY).build();
+        DeleteObjectResponse response = deleteObjectAPICall(deleteObjectRequest);
+        assertThat(regionsIntercepted(captureInterceptor.hosts())).isEqualTo(Arrays.asList(CROSS_REGION.id(),
+                                                                                           DEFAULT_REGION.id()));
+        assertThat(captureInterceptor.httpMethods()).isEqualTo(Arrays.asList(SdkHttpMethod.DELETE, SdkHttpMethod.DELETE));
+        assertThat(response).isNotNull();
+        assertThat(captureInterceptor.getServiceCalls()).isEqualTo(2);
+    }
+
+    @Test
+    void postApi_CrossRegionCall() {
+        s3.putObject(p -> p.bucket(bucketName()).checksumAlgorithm(ChecksumAlgorithm.CRC32).key(KEY), RequestBody.fromString(
+            "TEST_STRING"));
+        s3.putObject(p -> p.bucket(bucketName()).checksumAlgorithm(ChecksumAlgorithm.CRC32).key(KEY + "_1"),
+                     RequestBody.fromString("TEST_STRING"));
+        DeleteObjectsRequest deleteObjectsRequest =
+            DeleteObjectsRequest.builder().bucket(bucketName()).delete(d -> d.objects(o -> o.key(KEY), o -> o.key(KEY + "_1"))).build();
+        DeleteObjectsResponse response = postObjectAPICall(deleteObjectsRequest);
+        assertThat(regionsIntercepted(captureInterceptor.hosts())).isEqualTo(Arrays.asList(CROSS_REGION.id(),
+                                                                                           DEFAULT_REGION.id()));
+        assertThat(captureInterceptor.httpMethods()).isEqualTo(Arrays.asList(SdkHttpMethod.POST, SdkHttpMethod.POST));
+        assertThat(response).isNotNull();
+        assertThat(captureInterceptor.getServiceCalls()).isEqualTo(2);
+    }
+
+    @Test
+    void cachedRegionGetsUsed_when_CrossRegionCall() {
+        putAPICall(PutObjectRequest.builder().bucket(bucketName()).checksumAlgorithm(ChecksumAlgorithm.CRC32).key(KEY).build(),
+                   "TEST_STRING");
+        getAPICall(GetObjectRequest.builder().bucket(bucketName()).checksumMode(ChecksumMode.ENABLED).key(KEY).build());
+        assertThat(regionsIntercepted(captureInterceptor.hosts())).isEqualTo(Arrays.asList(CROSS_REGION.id(),
+                                                                                           DEFAULT_REGION.id(),
+                                                                                           DEFAULT_REGION.id()));
+        assertThat(captureInterceptor.httpMethods()).isEqualTo(Arrays.asList(SdkHttpMethod.PUT,
+                                                                             SdkHttpMethod.PUT,
+                                                                             SdkHttpMethod.GET));
+        assertThat(captureInterceptor.getServiceCalls()).isEqualTo(3);
+    }
+
+    @Test
+    void paginatedApi_CrossRegionCall() {
+        s3.deleteObject(p -> p.bucket(bucketName()).key(KEY));
+        int maxKeys = 3;
+        int totalKeys = maxKeys * 2 ;
+        IntStream.range(0, totalKeys )
+                 .forEach(
+                     i ->
+                         s3.putObject(p -> p.bucket(bucketName()).checksumAlgorithm(ChecksumAlgorithm.CRC32).key(KEY + "_" + i),
+                                      RequestBody.fromString("TEST_STRING"))
+                 );
+        ListObjectsV2Request listObjectsV2Request = ListObjectsV2Request.builder().bucket(bucketName()).maxKeys(maxKeys).build();
+        List<S3Object> s3ObjectList = paginatedAPICall(listObjectsV2Request);
+        assertThat(regionsIntercepted(captureInterceptor.hosts())).isEqualTo(Arrays.asList(CROSS_REGION.id(),
+                                                                                           DEFAULT_REGION.id(),
+                                                                                           DEFAULT_REGION.id()));
+
+        IntStream.range(0, totalKeys ).forEach(i -> s3.deleteObject(p -> p.bucket(bucketName()).key(KEY + "_" + i)));
+    }
+
+    @Test
+    void headApi_CrossRegionCall() {
+        s3.putObject(p -> p.bucket(bucketName()).checksumAlgorithm(ChecksumAlgorithm.CRC32).key(KEY), RequestBody.fromString(
+            "TEST_STRING"));
+        HeadBucketRequest headBucketRequest = HeadBucketRequest.builder().bucket(bucketName()).build();
+        HeadBucketResponse response = headAPICall(headBucketRequest);
+        assertThat(regionsIntercepted(captureInterceptor.hosts())).isEqualTo(Arrays.asList(CROSS_REGION.id(),
+                                                                                           DEFAULT_REGION.id()));
+        assertThat(captureInterceptor.httpMethods()).isEqualTo(Arrays.asList(SdkHttpMethod.HEAD, SdkHttpMethod.HEAD));
+        assertThat(response).isNotNull();
+        assertThat(captureInterceptor.getServiceCalls()).isEqualTo(2);
+    }
+
+    protected abstract List<S3Object> paginatedAPICall(ListObjectsV2Request listObjectsV2Request);
+
+    protected abstract DeleteObjectsResponse postObjectAPICall(DeleteObjectsRequest deleteObjectsRequest);
+
+    protected abstract HeadBucketResponse headAPICall(HeadBucketRequest headBucketRequest);
+
+    protected abstract DeleteObjectResponse deleteObjectAPICall(DeleteObjectRequest deleteObjectRequest);
+
+    protected abstract PutObjectResponse putAPICall(PutObjectRequest putObjectRequest, String testString);
+
+    protected abstract ResponseBytes<GetObjectResponse> getAPICall(GetObjectRequest getObjectRequest);
+
+    protected abstract String bucketName();
+
+    private List<String> regionsIntercepted(List<String> hosts) {
+        return hosts.stream()
+                    .map(req -> req.substring(bucketName().length() + 4, req.length() - 14))
+                    .collect(Collectors.toList());
+    }
+
+}