diff --git a/codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/CustomizationConfig.java b/codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/CustomizationConfig.java index 596d44bcf14..0bef67df786 100644 --- a/codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/CustomizationConfig.java +++ b/codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/CustomizationConfig.java @@ -227,6 +227,11 @@ public class CustomizationConfig { */ private String asyncClientDecorator; + /** + * Only for s3. A set of customization to related to multipart operations. + */ + private MultipartCustomization multipartCustomization; + /** * Whether to skip generating endpoint tests from endpoint-tests.json */ @@ -665,4 +670,12 @@ public Map getCustomClientContextParams() { public void setCustomClientContextParams(Map customClientContextParams) { this.customClientContextParams = customClientContextParams; } + + public MultipartCustomization getMultipartCustomization() { + return this.multipartCustomization; + } + + public void setMultipartCustomization(MultipartCustomization multipartCustomization) { + this.multipartCustomization = multipartCustomization; + } } diff --git a/codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/MultipartCustomization.java b/codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/MultipartCustomization.java new file mode 100644 index 00000000000..6f87cc50ce6 --- /dev/null +++ b/codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/MultipartCustomization.java @@ -0,0 +1,46 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.codegen.model.config.customization; + +public class MultipartCustomization { + private String multipartConfigurationClass; + private String multipartConfigMethodDoc; + private String multipartEnableMethodDoc; + + public String getMultipartConfigurationClass() { + return multipartConfigurationClass; + } + + public void setMultipartConfigurationClass(String multipartConfigurationClass) { + this.multipartConfigurationClass = multipartConfigurationClass; + } + + public String getMultipartConfigMethodDoc() { + return multipartConfigMethodDoc; + } + + public void setMultipartConfigMethodDoc(String multipartMethodDoc) { + this.multipartConfigMethodDoc = multipartMethodDoc; + } + + public String getMultipartEnableMethodDoc() { + return multipartEnableMethodDoc; + } + + public void setMultipartEnableMethodDoc(String multipartEnableMethodDoc) { + this.multipartEnableMethodDoc = multipartEnableMethodDoc; + } +} diff --git a/codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/ServiceConfig.java b/codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/ServiceConfig.java index b0c0db862df..3dc4b6291b5 100644 --- a/codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/ServiceConfig.java +++ b/codegen/src/main/java/software/amazon/awssdk/codegen/model/config/customization/ServiceConfig.java @@ -15,6 +15,8 @@ package software.amazon.awssdk.codegen.model.config.customization; +import software.amazon.awssdk.utils.ToString; + public class ServiceConfig { /** * Specifies the name of the client configuration class to use if a service @@ -112,4 +114,18 @@ public boolean hasAccelerateModeEnabledProperty() { public void setHasAccelerateModeEnabledProperty(boolean hasAccelerateModeEnabledProperty) { this.hasAccelerateModeEnabledProperty = hasAccelerateModeEnabledProperty; } + + @Override + public String toString() { + return ToString.builder("ServiceConfig") + .add("className", className) + .add("hasDualstackProperty", hasDualstackProperty) + .add("hasFipsProperty", hasFipsProperty) + .add("hasUseArnRegionProperty", hasUseArnRegionProperty) + .add("hasMultiRegionEnabledProperty", hasMultiRegionEnabledProperty) + .add("hasPathStyleAccessEnabledProperty", hasPathStyleAccessEnabledProperty) + .add("hasAccelerateModeEnabledProperty", hasAccelerateModeEnabledProperty) + .add("hasCrossRegionAccessEnabledProperty", hasCrossRegionAccessEnabledProperty) + .build(); + } } diff --git a/codegen/src/main/java/software/amazon/awssdk/codegen/poet/ClassSpec.java b/codegen/src/main/java/software/amazon/awssdk/codegen/poet/ClassSpec.java index a8265f0dc7f..59a719fb2c7 100644 --- a/codegen/src/main/java/software/amazon/awssdk/codegen/poet/ClassSpec.java +++ b/codegen/src/main/java/software/amazon/awssdk/codegen/poet/ClassSpec.java @@ -20,7 +20,7 @@ import java.util.Collections; /** - * Represents the a Poet generated class + * Represents a Poet generated class */ public interface ClassSpec { diff --git a/codegen/src/main/java/software/amazon/awssdk/codegen/poet/builder/AsyncClientBuilderClass.java b/codegen/src/main/java/software/amazon/awssdk/codegen/poet/builder/AsyncClientBuilderClass.java index 509a30c6c8d..efe81f2cb9a 100644 --- a/codegen/src/main/java/software/amazon/awssdk/codegen/poet/builder/AsyncClientBuilderClass.java +++ b/codegen/src/main/java/software/amazon/awssdk/codegen/poet/builder/AsyncClientBuilderClass.java @@ -17,6 +17,7 @@ import com.squareup.javapoet.ClassName; import com.squareup.javapoet.MethodSpec; +import com.squareup.javapoet.ParameterSpec; import com.squareup.javapoet.ParameterizedTypeName; import com.squareup.javapoet.TypeSpec; import java.net.URI; @@ -24,6 +25,7 @@ import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.auth.token.credentials.SdkTokenProvider; import software.amazon.awssdk.awscore.client.config.AwsClientOption; +import software.amazon.awssdk.codegen.model.config.customization.MultipartCustomization; import software.amazon.awssdk.codegen.model.intermediate.IntermediateModel; import software.amazon.awssdk.codegen.poet.ClassSpec; import software.amazon.awssdk.codegen.poet.PoetExtension; @@ -59,12 +61,12 @@ public AsyncClientBuilderClass(IntermediateModel model) { @Override public TypeSpec poetSpec() { TypeSpec.Builder builder = - PoetUtils.createClassBuilder(builderClassName) - .addAnnotation(SdkInternalApi.class) - .addModifiers(Modifier.FINAL) - .superclass(ParameterizedTypeName.get(builderBaseClassName, builderInterfaceName, clientInterfaceName)) - .addSuperinterface(builderInterfaceName) - .addJavadoc("Internal implementation of {@link $T}.", builderInterfaceName); + PoetUtils.createClassBuilder(builderClassName) + .addAnnotation(SdkInternalApi.class) + .addModifiers(Modifier.FINAL) + .superclass(ParameterizedTypeName.get(builderBaseClassName, builderInterfaceName, clientInterfaceName)) + .addSuperinterface(builderInterfaceName) + .addJavadoc("Internal implementation of {@link $T}.", builderInterfaceName); if (model.getEndpointOperation().isPresent()) { builder.addMethod(endpointDiscoveryEnabled()); @@ -80,6 +82,12 @@ public TypeSpec poetSpec() { builder.addMethod(bearerTokenProviderMethod()); } + MultipartCustomization multipartCustomization = model.getCustomizationConfig().getMultipartCustomization(); + if (multipartCustomization != null) { + builder.addMethod(multipartEnabledMethod(multipartCustomization)); + builder.addMethod(multipartConfigMethods(multipartCustomization)); + } + builder.addMethod(buildClientMethod()); builder.addMethod(initializeServiceClientConfigMethod()); @@ -124,15 +132,15 @@ private MethodSpec endpointProviderMethod() { private MethodSpec buildClientMethod() { MethodSpec.Builder builder = MethodSpec.methodBuilder("buildClient") - .addAnnotation(Override.class) - .addModifiers(Modifier.PROTECTED, Modifier.FINAL) - .returns(clientInterfaceName) - .addStatement("$T clientConfiguration = super.asyncClientConfiguration()", - SdkClientConfiguration.class).addStatement("this.validateClientOptions" - + "(clientConfiguration)") - .addStatement("$T serviceClientConfiguration = initializeServiceClientConfig" - + "(clientConfiguration)", - serviceConfigClassName); + .addAnnotation(Override.class) + .addModifiers(Modifier.PROTECTED, Modifier.FINAL) + .returns(clientInterfaceName) + .addStatement("$T clientConfiguration = super.asyncClientConfiguration()", + SdkClientConfiguration.class) + .addStatement("this.validateClientOptions(clientConfiguration)") + .addStatement("$T serviceClientConfiguration = initializeServiceClientConfig" + + "(clientConfiguration)", + serviceConfigClassName); builder.addStatement("$1T client = new $2T(serviceClientConfiguration, clientConfiguration)", clientInterfaceName, clientClassName); @@ -156,6 +164,34 @@ private MethodSpec bearerTokenProviderMethod() { .build(); } + private MethodSpec multipartEnabledMethod(MultipartCustomization multipartCustomization) { + ClassName mulitpartConfigClassName = + PoetUtils.classNameFromFqcn(multipartCustomization.getMultipartConfigurationClass()); + return MethodSpec.methodBuilder("multipartEnabled") + .addAnnotation(Override.class) + .addModifiers(Modifier.PUBLIC) + .returns(builderInterfaceName) + .addParameter(Boolean.class, "enabled") + .addStatement("clientContextParams.put($T.MULTIPART_ENABLED_KEY, enabled)", + mulitpartConfigClassName) + .addStatement("return this") + .build(); + } + + private MethodSpec multipartConfigMethods(MultipartCustomization multipartCustomization) { + ClassName mulitpartConfigClassName = + PoetUtils.classNameFromFqcn(multipartCustomization.getMultipartConfigurationClass()); + return MethodSpec.methodBuilder("multipartConfiguration") + .addAnnotation(Override.class) + .addModifiers(Modifier.PUBLIC) + .addParameter(ParameterSpec.builder(mulitpartConfigClassName, "multipartConfig").build()) + .returns(builderInterfaceName) + .addStatement("clientContextParams.put($T.MULTIPART_CONFIGURATION_KEY, multipartConfig)", + mulitpartConfigClassName) + .addStatement("return this") + .build(); + } + private MethodSpec initializeServiceClientConfigMethod() { return MethodSpec.methodBuilder("initializeServiceClientConfig").addModifiers(Modifier.PRIVATE) .addParameter(SdkClientConfiguration.class, "clientConfig") diff --git a/codegen/src/main/java/software/amazon/awssdk/codegen/poet/builder/AsyncClientBuilderInterface.java b/codegen/src/main/java/software/amazon/awssdk/codegen/poet/builder/AsyncClientBuilderInterface.java index 5348972b5df..df62f97ae7c 100644 --- a/codegen/src/main/java/software/amazon/awssdk/codegen/poet/builder/AsyncClientBuilderInterface.java +++ b/codegen/src/main/java/software/amazon/awssdk/codegen/poet/builder/AsyncClientBuilderInterface.java @@ -17,34 +17,97 @@ import com.squareup.javapoet.ClassName; import com.squareup.javapoet.CodeBlock; +import com.squareup.javapoet.MethodSpec; +import com.squareup.javapoet.ParameterSpec; import com.squareup.javapoet.ParameterizedTypeName; import com.squareup.javapoet.TypeSpec; +import java.util.function.Consumer; +import javax.lang.model.element.Modifier; import software.amazon.awssdk.awscore.client.builder.AwsAsyncClientBuilder; +import software.amazon.awssdk.codegen.model.config.customization.MultipartCustomization; import software.amazon.awssdk.codegen.model.intermediate.IntermediateModel; import software.amazon.awssdk.codegen.poet.ClassSpec; import software.amazon.awssdk.codegen.poet.PoetUtils; +import software.amazon.awssdk.utils.Logger; +import software.amazon.awssdk.utils.Validate; public class AsyncClientBuilderInterface implements ClassSpec { + private static final Logger log = Logger.loggerFor(AsyncClientBuilderInterface.class); + private final ClassName builderInterfaceName; private final ClassName clientInterfaceName; private final ClassName baseBuilderInterfaceName; + private final IntermediateModel model; public AsyncClientBuilderInterface(IntermediateModel model) { String basePackage = model.getMetadata().getFullClientPackageName(); this.clientInterfaceName = ClassName.get(basePackage, model.getMetadata().getAsyncInterface()); this.builderInterfaceName = ClassName.get(basePackage, model.getMetadata().getAsyncBuilderInterface()); this.baseBuilderInterfaceName = ClassName.get(basePackage, model.getMetadata().getBaseBuilderInterface()); + this.model = model; } @Override public TypeSpec poetSpec() { - return PoetUtils.createInterfaceBuilder(builderInterfaceName) - .addSuperinterface(ParameterizedTypeName.get(ClassName.get(AwsAsyncClientBuilder.class), - builderInterfaceName, clientInterfaceName)) - .addSuperinterface(ParameterizedTypeName.get(baseBuilderInterfaceName, - builderInterfaceName, clientInterfaceName)) - .addJavadoc(getJavadoc()) - .build(); + TypeSpec.Builder builder = PoetUtils + .createInterfaceBuilder(builderInterfaceName) + .addSuperinterface(ParameterizedTypeName.get(ClassName.get(AwsAsyncClientBuilder.class), + builderInterfaceName, clientInterfaceName)) + .addSuperinterface(ParameterizedTypeName.get(baseBuilderInterfaceName, + builderInterfaceName, clientInterfaceName)) + .addJavadoc(getJavadoc()); + + MultipartCustomization multipartCustomization = model.getCustomizationConfig().getMultipartCustomization(); + if (multipartCustomization != null) { + includeMultipartMethod(builder, multipartCustomization); + } + return builder.build(); + } + + private void includeMultipartMethod(TypeSpec.Builder builder, MultipartCustomization multipartCustomization) { + log.debug(() -> String.format("Adding multipart config methods to builder interface for service '%s'", + model.getMetadata().getServiceId())); + + // .multipartEnabled(Boolean) + builder.addMethod( + MethodSpec.methodBuilder("multipartEnabled") + .addModifiers(Modifier.DEFAULT, Modifier.PUBLIC) + .returns(builderInterfaceName) + .addParameter(Boolean.class, "enabled") + .addCode("throw new $T();", UnsupportedOperationException.class) + .addJavadoc(CodeBlock.of(multipartCustomization.getMultipartEnableMethodDoc())) + .build()); + + // .multipartConfiguration(MultipartConfiguration) + String multiPartConfigMethodName = "multipartConfiguration"; + String multipartConfigClass = Validate.notNull(multipartCustomization.getMultipartConfigurationClass(), + "'multipartConfigurationClass' must be defined"); + ClassName mulitpartConfigClassName = PoetUtils.classNameFromFqcn(multipartConfigClass); + builder.addMethod( + MethodSpec.methodBuilder(multiPartConfigMethodName) + .addModifiers(Modifier.DEFAULT, Modifier.PUBLIC) + .returns(builderInterfaceName) + .addParameter(ParameterSpec.builder(mulitpartConfigClassName, "multipartConfiguration").build()) + .addCode("throw new $T();", UnsupportedOperationException.class) + .addJavadoc(CodeBlock.of(multipartCustomization.getMultipartConfigMethodDoc())) + .build()); + + // .multipartConfiguration(Consumer) + ClassName mulitpartConfigBuilderClassName = PoetUtils.classNameFromFqcn(multipartConfigClass + ".Builder"); + ParameterizedTypeName consumerBuilderType = ParameterizedTypeName.get(ClassName.get(Consumer.class), + mulitpartConfigBuilderClassName); + builder.addMethod( + MethodSpec.methodBuilder(multiPartConfigMethodName) + .addModifiers(Modifier.DEFAULT, Modifier.PUBLIC) + .returns(builderInterfaceName) + .addParameter(ParameterSpec.builder(consumerBuilderType, "multipartConfiguration").build()) + .addStatement("$T builder = $T.builder()", + mulitpartConfigBuilderClassName, + mulitpartConfigClassName) + .addStatement("multipartConfiguration.accept(builder)") + .addStatement("return multipartConfiguration(builder.build())") + .addJavadoc(CodeBlock.of(multipartCustomization.getMultipartConfigMethodDoc())) + .build()); } @Override diff --git a/services/s3/src/it/java/software/amazon/awssdk/services/s3/S3IntegrationTestBase.java b/services/s3/src/it/java/software/amazon/awssdk/services/s3/S3IntegrationTestBase.java index 63dcf2ddc88..03cf42afe5d 100644 --- a/services/s3/src/it/java/software/amazon/awssdk/services/s3/S3IntegrationTestBase.java +++ b/services/s3/src/it/java/software/amazon/awssdk/services/s3/S3IntegrationTestBase.java @@ -117,7 +117,7 @@ protected static void deleteBucketAndAllContents(String bucketName) { S3TestUtils.deleteBucketAndAllContents(s3, bucketName); } - private static class UserAgentVerifyingExecutionInterceptor implements ExecutionInterceptor { + protected static class UserAgentVerifyingExecutionInterceptor implements ExecutionInterceptor { private final String clientName; private final ClientType clientType; diff --git a/services/s3/src/it/java/software/amazon/awssdk/services/s3/crt/S3CrossRegionCrtIntegrationTest.java b/services/s3/src/it/java/software/amazon/awssdk/services/s3/crt/S3CrossRegionCrtIntegrationTest.java index 72c6fce095c..45374b26529 100644 --- a/services/s3/src/it/java/software/amazon/awssdk/services/s3/crt/S3CrossRegionCrtIntegrationTest.java +++ b/services/s3/src/it/java/software/amazon/awssdk/services/s3/crt/S3CrossRegionCrtIntegrationTest.java @@ -31,7 +31,6 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ThreadLocalRandom; -import org.assertj.core.api.Assertions; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; diff --git a/services/s3/src/it/java/software/amazon/awssdk/services/s3/multipart/MultipartClientUserAgentTest.java b/services/s3/src/it/java/software/amazon/awssdk/services/s3/multipart/MultipartClientUserAgentTest.java new file mode 100644 index 00000000000..a76796e02fa --- /dev/null +++ b/services/s3/src/it/java/software/amazon/awssdk/services/s3/multipart/MultipartClientUserAgentTest.java @@ -0,0 +1,78 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.multipart; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.net.URI; +import java.util.ArrayList; +import java.util.List; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.core.ApiName; +import software.amazon.awssdk.core.async.AsyncRequestBody; +import software.amazon.awssdk.core.interceptor.Context; +import software.amazon.awssdk.core.interceptor.ExecutionAttributes; +import software.amazon.awssdk.core.interceptor.ExecutionInterceptor; +import software.amazon.awssdk.http.HttpExecuteResponse; +import software.amazon.awssdk.http.SdkHttpResponse; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.testutils.service.http.MockAsyncHttpClient; + +class MultipartClientUserAgentTest { + + private MockAsyncHttpClient mockAsyncHttpClient; + private UserAgentInterceptor userAgentInterceptor; + private S3AsyncClient s3Client; + + @BeforeEach + void init() { + this.mockAsyncHttpClient = new MockAsyncHttpClient(); + this.userAgentInterceptor = new UserAgentInterceptor(); + s3Client = S3AsyncClient.builder() + .httpClient(mockAsyncHttpClient) + .endpointOverride(URI.create("http://localhost")) + .overrideConfiguration(c -> c.addExecutionInterceptor(userAgentInterceptor)) + .multipartEnabled(true) + .multipartConfiguration(c -> c.minimumPartSizeInBytes(1024L).thresholdInBytes(1024L)) + .region(Region.US_EAST_1) + .build(); + } + + @Test + void validateUserAgent_put_oneChunk() throws Exception { + HttpExecuteResponse response = HttpExecuteResponse.builder() + .response(SdkHttpResponse.builder().statusCode(200).build()) + .build(); + mockAsyncHttpClient.stubResponses(response); + + s3Client.putObject(req -> req.key("key").bucket("bucket"), AsyncRequestBody.fromString("12345678")).get(); + + assertThat(userAgentInterceptor.apiNames).isNotNull(); + assertThat(userAgentInterceptor.apiNames) + .anyMatch(api -> "hll".equals(api.name()) && "s3Multipart".equals(api.version())); + } + + private static final class UserAgentInterceptor implements ExecutionInterceptor { + private final List apiNames = new ArrayList<>(); + + @Override + public void beforeTransmission(Context.BeforeTransmission context, ExecutionAttributes executionAttributes) { + context.request().overrideConfiguration().ifPresent(c -> apiNames.addAll(c.apiNames())); + } + } +} diff --git a/services/s3/src/it/java/software/amazon/awssdk/services/s3/multipart/S3ClientMultiPartCopyIntegrationTest.java b/services/s3/src/it/java/software/amazon/awssdk/services/s3/multipart/S3ClientMultiPartCopyIntegrationTest.java index 6db434526fb..fc4f31b76b1 100644 --- a/services/s3/src/it/java/software/amazon/awssdk/services/s3/multipart/S3ClientMultiPartCopyIntegrationTest.java +++ b/services/s3/src/it/java/software/amazon/awssdk/services/s3/multipart/S3ClientMultiPartCopyIntegrationTest.java @@ -31,17 +31,16 @@ import javax.crypto.KeyGenerator; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Timeout; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; +import software.amazon.awssdk.core.ClientType; import software.amazon.awssdk.core.ResponseBytes; import software.amazon.awssdk.core.async.AsyncRequestBody; import software.amazon.awssdk.core.sync.ResponseTransformer; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.S3IntegrationTestBase; import software.amazon.awssdk.services.s3.internal.crt.S3CrtAsyncClient; -import software.amazon.awssdk.services.s3.internal.multipart.MultipartS3AsyncClient; import software.amazon.awssdk.services.s3.model.CopyObjectResponse; import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.services.s3.model.MetadataDirective; @@ -58,6 +57,7 @@ public class S3ClientMultiPartCopyIntegrationTest extends S3IntegrationTestBase private static final long SMALL_OBJ_SIZE = 1024 * 1024; private static S3AsyncClient s3CrtAsyncClient; private static S3AsyncClient s3MpuClient; + @BeforeAll public static void setUp() throws Exception { S3IntegrationTestBase.setUp(); @@ -66,7 +66,13 @@ public static void setUp() throws Exception { .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) .region(DEFAULT_REGION) .build(); - s3MpuClient = new MultipartS3AsyncClient(s3Async); + s3MpuClient = S3AsyncClient.builder() + .region(DEFAULT_REGION) + .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) + .overrideConfiguration(o -> o.addExecutionInterceptor( + new UserAgentVerifyingExecutionInterceptor("NettyNio", ClientType.ASYNC))) + .multipartEnabled(true) + .build(); } @AfterAll @@ -158,7 +164,7 @@ private static byte[] generateSecretKey() { private void createOriginalObject(byte[] originalContent, String originalKey) { s3CrtAsyncClient.putObject(r -> r.bucket(BUCKET) - .key(originalKey), + .key(originalKey), AsyncRequestBody.fromBytes(originalContent)).join(); } diff --git a/services/s3/src/it/java/software/amazon/awssdk/services/s3/multipart/S3MultipartClientBuilderTest.java b/services/s3/src/it/java/software/amazon/awssdk/services/s3/multipart/S3MultipartClientBuilderTest.java new file mode 100644 index 00000000000..c922aa51b2a --- /dev/null +++ b/services/s3/src/it/java/software/amazon/awssdk/services/s3/multipart/S3MultipartClientBuilderTest.java @@ -0,0 +1,63 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.multipart; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.internal.multipart.MultipartS3AsyncClient; + +class S3MultipartClientBuilderTest { + + @Test + void multipartEnabledWithConfig_shouldBuildMultipartClient() { + S3AsyncClient client = S3AsyncClient.builder() + .multipartEnabled(true) + .multipartConfiguration(MultipartConfiguration.builder().build()) + .region(Region.US_EAST_1) + .build(); + assertThat(client).isInstanceOf(MultipartS3AsyncClient.class); + } + + @Test + void multipartEnabledWithoutConfig_shouldBuildMultipartClient() { + S3AsyncClient client = S3AsyncClient.builder() + .multipartEnabled(true) + .region(Region.US_EAST_1) + .build(); + assertThat(client).isInstanceOf(MultipartS3AsyncClient.class); + } + + @Test + void multipartDisabledWithConfig_shouldNotBuildMultipartClient() { + S3AsyncClient client = S3AsyncClient.builder() + .multipartEnabled(false) + .multipartConfiguration(b -> b.maximumMemoryUsageInBytes(1024L)) + .region(Region.US_EAST_1) + .build(); + assertThat(client).isNotInstanceOf(MultipartS3AsyncClient.class); + } + + @Test + void noMultipart_shouldNotBeMultipartClient() { + S3AsyncClient client = S3AsyncClient.builder() + .region(Region.US_EAST_1) + .build(); + assertThat(client).isNotInstanceOf(MultipartS3AsyncClient.class); + } +} diff --git a/services/s3/src/it/java/software/amazon/awssdk/services/s3/multipart/S3MultipartClientPutObjectIntegrationTest.java b/services/s3/src/it/java/software/amazon/awssdk/services/s3/multipart/S3MultipartClientPutObjectIntegrationTest.java index f791b4b3c26..f9d8a72b41e 100644 --- a/services/s3/src/it/java/software/amazon/awssdk/services/s3/multipart/S3MultipartClientPutObjectIntegrationTest.java +++ b/services/s3/src/it/java/software/amazon/awssdk/services/s3/multipart/S3MultipartClientPutObjectIntegrationTest.java @@ -15,7 +15,6 @@ package software.amazon.awssdk.services.s3.multipart; -import static java.util.concurrent.TimeUnit.MINUTES; import static java.util.concurrent.TimeUnit.SECONDS; import static org.assertj.core.api.Assertions.assertThat; import static software.amazon.awssdk.testutils.service.S3BucketUtils.temporaryBucketName; @@ -25,22 +24,19 @@ import java.nio.charset.Charset; import java.nio.file.Files; import java.util.UUID; -import java.util.concurrent.ThreadLocalRandom; import org.apache.commons.lang3.RandomStringUtils; -import org.assertj.core.api.Assertions; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; +import software.amazon.awssdk.core.ClientType; import software.amazon.awssdk.core.ResponseInputStream; import software.amazon.awssdk.core.async.AsyncRequestBody; import software.amazon.awssdk.core.sync.ResponseTransformer; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.S3IntegrationTestBase; -import software.amazon.awssdk.services.s3.internal.multipart.MultipartS3AsyncClient; import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.services.s3.utils.ChecksumUtils; -import software.amazon.awssdk.testutils.RandomTempFile; public class S3MultipartClientPutObjectIntegrationTest extends S3IntegrationTestBase { @@ -60,7 +56,13 @@ public static void setup() throws Exception { testFile = File.createTempFile("SplittingPublisherTest", UUID.randomUUID().toString()); Files.write(testFile.toPath(), CONTENT); - mpuS3Client = new MultipartS3AsyncClient(s3Async); + mpuS3Client = S3AsyncClient.builder() + .region(DEFAULT_REGION) + .credentialsProvider(CREDENTIALS_PROVIDER_CHAIN) + .overrideConfiguration(o -> o.addExecutionInterceptor( + new UserAgentVerifyingExecutionInterceptor("NettyNio", ClientType.ASYNC))) + .multipartEnabled(true) + .build(); } @AfterAll @@ -71,7 +73,7 @@ public static void teardown() throws Exception { } @Test - @Timeout(value = 20, unit = SECONDS) + @Timeout(value = 60, unit = SECONDS) void putObject_fileRequestBody_objectSentCorrectly() throws Exception { AsyncRequestBody body = AsyncRequestBody.fromFile(testFile.toPath()); mpuS3Client.putObject(r -> r.bucket(TEST_BUCKET).key(TEST_KEY), body).join(); @@ -85,7 +87,7 @@ void putObject_fileRequestBody_objectSentCorrectly() throws Exception { } @Test - @Timeout(value = 30, unit = SECONDS) + @Timeout(value = 60, unit = SECONDS) void putObject_byteAsyncRequestBody_objectSentCorrectly() throws Exception { byte[] bytes = RandomStringUtils.randomAscii(OBJ_SIZE).getBytes(Charset.defaultCharset()); AsyncRequestBody body = AsyncRequestBody.fromBytes(bytes); diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/client/S3AsyncClientDecorator.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/client/S3AsyncClientDecorator.java index 2dbb61091da..89c6982bb83 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/client/S3AsyncClientDecorator.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/client/S3AsyncClientDecorator.java @@ -15,6 +15,9 @@ package software.amazon.awssdk.services.s3.internal.client; +import static software.amazon.awssdk.services.s3.multipart.MultipartConfiguration.MULTIPART_CONFIGURATION_KEY; +import static software.amazon.awssdk.services.s3.multipart.MultipartConfiguration.MULTIPART_ENABLED_KEY; + import java.util.ArrayList; import java.util.List; import java.util.function.Predicate; @@ -23,6 +26,8 @@ import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.endpoints.S3ClientContextParams; import software.amazon.awssdk.services.s3.internal.crossregion.S3CrossRegionAsyncClient; +import software.amazon.awssdk.services.s3.internal.multipart.MultipartS3AsyncClient; +import software.amazon.awssdk.services.s3.multipart.MultipartConfiguration; import software.amazon.awssdk.utils.AttributeMap; import software.amazon.awssdk.utils.ConditionalDecorator; @@ -36,14 +41,26 @@ public S3AsyncClient decorate(S3AsyncClient base, SdkClientConfiguration clientConfiguration, AttributeMap clientContextParams) { List> decorators = new ArrayList<>(); - decorators.add(ConditionalDecorator.create(isCrossRegionEnabledAsync(clientContextParams), - S3CrossRegionAsyncClient::new)); + decorators.add(ConditionalDecorator.create( + isCrossRegionEnabledAsync(clientContextParams), + S3CrossRegionAsyncClient::new)); + + decorators.add(ConditionalDecorator.create( + isMultipartEnable(clientContextParams), + client -> { + MultipartConfiguration multipartConfiguration = clientContextParams.get(MULTIPART_CONFIGURATION_KEY); + return new MultipartS3AsyncClient(client, multipartConfiguration); + })); return ConditionalDecorator.decorate(base, decorators); } private Predicate isCrossRegionEnabledAsync(AttributeMap clientContextParams) { Boolean crossRegionEnabled = clientContextParams.get(S3ClientContextParams.CROSS_REGION_ACCESS_ENABLED); - return client -> crossRegionEnabled != null && crossRegionEnabled.booleanValue(); + return client -> crossRegionEnabled != null && crossRegionEnabled.booleanValue(); } + private Predicate isMultipartEnable(AttributeMap clientContextParams) { + Boolean multipartEnabled = clientContextParams.get(MULTIPART_ENABLED_KEY); + return client -> multipartEnabled != null && multipartEnabled.booleanValue(); + } } diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/CopyObjectHelper.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/CopyObjectHelper.java index 31b947bb89c..46f9da4ac94 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/CopyObjectHelper.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/CopyObjectHelper.java @@ -16,6 +16,8 @@ package software.amazon.awssdk.services.s3.internal.multipart; +import static software.amazon.awssdk.services.s3.internal.multipart.MultipartS3AsyncClient.USER_AGENT_API_NAME; + import java.util.ArrayList; import java.util.List; import java.util.concurrent.CompletableFuture; @@ -23,9 +25,8 @@ import java.util.stream.IntStream; import software.amazon.awssdk.annotations.SdkInternalApi; import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.internal.UserAgentUtils; import software.amazon.awssdk.services.s3.internal.crt.UploadPartCopyRequestIterable; -import software.amazon.awssdk.services.s3.internal.multipart.GenericMultipartHelper; -import software.amazon.awssdk.services.s3.internal.multipart.SdkPojoConversionUtils; import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest; import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadResponse; import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload; @@ -68,7 +69,9 @@ public CompletableFuture copyObject(CopyObjectRequest copyOb try { CompletableFuture headFuture = - s3AsyncClient.headObject(SdkPojoConversionUtils.toHeadObjectRequest(copyObjectRequest)); + s3AsyncClient.headObject( + UserAgentUtils.applyUserAgentInfo(SdkPojoConversionUtils.toHeadObjectRequest(copyObjectRequest), + b -> b.addApiName(USER_AGENT_API_NAME))); // Ensure cancellations are forwarded to the head future CompletableFutureUtils.forwardExceptionTo(returnFuture, headFuture); @@ -106,6 +109,7 @@ private void copyInParts(CopyObjectRequest copyObjectRequest, CompletableFuture returnFuture) { CreateMultipartUploadRequest request = SdkPojoConversionUtils.toCreateMultipartUploadRequest(copyObjectRequest); + request = UserAgentUtils.applyUserAgentInfo(request, b -> b.addApiName(USER_AGENT_API_NAME)); CompletableFuture createMultipartUploadFuture = s3AsyncClient.createMultipartUpload(request); @@ -128,6 +132,10 @@ private void doCopyInParts(CopyObjectRequest copyObjectRequest, String uploadId) { long optimalPartSize = genericMultipartHelper.calculateOptimalPartSizeFor(contentLength, partSizeInBytes); + if (optimalPartSize > partSizeInBytes) { + log.info(() -> String.format("Configured partSize is %d, but using %d to prevent reaching maximum number of parts " + + "allowed", partSizeInBytes, optimalPartSize)); + } int partCount = genericMultipartHelper.determinePartCount(contentLength, optimalPartSize); @@ -171,7 +179,8 @@ private CompletableFuture completeMultipartUplo .build()) .build(); - return s3AsyncClient.completeMultipartUpload(completeMultipartUploadRequest); + return s3AsyncClient.completeMultipartUpload(UserAgentUtils.applyUserAgentInfo(completeMultipartUploadRequest, + b -> b.addApiName(USER_AGENT_API_NAME))); } private List> sendUploadPartCopyRequests(CopyObjectRequest copyObjectRequest, @@ -201,7 +210,8 @@ private void sendIndividualUploadPartCopy(String uploadId, log.debug(() -> "Sending uploadPartCopyRequest with range: " + uploadPartCopyRequest.copySourceRange() + " uploadId: " + uploadId); - CompletableFuture uploadPartCopyFuture = s3AsyncClient.uploadPartCopy(uploadPartCopyRequest); + CompletableFuture uploadPartCopyFuture = s3AsyncClient.uploadPartCopy( + UserAgentUtils.applyUserAgentInfo(uploadPartCopyRequest, b -> b.addApiName(USER_AGENT_API_NAME))); CompletableFuture convertFuture = uploadPartCopyFuture.thenApply(uploadPartCopyResponse -> @@ -225,8 +235,8 @@ private static CompletedPart convertUploadPartCopyResponse(AtomicReferenceArray< private void copyInOneChunk(CopyObjectRequest copyObjectRequest, CompletableFuture returnFuture) { - CompletableFuture copyObjectFuture = - s3AsyncClient.copyObject(copyObjectRequest); + CompletableFuture copyObjectFuture = s3AsyncClient.copyObject( + UserAgentUtils.applyUserAgentInfo(copyObjectRequest, b -> b.addApiName(USER_AGENT_API_NAME))); CompletableFutureUtils.forwardExceptionTo(returnFuture, copyObjectFuture); CompletableFutureUtils.forwardResultTo(copyObjectFuture, returnFuture); } diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClient.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClient.java index 869eb404814..4f96604fc15 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClient.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartS3AsyncClient.java @@ -15,33 +15,51 @@ package software.amazon.awssdk.services.s3.internal.multipart; - import java.util.concurrent.CompletableFuture; import software.amazon.awssdk.annotations.SdkInternalApi; +import software.amazon.awssdk.core.ApiName; import software.amazon.awssdk.core.async.AsyncRequestBody; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; import software.amazon.awssdk.services.s3.DelegatingS3AsyncClient; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.model.CopyObjectRequest; import software.amazon.awssdk.services.s3.model.CopyObjectResponse; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; import software.amazon.awssdk.services.s3.model.PutObjectRequest; import software.amazon.awssdk.services.s3.model.PutObjectResponse; +import software.amazon.awssdk.services.s3.multipart.MultipartConfiguration; +import software.amazon.awssdk.utils.Validate; -// This is just a temporary class for testing -//TODO: change this @SdkInternalApi -public class MultipartS3AsyncClient extends DelegatingS3AsyncClient { - private static final long DEFAULT_PART_SIZE_IN_BYTES = 8L * 1024 * 1024; +public final class MultipartS3AsyncClient extends DelegatingS3AsyncClient { + + public static final ApiName USER_AGENT_API_NAME = ApiName.builder().name("hll").version("s3Multipart").build(); + + private static final long DEFAULT_MIN_PART_SIZE_IN_BYTES = 8L * 1024 * 1024; private static final long DEFAULT_THRESHOLD = 8L * 1024 * 1024; + private static final long DEFAULT_MAX_MEMORY = DEFAULT_MIN_PART_SIZE_IN_BYTES * 2; - private static final long DEFAULT_MAX_MEMORY = DEFAULT_PART_SIZE_IN_BYTES * 2; private final MultipartUploadHelper mpuHelper; private final CopyObjectHelper copyObjectHelper; - public MultipartS3AsyncClient(S3AsyncClient delegate) { + public MultipartS3AsyncClient(S3AsyncClient delegate, MultipartConfiguration multipartConfiguration) { super(delegate); - // TODO: pass a config object to the upload helper instead - mpuHelper = new MultipartUploadHelper(delegate, DEFAULT_PART_SIZE_IN_BYTES, DEFAULT_THRESHOLD, DEFAULT_MAX_MEMORY); - copyObjectHelper = new CopyObjectHelper(delegate, DEFAULT_PART_SIZE_IN_BYTES, DEFAULT_THRESHOLD); + MultipartConfiguration validConfiguration = Validate.getOrDefault(multipartConfiguration, + MultipartConfiguration.builder()::build); + long minPartSizeInBytes = Validate.getOrDefault(validConfiguration.minimumPartSizeInBytes(), + () -> DEFAULT_MIN_PART_SIZE_IN_BYTES); + long threshold = Validate.getOrDefault(validConfiguration.thresholdInBytes(), + () -> DEFAULT_THRESHOLD); + long maximumMemoryUsageInBytes = Validate.getOrDefault(validConfiguration.maximumMemoryUsageInBytes(), + () -> computeMaxMemoryUsage(validConfiguration)); + this.mpuHelper = new MultipartUploadHelper(delegate, minPartSizeInBytes, threshold, maximumMemoryUsageInBytes); + this.copyObjectHelper = new CopyObjectHelper(delegate, minPartSizeInBytes, threshold); + } + + private long computeMaxMemoryUsage(MultipartConfiguration multipartConfiguration) { + return multipartConfiguration.minimumPartSizeInBytes() != null ? multipartConfiguration.minimumPartSizeInBytes() * 2 + : DEFAULT_MAX_MEMORY; } @Override @@ -54,6 +72,13 @@ public CompletableFuture copyObject(CopyObjectRequest copyOb return copyObjectHelper.copyObject(copyObjectRequest); } + @Override + public CompletableFuture getObject( + GetObjectRequest getObjectRequest, AsyncResponseTransformer asyncResponseTransformer) { + throw new UnsupportedOperationException( + "Multipart download is not yet supported. Instead use the CRT based S3 client for multipart download."); + } + @Override public void close() { delegate().close(); diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartUploadHelper.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartUploadHelper.java index a3aea4a9bdf..ee6b16443d9 100644 --- a/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartUploadHelper.java +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/internal/multipart/MultipartUploadHelper.java @@ -16,6 +16,7 @@ package software.amazon.awssdk.services.s3.internal.multipart; +import static software.amazon.awssdk.services.s3.internal.multipart.MultipartS3AsyncClient.USER_AGENT_API_NAME; import static software.amazon.awssdk.services.s3.internal.multipart.SdkPojoConversionUtils.toAbortMultipartUploadRequest; import java.util.Collection; @@ -27,6 +28,7 @@ import software.amazon.awssdk.core.async.AsyncRequestBody; import software.amazon.awssdk.core.async.SplitAsyncRequestBodyResponse; import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.internal.UserAgentUtils; import software.amazon.awssdk.services.s3.model.CompletedPart; import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse; @@ -97,6 +99,7 @@ private void uploadInParts(PutObjectRequest putObjectRequest, long contentLength CompletableFuture returnFuture) { CreateMultipartUploadRequest request = SdkPojoConversionUtils.toCreateMultipartUploadRequest(putObjectRequest); + request = UserAgentUtils.applyUserAgentInfo(request, b -> b.addApiName(USER_AGENT_API_NAME)); CompletableFuture createMultipartUploadFuture = s3AsyncClient.createMultipartUpload(request); @@ -120,6 +123,10 @@ private void doUploadInParts(Pair request, String uploadId) { long optimalPartSize = genericMultipartHelper.calculateOptimalPartSizeFor(contentLength, partSizeInBytes); + if (optimalPartSize > partSizeInBytes) { + log.info(() -> String.format("Configured partSize is %d, but using %d to prevent reaching maximum number of parts " + + "allowed", partSizeInBytes, optimalPartSize)); + } int partCount = genericMultipartHelper.determinePartCount(contentLength, optimalPartSize); log.debug(() -> String.format("Starting multipart upload with partCount: %d, optimalPartSize: %d", partCount, @@ -169,8 +176,6 @@ private CompletableFuture sendUploadPartRequests(MpuRequestContext mpuRequ CompletableFuture returnFuture, Collection> futures) { - - AsyncRequestBody asyncRequestBody = mpuRequestContext.request.right(); SplitAsyncRequestBodyResponse result = asyncRequestBody.split(mpuRequestContext.partSize, maxMemoryUsageInBytes); @@ -197,7 +202,8 @@ private void sendIndividualUploadPartRequest(String uploadId, Collection> futures, Pair requestPair, CompletableFuture sendUploadPartRequestsFuture) { - UploadPartRequest uploadPartRequest = requestPair.left(); + UploadPartRequest uploadPartRequest = UserAgentUtils.applyUserAgentInfo(requestPair.left(), + b -> b.addApiName(USER_AGENT_API_NAME)); Integer partNumber = uploadPartRequest.partNumber(); log.debug(() -> "Sending uploadPartRequest: " + uploadPartRequest.partNumber() + " uploadId: " + uploadId + " " + "contentLength " + requestPair.right().contentLength()); @@ -224,6 +230,7 @@ private static CompletedPart convertUploadPartResponse(AtomicReferenceArray returnFuture) { + putObjectRequest = UserAgentUtils.applyUserAgentInfo(putObjectRequest, b -> b.addApiName(USER_AGENT_API_NAME)); CompletableFuture putObjectResponseCompletableFuture = s3AsyncClient.putObject(putObjectRequest, asyncRequestBody); CompletableFutureUtils.forwardExceptionTo(returnFuture, putObjectResponseCompletableFuture); diff --git a/services/s3/src/main/java/software/amazon/awssdk/services/s3/multipart/MultipartConfiguration.java b/services/s3/src/main/java/software/amazon/awssdk/services/s3/multipart/MultipartConfiguration.java new file mode 100644 index 00000000000..64e85858be9 --- /dev/null +++ b/services/s3/src/main/java/software/amazon/awssdk/services/s3/multipart/MultipartConfiguration.java @@ -0,0 +1,206 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.services.s3.multipart; + +import java.util.function.Consumer; +import software.amazon.awssdk.annotations.SdkPublicApi; +import software.amazon.awssdk.core.async.AsyncRequestBody; +import software.amazon.awssdk.core.async.AsyncResponseTransformer; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.S3AsyncClientBuilder; +import software.amazon.awssdk.services.s3.model.CopyObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.utils.AttributeMap; +import software.amazon.awssdk.utils.builder.CopyableBuilder; +import software.amazon.awssdk.utils.builder.ToCopyableBuilder; + +/** + * Class that hold configuration properties related to multipart operation for a {@link S3AsyncClient}. Passing this class to the + * {@link S3AsyncClientBuilder#multipartConfiguration(MultipartConfiguration)} will enable automatic conversion of + * {@link S3AsyncClient#putObject(Consumer, AsyncRequestBody)}, {@link S3AsyncClient#copyObject(CopyObjectRequest)} to their + * respective multipart operation. + *

+ * Note: The multipart operation for {@link S3AsyncClient#getObject(GetObjectRequest, AsyncResponseTransformer)} is + * temporarily disabled and will result in throwing a {@link UnsupportedOperationException} if called when configured for + * multipart operation. + */ +@SdkPublicApi +public final class MultipartConfiguration implements ToCopyableBuilder { + public static final AttributeMap.Key MULTIPART_CONFIGURATION_KEY = + new AttributeMap.Key(MultipartConfiguration.class){}; + public static final AttributeMap.Key MULTIPART_ENABLED_KEY = + new AttributeMap.Key(Boolean.class){}; + + private final Long thresholdInBytes; + private final Long minimumPartSizeInBytes; + private final Long maximumMemoryUsageInBytes; + + private MultipartConfiguration(DefaultMultipartConfigBuilder builder) { + this.thresholdInBytes = builder.thresholdInBytes; + this.minimumPartSizeInBytes = builder.minimumPartSizeInBytes; + this.maximumMemoryUsageInBytes = builder.maximumMemoryUsageInBytes; + } + + public static Builder builder() { + return new DefaultMultipartConfigBuilder(); + } + + @Override + public Builder toBuilder() { + return builder() + .maximumMemoryUsageInBytes(maximumMemoryUsageInBytes) + .minimumPartSizeInBytes(minimumPartSizeInBytes) + .thresholdInBytes(thresholdInBytes); + } + + /** + * Indicates the value of the configured threshold, in bytes. Any request whose size is less than the configured value will + * not + * use multipart operation + * @return the value of the configured threshold. + */ + public Long thresholdInBytes() { + return this.thresholdInBytes; + } + + /** + * Indicated the size, in bytes, of each individual part of the part requests. The actual part size used might be bigger to + * conforms to + * the maximum + * number of parts allowed per multipart requests. + * @return the value of the configured part size. + */ + public Long minimumPartSizeInBytes() { + return this.minimumPartSizeInBytes; + } + + /** + * The maximum memory, in bytes, that the SDK will use to buffer requests content into memory. + * @return the value of the configured maximum memory usage. + */ + public Long maximumMemoryUsageInBytes() { + return this.maximumMemoryUsageInBytes; + } + + /** + * Builder for a {@link MultipartConfiguration}. + */ + public interface Builder extends CopyableBuilder { + + /** + * Configures the minimum number of bytes of the body of the request required for requests to be converted to their + * multipart equivalent. Only taken into account when converting {@code putObject} and {@code copyObject} requests. + * Any request whose size is less than the configured value will not use multipart operation, + * even if multipart is enabled via {@link S3AsyncClientBuilder#multipartEnabled(Boolean)}. + *

+ * + * Default value: 8 Mib + * + * @param thresholdInBytes the value of the threshold to set. + * @return an instance of this builder. + */ + Builder thresholdInBytes(Long thresholdInBytes); + + /** + * Indicates the value of the configured threshold. + * @return the value of the threshold. + */ + Long thresholdInBytes(); + + /** + * Configures the part size, in bytes, to be used in each individual part requests. + *

+ * When uploading large payload, the size of the payload of each individual part requests might actually be + * bigger than + * the configured value since there is a limit to the maximum number of parts possible per multipart request. If the + * configured part size would lead to a number of parts higher than the maximum allowed, a larger part size will be + * calculated instead to allow fewer part to be uploaded, to avoid the limit imposed on the maximum number of parts. + *

+ * In the case where the {@code minimumPartSizeInBytes} is set to a value higher than the {@code thresholdInBytes}, when + * the client receive a request with a size smaller than a single part multipart operation will NOT be performed + * even if the size of the request is larger than the threshold. + *

+ * Default value: 8 Mib + * + * @param minimumPartSizeInBytes the value of the part size to set + * @return an instance of this builder. + */ + Builder minimumPartSizeInBytes(Long minimumPartSizeInBytes); + + /** + * Indicated the value of the part configured size. + * @return the value of the part size + */ + Long minimumPartSizeInBytes(); + + /** + * Configures the maximum amount of memory, in bytes, the SDK will use to buffer content of requests in memory. + * Increasing this value my lead to better performance at the cost of using more memory. + *

+ * Default value: If not specified, the SDK will use the equivalent of two parts worth of memory, so 16 Mib by default. + * + * @param maximumMemoryUsageInBytes the value of the maximum memory usage. + * @return an instance of this builder. + */ + Builder maximumMemoryUsageInBytes(Long maximumMemoryUsageInBytes); + + /** + * Indicates the value of the maximum memory usage that the SDK will use. + * @return the value of the maximum memory usage. + */ + Long maximumMemoryUsageInBytes(); + } + + private static class DefaultMultipartConfigBuilder implements Builder { + private Long thresholdInBytes; + private Long minimumPartSizeInBytes; + private Long maximumMemoryUsageInBytes; + + public Builder thresholdInBytes(Long thresholdInBytes) { + this.thresholdInBytes = thresholdInBytes; + return this; + } + + public Long thresholdInBytes() { + return this.thresholdInBytes; + } + + public Builder minimumPartSizeInBytes(Long minimumPartSizeInBytes) { + this.minimumPartSizeInBytes = minimumPartSizeInBytes; + return this; + } + + public Long minimumPartSizeInBytes() { + return this.minimumPartSizeInBytes; + } + + @Override + public Builder maximumMemoryUsageInBytes(Long maximumMemoryUsageInBytes) { + this.maximumMemoryUsageInBytes = maximumMemoryUsageInBytes; + return this; + } + + @Override + public Long maximumMemoryUsageInBytes() { + return maximumMemoryUsageInBytes; + } + + @Override + public MultipartConfiguration build() { + return new MultipartConfiguration(this); + } + } +} diff --git a/services/s3/src/main/resources/codegen-resources/customization.config b/services/s3/src/main/resources/codegen-resources/customization.config index 1a1efb76c5f..053ef58171a 100644 --- a/services/s3/src/main/resources/codegen-resources/customization.config +++ b/services/s3/src/main/resources/codegen-resources/customization.config @@ -236,6 +236,11 @@ "syncClientDecorator": "software.amazon.awssdk.services.s3.internal.client.S3SyncClientDecorator", "asyncClientDecorator": "software.amazon.awssdk.services.s3.internal.client.S3AsyncClientDecorator", "useGlobalEndpoint": true, + "multipartCustomization": { + "multipartConfigurationClass": "software.amazon.awssdk.services.s3.multipart.MultipartConfiguration", + "multipartConfigMethodDoc": "Configuration for multipart operation of this client.", + "multipartEnableMethodDoc": "Enables automatic conversion of put and copy method to their equivalent multipart operation." + }, "interceptors": [ "software.amazon.awssdk.services.s3.internal.handlers.PutObjectInterceptor", "software.amazon.awssdk.services.s3.internal.handlers.CreateBucketInterceptor",