From 000ee9fe910b05167dae3bdb3a6e7a409339470d Mon Sep 17 00:00:00 2001 From: Tianyi Wang Date: Wed, 22 Nov 2023 18:02:47 -0500 Subject: [PATCH 1/7] rebase to main branch --- aws/config.go | 12 +++ .../aws/go/codegen/AddAwsConfigFields.java | 19 +++++ .../aws/go/codegen/AwsProtocolUtils.java | 36 --------- config/config.go | 8 +- config/env_config.go | 54 +++++++++++++ config/env_config_test.go | 40 ++++++++++ config/load_options.go | 46 +++++++++++ config/provider.go | 34 ++++++++ config/resolve.go | 27 +++++++ config/resolve_test.go | 80 +++++++++++++++++++ config/shared_config.go | 69 ++++++++++++++++ config/shared_config_test.go | 25 ++++++ config/testdata/shared_config | 16 ++++ .../protocoltest/awsrestjson/api_client.go | 24 ++++-- .../api_op_PutWithContentEncoding.go | 3 + .../api_op_PutWithContentEncoding_test.go | 8 -- internal/protocoltest/awsrestjson/options.go | 7 ++ internal/protocoltest/ec2query/api_client.go | 24 ++++-- .../ec2query/api_op_PutWithContentEncoding.go | 3 + .../api_op_PutWithContentEncoding_test.go | 8 -- internal/protocoltest/ec2query/options.go | 7 ++ internal/protocoltest/jsonrpc/api_client.go | 26 +++--- .../jsonrpc/api_op_PutWithContentEncoding.go | 3 + .../api_op_PutWithContentEncoding_test.go | 8 -- internal/protocoltest/jsonrpc/options.go | 7 ++ internal/protocoltest/jsonrpc10/api_client.go | 24 ++++-- .../api_op_PutWithContentEncoding.go | 3 + .../api_op_PutWithContentEncoding_test.go | 8 -- internal/protocoltest/jsonrpc10/options.go | 7 ++ internal/protocoltest/query/api_client.go | 24 ++++-- .../query/api_op_PutWithContentEncoding.go | 3 + .../api_op_PutWithContentEncoding_test.go | 8 -- internal/protocoltest/query/options.go | 7 ++ internal/protocoltest/restxml/api_client.go | 24 ++++-- .../restxml/api_op_PutWithContentEncoding.go | 3 + .../api_op_PutWithContentEncoding_test.go | 8 -- internal/protocoltest/restxml/options.go | 7 ++ 37 files changed, 586 insertions(+), 134 deletions(-) diff --git a/aws/config.go b/aws/config.go index b361c13867c..1ee54cfe0f6 100644 --- a/aws/config.go +++ b/aws/config.go @@ -150,6 +150,18 @@ type Config struct { // BaseEndpoint is an intermediary transfer location to a service specific // BaseEndpoint on a service's Options. BaseEndpoint *string + + // DisableRequestCompression toggles if an operation request could be + // compressed or not. Will be set to false by default. This variable is sourced from + // environment variable AWS_DISABLE_REQUEST_COMPRESSION or the shared config profile attribute + // disable_request_compression + DisableRequestCompression bool + + // RequestMinCompressSizeBytes sets the inclusive min bytes of a request body that could be + // compressed. Will be set to 10240 by default and must be within 0 and 10485760 bytes inclusively. + // This variable is sourced from environment variable AWS_REQUEST_MIN_COMPRESSION_SIZE_BYTES or + // the shared config profile attribute request_min_compression_size_bytes + RequestMinCompressSizeBytes int64 } // NewConfig returns a new Config pointer that can be chained with builder diff --git a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AddAwsConfigFields.java b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AddAwsConfigFields.java index 3c89d0a9c5f..1383ec4c542 100644 --- a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AddAwsConfigFields.java +++ b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AddAwsConfigFields.java @@ -37,6 +37,7 @@ import software.amazon.smithy.go.codegen.integration.ConfigFieldResolver; import software.amazon.smithy.go.codegen.integration.GoIntegration; import software.amazon.smithy.go.codegen.integration.RuntimeClientPlugin; +import software.amazon.smithy.go.codegen.requestcompression.RequestCompression; import software.amazon.smithy.model.Model; import software.amazon.smithy.model.shapes.ServiceShape; import software.amazon.smithy.model.traits.HttpBearerAuthTrait; @@ -74,6 +75,10 @@ public class AddAwsConfigFields implements GoIntegration { private static final String SDK_APP_ID = "AppID"; + private static final String DISABLE_REQUEST_COMPRESSION = "DisableRequestCompression"; + + private static final String REQUEST_MIN_COMPRESSION_SIZE_BYTES = "RequestMinCompressSizeBytes"; + private static final List AWS_CONFIG_FIELDS = ListUtils.of( AwsConfigField.builder() .name(REGION_CONFIG_NAME) @@ -209,6 +214,20 @@ public class AddAwsConfigFields implements GoIntegration { .type(getUniversalSymbol("string")) .documentation("The optional application specific identifier appended to the User-Agent header.") .generatedOnClient(false) + .build(), + AwsConfigField.builder() + .name(DISABLE_REQUEST_COMPRESSION) + .type(getUniversalSymbol("bool")) + .documentation("Configure whether or not a operation request could be compressed.") + .servicePredicate(RequestCompression::isRequestCompressionService) + .generatedOnClient(false) + .build(), + AwsConfigField.builder() + .name(REQUEST_MIN_COMPRESSION_SIZE_BYTES) + .type(getUniversalSymbol("int64")) + .documentation("The inclusive min request body size to be compressed.") + .servicePredicate(RequestCompression::isRequestCompressionService) + .generatedOnClient(false) .build() ); diff --git a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsProtocolUtils.java b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsProtocolUtils.java index a6d5017114e..4cfe50ee6e3 100644 --- a/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsProtocolUtils.java +++ b/codegen/smithy-aws-go-codegen/src/main/java/software/amazon/smithy/aws/go/codegen/AwsProtocolUtils.java @@ -107,42 +107,6 @@ static void generateHttpProtocolTests(GenerationContext context) { // skip request compression tests, not yet implemented in the SDK Set inputSkipTests = new TreeSet<>(SetUtils.of( - HttpProtocolUnitTestGenerator.SkipTest.builder() - .service(ShapeId.from("aws.protocoltests.json10#JsonRpc10")) - .operation(ShapeId.from("aws.protocoltests.json10#PutWithContentEncoding")) - .addTestName("SDKAppliedContentEncoding_awsJson1_0") - .addTestName("SDKAppendsGzipAndIgnoresHttpProvidedEncoding_awsJson1_0") - .build(), - HttpProtocolUnitTestGenerator.SkipTest.builder() - .service(ShapeId.from("aws.protocoltests.json#JsonProtocol")) - .operation(ShapeId.from("aws.protocoltests.json#PutWithContentEncoding")) - .addTestName("SDKAppliedContentEncoding_awsJson1_1") - .addTestName("SDKAppendsGzipAndIgnoresHttpProvidedEncoding_awsJson1_1") - .build(), - HttpProtocolUnitTestGenerator.SkipTest.builder() - .service(ShapeId.from("aws.protocoltests.query#AwsQuery")) - .operation(ShapeId.from("aws.protocoltests.query#PutWithContentEncoding")) - .addTestName("SDKAppliedContentEncoding_awsQuery") - .addTestName("SDKAppendsGzipAndIgnoresHttpProvidedEncoding_awsQuery") - .build(), - HttpProtocolUnitTestGenerator.SkipTest.builder() - .service(ShapeId.from("aws.protocoltests.ec2#AwsEc2")) - .operation(ShapeId.from("aws.protocoltests.ec2#PutWithContentEncoding")) - .addTestName("SDKAppliedContentEncoding_ec2Query") - .addTestName("SDKAppendsGzipAndIgnoresHttpProvidedEncoding_ec2Query") - .build(), - HttpProtocolUnitTestGenerator.SkipTest.builder() - .service(ShapeId.from("aws.protocoltests.restjson#RestJson")) - .operation(ShapeId.from("aws.protocoltests.restjson#PutWithContentEncoding")) - .addTestName("SDKAppliedContentEncoding_restJson1") - .addTestName("SDKAppendedGzipAfterProvidedEncoding_restJson1") - .build(), - HttpProtocolUnitTestGenerator.SkipTest.builder() - .service(ShapeId.from("aws.protocoltests.restxml#RestXml")) - .operation(ShapeId.from("aws.protocoltests.restxml#PutWithContentEncoding")) - .addTestName("SDKAppliedContentEncoding_restXml") - .addTestName("SDKAppendedGzipAfterProvidedEncoding_restXml") - .build(), HttpProtocolUnitTestGenerator.SkipTest.builder() .service(ShapeId.from("aws.protocoltests.restxml#RestXml")) .operation(ShapeId.from("aws.protocoltests.restxml#HttpPayloadWithUnion")) diff --git a/config/config.go b/config/config.go index dfe62973221..50582d89d54 100644 --- a/config/config.go +++ b/config/config.go @@ -70,10 +70,16 @@ var defaultAWSConfigResolvers = []awsConfigResolver{ // httpBearerAuth authentication scheme. resolveBearerAuthToken, - // Sets the sdk app ID if present in shared config profile + // Sets the sdk app ID if present in env var or shared config profile resolveAppID, resolveBaseEndpoint, + + // Sets the DisableRequestCompression if present in env var or shared config profile + resolveDisableRequestCompression, + + // Sets the RequestMinCompressSizeBytes if present in env var or shared config profile + resolveRequestMinCompressSizeBytes, } // A Config represents a generic configuration value or set of values. This type diff --git a/config/env_config.go b/config/env_config.go index 78bc1493372..ccd9094fec1 100644 --- a/config/env_config.go +++ b/config/env_config.go @@ -74,6 +74,9 @@ const ( awsIgnoreConfiguredEndpoints = "AWS_IGNORE_CONFIGURED_ENDPOINT_URLS" awsEndpointURL = "AWS_ENDPOINT_URL" + + awsDisableRequestCompression = "AWS_DISABLE_REQUEST_COMPRESSION" + awsRequestMinCompressionSizeBytes = "AWS_REQUEST_MIN_COMPRESSION_SIZE_BYTES" ) var ( @@ -268,6 +271,15 @@ type EnvConfig struct { // Value to contain configured endpoints to be propagated to // corresponding endpoint resolution field. BaseEndpoint string + + // determine if request compression is allowed, default to false + // retrieved from env var AWS_DISABLE_REQUEST_COMPRESSION + DisableRequestCompression *bool + + // inclusive threshold request body size to trigger compression, + // default to 10240 and must be within 0 and 10485760 bytes inclusive + // retrieved from env var AWS_REQUEST_MIN_COMPRESSION_SIZE_BYTES + RequestMinCompressSizeBytes *int64 } // loadEnvConfig reads configuration values from the OS's environment variables. @@ -310,6 +322,13 @@ func NewEnvConfig() (EnvConfig, error) { cfg.AppID = os.Getenv(awsSdkAppID) + if err := setBoolPtrFromEnvVal(&cfg.DisableRequestCompression, []string{awsDisableRequestCompression}); err != nil { + return cfg, err + } + if err := setRequestMinCompressSizeBytes(&cfg.RequestMinCompressSizeBytes); err != nil { + return cfg, err + } + if err := setEndpointDiscoveryTypeFromEnvVal(&cfg.EnableEndpointDiscovery, []string{awsEnableEndpointDiscoveryEnvVar}); err != nil { return cfg, err } @@ -370,6 +389,41 @@ func (c EnvConfig) getAppID(context.Context) (string, bool, error) { return c.AppID, len(c.AppID) > 0, nil } +func setRequestMinCompressSizeBytes(bytes **int64) error { + b := os.Getenv(awsRequestMinCompressionSizeBytes) + if b == "" { + return nil + } + + byte, err := strconv.ParseInt(b, 10, 64) + if err != nil { + return fmt.Errorf("invalid value for env var, %s=%s, need int64", + awsRequestMinCompressionSizeBytes, b) + } else if byte < 0 || byte > 10485760 { + return fmt.Errorf("invalid range for env var min request compression size bytes %q, must be within 0 and 10485760 inclusively", byte) + } + if *bytes == nil { + *bytes = new(int64) + } + **bytes = byte + + return nil +} + +func (c EnvConfig) getDisableRequestCompression(context.Context) (bool, bool, error) { + if c.DisableRequestCompression == nil { + return false, false, nil + } + return *c.DisableRequestCompression, true, nil +} + +func (c EnvConfig) getRequestMinCompressSizeBytes(context.Context) (int64, bool, error) { + if c.RequestMinCompressSizeBytes == nil { + return 0, false, nil + } + return *c.RequestMinCompressSizeBytes, true, nil +} + // GetRetryMaxAttempts returns the value of AWS_MAX_ATTEMPTS if was specified, // and not 0. func (c EnvConfig) GetRetryMaxAttempts(ctx context.Context) (int, bool, error) { diff --git a/config/env_config_test.go b/config/env_config_test.go index 14bead837a1..af232da8b18 100644 --- a/config/env_config_test.go +++ b/config/env_config_test.go @@ -456,6 +456,46 @@ func TestNewEnvConfig(t *testing.T) { }, WantErr: true, }, + 42: { + Env: map[string]string{ + "AWS_DISABLE_REQUEST_COMPRESSION": "true", + "AWS_REQUEST_MIN_COMPRESSION_SIZE_BYTES": "12345", + }, + Config: EnvConfig{ + DisableRequestCompression: aws.Bool(true), + RequestMinCompressSizeBytes: aws.Int64(12345), + }, + }, + 43: { + Env: map[string]string{ + "AWS_DISABLE_REQUEST_COMPRESSION": "blabla", + "AWS_REQUEST_MIN_COMPRESSION_SIZE_BYTES": "12345", + }, + Config: EnvConfig{ + DisableRequestCompression: aws.Bool(false), + }, + WantErr: true, + }, + 44: { + Env: map[string]string{ + "AWS_DISABLE_REQUEST_COMPRESSION": "true", + "AWS_REQUEST_MIN_COMPRESSION_SIZE_BYTES": "1.1", + }, + Config: EnvConfig{ + DisableRequestCompression: aws.Bool(true), + }, + WantErr: true, + }, + 45: { + Env: map[string]string{ + "AWS_DISABLE_REQUEST_COMPRESSION": "false", + "AWS_REQUEST_MIN_COMPRESSION_SIZE_BYTES": "10485761", + }, + Config: EnvConfig{ + DisableRequestCompression: aws.Bool(false), + }, + WantErr: true, + }, } for i, c := range cases { diff --git a/config/load_options.go b/config/load_options.go index 7480bb45ed1..1ad93a76b31 100644 --- a/config/load_options.go +++ b/config/load_options.go @@ -206,6 +206,12 @@ type LoadOptions struct { // The sdk app ID retrieved from env var or shared config to be added to request user agent header AppID string + + // Specifies whether an operation request could be compressed + DisableRequestCompression *bool + + // The inclusive min bytes of a request body that could be compressed + RequestMinCompressSizeBytes *int64 } func (o LoadOptions) getDefaultsMode(ctx context.Context) (aws.DefaultsMode, bool, error) { @@ -253,6 +259,22 @@ func (o LoadOptions) getAppID(ctx context.Context) (string, bool, error) { return o.AppID, len(o.AppID) > 0, nil } +// getDisableRequestCompression returns DisableRequestCompression from config's LoadOptions +func (o LoadOptions) getDisableRequestCompression(ctx context.Context) (bool, bool, error) { + if o.DisableRequestCompression == nil { + return false, false, nil + } + return *o.DisableRequestCompression, true, nil +} + +// getRequestMinCompressSizeBytes returns RequestMinCompressSizeBytes from config's LoadOptions +func (o LoadOptions) getRequestMinCompressSizeBytes(ctx context.Context) (int64, bool, error) { + if o.RequestMinCompressSizeBytes == nil { + return 0, false, nil + } + return *o.RequestMinCompressSizeBytes, true, nil +} + // WithRegion is a helper function to construct functional options // that sets Region on config's LoadOptions. Setting the region to // an empty string, will result in the region value being ignored. @@ -274,6 +296,30 @@ func WithAppID(ID string) LoadOptionsFunc { } } +// WithDisableRequestCompression is a helper function to construct functional options +// that sets DisableRequestCompression on config's LoadOptions. +func WithDisableRequestCompression(DisableRequestCompression *bool) LoadOptionsFunc { + return func(o *LoadOptions) error { + if DisableRequestCompression == nil { + return nil + } + o.DisableRequestCompression = DisableRequestCompression + return nil + } +} + +// WithRequestMinCompressSizeBytes is a helper function to construct functional options +// that sets RequestMinCompressSizeBytes on config's LoadOptions. +func WithRequestMinCompressSizeBytes(RequestMinCompressSizeBytes *int64) LoadOptionsFunc { + return func(o *LoadOptions) error { + if RequestMinCompressSizeBytes == nil { + return nil + } + o.RequestMinCompressSizeBytes = RequestMinCompressSizeBytes + return nil + } +} + // getDefaultRegion returns DefaultRegion from config's LoadOptions func (o LoadOptions) getDefaultRegion(ctx context.Context) (string, bool, error) { if len(o.DefaultRegion) == 0 { diff --git a/config/provider.go b/config/provider.go index d5235846011..13745fc98fd 100644 --- a/config/provider.go +++ b/config/provider.go @@ -191,6 +191,40 @@ func getAppID(ctx context.Context, configs configs) (value string, found bool, e return } +// disableRequestCompressionProvider provides access to the DisableRequestCompression +type disableRequestCompressionProvider interface { + getDisableRequestCompression(context.Context) (bool, bool, error) +} + +func getDisableRequestCompression(ctx context.Context, configs configs) (value bool, found bool, err error) { + for _, cfg := range configs { + if p, ok := cfg.(disableRequestCompressionProvider); ok { + value, found, err = p.getDisableRequestCompression(ctx) + if err != nil || found { + break + } + } + } + return +} + +// requestMinCompressSizeBytesProvider provides access to the MinCompressSizeBytes +type requestMinCompressSizeBytesProvider interface { + getRequestMinCompressSizeBytes(context.Context) (int64, bool, error) +} + +func getRequestMinCompressSizeBytes(ctx context.Context, configs configs) (value int64, found bool, err error) { + for _, cfg := range configs { + if p, ok := cfg.(requestMinCompressSizeBytesProvider); ok { + value, found, err = p.getRequestMinCompressSizeBytes(ctx) + if err != nil || found { + break + } + } + } + return +} + // ec2IMDSRegionProvider provides access to the ec2 imds region // configuration value type ec2IMDSRegionProvider interface { diff --git a/config/resolve.go b/config/resolve.go index b3b2c93cdc4..fde2e3980e0 100644 --- a/config/resolve.go +++ b/config/resolve.go @@ -139,6 +139,33 @@ func resolveAppID(ctx context.Context, cfg *aws.Config, configs configs) error { return nil } +// resolveDisableRequestCompression extracts the DisableRequestCompression from the configs slice's +// SharedConfig or EnvConfig +func resolveDisableRequestCompression(ctx context.Context, cfg *aws.Config, configs configs) error { + disable, _, err := getDisableRequestCompression(ctx, configs) + if err != nil { + return err + } + + cfg.DisableRequestCompression = disable + return nil +} + +// resolveRequestMinCompressSizeBytes extracts the RequestMinCompressSizeBytes from the configs slice's +// SharedConfig or EnvConfig +func resolveRequestMinCompressSizeBytes(ctx context.Context, cfg *aws.Config, configs configs) error { + minBytes, found, err := getRequestMinCompressSizeBytes(ctx, configs) + if err != nil { + return err + } + // must set a default min size 10240 if not configured + if !found { + minBytes = 10240 + } + cfg.RequestMinCompressSizeBytes = minBytes + return nil +} + // resolveDefaultRegion extracts the first instance of a default region and sets `aws.Config.Region` to the default // region if region had not been resolved from other sources. func resolveDefaultRegion(ctx context.Context, cfg *aws.Config, configs configs) error { diff --git a/config/resolve_test.go b/config/resolve_test.go index 773e8285c94..1b33004379c 100644 --- a/config/resolve_test.go +++ b/config/resolve_test.go @@ -150,6 +150,86 @@ func TestResolveAppID(t *testing.T) { } } +func TestResolveRequestMinCompressSizeBytes(t *testing.T) { + cases := map[string]struct { + RequestMinCompressSizeBytes *int64 + ExpectMinBytes int64 + }{ + "min requet size of 100 bytes": { + RequestMinCompressSizeBytes: aws.Int64(100), + ExpectMinBytes: 100, + }, + "min request size unset": { + ExpectMinBytes: 10240, + }, + } + + for name, c := range cases { + t.Run(name, func(t *testing.T) { + var options LoadOptions + optFns := []func(options *LoadOptions) error{ + WithRequestMinCompressSizeBytes(c.RequestMinCompressSizeBytes), + } + + for _, optFn := range optFns { + optFn(&options) + } + + configs := configs{options} + + var cfg aws.Config + + if err := resolveRequestMinCompressSizeBytes(context.Background(), &cfg, configs); err != nil { + t.Fatalf("expect no error, got %v", err) + } + + if e, a := c.ExpectMinBytes, cfg.RequestMinCompressSizeBytes; e != a { + t.Errorf("expect RequestMinCompressSizeBytes to be %v , got %v", e, a) + } + }) + } +} + +func TestResolveDisableRequestCompression(t *testing.T) { + cases := map[string]struct { + DisableRequestCompression *bool + ExpectDisable bool + }{ + "disable request compression": { + DisableRequestCompression: aws.Bool(true), + ExpectDisable: true, + }, + "disable request compression unset": { + ExpectDisable: false, + }, + } + + for name, c := range cases { + t.Run(name, func(t *testing.T) { + var options LoadOptions + optFns := []func(options *LoadOptions) error{ + WithDisableRequestCompression(c.DisableRequestCompression), + } + + for _, optFn := range optFns { + optFn(&options) + } + + configs := configs{options} + + var cfg aws.Config + + if err := resolveDisableRequestCompression(context.Background(), &cfg, configs); err != nil { + t.Fatalf("expect no error, got %v", err) + } + + if e, a := c.ExpectDisable, cfg.DisableRequestCompression; e != a { + t.Errorf("expect DisableRequestCompression to be %v , got %v", e, a) + } + }) + } +} + func TestResolveCredentialsProvider(t *testing.T) { var options LoadOptions optFns := []func(options *LoadOptions) error{ diff --git a/config/shared_config.go b/config/shared_config.go index 20683bf5f07..b80039b092b 100644 --- a/config/shared_config.go +++ b/config/shared_config.go @@ -107,6 +107,9 @@ const ( ignoreConfiguredEndpoints = "ignore_configured_endpoint_urls" endpointURL = "endpoint_url" + + disableRequestCompression = "disable_request_compression" + requestMinCompressionSizeBytes = "request_min_compression_size_bytes" ) // defaultSharedConfigProfile allows for swapping the default profile for testing @@ -316,6 +319,15 @@ type SharedConfig struct { // Value to contain services section content. Services Services + + // determine if request compression is allowed, default to false + // retrieved from config file's profile field disable_request_compression + DisableRequestCompression *bool + + // inclusive threshold request body size to trigger compression, + // default to 10240 and must be within 0 and 10485760 bytes inclusive + // retrieved from config file's profile field request_min_compression_size_bytes + RequestMinCompressSizeBytes *int64 } func (c SharedConfig) getDefaultsMode(ctx context.Context) (value aws.DefaultsMode, ok bool, err error) { @@ -1084,6 +1096,13 @@ func (c *SharedConfig) setFromIniSection(profile string, section ini.Section) er updateString(&c.BaseEndpoint, section, endpointURL) + if err := updateDisableRequestCompression(&c.DisableRequestCompression, section, disableRequestCompression); err != nil { + return fmt.Errorf("failed to load %s from shared config, %w", disableRequestCompression, err) + } + if err := updateRequestMinCompressSizeBytes(&c.RequestMinCompressSizeBytes, section, requestMinCompressionSizeBytes); err != nil { + return fmt.Errorf("failed to load %s from shared config, %w", requestMinCompressionSizeBytes, err) + } + // Shared Credentials creds := aws.Credentials{ AccessKeyID: section.String(accessKeyIDKey), @@ -1099,6 +1118,56 @@ func (c *SharedConfig) setFromIniSection(profile string, section ini.Section) er return nil } +func updateRequestMinCompressSizeBytes(bytes **int64, sec ini.Section, key string) error { + if !sec.Has(key) { + return nil + } + + v, ok := sec.Int(key) + if !ok { + return fmt.Errorf("invalid value for min request compression size bytes %s, need int64", sec.String(key)) + } + if v < 0 || v > 10485760 { + return fmt.Errorf("invalid range for min request compression size bytes %d, must be within 0 and 10485760 inclusively", v) + } + *bytes = new(int64) + **bytes = v + return nil +} + +func updateDisableRequestCompression(disable **bool, sec ini.Section, key string) error { + if !sec.Has(key) { + return nil + } + + v := sec.String(key) + switch { + case v == "true": + *disable = new(bool) + **disable = true + case v == "false": + *disable = new(bool) + **disable = false + default: + return fmt.Errorf("invalid value for shared config profile field, %s=%s, need true or false", key, v) + } + return nil +} + +func (c SharedConfig) getRequestMinCompressSizeBytes(ctx context.Context) (int64, bool, error) { + if c.RequestMinCompressSizeBytes == nil { + return 0, false, nil + } + return *c.RequestMinCompressSizeBytes, true, nil +} + +func (c SharedConfig) getDisableRequestCompression(ctx context.Context) (bool, bool, error) { + if c.DisableRequestCompression == nil { + return false, false, nil + } + return *c.DisableRequestCompression, true, nil +} + func updateDefaultsMode(mode *aws.DefaultsMode, section ini.Section, key string) error { if !section.Has(key) { return nil diff --git a/config/shared_config_test.go b/config/shared_config_test.go index 359a28b894c..b99fb0c2470 100644 --- a/config/shared_config_test.go +++ b/config/shared_config_test.go @@ -686,6 +686,31 @@ func TestNewSharedConfig(t *testing.T) { EC2IMDSv1Disabled: aws.Bool(false), }, }, + "profile configuring request compression": { + ConfigFilenames: []string{testConfigFilename}, + Profile: "request_compression", + Expected: SharedConfig{ + Profile: "request_compression", + DisableRequestCompression: aws.Bool(true), + RequestMinCompressSizeBytes: aws.Int64(12345), + }, + }, + "profile with invalid disableRequestCompression": { + ConfigFilenames: []string{testConfigFilename}, + Profile: "request_compression_invalid_disable", + Err: fmt.Errorf("invalid value for shared config profile field, %s=%s, need true or false", + disableRequestCompression, "blabla"), + }, + "profile with non-int requestMinCompressSizeBytes": { + ConfigFilenames: []string{testConfigFilename}, + Profile: "request_compression_non_int_min_request", + Err: fmt.Errorf("invalid value for min request compression size bytes hahaha, need int64"), + }, + "profile with requestMinCompressSizeBytes out of bounds": { + ConfigFilenames: []string{testConfigFilename}, + Profile: "request_compression_min_request_out_of_bounds", + Err: fmt.Errorf("invalid range for min request compression size bytes 10485761, must be within 0 and 10485760 inclusively"), + }, } for name, c := range cases { diff --git a/config/testdata/shared_config b/config/testdata/shared_config index eefb5158262..96122cf242f 100644 --- a/config/testdata/shared_config +++ b/config/testdata/shared_config @@ -291,3 +291,19 @@ ec2_metadata_v1_disabled=True [profile ec2-metadata-v1-disabled-invalid] ec2_metadata_v1_disabled=invalid + +[profile request_compression] +disable_request_compression = true +request_min_compression_size_bytes = 12345 + +[profile request_compression_invalid_disable] +disable_request_compression = blabla +request_min_compression_size_bytes = 12345 + +[profile request_compression_non_int_min_request] +disable_request_compression = false +request_min_compression_size_bytes = hahaha + +[profile request_compression_min_request_out_of_bounds] +disable_request_compression = false +request_min_compression_size_bytes = 10485761 diff --git a/internal/protocoltest/awsrestjson/api_client.go b/internal/protocoltest/awsrestjson/api_client.go index b7a5e54abbc..28ce7e8dbd7 100644 --- a/internal/protocoltest/awsrestjson/api_client.go +++ b/internal/protocoltest/awsrestjson/api_client.go @@ -16,6 +16,7 @@ import ( smithydocument "github.com/aws/smithy-go/document" "github.com/aws/smithy-go/logging" "github.com/aws/smithy-go/middleware" + smithyrequestcompression "github.com/aws/smithy-go/private/requestcompression" smithyrand "github.com/aws/smithy-go/rand" smithyhttp "github.com/aws/smithy-go/transport/http" "net" @@ -206,14 +207,16 @@ func setResolvedDefaultsMode(o *Options) { // NewFromConfig returns a new client from the provided config. func NewFromConfig(cfg aws.Config, optFns ...func(*Options)) *Client { opts := Options{ - Region: cfg.Region, - DefaultsMode: cfg.DefaultsMode, - RuntimeEnvironment: cfg.RuntimeEnvironment, - HTTPClient: cfg.HTTPClient, - APIOptions: cfg.APIOptions, - Logger: cfg.Logger, - ClientLogMode: cfg.ClientLogMode, - AppID: cfg.AppID, + Region: cfg.Region, + DefaultsMode: cfg.DefaultsMode, + RuntimeEnvironment: cfg.RuntimeEnvironment, + HTTPClient: cfg.HTTPClient, + APIOptions: cfg.APIOptions, + Logger: cfg.Logger, + ClientLogMode: cfg.ClientLogMode, + AppID: cfg.AppID, + DisableRequestCompression: cfg.DisableRequestCompression, + RequestMinCompressSizeBytes: cfg.RequestMinCompressSizeBytes, } resolveAWSRetryerProvider(cfg, &opts) resolveAWSRetryMaxAttempts(cfg, &opts) @@ -385,6 +388,11 @@ func resolveUseFIPSEndpoint(cfg aws.Config, o *Options) error { return nil } +func addRequestCompression(stack *middleware.Stack, options Options) error { + return smithyrequestcompression.AddRequestCompression(stack, options.DisableRequestCompression, options.RequestMinCompressSizeBytes, + "gzip") +} + // IdempotencyTokenProvider interface for providing idempotency token type IdempotencyTokenProvider interface { GetIdempotencyToken() (string, error) diff --git a/internal/protocoltest/awsrestjson/api_op_PutWithContentEncoding.go b/internal/protocoltest/awsrestjson/api_op_PutWithContentEncoding.go index 0ea0e7c5097..e92e4ca40ce 100644 --- a/internal/protocoltest/awsrestjson/api_op_PutWithContentEncoding.go +++ b/internal/protocoltest/awsrestjson/api_op_PutWithContentEncoding.go @@ -92,6 +92,9 @@ func (c *Client) addOperationPutWithContentEncodingMiddlewares(stack *middleware if err = addSetLegacyContextSigningOptionsMiddleware(stack); err != nil { return err } + if err = addRequestCompression(stack, options); err != nil { + return err + } if err = stack.Initialize.Add(newServiceMetadataMiddleware_opPutWithContentEncoding(options.Region), middleware.Before); err != nil { return err } diff --git a/internal/protocoltest/awsrestjson/api_op_PutWithContentEncoding_test.go b/internal/protocoltest/awsrestjson/api_op_PutWithContentEncoding_test.go index e4196233a43..016e9350589 100644 --- a/internal/protocoltest/awsrestjson/api_op_PutWithContentEncoding_test.go +++ b/internal/protocoltest/awsrestjson/api_op_PutWithContentEncoding_test.go @@ -62,14 +62,6 @@ func TestClient_PutWithContentEncoding_awsRestjson1Serialize(t *testing.T) { } for name, c := range cases { t.Run(name, func(t *testing.T) { - if name == "SDKAppliedContentEncoding_restJson1" { - t.Skip("disabled test aws.protocoltests.restjson#RestJson aws.protocoltests.restjson#PutWithContentEncoding") - } - - if name == "SDKAppendedGzipAfterProvidedEncoding_restJson1" { - t.Skip("disabled test aws.protocoltests.restjson#RestJson aws.protocoltests.restjson#PutWithContentEncoding") - } - actualReq := &http.Request{} serverURL := "http://localhost:8888/" if c.Host != nil { diff --git a/internal/protocoltest/awsrestjson/options.go b/internal/protocoltest/awsrestjson/options.go index 52222eef1a6..0ffb6b769e9 100644 --- a/internal/protocoltest/awsrestjson/options.go +++ b/internal/protocoltest/awsrestjson/options.go @@ -36,6 +36,9 @@ type Options struct { // clients initial default settings. DefaultsMode aws.DefaultsMode + // Whether to disable automatic request compression for supported operations. + DisableRequestCompression bool + // The endpoint options to be used when attempting to resolve an endpoint. EndpointOptions EndpointResolverOptions @@ -62,6 +65,10 @@ type Options struct { // The region to send requests to. (Required) Region string + // Inclusive threshold request body size to trigger compression, default to 10240 + // and must be within 0 and 10485760 bytes inclusively + RequestMinCompressSizeBytes int64 + // RetryMaxAttempts specifies the maximum number attempts an API client will call // an operation that fails with a retryable error. A value of 0 is ignored, and // will not be used to configure the API client created default retryer, or modify diff --git a/internal/protocoltest/ec2query/api_client.go b/internal/protocoltest/ec2query/api_client.go index 8abda987cd6..536505c1ed2 100644 --- a/internal/protocoltest/ec2query/api_client.go +++ b/internal/protocoltest/ec2query/api_client.go @@ -16,6 +16,7 @@ import ( smithydocument "github.com/aws/smithy-go/document" "github.com/aws/smithy-go/logging" "github.com/aws/smithy-go/middleware" + smithyrequestcompression "github.com/aws/smithy-go/private/requestcompression" smithyrand "github.com/aws/smithy-go/rand" smithyhttp "github.com/aws/smithy-go/transport/http" "net" @@ -206,14 +207,16 @@ func setResolvedDefaultsMode(o *Options) { // NewFromConfig returns a new client from the provided config. func NewFromConfig(cfg aws.Config, optFns ...func(*Options)) *Client { opts := Options{ - Region: cfg.Region, - DefaultsMode: cfg.DefaultsMode, - RuntimeEnvironment: cfg.RuntimeEnvironment, - HTTPClient: cfg.HTTPClient, - APIOptions: cfg.APIOptions, - Logger: cfg.Logger, - ClientLogMode: cfg.ClientLogMode, - AppID: cfg.AppID, + Region: cfg.Region, + DefaultsMode: cfg.DefaultsMode, + RuntimeEnvironment: cfg.RuntimeEnvironment, + HTTPClient: cfg.HTTPClient, + APIOptions: cfg.APIOptions, + Logger: cfg.Logger, + ClientLogMode: cfg.ClientLogMode, + AppID: cfg.AppID, + DisableRequestCompression: cfg.DisableRequestCompression, + RequestMinCompressSizeBytes: cfg.RequestMinCompressSizeBytes, } resolveAWSRetryerProvider(cfg, &opts) resolveAWSRetryMaxAttempts(cfg, &opts) @@ -385,6 +388,11 @@ func resolveUseFIPSEndpoint(cfg aws.Config, o *Options) error { return nil } +func addRequestCompression(stack *middleware.Stack, options Options) error { + return smithyrequestcompression.AddRequestCompression(stack, options.DisableRequestCompression, options.RequestMinCompressSizeBytes, + "gzip") +} + // IdempotencyTokenProvider interface for providing idempotency token type IdempotencyTokenProvider interface { GetIdempotencyToken() (string, error) diff --git a/internal/protocoltest/ec2query/api_op_PutWithContentEncoding.go b/internal/protocoltest/ec2query/api_op_PutWithContentEncoding.go index cac98bd6609..df42870ce81 100644 --- a/internal/protocoltest/ec2query/api_op_PutWithContentEncoding.go +++ b/internal/protocoltest/ec2query/api_op_PutWithContentEncoding.go @@ -92,6 +92,9 @@ func (c *Client) addOperationPutWithContentEncodingMiddlewares(stack *middleware if err = addSetLegacyContextSigningOptionsMiddleware(stack); err != nil { return err } + if err = addRequestCompression(stack, options); err != nil { + return err + } if err = stack.Initialize.Add(newServiceMetadataMiddleware_opPutWithContentEncoding(options.Region), middleware.Before); err != nil { return err } diff --git a/internal/protocoltest/ec2query/api_op_PutWithContentEncoding_test.go b/internal/protocoltest/ec2query/api_op_PutWithContentEncoding_test.go index aa4434875c0..c4aa7a2eee4 100644 --- a/internal/protocoltest/ec2query/api_op_PutWithContentEncoding_test.go +++ b/internal/protocoltest/ec2query/api_op_PutWithContentEncoding_test.go @@ -62,14 +62,6 @@ func TestClient_PutWithContentEncoding_awsEc2querySerialize(t *testing.T) { } for name, c := range cases { t.Run(name, func(t *testing.T) { - if name == "SDKAppliedContentEncoding_ec2Query" { - t.Skip("disabled test aws.protocoltests.ec2#AwsEc2 aws.protocoltests.ec2#PutWithContentEncoding") - } - - if name == "SDKAppendsGzipAndIgnoresHttpProvidedEncoding_ec2Query" { - t.Skip("disabled test aws.protocoltests.ec2#AwsEc2 aws.protocoltests.ec2#PutWithContentEncoding") - } - actualReq := &http.Request{} serverURL := "http://localhost:8888/" if c.Host != nil { diff --git a/internal/protocoltest/ec2query/options.go b/internal/protocoltest/ec2query/options.go index 28519d9971a..b32d4862b59 100644 --- a/internal/protocoltest/ec2query/options.go +++ b/internal/protocoltest/ec2query/options.go @@ -36,6 +36,9 @@ type Options struct { // clients initial default settings. DefaultsMode aws.DefaultsMode + // Whether to disable automatic request compression for supported operations. + DisableRequestCompression bool + // The endpoint options to be used when attempting to resolve an endpoint. EndpointOptions EndpointResolverOptions @@ -62,6 +65,10 @@ type Options struct { // The region to send requests to. (Required) Region string + // Inclusive threshold request body size to trigger compression, default to 10240 + // and must be within 0 and 10485760 bytes inclusively + RequestMinCompressSizeBytes int64 + // RetryMaxAttempts specifies the maximum number attempts an API client will call // an operation that fails with a retryable error. A value of 0 is ignored, and // will not be used to configure the API client created default retryer, or modify diff --git a/internal/protocoltest/jsonrpc/api_client.go b/internal/protocoltest/jsonrpc/api_client.go index 4314c505fee..df59e71750f 100644 --- a/internal/protocoltest/jsonrpc/api_client.go +++ b/internal/protocoltest/jsonrpc/api_client.go @@ -18,6 +18,7 @@ import ( smithydocument "github.com/aws/smithy-go/document" "github.com/aws/smithy-go/logging" "github.com/aws/smithy-go/middleware" + smithyrequestcompression "github.com/aws/smithy-go/private/requestcompression" smithyhttp "github.com/aws/smithy-go/transport/http" "net" "net/http" @@ -217,15 +218,17 @@ func setResolvedDefaultsMode(o *Options) { // NewFromConfig returns a new client from the provided config. func NewFromConfig(cfg aws.Config, optFns ...func(*Options)) *Client { opts := Options{ - Region: cfg.Region, - DefaultsMode: cfg.DefaultsMode, - RuntimeEnvironment: cfg.RuntimeEnvironment, - HTTPClient: cfg.HTTPClient, - Credentials: cfg.Credentials, - APIOptions: cfg.APIOptions, - Logger: cfg.Logger, - ClientLogMode: cfg.ClientLogMode, - AppID: cfg.AppID, + Region: cfg.Region, + DefaultsMode: cfg.DefaultsMode, + RuntimeEnvironment: cfg.RuntimeEnvironment, + HTTPClient: cfg.HTTPClient, + Credentials: cfg.Credentials, + APIOptions: cfg.APIOptions, + Logger: cfg.Logger, + ClientLogMode: cfg.ClientLogMode, + AppID: cfg.AppID, + DisableRequestCompression: cfg.DisableRequestCompression, + RequestMinCompressSizeBytes: cfg.RequestMinCompressSizeBytes, } resolveAWSRetryerProvider(cfg, &opts) resolveAWSRetryMaxAttempts(cfg, &opts) @@ -408,6 +411,11 @@ func resolveUseFIPSEndpoint(cfg aws.Config, o *Options) error { return nil } +func addRequestCompression(stack *middleware.Stack, options Options) error { + return smithyrequestcompression.AddRequestCompression(stack, options.DisableRequestCompression, options.RequestMinCompressSizeBytes, + "gzip") +} + func addRequestIDRetrieverMiddleware(stack *middleware.Stack) error { return awsmiddleware.AddRequestIDRetrieverMiddleware(stack) } diff --git a/internal/protocoltest/jsonrpc/api_op_PutWithContentEncoding.go b/internal/protocoltest/jsonrpc/api_op_PutWithContentEncoding.go index e667a9759f3..701a96c36e7 100644 --- a/internal/protocoltest/jsonrpc/api_op_PutWithContentEncoding.go +++ b/internal/protocoltest/jsonrpc/api_op_PutWithContentEncoding.go @@ -96,6 +96,9 @@ func (c *Client) addOperationPutWithContentEncodingMiddlewares(stack *middleware if err = addSetLegacyContextSigningOptionsMiddleware(stack); err != nil { return err } + if err = addRequestCompression(stack, options); err != nil { + return err + } if err = stack.Initialize.Add(newServiceMetadataMiddleware_opPutWithContentEncoding(options.Region), middleware.Before); err != nil { return err } diff --git a/internal/protocoltest/jsonrpc/api_op_PutWithContentEncoding_test.go b/internal/protocoltest/jsonrpc/api_op_PutWithContentEncoding_test.go index a89d4b8cc01..5f8e353057f 100644 --- a/internal/protocoltest/jsonrpc/api_op_PutWithContentEncoding_test.go +++ b/internal/protocoltest/jsonrpc/api_op_PutWithContentEncoding_test.go @@ -61,14 +61,6 @@ func TestClient_PutWithContentEncoding_awsAwsjson11Serialize(t *testing.T) { } for name, c := range cases { t.Run(name, func(t *testing.T) { - if name == "SDKAppliedContentEncoding_awsJson1_1" { - t.Skip("disabled test aws.protocoltests.json#JsonProtocol aws.protocoltests.json#PutWithContentEncoding") - } - - if name == "SDKAppendsGzipAndIgnoresHttpProvidedEncoding_awsJson1_1" { - t.Skip("disabled test aws.protocoltests.json#JsonProtocol aws.protocoltests.json#PutWithContentEncoding") - } - actualReq := &http.Request{} serverURL := "http://localhost:8888/" if c.Host != nil { diff --git a/internal/protocoltest/jsonrpc/options.go b/internal/protocoltest/jsonrpc/options.go index 881ecd9b447..5f8fb155bd2 100644 --- a/internal/protocoltest/jsonrpc/options.go +++ b/internal/protocoltest/jsonrpc/options.go @@ -42,6 +42,9 @@ type Options struct { // clients initial default settings. DefaultsMode aws.DefaultsMode + // Whether to disable automatic request compression for supported operations. + DisableRequestCompression bool + // The endpoint options to be used when attempting to resolve an endpoint. EndpointOptions EndpointResolverOptions @@ -67,6 +70,10 @@ type Options struct { // The region to send requests to. (Required) Region string + // Inclusive threshold request body size to trigger compression, default to 10240 + // and must be within 0 and 10485760 bytes inclusively + RequestMinCompressSizeBytes int64 + // RetryMaxAttempts specifies the maximum number attempts an API client will call // an operation that fails with a retryable error. A value of 0 is ignored, and // will not be used to configure the API client created default retryer, or modify diff --git a/internal/protocoltest/jsonrpc10/api_client.go b/internal/protocoltest/jsonrpc10/api_client.go index e5923d03f9f..71748a0bd71 100644 --- a/internal/protocoltest/jsonrpc10/api_client.go +++ b/internal/protocoltest/jsonrpc10/api_client.go @@ -15,6 +15,7 @@ import ( smithydocument "github.com/aws/smithy-go/document" "github.com/aws/smithy-go/logging" "github.com/aws/smithy-go/middleware" + smithyrequestcompression "github.com/aws/smithy-go/private/requestcompression" smithyhttp "github.com/aws/smithy-go/transport/http" "net" "net/http" @@ -202,14 +203,16 @@ func setResolvedDefaultsMode(o *Options) { // NewFromConfig returns a new client from the provided config. func NewFromConfig(cfg aws.Config, optFns ...func(*Options)) *Client { opts := Options{ - Region: cfg.Region, - DefaultsMode: cfg.DefaultsMode, - RuntimeEnvironment: cfg.RuntimeEnvironment, - HTTPClient: cfg.HTTPClient, - APIOptions: cfg.APIOptions, - Logger: cfg.Logger, - ClientLogMode: cfg.ClientLogMode, - AppID: cfg.AppID, + Region: cfg.Region, + DefaultsMode: cfg.DefaultsMode, + RuntimeEnvironment: cfg.RuntimeEnvironment, + HTTPClient: cfg.HTTPClient, + APIOptions: cfg.APIOptions, + Logger: cfg.Logger, + ClientLogMode: cfg.ClientLogMode, + AppID: cfg.AppID, + DisableRequestCompression: cfg.DisableRequestCompression, + RequestMinCompressSizeBytes: cfg.RequestMinCompressSizeBytes, } resolveAWSRetryerProvider(cfg, &opts) resolveAWSRetryMaxAttempts(cfg, &opts) @@ -374,6 +377,11 @@ func resolveUseFIPSEndpoint(cfg aws.Config, o *Options) error { return nil } +func addRequestCompression(stack *middleware.Stack, options Options) error { + return smithyrequestcompression.AddRequestCompression(stack, options.DisableRequestCompression, options.RequestMinCompressSizeBytes, + "gzip") +} + func addRequestIDRetrieverMiddleware(stack *middleware.Stack) error { return awsmiddleware.AddRequestIDRetrieverMiddleware(stack) } diff --git a/internal/protocoltest/jsonrpc10/api_op_PutWithContentEncoding.go b/internal/protocoltest/jsonrpc10/api_op_PutWithContentEncoding.go index a6c07e8c478..4ec5315463f 100644 --- a/internal/protocoltest/jsonrpc10/api_op_PutWithContentEncoding.go +++ b/internal/protocoltest/jsonrpc10/api_op_PutWithContentEncoding.go @@ -92,6 +92,9 @@ func (c *Client) addOperationPutWithContentEncodingMiddlewares(stack *middleware if err = addSetLegacyContextSigningOptionsMiddleware(stack); err != nil { return err } + if err = addRequestCompression(stack, options); err != nil { + return err + } if err = stack.Initialize.Add(newServiceMetadataMiddleware_opPutWithContentEncoding(options.Region), middleware.Before); err != nil { return err } diff --git a/internal/protocoltest/jsonrpc10/api_op_PutWithContentEncoding_test.go b/internal/protocoltest/jsonrpc10/api_op_PutWithContentEncoding_test.go index e930a32b999..bb5aff04953 100644 --- a/internal/protocoltest/jsonrpc10/api_op_PutWithContentEncoding_test.go +++ b/internal/protocoltest/jsonrpc10/api_op_PutWithContentEncoding_test.go @@ -61,14 +61,6 @@ func TestClient_PutWithContentEncoding_awsAwsjson10Serialize(t *testing.T) { } for name, c := range cases { t.Run(name, func(t *testing.T) { - if name == "SDKAppliedContentEncoding_awsJson1_0" { - t.Skip("disabled test aws.protocoltests.json10#JsonRpc10 aws.protocoltests.json10#PutWithContentEncoding") - } - - if name == "SDKAppendsGzipAndIgnoresHttpProvidedEncoding_awsJson1_0" { - t.Skip("disabled test aws.protocoltests.json10#JsonRpc10 aws.protocoltests.json10#PutWithContentEncoding") - } - actualReq := &http.Request{} serverURL := "http://localhost:8888/" if c.Host != nil { diff --git a/internal/protocoltest/jsonrpc10/options.go b/internal/protocoltest/jsonrpc10/options.go index 48c6c62e654..67c2c683692 100644 --- a/internal/protocoltest/jsonrpc10/options.go +++ b/internal/protocoltest/jsonrpc10/options.go @@ -36,6 +36,9 @@ type Options struct { // clients initial default settings. DefaultsMode aws.DefaultsMode + // Whether to disable automatic request compression for supported operations. + DisableRequestCompression bool + // The endpoint options to be used when attempting to resolve an endpoint. EndpointOptions EndpointResolverOptions @@ -58,6 +61,10 @@ type Options struct { // The region to send requests to. (Required) Region string + // Inclusive threshold request body size to trigger compression, default to 10240 + // and must be within 0 and 10485760 bytes inclusively + RequestMinCompressSizeBytes int64 + // RetryMaxAttempts specifies the maximum number attempts an API client will call // an operation that fails with a retryable error. A value of 0 is ignored, and // will not be used to configure the API client created default retryer, or modify diff --git a/internal/protocoltest/query/api_client.go b/internal/protocoltest/query/api_client.go index a3bc51d9fff..a62926ba681 100644 --- a/internal/protocoltest/query/api_client.go +++ b/internal/protocoltest/query/api_client.go @@ -16,6 +16,7 @@ import ( smithydocument "github.com/aws/smithy-go/document" "github.com/aws/smithy-go/logging" "github.com/aws/smithy-go/middleware" + smithyrequestcompression "github.com/aws/smithy-go/private/requestcompression" smithyrand "github.com/aws/smithy-go/rand" smithyhttp "github.com/aws/smithy-go/transport/http" "net" @@ -206,14 +207,16 @@ func setResolvedDefaultsMode(o *Options) { // NewFromConfig returns a new client from the provided config. func NewFromConfig(cfg aws.Config, optFns ...func(*Options)) *Client { opts := Options{ - Region: cfg.Region, - DefaultsMode: cfg.DefaultsMode, - RuntimeEnvironment: cfg.RuntimeEnvironment, - HTTPClient: cfg.HTTPClient, - APIOptions: cfg.APIOptions, - Logger: cfg.Logger, - ClientLogMode: cfg.ClientLogMode, - AppID: cfg.AppID, + Region: cfg.Region, + DefaultsMode: cfg.DefaultsMode, + RuntimeEnvironment: cfg.RuntimeEnvironment, + HTTPClient: cfg.HTTPClient, + APIOptions: cfg.APIOptions, + Logger: cfg.Logger, + ClientLogMode: cfg.ClientLogMode, + AppID: cfg.AppID, + DisableRequestCompression: cfg.DisableRequestCompression, + RequestMinCompressSizeBytes: cfg.RequestMinCompressSizeBytes, } resolveAWSRetryerProvider(cfg, &opts) resolveAWSRetryMaxAttempts(cfg, &opts) @@ -385,6 +388,11 @@ func resolveUseFIPSEndpoint(cfg aws.Config, o *Options) error { return nil } +func addRequestCompression(stack *middleware.Stack, options Options) error { + return smithyrequestcompression.AddRequestCompression(stack, options.DisableRequestCompression, options.RequestMinCompressSizeBytes, + "gzip") +} + // IdempotencyTokenProvider interface for providing idempotency token type IdempotencyTokenProvider interface { GetIdempotencyToken() (string, error) diff --git a/internal/protocoltest/query/api_op_PutWithContentEncoding.go b/internal/protocoltest/query/api_op_PutWithContentEncoding.go index 858373d7b75..9518f5b41e9 100644 --- a/internal/protocoltest/query/api_op_PutWithContentEncoding.go +++ b/internal/protocoltest/query/api_op_PutWithContentEncoding.go @@ -92,6 +92,9 @@ func (c *Client) addOperationPutWithContentEncodingMiddlewares(stack *middleware if err = addSetLegacyContextSigningOptionsMiddleware(stack); err != nil { return err } + if err = addRequestCompression(stack, options); err != nil { + return err + } if err = stack.Initialize.Add(newServiceMetadataMiddleware_opPutWithContentEncoding(options.Region), middleware.Before); err != nil { return err } diff --git a/internal/protocoltest/query/api_op_PutWithContentEncoding_test.go b/internal/protocoltest/query/api_op_PutWithContentEncoding_test.go index 31c033483ce..040d583408b 100644 --- a/internal/protocoltest/query/api_op_PutWithContentEncoding_test.go +++ b/internal/protocoltest/query/api_op_PutWithContentEncoding_test.go @@ -62,14 +62,6 @@ func TestClient_PutWithContentEncoding_awsAwsquerySerialize(t *testing.T) { } for name, c := range cases { t.Run(name, func(t *testing.T) { - if name == "SDKAppliedContentEncoding_awsQuery" { - t.Skip("disabled test aws.protocoltests.query#AwsQuery aws.protocoltests.query#PutWithContentEncoding") - } - - if name == "SDKAppendsGzipAndIgnoresHttpProvidedEncoding_awsQuery" { - t.Skip("disabled test aws.protocoltests.query#AwsQuery aws.protocoltests.query#PutWithContentEncoding") - } - actualReq := &http.Request{} serverURL := "http://localhost:8888/" if c.Host != nil { diff --git a/internal/protocoltest/query/options.go b/internal/protocoltest/query/options.go index 03d34b92ed1..1fcd86e987d 100644 --- a/internal/protocoltest/query/options.go +++ b/internal/protocoltest/query/options.go @@ -36,6 +36,9 @@ type Options struct { // clients initial default settings. DefaultsMode aws.DefaultsMode + // Whether to disable automatic request compression for supported operations. + DisableRequestCompression bool + // The endpoint options to be used when attempting to resolve an endpoint. EndpointOptions EndpointResolverOptions @@ -62,6 +65,10 @@ type Options struct { // The region to send requests to. (Required) Region string + // Inclusive threshold request body size to trigger compression, default to 10240 + // and must be within 0 and 10485760 bytes inclusively + RequestMinCompressSizeBytes int64 + // RetryMaxAttempts specifies the maximum number attempts an API client will call // an operation that fails with a retryable error. A value of 0 is ignored, and // will not be used to configure the API client created default retryer, or modify diff --git a/internal/protocoltest/restxml/api_client.go b/internal/protocoltest/restxml/api_client.go index d1d31c7b9e7..be34ee868cb 100644 --- a/internal/protocoltest/restxml/api_client.go +++ b/internal/protocoltest/restxml/api_client.go @@ -16,6 +16,7 @@ import ( smithydocument "github.com/aws/smithy-go/document" "github.com/aws/smithy-go/logging" "github.com/aws/smithy-go/middleware" + smithyrequestcompression "github.com/aws/smithy-go/private/requestcompression" smithyrand "github.com/aws/smithy-go/rand" smithyhttp "github.com/aws/smithy-go/transport/http" "net" @@ -206,14 +207,16 @@ func setResolvedDefaultsMode(o *Options) { // NewFromConfig returns a new client from the provided config. func NewFromConfig(cfg aws.Config, optFns ...func(*Options)) *Client { opts := Options{ - Region: cfg.Region, - DefaultsMode: cfg.DefaultsMode, - RuntimeEnvironment: cfg.RuntimeEnvironment, - HTTPClient: cfg.HTTPClient, - APIOptions: cfg.APIOptions, - Logger: cfg.Logger, - ClientLogMode: cfg.ClientLogMode, - AppID: cfg.AppID, + Region: cfg.Region, + DefaultsMode: cfg.DefaultsMode, + RuntimeEnvironment: cfg.RuntimeEnvironment, + HTTPClient: cfg.HTTPClient, + APIOptions: cfg.APIOptions, + Logger: cfg.Logger, + ClientLogMode: cfg.ClientLogMode, + AppID: cfg.AppID, + DisableRequestCompression: cfg.DisableRequestCompression, + RequestMinCompressSizeBytes: cfg.RequestMinCompressSizeBytes, } resolveAWSRetryerProvider(cfg, &opts) resolveAWSRetryMaxAttempts(cfg, &opts) @@ -385,6 +388,11 @@ func resolveUseFIPSEndpoint(cfg aws.Config, o *Options) error { return nil } +func addRequestCompression(stack *middleware.Stack, options Options) error { + return smithyrequestcompression.AddRequestCompression(stack, options.DisableRequestCompression, options.RequestMinCompressSizeBytes, + "gzip") +} + // IdempotencyTokenProvider interface for providing idempotency token type IdempotencyTokenProvider interface { GetIdempotencyToken() (string, error) diff --git a/internal/protocoltest/restxml/api_op_PutWithContentEncoding.go b/internal/protocoltest/restxml/api_op_PutWithContentEncoding.go index d7663ec01ba..c9c46387ff4 100644 --- a/internal/protocoltest/restxml/api_op_PutWithContentEncoding.go +++ b/internal/protocoltest/restxml/api_op_PutWithContentEncoding.go @@ -92,6 +92,9 @@ func (c *Client) addOperationPutWithContentEncodingMiddlewares(stack *middleware if err = addSetLegacyContextSigningOptionsMiddleware(stack); err != nil { return err } + if err = addRequestCompression(stack, options); err != nil { + return err + } if err = stack.Initialize.Add(newServiceMetadataMiddleware_opPutWithContentEncoding(options.Region), middleware.Before); err != nil { return err } diff --git a/internal/protocoltest/restxml/api_op_PutWithContentEncoding_test.go b/internal/protocoltest/restxml/api_op_PutWithContentEncoding_test.go index 571b34f314e..9dd09be9fc1 100644 --- a/internal/protocoltest/restxml/api_op_PutWithContentEncoding_test.go +++ b/internal/protocoltest/restxml/api_op_PutWithContentEncoding_test.go @@ -62,14 +62,6 @@ func TestClient_PutWithContentEncoding_awsRestxmlSerialize(t *testing.T) { } for name, c := range cases { t.Run(name, func(t *testing.T) { - if name == "SDKAppliedContentEncoding_restXml" { - t.Skip("disabled test aws.protocoltests.restxml#RestXml aws.protocoltests.restxml#PutWithContentEncoding") - } - - if name == "SDKAppendedGzipAfterProvidedEncoding_restXml" { - t.Skip("disabled test aws.protocoltests.restxml#RestXml aws.protocoltests.restxml#PutWithContentEncoding") - } - actualReq := &http.Request{} serverURL := "http://localhost:8888/" if c.Host != nil { diff --git a/internal/protocoltest/restxml/options.go b/internal/protocoltest/restxml/options.go index ebab9939b6e..0f02a869eb0 100644 --- a/internal/protocoltest/restxml/options.go +++ b/internal/protocoltest/restxml/options.go @@ -36,6 +36,9 @@ type Options struct { // clients initial default settings. DefaultsMode aws.DefaultsMode + // Whether to disable automatic request compression for supported operations. + DisableRequestCompression bool + // The endpoint options to be used when attempting to resolve an endpoint. EndpointOptions EndpointResolverOptions @@ -62,6 +65,10 @@ type Options struct { // The region to send requests to. (Required) Region string + // Inclusive threshold request body size to trigger compression, default to 10240 + // and must be within 0 and 10485760 bytes inclusively + RequestMinCompressSizeBytes int64 + // RetryMaxAttempts specifies the maximum number attempts an API client will call // an operation that fails with a retryable error. A value of 0 is ignored, and // will not be used to configure the API client created default retryer, or modify From 703c6e9d651c0dbb8233b96e57d770a92b5f74bc Mon Sep 17 00:00:00 2001 From: Tianyi Wang Date: Wed, 22 Nov 2023 18:08:18 -0500 Subject: [PATCH 2/7] Add changelog --- .changelog/a96149e35456411aaca3224016feff3c.json | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 .changelog/a96149e35456411aaca3224016feff3c.json diff --git a/.changelog/a96149e35456411aaca3224016feff3c.json b/.changelog/a96149e35456411aaca3224016feff3c.json new file mode 100644 index 00000000000..bb52d01af81 --- /dev/null +++ b/.changelog/a96149e35456411aaca3224016feff3c.json @@ -0,0 +1,16 @@ +{ + "id": "a96149e3-5456-411a-aca3-224016feff3c", + "type": "feature", + "collapse": true, + "description": "Support modeled request compression. The only algorithm supported at this time is `gzip`.", + "modules": [ + ".", + "config", + "internal/protocoltest/awsrestjson", + "internal/protocoltest/ec2query", + "internal/protocoltest/jsonrpc", + "internal/protocoltest/jsonrpc10", + "internal/protocoltest/query", + "internal/protocoltest/restxml" + ] +} \ No newline at end of file From a39d14369de477e3521c093b2fb8247463ad9cb9 Mon Sep 17 00:00:00 2001 From: Tianyi Wang Date: Tue, 28 Nov 2023 13:41:38 -0500 Subject: [PATCH 3/7] Change request compression codegen to operation level --- config/env_config_test.go | 1 + internal/protocoltest/awsrestjson/api_client.go | 6 ------ .../awsrestjson/api_op_PutWithContentEncoding.go | 8 +++++++- internal/protocoltest/ec2query/api_client.go | 6 ------ .../ec2query/api_op_PutWithContentEncoding.go | 8 +++++++- internal/protocoltest/jsonrpc/api_client.go | 6 ------ .../protocoltest/jsonrpc/api_op_PutWithContentEncoding.go | 8 +++++++- internal/protocoltest/jsonrpc10/api_client.go | 6 ------ .../jsonrpc10/api_op_PutWithContentEncoding.go | 8 +++++++- internal/protocoltest/query/api_client.go | 6 ------ .../protocoltest/query/api_op_PutWithContentEncoding.go | 8 +++++++- internal/protocoltest/restxml/api_client.go | 6 ------ .../protocoltest/restxml/api_op_PutWithContentEncoding.go | 8 +++++++- 13 files changed, 43 insertions(+), 42 deletions(-) diff --git a/config/env_config_test.go b/config/env_config_test.go index af232da8b18..bf874552b4b 100644 --- a/config/env_config_test.go +++ b/config/env_config_test.go @@ -486,6 +486,7 @@ func TestNewEnvConfig(t *testing.T) { }, WantErr: true, }, + // expect err detected due to AWS_REQUEST_MIN_COMPRESSION_SIZE_BYTES exceeding max 10485760 45: { Env: map[string]string{ "AWS_DISABLE_REQUEST_COMPRESSION": "false", diff --git a/internal/protocoltest/awsrestjson/api_client.go b/internal/protocoltest/awsrestjson/api_client.go index 28ce7e8dbd7..4705a8ee742 100644 --- a/internal/protocoltest/awsrestjson/api_client.go +++ b/internal/protocoltest/awsrestjson/api_client.go @@ -16,7 +16,6 @@ import ( smithydocument "github.com/aws/smithy-go/document" "github.com/aws/smithy-go/logging" "github.com/aws/smithy-go/middleware" - smithyrequestcompression "github.com/aws/smithy-go/private/requestcompression" smithyrand "github.com/aws/smithy-go/rand" smithyhttp "github.com/aws/smithy-go/transport/http" "net" @@ -388,11 +387,6 @@ func resolveUseFIPSEndpoint(cfg aws.Config, o *Options) error { return nil } -func addRequestCompression(stack *middleware.Stack, options Options) error { - return smithyrequestcompression.AddRequestCompression(stack, options.DisableRequestCompression, options.RequestMinCompressSizeBytes, - "gzip") -} - // IdempotencyTokenProvider interface for providing idempotency token type IdempotencyTokenProvider interface { GetIdempotencyToken() (string, error) diff --git a/internal/protocoltest/awsrestjson/api_op_PutWithContentEncoding.go b/internal/protocoltest/awsrestjson/api_op_PutWithContentEncoding.go index e92e4ca40ce..01ef7acc235 100644 --- a/internal/protocoltest/awsrestjson/api_op_PutWithContentEncoding.go +++ b/internal/protocoltest/awsrestjson/api_op_PutWithContentEncoding.go @@ -7,6 +7,7 @@ import ( "fmt" awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware" "github.com/aws/smithy-go/middleware" + smithyrequestcompression "github.com/aws/smithy-go/private/requestcompression" smithyhttp "github.com/aws/smithy-go/transport/http" ) @@ -92,7 +93,7 @@ func (c *Client) addOperationPutWithContentEncodingMiddlewares(stack *middleware if err = addSetLegacyContextSigningOptionsMiddleware(stack); err != nil { return err } - if err = addRequestCompression(stack, options); err != nil { + if err = addOperationPutWithContentEncodingRequestCompressionMiddleware(stack, options); err != nil { return err } if err = stack.Initialize.Add(newServiceMetadataMiddleware_opPutWithContentEncoding(options.Region), middleware.Before); err != nil { @@ -116,6 +117,11 @@ func (c *Client) addOperationPutWithContentEncodingMiddlewares(stack *middleware return nil } +func addOperationPutWithContentEncodingRequestCompressionMiddleware(stack *middleware.Stack, options Options) error { + return smithyrequestcompression.AddRequestCompression(stack, options.DisableRequestCompression, options.RequestMinCompressSizeBytes, + []string{"gzip"}) +} + func newServiceMetadataMiddleware_opPutWithContentEncoding(region string) *awsmiddleware.RegisterServiceMetadata { return &awsmiddleware.RegisterServiceMetadata{ Region: region, diff --git a/internal/protocoltest/ec2query/api_client.go b/internal/protocoltest/ec2query/api_client.go index 536505c1ed2..c6e8a595ba0 100644 --- a/internal/protocoltest/ec2query/api_client.go +++ b/internal/protocoltest/ec2query/api_client.go @@ -16,7 +16,6 @@ import ( smithydocument "github.com/aws/smithy-go/document" "github.com/aws/smithy-go/logging" "github.com/aws/smithy-go/middleware" - smithyrequestcompression "github.com/aws/smithy-go/private/requestcompression" smithyrand "github.com/aws/smithy-go/rand" smithyhttp "github.com/aws/smithy-go/transport/http" "net" @@ -388,11 +387,6 @@ func resolveUseFIPSEndpoint(cfg aws.Config, o *Options) error { return nil } -func addRequestCompression(stack *middleware.Stack, options Options) error { - return smithyrequestcompression.AddRequestCompression(stack, options.DisableRequestCompression, options.RequestMinCompressSizeBytes, - "gzip") -} - // IdempotencyTokenProvider interface for providing idempotency token type IdempotencyTokenProvider interface { GetIdempotencyToken() (string, error) diff --git a/internal/protocoltest/ec2query/api_op_PutWithContentEncoding.go b/internal/protocoltest/ec2query/api_op_PutWithContentEncoding.go index df42870ce81..798fd9e980b 100644 --- a/internal/protocoltest/ec2query/api_op_PutWithContentEncoding.go +++ b/internal/protocoltest/ec2query/api_op_PutWithContentEncoding.go @@ -7,6 +7,7 @@ import ( "fmt" awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware" "github.com/aws/smithy-go/middleware" + smithyrequestcompression "github.com/aws/smithy-go/private/requestcompression" smithyhttp "github.com/aws/smithy-go/transport/http" ) @@ -92,7 +93,7 @@ func (c *Client) addOperationPutWithContentEncodingMiddlewares(stack *middleware if err = addSetLegacyContextSigningOptionsMiddleware(stack); err != nil { return err } - if err = addRequestCompression(stack, options); err != nil { + if err = addOperationPutWithContentEncodingRequestCompressionMiddleware(stack, options); err != nil { return err } if err = stack.Initialize.Add(newServiceMetadataMiddleware_opPutWithContentEncoding(options.Region), middleware.Before); err != nil { @@ -116,6 +117,11 @@ func (c *Client) addOperationPutWithContentEncodingMiddlewares(stack *middleware return nil } +func addOperationPutWithContentEncodingRequestCompressionMiddleware(stack *middleware.Stack, options Options) error { + return smithyrequestcompression.AddRequestCompression(stack, options.DisableRequestCompression, options.RequestMinCompressSizeBytes, + []string{"gzip"}) +} + func newServiceMetadataMiddleware_opPutWithContentEncoding(region string) *awsmiddleware.RegisterServiceMetadata { return &awsmiddleware.RegisterServiceMetadata{ Region: region, diff --git a/internal/protocoltest/jsonrpc/api_client.go b/internal/protocoltest/jsonrpc/api_client.go index df59e71750f..afa19b368a6 100644 --- a/internal/protocoltest/jsonrpc/api_client.go +++ b/internal/protocoltest/jsonrpc/api_client.go @@ -18,7 +18,6 @@ import ( smithydocument "github.com/aws/smithy-go/document" "github.com/aws/smithy-go/logging" "github.com/aws/smithy-go/middleware" - smithyrequestcompression "github.com/aws/smithy-go/private/requestcompression" smithyhttp "github.com/aws/smithy-go/transport/http" "net" "net/http" @@ -411,11 +410,6 @@ func resolveUseFIPSEndpoint(cfg aws.Config, o *Options) error { return nil } -func addRequestCompression(stack *middleware.Stack, options Options) error { - return smithyrequestcompression.AddRequestCompression(stack, options.DisableRequestCompression, options.RequestMinCompressSizeBytes, - "gzip") -} - func addRequestIDRetrieverMiddleware(stack *middleware.Stack) error { return awsmiddleware.AddRequestIDRetrieverMiddleware(stack) } diff --git a/internal/protocoltest/jsonrpc/api_op_PutWithContentEncoding.go b/internal/protocoltest/jsonrpc/api_op_PutWithContentEncoding.go index 701a96c36e7..7cdc7e9f081 100644 --- a/internal/protocoltest/jsonrpc/api_op_PutWithContentEncoding.go +++ b/internal/protocoltest/jsonrpc/api_op_PutWithContentEncoding.go @@ -8,6 +8,7 @@ import ( awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware" "github.com/aws/aws-sdk-go-v2/aws/signer/v4" "github.com/aws/smithy-go/middleware" + smithyrequestcompression "github.com/aws/smithy-go/private/requestcompression" smithyhttp "github.com/aws/smithy-go/transport/http" ) @@ -96,7 +97,7 @@ func (c *Client) addOperationPutWithContentEncodingMiddlewares(stack *middleware if err = addSetLegacyContextSigningOptionsMiddleware(stack); err != nil { return err } - if err = addRequestCompression(stack, options); err != nil { + if err = addOperationPutWithContentEncodingRequestCompressionMiddleware(stack, options); err != nil { return err } if err = stack.Initialize.Add(newServiceMetadataMiddleware_opPutWithContentEncoding(options.Region), middleware.Before); err != nil { @@ -120,6 +121,11 @@ func (c *Client) addOperationPutWithContentEncodingMiddlewares(stack *middleware return nil } +func addOperationPutWithContentEncodingRequestCompressionMiddleware(stack *middleware.Stack, options Options) error { + return smithyrequestcompression.AddRequestCompression(stack, options.DisableRequestCompression, options.RequestMinCompressSizeBytes, + []string{"gzip"}) +} + func newServiceMetadataMiddleware_opPutWithContentEncoding(region string) *awsmiddleware.RegisterServiceMetadata { return &awsmiddleware.RegisterServiceMetadata{ Region: region, diff --git a/internal/protocoltest/jsonrpc10/api_client.go b/internal/protocoltest/jsonrpc10/api_client.go index 71748a0bd71..9e25a8871c7 100644 --- a/internal/protocoltest/jsonrpc10/api_client.go +++ b/internal/protocoltest/jsonrpc10/api_client.go @@ -15,7 +15,6 @@ import ( smithydocument "github.com/aws/smithy-go/document" "github.com/aws/smithy-go/logging" "github.com/aws/smithy-go/middleware" - smithyrequestcompression "github.com/aws/smithy-go/private/requestcompression" smithyhttp "github.com/aws/smithy-go/transport/http" "net" "net/http" @@ -377,11 +376,6 @@ func resolveUseFIPSEndpoint(cfg aws.Config, o *Options) error { return nil } -func addRequestCompression(stack *middleware.Stack, options Options) error { - return smithyrequestcompression.AddRequestCompression(stack, options.DisableRequestCompression, options.RequestMinCompressSizeBytes, - "gzip") -} - func addRequestIDRetrieverMiddleware(stack *middleware.Stack) error { return awsmiddleware.AddRequestIDRetrieverMiddleware(stack) } diff --git a/internal/protocoltest/jsonrpc10/api_op_PutWithContentEncoding.go b/internal/protocoltest/jsonrpc10/api_op_PutWithContentEncoding.go index 4ec5315463f..79b8ced7f46 100644 --- a/internal/protocoltest/jsonrpc10/api_op_PutWithContentEncoding.go +++ b/internal/protocoltest/jsonrpc10/api_op_PutWithContentEncoding.go @@ -7,6 +7,7 @@ import ( "fmt" awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware" "github.com/aws/smithy-go/middleware" + smithyrequestcompression "github.com/aws/smithy-go/private/requestcompression" smithyhttp "github.com/aws/smithy-go/transport/http" ) @@ -92,7 +93,7 @@ func (c *Client) addOperationPutWithContentEncodingMiddlewares(stack *middleware if err = addSetLegacyContextSigningOptionsMiddleware(stack); err != nil { return err } - if err = addRequestCompression(stack, options); err != nil { + if err = addOperationPutWithContentEncodingRequestCompressionMiddleware(stack, options); err != nil { return err } if err = stack.Initialize.Add(newServiceMetadataMiddleware_opPutWithContentEncoding(options.Region), middleware.Before); err != nil { @@ -116,6 +117,11 @@ func (c *Client) addOperationPutWithContentEncodingMiddlewares(stack *middleware return nil } +func addOperationPutWithContentEncodingRequestCompressionMiddleware(stack *middleware.Stack, options Options) error { + return smithyrequestcompression.AddRequestCompression(stack, options.DisableRequestCompression, options.RequestMinCompressSizeBytes, + []string{"gzip"}) +} + func newServiceMetadataMiddleware_opPutWithContentEncoding(region string) *awsmiddleware.RegisterServiceMetadata { return &awsmiddleware.RegisterServiceMetadata{ Region: region, diff --git a/internal/protocoltest/query/api_client.go b/internal/protocoltest/query/api_client.go index a62926ba681..c6d7758d43d 100644 --- a/internal/protocoltest/query/api_client.go +++ b/internal/protocoltest/query/api_client.go @@ -16,7 +16,6 @@ import ( smithydocument "github.com/aws/smithy-go/document" "github.com/aws/smithy-go/logging" "github.com/aws/smithy-go/middleware" - smithyrequestcompression "github.com/aws/smithy-go/private/requestcompression" smithyrand "github.com/aws/smithy-go/rand" smithyhttp "github.com/aws/smithy-go/transport/http" "net" @@ -388,11 +387,6 @@ func resolveUseFIPSEndpoint(cfg aws.Config, o *Options) error { return nil } -func addRequestCompression(stack *middleware.Stack, options Options) error { - return smithyrequestcompression.AddRequestCompression(stack, options.DisableRequestCompression, options.RequestMinCompressSizeBytes, - "gzip") -} - // IdempotencyTokenProvider interface for providing idempotency token type IdempotencyTokenProvider interface { GetIdempotencyToken() (string, error) diff --git a/internal/protocoltest/query/api_op_PutWithContentEncoding.go b/internal/protocoltest/query/api_op_PutWithContentEncoding.go index 9518f5b41e9..5778a94674f 100644 --- a/internal/protocoltest/query/api_op_PutWithContentEncoding.go +++ b/internal/protocoltest/query/api_op_PutWithContentEncoding.go @@ -7,6 +7,7 @@ import ( "fmt" awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware" "github.com/aws/smithy-go/middleware" + smithyrequestcompression "github.com/aws/smithy-go/private/requestcompression" smithyhttp "github.com/aws/smithy-go/transport/http" ) @@ -92,7 +93,7 @@ func (c *Client) addOperationPutWithContentEncodingMiddlewares(stack *middleware if err = addSetLegacyContextSigningOptionsMiddleware(stack); err != nil { return err } - if err = addRequestCompression(stack, options); err != nil { + if err = addOperationPutWithContentEncodingRequestCompressionMiddleware(stack, options); err != nil { return err } if err = stack.Initialize.Add(newServiceMetadataMiddleware_opPutWithContentEncoding(options.Region), middleware.Before); err != nil { @@ -116,6 +117,11 @@ func (c *Client) addOperationPutWithContentEncodingMiddlewares(stack *middleware return nil } +func addOperationPutWithContentEncodingRequestCompressionMiddleware(stack *middleware.Stack, options Options) error { + return smithyrequestcompression.AddRequestCompression(stack, options.DisableRequestCompression, options.RequestMinCompressSizeBytes, + []string{"gzip"}) +} + func newServiceMetadataMiddleware_opPutWithContentEncoding(region string) *awsmiddleware.RegisterServiceMetadata { return &awsmiddleware.RegisterServiceMetadata{ Region: region, diff --git a/internal/protocoltest/restxml/api_client.go b/internal/protocoltest/restxml/api_client.go index be34ee868cb..a85553905f2 100644 --- a/internal/protocoltest/restxml/api_client.go +++ b/internal/protocoltest/restxml/api_client.go @@ -16,7 +16,6 @@ import ( smithydocument "github.com/aws/smithy-go/document" "github.com/aws/smithy-go/logging" "github.com/aws/smithy-go/middleware" - smithyrequestcompression "github.com/aws/smithy-go/private/requestcompression" smithyrand "github.com/aws/smithy-go/rand" smithyhttp "github.com/aws/smithy-go/transport/http" "net" @@ -388,11 +387,6 @@ func resolveUseFIPSEndpoint(cfg aws.Config, o *Options) error { return nil } -func addRequestCompression(stack *middleware.Stack, options Options) error { - return smithyrequestcompression.AddRequestCompression(stack, options.DisableRequestCompression, options.RequestMinCompressSizeBytes, - "gzip") -} - // IdempotencyTokenProvider interface for providing idempotency token type IdempotencyTokenProvider interface { GetIdempotencyToken() (string, error) diff --git a/internal/protocoltest/restxml/api_op_PutWithContentEncoding.go b/internal/protocoltest/restxml/api_op_PutWithContentEncoding.go index c9c46387ff4..d5d6b28f1ab 100644 --- a/internal/protocoltest/restxml/api_op_PutWithContentEncoding.go +++ b/internal/protocoltest/restxml/api_op_PutWithContentEncoding.go @@ -7,6 +7,7 @@ import ( "fmt" awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware" "github.com/aws/smithy-go/middleware" + smithyrequestcompression "github.com/aws/smithy-go/private/requestcompression" smithyhttp "github.com/aws/smithy-go/transport/http" ) @@ -92,7 +93,7 @@ func (c *Client) addOperationPutWithContentEncodingMiddlewares(stack *middleware if err = addSetLegacyContextSigningOptionsMiddleware(stack); err != nil { return err } - if err = addRequestCompression(stack, options); err != nil { + if err = addOperationPutWithContentEncodingRequestCompressionMiddleware(stack, options); err != nil { return err } if err = stack.Initialize.Add(newServiceMetadataMiddleware_opPutWithContentEncoding(options.Region), middleware.Before); err != nil { @@ -116,6 +117,11 @@ func (c *Client) addOperationPutWithContentEncodingMiddlewares(stack *middleware return nil } +func addOperationPutWithContentEncodingRequestCompressionMiddleware(stack *middleware.Stack, options Options) error { + return smithyrequestcompression.AddRequestCompression(stack, options.DisableRequestCompression, options.RequestMinCompressSizeBytes, + []string{"gzip"}) +} + func newServiceMetadataMiddleware_opPutWithContentEncoding(region string) *awsmiddleware.RegisterServiceMetadata { return &awsmiddleware.RegisterServiceMetadata{ Region: region, From 93fb405467dc571de83f743455bb4e8faa5476cf Mon Sep 17 00:00:00 2001 From: Tianyi Wang Date: Tue, 28 Nov 2023 14:40:56 -0500 Subject: [PATCH 4/7] Change requestion compression comment --- internal/protocoltest/awsrestjson/options.go | 4 ++-- internal/protocoltest/ec2query/options.go | 4 ++-- internal/protocoltest/jsonrpc/options.go | 4 ++-- internal/protocoltest/jsonrpc10/options.go | 4 ++-- internal/protocoltest/query/options.go | 4 ++-- internal/protocoltest/restxml/options.go | 4 ++-- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/internal/protocoltest/awsrestjson/options.go b/internal/protocoltest/awsrestjson/options.go index 0ffb6b769e9..0e164cc88a6 100644 --- a/internal/protocoltest/awsrestjson/options.go +++ b/internal/protocoltest/awsrestjson/options.go @@ -65,8 +65,8 @@ type Options struct { // The region to send requests to. (Required) Region string - // Inclusive threshold request body size to trigger compression, default to 10240 - // and must be within 0 and 10485760 bytes inclusively + // The minimum request body size, in bytes, at which compression should occur. The + // default value is 10 KiB. Values must fall within [0, 1MiB]. RequestMinCompressSizeBytes int64 // RetryMaxAttempts specifies the maximum number attempts an API client will call diff --git a/internal/protocoltest/ec2query/options.go b/internal/protocoltest/ec2query/options.go index b32d4862b59..b43e137b70f 100644 --- a/internal/protocoltest/ec2query/options.go +++ b/internal/protocoltest/ec2query/options.go @@ -65,8 +65,8 @@ type Options struct { // The region to send requests to. (Required) Region string - // Inclusive threshold request body size to trigger compression, default to 10240 - // and must be within 0 and 10485760 bytes inclusively + // The minimum request body size, in bytes, at which compression should occur. The + // default value is 10 KiB. Values must fall within [0, 1MiB]. RequestMinCompressSizeBytes int64 // RetryMaxAttempts specifies the maximum number attempts an API client will call diff --git a/internal/protocoltest/jsonrpc/options.go b/internal/protocoltest/jsonrpc/options.go index 5f8fb155bd2..9afe73b87c7 100644 --- a/internal/protocoltest/jsonrpc/options.go +++ b/internal/protocoltest/jsonrpc/options.go @@ -70,8 +70,8 @@ type Options struct { // The region to send requests to. (Required) Region string - // Inclusive threshold request body size to trigger compression, default to 10240 - // and must be within 0 and 10485760 bytes inclusively + // The minimum request body size, in bytes, at which compression should occur. The + // default value is 10 KiB. Values must fall within [0, 1MiB]. RequestMinCompressSizeBytes int64 // RetryMaxAttempts specifies the maximum number attempts an API client will call diff --git a/internal/protocoltest/jsonrpc10/options.go b/internal/protocoltest/jsonrpc10/options.go index 67c2c683692..289110712bc 100644 --- a/internal/protocoltest/jsonrpc10/options.go +++ b/internal/protocoltest/jsonrpc10/options.go @@ -61,8 +61,8 @@ type Options struct { // The region to send requests to. (Required) Region string - // Inclusive threshold request body size to trigger compression, default to 10240 - // and must be within 0 and 10485760 bytes inclusively + // The minimum request body size, in bytes, at which compression should occur. The + // default value is 10 KiB. Values must fall within [0, 1MiB]. RequestMinCompressSizeBytes int64 // RetryMaxAttempts specifies the maximum number attempts an API client will call diff --git a/internal/protocoltest/query/options.go b/internal/protocoltest/query/options.go index 1fcd86e987d..ff78bf6632b 100644 --- a/internal/protocoltest/query/options.go +++ b/internal/protocoltest/query/options.go @@ -65,8 +65,8 @@ type Options struct { // The region to send requests to. (Required) Region string - // Inclusive threshold request body size to trigger compression, default to 10240 - // and must be within 0 and 10485760 bytes inclusively + // The minimum request body size, in bytes, at which compression should occur. The + // default value is 10 KiB. Values must fall within [0, 1MiB]. RequestMinCompressSizeBytes int64 // RetryMaxAttempts specifies the maximum number attempts an API client will call diff --git a/internal/protocoltest/restxml/options.go b/internal/protocoltest/restxml/options.go index 0f02a869eb0..7bcf654e66a 100644 --- a/internal/protocoltest/restxml/options.go +++ b/internal/protocoltest/restxml/options.go @@ -65,8 +65,8 @@ type Options struct { // The region to send requests to. (Required) Region string - // Inclusive threshold request body size to trigger compression, default to 10240 - // and must be within 0 and 10485760 bytes inclusively + // The minimum request body size, in bytes, at which compression should occur. The + // default value is 10 KiB. Values must fall within [0, 1MiB]. RequestMinCompressSizeBytes int64 // RetryMaxAttempts specifies the maximum number attempts an API client will call From 4598e4b53e52a7dffcfc001f0402343ce8f75d94 Mon Sep 17 00:00:00 2001 From: Tianyi Wang Date: Wed, 29 Nov 2023 14:19:21 -0500 Subject: [PATCH 5/7] regenerate sdk --- .../protocoltest/awsrestjson/api_op_PutWithContentEncoding.go | 4 +++- .../protocoltest/ec2query/api_op_PutWithContentEncoding.go | 4 +++- .../protocoltest/jsonrpc/api_op_PutWithContentEncoding.go | 4 +++- .../protocoltest/jsonrpc10/api_op_PutWithContentEncoding.go | 4 +++- internal/protocoltest/query/api_op_PutWithContentEncoding.go | 4 +++- .../protocoltest/restxml/api_op_PutWithContentEncoding.go | 4 +++- 6 files changed, 18 insertions(+), 6 deletions(-) diff --git a/internal/protocoltest/awsrestjson/api_op_PutWithContentEncoding.go b/internal/protocoltest/awsrestjson/api_op_PutWithContentEncoding.go index 01ef7acc235..95b463902db 100644 --- a/internal/protocoltest/awsrestjson/api_op_PutWithContentEncoding.go +++ b/internal/protocoltest/awsrestjson/api_op_PutWithContentEncoding.go @@ -119,7 +119,9 @@ func (c *Client) addOperationPutWithContentEncodingMiddlewares(stack *middleware func addOperationPutWithContentEncodingRequestCompressionMiddleware(stack *middleware.Stack, options Options) error { return smithyrequestcompression.AddRequestCompression(stack, options.DisableRequestCompression, options.RequestMinCompressSizeBytes, - []string{"gzip"}) + []string{ + "gzip", + }) } func newServiceMetadataMiddleware_opPutWithContentEncoding(region string) *awsmiddleware.RegisterServiceMetadata { diff --git a/internal/protocoltest/ec2query/api_op_PutWithContentEncoding.go b/internal/protocoltest/ec2query/api_op_PutWithContentEncoding.go index 798fd9e980b..70c71708f6a 100644 --- a/internal/protocoltest/ec2query/api_op_PutWithContentEncoding.go +++ b/internal/protocoltest/ec2query/api_op_PutWithContentEncoding.go @@ -119,7 +119,9 @@ func (c *Client) addOperationPutWithContentEncodingMiddlewares(stack *middleware func addOperationPutWithContentEncodingRequestCompressionMiddleware(stack *middleware.Stack, options Options) error { return smithyrequestcompression.AddRequestCompression(stack, options.DisableRequestCompression, options.RequestMinCompressSizeBytes, - []string{"gzip"}) + []string{ + "gzip", + }) } func newServiceMetadataMiddleware_opPutWithContentEncoding(region string) *awsmiddleware.RegisterServiceMetadata { diff --git a/internal/protocoltest/jsonrpc/api_op_PutWithContentEncoding.go b/internal/protocoltest/jsonrpc/api_op_PutWithContentEncoding.go index 7cdc7e9f081..175c2ac7c9f 100644 --- a/internal/protocoltest/jsonrpc/api_op_PutWithContentEncoding.go +++ b/internal/protocoltest/jsonrpc/api_op_PutWithContentEncoding.go @@ -123,7 +123,9 @@ func (c *Client) addOperationPutWithContentEncodingMiddlewares(stack *middleware func addOperationPutWithContentEncodingRequestCompressionMiddleware(stack *middleware.Stack, options Options) error { return smithyrequestcompression.AddRequestCompression(stack, options.DisableRequestCompression, options.RequestMinCompressSizeBytes, - []string{"gzip"}) + []string{ + "gzip", + }) } func newServiceMetadataMiddleware_opPutWithContentEncoding(region string) *awsmiddleware.RegisterServiceMetadata { diff --git a/internal/protocoltest/jsonrpc10/api_op_PutWithContentEncoding.go b/internal/protocoltest/jsonrpc10/api_op_PutWithContentEncoding.go index 79b8ced7f46..33ddc6fa9ef 100644 --- a/internal/protocoltest/jsonrpc10/api_op_PutWithContentEncoding.go +++ b/internal/protocoltest/jsonrpc10/api_op_PutWithContentEncoding.go @@ -119,7 +119,9 @@ func (c *Client) addOperationPutWithContentEncodingMiddlewares(stack *middleware func addOperationPutWithContentEncodingRequestCompressionMiddleware(stack *middleware.Stack, options Options) error { return smithyrequestcompression.AddRequestCompression(stack, options.DisableRequestCompression, options.RequestMinCompressSizeBytes, - []string{"gzip"}) + []string{ + "gzip", + }) } func newServiceMetadataMiddleware_opPutWithContentEncoding(region string) *awsmiddleware.RegisterServiceMetadata { diff --git a/internal/protocoltest/query/api_op_PutWithContentEncoding.go b/internal/protocoltest/query/api_op_PutWithContentEncoding.go index 5778a94674f..c5e605d02ae 100644 --- a/internal/protocoltest/query/api_op_PutWithContentEncoding.go +++ b/internal/protocoltest/query/api_op_PutWithContentEncoding.go @@ -119,7 +119,9 @@ func (c *Client) addOperationPutWithContentEncodingMiddlewares(stack *middleware func addOperationPutWithContentEncodingRequestCompressionMiddleware(stack *middleware.Stack, options Options) error { return smithyrequestcompression.AddRequestCompression(stack, options.DisableRequestCompression, options.RequestMinCompressSizeBytes, - []string{"gzip"}) + []string{ + "gzip", + }) } func newServiceMetadataMiddleware_opPutWithContentEncoding(region string) *awsmiddleware.RegisterServiceMetadata { diff --git a/internal/protocoltest/restxml/api_op_PutWithContentEncoding.go b/internal/protocoltest/restxml/api_op_PutWithContentEncoding.go index d5d6b28f1ab..4a38ef02c34 100644 --- a/internal/protocoltest/restxml/api_op_PutWithContentEncoding.go +++ b/internal/protocoltest/restxml/api_op_PutWithContentEncoding.go @@ -119,7 +119,9 @@ func (c *Client) addOperationPutWithContentEncodingMiddlewares(stack *middleware func addOperationPutWithContentEncodingRequestCompressionMiddleware(stack *middleware.Stack, options Options) error { return smithyrequestcompression.AddRequestCompression(stack, options.DisableRequestCompression, options.RequestMinCompressSizeBytes, - []string{"gzip"}) + []string{ + "gzip", + }) } func newServiceMetadataMiddleware_opPutWithContentEncoding(region string) *awsmiddleware.RegisterServiceMetadata { From ffeed56a647598e0da245f26a86a101c07204e16 Mon Sep 17 00:00:00 2001 From: Tianyi Wang Date: Mon, 4 Dec 2023 23:46:53 -0500 Subject: [PATCH 6/7] Add request compression assertion to protocol unit tests --- .../api_op_PutWithContentEncoding_test.go | 12 ++++++++++++ .../ec2query/api_op_PutWithContentEncoding_test.go | 12 ++++++++++++ .../jsonrpc/api_op_PutWithContentEncoding_test.go | 12 ++++++++++++ .../jsonrpc10/api_op_PutWithContentEncoding_test.go | 12 ++++++++++++ .../query/api_op_PutWithContentEncoding_test.go | 12 ++++++++++++ .../restxml/api_op_PutWithContentEncoding_test.go | 12 ++++++++++++ 6 files changed, 72 insertions(+) diff --git a/internal/protocoltest/awsrestjson/api_op_PutWithContentEncoding_test.go b/internal/protocoltest/awsrestjson/api_op_PutWithContentEncoding_test.go index 016e9350589..d0b44c924cf 100644 --- a/internal/protocoltest/awsrestjson/api_op_PutWithContentEncoding_test.go +++ b/internal/protocoltest/awsrestjson/api_op_PutWithContentEncoding_test.go @@ -3,11 +3,13 @@ package awsrestjson import ( + "bytes" "context" "github.com/aws/aws-sdk-go-v2/aws" protocoltesthttp "github.com/aws/aws-sdk-go-v2/internal/protocoltest" "github.com/aws/smithy-go/middleware" smithyprivateprotocol "github.com/aws/smithy-go/private/protocol" + smithyrequestcompression "github.com/aws/smithy-go/private/requestcompression" "github.com/aws/smithy-go/ptr" smithyrand "github.com/aws/smithy-go/rand" smithytesting "github.com/aws/smithy-go/testing" @@ -63,6 +65,7 @@ func TestClient_PutWithContentEncoding_awsRestjson1Serialize(t *testing.T) { for name, c := range cases { t.Run(name, func(t *testing.T) { actualReq := &http.Request{} + rawBodyBuf := &bytes.Buffer{} serverURL := "http://localhost:8888/" if c.Host != nil { u, err := url.Parse(serverURL) @@ -95,7 +98,12 @@ func TestClient_PutWithContentEncoding_awsRestjson1Serialize(t *testing.T) { options.APIOptions = append(options.APIOptions, func(stack *middleware.Stack) error { return smithyprivateprotocol.AddCaptureRequestMiddleware(stack, actualReq) }) + options.APIOptions = append(options.APIOptions, func(stack *middleware.Stack) error { + return smithyrequestcompression.AddCaptureUncompressedRequestMiddleware(stack, rawBodyBuf) + }) }) + disable := client.Options().DisableRequestCompression + min := client.Options().RequestMinCompressSizeBytes if err != nil { t.Fatalf("expect nil err, got %v", err) } @@ -120,6 +128,10 @@ func TestClient_PutWithContentEncoding_awsRestjson1Serialize(t *testing.T) { t.Errorf("expect body equal, got %v", err) } } + if err := smithytesting.CompareCompressedBytes(rawBodyBuf, actualReq.Body, + disable, min, "gzip"); err != nil { + t.Errorf("unzipped request body not match: %q", err) + } }) } } diff --git a/internal/protocoltest/ec2query/api_op_PutWithContentEncoding_test.go b/internal/protocoltest/ec2query/api_op_PutWithContentEncoding_test.go index c4aa7a2eee4..aaa4876093e 100644 --- a/internal/protocoltest/ec2query/api_op_PutWithContentEncoding_test.go +++ b/internal/protocoltest/ec2query/api_op_PutWithContentEncoding_test.go @@ -3,11 +3,13 @@ package ec2query import ( + "bytes" "context" "github.com/aws/aws-sdk-go-v2/aws" protocoltesthttp "github.com/aws/aws-sdk-go-v2/internal/protocoltest" "github.com/aws/smithy-go/middleware" smithyprivateprotocol "github.com/aws/smithy-go/private/protocol" + smithyrequestcompression "github.com/aws/smithy-go/private/requestcompression" "github.com/aws/smithy-go/ptr" smithyrand "github.com/aws/smithy-go/rand" smithytesting "github.com/aws/smithy-go/testing" @@ -63,6 +65,7 @@ func TestClient_PutWithContentEncoding_awsEc2querySerialize(t *testing.T) { for name, c := range cases { t.Run(name, func(t *testing.T) { actualReq := &http.Request{} + rawBodyBuf := &bytes.Buffer{} serverURL := "http://localhost:8888/" if c.Host != nil { u, err := url.Parse(serverURL) @@ -95,7 +98,12 @@ func TestClient_PutWithContentEncoding_awsEc2querySerialize(t *testing.T) { options.APIOptions = append(options.APIOptions, func(stack *middleware.Stack) error { return smithyprivateprotocol.AddCaptureRequestMiddleware(stack, actualReq) }) + options.APIOptions = append(options.APIOptions, func(stack *middleware.Stack) error { + return smithyrequestcompression.AddCaptureUncompressedRequestMiddleware(stack, rawBodyBuf) + }) }) + disable := client.Options().DisableRequestCompression + min := client.Options().RequestMinCompressSizeBytes if err != nil { t.Fatalf("expect nil err, got %v", err) } @@ -120,6 +128,10 @@ func TestClient_PutWithContentEncoding_awsEc2querySerialize(t *testing.T) { t.Errorf("expect body equal, got %v", err) } } + if err := smithytesting.CompareCompressedBytes(rawBodyBuf, actualReq.Body, + disable, min, "gzip"); err != nil { + t.Errorf("unzipped request body not match: %q", err) + } }) } } diff --git a/internal/protocoltest/jsonrpc/api_op_PutWithContentEncoding_test.go b/internal/protocoltest/jsonrpc/api_op_PutWithContentEncoding_test.go index 5f8e353057f..c7ba37eaae2 100644 --- a/internal/protocoltest/jsonrpc/api_op_PutWithContentEncoding_test.go +++ b/internal/protocoltest/jsonrpc/api_op_PutWithContentEncoding_test.go @@ -3,11 +3,13 @@ package jsonrpc import ( + "bytes" "context" "github.com/aws/aws-sdk-go-v2/aws" protocoltesthttp "github.com/aws/aws-sdk-go-v2/internal/protocoltest" "github.com/aws/smithy-go/middleware" smithyprivateprotocol "github.com/aws/smithy-go/private/protocol" + smithyrequestcompression "github.com/aws/smithy-go/private/requestcompression" "github.com/aws/smithy-go/ptr" smithytesting "github.com/aws/smithy-go/testing" "io" @@ -62,6 +64,7 @@ func TestClient_PutWithContentEncoding_awsAwsjson11Serialize(t *testing.T) { for name, c := range cases { t.Run(name, func(t *testing.T) { actualReq := &http.Request{} + rawBodyBuf := &bytes.Buffer{} serverURL := "http://localhost:8888/" if c.Host != nil { u, err := url.Parse(serverURL) @@ -93,7 +96,12 @@ func TestClient_PutWithContentEncoding_awsAwsjson11Serialize(t *testing.T) { options.APIOptions = append(options.APIOptions, func(stack *middleware.Stack) error { return smithyprivateprotocol.AddCaptureRequestMiddleware(stack, actualReq) }) + options.APIOptions = append(options.APIOptions, func(stack *middleware.Stack) error { + return smithyrequestcompression.AddCaptureUncompressedRequestMiddleware(stack, rawBodyBuf) + }) }) + disable := client.Options().DisableRequestCompression + min := client.Options().RequestMinCompressSizeBytes if err != nil { t.Fatalf("expect nil err, got %v", err) } @@ -118,6 +126,10 @@ func TestClient_PutWithContentEncoding_awsAwsjson11Serialize(t *testing.T) { t.Errorf("expect body equal, got %v", err) } } + if err := smithytesting.CompareCompressedBytes(rawBodyBuf, actualReq.Body, + disable, min, "gzip"); err != nil { + t.Errorf("unzipped request body not match: %q", err) + } }) } } diff --git a/internal/protocoltest/jsonrpc10/api_op_PutWithContentEncoding_test.go b/internal/protocoltest/jsonrpc10/api_op_PutWithContentEncoding_test.go index bb5aff04953..f10dec03f98 100644 --- a/internal/protocoltest/jsonrpc10/api_op_PutWithContentEncoding_test.go +++ b/internal/protocoltest/jsonrpc10/api_op_PutWithContentEncoding_test.go @@ -3,11 +3,13 @@ package jsonrpc10 import ( + "bytes" "context" "github.com/aws/aws-sdk-go-v2/aws" protocoltesthttp "github.com/aws/aws-sdk-go-v2/internal/protocoltest" "github.com/aws/smithy-go/middleware" smithyprivateprotocol "github.com/aws/smithy-go/private/protocol" + smithyrequestcompression "github.com/aws/smithy-go/private/requestcompression" "github.com/aws/smithy-go/ptr" smithytesting "github.com/aws/smithy-go/testing" "io" @@ -62,6 +64,7 @@ func TestClient_PutWithContentEncoding_awsAwsjson10Serialize(t *testing.T) { for name, c := range cases { t.Run(name, func(t *testing.T) { actualReq := &http.Request{} + rawBodyBuf := &bytes.Buffer{} serverURL := "http://localhost:8888/" if c.Host != nil { u, err := url.Parse(serverURL) @@ -93,7 +96,12 @@ func TestClient_PutWithContentEncoding_awsAwsjson10Serialize(t *testing.T) { options.APIOptions = append(options.APIOptions, func(stack *middleware.Stack) error { return smithyprivateprotocol.AddCaptureRequestMiddleware(stack, actualReq) }) + options.APIOptions = append(options.APIOptions, func(stack *middleware.Stack) error { + return smithyrequestcompression.AddCaptureUncompressedRequestMiddleware(stack, rawBodyBuf) + }) }) + disable := client.Options().DisableRequestCompression + min := client.Options().RequestMinCompressSizeBytes if err != nil { t.Fatalf("expect nil err, got %v", err) } @@ -118,6 +126,10 @@ func TestClient_PutWithContentEncoding_awsAwsjson10Serialize(t *testing.T) { t.Errorf("expect body equal, got %v", err) } } + if err := smithytesting.CompareCompressedBytes(rawBodyBuf, actualReq.Body, + disable, min, "gzip"); err != nil { + t.Errorf("unzipped request body not match: %q", err) + } }) } } diff --git a/internal/protocoltest/query/api_op_PutWithContentEncoding_test.go b/internal/protocoltest/query/api_op_PutWithContentEncoding_test.go index 040d583408b..30156132795 100644 --- a/internal/protocoltest/query/api_op_PutWithContentEncoding_test.go +++ b/internal/protocoltest/query/api_op_PutWithContentEncoding_test.go @@ -3,11 +3,13 @@ package query import ( + "bytes" "context" "github.com/aws/aws-sdk-go-v2/aws" protocoltesthttp "github.com/aws/aws-sdk-go-v2/internal/protocoltest" "github.com/aws/smithy-go/middleware" smithyprivateprotocol "github.com/aws/smithy-go/private/protocol" + smithyrequestcompression "github.com/aws/smithy-go/private/requestcompression" "github.com/aws/smithy-go/ptr" smithyrand "github.com/aws/smithy-go/rand" smithytesting "github.com/aws/smithy-go/testing" @@ -63,6 +65,7 @@ func TestClient_PutWithContentEncoding_awsAwsquerySerialize(t *testing.T) { for name, c := range cases { t.Run(name, func(t *testing.T) { actualReq := &http.Request{} + rawBodyBuf := &bytes.Buffer{} serverURL := "http://localhost:8888/" if c.Host != nil { u, err := url.Parse(serverURL) @@ -95,7 +98,12 @@ func TestClient_PutWithContentEncoding_awsAwsquerySerialize(t *testing.T) { options.APIOptions = append(options.APIOptions, func(stack *middleware.Stack) error { return smithyprivateprotocol.AddCaptureRequestMiddleware(stack, actualReq) }) + options.APIOptions = append(options.APIOptions, func(stack *middleware.Stack) error { + return smithyrequestcompression.AddCaptureUncompressedRequestMiddleware(stack, rawBodyBuf) + }) }) + disable := client.Options().DisableRequestCompression + min := client.Options().RequestMinCompressSizeBytes if err != nil { t.Fatalf("expect nil err, got %v", err) } @@ -120,6 +128,10 @@ func TestClient_PutWithContentEncoding_awsAwsquerySerialize(t *testing.T) { t.Errorf("expect body equal, got %v", err) } } + if err := smithytesting.CompareCompressedBytes(rawBodyBuf, actualReq.Body, + disable, min, "gzip"); err != nil { + t.Errorf("unzipped request body not match: %q", err) + } }) } } diff --git a/internal/protocoltest/restxml/api_op_PutWithContentEncoding_test.go b/internal/protocoltest/restxml/api_op_PutWithContentEncoding_test.go index 9dd09be9fc1..cef29870146 100644 --- a/internal/protocoltest/restxml/api_op_PutWithContentEncoding_test.go +++ b/internal/protocoltest/restxml/api_op_PutWithContentEncoding_test.go @@ -3,11 +3,13 @@ package restxml import ( + "bytes" "context" "github.com/aws/aws-sdk-go-v2/aws" protocoltesthttp "github.com/aws/aws-sdk-go-v2/internal/protocoltest" "github.com/aws/smithy-go/middleware" smithyprivateprotocol "github.com/aws/smithy-go/private/protocol" + smithyrequestcompression "github.com/aws/smithy-go/private/requestcompression" "github.com/aws/smithy-go/ptr" smithyrand "github.com/aws/smithy-go/rand" smithytesting "github.com/aws/smithy-go/testing" @@ -63,6 +65,7 @@ func TestClient_PutWithContentEncoding_awsRestxmlSerialize(t *testing.T) { for name, c := range cases { t.Run(name, func(t *testing.T) { actualReq := &http.Request{} + rawBodyBuf := &bytes.Buffer{} serverURL := "http://localhost:8888/" if c.Host != nil { u, err := url.Parse(serverURL) @@ -95,7 +98,12 @@ func TestClient_PutWithContentEncoding_awsRestxmlSerialize(t *testing.T) { options.APIOptions = append(options.APIOptions, func(stack *middleware.Stack) error { return smithyprivateprotocol.AddCaptureRequestMiddleware(stack, actualReq) }) + options.APIOptions = append(options.APIOptions, func(stack *middleware.Stack) error { + return smithyrequestcompression.AddCaptureUncompressedRequestMiddleware(stack, rawBodyBuf) + }) }) + disable := client.Options().DisableRequestCompression + min := client.Options().RequestMinCompressSizeBytes if err != nil { t.Fatalf("expect nil err, got %v", err) } @@ -120,6 +128,10 @@ func TestClient_PutWithContentEncoding_awsRestxmlSerialize(t *testing.T) { t.Errorf("expect body equal, got %v", err) } } + if err := smithytesting.CompareCompressedBytes(rawBodyBuf, actualReq.Body, + disable, min, "gzip"); err != nil { + t.Errorf("unzipped request body not match: %q", err) + } }) } } From 70bf4107034085068fb6847667657fbb78367be4 Mon Sep 17 00:00:00 2001 From: Tianyi Wang Date: Wed, 6 Dec 2023 17:04:55 -0500 Subject: [PATCH 7/7] Change config loading logic --- config/env_config.go | 48 ++++++++++++++++++++++------------------- config/shared_config.go | 3 ++- 2 files changed, 28 insertions(+), 23 deletions(-) diff --git a/config/env_config.go b/config/env_config.go index 1a640ce47bf..88550198cce 100644 --- a/config/env_config.go +++ b/config/env_config.go @@ -12,6 +12,7 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/feature/ec2/imds" + smithyrequestcompression "github.com/aws/smithy-go/private/requestcompression" ) // CredentialsSourceName provides a name of the provider when config is @@ -334,7 +335,7 @@ func NewEnvConfig() (EnvConfig, error) { if err := setBoolPtrFromEnvVal(&cfg.DisableRequestCompression, []string{awsDisableRequestCompression}); err != nil { return cfg, err } - if err := setRequestMinCompressSizeBytes(&cfg.RequestMinCompressSizeBytes); err != nil { + if err := setInt64PtrFromEnvVal(&cfg.RequestMinCompressSizeBytes, []string{awsRequestMinCompressionSizeBytes}, smithyrequestcompression.MaxRequestMinCompressSizeBytes); err != nil { return cfg, err } @@ -402,27 +403,6 @@ func (c EnvConfig) getAppID(context.Context) (string, bool, error) { return c.AppID, len(c.AppID) > 0, nil } -func setRequestMinCompressSizeBytes(bytes **int64) error { - b := os.Getenv(awsRequestMinCompressionSizeBytes) - if b == "" { - return nil - } - - byte, err := strconv.ParseInt(b, 10, 64) - if err != nil { - return fmt.Errorf("invalid value for env var, %s=%s, need int64", - awsRequestMinCompressionSizeBytes, b) - } else if byte < 0 || byte > 10485760 { - return fmt.Errorf("invalid range for env var min request compression size bytes %q, must be within 0 and 10485760 inclusively", byte) - } - if *bytes == nil { - *bytes = new(int64) - } - **bytes = byte - - return nil -} - func (c EnvConfig) getDisableRequestCompression(context.Context) (bool, bool, error) { if c.DisableRequestCompression == nil { return false, false, nil @@ -693,6 +673,30 @@ func setBoolPtrFromEnvVal(dst **bool, keys []string) error { return nil } +func setInt64PtrFromEnvVal(dst **int64, keys []string, max int64) error { + for _, k := range keys { + value := os.Getenv(k) + if len(value) == 0 { + continue + } + + v, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return fmt.Errorf("invalid value for env var, %s=%s, need int64", k, value) + } else if v < 0 || v > max { + return fmt.Errorf("invalid range for env var min request compression size bytes %q, must be within 0 and 10485760 inclusively", v) + } + if *dst == nil { + *dst = new(int64) + } + + **dst = v + break + } + + return nil +} + func setEndpointDiscoveryTypeFromEnvVal(dst *aws.EndpointDiscoveryEnableState, keys []string) error { for _, k := range keys { value := os.Getenv(k) diff --git a/config/shared_config.go b/config/shared_config.go index 177051480c0..422dfc55bea 100644 --- a/config/shared_config.go +++ b/config/shared_config.go @@ -17,6 +17,7 @@ import ( "github.com/aws/aws-sdk-go-v2/internal/ini" "github.com/aws/aws-sdk-go-v2/internal/shareddefaults" "github.com/aws/smithy-go/logging" + smithyrequestcompression "github.com/aws/smithy-go/private/requestcompression" ) const ( @@ -1147,7 +1148,7 @@ func updateRequestMinCompressSizeBytes(bytes **int64, sec ini.Section, key strin if !ok { return fmt.Errorf("invalid value for min request compression size bytes %s, need int64", sec.String(key)) } - if v < 0 || v > 10485760 { + if v < 0 || v > smithyrequestcompression.MaxRequestMinCompressSizeBytes { return fmt.Errorf("invalid range for min request compression size bytes %d, must be within 0 and 10485760 inclusively", v) } *bytes = new(int64)