s3blob/blob: support additional endpoint query parameters

This commit adds the following query parameters for AWS:

1. `dualstack`
2. `fips`
3. `accelerate` (S3-only)

This avoids the need for users to specify `endpoint`. For example, if
my AWS S3 bucket is `my-bucket` in `us-east-1`, and you want to enable
transfer acceleration, dual-stack support, and/or FIPS, you would need
to configure `endpoint` with one of the following:

1. `my-bucket.s3-accelerate.amazonaws.com`
2. `my-bucket.s3-accelerate.dualstack.amazonaws.com`
3. `my-bucket.s3-fips.us-gov-east-1.amazonaws.com`
4. `my-bucket.s3-fips.dualstack.us-east-1.amazonaws.com`

For example, for the last option, users can use
`s3://my-bucket?fips=true&dualstack=true`.

Closes #3484

aws/aws.go

@@ -26,6 +26,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/client"
 	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/endpoints"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/google/wire"
 )
@@ -81,6 +82,8 @@ func (co ConfigOverrider) ClientConfig(serviceName string, cfgs ...*aws.Config)
 //   - endpoint: The endpoint URL (hostname only or fully qualified URI); sets aws.Config.Endpoint.
 //   - disableSSL: A value of "true" disables SSL when sending requests; sets aws.Config.DisableSSL.
 //   - s3ForcePathStyle: A value of "true" forces the request to use path-style addressing; sets aws.Config.S3ForcePathStyle.
+//   - dualstack: A value of "true" enables dual stack (IPv4 and IPv6) endpoints
+//   - fips: A value of "true" enables the use of FIPS endpoints
 func ConfigFromURLParams(q url.Values) (*aws.Config, error) {
 	var cfg aws.Config
 	for param, values := range q {
@@ -102,6 +105,20 @@ func ConfigFromURLParams(q url.Values) (*aws.Config, error) {
 				return nil, fmt.Errorf("invalid value for query parameter %q: %v", param, err)
 			}
 			cfg.S3ForcePathStyle = aws.Bool(b)
+		case "dualstack":
+			b, err := strconv.ParseBool(value)
+			if err != nil {
+				return nil, fmt.Errorf("invalid value for query parameter %q: %v", param, err)
+			}
+			cfg.UseDualStack = aws.Bool(b)
+		case "fips":
+			b, err := strconv.ParseBool(value)
+			if err != nil {
+				return nil, fmt.Errorf("invalid value for query parameter %q: %v", param, err)
+			}
+			if b {
+				cfg.UseFIPSEndpoint = endpoints.FIPSEndpointStateEnabled
+			}
 		case "awssdk":
 			// ignore, should be handled before this
 		default:
@@ -178,6 +195,8 @@ func NewDefaultV2Config(ctx context.Context) (awsv2.Config, error) {
 //   - profile: The shared config profile to use; sets SharedConfigProfile.
 //   - endpoint: The AWS service endpoint to send HTTP request.
 //   - hostname_immutable: Make the hostname immutable, only works if endpoint is also set.
+//   - dualstack: A value of "true" enables dual stack (IPv4 and IPv6) endpoints.
+//   - fips: A value of "true" enables the use of FIPS endpoints.
 func V2ConfigFromURLParams(ctx context.Context, q url.Values) (awsv2.Config, error) {
 	var endpoint string
 	var hostnameImmutable bool
@@ -197,6 +216,22 @@ func V2ConfigFromURLParams(ctx context.Context, q url.Values) (awsv2.Config, err
 			endpoint = value
 		case "profile":
 			opts = append(opts, awsv2cfg.WithSharedConfigProfile(value))
+		case "dualstack":
+			dualStack, err := strconv.ParseBool(value)
+			if err != nil {
+				return awsv2.Config{}, fmt.Errorf("invalid value for dualstack: %w", err)
+			}
+			if dualStack {
+				opts = append(opts, awsv2cfg.WithUseDualStackEndpoint(awsv2.DualStackEndpointStateEnabled))
+			}
+		case "fips":
+			fips, err := strconv.ParseBool(value)
+			if err != nil {
+				return awsv2.Config{}, fmt.Errorf("invalid value for fips: %w", err)
+			}
+			if fips {
+				opts = append(opts, awsv2cfg.WithUseFIPSEndpoint(awsv2.FIPSEndpointStateEnabled))
+			}
 		case "awssdk":
 			// ignore, should be handled before this
 		default:
@@ -215,5 +250,6 @@ func V2ConfigFromURLParams(ctx context.Context, q url.Values) (awsv2.Config, err
 			})
 		opts = append(opts, awsv2cfg.WithEndpointResolverWithOptions(customResolver))
 	}
+
 	return awsv2cfg.LoadDefaultConfig(ctx, opts...)
 }

aws/aws_test.go

@@ -184,6 +184,10 @@ func TestV2ConfigFromURLParams(t *testing.T) {
 				HostnameImmutable: true,
 			},
 		},
+		{
+			name:  "FIPS and dual stack",
+			query: url.Values{"fips": {"true"}, "dualstack": {"true"}},
+		},
 		// Can't test "profile", since AWS validates that the profile exists.
 	}
 

blob/s3blob/s3blob.go

@@ -130,6 +130,11 @@ const Scheme = "s3"
 // Use "awssdk=v1" to force using AWS SDK v1, "awssdk=v2" to force using AWS SDK v2,
 // or anything else to accept the default.
 //
+// The following S3-specific query options are also supported:
+//   - ssetype: The type of server side encryption used (AES256, aws:kms, aws:kms:dsse)
+//   - kmskeyid: The KMS key ID for server side encryption
+//   - accelerate: A value of "true" uses the S3 Transfer Accleration endpoints
+//
 // For V1, see gocloud.dev/aws/ConfigFromURLParams for supported query parameters
 // for overriding the aws.Session from the URL.
 // For V2, see gocloud.dev/aws/V2ConfigFromURLParams.
@@ -145,8 +150,9 @@ type URLOpener struct {
 }
 
 const (
-	sseTypeParamKey  = "ssetype"
-	kmsKeyIdParamKey = "kmskeyid"
+	sseTypeParamKey    = "ssetype"
+	kmsKeyIdParamKey   = "kmskeyid"
+	accelerateParamKey = "accelerate"
 )
 
 func toServerSideEncryptionType(value string) (typesv2.ServerSideEncryption, error) {
@@ -178,12 +184,24 @@ func (o *URLOpener) OpenBucketURL(ctx context.Context, u *url.URL) (*blob.Bucket
 		o.Options.KMSEncryptionID = kmsKeyID
 	}
 
+	accelerate := false
+	if accelerateParam := q.Get(accelerateParamKey); accelerateParam != "" {
+		q.Del(accelerateParamKey)
+		var err error
+		accelerate, err = strconv.ParseBool(accelerateParam)
+		if err != nil {
+			return nil, fmt.Errorf("invalid value for %q: %v", accelerateParamKey, err)
+		}
+	}
+
 	if o.UseV2 {
 		cfg, err := gcaws.V2ConfigFromURLParams(ctx, q)
 		if err != nil {
 			return nil, fmt.Errorf("open bucket %v: %v", u, err)
 		}
-		clientV2 := s3v2.NewFromConfig(cfg)
+		clientV2 := s3v2.NewFromConfig(cfg, func(o *s3v2.Options) {
+			o.UseAccelerate = accelerate
+		})
 
 		return OpenBucketV2(ctx, clientV2, u.Host, &o.Options)
 	}
@@ -194,6 +212,8 @@ func (o *URLOpener) OpenBucketURL(ctx context.Context, u *url.URL) (*blob.Bucket
 	if err != nil {
 		return nil, fmt.Errorf("open bucket %v: %v", u, err)
 	}
+
+	overrideCfg.S3UseAccelerate = &accelerate
 	configProvider.Configs = append(configProvider.Configs, overrideCfg)
 
 	return OpenBucket(ctx, configProvider, u.Host, &o.Options)

blob/s3blob/s3blob_test.go

@@ -476,6 +476,26 @@ func TestOpenBucketFromURL(t *testing.T) {
 		{"s3://mybucket?awssdk=v2", false},
 		// OK, use KMS Server Side Encryption
 		{"s3://mybucket?ssetype=aws:kms&kmskeyid=arn:aws:us-east-1:12345:key/1-a-2-b", false},
+		// OK, use S3 Transfer acceleration and dual stack endpoints
+		{"s3://mybucket?accelerate=true&dualstack=true", false},
+		// OK, use FIPS endpoints
+		{"s3://mybucket?fips=true", false},
+		// OK, use S3 Transfer accleration and dual stack endpoints (v1)
+		{"s3://mybucket?awssdk=v1&accelerate=true&dualstack=true", false},
+		// OK, use FIPS endpoints (v1)
+		{"s3://mybucket?awssdk=v1&fips=true", false},
+		// Invalid accelerate (v1)
+		{"s3://mybucket?awssdk=v1&accelerate=bogus", true},
+		// Invalid accelerate (v2)
+		{"s3://mybucket?accelerate=bogus", true},
+		// Invalid FIPS (v1)
+		{"s3://mybucket?awssdk=v1&fips=bogus", true},
+		// Invalid FIPS (v2)
+		{"s3://mybucket?fips=bogus", true},
+		// Invalid dualstack (v1)
+		{"s3://mybucket?awssdk=v1&dualstack=bad", true},
+		// Invalid dualstack (v2)
+		{"s3://mybucket?dualstack=bad", true},
 		// Invalid ssetype
 		{"s3://mybucket?ssetype=aws:notkmsoraes&kmskeyid=arn:aws:us-east-1:12345:key/1-a-2-b", true},
 		// Invalid parameter together with a valid one.