service/s3: Add support for Access Point resources (#2985)

Adds support for using Access Point resource with Amazon S3 API
operation calls. The Access Point resource are identified by an Amazon
Resource Name (ARN).

To make operation calls to an S3 Access Point instead of a S3 Bucket,
provide the Access Point ARN string as the value of the Bucket
parameter. You can create an Access Point for your bucket with the Amazon
S3 Control API. The Access Point ARN can be obtained from the S3 Control
API. You should avoid building the ARN directly.

```go
apResp, err := ctrClient.CreateAccessPoint(&s3control.CreateAccessPointInput{
    AccountId: aws.String("123456789012"),
    Bucket:    aws.String("myBucket"),
    Name:      aws.String("myAccessPoint"),
})

resp, err := client.GetObject(&s3.GetObjectInput{
    Bucket: aws.String("arn:aws:s3:us-west-2:123456789012:accesspoint/myAccessPoint"),
    Key: aws.String("myObject"),
})
```

See https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html
for more information on ARNs.

CHANGELOG_PENDING.md

@@ -1,5 +1,8 @@
 ### SDK Features
 
 ### SDK Enhancements
+* `service/s3`: Add support for Access Point resources
+  * Adds support for using Access Point resource with Amazon S3 API operation calls. The Access Point resource are identified by an Amazon Resource Name (ARN).
+  * To make operation calls to an S3 Access Point instead of a S3 Bucket, provide the Access Point ARN string as the value of the Bucket parameter. You can create an Access Point for your bucket with the Amazon S3 Control API. The Access Point ARN can be obtained from the S3 Control API. You should avoid building the ARN directly.
 
 ### SDK Bugs

Makefile

@@ -132,9 +132,9 @@ sandbox-test-go1.6: sandbox-build-go1.6
 
 sandbox-build-go1.7:
 	docker build -f ./awstesting/sandbox/Dockerfile.test.go1.7 -t "aws-sdk-go-1.7" .
-sandbox-go1.7: sandbox-build-go17
+sandbox-go1.7: sandbox-build-go1.7
 	docker run -i -t aws-sdk-go-1.7 bash
-sandbox-test-go1.7: sandbox-build-go17
+sandbox-test-go1.7: sandbox-build-go1.7
 	docker run -t aws-sdk-go-1.7
 
 sandbox-build-go1.8:

aws/arn/arn.go

@@ -75,6 +75,12 @@ func Parse(arn string) (ARN, error) {
 	}, nil
 }
 
+// IsARN returns whether the given string is an arn
+// by looking for whether the string starts with arn:
+func IsARN(arn string) bool {
+	return strings.HasPrefix(arn, arnPrefix) && strings.Count(arn, ":") > arnSections-1
+}
+
 // String returns the canonical representation of the ARN
 func (arn ARN) String() string {
 	return arnPrefix +

aws/config.go

@@ -161,6 +161,10 @@ type Config struct {
 	// on GetObject API calls.
 	S3DisableContentMD5Validation *bool
 
+	// Set this to `true` to have the S3 service client to use the region specified
+	// in the ARN, when an ARN is provided as an argument to a bucket parameter.
+	S3UseARNRegion *bool
+
 	// Set this to `true` to disable the EC2Metadata client from overriding the
 	// default http.Client's Timeout. This is helpful if you do not want the
 	// EC2Metadata client to create a new http.Client. This options is only
@@ -385,6 +389,13 @@ func (c *Config) WithS3DisableContentMD5Validation(enable bool) *Config {
 
 }
 
+// WithS3UseARNRegion sets a config S3UseARNRegion value and
+// returning a Config pointer for chaining
+func (c *Config) WithS3UseARNRegion(enable bool) *Config {
+	c.S3UseARNRegion = &enable
+	return c
+}
+
 // WithUseDualStack sets a config UseDualStack value returning a Config
 // pointer for chaining.
 func (c *Config) WithUseDualStack(enable bool) *Config {
@@ -513,6 +524,10 @@ func mergeInConfig(dst *Config, other *Config) {
 		dst.S3DisableContentMD5Validation = other.S3DisableContentMD5Validation
 	}
 
+	if other.S3UseARNRegion != nil {
+		dst.S3UseARNRegion = other.S3UseARNRegion
+	}
+
 	if other.UseDualStack != nil {
 		dst.UseDualStack = other.UseDualStack
 	}

aws/session/env_config.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 	"strconv"
+	"strings"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/credentials"
@@ -141,6 +142,12 @@ type envConfig struct {
 	// AWS_S3_US_EAST_1_REGIONAL_ENDPOINT=regional
 	// This can take value as `regional` or `legacy`
 	S3UsEast1RegionalEndpoint endpoints.S3UsEast1RegionalEndpoint
+
+	// Specifies if the S3 service should allow ARNs to direct the region
+	// the client's requests are sent to.
+	//
+	// AWS_S3_USE_ARN_REGION=true
+	S3UseARNRegion bool
 }
 
 var (
@@ -201,6 +208,9 @@ var (
 	s3UsEast1RegionalEndpoint = []string{
 		"AWS_S3_US_EAST_1_REGIONAL_ENDPOINT",
 	}
+	s3UseARNRegionEnvKey = []string{
+		"AWS_S3_USE_ARN_REGION",
+	}
 )
 
 // loadEnvConfig retrieves the SDK's environment configuration.
@@ -307,6 +317,21 @@ func envConfigLoad(enableSharedConfig bool) (envConfig, error) {
 		}
 	}
 
+	var s3UseARNRegion string
+	setFromEnvVal(&s3UseARNRegion, s3UseARNRegionEnvKey)
+	if len(s3UseARNRegion) != 0 {
+		switch {
+		case strings.EqualFold(s3UseARNRegion, "false"):
+			cfg.S3UseARNRegion = false
+		case strings.EqualFold(s3UseARNRegion, "true"):
+			cfg.S3UseARNRegion = true
+		default:
+			return envConfig{}, fmt.Errorf(
+				"invalid value for environment variable, %s=%s, need true or false",
+				s3UseARNRegionEnvKey[0], s3UseARNRegion)
+		}
+	}
+
 	return cfg, nil
 }
 

aws/session/env_config_test.go

@@ -292,6 +292,16 @@ func TestLoadEnvConfig(t *testing.T) {
 				SharedConfigFile:          shareddefaults.SharedConfigFilename(),
 			},
 		},
+		{
+			Env: map[string]string{
+				"AWS_S3_USE_ARN_REGION": "true",
+			},
+			Config: envConfig{
+				S3UseARNRegion:        true,
+				SharedCredentialsFile: shareddefaults.SharedCredentialsFilename(),
+				SharedConfigFile:      shareddefaults.SharedConfigFilename(),
+			},
+		},
 	}
 
 	for i, c := range cases {

aws/session/session.go

@@ -580,6 +580,14 @@ func mergeConfigSrcs(cfg, userCfg *aws.Config,
 		cfg.Credentials = creds
 	}
 
+	cfg.S3UseARNRegion = userCfg.S3UseARNRegion
+	if cfg.S3UseARNRegion == nil {
+		cfg.S3UseARNRegion = &envCfg.S3UseARNRegion
+	}
+	if cfg.S3UseARNRegion == nil {
+		cfg.S3UseARNRegion = &sharedCfg.S3UseARNRegion
+	}
+
 	return nil
 }
 
@@ -643,6 +651,7 @@ func (s *Session) ClientConfig(service string, cfgs ...*aws.Config) client.Confi
 	return client.Config{
 		Config:             s.Config,
 		Handlers:           s.Handlers,
+		PartitionID:        resolved.PartitionID,
 		Endpoint:           resolved.URL,
 		SigningRegion:      resolved.SigningRegion,
 		SigningNameDerived: resolved.SigningNameDerived,

aws/session/shared_config.go

@@ -51,6 +51,9 @@ const (
 	// loading configuration from the config files if another profile name
 	// is not provided.
 	DefaultSharedConfigProfile = `default`
+
+	// S3 ARN Region Usage
+	s3UseARNRegionKey = "s3_use_arn_region"
 )
 
 // sharedConfig represents the configuration fields of the SDK config files.
@@ -89,6 +92,7 @@ type sharedConfig struct {
 	//
 	//	endpoint_discovery_enabled = true
 	EnableEndpointDiscovery *bool
+
 	// CSM Options
 	CSMEnabled  *bool
 	CSMHost     string
@@ -106,6 +110,12 @@ type sharedConfig struct {
 	// s3_us_east_1_regional_endpoint = regional
 	// This can take value as `LegacyS3UsEast1Endpoint` or `RegionalS3UsEast1Endpoint`
 	S3UsEast1RegionalEndpoint endpoints.S3UsEast1RegionalEndpoint
+
+	// Specifies if the S3 service should allow ARNs to direct the region
+	// the client's requests are sent to.
+	//
+	// s3_use_arn_region=true
+	S3UseARNRegion bool
 }
 
 type sharedConfigFile struct {
@@ -306,6 +316,8 @@ func (cfg *sharedConfig) setFromIniFile(profile string, file sharedConfigFile, e
 	updateString(&cfg.CSMPort, section, csmPortKey)
 	updateString(&cfg.CSMClientID, section, csmClientIDKey)
 
+	updateBool(&cfg.S3UseARNRegion, section, s3UseARNRegionKey)
+
 	return nil
 }
 
@@ -398,6 +410,15 @@ func updateString(dst *string, section ini.Section, key string) {
 	*dst = section.String(key)
 }
 
+// updateBool will only update the dst with the value in the section key, key
+// is present in the section.
+func updateBool(dst *bool, section ini.Section, key string) {
+	if !section.Has(key) {
+		return
+	}
+	*dst = section.Bool(key)
+}
+
 // updateBoolPtr will only update the dst with the value in the section key,
 // key is present in the section.
 func updateBoolPtr(dst **bool, section ini.Section, key string) {

aws/session/shared_config_test.go

@@ -303,6 +303,12 @@ func TestLoadSharedConfigFromFile(t *testing.T) {
 			Profile: "does_not_exists",
 			Err:     SharedConfigProfileNotExistsError{Profile: "does_not_exists"},
 		},
+		{
+			Profile: "valid_arn_region",
+			Expected: sharedConfig{
+				S3UseARNRegion: true,
+			},
+		},
 	}
 
 	for i, c := range cases {

aws/session/testdata/shared_config

@@ -85,3 +85,6 @@ sts_regional_endpoints = regional
 
 [with_s3_us_east_1_regional]
 s3_us_east_1_regional_endpoint = regional
+
+[valid_arn_region]
+s3_use_arn_region=true

awstesting/integration/integration.go

@@ -16,7 +16,9 @@ import (
 )
 
 // Session is a shared session for all integration tests to use.
-var Session = session.Must(session.NewSession())
+var Session = session.Must(session.NewSession(&aws.Config{
+	CredentialsChainVerboseErrors: aws.Bool(true),
+}))
 
 func init() {
 	logLevel := Session.Config.LogLevel

awstesting/integration/s3integ/shared.go

@@ -7,6 +7,9 @@ import (
 
 	"github.com/aws/aws-sdk-go/awstesting/integration"
 	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3iface"
+	"github.com/aws/aws-sdk-go/service/s3control"
+	"github.com/aws/aws-sdk-go/service/s3control/s3controliface"
 )
 
 // BucketPrefix is the root prefix of integration test buckets.
@@ -18,8 +21,8 @@ func GenerateBucketName() string {
 		BucketPrefix, integration.UniqueID())
 }
 
-// SetupTest returns a test bucket created for the integration tests.
-func SetupTest(svc *s3.S3, bucketName string) (err error) {
+// SetupBucket returns a test bucket created for the integration tests.
+func SetupBucket(svc s3iface.S3API, bucketName string) (err error) {
 
 	fmt.Println("Setup: Creating test bucket,", bucketName)
 	_, err = svc.CreateBucket(&s3.CreateBucketInput{Bucket: &bucketName})
@@ -37,9 +40,9 @@ func SetupTest(svc *s3.S3, bucketName string) (err error) {
 	return nil
 }
 
-// CleanupTest deletes the contents of a S3 bucket, before deleting the bucket
+// CleanupBucket deletes the contents of a S3 bucket, before deleting the bucket
 // it self.
-func CleanupTest(svc *s3.S3, bucketName string) error {
+func CleanupBucket(svc s3iface.S3API, bucketName string) error {
 	errs := []error{}
 
 	fmt.Println("TearDown: Deleting objects from test bucket,", bucketName)
@@ -91,3 +94,30 @@ func CleanupTest(svc *s3.S3, bucketName string) error {
 
 	return nil
 }
+
+// SetupAccessPoint returns an access point for the given bucket for testing
+func SetupAccessPoint(svc s3controliface.S3ControlAPI, account, bucket, accessPoint string) error {
+	fmt.Printf("Setup: creating access point %q for bucket %q\n", accessPoint, bucket)
+	_, err := svc.CreateAccessPoint(&s3control.CreateAccessPointInput{
+		AccountId: &account,
+		Bucket:    &bucket,
+		Name:      &accessPoint,
+	})
+	if err != nil {
+		return fmt.Errorf("failed to create access point: %v", err)
+	}
+	return nil
+}
+
+// CleanupAccessPoint deletes the given access point
+func CleanupAccessPoint(svc s3controliface.S3ControlAPI, account, accessPoint string) error {
+	fmt.Printf("TearDown: Deleting access point %q\n", accessPoint)
+	_, err := svc.DeleteAccessPoint(&s3control.DeleteAccessPointInput{
+		AccountId: &account,
+		Name:      &accessPoint,
+	})
+	if err != nil {
+		return fmt.Errorf("failed to delete access point: %v", err)
+	}
+	return nil
+}

example/service/s3/usingAccessPoints/README.md

@@ -0,0 +1,12 @@
+# Example
+
+This example demonstrates how you can use the AWS SDK for Go's Amazon S3 client
+to create and use S3 Access Points resources for working with to S3 buckets.
+
+# Usage
+
+The example will create a bucket of the name provided in code. Replace the value of the `accountID` const with the account ID for your AWS account. The `bucket`, `keyName`, and `accessPoint` const variables need to be updated to match the name of the Bucket, Object Key, and Access Point that will be created by the example.
+
+```sh
+AWS_REGION=<region> go run -tags example usingAccessPoints.go
+```