Skip to content

Commit

Permalink
Merge pull request #23052 from hashicorp/endpoint-envvars
Browse files Browse the repository at this point in the history
provider: Add environment variable support for DynamoDB, IAM, S3, STS endpoints
  • Loading branch information
gdavison committed Feb 9, 2022
2 parents e567778 + 792c7e6 commit 1cd3073
Show file tree
Hide file tree
Showing 5 changed files with 266 additions and 19 deletions.
3 changes: 3 additions & 0 deletions .changelog/23052.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
```release-note:enhancement
provider: Add environment variables `TF_AWS_DYNAMODB_ENDPOINT`, `TF_AWS_IAM_ENDPOINT`, `TF_AWS_S3_ENDPOINT`, and `TF_AWS_STS_ENDPOINT`.
```
26 changes: 22 additions & 4 deletions internal/conns/conns.go
Original file line number Diff line number Diff line change
Expand Up @@ -572,6 +572,8 @@ type ServiceDatum struct {
AWSServiceID string
ProviderNameUpper string
HCLKeys []string
EnvVar string
DeprecatedEnvVar string
}

var serviceData map[string]*ServiceDatum
Expand Down Expand Up @@ -655,7 +657,7 @@ func init() {
serviceData[DMS] = &ServiceDatum{AWSClientName: "DatabaseMigrationService", AWSServiceName: databasemigrationservice.ServiceName, AWSEndpointsID: databasemigrationservice.EndpointsID, AWSServiceID: databasemigrationservice.ServiceID, ProviderNameUpper: "DMS", HCLKeys: []string{"dms", "databasemigration", "databasemigrationservice"}}
serviceData[DocDB] = &ServiceDatum{AWSClientName: "DocDB", AWSServiceName: docdb.ServiceName, AWSEndpointsID: docdb.EndpointsID, AWSServiceID: docdb.ServiceID, ProviderNameUpper: "DocDB", HCLKeys: []string{"docdb"}}
serviceData[DS] = &ServiceDatum{AWSClientName: "DirectoryService", AWSServiceName: directoryservice.ServiceName, AWSEndpointsID: directoryservice.EndpointsID, AWSServiceID: directoryservice.ServiceID, ProviderNameUpper: "DS", HCLKeys: []string{"ds"}}
serviceData[DynamoDB] = &ServiceDatum{AWSClientName: "DynamoDB", AWSServiceName: dynamodb.ServiceName, AWSEndpointsID: dynamodb.EndpointsID, AWSServiceID: dynamodb.ServiceID, ProviderNameUpper: "DynamoDB", HCLKeys: []string{"dynamodb"}}
serviceData[DynamoDB] = &ServiceDatum{AWSClientName: "DynamoDB", AWSServiceName: dynamodb.ServiceName, AWSEndpointsID: dynamodb.EndpointsID, AWSServiceID: dynamodb.ServiceID, ProviderNameUpper: "DynamoDB", HCLKeys: []string{"dynamodb"}, EnvVar: "TF_AWS_DYNAMODB_ENDPOINT", DeprecatedEnvVar: "AWS_DYNAMODB_ENDPOINT"}
serviceData[DynamoDBStreams] = &ServiceDatum{AWSClientName: "DynamoDBStreams", AWSServiceName: dynamodbstreams.ServiceName, AWSEndpointsID: dynamodbstreams.EndpointsID, AWSServiceID: dynamodbstreams.ServiceID, ProviderNameUpper: "DynamoDBStreams", HCLKeys: []string{"dynamodbstreams"}}
serviceData[EC2] = &ServiceDatum{AWSClientName: "EC2", AWSServiceName: ec2.ServiceName, AWSEndpointsID: ec2.EndpointsID, AWSServiceID: ec2.ServiceID, ProviderNameUpper: "EC2", HCLKeys: []string{"ec2"}}
serviceData[EC2InstanceConnect] = &ServiceDatum{AWSClientName: "EC2InstanceConnect", AWSServiceName: ec2instanceconnect.ServiceName, AWSEndpointsID: ec2instanceconnect.EndpointsID, AWSServiceID: ec2instanceconnect.ServiceID, ProviderNameUpper: "EC2InstanceConnect", HCLKeys: []string{"ec2instanceconnect"}}
Expand Down Expand Up @@ -696,7 +698,7 @@ func init() {
serviceData[Health] = &ServiceDatum{AWSClientName: "Health", AWSServiceName: health.ServiceName, AWSEndpointsID: health.EndpointsID, AWSServiceID: health.ServiceID, ProviderNameUpper: "Health", HCLKeys: []string{"health"}}
serviceData[HealthLake] = &ServiceDatum{AWSClientName: "HealthLake", AWSServiceName: healthlake.ServiceName, AWSEndpointsID: healthlake.EndpointsID, AWSServiceID: healthlake.ServiceID, ProviderNameUpper: "HealthLake", HCLKeys: []string{"healthlake"}}
serviceData[Honeycode] = &ServiceDatum{AWSClientName: "Honeycode", AWSServiceName: honeycode.ServiceName, AWSEndpointsID: honeycode.EndpointsID, AWSServiceID: honeycode.ServiceID, ProviderNameUpper: "Honeycode", HCLKeys: []string{"honeycode"}}
serviceData[IAM] = &ServiceDatum{AWSClientName: "IAM", AWSServiceName: iam.ServiceName, AWSEndpointsID: iam.EndpointsID, AWSServiceID: iam.ServiceID, ProviderNameUpper: "IAM", HCLKeys: []string{"iam"}}
serviceData[IAM] = &ServiceDatum{AWSClientName: "IAM", AWSServiceName: iam.ServiceName, AWSEndpointsID: iam.EndpointsID, AWSServiceID: iam.ServiceID, ProviderNameUpper: "IAM", HCLKeys: []string{"iam"}, EnvVar: "TF_AWS_IAM_ENDPOINT", DeprecatedEnvVar: "AWS_IAM_ENDPOINT"}
serviceData[IdentityStore] = &ServiceDatum{AWSClientName: "IdentityStore", AWSServiceName: identitystore.ServiceName, AWSEndpointsID: identitystore.EndpointsID, AWSServiceID: identitystore.ServiceID, ProviderNameUpper: "IdentityStore", HCLKeys: []string{"identitystore"}}
serviceData[ImageBuilder] = &ServiceDatum{AWSClientName: "ImageBuilder", AWSServiceName: imagebuilder.ServiceName, AWSEndpointsID: imagebuilder.EndpointsID, AWSServiceID: imagebuilder.ServiceID, ProviderNameUpper: "ImageBuilder", HCLKeys: []string{"imagebuilder"}}
serviceData[Inspector] = &ServiceDatum{AWSClientName: "Inspector", AWSServiceName: inspector.ServiceName, AWSEndpointsID: inspector.EndpointsID, AWSServiceID: inspector.ServiceID, ProviderNameUpper: "Inspector", HCLKeys: []string{"inspector"}}
Expand Down Expand Up @@ -797,7 +799,7 @@ func init() {
serviceData[Route53RecoveryControlConfig] = &ServiceDatum{AWSClientName: "Route53RecoveryControlConfig", AWSServiceName: route53recoverycontrolconfig.ServiceName, AWSEndpointsID: route53recoverycontrolconfig.EndpointsID, AWSServiceID: route53recoverycontrolconfig.ServiceID, ProviderNameUpper: "Route53RecoveryControlConfig", HCLKeys: []string{"route53recoverycontrolconfig"}}
serviceData[Route53RecoveryReadiness] = &ServiceDatum{AWSClientName: "Route53RecoveryReadiness", AWSServiceName: route53recoveryreadiness.ServiceName, AWSEndpointsID: route53recoveryreadiness.EndpointsID, AWSServiceID: route53recoveryreadiness.ServiceID, ProviderNameUpper: "Route53RecoveryReadiness", HCLKeys: []string{"route53recoveryreadiness"}}
serviceData[Route53Resolver] = &ServiceDatum{AWSClientName: "Route53Resolver", AWSServiceName: route53resolver.ServiceName, AWSEndpointsID: route53resolver.EndpointsID, AWSServiceID: route53resolver.ServiceID, ProviderNameUpper: "Route53Resolver", HCLKeys: []string{"route53resolver"}}
serviceData[S3] = &ServiceDatum{AWSClientName: "S3", AWSServiceName: s3.ServiceName, AWSEndpointsID: s3.EndpointsID, AWSServiceID: s3.ServiceID, ProviderNameUpper: "S3", HCLKeys: []string{"s3"}}
serviceData[S3] = &ServiceDatum{AWSClientName: "S3", AWSServiceName: s3.ServiceName, AWSEndpointsID: s3.EndpointsID, AWSServiceID: s3.ServiceID, ProviderNameUpper: "S3", HCLKeys: []string{"s3"}, EnvVar: "TF_AWS_S3_ENDPOINT", DeprecatedEnvVar: "AWS_S3_ENDPOINT"}
serviceData[S3Control] = &ServiceDatum{AWSClientName: "S3Control", AWSServiceName: s3control.ServiceName, AWSEndpointsID: s3control.EndpointsID, AWSServiceID: s3control.ServiceID, ProviderNameUpper: "S3Control", HCLKeys: []string{"s3control"}}
serviceData[S3Outposts] = &ServiceDatum{AWSClientName: "S3Outposts", AWSServiceName: s3outposts.ServiceName, AWSEndpointsID: s3outposts.EndpointsID, AWSServiceID: s3outposts.ServiceID, ProviderNameUpper: "S3Outposts", HCLKeys: []string{"s3outposts"}}
serviceData[SageMaker] = &ServiceDatum{AWSClientName: "SageMaker", AWSServiceName: sagemaker.ServiceName, AWSEndpointsID: sagemaker.EndpointsID, AWSServiceID: sagemaker.ServiceID, ProviderNameUpper: "SageMaker", HCLKeys: []string{"sagemaker"}}
Expand Down Expand Up @@ -829,7 +831,7 @@ func init() {
serviceData[SSOAdmin] = &ServiceDatum{AWSClientName: "SSOAdmin", AWSServiceName: ssoadmin.ServiceName, AWSEndpointsID: ssoadmin.EndpointsID, AWSServiceID: ssoadmin.ServiceID, ProviderNameUpper: "SSOAdmin", HCLKeys: []string{"ssoadmin"}}
serviceData[SSOOIDC] = &ServiceDatum{AWSClientName: "SSOOIDC", AWSServiceName: ssooidc.ServiceName, AWSEndpointsID: ssooidc.EndpointsID, AWSServiceID: ssooidc.ServiceID, ProviderNameUpper: "SSOOIDC", HCLKeys: []string{"ssooidc"}}
serviceData[StorageGateway] = &ServiceDatum{AWSClientName: "StorageGateway", AWSServiceName: storagegateway.ServiceName, AWSEndpointsID: storagegateway.EndpointsID, AWSServiceID: storagegateway.ServiceID, ProviderNameUpper: "StorageGateway", HCLKeys: []string{"storagegateway"}}
serviceData[STS] = &ServiceDatum{AWSClientName: "STS", AWSServiceName: sts.ServiceName, AWSEndpointsID: sts.EndpointsID, AWSServiceID: sts.ServiceID, ProviderNameUpper: "STS", HCLKeys: []string{"sts"}}
serviceData[STS] = &ServiceDatum{AWSClientName: "STS", AWSServiceName: sts.ServiceName, AWSEndpointsID: sts.EndpointsID, AWSServiceID: sts.ServiceID, ProviderNameUpper: "STS", HCLKeys: []string{"sts"}, EnvVar: "TF_AWS_STS_ENDPOINT", DeprecatedEnvVar: "AWS_STS_ENDPOINT"}
serviceData[Support] = &ServiceDatum{AWSClientName: "Support", AWSServiceName: support.ServiceName, AWSEndpointsID: support.EndpointsID, AWSServiceID: support.ServiceID, ProviderNameUpper: "Support", HCLKeys: []string{"support"}}
serviceData[SWF] = &ServiceDatum{AWSClientName: "SWF", AWSServiceName: swf.ServiceName, AWSEndpointsID: swf.EndpointsID, AWSServiceID: swf.ServiceID, ProviderNameUpper: "SWF", HCLKeys: []string{"swf"}}
serviceData[Synthetics] = &ServiceDatum{AWSClientName: "Synthetics", AWSServiceName: synthetics.ServiceName, AWSEndpointsID: synthetics.EndpointsID, AWSServiceID: synthetics.ServiceID, ProviderNameUpper: "Synthetics", HCLKeys: []string{"synthetics"}}
Expand Down Expand Up @@ -1973,3 +1975,19 @@ func ServiceProviderNameUpper(key string) (string, error) {

return "", fmt.Errorf("no service data found for %s", key)
}

func ServiceDeprecatedEnvVar(key string) string {
if v, ok := serviceData[key]; ok {
return v.DeprecatedEnvVar
}

return ""
}

func ServiceEnvVar(key string) string {
if v, ok := serviceData[key]; ok {
return v.EnvVar
}

return ""
}
58 changes: 43 additions & 15 deletions internal/provider/provider.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package provider
import (
"fmt"
"log"
"os"
"regexp"
"time"

Expand Down Expand Up @@ -1899,21 +1900,8 @@ func providerConfigure(d *schema.ResourceData, terraformVersion string) (interfa
}

endpointsSet := d.Get("endpoints").(*schema.Set)

for _, endpointsSetI := range endpointsSet.List() {
endpoints := endpointsSetI.(map[string]interface{})

for _, hclKey := range conns.HCLKeys() {
var serviceKey string
var err error
if serviceKey, err = conns.ServiceForHCLKey(hclKey); err != nil {
return nil, fmt.Errorf("failed to assign endpoint (%s): %w", hclKey, err)
}

if config.Endpoints[serviceKey] == "" && endpoints[hclKey].(string) != "" {
config.Endpoints[serviceKey] = endpoints[hclKey].(string)
}
}
if err := expandEndpoints(endpointsSet.List(), config.Endpoints); err != nil {
return nil, err
}

if v, ok := d.GetOk("allowed_account_ids"); ok {
Expand Down Expand Up @@ -2117,3 +2105,43 @@ func expandProviderIgnoreTags(l []interface{}) *tftags.IgnoreConfig {

return ignoreConfig
}

func expandEndpoints(endpointsSetList []interface{}, out map[string]string) error {
for _, endpointsSetI := range endpointsSetList {
endpoints := endpointsSetI.(map[string]interface{})

for _, hclKey := range conns.HCLKeys() {
var serviceKey string
var err error
if serviceKey, err = conns.ServiceForHCLKey(hclKey); err != nil {
return fmt.Errorf("failed to assign endpoint (%s): %w", hclKey, err)
}

if out[serviceKey] == "" && endpoints[hclKey].(string) != "" {
out[serviceKey] = endpoints[hclKey].(string)
}
}
}

for _, service := range conns.ServiceKeys() {
if out[service] != "" {
continue
}

envvar := conns.ServiceEnvVar(service)
if envvar != "" {
if v := os.Getenv(envvar); v != "" {
out[service] = v
continue
}
}
if envvarDeprecated := conns.ServiceDeprecatedEnvVar(service); envvarDeprecated != "" {
if v := os.Getenv(envvarDeprecated); v != "" {
log.Printf("[WARN] The environment variable %q is deprecated. Use %q instead.", envvarDeprecated, envvar)
out[service] = v
}
}
}

return nil
}
190 changes: 190 additions & 0 deletions internal/provider/provider_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,190 @@
package provider

import (
"os"
"strings"
"testing"

"github.com/hashicorp/terraform-provider-aws/internal/conns"
)

func TestExpandEndpoints(t *testing.T) {
oldEnv := stashEnv()
defer popEnv(oldEnv)

endpoints := make(map[string]interface{})
for _, serviceKey := range conns.HCLKeys() {
endpoints[serviceKey] = ""
}
endpoints["sts"] = "https://sts.fake.test"

results := make(map[string]string)

err := expandEndpoints([]interface{}{endpoints}, results)
if err != nil {
t.Fatalf("Unexpected error: %s", err)
}

if len(results) != 1 {
t.Errorf("Expected 1 endpoint, got %d", len(results))
}

if v := results["sts"]; v != "https://sts.fake.test" {
t.Errorf("Expected endpoint %q, got %v", "https://sts.fake.test", results)
}
}

func TestEndpointMultipleKeys(t *testing.T) {
testcases := []struct {
endpoints map[string]string
expectedService string
expectedEndpoint string
}{
{
endpoints: map[string]string{
"transcribe": "https://transcribe.fake.test",
},
expectedService: conns.Transcribe,
expectedEndpoint: "https://transcribe.fake.test",
},
{
endpoints: map[string]string{
"transcribeservice": "https://transcribe.fake.test",
},
expectedService: conns.Transcribe,
expectedEndpoint: "https://transcribe.fake.test",
},
{
endpoints: map[string]string{
"transcribe": "https://transcribe.fake.test",
"transcribeservice": "https://transcribeservice.fake.test",
},
expectedService: conns.Transcribe,
expectedEndpoint: "https://transcribe.fake.test",
},
}

for _, testcase := range testcases {
oldEnv := stashEnv()
defer popEnv(oldEnv)

endpoints := make(map[string]interface{})
for _, serviceKey := range conns.HCLKeys() {
endpoints[serviceKey] = ""
}
for k, v := range testcase.endpoints {
endpoints[k] = v
}

results := make(map[string]string)

err := expandEndpoints([]interface{}{endpoints}, results)
if err != nil {
t.Fatalf("Unexpected error: %s", err)
}

if a, e := len(results), 1; a != e {
t.Errorf("Expected 1 endpoint, got %d", len(results))
}

if v := results[testcase.expectedService]; v != testcase.expectedEndpoint {
t.Errorf("Expected endpoint[%s] to be %q, got %v", testcase.expectedService, testcase.expectedEndpoint, results)
}
}
}

func TestEndpointEnvVarPrecedence(t *testing.T) {
testcases := []struct {
endpoints map[string]string
envvars map[string]string
expectedService string
expectedEndpoint string
}{
{
endpoints: map[string]string{},
envvars: map[string]string{
"TF_AWS_STS_ENDPOINT": "https://sts.fake.test",
},
expectedService: conns.STS,
expectedEndpoint: "https://sts.fake.test",
},
{
endpoints: map[string]string{},
envvars: map[string]string{
"AWS_STS_ENDPOINT": "https://sts-deprecated.fake.test",
},
expectedService: conns.STS,
expectedEndpoint: "https://sts-deprecated.fake.test",
},
{
endpoints: map[string]string{},
envvars: map[string]string{
"TF_AWS_STS_ENDPOINT": "https://sts.fake.test",
"AWS_STS_ENDPOINT": "https://sts-deprecated.fake.test",
},
expectedService: conns.STS,
expectedEndpoint: "https://sts.fake.test",
},
{
endpoints: map[string]string{
"sts": "https://sts-config.fake.test",
},
envvars: map[string]string{
"TF_AWS_STS_ENDPOINT": "https://sts-env.fake.test",
},
expectedService: conns.STS,
expectedEndpoint: "https://sts-config.fake.test",
},
}

for _, testcase := range testcases {
oldEnv := stashEnv()
defer popEnv(oldEnv)

for k, v := range testcase.envvars {
os.Setenv(k, v)
}

endpoints := make(map[string]interface{})
for _, serviceKey := range conns.HCLKeys() {
endpoints[serviceKey] = ""
}
for k, v := range testcase.endpoints {
endpoints[k] = v
}

results := make(map[string]string)

err := expandEndpoints([]interface{}{endpoints}, results)
if err != nil {
t.Fatalf("Unexpected error: %s", err)
}

if a, e := len(results), 1; a != e {
t.Errorf("Expected 1 endpoint, got %d", len(results))
}

if v := results[testcase.expectedService]; v != testcase.expectedEndpoint {
t.Errorf("Expected endpoint[%s] to be %q, got %v", testcase.expectedService, testcase.expectedEndpoint, results)
}
}
}

func stashEnv() []string {
env := os.Environ()
os.Clearenv()
return env
}

func popEnv(env []string) {
os.Clearenv()

for _, e := range env {
p := strings.SplitN(e, "=", 2)
k, v := p[0], ""
if len(p) > 1 {
v = p[1]
}
os.Setenv(k, v)
}
}
8 changes: 8 additions & 0 deletions website/docs/guides/custom-service-endpoints.html.md
Original file line number Diff line number Diff line change
Expand Up @@ -343,6 +343,14 @@ provider "aws" {
</div>
<!-- markdownlint-enable MD033 -->

As a convenience, for compatibility with the [Terraform S3 Backend](https://www.terraform.io/language/settings/backends/s3),
the following service endpoints can be configured using environment variables:

* DynamoDB: `TF_AWS_DYNAMODB_ENDPOINT` (or **Deprecated** `AWS_DYNAMODB_ENDPOINT`)
* IAM: `TF_AWS_IAM_ENDPOINT` (or **Deprecated** `AWS_IAM_ENDPOINT`)
* S3: `TF_AWS_S3_ENDPOINT` (or **Deprecated** `AWS_S3_ENDPOINT`)
* STS: `TF_AWS_STS_ENDPOINT` (or **Deprecated** `AWS_STS_ENDPOINT`)

## Connecting to Local AWS Compatible Solutions

~> **NOTE:** This information is not intended to be exhaustive for all local AWS compatible solutions or necessarily authoritative configurations for those documented. Check the documentation for each of these solutions for the most up to date information.
Expand Down

0 comments on commit 1cd3073

Please sign in to comment.