Retry AWS commands that may fail and increase insufficient timeouts

AWS is lately experiencing increasing flakiness, or "eventual consistency",
that forces to retry a lot of operations.
Describing the resources until they show as available is not even enough
as a following call to eg. setTags will fail because the resource doesn't
exist.

builtin/providers/aws/resource_aws_internet_gateway.go

@@ -49,6 +49,7 @@ func resourceAwsInternetGatewayCreate(d *schema.ResourceData, meta interface{})
 	log.Printf("[INFO] InternetGateway ID: %s", d.Id())
 
 	resource.Retry(5*time.Minute, func() *resource.RetryError {
+		log.Printf("[INFO] Refreshing InternetGateway ID: %s", d.Id())
 		igRaw, _, err := IGStateRefreshFunc(conn, d.Id())()
 		if igRaw != nil {
 			return nil
@@ -60,6 +61,7 @@ func resourceAwsInternetGatewayCreate(d *schema.ResourceData, meta interface{})
 		}
 	})
 
+	log.Printf("[INFO] Setting tags InternetGateway ID: %s", d.Id())
 	err = setTags(conn, d)
 	if err != nil {
 		return err
@@ -168,9 +170,21 @@ func resourceAwsInternetGatewayAttach(d *schema.ResourceData, meta interface{})
 		d.Id(),
 		d.Get("vpc_id").(string))
 
-	_, err := conn.AttachInternetGateway(&ec2.AttachInternetGatewayInput{
-		InternetGatewayId: aws.String(d.Id()),
-		VpcId:             aws.String(d.Get("vpc_id").(string)),
+	err := resource.Retry(2*time.Minute, func() *resource.RetryError {
+		_, err := conn.AttachInternetGateway(&ec2.AttachInternetGatewayInput{
+			InternetGatewayId: aws.String(d.Id()),
+			VpcId:             aws.String(d.Get("vpc_id").(string)),
+		})
+		if err == nil {
+			return nil
+		}
+		if ec2err, ok := err.(awserr.Error); ok {
+			switch ec2err.Code() {
+			case "InvalidInternetGatewayID.NotFound":
+				return resource.RetryableError(err) // retry
+			}
+		}
+		return resource.NonRetryableError(err)
 	})
 	if err != nil {
 		return err
@@ -187,7 +201,7 @@ func resourceAwsInternetGatewayAttach(d *schema.ResourceData, meta interface{})
 		Pending: []string{"detached", "attaching"},
 		Target:  []string{"available"},
 		Refresh: IGAttachStateRefreshFunc(conn, d.Id(), "available"),
-		Timeout: 1 * time.Minute,
+		Timeout: 2 * time.Minute,
 	}
 	if _, err := stateConf.WaitForState(); err != nil {
 		return fmt.Errorf(

builtin/providers/aws/resource_aws_vpc.go

@@ -122,6 +122,8 @@ func resourceAwsVpcCreate(d *schema.ResourceData, meta interface{}) error {
 			d.Id(), err)
 	}
 
+	log.Printf("[INFO] VPC (%s) is now available", d.Id())
+
 	// Update our attributes and return
 	return resourceAwsVpcUpdate(d, meta)
 }
@@ -300,6 +302,7 @@ func resourceAwsVpcUpdate(d *schema.ResourceData, meta interface{}) error {
 	} else {
 		d.SetPartial("tags")
 	}
+	log.Printf("[INFO] Done set tags for VPC ID: %s", d.Id())
 
 	d.Partial(false)
 	return resourceAwsVpcRead(d, meta)
@@ -316,8 +319,10 @@ func resourceAwsVpcDelete(d *schema.ResourceData, meta interface{}) error {
 	return resource.Retry(5*time.Minute, func() *resource.RetryError {
 		_, err := conn.DeleteVpc(DeleteVpcOpts)
 		if err == nil {
+			log.Printf("[INFO] Deleted VPC ID: %s", d.Id())
 			return nil
 		}
+		log.Printf("[INFO] Error deleting VPC ID: %s %s", d.Id(), err)
 
 		ec2err, ok := err.(awserr.Error)
 		if !ok {
@@ -326,8 +331,10 @@ func resourceAwsVpcDelete(d *schema.ResourceData, meta interface{}) error {
 
 		switch ec2err.Code() {
 		case "InvalidVpcID.NotFound":
+			log.Printf("[INFO] Deleted VPC ID (no longer exists): %s", d.Id())
 			return nil
 		case "DependencyViolation":
+			log.Printf("[INFO] Will retry DependencyViolation deleting VPC ID: %s", d.Id())
 			return resource.RetryableError(err)
 		}
 

builtin/providers/aws/tags.go

@@ -2,9 +2,13 @@ package aws
 
 import (
 	"log"
+	"strings"
+	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/hashicorp/terraform/helper/resource"
 	"github.com/hashicorp/terraform/helper/schema"
 )
 
@@ -28,25 +32,47 @@ func setTags(conn *ec2.EC2, d *schema.ResourceData) error {
 
 		// Set tags
 		if len(remove) > 0 {
-			log.Printf("[DEBUG] Removing tags: %#v from %s", remove, d.Id())
-			_, err := conn.DeleteTags(&ec2.DeleteTagsInput{
-				Resources: []*string{aws.String(d.Id())},
-				Tags:      remove,
+			err := resource.Retry(2*time.Minute, func() *resource.RetryError {
+				log.Printf("[DEBUG] Removing tags: %#v from %s", remove, d.Id())
+				_, err := conn.DeleteTags(&ec2.DeleteTagsInput{
+					Resources: []*string{aws.String(d.Id())},
+					Tags:      remove,
+				})
+				if err != nil {
+					ec2err, ok := err.(awserr.Error)
+					if ok && strings.Contains(ec2err.Code(), ".NotFound") {
+						return resource.RetryableError(err) // retry
+					}
+					return resource.NonRetryableError(err)
+				}
+				return nil
 			})
 			if err != nil {
 				return err
 			}
 		}
+
 		if len(create) > 0 {
-			log.Printf("[DEBUG] Creating tags: %s for %s", create, d.Id())
-			_, err := conn.CreateTags(&ec2.CreateTagsInput{
-				Resources: []*string{aws.String(d.Id())},
-				Tags:      create,
+			err := resource.Retry(2*time.Minute, func() *resource.RetryError {
+				log.Printf("[DEBUG] Creating tags: %s for %s", create, d.Id())
+				_, err := conn.CreateTags(&ec2.CreateTagsInput{
+					Resources: []*string{aws.String(d.Id())},
+					Tags:      create,
+				})
+				if err != nil {
+					ec2err, ok := err.(awserr.Error)
+					if ok && strings.Contains(ec2err.Code(), ".NotFound") {
+						return resource.RetryableError(err) // retry
+					}
+					return resource.NonRetryableError(err)
+				}
+				return nil
 			})
 			if err != nil {
 				return err
 			}
 		}
+
 	}
 
 	return nil