From 004e9e45c2affde0176c3362ac8ffd6c9f348dd0 Mon Sep 17 00:00:00 2001 From: Abhinav Dahiya Date: Thu, 1 Nov 2018 10:34:28 -0700 Subject: [PATCH] awstagdeprovision: add support for deleting V2 LBs and LB Target groups To delete these load balancers: * deleting V2 load balancers ```console aws elbv2 describe-load-balancers { "LoadBalancers": [ { "DNSName": "crawford-aws-int-a85653d6d18c082c.elb.us-east-1.amazonaws.com", "VpcId": "vpc-06de9aca2fabbf1b7", "State": { "Code": "active" }, "Type": "network", "AvailabilityZones": [ { "ZoneName": "us-east-1c", "SubnetId": "subnet-061048a5f17adb00b", "LoadBalancerAddresses": [ {} ] }, { "ZoneName": "us-east-1d", "SubnetId": "subnet-03506a45cf76bbb78", "LoadBalancerAddresses": [ {} ] }, { "ZoneName": "us-east-1a", "SubnetId": "subnet-0d36ad7ab10e8a721", "LoadBalancerAddresses": [ {} ] }, { "ZoneName": "us-east-1f", "SubnetId": "subnet-00f19af286407f964", "LoadBalancerAddresses": [ {} ] }, { "ZoneName": "us-east-1e", "SubnetId": "subnet-0adbb874efe59de3f", "LoadBalancerAddresses": [ {} ] }, { "ZoneName": "us-east-1b", "SubnetId": "subnet-0122ba9d35b720814", "LoadBalancerAddresses": [ {} ] } ], "IpAddressType": "ipv4" }, { "DNSName": "crawford-aws-ext-e443ac9111e22219.elb.us-east-1.amazonaws.com", "CanonicalHostedZoneId": "Z26RNL4JYFTOTI", "CreatedTime": "2018-11-01T15:30:22.577Z", "LoadBalancerName": "crawford-aws-ext", "Scheme": "internet-facing", "VpcId": "vpc-06de9aca2fabbf1b7", "State": { "Code": "active" }, "Type": "network", "AvailabilityZones": [ { "ZoneName": "us-east-1e", "SubnetId": "subnet-0adbb874efe59de3f", "LoadBalancerAddresses": [ {} ] }, { "ZoneName": "us-east-1a", "SubnetId": "subnet-0d36ad7ab10e8a721", "LoadBalancerAddresses": [ {} ] }, { "ZoneName": "us-east-1b", "SubnetId": "subnet-0122ba9d35b720814", "LoadBalancerAddresses": [ {} ] }, { "ZoneName": "us-east-1f", "SubnetId": "subnet-00f19af286407f964", "LoadBalancerAddresses": [ {} ] }, { "ZoneName": "us-east-1d", "SubnetId": "subnet-03506a45cf76bbb78", "LoadBalancerAddresses": [ {} ] }, { "ZoneName": "us-east-1c", "SubnetId": "subnet-061048a5f17adb00b", "LoadBalancerAddresses": [ {} ] } ], "IpAddressType": "ipv4" } ] } ``` These load balancers are tagged, but the api does not allow filters based on tags, so we do that same as the V1 Lbs, filter on VpcID ```console aws elbv2 delete-load-balancer --load-balancer-arn ``` * deleting target groups ```console aws elbv2 describe-target-groups { "TargetGroups": [ { "TargetGroupName": "crawford-aws-api-external", "Protocol": "TCP", "Port": 6443, "VpcId": "vpc-06de9aca2fabbf1b7", "HealthCheckProtocol": "TCP", "HealthCheckPort": "6443", "HealthCheckIntervalSeconds": 10, "HealthCheckTimeoutSeconds": 10, "HealthyThresholdCount": 3, "UnhealthyThresholdCount": 3, "LoadBalancerArns": [], "TargetType": "ip" }, { "TargetGroupName": "crawford-aws-api-internal", "Protocol": "TCP", "Port": 6443, "VpcId": "vpc-06de9aca2fabbf1b7", "HealthCheckProtocol": "TCP", "HealthCheckPort": "6443", "HealthCheckIntervalSeconds": 10, "HealthCheckTimeoutSeconds": 10, "HealthyThresholdCount": 3, "UnhealthyThresholdCount": 3, "LoadBalancerArns": [ "arn:aws:elasticloadbalancing:us-east-1:816138690521:loadbalancer/net/crawford-aws-int/a85653d6d18c082c" ], "TargetType": "ip" }, { "TargetGroupName": "crawford-aws-services", "Protocol": "TCP", "Port": 49500, "VpcId": "vpc-06de9aca2fabbf1b7", "HealthCheckProtocol": "TCP", "HealthCheckPort": "49500", "HealthCheckIntervalSeconds": 10, "HealthCheckTimeoutSeconds": 10, "HealthyThresholdCount": 3, "UnhealthyThresholdCount": 3, "LoadBalancerArns": [ "arn:aws:elasticloadbalancing:us-east-1:816138690521:loadbalancer/net/crawford-aws-int/a85653d6d18c082c" ], "TargetType": "ip" } ] } ``` These target groups can be tagged, but i couldn't find a way to get those tags, and the describe api doesn't allow for filtering on tags anyways, so we filter on our VpcId ```console aws elbv2 delete-target-group --target-group-arn ``` deleting target groups without deleting all the load balancers that have listeners to this target groups returns ```console An error occurred (ResourceInUse) when calling the DeleteTargetGroup operation: Target group 'REDACTED' is currently in use by a listener or a rule ``` So we call delete target groups after we have completed delete of all v2 LBs. --- .../awstagdeprovision/awstagdeprovision.go | 106 +++++++++++++++++- 1 file changed, 104 insertions(+), 2 deletions(-) diff --git a/contrib/pkg/awstagdeprovision/awstagdeprovision.go b/contrib/pkg/awstagdeprovision/awstagdeprovision.go index d2658b694b1..5eb68866d9e 100644 --- a/contrib/pkg/awstagdeprovision/awstagdeprovision.go +++ b/contrib/pkg/awstagdeprovision/awstagdeprovision.go @@ -29,6 +29,7 @@ import ( "github.com/aws/aws-sdk-go/service/autoscaling" "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/elb" + "github.com/aws/aws-sdk-go/service/elbv2" "github.com/aws/aws-sdk-go/service/iam" "github.com/aws/aws-sdk-go/service/route53" "github.com/aws/aws-sdk-go/service/s3" @@ -266,6 +267,106 @@ func deleteLBs(vpc *ec2.Vpc, awsSession *session.Session, logger log.FieldLogger return false } +// filterV2LBsByVPC will find all the load balancers in the provided list that are under the provided VPC +func filterV2LBsByVPC(lbs []*elbv2.LoadBalancer, vpc *ec2.Vpc, logger log.FieldLogger) []*elbv2.LoadBalancer { + filteredLBs := []*elbv2.LoadBalancer{} + + for _, lb := range lbs { + if *lb.VpcId == *vpc.VpcId { + filteredLBs = append(filteredLBs, lb) + } + } + + return filteredLBs +} + +func deleteV2LBs(vpc *ec2.Vpc, awsSession *session.Session, logger log.FieldLogger) bool { + logger.Debugf("Deleting V2 load balancers (%s)", *vpc.VpcId) + defer logger.Debugf("Exiting deleting V2 load balancers (%s)", *vpc.VpcId) + elbv2Client := elbv2.New(awsSession) + + total := 0 + filteredLBs := []*elbv2.LoadBalancer{} + if err := elbv2Client.DescribeLoadBalancersPages(&elbv2.DescribeLoadBalancersInput{}, func(results *elbv2.DescribeLoadBalancersOutput, lastPage bool) bool { + total += len(results.LoadBalancers) + filteredLBs = append(filteredLBs, filterV2LBsByVPC(results.LoadBalancers, vpc, logger)...) + return lastPage + }); err != nil { + logger.Errorf("Error listing V2 load balancers: %v", err) + return false + } + logger.Debugf("from %d total V2 load balancers, %d scheduled for deletion", total, len(filteredLBs)) + + if len(filteredLBs) == 0 { + // no items left to delete + // see if we can delete target groups. + return deleteTargetGroups(vpc, awsSession, logger) + } + + for _, lb := range filteredLBs { + logger.Debugf("Deleting V2 load balancer: %v", *lb.LoadBalancerName) + _, err := elbv2Client.DeleteLoadBalancer(&elbv2.DeleteLoadBalancerInput{ + LoadBalancerArn: lb.LoadBalancerArn, + }) + if err != nil { + logger.Debugf("Error deleting V2 load balancer %v: %v", *lb.LoadBalancerName, err) + } else { + logger.WithField("name", *lb.LoadBalancerName).Info("Deleted load balancer") + } + } + // cleanup target groups + return deleteTargetGroups(vpc, awsSession, logger) +} + +// filterTargetGroupsByVPC will find all the target groups in the provided list that are under the provided VPC +func filterTargetGroupsByVPC(tgs []*elbv2.TargetGroup, vpc *ec2.Vpc, logger log.FieldLogger) []*elbv2.TargetGroup { + filteredTGs := []*elbv2.TargetGroup{} + + for _, tg := range tgs { + if *tg.VpcId == *vpc.VpcId { + filteredTGs = append(filteredTGs, tg) + } + } + + return filteredTGs +} + +func deleteTargetGroups(vpc *ec2.Vpc, awsSession *session.Session, logger log.FieldLogger) bool { + logger.Debugf("Deleting target groups (%s)", *vpc.VpcId) + defer logger.Debugf("Exiting deleting target groups (%s)", *vpc.VpcId) + elbv2Client := elbv2.New(awsSession) + + total := 0 + filteredTGs := []*elbv2.TargetGroup{} + if err := elbv2Client.DescribeTargetGroupsPages(&elbv2.DescribeTargetGroupsInput{}, func(results *elbv2.DescribeTargetGroupsOutput, lastPage bool) bool { + total += len(results.TargetGroups) + filteredTGs = append(filteredTGs, filterTargetGroupsByVPC(results.TargetGroups, vpc, logger)...) + return lastPage + }); err != nil { + logger.Errorf("Error listing target groups: %v", err) + return false + } + logger.Debugf("from %d total target groups, %d scheduled for deletion", total, len(filteredTGs)) + + if len(filteredTGs) == 0 { + // no items left to delete + return true + } + + for _, tg := range filteredTGs { + logger.Debugf("Deleting target groups: %v", *tg.TargetGroupName) + _, err := elbv2Client.DeleteTargetGroup(&elbv2.DeleteTargetGroupInput{ + TargetGroupArn: tg.TargetGroupArn, + }) + if err != nil { + logger.Debugf("Error deleting target groups %v: %v", *tg.TargetGroupName, err) + } else { + logger.WithField("name", *tg.TargetGroupName).Info("Deleted target group") + } + } + return false +} + // rtHasMainAssociation will check whether a given route table has an association marked 'Main' func rtHasMainAssociation(rt *ec2.RouteTable) bool { for _, association := range rt.Associations { @@ -348,8 +449,9 @@ func deleteVPCs(awsSession *session.Session, filters AWSFilter, clusterName stri for _, vpc := range results.Vpcs { // first delete any Load Balancers under this VPC (not all of them are tagged) - complete := deleteLBs(vpc, awsSession, logger) - if !complete { + v1lbcomplete := deleteLBs(vpc, awsSession, logger) + v2lbcomplete := deleteV2LBs(vpc, awsSession, logger) + if !v1lbcomplete || !v2lbcomplete { logger.Debugf("not finished deleting load balancers, will need to retry") return false, nil }