diff --git a/data/data/aws/main.tf b/data/data/aws/main.tf index 6db04c0f6c5..61e060e53d9 100644 --- a/data/data/aws/main.tf +++ b/data/data/aws/main.tf @@ -90,6 +90,7 @@ module "dns" { cluster_domain = var.cluster_domain cluster_id = var.cluster_id tags = local.tags + internal_zone = var.aws_internal_zone vpc_id = module.vpc.vpc_id region = var.aws_region publish_strategy = var.aws_publish_strategy diff --git a/data/data/aws/route53/base.tf b/data/data/aws/route53/base.tf index b457cf47ac3..9cd2af799b8 100644 --- a/data/data/aws/route53/base.tf +++ b/data/data/aws/route53/base.tf @@ -15,7 +15,13 @@ data "aws_route53_zone" "public" { name = var.base_domain } -resource "aws_route53_zone" "int" { +data "aws_route53_zone" "int" { + zone_id = var.internal_zone == null ? aws_route53_zone.new_int[0].id : var.internal_zone +} + +resource "aws_route53_zone" "new_int" { + count = var.internal_zone == null ? 1 : 0 + name = var.cluster_domain force_destroy = true @@ -50,7 +56,7 @@ resource "aws_route53_record" "api_external_alias" { resource "aws_route53_record" "api_internal_alias" { count = local.use_alias ? 1 : 0 - zone_id = aws_route53_zone.int.zone_id + zone_id = data.aws_route53_zone.int.zone_id name = "api-int.${var.cluster_domain}" type = "A" @@ -64,7 +70,7 @@ resource "aws_route53_record" "api_internal_alias" { resource "aws_route53_record" "api_external_internal_zone_alias" { count = local.use_alias ? 1 : 0 - zone_id = aws_route53_zone.int.zone_id + zone_id = data.aws_route53_zone.int.zone_id name = "api.${var.cluster_domain}" type = "A" @@ -89,7 +95,7 @@ resource "aws_route53_record" "api_external_cname" { resource "aws_route53_record" "api_internal_cname" { count = local.use_cname ? 1 : 0 - zone_id = aws_route53_zone.int.zone_id + zone_id = data.aws_route53_zone.int.zone_id name = "api-int.${var.cluster_domain}" type = "CNAME" ttl = 10 @@ -100,7 +106,7 @@ resource "aws_route53_record" "api_internal_cname" { resource "aws_route53_record" "api_external_internal_zone_cname" { count = local.use_cname ? 1 : 0 - zone_id = aws_route53_zone.int.zone_id + zone_id = data.aws_route53_zone.int.zone_id name = "api.${var.cluster_domain}" type = "CNAME" ttl = 10 diff --git a/data/data/aws/route53/variables.tf b/data/data/aws/route53/variables.tf index b06db70d97f..67f938ed2c9 100644 --- a/data/data/aws/route53/variables.tf +++ b/data/data/aws/route53/variables.tf @@ -23,6 +23,11 @@ variable "tags" { description = "AWS tags to be applied to created resources." } +variable "internal_zone" { + type = string + description = "An existing hosted zone (zone ID) to use for the internal API." +} + variable "api_external_lb_dns_name" { description = "External API's LB DNS name" type = string diff --git a/data/data/aws/variables-aws.tf b/data/data/aws/variables-aws.tf index a7e50cb4eed..0bbbb587ba8 100644 --- a/data/data/aws/variables-aws.tf +++ b/data/data/aws/variables-aws.tf @@ -126,6 +126,12 @@ variable "aws_private_subnets" { description = "(optional) Existing private subnets into which the cluster should be installed." } +variable "aws_internal_zone" { + type = string + default = null + description = "(optional) An existing hosted zone (zone ID) to use for the internal API." +} + variable "aws_publish_strategy" { type = string description = "The cluster publishing strategy, either Internal or External" diff --git a/data/data/install.openshift.io_installconfigs.yaml b/data/data/install.openshift.io_installconfigs.yaml index 5c9d21ecd13..46b1ee607b7 100644 --- a/data/data/install.openshift.io_installconfigs.yaml +++ b/data/data/install.openshift.io_installconfigs.yaml @@ -989,6 +989,14 @@ spec: type: string type: array type: object + hostedZone: + description: HostedZone is the ID of an existing hosted zone into + which to add DNS records for the cluster's internal API. An + existing hosted zone can only be used when also using existing + subnets. The hosted zone must be associated with the VPC containing + the subnets. Leave the hosted zone unset to have the installer + create the hosted zone on your behalf. + type: string region: description: Region specifies the AWS region where the cluster will be created. diff --git a/pkg/asset/cluster/aws/aws.go b/pkg/asset/cluster/aws/aws.go index 56b70d9bc93..932fe6ce5ec 100644 --- a/pkg/asset/cluster/aws/aws.go +++ b/pkg/asset/cluster/aws/aws.go @@ -8,6 +8,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/iam" + "github.com/aws/aws-sdk-go/service/route53" "github.com/pkg/errors" "github.com/openshift/installer/pkg/asset/installconfig" @@ -25,6 +26,7 @@ func Metadata(clusterID, infraID string, config *types.InstallConfig) *awstypes. "openshiftClusterID": clusterID, }}, ServiceEndpoints: config.AWS.ServiceEndpoints, + ClusterDomain: config.ClusterDomain(), } } @@ -32,18 +34,18 @@ func Metadata(clusterID, infraID string, config *types.InstallConfig) *awstypes. // happen before Terraform creates the remaining infrastructure. func PreTerraform(ctx context.Context, clusterID string, installConfig *installconfig.InstallConfig) error { - if err := tagSubnetEC2Instances(ctx, clusterID, installConfig); err != nil { + if err := tagSharedVPCResources(ctx, clusterID, installConfig); err != nil { return err } - if err := tagIamRoles(ctx, clusterID, installConfig); err != nil { + if err := tagSharedIAMRoles(ctx, clusterID, installConfig); err != nil { return err } return nil } -func tagSubnetEC2Instances(ctx context.Context, clusterID string, installConfig *installconfig.InstallConfig) error { +func tagSharedVPCResources(ctx context.Context, clusterID string, installConfig *installconfig.InstallConfig) error { if len(installConfig.Config.Platform.AWS.Subnets) == 0 { return nil } @@ -58,7 +60,6 @@ func tagSubnetEC2Instances(ctx context.Context, clusterID string, installConfig return err } - //arns := make([]*string, 0, len(privateSubnets)+len(publicSubnets)) ids := make([]*string, 0, len(privateSubnets)+len(publicSubnets)) for id := range privateSubnets { ids = append(ids, aws.String(id)) @@ -69,23 +70,34 @@ func tagSubnetEC2Instances(ctx context.Context, clusterID string, installConfig session, err := installConfig.AWS.Session(ctx) if err != nil { - return err + return errors.Wrap(err, "could not create AWS session") } - key, value := sharedTag(clusterID) - ec2Tags := []*ec2.Tag{{Key: &key, Value: &value}} - ec2Client := ec2.New(session, aws.NewConfig().WithRegion(installConfig.Config.Platform.AWS.Region)) + tagKey, tagValue := sharedTag(clusterID) + + ec2Client := ec2.New(session, aws.NewConfig().WithRegion(installConfig.Config.Platform.AWS.Region)) if _, err = ec2Client.CreateTagsWithContext(ctx, &ec2.CreateTagsInput{ Resources: ids, - Tags: ec2Tags, + Tags: []*ec2.Tag{{Key: &tagKey, Value: &tagValue}}, }); err != nil { - return err + return errors.Wrap(err, "could not add tags to subnets") + } + + if zone := installConfig.Config.AWS.HostedZone; zone != "" { + route53Client := route53.New(session) + if _, err := route53Client.ChangeTagsForResourceWithContext(ctx, &route53.ChangeTagsForResourceInput{ + ResourceType: aws.String("hostedzone"), + ResourceId: aws.String(zone), + AddTags: []*route53.Tag{{Key: &tagKey, Value: &tagValue}}, + }); err != nil { + return errors.Wrap(err, "could not add tags to hosted zone") + } } return nil } -func tagIamRoles(ctx context.Context, clusterID string, installConfig *installconfig.InstallConfig) error { +func tagSharedIAMRoles(ctx context.Context, clusterID string, installConfig *installconfig.InstallConfig) error { var iamRoleNames []*string if mp := installConfig.Config.ControlPlane; mp != nil { awsMP := &awstypes.MachinePool{} diff --git a/pkg/asset/cluster/tfvars.go b/pkg/asset/cluster/tfvars.go index ca510aaa574..77600abcc6a 100644 --- a/pkg/asset/cluster/tfvars.go +++ b/pkg/asset/cluster/tfvars.go @@ -255,6 +255,7 @@ func (t *TerraformVariables) Generate(parents asset.Parents) error { VPC: vpc, PrivateSubnets: privateSubnets, PublicSubnets: publicSubnets, + InternalZone: installConfig.Config.AWS.HostedZone, Services: installConfig.Config.AWS.ServiceEndpoints, Publish: installConfig.Config.Publish, MasterConfigs: masterConfigs, diff --git a/pkg/asset/installconfig/aws/validation.go b/pkg/asset/installconfig/aws/validation.go index 10967936aea..9eeeb454e2a 100644 --- a/pkg/asset/installconfig/aws/validation.go +++ b/pkg/asset/installconfig/aws/validation.go @@ -6,8 +6,12 @@ import ( "net" "net/url" "sort" + "strings" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/endpoints" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/route53" "github.com/pkg/errors" utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/sets" @@ -262,3 +266,87 @@ var requiredServices = []string{ "sts", "tagging", } + +// ValidateForProvisioning validates if the install config is valid for provisioning the cluster. +func ValidateForProvisioning(session *session.Session, ic *types.InstallConfig, metadata *Metadata) error { + allErrs := field.ErrorList{} + allErrs = append(allErrs, validateExistingHostedZone(session, ic, metadata)...) + return allErrs.ToAggregate() +} + +func validateExistingHostedZone(session *session.Session, ic *types.InstallConfig, metadata *Metadata) field.ErrorList { + if ic.AWS.HostedZone == "" { + return nil + } + + // validate that the hosted zone exists + hostedZonePath := field.NewPath("aws", "hostedZone") + client := route53.New(session) + zone, err := client.GetHostedZone(&route53.GetHostedZoneInput{Id: aws.String(ic.AWS.HostedZone)}) + if err != nil { + return field.ErrorList{ + field.Invalid(hostedZonePath, ic.AWS.HostedZone, "cannot find hosted zone"), + } + } + + allErrs := field.ErrorList{} + + // validate that the hosted zone is associated with the VPC containing the existing subnets for the cluster + vpcID, err := metadata.VPC(context.TODO()) + if err == nil { + if !isHostedZoneAssociatedWithVPC(zone, vpcID) { + allErrs = append(allErrs, field.Invalid(hostedZonePath, ic.AWS.HostedZone, "hosted zone is not associated with the VPC")) + } + } else { + allErrs = append(allErrs, field.Invalid(hostedZonePath, ic.AWS.HostedZone, "no VPC found")) + } + + // validate that the hosted zone does not already have any record sets for the cluster domain + dottedClusterDomain := ic.ClusterDomain() + "." + var problematicRecords []string + if err := client.ListResourceRecordSetsPages( + &route53.ListResourceRecordSetsInput{HostedZoneId: zone.HostedZone.Id}, + func(out *route53.ListResourceRecordSetsOutput, lastPage bool) bool { + for _, recordSet := range out.ResourceRecordSets { + name := aws.StringValue(recordSet.Name) + // skip record sets that are not sub-domains of the cluster domain. Such record sets may exist for + // hosted zones that are used for other clusters or other purposes. + if !strings.HasSuffix(name, dottedClusterDomain) { + continue + } + // skip record sets that are the cluster domain. Record sets for the cluster domain are fine. If the + // hosted zone has the name of the cluster domain, then there will be NS and SOA record sets for the + // cluster domain. + if len(name) == len(dottedClusterDomain) { + continue + } + problematicRecords = append(problematicRecords, fmt.Sprintf("%s (%s)", name, aws.StringValue(recordSet.Type))) + } + return !lastPage + }, + ); err != nil { + allErrs = append(allErrs, field.InternalError(hostedZonePath, + errors.Wrapf(err, "could not list record sets for hosted zone %q", ic.AWS.HostedZone))) + } + if len(problematicRecords) > 0 { + detail := fmt.Sprintf( + "hosted zone already has record sets for the domain of the cluster: [%s]", + strings.Join(problematicRecords, ", "), + ) + allErrs = append(allErrs, field.Invalid(hostedZonePath, ic.AWS.HostedZone, detail)) + } + + return allErrs +} + +func isHostedZoneAssociatedWithVPC(hostedZone *route53.GetHostedZoneOutput, vpcID string) bool { + if vpcID == "" { + return false + } + for _, vpc := range hostedZone.VPCs { + if aws.StringValue(vpc.VPCId) == vpcID { + return true + } + } + return false +} diff --git a/pkg/asset/installconfig/platformprovisioncheck.go b/pkg/asset/installconfig/platformprovisioncheck.go index 87d45710825..88f2232f090 100644 --- a/pkg/asset/installconfig/platformprovisioncheck.go +++ b/pkg/asset/installconfig/platformprovisioncheck.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/openshift/installer/pkg/asset" + awsconfig "github.com/openshift/installer/pkg/asset/installconfig/aws" azconfig "github.com/openshift/installer/pkg/asset/installconfig/azure" bmconfig "github.com/openshift/installer/pkg/asset/installconfig/baremetal" gcpconfig "github.com/openshift/installer/pkg/asset/installconfig/gcp" @@ -45,6 +46,12 @@ func (a *PlatformProvisionCheck) Generate(dependencies asset.Parents) error { var err error platform := ic.Config.Platform.Name() switch platform { + case aws.Name: + session, err := ic.AWS.Session(context.TODO()) + if err != nil { + return err + } + return awsconfig.ValidateForProvisioning(session, ic.Config, ic.AWS) case azure.Name: dnsConfig, err := ic.Azure.DNSConfig() if err != nil { @@ -92,7 +99,7 @@ func (a *PlatformProvisionCheck) Generate(dependencies asset.Parents) error { if err != nil { return err } - case aws.Name, libvirt.Name, none.Name, ovirt.Name: + case libvirt.Name, none.Name, ovirt.Name: // no special provisioning requirements to check default: err = fmt.Errorf("unknown platform type %q", platform) diff --git a/pkg/asset/manifests/dns.go b/pkg/asset/manifests/dns.go index 499cdfb1c01..1fa6a68e6f0 100644 --- a/pkg/asset/manifests/dns.go +++ b/pkg/asset/manifests/dns.go @@ -91,10 +91,14 @@ func (d *DNS) Generate(dependencies asset.Parents) error { } config.Spec.PublicZone = &configv1.DNSZone{ID: strings.TrimPrefix(*zone.Id, "/hostedzone/")} } - config.Spec.PrivateZone = &configv1.DNSZone{Tags: map[string]string{ - fmt.Sprintf("kubernetes.io/cluster/%s", clusterID.InfraID): "owned", - "Name": fmt.Sprintf("%s-int", clusterID.InfraID), - }} + if hostedZone := installConfig.Config.AWS.HostedZone; hostedZone == "" { + config.Spec.PrivateZone = &configv1.DNSZone{Tags: map[string]string{ + fmt.Sprintf("kubernetes.io/cluster/%s", clusterID.InfraID): "owned", + "Name": fmt.Sprintf("%s-int", clusterID.InfraID), + }} + } else { + config.Spec.PrivateZone = &configv1.DNSZone{ID: hostedZone} + } case azuretypes.Name: dnsConfig, err := installConfig.Azure.DNSConfig() if err != nil { diff --git a/pkg/destroy/aws/aws.go b/pkg/destroy/aws/aws.go index bd8605866c6..208918fa40d 100644 --- a/pkg/destroy/aws/aws.go +++ b/pkg/destroy/aws/aws.go @@ -59,10 +59,11 @@ type ClusterUninstaller struct { // } // // will match resources with (a:b and c:d) or d:e. - Filters []Filter // filter(s) we will be searching for - Logger logrus.FieldLogger - Region string - ClusterID string + Filters []Filter // filter(s) we will be searching for + Logger logrus.FieldLogger + Region string + ClusterID string + ClusterDomain string // Session is the AWS session to be used for deletion. If nil, a // new session will be created based on the usual credential @@ -86,11 +87,12 @@ func New(logger logrus.FieldLogger, metadata *types.ClusterMetadata) (providers. } return &ClusterUninstaller{ - Filters: filters, - Region: region, - Logger: logger, - ClusterID: metadata.InfraID, - Session: session, + Filters: filters, + Region: region, + Logger: logger, + ClusterID: metadata.InfraID, + ClusterDomain: metadata.AWS.ClusterDomain, + Session: session, }, nil } @@ -248,7 +250,7 @@ func (o *ClusterUninstaller) RunWithContext(ctx context.Context) ([]string, erro return resourcesToDelete.UnsortedList(), err } - err = removeSharedTags(ctx, tagClients, o.Filters, o.Logger) + err = o.removeSharedTags(ctx, awsSession, tagClients, tracker) if err != nil { return nil, err } @@ -626,9 +628,9 @@ func (search *iamUserSearch) arns(ctx context.Context) ([]string, error) { return arns, err } -// getSharedHostedZone will find the ID of the non-Terraform-managed public route53 zone given the +// getPublicHostedZone will find the ID of the non-Terraform-managed public route53 zone given the // Terraform-managed zone's privateID. -func getSharedHostedZone(ctx context.Context, client *route53.Route53, privateID string, logger logrus.FieldLogger) (string, error) { +func getPublicHostedZone(ctx context.Context, client *route53.Route53, privateID string, logger logrus.FieldLogger) (string, error) { response, err := client.GetHostedZoneWithContext(ctx, &route53.GetHostedZoneInput{ Id: aws.String(privateID), }) @@ -640,33 +642,32 @@ func getSharedHostedZone(ctx context.Context, client *route53.Route53, privateID if response.HostedZone.Config != nil && response.HostedZone.Config.PrivateZone != nil { if !*response.HostedZone.Config.PrivateZone { - return "", errors.Errorf("getSharedHostedZone requires a private ID, but was passed the public %s", privateID) + return "", errors.Errorf("getPublicHostedZone requires a private ID, but was passed the public %s", privateID) } } else { logger.WithField("hosted zone", privateName).Warn("could not determine whether hosted zone is private") } - domain := privateName - parents := []string{domain} - for { - idx := strings.Index(domain, ".") - if idx == -1 { - break - } - if len(domain[idx+1:]) > 0 { - parents = append(parents, domain[idx+1:]) - } - domain = domain[idx+1:] - } + return findAncestorPublicRoute53(ctx, client, privateName, logger) +} - for _, p := range parents { - sZone, err := findPublicRoute53(ctx, client, p, logger) +// findAncestorPublicRoute53 finds a public route53 zone with the closest ancestor or match to dnsName. +// It returns "", when no public route53 zone could be found. +func findAncestorPublicRoute53(ctx context.Context, client *route53.Route53, dnsName string, logger logrus.FieldLogger) (string, error) { + for len(dnsName) > 0 { + sZone, err := findPublicRoute53(ctx, client, dnsName, logger) if err != nil { return "", err } if sZone != "" { return sZone, nil } + + idx := strings.Index(dnsName, ".") + if idx == -1 { + break + } + dnsName = dnsName[idx+1:] } return "", nil } @@ -1816,7 +1817,7 @@ func deleteRoute53(ctx context.Context, session *session.Session, arn arn.ARN, l client := route53.New(session) - sharedZoneID, err := getSharedHostedZone(ctx, client, id, logger) + publicZoneID, err := getPublicHostedZone(ctx, client, id, logger) if err != nil { // In some cases AWS may return the zone in the list of tagged resources despite the fact // it no longer exists. @@ -1830,15 +1831,15 @@ func deleteRoute53(ctx context.Context, session *session.Session, arn arn.ARN, l return fmt.Sprintf("%s %s", *recordSet.Type, *recordSet.Name) } - sharedEntries := map[string]*route53.ResourceRecordSet{} - if len(sharedZoneID) != 0 { + publicEntries := map[string]*route53.ResourceRecordSet{} + if len(publicZoneID) != 0 { err = client.ListResourceRecordSetsPagesWithContext( ctx, - &route53.ListResourceRecordSetsInput{HostedZoneId: aws.String(sharedZoneID)}, + &route53.ListResourceRecordSetsInput{HostedZoneId: aws.String(publicZoneID)}, func(results *route53.ListResourceRecordSetsOutput, lastPage bool) bool { for _, recordSet := range results.ResourceRecordSets { key := recordSetKey(recordSet) - sharedEntries[key] = recordSet + publicEntries[key] = recordSet } return !lastPage @@ -1862,14 +1863,17 @@ func deleteRoute53(ctx context.Context, session *session.Session, arn arn.ARN, l continue } key := recordSetKey(recordSet) - if sharedEntry, ok := sharedEntries[key]; ok { - err := deleteRoute53RecordSet(ctx, client, sharedZoneID, sharedEntry, logger.WithField("public zone", sharedZoneID)) + if publicEntry, ok := publicEntries[key]; ok { + err := deleteRoute53RecordSet(ctx, client, publicZoneID, publicEntry, logger.WithField("public zone", publicZoneID)) if err != nil { if lastError != nil { logger.Debug(lastError) } - lastError = errors.Wrapf(err, "deleting public zone %s", sharedZoneID) + lastError = errors.Wrapf(err, "deleting record set %#v from public zone %s", publicEntry, publicZoneID) } + // do not delete the record set in the private zone if the delete failed in the public zone; + // otherwise the record set in the public zone will get leaked + continue } err = deleteRoute53RecordSet(ctx, client, id, recordSet, logger) @@ -2010,86 +2014,3 @@ func isBucketNotFound(err interface{}) bool { } return false } - -func removeSharedTags(ctx context.Context, tagClients []*resourcegroupstaggingapi.ResourceGroupsTaggingAPI, filters []Filter, logger logrus.FieldLogger) error { - for _, filter := range filters { - for key, value := range filter { - if strings.HasPrefix(key, "kubernetes.io/cluster/") { - if value == "owned" { - if err := removeSharedTag(ctx, tagClients, key, logger); err != nil { - return err - } - } else { - logger.Warnf("Ignoring non-owned cluster key %s: %s for shared-tag removal", key, value) - } - } - } - } - - return nil -} - -func removeSharedTag(ctx context.Context, tagClients []*resourcegroupstaggingapi.ResourceGroupsTaggingAPI, key string, logger logrus.FieldLogger) error { - request := &resourcegroupstaggingapi.UntagResourcesInput{ - TagKeys: []*string{aws.String(key)}, - } - - removed := map[string]struct{}{} - tagClients = append([]*resourcegroupstaggingapi.ResourceGroupsTaggingAPI(nil), tagClients...) - for len(tagClients) > 0 { - nextTagClients := tagClients[:0] - for _, tagClient := range tagClients { - logger.Debugf("Search for and remove tags in %s matching %s: shared", *tagClient.Config.Region, key) - arns := []string{} - err := tagClient.GetResourcesPagesWithContext( - ctx, - &resourcegroupstaggingapi.GetResourcesInput{TagFilters: []*resourcegroupstaggingapi.TagFilter{{ - Key: aws.String(key), - Values: []*string{aws.String("shared")}, - }}}, - func(results *resourcegroupstaggingapi.GetResourcesOutput, lastPage bool) bool { - for _, resource := range results.ResourceTagMappingList { - arn := *resource.ResourceARN - if _, ok := removed[arn]; !ok { - arns = append(arns, arn) - } - } - - return !lastPage - }, - ) - if err != nil { - err = errors.Wrap(err, "get tagged resources") - logger.Info(err) - nextTagClients = append(nextTagClients, tagClient) - continue - } - if len(arns) == 0 { - logger.Debugf("No matches in %s for %s: shared, removing client", *tagClient.Config.Region, key) - continue - } - nextTagClients = append(nextTagClients, tagClient) - - for i := 0; i < len(arns); i += 20 { - request.ResourceARNList = make([]*string, 0, 20) - for j := 0; i+j < len(arns) && j < 20; j++ { - request.ResourceARNList = append(request.ResourceARNList, aws.String(arns[i+j])) - } - _, err = tagClient.UntagResourcesWithContext(ctx, request) - if err != nil { - err = errors.Wrap(err, "untag shared resources") - logger.Info(err) - continue - } - for j := 0; i+j < len(arns) && j < 20; j++ { - arn := arns[i+j] - logger.WithField("arn", arn).Infof("Removed tag %s: shared", key) - removed[arn] = exists - } - } - } - tagClients = nextTagClients - } - - return nil -} diff --git a/pkg/destroy/aws/shared.go b/pkg/destroy/aws/shared.go new file mode 100644 index 00000000000..91f72d9c6d3 --- /dev/null +++ b/pkg/destroy/aws/shared.go @@ -0,0 +1,240 @@ +package aws + +import ( + "context" + "fmt" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/arn" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi" + "github.com/aws/aws-sdk-go/service/route53" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +func (o *ClusterUninstaller) removeSharedTags( + ctx context.Context, + session *session.Session, + tagClients []*resourcegroupstaggingapi.ResourceGroupsTaggingAPI, + tracker *errorTracker, +) error { + for _, key := range o.clusterOwnedKeys() { + if err := o.removeSharedTag(ctx, session, tagClients, key, tracker); err != nil { + return err + } + } + return nil +} + +func (o *ClusterUninstaller) clusterOwnedKeys() []string { + var keys []string + for _, filter := range o.Filters { + for key, value := range filter { + if !strings.HasPrefix(key, "kubernetes.io/cluster/") { + continue + } + if value != "owned" { + o.Logger.Warnf("Ignoring non-owned cluster key %s: %s for shared-tag removal", key, value) + } + keys = append(keys, key) + } + } + return keys +} + +func (o *ClusterUninstaller) removeSharedTag(ctx context.Context, session *session.Session, tagClients []*resourcegroupstaggingapi.ResourceGroupsTaggingAPI, key string, tracker *errorTracker) error { + request := &resourcegroupstaggingapi.UntagResourcesInput{ + TagKeys: []*string{aws.String(key)}, + } + + removed := map[string]struct{}{} + tagClients = append([]*resourcegroupstaggingapi.ResourceGroupsTaggingAPI(nil), tagClients...) + for len(tagClients) > 0 { + nextTagClients := tagClients[:0] + for _, tagClient := range tagClients { + o.Logger.Debugf("Search for and remove tags in %s matching %s: shared", *tagClient.Config.Region, key) + var arns []string + err := tagClient.GetResourcesPagesWithContext( + ctx, + &resourcegroupstaggingapi.GetResourcesInput{TagFilters: []*resourcegroupstaggingapi.TagFilter{{ + Key: aws.String(key), + Values: []*string{aws.String("shared")}, + }}}, + func(results *resourcegroupstaggingapi.GetResourcesOutput, lastPage bool) bool { + for _, resource := range results.ResourceTagMappingList { + arnString := aws.StringValue(resource.ResourceARN) + logger := o.Logger.WithField("arn", arnString) + parsedARN, err := arn.Parse(arnString) + if err != nil { + logger.WithError(err).Debug("could not parse ARN") + continue + } + if _, ok := removed[arnString]; !ok { + if err := o.cleanSharedARN(ctx, session, parsedARN, logger); err != nil { + tracker.suppressWarning(arnString, err, logger) + if err := ctx.Err(); err != nil { + return false + } + continue + } + arns = append(arns, arnString) + } + } + + return !lastPage + }, + ) + if err != nil { + err = errors.Wrap(err, "get tagged resources") + o.Logger.Info(err) + nextTagClients = append(nextTagClients, tagClient) + continue + } + if len(arns) == 0 { + o.Logger.Debugf("No matches in %s for %s: shared, removing client", *tagClient.Config.Region, key) + continue + } + nextTagClients = append(nextTagClients, tagClient) + + for i := 0; i < len(arns); i += 20 { + request.ResourceARNList = make([]*string, 0, 20) + for j := 0; i+j < len(arns) && j < 20; j++ { + request.ResourceARNList = append(request.ResourceARNList, aws.String(arns[i+j])) + } + _, err = tagClient.UntagResourcesWithContext(ctx, request) + if err != nil { + err = errors.Wrap(err, "untag shared resources") + o.Logger.Info(err) + continue + } + for j := 0; i+j < len(arns) && j < 20; j++ { + arn := arns[i+j] + o.Logger.WithField("arn", arn).Infof("Removed tag %s: shared", key) + removed[arn] = exists + } + } + } + tagClients = nextTagClients + } + + return nil +} + +func (o *ClusterUninstaller) cleanSharedARN(ctx context.Context, session *session.Session, arn arn.ARN, logger logrus.FieldLogger) error { + switch service := arn.Service; service { + case "route53": + return o.cleanSharedRoute53(ctx, session, arn, logger) + default: + logger.Debugf("Nothing to clean for shared %s resource", service) + return nil + } +} + +func (o *ClusterUninstaller) cleanSharedRoute53(ctx context.Context, session *session.Session, arn arn.ARN, logger logrus.FieldLogger) error { + client := route53.New(session) + + resourceType, id, err := splitSlash("resource", arn.Resource) + if err != nil { + return err + } + logger = logger.WithField("id", id) + + switch resourceType { + case "hostedzone": + return o.cleanSharedHostedZone(ctx, client, id, logger) + default: + logger.Debugf("Nothing to clean for shared %s resource", resourceType) + return nil + } +} + +func (o *ClusterUninstaller) cleanSharedHostedZone(ctx context.Context, client *route53.Route53, id string, logger logrus.FieldLogger) error { + if o.ClusterDomain == "" { + logger.Debug("No cluster domain specified in metadata; cannot clean the shared hosted zone") + return nil + } + dottedClusterDomain := o.ClusterDomain + "." + + publicZoneID, err := findAncestorPublicRoute53(ctx, client, o.ClusterDomain, logger) + if err != nil { + return err + } + + var lastError error + err = client.ListResourceRecordSetsPagesWithContext( + ctx, + &route53.ListResourceRecordSetsInput{HostedZoneId: aws.String(id)}, + func(results *route53.ListResourceRecordSetsOutput, lastPage bool) bool { + for _, recordSet := range results.ResourceRecordSets { + // skip record sets that are not part of the cluster + name := aws.StringValue(recordSet.Name) + if !strings.HasSuffix(name, dottedClusterDomain) { + continue + } + if len(name) == len(dottedClusterDomain) { + continue + } + recordSetLogger := logger.WithField( + "recordset", + fmt.Sprintf("%s (%s)", aws.StringValue(recordSet.Name), aws.StringValue(recordSet.Type)), + ) + // delete any matching record sets in the public hosted zone + if publicZoneID != "" { + if err := deleteMatchingRecordSetInPublicZone(ctx, client, publicZoneID, recordSet, logger); err != nil { + if lastError != nil { + logger.Debug(lastError) + } + lastError = errors.Wrapf(err, "deleting record set matching %#v from public zone %s", recordSet, publicZoneID) + // do not delete the record set in the private zone if the delete failed in the public zone; + // otherwise the record set in the public zone will get leaked + continue + } + recordSetLogger.Debug("Deleted from public zone") + } + // delete the record set + if err := deleteRoute53RecordSet(ctx, client, id, recordSet, logger); err != nil { + if lastError != nil { + logger.Debug(lastError) + } + lastError = errors.Wrapf(err, "deleting record set %#v from zone %s", recordSet, id) + } + recordSetLogger.Debug("Deleted") + } + return !lastPage + }, + ) + + if lastError != nil { + return lastError + } + if err != nil { + return err + } + + logger.Info("Cleaned record sets from hosted zone") + return nil +} + +func deleteMatchingRecordSetInPublicZone(ctx context.Context, client *route53.Route53, zoneID string, recordSet *route53.ResourceRecordSet, logger logrus.FieldLogger) error { + in := &route53.ListResourceRecordSetsInput{ + HostedZoneId: aws.String(zoneID), + MaxItems: aws.String("1"), + StartRecordName: recordSet.Name, + StartRecordType: recordSet.Type, + } + out, err := client.ListResourceRecordSetsWithContext(ctx, in) + if err != nil { + return err + } + if len(out.ResourceRecordSets) == 0 { + return nil + } + matchingRecordSet := out.ResourceRecordSets[0] + if aws.StringValue(matchingRecordSet.Name) != aws.StringValue(recordSet.Name) || + aws.StringValue(matchingRecordSet.Type) != aws.StringValue(recordSet.Type) { + return nil + } + return deleteRoute53RecordSet(ctx, client, zoneID, matchingRecordSet, logger) +} diff --git a/pkg/explain/printer_test.go b/pkg/explain/printer_test.go index 96dc509b637..e57e330108c 100644 --- a/pkg/explain/printer_test.go +++ b/pkg/explain/printer_test.go @@ -122,6 +122,9 @@ func Test_PrintFields(t *testing.T) { defaultMachinePlatform DefaultMachinePlatform is the default configuration used when installing on AWS for machine pools which do not define their own platform configuration. + hostedZone + HostedZone is the ID of an existing hosted zone into which to add DNS records for the cluster's internal API. An existing hosted zone can only be used when also using existing subnets. The hosted zone must be associated with the VPC containing the subnets. Leave the hosted zone unset to have the installer create the hosted zone on your behalf. + region -required- Region specifies the AWS region where the cluster will be created. diff --git a/pkg/tfvars/aws/aws.go b/pkg/tfvars/aws/aws.go index 01b7194120d..49d8fc91d8b 100644 --- a/pkg/tfvars/aws/aws.go +++ b/pkg/tfvars/aws/aws.go @@ -32,6 +32,7 @@ type config struct { VPC string `json:"aws_vpc,omitempty"` PrivateSubnets []string `json:"aws_private_subnets,omitempty"` PublicSubnets *[]string `json:"aws_public_subnets,omitempty"` + InternalZone string `json:"aws_internal_zone,omitempty"` PublishStrategy string `json:"aws_publish_strategy,omitempty"` SkipRegionCheck bool `json:"aws_skip_region_validation"` IgnitionBucket string `json:"aws_ignition_bucket"` @@ -44,6 +45,7 @@ type config struct { type TFVarsSources struct { VPC string PrivateSubnets, PublicSubnets []string + InternalZone string Services []typesaws.ServiceEndpoint Publish types.PublishingStrategy @@ -124,6 +126,7 @@ func TFVars(sources TFVarsSources) ([]byte, error) { Type: *rootVolume.EBS.VolumeType, VPC: sources.VPC, PrivateSubnets: sources.PrivateSubnets, + InternalZone: sources.InternalZone, PublishStrategy: string(sources.Publish), SkipRegionCheck: !configaws.IsKnownRegion(masterConfig.Placement.Region), IgnitionBucket: sources.IgnitionBucket, diff --git a/pkg/types/aws/metadata.go b/pkg/types/aws/metadata.go index 300077f037f..609c48baaee 100644 --- a/pkg/types/aws/metadata.go +++ b/pkg/types/aws/metadata.go @@ -15,4 +15,7 @@ type Metadata struct { // resource matches the map if all of the key/value pairs are in its // tags. A resource matches Identifier if it matches any of the maps. Identifier []map[string]string `json:"identifier"` + + // ClusterDomain is the domain for the cluster. + ClusterDomain string `json:"clusterDomain"` } diff --git a/pkg/types/aws/platform.go b/pkg/types/aws/platform.go index b29aaaddbb9..f5840f9a159 100644 --- a/pkg/types/aws/platform.go +++ b/pkg/types/aws/platform.go @@ -26,6 +26,15 @@ type Platform struct { // +optional Subnets []string `json:"subnets,omitempty"` + // HostedZone is the ID of an existing hosted zone into which to add DNS + // records for the cluster's internal API. An existing hosted zone can + // only be used when also using existing subnets. The hosted zone must be + // associated with the VPC containing the subnets. + // Leave the hosted zone unset to have the installer create the hosted zone + // on your behalf. + // +optional + HostedZone string `json:"hostedZone,omitempty"` + // UserTags additional keys and values that the installer will add // as tags to all resources that it creates. Resources created by the // cluster itself may not include these tags. diff --git a/pkg/types/aws/validation/platform.go b/pkg/types/aws/validation/platform.go index b962219eb92..f7b96efde98 100644 --- a/pkg/types/aws/validation/platform.go +++ b/pkg/types/aws/validation/platform.go @@ -20,6 +20,12 @@ func ValidatePlatform(p *aws.Platform, fldPath *field.Path) field.ErrorList { allErrs = append(allErrs, field.Required(fldPath.Child("region"), "region must be specified")) } + if p.HostedZone != "" { + if len(p.Subnets) == 0 { + allErrs = append(allErrs, field.Invalid(fldPath.Child("hostedZone"), p.HostedZone, "may not use an existing hosted zone when not using existing subnets")) + } + } + allErrs = append(allErrs, validateServiceEndpoints(p.ServiceEndpoints, fldPath.Child("serviceEndpoints"))...) allErrs = append(allErrs, validateUserTags(p.UserTags, fldPath.Child("userTags"))...) diff --git a/pkg/types/aws/validation/platform_test.go b/pkg/types/aws/validation/platform_test.go index 66b3d5327e8..76785034ba4 100644 --- a/pkg/types/aws/validation/platform_test.go +++ b/pkg/types/aws/validation/platform_test.go @@ -28,6 +28,22 @@ func TestValidatePlatform(t *testing.T) { }, expected: `^test-path\.region: Required value: region must be specified$`, }, + { + name: "hosted zone with subnets", + platform: &aws.Platform{ + Region: "us-east-1", + Subnets: []string{"test-subnet"}, + HostedZone: "test-hosted-zone", + }, + }, + { + name: "hosted zone without subnets", + platform: &aws.Platform{ + Region: "us-east-1", + HostedZone: "test-hosted-zone", + }, + expected: `^test-path\.hostedZone: Invalid value: "test-hosted-zone": may not use an existing hosted zone when not using existing subnets$`, + }, { name: "invalid url for service endpoint", platform: &aws.Platform{