RFE-2782: Create edge compute pool to support AWS Local Zones

pkg/asset/installconfig/aws/availabilityzones.go

@@ -7,10 +7,12 @@ import (
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/pkg/errors"
+
+	typesaws "github.com/openshift/installer/pkg/types/aws"
 )
 
-// availabilityZones retrieves a list of availability zones for the given region.
-func availabilityZones(ctx context.Context, session *session.Session, region string) ([]string, error) {
+// describeAvailabilityZones retrieves a list of all zones for the given region.
+func describeAvailabilityZones(ctx context.Context, session *session.Session, region string) ([]*ec2.AvailabilityZone, error) {
 	client := ec2.New(session, aws.NewConfig().WithRegion(region))
 	resp, err := client.DescribeAvailabilityZonesWithContext(ctx, &ec2.DescribeAvailabilityZonesInput{
 		Filters: []*ec2.Filter{
@@ -25,12 +27,21 @@ func availabilityZones(ctx context.Context, session *session.Session, region str
 		},
 	})
 	if err != nil {
-		return nil, errors.Wrap(err, "fetching availability zones")
+		return nil, errors.Wrap(err, "fetching zones")
 	}
 
+	return resp.AvailabilityZones, nil
+}
+
+// availabilityZones retrieves a list of zones type 'availability-zone' for the region.
+func availabilityZones(ctx context.Context, session *session.Session, region string) ([]string, error) {
+	azs, err := describeAvailabilityZones(ctx, session, region)
+	if err != nil {
+		return nil, errors.Wrap(err, "fetching availability zones")
+	}
 	zones := []string{}
-	for _, zone := range resp.AvailabilityZones {
-		if *zone.ZoneType == "availability-zone" {
+	for _, zone := range azs {
+		if *zone.ZoneType == typesaws.AvailabilityZoneType {
 			zones = append(zones, *zone.ZoneName)
 		}
 	}

pkg/asset/installconfig/aws/metadata.go

@@ -18,14 +18,16 @@ type Metadata struct {
 	availabilityZones []string
 	privateSubnets    map[string]Subnet
 	publicSubnets     map[string]Subnet
+	edgeSubnets       map[string]Subnet
 	vpc               string
 	instanceTypes     map[string]InstanceType
 
 	Region   string                     `json:"region,omitempty"`
 	Subnets  []string                   `json:"subnets,omitempty"`
 	Services []typesaws.ServiceEndpoint `json:"services,omitempty"`
 
-	mutex sync.Mutex
+	mutex        sync.Mutex
+	mutexSubnets sync.Mutex
 }
 
 // NewMetadata initializes a new Metadata object.
@@ -74,13 +76,22 @@ func (m *Metadata) AvailabilityZones(ctx context.Context) ([]string, error) {
 	return m.availabilityZones, nil
 }
 
+// EdgeSubnets retrieves subnet metadata indexed by subnet ID, for
+// subnets that the cloud-provider logic considers to be edge
+// (i.e. Local Zone).
+func (m *Metadata) EdgeSubnets(ctx context.Context) (map[string]Subnet, error) {
+	err := m.populateSubnets(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	return m.edgeSubnets, nil
+}
+
 // PrivateSubnets retrieves subnet metadata indexed by subnet ID, for
 // subnets that the cloud-provider logic considers to be private
 // (i.e. not public).
 func (m *Metadata) PrivateSubnets(ctx context.Context) (map[string]Subnet, error) {
-	m.mutex.Lock()
-	defer m.mutex.Unlock()
-
 	err := m.populateSubnets(ctx)
 	if err != nil {
 		return nil, err
@@ -93,9 +104,6 @@ func (m *Metadata) PrivateSubnets(ctx context.Context) (map[string]Subnet, error
 // subnets that the cloud-provider logic considers to be public
 // (e.g. with suitable routing for hosting public load balancers).
 func (m *Metadata) PublicSubnets(ctx context.Context) (map[string]Subnet, error) {
-	m.mutex.Lock()
-	defer m.mutex.Unlock()
-
 	err := m.populateSubnets(ctx)
 	if err != nil {
 		return nil, err
@@ -113,12 +121,19 @@ func (m *Metadata) populateSubnets(ctx context.Context) error {
 		return errors.New("no subnets configured")
 	}
 
-	session, err := m.unlockedSession(ctx)
+	m.mutexSubnets.Lock()
+	defer m.mutexSubnets.Unlock()
+
+	session, err := m.Session(ctx)
 	if err != nil {
 		return err
 	}
 
-	m.vpc, m.privateSubnets, m.publicSubnets, err = subnets(ctx, session, m.Region, m.Subnets)
+	sb, err := subnets(ctx, session, m.Region, m.Subnets)
+	m.vpc = sb.VPC
+	m.privateSubnets = sb.Private
+	m.publicSubnets = sb.Public
+	m.edgeSubnets = sb.Edge
 	return err
 }
 

pkg/asset/installconfig/aws/subnet.go

@@ -11,10 +11,15 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
+
+	typesaws "github.com/openshift/installer/pkg/types/aws"
 )
 
 // Subnet holds metadata for a subnet.
 type Subnet struct {
+	// ID is the subnet's Identifier.
+	ID string
+
 	// ARN is the subnet's Amazon Resource Name.
 	ARN string
 
@@ -23,13 +28,47 @@ type Subnet struct {
 
 	// CIDR is the subnet's CIDR block.
 	CIDR string
+
+	// ZoneType is the type of subnet's availability zone.
+	// The valid values are availability-zone and local-zone.
+	ZoneType string
+
+	// ZoneGroupName is the AWS zone group name.
+	// For Availability Zones, this parameter has the same value as the Region name.
+	//
+	// For Local Zones, the name of the associated group, for example us-west-2-lax-1.
+	ZoneGroupName string
+
+	// Public is the flag to define the subnet public.
+	Public bool
+
+	// PreferredEdgeInstanceType is the preferred instance type on the subnet's zone.
+	// It's used for the edge pools which does not offer the same type across zone groups.
+	PreferredEdgeInstanceType string
+}
+
+// Subnets is the map for the Subnet metadata.
+type Subnets map[string]Subnet
+
+// SubnetGroups is the group of subnets used by installer.
+type SubnetGroups struct {
+	Public  Subnets
+	Private Subnets
+	Edge    Subnets
+	VPC     string
 }
 
 // subnets retrieves metadata for the given subnet(s).
-func subnets(ctx context.Context, session *session.Session, region string, ids []string) (vpc string, private map[string]Subnet, public map[string]Subnet, err error) {
+func subnets(ctx context.Context, session *session.Session, region string, ids []string) (subnetGroups SubnetGroups, err error) {
 	metas := make(map[string]Subnet, len(ids))
-	private = map[string]Subnet{}
-	public = map[string]Subnet{}
+	zoneNames := make([]*string, len(ids))
+	availabilityZones := make(map[string]*ec2.AvailabilityZone, len(ids))
+	subnetGroups = SubnetGroups{
+		Public:  make(map[string]Subnet, len(ids)),
+		Private: make(map[string]Subnet, len(ids)),
+		Edge:    make(map[string]Subnet, len(ids)),
+	}
+
 	var vpcFromSubnet string
 	client := ec2.New(session, aws.NewConfig().WithRegion(region))
 
@@ -60,19 +99,22 @@ func subnets(ctx context.Context, session *session.Session, region string, ids [
 					return false
 				}
 
-				if vpc == "" {
-					vpc = *subnet.VpcId
+				if subnetGroups.VPC == "" {
+					subnetGroups.VPC = *subnet.VpcId
 					vpcFromSubnet = *subnet.SubnetId
-				} else if *subnet.VpcId != vpc {
-					lastError = errors.Errorf("all subnets must belong to the same VPC: %s is from %s, but %s is from %s", *subnet.SubnetId, *subnet.VpcId, vpcFromSubnet, vpc)
+				} else if *subnet.VpcId != subnetGroups.VPC {
+					lastError = errors.Errorf("all subnets must belong to the same VPC: %s is from %s, but %s is from %s", *subnet.SubnetId, *subnet.VpcId, vpcFromSubnet, subnetGroups.VPC)
 					return false
 				}
 
 				metas[*subnet.SubnetId] = Subnet{
-					ARN:  *subnet.SubnetArn,
-					Zone: *subnet.AvailabilityZone,
-					CIDR: *subnet.CidrBlock,
+					ID:     *subnet.SubnetId,
+					ARN:    *subnet.SubnetArn,
+					Zone:   *subnet.AvailabilityZone,
+					CIDR:   *subnet.CidrBlock,
+					Public: false,
 				}
+				zoneNames = append(zoneNames, subnet.AvailabilityZone)
 			}
 			return !lastPage
 		},
@@ -81,7 +123,7 @@ func subnets(ctx context.Context, session *session.Session, region string, ids [
 		err = lastError
 	}
 	if err != nil {
-		return vpc, nil, nil, errors.Wrap(err, "describing subnets")
+		return subnetGroups, errors.Wrap(err, "describing subnets")
 	}
 
 	var routeTables []*ec2.RouteTable
@@ -90,7 +132,7 @@ func subnets(ctx context.Context, session *session.Session, region string, ids [
 		&ec2.DescribeRouteTablesInput{
 			Filters: []*ec2.Filter{{
 				Name:   aws.String("vpc-id"),
-				Values: []*string{aws.String(vpc)},
+				Values: []*string{aws.String(subnetGroups.VPC)},
 			}},
 		},
 		func(results *ec2.DescribeRouteTablesOutput, lastPage bool) bool {
@@ -99,38 +141,59 @@ func subnets(ctx context.Context, session *session.Session, region string, ids [
 		},
 	)
 	if err != nil {
-		return vpc, nil, nil, errors.Wrap(err, "describing route tables")
+		return subnetGroups, errors.Wrap(err, "describing route tables")
+	}
+
+	azs, err := client.DescribeAvailabilityZonesWithContext(ctx, &ec2.DescribeAvailabilityZonesInput{ZoneNames: zoneNames})
+	if err != nil {
+		return subnetGroups, errors.Wrap(err, "describing availability zones")
+	}
+	for _, az := range azs.AvailabilityZones {
+		availabilityZones[*az.ZoneName] = az
 	}
 
 	publicOnlySubnets := os.Getenv("OPENSHIFT_INSTALL_AWS_PUBLIC_ONLY") != ""
 
 	for _, id := range ids {
 		meta, ok := metas[id]
 		if !ok {
-			return vpc, nil, nil, errors.Errorf("failed to find %s", id)
+			return subnetGroups, errors.Errorf("failed to find %s", id)
 		}
 
 		isPublic, err := isSubnetPublic(routeTables, id)
 		if err != nil {
-			return vpc, nil, nil, err
+			return subnetGroups, err
 		}
-		if isPublic {
-			public[id] = meta
-		} else {
-			private[id] = meta
+		meta.Public = isPublic
+		meta.ZoneType = *availabilityZones[meta.Zone].ZoneType
+		meta.ZoneGroupName = *availabilityZones[meta.Zone].GroupName
+
+		// AWS Local Zones are grouped as Edge subnets
+		if meta.ZoneType == typesaws.LocalZoneType {
+			// Local Zones is supported only in Public subnets
+			if !meta.Public {
+				return subnetGroups, errors.Errorf("subnet tyoe local-zone must be associated with public route tables: subnet %s from availability zone %s[%s] is public[%v]", id, meta.Zone, meta.ZoneType, meta.Public)
+			}
+			subnetGroups.Edge[id] = meta
+			continue
 		}
-
-		// Let public subnets work as if they were private. This allows us to
-		// have clusters with public-only subnets without having to introduce a
-		// lot of changes in the installer. Such clusters can be used in a
-		// NAT-less GW scenario, therefore decreasing costs in cases where node
-		// security is not a concern (e.g, ephemeral clusters in CI)
-		if publicOnlySubnets && isPublic {
-			private[id] = meta
+		if meta.Public {
+			subnetGroups.Public[id] = meta
+
+			// Let public subnets work as if they were private. This allows us to
+			// have clusters with public-only subnets without having to introduce a
+			// lot of changes in the installer. Such clusters can be used in a
+			// NAT-less GW scenario, therefore decreasing costs in cases where node
+			// security is not a concern (e.g, ephemeral clusters in CI)
+			if publicOnlySubnets {
+				subnetGroups.Private[id] = meta
+			}
+			continue
 		}
+		// Subnet is grouped by default as private
+		subnetGroups.Private[id] = meta
 	}
-
-	return vpc, private, public, nil
+	return subnetGroups, nil
 }
 
 // https://github.com/kubernetes/kubernetes/blob/9f036cd43d35a9c41d7ac4ca82398a6d0bef957b/staging/src/k8s.io/legacy-cloud-providers/aws/aws.go#L3376-L3419

pkg/asset/installconfig/aws/validation.go

@@ -49,12 +49,30 @@ func Validate(ctx context.Context, meta *Metadata, config *types.InstallConfig)
 	allErrs = append(allErrs, validatePlatform(ctx, meta, field.NewPath("platform", "aws"), config.Platform.AWS, config.Networking, config.Publish)...)
 
 	if config.ControlPlane != nil && config.ControlPlane.Platform.AWS != nil {
-		allErrs = append(allErrs, validateMachinePool(ctx, meta, field.NewPath("controlPlane", "platform", "aws"), config.Platform.AWS, config.ControlPlane.Platform.AWS, controlPlaneReq)...)
+		allErrs = append(allErrs, validateMachinePool(ctx, meta, field.NewPath("controlPlane", "platform", "aws"), config.Platform.AWS, config.ControlPlane.Platform.AWS, controlPlaneReq, "")...)
 	}
+
 	for idx, compute := range config.Compute {
 		fldPath := field.NewPath("compute").Index(idx)
+
+		// Pool's specific validation.
+		// Edge Compute Pool: AWS Local Zones is valid only when installing in existing VPC.
+		if compute.Name == types.MachinePoolEdgeRoleName {
+			if len(config.Platform.AWS.Subnets) == 0 {
+				return errors.New(field.Required(fldPath, "invalid install config. edge machine pool is valid when installing in existing VPC").Error())
+			}
+			edgeSubnets, err := meta.EdgeSubnets(ctx)
+			if err != nil {
+				errMsg := fmt.Sprintf("%s pool. %v", compute.Name, err.Error())
+				return errors.New(field.Invalid(field.NewPath("platform", "aws", "subnets"), config.Platform.AWS.Subnets, errMsg).Error())
+			}
+			if len(edgeSubnets) == 0 {
+				return errors.New(field.Required(fldPath, "invalid install config. There is no valid subnets for edge machine pool").Error())
+			}
+		}
+
 		if compute.Platform.AWS != nil {
-			allErrs = append(allErrs, validateMachinePool(ctx, meta, fldPath.Child("platform", "aws"), config.Platform.AWS, compute.Platform.AWS, computeReq)...)
+			allErrs = append(allErrs, validateMachinePool(ctx, meta, fldPath.Child("platform", "aws"), config.Platform.AWS, compute.Platform.AWS, computeReq, compute.Name)...)
 		}
 	}
 	return allErrs.ToAggregate()
@@ -74,7 +92,7 @@ func validatePlatform(ctx context.Context, meta *Metadata, fldPath *field.Path,
 		allErrs = append(allErrs, validateSubnets(ctx, meta, fldPath.Child("subnets"), platform.Subnets, networking, publish)...)
 	}
 	if platform.DefaultMachinePlatform != nil {
-		allErrs = append(allErrs, validateMachinePool(ctx, meta, fldPath.Child("defaultMachinePlatform"), platform, platform.DefaultMachinePlatform, controlPlaneReq)...)
+		allErrs = append(allErrs, validateMachinePool(ctx, meta, fldPath.Child("defaultMachinePlatform"), platform, platform.DefaultMachinePlatform, controlPlaneReq, "")...)
 	}
 	return allErrs
 }
@@ -153,10 +171,22 @@ func validateSubnets(ctx context.Context, meta *Metadata, fldPath *field.Path, s
 		}
 	}
 
+	edgeSubnets, err := meta.EdgeSubnets(ctx)
+	if err != nil {
+		return append(allErrs, field.Invalid(fldPath, subnets, err.Error()))
+	}
+	edgeSubnetsIdx := map[string]int{}
+	for idx, id := range subnets {
+		if _, ok := edgeSubnets[id]; ok {
+			edgeSubnetsIdx[id] = idx
+		}
+	}
+
 	allErrs = append(allErrs, validateSubnetCIDR(fldPath, privateSubnets, privateSubnetsIdx, networking.MachineNetwork)...)
 	allErrs = append(allErrs, validateSubnetCIDR(fldPath, publicSubnets, publicSubnetsIdx, networking.MachineNetwork)...)
 	allErrs = append(allErrs, validateDuplicateSubnetZones(fldPath, privateSubnets, privateSubnetsIdx, "private")...)
 	allErrs = append(allErrs, validateDuplicateSubnetZones(fldPath, publicSubnets, publicSubnetsIdx, "public")...)
+	allErrs = append(allErrs, validateDuplicateSubnetZones(fldPath, edgeSubnets, edgeSubnetsIdx, "edge")...)
 
 	privateZones := sets.NewString()
 	publicZones := sets.NewString()
@@ -174,16 +204,23 @@ func validateSubnets(ctx context.Context, meta *Metadata, fldPath *field.Path, s
 	return allErrs
 }
 
-func validateMachinePool(ctx context.Context, meta *Metadata, fldPath *field.Path, platform *awstypes.Platform, pool *awstypes.MachinePool, req resourceRequirements) field.ErrorList {
+func validateMachinePool(ctx context.Context, meta *Metadata, fldPath *field.Path, platform *awstypes.Platform, pool *awstypes.MachinePool, req resourceRequirements, poolName string) field.ErrorList {
 	allErrs := field.ErrorList{}
 	if len(pool.Zones) > 0 {
 		availableZones := sets.String{}
 		if len(platform.Subnets) > 0 {
-			privateSubnets, err := meta.PrivateSubnets(ctx)
+			var err error
+			var subnets Subnets
+			switch poolName {
+			case types.MachinePoolEdgeRoleName:
+				subnets, err = meta.EdgeSubnets(ctx)
+			default:
+				subnets, err = meta.PrivateSubnets(ctx)
+			}
 			if err != nil {
 				return append(allErrs, field.InternalError(fldPath, err))
 			}
-			for _, subnet := range privateSubnets {
+			for _, subnet := range subnets {
 				availableZones.Insert(subnet.Zone)
 			}
 		} else {