diff --git a/docs/service_controller.md b/docs/service_controller.md index 0434d29ac3..600da7940b 100644 --- a/docs/service_controller.md +++ b/docs/service_controller.md @@ -17,6 +17,7 @@ The service controller is responsible for watch for service and node object chan | service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled | [true\|false] | - | With cross-zone load balancing, each load balancer node for your Classic Load Balancer distributes requests evenly across the registered instances in all enabled Availability Zones. If cross-zone load balancing is disabled, each load balancer node distributes requests evenly across the registered instances in its Availability Zone only. | | service.beta.kubernetes.io/aws-load-balancer-extra-security-groups | Comma-separated list | - | Specifies additional security groups to be added to ELB. | | service.beta.kubernetes.io/aws-load-balancer-security-groups | Comma-separated list | - | Specifies the security groups to be added to ELB. Differently from the annotation "service.beta.kubernetes.io/aws-load-balancer-extra-security-groups", this replaces all other security groups previously assigned to the ELB. | +| service.beta.kubernetes.io/aws-load-balancer-manage-security-group | Bool | - | Indicates that the controller creates and manages the lifecycle of a Security Group when creating a Network Load Balancer (NLB). This is evaluated only when the service Load Balancer type `nlb` is created. You can not specify security group when using this option. | | service.beta.kubernetes.io/aws-load-balancer-healthcheck-healthy-threshold | [2-10] | - | Specifies the number of successive successful health checks required for a backend to be considered healthy for traffic. For NLB, healthy-threshold and unhealthy-threshold must be equal. | | service.beta.kubernetes.io/aws-load-balancer-healthcheck-interval | [5-300] | 30 | Specifies, in seconds, the interval between health checks. | | service.beta.kubernetes.io/aws-load-balancer-healthcheck-timeout | [2-60] | 5 | The amount of time to wait when receiving a response from the health check, in seconds. | diff --git a/pkg/providers/v1/aws.go b/pkg/providers/v1/aws.go index 4416f4ab4f..1c0426b558 100644 --- a/pkg/providers/v1/aws.go +++ b/pkg/providers/v1/aws.go @@ -149,6 +149,12 @@ const ServiceAnnotationLoadBalancerExtraSecurityGroups = "service.beta.kubernete // "service.beta.kubernetes.io/aws-load-balancer-extra-security-groups", this replaces all other security groups previously assigned to the ELB. const ServiceAnnotationLoadBalancerSecurityGroups = "service.beta.kubernetes.io/aws-load-balancer-security-groups" +// ServiceAnnotationLoadBalancerManagedSecurityGroup is the annotation used +// on the service to specify the instruct CCM to manage the security group when creating a Network Load Balancer. When enabled, +// the CCM creates the security group and it's rules. This option can not be used with annotations +// "service.beta.kubernetes.io/aws-load-balancer-security-groups" and "service.beta.kubernetes.io/aws-load-balancer-extra-security-groups". +const ServiceAnnotationLoadBalancerManagedSecurityGroup = "service.beta.kubernetes.io/aws-load-balancer-managed-security-group" + // ServiceAnnotationLoadBalancerCertificate is the annotation used on the // service to request a secure listener. Value is a valid certificate ARN. // For more, see http://docs.aws.amazon.com/ElasticLoadBalancing/latest/DeveloperGuide/elb-listener-config.html @@ -1307,7 +1313,7 @@ func isEqualUserGroupPair(l, r *ec2.UserIdGroupPair, compareGroupUserIDs bool) b // Makes sure the security group ingress is exactly the specified permissions // Returns true if and only if changes were made // The security group must already exist -func (c *Cloud) setSecurityGroupIngress(securityGroupID string, permissions IPPermissionSet) (bool, error) { +func (c *Cloud) setSecurityGroupIngress(securityGroupID string, permissions IPPermissionSet, isEgress bool) (bool, error) { group, err := c.findSecurityGroup(securityGroupID) if err != nil { klog.Warningf("Error retrieving security group %q", err) @@ -1318,7 +1324,12 @@ func (c *Cloud) setSecurityGroupIngress(securityGroupID string, permissions IPPe return false, fmt.Errorf("security group not found: %s", securityGroupID) } - klog.V(2).Infof("Existing security group ingress: %s %v", securityGroupID, group.IpPermissions) + ruleType := "ingress" + if isEgress { + ruleType = "egress" + } + + klog.V(2).Infof("Existing security group %s: %s %v", ruleType, securityGroupID, group.IpPermissions) actual := NewIPPermissionSet(group.IpPermissions...) @@ -1348,24 +1359,43 @@ func (c *Cloud) setSecurityGroupIngress(securityGroupID string, permissions IPPe // how removing single permissions from compound rules works, and we // don't want to accidentally open more than intended while we're // applying changes. - if add.Len() != 0 { - klog.V(2).Infof("Adding security group ingress: %s %v", securityGroupID, add.List()) - request := &ec2.AuthorizeSecurityGroupIngressInput{} - request.GroupId = &securityGroupID - request.IpPermissions = add.List() - _, err = c.ec2.AuthorizeSecurityGroupIngress(request) + if add.Len() != 0 { + klog.V(2).Infof("Adding security group %s: %s %v", ruleType, securityGroupID, add.List()) + + if isEgress { + request := &ec2.AuthorizeSecurityGroupEgressInput{} + request.GroupId = &securityGroupID + request.IpPermissions = add.List() + // TODO check how to handle egress rules while AWS creats ALLOW ALL egress when SG is created. + // _, err = c.ec2.AuthorizeSecurityGroupEgress(request) + klog.V(2).Infof("Skipping adding security group egress: %s %v", securityGroupID, remove.List()) + } else { + request := &ec2.AuthorizeSecurityGroupIngressInput{} + request.GroupId = &securityGroupID + request.IpPermissions = add.List() + _, err = c.ec2.AuthorizeSecurityGroupIngress(request) + } if err != nil { - return false, fmt.Errorf("error authorizing security group ingress: %q", err) + return false, fmt.Errorf("error authorizing security group %s: %q", ruleType, err) } } if remove.Len() != 0 { klog.V(2).Infof("Remove security group ingress: %s %v", securityGroupID, remove.List()) - request := &ec2.RevokeSecurityGroupIngressInput{} - request.GroupId = &securityGroupID - request.IpPermissions = remove.List() - _, err = c.ec2.RevokeSecurityGroupIngress(request) + if isEgress { + request := &ec2.RevokeSecurityGroupEgressInput{} + request.GroupId = &securityGroupID + request.IpPermissions = remove.List() + // TODO check how to handle egress rules while AWS creats ALLOW ALL egress when SG is created. + // _, err = c.ec2.RevokeSecurityGroupEgress(request) + klog.V(2).Infof("Skipping remove security group egress: %s %v", securityGroupID, remove.List()) + } else { + request := &ec2.RevokeSecurityGroupIngressInput{} + request.GroupId = &securityGroupID + request.IpPermissions = remove.List() + _, err = c.ec2.RevokeSecurityGroupIngress(request) + } if err != nil { return false, fmt.Errorf("error revoking security group ingress: %q", err) } @@ -1393,7 +1423,7 @@ func (c *Cloud) addSecurityGroupIngress(securityGroupID string, addPermissions [ return false, fmt.Errorf("security group not found: %s", securityGroupID) } - klog.V(2).Infof("Existing security group ingress: %s %v", securityGroupID, group.IpPermissions) + klog.V(2).Infof("Existing ingress rules to security group %q: %v", securityGroupID, group.IpPermissions) changes := []*ec2.IpPermission{} for _, addPermission := range addPermissions { @@ -1421,7 +1451,7 @@ func (c *Cloud) addSecurityGroupIngress(securityGroupID string, addPermissions [ return false, nil } - klog.V(2).Infof("Adding security group ingress: %s %v", securityGroupID, changes) + klog.V(2).Infof("Adding ingress rules to security group %q: %v", securityGroupID, changes) request := &ec2.AuthorizeSecurityGroupIngressInput{} request.GroupId = &securityGroupID @@ -1540,6 +1570,8 @@ func (c *Cloud) ensureSecurityGroup(name string, description string, additionalT createRequest.GroupName = &name createRequest.Description = &description tags := c.tagging.buildTags(ResourceLifecycleOwned, additionalTags) + tags["Name"] = name + // tags["sigs.k8s.io/cloud-controller-manager"] = ResourceLifecycleOwned var awsTags []*ec2.Tag for k, v := range tags { tag := &ec2.Tag{ @@ -1913,7 +1945,7 @@ func getSGListFromAnnotation(annotatedSG string) []string { // Extra groups can be specified via annotation, as can extra tags for any // new groups. The annotation "ServiceAnnotationLoadBalancerSecurityGroups" allows for // setting the security groups specified. -func (c *Cloud) buildELBSecurityGroupList(serviceName types.NamespacedName, loadBalancerName string, annotations map[string]string) ([]string, bool, error) { +func (c *Cloud) buildELBSecurityGroupList(serviceName types.NamespacedName, loadBalancerName string, annotations map[string]string, skipDefault bool) ([]string, bool, error) { var err error var securityGroupID string // We do not want to make changes to a Global defined SG @@ -1925,9 +1957,12 @@ func (c *Cloud) buildELBSecurityGroupList(serviceName types.NamespacedName, load if len(sgList) == 0 { if c.cfg.Global.ElbSecurityGroup != "" { sgList = append(sgList, c.cfg.Global.ElbSecurityGroup) + } else if skipDefault { + klog.V(4).Infof("Skip creation of default load balancer security group") } else { // Create a security group for the load balancer sgName := "k8s-elb-" + loadBalancerName + klog.V(4).Infof("Creating load balancer security group: %s", sgName) sgDescription := fmt.Sprintf("Security group for Kubernetes ELB %s (%v)", loadBalancerName, serviceName) securityGroupID, err = c.ensureSecurityGroup(sgName, sgDescription, getKeyValuePropertiesFromAnnotation(annotations, ServiceAnnotationLoadBalancerAdditionalTags)) if err != nil { @@ -2144,6 +2179,82 @@ func (c *Cloud) buildNLBHealthCheckConfiguration(svc *v1.Service) (healthCheckCo return hc, nil } +// createSecurityGroupRulesForNLB creates and configures security group rules for a Network Load Balancer (NLB). +// +// This function sets up ingress and egress rules for the specified security group (sgID) based on the provided +// port mappings and source IP ranges. It removes any default egress rules before applying the new rules. +// +// Parameters: +// - sgID: The ID of the security group to configure. +// - ports: A slice of nlbPortMapping objects that define the port mappings for the NLB. +// - ec2SourceRanges: A slice of *ec2.IpRange objects specifying the source IP ranges for the rules. +// +// Returns: +// - error: An error if any issue occurs while creating or applying the security group rules. +// +// The function performs the following steps: +// 1. Removes all default egress rules for the specified security group. +// 2. Iterates over the provided port mappings to create ingress and egress rules. +// 3. Configures health check-specific egress rules if the health check configuration differs from the traffic configuration. +// 4. Applies the generated ingress and egress rules to the security group. +// +// Errors are returned if there are issues with removing default rules, parsing health check ports, or applying the new rules. +func (c *Cloud) createSecurityGroupRulesForNLB(sgID string, ports []nlbPortMapping, ec2SourceRanges []*ec2.IpRange) error { + ingressRules := IPPermissionSet{} + egressRules := IPPermissionSet{} + + // Remove all default egress rules. + if _, err := c.setSecurityGroupIngress(sgID, IPPermissionSet{"default": &ec2.IpPermission{ + IpProtocol: aws.String("-1"), + IpRanges: []*ec2.IpRange{{CidrIp: aws.String("0.0.0.0/0")}}, + }}, true); err != nil { + return fmt.Errorf("removing default egress rules for security group %q: %w", sgID, err) + } + + for _, mapping := range ports { + // create ingress permissions: iteract over Load Balancer Listener mappings to create ingress rules + ingressRules.Insert(&ec2.IpPermission{ + FromPort: aws.Int64(int64(mapping.FrontendPort)), + ToPort: aws.Int64(int64(mapping.FrontendPort)), + IpProtocol: aws.String(strings.ToLower(string((mapping.FrontendProtocol)))), + IpRanges: ec2SourceRanges, + }) + + // create egress permissions: iteract over port mapping to build the ingress and egress rules + egressRule := &ec2.IpPermission{ + FromPort: aws.Int64(int64(mapping.TrafficPort)), + ToPort: aws.Int64(int64(mapping.TrafficPort)), + IpProtocol: aws.String(strings.ToLower(string((mapping.TrafficProtocol)))), + IpRanges: ec2SourceRanges, + } + // service ports are different than health check. + if mapping.HealthCheckConfig.Port != fmt.Sprintf("%d", mapping.TrafficPort) || + mapping.HealthCheckConfig.Protocol != strings.ToLower(string((mapping.TrafficProtocol))) { + healthCheckPort, err := strconv.ParseInt(mapping.HealthCheckConfig.Port, 10, 64) + if err != nil { + return fmt.Errorf("error building security group rules: invalid health check port '%v': %v", mapping.HealthCheckConfig.Port, err) + } + egressRule = &ec2.IpPermission{ + FromPort: aws.Int64(healthCheckPort), + ToPort: aws.Int64(healthCheckPort), + IpProtocol: aws.String(strings.ToLower(string((mapping.HealthCheckConfig.Protocol)))), + IpRanges: ec2SourceRanges, + } + } + egressRules.Insert(egressRule) + } + + // Setup ingress rules + if _, err := c.setSecurityGroupIngress(sgID, ingressRules, false); err != nil { + return fmt.Errorf("creating ingress rules for security group %q: %w", sgID, err) + } + // Setup egress rules + if _, err := c.setSecurityGroupIngress(sgID, egressRules, true); err != nil { + return fmt.Errorf("creating ingress rules for security group %q: %w", sgID, err) + } + return nil +} + // EnsureLoadBalancer implements LoadBalancer.EnsureLoadBalancer func (c *Cloud) EnsureLoadBalancer(ctx context.Context, clusterName string, apiService *v1.Service, nodes []*v1.Node) (*v1.LoadBalancerStatus, error) { annotations := apiService.Annotations @@ -2168,6 +2279,16 @@ func (c *Cloud) EnsureLoadBalancer(ctx context.Context, clusterName string, apiS listeners := []*elb.Listener{} v2Mappings := []nlbPortMapping{} + // Get source ranges to build permission list + sourceRanges, err := servicehelpers.GetLoadBalancerSourceRanges(apiService) + if err != nil { + return nil, err + } + ec2SourceRanges := []*ec2.IpRange{} + for _, srcRange := range sourceRanges.StringSlice() { + ec2SourceRanges = append(ec2SourceRanges, &ec2.IpRange{CidrIp: aws.String(srcRange)}) + } + sslPorts := getPortSets(annotations[ServiceAnnotationLoadBalancerSSLPorts]) for _, port := range apiService.Spec.Ports { if err := checkProtocol(port, annotations); err != nil { @@ -2221,11 +2342,6 @@ func (c *Cloud) EnsureLoadBalancer(ctx context.Context, clusterName string, apiS return nil, err } - sourceRanges, err := servicehelpers.GetLoadBalancerSourceRanges(apiService) - if err != nil { - return nil, err - } - // Determine if this is tagged as an Internal ELB internalELB := false internalAnnotation := apiService.Annotations[ServiceAnnotationLoadBalancerInternal] @@ -2255,6 +2371,52 @@ func (c *Cloud) EnsureLoadBalancer(ctx context.Context, clusterName string, apiS instanceIDs = append(instanceIDs, string(id)) } + // Enable Creating NLB with Security Groups. + // Create NLB with security group support only when one of the following annotations is added: + // ServiceAnnotationLoadBalancerManagedSecurityGroup: managed security group. CCM creates and manage SG for NLB. + securityGroups := []*string{} + // if managedValue, present := annotations[ServiceAnnotationLoadBalancerManagedSecurityGroup]; present { // managed + // if managedValue != "true" { + // return nil, fmt.Errorf("invalid value for annotation %q: %s. valid value: 'true'", ServiceAnnotationLoadBalancerManagedSecurityGroup, managedValue) + // } + if c.cfg.Global.NLBSecurityGroupEnabled { + + // annotation conflict - don't support unmanaged/BYO path on NLB + // TODO: this would be overscoped in OCP feature, and it's not required for short-term. + // TODO/TBD: do we need to support BYO SG for NLBs? Would it be supported with ManagedSecurityGroup annotation? + unmanagedSecurityGroup := true + { + groupList, _, err := c.buildELBSecurityGroupList(serviceName, loadBalancerName, annotations, unmanagedSecurityGroup) + if err != nil { + return nil, fmt.Errorf("unable to retrieve Security Group from annotations: %w", err) + } + if len(groupList) == 0 { + unmanagedSecurityGroup = false + } + if unmanagedSecurityGroup { + return nil, fmt.Errorf("annotation conflict: managed security group and security groups list are mtual exclusive: %v", groupList) + } + } + + // build the list with CCM-created SG + groupList, setupSG, err := c.buildELBSecurityGroupList(serviceName, loadBalancerName, annotations, unmanagedSecurityGroup) + if err != nil { + return nil, fmt.Errorf("unable to create the managed security group: %w", err) + } + + // Check if SG ID has valid syntax + if !strings.HasPrefix(groupList[0], "sg-") { + return nil, fmt.Errorf("invalid security group prefix \"sg-\", got: %q", groupList[0]) + } + securityGroups = append(securityGroups, aws.String(groupList[0])) + + if setupSG { + if err := c.createSecurityGroupRulesForNLB(groupList[0], v2Mappings, ec2SourceRanges); err != nil { + return nil, fmt.Errorf("security group created for NLB is expected to prefix with sg-, got: %q", groupList[0]) + } + } + } + v2LoadBalancer, err := c.ensureLoadBalancerv2( serviceName, loadBalancerName, @@ -2263,6 +2425,7 @@ func (c *Cloud) EnsureLoadBalancer(ctx context.Context, clusterName string, apiS discoveredSubnetIDs, internalELB, annotations, + securityGroups, ) if err != nil { return nil, err @@ -2426,7 +2589,7 @@ func (c *Cloud) EnsureLoadBalancer(ctx context.Context, clusterName string, apiS loadBalancerName := c.GetLoadBalancerName(ctx, clusterName, apiService) serviceName := types.NamespacedName{Namespace: apiService.Namespace, Name: apiService.Name} - securityGroupIDs, setupSg, err := c.buildELBSecurityGroupList(serviceName, loadBalancerName, annotations) + securityGroupIDs, setupSg, err := c.buildELBSecurityGroupList(serviceName, loadBalancerName, annotations, false) if err != nil { return nil, err } @@ -2435,11 +2598,6 @@ func (c *Cloud) EnsureLoadBalancer(ctx context.Context, clusterName string, apiS } if setupSg { - ec2SourceRanges := []*ec2.IpRange{} - for _, sourceRange := range sourceRanges.StringSlice() { - ec2SourceRanges = append(ec2SourceRanges, &ec2.IpRange{CidrIp: aws.String(sourceRange)}) - } - permissions := NewIPPermissionSet() for _, port := range apiService.Spec.Ports { portInt64 := int64(port.Port) @@ -2465,7 +2623,7 @@ func (c *Cloud) EnsureLoadBalancer(ctx context.Context, clusterName string, apiS permissions.Insert(permission) } - _, err = c.setSecurityGroupIngress(securityGroupIDs[0], permissions) + _, err = c.setSecurityGroupIngress(securityGroupIDs[0], permissions, false) if err != nil { return nil, err } @@ -2872,15 +3030,14 @@ func (c *Cloud) EnsureLoadBalancerDeleted(ctx context.Context, clusterName strin // * Delete Load Balancer // * Delete target groups // * Clean up SecurityGroupRules + // * Clean up Security Groups { - targetGroups, err := c.elbv2.DescribeTargetGroups( &elbv2.DescribeTargetGroupsInput{LoadBalancerArn: lb.LoadBalancerArn}, ) if err != nil { return fmt.Errorf("error listing target groups before deleting load balancer: %q", err) } - _, err = c.elbv2.DeleteLoadBalancer( &elbv2.DeleteLoadBalancerInput{LoadBalancerArn: lb.LoadBalancerArn}, ) @@ -2898,7 +3055,61 @@ func (c *Cloud) EnsureLoadBalancerDeleted(ctx context.Context, clusterName strin } } - return c.updateInstanceSecurityGroupsForNLB(loadBalancerName, nil, nil, nil, nil) + if err = c.updateInstanceSecurityGroupsForNLB(loadBalancerName, nil, nil, nil, nil); err != nil { + return fmt.Errorf("error deleting instance security group rules: %w", err) + } + + // Do nothing if there is no Security Group managed annotation (default flow). + if managedValue, present := service.Annotations[ServiceAnnotationLoadBalancerManagedSecurityGroup]; !present || managedValue != "true" { + klog.Warningf("EnsureLoadBalancerDeleted - pre updateInstanceSecurityGroupsForNLB(): managedValue(%v) present(%t)", managedValue, present) + return nil + } + + // Ensure managed NLB security groups are deleted. + describeResponse, err := c.ec2.DescribeSecurityGroups(&ec2.DescribeSecurityGroupsInput{ + GroupIds: lb.SecurityGroups, + }) + if err != nil { + return fmt.Errorf("error describing security group of deleted LB: %w", err) + } + if len(describeResponse) == 0 { + klog.V(3).Infof("No security groups matching the Load Balancer Name to be deleted: %d", len(describeResponse)) + return nil + } + + klog.Infof("Deleting security group: %v", lb.SecurityGroups) + for _, sgID := range describeResponse { + if sgID != nil && c.tagging.hasClusterTag(sgID.Tags) { + klog.V(2).Infof("Deleting managed security group: %s", aws.StringValue(sgID.GroupId)) + backoff := 5 * time.Second + retryLimit := 10 + retryCount := 0 + for { + _, err := c.ec2.DeleteSecurityGroup(&ec2.DeleteSecurityGroupInput{GroupId: sgID.GroupId}) + if err == nil { + break + } + if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == "DependencyViolation" { + klog.V(2).Infof("Security group %s has dependencies, waiting before retrying deletion: %d/%d", aws.StringValue(sgID.GroupId), retryCount, retryLimit) + if retryCount > retryLimit { + return fmt.Errorf("exchausted retries while deleting managed security group %s: %w", aws.StringValue(sgID.GroupId), err) + } + retryCount++ + if backoff > 60*time.Second { + backoff = 10 * time.Second + } + time.Sleep(backoff) + backoff *= 2 + continue + } + return fmt.Errorf("error deleting managed security group %s: %w", aws.StringValue(sgID.GroupId), err) + } + } else { + klog.V(3).Infof("Skipping security group %q delettion as cluster tag does not", aws.StringValue(sgID.GroupId)) + } + } + + return nil } lb, err := c.describeLoadBalancer(loadBalancerName) diff --git a/pkg/providers/v1/aws_loadbalancer.go b/pkg/providers/v1/aws_loadbalancer.go index 5e170f1280..f08bbd929c 100644 --- a/pkg/providers/v1/aws_loadbalancer.go +++ b/pkg/providers/v1/aws_loadbalancer.go @@ -141,7 +141,7 @@ func getKeyValuePropertiesFromAnnotation(annotations map[string]string, annotati } // ensureLoadBalancerv2 ensures a v2 load balancer is created -func (c *Cloud) ensureLoadBalancerv2(namespacedName types.NamespacedName, loadBalancerName string, mappings []nlbPortMapping, instanceIDs, discoveredSubnetIDs []string, internalELB bool, annotations map[string]string) (*elbv2.LoadBalancer, error) { +func (c *Cloud) ensureLoadBalancerv2(namespacedName types.NamespacedName, loadBalancerName string, mappings []nlbPortMapping, instanceIDs, discoveredSubnetIDs []string, internalELB bool, annotations map[string]string, securityGroups []*string) (*elbv2.LoadBalancer, error) { loadBalancer, err := c.describeLoadBalancerv2(loadBalancerName) if err != nil { return nil, err @@ -177,6 +177,9 @@ func (c *Cloud) ensureLoadBalancerv2(namespacedName types.NamespacedName, loadBa // TODO: What happens if we have more than one subnet per AZ? createRequest.SubnetMappings = createSubnetMappings(discoveredSubnetIDs, allocationIDs) + // Enable provisioning NLB with security groups when the annotation(s) are set. + createRequest.SecurityGroups = securityGroups + for k, v := range tags { createRequest.Tags = append(createRequest.Tags, &elbv2.Tag{ Key: aws.String(k), Value: aws.String(v), @@ -402,6 +405,7 @@ func (c *Cloud) ensureLoadBalancerv2(namespacedName types.NamespacedName, loadBa func (c *Cloud) reconcileLBAttributes(loadBalancerArn string, annotations map[string]string) error { desiredLoadBalancerAttributes := map[string]string{} + //REVIEW on Update desiredLoadBalancerAttributes[lbAttrLoadBalancingCrossZoneEnabled] = "false" crossZoneLoadBalancingEnabledAnnotation := annotations[ServiceAnnotationLoadBalancerCrossZoneLoadBalancingEnabled] if crossZoneLoadBalancingEnabledAnnotation != "" { diff --git a/pkg/providers/v1/aws_test.go b/pkg/providers/v1/aws_test.go index 428690f0fb..1d2400e88c 100644 --- a/pkg/providers/v1/aws_test.go +++ b/pkg/providers/v1/aws_test.go @@ -2063,7 +2063,7 @@ func TestLBExtraSecurityGroupsAnnotation(t *testing.T) { t.Run(test.name, func(t *testing.T) { serviceName := types.NamespacedName{Namespace: "default", Name: "myservice"} - sgList, setupSg, err := c.buildELBSecurityGroupList(serviceName, "aid", test.annotations) + sgList, setupSg, err := c.buildELBSecurityGroupList(serviceName, "aid", test.annotations, false) assert.NoError(t, err, "buildELBSecurityGroupList failed") extraSGs := sgList[1:] assert.True(t, sets.NewString(test.expectedSGs...).Equal(sets.NewString(extraSGs...)), @@ -2097,7 +2097,7 @@ func TestLBSecurityGroupsAnnotation(t *testing.T) { t.Run(test.name, func(t *testing.T) { serviceName := types.NamespacedName{Namespace: "default", Name: "myservice"} - sgList, setupSg, err := c.buildELBSecurityGroupList(serviceName, "aid", test.annotations) + sgList, setupSg, err := c.buildELBSecurityGroupList(serviceName, "aid", test.annotations, false) assert.NoError(t, err, "buildELBSecurityGroupList failed") assert.True(t, sets.NewString(test.expectedSGs...).Equal(sets.NewString(sgList...)), "Security Groups expected=%q , returned=%q", test.expectedSGs, sgList) @@ -2106,6 +2106,57 @@ func TestLBSecurityGroupsAnnotation(t *testing.T) { } } +func TestLBManagedSecurityGroupAnnotation(t *testing.T) { + awsServices := newMockedFakeAWSServices(TestClusterID) + c, _ := newAWSCloud(config.CloudConfig{}, awsServices) + + loadBalancerName := "nlbsg" + managed := map[string]string{ServiceAnnotationLoadBalancerManagedSecurityGroup: "true"} + managedSG := "k8s-elb-" + loadBalancerName + unmanaged := map[string]string{ServiceAnnotationLoadBalancerManagedSecurityGroup: "false"} + + tests := []struct { + name string + annotations map[string]string + skipDefault bool + expectedSGs []string + wantSetupSG bool + }{ + { + "No NLB Managed SG", map[string]string{}, false, []string{}, true, + }, + { + "NLB Managed SG specified true skipDefault", managed, true, []string{}, false, + }, + { + "NLB Managed SG specified true", managed, false, []string{managedSG}, true, + }, + { + "NLB Managed SG specified false", unmanaged, true, []string{}, false, + }, + { + "NLB Managed SG specified true with securityGroupLabels", managed, true, []string{}, false, + }, + { + "NLB Managed SG specified true with extra securityGroupLabels", managed, true, []string{}, false, + }, + } + + awsServices.ec2.(*MockedFakeEC2).expectDescribeSecurityGroups(TestClusterID, managedSG) + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + serviceName := types.NamespacedName{Namespace: "default", Name: "myservice"} + + sgList, setupSg, err := c.buildELBSecurityGroupList(serviceName, loadBalancerName, test.annotations, test.skipDefault) + assert.NoError(t, err, "buildELBSecurityGroupList failed") + assert.True(t, sets.NewString(test.expectedSGs...).Equal(sets.NewString(sgList...)), + "Security Groups expected=%q , returned=%q", test.expectedSGs, sgList) + assert.False(t, setupSg, "Security Groups Setup Permissions Flag expected=%t , returned=%t", test.wantSetupSG, setupSg) + }) + } +} + // Test that we can add a load balancer tag func TestAddLoadBalancerTags(t *testing.T) { loadBalancerName := "test-elb" diff --git a/pkg/providers/v1/config/config.go b/pkg/providers/v1/config/config.go index efae450ed8..98ee44c0b5 100644 --- a/pkg/providers/v1/config/config.go +++ b/pkg/providers/v1/config/config.go @@ -19,6 +19,9 @@ const ( ClusterServiceLoadBalancerHealthProbeModeServiceNodePort = "ServiceNodePort" ) +// NLBSecurityGroupEnabled indicates whether the service loadbalancer type NLB is created with a Security Group. +type NLBSecurityGroupEnabled bool + // CloudConfig wraps the settings for the AWS cloud provider. // NOTE: Cloud config files should follow the same Kubernetes deprecation policy as // flags or CLIs. Config fields should not change behavior in incompatible ways and @@ -83,6 +86,10 @@ type CloudConfig struct { // ClusterServiceSharedLoadBalancerHealthProbePath defines the target path of the shared health probe. Default to `/healthz`. ClusterServiceSharedLoadBalancerHealthProbePath string `json:"clusterServiceSharedLoadBalancerHealthProbePath,omitempty" yaml:"clusterServiceSharedLoadBalancerHealthProbePath,omitempty"` + + // NLBSecurityGroupEnabled determines if the service type loadbalancer NLB creates and manages + // the resource with a security group (default behavior Classic Load Balancer). + NLBSecurityGroupEnabled NLBSecurityGroupEnabled `json:"nlbSecurityGroupEnabled,omitempty" yaml:"nlbSecurityGroupEnabled,omitempty"` } // [ServiceOverride "1"] // Service = s3 diff --git a/tests/e2e/go.mod b/tests/e2e/go.mod index 0c411c08e2..6e1014ec5a 100644 --- a/tests/e2e/go.mod +++ b/tests/e2e/go.mod @@ -3,6 +3,9 @@ module k8s.io/cloud-provider-aws/tests/e2e go 1.23.0 require ( + github.com/aws/aws-sdk-go-v2 v1.36.3 + github.com/aws/aws-sdk-go-v2/config v1.29.14 + github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.45.2 github.com/onsi/ginkgo/v2 v2.9.4 github.com/onsi/gomega v1.27.6 k8s.io/api v0.26.0 @@ -13,6 +16,17 @@ require ( ) require ( + github.com/aws/aws-sdk-go-v2/credentials v1.17.67 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 // indirect + github.com/aws/smithy-go v1.22.2 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect github.com/cenkalti/backoff/v4 v4.1.3 // indirect diff --git a/tests/e2e/go.sum b/tests/e2e/go.sum index 663324c896..4e8455f1bf 100644 --- a/tests/e2e/go.sum +++ b/tests/e2e/go.sum @@ -42,6 +42,34 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38yqWM= +github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg= +github.com/aws/aws-sdk-go-v2/config v1.29.14 h1:f+eEi/2cKCg9pqKBoAIwRGzVb70MRKqWX4dg1BDcSJM= +github.com/aws/aws-sdk-go-v2/config v1.29.14/go.mod h1:wVPHWcIFv3WO89w0rE10gzf17ZYy+UVS1Geq8Iei34g= +github.com/aws/aws-sdk-go-v2/credentials v1.17.67 h1:9KxtdcIA/5xPNQyZRgUSpYOE6j9Bc4+D7nZua0KGYOM= +github.com/aws/aws-sdk-go-v2/credentials v1.17.67/go.mod h1:p3C44m+cfnbv763s52gCqrjaqyPikj9Sg47kUVaNZQQ= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 h1:x793wxmUWVDhshP8WW2mlnXuFrO4cOd3HLBroh1paFw= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 h1:ZK5jHhnrioRkUNOc+hOgQKlUL5JeC3S6JgLxtQ+Rm0Q= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0ioo/gq8Mn6u9w19Mri8DnJ15Jf0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34/go.mod h1:dFZsC0BLo346mvKQLWmoJxT+Sjp+qcVR1tRVHQGOH9Q= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo= +github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.45.2 h1:vX70Z4lNSr7XsioU0uJq5yvxgI50sB66MvD+V/3buS4= +github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.45.2/go.mod h1:xnCC3vFBfOKpU6PcsCKL2ktgBTZfOwTGxj6V8/X3IS4= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 h1:eAh2A4b5IzM/lum78bZ590jy36+d/aFLgKF/4Vd1xPE= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3/go.mod h1:0yKJC/kb8sAnmlYa6Zs3QVYqaC8ug2AbnNChv5Ox3uA= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 h1:dM9/92u2F1JbDaGooxTq18wmmFzbJRfXfVfy96/1CXM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15/go.mod h1:SwFBy2vjtA0vZbjjaFtfN045boopadnoVPhu4Fv66vY= +github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 h1:1Gw+9ajCV1jogloEv1RRnvfRFia2cL6c9cuKV2Ps+G8= +github.com/aws/aws-sdk-go-v2/service/sso v1.25.3/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 h1:hXmVKytPfTy5axZ+fYbR5d0cFmC3JvwLm5kM83luako= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1/go.mod h1:MlYRNmYu/fGPoxBQVvBYr9nyr948aY/WLUvwBMBJubs= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 h1:1XuUZ8mYJw9B6lzAkXhqHlJd/XvaX32evhproijJEZY= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.19/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4= +github.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ= +github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= diff --git a/tests/e2e/loadbalancer.go b/tests/e2e/loadbalancer.go index 79b0e2625e..563131a7ee 100644 --- a/tests/e2e/loadbalancer.go +++ b/tests/e2e/loadbalancer.go @@ -14,15 +14,27 @@ limitations under the License. package e2e import ( + "context" + "fmt" + "strings" + + "github.com/onsi/ginkgo/v2" . "github.com/onsi/ginkgo/v2" v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" e2eservice "k8s.io/kubernetes/test/e2e/framework/service" + imageutils "k8s.io/kubernetes/test/utils/image" admissionapi "k8s.io/pod-security-admission/api" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + elbv2 "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2" ) +// loadbalancer tests var _ = Describe("[cloud-provider-aws-e2e] loadbalancer", func() { f := framework.NewDefaultFramework("cloud-provider-aws") f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged @@ -41,61 +53,290 @@ var _ = Describe("[cloud-provider-aws-e2e] loadbalancer", func() { // After each test }) - It("should configure the loadbalancer based on annotations", func() { - loadBalancerCreateTimeout := e2eservice.GetServiceLoadBalancerCreationTimeout(cs) - framework.Logf("Running tests against AWS with timeout %s", loadBalancerCreateTimeout) + type loadBalancerTestCases struct { + Name string + ResourceSuffix string + Annotations map[string]string + PostRunValidations func(cfg *configServiceLB, svc *v1.Service) + } + + cases := []loadBalancerTestCases{ + { + Name: "should configure the loadbalancer based on annotations", + ResourceSuffix: "", + Annotations: map[string]string{}, + }, + { + Name: "NLB should configure the loadbalancer based on annotations", + ResourceSuffix: "nlb", + Annotations: map[string]string{ + "service.beta.kubernetes.io/aws-load-balancer-type": "nlb", + }, + }, + { + Name: "NLB should configure the loadbalancer with target-node-labels", + ResourceSuffix: "sg-wk", + Annotations: map[string]string{ + "service.beta.kubernetes.io/aws-load-balancer-type": "nlb", + "service.beta.kubernetes.io/aws-load-balancer-target-node-labels": "node-role.kubernetes.io/worker=", + }, + PostRunValidations: func(cfg *configServiceLB, svc *v1.Service) { + j := cfg.LBJig + nodeList, err := j.Client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{ + LabelSelector: "node-role.kubernetes.io/worker=", + }) + framework.ExpectNoError(err, "failed to list worker nodes") + + workerNodes := len(nodeList.Items) + framework.Logf("Found %d worker nodes", workerNodes) + + // Validate in the TG if the node count matches with expected target-node-labels selector. + lbDNS := svc.Status.LoadBalancer.Ingress[0].Hostname + framework.ExpectNoError(getLBTargetCount(context.TODO(), lbDNS, workerNodes), "AWS LB target count validation failed") + }, + }, + } + + serviceNameBase := "lbconfig-test" + for _, tc := range cases { + It(tc.Name, func() { + loadBalancerCreateTimeout := e2eservice.GetServiceLoadBalancerCreationTimeout(cs) + framework.Logf("Running tests against AWS with timeout %s", loadBalancerCreateTimeout) + + // Create Configuration + serviceName := serviceNameBase + if len(tc.ResourceSuffix) > 0 { + serviceName = fmt.Sprintf("%s-%s", serviceName, tc.ResourceSuffix) + } + framework.Logf("namespace for load balancer conig test: %s", ns.Name) + + By("creating a TCP service " + serviceName + " with type=LoadBalancerType in namespace " + ns.Name) + lbConfig := newConfigServiceLB() + lbConfig.LBJig = e2eservice.NewTestJig(cs, ns.Name, serviceName) + lbServiceConfig := lbConfig.buildService(tc.Annotations) + + // Create Load Balancer + ginkgo.By("creating loadbalancer for service " + lbServiceConfig.Namespace + "/" + lbServiceConfig.Name) + _, err := lbConfig.LBJig.Client.CoreV1().Services(lbConfig.LBJig.Namespace).Create(context.TODO(), lbServiceConfig, metav1.CreateOptions{}) + framework.ExpectNoError(fmt.Errorf("failed to create LoadBalancer Service %q: %v", lbServiceConfig.Name, err)) + + ginkgo.By("waiting for loadbalancer for service " + lbServiceConfig.Namespace + "/" + lbServiceConfig.Name) + lbService, err := lbConfig.LBJig.WaitForLoadBalancer(loadBalancerCreateTimeout) + framework.ExpectNoError(err) + + // Run Workloads + By("creating a pod to be part of the TCP service " + serviceName) + _, err = lbConfig.LBJig.Run(lbConfig.buildReplicationController()) + framework.ExpectNoError(err) + + if tc.PostRunValidations != nil { + By("running post run validations") + tc.PostRunValidations(lbConfig, lbService) + } + + // Test the Service Endpoint + By("hitting the TCP service's LB External IP") + svcPort := int(lbService.Spec.Ports[0].Port) + ingressIP := e2eservice.GetIngressPoint(&lbService.Status.LoadBalancer.Ingress[0]) + framework.Logf("Load balancer's ingress IP: %s", ingressIP) + + e2eservice.TestReachableHTTP(ingressIP, svcPort, e2eservice.LoadBalancerLagTimeoutAWS) + + // Update the service to cluster IP + By("changing TCP service back to type=ClusterIP") + _, err = lbConfig.LBJig.UpdateService(func(s *v1.Service) { + s.Spec.Type = v1.ServiceTypeClusterIP + }) + framework.ExpectNoError(err) + + // Wait for the load balancer to be destroyed asynchronously + _, err = lbConfig.LBJig.WaitForLoadBalancerDestroy(ingressIP, svcPort, loadBalancerCreateTimeout) + framework.ExpectNoError(err) + }) + } +}) - serviceName := "lbconfig-test" - framework.Logf("namespace for load balancer conig test: %s", ns.Name) +// configServiceLB hold loadbalancer test configurations +type configServiceLB struct { + PodPort uint16 + PodProtocol v1.Protocol + DefaultAnnotations map[string]string - By("creating a TCP service " + serviceName + " with type=LoadBalancerType in namespace " + ns.Name) - lbJig := e2eservice.NewTestJig(cs, ns.Name, serviceName) + LBJig *e2eservice.TestJig +} - serviceUpdateFunc := func(svc *v1.Service) { - annotations := make(map[string]string) - annotations["aws-load-balancer-backend-protocol"] = "http" - annotations["aws-load-balancer-ssl-ports"] = "https" +func newConfigServiceLB() *configServiceLB { + return &configServiceLB{ + PodPort: 8080, + PodProtocol: v1.ProtocolTCP, + DefaultAnnotations: map[string]string{ + "aws-load-balancer-backend-protocol": "http", + "aws-load-balancer-ssl-ports": "https", + }, + } +} - svc.Annotations = annotations - svc.Spec.Ports = []v1.ServicePort{ +func (s *configServiceLB) buildService(extraAnnotations map[string]string) *v1.Service { + svc := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: s.LBJig.Namespace, + Name: s.LBJig.Name, + Labels: s.LBJig.Labels, + Annotations: make(map[string]string, len(s.DefaultAnnotations)+len(extraAnnotations)), + }, + Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeLoadBalancer, + SessionAffinity: v1.ServiceAffinityNone, + Selector: s.LBJig.Labels, + Ports: []v1.ServicePort{ { Name: "http", Protocol: v1.ProtocolTCP, Port: int32(80), - TargetPort: intstr.FromInt(80), + TargetPort: intstr.FromInt(int(s.PodPort)), }, { Name: "https", Protocol: v1.ProtocolTCP, Port: int32(443), - TargetPort: intstr.FromInt(80), + TargetPort: intstr.FromInt(int(s.PodPort)), }, - } + }, + }, + } + + // add default annotations - can be overriden by extra annotations + for aK, aV := range s.DefaultAnnotations { + svc.Annotations[aK] = aV + } + + // append test case annotations to the service + for aK, aV := range extraAnnotations { + svc.Annotations[aK] = aV + } + + return svc +} + +// buildReplicationController creates a replication controller wrapper for the test framework. +// buildReplicationController is basaed on newRCTemplate() from the test, which not provide +// customization to bind in non-privileged ports. +// TODO(mtulio): v1.33+[2] moved from RC to Deployments on tests, we must do the same to use Run() +// when the test framework is updated. +// [1] https://github.com/kubernetes/kubernetes/blob/89d95c9713a8fd189e8ad555120838b3c4f888d1/test/e2e/framework/service/jig.go#L636 +// [2] https://github.com/kubernetes/kubernetes/issues/119021 +func (s *configServiceLB) buildReplicationController() func(rc *v1.ReplicationController) { + return func(rc *v1.ReplicationController) { + var replicas int32 = 1 + var grace int64 = 3 // so we don't race with kube-proxy when scaling up/down + rc.ObjectMeta = metav1.ObjectMeta{ + Namespace: s.LBJig.Namespace, + Name: s.LBJig.Name, + Labels: s.LBJig.Labels, + } + rc.Spec = v1.ReplicationControllerSpec{ + Replicas: &replicas, + Selector: s.LBJig.Labels, + Template: &v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: s.LBJig.Labels, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "netexec", + Image: imageutils.GetE2EImage(imageutils.Agnhost), + Args: []string{ + "netexec", + fmt.Sprintf("--http-port=%d", s.PodPort), + fmt.Sprintf("--udp-port=%d", s.PodPort), + }, + ReadinessProbe: &v1.Probe{ + PeriodSeconds: 3, + ProbeHandler: v1.ProbeHandler{ + HTTPGet: &v1.HTTPGetAction{ + Port: intstr.FromInt(int(s.PodPort)), + Path: "/hostName", + }, + }, + }, + }, + }, + TerminationGracePeriodSeconds: &grace, + }, + }, } + } +} - lbService, err := lbJig.CreateLoadBalancerService(loadBalancerCreateTimeout, serviceUpdateFunc) - framework.ExpectNoError(err) +// getLBTargetCount verifies the number of registered targets for a given LBv2 DNS name matches the expected count. +// The steps includes: +// 1. Get Load Balancer ARN from DNS name extracted from service Status.LoadBalancer.Ingress[0].Hostname +// 2. List listeners for the load balancer +// 3. Get target groups attached to listeners +// 4. Count registered targets in target groups +// 5. Verify count matches number of worker nodes +func getLBTargetCount(ctx context.Context, lbDNSName string, expectedTargets int) error { + // Load AWS config + cfg, err := config.LoadDefaultConfig(ctx) + if err != nil { + return fmt.Errorf("unable to load AWS config: %v", err) + } + elbClient := elbv2.NewFromConfig(cfg) - By("creating a pod to be part of the TCP service " + serviceName) - _, err = lbJig.Run(nil) - framework.ExpectNoError(err) + // Get Load Balancer ARN from DNS name + describeLBs, err := elbClient.DescribeLoadBalancers(ctx, &elbv2.DescribeLoadBalancersInput{}) + if err != nil { + return fmt.Errorf("failed to describe load balancers: %v", err) + } + var lbARN string + for _, lb := range describeLBs.LoadBalancers { + if strings.EqualFold(aws.ToString(lb.DNSName), lbDNSName) { + lbARN = aws.ToString(lb.LoadBalancerArn) + break + } + } + if lbARN == "" { + return fmt.Errorf("could not find LB with DNS name: %s", lbDNSName) + } - By("hitting the TCP service's LB External IP") - svcPort := int(lbService.Spec.Ports[0].Port) - ingressIP := e2eservice.GetIngressPoint(&lbService.Status.LoadBalancer.Ingress[0]) - framework.Logf("Load balancer's ingress IP: %s", ingressIP) + // List listeners for the load balancer + listenersOut, err := elbClient.DescribeListeners(ctx, &elbv2.DescribeListenersInput{ + LoadBalancerArn: aws.String(lbARN), + }) + if err != nil { + return fmt.Errorf("failed to describe listeners: %v", err) + } - e2eservice.TestReachableHTTP(ingressIP, svcPort, e2eservice.LoadBalancerLagTimeoutAWS) + // Get target groups attached to listeners + targetGroupARNs := map[string]struct{}{} + for _, listener := range listenersOut.Listeners { + if len(targetGroupARNs) > 0 { + break + } + for _, action := range listener.DefaultActions { + if action.TargetGroupArn != nil { + targetGroupARNs[aws.ToString(action.TargetGroupArn)] = struct{}{} + break + } + } + } - // Update the service to cluster IP - By("changing TCP service back to type=ClusterIP") - _, err = lbJig.UpdateService(func(s *v1.Service) { - s.Spec.Type = v1.ServiceTypeClusterIP + // Count registered targets in target groups + totalTargets := 0 + for tgARN := range targetGroupARNs { + tgHealth, err := elbClient.DescribeTargetHealth(ctx, &elbv2.DescribeTargetHealthInput{ + TargetGroupArn: aws.String(tgARN), }) - framework.ExpectNoError(err) + if err != nil { + return fmt.Errorf("failed to describe target health for TG %s: %v", tgARN, err) + } + totalTargets += len(tgHealth.TargetHealthDescriptions) + } - // Wait for the load balancer to be destroyed asynchronously - _, err = lbJig.WaitForLoadBalancerDestroy(ingressIP, svcPort, loadBalancerCreateTimeout) - framework.ExpectNoError(err) - }) -}) + // Verify count matches number of worker nodes + if totalTargets != expectedTargets { + return fmt.Errorf("target count mismatch: expected %d, got %d", expectedTargets, totalTargets) + } + return nil +} diff --git a/tests/e2e/vendor/modules.txt b/tests/e2e/vendor/modules.txt index 4ae0f8ac24..5b6aa1496b 100644 --- a/tests/e2e/vendor/modules.txt +++ b/tests/e2e/vendor/modules.txt @@ -1,3 +1,105 @@ +# github.com/aws/aws-sdk-go-v2 v1.36.3 +## explicit; go 1.22 +github.com/aws/aws-sdk-go-v2/aws +github.com/aws/aws-sdk-go-v2/aws/defaults +github.com/aws/aws-sdk-go-v2/aws/middleware +github.com/aws/aws-sdk-go-v2/aws/protocol/query +github.com/aws/aws-sdk-go-v2/aws/protocol/restjson +github.com/aws/aws-sdk-go-v2/aws/protocol/xml +github.com/aws/aws-sdk-go-v2/aws/ratelimit +github.com/aws/aws-sdk-go-v2/aws/retry +github.com/aws/aws-sdk-go-v2/aws/signer/internal/v4 +github.com/aws/aws-sdk-go-v2/aws/signer/v4 +github.com/aws/aws-sdk-go-v2/aws/transport/http +github.com/aws/aws-sdk-go-v2/internal/auth +github.com/aws/aws-sdk-go-v2/internal/auth/smithy +github.com/aws/aws-sdk-go-v2/internal/context +github.com/aws/aws-sdk-go-v2/internal/endpoints +github.com/aws/aws-sdk-go-v2/internal/endpoints/awsrulesfn +github.com/aws/aws-sdk-go-v2/internal/middleware +github.com/aws/aws-sdk-go-v2/internal/rand +github.com/aws/aws-sdk-go-v2/internal/sdk +github.com/aws/aws-sdk-go-v2/internal/sdkio +github.com/aws/aws-sdk-go-v2/internal/shareddefaults +github.com/aws/aws-sdk-go-v2/internal/strings +github.com/aws/aws-sdk-go-v2/internal/sync/singleflight +github.com/aws/aws-sdk-go-v2/internal/timeconv +# github.com/aws/aws-sdk-go-v2/config v1.29.14 +## explicit; go 1.22 +github.com/aws/aws-sdk-go-v2/config +# github.com/aws/aws-sdk-go-v2/credentials v1.17.67 +## explicit; go 1.22 +github.com/aws/aws-sdk-go-v2/credentials +github.com/aws/aws-sdk-go-v2/credentials/ec2rolecreds +github.com/aws/aws-sdk-go-v2/credentials/endpointcreds +github.com/aws/aws-sdk-go-v2/credentials/endpointcreds/internal/client +github.com/aws/aws-sdk-go-v2/credentials/processcreds +github.com/aws/aws-sdk-go-v2/credentials/ssocreds +github.com/aws/aws-sdk-go-v2/credentials/stscreds +# github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 +## explicit; go 1.22 +github.com/aws/aws-sdk-go-v2/feature/ec2/imds +github.com/aws/aws-sdk-go-v2/feature/ec2/imds/internal/config +# github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 +## explicit; go 1.22 +github.com/aws/aws-sdk-go-v2/internal/configsources +# github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 +## explicit; go 1.22 +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 +# github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 +## explicit; go 1.22 +github.com/aws/aws-sdk-go-v2/internal/ini +# github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.45.2 +## explicit; go 1.22 +github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 +github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/internal/endpoints +github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2/types +# github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 +## explicit; go 1.22 +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding +# github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 +## explicit; go 1.22 +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url +# github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 +## explicit; go 1.22 +github.com/aws/aws-sdk-go-v2/service/sso +github.com/aws/aws-sdk-go-v2/service/sso/internal/endpoints +github.com/aws/aws-sdk-go-v2/service/sso/types +# github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 +## explicit; go 1.22 +github.com/aws/aws-sdk-go-v2/service/ssooidc +github.com/aws/aws-sdk-go-v2/service/ssooidc/internal/endpoints +github.com/aws/aws-sdk-go-v2/service/ssooidc/types +# github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 +## explicit; go 1.22 +github.com/aws/aws-sdk-go-v2/service/sts +github.com/aws/aws-sdk-go-v2/service/sts/internal/endpoints +github.com/aws/aws-sdk-go-v2/service/sts/types +# github.com/aws/smithy-go v1.22.2 +## explicit; go 1.21 +github.com/aws/smithy-go +github.com/aws/smithy-go/auth +github.com/aws/smithy-go/auth/bearer +github.com/aws/smithy-go/context +github.com/aws/smithy-go/document +github.com/aws/smithy-go/encoding +github.com/aws/smithy-go/encoding/httpbinding +github.com/aws/smithy-go/encoding/json +github.com/aws/smithy-go/encoding/xml +github.com/aws/smithy-go/endpoints +github.com/aws/smithy-go/internal/sync/singleflight +github.com/aws/smithy-go/io +github.com/aws/smithy-go/logging +github.com/aws/smithy-go/metrics +github.com/aws/smithy-go/middleware +github.com/aws/smithy-go/private/requestcompression +github.com/aws/smithy-go/ptr +github.com/aws/smithy-go/rand +github.com/aws/smithy-go/time +github.com/aws/smithy-go/tracing +github.com/aws/smithy-go/transport/http +github.com/aws/smithy-go/transport/http/internal/io +github.com/aws/smithy-go/waiter # github.com/beorn7/perks v1.0.1 ## explicit; go 1.11 github.com/beorn7/perks/quantile