Skip to content

Commit 6193d17

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request kubernetes-sigs#4438 from prometherion/issues/4437
✨ Supporting externally managed Control Plane
2 parents 6ffb575 + fc3cb07 commit 6193d17

19 files changed

+448
-47
lines changed

‎api/v1beta2/awscluster_types.go

+6-5
Original file line numberDiff line numberDiff line change
@@ -167,10 +167,11 @@ type Bastion struct {
167167
type LoadBalancerType string
168168

169169
var (
170-
LoadBalancerTypeClassic = LoadBalancerType("classic")
171-
LoadBalancerTypeELB = LoadBalancerType("elb")
172-
LoadBalancerTypeALB = LoadBalancerType("alb")
173-
LoadBalancerTypeNLB = LoadBalancerType("nlb")
170+
LoadBalancerTypeClassic = LoadBalancerType("classic")
171+
LoadBalancerTypeELB = LoadBalancerType("elb")
172+
LoadBalancerTypeALB = LoadBalancerType("alb")
173+
LoadBalancerTypeNLB = LoadBalancerType("nlb")
174+
LoadBalancerTypeDisabled = LoadBalancerType("disabled")
174175
)
175176

176177
// AWSLoadBalancerSpec defines the desired state of an AWS load balancer.
@@ -229,7 +230,7 @@ type AWSLoadBalancerSpec struct {
229230

230231
// LoadBalancerType sets the type for a load balancer. The default type is classic.
231232
// +kubebuilder:default=classic
232-
// +kubebuilder:validation:Enum:=classic;elb;alb;nlb
233+
// +kubebuilder:validation:Enum:=classic;elb;alb;nlb;disabled
233234
LoadBalancerType LoadBalancerType `json:"loadBalancerType,omitempty"`
234235

235236
// DisableHostsRewrite disabled the hair pinning issue solution that adds the NLB's address as 127.0.0.1 to the hosts

‎api/v1beta2/awscluster_webhook.go

+44
Original file line numberDiff line numberDiff line change
@@ -298,5 +298,49 @@ func (r *AWSCluster) validateControlPlaneLBs() field.ErrorList {
298298
}
299299
}
300300

301+
if r.Spec.ControlPlaneLoadBalancer.LoadBalancerType == LoadBalancerTypeDisabled {
302+
if r.Spec.ControlPlaneLoadBalancer.Name != nil {
303+
allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "controlPlaneLoadBalancer", "name"), r.Spec.ControlPlaneLoadBalancer.Name, "cannot configure a name if the LoadBalancer reconciliation is disabled"))
304+
}
305+
306+
if r.Spec.ControlPlaneLoadBalancer.CrossZoneLoadBalancing {
307+
allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "controlPlaneLoadBalancer", "crossZoneLoadBalancing"), r.Spec.ControlPlaneLoadBalancer.CrossZoneLoadBalancing, "cross-zone load balancing cannot be set if the LoadBalancer reconciliation is disabled"))
308+
}
309+
310+
if len(r.Spec.ControlPlaneLoadBalancer.Subnets) > 0 {
311+
allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "controlPlaneLoadBalancer", "subnets"), r.Spec.ControlPlaneLoadBalancer.Subnets, "subnets cannot be set if the LoadBalancer reconciliation is disabled"))
312+
}
313+
314+
if r.Spec.ControlPlaneLoadBalancer.HealthCheckProtocol != nil {
315+
allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "controlPlaneLoadBalancer", "healthCheckProtocol"), r.Spec.ControlPlaneLoadBalancer.HealthCheckProtocol, "healthcheck protocol cannot be set if the LoadBalancer reconciliation is disabled"))
316+
}
317+
318+
if len(r.Spec.ControlPlaneLoadBalancer.AdditionalSecurityGroups) > 0 {
319+
allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "controlPlaneLoadBalancer", "additionalSecurityGroups"), r.Spec.ControlPlaneLoadBalancer.AdditionalSecurityGroups, "additional Security Groups cannot be set if the LoadBalancer reconciliation is disabled"))
320+
}
321+
322+
if len(r.Spec.ControlPlaneLoadBalancer.AdditionalListeners) > 0 {
323+
allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "controlPlaneLoadBalancer", "additionalListeners"), r.Spec.ControlPlaneLoadBalancer.AdditionalListeners, "cannot set additional listeners if the LoadBalancer reconciliation is disabled"))
324+
}
325+
326+
if len(r.Spec.ControlPlaneLoadBalancer.IngressRules) > 0 {
327+
allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "controlPlaneLoadBalancer", "ingressRules"), r.Spec.ControlPlaneLoadBalancer.IngressRules, "ingress rules cannot be set if the LoadBalancer reconciliation is disabled"))
328+
}
329+
330+
if r.Spec.ControlPlaneLoadBalancer.PreserveClientIP {
331+
allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "controlPlaneLoadBalancer", "preserveClientIP"), r.Spec.ControlPlaneLoadBalancer.PreserveClientIP, "cannot preserve client IP if the LoadBalancer reconciliation is disabled"))
332+
}
333+
334+
if r.Spec.ControlPlaneLoadBalancer.DisableHostsRewrite {
335+
allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "controlPlaneLoadBalancer", "disableHostsRewrite"), r.Spec.ControlPlaneLoadBalancer.DisableHostsRewrite, "cannot disable hosts rewrite if the LoadBalancer reconciliation is disabled"))
336+
}
337+
}
338+
339+
for _, rule := range r.Spec.ControlPlaneLoadBalancer.IngressRules {
340+
if (rule.CidrBlocks != nil || rule.IPv6CidrBlocks != nil) && (rule.SourceSecurityGroupIDs != nil || rule.SourceSecurityGroupRoles != nil) {
341+
allErrs = append(allErrs, field.Invalid(field.NewPath("spec", "controlPlaneLoadBalancer", "ingressRules"), r.Spec.ControlPlaneLoadBalancer.IngressRules, "CIDR blocks and security group IDs or security group roles cannot be used together"))
342+
}
343+
}
344+
301345
return allErrs
302346
}

‎api/v1beta2/awscluster_webhook_test.go

+121
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import (
2626
. "github.com/onsi/gomega"
2727
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2828
utilfeature "k8s.io/component-base/featuregate/testing"
29+
"k8s.io/utils/ptr"
2930
"sigs.k8s.io/controller-runtime/pkg/client"
3031

3132
"sigs.k8s.io/cluster-api-provider-aws/v2/feature"
@@ -50,6 +51,126 @@ func TestAWSClusterValidateCreate(t *testing.T) {
5051
wantErr bool
5152
expect func(g *WithT, res *AWSLoadBalancerSpec)
5253
}{
54+
{
55+
name: "No options are allowed when LoadBalancer is disabled (name)",
56+
cluster: &AWSCluster{
57+
Spec: AWSClusterSpec{
58+
ControlPlaneLoadBalancer: &AWSLoadBalancerSpec{
59+
LoadBalancerType: LoadBalancerTypeDisabled,
60+
Name: ptr.To("name"),
61+
},
62+
},
63+
},
64+
wantErr: true,
65+
},
66+
{
67+
name: "No options are allowed when LoadBalancer is disabled (crossZoneLoadBalancing)",
68+
cluster: &AWSCluster{
69+
Spec: AWSClusterSpec{
70+
ControlPlaneLoadBalancer: &AWSLoadBalancerSpec{
71+
CrossZoneLoadBalancing: true,
72+
LoadBalancerType: LoadBalancerTypeDisabled,
73+
},
74+
},
75+
},
76+
wantErr: true,
77+
},
78+
{
79+
name: "No options are allowed when LoadBalancer is disabled (subnets)",
80+
cluster: &AWSCluster{
81+
Spec: AWSClusterSpec{
82+
ControlPlaneLoadBalancer: &AWSLoadBalancerSpec{
83+
Subnets: []string{"foo", "bar"},
84+
LoadBalancerType: LoadBalancerTypeDisabled,
85+
},
86+
},
87+
},
88+
wantErr: true,
89+
},
90+
{
91+
name: "No options are allowed when LoadBalancer is disabled (healthCheckProtocol)",
92+
cluster: &AWSCluster{
93+
Spec: AWSClusterSpec{
94+
ControlPlaneLoadBalancer: &AWSLoadBalancerSpec{
95+
HealthCheckProtocol: &ELBProtocolTCP,
96+
LoadBalancerType: LoadBalancerTypeDisabled,
97+
},
98+
},
99+
},
100+
wantErr: true,
101+
},
102+
{
103+
name: "No options are allowed when LoadBalancer is disabled (additionalSecurityGroups)",
104+
cluster: &AWSCluster{
105+
Spec: AWSClusterSpec{
106+
ControlPlaneLoadBalancer: &AWSLoadBalancerSpec{
107+
AdditionalSecurityGroups: []string{"foo", "bar"},
108+
LoadBalancerType: LoadBalancerTypeDisabled,
109+
},
110+
},
111+
},
112+
wantErr: true,
113+
},
114+
{
115+
name: "No options are allowed when LoadBalancer is disabled (additionalListeners)",
116+
cluster: &AWSCluster{
117+
Spec: AWSClusterSpec{
118+
ControlPlaneLoadBalancer: &AWSLoadBalancerSpec{
119+
AdditionalListeners: []AdditionalListenerSpec{
120+
{
121+
Port: 6443,
122+
Protocol: ELBProtocolTCP,
123+
},
124+
},
125+
LoadBalancerType: LoadBalancerTypeDisabled,
126+
},
127+
},
128+
},
129+
wantErr: true,
130+
},
131+
{
132+
name: "No options are allowed when LoadBalancer is disabled (ingressRules)",
133+
cluster: &AWSCluster{
134+
Spec: AWSClusterSpec{
135+
ControlPlaneLoadBalancer: &AWSLoadBalancerSpec{
136+
IngressRules: []IngressRule{
137+
{
138+
Description: "ingress rule",
139+
Protocol: SecurityGroupProtocolTCP,
140+
FromPort: 6443,
141+
ToPort: 6443,
142+
},
143+
},
144+
LoadBalancerType: LoadBalancerTypeDisabled,
145+
},
146+
},
147+
},
148+
wantErr: true,
149+
},
150+
{
151+
name: "No options are allowed when LoadBalancer is disabled (disableHostsRewrite)",
152+
cluster: &AWSCluster{
153+
Spec: AWSClusterSpec{
154+
ControlPlaneLoadBalancer: &AWSLoadBalancerSpec{
155+
DisableHostsRewrite: true,
156+
LoadBalancerType: LoadBalancerTypeDisabled,
157+
},
158+
},
159+
},
160+
wantErr: true,
161+
},
162+
{
163+
name: "No options are allowed when LoadBalancer is disabled (preserveClientIP)",
164+
cluster: &AWSCluster{
165+
Spec: AWSClusterSpec{
166+
ControlPlaneLoadBalancer: &AWSLoadBalancerSpec{
167+
PreserveClientIP: true,
168+
LoadBalancerType: LoadBalancerTypeDisabled,
169+
},
170+
},
171+
},
172+
wantErr: true,
173+
},
53174
// The SSHKeyName tests were moved to sshkeyname_test.go
54175
{
55176
name: "Supported schemes are 'internet-facing, Internet-facing, internal, or nil', rest will be rejected",

‎api/v1beta2/conditions_consts.go

+3
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,9 @@ const (
125125
LoadBalancerReadyCondition clusterv1.ConditionType = "LoadBalancerReady"
126126
// WaitForDNSNameReason used while waiting for a DNS name for the API server to be populated.
127127
WaitForDNSNameReason = "WaitForDNSName"
128+
// WaitForExternalControlPlaneEndpointReason is available when the AWS Cluster is waiting for an externally managed
129+
// Load Balancer, such as an external Control Plane provider.
130+
WaitForExternalControlPlaneEndpointReason = "WaitForExternalControlPlaneEndpoint"
128131
// WaitForDNSNameResolveReason used while waiting for DNS name to resolve.
129132
WaitForDNSNameResolveReason = "WaitForDNSNameResolve"
130133
// LoadBalancerFailedReason used when an error occurs during load balancer reconciliation.

‎config/crd/bases/infrastructure.cluster.x-k8s.io_awsclusters.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -1109,6 +1109,7 @@ spec:
11091109
- elb
11101110
- alb
11111111
- nlb
1112+
- disabled
11121113
type: string
11131114
name:
11141115
description: Name sets the name of the classic ELB load balancer.
@@ -1689,6 +1690,7 @@ spec:
16891690
- elb
16901691
- alb
16911692
- nlb
1693+
- disabled
16921694
type: string
16931695
name:
16941696
description: Name sets the name of the classic ELB load balancer.

‎config/crd/bases/infrastructure.cluster.x-k8s.io_awsclustertemplates.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -707,6 +707,7 @@ spec:
707707
- elb
708708
- alb
709709
- nlb
710+
- disabled
710711
type: string
711712
name:
712713
description: Name sets the name of the classic ELB load
@@ -1316,6 +1317,7 @@ spec:
13161317
- elb
13171318
- alb
13181319
- nlb
1320+
- disabled
13191321
type: string
13201322
name:
13211323
description: Name sets the name of the classic ELB load

‎config/rbac/role.yaml

+8
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,14 @@ rules:
119119
- get
120120
- list
121121
- watch
122+
- apiGroups:
123+
- controlplane.cluster.x-k8s.io
124+
resources:
125+
- '*'
126+
verbs:
127+
- get
128+
- list
129+
- watch
122130
- apiGroups:
123131
- controlplane.cluster.x-k8s.io
124132
resources:

‎controllers/awscluster_controller.go

+68-24
Original file line numberDiff line numberDiff line change
@@ -266,6 +266,45 @@ func (r *AWSClusterReconciler) reconcileDelete(ctx context.Context, clusterScope
266266
return nil
267267
}
268268

269+
func (r *AWSClusterReconciler) reconcileLoadBalancer(clusterScope *scope.ClusterScope, awsCluster *infrav1.AWSCluster) (*time.Duration, error) {
270+
retryAfterDuration := 15 * time.Second
271+
if clusterScope.AWSCluster.Spec.ControlPlaneLoadBalancer.LoadBalancerType == infrav1.LoadBalancerTypeDisabled {
272+
clusterScope.Debug("load balancer reconciliation shifted to external provider, checking external endpoint")
273+
274+
return r.checkForExternalControlPlaneLoadBalancer(clusterScope, awsCluster), nil
275+
}
276+
277+
elbService := r.getELBService(clusterScope)
278+
279+
if err := elbService.ReconcileLoadbalancers(); err != nil {
280+
clusterScope.Error(err, "failed to reconcile load balancer")
281+
conditions.MarkFalse(awsCluster, infrav1.LoadBalancerReadyCondition, infrav1.LoadBalancerFailedReason, infrautilconditions.ErrorConditionAfterInit(clusterScope.ClusterObj()), err.Error())
282+
return nil, err
283+
}
284+
285+
if awsCluster.Status.Network.APIServerELB.DNSName == "" {
286+
conditions.MarkFalse(awsCluster, infrav1.LoadBalancerReadyCondition, infrav1.WaitForDNSNameReason, clusterv1.ConditionSeverityInfo, "")
287+
clusterScope.Info("Waiting on API server ELB DNS name")
288+
return &retryAfterDuration, nil
289+
}
290+
291+
clusterScope.Debug("Looking up IP address for DNS", "dns", awsCluster.Status.Network.APIServerELB.DNSName)
292+
if _, err := net.LookupIP(awsCluster.Status.Network.APIServerELB.DNSName); err != nil {
293+
clusterScope.Error(err, "failed to get IP address for dns name", "dns", awsCluster.Status.Network.APIServerELB.DNSName)
294+
conditions.MarkFalse(awsCluster, infrav1.LoadBalancerReadyCondition, infrav1.WaitForDNSNameResolveReason, clusterv1.ConditionSeverityInfo, "")
295+
clusterScope.Info("Waiting on API server ELB DNS name to resolve")
296+
return &retryAfterDuration, nil
297+
}
298+
conditions.MarkTrue(awsCluster, infrav1.LoadBalancerReadyCondition)
299+
300+
awsCluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{
301+
Host: awsCluster.Status.Network.APIServerELB.DNSName,
302+
Port: clusterScope.APIServerPort(),
303+
}
304+
305+
return nil, nil
306+
}
307+
269308
func (r *AWSClusterReconciler) reconcileNormal(clusterScope *scope.ClusterScope) (reconcile.Result, error) {
270309
clusterScope.Info("Reconciling AWSCluster")
271310

@@ -280,7 +319,6 @@ func (r *AWSClusterReconciler) reconcileNormal(clusterScope *scope.ClusterScope)
280319
}
281320

282321
ec2Service := r.getEC2Service(clusterScope)
283-
elbService := r.getELBService(clusterScope)
284322
networkSvc := r.getNetworkService(*clusterScope)
285323
sgService := r.getSecurityGroupService(*clusterScope)
286324
s3Service := s3.NewService(clusterScope)
@@ -310,37 +348,17 @@ func (r *AWSClusterReconciler) reconcileNormal(clusterScope *scope.ClusterScope)
310348
}
311349
}
312350

313-
if err := elbService.ReconcileLoadbalancers(); err != nil {
314-
clusterScope.Error(err, "failed to reconcile load balancer")
315-
conditions.MarkFalse(awsCluster, infrav1.LoadBalancerReadyCondition, infrav1.LoadBalancerFailedReason, infrautilconditions.ErrorConditionAfterInit(clusterScope.ClusterObj()), err.Error())
351+
if requeueAfter, err := r.reconcileLoadBalancer(clusterScope, awsCluster); err != nil {
316352
return reconcile.Result{}, err
353+
} else if requeueAfter != nil {
354+
return reconcile.Result{RequeueAfter: *requeueAfter}, err
317355
}
318356

319357
if err := s3Service.ReconcileBucket(); err != nil {
320358
conditions.MarkFalse(awsCluster, infrav1.S3BucketReadyCondition, infrav1.S3BucketFailedReason, clusterv1.ConditionSeverityError, err.Error())
321359
return reconcile.Result{}, errors.Wrapf(err, "failed to reconcile S3 Bucket for AWSCluster %s/%s", awsCluster.Namespace, awsCluster.Name)
322360
}
323361

324-
if awsCluster.Status.Network.APIServerELB.DNSName == "" {
325-
conditions.MarkFalse(awsCluster, infrav1.LoadBalancerReadyCondition, infrav1.WaitForDNSNameReason, clusterv1.ConditionSeverityInfo, "")
326-
clusterScope.Info("Waiting on API server ELB DNS name")
327-
return reconcile.Result{RequeueAfter: 15 * time.Second}, nil
328-
}
329-
330-
clusterScope.Debug("looking up IP address for DNS", "dns", awsCluster.Status.Network.APIServerELB.DNSName)
331-
if _, err := net.LookupIP(awsCluster.Status.Network.APIServerELB.DNSName); err != nil {
332-
clusterScope.Error(err, "failed to get IP address for dns name", "dns", awsCluster.Status.Network.APIServerELB.DNSName)
333-
conditions.MarkFalse(awsCluster, infrav1.LoadBalancerReadyCondition, infrav1.WaitForDNSNameResolveReason, clusterv1.ConditionSeverityInfo, "")
334-
clusterScope.Info("Waiting on API server ELB DNS name to resolve")
335-
return reconcile.Result{RequeueAfter: 15 * time.Second}, nil
336-
}
337-
conditions.MarkTrue(awsCluster, infrav1.LoadBalancerReadyCondition)
338-
339-
awsCluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{
340-
Host: awsCluster.Status.Network.APIServerELB.DNSName,
341-
Port: clusterScope.APIServerPort(),
342-
}
343-
344362
for _, subnet := range clusterScope.Subnets().FilterPrivate() {
345363
found := false
346364
for _, az := range awsCluster.Status.Network.APIServerELB.AvailabilityZones {
@@ -447,3 +465,29 @@ func (r *AWSClusterReconciler) requeueAWSClusterForUnpausedCluster(_ context.Con
447465
}
448466
}
449467
}
468+
469+
func (r *AWSClusterReconciler) checkForExternalControlPlaneLoadBalancer(clusterScope *scope.ClusterScope, awsCluster *infrav1.AWSCluster) *time.Duration {
470+
requeueAfterPeriod := 15 * time.Second
471+
472+
switch {
473+
case len(awsCluster.Spec.ControlPlaneEndpoint.Host) == 0 && awsCluster.Spec.ControlPlaneEndpoint.Port == 0:
474+
clusterScope.Info("AWSCluster control plane endpoint is still non-populated")
475+
conditions.MarkFalse(awsCluster, infrav1.LoadBalancerReadyCondition, infrav1.WaitForExternalControlPlaneEndpointReason, clusterv1.ConditionSeverityInfo, "")
476+
477+
return &requeueAfterPeriod
478+
case len(awsCluster.Spec.ControlPlaneEndpoint.Host) == 0:
479+
clusterScope.Info("AWSCluster control plane endpoint host is still non-populated")
480+
conditions.MarkFalse(awsCluster, infrav1.LoadBalancerReadyCondition, infrav1.WaitForExternalControlPlaneEndpointReason, clusterv1.ConditionSeverityInfo, "")
481+
482+
return &requeueAfterPeriod
483+
case awsCluster.Spec.ControlPlaneEndpoint.Port == 0:
484+
clusterScope.Info("AWSCluster control plane endpoint port is still non-populated")
485+
conditions.MarkFalse(awsCluster, infrav1.LoadBalancerReadyCondition, infrav1.WaitForExternalControlPlaneEndpointReason, clusterv1.ConditionSeverityInfo, "")
486+
487+
return &requeueAfterPeriod
488+
default:
489+
conditions.MarkTrue(awsCluster, infrav1.LoadBalancerReadyCondition)
490+
491+
return nil
492+
}
493+
}

0 commit comments

Comments
 (0)