Merge pull request #4898 from AndiDog/secondary-vpc-cidr

✨ Support adding custom secondary VPC CIDR blocks in `AWSCluster`
kubernetes-sigs · Jul 30, 2024 · f99a356 · f99a356
2 parents f97d237 + 05d6299
commit f99a356
Show file tree

Hide file tree

Showing 36 changed files with 516 additions and 94 deletions.
diff --git a/api/v1beta1/awscluster_conversion.go b/api/v1beta1/awscluster_conversion.go
@@ -106,6 +106,7 @@ func (src *AWSCluster) ConvertTo(dstRaw conversion.Hub) error {
 	dst.Spec.NetworkSpec.VPC.PrivateDNSHostnameTypeOnLaunch = restored.Spec.NetworkSpec.VPC.PrivateDNSHostnameTypeOnLaunch
 	dst.Spec.NetworkSpec.VPC.CarrierGatewayID = restored.Spec.NetworkSpec.VPC.CarrierGatewayID
 	dst.Spec.NetworkSpec.VPC.SubnetSchema = restored.Spec.NetworkSpec.VPC.SubnetSchema
+	dst.Spec.NetworkSpec.VPC.SecondaryCidrBlocks = restored.Spec.NetworkSpec.VPC.SecondaryCidrBlocks
 
 	if restored.Spec.NetworkSpec.VPC.ElasticIPPool != nil {
 		if dst.Spec.NetworkSpec.VPC.ElasticIPPool == nil {

diff --git a/api/v1beta1/zz_generated.conversion.go b/api/v1beta1/zz_generated.conversion.go
diff --git a/api/v1beta2/awscluster_webhook.go b/api/v1beta2/awscluster_webhook.go
@@ -283,6 +283,14 @@ func (r *AWSCluster) validateNetwork() field.ErrorList {
 		}
 	}
 
+	secondaryCidrBlocks := r.Spec.NetworkSpec.VPC.SecondaryCidrBlocks
+	secondaryCidrBlocksField := field.NewPath("spec", "network", "vpc", "secondaryCidrBlocks")
+	for _, cidrBlock := range secondaryCidrBlocks {
+		if r.Spec.NetworkSpec.VPC.CidrBlock != "" && r.Spec.NetworkSpec.VPC.CidrBlock == cidrBlock.IPv4CidrBlock {
+			allErrs = append(allErrs, field.Invalid(secondaryCidrBlocksField, secondaryCidrBlocks, fmt.Sprintf("AWSCluster.spec.network.vpc.secondaryCidrBlocks must not contain the primary AWSCluster.spec.network.vpc.cidrBlock %v", r.Spec.NetworkSpec.VPC.CidrBlock)))
+		}
+	}
+
 	return allErrs
 }
 

diff --git a/api/v1beta2/network_types.go b/api/v1beta2/network_types.go
@@ -388,6 +388,13 @@ type IPAMPool struct {
 	NetmaskLength int64 `json:"netmaskLength,omitempty"`
 }
 
+// VpcCidrBlock defines the CIDR block and settings to associate with the managed VPC. Currently, only IPv4 is supported.
+type VpcCidrBlock struct {
+	// IPv4CidrBlock is the IPv4 CIDR block to associate with the managed VPC.
+	// +kubebuilder:validation:MinLength=1
+	IPv4CidrBlock string `json:"ipv4CidrBlock"`
+}
+
 // VPCSpec configures an AWS VPC.
 type VPCSpec struct {
 	// ID is the vpc-id of the VPC this provider should use to create resources.
@@ -398,6 +405,12 @@ type VPCSpec struct {
 	// Mutually exclusive with IPAMPool.
 	CidrBlock string `json:"cidrBlock,omitempty"`
 
+	// SecondaryCidrBlocks are additional CIDR blocks to be associated when the provider creates a managed VPC.
+	// Defaults to none. Mutually exclusive with IPAMPool. This makes sense to use if, for example, you want to use
+	// a separate IP range for pods (e.g. Cilium ENI mode).
+	// +optional
+	SecondaryCidrBlocks []VpcCidrBlock `json:"secondaryCidrBlocks,omitempty"`
+
 	// IPAMPool defines the IPAMv4 pool to be used for VPC.
 	// Mutually exclusive with CidrBlock.
 	IPAMPool *IPAMPool `json:"ipamPool,omitempty"`

diff --git a/api/v1beta2/zz_generated.deepcopy.go b/api/v1beta2/zz_generated.deepcopy.go
diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/cluster_api_controller.go b/cmd/clusterawsadm/cloudformation/bootstrap/cluster_api_controller.go
@@ -90,6 +90,7 @@ func (t Template) ControllersPolicy() *iamv1.PolicyDocument {
 				"ec2:AssignPrivateIpAddresses",
 				"ec2:UnassignPrivateIpAddresses",
 				"ec2:AssociateRouteTable",
+				"ec2:AssociateVpcCidrBlock",
 				"ec2:AttachInternetGateway",
 				"ec2:AuthorizeSecurityGroupIngress",
 				"ec2:CreateCarrierGateway",
@@ -104,6 +105,7 @@ func (t Template) ControllersPolicy() *iamv1.PolicyDocument {
 				"ec2:CreateTags",
 				"ec2:CreateVpc",
 				"ec2:CreateVpcEndpoint",
+				"ec2:DisassociateVpcCidrBlock",
 				"ec2:ModifyVpcAttribute",
 				"ec2:ModifyVpcEndpoint",
 				"ec2:DeleteCarrierGateway",

diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/customsuffix.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/customsuffix.yaml
@@ -149,6 +149,7 @@ Resources:
           - ec2:AssignPrivateIpAddresses
           - ec2:UnassignPrivateIpAddresses
           - ec2:AssociateRouteTable
+          - ec2:AssociateVpcCidrBlock
           - ec2:AttachInternetGateway
           - ec2:AuthorizeSecurityGroupIngress
           - ec2:CreateCarrierGateway
@@ -163,6 +164,7 @@ Resources:
           - ec2:CreateTags
           - ec2:CreateVpc
           - ec2:CreateVpcEndpoint
+          - ec2:DisassociateVpcCidrBlock
           - ec2:ModifyVpcAttribute
           - ec2:ModifyVpcEndpoint
           - ec2:DeleteCarrierGateway

diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/default.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/default.yaml
@@ -149,6 +149,7 @@ Resources:
           - ec2:AssignPrivateIpAddresses
           - ec2:UnassignPrivateIpAddresses
           - ec2:AssociateRouteTable
+          - ec2:AssociateVpcCidrBlock
           - ec2:AttachInternetGateway
           - ec2:AuthorizeSecurityGroupIngress
           - ec2:CreateCarrierGateway
@@ -163,6 +164,7 @@ Resources:
           - ec2:CreateTags
           - ec2:CreateVpc
           - ec2:CreateVpcEndpoint
+          - ec2:DisassociateVpcCidrBlock
           - ec2:ModifyVpcAttribute
           - ec2:ModifyVpcEndpoint
           - ec2:DeleteCarrierGateway

diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_all_secret_backends.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_all_secret_backends.yaml
@@ -155,6 +155,7 @@ Resources:
           - ec2:AssignPrivateIpAddresses
           - ec2:UnassignPrivateIpAddresses
           - ec2:AssociateRouteTable
+          - ec2:AssociateVpcCidrBlock
           - ec2:AttachInternetGateway
           - ec2:AuthorizeSecurityGroupIngress
           - ec2:CreateCarrierGateway
@@ -169,6 +170,7 @@ Resources:
           - ec2:CreateTags
           - ec2:CreateVpc
           - ec2:CreateVpcEndpoint
+          - ec2:DisassociateVpcCidrBlock
           - ec2:ModifyVpcAttribute
           - ec2:ModifyVpcEndpoint
           - ec2:DeleteCarrierGateway

diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_allow_assume_role.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_allow_assume_role.yaml
@@ -149,6 +149,7 @@ Resources:
           - ec2:AssignPrivateIpAddresses
           - ec2:UnassignPrivateIpAddresses
           - ec2:AssociateRouteTable
+          - ec2:AssociateVpcCidrBlock
           - ec2:AttachInternetGateway
           - ec2:AuthorizeSecurityGroupIngress
           - ec2:CreateCarrierGateway
@@ -163,6 +164,7 @@ Resources:
           - ec2:CreateTags
           - ec2:CreateVpc
           - ec2:CreateVpcEndpoint
+          - ec2:DisassociateVpcCidrBlock
           - ec2:ModifyVpcAttribute
           - ec2:ModifyVpcEndpoint
           - ec2:DeleteCarrierGateway

diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_bootstrap_user.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_bootstrap_user.yaml
@@ -155,6 +155,7 @@ Resources:
           - ec2:AssignPrivateIpAddresses
           - ec2:UnassignPrivateIpAddresses
           - ec2:AssociateRouteTable
+          - ec2:AssociateVpcCidrBlock
           - ec2:AttachInternetGateway
           - ec2:AuthorizeSecurityGroupIngress
           - ec2:CreateCarrierGateway
@@ -169,6 +170,7 @@ Resources:
           - ec2:CreateTags
           - ec2:CreateVpc
           - ec2:CreateVpcEndpoint
+          - ec2:DisassociateVpcCidrBlock
           - ec2:ModifyVpcAttribute
           - ec2:ModifyVpcEndpoint
           - ec2:DeleteCarrierGateway

diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_custom_bootstrap_user.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_custom_bootstrap_user.yaml
@@ -155,6 +155,7 @@ Resources:
           - ec2:AssignPrivateIpAddresses
           - ec2:UnassignPrivateIpAddresses
           - ec2:AssociateRouteTable
+          - ec2:AssociateVpcCidrBlock
           - ec2:AttachInternetGateway
           - ec2:AuthorizeSecurityGroupIngress
           - ec2:CreateCarrierGateway
@@ -169,6 +170,7 @@ Resources:
           - ec2:CreateTags
           - ec2:CreateVpc
           - ec2:CreateVpcEndpoint
+          - ec2:DisassociateVpcCidrBlock
           - ec2:ModifyVpcAttribute
           - ec2:ModifyVpcEndpoint
           - ec2:DeleteCarrierGateway

diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_different_instance_profiles.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_different_instance_profiles.yaml
@@ -149,6 +149,7 @@ Resources:
           - ec2:AssignPrivateIpAddresses
           - ec2:UnassignPrivateIpAddresses
           - ec2:AssociateRouteTable
+          - ec2:AssociateVpcCidrBlock
           - ec2:AttachInternetGateway
           - ec2:AuthorizeSecurityGroupIngress
           - ec2:CreateCarrierGateway
@@ -163,6 +164,7 @@ Resources:
           - ec2:CreateTags
           - ec2:CreateVpc
           - ec2:CreateVpcEndpoint
+          - ec2:DisassociateVpcCidrBlock
           - ec2:ModifyVpcAttribute
           - ec2:ModifyVpcEndpoint
           - ec2:DeleteCarrierGateway

diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_console.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_console.yaml
@@ -149,6 +149,7 @@ Resources:
           - ec2:AssignPrivateIpAddresses
           - ec2:UnassignPrivateIpAddresses
           - ec2:AssociateRouteTable
+          - ec2:AssociateVpcCidrBlock
           - ec2:AttachInternetGateway
           - ec2:AuthorizeSecurityGroupIngress
           - ec2:CreateCarrierGateway
@@ -163,6 +164,7 @@ Resources:
           - ec2:CreateTags
           - ec2:CreateVpc
           - ec2:CreateVpcEndpoint
+          - ec2:DisassociateVpcCidrBlock
           - ec2:ModifyVpcAttribute
           - ec2:ModifyVpcEndpoint
           - ec2:DeleteCarrierGateway

diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_default_roles.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_default_roles.yaml
@@ -149,6 +149,7 @@ Resources:
           - ec2:AssignPrivateIpAddresses
           - ec2:UnassignPrivateIpAddresses
           - ec2:AssociateRouteTable
+          - ec2:AssociateVpcCidrBlock
           - ec2:AttachInternetGateway
           - ec2:AuthorizeSecurityGroupIngress
           - ec2:CreateCarrierGateway
@@ -163,6 +164,7 @@ Resources:
           - ec2:CreateTags
           - ec2:CreateVpc
           - ec2:CreateVpcEndpoint
+          - ec2:DisassociateVpcCidrBlock
           - ec2:ModifyVpcAttribute
           - ec2:ModifyVpcEndpoint
           - ec2:DeleteCarrierGateway

diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_disable.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_disable.yaml
@@ -149,6 +149,7 @@ Resources:
           - ec2:AssignPrivateIpAddresses
           - ec2:UnassignPrivateIpAddresses
           - ec2:AssociateRouteTable
+          - ec2:AssociateVpcCidrBlock
           - ec2:AttachInternetGateway
           - ec2:AuthorizeSecurityGroupIngress
           - ec2:CreateCarrierGateway
@@ -163,6 +164,7 @@ Resources:
           - ec2:CreateTags
           - ec2:CreateVpc
           - ec2:CreateVpcEndpoint
+          - ec2:DisassociateVpcCidrBlock
           - ec2:ModifyVpcAttribute
           - ec2:ModifyVpcEndpoint
           - ec2:DeleteCarrierGateway

diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_kms_prefix.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_eks_kms_prefix.yaml
@@ -149,6 +149,7 @@ Resources:
           - ec2:AssignPrivateIpAddresses
           - ec2:UnassignPrivateIpAddresses
           - ec2:AssociateRouteTable
+          - ec2:AssociateVpcCidrBlock
           - ec2:AttachInternetGateway
           - ec2:AuthorizeSecurityGroupIngress
           - ec2:CreateCarrierGateway
@@ -163,6 +164,7 @@ Resources:
           - ec2:CreateTags
           - ec2:CreateVpc
           - ec2:CreateVpcEndpoint
+          - ec2:DisassociateVpcCidrBlock
           - ec2:ModifyVpcAttribute
           - ec2:ModifyVpcEndpoint
           - ec2:DeleteCarrierGateway

diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_extra_statements.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_extra_statements.yaml
@@ -155,6 +155,7 @@ Resources:
           - ec2:AssignPrivateIpAddresses
           - ec2:UnassignPrivateIpAddresses
           - ec2:AssociateRouteTable
+          - ec2:AssociateVpcCidrBlock
           - ec2:AttachInternetGateway
           - ec2:AuthorizeSecurityGroupIngress
           - ec2:CreateCarrierGateway
@@ -169,6 +170,7 @@ Resources:
           - ec2:CreateTags
           - ec2:CreateVpc
           - ec2:CreateVpcEndpoint
+          - ec2:DisassociateVpcCidrBlock
           - ec2:ModifyVpcAttribute
           - ec2:ModifyVpcEndpoint
           - ec2:DeleteCarrierGateway

diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_s3_bucket.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_s3_bucket.yaml
@@ -149,6 +149,7 @@ Resources:
           - ec2:AssignPrivateIpAddresses
           - ec2:UnassignPrivateIpAddresses
           - ec2:AssociateRouteTable
+          - ec2:AssociateVpcCidrBlock
           - ec2:AttachInternetGateway
           - ec2:AuthorizeSecurityGroupIngress
           - ec2:CreateCarrierGateway
@@ -163,6 +164,7 @@ Resources:
           - ec2:CreateTags
           - ec2:CreateVpc
           - ec2:CreateVpcEndpoint
+          - ec2:DisassociateVpcCidrBlock
           - ec2:ModifyVpcAttribute
           - ec2:ModifyVpcEndpoint
           - ec2:DeleteCarrierGateway

diff --git a/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_ssm_secret_backend.yaml b/cmd/clusterawsadm/cloudformation/bootstrap/fixtures/with_ssm_secret_backend.yaml
@@ -149,6 +149,7 @@ Resources:
           - ec2:AssignPrivateIpAddresses
           - ec2:UnassignPrivateIpAddresses
           - ec2:AssociateRouteTable
+          - ec2:AssociateVpcCidrBlock
           - ec2:AttachInternetGateway
           - ec2:AuthorizeSecurityGroupIngress
           - ec2:CreateCarrierGateway
@@ -163,6 +164,7 @@ Resources:
           - ec2:CreateTags
           - ec2:CreateVpc
           - ec2:CreateVpcEndpoint
+          - ec2:DisassociateVpcCidrBlock
           - ec2:ModifyVpcAttribute
           - ec2:ModifyVpcEndpoint
           - ec2:DeleteCarrierGateway

diff --git a/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml b/config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml
@@ -759,6 +759,25 @@ spec:
                         - ip-name
                         - resource-name
                         type: string
+                      secondaryCidrBlocks:
+                        description: |-
+                          SecondaryCidrBlocks are additional CIDR blocks to be associated when the provider creates a managed VPC.
+                          Defaults to none. Mutually exclusive with IPAMPool. This makes sense to use if, for example, you want to use
+                          a separate IP range for pods (e.g. Cilium ENI mode).
+                        items:
+                          description: VpcCidrBlock defines the CIDR block and settings
+                            to associate with the managed VPC. Currently, only IPv4
+                            is supported.
+                          properties:
+                            ipv4CidrBlock:
+                              description: IPv4CidrBlock is the IPv4 CIDR block to
+                                associate with the managed VPC.
+                              minLength: 1
+                              type: string
+                          required:
+                          - ipv4CidrBlock
+                          type: object
+                        type: array
                       subnetSchema:
                         default: PreferPrivate
                         description: |-
@@ -2766,6 +2785,25 @@ spec:
                         - ip-name
                         - resource-name
                         type: string
+                      secondaryCidrBlocks:
+                        description: |-
+                          SecondaryCidrBlocks are additional CIDR blocks to be associated when the provider creates a managed VPC.
+                          Defaults to none. Mutually exclusive with IPAMPool. This makes sense to use if, for example, you want to use
+                          a separate IP range for pods (e.g. Cilium ENI mode).
+                        items:
+                          description: VpcCidrBlock defines the CIDR block and settings
+                            to associate with the managed VPC. Currently, only IPv4
+                            is supported.
+                          properties:
+                            ipv4CidrBlock:
+                              description: IPv4CidrBlock is the IPv4 CIDR block to
+                                associate with the managed VPC.
+                              minLength: 1
+                              type: string
+                          required:
+                          - ipv4CidrBlock
+                          type: object
+                        type: array
                       subnetSchema:
                         default: PreferPrivate
                         description: |-