Add support for joining clusters to AKS Fleet

Makefile

@@ -158,7 +158,7 @@ WEBHOOK_ROOT ?= $(MANIFEST_ROOT)/webhook
 RBAC_ROOT ?= $(MANIFEST_ROOT)/rbac
 ASO_CRDS_PATH := $(MANIFEST_ROOT)/aso/crds.yaml
 ASO_VERSION := v2.5.0
-ASO_CRDS := resourcegroups.resources.azure.com natgateways.network.azure.com managedclusters.containerservice.azure.com managedclustersagentpools.containerservice.azure.com bastionhosts.network.azure.com virtualnetworks.network.azure.com virtualnetworkssubnets.network.azure.com privateendpoints.network.azure.com
+ASO_CRDS := resourcegroups.resources.azure.com natgateways.network.azure.com managedclusters.containerservice.azure.com managedclustersagentpools.containerservice.azure.com bastionhosts.network.azure.com virtualnetworks.network.azure.com virtualnetworkssubnets.network.azure.com privateendpoints.network.azure.com fleetsmembers.containerservice.azure.com
 
 # Allow overriding the imagePullPolicy
 PULL_POLICY ?= Always

api/v1beta1/azuremanagedcontrolplane_default.go

@@ -98,6 +98,15 @@ func (m *AzureManagedControlPlane) setDefaultSubnet() {
 	}
 }
 
+// setDefaultFleetsMember sets the default FleetsMember for an AzureManagedControlPlane.
+func setDefaultFleetsMember(fleetsMember *FleetsMember, labels map[string]string) *FleetsMember {
+	result := fleetsMember.DeepCopy()
+	if clusterName, ok := labels[clusterv1.ClusterNameLabel]; ok && fleetsMember != nil && fleetsMember.Name == "" {
+		result.Name = clusterName
+	}
+	return result
+}
+
 func setDefaultSku(sku *AKSSku) *AKSSku {
 	result := sku.DeepCopy()
 	if sku == nil {

api/v1beta1/azuremanagedcontrolplane_types.go

@@ -110,6 +110,13 @@ type AzureManagedControlPlaneSpec struct {
 	// Immutable.
 	// +optional
 	DNSPrefix *string `json:"dnsPrefix,omitempty"`
+
+	// FleetsMember is the spec for the fleet this cluster is a member of.
+	// See also [AKS doc].
+	//
+	// [AKS doc]: https://learn.microsoft.com/en-us/azure/templates/microsoft.containerservice/2023-03-15-preview/fleets/members
+	// +optional
+	FleetsMember *FleetsMember `json:"fleetsMember,omitempty"`
 }
 
 // HTTPProxyConfig is the HTTP proxy configuration for the cluster.

api/v1beta1/azuremanagedcontrolplane_webhook.go

@@ -83,6 +83,7 @@ func (mw *azureManagedControlPlaneWebhook) Default(ctx context.Context, obj runt
 	m.Spec.Version = setDefaultVersion(m.Spec.Version)
 	m.Spec.SKU = setDefaultSku(m.Spec.SKU)
 	m.Spec.AutoScalerProfile = setDefaultAutoScalerProfile(m.Spec.AutoScalerProfile)
+	m.Spec.FleetsMember = setDefaultFleetsMember(m.Spec.FleetsMember, m.Labels)
 
 	if err := m.setDefaultSSHPublicKey(); err != nil {
 		ctrl.Log.WithName("AzureManagedControlPlaneWebHookLogger").Error(err, "setDefaultSSHPublicKey failed")
@@ -260,6 +261,10 @@ func (mw *azureManagedControlPlaneWebhook) ValidateUpdate(ctx context.Context, o
 		allErrs = append(allErrs, errs...)
 	}
 
+	if errs := m.validateFleetsMember(old); len(errs) > 0 {
+		allErrs = append(allErrs, errs...)
+	}
+
 	if len(allErrs) == 0 {
 		return nil, m.Validate(mw.Client)
 	}
@@ -687,6 +692,25 @@ func (m *AzureManagedControlPlane) validateOIDCIssuerProfileUpdate(old *AzureMan
 	return allErrs
 }
 
+// validateFleetsMember validates a FleetsMember.
+func (m *AzureManagedControlPlane) validateFleetsMember(old *AzureManagedControlPlane) field.ErrorList {
+	var allErrs field.ErrorList
+
+	if old.Spec.FleetsMember == nil || m.Spec.FleetsMember == nil {
+		return allErrs
+	}
+	if old.Spec.FleetsMember.Name != "" && old.Spec.FleetsMember.Name != m.Spec.FleetsMember.Name {
+		allErrs = append(allErrs,
+			field.Forbidden(
+				field.NewPath("Spec", "FleetsMember", "Name"),
+				"Name is immutable",
+			),
+		)
+	}
+
+	return allErrs
+}
+
 func validateName(name string, fldPath *field.Path) field.ErrorList {
 	var allErrs field.ErrorList
 	if lName := strings.ToLower(name); strings.Contains(lName, "microsoft") ||

api/v1beta1/azuremanagedcontrolplane_webhook_test.go

@@ -37,6 +37,9 @@ func TestDefaultingWebhook(t *testing.T) {
 	amcp := &AzureManagedControlPlane{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: "fooName",
+			Labels: map[string]string{
+				clusterv1.ClusterNameLabel: "fooCluster",
+			},
 		},
 		Spec: AzureManagedControlPlaneSpec{
 			AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{
@@ -80,6 +83,7 @@ func TestDefaultingWebhook(t *testing.T) {
 		Enabled: ptr.To(true),
 	}
 	amcp.Spec.DNSPrefix = ptr.To("test-prefix")
+	amcp.Spec.FleetsMember = &FleetsMember{}
 
 	err = mcpw.Default(context.Background(), amcp)
 	g.Expect(err).NotTo(HaveOccurred())
@@ -94,6 +98,8 @@ func TestDefaultingWebhook(t *testing.T) {
 	g.Expect(*amcp.Spec.OIDCIssuerProfile.Enabled).To(BeTrue())
 	g.Expect(amcp.Spec.DNSPrefix).ToNot(BeNil())
 	g.Expect(*amcp.Spec.DNSPrefix).To(Equal("test-prefix"))
+	g.Expect(amcp.Spec.FleetsMember.Name).To(Equal("fooCluster"))
+
 	t.Logf("Testing amcp defaulting webhook with overlay")
 	amcp = &AzureManagedControlPlane{
 		ObjectMeta: metav1.ObjectMeta{

api/v1beta1/consts.go

@@ -130,6 +130,8 @@ const (
 	NetworkInterfaceReadyCondition clusterv1.ConditionType = "NetworkInterfacesReady"
 	// PrivateEndpointsReadyCondition means the private endpoints exist and are ready to be used.
 	PrivateEndpointsReadyCondition clusterv1.ConditionType = "PrivateEndpointsReady"
+	// FleetReadyCondition means the Fleet exists and is ready to be used.
+	FleetReadyCondition clusterv1.ConditionType = "FleetReady"
 
 	// CreatingReason means the resource is being created.
 	CreatingReason = "Creating"

api/v1beta1/types.go

@@ -1000,6 +1000,18 @@ type AzureBastion struct {
 	EnableTunneling bool `json:"enableTunneling,omitempty"`
 }
 
+// FleetsMember defines the fleets member configuration.
+// See also [AKS doc].
+//
+// [AKS doc]: https://learn.microsoft.com/en-us/azure/templates/microsoft.containerservice/2023-03-15-preview/fleets/members
+type FleetsMember struct {
+	// Name is the name of the member.
+	// +optional
+	Name string `json:"name,omitempty"`
+
+	FleetsMemberClassSpec `json:",inline"`
+}
+
 // BackendPool describes the backend pool of the load balancer.
 type BackendPool struct {
 	// Name specifies the name of backend pool for the load balancer. If not specified, the default name will

api/v1beta1/types_class.go

@@ -211,6 +211,13 @@ type AzureManagedControlPlaneClassSpec struct {
 	// DisableLocalAccounts disables getting static credentials for this cluster when set. Expected to only be used for AAD clusters.
 	// +optional
 	DisableLocalAccounts *bool `json:"disableLocalAccounts,omitempty"`
+
+	// FleetsMember is the spec for the fleet this cluster is a member of.
+	// See also [AKS doc].
+	//
+	// [AKS doc]: https://learn.microsoft.com/en-us/azure/templates/microsoft.containerservice/2023-03-15-preview/fleets/members
+	// +optional
+	FleetsMember *FleetsMemberClassSpec `json:"fleetsMember,omitempty"`
 }
 
 // AzureManagedMachinePoolClassSpec defines the AzureManagedMachinePool properties that may be shared across several Azure managed machinepools.
@@ -448,6 +455,19 @@ type LoadBalancerClassSpec struct {
 	IdleTimeoutInMinutes *int32 `json:"idleTimeoutInMinutes,omitempty"`
 }
 
+// FleetsMemberClassSpec defines the FleetsMemberSpec properties that may be shared across several Azure clusters.
+type FleetsMemberClassSpec struct {
+	// Group is the group this member belongs to for multi-cluster update management.
+	// +optional
+	Group string `json:"group,omitempty"`
+
+	// ManagerName is the name of the fleet manager.
+	ManagerName string `json:"managerName"`
+
+	// ManagerResourceGroup is the resource group of the fleet manager.
+	ManagerResourceGroup string `json:"managerResourceGroup"`
+}
+
 // SecurityGroupClass defines the SecurityGroup properties that may be shared across several Azure clusters.
 type SecurityGroupClass struct {
 	// +optional

azure/defaults.go

@@ -295,6 +295,11 @@ func ManagedClusterID(subscriptionID, resourceGroup, managedClusterName string)
 	return fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.ContainerService/managedClusters/%s", subscriptionID, resourceGroup, managedClusterName)
 }
 
+// FleetID returns the azure resource ID for a given fleet manager.
+func FleetID(subscriptionID, resourceGroup, fleetName string) string {
+	return fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.ContainerService/fleets/%s", subscriptionID, resourceGroup, fleetName)
+}
+
 // GetBootstrappingVMExtension returns the CAPZ Bootstrapping VM extension.
 // The CAPZ Bootstrapping extension is a simple clone of https://github.com/Azure/custom-script-extension-linux for Linux or
 // https://learn.microsoft.com/azure/virtual-machines/extensions/custom-script-windows for Windows.

azure/scope/managedcontrolplane.go

@@ -23,6 +23,7 @@ import (
 	"strings"
 	"time"
 
+	asocontainerservicev1preview "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20230315preview"
 	asocontainerservicev1 "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20231001"
 	asonetworkv1api20201101 "github.com/Azure/azure-service-operator/v2/api/network/v1api20201101"
 	asonetworkv1api20220701 "github.com/Azure/azure-service-operator/v2/api/network/v1api20220701"
@@ -38,6 +39,7 @@ import (
 	"k8s.io/utils/ptr"
 	infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1"
 	"sigs.k8s.io/cluster-api-provider-azure/azure"
+	"sigs.k8s.io/cluster-api-provider-azure/azure/services/fleetsmembers"
 	"sigs.k8s.io/cluster-api-provider-azure/azure/services/groups"
 	"sigs.k8s.io/cluster-api-provider-azure/azure/services/managedclusters"
 	"sigs.k8s.io/cluster-api-provider-azure/azure/services/privateendpoints"
@@ -204,6 +206,11 @@ func (s *ManagedControlPlaneScope) AdditionalTags() infrav1.Tags {
 	return tags
 }
 
+// AzureFleetMembership returns the cluster AzureFleetMembership.
+func (s *ManagedControlPlaneScope) AzureFleetMembership() *infrav1.FleetsMember {
+	return s.ControlPlane.Spec.FleetsMember
+}
+
 // SubscriptionID returns the Azure client Subscription ID.
 func (s *ManagedControlPlaneScope) SubscriptionID() string {
 	return s.AzureClients.SubscriptionID()
@@ -281,6 +288,23 @@ func (s *ManagedControlPlaneScope) VNetSpec() azure.ASOResourceSpecGetter[*asone
 	}
 }
 
+// AzureFleetsMemberSpec returns the fleet spec.
+func (s *ManagedControlPlaneScope) AzureFleetsMemberSpec() []azure.ASOResourceSpecGetter[*asocontainerservicev1preview.FleetsMember] {
+	if s.AzureFleetMembership() == nil {
+		return nil
+	}
+	return []azure.ASOResourceSpecGetter[*asocontainerservicev1preview.FleetsMember]{&fleetsmembers.AzureFleetsMemberSpec{
+		Name:                 s.AzureFleetMembership().Name,
+		Namespace:            s.Cluster.Namespace,
+		ClusterName:          s.ClusterName(),
+		ClusterResourceGroup: s.ResourceGroup(),
+		Group:                s.AzureFleetMembership().Group,
+		SubscriptionID:       s.SubscriptionID(),
+		ManagerName:          s.AzureFleetMembership().ManagerName,
+		ManagerResourceGroup: s.AzureFleetMembership().ManagerResourceGroup,
+	}}
+}
+
 // ControlPlaneRouteTable returns the cluster controlplane routetable.
 func (s *ManagedControlPlaneScope) ControlPlaneRouteTable() infrav1.RouteTable {
 	return infrav1.RouteTable{}

azure/services/bastionhosts/spec_test.go

@@ -134,10 +134,10 @@ func TestAzureBastionSpec_Parameters(t *testing.T) {
 					},
 				)
 
-				// ObjectMeta should be carried over from existing private endpoint.
+				// ObjectMeta should be carried over from existing bastion host.
 				g.Expect(result.ObjectMeta).To(Equal(resultantASOBastionHost.ObjectMeta))
 
-				// EnableTunneling addition is accepted.
+				// DisableCopyPaste addition is accepted.
 				g.Expect(result.Spec).To(Equal(resultantASOBastionHost.Spec))
 
 				// Status should be carried over.

azure/services/fleetsmembers/fleetsmembers.go

@@ -0,0 +1,40 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package fleetsmembers
+
+import (
+	asocontainerservicev1 "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20230315preview"
+	infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1"
+	"sigs.k8s.io/cluster-api-provider-azure/azure"
+	"sigs.k8s.io/cluster-api-provider-azure/azure/services/aso"
+)
+
+const serviceName = "fleetsmember"
+
+// FleetsMemberScope defines the scope interface for a Fleet host service.
+type FleetsMemberScope interface {
+	aso.Scope
+	AzureFleetsMemberSpec() []azure.ASOResourceSpecGetter[*asocontainerservicev1.FleetsMember]
+}
+
+// New creates a new service.
+func New(scope FleetsMemberScope) *aso.Service[*asocontainerservicev1.FleetsMember, FleetsMemberScope] {
+	svc := aso.NewService[*asocontainerservicev1.FleetsMember, FleetsMemberScope](serviceName, scope)
+	svc.Specs = scope.AzureFleetsMemberSpec()
+	svc.ConditionType = infrav1.FleetReadyCondition
+	return svc
+}