diff --git a/Makefile b/Makefile index 1bdb898fc9e..18c925b4f3d 100644 --- a/Makefile +++ b/Makefile @@ -158,7 +158,7 @@ WEBHOOK_ROOT ?= $(MANIFEST_ROOT)/webhook RBAC_ROOT ?= $(MANIFEST_ROOT)/rbac ASO_CRDS_PATH := $(MANIFEST_ROOT)/aso/crds.yaml ASO_VERSION := v2.5.0 -ASO_CRDS := resourcegroups.resources.azure.com natgateways.network.azure.com managedclusters.containerservice.azure.com managedclustersagentpools.containerservice.azure.com bastionhosts.network.azure.com virtualnetworks.network.azure.com virtualnetworkssubnets.network.azure.com privateendpoints.network.azure.com +ASO_CRDS := resourcegroups.resources.azure.com natgateways.network.azure.com managedclusters.containerservice.azure.com managedclustersagentpools.containerservice.azure.com bastionhosts.network.azure.com virtualnetworks.network.azure.com virtualnetworkssubnets.network.azure.com privateendpoints.network.azure.com fleetsmembers.containerservice.azure.com # Allow overriding the imagePullPolicy PULL_POLICY ?= Always diff --git a/api/v1beta1/azuremanagedcontrolplane_default.go b/api/v1beta1/azuremanagedcontrolplane_default.go index f4b0cd4f4c6..5f26197c08a 100644 --- a/api/v1beta1/azuremanagedcontrolplane_default.go +++ b/api/v1beta1/azuremanagedcontrolplane_default.go @@ -98,6 +98,15 @@ func (m *AzureManagedControlPlane) setDefaultSubnet() { } } +// setDefaultFleetsMember sets the default FleetsMember for an AzureManagedControlPlane. +func setDefaultFleetsMember(fleetsMember *FleetsMember, labels map[string]string) *FleetsMember { + result := fleetsMember.DeepCopy() + if clusterName, ok := labels[clusterv1.ClusterNameLabel]; ok && fleetsMember != nil && fleetsMember.Name == "" { + result.Name = clusterName + } + return result +} + func setDefaultSku(sku *AKSSku) *AKSSku { result := sku.DeepCopy() if sku == nil { diff --git a/api/v1beta1/azuremanagedcontrolplane_types.go b/api/v1beta1/azuremanagedcontrolplane_types.go index 09535d9e25d..5e274d40505 100644 --- a/api/v1beta1/azuremanagedcontrolplane_types.go +++ b/api/v1beta1/azuremanagedcontrolplane_types.go @@ -110,6 +110,13 @@ type AzureManagedControlPlaneSpec struct { // Immutable. // +optional DNSPrefix *string `json:"dnsPrefix,omitempty"` + + // FleetsMember is the spec for the fleet this cluster is a member of. + // See also [AKS doc]. + // + // [AKS doc]: https://learn.microsoft.com/en-us/azure/templates/microsoft.containerservice/2023-03-15-preview/fleets/members + // +optional + FleetsMember *FleetsMember `json:"fleetsMember,omitempty"` } // HTTPProxyConfig is the HTTP proxy configuration for the cluster. diff --git a/api/v1beta1/azuremanagedcontrolplane_webhook.go b/api/v1beta1/azuremanagedcontrolplane_webhook.go index 3285c059143..4c9da179082 100644 --- a/api/v1beta1/azuremanagedcontrolplane_webhook.go +++ b/api/v1beta1/azuremanagedcontrolplane_webhook.go @@ -83,6 +83,7 @@ func (mw *azureManagedControlPlaneWebhook) Default(ctx context.Context, obj runt m.Spec.Version = setDefaultVersion(m.Spec.Version) m.Spec.SKU = setDefaultSku(m.Spec.SKU) m.Spec.AutoScalerProfile = setDefaultAutoScalerProfile(m.Spec.AutoScalerProfile) + m.Spec.FleetsMember = setDefaultFleetsMember(m.Spec.FleetsMember, m.Labels) if err := m.setDefaultSSHPublicKey(); err != nil { ctrl.Log.WithName("AzureManagedControlPlaneWebHookLogger").Error(err, "setDefaultSSHPublicKey failed") @@ -260,6 +261,10 @@ func (mw *azureManagedControlPlaneWebhook) ValidateUpdate(ctx context.Context, o allErrs = append(allErrs, errs...) } + if errs := m.validateFleetsMember(old); len(errs) > 0 { + allErrs = append(allErrs, errs...) + } + if len(allErrs) == 0 { return nil, m.Validate(mw.Client) } @@ -687,6 +692,25 @@ func (m *AzureManagedControlPlane) validateOIDCIssuerProfileUpdate(old *AzureMan return allErrs } +// validateFleetsMember validates a FleetsMember. +func (m *AzureManagedControlPlane) validateFleetsMember(old *AzureManagedControlPlane) field.ErrorList { + var allErrs field.ErrorList + + if old.Spec.FleetsMember == nil || m.Spec.FleetsMember == nil { + return allErrs + } + if old.Spec.FleetsMember.Name != "" && old.Spec.FleetsMember.Name != m.Spec.FleetsMember.Name { + allErrs = append(allErrs, + field.Forbidden( + field.NewPath("Spec", "FleetsMember", "Name"), + "Name is immutable", + ), + ) + } + + return allErrs +} + func validateName(name string, fldPath *field.Path) field.ErrorList { var allErrs field.ErrorList if lName := strings.ToLower(name); strings.Contains(lName, "microsoft") || diff --git a/api/v1beta1/azuremanagedcontrolplane_webhook_test.go b/api/v1beta1/azuremanagedcontrolplane_webhook_test.go index 3c695a57a8e..bca8dd29efd 100644 --- a/api/v1beta1/azuremanagedcontrolplane_webhook_test.go +++ b/api/v1beta1/azuremanagedcontrolplane_webhook_test.go @@ -37,6 +37,9 @@ func TestDefaultingWebhook(t *testing.T) { amcp := &AzureManagedControlPlane{ ObjectMeta: metav1.ObjectMeta{ Name: "fooName", + Labels: map[string]string{ + clusterv1.ClusterNameLabel: "fooCluster", + }, }, Spec: AzureManagedControlPlaneSpec{ AzureManagedControlPlaneClassSpec: AzureManagedControlPlaneClassSpec{ @@ -80,6 +83,7 @@ func TestDefaultingWebhook(t *testing.T) { Enabled: ptr.To(true), } amcp.Spec.DNSPrefix = ptr.To("test-prefix") + amcp.Spec.FleetsMember = &FleetsMember{} err = mcpw.Default(context.Background(), amcp) g.Expect(err).NotTo(HaveOccurred()) @@ -94,6 +98,8 @@ func TestDefaultingWebhook(t *testing.T) { g.Expect(*amcp.Spec.OIDCIssuerProfile.Enabled).To(BeTrue()) g.Expect(amcp.Spec.DNSPrefix).ToNot(BeNil()) g.Expect(*amcp.Spec.DNSPrefix).To(Equal("test-prefix")) + g.Expect(amcp.Spec.FleetsMember.Name).To(Equal("fooCluster")) + t.Logf("Testing amcp defaulting webhook with overlay") amcp = &AzureManagedControlPlane{ ObjectMeta: metav1.ObjectMeta{ diff --git a/api/v1beta1/consts.go b/api/v1beta1/consts.go index 590da87c172..75e9d422f8f 100644 --- a/api/v1beta1/consts.go +++ b/api/v1beta1/consts.go @@ -130,6 +130,8 @@ const ( NetworkInterfaceReadyCondition clusterv1.ConditionType = "NetworkInterfacesReady" // PrivateEndpointsReadyCondition means the private endpoints exist and are ready to be used. PrivateEndpointsReadyCondition clusterv1.ConditionType = "PrivateEndpointsReady" + // FleetReadyCondition means the Fleet exists and is ready to be used. + FleetReadyCondition clusterv1.ConditionType = "FleetReady" // CreatingReason means the resource is being created. CreatingReason = "Creating" diff --git a/api/v1beta1/types.go b/api/v1beta1/types.go index 0e95ab0b7f4..1314d3bc6cf 100644 --- a/api/v1beta1/types.go +++ b/api/v1beta1/types.go @@ -1000,6 +1000,18 @@ type AzureBastion struct { EnableTunneling bool `json:"enableTunneling,omitempty"` } +// FleetsMember defines the fleets member configuration. +// See also [AKS doc]. +// +// [AKS doc]: https://learn.microsoft.com/en-us/azure/templates/microsoft.containerservice/2023-03-15-preview/fleets/members +type FleetsMember struct { + // Name is the name of the member. + // +optional + Name string `json:"name,omitempty"` + + FleetsMemberClassSpec `json:",inline"` +} + // BackendPool describes the backend pool of the load balancer. type BackendPool struct { // Name specifies the name of backend pool for the load balancer. If not specified, the default name will diff --git a/api/v1beta1/types_class.go b/api/v1beta1/types_class.go index 46913ab7be7..668dbdea3ad 100644 --- a/api/v1beta1/types_class.go +++ b/api/v1beta1/types_class.go @@ -211,6 +211,13 @@ type AzureManagedControlPlaneClassSpec struct { // DisableLocalAccounts disables getting static credentials for this cluster when set. Expected to only be used for AAD clusters. // +optional DisableLocalAccounts *bool `json:"disableLocalAccounts,omitempty"` + + // FleetsMember is the spec for the fleet this cluster is a member of. + // See also [AKS doc]. + // + // [AKS doc]: https://learn.microsoft.com/en-us/azure/templates/microsoft.containerservice/2023-03-15-preview/fleets/members + // +optional + FleetsMember *FleetsMemberClassSpec `json:"fleetsMember,omitempty"` } // AzureManagedMachinePoolClassSpec defines the AzureManagedMachinePool properties that may be shared across several Azure managed machinepools. @@ -448,6 +455,19 @@ type LoadBalancerClassSpec struct { IdleTimeoutInMinutes *int32 `json:"idleTimeoutInMinutes,omitempty"` } +// FleetsMemberClassSpec defines the FleetsMemberSpec properties that may be shared across several Azure clusters. +type FleetsMemberClassSpec struct { + // Group is the group this member belongs to for multi-cluster update management. + // +optional + Group string `json:"group,omitempty"` + + // ManagerName is the name of the fleet manager. + ManagerName string `json:"managerName"` + + // ManagerResourceGroup is the resource group of the fleet manager. + ManagerResourceGroup string `json:"managerResourceGroup"` +} + // SecurityGroupClass defines the SecurityGroup properties that may be shared across several Azure clusters. type SecurityGroupClass struct { // +optional diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 31b37491e4b..656d54f2338 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -1346,6 +1346,11 @@ func (in *AzureManagedControlPlaneClassSpec) DeepCopyInto(out *AzureManagedContr *out = new(bool) **out = **in } + if in.FleetsMember != nil { + in, out := &in.FleetsMember, &out.FleetsMember + *out = new(FleetsMemberClassSpec) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedControlPlaneClassSpec. @@ -1405,6 +1410,11 @@ func (in *AzureManagedControlPlaneSpec) DeepCopyInto(out *AzureManagedControlPla *out = new(string) **out = **in } + if in.FleetsMember != nil { + in, out := &in.FleetsMember, &out.FleetsMember + *out = new(FleetsMember) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedControlPlaneSpec. @@ -2213,6 +2223,37 @@ func (in *ExtendedLocationSpec) DeepCopy() *ExtendedLocationSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FleetsMember) DeepCopyInto(out *FleetsMember) { + *out = *in + out.FleetsMemberClassSpec = in.FleetsMemberClassSpec +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FleetsMember. +func (in *FleetsMember) DeepCopy() *FleetsMember { + if in == nil { + return nil + } + out := new(FleetsMember) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FleetsMemberClassSpec) DeepCopyInto(out *FleetsMemberClassSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FleetsMemberClassSpec. +func (in *FleetsMemberClassSpec) DeepCopy() *FleetsMemberClassSpec { + if in == nil { + return nil + } + out := new(FleetsMemberClassSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *FrontendIP) DeepCopyInto(out *FrontendIP) { *out = *in diff --git a/azure/defaults.go b/azure/defaults.go index c91495c2e4a..58e711379c7 100644 --- a/azure/defaults.go +++ b/azure/defaults.go @@ -295,6 +295,11 @@ func ManagedClusterID(subscriptionID, resourceGroup, managedClusterName string) return fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.ContainerService/managedClusters/%s", subscriptionID, resourceGroup, managedClusterName) } +// FleetID returns the azure resource ID for a given fleet manager. +func FleetID(subscriptionID, resourceGroup, fleetName string) string { + return fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.ContainerService/fleets/%s", subscriptionID, resourceGroup, fleetName) +} + // GetBootstrappingVMExtension returns the CAPZ Bootstrapping VM extension. // The CAPZ Bootstrapping extension is a simple clone of https://github.com/Azure/custom-script-extension-linux for Linux or // https://learn.microsoft.com/azure/virtual-machines/extensions/custom-script-windows for Windows. diff --git a/azure/scope/managedcontrolplane.go b/azure/scope/managedcontrolplane.go index bdc2e383b75..bbf7bca5c94 100644 --- a/azure/scope/managedcontrolplane.go +++ b/azure/scope/managedcontrolplane.go @@ -23,6 +23,7 @@ import ( "strings" "time" + asocontainerservicev1preview "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20230315preview" asocontainerservicev1 "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20231001" asonetworkv1api20201101 "github.com/Azure/azure-service-operator/v2/api/network/v1api20201101" asonetworkv1api20220701 "github.com/Azure/azure-service-operator/v2/api/network/v1api20220701" @@ -38,6 +39,7 @@ import ( "k8s.io/utils/ptr" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" "sigs.k8s.io/cluster-api-provider-azure/azure" + "sigs.k8s.io/cluster-api-provider-azure/azure/services/fleetsmembers" "sigs.k8s.io/cluster-api-provider-azure/azure/services/groups" "sigs.k8s.io/cluster-api-provider-azure/azure/services/managedclusters" "sigs.k8s.io/cluster-api-provider-azure/azure/services/privateendpoints" @@ -204,6 +206,11 @@ func (s *ManagedControlPlaneScope) AdditionalTags() infrav1.Tags { return tags } +// AzureFleetMembership returns the cluster AzureFleetMembership. +func (s *ManagedControlPlaneScope) AzureFleetMembership() *infrav1.FleetsMember { + return s.ControlPlane.Spec.FleetsMember +} + // SubscriptionID returns the Azure client Subscription ID. func (s *ManagedControlPlaneScope) SubscriptionID() string { return s.AzureClients.SubscriptionID() @@ -281,6 +288,23 @@ func (s *ManagedControlPlaneScope) VNetSpec() azure.ASOResourceSpecGetter[*asone } } +// AzureFleetsMemberSpec returns the fleet spec. +func (s *ManagedControlPlaneScope) AzureFleetsMemberSpec() []azure.ASOResourceSpecGetter[*asocontainerservicev1preview.FleetsMember] { + if s.AzureFleetMembership() == nil { + return nil + } + return []azure.ASOResourceSpecGetter[*asocontainerservicev1preview.FleetsMember]{&fleetsmembers.AzureFleetsMemberSpec{ + Name: s.AzureFleetMembership().Name, + Namespace: s.Cluster.Namespace, + ClusterName: s.ClusterName(), + ClusterResourceGroup: s.ResourceGroup(), + Group: s.AzureFleetMembership().Group, + SubscriptionID: s.SubscriptionID(), + ManagerName: s.AzureFleetMembership().ManagerName, + ManagerResourceGroup: s.AzureFleetMembership().ManagerResourceGroup, + }} +} + // ControlPlaneRouteTable returns the cluster controlplane routetable. func (s *ManagedControlPlaneScope) ControlPlaneRouteTable() infrav1.RouteTable { return infrav1.RouteTable{} diff --git a/azure/services/bastionhosts/spec_test.go b/azure/services/bastionhosts/spec_test.go index 2d8a99cc457..d59bbe7a979 100644 --- a/azure/services/bastionhosts/spec_test.go +++ b/azure/services/bastionhosts/spec_test.go @@ -134,10 +134,10 @@ func TestAzureBastionSpec_Parameters(t *testing.T) { }, ) - // ObjectMeta should be carried over from existing private endpoint. + // ObjectMeta should be carried over from existing bastion host. g.Expect(result.ObjectMeta).To(Equal(resultantASOBastionHost.ObjectMeta)) - // EnableTunneling addition is accepted. + // DisableCopyPaste addition is accepted. g.Expect(result.Spec).To(Equal(resultantASOBastionHost.Spec)) // Status should be carried over. diff --git a/azure/services/fleetsmembers/fleetsmembers.go b/azure/services/fleetsmembers/fleetsmembers.go new file mode 100644 index 00000000000..a6fbb735953 --- /dev/null +++ b/azure/services/fleetsmembers/fleetsmembers.go @@ -0,0 +1,40 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fleetsmembers + +import ( + asocontainerservicev1 "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20230315preview" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" + "sigs.k8s.io/cluster-api-provider-azure/azure" + "sigs.k8s.io/cluster-api-provider-azure/azure/services/aso" +) + +const serviceName = "fleetsmember" + +// FleetsMemberScope defines the scope interface for a Fleet host service. +type FleetsMemberScope interface { + aso.Scope + AzureFleetsMemberSpec() []azure.ASOResourceSpecGetter[*asocontainerservicev1.FleetsMember] +} + +// New creates a new service. +func New(scope FleetsMemberScope) *aso.Service[*asocontainerservicev1.FleetsMember, FleetsMemberScope] { + svc := aso.NewService[*asocontainerservicev1.FleetsMember, FleetsMemberScope](serviceName, scope) + svc.Specs = scope.AzureFleetsMemberSpec() + svc.ConditionType = infrav1.FleetReadyCondition + return svc +} diff --git a/azure/services/fleetsmembers/spec.go b/azure/services/fleetsmembers/spec.go new file mode 100644 index 00000000000..db3c88d62c6 --- /dev/null +++ b/azure/services/fleetsmembers/spec.go @@ -0,0 +1,74 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fleetsmembers + +import ( + "context" + + asocontainerservicev1 "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20230315preview" + "github.com/Azure/azure-service-operator/v2/pkg/genruntime" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + "sigs.k8s.io/cluster-api-provider-azure/azure" +) + +// AzureFleetsMemberSpec defines the specification for an Azure Fleets Member. +type AzureFleetsMemberSpec struct { + Name string + Namespace string + ClusterName string + ClusterResourceGroup string + Group string + SubscriptionID string + ManagerName string + ManagerResourceGroup string +} + +// ResourceRef implements azure.ASOResourceSpecGetter. +func (s *AzureFleetsMemberSpec) ResourceRef() *asocontainerservicev1.FleetsMember { + return &asocontainerservicev1.FleetsMember{ + ObjectMeta: metav1.ObjectMeta{ + Name: s.Name, + Namespace: s.Namespace, + }, + } +} + +// Parameters implements azure.ASOResourceSpecGetter. +func (s *AzureFleetsMemberSpec) Parameters(ctx context.Context, existingFleetsMember *asocontainerservicev1.FleetsMember) (parameters *asocontainerservicev1.FleetsMember, err error) { + fleetsMember := &asocontainerservicev1.FleetsMember{} + if existingFleetsMember != nil { + fleetsMember = existingFleetsMember + } + + fleetsMember.Spec.AzureName = s.Name + fleetsMember.Spec.Owner = &genruntime.KnownResourceReference{ + ARMID: azure.FleetID(s.SubscriptionID, s.ManagerResourceGroup, s.ManagerName), + } + fleetsMember.Spec.Group = ptr.To(s.Group) + fleetsMember.Spec.ClusterResourceReference = &genruntime.ResourceReference{ + ARMID: azure.ManagedClusterID(s.SubscriptionID, s.ClusterResourceGroup, s.ClusterName), + } + + return fleetsMember, nil +} + +// WasManaged implements azure.ASOResourceSpecGetter. +func (s *AzureFleetsMemberSpec) WasManaged(resource *asocontainerservicev1.FleetsMember) bool { + // returns false because previous versions of CAPZ did not support Fleets. + return false +} diff --git a/azure/services/fleetsmembers/spec_test.go b/azure/services/fleetsmembers/spec_test.go new file mode 100644 index 00000000000..68f23fafb2d --- /dev/null +++ b/azure/services/fleetsmembers/spec_test.go @@ -0,0 +1,134 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fleetsmembers + +import ( + "context" + "testing" + + asocontainerservicev1 "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20230315preview" + "github.com/Azure/azure-service-operator/v2/pkg/genruntime" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + "sigs.k8s.io/cluster-api-provider-azure/azure" +) + +var ( + fakeAzureFleetsMember = asocontainerservicev1.FleetsMember{ + Spec: asocontainerservicev1.Fleets_Member_Spec{ + AzureName: fakeAzureFleetsMemberSpec.Name, + Owner: &genruntime.KnownResourceReference{ + ARMID: azure.FleetID(fakeAzureFleetsMemberSpec.SubscriptionID, fakeAzureFleetsMemberSpec.ManagerResourceGroup, fakeAzureFleetsMemberSpec.ManagerName), + }, + Group: ptr.To(fakeAzureFleetsMemberSpec.Group), + ClusterResourceReference: &genruntime.ResourceReference{ + ARMID: azure.ManagedClusterID(fakeAzureFleetsMemberSpec.SubscriptionID, fakeAzureFleetsMemberSpec.ClusterResourceGroup, fakeAzureFleetsMemberSpec.ClusterName), + }, + }, + } + fakeAzureFleetsMemberSpec = AzureFleetsMemberSpec{ + Name: "fake-name", + Namespace: "fake-namespace", + ClusterName: "fake-cluster-name", + ClusterResourceGroup: "fake-cluster-resource-group", + Group: "fake-group", + SubscriptionID: "fake-subscription-id", + ManagerName: "fake-manager-name", + ManagerResourceGroup: "fake-manager-resource-group", + } + fakeFleetsMemberStatus = asocontainerservicev1.Fleets_Member_STATUS{ + Name: ptr.To(fakeAzureFleetsMemberSpec.Name), + ProvisioningState: ptr.To(asocontainerservicev1.FleetMemberProvisioningState_STATUS_Succeeded), + } +) + +func getASOFleetsMember(changes ...func(*asocontainerservicev1.FleetsMember)) *asocontainerservicev1.FleetsMember { + fleetsMember := fakeAzureFleetsMember.DeepCopy() + for _, change := range changes { + change(fleetsMember) + } + return fleetsMember +} + +func TestAzureFleetsMemberSpec_Parameters(t *testing.T) { + testcases := []struct { + name string + spec *AzureFleetsMemberSpec + existing *asocontainerservicev1.FleetsMember + expect func(g *WithT, result asocontainerservicev1.FleetsMember) + expectedError string + }{ + { + name: "Creating a new FleetsMember", + spec: &fakeAzureFleetsMemberSpec, + existing: nil, + expect: func(g *WithT, result asocontainerservicev1.FleetsMember) { + g.Expect(result).To(Not(BeNil())) + + // ObjectMeta is populated later in the codeflow + g.Expect(result.ObjectMeta).To(Equal(metav1.ObjectMeta{})) + + // Spec is populated from the spec passed in + g.Expect(result.Spec).To(Equal(getASOFleetsMember().Spec)) + }, + }, + { + name: "User updates to a FleetsMember group should be overwritten", + spec: &fakeAzureFleetsMemberSpec, + existing: getASOFleetsMember( + // user added group which should be overwritten by capz + func(fleetsMember *asocontainerservicev1.FleetsMember) { + fleetsMember.Spec.Group = ptr.To("fake-group-2") + }, + // user added Status + func(fleetsMember *asocontainerservicev1.FleetsMember) { + fleetsMember.Status = fakeFleetsMemberStatus + }, + ), + expect: func(g *WithT, result asocontainerservicev1.FleetsMember) { + g.Expect(result).To(Not(BeNil())) + resultantASOFleetsMember := getASOFleetsMember() + + // ObjectMeta should be carried over from existing fleets member. + g.Expect(result.ObjectMeta).To(Equal(resultantASOFleetsMember.ObjectMeta)) + + // Group addition is accepted. + g.Expect(result.Spec).To(Equal(resultantASOFleetsMember.Spec)) + + // Status should be carried over. + g.Expect(result.Status).To(Equal(fakeFleetsMemberStatus)) + }, + }, + } + for _, tc := range testcases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + t.Parallel() + + result, err := tc.spec.Parameters(context.TODO(), tc.existing) + if tc.expectedError != "" { + g.Expect(err).To(HaveOccurred()) + g.Expect(err).To(MatchError(tc.expectedError)) + } else { + g.Expect(err).NotTo(HaveOccurred()) + } + tc.expect(g, *result) + }) + } +} diff --git a/config/aso/crds.yaml b/config/aso/crds.yaml index df73499929c..b10f688d311 100644 --- a/config/aso/crds.yaml +++ b/config/aso/crds.yaml @@ -535,6 +535,350 @@ spec: --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: azureserviceoperator-system/azureserviceoperator-serving-cert + controller-gen.kubebuilder.io/version: v0.13.0 + labels: + app.kubernetes.io/name: azure-service-operator + app.kubernetes.io/version: v2.5.0 + name: fleetsmembers.containerservice.azure.com +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + name: azureserviceoperator-webhook-service + namespace: azureserviceoperator-system + path: /convert + port: 443 + conversionReviewVersions: + - v1 + group: containerservice.azure.com + names: + kind: FleetsMember + listKind: FleetsMemberList + plural: fleetsmembers + singular: fleetsmember + preserveUnknownFields: false + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.conditions[?(@.type=='Ready')].status + name: Ready + type: string + - jsonPath: .status.conditions[?(@.type=='Ready')].severity + name: Severity + type: string + - jsonPath: .status.conditions[?(@.type=='Ready')].reason + name: Reason + type: string + - jsonPath: .status.conditions[?(@.type=='Ready')].message + name: Message + type: string + name: v1api20230315preview + schema: + openAPIV3Schema: + description: 'Generator information: - Generated from: /containerservice/resource-manager/Microsoft.ContainerService/fleet/preview/2023-03-15-preview/fleets.json - ARM URI: /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.ContainerService/fleets/{fleetName}/members/{fleetMemberName}' + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + azureName: + description: 'AzureName: The name of the resource in Azure. This is often the same as the name of the resource in Kubernetes but it doesn''t have to be.' + maxLength: 50 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + clusterResourceReference: + description: 'ClusterResourceReference: The ARM resource id of the cluster that joins the Fleet. Must be a valid Azure resource id. e.g.: ''/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.ContainerService/managedClusters/{clusterName}''.' + properties: + armId: + description: ARMID is a string of the form /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/{resourceProviderNamespace}/{resourceType}/{resourceName}. The /resourcegroups/{resourceGroupName} bit is optional as some resources are scoped at the subscription level ARMID is mutually exclusive with Group, Kind, Namespace and Name. + pattern: (?i)(^(/subscriptions/([^/]+)(/resourcegroups/([^/]+))?)?/providers/([^/]+)/([^/]+/[^/]+)(/([^/]+/[^/]+))*$|^/subscriptions/([^/]+)(/resourcegroups/([^/]+))?$) + type: string + group: + description: Group is the Kubernetes group of the resource. + type: string + kind: + description: Kind is the Kubernetes kind of the resource. + type: string + name: + description: Name is the Kubernetes name of the resource. + type: string + type: object + group: + description: 'Group: The group this member belongs to for multi-cluster update management.' + maxLength: 50 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + type: string + owner: + description: 'Owner: The owner of the resource. The owner controls where the resource goes when it is deployed. The owner also controls the resources lifecycle. When the owner is deleted the resource will also be deleted. Owner is expected to be a reference to a containerservice.azure.com/Fleet resource' + properties: + armId: + pattern: (?i)(^(/subscriptions/([^/]+)(/resourcegroups/([^/]+))?)?/providers/([^/]+)/([^/]+/[^/]+)(/([^/]+/[^/]+))*$|^/subscriptions/([^/]+)(/resourcegroups/([^/]+))?$) + type: string + name: + description: This is the name of the Kubernetes resource to reference. + type: string + type: object + required: + - clusterResourceReference + - owner + type: object + status: + properties: + clusterResourceId: + description: 'ClusterResourceId: The ARM resource id of the cluster that joins the Fleet. Must be a valid Azure resource id. e.g.: ''/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.ContainerService/managedClusters/{clusterName}''.' + type: string + conditions: + description: 'Conditions: The observed state of the resource' + items: + description: Condition defines an extension to status (an observation) of a resource + properties: + lastTransitionTime: + description: LastTransitionTime is the last time the condition transitioned from one status to another. + format: date-time + type: string + message: + description: Message is a human readable message indicating details about the transition. This field may be empty. + type: string + observedGeneration: + description: ObservedGeneration is the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.condition[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance. + format: int64 + type: integer + reason: + description: Reason for the condition's last transition. Reasons are upper CamelCase (PascalCase) with no spaces. A reason is always provided, this field will not be empty. + type: string + severity: + description: Severity with which to treat failures of this type of condition. For conditions which have positive polarity (Status == True is their normal/healthy state), this will be omitted when Status == True For conditions which have negative polarity (Status == False is their normal/healthy state), this will be omitted when Status == False. This is omitted in all cases when Status == Unknown + type: string + status: + description: Status of the condition, one of True, False, or Unknown. + type: string + type: + description: Type of condition. + type: string + required: + - lastTransitionTime + - reason + - status + - type + type: object + type: array + eTag: + description: 'ETag: If eTag is provided in the response body, it may also be provided as a header per the normal etag convention. Entity tags are used for comparing two or more entities from the same requested resource. HTTP/1.1 uses entity tags in the etag (section 14.19), If-Match (section 14.24), If-None-Match (section 14.26), and If-Range (section 14.27) header fields.' + type: string + group: + description: 'Group: The group this member belongs to for multi-cluster update management.' + type: string + id: + description: 'Id: Fully qualified resource ID for the resource. Ex - /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/{resourceProviderNamespace}/{resourceType}/{resourceName}' + type: string + name: + description: 'Name: The name of the resource' + type: string + provisioningState: + description: 'ProvisioningState: The status of the last operation.' + type: string + systemData: + description: 'SystemData: Azure Resource Manager metadata containing createdBy and modifiedBy information.' + properties: + createdAt: + description: 'CreatedAt: The timestamp of resource creation (UTC).' + type: string + createdBy: + description: 'CreatedBy: The identity that created the resource.' + type: string + createdByType: + description: 'CreatedByType: The type of identity that created the resource.' + type: string + lastModifiedAt: + description: 'LastModifiedAt: The timestamp of resource last modification (UTC)' + type: string + lastModifiedBy: + description: 'LastModifiedBy: The identity that last modified the resource.' + type: string + lastModifiedByType: + description: 'LastModifiedByType: The type of identity that last modified the resource.' + type: string + type: object + type: + description: 'Type: The type of the resource. E.g. "Microsoft.Compute/virtualMachines" or "Microsoft.Storage/storageAccounts"' + type: string + type: object + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - jsonPath: .status.conditions[?(@.type=='Ready')].status + name: Ready + type: string + - jsonPath: .status.conditions[?(@.type=='Ready')].severity + name: Severity + type: string + - jsonPath: .status.conditions[?(@.type=='Ready')].reason + name: Reason + type: string + - jsonPath: .status.conditions[?(@.type=='Ready')].message + name: Message + type: string + name: v1api20230315previewstorage + schema: + openAPIV3Schema: + description: 'Storage version of v1api20230315preview.FleetsMember Generator information: - Generated from: /containerservice/resource-manager/Microsoft.ContainerService/fleet/preview/2023-03-15-preview/fleets.json - ARM URI: /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.ContainerService/fleets/{fleetName}/members/{fleetMemberName}' + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Storage version of v1api20230315preview.Fleets_Member_Spec + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + azureName: + description: 'AzureName: The name of the resource in Azure. This is often the same as the name of the resource in Kubernetes but it doesn''t have to be.' + type: string + clusterResourceReference: + description: 'ClusterResourceReference: The ARM resource id of the cluster that joins the Fleet. Must be a valid Azure resource id. e.g.: ''/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.ContainerService/managedClusters/{clusterName}''.' + properties: + armId: + description: ARMID is a string of the form /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/{resourceProviderNamespace}/{resourceType}/{resourceName}. The /resourcegroups/{resourceGroupName} bit is optional as some resources are scoped at the subscription level ARMID is mutually exclusive with Group, Kind, Namespace and Name. + pattern: (?i)(^(/subscriptions/([^/]+)(/resourcegroups/([^/]+))?)?/providers/([^/]+)/([^/]+/[^/]+)(/([^/]+/[^/]+))*$|^/subscriptions/([^/]+)(/resourcegroups/([^/]+))?$) + type: string + group: + description: Group is the Kubernetes group of the resource. + type: string + kind: + description: Kind is the Kubernetes kind of the resource. + type: string + name: + description: Name is the Kubernetes name of the resource. + type: string + type: object + group: + type: string + originalVersion: + type: string + owner: + description: 'Owner: The owner of the resource. The owner controls where the resource goes when it is deployed. The owner also controls the resources lifecycle. When the owner is deleted the resource will also be deleted. Owner is expected to be a reference to a containerservice.azure.com/Fleet resource' + properties: + armId: + pattern: (?i)(^(/subscriptions/([^/]+)(/resourcegroups/([^/]+))?)?/providers/([^/]+)/([^/]+/[^/]+)(/([^/]+/[^/]+))*$|^/subscriptions/([^/]+)(/resourcegroups/([^/]+))?$) + type: string + name: + description: This is the name of the Kubernetes resource to reference. + type: string + type: object + required: + - clusterResourceReference + - owner + type: object + status: + description: Storage version of v1api20230315preview.Fleets_Member_STATUS + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + clusterResourceId: + type: string + conditions: + items: + description: Condition defines an extension to status (an observation) of a resource + properties: + lastTransitionTime: + description: LastTransitionTime is the last time the condition transitioned from one status to another. + format: date-time + type: string + message: + description: Message is a human readable message indicating details about the transition. This field may be empty. + type: string + observedGeneration: + description: ObservedGeneration is the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.condition[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance. + format: int64 + type: integer + reason: + description: Reason for the condition's last transition. Reasons are upper CamelCase (PascalCase) with no spaces. A reason is always provided, this field will not be empty. + type: string + severity: + description: Severity with which to treat failures of this type of condition. For conditions which have positive polarity (Status == True is their normal/healthy state), this will be omitted when Status == True For conditions which have negative polarity (Status == False is their normal/healthy state), this will be omitted when Status == False. This is omitted in all cases when Status == Unknown + type: string + status: + description: Status of the condition, one of True, False, or Unknown. + type: string + type: + description: Type of condition. + type: string + required: + - lastTransitionTime + - reason + - status + - type + type: object + type: array + eTag: + type: string + group: + type: string + id: + type: string + name: + type: string + provisioningState: + type: string + systemData: + description: Storage version of v1api20230315preview.SystemData_STATUS Metadata pertaining to creation and last modification of the resource. + properties: + $propertyBag: + additionalProperties: + type: string + description: PropertyBag is an unordered set of stashed information that used for properties not directly supported by storage resources, allowing for full fidelity round trip conversions + type: object + createdAt: + type: string + createdBy: + type: string + createdByType: + type: string + lastModifiedAt: + type: string + lastModifiedBy: + type: string + lastModifiedByType: + type: string + type: object + type: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition metadata: annotations: cert-manager.io/inject-ca-from: azureserviceoperator-system/azureserviceoperator-serving-cert diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml index 345f8dfd7bb..66a944752b7 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml @@ -257,6 +257,28 @@ spec: DNS service. It must be within the Kubernetes service address range specified in serviceCidr. Immutable. type: string + fleetsMember: + description: "FleetsMember is the spec for the fleet this cluster + is a member of. See also [AKS doc]. \n [AKS doc]: https://learn.microsoft.com/en-us/azure/templates/microsoft.containerservice/2023-03-15-preview/fleets/members" + properties: + group: + description: Group is the group this member belongs to for multi-cluster + update management. + type: string + managerName: + description: ManagerName is the name of the fleet manager. + type: string + managerResourceGroup: + description: ManagerResourceGroup is the resource group of the + fleet manager. + type: string + name: + description: Name is the name of the member. + type: string + required: + - managerName + - managerResourceGroup + type: object httpProxyConfig: description: HTTPProxyConfig is the HTTP proxy configuration for the cluster. Immutable. diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanetemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanetemplates.yaml index f92a346175f..758e659ce85 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanetemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanetemplates.yaml @@ -260,6 +260,26 @@ spec: Kubernetes DNS service. It must be within the Kubernetes service address range specified in serviceCidr. Immutable. type: string + fleetsMember: + description: "FleetsMember is the spec for the fleet this + cluster is a member of. See also [AKS doc]. \n [AKS doc]: + https://learn.microsoft.com/en-us/azure/templates/microsoft.containerservice/2023-03-15-preview/fleets/members" + properties: + group: + description: Group is the group this member belongs to + for multi-cluster update management. + type: string + managerName: + description: ManagerName is the name of the fleet manager. + type: string + managerResourceGroup: + description: ManagerResourceGroup is the resource group + of the fleet manager. + type: string + required: + - managerName + - managerResourceGroup + type: object httpProxyConfig: description: HTTPProxyConfig is the HTTP proxy configuration for the cluster. Immutable. diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 13e63db5d17..ad06984359f 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -85,6 +85,26 @@ rules: - get - list - watch +- apiGroups: + - containerservice.azure.com + resources: + - fleetsmembers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - containerservice.azure.com + resources: + - fleetsmembers/status + verbs: + - get + - list + - watch - apiGroups: - containerservice.azure.com resources: diff --git a/controllers/azuremanagedcontrolplane_controller.go b/controllers/azuremanagedcontrolplane_controller.go index edc001b6031..d8559614d83 100644 --- a/controllers/azuremanagedcontrolplane_controller.go +++ b/controllers/azuremanagedcontrolplane_controller.go @@ -118,6 +118,8 @@ func (amcpr *AzureManagedControlPlaneReconciler) SetupWithManager(ctx context.Co // +kubebuilder:rbac:groups=containerservice.azure.com,resources=managedclusters/status,verbs=get;list;watch // +kubebuilder:rbac:groups=network.azure.com,resources=privateendpoints;virtualnetworks;virtualnetworkssubnets,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=network.azure.com,resources=privateendpoints/status;virtualnetworks/status;virtualnetworkssubnets/status,verbs=get;list;watch +// +kubebuilder:rbac:groups=containerservice.azure.com,resources=fleetsmembers,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=containerservice.azure.com,resources=fleetsmembers/status,verbs=get;list;watch // Reconcile idempotently gets, creates, and updates a managed control plane. func (amcpr *AzureManagedControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Result, reterr error) { diff --git a/controllers/azuremanagedcontrolplane_controller_test.go b/controllers/azuremanagedcontrolplane_controller_test.go index a25edb6365a..8073063c3b1 100644 --- a/controllers/azuremanagedcontrolplane_controller_test.go +++ b/controllers/azuremanagedcontrolplane_controller_test.go @@ -20,6 +20,7 @@ import ( "context" "testing" + asocontainerservicev1preview "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20230315preview" asocontainerservicev1 "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20231001" asonetworkv1 "github.com/Azure/azure-service-operator/v2/api/network/v1api20201101" asoresourcesv1 "github.com/Azure/azure-service-operator/v2/api/resources/v1api20200601" @@ -108,6 +109,7 @@ func TestAzureManagedControlPlaneReconcilePaused(t *testing.T) { asocontainerservicev1.AddToScheme, asonetworkv1.AddToScheme, corev1.AddToScheme, + asocontainerservicev1preview.AddToScheme, ) s := runtime.NewScheme() g.Expect(sb.AddToScheme(s)).To(Succeed()) @@ -187,6 +189,11 @@ func TestAzureManagedControlPlaneReconcilePaused(t *testing.T) { }, }, }, + FleetsMember: &infrav1.FleetsMemberClassSpec{ + Group: "fleets", + ManagerName: "fleets-manager", + ManagerResourceGroup: "fleets-manager-rg", + }, IdentityRef: &corev1.ObjectReference{ Name: "fake-identity", Namespace: "default", @@ -222,6 +229,14 @@ func TestAzureManagedControlPlaneReconcilePaused(t *testing.T) { } g.Expect(c.Create(ctx, vnet)).To(Succeed()) + fleetsMember := &asocontainerservicev1preview.FleetsMember{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + } + g.Expect(c.Create(ctx, fleetsMember)).To(Succeed()) + subnet := &asonetworkv1.VirtualNetworksSubnet{ ObjectMeta: metav1.ObjectMeta{ Name: name + "-subnet", diff --git a/controllers/azuremanagedcontrolplane_reconciler.go b/controllers/azuremanagedcontrolplane_reconciler.go index bd8ca504844..14f5f8660da 100644 --- a/controllers/azuremanagedcontrolplane_reconciler.go +++ b/controllers/azuremanagedcontrolplane_reconciler.go @@ -24,6 +24,7 @@ import ( "k8s.io/client-go/tools/clientcmd" "sigs.k8s.io/cluster-api-provider-azure/azure" "sigs.k8s.io/cluster-api-provider-azure/azure/scope" + "sigs.k8s.io/cluster-api-provider-azure/azure/services/fleetsmembers" "sigs.k8s.io/cluster-api-provider-azure/azure/services/groups" "sigs.k8s.io/cluster-api-provider-azure/azure/services/managedclusters" "sigs.k8s.io/cluster-api-provider-azure/azure/services/privateendpoints" @@ -58,6 +59,7 @@ func newAzureManagedControlPlaneReconciler(scope *scope.ManagedControlPlaneScope subnets.New(scope), managedclusters.New(scope), privateendpoints.New(scope), + fleetsmembers.New(scope), resourceHealthSvc, }, }, nil diff --git a/docs/book/src/topics/managedcluster.md b/docs/book/src/topics/managedcluster.md index fb4f533c62f..f6fa3842867 100644 --- a/docs/book/src/topics/managedcluster.md +++ b/docs/book/src/topics/managedcluster.md @@ -277,6 +277,31 @@ User can do that by adding the Service Principal to the appropriate group define add the corresponding group ID in `spec.aadProfile.adminGroupObjectIDs`. CAPI and CAPZ will be able to authenticate via AAD while accessing the target cluster. +### AKS Fleet Integration + +CAPZ supports joining your managed AKS clusters to a single AKS fleet. Azure Kubernetes Fleet Manager (Fleet) enables at-scale management of multiple Azure Kubernetes Service (AKS) clusters. For more documentation on Azure Kubernetes Fleet Manager, refer [AKS Docs](https://learn.microsoft.com/azure/kubernetes-fleet/overview) + +To join a CAPZ cluster to an AKS fleet, you must first create an AKS fleet manager. For more information on how to create an AKS fleet manager, refer [AKS Docs](https://learn.microsoft.com/en-us/azure/kubernetes-fleet/quickstart-create-fleet-and-members). This fleet manager will be your point of reference for managing any CAPZ clusters that you join to the fleet. + +Once you have created an AKS fleet manager, you can join your CAPZ cluster to the fleet by adding the `fleetsMember` field to your AzureManagedControlPlane resource spec: + +```yaml +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedControlPlane +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + fleetsMember: + group: fleet-update-group + managerName: fleet-manager-name + managerResourceGroup: fleet-manager-resource-group +``` + +The `managerName` and `managerResourceGroup` fields are the name and resource group of your AKS fleet manager. The `group` field is the name of the update group for the cluster, not to be confused with the resource group. + +When the `fleetMember` field is included, CAPZ will create an AKS fleet member resource which will join the CAPZ cluster to the AKS fleet. The AKS fleet member resource will be created in the same resource group as the CAPZ cluster. + ## Features AKS clusters deployed from CAPZ currently only support a limited, diff --git a/go.mod b/go.mod index 99724314e22..0af6d7b6d8b 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v2 v2.2.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.4.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4 v4.6.0 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservicefleet/armcontainerservicefleet v1.1.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/monitor/armmonitor v0.11.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v1.2.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4 v4.3.0 diff --git a/go.sum b/go.sum index c8d1764194a..699e1dc0f5c 100644 --- a/go.sum +++ b/go.sum @@ -56,6 +56,8 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5 v5.4 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice v1.0.0 h1:figxyQZXzZQIcP3njhC68bYUiTw45J8/SsHaLW8Ax0M= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4 v4.6.0 h1:AAIdAyPkFff6XTct2lQCxOWN/+LnA41S7kIkzKaMbyE= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4 v4.6.0/go.mod h1:noQIdW75SiQFB3mSFJBr4iRRH83S9skaFiBv4C0uEs0= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservicefleet/armcontainerservicefleet v1.1.0 h1:cv2cjvdh2V6AQynbMGUFMSivtqM04zH+PL8A9iK9IWk= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservicefleet/armcontainerservicefleet v1.1.0/go.mod h1:F2Ad7qGpAbSYv2AoupVMoc+JBpd5gMMY9V/NAIv4+48= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/cosmos/armcosmos v1.0.0 h1:Fv8iibGn1eSw0lt2V3cTsuokBEnOP+M//n8OiMcCgTM= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/eventgrid/armeventgrid v1.0.0 h1:w6b0+FygDpqM7g5cjbeyPoBzgxVHwwt2vCUvTz1oFY8= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0 h1:PTFGRSlMKCQelWwxUyYVEUqseBJVemLyqWJjvMyt0do= diff --git a/main.go b/main.go index 2baa1861fac..0852b54f0e6 100644 --- a/main.go +++ b/main.go @@ -26,6 +26,7 @@ import ( "time" // +kubebuilder:scaffold:imports + asocontainerservicev1preview "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20230315preview" asocontainerservicev1 "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20231001" asonetworkv1api20201101 "github.com/Azure/azure-service-operator/v2/api/network/v1api20201101" asonetworkv1api20220701 "github.com/Azure/azure-service-operator/v2/api/network/v1api20220701" @@ -79,6 +80,7 @@ func init() { _ = asocontainerservicev1.AddToScheme(scheme) _ = asonetworkv1api20220701.AddToScheme(scheme) _ = asonetworkv1api20201101.AddToScheme(scheme) + _ = asocontainerservicev1preview.AddToScheme(scheme) // +kubebuilder:scaffold:scheme } @@ -309,6 +311,7 @@ func main() { }), EventBroadcaster: broadcaster, }) + if err != nil { setupLog.Error(err, "unable to start manager") os.Exit(1) diff --git a/test/e2e/aks_fleets_member.go b/test/e2e/aks_fleets_member.go new file mode 100644 index 00000000000..66ae51ae0b7 --- /dev/null +++ b/test/e2e/aks_fleets_member.go @@ -0,0 +1,152 @@ +//go:build e2e +// +build e2e + +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "os" + + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservicefleet/armcontainerservicefleet" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" + "sigs.k8s.io/cluster-api-provider-azure/azure" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/cluster-api/util/conditions" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type AKSFleetsMemberInput struct { + Cluster *clusterv1.Cluster + WaitIntervals []interface{} +} + +const ( + fleetName = "capz-aks-fleets-manager" + updateGroupName = "capz-aks-fleets-member-update" +) + +func AKSFleetsMemberSpec(ctx context.Context, inputGetter func() AKSFleetsMemberInput) { + input := inputGetter() + + cred, err := azidentity.NewDefaultAzureCredential(nil) + Expect(err).NotTo(HaveOccurred()) + + mgmtClient := bootstrapClusterProxy.GetClient() + Expect(mgmtClient).NotTo(BeNil()) + + containerserviceClient, err := armcontainerservice.NewManagedClustersClient(getSubscriptionID(Default), cred, nil) + Expect(err).NotTo(HaveOccurred()) + + amcp := &infrav1.AzureManagedControlPlane{} + err = mgmtClient.Get(ctx, types.NamespacedName{ + Namespace: input.Cluster.Spec.ControlPlaneRef.Namespace, + Name: input.Cluster.Spec.ControlPlaneRef.Name, + }, amcp) + Expect(err).NotTo(HaveOccurred()) + + groupClient, err := armresources.NewResourceGroupsClient(getSubscriptionID(Default), cred, nil) + Expect(err).NotTo(HaveOccurred()) + + By("Creating a resource group") + groupName := "capz-aks-fleets-member-" + amcp.Spec.ResourceGroupName + _, err = groupClient.CreateOrUpdate(ctx, groupName, armresources.ResourceGroup{ + Location: ptr.To(os.Getenv(AzureLocation)), + Tags: map[string]*string{ + "jobName": ptr.To(os.Getenv(JobName)), + "creationTimestamp": ptr.To(os.Getenv(Timestamp)), + }, + }, nil) + Expect(err).NotTo(HaveOccurred()) + + fleetClient, err := armcontainerservicefleet.NewFleetsClient(getSubscriptionID(Default), cred, nil) + Expect(err).NotTo(HaveOccurred()) + + fleetsMemberClient, err := armcontainerservicefleet.NewFleetMembersClient(getSubscriptionID(Default), cred, nil) + Expect(err).NotTo(HaveOccurred()) + + By("Creating a fleet manager") + poller, err := fleetClient.BeginCreateOrUpdate(ctx, groupName, fleetName, armcontainerservicefleet.Fleet{ + Location: ptr.To(os.Getenv(AzureLocation)), + }, nil) + Expect(err).NotTo(HaveOccurred()) + _, err = poller.PollUntilDone(ctx, nil) + Expect(err).NotTo(HaveOccurred()) + + By("Joining the cluster to the fleet hub") + var infraControlPlane = &infrav1.AzureManagedControlPlane{} + Eventually(func(g Gomega) { + err = mgmtClient.Get(ctx, client.ObjectKey{Namespace: input.Cluster.Spec.ControlPlaneRef.Namespace, Name: input.Cluster.Spec.ControlPlaneRef.Name}, infraControlPlane) + g.Expect(err).NotTo(HaveOccurred()) + infraControlPlane.Spec.FleetsMember = &infrav1.FleetsMember{ + FleetsMemberClassSpec: infrav1.FleetsMemberClassSpec{ + ManagerName: fleetName, + ManagerResourceGroup: groupName, + Group: updateGroupName, + }, + } + g.Expect(mgmtClient.Update(ctx, infraControlPlane)).To(Succeed()) + g.Expect(conditions.IsTrue(infraControlPlane, infrav1.FleetReadyCondition)).To(BeTrue()) + }, input.WaitIntervals...).Should(Succeed()) + + By("Ensuring the fleet member is created and attached to the managed cluster") + Eventually(func(g Gomega) { + resp, err := fleetsMemberClient.Get(ctx, groupName, fleetName, input.Cluster.Name, nil) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(resp.Properties.ProvisioningState).To(Equal(ptr.To(armcontainerservicefleet.FleetMemberProvisioningStateSucceeded))) + fleetsMember := resp.FleetMember + g.Expect(fleetsMember.Properties).NotTo(BeNil()) + expectedID := azure.ManagedClusterID(getSubscriptionID(Default), infraControlPlane.Spec.ResourceGroupName, input.Cluster.Name) + g.Expect(fleetsMember.Properties.ClusterResourceID).To(Equal(ptr.To(expectedID))) + g.Expect(fleetsMember.Properties.ProvisioningState).To(Equal(ptr.To(armcontainerservicefleet.FleetMemberProvisioningStateSucceeded))) + }, input.WaitIntervals...).Should(Succeed()) + + By("Remove the FleetsMember spec from the AzureManagedControlPlane") + Eventually(func(g Gomega) { + err = mgmtClient.Get(ctx, client.ObjectKey{Namespace: input.Cluster.Spec.ControlPlaneRef.Namespace, Name: input.Cluster.Spec.ControlPlaneRef.Name}, infraControlPlane) + g.Expect(err).NotTo(HaveOccurred()) + infraControlPlane.Spec.FleetsMember = nil + g.Expect(mgmtClient.Update(ctx, infraControlPlane)).To(Succeed()) + }, input.WaitIntervals...).Should(Succeed()) + + By("Waiting for the managed cluster to finish updating") + Eventually(func(g Gomega) { + aks, err := containerserviceClient.Get(ctx, amcp.Spec.ResourceGroupName, amcp.Name, nil) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(aks.ManagedCluster.Properties.ProvisioningState).NotTo(Equal(ptr.To("Updating"))) + }, input.WaitIntervals...).Should(Succeed()) + + By("Deleting the fleets member") + fleetsMemberPoller, err := fleetsMemberClient.BeginDelete(ctx, groupName, fleetName, input.Cluster.Name, nil) + Expect(err).NotTo(HaveOccurred()) + _, err = fleetsMemberPoller.PollUntilDone(ctx, nil) + Expect(err).NotTo(HaveOccurred()) + + Logf("Deleting the fleet manager resource group %q", groupName) + grpPoller, err := groupClient.BeginDelete(ctx, groupName, nil) + Expect(err).NotTo(HaveOccurred()) + _, err = grpPoller.PollUntilDone(ctx, nil) + Expect(err).NotTo(HaveOccurred()) +} diff --git a/test/e2e/azure_test.go b/test/e2e/azure_test.go index 6a54343749f..3178862263a 100644 --- a/test/e2e/azure_test.go +++ b/test/e2e/azure_test.go @@ -678,6 +678,15 @@ var _ = Describe("Workload cluster creation", func() { }), ), result) + By("attaching the cluster to azure fleet", func() { + AKSFleetsMemberSpec(ctx, func() AKSFleetsMemberInput { + return AKSFleetsMemberInput{ + Cluster: result.Cluster, + WaitIntervals: e2eConfig.GetIntervals(specName, "wait-machine-pool-nodes"), + } + }) + }) + By("Upgrading the Kubernetes version of the cluster", func() { AKSUpgradeSpec(ctx, func() AKSUpgradeSpecInput { return AKSUpgradeSpecInput{