Chained upgrades

Author: fabriziopandini

api/core/v1beta1/conversion.go

@@ -248,6 +248,8 @@ func (src *ClusterClass) ConvertTo(dstRaw conversion.Hub) error {
 		dst.Status.Variables[i] = variable
 	}
 
+	dst.Spec.KubernetesVersions = restored.Spec.KubernetesVersions
+
 	return nil
 }
 

api/core/v1beta1/zz_generated.conversion.go

@@ -135,6 +135,18 @@ type ClusterClassSpec struct {
 	// +kubebuilder:validation:MinItems=1
 	// +kubebuilder:validation:MaxItems=1000
 	Patches []ClusterClassPatch `json:"patches,omitempty"`
+
+	// kubernetesVersions is the list of Kubernetes versions that can be
+	// used for clusters using this ClusterClass.
+	// The list of version must be ordered from the older to the newer version, and there should be
+	// at least one version for every minor in between the first and the last version.
+	// +optional
+	// +listType=atomic
+	// +kubebuilder:validation:MinItems=1
+	// +kubebuilder:validation:MaxItems=100
+	// +kubebuilder:validation:items:MinLength=1
+	// +kubebuilder:validation:items:MaxLength=256
+	KubernetesVersions []string `json:"kubernetesVersions,omitempty"`
 }
 
 // InfrastructureClass defines the class for the infrastructure cluster.

api/core/v1beta2/clusterclass_types.go

@@ -36,6 +36,10 @@ const (
 	// to track the name of the MachineDeployment topology it represents.
 	ClusterTopologyMachineDeploymentNameLabel = "topology.cluster.x-k8s.io/deployment-name"
 
+	// ClusterTopologyControlPlaneUpgradeStepAnnotation tracks version current upgrade step.
+	// NOTE: The annotation exists only during upgrades.
+	ClusterTopologyControlPlaneUpgradeStepAnnotation = "topology.cluster.x-k8s.io/upgrade-step"
+
 	// ClusterTopologyHoldUpgradeSequenceAnnotation can be used to hold the entire MachineDeployment upgrade sequence.
 	// If the annotation is set on a MachineDeployment topology in Cluster.spec.topology.workers, the Kubernetes upgrade
 	// for this MachineDeployment topology and all subsequent ones is deferred.

api/core/v1beta2/common_types.go

@@ -185,7 +185,16 @@ func (r *KubeadmControlPlaneReconciler) preflightChecks(ctx context.Context, con
 
 	if feature.Gates.Enabled(feature.ClusterTopology) {
 		// Block when we expect an upgrade to be propagated for topology clusters.
-		if controlPlane.Cluster.Spec.Topology.IsDefined() && controlPlane.Cluster.Spec.Topology.Version != controlPlane.KCP.Spec.Version {
+		// NOTE: in case the cluster is performing an upgrade, allow creation of machines for the intermediate step.
+		isUpgradeForVersion := false
+		if versions, ok := controlPlane.Cluster.GetAnnotations()[clusterv1.ClusterTopologyControlPlaneUpgradeStepAnnotation]; ok {
+			v := strings.Split(versions, ",")
+			if len(v) > 0 {
+				isUpgradeForVersion = strings.TrimSpace(v[0]) == controlPlane.KCP.Spec.Version
+			}
+		}
+
+		if controlPlane.Cluster.Spec.Topology.IsDefined() && controlPlane.Cluster.Spec.Topology.Version != controlPlane.KCP.Spec.Version && !isUpgradeForVersion {
 			logger.Info(fmt.Sprintf("Waiting for a version upgrade to %s to be propagated from Cluster.spec.topology", controlPlane.Cluster.Spec.Topology.Version))
 			controlPlane.PreflightCheckResults.TopologyVersionMismatch = true
 			return ctrl.Result{RequeueAfter: preflightFailedRequeueAfter}, nil

api/core/v1beta2/zz_generated.deepcopy.go

@@ -562,6 +562,37 @@ func TestPreflightChecks(t *testing.T) {
 				TopologyVersionMismatch:          true,
 			},
 		},
+		{
+			name: "control plane with a pending upgrade, but not yet at the current step of the upgrade plan, should requeue",
+			cluster: &clusterv1.Cluster{
+				ObjectMeta: metav1.ObjectMeta{
+					Annotations: map[string]string{
+						clusterv1.ClusterTopologyControlPlaneUpgradeStepAnnotation: "v1.32.0",
+					},
+				},
+				Spec: clusterv1.ClusterSpec{
+					Topology: clusterv1.Topology{
+						Version: "v1.33.0",
+					},
+				},
+			},
+			kcp: &controlplanev1.KubeadmControlPlane{
+				Spec: controlplanev1.KubeadmControlPlaneSpec{
+					Version: "v1.31.0",
+				},
+			},
+			machines: []*clusterv1.Machine{
+				{},
+			},
+
+			expectResult: ctrl.Result{RequeueAfter: preflightFailedRequeueAfter},
+			expectPreflight: internal.PreflightCheckResults{
+				HasDeletingMachine:               false,
+				ControlPlaneComponentsNotHealthy: false,
+				EtcdClusterNotHealthy:            false,
+				TopologyVersionMismatch:          true,
+			},
+		},
 		{
 			name: "control plane with a deleting machine should requeue",
 			kcp:  &controlplanev1.KubeadmControlPlane{},
@@ -687,6 +718,55 @@ func TestPreflightChecks(t *testing.T) {
 				TopologyVersionMismatch:          false,
 			},
 		},
+		{
+			name: "control plane with a pending upgrade, but already at the current step of the upgrade plan, should pass",
+			cluster: &clusterv1.Cluster{
+				ObjectMeta: metav1.ObjectMeta{
+					Annotations: map[string]string{
+						clusterv1.ClusterTopologyControlPlaneUpgradeStepAnnotation: "v1.32.0",
+					},
+				},
+				Spec: clusterv1.ClusterSpec{
+					Topology: clusterv1.Topology{
+						Version: "v1.33.0",
+					},
+				},
+			},
+			kcp: &controlplanev1.KubeadmControlPlane{
+				Spec: controlplanev1.KubeadmControlPlaneSpec{
+					Version: "v1.32.0",
+				}, Status: controlplanev1.KubeadmControlPlaneStatus{
+					Conditions: []metav1.Condition{
+						{Type: controlplanev1.KubeadmControlPlaneControlPlaneComponentsHealthyCondition, Status: metav1.ConditionTrue},
+						{Type: controlplanev1.KubeadmControlPlaneEtcdClusterHealthyCondition, Status: metav1.ConditionTrue},
+					},
+				},
+			},
+			machines: []*clusterv1.Machine{
+				{
+					Status: clusterv1.MachineStatus{
+						NodeRef: clusterv1.MachineNodeReference{
+							Name: "node-1",
+						},
+						Conditions: []metav1.Condition{
+							{Type: controlplanev1.KubeadmControlPlaneMachineAPIServerPodHealthyCondition, Status: metav1.ConditionTrue},
+							{Type: controlplanev1.KubeadmControlPlaneMachineControllerManagerPodHealthyCondition, Status: metav1.ConditionTrue},
+							{Type: controlplanev1.KubeadmControlPlaneMachineSchedulerPodHealthyCondition, Status: metav1.ConditionTrue},
+							{Type: controlplanev1.KubeadmControlPlaneMachineEtcdPodHealthyCondition, Status: metav1.ConditionTrue},
+							{Type: controlplanev1.KubeadmControlPlaneMachineEtcdMemberHealthyCondition, Status: metav1.ConditionTrue},
+						},
+					},
+				},
+			},
+
+			expectResult: ctrl.Result{},
+			expectPreflight: internal.PreflightCheckResults{
+				HasDeletingMachine:               false,
+				ControlPlaneComponentsNotHealthy: false,
+				EtcdClusterNotHealthy:            false,
+				TopologyVersionMismatch:          false,
+			},
+		},
 	}
 
 	for _, tt := range testCases {