diff --git a/exp/topology/desiredstate/desired_state.go b/exp/topology/desiredstate/desired_state.go index 9e4a817f62a3..7b414fa3f580 100644 --- a/exp/topology/desiredstate/desired_state.go +++ b/exp/topology/desiredstate/desired_state.go @@ -37,6 +37,7 @@ import ( runtimehooksv1 "sigs.k8s.io/cluster-api/api/runtime/hooks/v1alpha1" "sigs.k8s.io/cluster-api/controllers/clustercache" "sigs.k8s.io/cluster-api/controllers/external" + runtimecatalog "sigs.k8s.io/cluster-api/exp/runtime/catalog" runtimeclient "sigs.k8s.io/cluster-api/exp/runtime/client" "sigs.k8s.io/cluster-api/exp/topology/scope" "sigs.k8s.io/cluster-api/feature" @@ -552,23 +553,29 @@ func (g *generator) computeControlPlaneVersion(ctx context.Context, s *scope.Sco } // if the control plane is not upgrading, before making further considerations about if to pick up another version, - // we should call the AfterControlPlaneUpgrade hook if not already done. + // we should call the AfterControlPlaneUpgrade and the BeforeWorkersUpgrade hooks if not already done. if feature.Gates.Enabled(feature.RuntimeSDK) { - hookCompleted, err := g.callAfterControlPlaneUpgradeHook(ctx, s, currentVersion, topologyVersion) + // Note: calling the AfterControlPlaneUpgrade is the final step of a control plane upgrade. + hookCompleted, err := g.callAfterControlPlaneUpgradeHook(ctx, s, currentVersion) if err != nil { return "", err } if !hookCompleted { return *currentVersion, nil } - } - - // At this stage, we can assume the previous control plane upgrade is fully complete (including calling the AfterControlPlaneUpgrade). - // It is now possible to start making considerations if to pick up another version. - // If the control plane is not pending upgrade, then it is already at the desired version and there is no other version to pick up. - if !s.UpgradeTracker.ControlPlane.IsPendingUpgrade { - return *currentVersion, nil + // Note: calling the BeforeWorkersUpgrade is the first part of the execution of a worker upgrade step from the upgrade plan. + // The call to this hook is implemented in this function in order to ensure the hook is called + // after AfterControlPlaneUpgrade unblocks, and also to ensure that BeforeWorkersUpgrade + // can block the control plane upgrade to proceed in the upgrade plan. + // Note: this operation is a no-op if workers are not required to upgrade to the current control plane version. + hookCompleted, err = g.callBeforeWorkersUpgradeHook(ctx, s, &s.UpgradeTracker.MinWorkersVersion, *currentVersion) + if err != nil { + return "", err + } + if !hookCompleted { + return *currentVersion, nil + } } // Before considering picking up the next control plane version, check if workers are required @@ -595,8 +602,35 @@ func (g *generator) computeControlPlaneVersion(ctx context.Context, s *scope.Sco // At this point we can assume the control plane is stable and also MachineDeployments/MachinePools // are not upgrading/are not required to upgrade. + + // If not already done, call the AfterWorkersUpgrade hook before picking up the desired version. + // (this is the last step of the previous upgrade). + if feature.Gates.Enabled(feature.RuntimeSDK) { + // Note: calling the AfterWorkersUpgrade is the last step of workers upgrade. + // The call to this hook is implemented in this function in order to ensure that AfterWorkersUpgrade + // can block the control plane upgrade to proceed in the upgrade plan. + // Note: this operation is a no-op if workers are not required to upgrade to the current control plane version. + hookCompleted, err := g.callAfterWorkersUpgradeHook(ctx, s, currentVersion) + if err != nil { + return "", err + } + if !hookCompleted { + return *currentVersion, nil + } + } + + // At this stage, we can assume the previous control plane upgrade is fully complete (including calling the AfterControlPlaneUpgrade). + // It is now possible to start making considerations if to pick up another version. + + // If the control plane is not pending upgrade, then it is already at the desired version and there is no other version to pick up. + if !s.UpgradeTracker.ControlPlane.IsPendingUpgrade { + return *currentVersion, nil + } + // If not already done, call the BeforeClusterUpgrade hook before picking up the desired version. if feature.Gates.Enabled(feature.RuntimeSDK) { + // Note: calling the BeforeClusterUpgrade is the first step of an upgrade plan; + // this operation is a no-op for intermediate steps of an upgrade plan. hookCompleted, err := g.callBeforeClusterUpgradeHook(ctx, s, currentVersion, topologyVersion) if err != nil { return "", err @@ -604,22 +638,51 @@ func (g *generator) computeControlPlaneVersion(ctx context.Context, s *scope.Sco if !hookCompleted { return *currentVersion, nil } + + // After BeforeClusterUpgrade unblocked the upgrade, consider the upgrade started. + // As a consequence, the system start tracking the intent of calling AfterClusterUpgrade once the upgrade is complete. + // Note: this also prevent the BeforeClusterUpgrade to be called again (until after the upgrade is completed). + if err := hooks.MarkAsPending(ctx, g.Client, s.Current.Cluster, runtimehooksv1.AfterClusterUpgrade); err != nil { + return "", err + } } - // Control plane and machine deployments are stable. All the required hooks are called. + // Control plane and machine deployments are stable. The BeforeClusterUpgrade hook have been called. // Ready to pick up the next version in the upgrade plan. - // Track the intent of calling the AfterControlPlaneUpgrade and the AfterClusterUpgrade hooks once we are done with the upgrade. - if err := hooks.MarkAsPending(ctx, g.Client, s.Current.Cluster, runtimehooksv1.AfterControlPlaneUpgrade, runtimehooksv1.AfterClusterUpgrade); err != nil { - return "", err - } - - // Pick up the new version + // Select the next version for the control plane if len(s.UpgradeTracker.ControlPlane.UpgradePlan) == 0 { return "", errors.New("cannot compute the control plane version if the control plane is pending upgrade and the upgrade plan is not set") } nextVersion := s.UpgradeTracker.ControlPlane.UpgradePlan[0] + if feature.Gates.Enabled(feature.RuntimeSDK) { + // Note: calling the BeforeControlPlaneUpgrade is the first step of a control plan upgrade step from the upgrade plan. + hookCompleted, err := g.callBeforeControlPlaneUpgradeHook(ctx, s, currentVersion, nextVersion) + if err != nil { + return "", err + } + if !hookCompleted { + return *currentVersion, nil + } + + // After BeforeControlPlaneUpgrade unblocked the upgrade step, consider the upgrade step start started, + // As a consequence, the system start tracking the intent of calling other hooks for this upgrade step: + // - AfterControlPlaneUpgrade hook to be called after the control plane completes the upgrade step. + // - If workers are required to upgrade to the current control plane version: + // - BeforeWorkersUpgrade hook to be called before workers start the upgrade step. + // - AfterWorkersUpgrade hook to be called after workers completes the upgrade step. + hooksToBeCalled := []runtimecatalog.Hook{runtimehooksv1.AfterControlPlaneUpgrade} + machineDeploymentPendingUpgrade := len(s.UpgradeTracker.MachineDeployments.UpgradePlan) > 0 && s.UpgradeTracker.MachineDeployments.UpgradePlan[0] == nextVersion + machinePoolPendingUpgrade := len(s.UpgradeTracker.MachinePools.UpgradePlan) > 0 && s.UpgradeTracker.MachinePools.UpgradePlan[0] == nextVersion + if machineDeploymentPendingUpgrade || machinePoolPendingUpgrade { + hooksToBeCalled = append(hooksToBeCalled, runtimehooksv1.BeforeWorkersUpgrade, runtimehooksv1.AfterWorkersUpgrade) + } + if err := hooks.MarkAsPending(ctx, g.Client, s.Current.Cluster, hooksToBeCalled...); err != nil { + return "", err + } + } + // The upgrade is now starting in this reconcile and not pending anymore. // Note: it is important to unset IsPendingUpgrade, otherwise reconcileState will assume that we are still waiting for another upgrade (and thus defer the one we are starting). s.UpgradeTracker.ControlPlane.IsStartingUpgrade = true @@ -979,7 +1042,7 @@ func (g *generator) computeMachineDeploymentVersion(s *scope.Scope, machineDeplo // Example: join could fail if the load balancers are slow in detecting when CP machines are // being deleted. if currentMDState == nil || currentMDState.Object == nil { - if !s.UpgradeTracker.ControlPlane.IsControlPlaneStable() || s.HookResponseTracker.IsBlocking(runtimehooksv1.AfterControlPlaneUpgrade) { + if !s.UpgradeTracker.ControlPlane.IsControlPlaneStable() || s.HookResponseTracker.IsBlocking(runtimehooksv1.AfterControlPlaneUpgrade) || s.HookResponseTracker.IsBlocking(runtimehooksv1.BeforeWorkersUpgrade) { s.UpgradeTracker.MachineDeployments.MarkPendingCreate(machineDeploymentTopology.Name) } return topologyVersion, nil @@ -1007,6 +1070,12 @@ func (g *generator) computeMachineDeploymentVersion(s *scope.Scope, machineDeplo return currentVersion, nil } + // Return early if the BeforeWorkersUpgrade hook returns a blocking response. + if s.HookResponseTracker.IsBlocking(runtimehooksv1.BeforeWorkersUpgrade) { + s.UpgradeTracker.MachineDeployments.MarkPendingUpgrade(currentMDState.Object.Name) + return currentVersion, nil + } + // Return early if the upgrade concurrency is reached. if s.UpgradeTracker.MachineDeployments.UpgradeConcurrencyReached() { s.UpgradeTracker.MachineDeployments.MarkPendingUpgrade(currentMDState.Object.Name) @@ -1293,7 +1362,7 @@ func (g *generator) computeMachinePoolVersion(s *scope.Scope, machinePoolTopolog // Example: join could fail if the load balancers are slow in detecting when CP machines are // being deleted. if currentMPState == nil || currentMPState.Object == nil { - if !s.UpgradeTracker.ControlPlane.IsControlPlaneStable() || s.HookResponseTracker.IsBlocking(runtimehooksv1.AfterControlPlaneUpgrade) { + if !s.UpgradeTracker.ControlPlane.IsControlPlaneStable() || s.HookResponseTracker.IsBlocking(runtimehooksv1.AfterControlPlaneUpgrade) || s.HookResponseTracker.IsBlocking(runtimehooksv1.BeforeWorkersUpgrade) { s.UpgradeTracker.MachinePools.MarkPendingCreate(machinePoolTopology.Name) } return topologyVersion, nil @@ -1321,6 +1390,12 @@ func (g *generator) computeMachinePoolVersion(s *scope.Scope, machinePoolTopolog return currentVersion, nil } + // Return early if the BeforeWorkersUpgrade hook returns a blocking response. + if s.HookResponseTracker.IsBlocking(runtimehooksv1.BeforeWorkersUpgrade) { + s.UpgradeTracker.MachinePools.MarkPendingUpgrade(currentMPState.Object.Name) + return currentVersion, nil + } + // Return early if the upgrade concurrency is reached. if s.UpgradeTracker.MachinePools.UpgradeConcurrencyReached() { s.UpgradeTracker.MachinePools.MarkPendingUpgrade(currentMPState.Object.Name) diff --git a/exp/topology/desiredstate/desired_state_test.go b/exp/topology/desiredstate/desired_state_test.go index 1edecdfc6889..37fc927aefda 100644 --- a/exp/topology/desiredstate/desired_state_test.go +++ b/exp/topology/desiredstate/desired_state_test.go @@ -1089,9 +1089,93 @@ func TestComputeControlPlaneVersion(t *testing.T) { }, } + beforeControlPlaneUpgradeGVH, err := catalog.GroupVersionHook(runtimehooksv1.BeforeControlPlaneUpgrade) + if err != nil { + panic("unable to compute GVH") + } + nonBlockingBeforeControlPlaneUpgradeResponse := &runtimehooksv1.BeforeControlPlaneUpgradeResponse{ + CommonRetryResponse: runtimehooksv1.CommonRetryResponse{ + CommonResponse: runtimehooksv1.CommonResponse{ + Status: runtimehooksv1.ResponseStatusSuccess, + }, + }, + } + blockingBeforeControlPlaneUpgradeResponse := &runtimehooksv1.BeforeControlPlaneUpgradeResponse{ + CommonRetryResponse: runtimehooksv1.CommonRetryResponse{ + CommonResponse: runtimehooksv1.CommonResponse{ + Status: runtimehooksv1.ResponseStatusSuccess, + }, + RetryAfterSeconds: int32(10), + }, + } + failureBeforeControlPlaneUpgradeResponse := &runtimehooksv1.BeforeControlPlaneUpgradeResponse{ + CommonRetryResponse: runtimehooksv1.CommonRetryResponse{ + CommonResponse: runtimehooksv1.CommonResponse{ + Status: runtimehooksv1.ResponseStatusFailure, + }, + }, + } + + beforeWorkersUpgradeGVH, err := catalog.GroupVersionHook(runtimehooksv1.BeforeWorkersUpgrade) + if err != nil { + panic("unable to compute GVH") + } + nonBlockingBeforeWorkersUpgradeResponse := &runtimehooksv1.BeforeWorkersUpgradeResponse{ + CommonRetryResponse: runtimehooksv1.CommonRetryResponse{ + CommonResponse: runtimehooksv1.CommonResponse{ + Status: runtimehooksv1.ResponseStatusSuccess, + }, + }, + } + blockingBeforeWorkersUpgradeResponse := &runtimehooksv1.BeforeWorkersUpgradeResponse{ + CommonRetryResponse: runtimehooksv1.CommonRetryResponse{ + CommonResponse: runtimehooksv1.CommonResponse{ + Status: runtimehooksv1.ResponseStatusSuccess, + }, + RetryAfterSeconds: int32(10), + }, + } + failureBeforeWorkersUpgradeResponse := &runtimehooksv1.BeforeWorkersUpgradeResponse{ + CommonRetryResponse: runtimehooksv1.CommonRetryResponse{ + CommonResponse: runtimehooksv1.CommonResponse{ + Status: runtimehooksv1.ResponseStatusFailure, + }, + }, + } + + afterWorkersUpgradeGVH, err := catalog.GroupVersionHook(runtimehooksv1.AfterWorkersUpgrade) + if err != nil { + panic("unable to compute GVH") + } + nonBlockingAfterWorkersUpgradeResponse := &runtimehooksv1.AfterWorkersUpgradeResponse{ + CommonRetryResponse: runtimehooksv1.CommonRetryResponse{ + CommonResponse: runtimehooksv1.CommonResponse{ + Status: runtimehooksv1.ResponseStatusSuccess, + }, + }, + } + blockingAfterWorkersUpgradeResponse := &runtimehooksv1.AfterWorkersUpgradeResponse{ + CommonRetryResponse: runtimehooksv1.CommonRetryResponse{ + CommonResponse: runtimehooksv1.CommonResponse{ + Status: runtimehooksv1.ResponseStatusSuccess, + }, + RetryAfterSeconds: int32(10), + }, + } + failureAfterWorkersUpgradeResponse := &runtimehooksv1.AfterWorkersUpgradeResponse{ + CommonRetryResponse: runtimehooksv1.CommonRetryResponse{ + CommonResponse: runtimehooksv1.CommonResponse{ + Status: runtimehooksv1.ResponseStatusFailure, + }, + }, + } + tests := []struct { name string beforeClusterUpgradeResponse *runtimehooksv1.BeforeClusterUpgradeResponse + beforeControlPlaneUpgradeResponse *runtimehooksv1.BeforeControlPlaneUpgradeResponse + beforeWorkersUpgradeResponse *runtimehooksv1.BeforeWorkersUpgradeResponse + afterWorkersUpgradeResponse *runtimehooksv1.AfterWorkersUpgradeResponse topologyVersion string clusterModifier func(c *clusterv1.Cluster) controlPlaneObj *unstructured.Unstructured @@ -1171,9 +1255,12 @@ func TestComputeControlPlaneVersion(t *testing.T) { expectedIsStartingUpgrade: false, }, { - name: "should return cluster.spec.topology.version if control plane is not upgrading and not scaling and none of the MachineDeployments and MachinePools are upgrading - BeforeClusterUpgrade hook returns non blocking response", - beforeClusterUpgradeResponse: nonBlockingBeforeClusterUpgradeResponse, - topologyVersion: "v1.2.3", + name: "should return cluster.spec.topology.version if control plane is not upgrading and not scaling and none of the MachineDeployments and MachinePools are upgrading - BeforeClusterUpgrade, BeforeControlPlaneUpgrade, BeforeWorkersUpgrade and AfterWorkersUpgrade hooks returns non blocking response", + beforeClusterUpgradeResponse: nonBlockingBeforeClusterUpgradeResponse, + beforeControlPlaneUpgradeResponse: nonBlockingBeforeControlPlaneUpgradeResponse, + beforeWorkersUpgradeResponse: nonBlockingBeforeWorkersUpgradeResponse, + afterWorkersUpgradeResponse: nonBlockingAfterWorkersUpgradeResponse, + topologyVersion: "v1.2.3", controlPlaneObj: builder.ControlPlane("test1", "cp1"). WithSpecFields(map[string]interface{}{ "spec.version": "v1.2.2", @@ -1187,6 +1274,11 @@ func TestComputeControlPlaneVersion(t *testing.T) { "status.unavailableReplicas": int64(0), }). Build(), + clusterModifier: func(c *clusterv1.Cluster) { + c.Annotations = map[string]string{ + runtimev1.PendingHooksAnnotation: "BeforeWorkersUpgrade,AfterWorkersUpgrade", + } + }, controlPlaneUpgradePlan: []string{"v1.2.3"}, upgradingMachineDeployments: []string{}, upgradingMachinePools: []string{}, @@ -1195,9 +1287,10 @@ func TestComputeControlPlaneVersion(t *testing.T) { expectedIsStartingUpgrade: true, }, { - name: "should return cluster.spec.topology.version if the control plane is not upgrading or scaling and none of the MachineDeployments and MachinePools are upgrading - BeforeClusterUpgrade, BeforeControlPlaneUpgrade, BeforeWorkersUpgrade and AfterWorkersUpgrade hooks returns non blocking response", - beforeClusterUpgradeResponse: nonBlockingBeforeClusterUpgradeResponse, - topologyVersion: "v1.2.3", + name: "should return cluster.spec.topology.version if the control plane is not upgrading or scaling and none of the MachineDeployments and MachinePools are upgrading - BeforeClusterUpgrade, BeforeControlPlaneUpgrade hooks returns non blocking response", + beforeClusterUpgradeResponse: nonBlockingBeforeClusterUpgradeResponse, + beforeControlPlaneUpgradeResponse: nonBlockingBeforeControlPlaneUpgradeResponse, + topologyVersion: "v1.2.3", controlPlaneObj: builder.ControlPlane("test1", "cp1"). WithSpecFields(map[string]interface{}{ "spec.version": "v1.2.2", @@ -1211,20 +1304,16 @@ func TestComputeControlPlaneVersion(t *testing.T) { "status.unavailableReplicas": int64(0), }). Build(), - clusterModifier: func(c *clusterv1.Cluster) { - c.Annotations = map[string]string{ - runtimev1.PendingHooksAnnotation: "AfterWorkersUpgrade", - } - }, controlPlaneUpgradePlan: []string{"v1.2.3"}, expectedVersion: "v1.2.3", expectedIsPendingUpgrade: false, expectedIsStartingUpgrade: true, }, { - name: "should return an intermediate version when upgrading by more than 1 minor and control plane should perform the first step of the upgrade sequence", - beforeClusterUpgradeResponse: nonBlockingBeforeClusterUpgradeResponse, - topologyVersion: "v1.5.3", + name: "should return an intermediate version when upgrading by more than 1 minor and control plane should perform the first step of the upgrade sequence", + beforeClusterUpgradeResponse: nonBlockingBeforeClusterUpgradeResponse, + beforeControlPlaneUpgradeResponse: nonBlockingBeforeControlPlaneUpgradeResponse, + topologyVersion: "v1.5.3", controlPlaneObj: builder.ControlPlane("test1", "cp1"). WithSpecFields(map[string]interface{}{ "spec.version": "v1.2.2", @@ -1246,9 +1335,10 @@ func TestComputeControlPlaneVersion(t *testing.T) { expectedIsStartingUpgrade: true, }, { - name: "should return cluster.spec.topology.version when performing a multi step upgrade and control plane is at the second last minor in the upgrade sequence", - beforeClusterUpgradeResponse: nonBlockingBeforeClusterUpgradeResponse, - topologyVersion: "v1.5.3", + name: "should return cluster.spec.topology.version when performing a multi step upgrade and control plane is at the second last minor in the upgrade sequence", + beforeClusterUpgradeResponse: nonBlockingBeforeClusterUpgradeResponse, + beforeControlPlaneUpgradeResponse: nonBlockingBeforeControlPlaneUpgradeResponse, + topologyVersion: "v1.5.3", controlPlaneObj: builder.ControlPlane("test1", "cp1"). WithSpecFields(map[string]interface{}{ "spec.version": "v1.4.2", @@ -1391,6 +1481,162 @@ func TestComputeControlPlaneVersion(t *testing.T) { expectedIsStartingUpgrade: false, wantErr: false, }, + { + name: "should return the controlplane.spec.version if a BeforeControlPlaneUpgrade returns a blocking response", + beforeClusterUpgradeResponse: nonBlockingBeforeClusterUpgradeResponse, + beforeControlPlaneUpgradeResponse: blockingBeforeControlPlaneUpgradeResponse, + topologyVersion: "v1.2.3", + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.2.2", + "spec.replicas": int64(2), + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.2.2", + "status.replicas": int64(2), + "status.updatedReplicas": int64(2), + "status.readyReplicas": int64(2), + "status.unavailableReplicas": int64(0), + }). + Build(), + controlPlaneUpgradePlan: []string{"v1.2.3"}, + expectedVersion: "v1.2.2", + expectedIsPendingUpgrade: true, + expectedIsStartingUpgrade: false, + }, + { + name: "should fail if the BeforeControlPlaneUpgrade hooks returns a failure response", + beforeClusterUpgradeResponse: nonBlockingBeforeClusterUpgradeResponse, + beforeControlPlaneUpgradeResponse: failureBeforeControlPlaneUpgradeResponse, + topologyVersion: "v1.2.3", + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.2.2", + "spec.replicas": int64(2), + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.2.2", + "status.replicas": int64(2), + "status.updatedReplicas": int64(2), + "status.readyReplicas": int64(2), + "status.unavailableReplicas": int64(0), + }). + Build(), + controlPlaneUpgradePlan: []string{"v1.2.3"}, + wantErr: true, + }, + { + name: "should return the controlplane.spec.version if a AfterWorkersUpgrade returns a blocking response", + beforeClusterUpgradeResponse: nonBlockingBeforeClusterUpgradeResponse, + beforeControlPlaneUpgradeResponse: nonBlockingBeforeControlPlaneUpgradeResponse, + afterWorkersUpgradeResponse: blockingAfterWorkersUpgradeResponse, + topologyVersion: "v1.2.3", + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.2.2", + "spec.replicas": int64(2), + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.2.2", + "status.replicas": int64(2), + "status.updatedReplicas": int64(2), + "status.readyReplicas": int64(2), + "status.unavailableReplicas": int64(0), + }). + Build(), + clusterModifier: func(c *clusterv1.Cluster) { + c.Annotations = map[string]string{ + runtimev1.PendingHooksAnnotation: "AfterWorkersUpgrade", + } + }, + controlPlaneUpgradePlan: []string{"v1.2.3"}, + expectedVersion: "v1.2.2", + expectedIsPendingUpgrade: true, + expectedIsStartingUpgrade: false, + }, + { + name: "should fail if the AfterWorkersUpgrade hooks returns a failure response", + beforeClusterUpgradeResponse: nonBlockingBeforeClusterUpgradeResponse, + beforeControlPlaneUpgradeResponse: nonBlockingBeforeControlPlaneUpgradeResponse, + afterWorkersUpgradeResponse: failureAfterWorkersUpgradeResponse, + topologyVersion: "v1.2.3", + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.2.2", + "spec.replicas": int64(2), + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.2.2", + "status.replicas": int64(2), + "status.updatedReplicas": int64(2), + "status.readyReplicas": int64(2), + "status.unavailableReplicas": int64(0), + }). + Build(), + clusterModifier: func(c *clusterv1.Cluster) { + c.Annotations = map[string]string{ + runtimev1.PendingHooksAnnotation: "AfterWorkersUpgrade", + } + }, + controlPlaneUpgradePlan: []string{"v1.2.3"}, + wantErr: true, + }, + { + name: "should return the controlplane.spec.version if a BeforeWorkersUpgrade returns a blocking response", + beforeClusterUpgradeResponse: nonBlockingBeforeClusterUpgradeResponse, + beforeControlPlaneUpgradeResponse: nonBlockingBeforeControlPlaneUpgradeResponse, + beforeWorkersUpgradeResponse: blockingBeforeWorkersUpgradeResponse, + topologyVersion: "v1.2.3", + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.2.2", + "spec.replicas": int64(2), + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.2.2", + "status.replicas": int64(2), + "status.updatedReplicas": int64(2), + "status.readyReplicas": int64(2), + "status.unavailableReplicas": int64(0), + }). + Build(), + clusterModifier: func(c *clusterv1.Cluster) { + c.Annotations = map[string]string{ + runtimev1.PendingHooksAnnotation: "BeforeWorkersUpgrade", + } + }, + controlPlaneUpgradePlan: []string{"v1.2.3"}, + expectedVersion: "v1.2.2", + expectedIsPendingUpgrade: true, + expectedIsStartingUpgrade: false, + }, + { + name: "should fail if the BeforeWorkersUpgrade hooks returns a failure response", + beforeClusterUpgradeResponse: nonBlockingBeforeClusterUpgradeResponse, + beforeControlPlaneUpgradeResponse: nonBlockingBeforeControlPlaneUpgradeResponse, + beforeWorkersUpgradeResponse: failureBeforeWorkersUpgradeResponse, + topologyVersion: "v1.2.3", + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.2.2", + "spec.replicas": int64(2), + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.2.2", + "status.replicas": int64(2), + "status.updatedReplicas": int64(2), + "status.readyReplicas": int64(2), + "status.unavailableReplicas": int64(0), + }). + Build(), + clusterModifier: func(c *clusterv1.Cluster) { + c.Annotations = map[string]string{ + runtimev1.PendingHooksAnnotation: "BeforeWorkersUpgrade", + } + }, + controlPlaneUpgradePlan: []string{"v1.2.3"}, + wantErr: true, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -1459,7 +1705,10 @@ func TestComputeControlPlaneVersion(t *testing.T) { runtimeClient := fakeruntimeclient.NewRuntimeClientBuilder(). WithCatalog(catalog). WithCallAllExtensionResponses(map[runtimecatalog.GroupVersionHook]runtimehooksv1.ResponseObject{ - beforeClusterUpgradeGVH: tt.beforeClusterUpgradeResponse, + beforeClusterUpgradeGVH: tt.beforeClusterUpgradeResponse, + beforeControlPlaneUpgradeGVH: tt.beforeControlPlaneUpgradeResponse, + beforeWorkersUpgradeGVH: tt.beforeWorkersUpgradeResponse, + afterWorkersUpgradeGVH: tt.afterWorkersUpgradeResponse, }). WithCallAllExtensionValidations(validateClusterParameter(s.Current.Cluster)). Build() @@ -2496,6 +2745,7 @@ func TestComputeMachineDeploymentVersion(t *testing.T) { controlPlaneUpgrading bool controlPlaneProvisioning bool afterControlPlaneUpgradeHookBlocking bool + beforeWorkersUpgradeHookBlocking bool topologyVersion string upgradePlan []string expectedVersion string @@ -2621,6 +2871,17 @@ func TestComputeMachineDeploymentVersion(t *testing.T) { expectedVersion: "v1.2.2", expectPendingUpgrade: true, }, + { + name: "should return machine deployment's spec.template.spec.version if control plane is stable, other machine deployments are upgrading, concurrency limit not reached but BeforeWorkersUpgrade hook is blocking", + currentMachineDeploymentState: currentMachineDeploymentState, + upgradingMachineDeployments: []string{"upgrading-md1"}, + upgradeConcurrency: 2, + beforeWorkersUpgradeHookBlocking: true, + topologyVersion: "v1.2.3", + upgradePlan: []string{"v1.2.3"}, + expectedVersion: "v1.2.2", + expectPendingUpgrade: true, + }, { name: "should return cluster.spec.topology.version if control plane is stable, other machine deployments are upgrading, concurrency limit not reached", currentMachineDeploymentState: currentMachineDeploymentState, @@ -2671,6 +2932,13 @@ func TestComputeMachineDeploymentVersion(t *testing.T) { }, }) } + if tt.beforeWorkersUpgradeHookBlocking { + s.HookResponseTracker.Add(runtimehooksv1.BeforeWorkersUpgrade, &runtimehooksv1.BeforeWorkersUpgradeResponse{ + CommonRetryResponse: runtimehooksv1.CommonRetryResponse{ + RetryAfterSeconds: 10, + }, + }) + } s.UpgradeTracker.ControlPlane.IsStartingUpgrade = tt.controlPlaneStartingUpgrade s.UpgradeTracker.ControlPlane.IsUpgrading = tt.controlPlaneUpgrading s.UpgradeTracker.ControlPlane.IsProvisioning = tt.controlPlaneProvisioning @@ -2722,6 +2990,7 @@ func TestComputeMachinePoolVersion(t *testing.T) { controlPlaneUpgrading bool controlPlaneProvisioning bool afterControlPlaneUpgradeHookBlocking bool + beforeWorkersUpgradeHookBlocking bool topologyVersion string upgradePlan []string expectedVersion string @@ -2847,6 +3116,17 @@ func TestComputeMachinePoolVersion(t *testing.T) { expectedVersion: "v1.2.2", expectPendingUpgrade: true, }, + { + name: "should return MachinePool's spec.template.spec.version if control plane is stable, other MachinePools are upgrading, concurrency limit not reached but BeforeWorkersUpgrade hook is blocking", + currentMachinePoolState: currentMachinePoolState, + upgradingMachinePools: []string{"upgrading-mp1"}, + upgradeConcurrency: 2, + beforeWorkersUpgradeHookBlocking: true, + topologyVersion: "v1.2.3", + upgradePlan: []string{"v1.2.3"}, + expectedVersion: "v1.2.2", + expectPendingUpgrade: true, + }, { name: "should return cluster.spec.topology.version if control plane is stable, other MachinePools are upgrading, concurrency limit not reached", currentMachinePoolState: currentMachinePoolState, @@ -2894,6 +3174,13 @@ func TestComputeMachinePoolVersion(t *testing.T) { }, }) } + if tt.beforeWorkersUpgradeHookBlocking { + s.HookResponseTracker.Add(runtimehooksv1.BeforeWorkersUpgrade, &runtimehooksv1.BeforeWorkersUpgradeResponse{ + CommonRetryResponse: runtimehooksv1.CommonRetryResponse{ + RetryAfterSeconds: 10, + }, + }) + } s.UpgradeTracker.ControlPlane.IsStartingUpgrade = tt.controlPlaneStartingUpgrade s.UpgradeTracker.ControlPlane.IsUpgrading = tt.controlPlaneUpgrading s.UpgradeTracker.ControlPlane.IsProvisioning = tt.controlPlaneProvisioning diff --git a/exp/topology/desiredstate/lifecycle_hooks.go b/exp/topology/desiredstate/lifecycle_hooks.go index 4916ea00d8be..90eaa1e06f32 100644 --- a/exp/topology/desiredstate/lifecycle_hooks.go +++ b/exp/topology/desiredstate/lifecycle_hooks.go @@ -33,11 +33,12 @@ import ( "sigs.k8s.io/cluster-api/internal/hooks" ) +// callBeforeClusterUpgradeHook calls the BeforeClusterUpgrade at the beginning of an upgrade. +// NOTE: the hook should be called only at the beginning of an upgrade sequence (it should not be called when in the middle of a multistep upgrade sequence); +// to detect if we are at the beginning of an upgrade, the code checks if the intent to call the AfterClusterUpgrade is not yet tracked. func (g *generator) callBeforeClusterUpgradeHook(ctx context.Context, s *scope.Scope, currentVersion *string, topologyVersion string) (bool, error) { log := ctrl.LoggerFrom(ctx) - // NOTE: the hook should be called only at the beginning of either a regular upgrade or a multistep upgrade sequence (it should not be called when in the middle of a multistep upgrade sequence); - // to detect if we are at the beginning of an upgrade, we check if the intent to call the AfterClusterUpgrade is not yet tracked. if !hooks.IsPending(runtimehooksv1.AfterClusterUpgrade, s.Current.Cluster) { var hookAnnotations []string for key := range s.Current.Cluster.Annotations { @@ -64,7 +65,10 @@ func (g *generator) callBeforeClusterUpgradeHook(ctx context.Context, s *scope.S }, }) - log.Info(fmt.Sprintf("Cluster upgrade to version %q is blocked by %q hook (via annotations)", topologyVersion, runtimecatalog.HookName(runtimehooksv1.BeforeClusterUpgrade)), "hooks", strings.Join(hookAnnotations, ",")) + log.Info(fmt.Sprintf("Cluster upgrade from version %s to version %s is blocked by %s hook (via annotations)", *currentVersion, topologyVersion, runtimecatalog.HookName(runtimehooksv1.BeforeClusterUpgrade)), "hooks", strings.Join(hookAnnotations, ","), + "ControlPlaneUpgrades", toUpgradeStep(s.UpgradeTracker.ControlPlane.UpgradePlan), + "WorkersUpgrades", toUpgradeStep(s.UpgradeTracker.MachineDeployments.UpgradePlan, s.UpgradeTracker.MachinePools.UpgradePlan), + ) return false, nil } @@ -87,16 +91,63 @@ func (g *generator) callBeforeClusterUpgradeHook(ctx context.Context, s *scope.S } // Add the response to the tracker so we can later update condition or requeue when required. s.HookResponseTracker.Add(runtimehooksv1.BeforeClusterUpgrade, hookResponse) + if hookResponse.RetryAfterSeconds != 0 { // Cannot pickup the new version right now. Need to try again later. - log.Info(fmt.Sprintf("Cluster upgrade to version %q is blocked by %q hook", topologyVersion, runtimecatalog.HookName(runtimehooksv1.BeforeClusterUpgrade))) + log.Info(fmt.Sprintf("Cluster upgrade from version %s to version %s is blocked by %s hook", *currentVersion, topologyVersion, runtimecatalog.HookName(runtimehooksv1.BeforeClusterUpgrade)), + "ControlPlaneUpgrades", hookRequest.ControlPlaneUpgrades, + "WorkersUpgrades", hookRequest.WorkersUpgrades, + ) return false, nil } } return true, nil } -func (g *generator) callAfterControlPlaneUpgradeHook(ctx context.Context, s *scope.Scope, currentVersion *string, topologyVersion string) (bool, error) { +// callBeforeControlPlaneUpgradeHook calls the BeforeControlPlaneUpgrade before the control plane picks up a new control plane version, +// no matter if this is an intermediate versions of an upgrade plan or the target version of an upgrade plan. +// NOTE: when an upgrade starts, the hook should be called after the BeforeClusterUpgrade hook. +// NOTE: the hook doesn't need call intent tracking: it is always called before picking up a new control plane version. +func (g *generator) callBeforeControlPlaneUpgradeHook(ctx context.Context, s *scope.Scope, currentVersion *string, nextVersion string) (bool, error) { + log := ctrl.LoggerFrom(ctx) + + // NOTE: the hook should always be called before piking up a new version. + v1beta1Cluster := &clusterv1beta1.Cluster{} + // DeepCopy cluster because ConvertFrom has side effects like adding the conversion annotation. + if err := v1beta1Cluster.ConvertFrom(s.Current.Cluster.DeepCopy()); err != nil { + return false, errors.Wrap(err, "error converting Cluster to v1beta1 Cluster") + } + + hookRequest := &runtimehooksv1.BeforeControlPlaneUpgradeRequest{ + Cluster: *cleanupCluster(v1beta1Cluster), + FromKubernetesVersion: *currentVersion, + ToKubernetesVersion: nextVersion, + ControlPlaneUpgrades: toUpgradeStep(s.UpgradeTracker.ControlPlane.UpgradePlan), + WorkersUpgrades: toUpgradeStep(s.UpgradeTracker.MachineDeployments.UpgradePlan, s.UpgradeTracker.MachinePools.UpgradePlan), + } + hookResponse := &runtimehooksv1.BeforeControlPlaneUpgradeResponse{} + if err := g.RuntimeClient.CallAllExtensions(ctx, runtimehooksv1.BeforeControlPlaneUpgrade, s.Current.Cluster, hookRequest, hookResponse); err != nil { + return false, err + } + // Add the response to the tracker so we can later update condition or requeue when required. + s.HookResponseTracker.Add(runtimehooksv1.BeforeControlPlaneUpgrade, hookResponse) + + if hookResponse.RetryAfterSeconds != 0 { + // Cannot pickup the new version right now. Need to try again later. + log.Info(fmt.Sprintf("Control plane upgrade from version %s to version %s is blocked by %s hook", *currentVersion, nextVersion, runtimecatalog.HookName(runtimehooksv1.BeforeControlPlaneUpgrade)), + "ControlPlaneUpgrades", hookRequest.ControlPlaneUpgrades, + "WorkersUpgrades", hookRequest.WorkersUpgrades, + ) + return false, nil + } + + return true, nil +} + +// callAfterControlPlaneUpgradeHook calls the AfterControlPlaneUpgrade after the control plane upgrade is completed, +// no matter if this is an intermediate versions of an upgrade plan or the target version of an upgrade plan. +// NOTE: computeControlPlaneVersion records intent to call this hook when picking up a new control plane version. +func (g *generator) callAfterControlPlaneUpgradeHook(ctx context.Context, s *scope.Scope, currentVersion *string) (bool, error) { log := ctrl.LoggerFrom(ctx) // Call the hook only if we are tracking the intent to do so. If it is not tracked it means we don't need to call the @@ -122,15 +173,11 @@ func (g *generator) callAfterControlPlaneUpgradeHook(ctx context.Context, s *sco // Add the response to the tracker so we can later update condition or requeue when required. s.HookResponseTracker.Add(runtimehooksv1.AfterControlPlaneUpgrade, hookResponse) - // If the extension responds to hold off on starting Machine deployments upgrades, - // change the UpgradeTracker accordingly, otherwise the hook call is completed and we - // can remove this hook from the list of pending-hooks. if hookResponse.RetryAfterSeconds != 0 { - v := topologyVersion - if len(s.UpgradeTracker.ControlPlane.UpgradePlan) > 0 { - v = s.UpgradeTracker.ControlPlane.UpgradePlan[0] - } - log.Info(fmt.Sprintf("Upgrade to version %q is blocked by %q hook", v, runtimecatalog.HookName(runtimehooksv1.AfterControlPlaneUpgrade))) + log.Info(fmt.Sprintf("Cluster Upgrade is blocked after control plane upgrade to version %s by %s hook", *currentVersion, runtimecatalog.HookName(runtimehooksv1.AfterControlPlaneUpgrade)), + "ControlPlaneUpgrades", hookRequest.ControlPlaneUpgrades, + "WorkersUpgrades", hookRequest.WorkersUpgrades, + ) return false, nil } if err := hooks.MarkAsDone(ctx, g.Client, s.Current.Cluster, runtimehooksv1.AfterControlPlaneUpgrade); err != nil { @@ -140,6 +187,96 @@ func (g *generator) callAfterControlPlaneUpgradeHook(ctx context.Context, s *sco return true, nil } +// callBeforeWorkersUpgradeHook calls the BeforeWorkersUpgrade before workers starts picking up a new worker version, +// no matter if this is an intermediate versions of an upgrade plan or the target version of an upgrade plan. +// NOTE: computeControlPlaneVersion records intent to call this hook when picking up a new control plane version +// that exists also in the workers upgrade plan. +func (g *generator) callBeforeWorkersUpgradeHook(ctx context.Context, s *scope.Scope, currentVersion *string, nextVersion string) (bool, error) { + log := ctrl.LoggerFrom(ctx) + + // Call the hook only if we are tracking the intent to do so. If it is not tracked it means we don't need to call the + // hook because we didn't go through an upgrade or we already called the hook after the upgrade. + if hooks.IsPending(runtimehooksv1.BeforeWorkersUpgrade, s.Current.Cluster) { + v1beta1Cluster := &clusterv1beta1.Cluster{} + // DeepCopy cluster because ConvertFrom has side effects like adding the conversion annotation. + if err := v1beta1Cluster.ConvertFrom(s.Current.Cluster.DeepCopy()); err != nil { + return false, errors.Wrap(err, "error converting Cluster to v1beta1 Cluster") + } + + hookRequest := &runtimehooksv1.BeforeWorkersUpgradeRequest{ + Cluster: *cleanupCluster(v1beta1Cluster), + FromKubernetesVersion: *currentVersion, + ToKubernetesVersion: nextVersion, + ControlPlaneUpgrades: toUpgradeStep(s.UpgradeTracker.ControlPlane.UpgradePlan), + WorkersUpgrades: toUpgradeStep(s.UpgradeTracker.MachineDeployments.UpgradePlan, s.UpgradeTracker.MachinePools.UpgradePlan), + } + hookResponse := &runtimehooksv1.BeforeWorkersUpgradeResponse{} + if err := g.RuntimeClient.CallAllExtensions(ctx, runtimehooksv1.BeforeWorkersUpgrade, s.Current.Cluster, hookRequest, hookResponse); err != nil { + return false, err + } + // Add the response to the tracker so we can later update condition or requeue when required. + s.HookResponseTracker.Add(runtimehooksv1.BeforeWorkersUpgrade, hookResponse) + + if hookResponse.RetryAfterSeconds != 0 { + // Cannot pickup the new version right now. Need to try again later. + log.Info(fmt.Sprintf("Workers upgrade from version %s to version %s is blocked by %s hook", *currentVersion, nextVersion, runtimecatalog.HookName(runtimehooksv1.BeforeWorkersUpgrade)), + "ControlPlaneUpgrades", hookRequest.ControlPlaneUpgrades, + "WorkersUpgrades", hookRequest.WorkersUpgrades, + ) + return false, nil + } + if err := hooks.MarkAsDone(ctx, g.Client, s.Current.Cluster, runtimehooksv1.BeforeWorkersUpgrade); err != nil { + return false, err + } + } + + return true, nil +} + +// callAfterWorkersUpgradeHook calls the AfterWorkersUpgrade after the worker upgrade is completed, +// no matter if this is an intermediate versions of an upgrade plan or the target version of an upgrade plan. +// NOTE: computeControlPlaneVersion records intent to call this hook when picking up a new control plane version +// that exists also in the workers upgrade plan. +func (g *generator) callAfterWorkersUpgradeHook(ctx context.Context, s *scope.Scope, currentVersion *string) (bool, error) { + log := ctrl.LoggerFrom(ctx) + + // Call the hook only if we are tracking the intent to do so. If it is not tracked it means we don't need to call the + // hook because we didn't go through an upgrade or we already called the hook after the upgrade. + if hooks.IsPending(runtimehooksv1.AfterWorkersUpgrade, s.Current.Cluster) { + v1beta1Cluster := &clusterv1beta1.Cluster{} + // DeepCopy cluster because ConvertFrom has side effects like adding the conversion annotation. + if err := v1beta1Cluster.ConvertFrom(s.Current.Cluster.DeepCopy()); err != nil { + return false, errors.Wrap(err, "error converting Cluster to v1beta1 Cluster") + } + + // Call all the registered extension for the hook. + hookRequest := &runtimehooksv1.AfterWorkersUpgradeRequest{ + Cluster: *cleanupCluster(v1beta1Cluster), + KubernetesVersion: *currentVersion, + ControlPlaneUpgrades: toUpgradeStep(s.UpgradeTracker.ControlPlane.UpgradePlan), + WorkersUpgrades: toUpgradeStep(s.UpgradeTracker.MachineDeployments.UpgradePlan, s.UpgradeTracker.MachinePools.UpgradePlan), + } + hookResponse := &runtimehooksv1.AfterWorkersUpgradeResponse{} + if err := g.RuntimeClient.CallAllExtensions(ctx, runtimehooksv1.AfterWorkersUpgrade, s.Current.Cluster, hookRequest, hookResponse); err != nil { + return false, err + } + // Add the response to the tracker so we can later update condition or requeue when required. + s.HookResponseTracker.Add(runtimehooksv1.AfterWorkersUpgrade, hookResponse) + + if hookResponse.RetryAfterSeconds != 0 { + log.Info(fmt.Sprintf("Cluster upgrade is blocked after workers upgrade to version %s by %s hook", *currentVersion, runtimecatalog.HookName(runtimehooksv1.AfterWorkersUpgrade)), + "ControlPlaneUpgrades", hookRequest.ControlPlaneUpgrades, + "WorkersUpgrades", hookRequest.WorkersUpgrades, + ) + return false, nil + } + if err := hooks.MarkAsDone(ctx, g.Client, s.Current.Cluster, runtimehooksv1.AfterWorkersUpgrade); err != nil { + return false, err + } + } + return true, nil +} + // toUpgradeStep converts a list of version to a list of upgrade steps. // Note. when called for workers, the function will receive in input two plans one for the MachineDeployments if any, the other for MachinePools if any. // Considering that both plans, if defined, have to be equal, the function picks the first one not empty. diff --git a/exp/topology/desiredstate/lifecycle_hooks_test.go b/exp/topology/desiredstate/lifecycle_hooks_test.go index e8290a6473ec..0612cfc17ba1 100644 --- a/exp/topology/desiredstate/lifecycle_hooks_test.go +++ b/exp/topology/desiredstate/lifecycle_hooks_test.go @@ -93,6 +93,27 @@ func TestComputeControlPlaneVersion_LifecycleHooksSequences(t *testing.T) { }, } + beforeControlPlaneUpgradeGVH, err := catalog.GroupVersionHook(runtimehooksv1.BeforeControlPlaneUpgrade) + if err != nil { + panic("unable to compute GVH") + } + + blockingBeforeControlPlaneUpgradeResponse := &runtimehooksv1.BeforeControlPlaneUpgradeResponse{ + CommonRetryResponse: runtimehooksv1.CommonRetryResponse{ + CommonResponse: runtimehooksv1.CommonResponse{ + Status: runtimehooksv1.ResponseStatusSuccess, + }, + RetryAfterSeconds: int32(10), + }, + } + nonBlockingBeforeControlPlaneUpgradeResponse := &runtimehooksv1.BeforeControlPlaneUpgradeResponse{ + CommonRetryResponse: runtimehooksv1.CommonRetryResponse{ + CommonResponse: runtimehooksv1.CommonResponse{ + Status: runtimehooksv1.ResponseStatusSuccess, + }, + }, + } + afterControlPlaneUpgradeGVH, err := catalog.GroupVersionHook(runtimehooksv1.AfterControlPlaneUpgrade) if err != nil { panic("unable to compute GVH") @@ -114,25 +135,71 @@ func TestComputeControlPlaneVersion_LifecycleHooksSequences(t *testing.T) { }, } + beforeWorkersUpgradeGVH, err := catalog.GroupVersionHook(runtimehooksv1.BeforeWorkersUpgrade) + if err != nil { + panic("unable to compute GVH") + } + + blockingBeforeWorkersUpgradeResponse := &runtimehooksv1.BeforeWorkersUpgradeResponse{ + CommonRetryResponse: runtimehooksv1.CommonRetryResponse{ + CommonResponse: runtimehooksv1.CommonResponse{ + Status: runtimehooksv1.ResponseStatusSuccess, + }, + RetryAfterSeconds: int32(10), + }, + } + nonBlockingBeforeWorkersUpgradeResponse := &runtimehooksv1.BeforeWorkersUpgradeResponse{ + CommonRetryResponse: runtimehooksv1.CommonRetryResponse{ + CommonResponse: runtimehooksv1.CommonResponse{ + Status: runtimehooksv1.ResponseStatusSuccess, + }, + }, + } + + afterWorkersUpgradeGVH, err := catalog.GroupVersionHook(runtimehooksv1.AfterWorkersUpgrade) + if err != nil { + panic("unable to compute GVH") + } + blockingAfterWorkersUpgradeResponse := &runtimehooksv1.AfterWorkersUpgradeResponse{ + CommonRetryResponse: runtimehooksv1.CommonRetryResponse{ + CommonResponse: runtimehooksv1.CommonResponse{ + Status: runtimehooksv1.ResponseStatusSuccess, + }, + RetryAfterSeconds: int32(10), + }, + } + nonBlockingAfterWorkersUpgradeResponse := &runtimehooksv1.AfterWorkersUpgradeResponse{ + CommonRetryResponse: runtimehooksv1.CommonRetryResponse{ + CommonResponse: runtimehooksv1.CommonResponse{ + Status: runtimehooksv1.ResponseStatusSuccess, + }, + }, + } + tests := []struct { - name string - topologyVersion string - pendingHookAnnotation string - controlPlaneObj *unstructured.Unstructured - controlPlaneUpgradePlan []string - machineDeploymentsUpgradePlan []string - machinePoolsUpgradePlan []string - upgradingMachineDeployments []string - upgradingMachinePools []string - wantBeforeClusterUpgradeRequest *runtimehooksv1.BeforeClusterUpgradeRequest - beforeClusterUpgradeResponse *runtimehooksv1.BeforeClusterUpgradeResponse - wantAfterControlPlaneUpgradeRequest *runtimehooksv1.AfterControlPlaneUpgradeRequest - afterControlPlaneUpgradeResponse *runtimehooksv1.AfterControlPlaneUpgradeResponse - wantVersion string - wantIsPendingUpgrade bool - wantIsStartingUpgrade bool - wantIsWaitingForWorkersUpgrade bool - wantPendingHookAnnotation string + name string + topologyVersion string + pendingHookAnnotation string + controlPlaneObj *unstructured.Unstructured + controlPlaneUpgradePlan []string + minWorkersVersion string + machineDeploymentsUpgradePlan []string + machinePoolsUpgradePlan []string + wantBeforeClusterUpgradeRequest *runtimehooksv1.BeforeClusterUpgradeRequest + beforeClusterUpgradeResponse *runtimehooksv1.BeforeClusterUpgradeResponse + wantBeforeControlPlaneUpgradeRequest *runtimehooksv1.BeforeControlPlaneUpgradeRequest + beforeControlPlaneUpgradeResponse *runtimehooksv1.BeforeControlPlaneUpgradeResponse + wantAfterControlPlaneUpgradeRequest *runtimehooksv1.AfterControlPlaneUpgradeRequest + afterControlPlaneUpgradeResponse *runtimehooksv1.AfterControlPlaneUpgradeResponse + wantBeforeWorkersUpgradeRequest *runtimehooksv1.BeforeWorkersUpgradeRequest + beforeWorkersUpgradeResponse *runtimehooksv1.BeforeWorkersUpgradeResponse + wantAfterWorkersUpgradeRequest *runtimehooksv1.AfterWorkersUpgradeRequest + afterWorkersUpgradeResponse *runtimehooksv1.AfterWorkersUpgradeResponse + wantVersion string + wantIsPendingUpgrade bool + wantIsStartingUpgrade bool + wantIsWaitingForWorkersUpgrade bool + wantPendingHookAnnotation string }{ // Upgrade cluster with CP, MD, MP (upgrade by one minor) @@ -160,6 +227,7 @@ func TestComputeControlPlaneVersion_LifecycleHooksSequences(t *testing.T) { "status.version": "v1.2.2", }). Build(), + minWorkersVersion: "v1.2.2", controlPlaneUpgradePlan: []string{"v1.2.3"}, machineDeploymentsUpgradePlan: []string{"v1.2.3"}, machinePoolsUpgradePlan: []string{"v1.2.3"}, @@ -174,7 +242,7 @@ func TestComputeControlPlaneVersion_LifecycleHooksSequences(t *testing.T) { wantIsPendingUpgrade: true, }, { - name: "when an upgrade starts: pick up a new version when BeforeClusterUpgrade hook unblocks", + name: "when an upgrade starts: call the BeforeControlPlaneUpgrade hook when BeforeClusterUpgrade hook unblocks, blocking answer", topologyVersion: "v1.2.3", controlPlaneObj: builder.ControlPlane("test1", "cp1"). WithSpecFields(map[string]interface{}{ @@ -184,6 +252,7 @@ func TestComputeControlPlaneVersion_LifecycleHooksSequences(t *testing.T) { "status.version": "v1.2.2", }). Build(), + minWorkersVersion: "v1.2.2", controlPlaneUpgradePlan: []string{"v1.2.3"}, machineDeploymentsUpgradePlan: []string{"v1.2.3"}, machinePoolsUpgradePlan: []string{"v1.2.3"}, @@ -194,14 +263,48 @@ func TestComputeControlPlaneVersion_LifecycleHooksSequences(t *testing.T) { WorkersUpgrades: toUpgradeStep([]string{"v1.2.3"}), }, beforeClusterUpgradeResponse: nonBlockingBeforeClusterUpgradeResponse, - wantVersion: "v1.2.3", // changed from previous step - wantIsStartingUpgrade: true, - wantPendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade", // changed from previous step + wantBeforeControlPlaneUpgradeRequest: &runtimehooksv1.BeforeControlPlaneUpgradeRequest{ + FromKubernetesVersion: "v1.2.2", + ToKubernetesVersion: "v1.2.3", + ControlPlaneUpgrades: toUpgradeStep([]string{"v1.2.3"}), + WorkersUpgrades: toUpgradeStep([]string{"v1.2.3"}), + }, + beforeControlPlaneUpgradeResponse: blockingBeforeControlPlaneUpgradeResponse, + wantVersion: "v1.2.2", + wantIsPendingUpgrade: true, + wantPendingHookAnnotation: "AfterClusterUpgrade", // changed from previous step + }, + { + name: "when an upgrade starts: pick up a new version when BeforeControlPlaneUpgrade hook unblocks (does not call the BeforeClusterUpgrade hook when already done)", + topologyVersion: "v1.2.3", + pendingHookAnnotation: "AfterClusterUpgrade", + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.2.2", + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.2.2", + }). + Build(), + minWorkersVersion: "v1.2.2", + controlPlaneUpgradePlan: []string{"v1.2.3"}, + machineDeploymentsUpgradePlan: []string{"v1.2.3"}, + machinePoolsUpgradePlan: []string{"v1.2.3"}, + wantBeforeControlPlaneUpgradeRequest: &runtimehooksv1.BeforeControlPlaneUpgradeRequest{ + FromKubernetesVersion: "v1.2.2", + ToKubernetesVersion: "v1.2.3", + ControlPlaneUpgrades: toUpgradeStep([]string{"v1.2.3"}), + WorkersUpgrades: toUpgradeStep([]string{"v1.2.3"}), + }, + beforeControlPlaneUpgradeResponse: nonBlockingBeforeControlPlaneUpgradeResponse, + wantVersion: "v1.2.3", // changed from previous step + wantIsStartingUpgrade: true, + wantPendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade,AfterWorkersUpgrade,BeforeWorkersUpgrade", // changed from previous step }, { name: "when control plane is upgrading: do not call hooks", topologyVersion: "v1.2.3", - pendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade", + pendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade,AfterWorkersUpgrade,BeforeWorkersUpgrade", controlPlaneObj: builder.ControlPlane("test1", "cp1"). WithSpecFields(map[string]interface{}{ "spec.version": "v1.2.3", @@ -210,16 +313,17 @@ func TestComputeControlPlaneVersion_LifecycleHooksSequences(t *testing.T) { "status.version": "v1.2.2", }). Build(), + minWorkersVersion: "v1.2.2", controlPlaneUpgradePlan: []string{"v1.2.3"}, machineDeploymentsUpgradePlan: []string{"v1.2.3"}, machinePoolsUpgradePlan: []string{"v1.2.3"}, wantVersion: "v1.2.3", - wantPendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade", + wantPendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade,AfterWorkersUpgrade,BeforeWorkersUpgrade", }, { name: "after control plane is upgraded: call the AfterControlPlaneUpgrade hook, blocking answer", topologyVersion: "v1.2.3", - pendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade", + pendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade,AfterWorkersUpgrade,BeforeWorkersUpgrade", controlPlaneObj: builder.ControlPlane("test1", "cp1"). WithSpecFields(map[string]interface{}{ "spec.version": "v1.2.3", @@ -228,6 +332,7 @@ func TestComputeControlPlaneVersion_LifecycleHooksSequences(t *testing.T) { "status.version": "v1.2.3", // changed from previous step }). Build(), + minWorkersVersion: "v1.2.2", controlPlaneUpgradePlan: []string{}, machineDeploymentsUpgradePlan: []string{"v1.2.3"}, machinePoolsUpgradePlan: []string{"v1.2.3"}, @@ -238,12 +343,12 @@ func TestComputeControlPlaneVersion_LifecycleHooksSequences(t *testing.T) { }, afterControlPlaneUpgradeResponse: blockingAfterControlPlaneUpgradeResponse, wantVersion: "v1.2.3", - wantPendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade", + wantPendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade,AfterWorkersUpgrade,BeforeWorkersUpgrade", }, { - name: "after control plane is upgraded: AfterControlPlaneUpgrade hook unblocks", + name: "after control plane is upgraded: call the BeforeWorkersUpgrade hook when AfterControlPlaneUpgrade hook unblocks, blocking answer", topologyVersion: "v1.2.3", - pendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade", + pendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade,AfterWorkersUpgrade,BeforeWorkersUpgrade", controlPlaneObj: builder.ControlPlane("test1", "cp1"). WithSpecFields(map[string]interface{}{ "spec.version": "v1.2.3", @@ -252,6 +357,7 @@ func TestComputeControlPlaneVersion_LifecycleHooksSequences(t *testing.T) { "status.version": "v1.2.3", }). Build(), + minWorkersVersion: "v1.2.2", controlPlaneUpgradePlan: []string{}, machineDeploymentsUpgradePlan: []string{"v1.2.3"}, machinePoolsUpgradePlan: []string{"v1.2.3"}, @@ -261,13 +367,20 @@ func TestComputeControlPlaneVersion_LifecycleHooksSequences(t *testing.T) { WorkersUpgrades: toUpgradeStep([]string{"v1.2.3"}), }, afterControlPlaneUpgradeResponse: nonBlockingAfterControlPlaneUpgradeResponse, - wantVersion: "v1.2.3", - wantPendingHookAnnotation: "AfterClusterUpgrade", // changed from previous step + wantBeforeWorkersUpgradeRequest: &runtimehooksv1.BeforeWorkersUpgradeRequest{ + FromKubernetesVersion: "v1.2.2", + ToKubernetesVersion: "v1.2.3", + ControlPlaneUpgrades: toUpgradeStep([]string{}), + WorkersUpgrades: toUpgradeStep([]string{"v1.2.3"}), + }, + beforeWorkersUpgradeResponse: blockingBeforeWorkersUpgradeResponse, + wantVersion: "v1.2.3", + wantPendingHookAnnotation: "AfterClusterUpgrade,AfterWorkersUpgrade,BeforeWorkersUpgrade", // changed from previous step }, { - name: "when machine deployment are upgrading: do not call hooks", + name: "after control plane is upgraded: BeforeWorkersUpgrade hook unblocks (does not call the AfterControlPlaneUpgrade hook when already done)", topologyVersion: "v1.2.3", - pendingHookAnnotation: "AfterClusterUpgrade", + pendingHookAnnotation: "AfterClusterUpgrade,AfterWorkersUpgrade,BeforeWorkersUpgrade", controlPlaneObj: builder.ControlPlane("test1", "cp1"). WithSpecFields(map[string]interface{}{ "spec.version": "v1.2.3", @@ -276,16 +389,45 @@ func TestComputeControlPlaneVersion_LifecycleHooksSequences(t *testing.T) { "status.version": "v1.2.3", }). Build(), + minWorkersVersion: "v1.2.2", controlPlaneUpgradePlan: []string{}, machineDeploymentsUpgradePlan: []string{"v1.2.3"}, machinePoolsUpgradePlan: []string{"v1.2.3"}, - wantVersion: "v1.2.3", - wantPendingHookAnnotation: "AfterClusterUpgrade", + wantBeforeWorkersUpgradeRequest: &runtimehooksv1.BeforeWorkersUpgradeRequest{ + FromKubernetesVersion: "v1.2.2", + ToKubernetesVersion: "v1.2.3", + ControlPlaneUpgrades: toUpgradeStep([]string{}), + WorkersUpgrades: toUpgradeStep([]string{"v1.2.3"}), + }, + beforeWorkersUpgradeResponse: nonBlockingBeforeWorkersUpgradeResponse, + wantVersion: "v1.2.3", + wantIsWaitingForWorkersUpgrade: true, + wantPendingHookAnnotation: "AfterClusterUpgrade,AfterWorkersUpgrade", // changed from previous step + }, + { + name: "when machine deployment are upgrading: do not call hooks", + topologyVersion: "v1.2.3", + pendingHookAnnotation: "AfterClusterUpgrade,AfterWorkersUpgrade", + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.2.3", + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.2.3", + }). + Build(), + minWorkersVersion: "v1.2.2", + controlPlaneUpgradePlan: []string{}, + machineDeploymentsUpgradePlan: []string{"v1.2.3"}, + machinePoolsUpgradePlan: []string{"v1.2.3"}, + wantVersion: "v1.2.3", + wantIsWaitingForWorkersUpgrade: true, + wantPendingHookAnnotation: "AfterClusterUpgrade,AfterWorkersUpgrade", }, { name: "when machine pools are upgrading: do not call hooks", topologyVersion: "v1.2.3", - pendingHookAnnotation: "AfterClusterUpgrade", + pendingHookAnnotation: "AfterClusterUpgrade,AfterWorkersUpgrade", controlPlaneObj: builder.ControlPlane("test1", "cp1"). WithSpecFields(map[string]interface{}{ "spec.version": "v1.2.3", @@ -294,11 +436,63 @@ func TestComputeControlPlaneVersion_LifecycleHooksSequences(t *testing.T) { "status.version": "v1.2.3", }). Build(), + minWorkersVersion: "v1.2.2", + controlPlaneUpgradePlan: []string{}, + machineDeploymentsUpgradePlan: []string{}, // changed from previous step + machinePoolsUpgradePlan: []string{"v1.2.3"}, + wantVersion: "v1.2.3", + wantIsWaitingForWorkersUpgrade: true, + wantPendingHookAnnotation: "AfterClusterUpgrade,AfterWorkersUpgrade", + }, + { + name: "after workers are upgraded: call the AfterWorkersUpgrade hook, blocking answer", + topologyVersion: "v1.2.3", + pendingHookAnnotation: "AfterClusterUpgrade,AfterWorkersUpgrade", + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.2.3", + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.2.3", + }). + Build(), + minWorkersVersion: "v1.2.3", controlPlaneUpgradePlan: []string{}, - machineDeploymentsUpgradePlan: []string{}, // changed from previous step - machinePoolsUpgradePlan: []string{"v1.2.3"}, - wantVersion: "v1.2.3", - wantPendingHookAnnotation: "AfterClusterUpgrade", + machineDeploymentsUpgradePlan: []string{}, + machinePoolsUpgradePlan: []string{}, // changed from previous step + wantAfterWorkersUpgradeRequest: &runtimehooksv1.AfterWorkersUpgradeRequest{ + KubernetesVersion: "v1.2.3", + ControlPlaneUpgrades: toUpgradeStep([]string{}), + WorkersUpgrades: toUpgradeStep([]string{}), + }, + afterWorkersUpgradeResponse: blockingAfterWorkersUpgradeResponse, + wantVersion: "v1.2.3", + wantPendingHookAnnotation: "AfterClusterUpgrade,AfterWorkersUpgrade", + }, + { + name: "after workers are upgraded: AfterWorkersUpgrade hook unblocks", + topologyVersion: "v1.2.3", + pendingHookAnnotation: "AfterClusterUpgrade,AfterWorkersUpgrade", + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.2.3", + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.2.3", + }). + Build(), + minWorkersVersion: "v1.2.3", + controlPlaneUpgradePlan: []string{}, + machineDeploymentsUpgradePlan: []string{}, + machinePoolsUpgradePlan: []string{}, + wantAfterWorkersUpgradeRequest: &runtimehooksv1.AfterWorkersUpgradeRequest{ + KubernetesVersion: "v1.2.3", + ControlPlaneUpgrades: toUpgradeStep([]string{}), + WorkersUpgrades: toUpgradeStep([]string{}), + }, + afterWorkersUpgradeResponse: nonBlockingAfterWorkersUpgradeResponse, + wantVersion: "v1.2.3", + wantPendingHookAnnotation: "AfterClusterUpgrade", // changed from previous step }, // Note: After MP upgrade completes, the AfterClusterUpgrade is called from reconcile_state.go @@ -328,6 +522,7 @@ func TestComputeControlPlaneVersion_LifecycleHooksSequences(t *testing.T) { "status.version": "v1.2.2", }). Build(), + minWorkersVersion: "v1.2.2", controlPlaneUpgradePlan: []string{"v1.3.3", "v1.4.4"}, machineDeploymentsUpgradePlan: []string{"v1.4.4"}, machinePoolsUpgradePlan: []string{}, @@ -342,7 +537,7 @@ func TestComputeControlPlaneVersion_LifecycleHooksSequences(t *testing.T) { wantIsPendingUpgrade: true, }, { - name: "when an upgrade to the first minor starts: BeforeClusterUpgrade hook unblocks, pick up the new version", + name: "when an upgrade to the first minor starts: call the BeforeControlPlaneUpgrade hook when BeforeClusterUpgrade hook unblocks, blocking answer", topologyVersion: "v1.4.4", controlPlaneObj: builder.ControlPlane("test1", "cp1"). WithSpecFields(map[string]interface{}{ @@ -352,6 +547,7 @@ func TestComputeControlPlaneVersion_LifecycleHooksSequences(t *testing.T) { "status.version": "v1.2.2", }). Build(), + minWorkersVersion: "v1.2.2", controlPlaneUpgradePlan: []string{"v1.3.3", "v1.4.4"}, machineDeploymentsUpgradePlan: []string{"v1.4.4"}, machinePoolsUpgradePlan: []string{}, @@ -362,9 +558,43 @@ func TestComputeControlPlaneVersion_LifecycleHooksSequences(t *testing.T) { WorkersUpgrades: toUpgradeStep([]string{"v1.4.4"}), }, beforeClusterUpgradeResponse: nonBlockingBeforeClusterUpgradeResponse, - wantVersion: "v1.3.3", // changed from previous step - wantIsStartingUpgrade: true, - wantPendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade", // changed from previous step + wantBeforeControlPlaneUpgradeRequest: &runtimehooksv1.BeforeControlPlaneUpgradeRequest{ + FromKubernetesVersion: "v1.2.2", + ToKubernetesVersion: "v1.3.3", // CP picking up the first version in the plan + ControlPlaneUpgrades: toUpgradeStep([]string{"v1.3.3", "v1.4.4"}), + WorkersUpgrades: toUpgradeStep([]string{"v1.4.4"}), + }, + beforeControlPlaneUpgradeResponse: blockingBeforeControlPlaneUpgradeResponse, + wantVersion: "v1.2.2", + wantIsPendingUpgrade: true, + wantPendingHookAnnotation: "AfterClusterUpgrade", // changed from previous step + }, + { + name: "when an upgrade to the first minor starts: pick up a new version when BeforeControlPlaneUpgrade hook unblocks (does not call the BeforeClusterUpgrade hook when already done)", + topologyVersion: "v1.4.4", + pendingHookAnnotation: "AfterClusterUpgrade", + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.2.2", + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.2.2", + }). + Build(), + minWorkersVersion: "v1.2.2", + controlPlaneUpgradePlan: []string{"v1.3.3", "v1.4.4"}, + machineDeploymentsUpgradePlan: []string{"v1.4.4"}, + machinePoolsUpgradePlan: []string{}, + wantBeforeControlPlaneUpgradeRequest: &runtimehooksv1.BeforeControlPlaneUpgradeRequest{ + FromKubernetesVersion: "v1.2.2", + ToKubernetesVersion: "v1.3.3", // CP picking up the first version in the plan + ControlPlaneUpgrades: toUpgradeStep([]string{"v1.3.3", "v1.4.4"}), + WorkersUpgrades: toUpgradeStep([]string{"v1.4.4"}), + }, + beforeControlPlaneUpgradeResponse: nonBlockingBeforeControlPlaneUpgradeResponse, + wantVersion: "v1.3.3", // changed from previous step + wantIsStartingUpgrade: true, + wantPendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade", // changed from previous step }, { name: "when control plane is upgrading to the first minor: do not call hooks", @@ -378,6 +608,7 @@ func TestComputeControlPlaneVersion_LifecycleHooksSequences(t *testing.T) { "status.version": "v1.2.2", }). Build(), + minWorkersVersion: "v1.2.2", controlPlaneUpgradePlan: []string{"v1.4.4"}, // changed from previous step machineDeploymentsUpgradePlan: []string{"v1.4.4"}, machinePoolsUpgradePlan: []string{}, @@ -411,7 +642,7 @@ func TestComputeControlPlaneVersion_LifecycleHooksSequences(t *testing.T) { wantPendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade", }, { - name: "when an upgrade to the second minor starts: pick up a new version when AfterControlPlaneUpgrade hook unblocks", + name: "when an upgrade to the second minor starts: call the BeforeControlPlaneUpgrade after AfterControlPlaneUpgrade hook unblocks, blocking answer", topologyVersion: "v1.4.4", pendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade", controlPlaneObj: builder.ControlPlane("test1", "cp1"). @@ -422,6 +653,7 @@ func TestComputeControlPlaneVersion_LifecycleHooksSequences(t *testing.T) { "status.version": "v1.3.3", }). Build(), + minWorkersVersion: "v1.2.2", controlPlaneUpgradePlan: []string{"v1.4.4"}, machineDeploymentsUpgradePlan: []string{"v1.4.4"}, machinePoolsUpgradePlan: []string{}, @@ -431,14 +663,48 @@ func TestComputeControlPlaneVersion_LifecycleHooksSequences(t *testing.T) { WorkersUpgrades: toUpgradeStep([]string{"v1.4.4"}), }, afterControlPlaneUpgradeResponse: nonBlockingAfterControlPlaneUpgradeResponse, - wantVersion: "v1.4.4", // changed from previous step - wantIsStartingUpgrade: true, - wantPendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade", // changed from previous step + wantBeforeControlPlaneUpgradeRequest: &runtimehooksv1.BeforeControlPlaneUpgradeRequest{ + FromKubernetesVersion: "v1.3.3", + ToKubernetesVersion: "v1.4.4", // CP picking up the first version in the plan + ControlPlaneUpgrades: toUpgradeStep([]string{"v1.4.4"}), + WorkersUpgrades: toUpgradeStep([]string{"v1.4.4"}), + }, + beforeControlPlaneUpgradeResponse: blockingBeforeControlPlaneUpgradeResponse, + wantVersion: "v1.3.3", + wantIsPendingUpgrade: true, + wantPendingHookAnnotation: "AfterClusterUpgrade", // changed from previous step + }, + { + name: "when an upgrade to the second minor starts: pick up a new version when BeforeControlPlaneUpgrade hook unblocks (does not call the BeforeClusterUpgrade hook when already done)", + topologyVersion: "v1.4.4", + pendingHookAnnotation: "AfterClusterUpgrade", + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.3.3", + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.3.3", + }). + Build(), + minWorkersVersion: "v1.2.2", + controlPlaneUpgradePlan: []string{"v1.4.4"}, + machineDeploymentsUpgradePlan: []string{"v1.4.4"}, + machinePoolsUpgradePlan: []string{}, + wantBeforeControlPlaneUpgradeRequest: &runtimehooksv1.BeforeControlPlaneUpgradeRequest{ + FromKubernetesVersion: "v1.3.3", + ToKubernetesVersion: "v1.4.4", // CP picking up the first version in the plan + ControlPlaneUpgrades: toUpgradeStep([]string{"v1.4.4"}), + WorkersUpgrades: toUpgradeStep([]string{"v1.4.4"}), + }, + beforeControlPlaneUpgradeResponse: nonBlockingBeforeControlPlaneUpgradeResponse, + wantVersion: "v1.4.4", // changed from previous step + wantIsStartingUpgrade: true, + wantPendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade,AfterWorkersUpgrade,BeforeWorkersUpgrade", // changed from previous step }, { name: "when control plane is upgrading to the second minor: do not call hooks", topologyVersion: "v1.4.4", - pendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade", + pendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade,AfterWorkersUpgrade,BeforeWorkersUpgrade", controlPlaneObj: builder.ControlPlane("test1", "cp1"). WithSpecFields(map[string]interface{}{ "spec.version": "v1.4.4", @@ -447,16 +713,17 @@ func TestComputeControlPlaneVersion_LifecycleHooksSequences(t *testing.T) { "status.version": "v1.3.3", }). Build(), + minWorkersVersion: "v1.2.2", controlPlaneUpgradePlan: []string{}, // changed from previous step machineDeploymentsUpgradePlan: []string{"v1.4.4"}, machinePoolsUpgradePlan: []string{}, wantVersion: "v1.4.4", - wantPendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade", + wantPendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade,AfterWorkersUpgrade,BeforeWorkersUpgrade", }, { name: "after control plane is upgraded to the second minor: call the AfterControlPlaneUpgrade hook, blocking answer", topologyVersion: "v1.4.4", - pendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade", + pendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade,AfterWorkersUpgrade,BeforeWorkersUpgrade", controlPlaneObj: builder.ControlPlane("test1", "cp1"). WithSpecFields(map[string]interface{}{ "spec.version": "v1.4.4", @@ -465,6 +732,7 @@ func TestComputeControlPlaneVersion_LifecycleHooksSequences(t *testing.T) { "status.version": "v1.4.4", // changed from previous step }). Build(), + minWorkersVersion: "v1.2.2", controlPlaneUpgradePlan: []string{}, machineDeploymentsUpgradePlan: []string{"v1.4.4"}, machinePoolsUpgradePlan: []string{}, @@ -475,12 +743,44 @@ func TestComputeControlPlaneVersion_LifecycleHooksSequences(t *testing.T) { }, afterControlPlaneUpgradeResponse: blockingAfterControlPlaneUpgradeResponse, wantVersion: "v1.4.4", - wantPendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade", + wantPendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade,AfterWorkersUpgrade,BeforeWorkersUpgrade", }, { - name: "when machine deployment are upgrading to the second minor: do not call hooks", + name: "when starting workers upgrade to the second minor: call the BeforeWorkersUpgrade hook when AfterControlPlaneUpgradeRequest hook unblocks, blocking answer", topologyVersion: "v1.4.4", - pendingHookAnnotation: "AfterClusterUpgrade", + pendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade,AfterWorkersUpgrade,BeforeWorkersUpgrade", + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.4.4", + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.4.4", + }). + Build(), + minWorkersVersion: "v1.2.2", + controlPlaneUpgradePlan: []string{}, + machineDeploymentsUpgradePlan: []string{"v1.4.4"}, + machinePoolsUpgradePlan: []string{}, + wantAfterControlPlaneUpgradeRequest: &runtimehooksv1.AfterControlPlaneUpgradeRequest{ + KubernetesVersion: "v1.4.4", + ControlPlaneUpgrades: toUpgradeStep([]string{}), + WorkersUpgrades: toUpgradeStep([]string{"v1.4.4"}), + }, + afterControlPlaneUpgradeResponse: nonBlockingAfterControlPlaneUpgradeResponse, + wantBeforeWorkersUpgradeRequest: &runtimehooksv1.BeforeWorkersUpgradeRequest{ + FromKubernetesVersion: "v1.2.2", + ToKubernetesVersion: "v1.4.4", + ControlPlaneUpgrades: toUpgradeStep([]string{}), + WorkersUpgrades: toUpgradeStep([]string{"v1.4.4"}), + }, + beforeWorkersUpgradeResponse: blockingBeforeWorkersUpgradeResponse, + wantVersion: "v1.4.4", + wantPendingHookAnnotation: "AfterClusterUpgrade,AfterWorkersUpgrade,BeforeWorkersUpgrade", // changed from previous step + }, + { + name: "when starting workers upgrade to the second minor: BeforeWorkersUpgrade hook unblocks (does not call the AfterControlPlaneUpgrade hook when already done)", + topologyVersion: "v1.4.4", + pendingHookAnnotation: "AfterClusterUpgrade,AfterWorkersUpgrade,BeforeWorkersUpgrade", controlPlaneObj: builder.ControlPlane("test1", "cp1"). WithSpecFields(map[string]interface{}{ "spec.version": "v1.4.4", @@ -489,11 +789,357 @@ func TestComputeControlPlaneVersion_LifecycleHooksSequences(t *testing.T) { "status.version": "v1.4.4", }). Build(), + minWorkersVersion: "v1.2.2", controlPlaneUpgradePlan: []string{}, machineDeploymentsUpgradePlan: []string{"v1.4.4"}, machinePoolsUpgradePlan: []string{}, + wantBeforeWorkersUpgradeRequest: &runtimehooksv1.BeforeWorkersUpgradeRequest{ + FromKubernetesVersion: "v1.2.2", + ToKubernetesVersion: "v1.4.4", + ControlPlaneUpgrades: toUpgradeStep([]string{}), + WorkersUpgrades: toUpgradeStep([]string{"v1.4.4"}), + }, + beforeWorkersUpgradeResponse: nonBlockingBeforeWorkersUpgradeResponse, + wantVersion: "v1.4.4", + wantIsWaitingForWorkersUpgrade: true, + wantPendingHookAnnotation: "AfterClusterUpgrade,AfterWorkersUpgrade", // changed from previous step + }, + { + name: "when machine deployment are upgrading to the second minor: do not call hooks", + topologyVersion: "v1.4.4", + pendingHookAnnotation: "AfterClusterUpgrade,AfterWorkersUpgrade", + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.4.4", + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.4.4", + }). + Build(), + minWorkersVersion: "v1.2.2", + controlPlaneUpgradePlan: []string{}, + machineDeploymentsUpgradePlan: []string{"v1.4.4"}, + machinePoolsUpgradePlan: []string{}, + wantVersion: "v1.4.4", + wantIsWaitingForWorkersUpgrade: true, + wantPendingHookAnnotation: "AfterClusterUpgrade,AfterWorkersUpgrade", + }, + { + name: "after workers are upgraded to the second minor: call the AfterWorkersUpgrade hook, blocking answer", + topologyVersion: "v1.4.4", + pendingHookAnnotation: "AfterClusterUpgrade,AfterWorkersUpgrade", + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.4.4", + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.4.4", + }). + Build(), + minWorkersVersion: "v1.4.4", + controlPlaneUpgradePlan: []string{}, + machineDeploymentsUpgradePlan: []string{}, // changed from previous step + machinePoolsUpgradePlan: []string{}, + wantAfterWorkersUpgradeRequest: &runtimehooksv1.AfterWorkersUpgradeRequest{ + KubernetesVersion: "v1.4.4", + ControlPlaneUpgrades: toUpgradeStep([]string{}), + WorkersUpgrades: toUpgradeStep([]string{}), + }, + afterWorkersUpgradeResponse: blockingAfterWorkersUpgradeResponse, + wantVersion: "v1.4.4", + wantPendingHookAnnotation: "AfterClusterUpgrade,AfterWorkersUpgrade", + }, + { + name: "after workers are upgraded to the second minor: AfterWorkersUpgrade hook unblocks", + topologyVersion: "v1.4.4", + pendingHookAnnotation: "AfterClusterUpgrade,AfterWorkersUpgrade", + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.4.4", + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.4.4", + }). + Build(), + minWorkersVersion: "v1.4.4", + controlPlaneUpgradePlan: []string{}, + machineDeploymentsUpgradePlan: []string{}, // changed from previous step + machinePoolsUpgradePlan: []string{}, + wantAfterWorkersUpgradeRequest: &runtimehooksv1.AfterWorkersUpgradeRequest{ + KubernetesVersion: "v1.4.4", + ControlPlaneUpgrades: toUpgradeStep([]string{}), + WorkersUpgrades: toUpgradeStep([]string{}), + }, + afterWorkersUpgradeResponse: nonBlockingAfterWorkersUpgradeResponse, + wantVersion: "v1.4.4", + wantPendingHookAnnotation: "AfterClusterUpgrade", // changed from previous step + }, + // Note: After MD upgrade completes, the AfterClusterUpgrade is called from reconcile_state.go + + // Upgrade cluster with CP, no workers (upgrade by two minors) + + { + name: "no hook called before starting the upgrade", + topologyVersion: "v1.2.2", + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.2.2", + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.2.2", + }). + Build(), + wantVersion: "v1.2.2", + }, + { + name: "when an upgrade to the first minor starts: call the BeforeClusterUpgrade hook, blocking answer", + topologyVersion: "v1.4.4", // changed from previous step + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.2.2", + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.2.2", + }). + Build(), + controlPlaneUpgradePlan: []string{"v1.3.3", "v1.4.4"}, + machineDeploymentsUpgradePlan: []string{}, + machinePoolsUpgradePlan: []string{}, + wantBeforeClusterUpgradeRequest: &runtimehooksv1.BeforeClusterUpgradeRequest{ + FromKubernetesVersion: "v1.2.2", + ToKubernetesVersion: "v1.4.4", + ControlPlaneUpgrades: toUpgradeStep([]string{"v1.3.3", "v1.4.4"}), + WorkersUpgrades: toUpgradeStep([]string{}), + }, + beforeClusterUpgradeResponse: blockingBeforeClusterUpgradeResponse, + wantVersion: "v1.2.2", + wantIsPendingUpgrade: true, + }, + { + name: "when an upgrade to the first minor starts: call the BeforeControlPlaneUpgrade hook when BeforeClusterUpgrade hook unblocks, blocking answer", + topologyVersion: "v1.4.4", + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.2.2", + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.2.2", + }). + Build(), + controlPlaneUpgradePlan: []string{"v1.3.3", "v1.4.4"}, + machineDeploymentsUpgradePlan: []string{}, + machinePoolsUpgradePlan: []string{}, + wantBeforeClusterUpgradeRequest: &runtimehooksv1.BeforeClusterUpgradeRequest{ + FromKubernetesVersion: "v1.2.2", + ToKubernetesVersion: "v1.4.4", + ControlPlaneUpgrades: toUpgradeStep([]string{"v1.3.3", "v1.4.4"}), + WorkersUpgrades: toUpgradeStep([]string{}), + }, + beforeClusterUpgradeResponse: nonBlockingBeforeClusterUpgradeResponse, + wantBeforeControlPlaneUpgradeRequest: &runtimehooksv1.BeforeControlPlaneUpgradeRequest{ + FromKubernetesVersion: "v1.2.2", + ToKubernetesVersion: "v1.3.3", // CP picking up the first version in the plan + ControlPlaneUpgrades: toUpgradeStep([]string{"v1.3.3", "v1.4.4"}), + WorkersUpgrades: toUpgradeStep([]string{}), + }, + beforeControlPlaneUpgradeResponse: blockingBeforeControlPlaneUpgradeResponse, + wantVersion: "v1.2.2", + wantIsPendingUpgrade: true, + wantPendingHookAnnotation: "AfterClusterUpgrade", // changed from previous step + }, + { + name: "when an upgrade to the first minor starts: pick up a new version when BeforeControlPlaneUpgrade hook unblocks (does not call the BeforeClusterUpgrade hook when already done)", + topologyVersion: "v1.4.4", + pendingHookAnnotation: "AfterClusterUpgrade", + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.2.2", + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.2.2", + }). + Build(), + controlPlaneUpgradePlan: []string{"v1.3.3", "v1.4.4"}, + machineDeploymentsUpgradePlan: []string{}, + machinePoolsUpgradePlan: []string{}, + wantBeforeControlPlaneUpgradeRequest: &runtimehooksv1.BeforeControlPlaneUpgradeRequest{ + FromKubernetesVersion: "v1.2.2", + ToKubernetesVersion: "v1.3.3", // CP picking up the first version in the plan + ControlPlaneUpgrades: toUpgradeStep([]string{"v1.3.3", "v1.4.4"}), + WorkersUpgrades: toUpgradeStep([]string{}), + }, + beforeControlPlaneUpgradeResponse: nonBlockingBeforeControlPlaneUpgradeResponse, + wantVersion: "v1.3.3", // changed from previous step + wantIsStartingUpgrade: true, + wantPendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade", // changed from previous step + }, + { + name: "when control plane is upgrading to the first minor: do not call hooks", + topologyVersion: "v1.4.4", + pendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade", + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.3.3", + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.2.2", + }). + Build(), + controlPlaneUpgradePlan: []string{"v1.4.4"}, // changed from previous step + machineDeploymentsUpgradePlan: []string{}, + machinePoolsUpgradePlan: []string{}, + wantVersion: "v1.3.3", + wantIsPendingUpgrade: true, + wantPendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade", + }, + { + name: "after control plane is upgraded to the first minor: call the AfterControlPlaneUpgrade hook, blocking answer", + topologyVersion: "v1.4.4", + pendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade", + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.3.3", + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.3.3", // changed from previous step + }). + Build(), + controlPlaneUpgradePlan: []string{"v1.4.4"}, + machineDeploymentsUpgradePlan: []string{}, + machinePoolsUpgradePlan: []string{}, + wantAfterControlPlaneUpgradeRequest: &runtimehooksv1.AfterControlPlaneUpgradeRequest{ + KubernetesVersion: "v1.3.3", + ControlPlaneUpgrades: toUpgradeStep([]string{"v1.4.4"}), + WorkersUpgrades: toUpgradeStep([]string{}), + }, + afterControlPlaneUpgradeResponse: blockingAfterControlPlaneUpgradeResponse, + wantVersion: "v1.3.3", + wantIsPendingUpgrade: true, + wantPendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade", + }, + { + name: "when an upgrade to the second minor starts: call the BeforeControlPlaneUpgrade after AfterControlPlaneUpgrade hook unblocks, blocking answer", + topologyVersion: "v1.4.4", + pendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade", + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.3.3", + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.3.3", + }). + Build(), + controlPlaneUpgradePlan: []string{"v1.4.4"}, + machineDeploymentsUpgradePlan: []string{}, + machinePoolsUpgradePlan: []string{}, + wantAfterControlPlaneUpgradeRequest: &runtimehooksv1.AfterControlPlaneUpgradeRequest{ + KubernetesVersion: "v1.3.3", + ControlPlaneUpgrades: toUpgradeStep([]string{"v1.4.4"}), + WorkersUpgrades: toUpgradeStep([]string{}), + }, + afterControlPlaneUpgradeResponse: nonBlockingAfterControlPlaneUpgradeResponse, + wantBeforeControlPlaneUpgradeRequest: &runtimehooksv1.BeforeControlPlaneUpgradeRequest{ + FromKubernetesVersion: "v1.3.3", + ToKubernetesVersion: "v1.4.4", // CP picking up the first version in the plan + ControlPlaneUpgrades: toUpgradeStep([]string{"v1.4.4"}), + WorkersUpgrades: toUpgradeStep([]string{}), + }, + beforeControlPlaneUpgradeResponse: blockingBeforeControlPlaneUpgradeResponse, + wantVersion: "v1.3.3", + wantIsPendingUpgrade: true, + wantPendingHookAnnotation: "AfterClusterUpgrade", // changed from previous step + }, + { + name: "when an upgrade to the second minor starts: pick up a new version when BeforeControlPlaneUpgrade hook unblocks (does not call the BeforeClusterUpgrade hook when already done)", + topologyVersion: "v1.4.4", + pendingHookAnnotation: "AfterClusterUpgrade", + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.3.3", + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.3.3", + }). + Build(), + controlPlaneUpgradePlan: []string{"v1.4.4"}, + machineDeploymentsUpgradePlan: []string{}, + machinePoolsUpgradePlan: []string{}, + wantBeforeControlPlaneUpgradeRequest: &runtimehooksv1.BeforeControlPlaneUpgradeRequest{ + FromKubernetesVersion: "v1.3.3", + ToKubernetesVersion: "v1.4.4", // CP picking up the first version in the plan + ControlPlaneUpgrades: toUpgradeStep([]string{"v1.4.4"}), + WorkersUpgrades: toUpgradeStep([]string{}), + }, + beforeControlPlaneUpgradeResponse: nonBlockingBeforeControlPlaneUpgradeResponse, + wantVersion: "v1.4.4", // changed from previous step + wantIsStartingUpgrade: true, + wantPendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade", // changed from previous step + }, + { + name: "when control plane is upgrading to the second minor: do not call hooks", + topologyVersion: "v1.4.4", + pendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade", + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.4.4", + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.3.3", + }). + Build(), + controlPlaneUpgradePlan: []string{}, // changed from previous step + machineDeploymentsUpgradePlan: []string{}, + machinePoolsUpgradePlan: []string{}, wantVersion: "v1.4.4", - wantPendingHookAnnotation: "AfterClusterUpgrade", + wantPendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade", + }, + { + name: "after control plane is upgraded to the second minor: call the AfterControlPlaneUpgrade hook, blocking answer", + topologyVersion: "v1.4.4", + pendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade", + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.4.4", + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.4.4", // changed from previous step + }). + Build(), + controlPlaneUpgradePlan: []string{}, + machineDeploymentsUpgradePlan: []string{}, + machinePoolsUpgradePlan: []string{}, + wantAfterControlPlaneUpgradeRequest: &runtimehooksv1.AfterControlPlaneUpgradeRequest{ + KubernetesVersion: "v1.4.4", + ControlPlaneUpgrades: toUpgradeStep([]string{}), + WorkersUpgrades: toUpgradeStep([]string{}), + }, + afterControlPlaneUpgradeResponse: blockingAfterControlPlaneUpgradeResponse, + wantVersion: "v1.4.4", + wantPendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade", + }, + { + name: "after control plane is upgraded to the second minor: call the AfterControlPlaneUpgrade hook, non blocking answer", + topologyVersion: "v1.4.4", + pendingHookAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade", + controlPlaneObj: builder.ControlPlane("test1", "cp1"). + WithSpecFields(map[string]interface{}{ + "spec.version": "v1.4.4", + }). + WithStatusFields(map[string]interface{}{ + "status.version": "v1.4.4", // changed from previous step + }). + Build(), + controlPlaneUpgradePlan: []string{}, + machineDeploymentsUpgradePlan: []string{}, + machinePoolsUpgradePlan: []string{}, + wantAfterControlPlaneUpgradeRequest: &runtimehooksv1.AfterControlPlaneUpgradeRequest{ + KubernetesVersion: "v1.4.4", + ControlPlaneUpgrades: toUpgradeStep([]string{}), + WorkersUpgrades: toUpgradeStep([]string{}), + }, + afterControlPlaneUpgradeResponse: nonBlockingAfterControlPlaneUpgradeResponse, + wantVersion: "v1.4.4", + wantPendingHookAnnotation: "AfterClusterUpgrade", // changed from previous step }, // Note: After MD upgrade completes, the AfterClusterUpgrade is called from reconcile_state.go } @@ -548,6 +1194,8 @@ func TestComputeControlPlaneVersion_LifecycleHooksSequences(t *testing.T) { } s.Current.Cluster.Annotations[runtimev1.PendingHooksAnnotation] = tt.pendingHookAnnotation } + + s.UpgradeTracker.MinWorkersVersion = tt.minWorkersVersion if len(tt.controlPlaneUpgradePlan) > 0 { s.UpgradeTracker.ControlPlane.UpgradePlan = tt.controlPlaneUpgradePlan } @@ -557,12 +1205,6 @@ func TestComputeControlPlaneVersion_LifecycleHooksSequences(t *testing.T) { if len(tt.machinePoolsUpgradePlan) > 0 { s.UpgradeTracker.MachinePools.UpgradePlan = tt.machinePoolsUpgradePlan } - if len(tt.upgradingMachineDeployments) > 0 { - s.UpgradeTracker.MachineDeployments.MarkUpgrading(tt.upgradingMachineDeployments...) - } - if len(tt.upgradingMachinePools) > 0 { - s.UpgradeTracker.MachinePools.MarkUpgrading(tt.upgradingMachinePools...) - } hooksCalled := sets.Set[string]{} validateHookCall := func(request runtimehooksv1.RequestObject) error { @@ -572,11 +1214,26 @@ func TestComputeControlPlaneVersion_LifecycleHooksSequences(t *testing.T) { if err := validateHookRequest(request, tt.wantBeforeClusterUpgradeRequest); err != nil { return err } + case *runtimehooksv1.BeforeControlPlaneUpgradeRequest: + hooksCalled.Insert("BeforeControlPlaneUpgrade") + if err := validateHookRequest(request, tt.wantBeforeControlPlaneUpgradeRequest); err != nil { + return err + } case *runtimehooksv1.AfterControlPlaneUpgradeRequest: hooksCalled.Insert("AfterControlPlaneUpgrade") if err := validateHookRequest(request, tt.wantAfterControlPlaneUpgradeRequest); err != nil { return err } + case *runtimehooksv1.BeforeWorkersUpgradeRequest: + hooksCalled.Insert("BeforeWorkersUpgrade") + if err := validateHookRequest(request, tt.wantBeforeWorkersUpgradeRequest); err != nil { + return err + } + case *runtimehooksv1.AfterWorkersUpgradeRequest: + hooksCalled.Insert("AfterWorkersUpgrade") + if err := validateHookRequest(request, tt.wantAfterWorkersUpgradeRequest); err != nil { + return err + } default: return errors.Errorf("unhandled request type %T", request) } @@ -586,8 +1243,11 @@ func TestComputeControlPlaneVersion_LifecycleHooksSequences(t *testing.T) { runtimeClient := fakeruntimeclient.NewRuntimeClientBuilder(). WithCatalog(catalog). WithCallAllExtensionResponses(map[runtimecatalog.GroupVersionHook]runtimehooksv1.ResponseObject{ - beforeClusterUpgradeGVH: tt.beforeClusterUpgradeResponse, - afterControlPlaneUpgradeGVH: tt.afterControlPlaneUpgradeResponse, + beforeClusterUpgradeGVH: tt.beforeClusterUpgradeResponse, + beforeControlPlaneUpgradeGVH: tt.beforeControlPlaneUpgradeResponse, + afterControlPlaneUpgradeGVH: tt.afterControlPlaneUpgradeResponse, + beforeWorkersUpgradeGVH: tt.beforeWorkersUpgradeResponse, + afterWorkersUpgradeGVH: tt.afterWorkersUpgradeResponse, }). WithCallAllExtensionValidations(validateHookCall). Build() @@ -608,7 +1268,10 @@ func TestComputeControlPlaneVersion_LifecycleHooksSequences(t *testing.T) { // check call received g.Expect(hooksCalled.Has("BeforeClusterUpgrade")).To(Equal(tt.wantBeforeClusterUpgradeRequest != nil), "Unexpected call/missing call to BeforeClusterUpgrade") + g.Expect(hooksCalled.Has("BeforeControlPlaneUpgrade")).To(Equal(tt.wantBeforeControlPlaneUpgradeRequest != nil), "Unexpected call/missing call to BeforeControlPlaneUpgrade") g.Expect(hooksCalled.Has("AfterControlPlaneUpgrade")).To(Equal(tt.wantAfterControlPlaneUpgradeRequest != nil), "Unexpected call/missing call to AfterControlPlaneUpgrade") + g.Expect(hooksCalled.Has("BeforeWorkersUpgrade")).To(Equal(tt.wantBeforeWorkersUpgradeRequest != nil), "Unexpected call/missing call to BeforeWorkersUpgrade") + g.Expect(hooksCalled.Has("AfterWorkersUpgrade")).To(Equal(tt.wantAfterWorkersUpgradeRequest != nil), "Unexpected call/missing call to AfterWorkersUpgrade") // check intent to call hooks if tt.wantPendingHookAnnotation != "" { diff --git a/exp/topology/desiredstate/upgrade_plan.go b/exp/topology/desiredstate/upgrade_plan.go index c44245fe7e67..6462d88e1f7e 100644 --- a/exp/topology/desiredstate/upgrade_plan.go +++ b/exp/topology/desiredstate/upgrade_plan.go @@ -93,6 +93,7 @@ func ComputeUpgradePlan(ctx context.Context, s *scope.Scope, getUpgradePlan GetU if minWorkersSemVer != nil { minWorkersVersion = fmt.Sprintf("v%s", minWorkersSemVer.String()) } + s.UpgradeTracker.MinWorkersVersion = minWorkersVersion // If both control plane and workers are already at the desired version, there is no need to compute the upgrade plan. if controlPlaneSemVer.String() == desiredSemVer.String() && (minWorkersSemVer == nil || minWorkersSemVer.String() == desiredSemVer.String()) { diff --git a/exp/topology/scope/upgradetracker.go b/exp/topology/scope/upgradetracker.go index 8a1a47a306d1..7384dce71666 100644 --- a/exp/topology/scope/upgradetracker.go +++ b/exp/topology/scope/upgradetracker.go @@ -23,6 +23,7 @@ type UpgradeTracker struct { ControlPlane ControlPlaneUpgradeTracker MachineDeployments WorkerUpgradeTracker MachinePools WorkerUpgradeTracker + MinWorkersVersion string } // ControlPlaneUpgradeTracker holds the current upgrade status of the Control Plane. diff --git a/internal/controllers/topology/cluster/reconcile_state.go b/internal/controllers/topology/cluster/reconcile_state.go index 83c285b4f4f9..b57bfa9c2384 100644 --- a/internal/controllers/topology/cluster/reconcile_state.go +++ b/internal/controllers/topology/cluster/reconcile_state.go @@ -234,7 +234,8 @@ func isControlPlaneInitialized(cluster *clusterv1.Cluster) bool { func (r *Reconciler) callAfterClusterUpgrade(ctx context.Context, s *scope.Scope) error { // Call the hook only if we are tracking the intent to do so. If it is not tracked it means we don't need to call the // hook because we didn't go through an upgrade or we already called the hook after the upgrade. - if hooks.IsPending(runtimehooksv1.AfterClusterUpgrade, s.Current.Cluster) { + // Note: also check that the AfterControlPlaneUpgrade hooks and the AfterWorkersUpgrade hooks already have been called. + if hooks.IsPending(runtimehooksv1.AfterClusterUpgrade, s.Current.Cluster) && !hooks.IsPending(runtimehooksv1.AfterControlPlaneUpgrade, s.Current.Cluster) && !hooks.IsPending(runtimehooksv1.AfterWorkersUpgrade, s.Current.Cluster) { // Call the registered extensions for the hook after the cluster is fully upgraded. // A clusters is considered fully upgraded if: // - Control plane is stable (not upgrading, not scaling, not about to upgrade) diff --git a/internal/controllers/topology/cluster/reconcile_state_test.go b/internal/controllers/topology/cluster/reconcile_state_test.go index 70f89a72ea62..8c4468fa1852 100644 --- a/internal/controllers/topology/cluster/reconcile_state_test.go +++ b/internal/controllers/topology/cluster/reconcile_state_test.go @@ -1060,6 +1060,80 @@ func TestReconcile_callAfterClusterUpgrade(t *testing.T) { wantHookToBeCalled: false, wantError: false, }, + { + name: "hook should not be called if AfterControlPlaneUpgrade did not completed - hook is marked", + s: &scope.Scope{ + Blueprint: &scope.ClusterBlueprint{ + Topology: clusterv1.Topology{ + ControlPlane: clusterv1.ControlPlaneTopology{ + Replicas: ptr.To[int32](2), + }, + }, + }, + Current: &scope.ClusterState{ + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "test-ns", + Annotations: map[string]string{ + runtimev1.PendingHooksAnnotation: "AfterClusterUpgrade,AfterControlPlaneUpgrade", + }, + }, + Spec: clusterv1.ClusterSpec{ + Topology: clusterv1.Topology{ + Version: topologyVersion, + }, + }, + }, + ControlPlane: &scope.ControlPlaneState{ + Object: controlPlaneObj, + }, + }, + HookResponseTracker: scope.NewHookResponseTracker(), + UpgradeTracker: scope.NewUpgradeTracker(), + }, + wantMarked: true, + hookResponse: successResponse, + wantHookToBeCalled: false, + wantError: false, + }, + { + name: "hook should not be called if AfterWorkersUpgrade did not completed - hook is marked", + s: &scope.Scope{ + Blueprint: &scope.ClusterBlueprint{ + Topology: clusterv1.Topology{ + ControlPlane: clusterv1.ControlPlaneTopology{ + Replicas: ptr.To[int32](2), + }, + }, + }, + Current: &scope.ClusterState{ + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "test-ns", + Annotations: map[string]string{ + runtimev1.PendingHooksAnnotation: "AfterClusterUpgrade,AfterWorkersUpgrade", + }, + }, + Spec: clusterv1.ClusterSpec{ + Topology: clusterv1.Topology{ + Version: topologyVersion, + }, + }, + }, + ControlPlane: &scope.ControlPlaneState{ + Object: controlPlaneObj, + }, + }, + HookResponseTracker: scope.NewHookResponseTracker(), + UpgradeTracker: scope.NewUpgradeTracker(), + }, + wantMarked: true, + hookResponse: successResponse, + wantHookToBeCalled: false, + wantError: false, + }, { name: "hook should be called if the control plane, MDs, and MPs are stable at the topology version - success response should unmark the hook", s: &scope.Scope{ diff --git a/test/e2e/cluster_upgrade_runtimesdk.go b/test/e2e/cluster_upgrade_runtimesdk.go index 2658710db207..7367aebe22ff 100644 --- a/test/e2e/cluster_upgrade_runtimesdk.go +++ b/test/e2e/cluster_upgrade_runtimesdk.go @@ -249,8 +249,7 @@ func ClusterUpgradeWithRuntimeSDKSpec(ctx context.Context, inputGetter func() Cl beforeClusterCreateTestHandler(ctx, input.BootstrapClusterProxy.GetClient(), cluster, - input.ExtensionConfigName, - input.E2EConfig.GetIntervals(specName, "wait-cluster")) + input.ExtensionConfigName) }, WaitForClusterIntervals: input.E2EConfig.GetIntervals(specName, "wait-cluster"), WaitForControlPlaneIntervals: input.E2EConfig.GetIntervals(specName, "wait-control-plane"), @@ -287,7 +286,7 @@ func ClusterUpgradeWithRuntimeSDKSpec(ctx context.Context, inputGetter func() Cl log.Logf("Control Plane upgrade plan: %s", controlPlaneUpgradePlan) log.Logf("Workers upgrade plan: %s", workersUpgradePlan) - // Perform the upgrade and check every is going as expected. + // Perform the upgrade and check everything is working as expected. // More specifically: // - After control plane upgrade steps // - Check control plane machines are on the desired version, nodes are healthy @@ -333,16 +332,30 @@ func ClusterUpgradeWithRuntimeSDKSpec(ctx context.Context, inputGetter func() Cl // the control plane upgrade step by step using hooks to block/unblock every phase. // Check for the beforeClusterUpgrade being called, then unblock + Expect(controlPlaneUpgradePlan).ToNot(BeEmpty()) + firstControlPlaneVersion := controlPlaneUpgradePlan[0] + beforeClusterUpgradeTestHandler(ctx, input.BootstrapClusterProxy.GetClient(), clusterResources.Cluster, input.ExtensionConfigName, - fromVersion, - toVersion, - input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade")) + fromVersion, // Cluster fromVersion + toVersion, // Cluster toVersion + firstControlPlaneVersion, // firstControlPlaneVersion in the upgrade plan, used to check the BeforeControlPlaneUpgrade is not called before its time. + ) // Then check the upgrade is progressing step by step according to the upgrade plan - for _, version := range controlPlaneUpgradePlan { + for i, version := range controlPlaneUpgradePlan { + // make sure beforeControlPlaneUpgrade still blocks, then unblock the upgrade. + beforeControlPlaneUpgradeTestHandler(ctx, + input.BootstrapClusterProxy.GetClient(), + clusterResources.Cluster, + input.ExtensionConfigName, + controlPlaneVersion, // Control plane fromVersion for this upgrade step. + version, // Control plane toVersion for this upgrade step. + workersVersion, // Current workersVersion, used to check workers do not upgrade before its time. + ) + // Wait CP to update to version controlPlaneVersion = version waitControlPlaneVersion(ctx, input.BootstrapClusterProxy.GetClient(), clusterResources.Cluster, controlPlaneVersion, input.E2EConfig.GetIntervals(specName, "wait-control-plane-upgrade")) @@ -351,28 +364,59 @@ func ClusterUpgradeWithRuntimeSDKSpec(ctx context.Context, inputGetter func() Cl checkWorkersVersions(ctx, input.BootstrapClusterProxy.GetClient(), clusterResources.Cluster, workersVersion) // make sure afterControlPlaneUpgrade still blocks, then unblock the upgrade. + nextControlPlaneVersion := "" + if i < len(controlPlaneUpgradePlan)-1 { + nextControlPlaneVersion = controlPlaneUpgradePlan[i+1] + } afterControlPlaneUpgradeTestHandler(ctx, input.BootstrapClusterProxy.GetClient(), clusterResources.Cluster, input.ExtensionConfigName, - controlPlaneVersion, - workersVersion, - input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade")) + controlPlaneVersion, // Current controlPlaneVersion for this upgrade step. + workersVersion, // Current workersVersion, used to check workers do not upgrade before its time. + nextControlPlaneVersion, // nextControlPlaneVersion in the upgrade plan, used to check the BeforeControlPlaneUpgrade is not called before its time (in case workers do not perform this upgrade step). + toVersion, // toVersion of the upgrade, used to check the AfterClusterUpgrade is not called before its time (in case workers do not perform this upgrade step). + ) // If worker should not upgrade at this step, continue if !sets.New[string](workersUpgradePlan...).Has(version) { continue } + // make sure beforeWorkersUpgrade still blocks, then unblock the upgrade. + beforeWorkersUpgradeTestHandler(ctx, + input.BootstrapClusterProxy.GetClient(), + clusterResources.Cluster, + input.ExtensionConfigName, + workersVersion, // Current workersVersion for this upgrade step. + version, // Workers toVersion for this upgrade step. + ) + // Wait for workers to update to version workersVersion = version waitWorkersVersions(ctx, input.BootstrapClusterProxy.GetClient(), clusterResources.Cluster, workersVersion, input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade")) - // TODO(chained-upgrade): FP, we can't check next CP upgrade doesn't start, it starts immediately + // make sure afterWorkersUpgradeTestHandler still blocks, then unblock the upgrade. + afterWorkersUpgradeTestHandler(ctx, + input.BootstrapClusterProxy.GetClient(), + clusterResources.Cluster, + input.ExtensionConfigName, + controlPlaneVersion, // Current controlPlaneVersion for this upgrade step. + workersVersion, // Current workersVersion for this upgrade step. + nextControlPlaneVersion, // nextControlPlaneVersion in the upgrade plan, used to check the BeforeControlPlaneUpgrade is not called before its time (in case workers do not perform this upgrade step). + toVersion, // toVersion of the upgrade, used to check the AfterClusterUpgrade is not called before its time (in case workers do not perform this upgrade step). + ) } }, }) + // Check if the AfterClusterUpgrade hook is actually called. + hookName := computeHookName("AfterClusterUpgrade", []string{toVersion}) + Byf("Waiting for %s hook to be called", hookName) + Eventually(func(_ Gomega) error { + return checkLifecycleHooksCalledAtLeastOnce(ctx, input.BootstrapClusterProxy.GetClient(), clusterResources.Cluster, input.ExtensionConfigName, "AfterClusterUpgrade", []string{toVersion}) + }, 30*time.Second, 2*time.Second).Should(Succeed(), "%s has not been called", hookName) + By("Waiting until nodes are ready") workloadProxy := input.BootstrapClusterProxy.GetWorkloadCluster(ctx, namespace.Name, clusterResources.Cluster.Name) workloadClient := workloadProxy.GetClient() @@ -405,20 +449,28 @@ func ClusterUpgradeWithRuntimeSDKSpec(ctx context.Context, inputGetter func() Cl By("Dumping resources and deleting the workload cluster; deletion waits for BeforeClusterDeleteHook to gate the operation") dumpAndDeleteCluster(ctx, input.BootstrapClusterProxy, input.ClusterctlConfigPath, namespace.Name, clusterName, input.ArtifactFolder) - beforeClusterDeleteHandler(ctx, input.BootstrapClusterProxy.GetClient(), clusterResources.Cluster, input.ExtensionConfigName, input.E2EConfig.GetIntervals(specName, "wait-delete-cluster")) + beforeClusterDeleteHandler(ctx, input.BootstrapClusterProxy.GetClient(), clusterResources.Cluster, input.ExtensionConfigName) By("Checking all lifecycle hooks have been called") // Assert that each hook has been called and returned "Success" during the test. expectedHooks := map[string]string{ - computeHookName("BeforeClusterCreate", nil): "Status: Success, RetryAfterSeconds: 0", - computeHookName("BeforeClusterUpgrade", nil): "Status: Success, RetryAfterSeconds: 0", - computeHookName("BeforeClusterDelete", nil): "Status: Success, RetryAfterSeconds: 0", - computeHookName("AfterControlPlaneUpgrade", []string{toVersion}): "Status: Success, RetryAfterSeconds: 0", - computeHookName("AfterControlPlaneInitialized", nil): "Success", - computeHookName("AfterClusterUpgrade", nil): "Success", + computeHookName("BeforeClusterCreate", nil): "Status: Success, RetryAfterSeconds: 0", + computeHookName("AfterControlPlaneInitialized", nil): "Success", + computeHookName("BeforeClusterUpgrade", []string{fromVersion, toVersion}): "Status: Success, RetryAfterSeconds: 0", + computeHookName("AfterClusterUpgrade", []string{toVersion}): "Success", + computeHookName("BeforeClusterDelete", nil): "Status: Success, RetryAfterSeconds: 0", } + fromv := fromVersion for _, v := range controlPlaneUpgradePlan { + expectedHooks[computeHookName("BeforeControlPlaneUpgrade", []string{fromv, v})] = "Status: Success, RetryAfterSeconds: 0" expectedHooks[computeHookName("AfterControlPlaneUpgrade", []string{v})] = "Status: Success, RetryAfterSeconds: 0" + fromv = v + } + fromv = fromVersion + for _, v := range workersUpgradePlan { + expectedHooks[computeHookName("BeforeWorkersUpgrade", []string{fromv, v})] = "Status: Success, RetryAfterSeconds: 0" + expectedHooks[computeHookName("AfterWorkersUpgrade", []string{v})] = "Status: Success, RetryAfterSeconds: 0" + fromv = v } checkLifecycleHookResponses(ctx, input.BootstrapClusterProxy.GetClient(), clusterResources.Cluster, input.ExtensionConfigName, expectedHooks) @@ -646,12 +698,12 @@ func getLifecycleHookResponsesFromConfigMap(ctx context.Context, c client.Client // beforeClusterCreateTestHandler calls runtimeHookTestHandler with a blockedCondition function which returns false if // the Cluster has a control plane or an infrastructure reference. -func beforeClusterCreateTestHandler(ctx context.Context, c client.Client, cluster *clusterv1.Cluster, extensionConfigName string, intervals []interface{}) { +func beforeClusterCreateTestHandler(ctx context.Context, c client.Client, cluster *clusterv1.Cluster, extensionConfigName string) { hookName := "BeforeClusterCreate" + // for BeforeClusterCreate, the hook is blocking if Get Cluster keep returning a cluster without infraCluster and controlPlaneRef set. isBlockingCreate := func() bool { blocked := true - // This hook should block the Cluster from entering the "Provisioned" state. cluster := framework.GetClusterByName(ctx, framework.GetClusterByNameInput{Name: cluster.Name, Namespace: cluster.Namespace, Getter: c}) @@ -660,19 +712,19 @@ func beforeClusterCreateTestHandler(ctx context.Context, c client.Client, cluste } return blocked } + runtimeHookTestHandler(ctx, c, cluster, extensionConfigName, hookName, nil, isBlockingCreate, nil) - // Test the BeforeClusterCreate the hook. - runtimeHookTestHandler(ctx, c, cluster, extensionConfigName, hookName, nil, isBlockingCreate, nil, intervals) - - Byf("ClusterCreate unblocked") + Byf("BeforeClusterCreate unblocked") } // beforeClusterUpgradeTestHandler calls runtimeHookTestHandler with a blocking function which returns false if // any of the control plane, machine deployments or machine pools has been updated from the initial Kubernetes version. -func beforeClusterUpgradeTestHandler(ctx context.Context, c client.Client, cluster *clusterv1.Cluster, extensionConfigName string, fromVersion, toVersion string, intervals []interface{}) { +func beforeClusterUpgradeTestHandler(ctx context.Context, c client.Client, cluster *clusterv1.Cluster, extensionConfigName string, fromVersion, toVersion, firstControlPlaneVersion string) { hookName := "BeforeClusterUpgrade" beforeClusterUpgradeAnnotation := clusterv1.BeforeClusterUpgradeHookAnnotationPrefix + "/upgrade-test" + // for BeforeClusterUpgrade, the hook is blocking if both controlPlane and workers remain at the expected version (both controlPlane and workers should be at fromVersion) + // and the BeforeControlPlaneUpgrade hook is not called yet. isBlockingUpgrade := func() bool { controlPlane, err := external.GetObjectFromContractVersionedRef(ctx, c, cluster.Spec.ControlPlaneRef, cluster.Namespace) if err != nil { @@ -686,6 +738,14 @@ func beforeClusterUpgradeTestHandler(ctx context.Context, c client.Client, clust return false } + controlPlaneMachines := framework.GetControlPlaneMachinesByCluster(ctx, + framework.GetControlPlaneMachinesByClusterInput{Lister: c, ClusterName: cluster.Name, Namespace: cluster.Namespace}) + for _, machine := range controlPlaneMachines { + if machine.Spec.Version != fromVersion { + return false + } + } + mds := framework.GetMachineDeploymentsByCluster(ctx, framework.GetMachineDeploymentsByClusterInput{ClusterName: cluster.Name, Namespace: cluster.Namespace, Lister: c}) for _, md := range mds { @@ -694,30 +754,90 @@ func beforeClusterUpgradeTestHandler(ctx context.Context, c client.Client, clust } } + mps := framework.GetMachinePoolsByCluster(ctx, + framework.GetMachinePoolsByClusterInput{ClusterName: cluster.Name, Namespace: cluster.Namespace, Lister: c}) + for _, mp := range mps { + if mp.Spec.Template.Spec.Version != fromVersion { + return false + } + } + + // Check if the BeforeControlPlaneUpgrade hook has been called (this should not happen when BeforeClusterUpgrade is blocking). + // Note: BeforeClusterUpgrade hook is followed by BeforeControlPlaneUpgrade hook. + if err := checkLifecycleHooksCalledAtLeastOnce(ctx, c, cluster, extensionConfigName, "BeforeControlPlaneUpgrade", []string{fromVersion, firstControlPlaneVersion}); err == nil { + return false + } + + return true + } + + // BeforeClusterUpgrade can be blocked via an annotation hook. Check it works + annotationHookTestHandler(ctx, c, cluster, hookName, beforeClusterUpgradeAnnotation, isBlockingUpgrade) + + // Test the BeforeClusterUpgrade hook. + runtimeHookTestHandler(ctx, c, cluster, extensionConfigName, hookName, []string{fromVersion, toVersion}, isBlockingUpgrade, nil) + + Byf("BeforeClusterUpgrade from %s to %s unblocked", fromVersion, toVersion) +} + +// beforeControlPlaneUpgradeTestHandler calls runtimeHookTestHandler with a blocking function which returns false if +// any of the control plane, machine deployments or machine pools has been updated from the initial Kubernetes version. +func beforeControlPlaneUpgradeTestHandler(ctx context.Context, c client.Client, cluster *clusterv1.Cluster, extensionConfigName string, fromVersion, toVersion, workersVersion string) { + hookName := "BeforeControlPlaneUpgrade" + + // for BeforeControlPlaneUpgrade, the hook is blocking if both controlPlane and workers remain at the expected version (both controlPlane and workers should be at fromVersion) + isBlockingUpgrade := func() bool { + controlPlane, err := external.GetObjectFromContractVersionedRef(ctx, c, cluster.Spec.ControlPlaneRef, cluster.Namespace) + if err != nil { + return false + } + controlPlaneVersion, err := contract.ControlPlane().Version().Get(controlPlane) + if err != nil { + return false + } + if *controlPlaneVersion != fromVersion { + return false + } + controlPlaneMachines := framework.GetControlPlaneMachinesByCluster(ctx, framework.GetControlPlaneMachinesByClusterInput{Lister: c, ClusterName: cluster.Name, Namespace: cluster.Namespace}) for _, machine := range controlPlaneMachines { - if machine.Spec.Version == toVersion { + if machine.Spec.Version != fromVersion { + return false + } + } + + mds := framework.GetMachineDeploymentsByCluster(ctx, + framework.GetMachineDeploymentsByClusterInput{ClusterName: cluster.Name, Namespace: cluster.Namespace, Lister: c}) + for _, md := range mds { + if md.Spec.Template.Spec.Version != workersVersion { + return false + } + } + + mps := framework.GetMachinePoolsByCluster(ctx, + framework.GetMachinePoolsByClusterInput{ClusterName: cluster.Name, Namespace: cluster.Namespace, Lister: c}) + for _, mp := range mps { + if mp.Spec.Template.Spec.Version != workersVersion { return false } } + return true } - // BeforeClusterUpgrade can be blocked via an annotation hook. Check it works - annotationHookTestHandler(ctx, c, cluster, hookName, beforeClusterUpgradeAnnotation, isBlockingUpgrade, intervals) - - // Test the BeforeClusterUpgrade the hook. - runtimeHookTestHandler(ctx, c, cluster, extensionConfigName, hookName, nil, isBlockingUpgrade, nil, intervals) + runtimeHookTestHandler(ctx, c, cluster, extensionConfigName, hookName, []string{fromVersion, toVersion}, isBlockingUpgrade, nil) - Byf("ClusterUpgrade to %s unblocked", toVersion) + Byf("BeforeControlPlaneUpgrade from %s to %s unblocked", fromVersion, toVersion) } // afterControlPlaneUpgradeTestHandler calls runtimeHookTestHandler with a blocking function which returns false if any // MachineDeployment in the Cluster has upgraded to the target Kubernetes version. -func afterControlPlaneUpgradeTestHandler(ctx context.Context, c client.Client, cluster *clusterv1.Cluster, extensionConfigName string, controlPlaneVersion, workersVersion string, intervals []interface{}) { +func afterControlPlaneUpgradeTestHandler(ctx context.Context, c client.Client, cluster *clusterv1.Cluster, extensionConfigName string, controlPlaneVersion, workersVersion, nextControlPlaneVersion, topologyVersion string) { hookName := "AfterControlPlaneUpgrade" + // for AfterControlPlaneUpgrade, the hook is blocking if both controlPlane and workers remain at the expected version (controlPlaneVersion, workersVersion) + // and the BeforeWorkersUpgrade hook or the BeforeControlPlaneUpgrade is not called yet. isBlockingUpgrade := func() bool { controlPlane, err := external.GetObjectFromContractVersionedRef(ctx, c, cluster.Spec.ControlPlaneRef, cluster.Namespace) if err != nil { @@ -731,6 +851,14 @@ func afterControlPlaneUpgradeTestHandler(ctx context.Context, c client.Client, c return false } + controlPlaneMachines := framework.GetControlPlaneMachinesByCluster(ctx, + framework.GetControlPlaneMachinesByClusterInput{Lister: c, ClusterName: cluster.Name, Namespace: cluster.Namespace}) + for _, machine := range controlPlaneMachines { + if machine.Spec.Version != controlPlaneVersion { + return false + } + } + mds := framework.GetMachineDeploymentsByCluster(ctx, framework.GetMachineDeploymentsByClusterInput{ClusterName: cluster.Name, Namespace: cluster.Namespace, Lister: c}) for _, md := range mds { @@ -746,6 +874,31 @@ func afterControlPlaneUpgradeTestHandler(ctx context.Context, c client.Client, c return false } } + + // Check if the BeforeWorkersUpgrade hook has been called (this should not happen when AfterControlPlaneUpgrade is blocking). + // Note: AfterControlPlaneUpgrade hook can be followed by BeforeWorkersUpgrade hook in case also workers are performing this upgrade step. + if err := checkLifecycleHooksCalledAtLeastOnce(ctx, c, cluster, extensionConfigName, "BeforeWorkersUpgrade", []string{workersVersion, controlPlaneVersion}); err == nil { + return false + } + + // Check if the BeforeControlPlaneUpgrade hook has been called (this should not happen when AfterControlPlaneUpgrade is blocking). + // Note: AfterControlPlaneUpgrade hook can be followed by BeforeControlPlaneUpgrade hook in case there are still upgrade steps to perform and workers are skipping this minor. + // Note: nextControlPlaneVersion != "" when there are still upgrade steps to perform. + if nextControlPlaneVersion != "" { + if err := checkLifecycleHooksCalledAtLeastOnce(ctx, c, cluster, extensionConfigName, "BeforeControlPlaneUpgrade", []string{controlPlaneVersion, nextControlPlaneVersion}); err == nil { + return false + } + } + + // Check if the AfterClusterUpgrade hook has been called (this should not happen when AfterControlPlaneUpgrade is blocking). + // Note: AfterControlPlaneUpgrade hook can be followed by AfterClusterUpgrade hook in case the upgrade is completed. + // Note: nextControlPlaneVersion == "" in case the upgrade is completed. + if nextControlPlaneVersion == "" { + if err := checkLifecycleHooksCalledAtLeastOnce(ctx, c, cluster, extensionConfigName, "AfterClusterUpgrade", []string{topologyVersion}); err == nil { + return false + } + } + return true } @@ -754,33 +907,155 @@ func afterControlPlaneUpgradeTestHandler(ctx context.Context, c client.Client, c } // Test the AfterControlPlaneUpgrade hook and perform machine set preflight checks before unblocking. - runtimeHookTestHandler(ctx, c, cluster, extensionConfigName, hookName, []string{controlPlaneVersion}, isBlockingUpgrade, beforeUnblockingUpgrade, intervals) + runtimeHookTestHandler(ctx, c, cluster, extensionConfigName, hookName, []string{controlPlaneVersion}, isBlockingUpgrade, beforeUnblockingUpgrade) + + Byf("AfterControlPlaneUpgrade to %s unblocked", controlPlaneVersion) +} + +// beforeWorkersUpgradeTestHandler calls runtimeHookTestHandler with a blocking function which returns false if +// any of the control plane, machine deployments or machine pools has been updated from the initial Kubernetes version. +func beforeWorkersUpgradeTestHandler(ctx context.Context, c client.Client, cluster *clusterv1.Cluster, extensionConfigName string, fromVersion, toVersion string) { + hookName := "BeforeWorkersUpgrade" + + // for BeforeWorkersUpgrade, the hook is blocking if both controlPlane and workers remain at the expected version (controlPlaneVersion should be already at toVersion, workers should be at fromVersion) + isBlockingUpgrade := func() bool { + controlPlane, err := external.GetObjectFromContractVersionedRef(ctx, c, cluster.Spec.ControlPlaneRef, cluster.Namespace) + if err != nil { + return false + } + controlPlaneVersion, err := contract.ControlPlane().Version().Get(controlPlane) + if err != nil { + return false + } + if *controlPlaneVersion != toVersion { + return false + } + + controlPlaneMachines := framework.GetControlPlaneMachinesByCluster(ctx, + framework.GetControlPlaneMachinesByClusterInput{Lister: c, ClusterName: cluster.Name, Namespace: cluster.Namespace}) + for _, machine := range controlPlaneMachines { + if machine.Spec.Version != toVersion { + return false + } + } + + mds := framework.GetMachineDeploymentsByCluster(ctx, + framework.GetMachineDeploymentsByClusterInput{ClusterName: cluster.Name, Namespace: cluster.Namespace, Lister: c}) + for _, md := range mds { + if md.Spec.Template.Spec.Version != fromVersion { + return false + } + } + + mps := framework.GetMachinePoolsByCluster(ctx, + framework.GetMachinePoolsByClusterInput{ClusterName: cluster.Name, Namespace: cluster.Namespace, Lister: c}) + for _, mp := range mps { + if mp.Spec.Template.Spec.Version != fromVersion { + return false + } + } + + return true + } + + runtimeHookTestHandler(ctx, c, cluster, extensionConfigName, hookName, []string{fromVersion, toVersion}, isBlockingUpgrade, nil) + + Byf("BeforeWorkersUpgrade from %s to %s unblocked", fromVersion, toVersion) +} + +// afterWorkersUpgradeTestHandler calls runtimeHookTestHandler with a blocking function which returns false if any +// MachineDeployment in the Cluster has upgraded to the target Kubernetes version. +func afterWorkersUpgradeTestHandler(ctx context.Context, c client.Client, cluster *clusterv1.Cluster, extensionConfigName string, controlPlaneVersion, workersVersion, nextControlPlaneVersion, topologyVersion string) { + hookName := "AfterWorkersUpgrade" + + // for AfterWorkersUpgrade, the hook is blocking if both controlPlane and workers remain at the expected version (controlPlaneVersion, workersVersion) + // and the BeforeControlPlaneUpgrade hook or the AfterClusterUpgrade are not called yet. + isBlockingUpgrade := func() bool { + controlPlane, err := external.GetObjectFromContractVersionedRef(ctx, c, cluster.Spec.ControlPlaneRef, cluster.Namespace) + if err != nil { + return false + } + v, err := contract.ControlPlane().Version().Get(controlPlane) + if err != nil { + return false + } + if *v != controlPlaneVersion { + return false + } + + controlPlaneMachines := framework.GetControlPlaneMachinesByCluster(ctx, + framework.GetControlPlaneMachinesByClusterInput{Lister: c, ClusterName: cluster.Name, Namespace: cluster.Namespace}) + for _, machine := range controlPlaneMachines { + if machine.Spec.Version != controlPlaneVersion { + return false + } + } + + mds := framework.GetMachineDeploymentsByCluster(ctx, + framework.GetMachineDeploymentsByClusterInput{ClusterName: cluster.Name, Namespace: cluster.Namespace, Lister: c}) + for _, md := range mds { + if md.Spec.Template.Spec.Version != workersVersion { + return false + } + } + + mps := framework.GetMachinePoolsByCluster(ctx, + framework.GetMachinePoolsByClusterInput{ClusterName: cluster.Name, Namespace: cluster.Namespace, Lister: c}) + for _, mp := range mps { + if mp.Spec.Template.Spec.Version != workersVersion { + return false + } + } + + // Check if the BeforeControlPlaneUpgrade hook has been called (this should not happen when AfterWorkersUpgrade is blocking). + // Note: AfterWorkersUpgrade hook can be followed by BeforeControlPlaneUpgrade hook in case there are still upgrade steps to perform. + // Note: nextControlPlaneVersion != "" when there are still upgrade steps to perform. + if nextControlPlaneVersion != "" { + if err := checkLifecycleHooksCalledAtLeastOnce(ctx, c, cluster, extensionConfigName, "BeforeControlPlaneUpgrade", []string{controlPlaneVersion, nextControlPlaneVersion}); err == nil { + return false + } + } + + // Check if the AfterClusterUpgrade hook has been called (this should not happen when AfterWorkersUpgrade is blocking). + // Note: AfterWorkersUpgrade hook can be followed by AfterClusterUpgrade hook in case the upgrade is completed. + // Note: nextControlPlaneVersion == "" in case the upgrade is completed. + if nextControlPlaneVersion == "" { + if err := checkLifecycleHooksCalledAtLeastOnce(ctx, c, cluster, extensionConfigName, "AfterClusterUpgrade", []string{topologyVersion}); err == nil { + return false + } + } + + return true + } + + runtimeHookTestHandler(ctx, c, cluster, extensionConfigName, hookName, []string{workersVersion}, isBlockingUpgrade, nil) + + Byf("AfterWorkersUpgrade to %s unblocked", workersVersion) } // beforeClusterDeleteHandler calls runtimeHookTestHandler with a blocking function which returns false if the Cluster // cannot be found in the API server. -func beforeClusterDeleteHandler(ctx context.Context, c client.Client, cluster *clusterv1.Cluster, extensionConfigName string, intervals []interface{}) { +func beforeClusterDeleteHandler(ctx context.Context, c client.Client, cluster *clusterv1.Cluster, extensionConfigName string) { hookName := "BeforeClusterDelete" + // for BeforeClusterDelete, the hook is blocking if Get Cluster keep returning something different from IsNotFound error. isBlockingDelete := func() bool { var blocked = true - // If the Cluster is not found, it has been deleted and the hook is unblocked. if apierrors.IsNotFound(c.Get(ctx, client.ObjectKey{Name: cluster.Name, Namespace: cluster.Namespace}, &clusterv1.Cluster{})) { blocked = false } return blocked } - // Test the BeforeClusterDelete the hook. - runtimeHookTestHandler(ctx, c, cluster, extensionConfigName, hookName, nil, isBlockingDelete, nil, intervals) + runtimeHookTestHandler(ctx, c, cluster, extensionConfigName, hookName, nil, isBlockingDelete, nil) } // annotationHookTestHandler runs a series of tests in sequence to check if the annotation hook can block. // 1. Check if the annotation hook is blocking and if the TopologyReconciled condition reports if the annotation hook is blocking. // 2. Remove the annotation hook. // 3. Check if the TopologyReconciled condition stops reporting the annotation hook is blocking. -func annotationHookTestHandler(ctx context.Context, c client.Client, cluster *clusterv1.Cluster, hook, annotation string, blockingCondition func() bool, intervals []interface{}) { +func annotationHookTestHandler(ctx context.Context, c client.Client, cluster *clusterv1.Cluster, hook, annotation string, blockingCondition func() bool) { log.Logf("Blocking with the %s annotation hook for 60 seconds", hook) expectedBlockingMessage := fmt.Sprintf("hook %q is blocking: annotation [%s] is set", hook, annotation) @@ -811,9 +1086,9 @@ func annotationHookTestHandler(ctx context.Context, c client.Client, cluster *cl return false } return true - }, 30*time.Second).Should(BeTrue(), "%s (via annotation %s) did not block", hook, annotation) + }, 20*time.Second, 2*time.Second).Should(BeTrue(), "%s (via annotation %s) did not block", hook, annotation) - Byf("Validating %s hook (via annotation %s) consistently blocks progress in the reconciliation", hook, annotation) + Byf("Validating %s hook (via annotation %s) consistently blocks progress in the upgrade", hook, annotation) // Check if the annotation hook keeps blocking. Consistently(func(_ Gomega) bool { @@ -824,7 +1099,7 @@ func annotationHookTestHandler(ctx context.Context, c client.Client, cluster *cl return false } return true - }, 60*time.Second).Should(BeTrue(), + }, 30*time.Second, 2*time.Second).Should(BeTrue(), fmt.Sprintf("Cluster Topology reconciliation continued unexpectedly: hook %s (via annotation %s) is not blocking", hook, annotation)) // Patch the Cluster to remove the LifecycleHook annotation hook and unblock. @@ -848,7 +1123,7 @@ func annotationHookTestHandler(ctx context.Context, c client.Client, cluster *cl return fmt.Errorf("hook %s (via annotation %s) should not be blocking anymore with message: %s", hook, annotation, expectedBlockingMessage) } return nil - }, intervals...).Should(Succeed(), + }, 20*time.Second, 2*time.Second).Should(Succeed(), fmt.Sprintf("ClusterTopology reconcile did not proceed as expected when unblocking hook %s (via annotation %s)", hook, annotation)) } @@ -859,7 +1134,7 @@ func annotationHookTestHandler(ctx context.Context, c client.Client, cluster *cl // // Note: runtimeHookTestHandler assumes that the hook passed to it is currently returning a blocking response. // Updating the response to be non-blocking happens inline in the function. -func runtimeHookTestHandler(ctx context.Context, c client.Client, cluster *clusterv1.Cluster, extensionConfigName string, hook string, attributes []string, blockingCondition func() bool, beforeUnblocking func(), intervals []interface{}) { +func runtimeHookTestHandler(ctx context.Context, c client.Client, cluster *clusterv1.Cluster, extensionConfigName string, hook string, attributes []string, blockingCondition func() bool, beforeUnblocking func()) { hookName := computeHookName(hook, attributes) log.Logf("Blocking with the %s hook for 60 seconds", hookName) @@ -887,14 +1162,14 @@ func runtimeHookTestHandler(ctx context.Context, c client.Client, cluster *clust return errors.Errorf("Blocking condition for %s not found on Cluster object", hookName) } return nil - }, 30*time.Second).Should(Succeed(), "%s has not been called", hookName) + }, 30*time.Second, 2*time.Second).Should(Succeed(), "%s has not been called", hookName) - Byf("Validating %s hook consistently blocks progress in the reconciliation", hookName) + Byf("Validating %s hook consistently blocks progress in the upgrade", hookName) // Check if the hook keeps blocking. Consistently(func(_ Gomega) bool { return topologyConditionCheck() && blockingCondition() - }, 60*time.Second).Should(BeTrue(), + }, 30*time.Second, 5*time.Second).Should(BeTrue(), fmt.Sprintf("Cluster Topology reconciliation continued unexpectedly: hook %s not blocking", hookName)) if beforeUnblocking != nil { @@ -902,7 +1177,7 @@ func runtimeHookTestHandler(ctx context.Context, c client.Client, cluster *clust } // Patch the ConfigMap to set the hook response to "Success". - Byf("Setting %s response to Status:Success to unblock the reconciliation", hookName) + Byf("Setting %s response to Status:Success to unblock the upgrade", hookName) configMap := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: hookResponsesConfigMapName(cluster.Name, extensionConfigName), Namespace: cluster.Namespace}} Eventually(func() error { @@ -912,7 +1187,7 @@ func runtimeHookTestHandler(ctx context.Context, c client.Client, cluster *clust []byte(fmt.Sprintf(`{"data":{"%s-preloadedResponse":%s}}`, hookName, "\"{\\\"Status\\\": \\\"Success\\\"}\""))) Eventually(func() error { return c.Patch(ctx, configMap, patchData) - }).Should(Succeed(), "Failed to set %s response to Status:Success to unblock the reconciliation", hookName) + }).Should(Succeed(), "Failed to set %s response to Status:Success to unblock the upgrade", hookName) // Check if the hook stops blocking and if the TopologyReconciled condition stops reporting the hook is blocking. @@ -925,7 +1200,7 @@ func runtimeHookTestHandler(ctx context.Context, c client.Client, cluster *clust return blockingCondition() } return topologyConditionCheck() || blockingCondition() - }, intervals...).Should(BeFalse(), + }, 30*time.Second, 2*time.Second).Should(BeFalse(), fmt.Sprintf("ClusterTopology reconcile did not proceed as expected when calling %s", hookName)) } @@ -942,10 +1217,7 @@ func clusterConditionShowsHookBlocking(cluster *clusterv1.Cluster, hookName stri func waitControlPlaneVersion(ctx context.Context, c client.Client, cluster *clusterv1.Cluster, version string, intervals []interface{}) { Byf("Waiting for control plane to have version %s", version) - controlPlaneVersion(ctx, c, cluster, version, intervals...) -} -func controlPlaneVersion(ctx context.Context, c client.Client, cluster *clusterv1.Cluster, version string, intervals ...interface{}) { Eventually(func(_ Gomega) bool { controlPlane, err := external.GetObjectFromContractVersionedRef(ctx, c, cluster.Spec.ControlPlaneRef, cluster.Namespace) if err != nil { diff --git a/test/extension/handlers/lifecycle/handlers.go b/test/extension/handlers/lifecycle/handlers.go index b3f5cb0abfcb..0c45cefc7fbc 100644 --- a/test/extension/handlers/lifecycle/handlers.go +++ b/test/extension/handlers/lifecycle/handlers.go @@ -83,6 +83,29 @@ func (m *ExtensionHandlers) DoBeforeClusterCreate(ctx context.Context, request * } } +// DoAfterControlPlaneInitialized implements the HandlerFunc for the AfterControlPlaneInitialized hook. +// The hook answers with the response stored in a well know config map, thus allowing E2E tests to +// control the hook behaviour during a test. +// NOTE: custom RuntimeExtension, must implement the body of this func according to the specific use case. +func (m *ExtensionHandlers) DoAfterControlPlaneInitialized(ctx context.Context, request *runtimehooksv1.AfterControlPlaneInitializedRequest, response *runtimehooksv1.AfterControlPlaneInitializedResponse) { + log := ctrl.LoggerFrom(ctx).WithValues("Cluster", klog.KObj(&request.Cluster)) + ctx = ctrl.LoggerInto(ctx, log) + log.Info("AfterControlPlaneInitialized is called") + + settings := request.GetSettings() + + if err := m.readResponseFromConfigMap(ctx, &request.Cluster, runtimehooksv1.AfterControlPlaneInitialized, nil, settings, response); err != nil { + response.Status = runtimehooksv1.ResponseStatusFailure + response.Message = err.Error() + return + } + + if err := m.recordCallInConfigMap(ctx, &request.Cluster, runtimehooksv1.AfterControlPlaneInitialized, nil, settings, response); err != nil { + response.Status = runtimehooksv1.ResponseStatusFailure + response.Message = err.Error() + } +} + // DoBeforeClusterUpgrade implements the HandlerFunc for the BeforeClusterUpgrade hook. // The hook answers with the response stored in a well know config map, thus allowing E2E tests to // control the hook behaviour during a test. @@ -92,38 +115,40 @@ func (m *ExtensionHandlers) DoBeforeClusterUpgrade(ctx context.Context, request ctx = ctrl.LoggerInto(ctx, log) log.Info("BeforeClusterUpgrade is called") + attributes := []string{request.FromKubernetesVersion, request.ToKubernetesVersion} settings := request.GetSettings() - if err := m.readResponseFromConfigMap(ctx, &request.Cluster, runtimehooksv1.BeforeClusterUpgrade, nil, settings, response); err != nil { + if err := m.readResponseFromConfigMap(ctx, &request.Cluster, runtimehooksv1.BeforeClusterUpgrade, attributes, settings, response); err != nil { response.Status = runtimehooksv1.ResponseStatusFailure response.Message = err.Error() return } - if err := m.recordCallInConfigMap(ctx, &request.Cluster, runtimehooksv1.BeforeClusterUpgrade, nil, settings, response); err != nil { + if err := m.recordCallInConfigMap(ctx, &request.Cluster, runtimehooksv1.BeforeClusterUpgrade, attributes, settings, response); err != nil { response.Status = runtimehooksv1.ResponseStatusFailure response.Message = err.Error() } } -// DoAfterControlPlaneInitialized implements the HandlerFunc for the AfterControlPlaneInitialized hook. +// DoBeforeControlPlaneUpgrade implements the HandlerFunc for the ControlPlaneUpgrade hook. // The hook answers with the response stored in a well know config map, thus allowing E2E tests to // control the hook behaviour during a test. // NOTE: custom RuntimeExtension, must implement the body of this func according to the specific use case. -func (m *ExtensionHandlers) DoAfterControlPlaneInitialized(ctx context.Context, request *runtimehooksv1.AfterControlPlaneInitializedRequest, response *runtimehooksv1.AfterControlPlaneInitializedResponse) { +func (m *ExtensionHandlers) DoBeforeControlPlaneUpgrade(ctx context.Context, request *runtimehooksv1.BeforeControlPlaneUpgradeRequest, response *runtimehooksv1.BeforeControlPlaneUpgradeResponse) { log := ctrl.LoggerFrom(ctx).WithValues("Cluster", klog.KObj(&request.Cluster)) ctx = ctrl.LoggerInto(ctx, log) - log.Info("AfterControlPlaneInitialized is called") + log.Info("BeforeControlPlaneUpgrade is called") + attributes := []string{request.FromKubernetesVersion, request.ToKubernetesVersion} settings := request.GetSettings() - if err := m.readResponseFromConfigMap(ctx, &request.Cluster, runtimehooksv1.AfterControlPlaneInitialized, nil, settings, response); err != nil { + if err := m.readResponseFromConfigMap(ctx, &request.Cluster, runtimehooksv1.BeforeControlPlaneUpgrade, attributes, settings, response); err != nil { response.Status = runtimehooksv1.ResponseStatusFailure response.Message = err.Error() return } - if err := m.recordCallInConfigMap(ctx, &request.Cluster, runtimehooksv1.AfterControlPlaneInitialized, nil, settings, response); err != nil { + if err := m.recordCallInConfigMap(ctx, &request.Cluster, runtimehooksv1.BeforeControlPlaneUpgrade, attributes, settings, response); err != nil { response.Status = runtimehooksv1.ResponseStatusFailure response.Message = err.Error() } @@ -153,6 +178,54 @@ func (m *ExtensionHandlers) DoAfterControlPlaneUpgrade(ctx context.Context, requ } } +// DoBeforeWorkersUpgrade implements the HandlerFunc for the WorkersUpgrade hook. +// The hook answers with the response stored in a well know config map, thus allowing E2E tests to +// control the hook behaviour during a test. +// NOTE: custom RuntimeExtension, must implement the body of this func according to the specific use case. +func (m *ExtensionHandlers) DoBeforeWorkersUpgrade(ctx context.Context, request *runtimehooksv1.BeforeWorkersUpgradeRequest, response *runtimehooksv1.BeforeWorkersUpgradeResponse) { + log := ctrl.LoggerFrom(ctx).WithValues("Cluster", klog.KObj(&request.Cluster)) + ctx = ctrl.LoggerInto(ctx, log) + log.Info("BeforeWorkersUpgrade is called") + + attributes := []string{request.FromKubernetesVersion, request.ToKubernetesVersion} + settings := request.GetSettings() + + if err := m.readResponseFromConfigMap(ctx, &request.Cluster, runtimehooksv1.BeforeWorkersUpgrade, attributes, settings, response); err != nil { + response.Status = runtimehooksv1.ResponseStatusFailure + response.Message = err.Error() + return + } + + if err := m.recordCallInConfigMap(ctx, &request.Cluster, runtimehooksv1.BeforeWorkersUpgrade, attributes, settings, response); err != nil { + response.Status = runtimehooksv1.ResponseStatusFailure + response.Message = err.Error() + } +} + +// DoAfterWorkersUpgrade implements the HandlerFunc for the AfterWorkersUpgrade hook. +// The hook answers with the response stored in a well know config map, thus allowing E2E tests to +// control the hook behaviour during a test. +// NOTE: custom RuntimeExtension, must implement the body of this func according to the specific use case. +func (m *ExtensionHandlers) DoAfterWorkersUpgrade(ctx context.Context, request *runtimehooksv1.AfterWorkersUpgradeRequest, response *runtimehooksv1.AfterWorkersUpgradeResponse) { + log := ctrl.LoggerFrom(ctx).WithValues("Cluster", klog.KObj(&request.Cluster)) + ctx = ctrl.LoggerInto(ctx, log) + log.Info("AfterWorkersUpgrade is called") + + attributes := []string{request.KubernetesVersion} + settings := request.GetSettings() + + if err := m.readResponseFromConfigMap(ctx, &request.Cluster, runtimehooksv1.AfterWorkersUpgrade, attributes, settings, response); err != nil { + response.Status = runtimehooksv1.ResponseStatusFailure + response.Message = err.Error() + return + } + + if err := m.recordCallInConfigMap(ctx, &request.Cluster, runtimehooksv1.AfterWorkersUpgrade, attributes, settings, response); err != nil { + response.Status = runtimehooksv1.ResponseStatusFailure + response.Message = err.Error() + } +} + // DoAfterClusterUpgrade implements the HandlerFunc for the AfterClusterUpgrade hook. // The hook answers with the response stored in a well know config map, thus allowing E2E tests to // control the hook behaviour during a test. @@ -162,15 +235,16 @@ func (m *ExtensionHandlers) DoAfterClusterUpgrade(ctx context.Context, request * ctx = ctrl.LoggerInto(ctx, log) log.Info("AfterClusterUpgrade is called") + attributes := []string{request.KubernetesVersion} settings := request.GetSettings() - if err := m.readResponseFromConfigMap(ctx, &request.Cluster, runtimehooksv1.AfterClusterUpgrade, nil, settings, response); err != nil { + if err := m.readResponseFromConfigMap(ctx, &request.Cluster, runtimehooksv1.AfterClusterUpgrade, attributes, settings, response); err != nil { response.Status = runtimehooksv1.ResponseStatusFailure response.Message = err.Error() return } - if err := m.recordCallInConfigMap(ctx, &request.Cluster, runtimehooksv1.AfterClusterUpgrade, nil, settings, response); err != nil { + if err := m.recordCallInConfigMap(ctx, &request.Cluster, runtimehooksv1.AfterClusterUpgrade, attributes, settings, response); err != nil { response.Status = runtimehooksv1.ResponseStatusFailure response.Message = err.Error() } @@ -240,8 +314,14 @@ func (m *ExtensionHandlers) readResponseFromConfigMap(ctx context.Context, clust data = fmt.Sprintf(`{"Status": "Success", "RetryAfterSeconds": %d}`, retryAfterSeconds) case "BeforeClusterUpgrade": data = fmt.Sprintf(`{"Status": "Success", "RetryAfterSeconds": %d}`, retryAfterSeconds) + case "BeforeControlPlaneUpgrade": + data = fmt.Sprintf(`{"Status": "Success", "RetryAfterSeconds": %d}`, retryAfterSeconds) case "AfterControlPlaneUpgrade": data = fmt.Sprintf(`{"Status": "Success", "RetryAfterSeconds": %d}`, retryAfterSeconds) + case "BeforeWorkersUpgrade": + data = fmt.Sprintf(`{"Status": "Success", "RetryAfterSeconds": %d}`, retryAfterSeconds) + case "AfterWorkersUpgrade": + data = fmt.Sprintf(`{"Status": "Success", "RetryAfterSeconds": %d}`, retryAfterSeconds) case "BeforeClusterDelete": data = fmt.Sprintf(`{"Status": "Success", "RetryAfterSeconds": %d}`, retryAfterSeconds) diff --git a/test/extension/main.go b/test/extension/main.go index 5507252f73c2..3408921a17c3 100644 --- a/test/extension/main.go +++ b/test/extension/main.go @@ -344,6 +344,15 @@ func setupLifecycleHookHandlers(mgr ctrl.Manager, runtimeExtensionWebhookServer os.Exit(1) } + if err := runtimeExtensionWebhookServer.AddExtensionHandler(server.ExtensionHandler{ + Hook: runtimehooksv1.BeforeControlPlaneUpgrade, + Name: "before-control-plane-upgrade", + HandlerFunc: lifecycleExtensionHandlers.DoBeforeControlPlaneUpgrade, + }); err != nil { + setupLog.Error(err, "Error adding handler") + os.Exit(1) + } + if err := runtimeExtensionWebhookServer.AddExtensionHandler(server.ExtensionHandler{ Hook: runtimehooksv1.AfterControlPlaneUpgrade, Name: "after-control-plane-upgrade", @@ -353,6 +362,24 @@ func setupLifecycleHookHandlers(mgr ctrl.Manager, runtimeExtensionWebhookServer os.Exit(1) } + if err := runtimeExtensionWebhookServer.AddExtensionHandler(server.ExtensionHandler{ + Hook: runtimehooksv1.BeforeWorkersUpgrade, + Name: "before-workers-upgrade", + HandlerFunc: lifecycleExtensionHandlers.DoBeforeWorkersUpgrade, + }); err != nil { + setupLog.Error(err, "Error adding handler") + os.Exit(1) + } + + if err := runtimeExtensionWebhookServer.AddExtensionHandler(server.ExtensionHandler{ + Hook: runtimehooksv1.AfterWorkersUpgrade, + Name: "after-workers-upgrade", + HandlerFunc: lifecycleExtensionHandlers.DoAfterWorkersUpgrade, + }); err != nil { + setupLog.Error(err, "Error adding handler") + os.Exit(1) + } + if err := runtimeExtensionWebhookServer.AddExtensionHandler(server.ExtensionHandler{ Hook: runtimehooksv1.AfterClusterUpgrade, Name: "after-cluster-upgrade",