Implement forceAt and resetAt annotations

This makes the controller actually take the
`reconcile.fluxcd.io/forceAt` and `reconcile.fluxcd.io/resetAt` into
account.

For `reconcile.fluxcd.io/resetAt`, this means that the failure counts on
the `HelmRelease` object are reset when the token value of the
annotation equals `reconcile.fluxcd.io/requestedAt`. Allowing the
controller to start over with attempting to install or upgrade the
release until the retries count has been reached again.

For `reconcile.fluxcd.io/forceAt`, this means that a one-off Helm
install or upgrade is allowed to take place even if the object is out of
retries, in a failed state where it should be remediated, or in-sync.

Signed-off-by: Hidde Beydals <[email protected]>

Author: hiddeco

internal/action/reset.go

@@ -29,6 +29,7 @@ const (
 	differentGenerationReason = "generation differs from last attempt"
 	differentRevisionReason   = "chart version differs from last attempt"
 	differentValuesReason     = "values differ from last attempt"
+	resetRequestedReason      = "reset requested through annotation"
 )
 
 // MustResetFailures returns a reason and true if the HelmRelease's status
@@ -38,6 +39,12 @@ const (
 // For example, a change in generation, chart version, or values.
 // If no change is detected, an empty string is returned along with false.
 func MustResetFailures(obj *v2.HelmRelease, chart *chart.Metadata, values chartutil.Values) (string, bool) {
+	// Always check if a reset is requested.
+	// This is done first, so that the HelmReleaseStatus.LastHandledResetAt
+	// field is updated even if the reset request is not handled due to other
+	// diverging data.
+	resetRequested := v2.ShouldHandleResetRequest(obj)
+
 	switch {
 	case obj.Status.LastAttemptedGeneration != obj.Generation:
 		return differentGenerationReason, true
@@ -53,5 +60,10 @@ func MustResetFailures(obj *v2.HelmRelease, chart *chart.Metadata, values chartu
 			return differentValuesReason, true
 		}
 	}
+
+	if resetRequested {
+		return resetRequestedReason, true
+	}
+
 	return "", false
 }

internal/action/reset_test.go

@@ -24,6 +24,8 @@ import (
 	"helm.sh/helm/v3/pkg/chartutil"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
+	"github.com/fluxcd/pkg/apis/meta"
+
 	v2 "github.com/fluxcd/helm-controller/api/v2beta2"
 )
 
@@ -108,6 +110,31 @@ func TestMustResetFailures(t *testing.T) {
 			want:       true,
 			wantReason: differentValuesReason,
 		},
+		{
+			name: "on reset request through annotation",
+			obj: &v2.HelmRelease{
+				ObjectMeta: metav1.ObjectMeta{
+					Generation: 1,
+					Annotations: map[string]string{
+						meta.ReconcileRequestAnnotation: "a",
+						v2.ResetRequestAnnotation:       "a",
+					},
+				},
+				Status: v2.HelmReleaseStatus{
+					LastAttemptedGeneration:   1,
+					LastAttemptedRevision:     "1.0.0",
+					LastAttemptedConfigDigest: "sha256:1dabc4e3cbbd6a0818bd460f3a6c9855bfe95d506c74726bc0f2edb0aecb1f4e",
+				},
+			},
+			chart: &chart.Metadata{
+				Version: "1.0.0",
+			},
+			values: chartutil.Values{
+				"foo": "bar",
+			},
+			want:       true,
+			wantReason: resetRequestedReason,
+		},
 		{
 			name: "without change no reset",
 			obj: &v2.HelmRelease{

internal/controller/helmrelease_controller.go

@@ -154,6 +154,10 @@ func (r *HelmReleaseReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 
 	// Always attempt to patch the object after each reconciliation.
 	defer func() {
+		if v, ok := meta.ReconcileAnnotationValue(obj.GetAnnotations()); ok {
+			obj.Status.SetLastHandledReconcileRequest(v)
+		}
+
 		patchOpts := []patch.Option{
 			patch.WithFieldOwner(r.FieldManager),
 			patch.WithOwnedConditions{Conditions: intreconcile.OwnedConditions},

internal/reconcile/atomic_release.go

@@ -174,7 +174,7 @@ func (r *AtomicRelease) Reconcile(ctx context.Context, req *Request) error {
 			}
 			return fmt.Errorf("atomic release canceled: %w", ctx.Err())
 		default:
-			// Determine the next action to run based on the current state.
+			// Determine the current state of the Helm release.
 			log.V(logger.DebugLevel).Info("determining current state of Helm release")
 			state, err := DetermineReleaseState(ctx, r.configFactory, req)
 			if err != nil {
@@ -272,6 +272,13 @@ func (r *AtomicRelease) Reconcile(ctx context.Context, req *Request) error {
 func (r *AtomicRelease) actionForState(ctx context.Context, req *Request, state ReleaseState) (ActionReconciler, error) {
 	log := ctrl.LoggerFrom(ctx)
 
+	// Determine whether we may need to force a release action.
+	// We do this before determining the next action to run, as otherwise we may
+	// end up running a Helm upgrade (due to e.g. ReleaseStatusUnmanaged) and
+	// then forcing an upgrade (due to the release now being in
+	// ReleaseStatusInSync with a yet unhandled force request).
+	forceRequested := v2.ShouldHandleForceRequest(req.Object)
+
 	switch state.Status {
 	case ReleaseStatusInSync:
 		log.Info("release in-sync with desired state")
@@ -290,6 +297,11 @@ func (r *AtomicRelease) actionForState(ctx context.Context, req *Request, state
 		//  field, but should be removed in a future release.
 		req.Object.Status.LastAppliedRevision = req.Object.Status.History.Latest().ChartVersion
 
+		if forceRequested {
+			log.Info(msgWithReason("forcing upgrade for in-sync release", "force requested through annotation"))
+			return NewUpgrade(r.configFactory, r.eventRecorder), nil
+		}
+
 		return nil, nil
 	case ReleaseStatusLocked:
 		log.Info(msgWithReason("release locked", state.Reason))
@@ -298,6 +310,11 @@ func (r *AtomicRelease) actionForState(ctx context.Context, req *Request, state
 		log.Info(msgWithReason("release not installed", state.Reason))
 
 		if req.Object.GetInstall().GetRemediation().RetriesExhausted(req.Object) {
+			if forceRequested {
+				log.Info(msgWithReason("forcing install while out of retries", "force requested through annotation"))
+				return NewInstall(r.configFactory, r.eventRecorder), nil
+			}
+
 			return nil, fmt.Errorf("%w: cannot install release", ErrExceededMaxRetries)
 		}
 
@@ -313,6 +330,11 @@ func (r *AtomicRelease) actionForState(ctx context.Context, req *Request, state
 		log.Info(msgWithReason("release out-of-sync with desired state", state.Reason))
 
 		if req.Object.GetUpgrade().GetRemediation().RetriesExhausted(req.Object) {
+			if forceRequested {
+				log.Info(msgWithReason("forcing upgrade while out of retries", "force requested through annotation"))
+				return NewUpgrade(r.configFactory, r.eventRecorder), nil
+			}
+
 			return nil, fmt.Errorf("%w: cannot upgrade release", ErrExceededMaxRetries)
 		}
 
@@ -360,6 +382,13 @@ func (r *AtomicRelease) actionForState(ctx context.Context, req *Request, state
 			return NewUpgrade(r.configFactory, r.eventRecorder), nil
 		}
 
+		// If the force annotation is set, we can attempt to upgrade the release
+		// without any further checks.
+		if forceRequested {
+			log.Info(msgWithReason("forcing upgrade for failed release", "force requested through annotation"))
+			return NewUpgrade(r.configFactory, r.eventRecorder), nil
+		}
+
 		// We have exhausted the number of retries for the remediation
 		// strategy.
 		if remediation.RetriesExhausted(req.Object) && !remediation.MustRemediateLastFailure() {

internal/reconcile/atomic_release_test.go

@@ -1015,14 +1015,15 @@ func TestAtomicRelease_Reconcile_Scenarios(t *testing.T) {
 
 func TestAtomicRelease_actionForState(t *testing.T) {
 	tests := []struct {
-		name      string
-		releases  []*helmrelease.Release
-		spec      func(spec *v2.HelmReleaseSpec)
-		status    func(releases []*helmrelease.Release) v2.HelmReleaseStatus
-		state     ReleaseState
-		want      ActionReconciler
-		wantEvent *corev1.Event
-		wantErr   error
+		name        string
+		releases    []*helmrelease.Release
+		annotations map[string]string
+		spec        func(spec *v2.HelmReleaseSpec)
+		status      func(releases []*helmrelease.Release) v2.HelmReleaseStatus
+		state       ReleaseState
+		want        ActionReconciler
+		wantEvent   *corev1.Event
+		wantErr     error
 	}{
 		{
 			name: "in-sync release does not trigger any action",
@@ -1036,6 +1037,22 @@ func TestAtomicRelease_actionForState(t *testing.T) {
 			state: ReleaseState{Status: ReleaseStatusInSync},
 			want:  nil,
 		},
+		{
+			name:  "in-sync release with force annotation triggers upgrade action",
+			state: ReleaseState{Status: ReleaseStatusInSync},
+			annotations: map[string]string{
+				meta.ReconcileRequestAnnotation: "force",
+				v2.ForceRequestAnnotation:       "force",
+			},
+			status: func(releases []*helmrelease.Release) v2.HelmReleaseStatus {
+				return v2.HelmReleaseStatus{
+					History: v2.Snapshots{
+						{Version: 1},
+					},
+				}
+			},
+			want: &Upgrade{},
+		},
 		{
 			name:  "locked release triggers unlock action",
 			state: ReleaseState{Status: ReleaseStatusLocked},
@@ -1046,6 +1063,20 @@ func TestAtomicRelease_actionForState(t *testing.T) {
 			state: ReleaseState{Status: ReleaseStatusAbsent},
 			want:  &Install{},
 		},
+		{
+			name: "absent release without remaining retries and force annotation triggers install",
+			annotations: map[string]string{
+				meta.ReconcileRequestAnnotation: "force",
+				v2.ForceRequestAnnotation:       "force",
+			},
+			state: ReleaseState{Status: ReleaseStatusAbsent},
+			status: func(releases []*helmrelease.Release) v2.HelmReleaseStatus {
+				return v2.HelmReleaseStatus{
+					InstallFailures: 1,
+				}
+			},
+			want: &Install{},
+		},
 		{
 			name:  "absent release without remaining retries returns error",
 			state: ReleaseState{Status: ReleaseStatusAbsent},
@@ -1159,6 +1190,22 @@ func TestAtomicRelease_actionForState(t *testing.T) {
 			},
 			want: &Upgrade{},
 		},
+		{
+			name: "out-of-sync release with no remaining retries and force annotation triggers upgrade",
+			state: ReleaseState{
+				Status: ReleaseStatusOutOfSync,
+			},
+			annotations: map[string]string{
+				meta.ReconcileRequestAnnotation: "force",
+				v2.ForceRequestAnnotation:       "force",
+			},
+			status: func(releases []*helmrelease.Release) v2.HelmReleaseStatus {
+				return v2.HelmReleaseStatus{
+					UpgradeFailures: 1,
+				}
+			},
+			want: &Upgrade{},
+		},
 		{
 			name: "out-of-sync release with no remaining retries returns error",
 			state: ReleaseState{
@@ -1198,6 +1245,21 @@ func TestAtomicRelease_actionForState(t *testing.T) {
 			},
 			want: &Upgrade{},
 		},
+		{
+			name:  "failed release with exhausted retries and force annotation triggers upgrade",
+			state: ReleaseState{Status: ReleaseStatusFailed},
+			annotations: map[string]string{
+				meta.ReconcileRequestAnnotation: "force",
+				v2.ForceRequestAnnotation:       "force",
+			},
+			status: func(releases []*helmrelease.Release) v2.HelmReleaseStatus {
+				return v2.HelmReleaseStatus{
+					LastAttemptedReleaseAction: v2.ReleaseActionUpgrade,
+					UpgradeFailures:            1,
+				}
+			},
+			want: &Upgrade{},
+		},
 		{
 			name:  "failed release with exhausted retries returns error",
 			state: ReleaseState{Status: ReleaseStatusFailed},
@@ -1348,6 +1410,9 @@ func TestAtomicRelease_actionForState(t *testing.T) {
 			g := NewWithT(t)
 
 			obj := &v2.HelmRelease{
+				ObjectMeta: metav1.ObjectMeta{
+					Annotations: tt.annotations,
+				},
 				Spec: v2.HelmReleaseSpec{
 					ReleaseName:      mockReleaseName,
 					TargetNamespace:  mockReleaseNamespace,