End-to-end test conditional remediation

config/crd/bases/helm.toolkit.fluxcd.io_helmreleases.yaml

@@ -111,6 +111,28 @@ spec:
                     description: DisableWait disables the waiting for resources to
                       be ready after a Helm install has been performed.
                     type: boolean
+                  remediation:
+                    description: Remediation holds the remediation configuration for
+                      when the Helm install action for the HelmRelease fails. The
+                      default is to not perform any action.
+                    properties:
+                      ignoreTestFailures:
+                        description: IgnoreTestFailures tells the controller to skip
+                          remediation when the Helm tests are run after an install
+                          action but fail. Defaults to 'Test.IgnoreFailures'.
+                        type: boolean
+                      remediateLastFailure:
+                        description: RemediateLastFailure tells the controller to
+                          remediate the last failure, when no retries remain. Defaults
+                          to 'false'.
+                        type: boolean
+                      retries:
+                        description: Retries is the number of retries that should
+                          be attempted on failures before bailing. Remediation, using
+                          an uninstall, is performed between each attempt. Defaults
+                          to '0', a negative integer equals to unlimited retries.
+                        type: integer
+                    type: object
                   replace:
                     description: Replace tells the Helm install action to re-use the
                       'ReleaseName', but only if that name is a deleted release which
@@ -154,10 +176,6 @@ spec:
                     description: DisableWait disables the waiting for resources to
                       be ready after a Helm rollback has been performed.
                     type: boolean
-                  enable:
-                    description: Enable enables Helm rollback actions for this HelmRelease
-                      after an Helm install or upgrade action failure.
-                    type: boolean
                   force:
                     description: Force forces resource updates through a replacement
                       strategy.
@@ -189,6 +207,12 @@ spec:
                     description: Enable enables Helm test actions for this HelmRelease
                       after an Helm install or upgrade action has been performed.
                     type: boolean
+                  ignoreFailures:
+                    description: IgnoreFailures tells the controller to skip remediation
+                      when the Helm tests are run but fail. Can be overwritten for
+                      tests run after install or upgrade actions in 'Install.IgnoreTestFailures'
+                      and 'Upgrade.IgnoreTestFailures'.
+                    type: boolean
                   timeout:
                     description: Timeout is the time to wait for any individual Kubernetes
                       operation during the performance of a Helm test action. Defaults
@@ -243,16 +267,40 @@ spec:
                     description: Force forces resource updates through a replacement
                       strategy.
                     type: boolean
-                  maxRetries:
-                    description: MaxRetries is the number of retries that should be
-                      attempted on failures before bailing. Defaults to '0', a negative
-                      integer equals to unlimited retries.
-                    type: integer
                   preserveValues:
                     description: PreserveValues will make Helm reuse the last release's
                       values and merge in overrides from 'Values'. Setting this flag
                       makes the HelmRelease non-declarative.
                     type: boolean
+                  remediation:
+                    description: Remediation holds the remediation configuration for
+                      when the Helm upgrade action for the HelmRelease fails. The
+                      default is to not perform any action.
+                    properties:
+                      ignoreTestFailures:
+                        description: IgnoreTestFailures tells the controller to skip
+                          remediation when the Helm tests are run after an upgrade
+                          action but fail. Defaults to 'Test.IgnoreFailures'.
+                        type: boolean
+                      remediateLastFailure:
+                        description: RemediateLastFailure tells the controller to
+                          remediate the last failure, when no retries remain. Defaults
+                          to 'false' unless 'Retries' is greater than 0.
+                        type: boolean
+                      retries:
+                        description: Retries is the number of retries that should
+                          be attempted on failures before bailing. Remediation, using
+                          'Strategy', is performed between each attempt. Defaults
+                          to '0', a negative integer equals to unlimited retries.
+                        type: integer
+                      strategy:
+                        description: Strategy to use for failure remediation. Defaults
+                          to 'rollback'.
+                        enum:
+                        - rollback
+                        - uninstall
+                        type: string
+                    type: object
                   timeout:
                     description: Timeout is the time to wait for any individual Kubernetes
                       operation (like Jobs for hooks) during the performance of a
@@ -336,14 +384,17 @@ spec:
                   type: object
                 type: array
               failures:
-                description: Failures is the reconciliation failure count. It is reset
-                  after a successful reconciliation.
+                description: Failures is the reconciliation failure count.
                 format: int64
                 type: integer
               helmChart:
                 description: HelmChart is the namespaced name of the HelmChart resource
                   created by the controller for the HelmRelease.
                 type: string
+              installFailures:
+                description: InstallFailures is the install failure count.
+                format: int64
+                type: integer
               lastAppliedRevision:
                 description: LastAppliedRevision is the revision of the last successfully
                   applied source.
@@ -364,6 +415,10 @@ spec:
                 description: ObservedGeneration is the last reconciled generation.
                 format: int64
                 type: integer
+              upgradeFailures:
+                description: UpgradeFailures is the upgrade failure count.
+                format: int64
+                type: integer
             type: object
         type: object
     served: true

config/manager/kustomization.yaml

@@ -1,7 +1,7 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - deployment.yaml
+- deployment.yaml
 images:
   - name: fluxcd/helm-controller
     newName: fluxcd/helm-controller

config/testdata/dependencies/helmrelease-backend.yaml

@@ -11,10 +11,6 @@ spec:
       kind: HelmRepository
       name: webapp
     interval: 1m
-  test:
-    enable: true
-  rollback:
-    enable: true
   values:
     service:
       grpcService: backend

config/testdata/dependencies/helmrelease-frontend.yaml

@@ -13,10 +13,6 @@ spec:
     interval: 1m
   dependsOn:
     - backend
-  test:
-    enable: true
-  rollback:
-    enable: true
   values:
     backend: http://backend-podinfo:9898/echo
     resources:

config/testdata/install-fail-remediate/helmrelease.yaml

@@ -0,0 +1,27 @@
+apiVersion: helm.toolkit.fluxcd.io/v2alpha1
+kind: HelmRelease
+metadata:
+  name: install-fail-remediate
+spec:
+  interval: 5m
+  chart:
+    name: podinfo
+    version: '^4.0.0'
+    sourceRef:
+      kind: HelmRepository
+      name: podinfo
+    interval: 1m
+  install:
+    remediation:
+      remediateLastFailure: true
+  values:
+    resources:
+      requests:
+        cpu: 100m
+        memory: 64Mi
+    # Make wait fail. With single replica helm doesn't actually wait, see:
+    # https://github.com/helm/helm/issues/5814#issuecomment-567130226
+    replicaCount: 2
+    faults:
+      unready: true
+  timeout: 3s

config/testdata/install-fail-retry/helmrelease.yaml

@@ -0,0 +1,27 @@
+apiVersion: helm.toolkit.fluxcd.io/v2alpha1
+kind: HelmRelease
+metadata:
+  name: install-fail-retry
+spec:
+  interval: 5m
+  chart:
+    name: podinfo
+    version: '^4.0.0'
+    sourceRef:
+      kind: HelmRepository
+      name: podinfo
+    interval: 1m
+  install:
+    remediation:
+      retries: 1
+  values:
+    resources:
+      requests:
+        cpu: 100m
+        memory: 64Mi
+    # Make wait fail. With single replica helm doesn't actually wait, see:
+    # https://github.com/helm/helm/issues/5814#issuecomment-567130226
+    replicaCount: 2
+    faults:
+      unready: true
+  timeout: 3s

config/testdata/install-fail/helmrelease.yaml

@@ -0,0 +1,24 @@
+apiVersion: helm.toolkit.fluxcd.io/v2alpha1
+kind: HelmRelease
+metadata:
+  name: install-fail
+spec:
+  interval: 5m
+  chart:
+    name: podinfo
+    version: '^4.0.0'
+    sourceRef:
+      kind: HelmRepository
+      name: podinfo
+    interval: 1m
+  values:
+    resources:
+      requests:
+        cpu: 100m
+        memory: 64Mi
+    # Make wait fail. With single replica helm doesn't actually wait, see:
+    # https://github.com/helm/helm/issues/5814#issuecomment-567130226
+    replicaCount: 2
+    faults:
+      unready: true
+  timeout: 3s

config/testdata/install-test-fail-ignore/helmrelease.yaml

@@ -0,0 +1,23 @@
+apiVersion: helm.toolkit.fluxcd.io/v2alpha1
+kind: HelmRelease
+metadata:
+  name: install-test-fail-ignore
+spec:
+  interval: 5m
+  chart:
+    name: podinfo
+    version: '^4.0.0'
+    sourceRef:
+      kind: HelmRepository
+      name: podinfo
+    interval: 1m
+  test:
+    enable: true
+    ignoreFailures: true
+  values:
+    resources:
+      requests:
+        cpu: 100m
+        memory: 64Mi
+    faults:
+      testFail: true

config/testdata/install-test-fail/helmrelease.yaml

@@ -0,0 +1,22 @@
+apiVersion: helm.toolkit.fluxcd.io/v2alpha1
+kind: HelmRelease
+metadata:
+  name: install-test-fail
+spec:
+  interval: 5m
+  chart:
+    name: podinfo
+    version: '^4.0.0'
+    sourceRef:
+      kind: HelmRepository
+      name: podinfo
+    interval: 1m
+  test:
+    enable: true
+  values:
+    resources:
+      requests:
+        cpu: 100m
+        memory: 64Mi
+    faults:
+      testFail: true

config/testdata/podinfo/helmrelease.yaml

@@ -11,10 +11,6 @@ spec:
       kind: HelmRepository
       name: podinfo
     interval: 1m
-  test:
-    enable: true
-  rollback:
-    enable: true
   values:
     resources:
       requests:

config/testdata/upgrade-fail-remediate-uninstall/install.yaml

@@ -0,0 +1,18 @@
+apiVersion: helm.toolkit.fluxcd.io/v2alpha1
+kind: HelmRelease
+metadata:
+  name: upgrade-fail-remediate-uninstall
+spec:
+  interval: 5m
+  chart:
+    name: podinfo
+    version: '^4.0.0'
+    sourceRef:
+      kind: HelmRepository
+      name: podinfo
+    interval: 1m
+  values:
+    resources:
+      requests:
+        cpu: 100m
+        memory: 64Mi

config/testdata/upgrade-fail-remediate-uninstall/upgrade.yaml

@@ -0,0 +1,28 @@
+apiVersion: helm.toolkit.fluxcd.io/v2alpha1
+kind: HelmRelease
+metadata:
+  name: upgrade-fail-remediate-uninstall
+spec:
+  interval: 5m
+  chart:
+    name: podinfo
+    version: '^4.0.0'
+    sourceRef:
+      kind: HelmRepository
+      name: podinfo
+    interval: 1m
+  upgrade:
+    remediation:
+      remediateLastFailure: true
+      strategy: uninstall
+  values:
+    resources:
+      requests:
+        cpu: 100m
+        memory: 64Mi
+    # Make wait fail. With single replica helm doesn't actually wait, see:
+    # https://github.com/helm/helm/issues/5814#issuecomment-567130226
+    replicaCount: 2
+    faults:
+      unready: true
+  timeout: 3s

config/testdata/upgrade-fail-remediate/install.yaml

@@ -0,0 +1,18 @@
+apiVersion: helm.toolkit.fluxcd.io/v2alpha1
+kind: HelmRelease
+metadata:
+  name: upgrade-fail-remediate
+spec:
+  interval: 5m
+  chart:
+    name: podinfo
+    version: '^4.0.0'
+    sourceRef:
+      kind: HelmRepository
+      name: podinfo
+    interval: 1m
+  values:
+    resources:
+      requests:
+        cpu: 100m
+        memory: 64Mi

config/testdata/upgrade-fail-remediate/upgrade.yaml

@@ -0,0 +1,27 @@
+apiVersion: helm.toolkit.fluxcd.io/v2alpha1
+kind: HelmRelease
+metadata:
+  name: upgrade-fail-remediate
+spec:
+  interval: 5m
+  chart:
+    name: podinfo
+    version: '^4.0.0'
+    sourceRef:
+      kind: HelmRepository
+      name: podinfo
+    interval: 1m
+  upgrade:
+    remediation:
+      remediateLastFailure: true
+  values:
+    resources:
+      requests:
+        cpu: 100m
+        memory: 64Mi
+    # Make wait fail. With single replica helm doesn't actually wait, see:
+    # https://github.com/helm/helm/issues/5814#issuecomment-567130226
+    replicaCount: 2
+    faults:
+      unready: true
+  timeout: 3s