Test chart upgrades (#103)

* Test chart upgrades against previous version

Upgrade testing will be run if the --upgrade flag is set (default true)
and chart version increment does not indicate a breaking change
according to the SemVer 2.0 spec.

Any releases associated with previous chart versions which fail to roll
out or for which an initial `helm test` fails will be ignored.

Signed-off-by: Jacob LeGrone <[email protected]>

* fix(dep): add version constraint for github.com/otiai10/copy

Signed-off-by: Jacob LeGrone <[email protected]>

* refactor(git): checkout whole repository with worktree

Signed-off-by: Jacob LeGrone <[email protected]>

* Rename test* to do*

Signed-off-by: Jacob LeGrone <[email protected]>

* Return bool, error from BreakingChangeAllowed

Signed-off-by: Jacob LeGrone <[email protected]>

* Use errors.Wrapf

Co-Authored-By: jlegrone <[email protected]>
Signed-off-by: Jacob LeGrone <[email protected]>

* Explicitly disable upgrade when not install

Signed-off-by: Jacob LeGrone <[email protected]>

Author: jlegrone

Gopkg.lock

@@ -31,13 +31,17 @@ func newInstallCmd() *cobra.Command {
 		Use:   "install",
 		Short: "Install and test a chart",
 		Long: heredoc.Doc(`
-			Run 'helm install' and ' helm test' on
+			Run 'helm install', 'helm test', and optionally 'helm upgrade' on
 
 			* changed charts (default)
 			* specific charts (--charts)
 			* all charts (--all)
 
-			in given chart directories.
+			in given chart directories. If upgrade (--upgrade) is true, then this
+			command will validate that 'helm test' passes for the following upgrade paths:
+
+			* previous chart revision => current chart version (if non-breaking SemVer change)
+			* current chart version => current chart version
 
 			Charts may have multiple custom values files matching the glob pattern
 			'*-values.yaml' in a directory named 'ci' in the root of the chart's
@@ -61,12 +65,15 @@ func addInstallFlags(flags *flag.FlagSet) {
 	flags.String("helm-extra-args", "", heredoc.Doc(`
 		Additional arguments for Helm. Must be passed as a single quoted string
 		(e.g. "--timeout 500 --tiller-namespace tiller"`))
+	flags.Bool("upgrade", false, heredoc.Doc(`
+		Whether to test an in-place upgrade of each chart from its previous revision if the
+		current version should not introduce a breaking change according to the SemVer spec`))
 	flags.String("namespace", "", heredoc.Doc(`
 		Namespace to install the release(s) into. If not specified, each release will be
-		installed in its own randomly generated namespace.`))
+		installed in its own randomly generated namespace`))
 	flags.String("release-label", "app.kubernetes.io/instance", heredoc.Doc(`
 		The label to be used as a selector when inspecting resources created by charts.
-		This is only used if namespace is specified.`))
+		This is only used if namespace is specified`))
 }
 
 func install(cmd *cobra.Command, args []string) {

ct/cmd/install.go

@@ -16,9 +16,10 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/MakeNowJust/heredoc"
 	"os"
 
+	"github.com/MakeNowJust/heredoc"
+
 	"github.com/helm/chart-testing/pkg/chart"
 	"github.com/helm/chart-testing/pkg/config"
 	"github.com/spf13/cobra"

ct/cmd/listChanged.go

@@ -26,4 +26,4 @@ in given chart directories.
 * [ct list-changed](ct_list-changed.md)	 - List changed charts
 * [ct version](ct_version.md)	 - Print version information
 
-###### Auto generated by spf13/cobra on 31-Jan-2019
+###### Auto generated by spf13/cobra on 26-Feb-2019

doc/ct.md

@@ -4,13 +4,17 @@ Install and test a chart
 
 ### Synopsis
 
-Run 'helm install' and ' helm test' on
+Run 'helm install', 'helm test', and optionally 'helm upgrade' on
 
 * changed charts (default)
 * specific charts (--charts)
 * all charts (--all)
 
-in given chart directories.
+in given chart directories. If upgrade (--upgrade) is true, then this
+command will validate that 'helm test' passes for the following upgrade paths:
+
+* previous chart revision => current chart version (if non-breaking SemVer change)
+* current chart version => current chart version
 
 Charts may have multiple custom values files matching the glob pattern
 '*-values.yaml' in a directory named 'ci' in the root of the chart's
@@ -51,15 +55,17 @@ ct install [flags]
                                        multiple times or separate values with commas
   -h, --help                           help for install
       --namespace string               Namespace to install the release(s) into. If not specified, each release will be
-                                       installed in its own randomly generated namespace.
+                                       installed in its own randomly generated namespace
       --release-label string           The label to be used as a selector when inspecting resources created by charts.
-                                       This is only used if namespace is specified. (default "app.kubernetes.io/instance")
+                                       This is only used if namespace is specified (default "app.kubernetes.io/instance")
       --remote string                  The name of the Git remote used to identify changed charts (default "origin")
       --target-branch string           The name of the target branch used to identify changed charts (default "master")
+      --upgrade                        Whether to test an in-place upgrade of each chart from its previous revision if the
+                                       current version should not introduce a breaking change according to the SemVer spec
 ```
 
 ### SEE ALSO
 
 * [ct](ct.md)	 - The Helm chart testing tool
 
-###### Auto generated by spf13/cobra on 31-Jan-2019
+###### Auto generated by spf13/cobra on 26-Feb-2019

doc/ct_install.md

@@ -46,11 +46,13 @@ ct lint-and-install [flags]
                                        is searched in the current directory, '$HOME/.ct', and '/etc/ct', in
                                        that order
       --namespace string               Namespace to install the release(s) into. If not specified, each release will be
-                                       installed in its own randomly generated namespace.
+                                       installed in its own randomly generated namespace
       --release-label string           The label to be used as a selector when inspecting resources created by charts.
-                                       This is only used if namespace is specified. (default "app.kubernetes.io/instance")
+                                       This is only used if namespace is specified (default "app.kubernetes.io/instance")
       --remote string                  The name of the Git remote used to identify changed charts (default "origin")
       --target-branch string           The name of the target branch used to identify changed charts (default "master")
+      --upgrade                        Whether to test an in-place upgrade of each chart from its previous revision if the
+                                       current version should not introduce a breaking change according to the SemVer spec
       --validate-chart-schema          Enable schema validation of 'Chart.yaml' using Yamale (default: true) (default true)
       --validate-maintainers           Enable validation of maintainer account names in chart.yml (default: true).
                                        Works for GitHub, GitLab, and Bitbucket (default true)
@@ -61,4 +63,4 @@ ct lint-and-install [flags]
 
 * [ct](ct.md)	 - The Helm chart testing tool
 
-###### Auto generated by spf13/cobra on 31-Jan-2019
+###### Auto generated by spf13/cobra on 26-Feb-2019

doc/ct_lint-and-install.md

@@ -65,4 +65,4 @@ ct lint [flags]
 
 * [ct](ct.md)	 - The Helm chart testing tool
 
-###### Auto generated by spf13/cobra on 31-Jan-2019
+###### Auto generated by spf13/cobra on 26-Feb-2019

doc/ct_lint.md

@@ -28,4 +28,4 @@ ct list-changed [flags]
 
 * [ct](ct.md)	 - The Helm chart testing tool
 
-###### Auto generated by spf13/cobra on 31-Jan-2019
+###### Auto generated by spf13/cobra on 26-Feb-2019

doc/ct_list-changed.md

@@ -20,4 +20,4 @@ ct version [flags]
 
 * [ct](ct.md)	 - The Helm chart testing tool
 
-###### Auto generated by spf13/cobra on 31-Jan-2019
+###### Auto generated by spf13/cobra on 26-Feb-2019