feat: experiment, adds a command to generate a helmfile.yaml from a j…

…x-applications.yml

this is an experimenal feature which uses a seperate jx-apps.yml file to incorporate adding apps to a boot install.

relates to #6442

Signed-off-by: James Rawlings <[email protected]>

.secrets.baseline

@@ -3,7 +3,7 @@
     "files": "^pkg/jenkins/test_data/update_center.json.*$|^.secrets.baseline$|^.*test.*$",
     "lines": null
   },
-  "generated_at": "2020-01-20T14:45:13Z",
+  "generated_at": "2020-01-25T11:13:56Z",
   "plugins_used": [
     {
       "name": "AWSKeyDetector"

docs/contributing/experiments.md

@@ -3,12 +3,20 @@
 This page contains a list of experiments that are being worked on and how to enable them.  This list will be maintained and
 it will likely change how features are enabled as they mature past an experiment. 
 
+Experiments tend to go hand in hand with the Jenkins X enhancement process of which more details can be found in the git
+repository [https://github.com/jenkins-x/enhancements](https://github.com/jenkins-x/enhancements)
+
+# Current Experiments
+
 ## Helmfile
 
+https://github.com/jenkins-x/enhancements/pull/1
+
 We are experimenting with [Helmfile](https://github.com/roboll/helmfile) to see if we can make the `jx boot` implementation
-a bit more modular and leverage some of the extra fetures the OSS project has.  To enable this feature set the `JX_HELMFILE`
-environment variable:
+a bit more modular and leverage some of the extra fetures the OSS project has.  To enable this feature set a top level
+jx requirements value:
+
+```yaml
+helmfile: true
+```
 
-```bash
-export JX_HELMFILE=true
-```

pkg/cmd/cmd.go

@@ -91,6 +91,7 @@ func NewJXCommand(f clients.Factory, in terminal.FileReader, out terminal.FileWr
 	addCommands := add.NewCmdAdd(commonOpts)
 	createCommands := create.NewCmdCreate(commonOpts)
 	deleteCommands := deletecmd.NewCmdDelete(commonOpts)
+
 	getCommands := get.NewCmdGet(commonOpts)
 	editCommands := edit.NewCmdEdit(commonOpts)
 	updateCommands := update.NewCmdUpdate(commonOpts)

pkg/cmd/create/create.go

@@ -1,6 +1,7 @@
 package create
 
 import (
+	"github.com/jenkins-x/jx/pkg/cmd/create/helmfile"
 	"github.com/jenkins-x/jx/pkg/cmd/create/options"
 	"github.com/jenkins-x/jx/pkg/cmd/create/vault"
 	"github.com/jenkins-x/jx/pkg/cmd/helper"
@@ -70,6 +71,7 @@ func NewCmdCreate(commonOpts *opts.CommonOptions) *cobra.Command {
 	cmd.AddCommand(NewCmdCreateEtcHosts(commonOpts))
 	cmd.AddCommand(NewCmdCreateGkeServiceAccount(commonOpts))
 	cmd.AddCommand(NewCmdCreateGit(commonOpts))
+	cmd.AddCommand(helmfile.NewCmdCreateHelmfile(commonOpts))
 	cmd.AddCommand(NewCmdCreateIssue(commonOpts))
 	cmd.AddCommand(NewCmdCreateJenkins(commonOpts))
 	cmd.AddCommand(NewCmdCreateJHipster(commonOpts))

pkg/cmd/create/helmfile/create_helmfile.go

@@ -0,0 +1,185 @@
+package helmfile
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/url"
+	"path"
+
+	"github.com/jenkins-x/jx/pkg/config"
+	helmfile2 "github.com/jenkins-x/jx/pkg/helmfile"
+
+	"github.com/google/uuid"
+	"github.com/jenkins-x/jx/pkg/util"
+
+	"github.com/ghodss/yaml"
+
+	"github.com/jenkins-x/jx/pkg/cmd/create/options"
+	"github.com/jenkins-x/jx/pkg/cmd/helper"
+	"github.com/jenkins-x/jx/pkg/cmd/opts"
+	"github.com/jenkins-x/jx/pkg/cmd/templates"
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+)
+
+const (
+	helmfile = "helmfile.yaml"
+)
+
+var (
+	createHelmfileLong = templates.LongDesc(`
+		** EXPERIMENTAL COMMAND **
+
+		Creates a new helmfile.yaml from a jx-apps.yaml
+`)
+
+	createHelmfileExample = templates.Examples(`
+		** EXPERIMENTAL COMMAND **
+
+		# Create a new helmfile.yaml from a jx-apps.yaml
+		jx create helmfile
+	`)
+)
+
+// CreateHelmfileOptions the options for the create helmfile command
+type CreateHelmfileOptions struct {
+	options.CreateOptions
+	outputDir  string
+	dir        string
+	valueFiles []string
+}
+
+// NewCmdCreateHelmfile  creates a command object for the "create" command
+func NewCmdCreateHelmfile(commonOpts *opts.CommonOptions) *cobra.Command {
+	o := &CreateHelmfileOptions{
+		CreateOptions: options.CreateOptions{
+			CommonOptions: commonOpts,
+		},
+	}
+
+	cmd := &cobra.Command{
+		Use:     "helmfile",
+		Short:   "Create a new helmfile",
+		Long:    createHelmfileLong,
+		Example: createHelmfileExample,
+		Run: func(cmd *cobra.Command, args []string) {
+			o.Cmd = cmd
+			o.Args = args
+			err := o.Run()
+			helper.CheckErr(err)
+		},
+	}
+	cmd.Flags().StringVarP(&o.dir, "dir", "", ".", "the directory to look for a 'jx-apps.yml' file")
+	cmd.Flags().StringVarP(&o.outputDir, "outputDir", "", "", "The directory to write the helmfile.yaml file")
+	cmd.Flags().StringArrayVarP(&o.valueFiles, "values", "", []string{""}, "specify values in a YAML file or a URL(can specify multiple)")
+
+	return cmd
+}
+
+// Run implements the command
+func (o *CreateHelmfileOptions) Run() error {
+
+	apps, err := config.LoadApplicationsConfig(o.dir)
+	if err != nil {
+		return errors.Wrap(err, "failed to load applications")
+	}
+
+	helm := o.Helm()
+	localHelmRepos, err := helm.ListRepos()
+	if err != nil {
+		return errors.Wrap(err, "failed listing helm repos")
+	}
+
+	// contains the repo url and name to reference it by in the release spec
+	// use a map to dedupe repositories
+	repos := make(map[string]string)
+	for _, app := range apps.Applications {
+		_, err = url.ParseRequestURI(app.Repository)
+		if err != nil {
+			// if the repository isn't a valid URL lets just use whatever was supplied in the application repository field, probably it is a directory path
+			repos[app.Repository] = app.Repository
+		} else {
+			matched := false
+			// check if URL matches a repo in helms local list
+			for key, value := range localHelmRepos {
+				if app.Repository == value {
+					repos[app.Repository] = key
+					matched = true
+				}
+			}
+			if !matched {
+				repos[app.Repository] = uuid.New().String()
+			}
+		}
+	}
+
+	var repositories []helmfile2.RepositorySpec
+	for repoURL, name := range repos {
+		_, err = url.ParseRequestURI(repoURL)
+		// skip non URLs as they're probably local directories which don't need to be in the helmfile.repository section
+		if err == nil {
+			repository := helmfile2.RepositorySpec{
+				Name: name,
+				URL:  repoURL,
+			}
+			repositories = append(repositories, repository)
+		}
+
+	}
+
+	var releases []helmfile2.ReleaseSpec
+	for _, app := range apps.Applications {
+		if app.Namespace == "" {
+			app.Namespace = apps.DefaultNamespace
+		}
+
+		// check if a local directory and values file exists for the app
+		extraValuesFiles := o.valueFiles
+		extraValuesFiles = o.addExtraAppValues(app, extraValuesFiles, "values.yaml")
+		extraValuesFiles = o.addExtraAppValues(app, extraValuesFiles, "values.yaml.gotmpl")
+
+		chartName := fmt.Sprintf("%s/%s", repos[app.Repository], app.Name)
+		release := helmfile2.ReleaseSpec{
+			Name:      app.Name,
+			Namespace: app.Namespace,
+			Chart:     chartName,
+			Values:    extraValuesFiles,
+		}
+		releases = append(releases, release)
+	}
+
+	h := helmfile2.HelmState{
+		Bases: []string{"../environments.yaml"},
+		HelmDefaults: helmfile2.HelmSpec{
+			Atomic:  true,
+			Verify:  false,
+			Wait:    true,
+			Timeout: 180,
+			// need Force to be false https://github.com/helm/helm/issues/6378
+			Force: false,
+		},
+		Repositories: repositories,
+		Releases:     releases,
+	}
+
+	data, err := yaml.Marshal(h)
+	if err != nil {
+		return err
+	}
+
+	err = ioutil.WriteFile(path.Join(o.outputDir, helmfile), data, util.DefaultWritePermissions)
+	if err != nil {
+		return errors.Wrapf(err, "failed to save file %s", helmfile)
+	}
+
+	return nil
+}
+
+func (o *CreateHelmfileOptions) addExtraAppValues(app config.Application, newValuesFiles []string, valuesFilename string) []string {
+	fileName := path.Join(o.dir, "apps", app.Name, valuesFilename)
+	exists, _ := util.FileExists(fileName)
+	if exists {
+		newValuesFiles = append(newValuesFiles, path.Join(app.Name, valuesFilename))
+	}
+	return newValuesFiles
+}

pkg/cmd/create/helmfile/create_helmfile_test.go

@@ -0,0 +1,130 @@
+package helmfile
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	helmfile2 "github.com/jenkins-x/jx/pkg/helmfile"
+
+	"github.com/jenkins-x/jx/pkg/cmd/create/options"
+	"github.com/jenkins-x/jx/pkg/cmd/opts"
+	helm_test "github.com/jenkins-x/jx/pkg/helm/mocks"
+
+	"github.com/jenkins-x/jx/pkg/util"
+	"github.com/pkg/errors"
+	"github.com/stretchr/testify/assert"
+	"gopkg.in/yaml.v2"
+)
+
+func TestDedupeRepositories(t *testing.T) {
+	tempDir, err := ioutil.TempDir("", "test-applications-config")
+	assert.NoError(t, err, "should create a temporary config dir")
+
+	o := &CreateHelmfileOptions{
+		outputDir:     tempDir,
+		dir:           "test_data",
+		CreateOptions: *getCreateOptions(),
+	}
+	err = o.Run()
+	assert.NoError(t, err)
+
+	h, err := loadHelmfile(tempDir)
+	assert.NoError(t, err)
+
+	// assert there are 3 repos and not 4 as one of them in the jx-applications.yaml is a duplicate
+	assert.Equal(t, 3, len(h.Repositories))
+
+}
+
+func TestExtraAppValues(t *testing.T) {
+	tempDir, err := ioutil.TempDir("", "test-applications-config")
+	assert.NoError(t, err, "should create a temporary config dir")
+
+	o := &CreateHelmfileOptions{
+		outputDir:     tempDir,
+		dir:           path.Join("test_data", "extra-values"),
+		CreateOptions: *getCreateOptions(),
+	}
+	err = o.Run()
+	assert.NoError(t, err)
+
+	h, err := loadHelmfile(tempDir)
+	assert.NoError(t, err)
+
+	// assert we added the local values.yaml for the velero app
+	assert.Equal(t, "velero/values.yaml", h.Releases[0].Values[0])
+
+}
+
+func TestExtraFlagValues(t *testing.T) {
+	tempDir, err := ioutil.TempDir("", "test-applications-config")
+	assert.NoError(t, err, "should create a temporary config dir")
+
+	o := &CreateHelmfileOptions{
+		outputDir:     tempDir,
+		dir:           path.Join("test_data"),
+		valueFiles:    []string{"foo/bar.yaml"},
+		CreateOptions: *getCreateOptions(),
+	}
+	err = o.Run()
+	assert.NoError(t, err)
+
+	h, err := loadHelmfile(tempDir)
+	assert.NoError(t, err)
+
+	// assert we added the values file passed in as a CLI flag
+	assert.Equal(t, "foo/bar.yaml", h.Releases[0].Values[0])
+
+}
+
+func loadHelmfile(dir string) (*helmfile2.HelmState, error) {
+
+	fileName := helmfile
+	if dir != "" {
+		fileName = filepath.Join(dir, helmfile)
+	}
+
+	exists, err := util.FileExists(fileName)
+	if err != nil || !exists {
+		return nil, errors.Errorf("no %s found in directory %s", fileName, dir)
+	}
+
+	config := &helmfile2.HelmState{}
+
+	data, err := ioutil.ReadFile(fileName)
+	if err != nil {
+		return config, fmt.Errorf("Failed to load file %s due to %s", fileName, err)
+	}
+	validationErrors, err := util.ValidateYaml(config, data)
+	if err != nil {
+		return config, fmt.Errorf("failed to validate YAML file %s due to %s", fileName, err)
+	}
+	if len(validationErrors) > 0 {
+		return config, fmt.Errorf("Validation failures in YAML file %s:\n%s", fileName, strings.Join(validationErrors, "\n"))
+	}
+	err = yaml.Unmarshal(data, config)
+	if err != nil {
+		return config, fmt.Errorf("Failed to unmarshal YAML file %s due to %s", fileName, err)
+	}
+
+	return config, err
+}
+
+func getCreateOptions() *options.CreateOptions {
+
+	helmer := helm_test.NewMockHelmer()
+	co := &opts.CommonOptions{
+		In:  os.Stdin,
+		Out: os.Stdout,
+		Err: os.Stderr,
+	}
+	co.SetHelm(helmer)
+	return &options.CreateOptions{
+		CommonOptions: co,
+	}
+}

pkg/cmd/create/helmfile/test_data/extra-values/apps/velero/values.yaml

@@ -0,0 +1 @@
+foo: bar

pkg/cmd/create/helmfile/test_data/extra-values/jx-apps.yml

@@ -0,0 +1,5 @@
+defaultNamespace: jx
+applications:
+- name: velero
+  repository: https://kubernetes-charts.storage.googleapis.com
+  namespace: velero