diff --git a/applicationset/controllers/applicationset_controller.go b/applicationset/controllers/applicationset_controller.go index 99ffa1c617902..a14148a5820a2 100644 --- a/applicationset/controllers/applicationset_controller.go +++ b/applicationset/controllers/applicationset_controller.go @@ -430,7 +430,7 @@ func (r *ApplicationSetReconciler) generateApplications(applicationSetInfo argop var applicationSetReason argoprojiov1alpha1.ApplicationSetReasonType for _, requestedGenerator := range applicationSetInfo.Spec.Generators { - t, err := generators.Transform(requestedGenerator, r.Generators, applicationSetInfo.Spec.Template, &applicationSetInfo) + t, err := generators.Transform(requestedGenerator, r.Generators, applicationSetInfo.Spec.Template, &applicationSetInfo, map[string]string{}) if err != nil { log.WithError(err).WithField("generator", requestedGenerator). Error("error generating application from params") diff --git a/applicationset/generators/generator_spec_processor.go b/applicationset/generators/generator_spec_processor.go index 02091c5d30d5e..634dc69895db3 100644 --- a/applicationset/generators/generator_spec_processor.go +++ b/applicationset/generators/generator_spec_processor.go @@ -1,12 +1,14 @@ package generators import ( + "encoding/json" + "github.com/argoproj/argo-cd/v2/applicationset/utils" + "github.com/valyala/fasttemplate" "reflect" + argoprojiov1alpha1 "github.com/argoproj/argo-cd/v2/pkg/apis/applicationset/v1alpha1" "github.com/imdario/mergo" log "github.com/sirupsen/logrus" - - argoprojiov1alpha1 "github.com/argoproj/argo-cd/v2/pkg/apis/applicationset/v1alpha1" ) type TransformResult struct { @@ -15,9 +17,10 @@ type TransformResult struct { } //Transform a spec generator to list of paramSets and a template -func Transform(requestedGenerator argoprojiov1alpha1.ApplicationSetGenerator, allGenerators map[string]Generator, baseTemplate argoprojiov1alpha1.ApplicationSetTemplate, appSet *argoprojiov1alpha1.ApplicationSet) ([]TransformResult, error) { +func Transform(requestedGenerator argoprojiov1alpha1.ApplicationSetGenerator, allGenerators map[string]Generator, baseTemplate argoprojiov1alpha1.ApplicationSetTemplate, appSet *argoprojiov1alpha1.ApplicationSet, genParams map[string]string) ([]TransformResult, error) { res := []TransformResult{} var firstError error + interpolatedGenerator := requestedGenerator.DeepCopy() generators := GetRelevantGenerators(&requestedGenerator, allGenerators) for _, g := range generators { @@ -31,8 +34,20 @@ func Transform(requestedGenerator argoprojiov1alpha1.ApplicationSetGenerator, al } continue } - - params, err := g.GenerateParams(&requestedGenerator, appSet) + var params []map[string]string + if len(genParams) != 0 { + tempInterpolatedGenerator, err := interpolateGenerator(&requestedGenerator, genParams) + interpolatedGenerator = &tempInterpolatedGenerator + if err != nil { + log.WithError(err).WithField("genParams", genParams). + Error("error interpolating params for generator") + if firstError == nil { + firstError = err + } + continue + } + } + params, err = g.GenerateParams(interpolatedGenerator, appSet) if err != nil { log.WithError(err).WithField("generator", g). Error("error generating params") @@ -46,11 +61,9 @@ func Transform(requestedGenerator argoprojiov1alpha1.ApplicationSetGenerator, al Params: params, Template: mergedTemplate, }) - } return res, firstError - } func GetRelevantGenerators(requestedGenerator *argoprojiov1alpha1.ApplicationSetGenerator, generators map[string]Generator) []Generator { @@ -81,3 +94,29 @@ func mergeGeneratorTemplate(g Generator, requestedGenerator *argoprojiov1alpha1. return *dest, err } + +// Currently for Matrix Generator. Allows interpolating the matrix's 2nd child generator with values from the 1st child generator +// "params" parameter is an array, where each index corresponds to a generator. Each index contains a map w/ that generator's parameters. +func interpolateGenerator(requestedGenerator *argoprojiov1alpha1.ApplicationSetGenerator, params map[string]string) (argoprojiov1alpha1.ApplicationSetGenerator, error) { + interpolatedGenerator := requestedGenerator.DeepCopy() + tmplBytes, err := json.Marshal(interpolatedGenerator) + if err != nil { + log.WithError(err).WithField("requestedGenerator", interpolatedGenerator).Error("error marshalling requested generator for interpolation") + return *interpolatedGenerator, err + } + + render := utils.Render{} + fstTmpl := fasttemplate.New(string(tmplBytes), "{{", "}}") + replacedTmplStr, err := render.Replace(fstTmpl, params, true) + if err != nil { + log.WithError(err).WithField("interpolatedGeneratorString", replacedTmplStr).Error("error interpolating generator with other generator's parameter") + return *interpolatedGenerator, err + } + + err = json.Unmarshal([]byte(replacedTmplStr), interpolatedGenerator) + if err != nil { + log.WithError(err).WithField("requestedGenerator", interpolatedGenerator).Error("error unmarshalling requested generator for interpolation") + return *interpolatedGenerator, err + } + return *interpolatedGenerator, nil +} diff --git a/applicationset/generators/generator_spec_processor_test.go b/applicationset/generators/generator_spec_processor_test.go new file mode 100644 index 0000000000000..96e8497b482ee --- /dev/null +++ b/applicationset/generators/generator_spec_processor_test.go @@ -0,0 +1,180 @@ +package generators + +import ( + "context" + log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + kubefake "k8s.io/client-go/kubernetes/fake" + crtclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "testing" + + argoprojiov1alpha1 "github.com/argoproj/argo-cd/v2/pkg/apis/applicationset/v1alpha1" +) + +func getMockClusterGenerator() Generator { + clusters := []crtclient.Object{ + &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "staging-01", + Namespace: "namespace", + Labels: map[string]string{ + "argocd.argoproj.io/secret-type": "cluster", + "environment": "staging", + "org": "foo", + }, + Annotations: map[string]string{ + "foo.argoproj.io": "staging", + }, + }, + Data: map[string][]byte{ + "config": []byte("{}"), + "name": []byte("staging-01"), + "server": []byte("https://staging-01.example.com"), + }, + Type: corev1.SecretType("Opaque"), + }, + &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "production-01", + Namespace: "namespace", + Labels: map[string]string{ + "argocd.argoproj.io/secret-type": "cluster", + "environment": "production", + "org": "bar", + }, + Annotations: map[string]string{ + "foo.argoproj.io": "production", + }, + }, + Data: map[string][]byte{ + "config": []byte("{}"), + "name": []byte("production_01/west"), + "server": []byte("https://production-01.example.com"), + }, + Type: corev1.SecretType("Opaque"), + }, + } + runtimeClusters := []runtime.Object{} + appClientset := kubefake.NewSimpleClientset(runtimeClusters...) + + fakeClient := fake.NewClientBuilder().WithObjects(clusters...).Build() + return NewClusterGenerator(fakeClient, context.Background(), appClientset, "namespace") +} + +func getMockGitGenerator() Generator { + argoCDServiceMock := argoCDServiceMock{mock: &mock.Mock{}} + argoCDServiceMock.mock.On("GetDirectories", mock.Anything, mock.Anything, mock.Anything).Return([]string{"app1", "app2", "app_3", "p1/app4"}, nil) + var gitGenerator = NewGitGenerator(argoCDServiceMock) + return gitGenerator +} + +func TestGetRelevantGenerators(t *testing.T) { + + testGenerators := map[string]Generator{ + "Clusters": getMockClusterGenerator(), + "Git": getMockGitGenerator(), + } + + testGenerators["Matrix"] = NewMatrixGenerator(testGenerators) + testGenerators["Merge"] = NewMergeGenerator(testGenerators) + testGenerators["List"] = NewListGenerator() + + requestedGenerator := &argoprojiov1alpha1.ApplicationSetGenerator{ + List: &argoprojiov1alpha1.ListGenerator{ + Elements: []apiextensionsv1.JSON{{Raw: []byte(`{"cluster": "cluster","url": "url","values":{"foo":"bar"}}`)}}, + }} + + relevantGenerators := GetRelevantGenerators(requestedGenerator, testGenerators) + assert.Len(t, relevantGenerators, 1) + assert.IsType(t, &ListGenerator{}, relevantGenerators[0]) + + requestedGenerator = &argoprojiov1alpha1.ApplicationSetGenerator{ + Clusters: &argoprojiov1alpha1.ClusterGenerator{ + Selector: metav1.LabelSelector{}, + Template: argoprojiov1alpha1.ApplicationSetTemplate{}, + Values: nil, + }, + } + + relevantGenerators = GetRelevantGenerators(requestedGenerator, testGenerators) + assert.Len(t, relevantGenerators, 1) + assert.IsType(t, &ClusterGenerator{}, relevantGenerators[0]) + + requestedGenerator = &argoprojiov1alpha1.ApplicationSetGenerator{ + Git: &argoprojiov1alpha1.GitGenerator{ + RepoURL: "", + Directories: nil, + Files: nil, + Revision: "", + RequeueAfterSeconds: nil, + Template: argoprojiov1alpha1.ApplicationSetTemplate{}, + }, + } + + relevantGenerators = GetRelevantGenerators(requestedGenerator, testGenerators) + assert.Len(t, relevantGenerators, 1) + assert.IsType(t, &GitGenerator{}, relevantGenerators[0]) +} + +func TestInterpolateGenerator(t *testing.T) { + requestedGenerator := &argoprojiov1alpha1.ApplicationSetGenerator{ + Clusters: &argoprojiov1alpha1.ClusterGenerator{ + Selector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + "argocd.argoproj.io/secret-type": "cluster", + "path-basename": "{{path.basename}}", + "path-zero": "{{path[0]}}", + "path-full": "{{path}}", + }}, + }, + } + gitGeneratorParams := map[string]string{ + "path": "p1/p2/app3", "path.basename": "app3", "path[0]": "p1", "path[1]": "p2", "path.basenameNormalized": "app3", + } + interpolatedGenerator, err := interpolateGenerator(requestedGenerator, gitGeneratorParams) + if err != nil { + log.WithError(err).WithField("requestedGenerator", requestedGenerator).Error("error interpolating Generator") + return + } + assert.Equal(t, "app3", interpolatedGenerator.Clusters.Selector.MatchLabels["path-basename"]) + assert.Equal(t, "p1", interpolatedGenerator.Clusters.Selector.MatchLabels["path-zero"]) + assert.Equal(t, "p1/p2/app3", interpolatedGenerator.Clusters.Selector.MatchLabels["path-full"]) + + fileNamePath := argoprojiov1alpha1.GitFileGeneratorItem{ + Path: "{{name}}", + } + fileServerPath := argoprojiov1alpha1.GitFileGeneratorItem{ + Path: "{{server}}", + } + + requestedGenerator = &argoprojiov1alpha1.ApplicationSetGenerator{ + Git: &argoprojiov1alpha1.GitGenerator{ + Files: append([]argoprojiov1alpha1.GitFileGeneratorItem{}, fileNamePath, fileServerPath), + Template: argoprojiov1alpha1.ApplicationSetTemplate{}, + }, + } + clusterGeneratorParams := map[string]string{ + "name": "production_01/west", "server": "https://production-01.example.com", + } + interpolatedGenerator, err = interpolateGenerator(requestedGenerator, clusterGeneratorParams) + if err != nil { + log.WithError(err).WithField("requestedGenerator", requestedGenerator).Error("error interpolating Generator") + return + } + assert.Equal(t, "production_01/west", interpolatedGenerator.Git.Files[0].Path) + assert.Equal(t, "https://production-01.example.com", interpolatedGenerator.Git.Files[1].Path) +} diff --git a/applicationset/generators/matrix.go b/applicationset/generators/matrix.go index c4530bb8dea8a..33a14d34861f3 100644 --- a/applicationset/generators/matrix.go +++ b/applicationset/generators/matrix.go @@ -44,16 +44,15 @@ func (m *MatrixGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha1.App res := []map[string]string{} - g0, err := m.getParams(appSetGenerator.Matrix.Generators[0], appSet) + g0, err := m.getParams(appSetGenerator.Matrix.Generators[0], appSet, nil) if err != nil { return nil, err } - g1, err := m.getParams(appSetGenerator.Matrix.Generators[1], appSet) - if err != nil { - return nil, err - } - for _, a := range g0 { + g1, err := m.getParams(appSetGenerator.Matrix.Generators[1], appSet, a) + if err != nil { + return nil, err + } for _, b := range g1 { val, err := utils.CombineStringMaps(a, b) if err != nil { @@ -66,7 +65,7 @@ func (m *MatrixGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha1.App return res, nil } -func (m *MatrixGenerator) getParams(appSetBaseGenerator argoprojiov1alpha1.ApplicationSetNestedGenerator, appSet *argoprojiov1alpha1.ApplicationSet) ([]map[string]string, error) { +func (m *MatrixGenerator) getParams(appSetBaseGenerator argoprojiov1alpha1.ApplicationSetNestedGenerator, appSet *argoprojiov1alpha1.ApplicationSet, params map[string]string) ([]map[string]string, error) { var matrix *argoprojiov1alpha1.MatrixGenerator if appSetBaseGenerator.Matrix != nil { // Since nested matrix generator is represented as a JSON object in the CRD, we unmarshall it back to a Go struct here. @@ -104,7 +103,8 @@ func (m *MatrixGenerator) getParams(appSetBaseGenerator argoprojiov1alpha1.Appli }, m.supportedGenerators, argoprojiov1alpha1.ApplicationSetTemplate{}, - appSet) + appSet, + params) if err != nil { return nil, fmt.Errorf("child generator returned an error on parameter generation: %v", err) diff --git a/applicationset/generators/matrix_test.go b/applicationset/generators/matrix_test.go index c5c7fce24d131..cce467efcfa37 100644 --- a/applicationset/generators/matrix_test.go +++ b/applicationset/generators/matrix_test.go @@ -1,6 +1,13 @@ package generators import ( + "context" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + kubefake "k8s.io/client-go/kubernetes/fake" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" "testing" "time" @@ -127,7 +134,7 @@ func TestMatrixGenerate(t *testing.T) { testCaseCopy := testCase // Since tests may run in parallel t.Run(testCaseCopy.name, func(t *testing.T) { - mock := &generatorMock{} + genMock := &generatorMock{} appSet := &argoprojiov1alpha1.ApplicationSet{} for _, g := range testCaseCopy.baseGenerators { @@ -136,7 +143,7 @@ func TestMatrixGenerate(t *testing.T) { Git: g.Git, List: g.List, } - mock.On("GenerateParams", &gitGeneratorSpec, appSet).Return([]map[string]string{ + genMock.On("GenerateParams", mock.AnythingOfType("*v1alpha1.ApplicationSetGenerator"), appSet).Return([]map[string]string{ { "path": "app1", "path.basename": "app1", @@ -149,13 +156,13 @@ func TestMatrixGenerate(t *testing.T) { }, }, nil) - mock.On("GetTemplate", &gitGeneratorSpec). + genMock.On("GetTemplate", &gitGeneratorSpec). Return(&argoprojiov1alpha1.ApplicationSetTemplate{}) } var matrixGenerator = NewMatrixGenerator( map[string]Generator{ - "Git": mock, + "Git": genMock, "List": &ListGenerator{}, }, ) @@ -260,6 +267,156 @@ func TestMatrixGetRequeueAfter(t *testing.T) { } } +func TestInterpolatedMatrixGenerate(t *testing.T) { + interpolatedGitGenerator := &argoprojiov1alpha1.GitGenerator{ + RepoURL: "RepoURL", + Revision: "Revision", + Files: []argoprojiov1alpha1.GitFileGeneratorItem{ + {Path: "examples/git-generator-files-discovery/cluster-config/dev/config.json"}, + {Path: "examples/git-generator-files-discovery/cluster-config/prod/config.json"}, + }, + } + + interpolatedClusterGenerator := &argoprojiov1alpha1.ClusterGenerator{ + Selector: metav1.LabelSelector{ + MatchLabels: map[string]string{"environment": "{{path.basename}}"}, + MatchExpressions: nil, + }, + } + testCases := []struct { + name string + baseGenerators []argoprojiov1alpha1.ApplicationSetNestedGenerator + expectedErr error + expected []map[string]string + clientError bool + }{ + { + name: "happy flow - generate interpolated params", + baseGenerators: []argoprojiov1alpha1.ApplicationSetNestedGenerator{ + { + Git: interpolatedGitGenerator, + }, + { + Clusters: interpolatedClusterGenerator, + }, + }, + expected: []map[string]string{ + {"path": "examples/git-generator-files-discovery/cluster-config/dev/config.json", "path.basename": "dev", "path.basenameNormalized": "dev", "name": "dev-01", "nameNormalized": "dev-01", "server": "https://dev-01.example.com", "metadata.labels.environment": "dev", "metadata.labels.argocd.argoproj.io/secret-type": "cluster"}, + {"path": "examples/git-generator-files-discovery/cluster-config/prod/config.json", "path.basename": "prod", "path.basenameNormalized": "prod", "name": "prod-01", "nameNormalized": "prod-01", "server": "https://prod-01.example.com", "metadata.labels.environment": "prod", "metadata.labels.argocd.argoproj.io/secret-type": "cluster"}, + }, + clientError: false, + }, + } + clusters := []client.Object{ + &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "dev-01", + Namespace: "namespace", + Labels: map[string]string{ + "argocd.argoproj.io/secret-type": "cluster", + "environment": "dev", + }, + }, + Data: map[string][]byte{ + "config": []byte("{}"), + "name": []byte("dev-01"), + "server": []byte("https://dev-01.example.com"), + }, + Type: corev1.SecretType("Opaque"), + }, + &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "prod-01", + Namespace: "namespace", + Labels: map[string]string{ + "argocd.argoproj.io/secret-type": "cluster", + "environment": "prod", + }, + }, + Data: map[string][]byte{ + "config": []byte("{}"), + "name": []byte("prod-01"), + "server": []byte("https://prod-01.example.com"), + }, + Type: corev1.SecretType("Opaque"), + }, + } + // convert []client.Object to []runtime.Object, for use by kubefake package + runtimeClusters := []runtime.Object{} + for _, clientCluster := range clusters { + runtimeClusters = append(runtimeClusters, clientCluster) + } + + for _, testCase := range testCases { + testCaseCopy := testCase // Since tests may run in parallel + + t.Run(testCaseCopy.name, func(t *testing.T) { + genMock := &generatorMock{} + appSet := &argoprojiov1alpha1.ApplicationSet{} + + appClientset := kubefake.NewSimpleClientset(runtimeClusters...) + fakeClient := fake.NewClientBuilder().WithObjects(clusters...).Build() + cl := &possiblyErroringFakeCtrlRuntimeClient{ + fakeClient, + testCase.clientError, + } + var clusterGenerator = NewClusterGenerator(cl, context.Background(), appClientset, "namespace") + + for _, g := range testCaseCopy.baseGenerators { + + gitGeneratorSpec := argoprojiov1alpha1.ApplicationSetGenerator{ + Git: g.Git, + Clusters: g.Clusters, + } + genMock.On("GenerateParams", mock.AnythingOfType("*v1alpha1.ApplicationSetGenerator"), appSet).Return([]map[string]string{ + { + "path": "examples/git-generator-files-discovery/cluster-config/dev/config.json", + "path.basename": "dev", + "path.basenameNormalized": "dev", + }, + { + "path": "examples/git-generator-files-discovery/cluster-config/prod/config.json", + "path.basename": "prod", + "path.basenameNormalized": "prod", + }, + }, nil) + genMock.On("GetTemplate", &gitGeneratorSpec). + Return(&argoprojiov1alpha1.ApplicationSetTemplate{}) + } + var matrixGenerator = NewMatrixGenerator( + map[string]Generator{ + "Git": genMock, + "Clusters": clusterGenerator, + }, + ) + + got, err := matrixGenerator.GenerateParams(&argoprojiov1alpha1.ApplicationSetGenerator{ + Matrix: &argoprojiov1alpha1.MatrixGenerator{ + Generators: testCaseCopy.baseGenerators, + Template: argoprojiov1alpha1.ApplicationSetTemplate{}, + }, + }, appSet) + + if testCaseCopy.expectedErr != nil { + assert.EqualError(t, err, testCaseCopy.expectedErr.Error()) + } else { + assert.NoError(t, err) + assert.Equal(t, testCaseCopy.expected, got) + } + + }) + + } +} + type generatorMock struct { mock.Mock } diff --git a/applicationset/generators/merge.go b/applicationset/generators/merge.go index 8b2cefc3cd336..a3ad8d5cf2eb8 100644 --- a/applicationset/generators/merge.go +++ b/applicationset/generators/merge.go @@ -163,7 +163,8 @@ func (m *MergeGenerator) getParams(appSetBaseGenerator argoprojiov1alpha1.Applic }, m.supportedGenerators, argoprojiov1alpha1.ApplicationSetTemplate{}, - appSet) + appSet, + map[string]string{}) if err != nil { return nil, fmt.Errorf("child generator returned an error on parameter generation: %v", err) diff --git a/applicationset/utils/utils.go b/applicationset/utils/utils.go index fc4ae33595528..f3586ffdad29c 100644 --- a/applicationset/utils/utils.go +++ b/applicationset/utils/utils.go @@ -38,7 +38,7 @@ func (r *Render) RenderTemplateParams(tmpl *argoappsv1.Application, syncPolicy * } fstTmpl := fasttemplate.New(string(tmplBytes), "{{", "}}") - replacedTmplStr, err := r.replace(fstTmpl, params, true) + replacedTmplStr, err := r.Replace(fstTmpl, params, true) if err != nil { return nil, err } @@ -66,7 +66,7 @@ func (r *Render) RenderTemplateParams(tmpl *argoappsv1.Application, syncPolicy * // Replace executes basic string substitution of a template with replacement values. // 'allowUnresolved' indicates whether or not it is acceptable to have unresolved variables // remaining in the substituted template. -func (r *Render) replace(fstTmpl *fasttemplate.Template, replaceMap map[string]string, allowUnresolved bool) (string, error) { +func (r *Render) Replace(fstTmpl *fasttemplate.Template, replaceMap map[string]string, allowUnresolved bool) (string, error) { var unresolvedErr error replacedTmpl := fstTmpl.ExecuteFuncString(func(w io.Writer, tag string) (int, error) { diff --git a/docs/operator-manual/applicationset/Generators-Matrix.md b/docs/operator-manual/applicationset/Generators-Matrix.md index 3ed9b6469c84c..d844ff03b87e4 100644 --- a/docs/operator-manual/applicationset/Generators-Matrix.md +++ b/docs/operator-manual/applicationset/Generators-Matrix.md @@ -104,6 +104,68 @@ Finally, the Matrix generator will combine both sets of outputs, and produce: ``` (*The full example can be found [here](https://github.com/argoproj/argo-cd/tree/master/applicationset/examples/matrix).*) +## Using Parameters from one child generator in another child generator + +The Matrix generator allows using the parameters generated by one child generator inside another child generator. +Below is an example that uses a git-files generator in conjunction with a cluster generator. + +```yaml +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: cluster-git +spec: + generators: + # matrix 'parent' generator + - matrix: + generators: + # git generator, 'child' #1 + - git: + repoURL: https://github.com/argoproj/applicationset.git + revision: HEAD + files: + - path: "examples/git-generator-files-discovery/cluster-config/**/config.json" + # cluster generator, 'child' #2 + - clusters: + selector: + matchLabels: + argocd.argoproj.io/secret-type: cluster + kubernetes.io/environment: '{{path.basename}}' + template: + metadata: + name: '{{name}}-guestbook' + spec: + project: default + source: + repoURL: https://github.com/argoproj/applicationset.git + targetRevision: HEAD + path: "examples/git-generator-files-discovery/apps/guestbook" + destination: + server: '{{server}}' + namespace: guestbook +``` +Here is the corresponding folder structure for the git repository used by the git-files generator: + +``` +├── apps +│ └── guestbook +│ ├── guestbook-ui-deployment.yaml +│ ├── guestbook-ui-svc.yaml +│ └── kustomization.yaml +├── cluster-config +│ └── engineering +│ ├── dev +│ │ └── config.json +│ └── prod +│ └── config.json +└── git-generator-files.yaml +``` +In the above example, the `{{path.basename}}` parameters produced by the git-files generator will resolve to `dev` and `prod`. +In the 2nd child generator, the label selector with label `kubernetes.io/environment: {{path.basename}}` will resolve with the values produced by the first child generator's parameters (`kubernetes.io/environment: prod` and `kubernetes.io/environment: dev`). + +So in the above example, clusters with the label `kubernetes.io/environment: prod` will have only prod-specific configuration (ie. `prod/config.json`) applied to it, wheres clusters +with the label `kubernetes.io/environment: dev` will have only dev-specific configuration (ie. `dev/config.json`) + ## Restrictions 1. The Matrix generator currently only supports combining the outputs of only two child generators (eg does not support generating combinations for 3 or more). @@ -136,3 +198,39 @@ Finally, the Matrix generator will combine both sets of outputs, and produce: elements: - # (...) ``` +1. When using parameters from one child generator inside another child generator, the child generator that *consumes* the parameters **must come after** the child generator that *produces* the parameters. +For example, the below example would be invalid (cluster-generator must come after the git-files generator): +```yaml +- matrix: + generators: + # cluster generator, 'child' #1 + - clusters: + selector: + matchLabels: + argocd.argoproj.io/secret-type: cluster + kubernetes.io/environment: '{{path.basename}}' # {{path.basename}} is produced by git-files generator + # git generator, 'child' #2 + - git: + repoURL: https://github.com/argoproj/applicationset.git + revision: HEAD + files: + - path: "examples/git-generator-files-discovery/cluster-config/**/config.json" +``` +1. You cannot have both child generators consuming parameters from each another. In the example below, the cluster generator is consuming the `{{path.basename}}` parameter produced by the git-files generator, whereas the git-files generator is consuming the `{{name}}` parameter produced by the cluster generator. This will result in a circular dependency, which is invalid. +```yaml +- matrix: + generators: + # cluster generator, 'child' #1 + - clusters: + selector: + matchLabels: + argocd.argoproj.io/secret-type: cluster + kubernetes.io/environment: '{{path.basename}}' # {{path.basename}} is produced by git-files generator + # git generator, 'child' #2 + - git: + repoURL: https://github.com/argoproj/applicationset.git + revision: HEAD + files: + - path: "examples/git-generator-files-discovery/cluster-config/engineering/{{name}}**/config.json" # {{name}} is produced by cluster generator +``` + diff --git a/go.mod b/go.mod index 6a27e8144602c..1f6222f5ade5a 100644 --- a/go.mod +++ b/go.mod @@ -191,7 +191,7 @@ require ( github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sergi/go-diff v1.1.0 // indirect github.com/slack-go/slack v0.10.1 // indirect - github.com/stretchr/objx v0.2.0 // indirect + github.com/stretchr/objx v0.3.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/vmihailenco/go-tinylfu v0.2.1 // indirect github.com/vmihailenco/msgpack/v5 v5.3.4 // indirect diff --git a/go.sum b/go.sum index a0ab0faa96912..983fcf52a7a5b 100644 --- a/go.sum +++ b/go.sum @@ -1070,8 +1070,9 @@ github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3 github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/objx v0.3.0 h1:NGXK3lHquSN08v5vWalVI/L8XU9hdzE/G6xsrze47As= +github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v0.0.0-20161117074351-18a02ba4a312/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=