From d4da216d1c7e76d1390091dd0e8cda53e53fe74e Mon Sep 17 00:00:00 2001 From: jkulkarn Date: Tue, 12 Apr 2022 09:14:03 -0700 Subject: [PATCH 01/22] Upgraded Go to 1.18, & have changes related to matrix generator's child generators reading parameters. Specifically, 2nd generator reading the 1st generator's parameters. Signed-off-by: jkulkarn --- .../controllers/applicationset_controller.go | 2 +- .../generators/generator_spec_processor.go | 48 ++++++++++++++++--- applicationset/generators/matrix.go | 9 ++-- applicationset/generators/merge.go | 3 +- applicationset/utils/utils.go | 4 +- 5 files changed, 52 insertions(+), 14 deletions(-) diff --git a/applicationset/controllers/applicationset_controller.go b/applicationset/controllers/applicationset_controller.go index 99ffa1c617902..4469397415e9e 100644 --- a/applicationset/controllers/applicationset_controller.go +++ b/applicationset/controllers/applicationset_controller.go @@ -430,7 +430,7 @@ func (r *ApplicationSetReconciler) generateApplications(applicationSetInfo argop var applicationSetReason argoprojiov1alpha1.ApplicationSetReasonType for _, requestedGenerator := range applicationSetInfo.Spec.Generators { - t, err := generators.Transform(requestedGenerator, r.Generators, applicationSetInfo.Spec.Template, &applicationSetInfo) + t, err := generators.Transform(requestedGenerator, r.Generators, applicationSetInfo.Spec.Template, &applicationSetInfo, []map[string]string{}) if err != nil { log.WithError(err).WithField("generator", requestedGenerator). Error("error generating application from params") diff --git a/applicationset/generators/generator_spec_processor.go b/applicationset/generators/generator_spec_processor.go index 02091c5d30d5e..428a7e9814e81 100644 --- a/applicationset/generators/generator_spec_processor.go +++ b/applicationset/generators/generator_spec_processor.go @@ -1,12 +1,14 @@ package generators import ( + "encoding/json" + "github.com/argoproj/argo-cd/v2/applicationset/utils" + "github.com/valyala/fasttemplate" "reflect" + argoprojiov1alpha1 "github.com/argoproj/argo-cd/v2/pkg/apis/applicationset/v1alpha1" "github.com/imdario/mergo" log "github.com/sirupsen/logrus" - - argoprojiov1alpha1 "github.com/argoproj/argo-cd/v2/pkg/apis/applicationset/v1alpha1" ) type TransformResult struct { @@ -15,7 +17,7 @@ type TransformResult struct { } //Transform a spec generator to list of paramSets and a template -func Transform(requestedGenerator argoprojiov1alpha1.ApplicationSetGenerator, allGenerators map[string]Generator, baseTemplate argoprojiov1alpha1.ApplicationSetTemplate, appSet *argoprojiov1alpha1.ApplicationSet) ([]TransformResult, error) { +func Transform(requestedGenerator argoprojiov1alpha1.ApplicationSetGenerator, allGenerators map[string]Generator, baseTemplate argoprojiov1alpha1.ApplicationSetTemplate, appSet *argoprojiov1alpha1.ApplicationSet, genParams []map[string]string) ([]TransformResult, error) { res := []TransformResult{} var firstError error @@ -31,7 +33,17 @@ func Transform(requestedGenerator argoprojiov1alpha1.ApplicationSetGenerator, al } continue } - + if len(genParams) != 0 { + err := interpolateGenerator(&requestedGenerator, genParams) + if err != nil { + log.WithError(err).WithField("generator", g). + Error("error interpolating params for generator") + if firstError == nil { + firstError = err + } + continue + } + } params, err := g.GenerateParams(&requestedGenerator, appSet) if err != nil { log.WithError(err).WithField("generator", g). @@ -46,11 +58,9 @@ func Transform(requestedGenerator argoprojiov1alpha1.ApplicationSetGenerator, al Params: params, Template: mergedTemplate, }) - } return res, firstError - } func GetRelevantGenerators(requestedGenerator *argoprojiov1alpha1.ApplicationSetGenerator, generators map[string]Generator) []Generator { @@ -81,3 +91,29 @@ func mergeGeneratorTemplate(g Generator, requestedGenerator *argoprojiov1alpha1. return *dest, err } + +func interpolateGenerator(requestedGenerator *argoprojiov1alpha1.ApplicationSetGenerator, currParams []map[string]string) error { + tmplBytes, err := json.Marshal(requestedGenerator) + if err != nil { + return err + } + tmpParams := make(map[string]string) + for _, currParam := range currParams { + for k, v := range currParam { + tmpParams[k] = v + } + } + log.Info(tmplBytes) + + fstTmpl := fasttemplate.New(string(tmplBytes), "{{", "}}") + render := utils.Render{} + replacedTmplStr, err := render.Replace(fstTmpl, tmpParams, true) + if err != nil { + return err + } + err = json.Unmarshal([]byte(replacedTmplStr), requestedGenerator) + if err != nil { + return err + } + return nil +} diff --git a/applicationset/generators/matrix.go b/applicationset/generators/matrix.go index c4530bb8dea8a..d5cbfd78e7c26 100644 --- a/applicationset/generators/matrix.go +++ b/applicationset/generators/matrix.go @@ -44,11 +44,11 @@ func (m *MatrixGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha1.App res := []map[string]string{} - g0, err := m.getParams(appSetGenerator.Matrix.Generators[0], appSet) + g0, err := m.getParams(appSetGenerator.Matrix.Generators[0], appSet, []map[string]string{}) if err != nil { return nil, err } - g1, err := m.getParams(appSetGenerator.Matrix.Generators[1], appSet) + g1, err := m.getParams(appSetGenerator.Matrix.Generators[1], appSet, g0) if err != nil { return nil, err } @@ -66,7 +66,7 @@ func (m *MatrixGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha1.App return res, nil } -func (m *MatrixGenerator) getParams(appSetBaseGenerator argoprojiov1alpha1.ApplicationSetNestedGenerator, appSet *argoprojiov1alpha1.ApplicationSet) ([]map[string]string, error) { +func (m *MatrixGenerator) getParams(appSetBaseGenerator argoprojiov1alpha1.ApplicationSetNestedGenerator, appSet *argoprojiov1alpha1.ApplicationSet, params []map[string]string) ([]map[string]string, error) { var matrix *argoprojiov1alpha1.MatrixGenerator if appSetBaseGenerator.Matrix != nil { // Since nested matrix generator is represented as a JSON object in the CRD, we unmarshall it back to a Go struct here. @@ -104,7 +104,8 @@ func (m *MatrixGenerator) getParams(appSetBaseGenerator argoprojiov1alpha1.Appli }, m.supportedGenerators, argoprojiov1alpha1.ApplicationSetTemplate{}, - appSet) + appSet, + params) if err != nil { return nil, fmt.Errorf("child generator returned an error on parameter generation: %v", err) diff --git a/applicationset/generators/merge.go b/applicationset/generators/merge.go index 8b2cefc3cd336..9dbc87457f779 100644 --- a/applicationset/generators/merge.go +++ b/applicationset/generators/merge.go @@ -163,7 +163,8 @@ func (m *MergeGenerator) getParams(appSetBaseGenerator argoprojiov1alpha1.Applic }, m.supportedGenerators, argoprojiov1alpha1.ApplicationSetTemplate{}, - appSet) + appSet, + []map[string]string{}) if err != nil { return nil, fmt.Errorf("child generator returned an error on parameter generation: %v", err) diff --git a/applicationset/utils/utils.go b/applicationset/utils/utils.go index fc4ae33595528..f3586ffdad29c 100644 --- a/applicationset/utils/utils.go +++ b/applicationset/utils/utils.go @@ -38,7 +38,7 @@ func (r *Render) RenderTemplateParams(tmpl *argoappsv1.Application, syncPolicy * } fstTmpl := fasttemplate.New(string(tmplBytes), "{{", "}}") - replacedTmplStr, err := r.replace(fstTmpl, params, true) + replacedTmplStr, err := r.Replace(fstTmpl, params, true) if err != nil { return nil, err } @@ -66,7 +66,7 @@ func (r *Render) RenderTemplateParams(tmpl *argoappsv1.Application, syncPolicy * // Replace executes basic string substitution of a template with replacement values. // 'allowUnresolved' indicates whether or not it is acceptable to have unresolved variables // remaining in the substituted template. -func (r *Render) replace(fstTmpl *fasttemplate.Template, replaceMap map[string]string, allowUnresolved bool) (string, error) { +func (r *Render) Replace(fstTmpl *fasttemplate.Template, replaceMap map[string]string, allowUnresolved bool) (string, error) { var unresolvedErr error replacedTmpl := fstTmpl.ExecuteFuncString(func(w io.Writer, tag string) (int, error) { From 38a2b5667f8cca444e3bd5f5b530a91860e903b5 Mon Sep 17 00:00:00 2001 From: jkulkarn Date: Tue, 12 Apr 2022 10:47:33 -0700 Subject: [PATCH 02/22] Added import for applicationSet in matrix.go Signed-off-by: jkulkarn --- applicationset/generators/matrix.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/applicationset/generators/matrix.go b/applicationset/generators/matrix.go index d5cbfd78e7c26..e254c085e7cc1 100644 --- a/applicationset/generators/matrix.go +++ b/applicationset/generators/matrix.go @@ -2,8 +2,11 @@ package generators import ( "fmt" + "github.com/argoproj/argo-cd/v2/applicationset/utils" + argoprojiov1alpha1 "github.com/argoproj/argo-cd/v2/pkg/apis/applicationset/v1alpha1" "time" - +) +- "github.com/argoproj/argo-cd/v2/applicationset/utils" argoprojiov1alpha1 "github.com/argoproj/argo-cd/v2/pkg/apis/applicationset/v1alpha1" ) From 5c7ae5a413299ccc0d6dffdbe652846cc0792e9e Mon Sep 17 00:00:00 2001 From: jkulkarn Date: Tue, 12 Apr 2022 10:48:03 -0700 Subject: [PATCH 03/22] Added import for applicationSet in matrix.go Signed-off-by: jkulkarn --- applicationset/generators/matrix.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/applicationset/generators/matrix.go b/applicationset/generators/matrix.go index e254c085e7cc1..d5cbfd78e7c26 100644 --- a/applicationset/generators/matrix.go +++ b/applicationset/generators/matrix.go @@ -2,11 +2,8 @@ package generators import ( "fmt" - "github.com/argoproj/argo-cd/v2/applicationset/utils" - argoprojiov1alpha1 "github.com/argoproj/argo-cd/v2/pkg/apis/applicationset/v1alpha1" "time" -) -- + "github.com/argoproj/argo-cd/v2/applicationset/utils" argoprojiov1alpha1 "github.com/argoproj/argo-cd/v2/pkg/apis/applicationset/v1alpha1" ) From f09d41e1914ccf74dda16d8132dab8758e2cc51b Mon Sep 17 00:00:00 2001 From: jkulkarn Date: Tue, 12 Apr 2022 13:13:18 -0700 Subject: [PATCH 04/22] Added logging + small fixes for ArgoCD PR Signed-off-by: jkulkarn --- applicationset/generators/generator_spec_processor.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/applicationset/generators/generator_spec_processor.go b/applicationset/generators/generator_spec_processor.go index 428a7e9814e81..57ea8ef2367d0 100644 --- a/applicationset/generators/generator_spec_processor.go +++ b/applicationset/generators/generator_spec_processor.go @@ -36,7 +36,7 @@ func Transform(requestedGenerator argoprojiov1alpha1.ApplicationSetGenerator, al if len(genParams) != 0 { err := interpolateGenerator(&requestedGenerator, genParams) if err != nil { - log.WithError(err).WithField("generator", g). + log.WithError(err).WithField("genParams", genParams). Error("error interpolating params for generator") if firstError == nil { firstError = err @@ -92,27 +92,29 @@ func mergeGeneratorTemplate(g Generator, requestedGenerator *argoprojiov1alpha1. return *dest, err } -func interpolateGenerator(requestedGenerator *argoprojiov1alpha1.ApplicationSetGenerator, currParams []map[string]string) error { +func interpolateGenerator(requestedGenerator *argoprojiov1alpha1.ApplicationSetGenerator, params []map[string]string) error { tmplBytes, err := json.Marshal(requestedGenerator) if err != nil { + log.WithError(err).WithField("requestedGenerator", requestedGenerator).Error("error marshalling requested generator for interpolation") return err } tmpParams := make(map[string]string) - for _, currParam := range currParams { + for _, currParam := range params { for k, v := range currParam { tmpParams[k] = v } } - log.Info(tmplBytes) fstTmpl := fasttemplate.New(string(tmplBytes), "{{", "}}") render := utils.Render{} replacedTmplStr, err := render.Replace(fstTmpl, tmpParams, true) if err != nil { + log.WithError(err).WithField("interpolatedGeneratorString", replacedTmplStr).Error("error interpolating generator with other generator's parameter") return err } err = json.Unmarshal([]byte(replacedTmplStr), requestedGenerator) if err != nil { + log.WithError(err).WithField("requestedGenerator", requestedGenerator).Error("error unmarshalling requested generator for interpolation") return err } return nil From 491d3019789bf9c9daa727b9da7822df6004d3df Mon Sep 17 00:00:00 2001 From: jkulkarn Date: Wed, 13 Apr 2022 12:02:46 -0700 Subject: [PATCH 05/22] Added testing for GetRelevantGenerators Signed-off-by: jkulkarn --- .../generator_spec_processor_test.go | 256 ++++++++++++++++++ 1 file changed, 256 insertions(+) create mode 100644 applicationset/generators/generator_spec_processor_test.go diff --git a/applicationset/generators/generator_spec_processor_test.go b/applicationset/generators/generator_spec_processor_test.go new file mode 100644 index 0000000000000..c091aa516919f --- /dev/null +++ b/applicationset/generators/generator_spec_processor_test.go @@ -0,0 +1,256 @@ +package generators + +import ( + "context" + "fmt" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + kubefake "k8s.io/client-go/kubernetes/fake" + crtclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "testing" + + argoprojiov1alpha1 "github.com/argoproj/argo-cd/v2/pkg/apis/applicationset/v1alpha1" +) + +type genProcessorMock struct { + mock.Mock +} + +func (g *genProcessorMock) Transform(requestedGenerator argoprojiov1alpha1.ApplicationSetGenerator, allGenerators map[string]Generator, baseTemplate argoprojiov1alpha1.ApplicationSetTemplate, appSet *argoprojiov1alpha1.ApplicationSet, genParams []map[string]string) ([]TransformResult, error) { + args := g.Called(requestedGenerator) + + return args.Get(0).([]TransformResult), args.Error(1) +} + +func (g *genProcessorMock) GetRelevantGenerators(requestedGenerator argoprojiov1alpha1.ApplicationSetGenerator, generators map[string]Generator) []Generator { + args := g.Called(requestedGenerator) + + return args.Get(0).([]Generator) +} + +func (g *genProcessorMock) interpolateGenerator(requestedGenerator *argoprojiov1alpha1.ApplicationSetGenerator, params []map[string]string) error { + args := g.Called(requestedGenerator) + + return args.Error(1) +} + +func getMockClusterGenerator() Generator { + clusters := []crtclient.Object{ + &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "staging-01", + Namespace: "namespace", + Labels: map[string]string{ + "argocd.argoproj.io/secret-type": "cluster", + "environment": "staging", + "org": "foo", + }, + Annotations: map[string]string{ + "foo.argoproj.io": "staging", + }, + }, + Data: map[string][]byte{ + "config": []byte("{}"), + "name": []byte("staging-01"), + "server": []byte("https://staging-01.example.com"), + }, + Type: corev1.SecretType("Opaque"), + }, + &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "production-01", + Namespace: "namespace", + Labels: map[string]string{ + "argocd.argoproj.io/secret-type": "cluster", + "environment": "production", + "org": "bar", + }, + Annotations: map[string]string{ + "foo.argoproj.io": "production", + }, + }, + Data: map[string][]byte{ + "config": []byte("{}"), + "name": []byte("production_01/west"), + "server": []byte("https://production-01.example.com"), + }, + Type: corev1.SecretType("Opaque"), + }, + } + runtimeClusters := []runtime.Object{} + appClientset := kubefake.NewSimpleClientset(runtimeClusters...) + + fakeClient := fake.NewClientBuilder().WithObjects(clusters...).Build() + return NewClusterGenerator(fakeClient, context.Background(), appClientset, "namespace") +} + +func getMockGitGenerator() Generator { + cases := []struct { + name string + directories []argoprojiov1alpha1.GitDirectoryGeneratorItem + repoApps []string + repoError error + expected []map[string]string + expectedError error + }{ + { + name: "happy flow - created apps", + directories: []argoprojiov1alpha1.GitDirectoryGeneratorItem{{Path: "*"}}, + repoApps: []string{ + "app1", + "app2", + "app_3", + "p1/app4", + }, + repoError: nil, + expected: []map[string]string{ + {"path": "app1", "path.basename": "app1", "path.basenameNormalized": "app1"}, + {"path": "app2", "path.basename": "app2", "path.basenameNormalized": "app2"}, + {"path": "app_3", "path.basename": "app_3", "path.basenameNormalized": "app-3"}, + }, + expectedError: nil, + }, + { + name: "It filters application according to the paths", + directories: []argoprojiov1alpha1.GitDirectoryGeneratorItem{{Path: "p1/*"}, {Path: "p1/*/*"}}, + repoApps: []string{ + "app1", + "p1/app2", + "p1/p2/app3", + "p1/p2/p3/app4", + }, + repoError: nil, + expected: []map[string]string{ + {"path": "p1/app2", "path.basename": "app2", "path[0]": "p1", "path.basenameNormalized": "app2"}, + {"path": "p1/p2/app3", "path.basename": "app3", "path[0]": "p1", "path[1]": "p2", "path.basenameNormalized": "app3"}, + }, + expectedError: nil, + }, + { + name: "It filters application according to the paths with Exclude", + directories: []argoprojiov1alpha1.GitDirectoryGeneratorItem{{Path: "p1/*", Exclude: true}, {Path: "*"}, {Path: "*/*"}}, + repoApps: []string{ + "app1", + "app2", + "p1/app2", + "p1/app3", + "p2/app3", + }, + repoError: nil, + expected: []map[string]string{ + {"path": "app1", "path.basename": "app1", "path.basenameNormalized": "app1"}, + {"path": "app2", "path.basename": "app2", "path.basenameNormalized": "app2"}, + {"path": "p2/app3", "path.basename": "app3", "path[0]": "p2", "path.basenameNormalized": "app3"}, + }, + expectedError: nil, + }, + { + name: "Expecting same exclude behavior with different order", + directories: []argoprojiov1alpha1.GitDirectoryGeneratorItem{{Path: "*"}, {Path: "*/*"}, {Path: "p1/*", Exclude: true}}, + repoApps: []string{ + "app1", + "app2", + "p1/app2", + "p1/app3", + "p2/app3", + }, + repoError: nil, + expected: []map[string]string{ + {"path": "app1", "path.basename": "app1", "path.basenameNormalized": "app1"}, + {"path": "app2", "path.basename": "app2", "path.basenameNormalized": "app2"}, + {"path": "p2/app3", "path.basename": "app3", "path[0]": "p2", "path.basenameNormalized": "app3"}, + }, + expectedError: nil, + }, + { + name: "handles empty response from repo server", + directories: []argoprojiov1alpha1.GitDirectoryGeneratorItem{{Path: "*"}}, + repoApps: []string{}, + repoError: nil, + expected: []map[string]string{}, + expectedError: nil, + }, + { + name: "handles error from repo server", + directories: []argoprojiov1alpha1.GitDirectoryGeneratorItem{{Path: "*"}}, + repoApps: []string{}, + repoError: fmt.Errorf("error"), + expected: []map[string]string{}, + expectedError: fmt.Errorf("error"), + }, + } + argoCDServiceMock := argoCDServiceMock{mock: &mock.Mock{}} + + argoCDServiceMock.mock.On("GetDirectories", mock.Anything, mock.Anything, mock.Anything).Return(cases[0].repoApps, cases[0].repoError) + + var gitGenerator = NewGitGenerator(argoCDServiceMock) + return gitGenerator +} + +func getMockMatrixGenerator(generators map[string]Generator) Generator { + mockMatrixGenerator := NewMatrixGenerator(generators) + return mockMatrixGenerator +} + +func TestGetRelevantGenerators(t *testing.T) { + + testGenerators := map[string]Generator{ + "Clusters": getMockClusterGenerator(), + "Git": getMockGitGenerator(), + } + + mockMatrixGenerator := NewMatrixGenerator(testGenerators) + mockMergeGenerator := NewMergeGenerator(testGenerators) + testGenerators["Matrix"] = mockMatrixGenerator + testGenerators["Merge"] = mockMergeGenerator + testGenerators["List"] = NewListGenerator() + + requestedGenerator := &argoprojiov1alpha1.ApplicationSetGenerator{ + List: &argoprojiov1alpha1.ListGenerator{ + Elements: []apiextensionsv1.JSON{{Raw: []byte(`{"cluster": "cluster","url": "url","values":{"foo":"bar"}}`)}}, + }} + + relevantGenerators := GetRelevantGenerators(requestedGenerator, testGenerators) + assert.Len(t, relevantGenerators, 1) + assert.IsType(t, &ListGenerator{}, relevantGenerators[0]) + + requestedGenerator = &argoprojiov1alpha1.ApplicationSetGenerator{ + Clusters: &argoprojiov1alpha1.ClusterGenerator{ + Selector: metav1.LabelSelector{}, + Template: argoprojiov1alpha1.ApplicationSetTemplate{}, + Values: nil, + }, + } + + relevantGenerators = GetRelevantGenerators(requestedGenerator, testGenerators) + assert.Len(t, relevantGenerators, 1) + assert.IsType(t, &ClusterGenerator{}, relevantGenerators[0]) + + requestedGenerator = &argoprojiov1alpha1.ApplicationSetGenerator{ + Git: &argoprojiov1alpha1.GitGenerator{ + RepoURL: "", + Directories: nil, + Files: nil, + Revision: "", + RequeueAfterSeconds: nil, + Template: argoprojiov1alpha1.ApplicationSetTemplate{}, + }, + } + + relevantGenerators = GetRelevantGenerators(requestedGenerator, testGenerators) + assert.Len(t, relevantGenerators, 1) + assert.IsType(t, &GitGenerator{}, relevantGenerators[0]) +} From 535a3918654440a97d11e13844e6c24ed54e57b2 Mon Sep 17 00:00:00 2001 From: jkulkarn Date: Wed, 13 Apr 2022 13:23:12 -0700 Subject: [PATCH 06/22] Added a test for interpolating Generators Signed-off-by: jkulkarn --- .../generator_spec_processor_test.go | 35 ++++++++++++++----- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/applicationset/generators/generator_spec_processor_test.go b/applicationset/generators/generator_spec_processor_test.go index c091aa516919f..fdeafdc694444 100644 --- a/applicationset/generators/generator_spec_processor_test.go +++ b/applicationset/generators/generator_spec_processor_test.go @@ -200,11 +200,6 @@ func getMockGitGenerator() Generator { return gitGenerator } -func getMockMatrixGenerator(generators map[string]Generator) Generator { - mockMatrixGenerator := NewMatrixGenerator(generators) - return mockMatrixGenerator -} - func TestGetRelevantGenerators(t *testing.T) { testGenerators := map[string]Generator{ @@ -212,10 +207,8 @@ func TestGetRelevantGenerators(t *testing.T) { "Git": getMockGitGenerator(), } - mockMatrixGenerator := NewMatrixGenerator(testGenerators) - mockMergeGenerator := NewMergeGenerator(testGenerators) - testGenerators["Matrix"] = mockMatrixGenerator - testGenerators["Merge"] = mockMergeGenerator + testGenerators["Matrix"] = NewMatrixGenerator(testGenerators) + testGenerators["Merge"] = NewMergeGenerator(testGenerators) testGenerators["List"] = NewListGenerator() requestedGenerator := &argoprojiov1alpha1.ApplicationSetGenerator{ @@ -254,3 +247,27 @@ func TestGetRelevantGenerators(t *testing.T) { assert.Len(t, relevantGenerators, 1) assert.IsType(t, &GitGenerator{}, relevantGenerators[0]) } + +func TestInterpolateGenerator(t *testing.T) { + requestedGenerator := &argoprojiov1alpha1.ApplicationSetGenerator{ + Clusters: &argoprojiov1alpha1.ClusterGenerator{ + Selector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + "argocd.argoproj.io/secret-type": "cluster", + "path-basename": "{{path.basename}}", + "path-zero": "{{path[0]}}", + "path-full": "{{path}}", + }}, + Template: argoprojiov1alpha1.ApplicationSetTemplate{}, + Values: nil, + }, + } + //mockGitGenerator := getMockGitGenerator() + gitGeneratorParams := []map[string]string{ + {"path": "p1/p2/app3", "path.basename": "app3", "path[0]": "p1", "path[1]": "p2", "path.basenameNormalized": "app3"}, + } + interpolateGenerator(requestedGenerator, gitGeneratorParams) + assert.Equal(t, "app3", requestedGenerator.Clusters.Selector.MatchLabels["path-basename"]) + assert.Equal(t, "p1", requestedGenerator.Clusters.Selector.MatchLabels["path-zero"]) + assert.Equal(t, "p1/p2/app3", requestedGenerator.Clusters.Selector.MatchLabels["path-full"]) +} From 4dfa276faa52ba1c501ca2ecbd2b38e914cb5760 Mon Sep 17 00:00:00 2001 From: jkulkarn Date: Wed, 13 Apr 2022 14:54:04 -0700 Subject: [PATCH 07/22] Added a 2nd test for interpolating Generators Signed-off-by: jkulkarn --- .../generators/generator_spec_processor.go | 2 ++ .../generator_spec_processor_test.go | 33 +++++++++++++++++-- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/applicationset/generators/generator_spec_processor.go b/applicationset/generators/generator_spec_processor.go index 57ea8ef2367d0..f3fa1c804801e 100644 --- a/applicationset/generators/generator_spec_processor.go +++ b/applicationset/generators/generator_spec_processor.go @@ -92,6 +92,8 @@ func mergeGeneratorTemplate(g Generator, requestedGenerator *argoprojiov1alpha1. return *dest, err } +// Currently for Matrix Generator. Allows interpolating the matrix's 2nd child generator with values from the 1st child generator +// "params" parameter here are params from the 1st child generator. Uses similar process for interpreting values as the actual Application Template func interpolateGenerator(requestedGenerator *argoprojiov1alpha1.ApplicationSetGenerator, params []map[string]string) error { tmplBytes, err := json.Marshal(requestedGenerator) if err != nil { diff --git a/applicationset/generators/generator_spec_processor_test.go b/applicationset/generators/generator_spec_processor_test.go index fdeafdc694444..8680009b1da37 100644 --- a/applicationset/generators/generator_spec_processor_test.go +++ b/applicationset/generators/generator_spec_processor_test.go @@ -3,6 +3,7 @@ package generators import ( "context" "fmt" + log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" corev1 "k8s.io/api/core/v1" @@ -258,16 +259,42 @@ func TestInterpolateGenerator(t *testing.T) { "path-zero": "{{path[0]}}", "path-full": "{{path}}", }}, - Template: argoprojiov1alpha1.ApplicationSetTemplate{}, - Values: nil, }, } //mockGitGenerator := getMockGitGenerator() gitGeneratorParams := []map[string]string{ {"path": "p1/p2/app3", "path.basename": "app3", "path[0]": "p1", "path[1]": "p2", "path.basenameNormalized": "app3"}, } - interpolateGenerator(requestedGenerator, gitGeneratorParams) + err := interpolateGenerator(requestedGenerator, gitGeneratorParams) + if err != nil { + log.WithError(err).WithField("requestedGenerator", requestedGenerator).Error("error interpolating Generator") + return + } assert.Equal(t, "app3", requestedGenerator.Clusters.Selector.MatchLabels["path-basename"]) assert.Equal(t, "p1", requestedGenerator.Clusters.Selector.MatchLabels["path-zero"]) assert.Equal(t, "p1/p2/app3", requestedGenerator.Clusters.Selector.MatchLabels["path-full"]) + + fileNamePath := argoprojiov1alpha1.GitFileGeneratorItem{ + Path: "{{name}}", + } + fileServerPath := argoprojiov1alpha1.GitFileGeneratorItem{ + Path: "{{server}}", + } + + requestedGenerator = &argoprojiov1alpha1.ApplicationSetGenerator{ + Git: &argoprojiov1alpha1.GitGenerator{ + Files: append([]argoprojiov1alpha1.GitFileGeneratorItem{}, fileNamePath, fileServerPath), + Template: argoprojiov1alpha1.ApplicationSetTemplate{}, + }, + } + clusterGeneratorParams := []map[string]string{ + {"name": "production_01/west", "server": "https://production-01.example.com"}, + } + err = interpolateGenerator(requestedGenerator, clusterGeneratorParams) + if err != nil { + log.WithError(err).WithField("requestedGenerator", requestedGenerator).Error("error interpolating Generator") + return + } + assert.Equal(t, "production_01/west", requestedGenerator.Git.Files[0].Path) + assert.Equal(t, "https://production-01.example.com", requestedGenerator.Git.Files[1].Path) } From 4edb6fe1616e675cbd7215fc33403e87b6e30c12 Mon Sep 17 00:00:00 2001 From: jkulkarn Date: Thu, 14 Apr 2022 18:31:45 -0400 Subject: [PATCH 08/22] Updated Generators-Matrix.md documentation to include an example + restrictions regarding child generator B consuming parameters produced by child generator A/ Signed-off-by: jkulkarn --- .../applicationset/Generators-Matrix.md | 104 +++++++++++++++++- 1 file changed, 101 insertions(+), 3 deletions(-) diff --git a/docs/operator-manual/applicationset/Generators-Matrix.md b/docs/operator-manual/applicationset/Generators-Matrix.md index 4cc149060241b..e45c6a058c0d1 100644 --- a/docs/operator-manual/applicationset/Generators-Matrix.md +++ b/docs/operator-manual/applicationset/Generators-Matrix.md @@ -104,10 +104,72 @@ Finally, the Matrix generator will combine both sets of outputs, and produce: ``` (*The full example can be found [here](https://github.com/argoproj/argo-cd/tree/master/applicationset/examples/matrix).*) +## Using Parameters from one child generator in another child generator + +The Matrix generator allows using the parameters generated by one child generator inside another. +Below is an example that uses a git-files generator in conjunction with a cluster generator. + +```yaml +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: cluster-git +spec: + generators: + # matrix 'parent' generator + - matrix: + generators: + # git generator, 'child' #1 + - git: + repoURL: https://github.com/argoproj/applicationset.git + revision: HEAD + files: + - path: "examples/git-generator-files-discovery/cluster-config/**/config.json" + # cluster generator, 'child' #2 + - clusters: + selector: + matchLabels: + argocd.argoproj.io/secret-type: cluster + kubernetes.io/environment: '{{path.basename}}' + template: + metadata: + name: '{{name}}-guestbook' + spec: + project: default + source: + repoURL: https://github.com/argoproj/applicationset.git + targetRevision: HEAD + path: "examples/git-generator-files-discovery/apps/guestbook" + destination: + server: '{{server}}' + namespace: guestbook +``` +Here is the corresponding folder structure for the git repository used by the git-files generator: + +``` +├── apps +│ └── guestbook +│ ├── guestbook-ui-deployment.yaml +│ ├── guestbook-ui-svc.yaml +│ └── kustomization.yaml +├── cluster-config +│ └── engineering +│ ├── dev +│ │ └── config.json +│ └── prod +│ └── config.json +└── git-generator-files.yaml +``` +In the above example, the `{{path.basename}}` parameters produced by the git-files generator will resolve to `dev` and `prod`. +In the 2nd child generator, the label selector with label `kubernetes.io/environment: {{path.basename}}` will resolve with the values produced by the first child generator's parameters (`kubernetes.io/environment: prod` and `kubernetes.io/environment: dev`). + +So in the above example, clusters with the label `kubernetes.io/environment: prod` will have only prod-specific configuration (ie. `prod/config.json`) applied to it, wheres clusters +with the label `kubernetes.io/environment: dev` will have only dev-specific configuration (ie. `dev/config.json`) + ## Restrictions 1. The Matrix generator currently only supports combining the outputs of only two child generators (eg does not support generating combinations for 3 or more). -1. You should specify only a single generator per array entry, eg this is not valid: +2. You should specify only a single generator per array entry, eg this is not valid: ```yaml - matrix: generators: @@ -115,7 +177,7 @@ Finally, the Matrix generator will combine both sets of outputs, and produce: git: # (...) ``` - While this *will* be accepted by Kubernetes API validation, the controller will report an error on generation. Each generator should be specified in a separate array element, as in the examples above. -1. The Matrix generator does not currently support [`template` overrides](Template.md#generator-templates) specified on child generators, eg this `template` will not be processed: +3. The Matrix generator does not currently support [`template` overrides](Template.md#generator-templates) specified on child generators, eg this `template` will not be processed: ```yaml - matrix: generators: @@ -124,7 +186,7 @@ Finally, the Matrix generator will combine both sets of outputs, and produce: - # (...) template: { } # Not processed ``` -1. Combination-type generators (matrix or merge) can only be nested once. For example, this will not work: +4. Combination-type generators (matrix or merge) can only be nested once. For example, this will not work: ```yaml - matrix: generators: @@ -136,3 +198,39 @@ Finally, the Matrix generator will combine both sets of outputs, and produce: elements: - # (...) ``` +5. When using parameters from one child generator inside another child generator, the child generator that *consumes* the parameters **must come after** the child generator that *produces* the parameters. +For example, the below example would be invalid (cluster-generator must come after the git-files generator): +```yaml +- matrix: + generators: + # cluster generator, 'child' #1 + - clusters: + selector: + matchLabels: + argocd.argoproj.io/secret-type: cluster + kubernetes.io/environment: '{{path.basename}}' # {{path.basename}} is produced by git-files generator + # git generator, 'child' #2 + - git: + repoURL: https://github.com/argoproj/applicationset.git + revision: HEAD + files: + - path: "examples/git-generator-files-discovery/cluster-config/**/config.json" +``` +6. You cannot have both child generators consuming parameters from each another. In the example below, the cluster generator is consuming the `{{path.basename}}` parameter produced by the git-files generator, whereas the git-files generator is consuming the `{{name}}` parameter produced by the cluster generator. This will result in a circular dependency, which is invalid. +```yaml +- matrix: + generators: + # cluster generator, 'child' #1 + - clusters: + selector: + matchLabels: + argocd.argoproj.io/secret-type: cluster + kubernetes.io/environment: '{{path.basename}}' # {{path.basename}} is produced by git-files generator + # git generator, 'child' #2 + - git: + repoURL: https://github.com/argoproj/applicationset.git + revision: HEAD + files: + - path: "examples/git-generator-files-discovery/cluster-config/engineering/{{name}}**/config.json" # {{name}} is produced by cluster generator +``` + From ad633e2765d70795cd1f306bcaed7b994b103fc4 Mon Sep 17 00:00:00 2001 From: jkulkarn Date: Thu, 14 Apr 2022 18:37:32 -0400 Subject: [PATCH 09/22] Small wording fix. Signed-off-by: jkulkarn --- docs/operator-manual/applicationset/Generators-Matrix.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/operator-manual/applicationset/Generators-Matrix.md b/docs/operator-manual/applicationset/Generators-Matrix.md index e45c6a058c0d1..a715fb697532c 100644 --- a/docs/operator-manual/applicationset/Generators-Matrix.md +++ b/docs/operator-manual/applicationset/Generators-Matrix.md @@ -106,7 +106,7 @@ Finally, the Matrix generator will combine both sets of outputs, and produce: ## Using Parameters from one child generator in another child generator -The Matrix generator allows using the parameters generated by one child generator inside another. +The Matrix generator allows using the parameters generated by one child generator inside another child generator. Below is an example that uses a git-files generator in conjunction with a cluster generator. ```yaml From a7196388cd88b3d1e87649422587af0f34f9bb77 Mon Sep 17 00:00:00 2001 From: jkulkarn Date: Sat, 16 Apr 2022 14:29:48 -0400 Subject: [PATCH 10/22] Small change to generator_spec_processor.go Signed-off-by: jkulkarn --- .../generators/generator_spec_processor.go | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/applicationset/generators/generator_spec_processor.go b/applicationset/generators/generator_spec_processor.go index f3fa1c804801e..9cf2da6239cb5 100644 --- a/applicationset/generators/generator_spec_processor.go +++ b/applicationset/generators/generator_spec_processor.go @@ -93,27 +93,24 @@ func mergeGeneratorTemplate(g Generator, requestedGenerator *argoprojiov1alpha1. } // Currently for Matrix Generator. Allows interpolating the matrix's 2nd child generator with values from the 1st child generator -// "params" parameter here are params from the 1st child generator. Uses similar process for interpreting values as the actual Application Template +// "params" parameter is an array, where each index corresponds to a generator. Each index contains a map w/ that generator's parameters. +// Since the Matrix generator currently only allows 2 child generators, effectively this params array will always be size 1 (containing the 1st child generator's params) +// Uses similar process for interpreting values as the actual Application Template func interpolateGenerator(requestedGenerator *argoprojiov1alpha1.ApplicationSetGenerator, params []map[string]string) error { tmplBytes, err := json.Marshal(requestedGenerator) if err != nil { log.WithError(err).WithField("requestedGenerator", requestedGenerator).Error("error marshalling requested generator for interpolation") return err } - tmpParams := make(map[string]string) - for _, currParam := range params { - for k, v := range currParam { - tmpParams[k] = v - } - } - fstTmpl := fasttemplate.New(string(tmplBytes), "{{", "}}") render := utils.Render{} - replacedTmplStr, err := render.Replace(fstTmpl, tmpParams, true) + fstTmpl := fasttemplate.New(string(tmplBytes), "{{", "}}") + replacedTmplStr, err := render.Replace(fstTmpl, params[0], true) if err != nil { log.WithError(err).WithField("interpolatedGeneratorString", replacedTmplStr).Error("error interpolating generator with other generator's parameter") return err } + err = json.Unmarshal([]byte(replacedTmplStr), requestedGenerator) if err != nil { log.WithError(err).WithField("requestedGenerator", requestedGenerator).Error("error unmarshalling requested generator for interpolation") From a6a915b56abafa76b7c4777e076975eaea3415ea Mon Sep 17 00:00:00 2001 From: jkulkarn Date: Wed, 20 Apr 2022 12:15:08 -0400 Subject: [PATCH 11/22] Fixing Test case Signed-off-by: jkulkarn --- .../generator_spec_processor_test.go | 100 +----------------- 1 file changed, 1 insertion(+), 99 deletions(-) diff --git a/applicationset/generators/generator_spec_processor_test.go b/applicationset/generators/generator_spec_processor_test.go index 8680009b1da37..37d0e14d32f95 100644 --- a/applicationset/generators/generator_spec_processor_test.go +++ b/applicationset/generators/generator_spec_processor_test.go @@ -2,7 +2,6 @@ package generators import ( "context" - "fmt" log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -99,104 +98,8 @@ func getMockClusterGenerator() Generator { } func getMockGitGenerator() Generator { - cases := []struct { - name string - directories []argoprojiov1alpha1.GitDirectoryGeneratorItem - repoApps []string - repoError error - expected []map[string]string - expectedError error - }{ - { - name: "happy flow - created apps", - directories: []argoprojiov1alpha1.GitDirectoryGeneratorItem{{Path: "*"}}, - repoApps: []string{ - "app1", - "app2", - "app_3", - "p1/app4", - }, - repoError: nil, - expected: []map[string]string{ - {"path": "app1", "path.basename": "app1", "path.basenameNormalized": "app1"}, - {"path": "app2", "path.basename": "app2", "path.basenameNormalized": "app2"}, - {"path": "app_3", "path.basename": "app_3", "path.basenameNormalized": "app-3"}, - }, - expectedError: nil, - }, - { - name: "It filters application according to the paths", - directories: []argoprojiov1alpha1.GitDirectoryGeneratorItem{{Path: "p1/*"}, {Path: "p1/*/*"}}, - repoApps: []string{ - "app1", - "p1/app2", - "p1/p2/app3", - "p1/p2/p3/app4", - }, - repoError: nil, - expected: []map[string]string{ - {"path": "p1/app2", "path.basename": "app2", "path[0]": "p1", "path.basenameNormalized": "app2"}, - {"path": "p1/p2/app3", "path.basename": "app3", "path[0]": "p1", "path[1]": "p2", "path.basenameNormalized": "app3"}, - }, - expectedError: nil, - }, - { - name: "It filters application according to the paths with Exclude", - directories: []argoprojiov1alpha1.GitDirectoryGeneratorItem{{Path: "p1/*", Exclude: true}, {Path: "*"}, {Path: "*/*"}}, - repoApps: []string{ - "app1", - "app2", - "p1/app2", - "p1/app3", - "p2/app3", - }, - repoError: nil, - expected: []map[string]string{ - {"path": "app1", "path.basename": "app1", "path.basenameNormalized": "app1"}, - {"path": "app2", "path.basename": "app2", "path.basenameNormalized": "app2"}, - {"path": "p2/app3", "path.basename": "app3", "path[0]": "p2", "path.basenameNormalized": "app3"}, - }, - expectedError: nil, - }, - { - name: "Expecting same exclude behavior with different order", - directories: []argoprojiov1alpha1.GitDirectoryGeneratorItem{{Path: "*"}, {Path: "*/*"}, {Path: "p1/*", Exclude: true}}, - repoApps: []string{ - "app1", - "app2", - "p1/app2", - "p1/app3", - "p2/app3", - }, - repoError: nil, - expected: []map[string]string{ - {"path": "app1", "path.basename": "app1", "path.basenameNormalized": "app1"}, - {"path": "app2", "path.basename": "app2", "path.basenameNormalized": "app2"}, - {"path": "p2/app3", "path.basename": "app3", "path[0]": "p2", "path.basenameNormalized": "app3"}, - }, - expectedError: nil, - }, - { - name: "handles empty response from repo server", - directories: []argoprojiov1alpha1.GitDirectoryGeneratorItem{{Path: "*"}}, - repoApps: []string{}, - repoError: nil, - expected: []map[string]string{}, - expectedError: nil, - }, - { - name: "handles error from repo server", - directories: []argoprojiov1alpha1.GitDirectoryGeneratorItem{{Path: "*"}}, - repoApps: []string{}, - repoError: fmt.Errorf("error"), - expected: []map[string]string{}, - expectedError: fmt.Errorf("error"), - }, - } argoCDServiceMock := argoCDServiceMock{mock: &mock.Mock{}} - - argoCDServiceMock.mock.On("GetDirectories", mock.Anything, mock.Anything, mock.Anything).Return(cases[0].repoApps, cases[0].repoError) - + argoCDServiceMock.mock.On("GetDirectories", mock.Anything, mock.Anything, mock.Anything).Return([]string{"app1", "app2", "app_3", "p1/app4"}, nil) var gitGenerator = NewGitGenerator(argoCDServiceMock) return gitGenerator } @@ -261,7 +164,6 @@ func TestInterpolateGenerator(t *testing.T) { }}, }, } - //mockGitGenerator := getMockGitGenerator() gitGeneratorParams := []map[string]string{ {"path": "p1/p2/app3", "path.basename": "app3", "path[0]": "p1", "path[1]": "p2", "path.basenameNormalized": "app3"}, } From 6001908613506875b65c0421bfb568f2d9420940 Mon Sep 17 00:00:00 2001 From: jkulkarn Date: Wed, 20 Apr 2022 14:24:00 -0400 Subject: [PATCH 12/22] Small changes for matrix + generator_spec_processor.go Signed-off-by: jkulkarn --- applicationset/generators/generator_spec_processor.go | 7 +++++++ applicationset/generators/matrix.go | 9 ++++----- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/applicationset/generators/generator_spec_processor.go b/applicationset/generators/generator_spec_processor.go index 9cf2da6239cb5..6af4b739780b1 100644 --- a/applicationset/generators/generator_spec_processor.go +++ b/applicationset/generators/generator_spec_processor.go @@ -103,6 +103,13 @@ func interpolateGenerator(requestedGenerator *argoprojiov1alpha1.ApplicationSetG return err } + //tmpParams := make([]map[string]string, len(params)) + //for currIndex, currParam := range params { + // for k, v := range currParam { + // tmpParams[currIndex][k] = v + // } + //} + render := utils.Render{} fstTmpl := fasttemplate.New(string(tmplBytes), "{{", "}}") replacedTmplStr, err := render.Replace(fstTmpl, params[0], true) diff --git a/applicationset/generators/matrix.go b/applicationset/generators/matrix.go index d5cbfd78e7c26..98d47283101f0 100644 --- a/applicationset/generators/matrix.go +++ b/applicationset/generators/matrix.go @@ -48,12 +48,11 @@ func (m *MatrixGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha1.App if err != nil { return nil, err } - g1, err := m.getParams(appSetGenerator.Matrix.Generators[1], appSet, g0) - if err != nil { - return nil, err - } - for _, a := range g0 { + g1, err := m.getParams(appSetGenerator.Matrix.Generators[1], appSet, []map[string]string{a}) + if err != nil { + return nil, err + } for _, b := range g1 { val, err := utils.CombineStringMaps(a, b) if err != nil { From 295a6ed10309879de5b8f6615922960455f2548d Mon Sep 17 00:00:00 2001 From: jkulkarn Date: Fri, 22 Apr 2022 10:22:39 -0700 Subject: [PATCH 13/22] Fixed (I believe) the issue that @Lobstrosity mentioned. Signed-off-by: jkulkarn --- .../generators/generator_spec_processor.go | 33 +++++++++---------- .../generator_spec_processor_test.go | 14 ++++---- 2 files changed, 22 insertions(+), 25 deletions(-) diff --git a/applicationset/generators/generator_spec_processor.go b/applicationset/generators/generator_spec_processor.go index 6af4b739780b1..9f749fa4f8148 100644 --- a/applicationset/generators/generator_spec_processor.go +++ b/applicationset/generators/generator_spec_processor.go @@ -33,8 +33,9 @@ func Transform(requestedGenerator argoprojiov1alpha1.ApplicationSetGenerator, al } continue } + var params []map[string]string if len(genParams) != 0 { - err := interpolateGenerator(&requestedGenerator, genParams) + interpolatedGenerator, err := interpolateGenerator(&requestedGenerator, genParams) if err != nil { log.WithError(err).WithField("genParams", genParams). Error("error interpolating params for generator") @@ -43,8 +44,10 @@ func Transform(requestedGenerator argoprojiov1alpha1.ApplicationSetGenerator, al } continue } + params, err = g.GenerateParams(&interpolatedGenerator, appSet) + } else { + params, err = g.GenerateParams(&requestedGenerator, appSet) } - params, err := g.GenerateParams(&requestedGenerator, appSet) if err != nil { log.WithError(err).WithField("generator", g). Error("error generating params") @@ -96,32 +99,26 @@ func mergeGeneratorTemplate(g Generator, requestedGenerator *argoprojiov1alpha1. // "params" parameter is an array, where each index corresponds to a generator. Each index contains a map w/ that generator's parameters. // Since the Matrix generator currently only allows 2 child generators, effectively this params array will always be size 1 (containing the 1st child generator's params) // Uses similar process for interpreting values as the actual Application Template -func interpolateGenerator(requestedGenerator *argoprojiov1alpha1.ApplicationSetGenerator, params []map[string]string) error { - tmplBytes, err := json.Marshal(requestedGenerator) +func interpolateGenerator(requestedGenerator *argoprojiov1alpha1.ApplicationSetGenerator, params []map[string]string) (argoprojiov1alpha1.ApplicationSetGenerator, error) { + interpolatedGenerator := requestedGenerator.DeepCopy() + tmplBytes, err := json.Marshal(interpolatedGenerator) if err != nil { - log.WithError(err).WithField("requestedGenerator", requestedGenerator).Error("error marshalling requested generator for interpolation") - return err + log.WithError(err).WithField("requestedGenerator", interpolatedGenerator).Error("error marshalling requested generator for interpolation") + return *interpolatedGenerator, err } - //tmpParams := make([]map[string]string, len(params)) - //for currIndex, currParam := range params { - // for k, v := range currParam { - // tmpParams[currIndex][k] = v - // } - //} - render := utils.Render{} fstTmpl := fasttemplate.New(string(tmplBytes), "{{", "}}") replacedTmplStr, err := render.Replace(fstTmpl, params[0], true) if err != nil { log.WithError(err).WithField("interpolatedGeneratorString", replacedTmplStr).Error("error interpolating generator with other generator's parameter") - return err + return *interpolatedGenerator, err } - err = json.Unmarshal([]byte(replacedTmplStr), requestedGenerator) + err = json.Unmarshal([]byte(replacedTmplStr), interpolatedGenerator) if err != nil { - log.WithError(err).WithField("requestedGenerator", requestedGenerator).Error("error unmarshalling requested generator for interpolation") - return err + log.WithError(err).WithField("requestedGenerator", interpolatedGenerator).Error("error unmarshalling requested generator for interpolation") + return *interpolatedGenerator, err } - return nil + return *interpolatedGenerator, nil } diff --git a/applicationset/generators/generator_spec_processor_test.go b/applicationset/generators/generator_spec_processor_test.go index 37d0e14d32f95..6bc1034ca76e8 100644 --- a/applicationset/generators/generator_spec_processor_test.go +++ b/applicationset/generators/generator_spec_processor_test.go @@ -167,14 +167,14 @@ func TestInterpolateGenerator(t *testing.T) { gitGeneratorParams := []map[string]string{ {"path": "p1/p2/app3", "path.basename": "app3", "path[0]": "p1", "path[1]": "p2", "path.basenameNormalized": "app3"}, } - err := interpolateGenerator(requestedGenerator, gitGeneratorParams) + interpolatedGenerator, err := interpolateGenerator(requestedGenerator, gitGeneratorParams) if err != nil { log.WithError(err).WithField("requestedGenerator", requestedGenerator).Error("error interpolating Generator") return } - assert.Equal(t, "app3", requestedGenerator.Clusters.Selector.MatchLabels["path-basename"]) - assert.Equal(t, "p1", requestedGenerator.Clusters.Selector.MatchLabels["path-zero"]) - assert.Equal(t, "p1/p2/app3", requestedGenerator.Clusters.Selector.MatchLabels["path-full"]) + assert.Equal(t, "app3", interpolatedGenerator.Clusters.Selector.MatchLabels["path-basename"]) + assert.Equal(t, "p1", interpolatedGenerator.Clusters.Selector.MatchLabels["path-zero"]) + assert.Equal(t, "p1/p2/app3", interpolatedGenerator.Clusters.Selector.MatchLabels["path-full"]) fileNamePath := argoprojiov1alpha1.GitFileGeneratorItem{ Path: "{{name}}", @@ -192,11 +192,11 @@ func TestInterpolateGenerator(t *testing.T) { clusterGeneratorParams := []map[string]string{ {"name": "production_01/west", "server": "https://production-01.example.com"}, } - err = interpolateGenerator(requestedGenerator, clusterGeneratorParams) + interpolatedGenerator, err = interpolateGenerator(requestedGenerator, clusterGeneratorParams) if err != nil { log.WithError(err).WithField("requestedGenerator", requestedGenerator).Error("error interpolating Generator") return } - assert.Equal(t, "production_01/west", requestedGenerator.Git.Files[0].Path) - assert.Equal(t, "https://production-01.example.com", requestedGenerator.Git.Files[1].Path) + assert.Equal(t, "production_01/west", interpolatedGenerator.Git.Files[0].Path) + assert.Equal(t, "https://production-01.example.com", interpolatedGenerator.Git.Files[1].Path) } From 4bed874f29887ed2ed75f5aabb6a6e630befce9c Mon Sep 17 00:00:00 2001 From: jkulkarn Date: Fri, 22 Apr 2022 10:46:52 -0700 Subject: [PATCH 14/22] Refactored code to accept map[string]string instead of []map[string]string (for interpolateGenerator && Transform) Signed-off-by: jkulkarn --- applicationset/controllers/applicationset_controller.go | 2 +- applicationset/generators/generator_spec_processor.go | 6 +++--- applicationset/generators/matrix.go | 6 +++--- applicationset/generators/merge.go | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/applicationset/controllers/applicationset_controller.go b/applicationset/controllers/applicationset_controller.go index 4469397415e9e..a14148a5820a2 100644 --- a/applicationset/controllers/applicationset_controller.go +++ b/applicationset/controllers/applicationset_controller.go @@ -430,7 +430,7 @@ func (r *ApplicationSetReconciler) generateApplications(applicationSetInfo argop var applicationSetReason argoprojiov1alpha1.ApplicationSetReasonType for _, requestedGenerator := range applicationSetInfo.Spec.Generators { - t, err := generators.Transform(requestedGenerator, r.Generators, applicationSetInfo.Spec.Template, &applicationSetInfo, []map[string]string{}) + t, err := generators.Transform(requestedGenerator, r.Generators, applicationSetInfo.Spec.Template, &applicationSetInfo, map[string]string{}) if err != nil { log.WithError(err).WithField("generator", requestedGenerator). Error("error generating application from params") diff --git a/applicationset/generators/generator_spec_processor.go b/applicationset/generators/generator_spec_processor.go index 9f749fa4f8148..7537328272e87 100644 --- a/applicationset/generators/generator_spec_processor.go +++ b/applicationset/generators/generator_spec_processor.go @@ -17,7 +17,7 @@ type TransformResult struct { } //Transform a spec generator to list of paramSets and a template -func Transform(requestedGenerator argoprojiov1alpha1.ApplicationSetGenerator, allGenerators map[string]Generator, baseTemplate argoprojiov1alpha1.ApplicationSetTemplate, appSet *argoprojiov1alpha1.ApplicationSet, genParams []map[string]string) ([]TransformResult, error) { +func Transform(requestedGenerator argoprojiov1alpha1.ApplicationSetGenerator, allGenerators map[string]Generator, baseTemplate argoprojiov1alpha1.ApplicationSetTemplate, appSet *argoprojiov1alpha1.ApplicationSet, genParams map[string]string) ([]TransformResult, error) { res := []TransformResult{} var firstError error @@ -99,7 +99,7 @@ func mergeGeneratorTemplate(g Generator, requestedGenerator *argoprojiov1alpha1. // "params" parameter is an array, where each index corresponds to a generator. Each index contains a map w/ that generator's parameters. // Since the Matrix generator currently only allows 2 child generators, effectively this params array will always be size 1 (containing the 1st child generator's params) // Uses similar process for interpreting values as the actual Application Template -func interpolateGenerator(requestedGenerator *argoprojiov1alpha1.ApplicationSetGenerator, params []map[string]string) (argoprojiov1alpha1.ApplicationSetGenerator, error) { +func interpolateGenerator(requestedGenerator *argoprojiov1alpha1.ApplicationSetGenerator, params map[string]string) (argoprojiov1alpha1.ApplicationSetGenerator, error) { interpolatedGenerator := requestedGenerator.DeepCopy() tmplBytes, err := json.Marshal(interpolatedGenerator) if err != nil { @@ -109,7 +109,7 @@ func interpolateGenerator(requestedGenerator *argoprojiov1alpha1.ApplicationSetG render := utils.Render{} fstTmpl := fasttemplate.New(string(tmplBytes), "{{", "}}") - replacedTmplStr, err := render.Replace(fstTmpl, params[0], true) + replacedTmplStr, err := render.Replace(fstTmpl, params, true) if err != nil { log.WithError(err).WithField("interpolatedGeneratorString", replacedTmplStr).Error("error interpolating generator with other generator's parameter") return *interpolatedGenerator, err diff --git a/applicationset/generators/matrix.go b/applicationset/generators/matrix.go index 98d47283101f0..1c9d9a5be4f84 100644 --- a/applicationset/generators/matrix.go +++ b/applicationset/generators/matrix.go @@ -44,12 +44,12 @@ func (m *MatrixGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha1.App res := []map[string]string{} - g0, err := m.getParams(appSetGenerator.Matrix.Generators[0], appSet, []map[string]string{}) + g0, err := m.getParams(appSetGenerator.Matrix.Generators[0], appSet, map[string]string{}) if err != nil { return nil, err } for _, a := range g0 { - g1, err := m.getParams(appSetGenerator.Matrix.Generators[1], appSet, []map[string]string{a}) + g1, err := m.getParams(appSetGenerator.Matrix.Generators[1], appSet, a) if err != nil { return nil, err } @@ -65,7 +65,7 @@ func (m *MatrixGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha1.App return res, nil } -func (m *MatrixGenerator) getParams(appSetBaseGenerator argoprojiov1alpha1.ApplicationSetNestedGenerator, appSet *argoprojiov1alpha1.ApplicationSet, params []map[string]string) ([]map[string]string, error) { +func (m *MatrixGenerator) getParams(appSetBaseGenerator argoprojiov1alpha1.ApplicationSetNestedGenerator, appSet *argoprojiov1alpha1.ApplicationSet, params map[string]string) ([]map[string]string, error) { var matrix *argoprojiov1alpha1.MatrixGenerator if appSetBaseGenerator.Matrix != nil { // Since nested matrix generator is represented as a JSON object in the CRD, we unmarshall it back to a Go struct here. diff --git a/applicationset/generators/merge.go b/applicationset/generators/merge.go index 9dbc87457f779..a3ad8d5cf2eb8 100644 --- a/applicationset/generators/merge.go +++ b/applicationset/generators/merge.go @@ -164,7 +164,7 @@ func (m *MergeGenerator) getParams(appSetBaseGenerator argoprojiov1alpha1.Applic m.supportedGenerators, argoprojiov1alpha1.ApplicationSetTemplate{}, appSet, - []map[string]string{}) + map[string]string{}) if err != nil { return nil, fmt.Errorf("child generator returned an error on parameter generation: %v", err) From 54deaed8e02a7fd374e05213aae92288a6a7872f Mon Sep 17 00:00:00 2001 From: jkulkarn Date: Fri, 22 Apr 2022 10:54:39 -0700 Subject: [PATCH 15/22] Fixing test cases Signed-off-by: jkulkarn --- .../generators/generator_spec_processor_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/applicationset/generators/generator_spec_processor_test.go b/applicationset/generators/generator_spec_processor_test.go index 6bc1034ca76e8..14c267a63a124 100644 --- a/applicationset/generators/generator_spec_processor_test.go +++ b/applicationset/generators/generator_spec_processor_test.go @@ -164,8 +164,8 @@ func TestInterpolateGenerator(t *testing.T) { }}, }, } - gitGeneratorParams := []map[string]string{ - {"path": "p1/p2/app3", "path.basename": "app3", "path[0]": "p1", "path[1]": "p2", "path.basenameNormalized": "app3"}, + gitGeneratorParams := map[string]string{ + "path": "p1/p2/app3", "path.basename": "app3", "path[0]": "p1", "path[1]": "p2", "path.basenameNormalized": "app3", } interpolatedGenerator, err := interpolateGenerator(requestedGenerator, gitGeneratorParams) if err != nil { @@ -189,8 +189,8 @@ func TestInterpolateGenerator(t *testing.T) { Template: argoprojiov1alpha1.ApplicationSetTemplate{}, }, } - clusterGeneratorParams := []map[string]string{ - {"name": "production_01/west", "server": "https://production-01.example.com"}, + clusterGeneratorParams := map[string]string{ + "name": "production_01/west", "server": "https://production-01.example.com", } interpolatedGenerator, err = interpolateGenerator(requestedGenerator, clusterGeneratorParams) if err != nil { From 5785aab71f07cdc54fa4fd81aa70f9c38f99bd3f Mon Sep 17 00:00:00 2001 From: jkulkarn Date: Sun, 24 Apr 2022 22:08:04 -0700 Subject: [PATCH 16/22] Fixing lint error. Signed-off-by: jkulkarn --- applicationset/generators/generator_spec_processor.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/applicationset/generators/generator_spec_processor.go b/applicationset/generators/generator_spec_processor.go index 7537328272e87..9b04f35021cce 100644 --- a/applicationset/generators/generator_spec_processor.go +++ b/applicationset/generators/generator_spec_processor.go @@ -20,6 +20,7 @@ type TransformResult struct { func Transform(requestedGenerator argoprojiov1alpha1.ApplicationSetGenerator, allGenerators map[string]Generator, baseTemplate argoprojiov1alpha1.ApplicationSetTemplate, appSet *argoprojiov1alpha1.ApplicationSet, genParams map[string]string) ([]TransformResult, error) { res := []TransformResult{} var firstError error + interpolatedGenerator := requestedGenerator.DeepCopy() generators := GetRelevantGenerators(&requestedGenerator, allGenerators) for _, g := range generators { @@ -35,7 +36,8 @@ func Transform(requestedGenerator argoprojiov1alpha1.ApplicationSetGenerator, al } var params []map[string]string if len(genParams) != 0 { - interpolatedGenerator, err := interpolateGenerator(&requestedGenerator, genParams) + tempInterpolatedGenerator, err := interpolateGenerator(&requestedGenerator, genParams) + interpolatedGenerator = &tempInterpolatedGenerator if err != nil { log.WithError(err).WithField("genParams", genParams). Error("error interpolating params for generator") @@ -44,10 +46,8 @@ func Transform(requestedGenerator argoprojiov1alpha1.ApplicationSetGenerator, al } continue } - params, err = g.GenerateParams(&interpolatedGenerator, appSet) - } else { - params, err = g.GenerateParams(&requestedGenerator, appSet) } + params, err = g.GenerateParams(interpolatedGenerator, appSet) if err != nil { log.WithError(err).WithField("generator", g). Error("error generating params") From db5e8af2c19de2ad50449aa125395211408a34f9 Mon Sep 17 00:00:00 2001 From: jkulkarn Date: Tue, 24 May 2022 16:02:20 -0700 Subject: [PATCH 17/22] Using test-case suggestion from @rumstead. Signed-off-by: jkulkarn --- applicationset/generators/matrix_test.go | 8 ++++---- go.mod | 2 +- go.sum | 2 ++ 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/applicationset/generators/matrix_test.go b/applicationset/generators/matrix_test.go index c5c7fce24d131..0b02ff483ceaf 100644 --- a/applicationset/generators/matrix_test.go +++ b/applicationset/generators/matrix_test.go @@ -127,7 +127,7 @@ func TestMatrixGenerate(t *testing.T) { testCaseCopy := testCase // Since tests may run in parallel t.Run(testCaseCopy.name, func(t *testing.T) { - mock := &generatorMock{} + generatorMock := &generatorMock{} appSet := &argoprojiov1alpha1.ApplicationSet{} for _, g := range testCaseCopy.baseGenerators { @@ -136,7 +136,7 @@ func TestMatrixGenerate(t *testing.T) { Git: g.Git, List: g.List, } - mock.On("GenerateParams", &gitGeneratorSpec, appSet).Return([]map[string]string{ + generatorMock.On("GenerateParams", mock.AnythingOfType("*v1alpha1.ApplicationSetGenerator"), appSet).Return([]map[string]string{ { "path": "app1", "path.basename": "app1", @@ -149,13 +149,13 @@ func TestMatrixGenerate(t *testing.T) { }, }, nil) - mock.On("GetTemplate", &gitGeneratorSpec). + generatorMock.On("GetTemplate", &gitGeneratorSpec). Return(&argoprojiov1alpha1.ApplicationSetTemplate{}) } var matrixGenerator = NewMatrixGenerator( map[string]Generator{ - "Git": mock, + "Git": generatorMock, "List": &ListGenerator{}, }, ) diff --git a/go.mod b/go.mod index 49afec77dd5f6..9a34fe8e77ebe 100644 --- a/go.mod +++ b/go.mod @@ -191,7 +191,7 @@ require ( github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sergi/go-diff v1.1.0 // indirect github.com/slack-go/slack v0.10.1 // indirect - github.com/stretchr/objx v0.2.0 // indirect + github.com/stretchr/objx v0.3.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/vmihailenco/go-tinylfu v0.2.1 // indirect github.com/vmihailenco/msgpack/v5 v5.3.4 // indirect diff --git a/go.sum b/go.sum index ab6c56d62625e..02224444a75ee 100644 --- a/go.sum +++ b/go.sum @@ -1070,6 +1070,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/objx v0.3.0 h1:NGXK3lHquSN08v5vWalVI/L8XU9hdzE/G6xsrze47As= +github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v0.0.0-20161117074351-18a02ba4a312/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= From 0dae059829ab727ef3c0347a73e3aeef2c5bd778 Mon Sep 17 00:00:00 2001 From: jkulkarn Date: Tue, 24 May 2022 21:42:02 -0700 Subject: [PATCH 18/22] Changing up naming from testing. Signed-off-by: jkulkarn --- applicationset/generators/matrix_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/applicationset/generators/matrix_test.go b/applicationset/generators/matrix_test.go index 0b02ff483ceaf..e33043fe858c9 100644 --- a/applicationset/generators/matrix_test.go +++ b/applicationset/generators/matrix_test.go @@ -127,7 +127,7 @@ func TestMatrixGenerate(t *testing.T) { testCaseCopy := testCase // Since tests may run in parallel t.Run(testCaseCopy.name, func(t *testing.T) { - generatorMock := &generatorMock{} + genMock := &generatorMock{} appSet := &argoprojiov1alpha1.ApplicationSet{} for _, g := range testCaseCopy.baseGenerators { @@ -136,7 +136,7 @@ func TestMatrixGenerate(t *testing.T) { Git: g.Git, List: g.List, } - generatorMock.On("GenerateParams", mock.AnythingOfType("*v1alpha1.ApplicationSetGenerator"), appSet).Return([]map[string]string{ + genMock.On("GenerateParams", mock.AnythingOfType("*v1alpha1.ApplicationSetGenerator"), appSet).Return([]map[string]string{ { "path": "app1", "path.basename": "app1", @@ -149,13 +149,13 @@ func TestMatrixGenerate(t *testing.T) { }, }, nil) - generatorMock.On("GetTemplate", &gitGeneratorSpec). + genMock.On("GetTemplate", &gitGeneratorSpec). Return(&argoprojiov1alpha1.ApplicationSetTemplate{}) } var matrixGenerator = NewMatrixGenerator( map[string]Generator{ - "Git": generatorMock, + "Git": genMock, "List": &ListGenerator{}, }, ) From 2ec3e94a5d29322c6cfb9bf46d4a23d49a38143f Mon Sep 17 00:00:00 2001 From: jkulkarn Date: Tue, 24 May 2022 21:50:53 -0700 Subject: [PATCH 19/22] Updated go.sum Signed-off-by: jkulkarn --- go.sum | 1 - 1 file changed, 1 deletion(-) diff --git a/go.sum b/go.sum index a1a91d9a33556..10093541edca5 100644 --- a/go.sum +++ b/go.sum @@ -1070,7 +1070,6 @@ github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3 github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.3.0 h1:NGXK3lHquSN08v5vWalVI/L8XU9hdzE/G6xsrze47As= github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= From cd3602da359fb91e2735e0aca0ae9c06028468f7 Mon Sep 17 00:00:00 2001 From: jkulkarn Date: Tue, 24 May 2022 21:53:03 -0700 Subject: [PATCH 20/22] Cleaning up for linter. Signed-off-by: jkulkarn --- .../generator_spec_processor_test.go | 22 ------------------- 1 file changed, 22 deletions(-) diff --git a/applicationset/generators/generator_spec_processor_test.go b/applicationset/generators/generator_spec_processor_test.go index 14c267a63a124..96e8497b482ee 100644 --- a/applicationset/generators/generator_spec_processor_test.go +++ b/applicationset/generators/generator_spec_processor_test.go @@ -17,28 +17,6 @@ import ( argoprojiov1alpha1 "github.com/argoproj/argo-cd/v2/pkg/apis/applicationset/v1alpha1" ) -type genProcessorMock struct { - mock.Mock -} - -func (g *genProcessorMock) Transform(requestedGenerator argoprojiov1alpha1.ApplicationSetGenerator, allGenerators map[string]Generator, baseTemplate argoprojiov1alpha1.ApplicationSetTemplate, appSet *argoprojiov1alpha1.ApplicationSet, genParams []map[string]string) ([]TransformResult, error) { - args := g.Called(requestedGenerator) - - return args.Get(0).([]TransformResult), args.Error(1) -} - -func (g *genProcessorMock) GetRelevantGenerators(requestedGenerator argoprojiov1alpha1.ApplicationSetGenerator, generators map[string]Generator) []Generator { - args := g.Called(requestedGenerator) - - return args.Get(0).([]Generator) -} - -func (g *genProcessorMock) interpolateGenerator(requestedGenerator *argoprojiov1alpha1.ApplicationSetGenerator, params []map[string]string) error { - args := g.Called(requestedGenerator) - - return args.Error(1) -} - func getMockClusterGenerator() Generator { clusters := []crtclient.Object{ &corev1.Secret{ From b2d336e7cd3a41adc018743adb8a7bf02a09a7cf Mon Sep 17 00:00:00 2001 From: Jay P Kulkarni Date: Thu, 2 Jun 2022 13:31:58 -0700 Subject: [PATCH 21/22] Update Generators-Matrix.md Changed all numbering back to 1 in "Restrictions" section Signed-off-by: Jay P Kulkarni --- .../applicationset/Generators-Matrix.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/operator-manual/applicationset/Generators-Matrix.md b/docs/operator-manual/applicationset/Generators-Matrix.md index 40da3361f7a55..d844ff03b87e4 100644 --- a/docs/operator-manual/applicationset/Generators-Matrix.md +++ b/docs/operator-manual/applicationset/Generators-Matrix.md @@ -169,7 +169,7 @@ with the label `kubernetes.io/environment: dev` will have only dev-specific conf ## Restrictions 1. The Matrix generator currently only supports combining the outputs of only two child generators (eg does not support generating combinations for 3 or more). -2. You should specify only a single generator per array entry, eg this is not valid: +1. You should specify only a single generator per array entry, eg this is not valid: ```yaml - matrix: generators: @@ -177,7 +177,7 @@ with the label `kubernetes.io/environment: dev` will have only dev-specific conf git: # (...) ``` - While this *will* be accepted by Kubernetes API validation, the controller will report an error on generation. Each generator should be specified in a separate array element, as in the examples above. -3. The Matrix generator does not currently support [`template` overrides](Template.md#generator-templates) specified on child generators, eg this `template` will not be processed: +1. The Matrix generator does not currently support [`template` overrides](Template.md#generator-templates) specified on child generators, eg this `template` will not be processed: ```yaml - matrix: generators: @@ -186,7 +186,7 @@ with the label `kubernetes.io/environment: dev` will have only dev-specific conf - # (...) template: { } # Not processed ``` -4. Combination-type generators (matrix or merge) can only be nested once. For example, this will not work: +1. Combination-type generators (matrix or merge) can only be nested once. For example, this will not work: ```yaml - matrix: generators: @@ -198,7 +198,7 @@ with the label `kubernetes.io/environment: dev` will have only dev-specific conf elements: - # (...) ``` -5. When using parameters from one child generator inside another child generator, the child generator that *consumes* the parameters **must come after** the child generator that *produces* the parameters. +1. When using parameters from one child generator inside another child generator, the child generator that *consumes* the parameters **must come after** the child generator that *produces* the parameters. For example, the below example would be invalid (cluster-generator must come after the git-files generator): ```yaml - matrix: @@ -216,7 +216,7 @@ For example, the below example would be invalid (cluster-generator must come aft files: - path: "examples/git-generator-files-discovery/cluster-config/**/config.json" ``` -6. You cannot have both child generators consuming parameters from each another. In the example below, the cluster generator is consuming the `{{path.basename}}` parameter produced by the git-files generator, whereas the git-files generator is consuming the `{{name}}` parameter produced by the cluster generator. This will result in a circular dependency, which is invalid. +1. You cannot have both child generators consuming parameters from each another. In the example below, the cluster generator is consuming the `{{path.basename}}` parameter produced by the git-files generator, whereas the git-files generator is consuming the `{{name}}` parameter produced by the cluster generator. This will result in a circular dependency, which is invalid. ```yaml - matrix: generators: From c4c465446970a0859f8cbb829e85bebb9d244e74 Mon Sep 17 00:00:00 2001 From: jkulkarn Date: Fri, 3 Jun 2022 11:21:36 -0700 Subject: [PATCH 22/22] Added changes as asked by @crenshaw-dev. These include 1) Removing part of comment in generator_spec_processor.go that was for an earlier iteration of code 2) Returning nil instead of an empty map in matrix.go 3) Creating a full-test (TestInterpolatedMatrixGenerate) in matrix.go. This example is not exactly the same, but very similar, to the example appset I had written in the Matrix Docs (also part of this PR). Signed-off-by: jkulkarn --- .../generators/generator_spec_processor.go | 2 - applicationset/generators/matrix.go | 2 +- applicationset/generators/matrix_test.go | 157 ++++++++++++++++++ 3 files changed, 158 insertions(+), 3 deletions(-) diff --git a/applicationset/generators/generator_spec_processor.go b/applicationset/generators/generator_spec_processor.go index 9b04f35021cce..634dc69895db3 100644 --- a/applicationset/generators/generator_spec_processor.go +++ b/applicationset/generators/generator_spec_processor.go @@ -97,8 +97,6 @@ func mergeGeneratorTemplate(g Generator, requestedGenerator *argoprojiov1alpha1. // Currently for Matrix Generator. Allows interpolating the matrix's 2nd child generator with values from the 1st child generator // "params" parameter is an array, where each index corresponds to a generator. Each index contains a map w/ that generator's parameters. -// Since the Matrix generator currently only allows 2 child generators, effectively this params array will always be size 1 (containing the 1st child generator's params) -// Uses similar process for interpreting values as the actual Application Template func interpolateGenerator(requestedGenerator *argoprojiov1alpha1.ApplicationSetGenerator, params map[string]string) (argoprojiov1alpha1.ApplicationSetGenerator, error) { interpolatedGenerator := requestedGenerator.DeepCopy() tmplBytes, err := json.Marshal(interpolatedGenerator) diff --git a/applicationset/generators/matrix.go b/applicationset/generators/matrix.go index 1c9d9a5be4f84..33a14d34861f3 100644 --- a/applicationset/generators/matrix.go +++ b/applicationset/generators/matrix.go @@ -44,7 +44,7 @@ func (m *MatrixGenerator) GenerateParams(appSetGenerator *argoprojiov1alpha1.App res := []map[string]string{} - g0, err := m.getParams(appSetGenerator.Matrix.Generators[0], appSet, map[string]string{}) + g0, err := m.getParams(appSetGenerator.Matrix.Generators[0], appSet, nil) if err != nil { return nil, err } diff --git a/applicationset/generators/matrix_test.go b/applicationset/generators/matrix_test.go index e33043fe858c9..cce467efcfa37 100644 --- a/applicationset/generators/matrix_test.go +++ b/applicationset/generators/matrix_test.go @@ -1,6 +1,13 @@ package generators import ( + "context" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + kubefake "k8s.io/client-go/kubernetes/fake" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" "testing" "time" @@ -260,6 +267,156 @@ func TestMatrixGetRequeueAfter(t *testing.T) { } } +func TestInterpolatedMatrixGenerate(t *testing.T) { + interpolatedGitGenerator := &argoprojiov1alpha1.GitGenerator{ + RepoURL: "RepoURL", + Revision: "Revision", + Files: []argoprojiov1alpha1.GitFileGeneratorItem{ + {Path: "examples/git-generator-files-discovery/cluster-config/dev/config.json"}, + {Path: "examples/git-generator-files-discovery/cluster-config/prod/config.json"}, + }, + } + + interpolatedClusterGenerator := &argoprojiov1alpha1.ClusterGenerator{ + Selector: metav1.LabelSelector{ + MatchLabels: map[string]string{"environment": "{{path.basename}}"}, + MatchExpressions: nil, + }, + } + testCases := []struct { + name string + baseGenerators []argoprojiov1alpha1.ApplicationSetNestedGenerator + expectedErr error + expected []map[string]string + clientError bool + }{ + { + name: "happy flow - generate interpolated params", + baseGenerators: []argoprojiov1alpha1.ApplicationSetNestedGenerator{ + { + Git: interpolatedGitGenerator, + }, + { + Clusters: interpolatedClusterGenerator, + }, + }, + expected: []map[string]string{ + {"path": "examples/git-generator-files-discovery/cluster-config/dev/config.json", "path.basename": "dev", "path.basenameNormalized": "dev", "name": "dev-01", "nameNormalized": "dev-01", "server": "https://dev-01.example.com", "metadata.labels.environment": "dev", "metadata.labels.argocd.argoproj.io/secret-type": "cluster"}, + {"path": "examples/git-generator-files-discovery/cluster-config/prod/config.json", "path.basename": "prod", "path.basenameNormalized": "prod", "name": "prod-01", "nameNormalized": "prod-01", "server": "https://prod-01.example.com", "metadata.labels.environment": "prod", "metadata.labels.argocd.argoproj.io/secret-type": "cluster"}, + }, + clientError: false, + }, + } + clusters := []client.Object{ + &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "dev-01", + Namespace: "namespace", + Labels: map[string]string{ + "argocd.argoproj.io/secret-type": "cluster", + "environment": "dev", + }, + }, + Data: map[string][]byte{ + "config": []byte("{}"), + "name": []byte("dev-01"), + "server": []byte("https://dev-01.example.com"), + }, + Type: corev1.SecretType("Opaque"), + }, + &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "prod-01", + Namespace: "namespace", + Labels: map[string]string{ + "argocd.argoproj.io/secret-type": "cluster", + "environment": "prod", + }, + }, + Data: map[string][]byte{ + "config": []byte("{}"), + "name": []byte("prod-01"), + "server": []byte("https://prod-01.example.com"), + }, + Type: corev1.SecretType("Opaque"), + }, + } + // convert []client.Object to []runtime.Object, for use by kubefake package + runtimeClusters := []runtime.Object{} + for _, clientCluster := range clusters { + runtimeClusters = append(runtimeClusters, clientCluster) + } + + for _, testCase := range testCases { + testCaseCopy := testCase // Since tests may run in parallel + + t.Run(testCaseCopy.name, func(t *testing.T) { + genMock := &generatorMock{} + appSet := &argoprojiov1alpha1.ApplicationSet{} + + appClientset := kubefake.NewSimpleClientset(runtimeClusters...) + fakeClient := fake.NewClientBuilder().WithObjects(clusters...).Build() + cl := &possiblyErroringFakeCtrlRuntimeClient{ + fakeClient, + testCase.clientError, + } + var clusterGenerator = NewClusterGenerator(cl, context.Background(), appClientset, "namespace") + + for _, g := range testCaseCopy.baseGenerators { + + gitGeneratorSpec := argoprojiov1alpha1.ApplicationSetGenerator{ + Git: g.Git, + Clusters: g.Clusters, + } + genMock.On("GenerateParams", mock.AnythingOfType("*v1alpha1.ApplicationSetGenerator"), appSet).Return([]map[string]string{ + { + "path": "examples/git-generator-files-discovery/cluster-config/dev/config.json", + "path.basename": "dev", + "path.basenameNormalized": "dev", + }, + { + "path": "examples/git-generator-files-discovery/cluster-config/prod/config.json", + "path.basename": "prod", + "path.basenameNormalized": "prod", + }, + }, nil) + genMock.On("GetTemplate", &gitGeneratorSpec). + Return(&argoprojiov1alpha1.ApplicationSetTemplate{}) + } + var matrixGenerator = NewMatrixGenerator( + map[string]Generator{ + "Git": genMock, + "Clusters": clusterGenerator, + }, + ) + + got, err := matrixGenerator.GenerateParams(&argoprojiov1alpha1.ApplicationSetGenerator{ + Matrix: &argoprojiov1alpha1.MatrixGenerator{ + Generators: testCaseCopy.baseGenerators, + Template: argoprojiov1alpha1.ApplicationSetTemplate{}, + }, + }, appSet) + + if testCaseCopy.expectedErr != nil { + assert.EqualError(t, err, testCaseCopy.expectedErr.Error()) + } else { + assert.NoError(t, err) + assert.Equal(t, testCaseCopy.expected, got) + } + + }) + + } +} + type generatorMock struct { mock.Mock }